diff --git a/array.c b/array.c index 7b954d71d800ae..adb61a0d4cdeb9 100644 --- a/array.c +++ b/array.c @@ -1178,6 +1178,7 @@ rb_ary_initialize(int argc, VALUE *argv, VALUE ary) } /* recheck after argument conversion */ rb_ary_modify(ary); + ARY_SET_LEN(ary, 0); ary_resize_capa(ary, len); if (rb_block_given_p()) { long i; diff --git a/gc.c b/gc.c index b714a5394321a3..5896e06f344f75 100644 --- a/gc.c +++ b/gc.c @@ -4211,7 +4211,11 @@ rb_gc_vm_refresh_zombie_pages(void) void rb_gc_rest(void) { - rb_gc_impl_gc_rest(rb_gc_get_objspace()); + // Lock to keep assertions happy. This runs right after single-ractor mode is + // cancelled, but we can still free shareables like fstrings because ractor.cnt is 1. + RB_VM_LOCKING() { + rb_gc_impl_gc_rest(rb_gc_get_objspace()); + } } /* True while a zombie is being absorbed. The zombie's count is decremented before the diff --git a/gc/default/default.c b/gc/default/default.c index b7c4cb00d7256a..4299382d02199f 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -5202,16 +5202,6 @@ gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context } } - /* main's local GC is lock-free, but freeing a dead object can mutate VM-global state - * that other Ractors rewrite under the VM lock: weak tables (rb_gc_obj_free_vm_weak_ - * references: ci_table, fstring, symbol, cme). (JIT iseq frees are not reached here: - * iseqs are born shareable and a local GC never frees shareable objects.) Wrap the - * page's free loop in a no-barrier VM lock (FIXME). */ - const bool sweep_needs_vm_lock = - objspace == global_objspace->main_objspace && rb_gc_multi_ractor_p() && !objspace->flags.during_global_gc; - unsigned int sweep_lock_lev = 0; - if (sweep_needs_vm_lock) sweep_lock_lev = RB_GC_VM_LOCK_NO_BARRIER(); - for (int i = 0; i < bitmap_plane_count; i++) { bitset = ~bits[i]; if (bitset) { @@ -5220,8 +5210,6 @@ gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context p += BITS_BITLENGTH * slot_size; } - if (sweep_needs_vm_lock) RB_GC_VM_UNLOCK_NO_BARRIER(sweep_lock_lev); - /* Bulk-clear the freed slots' shareable and shref bits before the freelist is * published, so a reused slot is clean. Freed slots are exactly the unmarked ones, * so `bits &= mark_bits` keeps live shareable objects (which must stay pinned) and diff --git a/include/ruby/internal/attr/noalias.h b/include/ruby/internal/attr/noalias.h index 0790ef60e56786..8253a332cb1388 100644 --- a/include/ruby/internal/attr/noalias.h +++ b/include/ruby/internal/attr/noalias.h @@ -54,7 +54,7 @@ # /* # * `::llvm::Attribute::ArgMemOnly` was buggy before. Maybe because nobody # * actually seriously used it. It seems they somehow mitigated the situation -# * in LLVM 12. Still not found the exact changeset which fiexed the +# * in LLVM 12. Still not found the exact changeset which fixed the # * attribute, though. # * # * :FIXME: others (armclang, xlclang, ...) can also be affected? diff --git a/insns.def b/insns.def index 78ed3245a2230f..d5dbf1c08be002 100644 --- a/insns.def +++ b/insns.def @@ -145,7 +145,6 @@ getblockparamproxy (lindex_t idx, rb_num_t level) () (VALUE val) -// attr bool zjit_profile = true; { const VALUE *ep = vm_get_ep(GET_EP(), level); VM_ASSERT(VM_ENV_LOCAL_P(ep)); diff --git a/jit.c b/jit.c index df9accf87f0802..5e26e1b87e1afa 100644 --- a/jit.c +++ b/jit.c @@ -736,13 +736,32 @@ rb_jit_reserve_addr_space(uint32_t mem_size) #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE); uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_jit_reserve_addr_space; - uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX; - // Align the requested address to page size - uint8_t *req_addr = rb_jit_align_ptr(cfunc_sample_addr, page_size); + // 64MiB: balancing space probed and time spent probing. + const uintptr_t probe_stride = 64 * 1024 * 1024; + // Related to the stride. Any successful trial will be within INT32_MAX + // range with slack for the binary size. + const int max_probe_trials = 30; // Probe for addresses close to this function using MAP_FIXED_NOREPLACE // to improve odds of being in range for 32-bit relative call instructions. - do { + uint8_t *req_addr = cfunc_sample_addr; + for (int i = 0; i < max_probe_trials; i++) { + // The address space on x86-64/A64 Linux tends to look like: + // + // high addr +---------------+ + // | | [stack] | + // | | DSO text | + // | | [heap] | + // | | main exe text | + // v | 0 | + // low addr +---------------+ + // + // We always probe downwards from one of the program text areas + // to avoid getting in the way of the stack's downwards growth. + // If we happen to start from the main text, we also avoid the heap. + req_addr -= probe_stride; + req_addr = rb_jit_align_ptr(req_addr, page_size); + mem_block = mmap( req_addr, mem_size, @@ -757,13 +776,7 @@ rb_jit_reserve_addr_space(uint32_t mem_size) ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_jit_reserve_addr_space"); break; } - - // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux - // main_code_addr < heap_addr, and in case we are in a shared - // library mapped higher than the heap, downwards is still better - // since it's towards the end of the heap rather than the stack.) - req_addr -= 4 * 1024 * 1024; - } while (req_addr < probe_region_end); + } // On MacOS and other platforms #else diff --git a/spec/ruby/core/module/alias_method_spec.rb b/spec/ruby/core/module/alias_method_spec.rb index 57b7eea48e9bde..1a645f7b89a07d 100644 --- a/spec/ruby/core/module/alias_method_spec.rb +++ b/spec/ruby/core/module/alias_method_spec.rb @@ -120,11 +120,13 @@ def uno_refined_method -> { ModuleSpecs::ReopeningModule.foo2 }.should_not.raise(NoMethodError) end - it "accesses a method defined on Object from Kernel" do - Kernel.public_instance_methods(true).should_not.include?(:module_specs_public_method_on_object) + ruby_version_is ""..."4.2" do + it "accesses a method defined on Object from Kernel" do + Kernel.public_instance_methods(true).should_not.include?(:module_specs_public_method_on_object) - Kernel.public_instance_methods(false).should.include?(:module_specs_alias_on_kernel) - Object.public_instance_methods(true).should.include?(:module_specs_alias_on_kernel) + Kernel.public_instance_methods(false).should.include?(:module_specs_alias_on_kernel) + Object.public_instance_methods(true).should.include?(:module_specs_alias_on_kernel) + end end it "can call a method with super aliased twice" do diff --git a/spec/ruby/core/module/fixtures/classes.rb b/spec/ruby/core/module/fixtures/classes.rb index 964f64c593b8da..b29c0c61f5e508 100644 --- a/spec/ruby/core/module/fixtures/classes.rb +++ b/spec/ruby/core/module/fixtures/classes.rb @@ -643,7 +643,12 @@ def module_specs_public_method_on_object_for_kernel_private; end module Kernel def module_specs_public_method_on_kernel; end - alias_method :module_specs_alias_on_kernel, :module_specs_public_method_on_object + ruby_version_is ""..."4.2" do + deprecated, Warning[:deprecated] = Warning[:deprecated], false + alias_method :module_specs_alias_on_kernel, :module_specs_public_method_on_object + ensure + Warning[:deprecated] = deprecated + end public :module_specs_private_method_on_object_for_kernel_public protected :module_specs_public_method_on_object_for_kernel_protected diff --git a/test/ruby/test_array.rb b/test/ruby/test_array.rb index 3593143cfdf130..62373ab2f06722 100644 --- a/test/ruby/test_array.rb +++ b/test/ruby/test_array.rb @@ -2753,6 +2753,7 @@ def test_initialize assert_equal([1, 1, 1], Array.new(3, 1)) assert_equal([1, 1, 1], Array.new(3) { 1 }) assert_equal([1, 1, 1], assert_warning(/block supersedes default value argument/) {Array.new(3, 1) { 1 }}) + assert_equal([], [1].instance_eval { initialize(0) }) end def test_aset_error diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index be4653d19662ba..b91e15e418aec7 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -1005,40 +1005,39 @@ pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 222; pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 223; pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 224; pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 225; -pub const YARVINSN_zjit_getblockparamproxy: ruby_vminsn_type = 226; -pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 227; -pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 228; -pub const YARVINSN_zjit_splatkw: ruby_vminsn_type = 229; -pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 230; -pub const YARVINSN_zjit_send: ruby_vminsn_type = 231; -pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 232; -pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 233; -pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 234; -pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 235; -pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 236; -pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 237; -pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 238; -pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 239; -pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 240; -pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 241; -pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 242; -pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 243; -pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 244; -pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 245; -pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 246; -pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 247; -pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 248; -pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 249; -pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 250; -pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 251; -pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 252; -pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 253; -pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 254; -pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 255; -pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 256; -pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 257; -pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 258; -pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 259; +pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 226; +pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 227; +pub const YARVINSN_zjit_splatkw: ruby_vminsn_type = 228; +pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 229; +pub const YARVINSN_zjit_send: ruby_vminsn_type = 230; +pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 231; +pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 232; +pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 235; +pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 236; +pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 237; +pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 238; +pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 239; +pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 240; +pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 241; +pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 242; +pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 243; +pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 244; +pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 245; +pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 246; +pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 247; +pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 248; +pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 249; +pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 250; +pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 251; +pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 252; +pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 253; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 254; +pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 255; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 256; +pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 257; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 258; pub type ruby_vminsn_type = u32; pub const DEFINED_NOT_DEFINED: defined_type = 0; pub const DEFINED_NIL: defined_type = 1; diff --git a/zjit.c b/zjit.c index 9ae0ea6283a71b..4e44febf72b6bc 100644 --- a/zjit.c +++ b/zjit.c @@ -359,7 +359,6 @@ rb_zjit_class_has_default_allocator(VALUE klass) } -VALUE rb_vm_untag_block_handler(VALUE block_handler); VALUE rb_vm_get_untagged_block_handler(rb_control_frame_t *reg_cfp); bool rb_vm_once_done_value(ISE is, VALUE *result); diff --git a/zjit.rb b/zjit.rb index 3f6d18b205c731..1bebf983279686 100644 --- a/zjit.rb +++ b/zjit.rb @@ -110,7 +110,6 @@ def stats_string print_counters_with_prefix(prefix: 'getivar_fallback_', prompt: 'getivar fallback reasons', buf:, stats:, limit: 5) print_counters_with_prefix(prefix: 'definedivar_fallback_', prompt: 'definedivar fallback reasons', buf:, stats:, limit: 5) print_counters_with_prefix(prefix: 'invokeblock_handler_', prompt: 'invokeblock handler', buf:, stats:, limit: 10) - print_counters_with_prefix(prefix: 'getblockparamproxy_handler_', prompt: 'getblockparamproxy handler', buf:, stats:, limit: 10) print_counters_with_prefix(prefix: 'inline_reject_', prompt: 'HIR-level inlining rejection reasons', buf:, stats:, limit: 10) # Show most popular unsupported call features. Because each call can diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 99cb4432566d73..7da19ed0fa92c2 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -412,7 +412,6 @@ fn main() { .allowlist_function("rb_get_cfp_sp") .allowlist_function("rb_get_cfp_self") .allowlist_function("rb_get_cfp_ep") - .allowlist_function("rb_get_cfp_ep_level") .allowlist_function("rb_get_cme_def_type") .allowlist_function("rb_zjit_vm_search_method") .allowlist_function("rb_zjit_cme_is_cfunc") @@ -460,7 +459,6 @@ fn main() { .allowlist_function("rb_str_neq_internal") .allowlist_function("rb_yarv_ary_entry_internal") .allowlist_function("rb_vm_get_untagged_block_handler") - .allowlist_function("rb_vm_untag_block_handler") .allowlist_function("rb_FL_TEST") .allowlist_function("rb_FL_TEST_RAW") .allowlist_function("rb_RB_TYPE_P") @@ -473,7 +471,6 @@ fn main() { .allowlist_function("rb_RCLASS_ORIGIN") .allowlist_function("rb_method_basic_definition_p") .allowlist_function("rb_obj_class") - .allowlist_function("rb_obj_is_proc") .allowlist_function("rb_vm_base_ptr") .allowlist_function("rb_ec_stack_check") .allowlist_function("rb_vm_top_self") diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 9586fc0601fdef..1649c1006e1df7 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -364,6 +364,8 @@ fn gen_iseq(cb: &mut CodeBlock, iseq: IseqPtr, function: Option<&Function>) -> R Ok(code_ptrs) => { unsafe { version.as_mut() }.status = IseqStatus::Compiled(code_ptrs.clone()); incr_counter!(compiled_iseq_count); + // Give the new version a fresh budget of recompile exits. See exit_recompile(). + payload.num_exits_until_invalidate = get_option!(num_exits_until_invalidate); } Err(err) => { unsafe { version.as_mut() }.status = IseqStatus::CantCompile(err.clone()); @@ -756,7 +758,8 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, function, *id, opnd!(val), &function.frame_state(*state))), Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, function, *id, &function.frame_state(*state)), &Insn::IsBlockParamModified { flags } => gen_is_block_param_modified(asm, opnd!(flags)), - &Insn::GetBlockParam { ep_offset, level, state } => gen_getblockparam(jit, asm, function, ep_offset, level, &function.frame_state(state)), + &Insn::GetBlockParam { ep_offset, level, state } => gen_getblockparam(jit, asm, function, ep_offset, level, &function.frame_state(state), false), + &Insn::SymToProc { ep_offset, level, state } => gen_getblockparam(jit, asm, function, ep_offset, level, &function.frame_state(state), true), &Insn::SetLocal { val, ep_offset, level, .. } => no_output!(gen_setlocal(asm, opnd!(val), function.type_of(val), ep_offset, level)), Insn::GetConstant { klass, id, allow_nil, state } => gen_getconstant(jit, asm, function, opnd!(klass), *id, opnd!(allow_nil), &function.frame_state(*state)), Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, function, *ic, &function.frame_state(*state)), @@ -898,7 +901,11 @@ fn gen_is_block_param_modified(asm: &mut Assembler, flags: Opnd) -> Opnd { /// Get the block parameter as a Proc, write it to the environment, /// and mark the flag as modified. -fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, function: &Function, ep_offset: u32, level: u32, state: &FrameState) -> Opnd { +fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, function: &Function, ep_offset: u32, level: u32, state: &FrameState, known_symbol: bool) -> Opnd { + unsafe extern "C" { + fn rb_sym_to_proc(sym: VALUE) -> VALUE; + } + gen_prepare_leaf_call_with_gc(asm, state); // Bail out if write barrier is required. let ep = gen_get_ep(asm, level); @@ -906,9 +913,14 @@ fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, function: &Functio asm.test(flags, VM_ENV_FLAG_WB_REQUIRED.into()); asm.jnz(jit, side_exit(jit, function, state, SideExitReason::BlockParamWbRequired)); - // Convert block handler to Proc. + // Convert block handler to Proc. When the caller proved the block handler is a symbol, + // VM_BH_TO_SYMBOL() is an identity cast, so call rb_sym_to_proc() on it directly. let block_handler = asm.load(Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); - let proc = asm_ccall!(asm, rb_vm_bh_to_procval, EC, block_handler); + let proc = if known_symbol { + asm_ccall!(asm, rb_sym_to_proc, block_handler) + } else { + asm_ccall!(asm, rb_vm_bh_to_procval, EC, block_handler) + }; let local_ep_offset = c_int::try_from(ep_offset).unwrap_or_else(|_| { panic!("Could not convert local_ep_offset {ep_offset} to i32") @@ -3755,11 +3767,9 @@ c_callable! { /// the outer ISEQ's version holds the failing guard and must be invalidated to /// force a recompile. For non-inlined code, it is the same as the frame ISEQ. /// - /// The first exit invalidates the version right away. Invalidation resets the - /// ISEQ's call counter and re-stubs incoming JIT-to-JIT calls, so every entry - /// runs the profiling window in the interpreter before the next compile. - /// - /// TODO: Allow waiting for a configured number of exits before invalidating the ISEQ. + /// The version gets invalidated after `--zjit-num-exits-until-invalidate` recompile exits. + /// Invalidation resets the ISEQ's call counter and re-stubs incoming JIT-to-JIT calls, + /// so every entry runs the profiling window in the interpreter before the next compile. pub(crate) fn exit_recompile(compiled_iseq_raw: VALUE) { // Fast check before taking the VM lock: skip if the compiled unit is already // invalidated or at the version limit. This avoids expensive lock acquisition @@ -3774,6 +3784,12 @@ c_callable! { if already_done { return; } + + // Wait for the configured number of recompile exits before invalidation. + payload.num_exits_until_invalidate = payload.num_exits_until_invalidate.saturating_sub(1); + if payload.num_exits_until_invalidate > 0 { + return; + } } with_vm_lock(src_loc!(), || { diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index 26dde02c914406..b062f04b6dd418 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -6,7 +6,7 @@ use crate::backend::lir::Assembler; use crate::codegen::max_iseq_versions; use crate::cruby::*; use crate::hir::{Insn, iseq_to_hir}; -use crate::options::{CallThreshold, get_option, rb_zjit_prepare_options, set_call_threshold, set_inline_threshold, set_max_versions, set_mem_bytes}; +use crate::options::{CallThreshold, get_option, rb_zjit_prepare_options, set_call_threshold, set_inline_threshold, set_max_versions, set_mem_bytes, set_num_exits_until_invalidate}; use crate::payload::IseqVersion; use crate::hir::tests::hir_build_tests::assert_contains_opcode; use crate::payload::*; @@ -197,6 +197,7 @@ fn test_putobject() { #[test] fn test_recompile_exit_invalidates_on_first_exit() { set_call_threshold(2); + set_num_exits_until_invalidate(1); eval(" def recompile_on_first_exit(a, b) = a + b recompile_on_first_exit(1, 2) @@ -208,18 +209,49 @@ fn test_recompile_exit_invalidates_on_first_exit() { assert_eq!(1, payload.versions.len()); assert!(!unsafe { payload.versions.last().unwrap().as_ref() }.is_invalidated()); - // The first recompile exit invalidates the version right away, so subsequent - // calls re-profile every instruction in the interpreter before recompiling. + // With --zjit-num-exits-until-invalidate=1, the first recompile exit invalidates the version right + // away, so subsequent calls re-profile every instruction in the interpreter before recompiling. eval("recompile_on_first_exit(1.5, 2.5)"); let payload = get_or_create_iseq_payload(iseq); assert_eq!(1, payload.versions.len()); assert!(unsafe { payload.versions.last().unwrap().as_ref() }.is_invalidated()); } +#[test] +fn test_recompile_exit_waits_for_exit_budget() { + set_call_threshold(2); + set_num_exits_until_invalidate(3); + eval(" + def recompile_exit_budget(a, b) = a + b + recompile_exit_budget(1, 2) + recompile_exit_budget(1, 2) + "); + + let iseq = get_method_iseq("self", "recompile_exit_budget"); + let payload = get_or_create_iseq_payload(iseq); + assert_eq!(1, payload.versions.len()); + assert!(!unsafe { payload.versions.last().unwrap().as_ref() }.is_invalidated()); + + // The first two recompile exits only decrement the budget. The compiled version keeps running. + for _ in 0..2 { + eval("recompile_exit_budget(1.5, 2.5)"); + let payload = get_or_create_iseq_payload(iseq); + assert_eq!(1, payload.versions.len()); + assert!(!unsafe { payload.versions.last().unwrap().as_ref() }.is_invalidated()); + } + + // The third recompile exit exhausts the budget and invalidates the version. + eval("recompile_exit_budget(1.5, 2.5)"); + let payload = get_or_create_iseq_payload(iseq); + assert_eq!(1, payload.versions.len()); + assert!(unsafe { payload.versions.last().unwrap().as_ref() }.is_invalidated()); +} + #[test] fn test_function_stub_reprofiles_after_invalidation() { rb_zjit_prepare_options(); set_inline_threshold(0); + set_num_exits_until_invalidate(1); let num_profiles = get_option!(num_profiles); let call_threshold = CallThreshold::from(num_profiles) + 2; set_call_threshold(call_threshold); diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 7e0f2fdabb9732..3d8d561178bb09 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -180,7 +180,6 @@ pub use rb_get_ec_cfp as get_ec_cfp; pub use rb_get_cfp_iseq as get_cfp_iseq; pub use rb_get_cfp_pc as get_cfp_pc; pub use rb_get_cfp_sp as get_cfp_sp; -pub use rb_get_cfp_ep_level as get_cfp_ep_level; pub use rb_get_cme_def_type as get_cme_def_type; pub use rb_get_cme_def_body_attr_id as get_cme_def_body_attr_id; pub use rb_get_cme_def_body_optimized_type as get_cme_def_body_optimized_type; @@ -1742,7 +1741,6 @@ pub(crate) mod ids { name: freeze name: minusat content: b"-@" name: aref content: b"[]" - name: rb_obj_is_proc name: rb_ivar_get_at_no_ractor_check name: rb_jit_ruby2_keywords_splat_p name: RUBY_FL_FREEZE diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 6343cf69e10755..94d7bd4f24bfe5 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1980,40 +1980,39 @@ pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 222; pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 223; pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 224; pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 225; -pub const YARVINSN_zjit_getblockparamproxy: ruby_vminsn_type = 226; -pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 227; -pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 228; -pub const YARVINSN_zjit_splatkw: ruby_vminsn_type = 229; -pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 230; -pub const YARVINSN_zjit_send: ruby_vminsn_type = 231; -pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 232; -pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 233; -pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 234; -pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 235; -pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 236; -pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 237; -pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 238; -pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 239; -pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 240; -pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 241; -pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 242; -pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 243; -pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 244; -pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 245; -pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 246; -pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 247; -pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 248; -pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 249; -pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 250; -pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 251; -pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 252; -pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 253; -pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 254; -pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 255; -pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 256; -pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 257; -pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 258; -pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 259; +pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 226; +pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 227; +pub const YARVINSN_zjit_splatkw: ruby_vminsn_type = 228; +pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 229; +pub const YARVINSN_zjit_send: ruby_vminsn_type = 230; +pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 231; +pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 232; +pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 235; +pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 236; +pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 237; +pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 238; +pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 239; +pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 240; +pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 241; +pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 242; +pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 243; +pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 244; +pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 245; +pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 246; +pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 247; +pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 248; +pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 249; +pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 250; +pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 251; +pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 252; +pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 253; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 254; +pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 255; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 256; +pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 257; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 258; pub type ruby_vminsn_type = u32; #[repr(C)] #[repr(align(8))] @@ -2257,7 +2256,6 @@ unsafe extern "C" { pub fn rb_hash_aref(hash: VALUE, key: VALUE) -> VALUE; pub fn rb_hash_aset(hash: VALUE, key: VALUE, val: VALUE) -> VALUE; pub fn rb_hash_bulk_insert(argc: ::std::os::raw::c_long, argv: *const VALUE, hash: VALUE); - pub fn rb_obj_is_proc(recv: VALUE) -> VALUE; pub fn rb_protect( func: ::std::option::Option VALUE>, args: VALUE, @@ -2486,7 +2484,6 @@ unsafe extern "C" { pub fn rb_zjit_class_initialized_p(klass: VALUE) -> bool; pub fn rb_zjit_class_get_alloc_func(klass: VALUE) -> rb_alloc_func_t; pub fn rb_zjit_class_has_default_allocator(klass: VALUE) -> bool; - pub fn rb_vm_untag_block_handler(block_handler: VALUE) -> VALUE; pub fn rb_vm_get_untagged_block_handler(reg_cfp: *mut rb_control_frame_t) -> VALUE; pub fn rb_vm_once_done_value(is: ISE, result: *mut VALUE) -> bool; pub fn rb_iseq_encoded_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; @@ -2567,7 +2564,6 @@ unsafe extern "C" { pub fn rb_get_cfp_sp(cfp: *mut rb_control_frame_struct) -> *mut VALUE; pub fn rb_get_cfp_self(cfp: *mut rb_control_frame_struct) -> VALUE; pub fn rb_get_cfp_ep(cfp: *mut rb_control_frame_struct) -> *mut VALUE; - pub fn rb_get_cfp_ep_level(cfp: *mut rb_control_frame_struct, lv: u32) -> *const VALUE; pub fn rb_yarv_class_of(obj: VALUE) -> VALUE; pub fn rb_FL_TEST(obj: VALUE, flags: VALUE) -> VALUE; pub fn rb_FL_TEST_RAW(obj: VALUE, flags: VALUE) -> VALUE; diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index fef5b367a8ce54..082a0c57680334 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -650,11 +650,6 @@ pub enum SideExitReason { PatchPoint(Invariant), CalleeSideExit, Interrupt, - BlockParamProxyNotIseqOrIfunc, - BlockParamProxyNotNil, - BlockParamProxyNotProc, - BlockParamProxyFallbackMiss, - BlockParamProxyProfileNotCovered, InvokeBlockHandlerNotIseq, InvokeBlockIseqChanged, BlockParamWbRequired, @@ -680,7 +675,7 @@ pub enum SideExitReason { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Recompile; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum MethodType { Iseq, Cfunc, @@ -717,7 +712,7 @@ impl From for MethodType { } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum OptimizedMethodType { Send, Call, @@ -779,7 +774,7 @@ pub enum ReceiverTypeResolution { } /// Reason why a send-ish instruction cannot be optimized from a fallback instruction -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum SendFallbackReason { SendCfuncNotVariadic, SendNotOptimizedMethodTypeOptimized(OptimizedMethodType), @@ -1132,6 +1127,8 @@ pub enum Insn { IsBlockParamModified { flags: InsnId }, /// Get the block parameter as a Proc. GetBlockParam { level: u32, ep_offset: u32, state: InsnId }, + /// Get the block parameter, which is known to be a symbol, as a Proc. + SymToProc { level: u32, ep_offset: u32, state: InsnId }, /// Set a local variable in a higher scope or the heap SetLocal { level: u32, ep_offset: u32, val: InsnId, state: InsnId }, GetSpecialSymbol { symbol_type: SpecialBackrefSymbol, state: InsnId }, @@ -1387,6 +1384,7 @@ macro_rules! for_each_operand_impl { | Insn::CheckInterrupts { state } | Insn::PutSpecialObject { state, .. } | Insn::GetBlockParam { state, .. } + | Insn::SymToProc { state, .. } | Insn::GetConstantPath { state, .. } => { $visit_one!(*state); } @@ -1845,6 +1843,7 @@ impl Insn { Insn::SetClassVar { .. } => effects::Any, Insn::IsBlockParamModified { .. } => effects::Empty, Insn::GetBlockParam { .. } => effects::Any, + Insn::SymToProc { .. } => effects::Any, Insn::Snapshot { .. } => effects::Empty, Insn::Jump(_) => effects::Any, Insn::CondBranch { .. } => effects::Any, @@ -2352,6 +2351,12 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { .map_or(String::new(), |x| format!("{x}, ")); write!(f, "GetBlockParam {name}l{level}, EP@{ep_offset}") }, + &Insn::SymToProc { level, ep_offset, state, .. } => { + let iseq = self.fun.map(|fun| fun.frame_state_iseq(state)); + let name = get_local_var_name_for_printer(iseq, level, ep_offset) + .map_or(String::new(), |x| format!("{x}, ")); + write!(f, "SymToProc {name}l{level}, EP@{ep_offset}") + }, Insn::PatchPoint { invariant, .. } => { write!(f, "PatchPoint {}", invariant.print(self.ptr_map)) }, Insn::GetConstant { klass, id, allow_nil, .. } => { write!(f, "GetConstant {klass}, :{}, {allow_nil}", id.contents_lossy()) @@ -3598,7 +3603,13 @@ impl Function { | InvokeSuper { reason, .. } | InvokeSuperForward { reason, .. } | InvokeBlock { reason, .. } - => *reason = dynamic_send_reason, + => { + // Ignore the case where the instruction is intentionally a fallback for a + // polymorphic send. We already know that case is a lost cause. + if *reason != SendFallbackReason::SendPolymorphicFallback { + *reason = dynamic_send_reason; + } + } _ => unreachable!("unexpected instruction {} at {insn_id}", self.find(insn_id)) } } @@ -3777,6 +3788,7 @@ impl Function { Insn::AnyToString { .. } => types::StringExact, Insn::IsBlockParamModified { .. } => types::CBool, Insn::GetBlockParam { .. } => types::BasicObject, + Insn::SymToProc { .. } => types::BasicObject, // The type of Snapshot doesn't really matter; it's never materialized. It's used only // as a reference for FrameState, which we use to generate side-exit code. Insn::Snapshot { .. } => types::Any, @@ -4789,7 +4801,7 @@ impl Function { // blocks re-profiles the block arg and drops this speculation // (falling back to a dynamic send) instead of paying the guard // side exit repeatedly. This matches the receiver GuardType - // below and the getblockparamproxy BlockParamProxyNotNil guard. + // below. self.push_insn(block, Insn::GuardBitEquals { val: block_arg, expected: Const::Value(Qnil), @@ -7806,6 +7818,7 @@ impl Function { | Insn::GetSpecialNumber { .. } | Insn::GetSpecialSymbol { .. } | Insn::GetBlockParam { .. } + | Insn::SymToProc { .. } | Insn::StoreField { .. } => { Ok(()) } @@ -9047,36 +9060,6 @@ fn add_iseq_to_hir( } } } - } else if opcode == YARVINSN_getblockparamproxy || opcode == YARVINSN_trace_getblockparamproxy { - if get_option!(stats) { - let iseq_insn_idx = exit_state.insn_idx; - if let Some([block_handler_distribution]) = payload.profile.get_operand_types(iseq_insn_idx) { - let summary = TypeDistributionSummary::new(block_handler_distribution); - - if summary.is_monomorphic() { - let obj = summary.bucket(0).class(); - if unsafe { rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1} { - fun.count(block, Counter::getblockparamproxy_handler_iseq); - } else if unsafe { rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1} { - fun.count(block, Counter::getblockparamproxy_handler_ifunc); - } - else if obj.nil_p() { - fun.count(block, Counter::getblockparamproxy_handler_nil); - } - else if obj.symbol_p() { - fun.count(block, Counter::getblockparamproxy_handler_symbol); - } else if unsafe { rb_obj_is_proc(obj).test() } { - fun.count(block, Counter::getblockparamproxy_handler_proc); - } - } else if summary.is_polymorphic() || summary.is_skewed_polymorphic() { - fun.count(block, Counter::getblockparamproxy_handler_polymorphic); - } else if summary.is_megamorphic() || summary.is_skewed_megamorphic() { - fun.count(block, Counter::getblockparamproxy_handler_megamorphic); - } - } else { - fun.count(block, Counter::getblockparamproxy_handler_no_profiles); - } - } } else { profiles.profile_stack(exit_id, &exit_state); @@ -9744,37 +9727,13 @@ fn add_iseq_to_hir( }); } YARVINSN_getblockparamproxy => { - #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] - enum ProfiledBlockHandlerFamily { - Nil, - IseqOrIfunc, - Proc, - } - impl ProfiledBlockHandlerFamily { - fn from_profiled_type(profiled_type: ProfiledType) -> Option { - let obj = profiled_type.class(); - if obj.nil_p() { - Some(Self::Nil) - } else if unsafe { - rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1 - || rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1 - } { - Some(Self::IseqOrIfunc) - } else if unsafe { rb_obj_is_proc(obj).test() } { - Some(Self::Proc) - } else { - None - } - } - } - let ep_offset = get_arg(pc, 0).as_u32(); let level = get_arg(pc, 1).as_u32(); let branch_insn_idx = exit_state.insn_idx as u32; // `getblockparamproxy` has two semantic paths: // - modified: return the already-materialized block local from EP - // - unmodified: inspect the block handler and produce proxy/nil + // - unmodified: inspect the block handler and produce proxy/nil/proc let modified_block = fun.new_block(branch_insn_idx); let unmodified_block = fun.new_block(branch_insn_idx); let join_block = fun.new_block(insn_idx); @@ -9803,199 +9762,85 @@ fn add_iseq_to_hir( // does not accidentally accept symbol block handlers. const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); + let jump_to_join_block = |fun: &mut Function, from: BlockId, val: InsnId| { + let mut args = vec![val]; + if let Some(local) = original_local { args.push(local); } + fun.push_insn(from, Insn::Jump(BranchEdge { target: join_block, args })); + }; - let profiled_block_summary = payload.profile.get_operand_types(exit_state.insn_idx) - .and_then(|types| types.first()) - .map(TypeDistributionSummary::new); - - let mut profiled_handlers = Vec::new(); - if let Some(summary) = profiled_block_summary.as_ref() { - if summary.is_monomorphic() || summary.is_polymorphic() || summary.is_skewed_polymorphic() { - for &profiled_type in summary.buckets() { - if profiled_type.is_empty() { - break; - } - if let Some(profiled_handler) = ProfiledBlockHandlerFamily::from_profiled_type(profiled_type) { - if !profiled_handlers.contains(&profiled_handler) { - profiled_handlers.push(profiled_handler); - } - } - } - } - } - - match profiled_handlers.as_slice() { - // No supported profiled families. Keep the generic fallback iseq/ifunc fallback - // for sites we do not specialize, such as no-profile and megamorphic sites. - [] => { - let block_handler = fun.load_ep_env_field(unmodified_block, ep, FieldName::VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL, types::CInt64); - // This handles two cases which are nearly identical. - // Block handler is a tagged pointer. Look at the tag. - // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 - // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 - // So to check for either of those cases we can use: val & 0x1 == 0x1 - - // Bail out if the block handler is neither ISEQ nor ifunc - fun.push_insn(unmodified_block, Insn::GuardAnyBitSet { val: block_handler, mask: Const::CUInt64(0x1), mask_name: None, reason: Box::new(SideExitReason::BlockParamProxyFallbackMiss), state: exit_id, recompile: Some(Recompile) }); - // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing - let proxy_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); - let mut args = vec![proxy_val]; - if let Some(local) = original_local { - args.push(local); - } - fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); - } - // A single supported profiled family. Emit a monomorphic fast path - [profiled_handler] => match profiled_handler { - ProfiledBlockHandlerFamily::Nil => { - let block_handler = fun.load_ep_env_field(unmodified_block, ep, FieldName::VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL, types::CInt64); - fun.push_insn(unmodified_block, Insn::GuardBitEquals { val: block_handler, expected: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()), reason: Box::new(SideExitReason::BlockParamProxyNotNil), state: exit_id, recompile: Some(Recompile) }); - let nil_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(Qnil) }); - let mut args = vec![nil_val]; - if let Some(local) = original_local { - args.push(local); - } - fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); - } - ProfiledBlockHandlerFamily::IseqOrIfunc => { - let block_handler = fun.load_ep_env_field(unmodified_block, ep, FieldName::VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL, types::CInt64); - // This handles two cases which are nearly identical. - // Block handler is a tagged pointer. Look at the tag. - // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 - // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 - // So to check for either of those cases we can use: val & 0x1 == 0x1 - - // Bail out if the block handler is neither ISEQ nor ifunc - fun.push_insn(unmodified_block, Insn::GuardAnyBitSet { val: block_handler, mask: Const::CUInt64(0x1), mask_name: None, reason: Box::new(SideExitReason::BlockParamProxyNotIseqOrIfunc), state: exit_id, recompile: Some(Recompile) }); - // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing - let proxy_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); - let mut args = vec![proxy_val]; - if let Some(local) = original_local { - args.push(local); - } - fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); - } - ProfiledBlockHandlerFamily::Proc => { - let proc_val = fun.load_ep_env_field(unmodified_block, ep, FieldName::VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL, types::BasicObject); - let is_proc = fun.push_insn(unmodified_block, Insn::CCall { - cfunc: rb_obj_is_proc as *const u8, - recv: proc_val, - args: vec![], - name: ID!(rb_obj_is_proc), - owner: Qnil, - return_type: types::BasicObject, - elidable: true, - }); - fun.push_insn(unmodified_block, Insn::GuardBitEquals { val: is_proc, expected: Const::Value(Qtrue), reason: Box::new(SideExitReason::BlockParamProxyNotProc), state: exit_id, recompile: Some(Recompile) }); - let mut args = vec![proc_val]; - if let Some(local) = original_local { - args.push(local); - } - fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); - } - }, - // Multiple supported profiled families. Emit a polymorphic dispatch - _ => { - let block_handler = fun.load_ep_env_field(unmodified_block, ep, FieldName::VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL, types::CInt64); - let profiled_blocks = profiled_handlers.iter() - .map(|&kind| (kind, fun.new_block(branch_insn_idx))) - .collect::>(); - - let mut current_block = unmodified_block; - - for &(kind, profiled_block) in &profiled_blocks { - match kind { - ProfiledBlockHandlerFamily::Nil => { - let none_handler = fun.push_insn(current_block, Insn::Const { - val: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()), - }); - let is_none = fun.push_insn(current_block, Insn::IsBitEqual { - left: block_handler, - right: none_handler, - }); - - let next_block = fun.new_block(branch_insn_idx); - - fun.push_insn(current_block, Insn::CondBranch { - val: is_none, - if_true: BranchEdge { target: profiled_block, args: vec![] }, - if_false: BranchEdge { target: next_block, args: vec![] }, - }); - - current_block = next_block; - - let val = fun.push_insn(profiled_block, Insn::Const { val: Const::Value(Qnil) }); - let mut args = vec![val]; - if let Some(local) = original_local { args.push(local); } - fun.push_insn(profiled_block, Insn::Jump(BranchEdge { target: join_block, args })); + // Load a block_handler, which can be proxy to ISEQ/ifunc, nil, Proc, or something else. + let block_handler = fun.load_ep_env_field(unmodified_block, ep, FieldName::VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL, types::CInt64); + + // Handle two cases that use a tagged pointer: + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + let iseq_or_ifunc_block = fun.new_block(branch_insn_idx); + let nil_check_block = fun.new_block(branch_insn_idx); + let tag_mask = fun.push_insn(unmodified_block, Insn::Const { val: Const::CInt64(0x1) }); + let tag_bits = fun.push_insn(unmodified_block, Insn::IntAnd { left: block_handler, right: tag_mask }); + let is_iseq_or_ifunc = fun.push_insn(unmodified_block, Insn::IsBitEqual { left: tag_bits, right: tag_mask }); + fun.push_insn(unmodified_block, Insn::CondBranch { + val: is_iseq_or_ifunc, + if_true: BranchEdge { target: iseq_or_ifunc_block, args: vec![] }, + if_false: BranchEdge { target: nil_check_block, args: vec![] }, + }); + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + let proxy_val = fun.push_insn(iseq_or_ifunc_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); + jump_to_join_block(fun, iseq_or_ifunc_block, proxy_val); + + // Handle VM_BLOCK_HANDLER_NONE: the block param is nil. + let nil_block = fun.new_block(branch_insn_idx); + let sym_or_proc_block = fun.new_block(branch_insn_idx); + let none_handler = fun.push_insn(nil_check_block, Insn::Const { val: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()) }); + let is_none = fun.push_insn(nil_check_block, Insn::IsBitEqual { left: block_handler, right: none_handler }); + fun.push_insn(nil_check_block, Insn::CondBranch { + val: is_none, + if_true: BranchEdge { target: nil_block, args: vec![] }, + if_false: BranchEdge { target: sym_or_proc_block, args: vec![] }, + }); + let nil_val = fun.push_insn(nil_block, Insn::Const { val: Const::Value(Qnil) }); + jump_to_join_block(fun, nil_block, nil_val); + + // Prepare blocks for other cases: Everything left is a symbol or a Proc block handler. + let sym_block = fun.new_block(branch_insn_idx); + let dynsym_check_block = fun.new_block(branch_insn_idx); + let proc_block = fun.new_block(branch_insn_idx); + + // RB_STATIC_SYM_P(): (block_handler & 0xff) == RUBY_SYMBOL_FLAG + let sym_mask = fun.push_insn(sym_or_proc_block, Insn::Const { val: Const::CInt64((1 << RUBY_SPECIAL_SHIFT) - 1) }); + let sym_bits = fun.push_insn(sym_or_proc_block, Insn::IntAnd { left: block_handler, right: sym_mask }); + let sym_flag = fun.push_insn(sym_or_proc_block, Insn::Const { val: Const::CInt64(RUBY_SYMBOL_FLAG.into()) }); + let is_static_sym = fun.push_insn(sym_or_proc_block, Insn::IsBitEqual { left: sym_bits, right: sym_flag }); + fun.push_insn(sym_or_proc_block, Insn::CondBranch { + val: is_static_sym, + if_true: BranchEdge { target: sym_block, args: vec![] }, + if_false: BranchEdge { target: dynsym_check_block, args: vec![] }, + }); - } - ProfiledBlockHandlerFamily::IseqOrIfunc => { - // This handles two cases which are nearly identical. - // Block handler is a tagged pointer. Look at the tag. - // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 - // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 - // So to check for either of those cases we can use: val & 0x1 == 0x1 - let tag_mask = fun.push_insn(current_block, Insn::Const { val: Const::CInt64(0x1) }); - let tag_bits = fun.push_insn(current_block, Insn::IntAnd { - left: block_handler, - right: tag_mask, - }); - let is_iseq_or_ifunc = fun.push_insn(current_block, Insn::IsBitEqual { - left: tag_bits, - right: tag_mask, - }); - let next_block = fun.new_block(branch_insn_idx); - fun.push_insn(current_block, Insn::CondBranch { - val: is_iseq_or_ifunc, - if_true: BranchEdge { target: profiled_block, args: vec![] }, - if_false: BranchEdge { target: next_block, args: vec![] }, - }); - current_block = next_block; - - // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing - let val = fun.push_insn(profiled_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); - let mut args = vec![val]; - if let Some(local) = original_local { args.push(local); } - fun.push_insn(profiled_block, Insn::Jump(BranchEdge { target: join_block, args })); - }, - ProfiledBlockHandlerFamily::Proc => { - let proc_check_block = fun.new_block(branch_insn_idx); - let next_block = fun.new_block(branch_insn_idx); - fun.push_insn(current_block, Insn::Jump(BranchEdge { target: proc_check_block, args: vec![] })); - - let proc_val = fun.load_ep_env_field(proc_check_block, ep, FieldName::VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL, types::BasicObject); - let proc_result = fun.push_insn(proc_check_block, Insn::CCall { - cfunc: rb_obj_is_proc as *const u8, - recv: proc_val, - args: vec![], - name: ID!(rb_obj_is_proc), - owner: Qnil, - return_type: types::BasicObject, - elidable: true, - }); - let true_val = fun.push_insn(proc_check_block, Insn::Const { val: Const::Value(Qtrue) }); - let is_proc = fun.push_insn(proc_check_block, Insn::IsBitEqual { left: proc_result, right: true_val }); - fun.push_insn(proc_check_block, Insn::CondBranch { - val: is_proc, - if_true: BranchEdge { target: profiled_block, args: vec![] }, - if_false: BranchEdge { target: next_block, args: vec![] }, - }); - current_block = next_block; + // RB_DYNAMIC_SYM_P(): a dynamic symbol or a Proc is a heap object, so its builtin type can be read from the RBasic flags. + let rbasic_flags = fun.load_rbasic_flags(dynsym_check_block, block_handler); + let t_mask = fun.push_insn(dynsym_check_block, Insn::Const { val: Const::CUInt64(RUBY_T_MASK.into()) }); + let t_bits = fun.push_insn(dynsym_check_block, Insn::IntAnd { left: rbasic_flags, right: t_mask }); + let t_symbol = fun.push_insn(dynsym_check_block, Insn::Const { val: Const::CUInt64(RUBY_T_SYMBOL.into()) }); + let is_dynamic_sym = fun.push_insn(dynsym_check_block, Insn::IsBitEqual { left: t_bits, right: t_symbol }); + fun.push_insn(dynsym_check_block, Insn::CondBranch { + val: is_dynamic_sym, + if_true: BranchEdge { target: sym_block, args: vec![] }, + if_false: BranchEdge { target: proc_block, args: vec![] }, + }); - let mut args = vec![proc_val]; - if let Some(local) = original_local { args.push(local); } - fun.push_insn(profiled_block, Insn::Jump(BranchEdge { target: join_block, args })); - } - } - } + // block_handler_type_symbol: Let SymToProc call rb_sym_to_proc() and memoize the result in EP. + let sym_val = fun.push_insn(sym_block, Insn::SymToProc { ep_offset, level, state: exit_id }); + // Unlike the branches above, this wrote the Proc to the EP local. + let mut sym_args = vec![sym_val]; + if level == 0 { sym_args.push(sym_val); } + fun.push_insn(sym_block, Insn::Jump(BranchEdge { target: join_block, args: sym_args })); - fun.push_insn(current_block, Insn::SideExit { state: exit_id, reason: Box::new(SideExitReason::BlockParamProxyProfileNotCovered), recompile: None }); - } - } + // block_handler_type_proc: VM_BH_TO_PROC() is an identity cast, so no C call is needed. + let proc_val = fun.load_ep_env_field(proc_block, ep, FieldName::VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL, types::BasicObject); + jump_to_join_block(fun, proc_block, proc_val); - // Continue compilation from the merged continuation block at the next - // instruction. if let Some(local_param) = join_local { state.setlocal(ep_offset, local_param); } @@ -10302,8 +10147,55 @@ fn add_iseq_to_hir( None }; let caller_splat_length = fun.monomorphic_caller_splat_length(call_info, exit_id); - let send = fun.push_insn(block, Insn::Send { recv, cd, block: block_handler, args, caller_splat_length, state: exit_id, reason: Uncategorized(opcode.into()) }); - state.stack_push(send); + if let Some(summary) = fun.polymorphic_summary(&profiles, recv, exit_id) { + let join_block = fun.new_block(insn_idx); + let join_param = fun.push_insn(join_block, Insn::Param); + // Dedup by expected type so immediate/heap variants + // under the same Ruby class can still get separate branches. + let mut seen_types = Vec::with_capacity(summary.buckets().len()); + for &profiled_type in summary.buckets() { + if profiled_type.is_empty() { break; } + let expected = Type::from_profiled_type(profiled_type); + if seen_types.iter().any(|ty: &Type| ty.bit_equal(expected)) { + continue; + } + seen_types.push(expected); + let has_type = fun.push_insn(block, Insn::HasType { val: recv, expected }); + let iftrue_block = fun.new_block(insn_idx); + let fall_through = fun.new_block(insn_idx); + fun.push_insn(block, Insn::CondBranch { + val: has_type, + if_true: BranchEdge { target: iftrue_block, args: vec![] }, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + block = fall_through; + // Take a fresh Snapshot rather than + // reusing exit_id so type specialization resolves the receiver from + // its refined, exact type instead of the polymorphic profile that is + // keyed at exit_id. + let snapshot = fun.push_insn(iftrue_block, Insn::Snapshot { state: Box::new(exit_state.clone()) }); + // Keep the other operands' profile entries visible at the fresh + // Snapshot so the specialized send can still see argument profiles + // (e.g. Array#[] needs a Fixnum-profiled index to be inlined). Only + // the receiver's entry is dropped: it must resolve from its refined, + // exact type, and resolve_receiver_type prefers profiles over types. + profiles.copy_entries_except(exit_id, snapshot, recv, fun); + let refined_recv = fun.push_insn(iftrue_block, Insn::RefineType { val: recv, new_type: expected }); + let send = fun.push_insn(iftrue_block, Insn::Send { recv: refined_recv, cd, block: block_handler, args: args.clone(), caller_splat_length, state: snapshot, reason: Uncategorized(opcode.into()) }); + fun.push_insn(iftrue_block, Insn::Jump(BranchEdge { target: join_block, args: vec![send] })); + } + // In the fallthrough case, do a generic interpreter send and then join. + let reason = SendPolymorphicFallback; + let send = fun.push_insn(block, Insn::Send { recv, cd, block: block_handler, args, caller_splat_length, state: exit_id, reason }); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: vec![send] })); + state.stack_push(join_param); + // Continue compilation from the join block at the next instruction. + block = join_block; + } else { + // Maybe monomorphic; handled in type_specialize + let send = fun.push_insn(block, Insn::Send { recv, cd, block: block_handler, args, caller_splat_length, state: exit_id, reason: Uncategorized(opcode.into()) }); + state.stack_push(send); + } if let Some(BlockHandler::BlockIseq(blockiseq)) = block_handler { // Reload locals that may have been modified by the blockiseq. diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 1bc82cd59e8d50..637f85ea2c4c6b 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -4264,7 +4264,7 @@ mod hir_opt_tests { v13:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v52:NilClass = Const Value(nil) + v75:NilClass = Const Value(nil) PushInlineFrame :foo, v22 (0x1038), num_args=2 v34:CPtr = GetEP 0 v35:CUInt64 = LoadField v34, :VM_ENV_DATA_INDEX_FLAGS@0x1058 @@ -4275,14 +4275,44 @@ mod hir_opt_tests { Jump bb8(v38, v38) bb7(): v40:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x105a - v41:CInt64 = GuardAnyBitSet v40, CUInt64(1) recompile - v42:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) - Jump bb8(v42, v52) + v41:CInt64[1] = Const CInt64(1) + v42:CInt64 = IntAnd v40, v41 + v43:CBool = IsBitEqual v42, v41 + CondBranch v43, bb9(), bb10() + bb9(): + v45:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) + Jump bb8(v45, v75) + bb10(): + v47:CInt64[0] = Const CInt64(0) + v48:CBool = IsBitEqual v40, v47 + CondBranch v48, bb11(), bb12() + bb11(): + v50:NilClass = Const Value(nil) + Jump bb8(v50, v75) + bb12(): + v52:CInt64[255] = Const CInt64(255) + v53:CInt64 = IntAnd v40, v52 + v54:CInt64[12] = Const CInt64(12) + v55:CBool = IsBitEqual v53, v54 + CondBranch v55, bb13(), bb14() + bb14(): + v57:CUInt64 = LoadField v40, :RBASIC_FLAGS@0x1058 + v58:CUInt64[31] = Const CUInt64(31) + v59:CInt64 = IntAnd v57, v58 + v60:CUInt64[20] = Const CUInt64(20) + v61:CBool = IsBitEqual v59, v60 + CondBranch v61, bb13(), bb15() + bb13(): + v63:BasicObject = SymToProc :block, l0, EP@3 + Jump bb8(v63, v63) + bb15(): + v65:BasicObject = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x105a + Jump bb8(v65, v75) bb8(v32:BasicObject, v33:BasicObject): - v47:BasicObject = Send v32, :call, v11, v13 # SendFallbackReason: Send: unsupported optimized method type BlockCall + v70:BasicObject = Send v32, :call, v11, v13 # SendFallbackReason: Send: unsupported optimized method type BlockCall PopInlineFrame CheckInterrupts - Return v47 + Return v70 "); } @@ -4959,7 +4989,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v24:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v25:ArrayExact = NewArray v11, v13, v15 - v57:NilClass = Const Value(nil) + v80:NilClass = Const Value(nil) PushInlineFrame :foo, v24 (0x1038), num_args=1 v37:CPtr = GetEP 0 v38:CUInt64 = LoadField v37, :VM_ENV_DATA_INDEX_FLAGS@0x1058 @@ -4970,18 +5000,48 @@ mod hir_opt_tests { Jump bb8(v41, v41) bb7(): v43:CInt64 = LoadField v37, :VM_ENV_DATA_INDEX_SPECVAL@0x105a - v44:CInt64 = GuardAnyBitSet v43, CUInt64(1) recompile - v45:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) - Jump bb8(v45, v57) + v44:CInt64[1] = Const CInt64(1) + v45:CInt64 = IntAnd v43, v44 + v46:CBool = IsBitEqual v45, v44 + CondBranch v46, bb9(), bb10() + bb9(): + v48:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) + Jump bb8(v48, v80) + bb10(): + v50:CInt64[0] = Const CInt64(0) + v51:CBool = IsBitEqual v43, v50 + CondBranch v51, bb11(), bb12() + bb11(): + v53:NilClass = Const Value(nil) + Jump bb8(v53, v80) + bb12(): + v55:CInt64[255] = Const CInt64(255) + v56:CInt64 = IntAnd v43, v55 + v57:CInt64[12] = Const CInt64(12) + v58:CBool = IsBitEqual v56, v57 + CondBranch v58, bb13(), bb14() + bb14(): + v60:CUInt64 = LoadField v43, :RBASIC_FLAGS@0x1058 + v61:CUInt64[31] = Const CUInt64(31) + v62:CInt64 = IntAnd v60, v61 + v63:CUInt64[20] = Const CUInt64(20) + v64:CBool = IsBitEqual v62, v63 + CondBranch v64, bb13(), bb15() + bb13(): + v66:BasicObject = SymToProc :block, l0, EP@3 + Jump bb8(v66, v66) + bb15(): + v68:BasicObject = LoadField v37, :VM_ENV_DATA_INDEX_SPECVAL@0x105a + Jump bb8(v68, v80) bb8(v35:BasicObject, v36:BasicObject): PatchPoint NoSingletonClass(Array@0x1068) PatchPoint MethodRedefined(Array@0x1068, length@0x1070, cme:0x1078) - v66:CInt64 = ArrayLength v25 - v67:Fixnum = BoxFixnum v66 - v52:BasicObject = Send v35, :call, v67 # SendFallbackReason: Send: unsupported optimized method type BlockCall + v89:CInt64 = ArrayLength v25 + v90:Fixnum = BoxFixnum v89 + v75:BasicObject = Send v35, :call, v90 # SendFallbackReason: Send: unsupported optimized method type BlockCall PopInlineFrame CheckInterrupts - Return v52 + Return v75 "); } @@ -6781,9 +6841,39 @@ mod hir_opt_tests { Jump bb6(v21, v21) bb5(): v23:CInt64 = LoadField v17, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v24:CInt64 = GuardAnyBitSet v23, CUInt64(1) recompile - v25:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v25, v10) + v24:CInt64[1] = Const CInt64(1) + v25:CInt64 = IntAnd v23, v24 + v26:CBool = IsBitEqual v25, v24 + CondBranch v26, bb7(), bb8() + bb7(): + v28:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v28, v10) + bb8(): + v30:CInt64[0] = Const CInt64(0) + v31:CBool = IsBitEqual v23, v30 + CondBranch v31, bb9(), bb10() + bb9(): + v33:NilClass = Const Value(nil) + Jump bb6(v33, v10) + bb10(): + v35:CInt64[255] = Const CInt64(255) + v36:CInt64 = IntAnd v23, v35 + v37:CInt64[12] = Const CInt64(12) + v38:CBool = IsBitEqual v36, v37 + CondBranch v38, bb11(), bb12() + bb12(): + v40:CUInt64 = LoadField v23, :RBASIC_FLAGS@0x1001 + v41:CUInt64[31] = Const CUInt64(31) + v42:CInt64 = IntAnd v40, v41 + v43:CUInt64[20] = Const CUInt64(20) + v44:CBool = IsBitEqual v42, v43 + CondBranch v44, bb11(), bb13() + bb11(): + v46:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v46, v46) + bb13(): + v48:BasicObject = LoadField v17, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v48, v10) bb6(v15:BasicObject, v16:BasicObject): SideExit NoProfileSend recompile "); @@ -6822,14 +6912,44 @@ mod hir_opt_tests { v22:BasicObject = LoadField v18, :block@0x1002 Jump bb6(v22, v22) bb5(): - v24:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v25:BasicObject = CCall v24, :rb_obj_is_proc@0x1004 - v26:TrueClass = GuardBitEquals v25, Value(true) recompile - Jump bb6(v24, v10) + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() + bb7(): + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) + bb8(): + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb9(), bb10() + bb9(): + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb10(): + v36:CInt64[255] = Const CInt64(255) + v37:CInt64 = IntAnd v24, v36 + v38:CInt64[12] = Const CInt64(12) + v39:CBool = IsBitEqual v37, v38 + CondBranch v39, bb11(), bb12() + bb12(): + v41:CUInt64 = LoadField v24, :RBASIC_FLAGS@0x1001 + v42:CUInt64[31] = Const CUInt64(31) + v43:CInt64 = IntAnd v41, v42 + v44:CUInt64[20] = Const CUInt64(20) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb11(), bb13() + bb11(): + v47:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v47, v47) + bb13(): + v49:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v49, v10) bb6(v16:BasicObject, v17:BasicObject): - v29:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Send: block argument is not nil + v52:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Send: block argument is not nil CheckInterrupts - Return v29 + Return v52 "); } @@ -6878,21 +6998,47 @@ mod hir_opt_tests { Jump bb8(v32, v32) bb7(): v34:CInt64 = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 - v35:CInt64[0] = GuardBitEquals v34, CInt64(0) recompile - v36:NilClass = Const Value(nil) - Jump bb8(v36, v13) + v35:CInt64[1] = Const CInt64(1) + v36:CInt64 = IntAnd v34, v35 + v37:CBool = IsBitEqual v36, v35 + CondBranch v37, bb9(), bb10() + bb9(): + v39:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb8(v39, v13) + bb10(): + v41:CInt64[0] = Const CInt64(0) + v42:CBool = IsBitEqual v34, v41 + CondBranch v42, bb11(), bb12() + bb11(): + v44:NilClass = Const Value(nil) + Jump bb8(v44, v13) + bb12(): + v46:CInt64[255] = Const CInt64(255) + v47:CInt64 = IntAnd v34, v46 + v48:CInt64[12] = Const CInt64(12) + v49:CBool = IsBitEqual v47, v48 + CondBranch v49, bb13(), bb14() + bb14(): + v51:CUInt64 = LoadField v34, :RBASIC_FLAGS@0x1002 + v52:CUInt64[31] = Const CUInt64(31) + v53:CInt64 = IntAnd v51, v52 + v54:CUInt64[20] = Const CUInt64(20) + v55:CBool = IsBitEqual v53, v54 + CondBranch v55, bb13(), bb15() + bb13(): + v57:BasicObject = SymToProc :block, l0, EP@3 + Jump bb8(v57, v57) + bb15(): + v59:BasicObject = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + Jump bb8(v59, v13) bb8(v26:BasicObject, v27:BasicObject): - v56:NilClass = GuardBitEquals v26, Value(nil) recompile - PatchPoint MethodRedefined(Integer@0x1008, then@0x1010, cme:0x1018) - PushInlineFrame :then, v24 (0x1040), num_args=0 - v76:BasicObject = InvokeBuiltin , v24 - PopInlineFrame + v62:BasicObject = Send v24, &block, :then, v26 # SendFallbackReason: Send: block argument is not nil CheckInterrupts - Return v76 + Return v62 bb4(): - v50:StaticSymbol[:skip] = Const Value(VALUE(0x1060)) + v73:StaticSymbol[:skip] = Const Value(VALUE(0x1010)) CheckInterrupts - Return v50 + Return v73 "); } @@ -6919,7 +7065,7 @@ mod hir_opt_tests { v8:BasicObject = LoadArg :block@1 Jump bb3(v7, v8) bb3(v11:BasicObject, v12:BasicObject): - v50:NilClass = Const Value(nil) + v73:NilClass = Const Value(nil) v18:CPtr = GetEP 0 v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 v20:CBool = IsBlockParamModified v19 @@ -6940,9 +7086,39 @@ mod hir_opt_tests { Jump bb9(v36, v36) bb8(): v38:CInt64 = LoadField v32, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v39:CInt64 = GuardAnyBitSet v38, CUInt64(1) recompile - v40:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb9(v40, v17) + v39:CInt64[1] = Const CInt64(1) + v40:CInt64 = IntAnd v38, v39 + v41:CBool = IsBitEqual v40, v39 + CondBranch v41, bb10(), bb11() + bb10(): + v43:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v43, v17) + bb11(): + v45:CInt64[0] = Const CInt64(0) + v46:CBool = IsBitEqual v38, v45 + CondBranch v46, bb12(), bb13() + bb12(): + v48:NilClass = Const Value(nil) + Jump bb9(v48, v17) + bb13(): + v50:CInt64[255] = Const CInt64(255) + v51:CInt64 = IntAnd v38, v50 + v52:CInt64[12] = Const CInt64(12) + v53:CBool = IsBitEqual v51, v52 + CondBranch v53, bb14(), bb15() + bb15(): + v55:CUInt64 = LoadField v38, :RBASIC_FLAGS@0x1001 + v56:CUInt64[31] = Const CUInt64(31) + v57:CInt64 = IntAnd v55, v56 + v58:CUInt64[20] = Const CUInt64(20) + v59:CBool = IsBitEqual v57, v58 + CondBranch v59, bb14(), bb16() + bb14(): + v61:BasicObject = SymToProc :block, l0, EP@4 + Jump bb9(v61, v61) + bb16(): + v63:BasicObject = LoadField v32, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb9(v63, v17) bb9(v30:BasicObject, v31:BasicObject): SideExit NoProfileSend recompile "); @@ -6969,7 +7145,7 @@ mod hir_opt_tests { v5:BasicObject = LoadArg :self@0 Jump bb3(v5) bb3(v8:BasicObject): - v45:NilClass = Const Value(nil) + v68:NilClass = Const Value(nil) v14:CPtr = GetEP 1 v15:CUInt64 = LoadField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000 v16:CBool = IsBlockParamModified v15 @@ -6990,9 +7166,39 @@ mod hir_opt_tests { Jump bb9(v31) bb8(): v33:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 - v34:CInt64 = GuardAnyBitSet v33, CUInt64(1) recompile - v35:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb9(v35) + v34:CInt64[1] = Const CInt64(1) + v35:CInt64 = IntAnd v33, v34 + v36:CBool = IsBitEqual v35, v34 + CondBranch v36, bb10(), bb11() + bb10(): + v38:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v38) + bb11(): + v40:CInt64[0] = Const CInt64(0) + v41:CBool = IsBitEqual v33, v40 + CondBranch v41, bb12(), bb13() + bb12(): + v43:NilClass = Const Value(nil) + Jump bb9(v43) + bb13(): + v45:CInt64[255] = Const CInt64(255) + v46:CInt64 = IntAnd v33, v45 + v47:CInt64[12] = Const CInt64(12) + v48:CBool = IsBitEqual v46, v47 + CondBranch v48, bb14(), bb15() + bb15(): + v50:CUInt64 = LoadField v33, :RBASIC_FLAGS@0x1000 + v51:CUInt64[31] = Const CUInt64(31) + v52:CInt64 = IntAnd v50, v51 + v53:CUInt64[20] = Const CUInt64(20) + v54:CBool = IsBitEqual v52, v53 + CondBranch v54, bb14(), bb16() + bb14(): + v56:BasicObject = SymToProc :block, l1, EP@3 + Jump bb9(v56) + bb16(): + v58:BasicObject = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 + Jump bb9(v58) bb9(v26:BasicObject): SideExit NoProfileSend recompile "); @@ -7037,23 +7243,40 @@ mod hir_opt_tests { v25:CInt64[1] = Const CInt64(1) v26:CInt64 = IntAnd v24, v25 v27:CBool = IsBitEqual v26, v25 - CondBranch v27, bb7(), bb9() + CondBranch v27, bb7(), bb8() bb7(): v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) Jump bb6(v29, v10) - bb9(): + bb8(): v31:CInt64[0] = Const CInt64(0) v32:CBool = IsBitEqual v24, v31 - CondBranch v32, bb8(), bb10() - bb8(): + CondBranch v32, bb9(), bb10() + bb9(): v34:NilClass = Const Value(nil) Jump bb6(v34, v10) + bb10(): + v36:CInt64[255] = Const CInt64(255) + v37:CInt64 = IntAnd v24, v36 + v38:CInt64[12] = Const CInt64(12) + v39:CBool = IsBitEqual v37, v38 + CondBranch v39, bb11(), bb12() + bb12(): + v41:CUInt64 = LoadField v24, :RBASIC_FLAGS@0x1001 + v42:CUInt64[31] = Const CUInt64(31) + v43:CInt64 = IntAnd v41, v42 + v44:CUInt64[20] = Const CUInt64(20) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb11(), bb13() + bb11(): + v47:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v47, v47) + bb13(): + v49:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v49, v10) bb6(v16:BasicObject, v17:BasicObject): - v38:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Send: block argument is not nil + v52:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Send: block argument is not nil CheckInterrupts - Return v38 - bb10(): - SideExit BlockParamProxyProfileNotCovered + Return v52 "); } @@ -7094,34 +7317,43 @@ mod hir_opt_tests { Jump bb6(v22, v22) bb5(): v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v26:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v27:BasicObject = CCall v26, :rb_obj_is_proc@0x1004 - v28:TrueClass = Const Value(true) - v29:CBool = IsBitEqual v27, v28 - CondBranch v29, bb7(), bb11() + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() bb7(): - Jump bb6(v26, v10) - bb11(): - v32:CInt64[0] = Const CInt64(0) - v33:CBool = IsBitEqual v24, v32 - CondBranch v33, bb8(), bb12() + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) bb8(): - v35:NilClass = Const Value(nil) - Jump bb6(v35, v10) - bb12(): - v37:CInt64[1] = Const CInt64(1) - v38:CInt64 = IntAnd v24, v37 - v39:CBool = IsBitEqual v38, v37 - CondBranch v39, bb9(), bb13() + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb9(), bb10() bb9(): - v41:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v41, v10) + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb10(): + v36:CInt64[255] = Const CInt64(255) + v37:CInt64 = IntAnd v24, v36 + v38:CInt64[12] = Const CInt64(12) + v39:CBool = IsBitEqual v37, v38 + CondBranch v39, bb11(), bb12() + bb12(): + v41:CUInt64 = LoadField v24, :RBASIC_FLAGS@0x1001 + v42:CUInt64[31] = Const CUInt64(31) + v43:CInt64 = IntAnd v41, v42 + v44:CUInt64[20] = Const CUInt64(20) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb11(), bb13() + bb11(): + v47:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v47, v47) + bb13(): + v49:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v49, v10) bb6(v16:BasicObject, v17:BasicObject): - v45:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Send: block argument is not nil + v52:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Send: block argument is not nil CheckInterrupts - Return v45 - bb13(): - SideExit BlockParamProxyProfileNotCovered + Return v52 "); } @@ -9794,7 +10026,7 @@ mod hir_opt_tests { v60:TrueClass = Const Value(true) Jump bb7(v60) bb11(): - v48:BasicObject = Send v31, :! # SendFallbackReason: Send: polymorphic call site + v48:BasicObject = Send v31, :! # SendFallbackReason: Send: polymorphic fallback Jump bb7(v48) bb7(v35:BasicObject): CheckInterrupts @@ -10477,6 +10709,7 @@ mod hir_opt_tests { #[test] fn test_setivar_shape_guard_recompile() { set_max_versions(2); + set_num_exits_until_invalidate(1); // Call with one shape to compile, then call with a different shape to // trigger shape guard exits and recompilation. The recompiled version // specializes both profiled shapes. @@ -10580,7 +10813,7 @@ mod hir_opt_tests { SetIvar v24, :@foo, v17 Jump bb4(v17) bb6(): - v27:BasicObject = Send v10, :foo=, v17 # SendFallbackReason: Send: polymorphic call site + v27:BasicObject = Send v10, :foo=, v17 # SendFallbackReason: Send: polymorphic fallback Jump bb4(v27) bb4(v20:BasicObject): CheckInterrupts @@ -11247,7 +11480,7 @@ mod hir_opt_tests { v31:BasicObject = GetIvar v19, :@foo Jump bb4(v31) bb6(): - v22:BasicObject = Send v10, :foo # SendFallbackReason: Send: polymorphic call site + v22:BasicObject = Send v10, :foo # SendFallbackReason: Send: polymorphic fallback Jump bb4(v22) bb4(v15:BasicObject): CheckInterrupts @@ -11392,13 +11625,43 @@ mod hir_opt_tests { Jump bb6(v22, v22) bb5(): v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v25:CInt64 = GuardAnyBitSet v24, CUInt64(1) recompile - v26:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v26, v10) + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() + bb7(): + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) + bb8(): + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb9(), bb10() + bb9(): + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb10(): + v36:CInt64[255] = Const CInt64(255) + v37:CInt64 = IntAnd v24, v36 + v38:CInt64[12] = Const CInt64(12) + v39:CBool = IsBitEqual v37, v38 + CondBranch v39, bb11(), bb12() + bb12(): + v41:CUInt64 = LoadField v24, :RBASIC_FLAGS@0x1001 + v42:CUInt64[31] = Const CUInt64(31) + v43:CInt64 = IntAnd v41, v42 + v44:CUInt64[20] = Const CUInt64(20) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb11(), bb13() + bb11(): + v47:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v47, v47) + bb13(): + v49:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v49, v10) bb6(v16:BasicObject, v17:BasicObject): - v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Send: block argument is not nil + v52:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Send: block argument is not nil CheckInterrupts - Return v29 + Return v52 "); } @@ -11432,16 +11695,46 @@ mod hir_opt_tests { Jump bb6(v22, v22) bb5(): v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v25:CInt64[0] = GuardBitEquals v24, CInt64(0) recompile - v26:NilClass = Const Value(nil) - Jump bb6(v26, v10) + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() + bb7(): + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) + bb8(): + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb9(), bb10() + bb9(): + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb10(): + v36:CInt64[255] = Const CInt64(255) + v37:CInt64 = IntAnd v24, v36 + v38:CInt64[12] = Const CInt64(12) + v39:CBool = IsBitEqual v37, v38 + CondBranch v39, bb11(), bb12() + bb12(): + v41:CUInt64 = LoadField v24, :RBASIC_FLAGS@0x1001 + v42:CUInt64[31] = Const CUInt64(31) + v43:CInt64 = IntAnd v41, v42 + v44:CUInt64[20] = Const CUInt64(20) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb11(), bb13() + bb11(): + v47:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v47, v47) + bb13(): + v49:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v49, v10) bb6(v16:BasicObject, v17:BasicObject): - v35:NilClass = GuardBitEquals v16, Value(nil) recompile - PatchPoint NoSingletonClass(Array@0x1008) - PatchPoint MethodRedefined(Array@0x1008, map@0x1010, cme:0x1018) - v40:BasicObject = SendDirect v14, 0x0, :map (0x1040) + v58:NilClass = GuardBitEquals v16, Value(nil) recompile + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, map@0x1018, cme:0x1020) + v63:BasicObject = SendDirect v14, 0x0, :map (0x1048) CheckInterrupts - Return v40 + Return v63 "); } @@ -11476,13 +11769,43 @@ mod hir_opt_tests { Jump bb6(v17) bb5(): v19:CInt64 = LoadField v13, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 - v20:CInt64 = GuardAnyBitSet v19, CUInt64(1) recompile - v21:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v21) + v20:CInt64[1] = Const CInt64(1) + v21:CInt64 = IntAnd v19, v20 + v22:CBool = IsBitEqual v21, v20 + CondBranch v22, bb7(), bb8() + bb7(): + v24:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v24) + bb8(): + v26:CInt64[0] = Const CInt64(0) + v27:CBool = IsBitEqual v19, v26 + CondBranch v27, bb9(), bb10() + bb9(): + v29:NilClass = Const Value(nil) + Jump bb6(v29) + bb10(): + v31:CInt64[255] = Const CInt64(255) + v32:CInt64 = IntAnd v19, v31 + v33:CInt64[12] = Const CInt64(12) + v34:CBool = IsBitEqual v32, v33 + CondBranch v34, bb11(), bb12() + bb12(): + v36:CUInt64 = LoadField v19, :RBASIC_FLAGS@0x1000 + v37:CUInt64[31] = Const CUInt64(31) + v38:CInt64 = IntAnd v36, v37 + v39:CUInt64[20] = Const CUInt64(20) + v40:CBool = IsBitEqual v38, v39 + CondBranch v40, bb11(), bb13() + bb11(): + v42:BasicObject = SymToProc :block, l1, EP@3 + Jump bb6(v42) + bb13(): + v44:BasicObject = LoadField v13, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 + Jump bb6(v44) bb6(v12:BasicObject): - v24:BasicObject = Send v10, &block, :map, v12 # SendFallbackReason: Send: block argument is not nil + v47:BasicObject = Send v10, &block, :map, v12 # SendFallbackReason: Send: block argument is not nil CheckInterrupts - Return v24 + Return v47 "); } @@ -11548,7 +11871,7 @@ mod hir_opt_tests { bb3(v6:BasicObject): PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v71:NilClass = Const Value(nil) + v117:NilClass = Const Value(nil) PushInlineFrame :foo, v18 (0x1038), num_args=0 v28:CPtr = GetEP 0 v29:CUInt64 = LoadField v28, :VM_ENV_DATA_INDEX_FLAGS@0x1058 @@ -11559,37 +11882,97 @@ mod hir_opt_tests { Jump bb9(v32, v32) bb8(): v34:CInt64 = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x105a - v35:CInt64[0] = GuardBitEquals v34, CInt64(0) recompile - v36:NilClass = Const Value(nil) - Jump bb9(v36, v71) - bb9(v26:BasicObject, v27:BasicObject): - v39:CBool = Test v26 - CondBranch v39, bb10(), bb6() + v35:CInt64[1] = Const CInt64(1) + v36:CInt64 = IntAnd v34, v35 + v37:CBool = IsBitEqual v36, v35 + CondBranch v37, bb10(), bb11() bb10(): - v46:CPtr = GetEP 0 - v47:CUInt64 = LoadField v46, :VM_ENV_DATA_INDEX_FLAGS@0x1058 - v48:CBool = IsBlockParamModified v47 - CondBranch v48, bb11(), bb12() + v39:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) + Jump bb9(v39, v117) bb11(): - v50:BasicObject = LoadField v46, :blk@0x1059 - Jump bb13(v50, v50) + v41:CInt64[0] = Const CInt64(0) + v42:CBool = IsBitEqual v34, v41 + CondBranch v42, bb12(), bb13() bb12(): - v52:CInt64 = LoadField v46, :VM_ENV_DATA_INDEX_SPECVAL@0x105a - v53:CInt64 = GuardAnyBitSet v52, CUInt64(1) recompile - v54:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) - Jump bb13(v54, v27) - bb13(v44:BasicObject, v45:BasicObject): - v57:BasicObject = Send v44, :call # SendFallbackReason: Send: no profile data available - CheckInterrupts - Jump bb4(v57) + v44:NilClass = Const Value(nil) + Jump bb9(v44, v117) + bb13(): + v46:CInt64[255] = Const CInt64(255) + v47:CInt64 = IntAnd v34, v46 + v48:CInt64[12] = Const CInt64(12) + v49:CBool = IsBitEqual v47, v48 + CondBranch v49, bb14(), bb15() + bb15(): + v51:CUInt64 = LoadField v34, :RBASIC_FLAGS@0x1058 + v52:CUInt64[31] = Const CUInt64(31) + v53:CInt64 = IntAnd v51, v52 + v54:CUInt64[20] = Const CUInt64(20) + v55:CBool = IsBitEqual v53, v54 + CondBranch v55, bb14(), bb16() + bb14(): + v57:BasicObject = SymToProc :blk, l0, EP@3 + Jump bb9(v57, v57) + bb16(): + v59:BasicObject = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x105a + Jump bb9(v59, v117) + bb9(v26:BasicObject, v27:BasicObject): + v62:CBool = Test v26 + CondBranch v62, bb17(), bb6() + bb17(): + v69:CPtr = GetEP 0 + v70:CUInt64 = LoadField v69, :VM_ENV_DATA_INDEX_FLAGS@0x1058 + v71:CBool = IsBlockParamModified v70 + CondBranch v71, bb18(), bb19() + bb18(): + v73:BasicObject = LoadField v69, :blk@0x1059 + Jump bb20(v73, v73) + bb19(): + v75:CInt64 = LoadField v69, :VM_ENV_DATA_INDEX_SPECVAL@0x105a + v76:CInt64[1] = Const CInt64(1) + v77:CInt64 = IntAnd v75, v76 + v78:CBool = IsBitEqual v77, v76 + CondBranch v78, bb21(), bb22() + bb21(): + v80:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) + Jump bb20(v80, v27) + bb22(): + v82:CInt64[0] = Const CInt64(0) + v83:CBool = IsBitEqual v75, v82 + CondBranch v83, bb23(), bb24() + bb23(): + v85:NilClass = Const Value(nil) + Jump bb20(v85, v27) + bb24(): + v87:CInt64[255] = Const CInt64(255) + v88:CInt64 = IntAnd v75, v87 + v89:CInt64[12] = Const CInt64(12) + v90:CBool = IsBitEqual v88, v89 + CondBranch v90, bb25(), bb26() + bb26(): + v92:CUInt64 = LoadField v75, :RBASIC_FLAGS@0x1058 + v93:CUInt64[31] = Const CUInt64(31) + v94:CInt64 = IntAnd v92, v93 + v95:CUInt64[20] = Const CUInt64(20) + v96:CBool = IsBitEqual v94, v95 + CondBranch v96, bb25(), bb27() + bb25(): + v98:BasicObject = SymToProc :blk, l0, EP@3 + Jump bb20(v98, v98) + bb27(): + v100:BasicObject = LoadField v69, :VM_ENV_DATA_INDEX_SPECVAL@0x105a + Jump bb20(v100, v27) + bb20(v67:BasicObject, v68:BasicObject): + v103:BasicObject = Send v67, :call # SendFallbackReason: Send: no profile data available + CheckInterrupts + Jump bb4(v103) bb6(): - v66:Fixnum[42] = Const Value(42) + v112:Fixnum[42] = Const Value(42) CheckInterrupts - Jump bb4(v66) - bb4(v72:BasicObject): + Jump bb4(v112) + bb4(v118:BasicObject): PopInlineFrame CheckInterrupts - Return v72 + Return v118 "); } @@ -11719,16 +12102,46 @@ mod hir_opt_tests { Jump bb6(v21, v21) bb5(): v23:CInt64 = LoadField v17, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v24:CInt64[0] = GuardBitEquals v23, CInt64(0) recompile - v25:NilClass = Const Value(nil) - Jump bb6(v25, v10) + v24:CInt64[1] = Const CInt64(1) + v25:CInt64 = IntAnd v23, v24 + v26:CBool = IsBitEqual v25, v24 + CondBranch v26, bb7(), bb8() + bb7(): + v28:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v28, v10) + bb8(): + v30:CInt64[0] = Const CInt64(0) + v31:CBool = IsBitEqual v23, v30 + CondBranch v31, bb9(), bb10() + bb9(): + v33:NilClass = Const Value(nil) + Jump bb6(v33, v10) + bb10(): + v35:CInt64[255] = Const CInt64(255) + v36:CInt64 = IntAnd v23, v35 + v37:CInt64[12] = Const CInt64(12) + v38:CBool = IsBitEqual v36, v37 + CondBranch v38, bb11(), bb12() + bb12(): + v40:CUInt64 = LoadField v23, :RBASIC_FLAGS@0x1001 + v41:CUInt64[31] = Const CUInt64(31) + v42:CInt64 = IntAnd v40, v41 + v43:CUInt64[20] = Const CUInt64(20) + v44:CBool = IsBitEqual v42, v43 + CondBranch v44, bb11(), bb13() + bb11(): + v46:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v46, v46) + bb13(): + v48:BasicObject = LoadField v17, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v48, v10) bb6(v15:BasicObject, v16:BasicObject): - v34:NilClass = GuardBitEquals v15, Value(nil) recompile - PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v37:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v38:Fixnum[42] = Const Value(42) + v57:NilClass = GuardBitEquals v15, Value(nil) recompile + PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) + v60:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] recompile + v61:Fixnum[42] = Const Value(42) CheckInterrupts - Return v38 + Return v61 "); } @@ -15268,7 +15681,7 @@ mod hir_opt_tests { PopInlineFrame Jump bb4(v60) bb8(): - v34:BasicObject = Send v12, :target, v19 # SendFallbackReason: Send: polymorphic call site + v34:BasicObject = Send v12, :target, v19 # SendFallbackReason: Send: polymorphic fallback Jump bb4(v34) bb4(v21:BasicObject): CheckInterrupts @@ -15986,13 +16399,43 @@ mod hir_opt_tests { Jump bb6(v22, v22) bb5(): v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v25:CInt64 = GuardAnyBitSet v24, CUInt64(1) recompile - v26:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v26, v10) + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() + bb7(): + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) + bb8(): + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb9(), bb10() + bb9(): + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb10(): + v36:CInt64[255] = Const CInt64(255) + v37:CInt64 = IntAnd v24, v36 + v38:CInt64[12] = Const CInt64(12) + v39:CBool = IsBitEqual v37, v38 + CondBranch v39, bb11(), bb12() + bb12(): + v41:CUInt64 = LoadField v24, :RBASIC_FLAGS@0x1001 + v42:CUInt64[31] = Const CUInt64(31) + v43:CInt64 = IntAnd v41, v42 + v44:CUInt64[20] = Const CUInt64(20) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb11(), bb13() + bb11(): + v47:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v47, v47) + bb13(): + v49:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v49, v10) bb6(v16:BasicObject, v17:BasicObject): - v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Send: block argument is not nil + v52:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Send: block argument is not nil CheckInterrupts - Return v29 + Return v52 "); } @@ -16953,7 +17396,7 @@ mod hir_opt_tests { v46:TrueClass = Const Value(true) Jump bb4(v46) bb8(): - v31:BasicObject = Send v10, :is_a?, v16 # SendFallbackReason: Send: polymorphic call site + v31:BasicObject = Send v10, :is_a?, v16 # SendFallbackReason: Send: polymorphic fallback Jump bb4(v31) bb4(v18:BasicObject): CheckInterrupts @@ -19281,7 +19724,7 @@ mod hir_opt_tests { v45:Fixnum[4] = Const Value(4) Jump bb4(v45) bb8(): - v28:BasicObject = Send v10, :foo # SendFallbackReason: Send: polymorphic call site + v28:BasicObject = Send v10, :foo # SendFallbackReason: Send: polymorphic fallback Jump bb4(v28) bb4(v15:BasicObject): v31:Fixnum[2] = Const Value(2) @@ -19334,7 +19777,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Integer@0x1040, itself@0x1010, cme:0x1018) Jump bb4(v25) bb8(): - v28:BasicObject = Send v10, :itself # SendFallbackReason: Send: polymorphic call site + v28:BasicObject = Send v10, :itself # SendFallbackReason: Send: polymorphic fallback Jump bb4(v28) bb4(v15:BasicObject): CheckInterrupts @@ -19440,7 +19883,7 @@ mod hir_opt_tests { v54:BasicObject = HashAref v30, v13 Jump bb4(v54) bb8(): - v33:BasicObject = Send v12, :[], v13 # SendFallbackReason: Send: polymorphic call site + v33:BasicObject = Send v12, :[], v13 # SendFallbackReason: Send: polymorphic fallback Jump bb4(v33) bb4(v20:BasicObject): CheckInterrupts @@ -19496,7 +19939,7 @@ mod hir_opt_tests { v40:StringExact = CCallVariadic v25, :Integer#to_s@0x1040 Jump bb4(v40) bb8(): - v28:BasicObject = Send v10, :to_s # SendFallbackReason: Send: polymorphic call site + v28:BasicObject = Send v10, :to_s # SendFallbackReason: Send: polymorphic fallback Jump bb4(v28) bb4(v15:BasicObject): CheckInterrupts @@ -19549,7 +19992,7 @@ mod hir_opt_tests { v40:BasicObject = CCallWithFrame v25, :Float#to_s@0x1040 Jump bb4(v40) bb8(): - v28:BasicObject = Send v10, :to_s # SendFallbackReason: Send: polymorphic call site + v28:BasicObject = Send v10, :to_s # SendFallbackReason: Send: polymorphic fallback Jump bb4(v28) bb4(v15:BasicObject): CheckInterrupts @@ -19602,7 +20045,7 @@ mod hir_opt_tests { v38:StringExact = InvokeBuiltin leaf , v25 Jump bb4(v38) bb8(): - v28:BasicObject = Send v10, :to_s # SendFallbackReason: Send: polymorphic call site + v28:BasicObject = Send v10, :to_s # SendFallbackReason: Send: polymorphic fallback Jump bb4(v28) bb4(v15:BasicObject): CheckInterrupts @@ -19651,7 +20094,7 @@ mod hir_opt_tests { v31:Fixnum[3] = Const Value(3) Jump bb4(v31) bb6(): - v22:BasicObject = Send v10, :foo # SendFallbackReason: Send: polymorphic call site + v22:BasicObject = Send v10, :foo # SendFallbackReason: Send: polymorphic fallback Jump bb4(v22) bb4(v15:BasicObject): CheckInterrupts @@ -21177,7 +21620,7 @@ mod hir_opt_tests { v46:Float = FloatMul v30, v45 Jump bb4(v46) bb8(): - v33:BasicObject = Send v12, :*, v13 # SendFallbackReason: Send: polymorphic call site + v33:BasicObject = Send v12, :*, v13 # SendFallbackReason: Send: polymorphic fallback Jump bb4(v33) bb4(v20:BasicObject): CheckInterrupts @@ -22782,7 +23225,7 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): PatchPoint MethodRedefined(Object@0x1008, with_block_param@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v53:NilClass = Const Value(nil) + v76:NilClass = Const Value(nil) PushInlineFrame :with_block_param, v25 (0x1040), num_args=1 v36:CPtr = GetEP 0 v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1060 @@ -22793,15 +23236,45 @@ mod hir_opt_tests { Jump bb8(v40, v40) bb7(): v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 - v43:CInt64 = GuardAnyBitSet v42, CUInt64(1) recompile - v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) - Jump bb8(v44, v53) + v43:CInt64[1] = Const CInt64(1) + v44:CInt64 = IntAnd v42, v43 + v45:CBool = IsBitEqual v44, v43 + CondBranch v45, bb9(), bb10() + bb9(): + v47:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) + Jump bb8(v47, v76) + bb10(): + v49:CInt64[0] = Const CInt64(0) + v50:CBool = IsBitEqual v42, v49 + CondBranch v50, bb11(), bb12() + bb11(): + v52:NilClass = Const Value(nil) + Jump bb8(v52, v76) + bb12(): + v54:CInt64[255] = Const CInt64(255) + v55:CInt64 = IntAnd v42, v54 + v56:CInt64[12] = Const CInt64(12) + v57:CBool = IsBitEqual v55, v56 + CondBranch v57, bb13(), bb14() + bb14(): + v59:CUInt64 = LoadField v42, :RBASIC_FLAGS@0x1060 + v60:CUInt64[31] = Const CUInt64(31) + v61:CInt64 = IntAnd v59, v60 + v62:CUInt64[20] = Const CUInt64(20) + v63:CBool = IsBitEqual v61, v62 + CondBranch v63, bb13(), bb15() + bb13(): + v65:BasicObject = SymToProc :block, l0, EP@3 + Jump bb8(v65, v65) + bb15(): + v67:BasicObject = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 + Jump bb8(v67, v76) bb8(v34:BasicObject, v35:BasicObject): - v48:BasicObject = Send v34, :call, v10 # SendFallbackReason: Send: unsupported optimized method type BlockCall + v71:BasicObject = Send v34, :call, v10 # SendFallbackReason: Send: unsupported optimized method type BlockCall PopInlineFrame PatchPoint NoEPEscape(test) CheckInterrupts - Return v48 + Return v71 "); } @@ -22846,7 +23319,7 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): PatchPoint MethodRedefined(Object@0x1008, callee@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v54:NilClass = Const Value(nil) + v77:NilClass = Const Value(nil) PushInlineFrame :callee, v25 (0x1040), num_args=1 v38:CPtr = GetEP 0 v39:CUInt64 = LoadField v38, :VM_ENV_DATA_INDEX_FLAGS@0x1060 @@ -22857,15 +23330,45 @@ mod hir_opt_tests { Jump bb8(v42, v42) bb7(): v44:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 - v45:CInt64 = GuardAnyBitSet v44, CUInt64(1) recompile - v46:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) - Jump bb8(v46, v54) + v45:CInt64[1] = Const CInt64(1) + v46:CInt64 = IntAnd v44, v45 + v47:CBool = IsBitEqual v46, v45 + CondBranch v47, bb9(), bb10() + bb9(): + v49:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) + Jump bb8(v49, v77) + bb10(): + v51:CInt64[0] = Const CInt64(0) + v52:CBool = IsBitEqual v44, v51 + CondBranch v52, bb11(), bb12() + bb11(): + v54:NilClass = Const Value(nil) + Jump bb8(v54, v77) + bb12(): + v56:CInt64[255] = Const CInt64(255) + v57:CInt64 = IntAnd v44, v56 + v58:CInt64[12] = Const CInt64(12) + v59:CBool = IsBitEqual v57, v58 + CondBranch v59, bb13(), bb14() + bb14(): + v61:CUInt64 = LoadField v44, :RBASIC_FLAGS@0x1060 + v62:CUInt64[31] = Const CUInt64(31) + v63:CInt64 = IntAnd v61, v62 + v64:CUInt64[20] = Const CUInt64(20) + v65:CBool = IsBitEqual v63, v64 + CondBranch v65, bb13(), bb15() + bb13(): + v67:BasicObject = SymToProc :block, l0, EP@3 + Jump bb8(v67, v67) + bb15(): + v69:BasicObject = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 + Jump bb8(v69, v77) bb8(v36:BasicObject, v37:BasicObject): - v49:BasicObject = Send v25, &block, :inner, v10, v36 # SendFallbackReason: Send: block argument is not nil + v72:BasicObject = Send v25, &block, :inner, v10, v36 # SendFallbackReason: Send: block argument is not nil PopInlineFrame PatchPoint NoEPEscape(test) CheckInterrupts - Return v49 + Return v72 "); } @@ -23328,4 +23831,162 @@ mod hir_opt_tests { Return v37 "); } + + #[test] + fn test_specialize_polymorphic_send_with_block() { + set_call_threshold(4); + eval(r#" + class A + def foo = yield + end + class B < A; end + class C < A; end + def test(obj) + obj.foo { } + end + test(A.new) + test(B.new) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@:8: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :obj@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :obj@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:CBool = HasType v10, ObjectSubclass[class_exact:C] + CondBranch v16, bb5(), bb6() + bb5(): + v19:ObjectSubclass[class_exact:C] = RefineType v10, ObjectSubclass[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + PushInlineFrame :foo, v19 (0x1040), num_args=0 + v57:CPtr = GetEP 0 + v58:CInt64 = LoadField v57, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v59:CInt64[-4] = Const CInt64(-4) + v60:CInt64 = IntAnd v58, v59 + v61:BasicObject = InvokeBlockIseqDirect (0x1068), v60 + CheckInterrupts + PopInlineFrame + Jump bb4(v61) + bb6(): + v22:CBool = HasType v10, ObjectSubclass[class_exact:A] + CondBranch v22, bb7(), bb8() + bb7(): + v25:ObjectSubclass[class_exact:A] = RefineType v10, ObjectSubclass[class_exact:A] + PatchPoint NoSingletonClass(A@0x1088) + PatchPoint MethodRedefined(A@0x1088, foo@0x1010, cme:0x1018) + PushInlineFrame :foo, v25 (0x1040), num_args=0 + v75:CPtr = GetEP 0 + v76:CInt64 = LoadField v75, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v77:CInt64[-4] = Const CInt64(-4) + v78:CInt64 = IntAnd v76, v77 + v79:BasicObject = InvokeBlockIseqDirect (0x1068), v78 + CheckInterrupts + PopInlineFrame + Jump bb4(v79) + bb8(): + v28:CBool = HasType v10, ObjectSubclass[class_exact:B] + CondBranch v28, bb9(), bb10() + bb9(): + v31:ObjectSubclass[class_exact:B] = RefineType v10, ObjectSubclass[class_exact:B] + PatchPoint NoSingletonClass(B@0x1090) + PatchPoint MethodRedefined(B@0x1090, foo@0x1010, cme:0x1018) + PushInlineFrame :foo, v31 (0x1040), num_args=0 + v93:CPtr = GetEP 0 + v94:CInt64 = LoadField v93, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v95:CInt64[-4] = Const CInt64(-4) + v96:CInt64 = IntAnd v94, v95 + v97:BasicObject = InvokeBlockIseqDirect (0x1068), v96 + CheckInterrupts + PopInlineFrame + Jump bb4(v97) + bb10(): + v34:BasicObject = Send v10, 0x1068, :foo # SendFallbackReason: Send: polymorphic fallback + Jump bb4(v34) + bb4(v15:BasicObject): + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_specialize_polymorphic_nil_block() { + set_call_threshold(3); + eval(r#" + class A + def foo(&blk) = 42 + end + class B + def foo(&blk) = 43 + end + def test(obj, &blk) + obj.foo(&blk) + end + + test(A.new); test(B.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :obj@0x1000 + v4:BasicObject = LoadField v2, :blk@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :obj@1 + v9:BasicObject = LoadArg :blk@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:CPtr = GetEP 0 + v21:CUInt64 = LoadField v20, :VM_ENV_DATA_INDEX_FLAGS@0x1002 + v22:CBool = IsBlockParamModified v21 + CondBranch v22, bb4(), bb5() + bb4(): + v24:BasicObject = LoadField v20, :blk@0x1003 + Jump bb6(v24, v24) + bb5(): + v26:CInt64 = LoadField v20, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v27:CInt64[0] = GuardBitEquals v26, CInt64(0) recompile + v28:NilClass = Const Value(nil) + Jump bb6(v28, v13) + bb6(v18:BasicObject, v19:BasicObject): + v32:CBool = HasType v12, ObjectSubclass[class_exact:B] + CondBranch v32, bb8(), bb9() + bb8(): + v51:NilClass = GuardBitEquals v18, Value(nil) recompile + PatchPoint NoSingletonClass(B@0x1008) + PatchPoint MethodRedefined(B@0x1008, foo@0x1010, cme:0x1018) + v55:Fixnum[43] = Const Value(43) + Jump bb7(v55) + bb9(): + v38:CBool = HasType v12, ObjectSubclass[class_exact:A] + CondBranch v38, bb10(), bb11() + bb10(): + v56:NilClass = GuardBitEquals v18, Value(nil) recompile + PatchPoint NoSingletonClass(A@0x1040) + PatchPoint MethodRedefined(A@0x1040, foo@0x1010, cme:0x1048) + v60:Fixnum[42] = Const Value(42) + Jump bb7(v60) + bb11(): + v44:BasicObject = Send v12, &block, :foo, v18 # SendFallbackReason: Send: polymorphic fallback + Jump bb7(v44) + bb7(v31:BasicObject): + CheckInterrupts + Return v31 + "); + } } diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index 53c9f669000310..2894c25f5f299c 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -2303,13 +2303,43 @@ pub(crate) mod hir_build_tests { Jump bb6(v28, v28) bb5(): v30:CInt64 = LoadField v24, :VM_ENV_DATA_INDEX_SPECVAL@0x102a - v31:CInt64 = GuardAnyBitSet v30, CUInt64(1) recompile - v32:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1030)) - Jump bb6(v32, v10) + v31:CInt64[1] = Const CInt64(1) + v32:CInt64 = IntAnd v30, v31 + v33:CBool = IsBitEqual v32, v31 + CondBranch v33, bb7(), bb8() + bb7(): + v35:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1030)) + Jump bb6(v35, v10) + bb8(): + v37:CInt64[0] = Const CInt64(0) + v38:CBool = IsBitEqual v30, v37 + CondBranch v38, bb9(), bb10() + bb9(): + v40:NilClass = Const Value(nil) + Jump bb6(v40, v10) + bb10(): + v42:CInt64[255] = Const CInt64(255) + v43:CInt64 = IntAnd v30, v42 + v44:CInt64[12] = Const CInt64(12) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb11(), bb12() + bb12(): + v47:CUInt64 = LoadField v30, :RBASIC_FLAGS@0x1028 + v48:CUInt64[31] = Const CUInt64(31) + v49:CInt64 = IntAnd v47, v48 + v50:CUInt64[20] = Const CUInt64(20) + v51:CBool = IsBitEqual v49, v50 + CondBranch v51, bb11(), bb13() + bb11(): + v53:BasicObject = SymToProc :&, l0, EP@3 + Jump bb6(v53, v53) + bb13(): + v55:BasicObject = LoadField v24, :VM_ENV_DATA_INDEX_SPECVAL@0x102a + Jump bb6(v55, v10) bb6(v22:BasicObject, v23:BasicObject): - v35:BasicObject = Send v9, &block, :consume, v22 # SendFallbackReason: Uncategorized(send) + v58:BasicObject = Send v9, &block, :consume, v22 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v35 + Return v58 "); } @@ -2857,9 +2887,39 @@ pub(crate) mod hir_build_tests { Jump bb6(v40, v40) bb5(): v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 - v43:CInt64 = GuardAnyBitSet v42, CUInt64(1) recompile - v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v44, v21) + v43:CInt64[1] = Const CInt64(1) + v44:CInt64 = IntAnd v42, v43 + v45:CBool = IsBitEqual v44, v43 + CondBranch v45, bb7(), bb8() + bb7(): + v47:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v47, v21) + bb8(): + v49:CInt64[0] = Const CInt64(0) + v50:CBool = IsBitEqual v42, v49 + CondBranch v50, bb9(), bb10() + bb9(): + v52:NilClass = Const Value(nil) + Jump bb6(v52, v21) + bb10(): + v54:CInt64[255] = Const CInt64(255) + v55:CInt64 = IntAnd v42, v54 + v56:CInt64[12] = Const CInt64(12) + v57:CBool = IsBitEqual v55, v56 + CondBranch v57, bb11(), bb12() + bb12(): + v59:CUInt64 = LoadField v42, :RBASIC_FLAGS@0x1004 + v60:CUInt64[31] = Const CUInt64(31) + v61:CInt64 = IntAnd v59, v60 + v62:CUInt64[20] = Const CUInt64(20) + v63:CBool = IsBitEqual v61, v62 + CondBranch v63, bb11(), bb13() + bb11(): + v65:BasicObject = SymToProc :&, l0, EP@4 + Jump bb6(v65, v65) + bb13(): + v67:BasicObject = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + Jump bb6(v67, v21) bb6(v34:BasicObject, v35:BasicObject): SideExit SplatKwNotProfiled "); @@ -3819,13 +3879,43 @@ pub(crate) mod hir_build_tests { Jump bb6(v21, v21) bb5(): v23:CInt64 = LoadField v17, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v24:CInt64 = GuardAnyBitSet v23, CUInt64(1) recompile - v25:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v25, v10) + v24:CInt64[1] = Const CInt64(1) + v25:CInt64 = IntAnd v23, v24 + v26:CBool = IsBitEqual v25, v24 + CondBranch v26, bb7(), bb8() + bb7(): + v28:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v28, v10) + bb8(): + v30:CInt64[0] = Const CInt64(0) + v31:CBool = IsBitEqual v23, v30 + CondBranch v31, bb9(), bb10() + bb9(): + v33:NilClass = Const Value(nil) + Jump bb6(v33, v10) + bb10(): + v35:CInt64[255] = Const CInt64(255) + v36:CInt64 = IntAnd v23, v35 + v37:CInt64[12] = Const CInt64(12) + v38:CBool = IsBitEqual v36, v37 + CondBranch v38, bb11(), bb12() + bb12(): + v40:CUInt64 = LoadField v23, :RBASIC_FLAGS@0x1001 + v41:CUInt64[31] = Const CUInt64(31) + v42:CInt64 = IntAnd v40, v41 + v43:CUInt64[20] = Const CUInt64(20) + v44:CBool = IsBitEqual v42, v43 + CondBranch v44, bb11(), bb13() + bb11(): + v46:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v46, v46) + bb13(): + v48:BasicObject = LoadField v17, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v48, v10) bb6(v15:BasicObject, v16:BasicObject): - v28:BasicObject = Send v9, &block, :tap, v15 # SendFallbackReason: Uncategorized(send) + v51:BasicObject = Send v9, &block, :tap, v15 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v28 + Return v51 "); } @@ -3862,14 +3952,44 @@ pub(crate) mod hir_build_tests { v22:BasicObject = LoadField v18, :block@0x1002 Jump bb6(v22, v22) bb5(): - v24:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v25:BasicObject = CCall v24, :rb_obj_is_proc@0x1004 - v26:TrueClass = GuardBitEquals v25, Value(true) recompile - Jump bb6(v24, v10) + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() + bb7(): + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) + bb8(): + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb9(), bb10() + bb9(): + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb10(): + v36:CInt64[255] = Const CInt64(255) + v37:CInt64 = IntAnd v24, v36 + v38:CInt64[12] = Const CInt64(12) + v39:CBool = IsBitEqual v37, v38 + CondBranch v39, bb11(), bb12() + bb12(): + v41:CUInt64 = LoadField v24, :RBASIC_FLAGS@0x1001 + v42:CUInt64[31] = Const CUInt64(31) + v43:CInt64 = IntAnd v41, v42 + v44:CUInt64[20] = Const CUInt64(20) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb11(), bb13() + bb11(): + v47:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v47, v47) + bb13(): + v49:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v49, v10) bb6(v16:BasicObject, v17:BasicObject): - v29:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Uncategorized(send) + v52:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v29 + Return v52 "); } @@ -3918,13 +4038,43 @@ pub(crate) mod hir_build_tests { Jump bb9(v36, v36) bb8(): v38:CInt64 = LoadField v32, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v39:CInt64 = GuardAnyBitSet v38, CUInt64(1) recompile - v40:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb9(v40, v17) + v39:CInt64[1] = Const CInt64(1) + v40:CInt64 = IntAnd v38, v39 + v41:CBool = IsBitEqual v40, v39 + CondBranch v41, bb10(), bb11() + bb10(): + v43:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v43, v17) + bb11(): + v45:CInt64[0] = Const CInt64(0) + v46:CBool = IsBitEqual v38, v45 + CondBranch v46, bb12(), bb13() + bb12(): + v48:NilClass = Const Value(nil) + Jump bb9(v48, v17) + bb13(): + v50:CInt64[255] = Const CInt64(255) + v51:CInt64 = IntAnd v38, v50 + v52:CInt64[12] = Const CInt64(12) + v53:CBool = IsBitEqual v51, v52 + CondBranch v53, bb14(), bb15() + bb15(): + v55:CUInt64 = LoadField v38, :RBASIC_FLAGS@0x1001 + v56:CUInt64[31] = Const CUInt64(31) + v57:CInt64 = IntAnd v55, v56 + v58:CUInt64[20] = Const CUInt64(20) + v59:CBool = IsBitEqual v57, v58 + CondBranch v59, bb14(), bb16() + bb14(): + v61:BasicObject = SymToProc :block, l0, EP@4 + Jump bb9(v61, v61) + bb16(): + v63:BasicObject = LoadField v32, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb9(v63, v17) bb9(v30:BasicObject, v31:BasicObject): - v43:BasicObject = Send v11, &block, :tap, v30 # SendFallbackReason: Uncategorized(send) + v66:BasicObject = Send v11, &block, :tap, v30 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v43 + Return v66 "); } @@ -3971,13 +4121,43 @@ pub(crate) mod hir_build_tests { Jump bb9(v31) bb8(): v33:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 - v34:CInt64 = GuardAnyBitSet v33, CUInt64(1) recompile - v35:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb9(v35) + v34:CInt64[1] = Const CInt64(1) + v35:CInt64 = IntAnd v33, v34 + v36:CBool = IsBitEqual v35, v34 + CondBranch v36, bb10(), bb11() + bb10(): + v38:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v38) + bb11(): + v40:CInt64[0] = Const CInt64(0) + v41:CBool = IsBitEqual v33, v40 + CondBranch v41, bb12(), bb13() + bb12(): + v43:NilClass = Const Value(nil) + Jump bb9(v43) + bb13(): + v45:CInt64[255] = Const CInt64(255) + v46:CInt64 = IntAnd v33, v45 + v47:CInt64[12] = Const CInt64(12) + v48:CBool = IsBitEqual v46, v47 + CondBranch v48, bb14(), bb15() + bb15(): + v50:CUInt64 = LoadField v33, :RBASIC_FLAGS@0x1000 + v51:CUInt64[31] = Const CUInt64(31) + v52:CInt64 = IntAnd v50, v51 + v53:CUInt64[20] = Const CUInt64(20) + v54:CBool = IsBitEqual v52, v53 + CondBranch v54, bb14(), bb16() + bb14(): + v56:BasicObject = SymToProc :block, l1, EP@3 + Jump bb9(v56) + bb16(): + v58:BasicObject = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 + Jump bb9(v58) bb9(v26:BasicObject): - v38:BasicObject = Send v8, &block, :tap, v26 # SendFallbackReason: Uncategorized(send) + v61:BasicObject = Send v8, &block, :tap, v26 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v38 + Return v61 "); } @@ -4020,23 +4200,40 @@ pub(crate) mod hir_build_tests { v25:CInt64[1] = Const CInt64(1) v26:CInt64 = IntAnd v24, v25 v27:CBool = IsBitEqual v26, v25 - CondBranch v27, bb7(), bb9() + CondBranch v27, bb7(), bb8() bb7(): v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) Jump bb6(v29, v10) - bb9(): + bb8(): v31:CInt64[0] = Const CInt64(0) v32:CBool = IsBitEqual v24, v31 - CondBranch v32, bb8(), bb10() - bb8(): + CondBranch v32, bb9(), bb10() + bb9(): v34:NilClass = Const Value(nil) Jump bb6(v34, v10) + bb10(): + v36:CInt64[255] = Const CInt64(255) + v37:CInt64 = IntAnd v24, v36 + v38:CInt64[12] = Const CInt64(12) + v39:CBool = IsBitEqual v37, v38 + CondBranch v39, bb11(), bb12() + bb12(): + v41:CUInt64 = LoadField v24, :RBASIC_FLAGS@0x1001 + v42:CUInt64[31] = Const CUInt64(31) + v43:CInt64 = IntAnd v41, v42 + v44:CUInt64[20] = Const CUInt64(20) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb11(), bb13() + bb11(): + v47:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v47, v47) + bb13(): + v49:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v49, v10) bb6(v16:BasicObject, v17:BasicObject): - v38:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Uncategorized(send) + v52:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v38 - bb10(): - SideExit BlockParamProxyProfileNotCovered + Return v52 "); } @@ -4077,36 +4274,43 @@ pub(crate) mod hir_build_tests { Jump bb6(v22, v22) bb5(): v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - Jump bb10() - bb10(): - v26:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 - v27:BasicObject = CCall v26, :rb_obj_is_proc@0x1004 - v28:TrueClass = Const Value(true) - v29:CBool = IsBitEqual v27, v28 - CondBranch v29, bb7(), bb11() + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() bb7(): - Jump bb6(v26, v10) - bb11(): - v32:CInt64[0] = Const CInt64(0) - v33:CBool = IsBitEqual v24, v32 - CondBranch v33, bb8(), bb12() + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) bb8(): - v35:NilClass = Const Value(nil) - Jump bb6(v35, v10) - bb12(): - v37:CInt64[1] = Const CInt64(1) - v38:CInt64 = IntAnd v24, v37 - v39:CBool = IsBitEqual v38, v37 - CondBranch v39, bb9(), bb13() + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb9(), bb10() bb9(): - v41:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v41, v10) + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb10(): + v36:CInt64[255] = Const CInt64(255) + v37:CInt64 = IntAnd v24, v36 + v38:CInt64[12] = Const CInt64(12) + v39:CBool = IsBitEqual v37, v38 + CondBranch v39, bb11(), bb12() + bb12(): + v41:CUInt64 = LoadField v24, :RBASIC_FLAGS@0x1001 + v42:CUInt64[31] = Const CUInt64(31) + v43:CInt64 = IntAnd v41, v42 + v44:CUInt64[20] = Const CUInt64(20) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb11(), bb13() + bb11(): + v47:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v47, v47) + bb13(): + v49:BasicObject = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + Jump bb6(v49, v10) bb6(v16:BasicObject, v17:BasicObject): - v45:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Uncategorized(send) + v52:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v45 - bb13(): - SideExit BlockParamProxyProfileNotCovered + Return v52 "); } @@ -4244,9 +4448,39 @@ pub(crate) mod hir_build_tests { Jump bb6(v25, v25) bb5(): v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 - v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) recompile - v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v29, v13) + v28:CInt64[1] = Const CInt64(1) + v29:CInt64 = IntAnd v27, v28 + v30:CBool = IsBitEqual v29, v28 + CondBranch v30, bb7(), bb8() + bb7(): + v32:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v32, v13) + bb8(): + v34:CInt64[0] = Const CInt64(0) + v35:CBool = IsBitEqual v27, v34 + CondBranch v35, bb9(), bb10() + bb9(): + v37:NilClass = Const Value(nil) + Jump bb6(v37, v13) + bb10(): + v39:CInt64[255] = Const CInt64(255) + v40:CInt64 = IntAnd v27, v39 + v41:CInt64[12] = Const CInt64(12) + v42:CBool = IsBitEqual v40, v41 + CondBranch v42, bb11(), bb12() + bb12(): + v44:CUInt64 = LoadField v27, :RBASIC_FLAGS@0x1002 + v45:CUInt64[31] = Const CUInt64(31) + v46:CInt64 = IntAnd v44, v45 + v47:CUInt64[20] = Const CUInt64(20) + v48:CBool = IsBitEqual v46, v47 + CondBranch v48, bb11(), bb13() + bb11(): + v50:BasicObject = SymToProc :b, l0, EP@3 + Jump bb6(v50, v50) + bb13(): + v52:BasicObject = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + Jump bb6(v52, v13) bb6(v19:BasicObject, v20:BasicObject): SideExit SplatKwNotProfiled "); @@ -4293,14 +4527,44 @@ pub(crate) mod hir_build_tests { Jump bb6(v40, v40) bb5(): v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 - v43:CInt64[0] = GuardBitEquals v42, CInt64(0) recompile - v44:NilClass = Const Value(nil) - Jump bb6(v44, v21) + v43:CInt64[1] = Const CInt64(1) + v44:CInt64 = IntAnd v42, v43 + v45:CBool = IsBitEqual v44, v43 + CondBranch v45, bb7(), bb8() + bb7(): + v47:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v47, v21) + bb8(): + v49:CInt64[0] = Const CInt64(0) + v50:CBool = IsBitEqual v42, v49 + CondBranch v50, bb9(), bb10() + bb9(): + v52:NilClass = Const Value(nil) + Jump bb6(v52, v21) + bb10(): + v54:CInt64[255] = Const CInt64(255) + v55:CInt64 = IntAnd v42, v54 + v56:CInt64[12] = Const CInt64(12) + v57:CBool = IsBitEqual v55, v56 + CondBranch v57, bb11(), bb12() + bb12(): + v59:CUInt64 = LoadField v42, :RBASIC_FLAGS@0x1004 + v60:CUInt64[31] = Const CUInt64(31) + v61:CInt64 = IntAnd v59, v60 + v62:CUInt64[20] = Const CUInt64(20) + v63:CBool = IsBitEqual v61, v62 + CondBranch v63, bb11(), bb13() + bb11(): + v65:BasicObject = SymToProc :&, l0, EP@4 + Jump bb6(v65, v65) + bb13(): + v67:BasicObject = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + Jump bb6(v67, v21) bb6(v34:BasicObject, v35:BasicObject): - v47:NilClass = GuardType v20, NilClass - v49:BasicObject = Send v17, &block, :foo, v18, v29, v47, v34 # SendFallbackReason: Uncategorized(send) + v70:NilClass = GuardType v20, NilClass + v72:BasicObject = Send v17, &block, :foo, v18, v29, v70, v34 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v49 + Return v72 "); } @@ -4336,15 +4600,45 @@ pub(crate) mod hir_build_tests { v25:BasicObject = LoadField v21, :b@0x1003 Jump bb6(v25, v25) bb5(): - v27:BasicObject = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 - v28:BasicObject = CCall v27, :rb_obj_is_proc@0x1005 - v29:TrueClass = GuardBitEquals v28, Value(true) recompile - Jump bb6(v27, v13) + v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v28:CInt64[1] = Const CInt64(1) + v29:CInt64 = IntAnd v27, v28 + v30:CBool = IsBitEqual v29, v28 + CondBranch v30, bb7(), bb8() + bb7(): + v32:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v32, v13) + bb8(): + v34:CInt64[0] = Const CInt64(0) + v35:CBool = IsBitEqual v27, v34 + CondBranch v35, bb9(), bb10() + bb9(): + v37:NilClass = Const Value(nil) + Jump bb6(v37, v13) + bb10(): + v39:CInt64[255] = Const CInt64(255) + v40:CInt64 = IntAnd v27, v39 + v41:CInt64[12] = Const CInt64(12) + v42:CBool = IsBitEqual v40, v41 + CondBranch v42, bb11(), bb12() + bb12(): + v44:CUInt64 = LoadField v27, :RBASIC_FLAGS@0x1002 + v45:CUInt64[31] = Const CUInt64(31) + v46:CInt64 = IntAnd v44, v45 + v47:CUInt64[20] = Const CUInt64(20) + v48:CBool = IsBitEqual v46, v47 + CondBranch v48, bb11(), bb13() + bb11(): + v50:BasicObject = SymToProc :b, l0, EP@3 + Jump bb6(v50, v50) + bb13(): + v52:BasicObject = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + Jump bb6(v52, v13) bb6(v19:BasicObject, v20:BasicObject): - v32:HashExact = GuardType v12, HashExact - v34:BasicObject = Send v11, &block, :foo, v32, v19 # SendFallbackReason: Uncategorized(send) + v55:HashExact = GuardType v12, HashExact + v57:BasicObject = Send v11, &block, :foo, v55, v19 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v34 + Return v57 "); } @@ -4380,15 +4674,45 @@ pub(crate) mod hir_build_tests { v25:BasicObject = LoadField v21, :b@0x1003 Jump bb6(v25, v25) bb5(): - v27:BasicObject = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 - v28:BasicObject = CCall v27, :rb_obj_is_proc@0x1005 - v29:TrueClass = GuardBitEquals v28, Value(true) recompile - Jump bb6(v27, v13) + v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v28:CInt64[1] = Const CInt64(1) + v29:CInt64 = IntAnd v27, v28 + v30:CBool = IsBitEqual v29, v28 + CondBranch v30, bb7(), bb8() + bb7(): + v32:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v32, v13) + bb8(): + v34:CInt64[0] = Const CInt64(0) + v35:CBool = IsBitEqual v27, v34 + CondBranch v35, bb9(), bb10() + bb9(): + v37:NilClass = Const Value(nil) + Jump bb6(v37, v13) + bb10(): + v39:CInt64[255] = Const CInt64(255) + v40:CInt64 = IntAnd v27, v39 + v41:CInt64[12] = Const CInt64(12) + v42:CBool = IsBitEqual v40, v41 + CondBranch v42, bb11(), bb12() + bb12(): + v44:CUInt64 = LoadField v27, :RBASIC_FLAGS@0x1002 + v45:CUInt64[31] = Const CUInt64(31) + v46:CInt64 = IntAnd v44, v45 + v47:CUInt64[20] = Const CUInt64(20) + v48:CBool = IsBitEqual v46, v47 + CondBranch v48, bb11(), bb13() + bb11(): + v50:BasicObject = SymToProc :b, l0, EP@3 + Jump bb6(v50, v50) + bb13(): + v52:BasicObject = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + Jump bb6(v52, v13) bb6(v19:BasicObject, v20:BasicObject): - v32:HashExact = GuardType v12, HashExact - v34:BasicObject = Send v11, &block, :foo, v32, v19 # SendFallbackReason: Uncategorized(send) + v55:HashExact = GuardType v12, HashExact + v57:BasicObject = Send v11, &block, :foo, v55, v19 # SendFallbackReason: Uncategorized(send) CheckInterrupts - Return v34 + Return v57 "); } @@ -4435,9 +4759,39 @@ pub(crate) mod hir_build_tests { Jump bb6(v40, v40) bb5(): v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 - v43:CInt64[0] = GuardBitEquals v42, CInt64(0) recompile - v44:NilClass = Const Value(nil) - Jump bb6(v44, v21) + v43:CInt64[1] = Const CInt64(1) + v44:CInt64 = IntAnd v42, v43 + v45:CBool = IsBitEqual v44, v43 + CondBranch v45, bb7(), bb8() + bb7(): + v47:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v47, v21) + bb8(): + v49:CInt64[0] = Const CInt64(0) + v50:CBool = IsBitEqual v42, v49 + CondBranch v50, bb9(), bb10() + bb9(): + v52:NilClass = Const Value(nil) + Jump bb6(v52, v21) + bb10(): + v54:CInt64[255] = Const CInt64(255) + v55:CInt64 = IntAnd v42, v54 + v56:CInt64[12] = Const CInt64(12) + v57:CBool = IsBitEqual v55, v56 + CondBranch v57, bb11(), bb12() + bb12(): + v59:CUInt64 = LoadField v42, :RBASIC_FLAGS@0x1004 + v60:CUInt64[31] = Const CUInt64(31) + v61:CInt64 = IntAnd v59, v60 + v62:CUInt64[20] = Const CUInt64(20) + v63:CBool = IsBitEqual v61, v62 + CondBranch v63, bb11(), bb13() + bb11(): + v65:BasicObject = SymToProc :&, l0, EP@4 + Jump bb6(v65, v65) + bb13(): + v67:BasicObject = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + Jump bb6(v67, v21) bb6(v34:BasicObject, v35:BasicObject): SideExit SplatKwPolymorphic "); @@ -4478,9 +4832,39 @@ pub(crate) mod hir_build_tests { Jump bb6(v25, v25) bb5(): v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 - v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) recompile - v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb6(v29, v13) + v28:CInt64[1] = Const CInt64(1) + v29:CInt64 = IntAnd v27, v28 + v30:CBool = IsBitEqual v29, v28 + CondBranch v30, bb7(), bb8() + bb7(): + v32:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v32, v13) + bb8(): + v34:CInt64[0] = Const CInt64(0) + v35:CBool = IsBitEqual v27, v34 + CondBranch v35, bb9(), bb10() + bb9(): + v37:NilClass = Const Value(nil) + Jump bb6(v37, v13) + bb10(): + v39:CInt64[255] = Const CInt64(255) + v40:CInt64 = IntAnd v27, v39 + v41:CInt64[12] = Const CInt64(12) + v42:CBool = IsBitEqual v40, v41 + CondBranch v42, bb11(), bb12() + bb12(): + v44:CUInt64 = LoadField v27, :RBASIC_FLAGS@0x1002 + v45:CUInt64[31] = Const CUInt64(31) + v46:CInt64 = IntAnd v44, v45 + v47:CUInt64[20] = Const CUInt64(20) + v48:CBool = IsBitEqual v46, v47 + CondBranch v48, bb11(), bb13() + bb11(): + v50:BasicObject = SymToProc :block, l0, EP@3 + Jump bb6(v50, v50) + bb13(): + v52:BasicObject = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + Jump bb6(v52, v13) bb6(v19:BasicObject, v20:BasicObject): SideExit SplatKwNotNilOrHash "); @@ -5203,22 +5587,52 @@ pub(crate) mod hir_build_tests { Jump bb8(v39, v39) bb7(): v41:CInt64 = LoadField v35, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 - v42:CInt64 = GuardAnyBitSet v41, CUInt64(1) recompile - v43:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) - Jump bb8(v43, v22) - bb8(v33:BasicObject, v34:BasicObject): - v46:CBool = Test v33 - v47:Falsy = RefineType v33, Falsy - CondBranch v46, bb9(), bb4(v18, v19, v20, v21, v34, v27) + v42:CInt64[1] = Const CInt64(1) + v43:CInt64 = IntAnd v41, v42 + v44:CBool = IsBitEqual v43, v42 + CondBranch v44, bb9(), bb10() bb9(): - v49:Truthy = RefineType v33, Truthy - v53:BasicObject = InvokeBlock v27 # SendFallbackReason: InvokeBlock: not yet specialized - v56:BasicObject = InvokeBuiltin dir_s_close, v18, v27 + v46:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb8(v46, v22) + bb10(): + v48:CInt64[0] = Const CInt64(0) + v49:CBool = IsBitEqual v41, v48 + CondBranch v49, bb11(), bb12() + bb11(): + v51:NilClass = Const Value(nil) + Jump bb8(v51, v22) + bb12(): + v53:CInt64[255] = Const CInt64(255) + v54:CInt64 = IntAnd v41, v53 + v55:CInt64[12] = Const CInt64(12) + v56:CBool = IsBitEqual v54, v55 + CondBranch v56, bb13(), bb14() + bb14(): + v58:CUInt64 = LoadField v41, :RBASIC_FLAGS@0x1004 + v59:CUInt64[31] = Const CUInt64(31) + v60:CInt64 = IntAnd v58, v59 + v61:CUInt64[20] = Const CUInt64(20) + v62:CBool = IsBitEqual v60, v61 + CondBranch v62, bb13(), bb15() + bb13(): + v64:BasicObject = SymToProc :block, l0, EP@4 + Jump bb8(v64, v64) + bb15(): + v66:BasicObject = LoadField v35, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + Jump bb8(v66, v22) + bb8(v33:BasicObject, v34:BasicObject): + v69:CBool = Test v33 + v70:Falsy = RefineType v33, Falsy + CondBranch v69, bb16(), bb4(v18, v19, v20, v21, v34, v27) + bb16(): + v72:Truthy = RefineType v33, Truthy + v76:BasicObject = InvokeBlock v27 # SendFallbackReason: InvokeBlock: not yet specialized + v79:BasicObject = InvokeBuiltin dir_s_close, v18, v27 CheckInterrupts - Return v53 - bb4(v62:BasicObject, v63:BasicObject, v64:BasicObject, v65:BasicObject, v66:BasicObject, v67:BasicObject): + Return v76 + bb4(v85:BasicObject, v86:BasicObject, v87:BasicObject, v88:BasicObject, v89:BasicObject, v90:BasicObject): CheckInterrupts - Return v67 + Return v90 "); } diff --git a/zjit/src/options.rs b/zjit/src/options.rs index ac1e20bfdd3a17..a4b3d4cec3ea76 100644 --- a/zjit/src/options.rs +++ b/zjit/src/options.rs @@ -22,6 +22,10 @@ pub const DEFAULT_MAX_VERSIONS: usize = 4; const DEFAULT_NUM_PROFILES: NumProfiles = 5; pub type NumProfiles = u16; +/// Default --zjit-num-exits-until-invalidate +const DEFAULT_NUM_EXITS_UNTIL_INVALIDATE: NumExits = 5; +pub type NumExits = u32; + /// Default --zjit-call-threshold. This should be large enough to avoid compiling /// warmup code, but small enough to perform well on micro-benchmarks. pub const DEFAULT_CALL_THRESHOLD: CallThreshold = 30; @@ -80,6 +84,9 @@ pub struct Options { /// Number of times YARV instructions should be profiled. pub num_profiles: NumProfiles, + /// Number of recompile exits before invalidating the current version. See `exit_recompile`. + pub num_exits_until_invalidate: NumExits, + /// Enable ZJIT statistics pub stats: bool, @@ -198,6 +205,7 @@ impl Default for Options { exec_mem_bytes: 64 * 1024 * 1024, mem_bytes: 128 * 1024 * 1024, num_profiles: DEFAULT_NUM_PROFILES, + num_exits_until_invalidate: DEFAULT_NUM_EXITS_UNTIL_INVALIDATE, stats: false, print_stats: false, print_stats_file: None, @@ -442,6 +450,11 @@ fn parse_option(str_ptr: *const std::os::raw::c_char) -> Option<()> { Err(_) => return None, }, + ("num-exits-until-invalidate", _) => match opt_val.parse() { + Ok(n) => options.num_exits_until_invalidate = n, + Err(_) => return None, + }, + ("max-versions", _) => match opt_val.parse() { Ok(n) => options.max_versions = n, Err(_) => return None, @@ -677,6 +690,13 @@ pub fn set_call_threshold(call_threshold: CallThreshold) { update_profile_threshold(); } +/// Update --zjit-num-exits-until-invalidate for testing +#[cfg(test)] +pub fn set_num_exits_until_invalidate(num_exits_until_invalidate: NumExits) { + rb_zjit_prepare_options(); + unsafe { OPTIONS.as_mut().unwrap().num_exits_until_invalidate = num_exits_until_invalidate; } +} + /// Update --zjit-max-versions for testing #[cfg(test)] pub fn set_max_versions(max_versions: usize) { diff --git a/zjit/src/payload.rs b/zjit/src/payload.rs index 988b842990b347..07b9f351573c18 100644 --- a/zjit/src/payload.rs +++ b/zjit/src/payload.rs @@ -1,6 +1,7 @@ use std::ffi::c_void; use std::ptr::NonNull; use crate::codegen::IseqCallRef; +use crate::options::{get_option, NumExits}; use crate::stats::CompileError; use crate::{cruby::*, profile::IseqProfile, virtualmem::CodePtr}; @@ -24,6 +25,8 @@ pub struct IseqPayload { /// `BasicObject`) when the owner is unknown. /// See [`crate::cruby::iseq_self_is_heap_object`]. pub self_is_heap_object: bool, + /// Number of recompile exits before invalidating the current version. See `exit_recompile`. + pub num_exits_until_invalidate: NumExits, } impl IseqPayload { @@ -33,6 +36,7 @@ impl IseqPayload { versions: vec![], was_invalidated_for_singleton_class_creation: false, self_is_heap_object: false, + num_exits_until_invalidate: get_option!(num_exits_until_invalidate), } } } diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs index 9e1c62d98dbd01..1afdd64059e51c 100644 --- a/zjit/src/profile.rs +++ b/zjit/src/profile.rs @@ -93,7 +93,6 @@ fn profile_insn_sample( YARVINSN_opt_size => profile_operands(profiler, profile, 1), YARVINSN_opt_succ => profile_operands(profiler, profile, 1), YARVINSN_invokeblock => profile_block_handler(profiler, profile), - YARVINSN_getblockparamproxy => profile_getblockparamproxy(profiler, profile), YARVINSN_invokesuper => profile_invokesuper(profiler, profile), YARVINSN_opt_send_without_block | YARVINSN_send => { let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); @@ -222,22 +221,6 @@ fn profile_block_handler(profiler: &mut Profiler, profile: &mut IseqProfile) { entry.opnd_types[0].observe(ty); } -fn profile_getblockparamproxy(profiler: &mut Profiler, profile: &mut IseqProfile) { - let entry = profile.entry_mut(profiler.insn_idx); - if entry.opnd_types.is_empty() { - entry.opnd_types.resize(1, TypeDistribution::new()); - } - - let level = profiler.insn_opnd(1).as_u32(); - let ep = unsafe { get_cfp_ep_level(profiler.cfp, level) }; - let block_handler = unsafe { *ep.offset(VM_ENV_DATA_INDEX_SPECVAL as isize) }; - let untagged = unsafe { rb_vm_untag_block_handler(block_handler) }; - - let ty = ProfiledType::object(untagged); - VALUE::from(profiler.iseq).write_barrier(ty.class()); - entry.opnd_types[0].observe(ty); -} - fn profile_invokesuper(profiler: &mut Profiler, profile: &mut IseqProfile) { let cme = unsafe { rb_vm_frame_method_entry(profiler.cfp) }; let cme_value = VALUE(cme as usize); // CME is a T_IMEMO, which is a VALUE diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index 46faed063f30ff..c0fd8a83efd86f 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -232,11 +232,6 @@ make_counters! { exit_callee_side_exit, exit_interrupt, exit_stackoverflow, - exit_block_param_proxy_not_iseq_or_ifunc, - exit_block_param_proxy_not_nil, - exit_block_param_proxy_not_proc, - exit_block_param_proxy_fallback_miss, - exit_block_param_proxy_profile_not_covered, exit_invoke_block_handler_not_iseq, exit_invoke_block_iseq_changed, exit_block_param_wb_required, @@ -496,15 +491,6 @@ make_counters! { inline_reject_no_returns, inline_reject_budget_exceeded, - getblockparamproxy_handler_iseq, - getblockparamproxy_handler_ifunc, - getblockparamproxy_handler_symbol, - getblockparamproxy_handler_proc, - getblockparamproxy_handler_nil, - getblockparamproxy_handler_polymorphic, - getblockparamproxy_handler_megamorphic, - getblockparamproxy_handler_no_profiles, - total_native_stack_bytes, } @@ -640,11 +626,6 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { CalleeSideExit => exit_callee_side_exit, Interrupt => exit_interrupt, StackOverflow => exit_stackoverflow, - BlockParamProxyNotIseqOrIfunc => exit_block_param_proxy_not_iseq_or_ifunc, - BlockParamProxyNotNil => exit_block_param_proxy_not_nil, - BlockParamProxyNotProc => exit_block_param_proxy_not_proc, - BlockParamProxyFallbackMiss => exit_block_param_proxy_fallback_miss, - BlockParamProxyProfileNotCovered => exit_block_param_proxy_profile_not_covered, InvokeBlockHandlerNotIseq => exit_invoke_block_handler_not_iseq, InvokeBlockIseqChanged => exit_invoke_block_iseq_changed, BlockParamWbRequired => exit_block_param_wb_required,