// ============ Wasm Call Lowering (Standard) ============
///|
/// Call argument constant that can be rematerialized at the callsite.
priv enum CallConstArg {
Int(Int64)
F32(Int)
F64(Int64)
}
///|
fn wasm_func_symbol(func_idx : Int) -> @instr.CodeSymbol {
CodeSymbol("wasm.func", func_idx)
}
///|
fn emit_exact_indirect_type_check(
ctx : @lower.LoweringContext,
block : @block.Block,
actual_type_vreg : @abi.VReg,
expected_type_idx : Int,
) -> Unit {
let expected_type_vreg = ctx.machv_function().new_vreg(Int)
let load_expected = @instr.Inst(LoadConst(expected_type_idx.to_int64()))
load_expected.add_def({ reg: Virtual(expected_type_vreg) })
block.add_inst(load_expected)
let cmp = @instr.Inst(IntCmp(false))
cmp.add_use(Virtual(actual_type_vreg))
cmp.add_use(Virtual(expected_type_vreg))
block.add_inst(cmp)
let trap = @instr.Inst(TrapIf(Ne, 2))
block.add_inst(trap)
}
///|
/// Emit a constant load for a call argument, optionally constrained to a register.
fn emit_call_const(
block : @block.Block,
vreg : @abi.VReg,
value : CallConstArg,
preg? : @abi.PReg? = None,
) -> Unit {
let opcode = match value {
Int(v) => @instr.LoadConst(v)
F32(bits) => LoadConstF32(bits)
F64(bits) => LoadConstF64(bits)
}
let inst = @instr.Inst(opcode)
match preg {
Some(p) => inst.add_def_fixed({ reg: Virtual(vreg) }, p)
None => inst.add_def({ reg: Virtual(vreg) })
}
block.add_inst(inst)
}
///|
/// Emit StoreToStack for overflow arguments according to the Wasm ABI.
/// This also updates `max_outgoing_args_size` so the prologue reserves enough
/// space and SP doesn't move at call sites.
fn emit_wasm_overflow_arg_stores(
ctx : @lower.LoweringContext,
block : @block.Block,
int_args : Array[(@abi.VReg, @abi.RegClass)],
float_args : Array[(@abi.VReg, @abi.RegClass)],
) -> Int {
let call_conv = ctx.call_conv_layout()
let max_int_reg_args = call_conv.user_arg_gprs.length()
let max_float_reg_args = call_conv.arg_fprs.length()
let int_overflow = if int_args.length() > max_int_reg_args {
int_args.length() - max_int_reg_args
} else {
0
}
let float_overflow_classes : Array[@abi.RegClass] = []
for i in max_float_reg_args.. 0 {
ctx.machv_function().update_max_outgoing_args_size(total_bytes)
}
for i in 0.. Unit {
if !ctx.embedding_abi().reserve_context_role {
abort("unpinned VMContext ABI not implemented yet")
}
let call_conv = ctx.call_conv_layout()
// vmctx is pinned by the embedding ABI role supplied to lowering.
let vmctx_preg : @abi.PReg = @lower.context_preg(ctx)
// Classify user arguments by type
let int_args : Array[(@abi.VReg, @abi.RegClass)] = []
let float_args : Array[(@abi.VReg, @abi.RegClass)] = []
for arg in args {
match arg.class {
Int => int_args.push((arg, Int))
Float32 => float_args.push((arg, Float32))
Float64 => float_args.push((arg, Float64))
Vector => float_args.push((arg, Vector)) // SIMD uses Vn registers
}
}
// Materialize constant overflow args before storing to stack.
let max_int_reg_args = call_conv.user_arg_gprs.length()
let max_float_reg_args = call_conv.arg_fprs.length()
for i in max_int_reg_args.. ignore
// vmctx constrained to the ISA's vmctx-arg register.
call_inst.add_use_fixed(Physical(vmctx_preg), call_conv.context_arg)
// Register int args (ISA-specific).
let user_arg_gprs = call_conv.user_arg_gprs
let int_reg_count = if int_args.length() < max_int_reg_args {
int_args.length()
} else {
max_int_reg_args
}
for i in 0.. {
call_inst.add_def_fixed({ reg: Virtual(result) }, {
index: ret_gprs[int_result_idx].index,
class: Int,
})
int_result_idx += 1
}
Float32 | Float64 | Vector => {
// Float/Vector results go to V0, V1, ...
// Use actual class for proper move size (32/64/128 bits)
call_inst.add_def_fixed({ reg: Virtual(result) }, {
index: ret_fprs[float_result_idx].index,
class: result.class,
})
float_result_idx += 1
}
}
}
// Add clobbers for all caller-saved registers
@lower.add_call_clobbers(ctx, call_inst)
block.add_inst(call_inst)
@lower.invalidate_context_pointer_cache(ctx)
}
///|
/// Lower a Wasm function call using Standard approach.
/// All argument placement is done in lowering via FixedReg constraints.
/// The emit phase just emits BLR to the call target register.
///
/// Wasm ABI (Cranelift-style):
/// - X0 = vmctx
/// - X1-X7 = integer user args (up to 7)
/// - V0-V7 = float user args (up to 8)
/// - Overflow args go to stack
/// - X0/V0 = first result, etc.
fn lower_wasm_call(
ctx : @lower.LoweringContext,
block : @block.Block,
func_ptr : @abi.VReg,
args : Array[@abi.VReg],
results : Array[@abi.VReg],
const_args : Map[Int, CallConstArg],
) -> Unit {
let num_args = args.length()
let call_inst = @instr.Inst(
CallPtr(num_args, @instr.result_classes_for_vregs(results), Internal),
)
// Call target is constrained to SCRATCH_REG_2 (IP1).
call_inst.add_use_fixed(Virtual(func_ptr), {
index: @lower.scratch2_index(ctx),
class: Int,
})
lower_wasm_call_common(ctx, block, call_inst, args, results, const_args)
}
///|
/// Lower a direct Wasm function call.
fn lower_wasm_call_direct(
ctx : @lower.LoweringContext,
block : @block.Block,
func_idx : Int,
args : Array[@abi.VReg],
results : Array[@abi.VReg],
const_args : Map[Int, CallConstArg],
) -> Unit {
let num_args = args.length()
let call_inst = @instr.Inst(
CallDirect(
wasm_func_symbol(func_idx),
num_args,
@instr.result_classes_for_vregs(results),
Internal,
),
)
lower_wasm_call_common(ctx, block, call_inst, args, results, const_args)
}
// ============ Tail Call Lowering (Standard) ============
///|
/// Lower direct return_call (tail call optimization)
/// Note: parameters are handled in lowering phase
/// - Overflow args: StoreToStack instructions
/// - Register args: Fixed register constraints via add_use_fixed
/// - ReturnCallIndirect instruction only contains func_ptr use
pub fn lower_return_call(
ctx : @lower.LoweringContext,
inst : @milkir.Inst,
block : @block.Block,
func_idx : Int,
) -> Unit {
// Load function address (patched at JIT module load time)
let func_ptr_vreg = ctx.machv_function().new_vreg(Int)
let load_inst = @instr.Inst(LoadCodeAddr(wasm_func_symbol(func_idx)))
load_inst.add_def({ reg: Virtual(func_ptr_vreg) })
block.add_inst(load_inst)
// Get pinned vmctx.
let vmctx_preg : @abi.PReg = @lower.context_preg(ctx)
// Classify user arguments by type
let int_args : Array[(@abi.VReg, @abi.RegClass)] = []
let float_args : Array[(@abi.VReg, @abi.RegClass)] = []
for operand in inst.operands {
let vreg = ctx.get_vreg_for_use(operand, block)
match vreg.class {
Int => int_args.push((vreg, Int))
Float32 => float_args.push((vreg, Float32))
Float64 => float_args.push((vreg, Float64))
Vector => float_args.push((vreg, Vector)) // SIMD uses Vn registers
}
}
// Calculate overflow args count (user args, not including vmctx)
let stack_arg_bytes = emit_wasm_overflow_arg_stores(
ctx, block, int_args, float_args,
)
let call_conv = ctx.call_conv_layout()
// Create ReturnCallIndirect instruction with fixed register constraints
// Encode tail-call stack-bytes in opcode metadata.
let call_inst = @instr.Inst(ReturnCallIndirect(stack_arg_bytes, 0))
// Call target is constrained to SCRATCH_REG_2 (IP1).
call_inst.add_use_fixed(Virtual(func_ptr_vreg), {
index: @lower.scratch2_index(ctx),
class: Int,
})
// vmctx constrained to the ISA's vmctx-arg register.
call_inst.add_use_fixed(Physical(vmctx_preg), call_conv.context_arg)
// Register int args (ISA-specific).
let user_arg_gprs = call_conv.user_arg_gprs
let max_int_reg_args = user_arg_gprs.length()
let int_reg_count = if int_args.length() < max_int_reg_args {
int_args.length()
} else {
max_int_reg_args
}
for i in 0.. Unit {
// Get element index and load function pointer (similar to lower_call_indirect)
guard inst.operands.length() > 0 else { return }
let elem_idx_vreg = ctx.get_vreg_for_use(inst.operands[0], block)
// Load table pointer
let table_ptr_vreg : @abi.VReg = if table_idx == 0 {
load_context_slot_pointer_from_pinned_context(
ctx,
block,
options.context_slots.table0_base,
)
} else {
let indirect_tables_vreg = ctx.machv_function().new_vreg(Int)
let load_array_inst = @instr.Inst(
Load(I64, context_slot_offset(ctx, options.context_slots.table_directory)),
)
load_array_inst.add_def({ reg: Virtual(indirect_tables_vreg) })
load_array_inst.add_use(
Physical({ index: @lower.context_index(ctx), class: Int }),
)
block.add_inst(load_array_inst)
let table_offset = table_idx * options.context_slots.pointer_stride
let ptr_vreg = ctx.machv_function().new_vreg(Int)
let load_table_inst = @instr.Inst(Load(I64, table_offset))
load_table_inst.add_def({ reg: Virtual(ptr_vreg) })
load_table_inst.add_use(Virtual(indirect_tables_vreg))
block.add_inst(load_table_inst)
ptr_vreg
}
// Calculate address = table_ptr + (zext(elem_idx) << 4) (entry size = 16B).
// Match Cranelift-style folded add+shift shape.
let addr_vreg = ctx.machv_function().new_vreg(Int)
let add_inst = @instr.Inst(AddExtend(Uxtw, 4))
add_inst.add_def({ reg: Virtual(addr_vreg) })
add_inst.add_use(Virtual(table_ptr_vreg))
add_inst.add_use(Virtual(elem_idx_vreg))
block.add_inst(add_inst)
// Load raw function pointer
let raw_func_ptr_vreg = ctx.machv_function().new_vreg(Int)
let load_func_inst = @instr.Inst(Load(I64, 0))
load_func_inst.add_def({ reg: Virtual(raw_func_ptr_vreg) })
load_func_inst.add_use(Virtual(addr_vreg))
block.add_inst(load_func_inst)
// Clear FUNCREF_TAG (bit 61) from function pointer.
let func_ptr_vreg = ctx.machv_function().new_vreg(Int)
let and_inst = @instr.Inst(AndImm(0xDFFFFFFFFFFFFFFFL, true))
and_inst.add_def({ reg: Virtual(func_ptr_vreg) })
and_inst.add_use(Virtual(raw_func_ptr_vreg))
block.add_inst(and_inst)
let actual_type_vreg = ctx.machv_function().new_vreg(Int)
let load_type_inst = @instr.Inst(Load(I32, 8))
load_type_inst.add_def({ reg: Virtual(actual_type_vreg) })
load_type_inst.add_use(Virtual(addr_vreg))
block.add_inst(load_type_inst)
// Type check (fast path); align with Cranelift behavior:
// use exact check when subtype checks are unnecessary.
let expected_type_idx = options.canonicalize_indirect_type_idx(type_idx)
if options.use_subtype_indirect_check {
let type_check = @instr.Inst(
CallExternalIfI32NeImm(expected_type_idx, rt_gc_type_check_subtype(ctx)),
)
type_check.add_use(Virtual(actual_type_vreg))
@lower.add_c_call_clobbers(ctx, type_check)
block.add_inst(type_check)
} else {
emit_exact_indirect_type_check(
ctx, block, actual_type_vreg, expected_type_idx,
)
}
// Get pinned vmctx.
let vmctx_preg : @abi.PReg = @lower.context_preg(ctx)
// Classify user arguments (skip first operand which is elem_idx)
let int_args : Array[(@abi.VReg, @abi.RegClass)] = []
let float_args : Array[(@abi.VReg, @abi.RegClass)] = []
for i in 1.. int_args.push((vreg, Int))
Float32 => float_args.push((vreg, Float32))
Float64 => float_args.push((vreg, Float64))
Vector => float_args.push((vreg, Vector)) // SIMD uses Vn registers
}
}
let stack_arg_bytes = emit_wasm_overflow_arg_stores(
ctx, block, int_args, float_args,
)
let call_conv = ctx.call_conv_layout()
// Create ReturnCallIndirect with fixed constraints
// Encode tail-call stack-bytes in opcode metadata.
let call_inst = @instr.Inst(ReturnCallIndirect(stack_arg_bytes, 0))
// Call target is constrained to SCRATCH_REG_2 (IP1).
call_inst.add_use_fixed(Virtual(func_ptr_vreg), {
index: @lower.scratch2_index(ctx),
class: Int,
})
// vmctx constrained to the ISA's vmctx-arg register.
call_inst.add_use_fixed(Physical(vmctx_preg), call_conv.context_arg)
// Register int args (ISA-specific).
let user_arg_gprs = call_conv.user_arg_gprs
let max_int_reg_args = user_arg_gprs.length()
let int_reg_count = if int_args.length() < max_int_reg_args {
int_args.length()
} else {
max_int_reg_args
}
for i in 0.. Unit {
guard inst.operands.length() > 0 else { return }
let func_ref_vreg = ctx.get_vreg_for_use(inst.operands[0], block)
// Strip FUNCREF_TAG (0x2000000000000000) to get raw function pointer
// func_ptr = func_ref & 0xDFFFFFFFFFFFFFFF (clear bit 61)
let func_ptr_vreg = ctx.machv_function().new_vreg(Int)
let and_inst = @instr.Inst(AndImm(0xDFFFFFFFFFFFFFFFL, true))
and_inst.add_def({ reg: Virtual(func_ptr_vreg) })
and_inst.add_use(Virtual(func_ref_vreg))
block.add_inst(and_inst)
// Get vmctx
let vmctx_preg : @abi.PReg = @lower.context_preg(ctx)
// Classify user arguments (skip first operand which is func_ref)
let int_args : Array[(@abi.VReg, @abi.RegClass)] = []
let float_args : Array[(@abi.VReg, @abi.RegClass)] = []
for i in 1.. int_args.push((vreg, Int))
Float32 => float_args.push((vreg, Float32))
Float64 => float_args.push((vreg, Float64))
Vector => float_args.push((vreg, Vector)) // SIMD uses Vn registers
}
}
let stack_arg_bytes = emit_wasm_overflow_arg_stores(
ctx, block, int_args, float_args,
)
let call_conv = ctx.call_conv_layout()
// Create ReturnCallIndirect with fixed constraints
// Encode tail-call stack-bytes in opcode metadata.
let call_inst = @instr.Inst(ReturnCallIndirect(stack_arg_bytes, 0))
// Call target is constrained to SCRATCH_REG_2 (IP1).
call_inst.add_use_fixed(Virtual(func_ptr_vreg), {
index: @lower.scratch2_index(ctx),
class: Int,
})
// vmctx constrained to the ISA's vmctx-arg register.
call_inst.add_use_fixed(Physical(vmctx_preg), call_conv.context_arg)
// Register int args (ISA-specific).
let user_arg_gprs = call_conv.user_arg_gprs
let max_int_reg_args = user_arg_gprs.length()
let int_reg_count = if int_args.length() < max_int_reg_args {
int_args.length()
} else {
max_int_reg_args
}
for i in 0.. Unit {
// For call_indirect, the first operand is the element index within the table
// which we need to convert to a function pointer
if inst.operands.length() == 0 {
return
}
// First operand is the element index within the specific table
let elem_idx_vreg = ctx.get_vreg_for_use(inst.operands[0], block)
// Multi-table support: determine which table pointer to use
// Both table_idx == 0 and table_idx != 0 load the table pointer on-demand
// on-demand (no pre-loaded registers for table pointers)
let table_ptr_vreg : @abi.VReg = if table_idx == 0 {
// Fast path for table 0: load table0_base from the Wasm context on-demand.
load_context_slot_pointer_from_pinned_context(
ctx,
block,
options.context_slots.table0_base,
)
} else {
// Slow path: load indirect_tables[table_idx] from context
// 1. Load indirect_tables array pointer from the runtime context.
let indirect_tables_vreg = ctx.machv_function().new_vreg(Int)
let load_array_inst = @instr.Inst(
Load(I64, context_slot_offset(ctx, options.context_slots.table_directory)),
)
load_array_inst.add_def({ reg: Virtual(indirect_tables_vreg) })
load_array_inst.add_use(
Physical({ index: @lower.context_index(ctx), class: Int }),
)
block.add_inst(load_array_inst)
// 2. Calculate offset = table_idx * pointer size
let table_offset = table_idx * options.context_slots.pointer_stride
// 3. Load table pointer: [indirect_tables + offset]
let ptr_vreg = ctx.machv_function().new_vreg(Int)
let load_table_inst = @instr.Inst(Load(I64, table_offset))
load_table_inst.add_def({ reg: Virtual(ptr_vreg) })
load_table_inst.add_use(Virtual(indirect_tables_vreg))
block.add_inst(load_table_inst)
ptr_vreg
}
// Arguments are all operands except the first one
let arg_vregs : Array[@abi.VReg] = []
for i in 1.. Unit {
// First operand is the function reference (tagged function pointer)
if inst.operands.length() == 0 {
return
}
let func_ref_vreg = ctx.get_vreg_for_use(inst.operands[0], block)
// Arguments are all operands except the first one
let arg_vregs : Array[@abi.VReg] = []
for i in 1.. CallConstArg? {
if ctx.use_count(value.id) != 1 {
return None
}
if @lower.find_defining_inst(ctx, value) is Some(inst) {
match inst.opcode {
Scalar(IntConst(v)) => Some(Int(v))
Scalar(FloatConst32(bits)) => Some(F32(bits.reinterpret_as_int()))
Scalar(FloatConst64(bits)) => Some(F64(bits.reinterpret_as_int64()))
_ => None
}
} else {
None
}
}
///|
/// Collect call arguments and rematerializable constants.
fn collect_call_args_with_consts(
ctx : @lower.LoweringContext,
operands : Array[@milkir.Value],
block : @block.Block,
start_idx : Int,
) -> (Array[@abi.VReg], Map[Int, CallConstArg]) {
let arg_vregs : Array[@abi.VReg] = []
let const_args : Map[Int, CallConstArg] = Map([])
for i in start_idx..func_table` when call fixups are applied at module load time.
pub fn lower_call(
ctx : @lower.LoweringContext,
inst : @milkir.Inst,
block : @block.Block,
func_idx : Int,
) -> Unit {
// Collect argument vregs, rematerializing single-use constants only for
// direct calls. Indirect calls (imports or default path) keep original vregs.
let use_const_args = ctx.num_imports() >= 0 && func_idx >= ctx.num_imports()
let (arg_vregs, const_args) = if use_const_args {
collect_call_args_with_consts(ctx, inst.operands, block, 0)
} else {
let args : Array[@abi.VReg] = []
for operand in inst.operands {
args.push(ctx.get_vreg_for_use(operand, block))
}
(args, Map([]))
}
// Collect result vregs (skip when results are unused)
let result_vregs = @lower.collect_call_result_vregs(ctx, inst)
lower_wasm_call_direct(
ctx, block, func_idx, arg_vregs, result_vregs, const_args,
)
}