// Licensing scope: see ../../LICENSE and this directory's LICENSE and NOTICE.
// Copyright 2023 WebAssembly Community Group participants.
// Retained historical upstream notice; current derivation remains unconfirmed.
// Conservative attribution from pinned Binaryen version_130 counterpart:
// Copyright 2021 WebAssembly Community Group participants
// Counterpart: src/passes/Heap2Local.cpp (5d704ad5); origin remains unconfirmed.
// Potentially incorporated Binaryen expression retains Apache-2.0 terms.
// Original Starshine contributions use the scoped MIT grant in ../../LICENSE.
// Modified for Starshine through 2026: implementation and validation use
// Starshine's representations and APIs; this is not an unchanged upstream file.
// Conservative notice retention does not establish copying or independence.
///|
pub fn heap2local_descriptor() -> @ir.HotPassDescriptor {
@ir.HotPassDescriptor::new(
"heap2local",
requires=[@ir.HotAnalysis::use_def()],
invalidates=[
@ir.HotAnalysis::cfg(),
@ir.HotAnalysis::dominance(),
@ir.HotAnalysis::liveness(),
@ir.HotAnalysis::use_def(),
@ir.HotAnalysis::effects(),
@ir.HotAnalysis::loop_info(),
@ir.HotAnalysis::ssa(),
],
)
}
///|
pub fn heap2local_summary() -> String {
"Replace non-escaping struct locals and branch-selected variant flows with scalar field locals, including repeated loop-state writes."
}
///|
priv struct H2LUseSite {
source_node_id : Int32
user_node_id : Int32
}
///|
priv struct H2LCopySite {
source_node_id : Int32
write_node_id : Int32
}
///|
priv struct H2LCandidate {
write_node_id : Int32
init_node_id : Int32
fields : Array[@lib.FieldType]
use_sites : Array[H2LUseSite]
copy_sites : Array[H2LCopySite]
}
///|
priv struct H2LArrayUseSite {
source_node_id : Int32
user_node_id : Int32
index : Int32
is_set : Bool
}
///|
priv struct H2LArrayCandidate {
write_node_id : Int32
init_node_id : Int32
element : @lib.FieldType
length : Int32
use_sites : Array[H2LArrayUseSite]
copy_sites : Array[H2LCopySite]
}
///|
priv struct H2LDropOnlyStructOwnerCandidate {
write_node_id : Int32
init_node_id : Int32
read_node_ids : Array[Int32]
drop_node_ids : Array[Int32]
}
///|
priv struct H2LVariantField {
index : Int32
type_ : @lib.ValType
}
///|
priv struct H2LVariantUse {
user_node_id : Int32
field_index : Int32
source_path : Array[Int32]
}
///|
priv struct H2LVariantCandidate {
producer_user_node_id : Int32
producer_field_index : Int32
producer_source_path : Array[Int32]
owner_local : Int32
if_node_id : Int32
then_block_node_id : Int32
else_block_node_id : Int32
then_alloc_node_id : Int32
else_alloc_node_id : Int32
fields : Array[H2LVariantField]
uses : Array[H2LVariantUse]
}
///|
priv struct H2LVariantLocalProducer {
write_node_id : Int32
source_path : Array[Int32]
if_node_id : Int32
then_block_node_id : Int32
else_block_node_id : Int32
then_alloc_node_id : Int32
else_alloc_node_id : Int32
}
///|
priv struct H2LVariantLocalCandidate {
owner_local : Int32
producers : Array[H2LVariantLocalProducer]
fields : Array[H2LVariantField]
uses : Array[H2LVariantUse]
}
///|
fn h2l_build_default_value(func : @ir.HotFunc, vt : @lib.ValType) -> Int32? {
match vt {
@lib.NumTypeValType(@lib.I32NumType) =>
Some(@ir.hot_build_const_i32(func, @lib.I32(0)))
@lib.NumTypeValType(@lib.I64NumType) =>
Some(@ir.hot_build_const_i64(func, @lib.I64(0L)))
@lib.NumTypeValType(@lib.F32NumType) =>
Some(@ir.hot_build_const_f32(func, @lib.F32(0.0)))
@lib.NumTypeValType(@lib.F64NumType) =>
Some(@ir.hot_build_const_f64(func, @lib.F64(0.0)))
@lib.VecTypeValType => Some(@ir.hot_build_const_v128_zero(func))
@lib.RefTypeValType(ref_type) =>
if ref_type.is_nullable() {
Some(@ir.hot_build_ref_null(func, ref_type))
} else {
None
}
@lib.BotValType => None
}
}
///|
fn h2l_default_value_supported(vt : @lib.ValType) -> Bool {
match vt {
@lib.NumTypeValType(@lib.I32NumType)
| @lib.NumTypeValType(@lib.I64NumType)
| @lib.NumTypeValType(@lib.F32NumType)
| @lib.NumTypeValType(@lib.F64NumType)
| @lib.VecTypeValType => true
@lib.RefTypeValType(ref_type) => ref_type.is_nullable()
@lib.BotValType => false
}
}
///|
fn h2l_immediate_branch_block_value(
func : @ir.HotFunc,
node_id : Int32,
) -> Int32? {
match @ir.hot_node_get(func, node_id).op {
@ir.HotOp::Block => ()
_ => return None
}
let body_ref = @ir.hot_region_body_ref(func, node_id)
let root_count = @ir.hot_region_root_count(func, body_ref)
if root_count == 1 {
let branch_node_id = @ir.hot_region_get(func, body_ref, 0)
let branch = @ir.hot_node_get(func, branch_node_id)
match branch.op {
@ir.HotOp::Br =>
if @ir.hot_branch_target(func, branch_node_id) ==
Some(@ir.hot_control_node_label(func, node_id)) &&
branch.child_count == 1 {
return Some(@ir.hot_child_get(func, branch_node_id, 0))
}
_ => ()
}
return None
}
if root_count != 2 {
return None
}
let branch_node_id = @ir.hot_region_get(func, body_ref, 1)
let branch = @ir.hot_node_get(func, branch_node_id)
match branch.op {
@ir.HotOp::Br if @ir.hot_branch_target(func, branch_node_id) ==
Some(@ir.hot_control_node_label(func, node_id)) &&
branch.child_count == 0 => Some(@ir.hot_region_get(func, body_ref, 0))
_ => None
}
}
///|
fn h2l_allocation_source_node(func : @ir.HotFunc, node_id : Int32) -> Int32 {
match h2l_immediate_branch_block_value(func, node_id) {
Some(source_node_id) => source_node_id
None => node_id
}
}
///|
fn h2l_supported_struct_new_fields(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
) -> (@lib.TypeIdx, Array[@lib.FieldType], Bool)? {
match @ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::StructNew(type_idx)) =>
match @ir.hot_module_resolve_struct_fields(module_ctx, type_idx) {
Some(fields) =>
if fields.length() == @ir.hot_child_count(func, node_id) {
Some((type_idx, fields, false))
} else {
None
}
None => None
}
Some(@lib.Instruction::StructNewDefault(type_idx)) =>
match @ir.hot_module_resolve_struct_fields(module_ctx, type_idx) {
Some(fields) => {
for field in fields {
if !h2l_default_value_supported(field.unpack()) {
return None
}
}
Some((type_idx, fields, true))
}
None => None
}
_ => None
}
}
///|
fn h2l_is_direct_struct_alloc(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
) -> Bool {
match @ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::StructNew(type_idx)) =>
match @ir.hot_module_resolve_struct_fields(module_ctx, type_idx) {
Some(fields) => fields.length() == @ir.hot_child_count(func, node_id)
None => false
}
Some(@lib.Instruction::StructNewDefault(type_idx)) =>
match @ir.hot_module_resolve_struct_fields(module_ctx, type_idx) {
Some(_) => true
None => false
}
_ => false
}
}
///|
fn h2l_direct_descriptor_alloc_info(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
) -> (@lib.TypeIdx, Int32)? {
match @ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::StructNewDesc(type_idx)) =>
match
(
@ir.hot_module_resolve_subtype(module_ctx, type_idx),
@ir.hot_module_resolve_struct_fields(module_ctx, type_idx),
) {
(Some(subtype), Some(fields)) =>
match subtype.descriptor_type() {
Some(_) =>
if @ir.hot_child_count(func, node_id) == fields.length() + 1 {
Some(
(
type_idx,
@ir.hot_child_get(
func,
node_id,
@ir.hot_child_count(func, node_id) - 1,
),
),
)
} else {
None
}
None => None
}
_ => None
}
Some(@lib.Instruction::StructNewDefaultDesc(type_idx)) =>
match
(
@ir.hot_module_resolve_subtype(module_ctx, type_idx),
@ir.hot_module_resolve_struct_fields(module_ctx, type_idx),
) {
(Some(subtype), Some(_)) =>
match subtype.descriptor_type() {
Some(_) =>
if @ir.hot_child_count(func, node_id) == 1 {
Some((type_idx, @ir.hot_child_get(func, node_id, 0)))
} else {
None
}
None => None
}
_ => None
}
_ => None
}
}
///|
fn h2l_field_index(instr : @lib.Instruction) -> Int32? {
match instr {
@lib.StructGet(_, field_idx)
| @lib.StructGetS(_, field_idx)
| @lib.StructGetU(_, field_idx)
| @lib.StructSet(_, field_idx) => {
let @lib.U32(raw_field_idx) = field_idx
Some(raw_field_idx.reinterpret_as_int())
}
_ => None
}
}
///|
fn h2l_exact_field_user_matches(
func : @ir.HotFunc,
user_node_id : Int32,
struct_type : @lib.TypeIdx,
field_count : Int32,
) -> Bool {
let instr = match @ir.hot_node_exact_instr(func, user_node_id) {
Some(instr) => instr
None => return false
}
let user_type = match instr {
@lib.StructGet(type_idx, _)
| @lib.StructGetS(type_idx, _)
| @lib.StructGetU(type_idx, _)
| @lib.StructSet(type_idx, _) => type_idx
_ => return false
}
if user_type != struct_type {
return false
}
match h2l_field_index(instr) {
Some(field_idx) => field_idx >= 0 && field_idx < field_count
None => false
}
}
///|
fn h2l_local_can_hold_struct_ref(
func : @ir.HotFunc,
local_id : Int32,
struct_type : @lib.TypeIdx,
) -> Bool {
match @ir.hot_local_type(func, local_id) {
Some(@lib.RefTypeValType(ref_type)) => {
if !ref_type.is_nullable() {
return false
}
match ref_type.get_heap_type() {
@lib.HeapType(type_idx) => type_idx == struct_type
@lib.AbsHeapTypeHeapType(
@lib.AnyAbsHeapType
| @lib.EqAbsHeapType
| @lib.StructAbsHeapType
) => true
_ => false
}
}
_ => false
}
}
///|
fn h2l_local_can_hold_array_ref(
func : @ir.HotFunc,
local_id : Int32,
array_type : @lib.TypeIdx,
) -> Bool {
match @ir.hot_local_type(func, local_id) {
Some(@lib.RefTypeValType(ref_type)) =>
match ref_type.get_heap_type() {
@lib.HeapType(type_idx) => type_idx == array_type
@lib.AbsHeapTypeHeapType(
@lib.AnyAbsHeapType
| @lib.EqAbsHeapType
| @lib.ArrayAbsHeapType
) => ref_type.is_nullable()
_ => false
}
_ => false
}
}
///|
fn h2l_i32_const_value(func : @ir.HotFunc, node_id : Int32) -> Int32? {
match @ir.hot_node_get(func, node_id).op {
@ir.HotOp::Const =>
match @ir.hot_const_get(func, @ir.hot_node_get(func, node_id).imm0) {
@ir.HotConstPayload::I32Const(@lib.I32(value)) => Some(value)
_ => None
}
_ => None
}
}
///|
fn h2l_immediate_branch_flow_user(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
branch_node_id : Int32,
) -> (Int32, Int32)? {
let branch = @ir.hot_node_get(func, branch_node_id)
match branch.op {
@ir.HotOp::Br => ()
_ => return None
}
let branch_sites = use_def.node_uses[branch_node_id]
if branch_sites.length() != 1 {
return None
}
let branch_site = branch_sites[0]
let block_node_id = match branch_site.kind {
@ir.HotUseSlotKind::RootUseSlot if branch_site.user >= 0 => branch_site.user
_ => return None
}
match @ir.hot_node_get(func, block_node_id).op {
@ir.HotOp::Block => ()
_ => return None
}
if @ir.hot_branch_target(func, branch_node_id) !=
Some(@ir.hot_control_node_label(func, block_node_id)) {
return None
}
let body_ref = @ir.hot_region_body_ref(func, block_node_id)
let root_count = @ir.hot_region_root_count(func, body_ref)
if root_count == 0 || branch_site.slot != root_count - 1 {
return None
}
let block_sites = use_def.node_uses[block_node_id]
if block_sites.length() != 1 {
return None
}
let block_site = block_sites[0]
match block_site.kind {
@ir.HotUseSlotKind::ChildUseSlot if block_site.slot == 0 =>
Some((block_node_id, block_site.user))
_ => None
}
}
///|
fn h2l_collect_source_use(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
source_node_id : Int32,
current_local : Int32,
struct_type : @lib.TypeIdx,
field_count : Int32,
family_locals : Array[Int32],
use_sites : Array[H2LUseSite],
copy_sites : Array[H2LCopySite],
allow_copy : Bool,
) -> Bool {
if !@ir.hot_node_is_live(func, source_node_id) {
return false
}
let sites = use_def.node_uses[source_node_id]
if sites.length() != 1 {
return false
}
let site = sites[0]
let (flowed_source_node_id, flowed_user_node_id) = match site.kind {
@ir.HotUseSlotKind::ChildUseSlot =>
if site.slot != 0 {
return false
} else {
match h2l_immediate_branch_flow_user(func, use_def, site.user) {
Some(flow) => flow
None => (source_node_id, site.user)
}
}
@ir.HotUseSlotKind::RootUseSlot => {
if site.user < 0 {
return false
}
let block_node_id = site.user
let block_node = @ir.hot_node_get(func, block_node_id)
match block_node.op {
@ir.HotOp::Block => ()
_ => return false
}
let body_ref = @ir.hot_region_body_ref(func, block_node_id)
let root_count = @ir.hot_region_root_count(func, body_ref)
if root_count == 0 || site.slot != root_count - 1 {
return false
}
for root_idx = 0; root_idx < root_count - 1; root_idx = root_idx + 1 {
let root_node = @ir.hot_node_get(
func,
@ir.hot_region_get(func, body_ref, root_idx),
)
match root_node.op {
@ir.HotOp::Block
| @ir.HotOp::Loop
| @ir.HotOp::If
| @ir.HotOp::Try
| @ir.HotOp::TryTable
| @ir.HotOp::Br
| @ir.HotOp::BrIf
| @ir.HotOp::BrTable
| @ir.HotOp::BrOnNull
| @ir.HotOp::BrOnNonNull
| @ir.HotOp::BrOnCast
| @ir.HotOp::BrOnCastFail
| @ir.HotOp::Return
| @ir.HotOp::Throw
| @ir.HotOp::Rethrow
| @ir.HotOp::ThrowRef
| @ir.HotOp::Delegate
| @ir.HotOp::Catch
| @ir.HotOp::CatchAll => return false
_ => ()
}
}
let block_sites = use_def.node_uses[block_node_id]
if block_sites.length() != 1 {
return false
}
let block_site = block_sites[0]
match block_site.kind {
@ir.HotUseSlotKind::ChildUseSlot =>
if block_site.slot != 0 {
return false
}
_ => return false
}
(block_node_id, block_site.user)
}
}
if h2l_exact_field_user_matches(
func, flowed_user_node_id, struct_type, field_count,
) {
use_sites.push({
source_node_id: flowed_source_node_id,
user_node_id: flowed_user_node_id,
})
return true
}
if flowed_source_node_id != source_node_id {
return false
}
match @ir.hot_node_exact_instr(func, site.user) {
Some(@lib.Instruction::RefAsNonNull) => {
let flow_sites = use_def.node_uses[site.user]
if flow_sites.length() != 1 {
return false
}
let flow_site = flow_sites[0]
match flow_site.kind {
@ir.HotUseSlotKind::ChildUseSlot =>
if flow_site.slot != 0 {
return false
}
_ => return false
}
if h2l_exact_field_user_matches(
func,
flow_site.user,
struct_type,
field_count,
) {
use_sites.push({
source_node_id: site.user,
user_node_id: flow_site.user,
})
return true
}
return false
}
Some(@lib.Instruction::RefCast(_, _, cast_heap_type)) => {
if cast_heap_type != @lib.HeapType::new(struct_type) {
return false
}
let flow_sites = use_def.node_uses[site.user]
if flow_sites.length() != 1 {
return false
}
let flow_site = flow_sites[0]
match flow_site.kind {
@ir.HotUseSlotKind::ChildUseSlot =>
if flow_site.slot != 0 {
return false
}
_ => return false
}
if h2l_exact_field_user_matches(
func,
flow_site.user,
struct_type,
field_count,
) {
use_sites.push({
source_node_id: site.user,
user_node_id: flow_site.user,
})
return true
}
return false
}
_ => ()
}
if !allow_copy {
return false
}
let user_node = @ir.hot_node_get(func, site.user)
match user_node.op {
@ir.HotOp::LocalSet =>
if user_node.child_count != 1 || user_node.imm0 == current_local {
false
} else {
let target_local = user_node.imm0
if target_local < @ir.hot_param_count(func) {
return false
}
if !h2l_local_can_hold_struct_ref(func, target_local, struct_type) {
return false
}
let writes = use_def.local_writes[target_local]
if writes.length() != 1 || writes[0] != site.user {
return false
}
copy_sites.push({ source_node_id, write_node_id: site.user, })
if !family_locals.contains(target_local) {
family_locals.push(target_local)
}
true
}
_ => false
}
}
///|
fn h2l_find_candidate_for_local(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
use_def : @ir.HotUseDef,
local_id : Int32,
cfg : @ir.HotCfg,
dom : @ir.HotDominators,
order : H2LDominanceOrder,
) -> H2LCandidate? {
if local_id < @ir.hot_param_count(func) {
return None
}
let writes = use_def.local_writes[local_id]
if writes.length() != 1 {
return None
}
let write_node_id = writes[0]
if !@ir.hot_node_is_live(func, write_node_id) {
return None
}
let write_node = @ir.hot_node_get(func, write_node_id)
let write_is_tee = match write_node.op {
@ir.HotOp::LocalSet =>
if write_node.imm0 != local_id || write_node.child_count != 1 {
return None
} else {
false
}
@ir.HotOp::LocalTee =>
if write_node.imm0 != local_id || write_node.child_count != 1 {
return None
} else {
true
}
_ => return None
}
let init_node_id = h2l_allocation_source_node(
func,
@ir.hot_child_get(func, write_node_id, 0),
)
let (struct_type, fields, uses_default) = match
h2l_supported_struct_new_fields(func, module_ctx, init_node_id) {
Some(info) => info
None => return None
}
ignore(uses_default)
let owner_local_matches = h2l_local_can_hold_struct_ref(
func, local_id, struct_type,
)
if !owner_local_matches &&
(!write_is_tee || !use_def.local_reads[local_id].is_empty()) {
return None
}
let use_sites : Array[H2LUseSite] = []
let copy_sites : Array[H2LCopySite] = []
let family_locals : Array[Int32] = [local_id]
if write_is_tee &&
!h2l_collect_source_use(
func,
use_def,
write_node_id,
local_id,
struct_type,
fields.length(),
family_locals,
use_sites,
copy_sites,
false,
) {
return None
}
for family_idx = 0
family_idx < family_locals.length()
family_idx = family_idx + 1 {
let current_local = family_locals[family_idx]
let reads = use_def.local_reads[current_local]
let family_writes = use_def.local_writes[current_local]
if family_writes.length() != 1 {
return None
}
for read_node_id in reads {
if !h2l_write_dominates_read(
cfg,
dom,
order,
family_writes[0],
read_node_id,
) {
return None
}
if !h2l_collect_source_use(
func,
use_def,
read_node_id,
current_local,
struct_type,
fields.length(),
family_locals,
use_sites,
copy_sites,
true,
) {
return None
}
}
}
if use_sites.is_empty() {
return None
}
Some({ write_node_id, init_node_id, fields, use_sites, copy_sites, })
}
///|
fn h2l_find_drop_only_struct_owner_candidate(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
use_def : @ir.HotUseDef,
local_id : Int32,
) -> H2LDropOnlyStructOwnerCandidate? {
if local_id < @ir.hot_param_count(func) {
return None
}
let writes = use_def.local_writes[local_id]
if writes.length() != 1 {
return None
}
let write_node_id = writes[0]
let write_node = @ir.hot_node_get(func, write_node_id)
match write_node.op {
@ir.HotOp::LocalSet if write_node.imm0 == local_id &&
write_node.child_count == 1 => ()
_ => return None
}
let init_node_id = h2l_allocation_source_node(
func,
@ir.hot_child_get(func, write_node_id, 0),
)
let (struct_type, _, _) = match
h2l_supported_struct_new_fields(func, module_ctx, init_node_id) {
Some(info) => info
None => return None
}
if !h2l_local_can_hold_struct_ref(func, local_id, struct_type) {
return None
}
let reads = use_def.local_reads[local_id]
if reads.is_empty() {
return None
}
let read_node_ids : Array[Int32] = []
let drop_node_ids : Array[Int32] = []
for read_node_id in reads {
let sites = use_def.node_uses[read_node_id]
if sites.length() != 1 {
return None
}
let site = sites[0]
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot if site.slot == 0 => ()
_ => return None
}
match @ir.hot_node_get(func, site.user).op {
@ir.HotOp::Drop => ()
_ => return None
}
read_node_ids.push(read_node_id)
drop_node_ids.push(site.user)
}
Some({ write_node_id, init_node_id, read_node_ids, drop_node_ids, })
}
///|
fn h2l_build_discarded_allocation_operands(
func : @ir.HotFunc,
init_node_id : Int32,
) -> Int32 {
let roots : Array[Int32] = []
for child_idx = 0
child_idx < @ir.hot_child_count(func, init_node_id)
child_idx = child_idx + 1 {
roots.push(
@ir.hot_build_drop(func, @ir.hot_child_get(func, init_node_id, child_idx)),
)
}
match roots {
[] => @ir.hot_build_nop(func)
[root] => root
_ => @ir.hot_build_block(func, @lib.BlockType::void_(), roots)
}
}
///|
fn h2l_apply_drop_only_struct_owner_candidate(
func : @ir.HotFunc,
candidate : H2LDropOnlyStructOwnerCandidate,
detached_nodes : Array[Int32],
) -> Bool {
let write_source_node_id = @ir.hot_child_get(func, candidate.write_node_id, 0)
let replacement = h2l_build_discarded_allocation_operands(
func,
candidate.init_node_id,
)
h2l_replace_with_built(func, candidate.write_node_id, replacement)
if write_source_node_id == candidate.init_node_id {
detached_nodes.push(candidate.init_node_id)
}
for idx = 0; idx < candidate.drop_node_ids.length(); idx = idx + 1 {
@ir.hot_replace_node(
func,
candidate.drop_node_ids[idx],
@ir.hot_node_get(func, @ir.hot_build_nop(func)),
)
detached_nodes.push(candidate.read_node_ids[idx])
}
true
}
///|
fn h2l_subtree_contains_node(
func : @ir.HotFunc,
root_node_id : Int32,
target_node_id : Int32,
) -> Bool {
if root_node_id == target_node_id {
return true
}
if !@ir.hot_node_is_live(func, root_node_id) {
return false
}
let root_node = @ir.hot_node_get(func, root_node_id)
match root_node.op {
@ir.HotOp::Block | @ir.HotOp::Loop => {
let body_ref = @ir.hot_region_body_ref(func, root_node_id)
for idx = 0
idx < @ir.hot_region_root_count(func, body_ref)
idx = idx + 1 {
if h2l_subtree_contains_node(
func,
@ir.hot_region_get(func, body_ref, idx),
target_node_id,
) {
return true
}
}
return false
}
_ => ()
}
for slot = 0; slot < @ir.hot_child_count(func, root_node_id); slot = slot + 1 {
if h2l_subtree_contains_node(
func,
@ir.hot_child_get(func, root_node_id, slot),
target_node_id,
) {
return true
}
}
false
}
///|
fn h2l_root_slot_containing_node(func : @ir.HotFunc, node_id : Int32) -> Int32? {
let root_ref = @ir.HotRegionRef::root()
for idx = 0; idx < @ir.hot_region_root_count(func, root_ref); idx = idx + 1 {
if h2l_subtree_contains_node(
func,
@ir.hot_region_get(func, root_ref, idx),
node_id,
) {
return Some(idx)
}
}
None
}
///|
fn h2l_index_root_subtree(
func : @ir.HotFunc,
node_id : Int32,
root_slot : Int32,
slots : Array[Int32],
) -> Unit {
// Optional operands (for example a missing else arm) use a negative ID.
if node_id < 0 || node_id >= slots.length() {
return
}
if slots[node_id] != -1 {
return
}
slots[node_id] = root_slot
if !@ir.hot_node_is_live(func, node_id) {
return
}
match @ir.hot_node_get(func, node_id).op {
@ir.HotOp::Block | @ir.HotOp::Loop => {
let body_ref = @ir.hot_region_body_ref(func, node_id)
for idx = 0
idx < @ir.hot_region_root_count(func, body_ref)
idx = idx + 1 {
h2l_index_root_subtree(
func,
@ir.hot_region_get(func, body_ref, idx),
root_slot,
slots,
)
}
return
}
_ => ()
}
for child = 0; child < @ir.hot_child_count(func, node_id); child = child + 1 {
h2l_index_root_subtree(
func,
@ir.hot_child_get(func, node_id, child),
root_slot,
slots,
)
}
}
///|
fn h2l_build_root_slot_index(func : @ir.HotFunc) -> Array[Int32] {
let slots = Array::make(@ir.hot_node_count(func), -1)
let root_ref = @ir.HotRegionRef::root()
for root_slot = 0
root_slot < @ir.hot_region_root_count(func, root_ref)
root_slot = root_slot + 1 {
h2l_index_root_subtree(
func,
@ir.hot_region_get(func, root_ref, root_slot),
root_slot,
slots,
)
}
slots
}
///|
fn h2l_root_slot_from_index(slots : Array[Int32], node_id : Int32) -> Int32? {
if node_id < 0 || node_id >= slots.length() || slots[node_id] == -1 {
None
} else {
Some(slots[node_id])
}
}
///|
fn h2l_sort_sequential_write_slots(
writes : Array[(Int32, Int32)],
) -> Array[(Int32, Int32)] {
writes.sort_by(fn(left, right) { left.0.compare(right.0) })
writes
}
///|
fn h2l_self_copy_write_matches(
func : @ir.HotFunc,
write_node_id : Int32,
local_id : Int32,
) -> Bool {
if !@ir.hot_node_is_live(func, write_node_id) {
return false
}
let write_node = @ir.hot_node_get(func, write_node_id)
match write_node.op {
@ir.HotOp::LocalSet => ()
_ => return false
}
if write_node.imm0 != local_id || write_node.child_count != 1 {
return false
}
let source_node_id = @ir.hot_child_get(func, write_node_id, 0)
let source_node = @ir.hot_node_get(func, source_node_id)
match source_node.op {
@ir.HotOp::LocalGet => source_node.imm0 == local_id
_ => false
}
}
///|
fn h2l_struct_alloc_write_info(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
write_node_id : Int32,
local_id : Int32,
) -> (@lib.TypeIdx, Array[@lib.FieldType], Int32)? {
if !@ir.hot_node_is_live(func, write_node_id) {
return None
}
let write_node = @ir.hot_node_get(func, write_node_id)
match write_node.op {
@ir.HotOp::LocalSet => ()
_ => return None
}
if write_node.imm0 != local_id || write_node.child_count != 1 {
return None
}
let init_node_id = h2l_allocation_source_node(
func,
@ir.hot_child_get(func, write_node_id, 0),
)
match h2l_supported_struct_new_fields(func, module_ctx, init_node_id) {
Some((struct_type, fields, _)) =>
if h2l_local_can_hold_struct_ref(func, local_id, struct_type) {
Some((struct_type, fields, init_node_id))
} else {
None
}
None => None
}
}
///|
fn h2l_collect_sequential_source_use(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
source_node_id : Int32,
local_id : Int32,
struct_type : @lib.TypeIdx,
field_count : Int32,
use_sites : Array[H2LUseSite],
copy_sites : Array[H2LCopySite],
) -> Bool {
if !@ir.hot_node_is_live(func, source_node_id) {
return false
}
let sites = use_def.node_uses[source_node_id]
if sites.length() != 1 {
return false
}
let site = sites[0]
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot => ()
_ => return false
}
if site.slot == 0 && h2l_self_copy_write_matches(func, site.user, local_id) {
copy_sites.push({ source_node_id, write_node_id: site.user, })
return true
}
let family_locals : Array[Int32] = [local_id]
h2l_collect_source_use(
func, use_def, source_node_id, local_id, struct_type, field_count, family_locals,
use_sites, copy_sites, false,
)
}
///|
fn h2l_find_sequential_candidates_for_local(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
use_def : @ir.HotUseDef,
local_id : Int32,
root_slots : Array[Int32],
) -> Array[H2LCandidate] {
let candidates : Array[H2LCandidate] = []
if local_id < @ir.hot_param_count(func) {
return candidates
}
let writes = use_def.local_writes[local_id]
if writes.length() <= 1 {
return candidates
}
let allocation_writes : Array[(Int32, Int32)] = []
for write_node_id in writes {
let is_allocation = h2l_struct_alloc_write_info(
func, module_ctx, write_node_id, local_id,
).is_some()
let is_self_copy = h2l_self_copy_write_matches(
func, write_node_id, local_id,
)
if !is_allocation && !is_self_copy {
return []
}
match h2l_root_slot_from_index(root_slots, write_node_id) {
Some(root_slot) => {
// Conditional/nested allocation writes do not define straight-line
// epochs. A later read may observe either owner, not this candidate.
if is_allocation &&
@ir.hot_region_get(func, @ir.HotRegionRef::root(), root_slot) !=
write_node_id {
return []
}
if is_allocation {
allocation_writes.push((root_slot, write_node_id))
}
}
None => return []
}
}
let allocation_writes = h2l_sort_sequential_write_slots(allocation_writes)
let root_count = @ir.hot_root_count(func)
let reads_by_slot : Array[(Int32, Int32)] = []
for read_node_id in use_def.local_reads[local_id] {
let read_slot = match h2l_root_slot_from_index(root_slots, read_node_id) {
Some(read_slot) => read_slot
None => return []
}
reads_by_slot.push((read_slot, read_node_id))
}
reads_by_slot.sort_by(fn(a, b) { a.0.compare(b.0) })
let mut read_cursor = 0
for allocation_index, allocation_write in allocation_writes {
let (slot, write_node_id) = allocation_write
let (struct_type, fields, init_node_id) = match
h2l_struct_alloc_write_info(func, module_ctx, write_node_id, local_id) {
Some(info) => info
None => return []
}
let end_slot = if allocation_index + 1 < allocation_writes.length() {
allocation_writes[allocation_index + 1].0
} else {
root_count
}
let use_sites : Array[H2LUseSite] = []
let copy_sites : Array[H2LCopySite] = []
while read_cursor < reads_by_slot.length() &&
reads_by_slot[read_cursor].0 <= slot {
read_cursor += 1
}
while read_cursor < reads_by_slot.length() &&
reads_by_slot[read_cursor].0 <= end_slot {
let read_node_id = reads_by_slot[read_cursor].1
// Operands of the next allocation execute before its local write and
// still read the previous object. Rewrite these boundary reads too.
if !h2l_collect_sequential_source_use(
func,
use_def,
read_node_id,
local_id,
struct_type,
fields.length(),
use_sites,
copy_sites,
) {
return []
}
read_cursor += 1
}
if !use_sites.is_empty() {
candidates.push({
write_node_id,
init_node_id,
fields,
use_sites,
copy_sites,
})
}
}
candidates
}
///|
fn h2l_supported_array_init(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
) -> (@lib.TypeIdx, @lib.FieldType, Int32)? {
match @ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::ArrayNewDefault(type_idx)) =>
match @ir.hot_module_resolve_array_field(module_ctx, type_idx) {
Some(field) =>
if @ir.hot_child_count(func, node_id) != 1 {
None
} else {
match
h2l_i32_const_value(func, @ir.hot_child_get(func, node_id, 0)) {
Some(length) =>
if length >= 0 && length < 20 {
if h2l_default_value_supported(field.unpack()) {
Some((type_idx, field, length))
} else {
None
}
} else {
None
}
None => None
}
}
None => None
}
Some(@lib.Instruction::ArrayNew(type_idx)) =>
match @ir.hot_module_resolve_array_field(module_ctx, type_idx) {
Some(field) =>
if @ir.hot_child_count(func, node_id) != 2 {
None
} else {
match
h2l_i32_const_value(func, @ir.hot_child_get(func, node_id, 1)) {
Some(length) =>
if length >= 0 && length < 20 {
Some((type_idx, field, length))
} else {
None
}
None => None
}
}
None => None
}
Some(@lib.Instruction::ArrayNewFixed(type_idx, @lib.U32(raw_length))) =>
match @ir.hot_module_resolve_array_field(module_ctx, type_idx) {
Some(field) => {
let length = raw_length.reinterpret_as_int()
if length < 20 && @ir.hot_child_count(func, node_id) == length {
Some((type_idx, field, length))
} else {
None
}
}
None => None
}
_ => None
}
}
///|
fn h2l_collect_array_use(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
source_node_id : Int32,
array_type : @lib.TypeIdx,
_length : Int32,
use_sites : Array[H2LArrayUseSite],
) -> Bool {
if !@ir.hot_node_is_live(func, source_node_id) {
return false
}
let sites = use_def.node_uses[source_node_id]
if sites.length() != 1 {
return false
}
let site = sites[0]
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot => if site.slot != 0 { return false }
_ => return false
}
let user_node_id = site.user
let index = match @ir.hot_node_exact_instr(func, user_node_id) {
Some(@lib.Instruction::ArrayGet(type_idx))
| Some(@lib.Instruction::ArrayGetS(type_idx))
| Some(@lib.Instruction::ArrayGetU(type_idx)) =>
if type_idx != array_type || @ir.hot_child_count(func, user_node_id) != 2 {
return false
} else {
match
h2l_i32_const_value(func, @ir.hot_child_get(func, user_node_id, 1)) {
Some(index) => index
None => return false
}
}
Some(@lib.Instruction::ArraySet(type_idx)) =>
if type_idx != array_type || @ir.hot_child_count(func, user_node_id) != 3 {
return false
} else {
match
h2l_i32_const_value(func, @ir.hot_child_get(func, user_node_id, 1)) {
Some(index) => index
None => return false
}
}
_ => return false
}
use_sites.push({
source_node_id,
user_node_id,
index,
is_set: match @ir.hot_node_exact_instr(func, user_node_id) {
Some(@lib.Instruction::ArraySet(_)) => true
_ => false
},
})
true
}
///|
fn h2l_find_array_candidate_for_local(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
use_def : @ir.HotUseDef,
local_id : Int32,
cfg : @ir.HotCfg,
dom : @ir.HotDominators,
order : H2LDominanceOrder,
) -> H2LArrayCandidate? {
if local_id < @ir.hot_param_count(func) {
return None
}
let writes = use_def.local_writes[local_id]
if writes.length() != 1 {
return None
}
let write_node_id = writes[0]
if !@ir.hot_node_is_live(func, write_node_id) {
return None
}
let write_node = @ir.hot_node_get(func, write_node_id)
match write_node.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee =>
if write_node.imm0 != local_id || write_node.child_count != 1 {
return None
}
_ => return None
}
let init_node_id = h2l_allocation_source_node(
func,
@ir.hot_child_get(func, write_node_id, 0),
)
let (array_type, element, length) = match
h2l_supported_array_init(func, module_ctx, init_node_id) {
Some(info) => info
None => return None
}
if !h2l_local_can_hold_array_ref(func, local_id, array_type) {
return None
}
let use_sites : Array[H2LArrayUseSite] = []
if write_node.op is @ir.HotOp::LocalTee &&
!h2l_collect_array_use(
func, use_def, write_node_id, array_type, length, use_sites,
) {
return None
}
for read_node_id in use_def.local_reads[local_id] {
if !h2l_write_dominates_read(cfg, dom, order, write_node_id, read_node_id) {
return None
}
if !h2l_collect_array_use(
func, use_def, read_node_id, array_type, length, use_sites,
) {
return None
}
}
if use_sites.is_empty() {
return None
}
Some({
write_node_id,
init_node_id,
element,
length,
use_sites,
copy_sites: [],
})
}
///|
fn h2l_array_alloc_write_info(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
write_node_id : Int32,
local_id : Int32,
) -> (@lib.TypeIdx, @lib.FieldType, Int32, Int32)? {
if !@ir.hot_node_is_live(func, write_node_id) {
return None
}
let write_node = @ir.hot_node_get(func, write_node_id)
match write_node.op {
@ir.HotOp::LocalSet => ()
_ => return None
}
if write_node.imm0 != local_id || write_node.child_count != 1 {
return None
}
let init_node_id = h2l_allocation_source_node(
func,
@ir.hot_child_get(func, write_node_id, 0),
)
match h2l_supported_array_init(func, module_ctx, init_node_id) {
Some((array_type, element, length)) =>
if h2l_local_can_hold_array_ref(func, local_id, array_type) {
Some((array_type, element, length, init_node_id))
} else {
None
}
None => None
}
}
///|
/// HOT IR preserves local effects as region order rather than data edges. If
/// one scratch local carries more than one array allocation, rewriting another
/// allocation in the same function can let reconstruction move a later scratch
/// write before an earlier wrapper consumes it. This is unsafe even when both
/// arrays use the same eqref carrier because their logical element types can
/// differ. Keep unsafe scratch-carrier functions unchanged. Direct root writes
/// can instead define separate scalar epochs when their old owner is consumed
/// before the next write, including reads inside that next constructor.
fn h2l_array_allocation_write_type(
func : @ir.HotFunc,
write_node_id : Int32,
local_id : Int32,
) -> @lib.TypeIdx? {
let write = @ir.hot_node_get(func, write_node_id)
match write.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee =>
if write.imm0 != local_id || write.child_count != 1 {
return None
}
_ => return None
}
let source = h2l_allocation_source_node(
func,
@ir.hot_child_get(func, write_node_id, 0),
)
match @ir.hot_node_exact_instr(func, source) {
Some(@lib.Instruction::ArrayNew(type_idx))
| Some(@lib.Instruction::ArrayNewDefault(type_idx))
| Some(@lib.Instruction::ArrayNewFixed(type_idx, _)) => Some(type_idx)
_ => None
}
}
///|
fn h2l_repeated_array_allocation_write_locals(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
root_slots : Array[Int32],
) -> Array[Bool] {
let unsafe_locals = Array::make(@ir.hot_local_count(func), false)
for local_id = 0
local_id < @ir.hot_local_count(func)
local_id = local_id + 1 {
let allocation_slots : Array[Int32] = []
let mut missing_slot = false
for write_node_id in use_def.local_writes[local_id] {
match h2l_array_allocation_write_type(func, write_node_id, local_id) {
Some(_) =>
match h2l_root_slot_from_index(root_slots, write_node_id) {
Some(slot) => allocation_slots.push(slot)
None => missing_slot = true
}
None => ()
}
}
if missing_slot {
unsafe_locals[local_id] = true
continue
}
if allocation_slots.length() <= 1 {
continue
}
allocation_slots.sort()
let read_slots : Array[Int32] = []
for read_node_id in use_def.local_reads[local_id] {
match h2l_root_slot_from_index(root_slots, read_node_id) {
Some(read_slot) => read_slots.push(read_slot)
None => missing_slot = true
}
}
if missing_slot {
unsafe_locals[local_id] = true
continue
}
read_slots.sort()
let mut read_cursor = 0
for index = 0; index + 1 < allocation_slots.length(); index = index + 1 {
let write_slot = allocation_slots[index]
let next_write_slot = allocation_slots[index + 1]
while read_cursor < read_slots.length() &&
read_slots[read_cursor] < write_slot {
read_cursor += 1
}
// Constructor operands read the old owner before a direct root write.
// Nested writes and scratch tees do not establish straight-line epochs.
let boundary_read_before_write = write_slot < next_write_slot &&
read_cursor < read_slots.length() &&
read_slots[read_cursor] == next_write_slot &&
@ir.hot_node_op(func, @ir.hot_root_get(func, write_slot))
is @ir.HotOp::LocalSet &&
@ir.hot_node_op(func, @ir.hot_root_get(func, next_write_slot))
is @ir.HotOp::LocalSet &&
h2l_array_allocation_write_type(
func,
@ir.hot_root_get(func, write_slot),
local_id,
).is_some() &&
h2l_array_allocation_write_type(
func,
@ir.hot_root_get(func, next_write_slot),
local_id,
).is_some()
if read_cursor >= read_slots.length() ||
(
read_slots[read_cursor] >= next_write_slot &&
!boundary_read_before_write
) {
unsafe_locals[local_id] = true
break
}
}
}
unsafe_locals
}
///|
fn h2l_collect_sequential_array_source_use(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
source_node_id : Int32,
local_id : Int32,
array_type : @lib.TypeIdx,
length : Int32,
use_sites : Array[H2LArrayUseSite],
copy_sites : Array[H2LCopySite],
) -> Bool {
if !@ir.hot_node_is_live(func, source_node_id) {
return false
}
let sites = use_def.node_uses[source_node_id]
if sites.length() != 1 {
return false
}
let site = sites[0]
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot => ()
_ => return false
}
if site.slot == 0 && h2l_self_copy_write_matches(func, site.user, local_id) {
copy_sites.push({ source_node_id, write_node_id: site.user, })
return true
}
h2l_collect_array_use(
func, use_def, source_node_id, array_type, length, use_sites,
)
}
///|
fn h2l_find_sequential_array_candidates_for_local(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
use_def : @ir.HotUseDef,
local_id : Int32,
root_slots : Array[Int32],
) -> Array[H2LArrayCandidate] {
let candidates : Array[H2LArrayCandidate] = []
if local_id < @ir.hot_param_count(func) {
return candidates
}
let writes = use_def.local_writes[local_id]
if writes.length() <= 1 {
return candidates
}
let allocation_writes : Array[(Int32, Int32)] = []
for write_node_id in writes {
let is_allocation = h2l_array_alloc_write_info(
func, module_ctx, write_node_id, local_id,
).is_some()
let is_self_copy = h2l_self_copy_write_matches(
func, write_node_id, local_id,
)
if !is_allocation && !is_self_copy {
return []
}
match h2l_root_slot_from_index(root_slots, write_node_id) {
Some(root_slot) => {
// Conditional/nested allocation writes do not define straight-line
// epochs. A later read may observe either owner, not this candidate.
if is_allocation &&
@ir.hot_region_get(func, @ir.HotRegionRef::root(), root_slot) !=
write_node_id {
return []
}
if is_allocation {
allocation_writes.push((root_slot, write_node_id))
}
}
None => return []
}
}
let allocation_writes = h2l_sort_sequential_write_slots(allocation_writes)
let root_count = @ir.hot_root_count(func)
let reads_by_slot : Array[(Int32, Int32)] = []
for read_node_id in use_def.local_reads[local_id] {
let read_slot = match h2l_root_slot_from_index(root_slots, read_node_id) {
Some(read_slot) => read_slot
None => return []
}
reads_by_slot.push((read_slot, read_node_id))
}
reads_by_slot.sort_by(fn(a, b) { a.0.compare(b.0) })
let mut read_cursor = 0
for allocation_index, allocation_write in allocation_writes {
let (slot, write_node_id) = allocation_write
let (array_type, element, length, init_node_id) = match
h2l_array_alloc_write_info(func, module_ctx, write_node_id, local_id) {
Some(info) => info
None => return []
}
let end_slot = if allocation_index + 1 < allocation_writes.length() {
allocation_writes[allocation_index + 1].0
} else {
root_count
}
let use_sites : Array[H2LArrayUseSite] = []
let copy_sites : Array[H2LCopySite] = []
while read_cursor < reads_by_slot.length() &&
reads_by_slot[read_cursor].0 <= slot {
read_cursor += 1
}
while read_cursor < reads_by_slot.length() &&
reads_by_slot[read_cursor].0 <= end_slot {
let read_node_id = reads_by_slot[read_cursor].1
// Operands of the next allocation execute before its local write and
// still read the previous object. Rewrite these boundary reads too.
if !h2l_collect_sequential_array_source_use(
func, use_def, read_node_id, local_id, array_type, length, use_sites, copy_sites,
) {
return []
}
read_cursor += 1
}
if !use_sites.is_empty() {
candidates.push({
write_node_id,
init_node_id,
element,
length,
use_sites,
copy_sites,
})
}
}
candidates
}
///|
fn h2l_scalar_local_type(type_ : @lib.ValType) -> @lib.ValType {
match type_ {
@lib.RefTypeValType(ref_type) =>
@lib.ValType::ref_type(ref_type.make_nullable())
_ => type_
}
}
///|
fn h2l_build_scalar_local_get(
func : @ir.HotFunc,
local_id : Int32,
value_type : @lib.ValType,
) -> Int32 {
let read = @ir.hot_build_local_get(func, local_id)
match value_type {
@lib.RefTypeValType(ref_type) if ref_type.is_non_nullable() => {
let non_null = @ir.hot_build_unary(func, value_type, read)
@ir.hot_node_exact_instr_set(
func,
non_null,
@lib.Instruction::ref_as_non_null(),
)
non_null
}
_ => read
}
}
///|
fn h2l_alloc_field_locals(
func : @ir.HotFunc,
fields : Array[@lib.FieldType],
) -> Array[Int32] {
let field_locals : Array[Int32] = []
for field in fields {
let local_id = @ir.hot_local_count(func)
// Scalar writes can be nested in a value block while later field reads
// lie outside that block. Wasm does not carry non-defaultable local-init
// facts across that boundary; restore the field's non-null type at reads.
func.locals.locals.push(h2l_scalar_local_type(field.unpack()))
field_locals.push(local_id)
}
field_locals
}
///|
fn h2l_alloc_array_element_locals(
func : @ir.HotFunc,
element : @lib.FieldType,
length : Int32,
) -> Array[Int32] {
let element_locals : Array[Int32] = []
for _ in 0.. Int32? {
let init_roots : Array[Int32] = []
match @ir.hot_node_exact_instr(func, candidate.init_node_id) {
Some(@lib.Instruction::StructNew(_)) =>
for idx = 0; idx < candidate.fields.length(); idx = idx + 1 {
init_roots.push(
@ir.hot_build_local_set(
func,
field_locals[idx],
@ir.hot_child_get(func, candidate.init_node_id, idx),
),
)
}
Some(@lib.Instruction::StructNewDefault(_)) =>
for idx = 0; idx < candidate.fields.length(); idx = idx + 1 {
let init_value = match
h2l_build_default_value(func, candidate.fields[idx].unpack()) {
Some(node_id) => node_id
None => return None
}
init_roots.push(
@ir.hot_build_local_set(func, field_locals[idx], init_value),
)
}
_ => return None
}
Some(
if init_roots.length() == 1 {
init_roots[0]
} else {
@ir.hot_build_block(func, @lib.BlockType::void_(), init_roots)
},
)
}
///|
fn h2l_build_array_init_replacement(
func : @ir.HotFunc,
candidate : H2LArrayCandidate,
element_locals : Array[Int32],
) -> Int32? {
let init_roots : Array[Int32] = []
match @ir.hot_node_exact_instr(func, candidate.init_node_id) {
Some(@lib.Instruction::ArrayNewDefault(_)) =>
for element_local in element_locals {
let init_value = match
h2l_build_default_value(func, candidate.element.unpack()) {
Some(node_id) => node_id
None => return None
}
init_roots.push(
@ir.hot_build_local_set(func, element_local, init_value),
)
}
Some(@lib.Instruction::ArrayNew(_)) => {
let repeated_local = @ir.hot_local_count(func)
func.locals.locals.push(h2l_scalar_local_type(candidate.element.unpack()))
init_roots.push(
@ir.hot_build_local_set(
func,
repeated_local,
@ir.hot_child_get(func, candidate.init_node_id, 0),
),
)
for element_local in element_locals {
init_roots.push(
@ir.hot_build_local_set(
func,
element_local,
@ir.hot_build_local_get(func, repeated_local),
),
)
}
}
Some(@lib.Instruction::ArrayNewFixed(_, _)) =>
for idx = 0; idx < element_locals.length(); idx = idx + 1 {
init_roots.push(
@ir.hot_build_local_set(
func,
element_locals[idx],
@ir.hot_child_get(func, candidate.init_node_id, idx),
),
)
}
_ => return None
}
Some(
if init_roots.is_empty() {
@ir.hot_build_nop(func)
} else if init_roots.length() == 1 {
init_roots[0]
} else {
@ir.hot_build_block(func, @lib.BlockType::void_(), init_roots)
},
)
}
///|
fn h2l_wrap_replacement_with_init(
func : @ir.HotFunc,
candidate : H2LCandidate,
init_replacement : Int32,
field_idx : Int32,
replacement_id : Int32,
) -> Int32 {
let block_type = match @ir.hot_node_result_arity(func, replacement_id) {
0 => @lib.BlockType::void_()
_ => @lib.BlockType::val_type(candidate.fields[field_idx].unpack())
}
@ir.hot_build_block(func, block_type, [init_replacement, replacement_id])
}
///|
fn h2l_passthrough_child_node(
func : @ir.HotFunc,
source_node_id : Int32,
) -> Int32? {
match @ir.hot_node_exact_instr(func, source_node_id) {
Some(@lib.Instruction::RefAsNonNull) =>
if @ir.hot_child_count(func, source_node_id) == 1 {
Some(@ir.hot_child_get(func, source_node_id, 0))
} else {
None
}
Some(@lib.Instruction::RefCast(_, _, _))
| Some(@lib.Instruction::AnyConvertExtern)
| Some(@lib.Instruction::ExternConvertAny) =>
if @ir.hot_child_count(func, source_node_id) == 1 {
Some(@ir.hot_child_get(func, source_node_id, 0))
} else {
None
}
_ => None
}
}
///|
fn h2l_source_requires_tee_init(
func : @ir.HotFunc,
source_node_id : Int32,
write_node_id : Int32,
) -> Bool {
if source_node_id == write_node_id {
return true
}
match h2l_passthrough_child_node(func, source_node_id) {
Some(child_node_id) => child_node_id == write_node_id
None => false
}
}
///|
fn h2l_node_is_block(func : @ir.HotFunc, node_id : Int32) -> Bool {
match @ir.hot_node_get(func, node_id).op {
@ir.HotOp::Block => true
_ => false
}
}
///|
fn h2l_block_flow_tail_node(
func : @ir.HotFunc,
block_node_id : Int32,
) -> Int32? {
let block_node = @ir.hot_node_get(func, block_node_id)
match block_node.op {
@ir.HotOp::Block => ()
_ => return None
}
let body_ref = @ir.hot_region_body_ref(func, block_node_id)
let root_count = @ir.hot_region_root_count(func, body_ref)
if root_count == 0 {
None
} else {
Some(@ir.hot_region_get(func, body_ref, root_count - 1))
}
}
///|
fn h2l_wrap_replacement_with_block_prefix(
func : @ir.HotFunc,
candidate : H2LCandidate,
block_node_id : Int32,
field_idx : Int32,
replacement_id : Int32,
) -> Int32? {
let tail_node_id = match h2l_block_flow_tail_node(func, block_node_id) {
Some(node_id) => node_id
None => return None
}
if tail_node_id == block_node_id {
return None
}
let body_ref = @ir.hot_region_body_ref(func, block_node_id)
let root_count = @ir.hot_region_root_count(func, body_ref)
let roots : Array[Int32] = []
for root_idx = 0; root_idx < root_count - 1; root_idx = root_idx + 1 {
roots.push(@ir.hot_region_get(func, body_ref, root_idx))
}
roots.push(replacement_id)
let block_type = match @ir.hot_node_result_arity(func, replacement_id) {
0 => @lib.BlockType::void_()
_ => @lib.BlockType::val_type(candidate.fields[field_idx].unpack())
}
Some(@ir.hot_build_block(func, block_type, roots))
}
///|
fn h2l_ref_eq_null_replacement(
func : @ir.HotFunc,
alloc_node_id : Int32,
) -> Int32 {
let roots : Array[Int32] = []
let child_count = @ir.hot_child_count(func, alloc_node_id)
for child_idx = 0; child_idx < child_count; child_idx = child_idx + 1 {
roots.push(
@ir.hot_build_drop(
func,
@ir.hot_child_get(func, alloc_node_id, child_idx),
),
)
}
roots.push(@ir.hot_build_const_i32(func, @lib.I32(0)))
if roots.length() == 1 {
roots[0]
} else {
@ir.hot_build_block(
func,
@lib.BlockType::val_type(@lib.ValType::i32()),
roots,
)
}
}
///|
fn h2l_array_ref_test_replacement(
func : @ir.HotFunc,
alloc_node_id : Int32,
) -> Int32 {
let roots : Array[Int32] = []
let child_count = @ir.hot_child_count(func, alloc_node_id)
for child_idx = 0; child_idx < child_count; child_idx = child_idx + 1 {
roots.push(
@ir.hot_build_drop(
func,
@ir.hot_child_get(func, alloc_node_id, child_idx),
),
)
}
roots.push(@ir.hot_build_const_i32(func, @lib.I32(1)))
if roots.length() == 1 {
roots[0]
} else {
@ir.hot_build_block(
func,
@lib.BlockType::val_type(@lib.ValType::i32()),
roots,
)
}
}
///|
fn h2l_ref_get_desc_replacement(
func : @ir.HotFunc,
alloc_node_id : Int32,
) -> Int32 {
let child_count = @ir.hot_child_count(func, alloc_node_id)
if child_count <= 0 {
return @ir.hot_build_unreachable(func)
}
let roots : Array[Int32] = []
for child_idx = 0; child_idx < child_count - 1; child_idx = child_idx + 1 {
roots.push(
@ir.hot_build_drop(
func,
@ir.hot_child_get(func, alloc_node_id, child_idx),
),
)
}
let desc_node_id = @ir.hot_child_get(func, alloc_node_id, child_count - 1)
let desc_result_type = @ir.hot_type_results(
func,
@ir.hot_node_type(func, desc_node_id),
)[0]
let (checked_desc, checked_type) = match desc_result_type {
@lib.RefTypeValType(rt) if rt.is_nullable() => {
let checked_type = @lib.ValType::ref_type(
@lib.RefType::new(false, rt.get_heap_type(), exact=rt.is_exact()),
)
let checked_desc = @ir.hot_build_unary(func, checked_type, desc_node_id)
@ir.hot_node_exact_instr_set(
func,
checked_desc,
@lib.Instruction::ref_as_non_null(),
)
(checked_desc, checked_type)
}
_ => (desc_node_id, desc_result_type)
}
roots.push(checked_desc)
if roots.length() == 1 {
roots[0]
} else {
@ir.hot_build_block(func, @lib.BlockType::val_type(checked_type), roots)
}
}
///|
fn h2l_direct_packed_read_replacement(
func : @ir.HotFunc,
storage : @lib.StorageType,
signed_access : Bool?,
value_node_id : Int32,
) -> Int32? {
match storage {
@lib.ValTypeStorageType(_) =>
match signed_access {
None => Some(value_node_id)
Some(_) => None
}
@lib.PackTypeStorageType(pack_type) => {
let signed = match signed_access {
Some(signed) => signed
None => return None
}
let raw = match h2l_i32_const_value(func, value_node_id) {
Some(raw) => raw
None => return None
}
let folded = match (pack_type, signed) {
(@lib.I8PackType, true) => raw << 24 >> 24
(@lib.I8PackType, false) => raw & 0xff
(@lib.I16PackType, true) => raw << 16 >> 16
(@lib.I16PackType, false) => raw & 0xffff
}
Some(@ir.hot_build_const_i32(func, @lib.I32(folded)))
}
}
}
///|
fn h2l_direct_unused_struct_child_is_removable(
func : @ir.HotFunc,
node_id : Int32,
) -> Bool {
match @ir.hot_node_exact_instr(func, node_id) {
Some(
@lib.I32Const(_)
| @lib.I64Const(_)
| @lib.F32Const(_)
| @lib.F64Const(_)
| @lib.V128Const(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)
| @lib.RefNull(_)
| @lib.RefFunc(_)
) => true
_ => false
}
}
///|
fn h2l_try_fold_direct_struct_get(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
detached_nodes : Array[Int32],
) -> Bool {
if !@ir.hot_node_is_live(func, node_id) ||
@ir.hot_child_count(func, node_id) != 1 {
return false
}
let (access_type, field_idx, signed_access) = match
@ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::StructGet(type_idx, @lib.U32(raw_idx))) =>
(type_idx, raw_idx.reinterpret_as_int(), None)
Some(@lib.Instruction::StructGetS(type_idx, @lib.U32(raw_idx))) =>
(type_idx, raw_idx.reinterpret_as_int(), Some(true))
Some(@lib.Instruction::StructGetU(type_idx, @lib.U32(raw_idx))) =>
(type_idx, raw_idx.reinterpret_as_int(), Some(false))
_ => return false
}
let passthrough_nodes : Array[Int32] = []
let mut alloc_node_id = @ir.hot_child_get(func, node_id, 0)
while true {
match h2l_passthrough_child_node(func, alloc_node_id) {
Some(child_node_id) if child_node_id != alloc_node_id => {
passthrough_nodes.push(alloc_node_id)
alloc_node_id = child_node_id
}
_ => break
}
}
let (alloc_type, fields, _) = match
h2l_supported_struct_new_fields(func, module_ctx, alloc_node_id) {
Some(info) => info
None => return false
}
if alloc_type != access_type || field_idx < 0 || field_idx >= fields.length() {
return false
}
let allocation_ref = @lib.RefType::new(
false,
@lib.HeapType::new(alloc_type),
exact=true,
)
for wrapper in passthrough_nodes {
match @ir.hot_node_exact_instr(func, wrapper) {
Some(@lib.RefCast(nullable, exact, heap_type)) =>
if !oc_ref_always_matches(
module_ctx,
allocation_ref,
oc_target_ref_type(nullable, exact, heap_type),
) {
return false
}
_ => ()
}
}
let @lib.FieldType(storage, _) = fields[field_idx]
let original_node_count = @ir.hot_node_count(func)
let value_node_id = match @ir.hot_node_exact_instr(func, alloc_node_id) {
Some(@lib.Instruction::StructNewDefault(_)) =>
match h2l_build_default_value(func, fields[field_idx].unpack()) {
Some(node_id) => node_id
None => return false
}
Some(@lib.Instruction::StructNew(_)) if @ir.hot_child_count(
func, alloc_node_id,
) ==
fields.length() => {
for child_idx = 0; child_idx < fields.length(); child_idx = child_idx + 1 {
if child_idx != field_idx &&
!h2l_direct_unused_struct_child_is_removable(
func,
@ir.hot_child_get(func, alloc_node_id, child_idx),
) {
return false
}
}
@ir.hot_child_get(func, alloc_node_id, field_idx)
}
_ => return false
}
let replacement = match
h2l_direct_packed_read_replacement(
func, storage, signed_access, value_node_id,
) {
Some(node_id) => node_id
None => return false
}
h2l_replace_direct_value(func, node_id, replacement, original_node_count)
for passthrough_node in passthrough_nodes {
detached_nodes.push(passthrough_node)
}
detached_nodes.push(alloc_node_id)
// Cleanup retains borrowed values with incoming users, including a retained
// control child of the replacement wrapper.
detached_nodes.push(replacement)
true
}
///|
fn h2l_try_fold_direct_ref_is_null(
func : @ir.HotFunc,
node_id : Int32,
detached_nodes : Array[Int32],
) -> Bool {
if !@ir.hot_node_is_live(func, node_id) ||
@ir.hot_child_count(func, node_id) != 1 ||
(
!(@ir.hot_node_op(func, node_id) is @ir.HotOp::RefIsNull) &&
@ir.hot_node_exact_instr(func, node_id) !=
Some(@lib.Instruction::ref_is_null())
) {
return false
}
let operand = @ir.hot_child_get(func, node_id, 0)
match @ir.hot_node_exact_instr(func, operand) {
Some(@lib.StructNewDefault(_)) if @ir.hot_child_count(func, operand) == 0 =>
()
_ => return false
}
let replacement = @ir.hot_build_const_i32(func, @lib.I32(0))
@ir.hot_replace_node(func, node_id, @ir.hot_node_get(func, replacement))
detached_nodes.push(operand)
true
}
///|
fn h2l_try_fold_direct_array_get(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
detached_nodes : Array[Int32],
) -> Bool {
if !@ir.hot_node_is_live(func, node_id) ||
@ir.hot_child_count(func, node_id) != 2 {
return false
}
let (access_type, signed_access) = match
@ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::ArrayGet(type_idx)) => (type_idx, None)
Some(@lib.Instruction::ArrayGetS(type_idx)) => (type_idx, Some(true))
Some(@lib.Instruction::ArrayGetU(type_idx)) => (type_idx, Some(false))
_ => return false
}
let alloc_node_id = @ir.hot_child_get(func, node_id, 0)
let index_node_id = @ir.hot_child_get(func, node_id, 1)
let index = match h2l_i32_const_value(func, index_node_id) {
Some(index) => index
None => return false
}
let (array_type, field, length) = match
h2l_supported_array_init(func, module_ctx, alloc_node_id) {
Some(info) => info
None => return false
}
if array_type != access_type {
return false
}
let @lib.FieldType(storage, _) = field
let original_node_count = @ir.hot_node_count(func)
let replacement = if index < 0 || index >= length {
match @ir.hot_node_exact_instr(func, alloc_node_id) {
Some(@lib.Instruction::ArrayNewFixed(_, _)) => {
let roots : Array[Int32] = []
for child_idx = 0
child_idx < @ir.hot_child_count(func, alloc_node_id)
child_idx = child_idx + 1 {
roots.push(
@ir.hot_build_drop(
func,
@ir.hot_child_get(func, alloc_node_id, child_idx),
),
)
}
roots.push(@ir.hot_build_unreachable(func))
@ir.hot_build_block(
func,
@lib.BlockType::val_type(field.unpack()),
roots,
)
}
_ => return false
}
} else {
let value_node_id = match @ir.hot_node_exact_instr(func, alloc_node_id) {
Some(@lib.Instruction::ArrayNewFixed(_, _)) if length == 1 && index == 0 =>
@ir.hot_child_get(func, alloc_node_id, 0)
Some(@lib.Instruction::ArrayNew(_)) =>
@ir.hot_child_get(func, alloc_node_id, 0)
Some(@lib.Instruction::ArrayNewDefault(_)) =>
match h2l_build_default_value(func, field.unpack()) {
Some(node_id) => node_id
None => return false
}
_ => return false
}
match
h2l_direct_packed_read_replacement(
func, storage, signed_access, value_node_id,
) {
Some(node_id) => node_id
None => return false
}
}
h2l_replace_direct_value(func, node_id, replacement, original_node_count)
detached_nodes.push(alloc_node_id)
// Cleanup retains borrowed values with incoming users, including a retained
// control child of the replacement wrapper.
detached_nodes.push(replacement)
detached_nodes.push(index_node_id)
match @ir.hot_node_exact_instr(func, alloc_node_id) {
Some(@lib.Instruction::ArrayNew(_))
| Some(@lib.Instruction::ArrayNewDefault(_)) =>
detached_nodes.push(
@ir.hot_child_get(
func,
alloc_node_id,
@ir.hot_child_count(func, alloc_node_id) - 1,
),
)
_ => ()
}
true
}
///|
fn h2l_try_fold_direct_ref_eq(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
detached_nodes : Array[Int32],
) -> Bool {
if !@ir.hot_node_is_live(func, node_id) {
return false
}
match @ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::RefEq) => ()
_ => return false
}
if @ir.hot_child_count(func, node_id) != 2 {
return false
}
let lhs = @ir.hot_child_get(func, node_id, 0)
let rhs = @ir.hot_child_get(func, node_id, 1)
let lhs_alloc = h2l_is_direct_struct_alloc(func, module_ctx, lhs)
let rhs_alloc = h2l_is_direct_struct_alloc(func, module_ctx, rhs)
let lhs_is_null = match @ir.hot_node_get(func, lhs).op {
@ir.HotOp::RefNull => true
_ => false
}
let rhs_is_null = match @ir.hot_node_get(func, rhs).op {
@ir.HotOp::RefNull => true
_ => false
}
let (alloc_node_id, null_node_id) = match (lhs_alloc, rhs_is_null) {
(true, true) => (lhs, rhs)
_ =>
match (rhs_alloc, lhs_is_null) {
(true, true) => (rhs, lhs)
_ => return false
}
}
let replacement = h2l_ref_eq_null_replacement(func, alloc_node_id)
h2l_replace_with_built(func, node_id, replacement)
detached_nodes.push(alloc_node_id)
detached_nodes.push(null_node_id)
true
}
///|
fn h2l_try_fold_direct_struct_ref_test(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
detached_nodes : Array[Int32],
) -> Bool {
if !@ir.hot_node_is_live(func, node_id) {
return false
}
let target_heap_type = match @ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::RefTest(_, false, heap_type)) => heap_type
_ => return false
}
if @ir.hot_child_count(func, node_id) != 1 {
return false
}
let alloc_node_id = @ir.hot_child_get(func, node_id, 0)
let (struct_type, _, _) = match
h2l_supported_struct_new_fields(func, module_ctx, alloc_node_id) {
Some(info) => info
None => return false
}
if target_heap_type != @lib.HeapType::new(struct_type) {
return false
}
let replacement = h2l_array_ref_test_replacement(func, alloc_node_id)
h2l_replace_with_built(func, node_id, replacement)
detached_nodes.push(alloc_node_id)
true
}
///|
fn h2l_try_fold_direct_ref_get_desc(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
detached_nodes : Array[Int32],
) -> Bool {
if !@ir.hot_node_is_live(func, node_id) {
return false
}
let target_type = match @ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::RefGetDesc(type_idx)) => type_idx
_ => return false
}
if @ir.hot_child_count(func, node_id) != 1 {
return false
}
let alloc_node_id = @ir.hot_child_get(func, node_id, 0)
let (alloc_type, _) = match
h2l_direct_descriptor_alloc_info(func, module_ctx, alloc_node_id) {
Some(info) => info
None => return false
}
if alloc_type != target_type {
return false
}
let original_node_count = @ir.hot_node_count(func)
let replacement = h2l_ref_get_desc_replacement(func, alloc_node_id)
h2l_replace_direct_value(func, node_id, replacement, original_node_count)
detached_nodes.push(alloc_node_id)
true
}
///|
fn h2l_try_fold_direct_array_ref_test(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
detached_nodes : Array[Int32],
) -> Bool {
if !@ir.hot_node_is_live(func, node_id) {
return false
}
let target_heap_type = match @ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::RefTest(_, false, heap_type)) => heap_type
_ => return false
}
if @ir.hot_child_count(func, node_id) != 1 {
return false
}
let alloc_node_id = @ir.hot_child_get(func, node_id, 0)
let (array_type, _, _) = match
h2l_supported_array_init(func, module_ctx, alloc_node_id) {
Some(info) => info
None => return false
}
if target_heap_type != @lib.HeapType::new(array_type) {
return false
}
let replacement = h2l_array_ref_test_replacement(func, alloc_node_id)
h2l_replace_with_built(func, node_id, replacement)
detached_nodes.push(alloc_node_id)
true
}
///|
fn h2l_build_array_local_read(
func : @ir.HotFunc,
element : @lib.FieldType,
local_idx : Int32,
access : @lib.Instruction?,
) -> Int32? {
let local_get = h2l_build_scalar_local_get(func, local_idx, element.unpack())
match (element.get_storage_type(), access) {
(@lib.ValTypeStorageType(_), Some(@lib.ArrayGet(_) | @lib.StructGet(_, _))) =>
Some(local_get)
(
@lib.PackTypeStorageType(@lib.I8PackType),
Some(@lib.ArrayGetS(_) | @lib.StructGetS(_, _)),
) => {
let extended = @ir.hot_build_unary(func, @lib.ValType::i32(), local_get)
@ir.hot_node_exact_instr_set(
func,
extended,
@lib.Instruction::i32_extend8s(),
)
Some(extended)
}
(
@lib.PackTypeStorageType(@lib.I16PackType),
Some(@lib.ArrayGetS(_) | @lib.StructGetS(_, _)),
) => {
let extended = @ir.hot_build_unary(func, @lib.ValType::i32(), local_get)
@ir.hot_node_exact_instr_set(
func,
extended,
@lib.Instruction::i32_extend16s(),
)
Some(extended)
}
(
@lib.PackTypeStorageType(pack_type),
Some(@lib.ArrayGetU(_) | @lib.StructGetU(_, _)),
) => {
let mask = @ir.hot_build_const_i32(
func,
@lib.I32(
match pack_type {
@lib.I8PackType => 0xff
@lib.I16PackType => 0xffff
},
),
)
let masked = @ir.hot_build_binary(
func,
@lib.ValType::i32(),
local_get,
mask,
)
@ir.hot_node_exact_instr_set(func, masked, @lib.Instruction::i32_and())
Some(masked)
}
_ => None
}
}
///|
fn h2l_apply_array_candidate(
func : @ir.HotFunc,
candidate : H2LArrayCandidate,
detached_nodes : Array[Int32],
) -> Bool {
let element_locals = h2l_alloc_array_element_locals(
func,
candidate.element,
candidate.length,
)
let init_replacement = match
h2l_build_array_init_replacement(func, candidate, element_locals) {
Some(node_id) => node_id
None => return false
}
let write_is_tee = @ir.hot_node_op(func, candidate.write_node_id)
is @ir.HotOp::LocalTee
if write_is_tee {
// Retain the tee's execution position. A fresh region order would make
// lowering carry the already lifted store value before construction.
@ir.hot_preserve_replacement_position(
func,
init_replacement,
candidate.write_node_id,
)
} else {
let write_source_node_id = @ir.hot_child_get(
func,
candidate.write_node_id,
0,
)
h2l_replace_with_built(func, candidate.write_node_id, init_replacement)
if write_source_node_id == candidate.init_node_id {
detached_nodes.push(candidate.init_node_id)
}
}
for copy_site in candidate.copy_sites {
@ir.hot_replace_node(
func,
copy_site.write_node_id,
@ir.hot_node_get(func, @ir.hot_build_nop(func)),
)
detached_nodes.push(copy_site.source_node_id)
}
for use_site in candidate.use_sites {
let replacement = if use_site.index < 0 ||
use_site.index >= candidate.length {
// A fresh fixed-length owner cannot fail a null check. Keep the known
// bounds trap after evaluating an array.set value, when one is present.
let roots : Array[Int32] = []
if use_site.is_set {
roots.push(
@ir.hot_build_drop(
func,
@ir.hot_child_get(func, use_site.user_node_id, 2),
),
)
}
roots.push(@ir.hot_build_unreachable(func))
Some(
@ir.hot_build_block(
func,
if use_site.is_set {
@lib.BlockType::void_()
} else {
@lib.BlockType::val_type(candidate.element.unpack())
},
roots,
),
)
} else if use_site.is_set {
Some(
@ir.hot_build_local_set(
func,
element_locals[use_site.index],
@ir.hot_child_get(func, use_site.user_node_id, 2),
),
)
} else {
h2l_build_array_local_read(
func,
candidate.element,
element_locals[use_site.index],
@ir.hot_node_exact_instr(func, use_site.user_node_id),
)
}
match replacement {
Some(replacement_id) => {
// A tee evaluates the allocation at this consuming access, before its
// value operand and any bounds trap. Other local reads must not repeat it.
let replacement_id = if write_is_tee &&
use_site.source_node_id == candidate.write_node_id {
let wrapper = @ir.hot_build_block(
func,
if use_site.is_set {
@lib.BlockType::void_()
} else {
@lib.BlockType::val_type(candidate.element.unpack())
},
[init_replacement, replacement_id],
)
@ir.hot_preserve_replacement_position(
func,
wrapper,
use_site.user_node_id,
)
wrapper
} else {
replacement_id
}
h2l_replace_with_built(func, use_site.user_node_id, replacement_id)
detached_nodes.push(use_site.source_node_id)
}
None => return false
}
}
if write_is_tee {
detached_nodes.push(candidate.write_node_id)
if @ir.hot_child_get(func, candidate.write_node_id, 0) ==
candidate.init_node_id {
detached_nodes.push(candidate.init_node_id)
}
}
true
}
///|
fn h2l_replace_with_built(
func : @ir.HotFunc,
target_id : Int32,
replacement_id : Int32,
) -> Unit {
let replacement = @ir.hot_node_get(func, replacement_id)
if @ir.hot_node_kind_is_control(func, replacement_id) {
@ir.hot_label_owner_set(
func,
@ir.hot_control_node_label(func, replacement_id),
target_id,
)
}
// This fresh wrapper has no incoming use. Transfer its child span rather
// than leave a second live owner whose body later passes can also rewrite.
@ir.hot_delete_detached_node(func, replacement_id)
@ir.hot_replace_node(func, target_id, replacement)
}
///|
fn h2l_replace_direct_value(
func : @ir.HotFunc,
target_id : Int32,
replacement_id : Int32,
original_node_count : Int32,
) -> Unit {
if replacement_id >= original_node_count {
h2l_replace_with_built(func, target_id, replacement_id)
} else if @ir.hot_node_kind_is_control(func, replacement_id) {
// This value may have another live user. Retain its original control and
// label as a child, preserving its evaluation behavior for other consumers.
let value_type = @ir.hot_type_results(
func,
@ir.hot_node_type(func, replacement_id),
)[0]
let wrapper = @ir.hot_build_block(
func,
@lib.BlockType::val_type(value_type),
[replacement_id],
)
@ir.hot_preserve_replacement_position(func, wrapper, replacement_id)
h2l_replace_with_built(func, target_id, wrapper)
@ir.hot_preserve_replacement_position(func, target_id, replacement_id)
} else {
@ir.hot_replace_node(
func,
target_id,
@ir.hot_node_get(func, replacement_id),
)
}
}
///|
fn h2l_apply_candidate(
func : @ir.HotFunc,
candidate : H2LCandidate,
detached_nodes : Array[Int32],
) -> Bool {
let field_locals = h2l_alloc_field_locals(func, candidate.fields)
let init_replacement = match
h2l_build_init_replacement(func, candidate, field_locals) {
Some(node_id) => node_id
None => return false
}
let write_is_tee = match @ir.hot_node_get(func, candidate.write_node_id).op {
@ir.HotOp::LocalTee => true
_ => false
}
if !write_is_tee {
let write_source_node_id = @ir.hot_child_get(
func,
candidate.write_node_id,
0,
)
h2l_replace_with_built(func, candidate.write_node_id, init_replacement)
if write_source_node_id == candidate.init_node_id {
detached_nodes.push(candidate.init_node_id)
}
}
for copy_site in candidate.copy_sites {
@ir.hot_replace_node(
func,
copy_site.write_node_id,
@ir.hot_node_get(func, @ir.hot_build_nop(func)),
)
detached_nodes.push(copy_site.source_node_id)
}
for use_site in candidate.use_sites {
let replacement = match
@ir.hot_node_exact_instr(func, use_site.user_node_id) {
Some(instr) =>
match h2l_field_index(instr) {
Some(field_idx) =>
match instr {
@lib.StructGet(_, _)
| @lib.StructGetS(_, _)
| @lib.StructGetU(_, _) =>
Some(
{
let local_get = match
h2l_build_array_local_read(
func,
candidate.fields[field_idx],
field_locals[field_idx],
Some(instr),
) {
Some(node) => node
None => return false
}
if write_is_tee &&
h2l_source_requires_tee_init(
func,
use_site.source_node_id,
candidate.write_node_id,
) {
h2l_wrap_replacement_with_init(
func, candidate, init_replacement, field_idx, local_get,
)
} else if h2l_node_is_block(func, use_site.source_node_id) {
match
h2l_wrap_replacement_with_block_prefix(
func,
candidate,
use_site.source_node_id,
field_idx,
local_get,
) {
Some(node_id) => node_id
None => return false
}
} else {
local_get
}
},
)
@lib.StructSet(_, _) =>
Some(
{
let local_set = @ir.hot_build_local_set(
func,
field_locals[field_idx],
@ir.hot_child_get(func, use_site.user_node_id, 1),
)
if write_is_tee &&
h2l_source_requires_tee_init(
func,
use_site.source_node_id,
candidate.write_node_id,
) {
h2l_wrap_replacement_with_init(
func, candidate, init_replacement, field_idx, local_set,
)
} else if h2l_node_is_block(func, use_site.source_node_id) {
match
h2l_wrap_replacement_with_block_prefix(
func,
candidate,
use_site.source_node_id,
field_idx,
local_set,
) {
Some(node_id) => node_id
None => return false
}
} else {
local_set
}
},
)
_ => None
}
None => None
}
None => None
}
match replacement {
Some(replacement_id) => {
h2l_replace_with_built(func, use_site.user_node_id, replacement_id)
detached_nodes.push(use_site.source_node_id)
match h2l_passthrough_child_node(func, use_site.source_node_id) {
Some(child_node_id) => detached_nodes.push(child_node_id)
None => ()
}
if h2l_node_is_block(func, use_site.source_node_id) {
match h2l_block_flow_tail_node(func, use_site.source_node_id) {
Some(tail_node_id) => detached_nodes.push(tail_node_id)
None => return false
}
}
}
None => return false
}
}
if write_is_tee {
detached_nodes.push(candidate.write_node_id)
if @ir.hot_child_get(func, candidate.write_node_id, 0) ==
candidate.init_node_id {
detached_nodes.push(candidate.init_node_id)
}
}
true
}
///|
fn h2l_variant_struct_get_field(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
node_id : Int32,
) -> H2LVariantField? {
if !@ir.hot_node_is_live(func, node_id) {
return None
}
let (type_idx, field_index) = match @ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::StructGet(type_idx, @lib.U32(raw_index))) =>
(type_idx, raw_index.reinterpret_as_int())
_ => return None
}
let fields = match
@ir.hot_module_resolve_struct_fields(module_ctx, type_idx) {
Some(fields) => fields
None => return None
}
if field_index < 0 || field_index >= fields.length() {
return None
}
let @lib.FieldType(storage, _) = fields[field_index]
match storage {
@lib.ValTypeStorageType(type_) => Some({ index: field_index, type_, })
@lib.PackTypeStorageType(_) => None
}
}
///|
fn h2l_variant_backward_to_tee(
func : @ir.HotFunc,
user_node_id : Int32,
) -> (Int32, Array[Int32])? {
if @ir.hot_child_count(func, user_node_id) != 1 {
return None
}
let path : Array[Int32] = []
let mut current = @ir.hot_child_get(func, user_node_id, 0)
while true {
path.push(current)
match @ir.hot_node_get(func, current).op {
@ir.HotOp::LocalTee => return Some((current, path))
_ =>
match h2l_passthrough_child_node(func, current) {
Some(child) => current = child
None => return None
}
}
}
None
}
///|
fn h2l_variant_forward_to_struct_get(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
source_node_id : Int32,
) -> (Int32, Array[Int32])? {
let path : Array[Int32] = [source_node_id]
let mut current = source_node_id
while true {
let sites = use_def.node_uses[current]
if sites.length() != 1 {
return None
}
let site = sites[0]
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot if site.slot == 0 => ()
_ => return None
}
let user = site.user
match @ir.hot_node_exact_instr(func, user) {
Some(@lib.Instruction::StructGet(_, _)) => return Some((user, path))
Some(@lib.Instruction::RefCast(_, _, _))
| Some(@lib.Instruction::RefAsNonNull) => {
path.push(user)
current = user
}
_ => return None
}
}
None
}
///|
fn h2l_variant_single_branch_alloc(
func : @ir.HotFunc,
block_node_id : Int32,
) -> Int32? {
match @ir.hot_node_get(func, block_node_id).op {
@ir.HotOp::Block => ()
_ => return None
}
let body_ref = @ir.hot_region_body_ref(func, block_node_id)
let root_count = @ir.hot_region_root_count(func, body_ref)
if root_count == 0 {
return None
}
let node_id = @ir.hot_region_get(func, body_ref, root_count - 1)
match @ir.hot_node_exact_instr(func, node_id) {
Some(@lib.Instruction::StructNew(_))
| Some(@lib.Instruction::StructNewDefault(_)) => Some(node_id)
_ => None
}
}
///|
fn h2l_variant_add_field(
fields : Array[H2LVariantField],
field : H2LVariantField,
) -> Bool {
for current in fields {
if current.index == field.index {
return current.type_ == field.type_
}
}
fields.push(field)
true
}
///|
fn h2l_variant_alloc_supports_fields(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
alloc_node_id : Int32,
fields : Array[H2LVariantField],
) -> Bool {
let (_, alloc_fields, _) = match
h2l_supported_struct_new_fields(func, module_ctx, alloc_node_id) {
Some(info) => info
None => return false
}
for field in fields {
if field.index >= alloc_fields.length() {
// A base allocation may lack a subtype suffix field only when the cast
// path checks below prove that allocation unreachable at every such use.
continue
}
if alloc_fields[field.index].unpack() != field.type_ {
return false
}
}
true
}
///|
fn h2l_variant_cast_path_is_nontrapping(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
if_node_id : Int32,
then_alloc : Int32,
else_alloc : Int32,
path : Array[Int32],
) -> Bool {
let condition = h2l_i32_const_value(
func,
@ir.hot_child_get(func, if_node_id, 0),
)
let allocations = match condition {
Some(0) => [else_alloc]
Some(_) => [then_alloc]
None => [then_alloc, else_alloc]
}
for alloc in allocations {
let (alloc_type, _, _) = match
h2l_supported_struct_new_fields(func, module_ctx, alloc) {
Some(info) => info
None => return false
}
let source = @lib.RefType::new(
false,
@lib.HeapType::new(alloc_type),
exact=true,
)
for node in path {
match @ir.hot_node_exact_instr(func, node) {
Some(@lib.RefCast(nullable, exact, heap_type)) =>
if !oc_ref_always_matches(
module_ctx,
source,
oc_target_ref_type(nullable, exact, heap_type),
) {
return false
}
_ => ()
}
}
}
true
}
///|
fn h2l_variant_source_path_has_used_label(
func : @ir.HotFunc,
path : Array[Int32],
label_used : @bitset.BitSet,
) -> Bool {
for node_id in path {
if pass_label_is_used(@ir.hot_control_node_label(func, node_id), label_used) {
return true
}
}
false
}
///|
fn h2l_variant_producer_has_used_label(
func : @ir.HotFunc,
if_node_id : Int32,
then_block_node_id : Int32,
else_block_node_id : Int32,
source_path : Array[Int32],
label_used : @bitset.BitSet,
) -> Bool {
// Scalar variants replace the result-if and its arm blocks and remove the
// source wrappers. A reference-valued transfer to one of those labels needs
// its own scalar payload representation, which this rewrite does not have.
pass_label_is_used(@ir.hot_control_node_label(func, if_node_id), label_used) ||
pass_label_is_used(
@ir.hot_control_node_label(func, then_block_node_id),
label_used,
) ||
pass_label_is_used(
@ir.hot_control_node_label(func, else_block_node_id),
label_used,
) ||
h2l_variant_source_path_has_used_label(func, source_path, label_used)
}
///|
fn h2l_variant_uses_have_used_label(
func : @ir.HotFunc,
uses : Array[H2LVariantUse],
label_used : @bitset.BitSet,
) -> Bool {
uses.any(fn(use_) {
h2l_variant_source_path_has_used_label(func, use_.source_path, label_used)
})
}
///|
fn h2l_find_variant_candidate(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
use_def : @ir.HotUseDef,
producer_user_node_id : Int32,
cfg : @ir.HotCfg,
dom : @ir.HotDominators,
order : H2LDominanceOrder,
) -> H2LVariantCandidate? {
let producer_field = match
h2l_variant_struct_get_field(func, module_ctx, producer_user_node_id) {
Some(field) => field
None => return None
}
let (tee_node_id, producer_source_path) = match
h2l_variant_backward_to_tee(func, producer_user_node_id) {
Some(value) => value
None => return None
}
let tee = @ir.hot_node_get(func, tee_node_id)
if tee.child_count != 1 {
return None
}
let owner_local = tee.imm0
if owner_local < @ir.hot_param_count(func) ||
use_def.local_writes[owner_local].length() != 1 ||
use_def.local_writes[owner_local][0] != tee_node_id {
return None
}
let mut init_node_id = @ir.hot_child_get(func, tee_node_id, 0)
while true {
match h2l_passthrough_child_node(func, init_node_id) {
Some(child) => {
producer_source_path.push(init_node_id)
init_node_id = child
}
None => break
}
}
let if_node = @ir.hot_node_get(func, init_node_id)
match if_node.op {
@ir.HotOp::If => ()
_ => return None
}
if if_node.child_count != 3 {
return None
}
let then_block_node_id = @ir.hot_child_get(func, init_node_id, 1)
let else_block_node_id = @ir.hot_child_get(func, init_node_id, 2)
if else_block_node_id < 0 {
return None
}
let then_alloc_node_id = match
h2l_variant_single_branch_alloc(func, then_block_node_id) {
Some(node_id) => node_id
None => return None
}
let else_alloc_node_id = match
h2l_variant_single_branch_alloc(func, else_block_node_id) {
Some(node_id) => node_id
None => return None
}
let fields : Array[H2LVariantField] = []
if !h2l_variant_add_field(fields, producer_field) {
return None
}
let uses : Array[H2LVariantUse] = []
for read_node_id in use_def.local_reads[owner_local] {
if !h2l_write_dominates_read(cfg, dom, order, tee_node_id, read_node_id) {
return None
}
let (user_node_id, source_path) = match
h2l_variant_forward_to_struct_get(func, use_def, read_node_id) {
Some(value) => value
None => return None
}
let field = match
h2l_variant_struct_get_field(func, module_ctx, user_node_id) {
Some(field) => field
None => return None
}
if !h2l_variant_add_field(fields, field) {
return None
}
uses.push({ user_node_id, field_index: field.index, source_path, })
}
if !h2l_variant_cast_path_is_nontrapping(
func, module_ctx, init_node_id, then_alloc_node_id, else_alloc_node_id, producer_source_path,
) {
return None
}
for use_ in uses {
if !h2l_variant_cast_path_is_nontrapping(
func,
module_ctx,
init_node_id,
then_alloc_node_id,
else_alloc_node_id,
use_.source_path,
) {
return None
}
}
if !h2l_variant_alloc_supports_fields(
func, module_ctx, then_alloc_node_id, fields,
) ||
!h2l_variant_alloc_supports_fields(
func, module_ctx, else_alloc_node_id, fields,
) {
return None
}
Some({
producer_user_node_id,
producer_field_index: producer_field.index,
producer_source_path,
owner_local,
if_node_id: init_node_id,
then_block_node_id,
else_block_node_id,
then_alloc_node_id,
else_alloc_node_id,
fields,
uses,
})
}
///|
fn h2l_variant_local_producer(
func : @ir.HotFunc,
write_node_id : Int32,
owner_local : Int32,
) -> H2LVariantLocalProducer? {
let write = @ir.hot_node_get(func, write_node_id)
match write.op {
@ir.HotOp::LocalSet if write.imm0 == owner_local && write.child_count == 1 =>
()
_ => return None
}
let source_path : Array[Int32] = []
let mut current = @ir.hot_child_get(func, write_node_id, 0)
while true {
match h2l_passthrough_child_node(func, current) {
Some(child) => {
source_path.push(current)
current = child
}
None => break
}
}
let if_node = @ir.hot_node_get(func, current)
match if_node.op {
@ir.HotOp::If => ()
_ => return None
}
if if_node.child_count != 3 {
return None
}
let then_block_node_id = @ir.hot_child_get(func, current, 1)
let else_block_node_id = @ir.hot_child_get(func, current, 2)
if else_block_node_id < 0 {
return None
}
let then_alloc_node_id = match
h2l_variant_single_branch_alloc(func, then_block_node_id) {
Some(node_id) => node_id
None => return None
}
let else_alloc_node_id = match
h2l_variant_single_branch_alloc(func, else_block_node_id) {
Some(node_id) => node_id
None => return None
}
Some({
write_node_id,
source_path,
if_node_id: current,
then_block_node_id,
else_block_node_id,
then_alloc_node_id,
else_alloc_node_id,
})
}
///|
fn h2l_find_variant_local_candidate(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
use_def : @ir.HotUseDef,
owner_local : Int32,
cfg : @ir.HotCfg,
dom : @ir.HotDominators,
order : H2LDominanceOrder,
) -> H2LVariantLocalCandidate? {
if owner_local < @ir.hot_param_count(func) {
return None
}
let writes = use_def.local_writes[owner_local]
if writes.is_empty() {
return None
}
let producers : Array[H2LVariantLocalProducer] = []
for write_node_id in writes {
match h2l_variant_local_producer(func, write_node_id, owner_local) {
Some(producer) => producers.push(producer)
None => return None
}
}
let fields : Array[H2LVariantField] = []
let uses : Array[H2LVariantUse] = []
for read_node_id in use_def.local_reads[owner_local] {
if !writes.any(fn(write) {
h2l_write_dominates_read(cfg, dom, order, write, read_node_id)
}) {
return None
}
let (user_node_id, source_path) = match
h2l_variant_forward_to_struct_get(func, use_def, read_node_id) {
Some(value) => value
None => return None
}
let field = match
h2l_variant_struct_get_field(func, module_ctx, user_node_id) {
Some(field) => field
None => return None
}
if !h2l_variant_add_field(fields, field) {
return None
}
uses.push({ user_node_id, field_index: field.index, source_path, })
}
if uses.is_empty() {
return None
}
for producer in producers {
if !h2l_variant_cast_path_is_nontrapping(
func,
module_ctx,
producer.if_node_id,
producer.then_alloc_node_id,
producer.else_alloc_node_id,
producer.source_path,
) {
return None
}
for use_ in uses {
if !h2l_variant_cast_path_is_nontrapping(
func,
module_ctx,
producer.if_node_id,
producer.then_alloc_node_id,
producer.else_alloc_node_id,
use_.source_path,
) {
return None
}
}
if !h2l_variant_alloc_supports_fields(
func,
module_ctx,
producer.then_alloc_node_id,
fields,
) ||
!h2l_variant_alloc_supports_fields(
func,
module_ctx,
producer.else_alloc_node_id,
fields,
) {
return None
}
}
Some({ owner_local, producers, fields, uses, })
}
///|
fn h2l_variant_field_local(
fields : Array[H2LVariantField],
field_locals : Array[Int32],
field_index : Int32,
) -> Int32? {
for idx = 0; idx < fields.length(); idx = idx + 1 {
if fields[idx].index == field_index {
return Some(field_locals[idx])
}
}
None
}
///|
fn h2l_variant_branch_roots(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
block_node_id : Int32,
alloc_node_id : Int32,
fields : Array[H2LVariantField],
field_locals : Array[Int32],
) -> Array[Int32]? {
let (_, alloc_fields, _) = match
h2l_supported_struct_new_fields(func, module_ctx, alloc_node_id) {
Some(info) => info
None => return None
}
let roots : Array[Int32] = []
let body_ref = @ir.hot_region_body_ref(func, block_node_id)
let root_count = @ir.hot_region_root_count(func, body_ref)
for root_index = 0; root_index < root_count - 1; root_index = root_index + 1 {
roots.push(@ir.hot_region_get(func, body_ref, root_index))
}
match @ir.hot_node_exact_instr(func, alloc_node_id) {
Some(@lib.Instruction::StructNew(_)) => {
if @ir.hot_child_count(func, alloc_node_id) != alloc_fields.length() {
return None
}
for field_index = 0
field_index < alloc_fields.length()
field_index = field_index + 1 {
let child = @ir.hot_child_get(func, alloc_node_id, field_index)
match h2l_variant_field_local(fields, field_locals, field_index) {
Some(local_id) =>
roots.push(@ir.hot_build_local_set(func, local_id, child))
None => roots.push(@ir.hot_build_drop(func, child))
}
}
}
Some(@lib.Instruction::StructNewDefault(_)) =>
for field_index = 0
field_index < alloc_fields.length()
field_index = field_index + 1 {
match h2l_variant_field_local(fields, field_locals, field_index) {
Some(local_id) => {
let value = match
h2l_build_default_value(func, alloc_fields[field_index].unpack()) {
Some(value) => value
None => return None
}
roots.push(@ir.hot_build_local_set(func, local_id, value))
}
None => ()
}
}
_ => return None
}
Some(roots)
}
///|
fn h2l_apply_variant_candidate(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
candidate : H2LVariantCandidate,
detached_nodes : Array[Int32],
) -> Bool {
let field_locals : Array[Int32] = []
for field in candidate.fields {
let local_id = @ir.hot_local_count(func)
func.locals.locals.push(h2l_scalar_local_type(field.type_))
field_locals.push(local_id)
}
let then_roots = match
h2l_variant_branch_roots(
func,
module_ctx,
candidate.then_block_node_id,
candidate.then_alloc_node_id,
candidate.fields,
field_locals,
) {
Some(roots) => roots
None => return false
}
let else_roots = match
h2l_variant_branch_roots(
func,
module_ctx,
candidate.else_block_node_id,
candidate.else_alloc_node_id,
candidate.fields,
field_locals,
) {
Some(roots) => roots
None => return false
}
let condition = @ir.hot_child_get(func, candidate.if_node_id, 0)
let scalar_if = @ir.hot_build_if(
func,
@lib.BlockType::void_(),
condition,
then_roots,
else_body=Some(else_roots),
)
let producer_local = match
h2l_variant_field_local(
candidate.fields,
field_locals,
candidate.producer_field_index,
) {
Some(local_id) => local_id
None => return false
}
let mut producer_type : @lib.ValType? = None
for field in candidate.fields {
if field.index == candidate.producer_field_index {
producer_type = Some(field.type_)
break
}
}
let producer_type = match producer_type {
Some(type_) => type_
None => return false
}
let producer_replacement = @ir.hot_build_block(
func,
@lib.BlockType::val_type(producer_type),
[scalar_if, h2l_build_scalar_local_get(func, producer_local, producer_type)],
)
h2l_replace_with_built(
func,
candidate.producer_user_node_id,
producer_replacement,
)
for field_use in candidate.uses {
let local_id = match
h2l_variant_field_local(
candidate.fields,
field_locals,
field_use.field_index,
) {
Some(local_id) => local_id
None => return false
}
let value_type = @ir.hot_type_results(
func,
@ir.hot_node_type(func, field_use.user_node_id),
)[0]
let replacement = h2l_build_scalar_local_get(func, local_id, value_type)
@ir.hot_replace_node(
func,
field_use.user_node_id,
@ir.hot_node_get(func, replacement),
)
for idx = field_use.source_path.length() - 1; idx >= 0; idx = idx - 1 {
detached_nodes.push(field_use.source_path[idx])
}
}
for node_id in candidate.producer_source_path {
detached_nodes.push(node_id)
}
detached_nodes.append([
candidate.if_node_id,
candidate.then_block_node_id,
candidate.else_block_node_id,
candidate.then_alloc_node_id,
candidate.else_alloc_node_id,
])
true
}
///|
fn h2l_apply_variant_local_candidate(
func : @ir.HotFunc,
module_ctx : @ir.HotModuleContext,
candidate : H2LVariantLocalCandidate,
detached_nodes : Array[Int32],
) -> Bool {
let field_locals : Array[Int32] = []
for field in candidate.fields {
let local_id = @ir.hot_local_count(func)
func.locals.locals.push(h2l_scalar_local_type(field.type_))
field_locals.push(local_id)
}
for producer in candidate.producers {
// A replacement constructor can read the previous owner's fields. Keep
// those fields intact until every operand has been evaluated, including
// operands that throw or perform an update through nested control flow.
// Each producer owns its scratch locals so nested producers cannot change
// an already evaluated operand of the enclosing constructor.
let staged_fields : Array[Int32] = []
for field in candidate.fields {
let local_id = @ir.hot_local_count(func)
func.locals.locals.push(h2l_scalar_local_type(field.type_))
staged_fields.push(local_id)
}
let then_roots = match
h2l_variant_branch_roots(
func,
module_ctx,
producer.then_block_node_id,
producer.then_alloc_node_id,
candidate.fields,
staged_fields,
) {
Some(roots) => roots
None => return false
}
let else_roots = match
h2l_variant_branch_roots(
func,
module_ctx,
producer.else_block_node_id,
producer.else_alloc_node_id,
candidate.fields,
staged_fields,
) {
Some(roots) => roots
None => return false
}
let scalar_if = @ir.hot_build_if(
func,
@lib.BlockType::void_(),
@ir.hot_child_get(func, producer.if_node_id, 0),
then_roots,
else_body=Some(else_roots),
)
let publish_roots = [scalar_if]
for field_index = 0
field_index < candidate.fields.length()
field_index = field_index + 1 {
publish_roots.push(
@ir.hot_build_local_set(
func,
field_locals[field_index],
h2l_build_scalar_local_get(
func,
staged_fields[field_index],
candidate.fields[field_index].type_,
),
),
)
}
let publication = @ir.hot_build_block(
func,
@lib.BlockType::void_(),
publish_roots,
)
h2l_replace_with_built(func, producer.write_node_id, publication)
for node_id in producer.source_path {
detached_nodes.push(node_id)
}
detached_nodes.append([
producer.if_node_id,
producer.then_block_node_id,
producer.else_block_node_id,
producer.then_alloc_node_id,
producer.else_alloc_node_id,
])
}
for field_use in candidate.uses {
let local_id = match
h2l_variant_field_local(
candidate.fields,
field_locals,
field_use.field_index,
) {
Some(local_id) => local_id
None => return false
}
let value_type = @ir.hot_type_results(
func,
@ir.hot_node_type(func, field_use.user_node_id),
)[0]
let replacement = h2l_build_scalar_local_get(func, local_id, value_type)
@ir.hot_replace_node(
func,
field_use.user_node_id,
@ir.hot_node_get(func, replacement),
)
for idx = field_use.source_path.length() - 1; idx >= 0; idx = idx - 1 {
detached_nodes.push(field_use.source_path[idx])
}
}
true
}
///|
fn h2l_delete_detached_live_nodes(
func : @ir.HotFunc,
detached_nodes : Array[Int32],
) -> Unit {
if detached_nodes.is_empty() {
return
}
let node_count = @ir.hot_node_count(func)
let selected = @bitset.BitSet::new(node_count)
for node_id in detached_nodes {
if node_id >= 0 &&
node_id < node_count &&
@ir.hot_node_is_live(func, node_id) {
selected[node_id] = true
}
}
// A node detached by one rewrite can still be used by another live owner.
// Retain its selected descendants as well, so the final set has no incoming
// references from roots or nodes outside the set.
let retained : Array[Int32] = []
for index in 0..<@ir.hot_root_count(func) {
let root = @ir.hot_root_get(func, index)
if root >= 0 && root < node_count && selected[root] {
selected[root] = false
retained.push(root)
}
}
for owner in 0..= 0 && child < node_count && selected[child] {
selected[child] = false
retained.push(child)
}
}
}
for cursor = 0; cursor < retained.length(); cursor = cursor + 1 {
let owner = retained[cursor]
for slot in 0..<@ir.hot_child_count(func, owner) {
let child = @ir.hot_child_get(func, owner, slot)
if child >= 0 && child < node_count && selected[child] {
selected[child] = false
retained.push(child)
}
}
}
let deletable : Array[Int32] = []
for node_id in detached_nodes {
if node_id >= 0 && node_id < node_count && selected[node_id] {
selected[node_id] = false
deletable.push(node_id)
}
}
@ir.hot_delete_detached_nodes(func, deletable)
}
///|
pub fn heap2local_run(
ctx : HotPassContext,
func : @ir.HotFunc,
) -> HotPassResult {
let module_ctx = match ctx.module_ctx {
Some(module_ctx) => module_ctx
None => return HotPassResult::unchanged()
}
let use_def = pass_require_use_def(ctx, func)
let root_slots = h2l_build_root_slot_index(func)
let unsafe_array_locals = h2l_repeated_array_allocation_write_locals(
func, use_def, root_slots,
)
let cfg = @ir.cfg_build(func, expand_operand_control=true)
let dom = @ir.dom_build(cfg)
let order = H2LDominanceOrder::new(@ir.hot_node_count(func))
let variant_candidates : Array[H2LVariantCandidate] = []
let variant_local_candidates : Array[H2LVariantLocalCandidate] = []
let variant_owner_locals : Array[Int32] = []
// Only variants remove these control owners. Functions without a variant
// candidate need no label scan, and all candidates share one snapshot.
let mut variant_label_used : @bitset.BitSet? = None
for local_id = 0
local_id < @ir.hot_local_count(func)
local_id = local_id + 1 {
if unsafe_array_locals[local_id] {
continue
}
match
h2l_find_variant_local_candidate(
func, module_ctx, use_def, local_id, cfg, dom, order,
) {
Some(candidate) => {
let label_used = match variant_label_used {
Some(labels) => labels
None => {
let labels = pass_compute_label_used(func)
variant_label_used = Some(labels)
labels
}
}
if !h2l_variant_uses_have_used_label(func, candidate.uses, label_used) &&
!candidate.producers.any(fn(producer) {
h2l_variant_producer_has_used_label(
func,
producer.if_node_id,
producer.then_block_node_id,
producer.else_block_node_id,
producer.source_path,
label_used,
)
}) {
variant_local_candidates.push(candidate)
variant_owner_locals.push(candidate.owner_local)
}
}
None => ()
}
}
let original_node_count = @ir.hot_node_count(func)
for node_id = 0; node_id < original_node_count; node_id = node_id + 1 {
match
h2l_find_variant_candidate(
func, module_ctx, use_def, node_id, cfg, dom, order,
) {
Some(candidate) if !variant_owner_locals.contains(candidate.owner_local) => {
let label_used = match variant_label_used {
Some(labels) => labels
None => {
let labels = pass_compute_label_used(func)
variant_label_used = Some(labels)
labels
}
}
if !unsafe_array_locals[candidate.owner_local] &&
!h2l_variant_producer_has_used_label(
func,
candidate.if_node_id,
candidate.then_block_node_id,
candidate.else_block_node_id,
candidate.producer_source_path,
label_used,
) &&
!h2l_variant_uses_have_used_label(func, candidate.uses, label_used) {
variant_candidates.push(candidate)
variant_owner_locals.push(candidate.owner_local)
}
}
Some(_) => ()
None => ()
}
}
let candidates : Array[H2LCandidate] = []
let array_candidates : Array[H2LArrayCandidate] = []
let drop_only_struct_candidates : Array[H2LDropOnlyStructOwnerCandidate] = []
for local_id = 0
local_id < @ir.hot_local_count(func)
local_id = local_id + 1 {
if variant_owner_locals.contains(local_id) || unsafe_array_locals[local_id] {
continue
}
match
h2l_find_candidate_for_local(
func, module_ctx, use_def, local_id, cfg, dom, order,
) {
Some(candidate) => candidates.push(candidate)
None => ()
}
match
h2l_find_drop_only_struct_owner_candidate(
func, module_ctx, use_def, local_id,
) {
Some(candidate) => drop_only_struct_candidates.push(candidate)
None => ()
}
for
candidate in h2l_find_sequential_candidates_for_local(
func, module_ctx, use_def, local_id, root_slots,
) {
candidates.push(candidate)
}
match
h2l_find_array_candidate_for_local(
func, module_ctx, use_def, local_id, cfg, dom, order,
) {
Some(candidate) => array_candidates.push(candidate)
None => ()
}
for
candidate in h2l_find_sequential_array_candidates_for_local(
func, module_ctx, use_def, local_id, root_slots,
) {
array_candidates.push(candidate)
}
}
let detached_nodes : Array[Int32] = []
let mut changed = false
for candidate in variant_local_candidates {
if h2l_apply_variant_local_candidate(
func, module_ctx, candidate, detached_nodes,
) {
changed = true
}
}
for candidate in variant_candidates {
if h2l_apply_variant_candidate(func, module_ctx, candidate, detached_nodes) {
changed = true
}
}
for candidate in candidates {
if h2l_apply_candidate(func, candidate, detached_nodes) {
changed = true
}
}
for candidate in array_candidates {
if h2l_apply_array_candidate(func, candidate, detached_nodes) {
changed = true
}
}
for candidate in drop_only_struct_candidates {
if h2l_apply_drop_only_struct_owner_candidate(
func, candidate, detached_nodes,
) {
changed = true
}
}
let node_count = @ir.hot_node_count(func)
for node_id = 0; node_id < node_count; node_id = node_id + 1 {
if h2l_try_fold_direct_struct_get(func, module_ctx, node_id, detached_nodes) {
changed = true
}
if h2l_try_fold_direct_ref_is_null(func, node_id, detached_nodes) {
changed = true
}
if h2l_try_fold_direct_array_get(func, module_ctx, node_id, detached_nodes) {
changed = true
}
if h2l_try_fold_direct_ref_eq(func, module_ctx, node_id, detached_nodes) {
changed = true
}
if h2l_try_fold_direct_ref_get_desc(
func, module_ctx, node_id, detached_nodes,
) {
changed = true
}
if h2l_try_fold_direct_struct_ref_test(
func, module_ctx, node_id, detached_nodes,
) {
changed = true
}
if h2l_try_fold_direct_array_ref_test(
func, module_ctx, node_id, detached_nodes,
) {
changed = true
}
}
if !changed {
return HotPassResult::unchanged()
}
h2l_delete_detached_live_nodes(func, detached_nodes)
pass_mark_mutated(ctx, func)
HotPassResult::changed()
}
///|
struct H2LDominanceOrder {
node_count : Int32
mut long_scans : Int32
mut positions : Array[Int32]?
}
///|
fn H2LDominanceOrder::new(node_count : Int32) -> H2LDominanceOrder {
{ node_count, long_scans: 0, positions: None, }
}
///|
fn h2l_write_dominates_read(
cfg : @ir.HotCfg,
dom : @ir.HotDominators,
order : H2LDominanceOrder,
write_id : Int32,
read_id : Int32,
) -> Bool {
let write_block = match @ir.cfg_block_for_node(cfg, write_id) {
Some(block) => block
None => return false
}
let read_block = match @ir.cfg_block_for_node(cfg, read_id) {
Some(block) => block
None => return false
}
if write_block != read_block {
return @ir.dom_dominates(dom, write_block, read_block)
}
let nodes = @ir.cfg_block_get(cfg, read_block).nodes
for idx = 0; idx < nodes.length() && idx < 4; idx = idx + 1 {
if nodes[idx] == read_id {
return false
}
if nodes[idx] == write_id {
return true
}
}
match order.positions {
Some(positions) =>
return positions[write_id] >= 0 &&
positions[write_id] < positions[read_id]
None => ()
}
// CFG block nodes are in operand-before-user execution order.
let mut scanned = 4
let mut write_before_read = false
for idx = 4; idx < nodes.length(); idx = idx + 1 {
let node = nodes[idx]
if node == read_id {
if scanned < 64 {
return false
}
break
}
if node == write_id {
if scanned < 64 {
return true
}
write_before_read = true
break
}
scanned += 1
}
if scanned >= 64 {
order.long_scans += 1
if order.long_scans == 2 {
let positions = Array::make(order.node_count, -1)
for block = 0; block < @ir.cfg_block_count(cfg); block = block + 1 {
let nodes = @ir.cfg_block_get(cfg, block).nodes
for idx = 0; idx < nodes.length(); idx = idx + 1 {
positions[nodes[idx]] = idx
}
}
order.positions = Some(positions)
return positions[write_id] >= 0 &&
positions[write_id] < positions[read_id]
}
}
write_before_read
}
///|
fn heap2local_cleanup_module_types(mod_ : @lib.Module) -> @lib.Module {
// Scalarization can leave whole type groups unused, including in functions
// admitted by the no-allocation fast path. Run after complete module writeback.
pass_cleanup_unused_type_groups(mod_)
}