///|
#valtype
priv struct HotLowerStackValue {
node_id : NodeId
val_type : ValType
}
///|
priv struct HotLowerBodyResult {
body : Array[Instruction]
extra_locals : Array[ValType]
final_stack : HotLowerValueStack
}
///|
pub struct HotLowerOptions {
preserve_if_arm_single_nops : Bool
preserve_dead_local_tee_drops : Bool
preserve_nontrailing_nops : Bool
preserve_trailing_nops : Bool
preserve_typed_loop_const_prefix_spills : Bool
preserve_flat_ir_spills : Bool
flatten_unused_result_blocks : Bool
preserve_struct_set_value_blocks : Bool
}
///|
pub fn HotLowerOptions::new(
preserve_if_arm_single_nops~ : Bool = false,
preserve_dead_local_tee_drops~ : Bool = false,
preserve_nontrailing_nops~ : Bool = false,
preserve_trailing_nops~ : Bool = false,
preserve_typed_loop_const_prefix_spills~ : Bool = false,
preserve_flat_ir_spills~ : Bool = false,
flatten_unused_result_blocks~ : Bool = false,
preserve_struct_set_value_blocks~ : Bool = false,
) -> HotLowerOptions {
{
preserve_if_arm_single_nops,
preserve_dead_local_tee_drops,
preserve_nontrailing_nops,
preserve_trailing_nops,
preserve_typed_loop_const_prefix_spills,
preserve_flat_ir_spills,
flatten_unused_result_blocks,
preserve_struct_set_value_blocks,
}
}
///|
#valtype
priv struct HotLowerLegacyCatchFrame {
label_id : LabelId
local_idx : LocalIdx?
}
///|
priv struct HotLowerTupleLanePairs {
extra_locals : Array[ValType]
pairs : Array[(ValType, ValType, LocalIdx, LocalIdx)]
}
///|
priv struct HotLowerScopeQuery {
mut target : NodeId
answers : @hashmap.HashMap[NodeId, Bool]
nodes : Array[NodeId]
slots : Array[Int32]
}
///|
fn HotLowerScopeQuery::new(target : NodeId) -> HotLowerScopeQuery {
{ target, answers: @hashmap.new(), nodes: [], slots: [], }
}
///|
priv struct HotLowerLabelFrames {
active : Array[LabelId]
depths : Array[Int32]
legacy_rethrow_frames : Array[HotLowerLegacyCatchFrame]
used_labels : @bitset.BitSet
mut source_order : HotSourceOrderFacts?
mut node_effects : Array[EffectMask]?
mut operand_minima : Array[Int32]?
mut requested_nodes : @bitset.BitSet?
mut tuple_lane_pairs : HotLowerTupleLanePairs?
scope_query_work : Array[Int32]?
mut scope_query : HotLowerScopeQuery?
}
///|
fn hot_lower_impl_new_label_frames(
func : HotFunc,
scope_query_work~ : Array[Int32]? = None,
) -> HotLowerLabelFrames {
{
active: [],
depths: [],
legacy_rethrow_frames: [],
used_labels: hot_lower_impl_build_used_labels(func),
source_order: None,
node_effects: None,
operand_minima: None,
requested_nodes: None,
tuple_lane_pairs: None,
scope_query_work,
scope_query: None,
}
}
///|
// Initialize before any nested emission can suspend this stack. Requests are
// an over-approximation: a value never requested or introduced as a control
// input cannot already be buried. Repeated requests retain the existing search.
fn hot_lower_impl_root_was_requested(
func : HotFunc,
frames : HotLowerLabelFrames,
id : NodeId,
stack : HotLowerValueStack,
) -> Bool {
let requested = match frames.requested_nodes {
Some(requested) => requested
None => {
let requested = @bitset.BitSet::new(hot_node_count(func))
for value in stack {
// Bookkeeping must not add a new diagnostic for an unknown stack
// sentinel; normal node admission owns errors for requested nodes.
if value.node_id >= 0 && value.node_id < requested.length() {
requested[value.node_id] = true
}
}
frames.requested_nodes = Some(requested)
requested
}
}
let was_requested = requested[id]
requested[id] = true
was_requested
}
///|
fn hot_lower_impl_source_order(
func : HotFunc,
frames : HotLowerLabelFrames,
) -> HotSourceOrderFacts {
match frames.source_order {
Some(facts) => facts
None => {
// Minima are no longer queried after full facts exist. Retire this scratch
// before building the larger rows; both are private to this immutable lower.
frames.operand_minima = None
let facts = HotSourceOrderFacts::new(
func,
node_effects=frames.node_effects,
)
frames.node_effects = None
frames.source_order = Some(facts)
facts
}
}
}
///|
fn hot_lower_impl_node_effects(
func : HotFunc,
frames : HotLowerLabelFrames,
) -> Array[EffectMask] {
match frames.source_order {
Some(facts) => facts.node_effects
None =>
match frames.node_effects {
Some(masks) => masks
None => {
let masks = effects_build_node_masks(func)
frames.node_effects = Some(masks)
masks
}
}
}
}
///|
#valtype
priv struct HotLowerFutureDependencies {
required : Bool
suffix_minima : HotSourceOrderRootIndex?
}
///|
fn HotLowerFutureDependencies::new(
required : Bool,
suffix_minima~ : HotSourceOrderRootIndex? = None,
) -> HotLowerFutureDependencies {
{ required, suffix_minima, }
}
///|
fn hot_lower_impl_push_label(
frames : HotLowerLabelFrames,
label_stack_head : Int32,
label_id : LabelId,
) -> Int32 {
ignore(label_stack_head)
while frames.depths.length() <= label_id {
frames.depths.push(-1)
}
let next = frames.active.length()
frames.depths[label_id] = next
frames.active.push(label_id)
next
}
///|
fn hot_lower_impl_pop_label(
frames : HotLowerLabelFrames,
label_stack_head : Int32,
) -> Unit {
ignore(label_stack_head)
match frames.active.pop() {
Some(label_id) => frames.depths[label_id] = -1
None => abort("hot lower label pop requires a non-empty active label stack")
}
}
///|
fn hot_lower_impl_label_depth(
frames : HotLowerLabelFrames,
label_stack_head : Int32,
label_id : LabelId,
) -> LabelIdx {
if label_id == HOT_IMPLICIT_FUNCTION_LABEL {
return LabelIdx::new((label_stack_head + 1).reinterpret_as_uint())
}
if label_id >= 0 && label_id < frames.depths.length() {
let absolute_depth = frames.depths[label_id]
if absolute_depth >= 0 &&
absolute_depth < frames.active.length() &&
absolute_depth <= label_stack_head &&
frames.active[absolute_depth] == label_id {
let relative_depth = label_stack_head - absolute_depth
return LabelIdx::new(relative_depth.reinterpret_as_uint())
}
}
abort(
"hot lower target label must be present in the active boundary label stack",
)
}
///|
fn hot_lower_impl_control_block_type(func : HotFunc, id : NodeId) -> BlockType {
match hot_type_get(func, hot_node_get(func, id).ty) {
BlockResult(block_type) => block_type
BlockResultResolved(block_type, _, _) => block_type
_ => BlockType::void_()
}
}
///|
fn hot_lower_impl_control_param_types(
func : HotFunc,
id : NodeId,
) -> Array[ValType] {
match hot_type_get(func, hot_node_get(func, id).ty) {
// Private read-only borrow: lowering never changes the interned signature.
BlockResultResolved(_, params, _) => params
_ => []
}
}
///|
fn hot_lower_impl_build_used_labels(func : HotFunc) -> @bitset.BitSet {
let used = @bitset.BitSet::new(func.labels.length())
let mark = fn(label_id : LabelId) {
if label_id >= 0 && label_id < used.length() {
used[label_id] = true
}
}
for node_id = 0; node_id < hot_node_count(func); node_id = node_id + 1 {
if !hot_node_is_live(func, node_id) {
continue
}
let node = hot_node_get(func, node_id)
match node.op {
HotOp::Br
| HotOp::BrIf
| HotOp::BrOnNull
| HotOp::BrOnNonNull
| HotOp::BrOnCast
| HotOp::BrOnCastFail
| HotOp::Delegate => mark(node.imm0)
HotOp::BrTable => {
mark(hot_branch_table_default_target(func, node.imm0))
for target in hot_branch_table_targets(func, node.imm0) {
mark(target)
}
}
HotOp::Continuation =>
for target in hot_continuation_targets(func, node_id) {
mark(target)
}
HotOp::TryTable => {
let info = hot_catch_info_get(func, node.imm1)
for catch_ in info.catches {
match catch_ {
HotCatchArm::CatchArm(_, target)
| HotCatchArm::CatchRefArm(_, target)
| HotCatchArm::CatchAllArm(target)
| HotCatchArm::CatchAllRefArm(target) => mark(target)
}
}
}
_ => ()
}
}
used
}
///|
fn hot_lower_impl_label_is_used(
label_frames : HotLowerLabelFrames,
label_id : LabelId,
) -> Bool {
label_id >= 0 &&
label_id < label_frames.used_labels.length() &&
label_frames.used_labels[label_id]
}
///|
fn hot_lower_impl_exact_instr(func : HotFunc, id : NodeId) -> Instruction {
match hot_node_exact_instr(func, id) {
Some(instr) => instr
None =>
abort(
"hot lower exact-family node must carry an exact instruction payload",
)
}
}
///|
fn hot_lower_impl_const(
func : HotFunc,
id : NodeId,
out : Array[Instruction],
) -> Unit {
let node = hot_node_get(func, id)
match hot_const_get(func, node.imm0) {
HotConstPayload::I32Const(v) => out.push(Instruction::i32_const(v))
HotConstPayload::I64Const(v) => out.push(Instruction::i64_const(v))
HotConstPayload::F32Const(v) => out.push(Instruction::f32_const(v))
HotConstPayload::F64Const(v) => out.push(Instruction::f64_const(v))
HotConstPayload::RefNullConst(rt) =>
out.push(Instruction::ref_null_type(rt))
HotConstPayload::StringConst(bytes) =>
out.push(Instruction::string_const(bytes))
}
}
///|
fn hot_lower_impl_call_sig(func : HotFunc, id : SideTableId) -> HotCallSig {
hot_call_sig_get(func, id)
}
///|
fn hot_lower_impl_table_idx(raw : Int32) -> TableIdx {
TableIdx::new(raw.reinterpret_as_uint())
}
///|
fn hot_lower_impl_mem_idx(raw : Int32) -> MemIdx {
MemIdx::new(raw.reinterpret_as_uint())
}
///|
#valtype
priv struct HotLowerInputs {
nodes : ArrayView[NodeId]
pop_count : Int32
}
///|
fn hot_lower_impl_input_info(
func : HotFunc,
id : NodeId,
copied_slots~ : Array[Int32]? = None,
) -> HotLowerInputs {
// Keep the same checked admission without returning a complete header
// across a function boundary before reading its fields here.
hot_node_require_live_read(func, id)
let node = func.nodes[id]
match node.op {
HotOp::Block => {
let count = if node.imm1 > 0 { node.imm1 } else { 0 }
// Void entry effects remain in the prefix; only surviving parameters pop.
let pop_count = if count == 0 {
0
} else {
match hot_type_get(func, node.ty) {
BlockResultResolved(_, params, _) => params.length()
_ => count
}
}
{
nodes: func.children[node.child_base:node.child_base + count],
pop_count,
}
}
HotOp::Try
| HotOp::TryTable
| HotOp::Nop
| HotOp::Unreachable
| HotOp::Const
| HotOp::RefNull
| HotOp::LocalGet
| HotOp::GlobalGet
| HotOp::MemorySize
| HotOp::TableSize
| HotOp::DataDrop
| HotOp::ElemDrop
| HotOp::RefFunc => { nodes: func.children[0:0], pop_count: 0, }
HotOp::Loop => {
let input_count = hot_label_branch_arity(
func,
hot_control_node_label(func, id),
)
{
nodes: func.children[node.child_base:node.child_base + input_count],
pop_count: input_count,
}
}
HotOp::If => {
if node.child_count == 3 {
return {
nodes: func.children[node.child_base:node.child_base + 1],
pop_count: 1,
}
}
let inputs : Array[NodeId] = []
for slot in 3.. {
// Lowering and source-order analysis only read the current HOT snapshot.
// Dense child storage can be borrowed; absent slots need compact storage.
let children = func.children[node.child_base:node.child_base +
node.child_count]
let nodes = if children.any(fn(child) { child == HOT_NONE_NODE }) {
let kept : Array[NodeId] = []
for child in children {
if child != HOT_NONE_NODE {
match copied_slots {
Some(count) => count[0] += 1
None => ()
}
kept.push(child)
}
}
kept[:]
} else {
children
}
let pop_count = match node.op {
HotOp::BrIf | HotOp::BrTable | HotOp::BrOnNull | HotOp::BrOnNonNull => 1
HotOp::BrOnCast | HotOp::BrOnCastFail =>
match hot_node_exact_instr(func, id) {
Some(@lib.BrOnCastDescEq(_, _, _))
| Some(@lib.BrOnCastDescEqFail(_, _, _)) => 2
_ => 1
}
_ => nodes.length()
}
{ nodes, pop_count, }
}
}
}
///|
fn hot_lower_impl_clear_stack_after_emit(op : HotOp) -> Bool {
match op {
HotOp::Br
| HotOp::BrTable
| HotOp::Return
| HotOp::ReturnCall
| HotOp::ReturnCallIndirect
| HotOp::ReturnCallRef
| HotOp::Throw
| HotOp::ThrowRef
| HotOp::Rethrow
| HotOp::Delegate
| HotOp::Unreachable => true
_ => false
}
}
///|
fn hot_lower_impl_push_results(
func : HotFunc,
id : NodeId,
stack : HotLowerValueStack,
) -> Unit {
// Stack bookkeeping only reads interned result rows. Keep the public
// hot_type_results query owned, but avoid its scalar arrays and tuple copies.
match hot_type_get(func, hot_node_type(func, id)) {
VoidType | BlockResult(VoidBlockType) => ()
Value(val_type) | BlockResult(ValTypeBlockType(val_type)) =>
stack.push({ node_id: id, val_type, })
BlockResult(TypeIdxBlockType(_)) =>
abort(
"type-index block types are not supported by the current hot type queries",
)
BlockResultResolved(_, _, values) | Results(values) =>
for val_type in values {
stack.push({ node_id: id, val_type, })
}
}
}
///|
fn hot_lower_impl_matching_input_prefix(
stack : HotLowerValueStack,
inputs : ArrayView[NodeId],
comparisons~ : Array[Int32]? = None,
) -> Int32 {
let max_count = if stack.length() < inputs.length() {
stack.length()
} else {
inputs.length()
}
// Prefer the complete already-stacked input vector, especially tuple lanes
// which share one producer id. The first match is the longest valid prefix.
for count = max_count; count > 0; count = count - 1 {
let stack_start = stack.length() - count
let mut ok = true
for i = 0; i < count; i = i + 1 {
match comparisons {
Some(counts) => counts[0] += 1
None => ()
}
if stack[stack_start + i].node_id != inputs[i] {
ok = false
break
}
}
if ok {
return count
}
}
0
}
///|
fn hot_lower_impl_update_stack(
func : HotFunc,
id : NodeId,
pop_count : Int32,
stack : HotLowerValueStack,
) -> Unit {
let node = hot_node_get(func, id)
if hot_lower_impl_clear_stack_after_emit(node.op) {
stack.clear()
return
}
if pop_count > 0 {
// Discard the consumed suffix without boxing an optional packed value for
// each lane. Preserve the old saturating-pop behavior on a shorter stack.
if pop_count >= stack.length() {
stack.clear()
} else {
stack.truncate(stack.length() - pop_count)
}
}
hot_lower_impl_push_results(func, id, stack)
}
///|
fn hot_lower_impl_lower_br_on_cast(
func : HotFunc,
id : NodeId,
label_frames : HotLowerLabelFrames,
label_stack_head : Int32,
) -> Instruction {
let node = hot_node_get(func, id)
let label_idx = hot_lower_impl_label_depth(
label_frames,
label_stack_head,
node.imm0,
)
match hot_lower_impl_exact_instr(func, id) {
Instruction::BrOnCast(
_,
CastOp(src_nullable, dst_nullable, src_exact, dst_exact),
src_ht,
dst_ht
) =>
Instruction::br_on_cast(
label_idx,
src_nullable,
src_ht,
dst_nullable,
dst_ht,
source_exact=src_exact,
target_exact=dst_exact,
)
Instruction::BrOnCastFail(
_,
CastOp(src_nullable, dst_nullable, src_exact, dst_exact),
src_ht,
dst_ht
) =>
Instruction::br_on_cast_fail(
label_idx,
src_nullable,
src_ht,
dst_nullable,
dst_ht,
source_exact=src_exact,
target_exact=dst_exact,
)
Instruction::BrOnCastDescEq(_, source, target) =>
Instruction::br_on_cast_desc_eq(label_idx, source, target)
Instruction::BrOnCastDescEqFail(_, source, target) =>
Instruction::br_on_cast_desc_eq_fail(label_idx, source, target)
other =>
abort(
"hot lower expected br_on_cast exact payload for \{node.op}, got \{other}",
)
}
}
///|
fn hot_lower_impl_try_catches(
func : HotFunc,
id : NodeId,
label_frames : HotLowerLabelFrames,
label_stack_head : Int32,
) -> Array[Catch] {
let node = hot_node_get(func, id)
let info = hot_catch_info_get(func, node.imm1)
let catches : Array[Catch] = []
for arm in info.catches {
match arm {
HotCatchArm::CatchArm(tag_idx, label_id) =>
catches.push(
Catch::new(
tag_idx,
hot_lower_impl_label_depth(label_frames, label_stack_head, label_id),
),
)
HotCatchArm::CatchRefArm(tag_idx, label_id) =>
catches.push(
Catch::ref_(
tag_idx,
hot_lower_impl_label_depth(label_frames, label_stack_head, label_id),
),
)
HotCatchArm::CatchAllArm(label_id) =>
catches.push(
Catch::all(
hot_lower_impl_label_depth(label_frames, label_stack_head, label_id),
),
)
HotCatchArm::CatchAllRefArm(label_id) =>
catches.push(
Catch::all_ref(
hot_lower_impl_label_depth(label_frames, label_stack_head, label_id),
),
)
}
}
catches
}
///|
priv struct HotLowerCatchPayload {
tag_idx : TagIdx
val_type : ValType
}
///|
fn hot_lower_impl_catch_payload_prefix(
func : HotFunc,
region_id : NodeId,
) -> Array[HotLowerCatchPayload] {
let out : Array[HotLowerCatchPayload] = []
if region_id == HOT_NONE_NODE {
return out
}
let region_ref = hot_region_body_ref(func, region_id)
for slot = 0; slot < hot_region_root_count(func, region_ref); slot = slot + 1 {
let node_id = hot_region_get(func, region_ref, slot)
if hot_query_op(func, node_id) != HotOp::Catch {
break
}
let types = hot_type_results(func, hot_node_type(func, node_id))
if types.length() != 1 {
abort("hot lower typed catch payload requires one scalar result")
}
out.push({
tag_idx: hot_catch_payload_tag(func, node_id),
val_type: types[0],
})
}
out
}
///|
priv struct HotLowerCatchVisits {
mut rows : Array[UInt64]
mut size : Int32
}
///|
fn hot_lower_impl_catch_visit_slot(
rows : Array[UInt64],
encoded : UInt64,
) -> Int32 {
let key = encoded - 1UL
let mut hash = (key ^ (key >> 33)) * 0xff51afd7ed558ccdUL
hash = (hash ^ (hash >> 33)) * 0xc4ceb9fe1a85ec53UL
hash = hash ^ (hash >> 33)
let mask = rows.length() - 1
let mut slot = hash.to_int() & mask
while rows[slot] != 0UL && rows[slot] != encoded {
slot = (slot + 1) & mask
}
slot
}
///|
fn HotLowerCatchVisits::add(self : HotLowerCatchVisits, key : UInt64) -> Bool {
// Node ids and catch depths are nonnegative Int values. Adding one cannot
// overflow the packed pair and reserves zero for an empty slot.
let encoded = key + 1UL
let mut slot = hot_lower_impl_catch_visit_slot(self.rows, encoded)
if self.rows[slot] != 0UL {
return false
}
if self.size >= self.rows.length() / 2 {
let next = Array::make(self.rows.length() * 2, 0UL)
for row in self.rows {
if row != 0UL {
next[hot_lower_impl_catch_visit_slot(next, row)] = row
}
}
self.rows = next
slot = hot_lower_impl_catch_visit_slot(next, encoded)
}
self.rows[slot] = encoded
self.size += 1
true
}
///|
fn hot_lower_impl_catch_rethrow_count(
func : HotFunc,
region_ref : HotRegionRef,
target_depth : Int32,
) -> Int32 {
// Shared operands are visited once at each catch depth. Pack the node and
// depth into primitive rows and worklist keys. Unlike the linked set, this
// allocates no entry object for each node. Scratch follows the reached
// subgraph size. Ordinary control labels do not change this depth.
let pending : Array[UInt64] = []
let seen : HotLowerCatchVisits = { rows: Array::make(16, 0UL), size: 0, }
for slot = 0; slot < hot_region_root_count(func, region_ref); slot = slot + 1 {
let node_id = hot_region_get(func, region_ref, slot)
let key = (node_id.to_uint64() << 32) | target_depth.to_uint64()
if seen.add(key) {
pending.push(key)
}
}
let mut count = 0
while !pending.is_empty() {
let key = pending.pop().unwrap()
let node_id = (key >> 32).to_int()
let depth = key.to_int()
let node = hot_node_get(func, node_id)
if node.op == HotOp::Rethrow && node.imm0 == depth {
count += 1
}
for slot = 0; slot < node.child_count; slot = slot + 1 {
let child = func.children[node.child_base + slot]
if child == HOT_NONE_NODE {
continue
}
let child_depth = if node.op == HotOp::Try && slot == 1 {
depth + 1
} else {
depth
}
let child_key = (child.to_uint64() << 32) | child_depth.to_uint64()
if seen.add(child_key) {
pending.push(child_key)
}
}
}
count
}
///|
fn hot_lower_impl_strict_selected_delegate_if_root(
func : HotFunc,
if_id : NodeId,
label_frames : HotLowerLabelFrames,
) -> NodeId? {
match hot_node_get(func, if_id).op {
HotOp::If => ()
_ => return None
}
if hot_node_result_arity(func, if_id) != 0 {
return None
}
let label_id = hot_control_node_label(func, if_id)
if label_id < 0 || hot_lower_impl_label_is_used(label_frames, label_id) {
return None
}
let condition = hot_child_get(func, if_id, 0)
let selected_then = match hot_node_get(func, condition).op {
HotOp::Const =>
match hot_const_get(func, hot_node_get(func, condition).imm0) {
HotConstPayload::I32Const(@lib.I32(raw)) => raw != 0
_ => return None
}
_ => return None
}
let then_ref = hot_region_then_ref(func, if_id)
let else_ref = match hot_region_else_ref(func, if_id) {
Some(else_ref) => else_ref
None => return None
}
let selected_ref = if selected_then { then_ref } else { else_ref }
let opposite_ref = if selected_then { else_ref } else { then_ref }
if hot_region_root_count(func, selected_ref) != 1 {
return None
}
let selected_root = hot_region_get(func, selected_ref, 0)
if !hot_region_is_strict_no_work_forest(func, opposite_ref, fn(label_id) {
hot_lower_impl_label_is_used(label_frames, label_id)
}) {
return None
}
Some(selected_root)
}
///|
fn hot_lower_impl_direct_delegate_target(
func : HotFunc,
region_id : NodeId,
label_frames : HotLowerLabelFrames,
) -> LabelId? {
if region_id == HOT_NONE_NODE {
return None
}
let region_ref = hot_region_body_ref(func, region_id)
if hot_region_root_count(func, region_ref) != 1 {
return None
}
let mut root_id = hot_region_get(func, region_ref, 0)
while true {
let root = hot_node_get(func, root_id)
match root.op {
HotOp::Delegate => return Some(root.imm0)
HotOp::Block => {
if hot_node_result_arity(func, root_id) != 0 {
return None
}
let label_id = hot_control_node_label(func, root_id)
if label_id < 0 || hot_lower_impl_label_is_used(label_frames, label_id) {
return None
}
let body_ref = hot_region_body_ref(func, root_id)
if hot_region_root_count(func, body_ref) != 1 {
return None
}
root_id = hot_region_get(func, body_ref, 0)
}
HotOp::If =>
match
hot_lower_impl_strict_selected_delegate_if_root(
func, root_id, label_frames,
) {
Some(selected_root) => root_id = selected_root
None => return None
}
_ => return None
}
}
None
}
///|
fn hot_lower_impl_stack_has_node(
stack : HotLowerValueStack,
node_id : NodeId,
) -> Bool {
stack.any(fn(value) { value.node_id == node_id })
}
///|
fn hot_lower_impl_has_external_effects(mask : EffectMask) -> Bool {
mask != 0U && mask != EFFECT_MASK_LOCAL_STATE
}
///|
fn hot_lower_impl_build_first_effect_orders(func : HotFunc) -> Array[Int32] {
let none = hot_node_count(func)
let orders = Array::make(none, -1)
fn visit(node_id : NodeId) -> Int32 {
if node_id < 0 || !hot_node_is_live(func, node_id) {
return none
}
if orders[node_id] >= 0 {
return orders[node_id]
}
let own_effects = effects_mask_for_flags(hot_node_flags(func, node_id)) |
effects_mask_for_node_exact(func, node_id)
orders[node_id] = if hot_lower_impl_has_external_effects(own_effects) {
hot_node_get(func, node_id).order
} else {
none
}
for input_id in hot_lower_impl_input_info(func, node_id).nodes {
let input_order = visit(input_id)
if input_order < orders[node_id] {
orders[node_id] = input_order
}
}
orders[node_id]
}
for node_id = 0; node_id < none; node_id = node_id + 1 {
ignore(visit(node_id))
}
orders
}
///|
/// First source-ordered external effect in each node's operand tree.
/// Region bodies are separate scopes. The node count denotes no effect.
pub fn hot_source_first_effect_orders(func : HotFunc) -> Array[Int32] {
hot_lower_impl_build_first_effect_orders(func)
}
///|
fn hot_lower_impl_collect_preceding_dependency(
func : HotFunc,
node_id : NodeId,
before_node_id : NodeId,
consumer_id : NodeId,
value_orders : Array[Int32],
node_effects : Array[EffectMask],
seen_consumer : Array[Int32],
consumer_bounds : Array[Int32],
touched : Array[NodeId],
out : Array[NodeId],
) -> Unit {
hot_lower_impl_collect_preceding_dependency_indexed(
func,
node_id,
before_node_id,
consumer_id,
value_orders,
node_effects,
seen_consumer,
consumer_bounds,
touched,
out,
None,
None,
)
}
///|
fn hot_lower_impl_collect_preceding_dependency_indexed(
func : HotFunc,
node_id : NodeId,
before_node_id : NodeId,
consumer_id : NodeId,
value_orders : Array[Int32],
node_effects : Array[EffectMask],
seen_consumer : Array[Int32],
consumer_bounds : Array[Int32],
touched : Array[NodeId],
out : Array[NodeId],
minimum_cache : Array[Int32]?,
input_queries : Array[Int32]?,
) -> Unit {
if node_id < 0 ||
node_id >= seen_consumer.length() ||
seen_consumer[node_id] <= consumer_id ||
!hot_node_is_live(func, node_id) {
return
}
// Subtree masks include every operand: a pure tree cannot contain an
// effectful value that needs to be carried across this root.
if effects_is_pure(node_effects[node_id]) {
return
}
let carries_value = value_orders[node_id] < before_node_id &&
hot_node_result_arity(func, node_id) > 0
if !carries_value {
match minimum_cache {
Some(cache) =>
if hot_lower_impl_preceding_dependency_min_node(
func, node_id, cache, value_orders, node_effects,
) >=
before_node_id {
// The minimum covers every eligible value in this operand subtree.
// No consumer can carry one of them across this source-order bound.
return
}
None => ()
}
}
if seen_consumer[node_id] == seen_consumer.length() {
touched.push(node_id)
}
seen_consumer[node_id] = consumer_id
if carries_value {
if consumer_bounds[node_id] < 0 {
out.push(node_id)
}
consumer_bounds[node_id] = consumer_id
return
}
match input_queries {
Some(count) => count[0] += 1
None => ()
}
let inputs = hot_lower_impl_input_info(func, node_id)
for input_id in inputs.nodes {
hot_lower_impl_collect_preceding_dependency_indexed(
func, input_id, before_node_id, consumer_id, value_orders, node_effects, seen_consumer,
consumer_bounds, touched, out, minimum_cache, input_queries,
)
}
}
///|
fn hot_lower_impl_local_state_child_count(
func : HotFunc,
node_id : NodeId,
) -> Int32 {
let count = hot_child_count(func, node_id)
match hot_node_op(func, node_id) {
HotOp::Block | HotOp::Loop => {
// Dead region tails cannot observe or overwrite a carried value.
for slot = 0; slot < count; slot = slot + 1 {
let child = hot_child_get(func, node_id, slot)
if child >= 0 &&
hot_node_is_live(func, child) &&
hot_is_terminator_node(func, child) {
return slot + 1
}
}
count
}
_ => count
}
}
///|
fn hot_lower_impl_preceding_dependency_min_node(
func : HotFunc,
node_id : NodeId,
cache : Array[Int32],
value_orders : Array[Int32],
node_effects : Array[EffectMask],
) -> Int32 {
let none = hot_node_count(func)
if node_id < 0 ||
node_id >= cache.length() ||
!hot_node_is_live(func, node_id) {
return none
}
if cache[node_id] != -2 {
return cache[node_id]
}
if effects_is_pure(node_effects[node_id]) {
cache[node_id] = none
return none
}
let mut minimum = if hot_node_result_arity(func, node_id) > 0 &&
!effects_is_pure(node_effects[node_id]) {
value_orders[node_id]
} else {
none
}
// An in-progress cycle has no proved lower bound. Preserve the collector's
// conservative traversal rather than allowing a recursive cache fill to loop.
cache[node_id] = -1
for input_id in hot_lower_impl_input_info(func, node_id).nodes {
let input_minimum = hot_lower_impl_preceding_dependency_min_node(
func, input_id, cache, value_orders, node_effects,
)
if input_minimum < minimum {
minimum = input_minimum
}
}
cache[node_id] = minimum
minimum
}
///|
fn hot_lower_impl_build_preceding_dependency_suffix_minima(
func : HotFunc,
roots : Array[NodeId],
cache : Array[Int32],
value_orders : Array[Int32],
node_effects : Array[EffectMask],
) -> Array[Int32] {
let none = hot_node_count(func)
let suffix_minima = Array::make(roots.length() + 1, none)
let mut idx = roots.length() - 1
while idx >= 0 {
let root_minimum = hot_lower_impl_preceding_dependency_min_node(
func,
roots[idx],
cache,
value_orders,
node_effects,
)
suffix_minima[idx] = if root_minimum < suffix_minima[idx + 1] {
root_minimum
} else {
suffix_minima[idx + 1]
}
idx -= 1
}
suffix_minima
}
///|
fn hot_lower_impl_node_accesses_local(
func : HotFunc,
node_id : NodeId,
cache : Array[Int32],
) -> Bool {
if node_id < 0 ||
node_id >= cache.length() ||
!hot_node_is_live(func, node_id) {
return false
}
if cache[node_id] >= 0 {
return cache[node_id] == 1
}
cache[node_id] = 0
match hot_node_get(func, node_id).op {
HotOp::LocalGet | HotOp::LocalSet | HotOp::LocalTee => {
cache[node_id] = 1
return true
}
_ => ()
}
let child_count = hot_lower_impl_local_state_child_count(func, node_id)
for slot = 0; slot < child_count; slot = slot + 1 {
let input_id = hot_child_get(func, node_id, slot)
if hot_lower_impl_node_accesses_local(func, input_id, cache) {
cache[node_id] = 1
return true
}
}
false
}
///|
fn hot_lower_impl_roots_may_need_future_dependencies(
func : HotFunc,
roots : Array[NodeId],
local_access_cache : Array[Int32],
node_effects : Array[EffectMask],
) -> Bool {
for root_id in roots {
if hot_lower_impl_has_external_effects(node_effects[root_id]) {
return true
}
if hot_node_result_arity(func, root_id) == 0 &&
(match hot_query_op(func, root_id) {
HotOp::Call | HotOp::CallIndirect | HotOp::CallRef => true
_ => false
}) {
return true
}
if hot_lower_impl_node_accesses_local(func, root_id, local_access_cache) {
return true
}
}
false
}
///|
fn hot_lower_impl_future_dependencies(
func : HotFunc,
roots : Array[NodeId],
label_frames : HotLowerLabelFrames,
) -> HotLowerFutureDependencies {
if roots.length() < 2 {
return HotLowerFutureDependencies::new(false)
}
if label_frames.source_order is None {
let minima = match label_frames.operand_minima {
Some(minima) => minima
None => {
let minima = Array::make(hot_node_count(func), -1)
label_frames.operand_minima = Some(minima)
minima
}
}
// Reuse the complete CFG operand-order proof for this immutable lower.
// Shared operands are visited once; all unproved regions keep full facts.
if !cfg_region_may_need_source_order(func, roots, minima) {
return HotLowerFutureDependencies::new(false)
}
}
HotLowerFutureDependencies::new(
true,
suffix_minima=hot_source_order_future_suffix_minima(
func,
roots,
hot_lower_impl_source_order(func, label_frames),
),
)
}
///|
fn hot_lower_impl_carried_local_write_conflicts(
func : HotFunc,
carried_id : NodeId,
current_root : NodeId,
facts : HotSourceOrderFacts,
after_order~ : Int32 = -1,
before_access_order~ : Int32 = -1,
access_before_order~ : Int32? = None,
access_visits~ : Array[Int32]? = None,
preserve_before_escape~ : Bool = false,
) -> Bool {
// Reuse sparse local-access summaries. Allocating function-sized visited
// and local arrays for every candidate pair makes wide constructors cubic.
let writes = hot_source_order_local_access_nodes(func, carried_id, facts)
let mut first_access = hot_node_count(func)
let mut indexed_orders : @hashmap.HashMap[Int32, Int32]? = None
let mut bounded_orders : @hashmap.HashMap[Int32, Int32]? = None
for write_id in writes {
let write = hot_node_get(func, write_id)
if (write.op is HotOp::LocalSet || write.op is HotOp::LocalTee) &&
write.order > after_order &&
write.imm0 >= 0 &&
write.imm0 < hot_local_count(func) {
// An escape can skip a write whose value is consumed on fallthrough,
// even when the escape itself does not read that local.
if preserve_before_escape &&
write.order < facts.first_effect_orders[current_root] {
return true
}
let accesses = hot_source_order_local_access_nodes(
func, current_root, facts,
)
let use_index = accesses.length() > 4 &&
(
writes.length() > 1 ||
(accesses.length() > 8 && facts.last_wide_access_root == current_root) ||
facts.first_local_access_orders.contains(current_root)
)
if accesses.length() > 8 {
// A sequence of singleton writers otherwise rescans the same wide
// consumer once per writer. Keep the first query allocation-free;
// repeated consumers use the existing snapshot-owned minimum index.
facts.last_wide_access_root = current_root
}
if use_index {
// Cache the minimum for each local, not merely an access that happens
// to fall inside this query's interval. Later accesses cannot replace
// an earlier conflicting access when applying the order bounds.
let orders = match indexed_orders {
Some(orders) => orders
None => {
let orders = hot_source_order_first_local_access_orders(
func,
current_root,
accesses,
facts,
visits=access_visits,
)
indexed_orders = Some(orders)
orders
}
}
let order = orders.get(write.imm0).or(hot_node_count(func))
if order > before_access_order && order < first_access {
first_access = order
} else if order <= before_access_order {
// The first access to this local is outside the requested interval.
// A later access to the same local can still conflict.
let later = match bounded_orders {
Some(later) => later
None => {
let later : @hashmap.HashMap[Int32, Int32] = @hashmap.HashMap::new()
for access_id in accesses {
match access_visits {
Some(count) => count[0] += 1
None => ()
}
let access = hot_node_get(func, access_id)
if access.order > before_access_order &&
access.order < later.get(access.imm0).or(hot_node_count(func)) {
later.set(access.imm0, access.order)
}
}
bounded_orders = Some(later)
later
}
}
let later_order = later.get(write.imm0).or(hot_node_count(func))
if later_order < first_access {
first_access = later_order
}
}
} else {
for access_id in accesses {
match access_visits {
Some(count) => count[0] += 1
None => ()
}
let access = hot_node_get(func, access_id)
if access.imm0 == write.imm0 &&
access.order > before_access_order &&
access.order < first_access {
first_access = access.order
}
}
}
}
}
first_access < access_before_order.or(hot_node_count(func)) &&
first_access > before_access_order
}
///|
// An exact store run consumes an already computed tuple one lane at a time.
// The source query can select only that available producer when all later
// roots have no older dependency. Cache both success and scanned rejection.
fn hot_lower_impl_available_tuple_store_run(
func : HotFunc,
current_root : NodeId,
roots : Array[NodeId],
future_start : Int32,
index : HotSourceOrderRootIndex?,
facts : HotSourceOrderFacts,
stack : HotLowerValueStack,
ordering_work~ : Array[Int32]? = None,
) -> Bool {
let index = match index {
Some(index) => index
None => return false
}
let position = future_start - 1
if position < 0 ||
position >= roots.length() ||
index.length != roots.length() ||
roots[position] != current_root {
return false
}
if position == index.lower_store_next && position < index.lower_store_end {
match ordering_work {
Some(count) => count[0] += 1
None => ()
}
index.lower_store_next = position + 1
return index.lower_store_admitted &&
stack.length() > 0 &&
stack[stack.length() - 1].node_id == index.lower_store_producer
}
match ordering_work {
Some(count) => count[0] += 1
None => ()
}
let current = hot_node_get(func, current_root)
if current.op != HotOp::LocalSet ||
current.child_count != 1 ||
hot_node_result_arity(func, current_root) != 0 ||
stack.is_empty() {
return false
}
let producer = hot_child_get(func, current_root, 0)
if stack[stack.length() - 1].node_id != producer ||
!hot_node_is_live(func, producer) ||
effects_is_pure(facts.node_effects[producer]) ||
facts.preceding_dependency_min_cache[producer] < 0 ||
hot_node_result_arity(func, producer) < 2 {
return false
}
let mut end = position
let mut minimum = hot_node_count(func)
let mut maximum = -1
while end < roots.length() {
match ordering_work {
Some(count) => count[0] += 1
None => ()
}
let store = hot_node_get(func, roots[end])
if store.op != HotOp::LocalSet ||
store.child_count != 1 ||
hot_child_get(func, roots[end], 0) != producer ||
hot_node_result_arity(func, roots[end]) != 0 {
break
}
let order = store.order.min(roots[end])
minimum = minimum.min(order)
maximum = maximum.max(order)
end += 1
}
let run_length = end - position
let mut admitted = run_length > 0 &&
run_length <= stack.length() &&
facts.value_orders[producer] < minimum &&
index.suffix_minima[end] >= maximum
if admitted {
// Count only the lanes this run consumes. Scanning every remaining lane
// for separated short runs would add another quadratic fallback cost.
for offset in 0.. count[0] += 1
None => ()
}
if stack[stack.length() - 1 - offset].node_id != producer {
admitted = false
break
}
}
}
index.lower_store_next = position + 1
index.lower_store_end = end
index.lower_store_producer = producer
index.lower_store_admitted = admitted
admitted
}
///|
fn hot_lower_impl_emit_preceding_future_dependencies(
func : HotFunc,
current_root : NodeId,
future_roots : Array[NodeId],
future_start : Int32,
future_dependency_suffix_minima : HotLowerFutureDependencies,
use_def : HotNodeUseCounts,
label_frames : HotLowerLabelFrames,
label_stack_head : Int32,
stack : HotLowerValueStack,
out : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
ordering_work~ : Array[Int32]? = None,
) -> Unit {
if !future_dependency_suffix_minima.required ||
future_start >= future_roots.length() {
return
}
// A missing suffix index alone is not an empty-demand proof. Required plans
// always own complete ordering facts, regardless of earlier effect-only work.
guard label_frames.source_order is Some(facts) else {
abort("required future dependency plan must own complete ordering facts")
}
if future_dependency_suffix_minima.suffix_minima is Some(_) &&
hot_lower_impl_available_tuple_store_run(
func,
current_root,
future_roots,
future_start,
future_dependency_suffix_minima.suffix_minima,
facts,
stack,
ordering_work~,
) {
return
}
for
node_id in hot_source_order_preceding_dependencies(
func,
current_root,
future_roots,
future_start,
future_dependency_suffix_minima.suffix_minima,
facts,
root_visits=ordering_work,
) {
if !hot_lower_impl_stack_has_node(stack, node_id) {
hot_lower_impl_emit_value(
func, node_id, use_def, label_frames, label_stack_head, stack, out, extra_locals,
options,
)
}
}
}
///|
fn hot_lower_impl_push_control_prefix(
func : HotFunc,
id : NodeId,
count : Int32,
value_stack : HotLowerValueStack,
label_frames : HotLowerLabelFrames,
) -> Unit {
let node = hot_node_get(func, id)
let mut slot = 0
while slot < count {
let input_id = func.children[node.child_base + slot]
let mut end = slot + 1
while end < count && func.children[node.child_base + end] == input_id {
end += 1
}
// The verifier and input emitter group consecutive producers. Partial
// tuples use the suffix of their result row; a later run starts afresh.
// Borrow interned rows instead of copying one row for each entry lane.
let input_type = hot_type_get(func, hot_node_type(func, input_id))
// Check the original node/type boundary before bitmap bookkeeping.
// Catch and control inputs can lack an emit_root request of their own.
match label_frames.requested_nodes {
Some(requested) => requested[input_id] = true
None => ()
}
match input_type {
VoidType | BlockResult(VoidBlockType) => ()
Value(val_type) | BlockResult(ValTypeBlockType(val_type)) =>
for _ in slot..
abort("unresolved control prefix result type")
BlockResultResolved(_, _, results) | Results(results) => {
let arity = results.length()
let lanes = end - slot
if lanes <= arity {
for lane in (arity - lanes).. 0 {
// Loop entries can repeat a complete producer. Mirror the emitter's
// repeated evaluations and retain the final requested lanes.
let start = (arity - lanes % arity) % arity
for lane in 0.. Array[Instruction] {
let out : Array[Instruction] = []
let mut next_stack = label_stack_head
let label_id = hot_control_node_label(func, id)
if label_id >= 0 &&
hot_label_kind(func, label_id) != HotLabelKind::RegionLabel {
next_stack = hot_lower_impl_push_label(
label_frames, label_stack_head, label_id,
)
}
let node = hot_node_get(func, id)
let body_base = if node.op == HotOp::Loop {
hot_label_branch_arity(func, label_id)
} else if node.op == HotOp::Block && node.imm1 > 0 {
node.imm1
} else if node.op == HotOp::Block &&
label_id >= 0 &&
hot_label_kind(func, label_id) == HotLabelKind::RegionLabel {
hot_lower_impl_catch_payload_prefix(func, id).length()
} else {
0
}
let region_keep_count = if node.op == HotOp::Loop {
hot_node_result_arity(func, id)
} else {
hot_label_branch_arity(func, label_id)
}
let value_stack : HotLowerValueStack = match entry_stack {
Some(values) => values.copy()
None => hot_lower_value_stack_new()
}
hot_lower_impl_push_control_prefix(
func, id, body_base, value_stack, label_frames,
)
let root_ids : Array[NodeId] = []
for i = body_base; i < node.child_count; i = i + 1 {
let child = func.children[node.child_base + i]
if child != HOT_NONE_NODE {
root_ids.push(child)
}
}
let preserve_region_results = hot_lower_impl_preserve_region_result_roots(
func, use_def, root_ids, region_keep_count,
)
let future_dependency_suffix_minima = hot_lower_impl_future_dependencies(
func, root_ids, label_frames,
)
// An arm can forward its earlier entry lanes as leading result roots. They
// are already in the right order below the remaining entry values. Emitting
// them again duplicates constants or spills effectful values unnecessarily.
let mut entry_root_prefix = 0
if entry_stack is Some(_) || (node.op == HotOp::Block && node.imm1 > 0) {
// Each forwarded root denotes one entry lane, including tuple suffixes.
for root in root_ids {
if entry_root_prefix >= value_stack.length() ||
hot_node_result_arity(func, root) == 0 ||
value_stack[entry_root_prefix].node_id != root {
break
}
entry_root_prefix += 1
}
}
let mut root_idx = 0
for i = body_base; i < node.child_count; i = i + 1 {
let child = func.children[node.child_base + i]
if child != HOT_NONE_NODE {
if root_idx < entry_root_prefix {
root_idx += 1
continue
}
if hot_query_op(func, child) == HotOp::Unreachable {
hot_lower_impl_drop_dead_stack_before_unreachable(
child, value_stack, out,
)
}
hot_lower_impl_emit_preceding_future_dependencies(
func,
child,
root_ids,
root_idx + 1,
future_dependency_suffix_minima,
use_def,
label_frames,
next_stack,
value_stack,
out,
extra_locals,
options,
)
hot_lower_impl_emit_root(
func, child, use_def, label_frames, next_stack, value_stack, out, extra_locals,
options,
)
let is_final_root = i + 1 == node.child_count
hot_lower_impl_drop_dead_nonfinal_root_results(
func,
child,
use_def,
preserve_region_results[root_idx],
is_final_root,
value_stack,
out,
)
root_idx += 1
}
}
hot_lower_impl_finalize_region_stack(
func, region_keep_count, value_stack, out, extra_locals,
)
if next_stack != label_stack_head {
hot_lower_impl_pop_label(label_frames, next_stack)
}
out
}
///|
fn hot_lower_impl_region_body(
func : HotFunc,
id : NodeId,
use_def : HotNodeUseCounts,
label_frames : HotLowerLabelFrames,
label_stack_head : Int32,
extra_locals : Array[ValType],
options : HotLowerOptions,
entry_stack~ : HotLowerValueStack? = None,
) -> Array[Instruction] {
if id == HOT_NONE_NODE {
return []
}
let label_id = hot_control_node_label(func, id)
let node = hot_node_get(func, id)
if node.op == HotOp::Block &&
label_id >= 0 &&
hot_label_kind(func, label_id) == HotLabelKind::RegionLabel {
return hot_lower_impl_body_items(
func,
id,
use_def,
label_frames,
label_stack_head,
extra_locals,
options,
entry_stack~,
)
}
let out : Array[Instruction] = []
let value_stack : HotLowerValueStack = match entry_stack {
Some(values) => values.copy()
None => hot_lower_value_stack_new()
}
hot_lower_impl_emit_root(
func, id, use_def, label_frames, label_stack_head, value_stack, out, extra_locals,
options,
)
out
}
///|
fn hot_lower_impl_result_only_block_type_conflicts_with_param_arity(
func : HotFunc,
block_type : BlockType,
) -> Bool {
match block_type {
TypeIdxBlockType(type_idx) => {
for type_id = 0; type_id < func.types.length(); type_id = type_id + 1 {
match hot_type_get(func, type_id) {
BlockResultResolved(TypeIdxBlockType(candidate_idx), params, _) =>
if candidate_idx == type_idx && params.length() > 0 {
return true
}
_ => ()
}
}
if hot_param_count(func) > 0 {
match func.func_type_idx {
Some(func_type_idx) if func_type_idx == type_idx => return true
_ => ()
}
}
false
}
_ => false
}
}
///|
fn hot_lower_impl_result_only_block_type_candidate(
func : HotFunc,
block_type : BlockType,
params : Array[ValType],
results : Array[ValType],
expected_results : Array[ValType],
) -> BlockType? {
if params.length() != 0 || results != expected_results {
return None
}
if hot_lower_impl_result_only_block_type_conflicts_with_param_arity(
func, block_type,
) {
return None
}
Some(block_type)
}
///|
fn hot_lower_impl_result_only_block_type(
func : HotFunc,
results : Array[ValType],
) -> BlockType? {
let body_result_ty = hot_body_result_type(func)
match hot_type_get(func, body_result_ty) {
BlockResultResolved(block_type, params, body_results) =>
match
hot_lower_impl_result_only_block_type_candidate(
func, block_type, params, body_results, results,
) {
Some(candidate) => return Some(candidate)
None => ()
}
BlockResult(block_type) =>
if hot_type_results(func, body_result_ty) == results &&
!hot_lower_impl_result_only_block_type_conflicts_with_param_arity(
func, block_type,
) {
return Some(block_type)
}
_ => ()
}
for type_id = 0; type_id < func.types.length(); type_id = type_id + 1 {
if type_id == body_result_ty {
continue
}
match hot_type_get(func, type_id) {
BlockResultResolved(block_type, params, candidate_results) =>
match
hot_lower_impl_result_only_block_type_candidate(
func, block_type, params, candidate_results, results,
) {
Some(candidate) => return Some(candidate)
None => ()
}
BlockResult(block_type) =>
if hot_type_results(func, type_id) == results &&
!hot_lower_impl_result_only_block_type_conflicts_with_param_arity(
func, block_type,
) {
return Some(block_type)
}
_ => ()
}
}
None
}
///|
fn hot_lower_impl_drop_dead_nonfinal_root_results(
func : HotFunc,
id : NodeId,
use_def : HotNodeUseCounts,
preserve_region_result : Bool,
is_final_root : Bool,
stack : HotLowerValueStack,
out : Array[Instruction],
) -> Unit {
if is_final_root || preserve_region_result {
return
}
let result_arity = hot_node_result_arity(func, id)
if result_arity == 0 || hot_node_use_count(use_def, id) != 1 {
return
}
for _ in 0.. Unit {
ignore(id)
while stack.length() > 0 {
out.push(Instruction::drop())
ignore(stack.pop())
}
}
///|
priv struct HotLowerTypedLoopConstPrefixLowering {
outer_locals : Array[LocalIdx]
raw_block_type : BlockType
rewritten_body : Array[Instruction]
lower_to_block : Bool
}
///|
fn hot_lower_impl_typed_loop_const_prefix_head_instr(
instr : Instruction,
) -> Bool {
match instr {
I32Const(_)
| I64Const(_)
| F32Const(_)
| F64Const(_)
| RefNull(_)
| StringConst(_)
| Drop => true
_ => false
}
}
///|
fn hot_lower_impl_typed_loop_const_prefix_reuses_existing_spill(
body : Array[Instruction],
) -> Bool {
if body.length() < 4 {
return false
}
match body[0] {
LocalGet(source_local) =>
match body[1] {
LocalSet(spill_local) if spill_local != source_local =>
match body[2] {
LocalGet(reload_local) if reload_local == spill_local =>
hot_lower_impl_typed_loop_const_prefix_head_instr(body[3])
_ => false
}
_ => false
}
_ => false
}
}
///|
fn hot_lower_impl_typed_loop_const_prefix_lowering(
func : HotFunc,
id : NodeId,
body : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
label_frames : HotLowerLabelFrames,
) -> HotLowerTypedLoopConstPrefixLowering? {
if !options.preserve_typed_loop_const_prefix_spills {
return None
}
let label_id = hot_control_node_label(func, id)
if hot_lower_impl_label_is_used(label_frames, label_id) {
return None
}
let results = hot_type_results(func, hot_node_type(func, id))
let raw_block_type = match results {
[] => BlockType::void_()
[result_type] => BlockType::val_type(result_type)
_ =>
match hot_lower_impl_result_only_block_type(func, results) {
Some(block_type) => block_type
None => return None
}
}
if hot_lower_impl_typed_loop_const_prefix_reuses_existing_spill(body) {
return Some({
outer_locals: [],
raw_block_type,
rewritten_body: body,
lower_to_block: false,
})
}
let param_types = hot_lower_impl_control_param_types(func, id)
if results.length() == 0 &&
param_types.length() > 1 &&
!hot_lower_impl_body_targets_removed_label(body, 0) {
let block_type = hot_lower_impl_control_block_type(func, id)
return Some({
outer_locals: [],
raw_block_type,
rewritten_body: [Instruction::block(block_type, Expr::new(body))],
lower_to_block: true,
})
}
if param_types.length() != 1 {
return None
}
match body {
[I32Const(_), ..]
| [I64Const(_), ..]
| [F32Const(_), ..]
| [F64Const(_), ..]
| [RefNull(_), ..]
| [StringConst(_), ..]
| [Drop, ..] => {
let outer_local = hot_lower_impl_alloc_scratch_local(
func,
extra_locals,
param_types[0],
)
let inner_local = hot_lower_impl_alloc_scratch_local(
func,
extra_locals,
param_types[0],
)
let rewritten_body : Array[Instruction] = [
Instruction::local_get(outer_local),
Instruction::local_set(inner_local),
Instruction::local_get(inner_local),
]
for instr in body {
rewritten_body.push(instr)
}
Some({
outer_locals: [outer_local],
raw_block_type,
rewritten_body,
lower_to_block: false,
})
}
_ => None
}
}
///|
fn hot_lower_impl_alloc_scratch_local(
func : HotFunc,
extra_locals : Array[ValType],
val_type : ValType,
) -> LocalIdx {
let local_id = hot_local_count(func) + extra_locals.length()
extra_locals.push(val_type)
LocalIdx::new(local_id.reinterpret_as_uint())
}
///|
fn hot_lower_impl_closed_control_proves_terminal(instr : Instruction) -> Bool {
match instr {
Block(_, Expr(body)) | Loop(_, Expr(body)) =>
!hot_lower_impl_body_targets_removed_label(body, 0) &&
hot_lower_impl_tail_proves_terminal(body)
If(_, then_, Some(else_)) =>
!hot_lower_impl_body_targets_removed_label(then_, 0) &&
!hot_lower_impl_body_targets_removed_label(else_, 0) &&
hot_lower_impl_tail_proves_terminal(then_) &&
hot_lower_impl_tail_proves_terminal(else_)
_ => false
}
}
///|
fn hot_lower_impl_tail_proves_terminal(body : Array[Instruction]) -> Bool {
if body.is_empty() {
return false
}
match body[body.length() - 1] {
Return
| ReturnCall(_)
| ReturnCallIndirect(_, _)
| ReturnCallRef(_)
| Throw(_)
| ThrowRef
| Rethrow(_)
| Unreachable => true
instr => hot_lower_impl_closed_control_proves_terminal(instr)
}
}
///|
fn hot_lower_impl_finalize_region_stack(
func : HotFunc,
keep_count : Int32,
stack : HotLowerValueStack,
out : Array[Instruction],
extra_locals : Array[ValType],
) -> Unit {
if stack.length() < keep_count &&
out.length() > 0 &&
hot_lower_impl_closed_control_proves_terminal(out[out.length() - 1]) {
// A terminal replacement region can have a void signature. Closing it
// restores a concrete parent frame; the proven dead continuation needs
// its own terminal marker to satisfy the parent's declared results.
out.push(Instruction::unreachable_())
stack.clear()
return
}
if stack.length() <= keep_count {
return
}
if out.length() > 0 {
match out[out.length() - 1] {
Return | Throw(_) | ThrowRef | Unreachable => return
// A closed control frame contributes its declared concrete results to
// its parent even when execution escapes from inside that frame.
_ => ()
}
}
let preserved : Array[LocalIdx] = []
let preserved_count = if keep_count < stack.length() {
keep_count
} else {
stack.length()
}
for _ in 0.. 0 {
out.push(Instruction::drop())
ignore(stack.pop())
}
for i = preserved.length() - 1; i >= 0; i = i - 1 {
out.push(Instruction::local_get(preserved[i]))
}
}
///|
fn hot_lower_impl_preserve_region_result_roots(
func : HotFunc,
use_def : HotNodeUseCounts,
root_ids : Array[NodeId],
region_result_arity : Int32,
) -> @bitset.BitSet {
let preserve = @bitset.BitSet::new(root_ids.length())
let mut remaining = region_result_arity
for i = root_ids.length() - 1; i >= 0; i = i - 1 {
let root_id = root_ids[i]
if hot_lower_impl_clear_stack_after_emit(hot_query_op(func, root_id)) {
remaining = 0
continue
}
if remaining <= 0 {
break
}
if hot_node_use_count(use_def, root_id) != 1 {
continue
}
let result_arity = hot_node_result_arity(func, root_id)
if result_arity == 0 {
continue
}
preserve[i] = true
if result_arity >= remaining {
remaining = 0
} else {
remaining = remaining - result_arity
}
}
preserve
}
///|
fn hot_lower_impl_emit_instruction(
func : HotFunc,
id : NodeId,
use_def : HotNodeUseCounts,
label_frames : HotLowerLabelFrames,
label_stack_head : Int32,
out : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
) -> Unit {
let node = hot_node_get(func, id)
match node.op {
HotOp::Nop => out.push(Instruction::nop())
HotOp::Catch => ()
HotOp::Unreachable => out.push(Instruction::unreachable_())
HotOp::Const | HotOp::RefNull => hot_lower_impl_const(func, id, out)
HotOp::LocalGet =>
out.push(
Instruction::local_get(LocalIdx::new(node.imm0.reinterpret_as_uint())),
)
HotOp::LocalSet =>
out.push(
Instruction::local_set(LocalIdx::new(node.imm0.reinterpret_as_uint())),
)
HotOp::LocalTee =>
out.push(
Instruction::local_tee(LocalIdx::new(node.imm0.reinterpret_as_uint())),
)
HotOp::GlobalGet =>
out.push(
Instruction::global_get(GlobalIdx::new(node.imm0.reinterpret_as_uint())),
)
HotOp::GlobalSet =>
out.push(
Instruction::global_set(GlobalIdx::new(node.imm0.reinterpret_as_uint())),
)
HotOp::TableGet =>
out.push(Instruction::table_get(hot_lower_impl_table_idx(node.imm0)))
HotOp::TableSet =>
out.push(Instruction::table_set(hot_lower_impl_table_idx(node.imm0)))
HotOp::TableSize =>
out.push(Instruction::table_size(hot_lower_impl_table_idx(node.imm0)))
HotOp::TableGrow =>
out.push(Instruction::table_grow(hot_lower_impl_table_idx(node.imm0)))
HotOp::TableFill =>
out.push(Instruction::table_fill(hot_lower_impl_table_idx(node.imm0)))
HotOp::TableCopy =>
out.push(
Instruction::table_copy(
hot_lower_impl_table_idx(node.imm0),
hot_lower_impl_table_idx(node.imm1),
),
)
HotOp::TableInit =>
out.push(
Instruction::table_init(
ElemIdx::new(node.imm0.reinterpret_as_uint()),
hot_lower_impl_table_idx(node.imm1),
),
)
HotOp::ElemDrop =>
out.push(
Instruction::elem_drop(ElemIdx::new(node.imm0.reinterpret_as_uint())),
)
HotOp::MemorySize =>
out.push(Instruction::memory_size(hot_lower_impl_mem_idx(node.imm0)))
HotOp::MemoryGrow =>
out.push(Instruction::memory_grow(hot_lower_impl_mem_idx(node.imm0)))
HotOp::MemoryCopy =>
out.push(
Instruction::memory_copy(
hot_lower_impl_mem_idx(node.imm0),
hot_lower_impl_mem_idx(node.imm1),
),
)
HotOp::MemoryFill =>
out.push(Instruction::memory_fill(hot_lower_impl_mem_idx(node.imm0)))
HotOp::MemoryInit =>
out.push(
Instruction::memory_init(
DataIdx::new(node.imm0.reinterpret_as_uint()),
hot_lower_impl_mem_idx(node.imm1),
),
)
HotOp::DataDrop =>
out.push(
Instruction::data_drop(DataIdx::new(node.imm0.reinterpret_as_uint())),
)
HotOp::Drop => out.push(Instruction::drop())
HotOp::Call =>
out.push(Instruction::call(FuncIdx::new(node.imm0.reinterpret_as_uint())))
HotOp::CallIndirect => {
let sig = hot_lower_impl_call_sig(func, node.imm0)
let table_idx = match sig.table_idx {
Some(table_idx) => table_idx
None => abort("hot lower call_indirect requires a table index")
}
out.push(Instruction::call_indirect(sig.type_idx, table_idx))
}
HotOp::CallRef =>
out.push(
Instruction::call_ref(hot_lower_impl_call_sig(func, node.imm0).type_idx),
)
HotOp::ReturnCall =>
out.push(
Instruction::return_call(FuncIdx::new(node.imm0.reinterpret_as_uint())),
)
HotOp::ReturnCallIndirect => {
let sig = hot_lower_impl_call_sig(func, node.imm0)
let table_idx = match sig.table_idx {
Some(table_idx) => table_idx
None => abort("hot lower return_call_indirect requires a table index")
}
out.push(Instruction::return_call_indirect(sig.type_idx, table_idx))
}
HotOp::ReturnCallRef =>
out.push(
Instruction::return_call_ref(
hot_lower_impl_call_sig(func, node.imm0).type_idx,
),
)
HotOp::Br =>
out.push(
Instruction::br(
hot_lower_impl_label_depth(label_frames, label_stack_head, node.imm0),
),
)
HotOp::BrIf =>
out.push(
Instruction::br_if(
hot_lower_impl_label_depth(label_frames, label_stack_head, node.imm0),
),
)
HotOp::BrTable => {
let targets : Array[LabelIdx] = []
for label_id in hot_branch_table_targets(func, node.imm0) {
targets.push(
hot_lower_impl_label_depth(label_frames, label_stack_head, label_id),
)
}
out.push(
Instruction::br_table(
targets,
hot_lower_impl_label_depth(
label_frames,
label_stack_head,
hot_branch_table_default_target(func, node.imm0),
),
),
)
}
HotOp::BrOnNull =>
out.push(
Instruction::br_on_null(
hot_lower_impl_label_depth(label_frames, label_stack_head, node.imm0),
),
)
HotOp::BrOnNonNull =>
out.push(
Instruction::br_on_non_null(
hot_lower_impl_label_depth(label_frames, label_stack_head, node.imm0),
),
)
HotOp::BrOnCast | HotOp::BrOnCastFail =>
out.push(
hot_lower_impl_lower_br_on_cast(
func, id, label_frames, label_stack_head,
),
)
HotOp::Return => out.push(Instruction::return_())
HotOp::RefIsNull => out.push(Instruction::ref_is_null())
HotOp::RefFunc =>
out.push(
Instruction::ref_func(FuncIdx::new(node.imm0.reinterpret_as_uint())),
)
HotOp::Select => {
let instr = hot_lower_impl_exact_instr(func, id)
out.push(
match instr {
Select(Some(types)) => Instruction::select(types=Some(types.copy()))
_ => instr
},
)
}
HotOp::Load
| HotOp::Store
| HotOp::RefTest
| HotOp::RefCast
| HotOp::RefTestDesc
| HotOp::RefCastDescEq
| HotOp::Heap
| HotOp::Unary
| HotOp::Binary
| HotOp::Compare
| HotOp::Convert
| HotOp::StringOp
| HotOp::Atomic
| HotOp::Simd => out.push(hot_lower_impl_exact_instr(func, id))
HotOp::Continuation => {
let targets = hot_continuation_targets(func, id)
let depths = targets.map(fn(target) {
hot_lower_impl_label_depth(label_frames, label_stack_head, target)
})
out.push(
hot_continuation_with_handler_labels(
hot_lower_impl_exact_instr(func, id),
depths,
),
)
}
HotOp::Throw =>
out.push(
Instruction::throw_(TagIdx::new(node.imm0.reinterpret_as_uint())),
)
HotOp::ThrowRef => out.push(Instruction::throw_ref())
HotOp::Rethrow => {
let slot = label_frames.legacy_rethrow_frames.length() - 1 - node.imm0
let local_idx = if slot < 0 ||
slot >= label_frames.legacy_rethrow_frames.length() {
abort("hot lower legacy rethrow requires an active catch at its depth")
} else {
match label_frames.legacy_rethrow_frames[slot].local_idx {
Some(local_idx) => local_idx
None => {
out.push(
Instruction::rethrow(
hot_lower_impl_label_depth(
label_frames,
label_stack_head,
label_frames.legacy_rethrow_frames[slot].label_id,
),
),
)
return
}
}
}
out.push(Instruction::local_get(local_idx))
out.push(Instruction::throw_ref())
}
HotOp::Block =>
out.push(
Instruction::block(
hot_lower_impl_control_block_type(func, id),
Expr::new(
hot_lower_impl_body_items(
func, id, use_def, label_frames, label_stack_head, extra_locals, options,
),
),
),
)
HotOp::Loop => {
let loop_body = hot_lower_impl_body_items(
func, id, use_def, label_frames, label_stack_head, extra_locals, options,
)
match
hot_lower_impl_typed_loop_const_prefix_lowering(
func, id, loop_body, extra_locals, options, label_frames,
) {
Some(lowering) => {
for idx = lowering.outer_locals.length() - 1; idx >= 0; idx = idx - 1 {
out.push(Instruction::local_set(lowering.outer_locals[idx]))
}
if lowering.lower_to_block {
for outer_local in lowering.outer_locals {
out.push(Instruction::local_get(outer_local))
}
out.push(
Instruction::block(
hot_lower_impl_control_block_type(func, id),
Expr::new(lowering.rewritten_body),
),
)
} else {
match lowering.raw_block_type {
VoidBlockType if !hot_lower_impl_body_targets_removed_label(
lowering.rewritten_body,
0,
) =>
for
instr in hot_lower_impl_rebase_body(
lowering.rewritten_body,
0,
) {
out.push(instr)
}
_ =>
out.push(
Instruction::loop_(
lowering.raw_block_type,
Expr::new(lowering.rewritten_body),
),
)
}
}
}
None =>
out.push(
Instruction::loop_(
hot_lower_impl_control_block_type(func, id),
Expr::new(loop_body),
),
)
}
}
HotOp::If => {
let label_id = hot_control_node_label(func, id)
let next_stack = if label_id >= 0 {
hot_lower_impl_push_label(label_frames, label_stack_head, label_id)
} else {
label_stack_head
}
let condition_region_base = node.child_base
let then_id = func.children[condition_region_base + 1]
let else_id = func.children[condition_region_base + 2]
let entry_stack = if node.child_count == 3 {
None
} else {
let params = hot_lower_impl_control_param_types(func, id)
Some(
hot_lower_value_stack_makei(params.length(), fn(index) {
{
node_id: func.children[condition_region_base + 3 + index],
val_type: params[index],
}
}),
)
}
out.push(
Instruction::if_(
hot_lower_impl_control_block_type(func, id),
hot_lower_impl_region_body(
func,
then_id,
use_def,
label_frames,
next_stack,
extra_locals,
options,
entry_stack~,
),
if else_id == HOT_NONE_NODE {
None
} else {
Some(
hot_lower_impl_region_body(
func,
else_id,
use_def,
label_frames,
next_stack,
extra_locals,
options,
entry_stack~,
),
)
},
),
)
if next_stack != label_stack_head {
hot_lower_impl_pop_label(label_frames, next_stack)
}
}
HotOp::Try => {
let catch_region = func.children[node.child_base + 1]
if node.imm1 != 0 {
if catch_region == HOT_NONE_NODE {
abort("hot lower lifted legacy try requires a catch region")
}
let label_id = hot_control_node_label(func, id)
let try_stack = hot_lower_impl_push_label(
label_frames, label_stack_head, label_id,
)
let body = hot_lower_impl_region_body(
func,
func.children[node.child_base],
use_def,
label_frames,
try_stack,
extra_locals,
options,
)
let catches : Array[@lib.LegacyCatch] = []
let mut delegate = None
if node.imm1 == -2 {
let target = match
hot_lower_impl_direct_delegate_target(
func, catch_region, label_frames,
) {
Some(target) => target
None =>
abort(
"hot lower lifted legacy delegate requires a direct delegate root",
)
}
delegate = Some(
hot_lower_impl_label_depth(label_frames, label_stack_head, target),
)
} else {
label_frames.legacy_rethrow_frames.push({ label_id, local_idx: None, })
let catch_body = hot_lower_impl_region_body(
func, catch_region, use_def, label_frames, try_stack, extra_locals, options,
)
ignore(label_frames.legacy_rethrow_frames.pop())
if node.imm1 == -1 {
catches.push(@lib.LegacyCatch::all(Expr::new(catch_body)))
} else if node.imm1 > 0 {
catches.push(
@lib.LegacyCatch::tagged(
TagIdx::new((node.imm1 - 1).reinterpret_as_uint()),
Expr::new(catch_body),
),
)
} else {
abort("hot lower invalid lifted legacy catch kind")
}
}
hot_lower_impl_pop_label(label_frames, try_stack)
out.push(
Instruction::try_(
hot_lower_impl_control_block_type(func, id),
Expr::new(body),
catches,
delegate~,
),
)
return
}
if hot_node_result_arity(func, id) != 0 {
abort(
"hot lower only supports resultless synthetic catch-all try nodes",
)
}
if catch_region == HOT_NONE_NODE {
abort("hot lower synthetic catch-all try requires a catch region")
}
let body_ref = hot_region_body_ref(func, id)
let label_id = hot_control_node_label(func, id)
match
hot_lower_impl_direct_delegate_target(func, catch_region, label_frames) {
Some(target_label) => {
if target_label < 0 ||
hot_label_owner(func, target_label) == id ||
hot_lower_impl_label_is_used(label_frames, label_id) {
abort(
"hot lower direct delegate requires an unused inner try label and outer target",
)
}
ignore(
hot_lower_impl_label_depth(
label_frames, label_stack_head, target_label,
),
)
for
instr in hot_lower_impl_region_body(
func,
func.children[node.child_base],
use_def,
label_frames,
label_stack_head,
extra_locals,
options,
) {
out.push(instr)
}
return
}
None => ()
}
let catch_payloads = hot_lower_impl_catch_payload_prefix(
func, catch_region,
)
let rethrow_count = hot_lower_impl_catch_rethrow_count(
func,
hot_region_body_ref(func, catch_region),
0,
)
let rethrow_local = if rethrow_count > 0 {
Some(
hot_lower_impl_alloc_scratch_local(
func,
extra_locals,
ValType::ref_null_exn(),
),
)
} else {
None
}
if catch_payloads.length() > 1 {
let first_tag = catch_payloads[0].tag_idx
for payload in catch_payloads {
if payload.tag_idx != first_tag {
abort("hot lower typed legacy try payload lanes must share one tag")
}
}
}
if !effects_may_throw(effects_for_region(func, body_ref)) {
if hot_lower_impl_label_is_used(label_frames, label_id) {
let body_stack = hot_lower_impl_push_label(
label_frames, label_stack_head, label_id,
)
let body = hot_lower_impl_region_body(
func,
func.children[node.child_base],
use_def,
label_frames,
body_stack,
extra_locals,
options,
)
hot_lower_impl_pop_label(label_frames, body_stack)
out.push(Instruction::block(BlockType::void_(), Expr::new(body)))
} else {
for
instr in hot_lower_impl_region_body(
func,
func.children[node.child_base],
use_def,
label_frames,
label_stack_head,
extra_locals,
options,
) {
out.push(instr)
}
}
return
}
let try_stack = hot_lower_impl_push_label(
label_frames, label_stack_head, label_id,
)
let handler_label = func.labels.length() + label_frames.active.length()
let handler_stack = hot_lower_impl_push_label(
label_frames, try_stack, handler_label,
)
let table_label = func.labels.length() + label_frames.active.length()
let table_stack = hot_lower_impl_push_label(
label_frames, handler_stack, table_label,
)
let body = hot_lower_impl_region_body(
func,
func.children[node.child_base],
use_def,
label_frames,
table_stack,
extra_locals,
options,
)
hot_lower_impl_pop_label(label_frames, table_stack)
hot_lower_impl_pop_label(label_frames, handler_stack)
label_frames.legacy_rethrow_frames.push({
label_id,
local_idx: rethrow_local,
})
let catch_body = hot_lower_impl_region_body(
func, catch_region, use_def, label_frames, try_stack, extra_locals, options,
)
ignore(label_frames.legacy_rethrow_frames.pop())
hot_lower_impl_pop_label(label_frames, try_stack)
let handler_type = match rethrow_local {
Some(_) =>
match catch_payloads {
[] => BlockType::val_type(ValType::ref_null_exn())
[_, ..] => {
let handler_results : Array[ValType] = []
for lane in catch_payloads {
handler_results.push(lane.val_type)
}
handler_results.push(ValType::ref_null_exn())
match
hot_lower_impl_result_only_block_type(func, handler_results) {
Some(block_type) => block_type
None =>
abort(
"hot lower typed direct catch rethrow requires an existing payload-vector-plus-exnref handler type",
)
}
}
}
None =>
match catch_payloads {
[] => BlockType::void_()
[payload] => BlockType::val_type(payload.val_type)
_ => {
let payload_types : Array[ValType] = []
for payload in catch_payloads {
payload_types.push(payload.val_type)
}
match hot_lower_impl_result_only_block_type(func, payload_types) {
Some(block_type) => block_type
None =>
abort(
"hot lower typed legacy try requires an existing result-only multivalue handler type",
)
}
}
}
}
let catches = match rethrow_local {
Some(_) =>
match catch_payloads {
[] => [Catch::all_ref(LabelIdx::new(0U))]
[payload, ..] => [Catch::ref_(payload.tag_idx, LabelIdx::new(0U))]
}
None =>
match catch_payloads {
[] => [Catch::all(LabelIdx::new(0U))]
[payload, ..] => [Catch::new(payload.tag_idx, LabelIdx::new(0U))]
}
}
let handler_body : Array[Instruction] = [
Instruction::block(
handler_type,
Expr::new([
Instruction::try_table(BlockType::void_(), catches, Expr::new(body)),
Instruction::br(LabelIdx::new(1U)),
]),
),
]
match rethrow_local {
Some(local_idx) => handler_body.push(Instruction::local_set(local_idx))
None => ()
}
handler_body.append(catch_body)
out.push(Instruction::block(BlockType::void_(), Expr::new(handler_body)))
}
HotOp::TryTable => {
let label_id = hot_control_node_label(func, id)
let next_stack = if label_id >= 0 {
hot_lower_impl_push_label(label_frames, label_stack_head, label_id)
} else {
label_stack_head
}
out.push(
Instruction::try_table(
hot_lower_impl_control_block_type(func, id),
hot_lower_impl_try_catches(func, id, label_frames, label_stack_head),
Expr::new(
hot_lower_impl_region_body(
func,
func.children[node.child_base],
use_def,
label_frames,
next_stack,
extra_locals,
options,
),
),
),
)
if next_stack != label_stack_head {
hot_lower_impl_pop_label(label_frames, next_stack)
}
}
_ => abort("unsupported node for hot lowering: id=\{id} op=\{node.op}")
}
}
///|
fn hot_lower_impl_emit_value(
func : HotFunc,
id : NodeId,
use_def : HotNodeUseCounts,
label_frames : HotLowerLabelFrames,
label_stack_head : Int32,
stack : HotLowerValueStack,
out : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
force_emit~ : Bool = false,
) -> Unit {
hot_lower_impl_emit_root(
func,
id,
use_def,
label_frames,
label_stack_head,
stack,
out,
extra_locals,
options,
allow_stack_top_reuse=!force_emit,
)
}
///|
fn hot_lower_impl_emit_inputs(
func : HotFunc,
inputs : ArrayView[NodeId],
use_def : HotNodeUseCounts,
label_frames : HotLowerLabelFrames,
label_stack_head : Int32,
stack : HotLowerValueStack,
out : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
) -> Unit {
let matched = hot_lower_impl_matching_input_prefix(stack, inputs)
let mut idx = matched
while idx < inputs.length() {
let input_id = inputs[idx]
let mut run_len = 1
while idx + run_len < inputs.length() && inputs[idx + run_len] == input_id {
run_len += 1
}
let result_arity = hot_node_result_arity(func, input_id)
let emit_span = if result_arity > 0 { result_arity } else { 1 }
let emit_count = (run_len + emit_span - 1) / emit_span
for emit_idx = 0; emit_idx < emit_count; emit_idx = emit_idx + 1 {
hot_lower_impl_emit_value(
func,
input_id,
use_def,
label_frames,
label_stack_head,
stack,
out,
extra_locals,
options,
force_emit=emit_idx > 0,
)
}
idx += run_len
}
}
///|
fn hot_lower_impl_emit_typed_param_block_with_entry_prefix(
func : HotFunc,
id : NodeId,
use_def : HotNodeUseCounts,
label_frames : HotLowerLabelFrames,
label_stack_head : Int32,
stack : HotLowerValueStack,
out : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
) -> Unit {
let param_types = hot_lower_impl_control_param_types(func, id)
if param_types.length() == 0 {
abort("typed param block lowering requires resolved block params")
}
let node = hot_node_get(func, id)
let root_ids : Array[NodeId] = []
for i = 0; i < node.child_count; i = i + 1 {
let child = func.children[node.child_base + i]
if child != HOT_NONE_NODE {
root_ids.push(child)
}
}
let entry_values : HotLowerValueStack = hot_lower_value_stack_new()
let mut entry_root_count = 0
let mut prefix_drop_root_count = 0
let mut used_carrier_child_prefix = false
if stack.length() >= param_types.length() {
let start = stack.length() - param_types.length()
for idx = start; idx < stack.length(); idx = idx + 1 {
entry_values.push(stack[idx])
}
for _ in 0.. 0 &&
hot_child_count(func, root_ids[0]) >= param_types.length() {
used_carrier_child_prefix = true
let carrier_root = root_ids[0]
for idx = 0; idx < param_types.length(); idx = idx + 1 {
hot_lower_impl_emit_root(
func,
hot_child_get(func, carrier_root, idx),
use_def,
label_frames,
label_stack_head,
entry_stack,
out,
extra_locals,
options,
)
}
} else if root_ids.length() >= param_types.length() {
let prefix_inputs : Array[NodeId] = []
let mut prefix_ok = true
for idx = 0; idx < param_types.length(); idx = idx + 1 {
let root_id = root_ids[idx]
if hot_query_op(func, root_id) != HotOp::Drop ||
hot_child_count(func, root_id) != 1 {
prefix_ok = false
break
}
prefix_inputs.push(hot_child_get(func, root_id, 0))
}
if prefix_ok {
for idx = prefix_inputs.length() - 1; idx >= 0; idx = idx - 1 {
hot_lower_impl_emit_root(
func,
prefix_inputs[idx],
use_def,
label_frames,
label_stack_head,
entry_stack,
out,
extra_locals,
options,
)
}
if entry_stack.length() == param_types.length() {
prefix_drop_root_count = param_types.length()
}
} else {
while entry_stack.length() < param_types.length() &&
entry_root_count < root_ids.length() {
let entry_root = root_ids[entry_root_count]
hot_lower_impl_emit_root(
func, entry_root, use_def, label_frames, label_stack_head, entry_stack,
out, extra_locals, options,
)
entry_root_count += 1
}
}
} else {
while entry_stack.length() < param_types.length() &&
entry_root_count < root_ids.length() {
let entry_root = root_ids[entry_root_count]
hot_lower_impl_emit_root(
func, entry_root, use_def, label_frames, label_stack_head, entry_stack,
out, extra_locals, options,
)
entry_root_count += 1
}
}
if entry_stack.length() != param_types.length() {
abort(
"typed param block lowering requires entry roots or stack values for all params",
)
}
for value in entry_stack {
entry_values.push(value)
}
}
let body_root_ids : Array[NodeId] = []
let body_root_start = if prefix_drop_root_count > 0 {
prefix_drop_root_count
} else {
entry_root_count
}
for idx = body_root_start; idx < root_ids.length(); idx = idx + 1 {
body_root_ids.push(root_ids[idx])
}
let body_label_stack = hot_lower_impl_push_label(
label_frames,
label_stack_head,
hot_control_node_label(func, id),
)
let block_out : Array[Instruction] = []
let body_stack = entry_values.copy()
for _ in 0.. Unit {
let param_types = hot_lower_impl_control_param_types(func, id)
if param_types.length() == 0 {
abort("transparent param block lowering requires resolved block params")
}
let label_id = hot_control_node_label(func, id)
let node = hot_node_get(func, id)
let root_ids : Array[NodeId] = []
for i = 0; i < node.child_count; i = i + 1 {
let child = func.children[node.child_base + i]
if child != HOT_NONE_NODE {
root_ids.push(child)
}
}
let entry_values : HotLowerValueStack = hot_lower_value_stack_new()
let mut entry_root_count = 0
let mut used_carrier_child_prefix = false
if stack.length() >= param_types.length() {
let start = stack.length() - param_types.length()
for idx = start; idx < stack.length(); idx = idx + 1 {
entry_values.push(stack[idx])
}
for _ in 0.. 0 &&
hot_child_count(func, root_ids[0]) >= param_types.length() {
used_carrier_child_prefix = true
let carrier_root = root_ids[0]
for idx = 0; idx < param_types.length(); idx = idx + 1 {
hot_lower_impl_emit_root(
func,
hot_child_get(func, carrier_root, idx),
use_def,
label_frames,
label_stack_head,
entry_stack,
out,
extra_locals,
options,
)
}
} else {
while entry_stack.length() < param_types.length() &&
entry_root_count < root_ids.length() {
let entry_root = root_ids[entry_root_count]
hot_lower_impl_emit_root(
func, entry_root, use_def, label_frames, label_stack_head, entry_stack,
out, extra_locals, options,
)
entry_root_count += 1
}
}
if entry_stack.length() != param_types.length() {
abort(
"transparent param block lowering requires entry roots or stack values for all params",
)
}
for value in entry_stack {
entry_values.push(value)
}
}
let scratch_locals : Array[LocalIdx] = []
for param_type in param_types {
scratch_locals.push(
hot_lower_impl_alloc_scratch_local(func, extra_locals, param_type),
)
}
for idx = scratch_locals.length() - 1; idx >= 0; idx = idx - 1 {
out.push(Instruction::local_set(scratch_locals[idx]))
}
let body_stack : HotLowerValueStack = hot_lower_value_stack_new()
for idx = 0; idx < scratch_locals.length(); idx = idx + 1 {
out.push(Instruction::local_get(scratch_locals[idx]))
body_stack.push(entry_values[idx])
}
let branch_arity = hot_label_branch_arity(func, label_id)
if body_stack.length() == 1 &&
(branch_arity != 1 || used_carrier_child_prefix) {
let carried = body_stack.pop().unwrap()
let scratch = hot_lower_impl_alloc_scratch_local(
func,
extra_locals,
carried.val_type,
)
out.push(Instruction::local_set(scratch))
out.push(Instruction::local_get(scratch))
body_stack.push(carried)
}
let body_root_ids : Array[NodeId] = []
for idx = entry_root_count; idx < root_ids.length(); idx = idx + 1 {
body_root_ids.push(root_ids[idx])
}
let preserve_region_results = hot_lower_impl_preserve_region_result_roots(
func, use_def, body_root_ids, branch_arity,
)
let mut root_idx = 0
for child in body_root_ids {
hot_lower_impl_emit_root(
func, child, use_def, label_frames, label_stack_head, body_stack, out, extra_locals,
options,
)
let is_final_root = root_idx + 1 == body_root_ids.length()
hot_lower_impl_drop_dead_nonfinal_root_results(
func,
child,
use_def,
preserve_region_results[root_idx],
is_final_root,
body_stack,
out,
)
root_idx += 1
}
hot_lower_impl_finalize_region_stack(
func, branch_arity, body_stack, out, extra_locals,
)
hot_lower_impl_push_results(func, id, stack)
}
///|
fn hot_lower_impl_scope_parent_witness(
func : HotFunc,
id : NodeId,
stack : HotLowerValueStack,
work : Array[Int32]?,
) -> NodeId {
let marks : @hashmap.HashMap[NodeId, Int32] = @hashmap.new()
for value in stack {
if value.node_id >= 0 {
marks.set(value.node_id, 1)
}
}
let pending : Array[NodeId] = []
for slot in 0.. work[0] += 1
None => ()
}
let value = pending.pop().unwrap()
if value < 0 {
continue
}
match marks.get(value) {
Some(1) => return value
Some(_) => continue
None => ()
}
marks.set(value, 2)
for slot in 0.. Bool {
match work {
Some(work) => work[0] += 1
None => ()
}
if root < 0 {
return false
}
if root == self.target {
return true
}
match self.answers.get(root) {
Some(answer) => return answer
None => ()
}
self.nodes.push(root)
self.slots.push(0)
while !self.nodes.is_empty() {
let top = self.nodes.length() - 1
let node = self.nodes[top]
let slot = self.slots[top]
if slot == hot_child_count(func, node) {
self.answers.set(node, false)
ignore(self.nodes.pop())
ignore(self.slots.pop())
continue
}
self.slots[top] = slot + 1
let child = hot_child_get(func, node, slot)
match work {
Some(work) => work[0] += 1
None => ()
}
if child < 0 {
continue
}
if child != self.target {
match self.answers.get(child) {
Some(true) => ()
Some(false) => continue
None => {
self.nodes.push(child)
self.slots.push(0)
continue
}
}
}
// Every active ancestor reaches this same target. Publish those exact
// facts so the following inner-scope queries need only cache lookups.
for ancestor in self.nodes {
self.answers.set(ancestor, true)
}
self.nodes.clear()
self.slots.clear()
return true
}
false
}
///|
// A cached answer is useful only for a producer still on this actual stack.
// Different local.get nodes never share an answer merely by local index.
fn hot_lower_impl_scope_uses_parent(
func : HotFunc,
id : NodeId,
frames : HotLowerLabelFrames,
stack : HotLowerValueStack,
) -> Bool {
let mut target = HOT_NONE_NODE
let previous = match frames.scope_query {
Some(query) => query.target
None => HOT_NONE_NODE
}
for value in stack {
if value.node_id >= 0 {
if value.node_id == previous {
target = previous
break
}
if target < 0 {
target = value.node_id
}
}
}
if target < 0 {
return false
}
let query = match frames.scope_query {
Some(query) => {
if query.target != target {
query.target = target
query.answers.clear()
}
query
}
None => {
let query = HotLowerScopeQuery::new(target)
frames.scope_query = Some(query)
query
}
}
for slot in 0..= 0 && value.node_id != target }) {
return false
}
// A second held producer can still be used even when the cached target is
// absent. Retain the exact all-stack proof and cache its actual witness next.
let witness = hot_lower_impl_scope_parent_witness(
func,
id,
stack,
frames.scope_query_work,
)
if witness < 0 {
return false
}
query.target = witness
query.answers.clear()
true
}
///|
// An inserted ordered expression scope can refer to an already evaluated
// operand in its parent stack. A fresh region stack would evaluate that
// producer again; a typed loop entry then reads its initializer instead of the
// value carried by the backedge. An unused, parameter-free label can be omitted
// when every prefix root is void and the final root supplies the exact result.
fn hot_lower_impl_emit_ordered_stack_scope(
func : HotFunc,
id : NodeId,
use_def : HotNodeUseCounts,
label_frames : HotLowerLabelFrames,
label_stack_head : Int32,
stack : HotLowerValueStack,
out : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
) -> Bool {
let node = hot_node_get(func, id)
let result_arity = hot_node_result_arity(func, id)
if node.child_count == 0 || result_arity > 1 {
return false
}
let children = func.children[node.child_base:node.child_base +
node.child_count]
for index in 0.. 0 {
if stack.is_empty() {
abort("ordered expression scope did not emit its scalar result")
}
let result = stack[stack.length() - 1]
stack[stack.length() - 1] = { node_id: id, val_type: result.val_type, }
}
true
}
///|
fn hot_lower_impl_transparent_unused_result_block_roots(
func : HotFunc,
block_id : NodeId,
label_frames : HotLowerLabelFrames,
) -> Array[NodeId]? {
let block = hot_node_get(func, block_id)
let result_arity = hot_node_result_arity(func, block_id)
if block.op != HotOp::Block ||
result_arity == 0 ||
hot_lower_impl_control_param_types(func, block_id).length() != 0 {
return None
}
let label_id = hot_control_node_label(func, block_id)
if label_id < 0 || hot_lower_impl_label_is_used(label_frames, label_id) {
return None
}
let value_roots : Array[NodeId] = []
let mut seen_arity = 0
for slot = 0; slot < block.child_count; slot = slot + 1 {
let root_id = hot_child_get(func, block_id, slot)
let root_arity = hot_node_result_arity(func, root_id)
if root_arity > 0 {
// The original single-result path remains permissive. Multi-result block
// flattening starts with independent literal roots, the exact dropped
// multivalue family owned by remove-unused-brs.
if result_arity > 1 {
match hot_query_op(func, root_id) {
HotOp::Const | HotOp::RefNull => ()
_ => return None
}
}
value_roots.push(root_id)
seen_arity += root_arity
continue
}
if !hot_root_is_strict_no_work_block_chain(func, root_id, fn(root_label) {
hot_lower_impl_label_is_used(label_frames, root_label)
}) {
return None
}
}
if value_roots.length() == 0 || seen_arity != result_arity {
None
} else {
Some(value_roots)
}
}
///|
// A value emitted before a side effect can sit below a later operand. Move
// that existing value to the top; evaluating its producer again changes reads,
// calls, traps, and allocation identity. Scratch locals preserve the values
// above it in their original order.
fn hot_lower_impl_reuse_buried_scalar(
func : HotFunc,
id : NodeId,
stack : HotLowerValueStack,
out : Array[Instruction],
extra_locals : Array[ValType],
single_tuple_lane~ : Bool = false,
label_frames~ : HotLowerLabelFrames? = None,
) -> Bool {
// Only the sole-input LocalSet path can request one physical tuple lane.
// General value/root requests must still require a scalar producer.
if (!single_tuple_lane && hot_node_result_arity(func, id) != 1) ||
stack.length() < 2 {
return false
}
let mut position = stack.length() - 2
while position >= 0 && stack[position].node_id != id {
position -= 1
}
if position < 0 {
return false
}
if single_tuple_lane && position + 2 == stack.length() {
match label_frames {
Some(frames) => {
let lane = stack[position]
let cover = stack[position + 1]
let pairs = match frames.tuple_lane_pairs {
Some(cache) if physical_equal(cache.extra_locals, extra_locals) =>
cache.pairs
_ => {
let pairs = []
frames.tuple_lane_pairs = Some({ extra_locals, pairs, })
pairs
}
}
let pair = pairs.find_index(fn(pair) {
pair.0 == lane.val_type && pair.1 == cover.val_type
})
let (lane_local, cover_local) = match pair {
Some(index) => (pairs[index].2, pairs[index].3)
None => {
let lane_local = hot_lower_impl_alloc_scratch_local(
func,
extra_locals,
lane.val_type,
)
let cover_local = hot_lower_impl_alloc_scratch_local(
func,
extra_locals,
cover.val_type,
)
// This pair is live only within the four flat instructions below.
// No recursive emission, branch, or trap can overlap a later use.
// Bound lookup and retained metadata for uncommon type mixtures.
if pairs.length() < 8 {
pairs.push(
(lane.val_type, cover.val_type, lane_local, cover_local),
)
}
(lane_local, cover_local)
}
}
out.push(Instruction::local_set(cover_local))
out.push(Instruction::local_set(lane_local))
out.push(Instruction::local_get(cover_local))
out.push(Instruction::local_get(lane_local))
stack[position] = cover
stack[position + 1] = lane
return true
}
None => ()
}
}
let above : Array[(HotLowerStackValue, LocalIdx)] = []
while stack.length() > position + 1 {
let value = stack.pop().unwrap()
let local_ = hot_lower_impl_alloc_scratch_local(
func,
extra_locals,
value.val_type,
)
out.push(Instruction::local_set(local_))
above.push((value, local_))
}
let value = stack.pop().unwrap()
let local_ = hot_lower_impl_alloc_scratch_local(
func,
extra_locals,
value.val_type,
)
out.push(Instruction::local_set(local_))
for index = above.length() - 1; index >= 0; index = index - 1 {
let (saved, saved_local) = above[index]
out.push(Instruction::local_get(saved_local))
stack.push(saved)
}
out.push(Instruction::local_get(local_))
stack.push(value)
true
}
///|
fn hot_lower_impl_emit_root(
func : HotFunc,
id : NodeId,
use_def : HotNodeUseCounts,
label_frames : HotLowerLabelFrames,
label_stack_head : Int32,
stack : HotLowerValueStack,
out : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
allow_stack_top_reuse~ : Bool = true,
) -> Unit {
let node = hot_node_get(func, id)
let result_arity = if allow_stack_top_reuse {
hot_node_result_arity(func, id)
} else {
0
}
let was_requested = hot_lower_impl_root_was_requested(
func, label_frames, id, stack,
)
if result_arity > 0 {
if stack.length() > 0 && stack[stack.length() - 1].node_id == id {
return
}
// Pure values can be rematerialized cheaply. A saved effectful value
// must be reused even when a later operand has covered it on the stack.
if was_requested &&
stack.length() >= 2 &&
result_arity == 1 &&
!effects_is_pure(hot_lower_impl_node_effects(func, label_frames)[id]) &&
hot_lower_impl_reuse_buried_scalar(func, id, stack, out, extra_locals) {
return
}
}
if node.op == HotOp::Block &&
node.imm1 == 0 &&
hot_lower_impl_label_is_used(label_frames, node.imm0) &&
hot_lower_impl_control_param_types(func, id).length() == 1 {
hot_lower_impl_emit_typed_param_block_with_entry_prefix(
func, id, use_def, label_frames, label_stack_head, stack, out, extra_locals,
options,
)
return
}
if node.op == HotOp::Block &&
node.imm1 == 0 &&
!hot_lower_impl_label_is_used(label_frames, node.imm0) {
let param_types = hot_lower_impl_control_param_types(func, id)
if !stack.is_empty() &&
param_types.length() == 0 &&
hot_lower_impl_emit_ordered_stack_scope(
func, id, use_def, label_frames, label_stack_head, stack, out, extra_locals,
options,
) {
return
}
if options.flatten_unused_result_blocks && param_types.length() == 0 {
match
hot_lower_impl_transparent_unused_result_block_roots(
func, id, label_frames,
) {
Some(value_roots) => {
for value_root in value_roots {
hot_lower_impl_emit_root(
func, value_root, use_def, label_frames, label_stack_head, stack, out,
extra_locals, options,
)
}
let result_arity = hot_node_result_arity(func, id)
if stack.length() < result_arity {
abort("transparent unused result block did not emit all results")
}
for stack_idx = stack.length() - result_arity
stack_idx < stack.length()
stack_idx = stack_idx + 1 {
let value = stack[stack_idx]
stack[stack_idx] = { node_id: id, val_type: value.val_type, }
}
return
}
None => ()
}
}
if param_types.length() == 1 {
hot_lower_impl_emit_transparent_param_block(
func, id, use_def, label_frames, label_stack_head, stack, out, extra_locals,
options,
)
return
}
if param_types.length() > 1 && hot_node_result_arity(func, id) == 0 {
hot_lower_impl_emit_typed_param_block_with_entry_prefix(
func, id, use_def, label_frames, label_stack_head, stack, out, extra_locals,
options,
)
return
}
}
let inputs = hot_lower_impl_input_info(func, id)
if node.op == HotOp::LocalSet &&
inputs.pop_count == 1 &&
inputs.nodes.length() == 1 &&
stack.length() >= 2 &&
stack[stack.length() - 1].node_id != inputs.nodes[0] &&
hot_node_result_arity(func, id) == 0 &&
hot_node_result_arity(func, inputs.nodes[0]) > 1 {
// A carried old read can cover the next lane of an evaluated tuple.
// This consumer needs exactly one lane, with no earlier input prefix to
// protect. Move that row instead of replaying the whole producer.
ignore(
hot_lower_impl_reuse_buried_scalar(
func,
inputs.nodes[0],
stack,
out,
extra_locals,
single_tuple_lane=true,
label_frames=Some(label_frames),
),
)
}
hot_lower_impl_emit_inputs(
func,
inputs.nodes,
use_def,
label_frames,
label_stack_head,
stack,
out,
extra_locals,
options,
)
hot_lower_impl_emit_instruction(
func, id, use_def, label_frames, label_stack_head, out, extra_locals, options,
)
hot_lower_impl_update_stack(func, id, inputs.pop_count, stack)
}
///|
fn hot_lower_impl_declared_locals(
func : HotFunc,
extra_locals : Array[ValType],
) -> Array[ValType] {
let out : Array[ValType] = []
for local_ in func.locals.locals {
out.push(local_)
}
for local_ in extra_locals {
out.push(local_)
}
out
}
///|
fn hot_lower_impl_decoded_label_depth(li : LabelIdx) -> Int32 {
let @lib.LabelIdx(depth) = li
depth.reinterpret_as_int()
}
///|
fn hot_lower_impl_rebased_label_depth(
li : LabelIdx,
owner_depth : Int32,
) -> LabelIdx {
let depth = hot_lower_impl_decoded_label_depth(li)
if depth > owner_depth {
LabelIdx::new((depth - 1).reinterpret_as_uint())
} else {
li
}
}
///|
fn hot_lower_impl_catch_targets_removed_label(
catch_ : Catch,
owner_depth : Int32,
) -> Bool {
match catch_ {
Catch(_, li) | CatchRef(_, li) | CatchAll(li) | CatchAllRef(li) =>
hot_lower_impl_decoded_label_depth(li) == owner_depth
}
}
///|
fn hot_lower_impl_rebase_catch(catch_ : Catch, owner_depth : Int32) -> Catch {
match catch_ {
Catch(tag, li) =>
Catch::new(tag, hot_lower_impl_rebased_label_depth(li, owner_depth))
CatchRef(tag, li) =>
Catch::ref_(tag, hot_lower_impl_rebased_label_depth(li, owner_depth))
CatchAll(li) =>
Catch::all(hot_lower_impl_rebased_label_depth(li, owner_depth))
CatchAllRef(li) =>
Catch::all_ref(hot_lower_impl_rebased_label_depth(li, owner_depth))
}
}
///|
fn hot_lower_impl_instr_targets_removed_label(
instr : Instruction,
owner_depth : Int32,
) -> Bool {
match instr {
Br(li) | BrIf(li) | BrOnNull(li) | BrOnNonNull(li) | Rethrow(li) =>
hot_lower_impl_decoded_label_depth(li) == owner_depth
BrTable(labels, fallback) => {
if hot_lower_impl_decoded_label_depth(fallback) == owner_depth {
return true
}
for li in labels {
if hot_lower_impl_decoded_label_depth(li) == owner_depth {
return true
}
}
false
}
Block(_, Expr(body)) | Loop(_, Expr(body)) =>
hot_lower_impl_body_targets_removed_label(body, owner_depth + 1)
If(_, then_, else_) => {
if hot_lower_impl_body_targets_removed_label(then_, owner_depth + 1) {
return true
}
match else_ {
Some(else_body) =>
hot_lower_impl_body_targets_removed_label(else_body, owner_depth + 1)
None => false
}
}
Try(_, Expr(body), catches, delegate) => {
if delegate is Some(li) &&
hot_lower_impl_decoded_label_depth(li) == owner_depth {
return true
}
if hot_lower_impl_body_targets_removed_label(body, owner_depth + 1) {
return true
}
for arm in catches {
let Expr(catch_body) = match arm {
@lib.LegacyCatch(_, body) | @lib.LegacyCatchAll(body) => body
}
if hot_lower_impl_body_targets_removed_label(
catch_body,
owner_depth + 1,
) {
return true
}
}
false
}
TryTable(_, catches, Expr(body)) => {
for catch_ in catches {
if hot_lower_impl_catch_targets_removed_label(catch_, owner_depth) {
return true
}
}
hot_lower_impl_body_targets_removed_label(body, owner_depth + 1)
}
_ => false
}
}
///|
fn hot_lower_impl_body_targets_removed_label(
body : Array[Instruction],
owner_depth : Int32,
) -> Bool {
for instr in body {
if hot_lower_impl_instr_targets_removed_label(instr, owner_depth) {
return true
}
}
false
}
///|
fn hot_lower_impl_rebase_instr(
instr : Instruction,
owner_depth : Int32,
) -> Instruction {
match instr {
Rethrow(li) =>
Instruction::rethrow(hot_lower_impl_rebased_label_depth(li, owner_depth))
Br(li) =>
Instruction::br(hot_lower_impl_rebased_label_depth(li, owner_depth))
BrIf(li) =>
Instruction::br_if(hot_lower_impl_rebased_label_depth(li, owner_depth))
BrTable(labels, fallback) => {
let rebased : Array[LabelIdx] = []
for li in labels {
rebased.push(hot_lower_impl_rebased_label_depth(li, owner_depth))
}
Instruction::br_table(
rebased,
hot_lower_impl_rebased_label_depth(fallback, owner_depth),
)
}
BrOnNull(li) =>
Instruction::br_on_null(
hot_lower_impl_rebased_label_depth(li, owner_depth),
)
BrOnNonNull(li) =>
Instruction::br_on_non_null(
hot_lower_impl_rebased_label_depth(li, owner_depth),
)
Block(bt, Expr(body)) =>
Instruction::block(
bt,
Expr::new(hot_lower_impl_rebase_body(body, owner_depth + 1)),
)
Loop(bt, Expr(body)) =>
Instruction::loop_(
bt,
Expr::new(hot_lower_impl_rebase_body(body, owner_depth + 1)),
)
If(bt, then_, else_) =>
Instruction::if_(
bt,
hot_lower_impl_rebase_body(then_, owner_depth + 1),
match else_ {
Some(else_body) =>
Some(hot_lower_impl_rebase_body(else_body, owner_depth + 1))
None => None
},
)
Try(bt, Expr(body), catches, delegate) =>
Instruction::try_(
bt,
Expr::new(hot_lower_impl_rebase_body(body, owner_depth + 1)),
catches.map(fn(arm) {
match arm {
@lib.LegacyCatch(tag, Expr(catch_body)) =>
@lib.LegacyCatch::tagged(
tag,
Expr::new(
hot_lower_impl_rebase_body(catch_body, owner_depth + 1),
),
)
@lib.LegacyCatchAll(Expr(catch_body)) =>
@lib.LegacyCatch::all(
Expr::new(
hot_lower_impl_rebase_body(catch_body, owner_depth + 1),
),
)
}
}),
delegate=delegate.map(fn(li) {
hot_lower_impl_rebased_label_depth(li, owner_depth)
}),
)
TryTable(bt, catches, Expr(body)) => {
let rebased_catches : Array[Catch] = []
for catch_ in catches {
rebased_catches.push(hot_lower_impl_rebase_catch(catch_, owner_depth))
}
Instruction::try_table(
bt,
rebased_catches,
Expr::new(hot_lower_impl_rebase_body(body, owner_depth + 1)),
)
}
_ => instr
}
}
///|
fn hot_lower_impl_rebase_body(
body : Array[Instruction],
owner_depth : Int32,
) -> Array[Instruction] {
let rebased : Array[Instruction] = []
for instr in body {
rebased.push(hot_lower_impl_rebase_instr(instr, owner_depth))
}
rebased
}
///|
fn hot_lower_impl_shift_positive_label_depth(li : LabelIdx) -> LabelIdx {
let depth = hot_lower_impl_decoded_label_depth(li)
if depth > 0 {
LabelIdx::new((depth - 1).reinterpret_as_uint())
} else {
li
}
}
///|
fn hot_lower_impl_shift_positive_catch_depth(catch_ : Catch) -> Catch {
match catch_ {
Catch(tag, li) =>
Catch::new(tag, hot_lower_impl_shift_positive_label_depth(li))
CatchRef(tag, li) =>
Catch::ref_(tag, hot_lower_impl_shift_positive_label_depth(li))
CatchAll(li) => Catch::all(hot_lower_impl_shift_positive_label_depth(li))
CatchAllRef(li) =>
Catch::all_ref(hot_lower_impl_shift_positive_label_depth(li))
}
}
///|
fn hot_lower_impl_shift_positive_branch_depths_instr(
instr : Instruction,
) -> Instruction {
match instr {
Br(li) => Instruction::br(hot_lower_impl_shift_positive_label_depth(li))
BrIf(li) =>
Instruction::br_if(hot_lower_impl_shift_positive_label_depth(li))
BrTable(labels, fallback) => {
let shifted : Array[LabelIdx] = []
for li in labels {
shifted.push(hot_lower_impl_shift_positive_label_depth(li))
}
Instruction::br_table(
shifted,
hot_lower_impl_shift_positive_label_depth(fallback),
)
}
BrOnNull(li) =>
Instruction::br_on_null(hot_lower_impl_shift_positive_label_depth(li))
BrOnNonNull(li) =>
Instruction::br_on_non_null(hot_lower_impl_shift_positive_label_depth(li))
Block(bt, Expr(body)) =>
Instruction::block(
bt,
Expr::new(hot_lower_impl_shift_positive_branch_depths(body)),
)
Loop(bt, Expr(body)) =>
Instruction::loop_(
bt,
Expr::new(hot_lower_impl_shift_positive_branch_depths(body)),
)
If(bt, then_, else_) =>
Instruction::if_(
bt,
hot_lower_impl_shift_positive_branch_depths(then_),
match else_ {
Some(else_body) =>
Some(hot_lower_impl_shift_positive_branch_depths(else_body))
None => None
},
)
TryTable(bt, catches, Expr(body)) => {
let shifted_catches : Array[Catch] = []
for catch_ in catches {
shifted_catches.push(hot_lower_impl_shift_positive_catch_depth(catch_))
}
Instruction::try_table(
bt,
shifted_catches,
Expr::new(hot_lower_impl_shift_positive_branch_depths(body)),
)
}
_ => instr
}
}
///|
fn hot_lower_impl_shift_positive_branch_depths(
body : Array[Instruction],
) -> Array[Instruction] {
let shifted : Array[Instruction] = []
for instr in body {
shifted.push(hot_lower_impl_shift_positive_branch_depths_instr(instr))
}
shifted
}
///|
fn hot_lower_impl_expand_branch_depth_above(
li : LabelIdx,
min_depth : Int32,
) -> LabelIdx {
let depth = hot_lower_impl_decoded_label_depth(li)
if depth > min_depth {
LabelIdx::new((depth + 1).reinterpret_as_uint())
} else {
li
}
}
///|
fn hot_lower_impl_expand_catch_depth_above(
catch_ : Catch,
min_depth : Int32,
) -> Catch {
match catch_ {
Catch(tag, li) =>
Catch::new(tag, hot_lower_impl_expand_branch_depth_above(li, min_depth))
CatchRef(tag, li) =>
Catch::ref_(tag, hot_lower_impl_expand_branch_depth_above(li, min_depth))
CatchAll(li) =>
Catch::all(hot_lower_impl_expand_branch_depth_above(li, min_depth))
CatchAllRef(li) =>
Catch::all_ref(hot_lower_impl_expand_branch_depth_above(li, min_depth))
}
}
///|
fn hot_lower_impl_expand_branch_depths_above_instr(
instr : Instruction,
min_depth : Int32,
) -> Instruction {
match instr {
Rethrow(li) =>
Instruction::rethrow(
hot_lower_impl_expand_branch_depth_above(li, min_depth),
)
Br(li) =>
Instruction::br(hot_lower_impl_expand_branch_depth_above(li, min_depth))
BrIf(li) =>
Instruction::br_if(
hot_lower_impl_expand_branch_depth_above(li, min_depth),
)
BrTable(labels, fallback) => {
let shifted : Array[LabelIdx] = []
for li in labels {
shifted.push(hot_lower_impl_expand_branch_depth_above(li, min_depth))
}
Instruction::br_table(
shifted,
hot_lower_impl_expand_branch_depth_above(fallback, min_depth),
)
}
BrOnNull(li) =>
Instruction::br_on_null(
hot_lower_impl_expand_branch_depth_above(li, min_depth),
)
BrOnNonNull(li) =>
Instruction::br_on_non_null(
hot_lower_impl_expand_branch_depth_above(li, min_depth),
)
BrOnCast(
li,
@lib.CastOp(source_null, target_null, source_exact, target_exact),
source,
target
) =>
Instruction::br_on_cast(
hot_lower_impl_expand_branch_depth_above(li, min_depth),
source_null,
source,
target_null,
target,
source_exact~,
target_exact~,
)
BrOnCastFail(
li,
@lib.CastOp(source_null, target_null, source_exact, target_exact),
source,
target
) =>
Instruction::br_on_cast_fail(
hot_lower_impl_expand_branch_depth_above(li, min_depth),
source_null,
source,
target_null,
target,
source_exact~,
target_exact~,
)
BrOnCastDescEq(li, source, target) =>
Instruction::br_on_cast_desc_eq(
hot_lower_impl_expand_branch_depth_above(li, min_depth),
source,
target,
)
BrOnCastDescEqFail(li, source, target) =>
Instruction::br_on_cast_desc_eq_fail(
hot_lower_impl_expand_branch_depth_above(li, min_depth),
source,
target,
)
Block(bt, Expr(body)) =>
Instruction::block(
bt,
Expr::new(
hot_lower_impl_expand_branch_depths_above(body, min_depth + 1),
),
)
Loop(bt, Expr(body)) =>
Instruction::loop_(
bt,
Expr::new(
hot_lower_impl_expand_branch_depths_above(body, min_depth + 1),
),
)
If(bt, then_, else_) =>
Instruction::if_(
bt,
hot_lower_impl_expand_branch_depths_above(then_, min_depth + 1),
match else_ {
Some(else_body) =>
Some(
hot_lower_impl_expand_branch_depths_above(
else_body,
min_depth + 1,
),
)
None => None
},
)
Try(bt, Expr(body), catches, delegate) =>
Instruction::try_(
bt,
Expr::new(
hot_lower_impl_expand_branch_depths_above(body, min_depth + 1),
),
catches.map(fn(arm) {
match arm {
@lib.LegacyCatch(tag, Expr(catch_body)) =>
@lib.LegacyCatch::tagged(
tag,
Expr::new(
hot_lower_impl_expand_branch_depths_above(
catch_body,
min_depth + 1,
),
),
)
@lib.LegacyCatchAll(Expr(catch_body)) =>
@lib.LegacyCatch::all(
Expr::new(
hot_lower_impl_expand_branch_depths_above(
catch_body,
min_depth + 1,
),
),
)
}
}),
delegate=delegate.map(fn(li) {
hot_lower_impl_expand_branch_depth_above(li, min_depth)
}),
)
TryTable(bt, catches, Expr(body)) => {
let shifted_catches : Array[Catch] = []
for catch_ in catches {
shifted_catches.push(
hot_lower_impl_expand_catch_depth_above(catch_, min_depth),
)
}
Instruction::try_table(
bt,
shifted_catches,
Expr::new(
hot_lower_impl_expand_branch_depths_above(body, min_depth + 1),
),
)
}
_ => instr
}
}
///|
fn hot_lower_impl_expand_branch_depths_above(
body : Array[Instruction],
min_depth : Int32,
) -> Array[Instruction] {
let shifted : Array[Instruction] = []
for instr in body {
shifted.push(
hot_lower_impl_expand_branch_depths_above_instr(instr, min_depth),
)
}
shifted
}
///|
fn hot_lower_impl_has_control_transfer(instr : Instruction) -> Bool {
match instr {
Block(_, _)
| Loop(_, _)
| If(_, _, _)
| Try(_, _, _, _)
| Rethrow(_)
| TryTable(_, _, _)
| Br(_)
| BrIf(_)
| BrTable(_, _)
| BrOnNull(_)
| BrOnNonNull(_)
| Return
| Throw(_)
| ThrowRef
| Unreachable => true
_ => false
}
}
///|
fn hot_lower_impl_is_simple_parent_exit_body(
body : Array[Instruction],
parent_depth : Int32,
) -> Bool {
if body.length() == 0 {
return false
}
for i = 0; i < body.length() - 1; i = i + 1 {
if hot_lower_impl_has_control_transfer(body[i]) {
return false
}
}
match body[body.length() - 1] {
Br(li) => hot_lower_impl_decoded_label_depth(li) == parent_depth
_ => false
}
}
///|
fn hot_lower_impl_is_parent_exit_payload_body(
bt : BlockType,
body : Array[Instruction],
parent_depth : Int32,
) -> Bool {
if body.length() != 2 || hot_lower_impl_body_targets_removed_label(body, 0) {
return false
}
match body[0] {
Block(inner_bt, _) if inner_bt == bt => ()
_ => return false
}
match body[1] {
Br(li) => hot_lower_impl_decoded_label_depth(li) == parent_depth
_ => false
}
}
///|
fn hot_lower_impl_is_same_typed_single_if_wrapper(
bt : BlockType,
body : Array[Instruction],
) -> Bool {
if body.length() == 0 || hot_lower_impl_body_targets_removed_label(body, 0) {
return false
}
for i = 0; i < body.length() - 1; i = i + 1 {
if hot_lower_impl_has_control_transfer(body[i]) {
return false
}
}
match body[body.length() - 1] {
If(inner_bt, _, _) => bt == inner_bt
_ => false
}
}
///|
fn hot_lower_impl_has_nonfallthrough_tail(body : Array[Instruction]) -> Bool {
if body.length() == 0 {
return false
}
match body[body.length() - 1] {
Br(_)
| BrIf(_)
| BrTable(_, _)
| BrOnNull(_)
| BrOnNonNull(_)
| Return
| Throw(_)
| ThrowRef
| Unreachable => true
_ => false
}
}
///|
fn hot_lower_impl_instr_is_nonfallthrough(instr : Instruction) -> Bool {
match instr {
Return | Throw(_) | ThrowRef | Unreachable => true
Block(VoidBlockType, _) => false
Block(_, Expr(body)) => hot_lower_impl_body_is_nonfallthrough(body)
Loop(VoidBlockType, _) => false
Loop(_, Expr(body)) => hot_lower_impl_body_is_nonfallthrough(body)
If(VoidBlockType, _, _) => false
If(_, then_, Some(else_)) =>
hot_lower_impl_body_is_nonfallthrough(then_) &&
hot_lower_impl_body_is_nonfallthrough(else_)
_ => false
}
}
///|
fn hot_lower_impl_body_is_nonfallthrough(body : Array[Instruction]) -> Bool {
if body.length() == 0 {
return false
}
hot_lower_impl_instr_is_nonfallthrough(body[body.length() - 1])
}
///|
fn hot_lower_impl_body_has_terminal_exit(body : Array[Instruction]) -> Bool {
if body.length() == 0 {
return false
}
match body[body.length() - 1] {
Br(_) | BrTable(_, _) | Return | Throw(_) | ThrowRef | Unreachable => true
Block(_, Expr(inner)) | Loop(_, Expr(inner)) =>
hot_lower_impl_body_has_terminal_exit(inner)
If(_, then_, Some(else_)) =>
hot_lower_impl_body_has_terminal_exit(then_) &&
hot_lower_impl_body_has_terminal_exit(else_)
_ => false
}
}
///|
fn hot_lower_impl_body_terminal_branch_depth(
body : Array[Instruction],
) -> Int32? {
if body.length() == 0 {
return None
}
match body[body.length() - 1] {
Br(li) => Some(hot_lower_impl_decoded_label_depth(li))
_ => None
}
}
///|
fn hot_lower_impl_needs_explicit_unreachable_after_peeled_prefix(
body : Array[Instruction],
) -> Bool {
if body.length() == 0 {
return false
}
match body[body.length() - 1] {
If(VoidBlockType, then_, Some(else_)) => {
if !hot_lower_impl_body_has_terminal_exit(then_) ||
!hot_lower_impl_body_has_terminal_exit(else_) {
return false
}
match
(
hot_lower_impl_body_terminal_branch_depth(then_),
hot_lower_impl_body_terminal_branch_depth(else_),
) {
(Some(then_depth), Some(else_depth)) => then_depth != else_depth
(Some(_), None) => true
(None, Some(_)) => true
(None, None) => false
}
}
_ => false
}
}
///|
fn hot_lower_impl_append_unreachable_if_needed(
body : Array[Instruction],
) -> Array[Instruction] {
if !hot_lower_impl_needs_explicit_unreachable_after_peeled_prefix(body) {
return body
}
match body[body.length() - 1] {
Unreachable => body
_ => {
let rewritten : Array[Instruction] = []
for instr in body {
rewritten.push(instr)
}
rewritten.push(Instruction::unreachable_())
rewritten
}
}
}
///|
fn hot_lower_impl_has_mixed_terminal_if_exit_depths(
body : Array[Instruction],
) -> Bool {
match body {
[.., Unreachable] => {
let peeled : Array[Instruction] = []
for i = 0; i < body.length() - 1; i = i + 1 {
peeled.push(body[i])
}
hot_lower_impl_needs_explicit_unreachable_after_peeled_prefix(peeled)
}
_ => hot_lower_impl_needs_explicit_unreachable_after_peeled_prefix(body)
}
}
///|
fn hot_lower_impl_trim_redundant_unreachable_tail(
body : Array[Instruction],
) -> Array[Instruction] {
if body.length() < 2 {
return body
}
for i = 0; i + 1 < body.length(); i = i + 1 {
match body[i] {
Unreachable => {
let mut only_dead_debris = true
for j = i + 1; j < body.length(); j = j + 1 {
match body[j] {
Nop | Drop | Block(_, Expr([])) => ()
_ => only_dead_debris = false
}
}
if only_dead_debris {
let trimmed : Array[Instruction] = []
for j = 0; j <= i; j = j + 1 {
trimmed.push(body[j])
}
return trimmed
}
}
_ => ()
}
}
match body[body.length() - 1] {
Unreachable => {
let prev = body[body.length() - 2]
if !hot_lower_impl_instr_is_nonfallthrough(prev) {
return body
}
match prev {
Loop(_, _) => return body
If(_, _, _) | Block(_, _) => return body
Br(_) =>
if body.length() >= 3 {
match body[body.length() - 3] {
If(VoidBlockType, _, _) => ()
If(_, _, _) => return body
_ => ()
}
}
Return | Throw(_) | ThrowRef | Unreachable => ()
_ => return body
}
let trimmed : Array[Instruction] = []
for i = 0; i < body.length() - 1; i = i + 1 {
trimmed.push(body[i])
}
trimmed
}
_ => body
}
}
///|
fn hot_lower_impl_append_explicit_unreachable(
body : Array[Instruction],
) -> Array[Instruction] {
match body {
[.., Unreachable] => body
_ => {
let trimmed : Array[Instruction] = []
for instr in body {
trimmed.push(instr)
}
trimmed.push(Instruction::unreachable_())
trimmed
}
}
}
///|
fn hot_lower_impl_try_sink_terminal_exit_into_arm(
body : Array[Instruction],
exit_ : Instruction,
) -> Array[Instruction]? {
if hot_lower_impl_has_nonfallthrough_tail(body) {
return Some(body)
}
if body.length() == 0 {
return None
}
for i = 0; i < body.length() - 1; i = i + 1 {
if hot_lower_impl_has_control_transfer(body[i]) {
return None
}
}
if hot_lower_impl_has_control_transfer(body[body.length() - 1]) {
return None
}
let rewritten : Array[Instruction] = []
for instr in body {
rewritten.push(instr)
}
rewritten.push(exit_)
Some(rewritten)
}
///|
fn hot_lower_impl_try_voidify_parent_exit_if_forwarder(
bt : BlockType,
body : Array[Instruction],
) -> Array[Instruction]? {
match bt {
VoidBlockType => ()
_ => return None
}
if body.length() < 2 {
return None
}
for i = 0; i < body.length() - 2; i = i + 1 {
if hot_lower_impl_has_control_transfer(body[i]) {
return None
}
}
match body[body.length() - 1] {
Br(li) if hot_lower_impl_decoded_label_depth(li) == 1 => return None
Return => ()
_ => return None
}
let rewritten : Array[Instruction] = []
for i = 0; i < body.length() - 2; i = i + 1 {
rewritten.push(body[i])
}
let append_unreachable = false
match body[body.length() - 2] {
If(inner_bt, then_, Some(else_)) if inner_bt != @lib.BlockType::void_() => {
if body.length() > 2 {
match
(
hot_lower_impl_body_terminal_branch_depth(then_),
hot_lower_impl_body_terminal_branch_depth(else_),
) {
(Some(then_depth), Some(else_depth)) if then_depth != else_depth =>
return None
(Some(_), None) => return None
(None, Some(_)) => return None
_ => ()
}
}
match
hot_lower_impl_try_sink_terminal_exit_into_arm(
then_,
body[body.length() - 1],
) {
Some(rewritten_then) =>
match
hot_lower_impl_try_sink_terminal_exit_into_arm(
else_,
body[body.length() - 1],
) {
Some(rewritten_else) => {
rewritten.push(
Instruction::if_(
@lib.BlockType::void_(),
rewritten_then,
Some(rewritten_else),
),
)
if append_unreachable {
rewritten.push(Instruction::unreachable_())
}
Some(rewritten)
}
None => None
}
None => None
}
}
_ => None
}
}
///|
fn hot_lower_impl_try_voidify_terminal_return_if_forwarder(
_body : Array[Instruction],
) -> Array[Instruction]? {
None
}
///|
fn hot_lower_impl_try_voidify_terminal_drop_if_forwarder(
body : Array[Instruction],
) -> Array[Instruction]? {
if body.length() < 2 {
return None
}
match body[body.length() - 1] {
Drop => ()
_ => return None
}
let rewritten : Array[Instruction] = []
for i = 0; i < body.length() - 2; i = i + 1 {
rewritten.push(body[i])
}
match body[body.length() - 2] {
If(inner_bt, then_, Some(else_)) if inner_bt != @lib.BlockType::void_() &&
hot_lower_impl_body_has_terminal_exit(then_) &&
hot_lower_impl_body_has_terminal_exit(else_) => {
rewritten.push(
Instruction::if_(@lib.BlockType::void_(), then_, Some(else_)),
)
Some(hot_lower_impl_append_explicit_unreachable(rewritten))
}
_ => None
}
}
///|
fn hot_lower_impl_try_pack_split_parent_exit_payload(
bt : BlockType,
body : Array[Instruction],
) -> Array[Instruction]? {
match bt {
VoidBlockType => return None
_ => ()
}
match body {
[Block(VoidBlockType, Expr(inner_body))] => {
if inner_body.length() != 3 {
return None
}
let prefix = inner_body[0]
let payload = inner_body[1]
let exit_ = inner_body[2]
if hot_lower_impl_has_control_transfer(payload) {
return None
}
match exit_ {
Br(li) if hot_lower_impl_decoded_label_depth(li) == 1 => ()
_ => return None
}
match prefix {
Block(VoidBlockType, Expr(prefix_body)) => {
if prefix_body.length() <= 2 {
return None
}
match prefix_body[prefix_body.length() - 1] {
Unreachable => ()
_ => return None
}
if hot_lower_impl_body_targets_removed_label(prefix_body, 1) {
return None
}
}
_ => return None
}
let packed = Instruction::block(bt, Expr::new([prefix, payload]))
if inner_body.length() == 3 {
let prefix_kind = match prefix {
Block(_, _) => "Block"
_ => "Other"
}
let payload_kind = match payload {
LocalGet(_) => "LocalGet"
Call(_) => "Call"
_ => "Other"
}
ignore(prefix_kind)
ignore(payload_kind)
}
Some([
Instruction::block(@lib.BlockType::void_(), Expr::new([packed, exit_])),
])
}
_ => None
}
}
///|
fn hot_lower_impl_parent_exit_payload_has_mixed_child_exits(
bt : BlockType,
body : Array[Instruction],
) -> Bool {
match body {
[Block(inner_bt, Expr(inner_payload)), Br(li)] if inner_bt == bt &&
hot_lower_impl_decoded_label_depth(li) == 1 =>
match inner_payload {
[Block(VoidBlockType, Expr(child_body)), _] =>
hot_lower_impl_has_mixed_terminal_if_exit_depths(child_body)
_ => false
}
_ => false
}
}
///|
fn hot_lower_impl_split_parent_exit_payload_has_mixed_child_exits(
body : Array[Instruction],
) -> Bool {
match body {
[Block(VoidBlockType, Expr(inner_body))] =>
match inner_body {
[Block(VoidBlockType, Expr(child_body)), _, Br(li)] =>
hot_lower_impl_decoded_label_depth(li) == 1 &&
hot_lower_impl_has_mixed_terminal_if_exit_depths(child_body)
_ => false
}
_ => false
}
}
///|
fn hot_lower_impl_try_pack_split_if_payload_forwarder(
bt : BlockType,
body : Array[Instruction],
) -> Array[Instruction]? {
match bt {
VoidBlockType => return None
_ => ()
}
match body {
[Block(VoidBlockType, Expr(inner_body)), LocalGet(carried_local)] => {
if inner_body.length() < 2 {
return None
}
let inner_if = inner_body[inner_body.length() - 2]
match inner_body[inner_body.length() - 1] {
Unreachable => ()
_ => return None
}
let prefix : Array[Instruction] = []
for i = 0; i < inner_body.length() - 2; i = i + 1 {
if hot_lower_impl_has_control_transfer(inner_body[i]) {
return None
}
prefix.push(inner_body[i])
}
let extract_dropped_value_arm = fn(
arm : Array[Instruction],
) -> Array[Instruction]? {
if arm.length() < 2 {
return None
}
match arm[arm.length() - 1] {
Br(li) if ({
let depth = hot_lower_impl_decoded_label_depth(li)
depth == 0 || depth == 1
}) => ()
_ => return None
}
let value_end = match arm[arm.length() - 2] {
Drop => arm.length() - 2
_ => if arm.length() == 2 { 1 } else { return None }
}
let value_instrs : Array[Instruction] = []
for i = 0; i < value_end; i = i + 1 {
if hot_lower_impl_has_control_transfer(arm[i]) {
return None
}
value_instrs.push(arm[i])
}
if value_instrs.length() == 0 {
return None
}
Some(value_instrs)
}
let arm_is_forwarding = fn(arm : Array[Instruction]) -> Bool {
if arm.length() == 0 {
return false
}
match arm[arm.length() - 1] {
Br(li) if ({
let depth = hot_lower_impl_decoded_label_depth(li)
depth == 0 || depth == 1
}) => ()
_ => return false
}
for i = 0; i < arm.length() - 1; i = i + 1 {
if hot_lower_impl_has_control_transfer(arm[i]) {
return false
}
}
true
}
match inner_if {
If(VoidBlockType, then_, Some(else_)) => {
match
(
hot_lower_impl_body_terminal_branch_depth(then_),
hot_lower_impl_body_terminal_branch_depth(else_),
) {
(Some(then_depth), Some(else_depth)) if then_depth != else_depth =>
return None
_ => ()
}
let rebuilt = match extract_dropped_value_arm(then_) {
Some(then_value) if arm_is_forwarding(else_) =>
Some(Instruction::if_(bt, then_value, Some(else_)))
_ =>
match extract_dropped_value_arm(else_) {
Some(else_value) if arm_is_forwarding(then_) =>
Some(Instruction::if_(bt, then_, Some(else_value)))
_ => None
}
}
match rebuilt {
Some(rebuilt_if) =>
Some([
Instruction::block(
@lib.BlockType::void_(),
Expr::new(
prefix +
[
rebuilt_if,
Instruction::br(LabelIdx::new(1U)),
Instruction::unreachable_(),
],
),
),
Instruction::local_get(carried_local),
])
None => None
}
}
_ => None
}
}
_ => None
}
}
///|
fn hot_lower_impl_try_unpack_parent_exit_payload_wrapper(
bt : BlockType,
body : Array[Instruction],
) -> Array[Instruction]? {
match bt {
VoidBlockType => ()
_ => return None
}
match body {
[Block(inner_bt, Expr(inner_body)), Br(li)] => {
match inner_bt {
VoidBlockType => return None
_ => ()
}
if hot_lower_impl_decoded_label_depth(li) != 1 {
return None
}
match inner_body {
[child, payload] => {
if hot_lower_impl_has_control_transfer(payload) {
return None
}
match child {
Block(VoidBlockType, Expr(child_body)) => {
if child_body.length() == 0 {
return None
}
if hot_lower_impl_has_mixed_terminal_if_exit_depths(child_body) {
return None
}
match child_body[child_body.length() - 1] {
Unreachable =>
Some([
Instruction::block(
@lib.BlockType::void_(),
Expr::new(hot_lower_impl_rebase_body(child_body, 1)),
),
payload,
Instruction::br(li),
])
_ => None
}
}
_ => None
}
}
_ => None
}
}
_ => None
}
}
///|
fn hot_lower_impl_try_fix_split_payload_wrapper_with_tail(
bt : BlockType,
body : Array[Instruction],
) -> Array[Instruction]? {
match bt {
VoidBlockType => return None
_ => ()
}
if body.length() < 2 {
return None
}
let child_body_needs_depth_fix = fn(body : Array[Instruction]) -> Bool {
if body.length() == 0 {
return false
}
let tail_idx = body.length() - 1
let if_idx = match body[tail_idx] {
Unreachable if tail_idx > 0 => tail_idx - 1
_ => tail_idx
}
match body[if_idx] {
If(VoidBlockType, then_, Some(else_)) =>
match
(
hot_lower_impl_body_terminal_branch_depth(then_),
hot_lower_impl_body_terminal_branch_depth(else_),
) {
(Some(1), Some(2)) | (Some(2), Some(1)) => true
_ => false
}
_ => false
}
}
match body[0] {
Block(VoidBlockType, Expr(wrapper_body)) =>
match wrapper_body {
[Block(inner_bt, Expr(inner_body)), Br(li)] if inner_bt == bt &&
hot_lower_impl_decoded_label_depth(li) == 1 =>
match inner_body {
[Block(VoidBlockType, Expr(child_body)), payload] if !hot_lower_impl_has_control_transfer(
payload,
) &&
child_body_needs_depth_fix(child_body) => {
let rewritten : Array[Instruction] = [
Instruction::block(
@lib.BlockType::void_(),
Expr::new([
Instruction::block(
inner_bt,
Expr::new([
Instruction::block(
@lib.BlockType::void_(),
Expr::new(
hot_lower_impl_shift_positive_branch_depths(
child_body,
),
),
),
payload,
]),
),
Instruction::br(li),
]),
),
]
for i = 1; i < body.length(); i = i + 1 {
rewritten.push(body[i])
}
Some(rewritten)
}
_ => None
}
_ => None
}
_ => None
}
}
///|
fn hot_lower_impl_try_fix_direct_mixed_depth_payload_wrapper_with_tail(
bt : BlockType,
body : Array[Instruction],
) -> Array[Instruction]? {
match bt {
VoidBlockType => return None
_ => ()
}
match body {
[Block(VoidBlockType, Expr(inner_body)), payload] if !hot_lower_impl_has_control_transfer(
payload,
) =>
if inner_body.length() == 0 {
None
} else {
let tail_idx = inner_body.length() - 1
let if_idx = match inner_body[tail_idx] {
Unreachable if tail_idx > 0 => tail_idx - 1
_ => tail_idx
}
match inner_body[if_idx] {
If(VoidBlockType, then_, Some(else_)) =>
match
(
hot_lower_impl_body_terminal_branch_depth(then_),
hot_lower_impl_body_terminal_branch_depth(else_),
) {
(Some(1), Some(2)) | (Some(2), Some(1)) =>
Some([
Instruction::block(
@lib.BlockType::void_(),
Expr::new(
hot_lower_impl_expand_branch_depths_above(inner_body, 1),
),
),
payload,
])
_ => None
}
_ => None
}
}
_ => None
}
}
///|
fn hot_lower_impl_restore_single_if_arm_nop(
original : Array[Instruction],
canonical : Array[Instruction],
options : HotLowerOptions,
) -> Array[Instruction] {
if !options.preserve_if_arm_single_nops {
return canonical
}
match original {
[Nop] if canonical.length() == 0 => [Instruction::nop()]
_ => canonical
}
}
///|
fn hot_lower_impl_canonicalize_instr(
instr : Instruction,
options : HotLowerOptions,
) -> Instruction {
match instr {
Block(bt, Expr(body)) =>
Instruction::block(
bt,
Expr::new(hot_lower_impl_canonicalize_block_body(bt, body, options)),
)
Loop(bt, Expr(body)) =>
Instruction::loop_(
bt,
Expr::new(hot_lower_impl_canonicalize_body(body, options)),
)
If(bt, then_, else_) => {
let canonical_then = hot_lower_impl_restore_single_if_arm_nop(
then_,
hot_lower_impl_canonicalize_body(then_, options),
options,
)
let canonical_else = match else_ {
Some(else_body) =>
Some(
hot_lower_impl_restore_single_if_arm_nop(
else_body,
hot_lower_impl_canonicalize_body(else_body, options),
options,
),
)
None => None
}
Instruction::if_(bt, canonical_then, canonical_else)
}
TryTable(bt, catches, Expr(body)) =>
Instruction::try_table(
bt,
catches,
Expr::new(hot_lower_impl_canonicalize_body(body, options)),
)
_ => instr
}
}
///|
fn hot_lower_impl_canonicalize_block_body(
bt : BlockType,
body : Array[Instruction],
options : HotLowerOptions,
) -> Array[Instruction] {
let canonical = match hot_lower_impl_canonicalize_body(body, options) {
[Block(VoidBlockType, Expr(prefix_body)), tail] =>
[
Instruction::block(
@lib.BlockType::void_(),
Expr::new(hot_lower_impl_append_unreachable_if_needed(prefix_body)),
),
tail,
]
canonical => canonical
}
match hot_lower_impl_try_unpack_parent_exit_payload_wrapper(bt, canonical) {
Some(rewritten) => return rewritten
None => ()
}
match hot_lower_impl_try_voidify_parent_exit_if_forwarder(bt, canonical) {
Some(rewritten) => return rewritten
None => ()
}
match hot_lower_impl_try_pack_split_if_payload_forwarder(bt, canonical) {
Some(rewritten) => return rewritten
None => ()
}
match
hot_lower_impl_try_fix_direct_mixed_depth_payload_wrapper_with_tail(
bt, canonical,
) {
Some(rewritten) => return rewritten
None => ()
}
match hot_lower_impl_try_fix_split_payload_wrapper_with_tail(bt, canonical) {
Some(rewritten) => return rewritten
None => ()
}
let peeled = match canonical {
[Block(inner_bt, Expr(inner_body))] if bt == inner_bt &&
hot_lower_impl_is_simple_parent_exit_body(inner_body, 1) &&
!hot_lower_impl_body_targets_removed_label(inner_body, 0) =>
hot_lower_impl_rebase_body(inner_body, 0)
[Block(VoidBlockType, Expr(inner_body))] if hot_lower_impl_is_parent_exit_payload_body(
bt, inner_body, 1,
) &&
!hot_lower_impl_parent_exit_payload_has_mixed_child_exits(bt, inner_body) =>
hot_lower_impl_rebase_body(inner_body, 0)
[Block(inner_bt, Expr(inner_body))] if bt == inner_bt &&
hot_lower_impl_is_same_typed_single_if_wrapper(bt, inner_body) =>
hot_lower_impl_rebase_body(inner_body, 0)
_ => canonical
}
let spliced : Array[Instruction] = []
let mut loop_exit_idx = -1
for i = 0; i < peeled.length(); i = i + 1 {
let instr = peeled[i]
match instr {
Block(VoidBlockType, Expr(inner_body)) if hot_lower_impl_is_simple_parent_exit_body(
inner_body, 1,
) &&
!hot_lower_impl_body_targets_removed_label(inner_body, 0) =>
for inner in hot_lower_impl_rebase_body(inner_body, 0) {
spliced.push(inner)
}
Block(VoidBlockType, Expr(inner_body)) if hot_lower_impl_is_parent_exit_payload_body(
bt, inner_body, 1,
) &&
!hot_lower_impl_parent_exit_payload_has_mixed_child_exits(
bt, inner_body,
) =>
for inner in hot_lower_impl_rebase_body(inner_body, 0) {
spliced.push(inner)
}
Block(VoidBlockType, Expr(inner_body)) => {
let packed = if i == peeled.length() - 1 ||
!hot_lower_impl_split_parent_exit_payload_has_mixed_child_exits([
instr,
]) {
hot_lower_impl_try_pack_split_parent_exit_payload(bt, [instr])
} else {
None
}
match packed {
Some([rewritten]) => spliced.push(rewritten)
_ =>
spliced.push(
Instruction::block(@lib.BlockType::void_(), Expr::new(inner_body)),
)
}
}
Loop(_, Expr(loop_body)) if hot_lower_impl_is_simple_parent_exit_body(
loop_body, 1,
) => {
spliced.push(instr)
loop_exit_idx = spliced.length() - 1
}
_ => spliced.push(instr)
}
}
let trimmed : Array[Instruction] = []
for instr in spliced {
trimmed.push(instr)
match instr {
Br(li) if hot_lower_impl_decoded_label_depth(li) == 0 => return trimmed
_ => ()
}
}
if loop_exit_idx >= 0 {
let prefix : Array[Instruction] = []
for i = 0; i <= loop_exit_idx; i = i + 1 {
prefix.push(spliced[i])
}
return match bt {
VoidBlockType => prefix
_ => {
prefix.push(Instruction::unreachable_())
prefix
}
}
}
trimmed
}
///|
fn hot_lower_impl_can_inline_value_wrapper(
bt : BlockType,
body : Array[Instruction],
) -> Bool {
match bt {
VoidBlockType => false
_ =>
match body {
[LocalGet(_)] => !hot_lower_impl_body_targets_removed_label(body, 0)
_ => false
}
}
}
///|
fn hot_lower_impl_instr_is_pure_dead_drop_leaf(instr : Instruction) -> Bool {
match instr {
I32Const(_)
| I64Const(_)
| F32Const(_)
| F64Const(_)
| V128Const(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)
| RefNull(_)
| RefFunc(_)
| StringConst(_)
| LocalGet(_)
| GlobalGet(_) => true
_ => false
}
}
///|
fn hot_lower_impl_instr_is_pure_dead_drop_unary(instr : Instruction) -> Bool {
match instr {
F32Ceil | I32x4TruncSatF64x2SZero | I32x4TruncSatF64x2UZero => true
_ => false
}
}
///|
fn hot_lower_impl_remove_pure_drop_between_unreachables(
body : Array[Instruction],
) -> Array[Instruction] {
match body {
[Unreachable, ..] => ()
_ => return body
}
let rewritten : Array[Instruction] = []
let mut changed = false
let mut idx = 0
while idx < body.length() {
if idx > 0 && idx + 2 < body.length() {
match body[idx + 2] {
Drop =>
if hot_lower_impl_instr_is_pure_dead_drop_leaf(body[idx]) &&
hot_lower_impl_instr_is_pure_dead_drop_unary(body[idx + 1]) {
let mut has_later_unreachable = false
for probe = idx + 3; probe < body.length(); probe = probe + 1 {
match body[probe] {
Unreachable => has_later_unreachable = true
_ => ()
}
}
if has_later_unreachable {
changed = true
idx = idx + 3
continue
}
}
_ => ()
}
}
rewritten.push(body[idx])
idx = idx + 1
}
if changed {
rewritten
} else {
body
}
}
///|
fn hot_lower_impl_instr_is_const_like_dead_value_before_unreachable(
instr : Instruction,
) -> Bool {
match instr {
I32Const(_)
| I64Const(_)
| F32Const(_)
| F64Const(_)
| V128Const(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)
| RefNull(_)
| StringConst(_) => true
_ => false
}
}
///|
fn hot_lower_impl_hoist_dropped_values_before_leading_unreachable(
body : Array[Instruction],
) -> Array[Instruction] {
match body {
[Unreachable, ..] => ()
_ => return body
}
let moved : Array[Instruction] = []
let mut idx = 1
while idx + 1 < body.length() {
match body[idx + 1] {
Drop if !hot_lower_impl_has_control_transfer(body[idx]) => {
moved.push(body[idx])
moved.push(body[idx + 1])
idx = idx + 2
}
_ => break
}
}
if moved.length() == 0 {
return body
}
let rewritten : Array[Instruction] = []
for instr in moved {
rewritten.push(instr)
}
rewritten.push(Instruction::unreachable_())
for tail_idx = idx; tail_idx < body.length(); tail_idx = tail_idx + 1 {
rewritten.push(body[tail_idx])
}
rewritten
}
///|
fn hot_lower_impl_materialize_dead_const_prefix_before_unreachable(
body : Array[Instruction],
) -> Array[Instruction] {
if body.length() < 2 {
return body
}
match body[body.length() - 1] {
Unreachable => ()
_ => return body
}
let mut suffix_start = body.length() - 1
while suffix_start > 0 &&
hot_lower_impl_instr_is_const_like_dead_value_before_unreachable(
body[suffix_start - 1],
) {
suffix_start -= 1
}
if suffix_start == body.length() - 1 {
return body
}
let rewritten : Array[Instruction] = []
for i = 0; i < body.length() - 1; i = i + 1 {
rewritten.push(body[i])
if i >= suffix_start {
rewritten.push(Instruction::drop())
}
}
rewritten.push(Instruction::unreachable_())
rewritten
}
///|
fn hot_lower_impl_strip_redundant_nops_and_dead_local_tee_drops(
body : Array[Instruction],
options : HotLowerOptions,
) -> Array[Instruction] {
let stripped : Array[Instruction] = []
let mut cursor = 0
let mut i = 0
while i < body.length() {
match body[i] {
Unreachable => {
let after_unreachable = i + 1
i = after_unreachable
while i < body.length() {
match body[i] {
Nop | Drop | Block(_, Expr([])) => i = i + 1
_ => break
}
}
if i > after_unreachable {
for prefix = cursor; prefix < after_unreachable; prefix = prefix + 1 {
stripped.push(body[prefix])
}
cursor = i
}
continue
}
Nop => {
if !((options.preserve_nontrailing_nops && i + 1 < body.length()) ||
(options.preserve_trailing_nops && i + 1 == body.length())) {
for prefix = cursor; prefix < i; prefix = prefix + 1 {
stripped.push(body[prefix])
}
cursor = i + 1
}
i = i + 1
continue
}
LocalTee(local_) if i + 1 < body.length() =>
match body[i + 1] {
Drop => {
if options.preserve_dead_local_tee_drops {
i = i + 1
continue
}
if hot_lower_impl_body_has_later_local_get(body, i + 2, local_) {
i = i + 1
continue
}
for prefix = cursor; prefix < i; prefix = prefix + 1 {
stripped.push(body[prefix])
}
stripped.push(Instruction::local_set(local_))
i = i + 2
cursor = i
continue
}
_ => ()
}
_ => ()
}
i = i + 1
}
// A rewrite always advances cursor past at least one removed instruction.
// Unchanged rows remain read-only; changed rows are independently owned.
// Public lowering still owns the preceding emitted/canonicalized rows.
if cursor == 0 {
return body
}
for tail = cursor; tail < body.length(); tail = tail + 1 {
stripped.push(body[tail])
}
stripped
}
///|
fn hot_lower_impl_canonicalize_body(
body : Array[Instruction],
options : HotLowerOptions,
) -> Array[Instruction] {
let canonical : Array[Instruction] = []
for i = 0; i < body.length(); i = i + 1 {
match hot_lower_impl_canonicalize_instr(body[i], options) {
Block(bt, Expr(inner_body)) if hot_lower_impl_can_inline_value_wrapper(
bt, inner_body,
) =>
for inner in inner_body {
canonical.push(inner)
}
Block(bt, Expr(inner_body)) if i == body.length() - 1 => {
let mut peeled = false
match bt {
VoidBlockType => ()
_ =>
match inner_body {
[Block(VoidBlockType, Expr(prefix_body)), LocalGet(_)] =>
if !hot_lower_impl_body_targets_removed_label(prefix_body, 1) &&
!hot_lower_impl_needs_explicit_unreachable_after_peeled_prefix(
prefix_body,
) {
let peeled_prefix : Array[Instruction] = []
for prefix_instr in prefix_body {
peeled_prefix.push(prefix_instr)
}
if hot_lower_impl_needs_explicit_unreachable_after_peeled_prefix(
prefix_body,
) {
peeled_prefix.push(Instruction::unreachable_())
}
canonical.push(
Instruction::block(
@lib.BlockType::void_(),
Expr::new(peeled_prefix),
),
)
canonical.push(inner_body[1])
peeled = true
}
[Block(VoidBlockType, Expr(_)), Block(_, _)] =>
if !hot_lower_impl_body_targets_removed_label(inner_body, 0) {
match inner_body[0] {
Block(VoidBlockType, Expr(prefix_body)) if prefix_body.any(fn(
instr,
) {
hot_lower_impl_has_control_transfer(instr)
}) =>
match inner_body[1] {
Block(tail_bt, Expr(_)) if tail_bt == bt => {
for inner in inner_body {
canonical.push(inner)
}
peeled = true
}
_ => ()
}
_ => ()
}
}
_ => ()
}
}
if !peeled {
match inner_body {
[Block(VoidBlockType, Expr(prefix_body)), tail] => {
let rewritten_prefix = hot_lower_impl_append_unreachable_if_needed(
prefix_body,
)
canonical.push(
Instruction::block(
bt,
Expr::new([
Instruction::block(
@lib.BlockType::void_(),
Expr::new(rewritten_prefix),
),
tail,
]),
),
)
}
_ => canonical.push(Instruction::block(bt, Expr::new(inner_body)))
}
}
}
canonical_instr => canonical.push(canonical_instr)
}
}
let canonical = hot_lower_impl_remove_pure_drop_between_unreachables(
canonical,
)
let canonical = hot_lower_impl_hoist_dropped_values_before_leading_unreachable(
canonical,
)
let canonical = hot_lower_impl_materialize_dead_const_prefix_before_unreachable(
canonical,
)
let canonical = hot_lower_impl_strip_redundant_nops_and_dead_local_tee_drops(
canonical, options,
)
let canonical = match
hot_lower_impl_try_voidify_terminal_return_if_forwarder(canonical) {
Some(rewritten) => rewritten
None => canonical
}
let canonical = match
hot_lower_impl_try_voidify_terminal_drop_if_forwarder(canonical) {
Some(rewritten) => rewritten
None => canonical
}
hot_lower_impl_trim_redundant_unreachable_tail(
hot_lower_impl_strip_redundant_nops_and_dead_local_tee_drops(
canonical, options,
),
)
}
///|
fn hot_lower_impl_count_local_gets_in_instr(
instr : Instruction,
counts : Array[Int32],
) -> Unit {
match instr {
LocalGet(li) => {
let @lib.LocalIdx(raw) = li
let idx = raw.reinterpret_as_int()
if idx >= 0 && idx < counts.length() {
counts[idx] = counts[idx] + 1
}
}
Block(_, Expr(body)) | Loop(_, Expr(body)) | TryTable(_, _, Expr(body)) =>
hot_lower_impl_count_local_gets_in_body(body, counts)
If(_, then_, else_) => {
hot_lower_impl_count_local_gets_in_body(then_, counts)
match else_ {
Some(else_body) =>
hot_lower_impl_count_local_gets_in_body(else_body, counts)
None => ()
}
}
_ => ()
}
}
///|
fn hot_lower_impl_count_local_gets_in_body(
body : Array[Instruction],
counts : Array[Int32],
) -> Unit {
// The emitted dead tail still participates in validation. Its reads keep
// non-defaultable local initialization writes alive even after a trap.
for instr in body {
hot_lower_impl_count_local_gets_in_instr(instr, counts)
}
}
///|
// These middle passes consume private emitted/canonicalized rows. A later
// canonicalization still supplies the public lower result's owned containers.
fn hot_lower_impl_changed_body_prefix(
body : Array[Instruction],
end : Int32,
reserve_tail~ : Bool = true,
) -> Array[Instruction] {
let copy = Array::new(capacity=if reserve_tail { body.length() } else { end })
for index in 0.. Instruction {
match instr {
Block(bt, Expr(body)) => {
let next = hot_lower_impl_prune_dead_local_tees_in_body(
body, counts, options,
)
if physical_equal(next, body) {
instr
} else {
Instruction::block(bt, Expr::new(next))
}
}
Loop(bt, Expr(body)) => {
let next = hot_lower_impl_prune_dead_local_tees_in_body(
body, counts, options,
)
if physical_equal(next, body) {
instr
} else {
Instruction::loop_(bt, Expr::new(next))
}
}
If(bt, then_, else_) => {
let next_then = hot_lower_impl_prune_dead_local_tees_in_body(
then_, counts, options,
)
let mut next_else = else_
if else_ is Some(body) {
let next = hot_lower_impl_prune_dead_local_tees_in_body(
body, counts, options,
)
if !physical_equal(next, body) {
next_else = Some(next)
}
}
if physical_equal(next_then, then_) && physical_equal(next_else, else_) {
instr
} else {
Instruction::if_(bt, next_then, next_else)
}
}
TryTable(bt, catches, Expr(body)) => {
let next = hot_lower_impl_prune_dead_local_tees_in_body(
body, counts, options,
)
if physical_equal(next, body) {
instr
} else {
Instruction::try_table(bt, catches, Expr::new(next))
}
}
_ => instr
}
}
///|
fn hot_lower_impl_prune_dead_local_tees_in_body(
body : Array[Instruction],
counts : Array[Int32],
options : HotLowerOptions,
) -> Array[Instruction] {
let mut pruned : Array[Instruction]? = None
for i = 0; i < body.length(); i = i + 1 {
let instr = body[i]
if instr is LocalTee(li) {
let @lib.LocalIdx(raw) = li
let idx = raw.reinterpret_as_int()
let dead = idx >= 0 && idx < counts.length() && counts[idx] == 0
let preserve_dead_drop = dead &&
options.preserve_dead_local_tee_drops &&
i + 1 < body.length() &&
body[i + 1] is Drop
if dead && !preserve_dead_drop {
if pruned is None {
pruned = Some(
hot_lower_impl_changed_body_prefix(body, i, reserve_tail=false),
)
}
continue
}
}
let next = hot_lower_impl_prune_dead_local_tees_in_instr(
instr, counts, options,
)
match pruned {
Some(rows) => rows.push(next)
None =>
if !physical_equal(next, instr) {
let rows = hot_lower_impl_changed_body_prefix(body, i)
rows.push(next)
pruned = Some(rows)
}
}
}
pruned.unwrap_or(body)
}
///|
fn hot_lower_impl_prune_dead_local_tees(
func : HotFunc,
body : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
) -> Array[Instruction] {
let counts = Array::make(hot_local_count(func) + extra_locals.length(), 0)
hot_lower_impl_count_local_gets_in_body(body, counts)
hot_lower_impl_prune_dead_local_tees_in_body(body, counts, options)
}
///|
fn hot_lower_impl_stackify_struct_set_value_tail_start(
body : Array[Instruction],
) -> Int32? {
if body.length() < 2 || hot_lower_impl_body_targets_removed_label(body, 0) {
return None
}
let tail_start = match body[body.length() - 1] {
If(_, _, _) if body.length() >= 3 &&
!hot_lower_impl_has_control_transfer(body[body.length() - 2]) =>
body.length() - 2
_ => body.length() - 1
}
for i = 0; i < tail_start; i = i + 1 {
if hot_lower_impl_has_control_transfer(body[i]) {
return None
}
}
Some(tail_start)
}
///|
fn hot_lower_impl_instr_has_later_local_get(
instr : Instruction,
local_ : LocalIdx,
) -> Bool {
match instr {
LocalGet(found) => found == local_
Block(_, Expr(body)) | Loop(_, Expr(body)) =>
hot_lower_impl_body_has_later_local_get(body, 0, local_)
If(_, then_, Some(else_)) =>
hot_lower_impl_body_has_later_local_get(then_, 0, local_) ||
hot_lower_impl_body_has_later_local_get(else_, 0, local_)
If(_, then_, None) =>
hot_lower_impl_body_has_later_local_get(then_, 0, local_)
TryTable(_, _, Expr(body)) =>
hot_lower_impl_body_has_later_local_get(body, 0, local_)
_ => false
}
}
///|
fn hot_lower_impl_body_has_later_local_get(
body : Array[Instruction],
start_idx : Int32,
local_ : LocalIdx,
) -> Bool {
for idx = start_idx; idx < body.length(); idx = idx + 1 {
if hot_lower_impl_instr_has_later_local_get(body[idx], local_) {
return true
}
}
false
}
///|
fn hot_lower_impl_instr_writes_local(
instr : Instruction,
local_ : LocalIdx,
) -> Bool {
match instr {
LocalSet(found) | LocalTee(found) => found == local_
Block(_, Expr(body)) | Loop(_, Expr(body)) =>
hot_lower_impl_body_writes_local(body, local_)
If(_, then_, Some(else_)) =>
hot_lower_impl_body_writes_local(then_, local_) ||
hot_lower_impl_body_writes_local(else_, local_)
If(_, then_, None) => hot_lower_impl_body_writes_local(then_, local_)
TryTable(_, _, Expr(body)) => hot_lower_impl_body_writes_local(body, local_)
_ => false
}
}
///|
fn hot_lower_impl_body_writes_local(
body : Array[Instruction],
local_ : LocalIdx,
) -> Bool {
for instr in body {
if hot_lower_impl_instr_writes_local(instr, local_) {
return true
}
}
false
}
///|
fn hot_lower_impl_catch_targets_enclosing_label_from_min_depth(
catch_ : Catch,
owner_depth : Int32,
min_owner_depth : Int32,
) -> Bool {
match catch_ {
Catch(_, li) | CatchRef(_, li) | CatchAll(li) | CatchAllRef(li) => {
let depth = hot_lower_impl_decoded_label_depth(li)
depth == owner_depth && depth >= min_owner_depth
}
}
}
///|
fn hot_lower_impl_instr_targets_enclosing_label_from_min_depth(
instr : Instruction,
owner_depth : Int32,
min_owner_depth : Int32,
) -> Bool {
match instr {
Br(li) | BrIf(li) | BrOnNull(li) | BrOnNonNull(li) => {
let depth = hot_lower_impl_decoded_label_depth(li)
depth == owner_depth && depth >= min_owner_depth
}
BrTable(labels, fallback) => {
let fallback_depth = hot_lower_impl_decoded_label_depth(fallback)
if fallback_depth == owner_depth && fallback_depth >= min_owner_depth {
return true
}
for li in labels {
let depth = hot_lower_impl_decoded_label_depth(li)
if depth == owner_depth && depth >= min_owner_depth {
return true
}
}
false
}
Block(_, Expr(body)) | Loop(_, Expr(body)) =>
hot_lower_impl_body_targets_enclosing_label_from_min_depth(
body,
owner_depth + 1,
min_owner_depth,
)
If(_, then_, Some(else_)) =>
hot_lower_impl_body_targets_enclosing_label_from_min_depth(
then_,
owner_depth + 1,
min_owner_depth,
) ||
hot_lower_impl_body_targets_enclosing_label_from_min_depth(
else_,
owner_depth + 1,
min_owner_depth,
)
If(_, then_, None) =>
hot_lower_impl_body_targets_enclosing_label_from_min_depth(
then_,
owner_depth + 1,
min_owner_depth,
)
TryTable(_, catches, Expr(body)) => {
for catch_ in catches {
if hot_lower_impl_catch_targets_enclosing_label_from_min_depth(
catch_, owner_depth, min_owner_depth,
) {
return true
}
}
hot_lower_impl_body_targets_enclosing_label_from_min_depth(
body,
owner_depth + 1,
min_owner_depth,
)
}
_ => false
}
}
///|
fn hot_lower_impl_body_targets_enclosing_label_from_min_depth(
body : Array[Instruction],
owner_depth : Int32,
min_owner_depth : Int32,
) -> Bool {
for instr in body {
if hot_lower_impl_instr_targets_enclosing_label_from_min_depth(
instr, owner_depth, min_owner_depth,
) {
return true
}
}
false
}
///|
fn hot_lower_impl_body_targets_own_label_from_min_depth(
body : Array[Instruction],
min_owner_depth : Int32,
) -> Bool {
hot_lower_impl_body_targets_enclosing_label_from_min_depth(
body, 0, min_owner_depth,
)
}
///|
fn hot_lower_impl_stackify_wrapped_struct_set_prefixes(
func : HotFunc,
body : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
) -> Array[Instruction] {
let mut output : Array[Instruction]? = None
let mut idx = 0
while idx < body.length() {
if idx + 2 < body.length() {
match body[idx] {
Block(VoidBlockType, Expr(prefix_body)) =>
if prefix_body.length() > 0 {
match prefix_body[prefix_body.length() - 1] {
Unreachable =>
match body[idx + 1] {
LocalGet(source_local) =>
match body[idx + 2] {
LocalSet(target_local) | LocalTee(target_local) => {
let prefix_writes_source_local = hot_lower_impl_body_writes_local(
prefix_body, source_local,
)
let prefix_targets_removed_enclosing_label_deep = hot_lower_impl_body_targets_removed_label(
prefix_body, 2,
)
let prefix_targets_own_label_deep = hot_lower_impl_body_targets_own_label_from_min_depth(
prefix_body, 2,
)
if source_local != target_local &&
!hot_lower_impl_body_has_later_local_get(
body,
idx + 3,
source_local,
) &&
!hot_lower_impl_body_targets_removed_label(
prefix_body, 1,
) &&
!prefix_targets_own_label_deep &&
!(prefix_writes_source_local &&
prefix_targets_removed_enclosing_label_deep) {
match target_local {
LocalIdx(raw_local) => {
let local_id = raw_local.reinterpret_as_int()
match hot_local_type(func, local_id) {
Some(local_ty) => {
let wrapped_body = hot_lower_impl_canonicalize_block_body(
BlockType::val_type(local_ty),
[body[idx], body[idx + 1]],
HotLowerOptions::new(),
)
let wrapped_body = hot_lower_impl_expand_branch_depths_above(
wrapped_body, -1,
)
let rewritten = match output {
Some(rows) => rows
None => {
let rows = hot_lower_impl_changed_body_prefix(
body, idx,
)
output = Some(rows)
rows
}
}
rewritten.push(
Instruction::block(
BlockType::val_type(local_ty),
Expr::new(wrapped_body),
),
)
rewritten.push(body[idx + 2])
idx = idx + 3
continue
}
None => ()
}
}
}
}
}
_ => ()
}
_ => ()
}
_ => ()
}
}
_ => ()
}
}
if !options.preserve_struct_set_value_blocks && idx + 2 < body.length() {
match body[idx] {
LocalGet(local_idx) =>
match body[idx + 1] {
Block(_, Expr(value_body)) =>
match
hot_lower_impl_stackify_struct_set_value_tail_start(value_body) {
Some(tail_start) =>
match body[idx + 2] {
StructSet(_, _) =>
match local_idx {
LocalIdx(raw_local) => {
let local_id = raw_local.reinterpret_as_int()
match hot_local_type(func, local_id) {
Some(local_ty) => {
let scratch_local = LocalIdx::new(
(hot_local_count(func) + extra_locals.length()).reinterpret_as_uint(),
)
extra_locals.push(local_ty)
let wrapper_body : Array[Instruction] = [
Instruction::local_get(local_idx),
Instruction::local_set(scratch_local),
]
for prefix_idx = 0
prefix_idx < tail_start
prefix_idx = prefix_idx + 1 {
wrapper_body.push(value_body[prefix_idx])
}
wrapper_body.push(
Instruction::local_get(scratch_local),
)
let rewritten = match output {
Some(rows) => rows
None => {
let rows = hot_lower_impl_changed_body_prefix(
body, idx,
)
output = Some(rows)
rows
}
}
rewritten.push(
Instruction::block(
BlockType::val_type(local_ty),
Expr::new(
hot_lower_impl_canonicalize_block_body(
BlockType::val_type(local_ty),
wrapper_body,
HotLowerOptions::new(),
),
),
),
)
for tail_idx = tail_start
tail_idx < value_body.length()
tail_idx = tail_idx + 1 {
rewritten.push(value_body[tail_idx])
}
rewritten.push(body[idx + 2])
idx = idx + 3
continue
}
None => ()
}
}
}
_ => ()
}
None => ()
}
_ => ()
}
_ => ()
}
}
match output {
Some(rows) => rows.push(body[idx])
None => ()
}
idx = idx + 1
}
output.unwrap_or(body)
}
///|
fn hot_lower_impl_stackify_nested_instr(
func : HotFunc,
instr : Instruction,
extra_locals : Array[ValType],
options : HotLowerOptions,
) -> Instruction {
match instr {
Block(bt, Expr(body)) => {
let next = hot_lower_impl_stackify_nested_body(
func, body, extra_locals, options,
)
if physical_equal(next, body) {
instr
} else {
Instruction::block(bt, Expr::new(next))
}
}
Loop(bt, Expr(body)) => {
let next = hot_lower_impl_stackify_nested_body(
func, body, extra_locals, options,
)
if physical_equal(next, body) {
instr
} else {
Instruction::loop_(bt, Expr::new(next))
}
}
If(bt, then_, else_) => {
let next_then = hot_lower_impl_stackify_nested_body(
func, then_, extra_locals, options,
)
let mut next_else = else_
if else_ is Some(body) {
let next = hot_lower_impl_stackify_nested_body(
func, body, extra_locals, options,
)
if !physical_equal(next, body) {
next_else = Some(next)
}
}
if physical_equal(next_then, then_) && physical_equal(next_else, else_) {
instr
} else {
Instruction::if_(bt, next_then, next_else)
}
}
TryTable(bt, catches, Expr(body)) => {
let next = hot_lower_impl_stackify_nested_body(
func, body, extra_locals, options,
)
if physical_equal(next, body) {
instr
} else {
Instruction::try_table(bt, catches, Expr::new(next))
}
}
_ => instr
}
}
///|
fn hot_lower_impl_stackify_nested_body(
func : HotFunc,
body : Array[Instruction],
extra_locals : Array[ValType],
options : HotLowerOptions,
) -> Array[Instruction] {
let mut rewritten : Array[Instruction]? = None
for index in 0.. rows.push(next)
None =>
if !physical_equal(next, instr) {
let rows = hot_lower_impl_changed_body_prefix(body, index)
rows.push(next)
rewritten = Some(rows)
}
}
}
hot_lower_impl_stackify_wrapped_struct_set_prefixes(
func,
rewritten.unwrap_or(body),
extra_locals,
options,
)
}
///|
fn hot_lower_impl_fix_split_payload_wrappers_in_instr(
instr : Instruction,
) -> Instruction {
match instr {
Block(bt, Expr(body)) => {
let fixed_body = hot_lower_impl_fix_split_payload_wrappers_in_body(body)
let fixed_body = match
hot_lower_impl_try_fix_split_payload_wrapper_with_tail(bt, fixed_body) {
Some(rewritten) => rewritten
None => fixed_body
}
if physical_equal(fixed_body, body) {
instr
} else {
Instruction::block(bt, Expr::new(fixed_body))
}
}
Loop(bt, Expr(body)) => {
let next = hot_lower_impl_fix_split_payload_wrappers_in_body(body)
if physical_equal(next, body) {
instr
} else {
Instruction::loop_(bt, Expr::new(next))
}
}
If(bt, then_, else_) => {
let next_then = hot_lower_impl_fix_split_payload_wrappers_in_body(then_)
let mut next_else = else_
if else_ is Some(body) {
let next = hot_lower_impl_fix_split_payload_wrappers_in_body(body)
if !physical_equal(next, body) {
next_else = Some(next)
}
}
if physical_equal(next_then, then_) && physical_equal(next_else, else_) {
instr
} else {
Instruction::if_(bt, next_then, next_else)
}
}
TryTable(bt, catches, Expr(body)) => {
let next = hot_lower_impl_fix_split_payload_wrappers_in_body(body)
if physical_equal(next, body) {
instr
} else {
Instruction::try_table(bt, catches, Expr::new(next))
}
}
_ => instr
}
}
///|
fn hot_lower_impl_fix_split_payload_wrappers_in_body(
body : Array[Instruction],
) -> Array[Instruction] {
let mut fixed : Array[Instruction]? = None
for index in 0.. rows.push(next)
None =>
if !physical_equal(next, instr) {
let rows = hot_lower_impl_changed_body_prefix(body, index)
rows.push(next)
fixed = Some(rows)
}
}
}
fixed.unwrap_or(body)
}
///|
fn hot_lower_impl_rewrite_binary_const_result_spill_in_instr(
func : HotFunc,
instr : Instruction,
) -> Instruction {
match instr {
Block(bt, Expr(body)) => {
let next = hot_lower_impl_rewrite_binary_const_result_spills_in_body(
func, body,
)
if physical_equal(next, body) {
instr
} else {
Instruction::block(bt, Expr::new(next))
}
}
Loop(bt, Expr(body)) => {
let next = hot_lower_impl_rewrite_binary_const_result_spills_in_body(
func, body,
)
if physical_equal(next, body) {
instr
} else {
Instruction::loop_(bt, Expr::new(next))
}
}
If(bt, then_, else_) => {
let next_then = hot_lower_impl_rewrite_binary_const_result_spills_in_body(
func, then_,
)
let mut next_else = else_
if else_ is Some(body) {
let next = hot_lower_impl_rewrite_binary_const_result_spills_in_body(
func, body,
)
if !physical_equal(next, body) {
next_else = Some(next)
}
}
if physical_equal(next_then, then_) && physical_equal(next_else, else_) {
instr
} else {
Instruction::if_(bt, next_then, next_else)
}
}
TryTable(bt, catches, Expr(body)) => {
let next = hot_lower_impl_rewrite_binary_const_result_spills_in_body(
func, body,
)
if physical_equal(next, body) {
instr
} else {
Instruction::try_table(bt, catches, Expr::new(next))
}
}
_ => instr
}
}
///|
fn hot_lower_impl_rewrite_binary_const_result_spills_in_body(
func : HotFunc,
body : Array[Instruction],
) -> Array[Instruction] {
// This private cleanup reads owned lowering rows. Copy only the first
// changed path; every flat rewrite permutes exactly five existing entries.
let mut nested = body
for index = 0; index < body.length(); index = index + 1 {
let instr = body[index]
let next = hot_lower_impl_rewrite_binary_const_result_spill_in_instr(
func, instr,
)
if !physical_equal(next, instr) {
if physical_equal(nested, body) {
nested = body.copy()
}
nested[index] = next
}
}
let mut rewritten = nested
let mut i = 0
while i < nested.length() {
if i + 4 < nested.length() {
let lhs = nested[i]
let rhs = nested[i + 1]
let op_instr = nested[i + 2]
let set_instr = nested[i + 3]
let get_instr = nested[i + 4]
let rhs_is_const = match rhs {
I32Const(_)
| I64Const(_)
| F32Const(_)
| F64Const(_)
| RefNull(_)
| StringConst(_) => true
_ => false
}
if rhs_is_const && !hot_lower_impl_has_control_transfer(op_instr) {
match lhs {
LocalGet(source_local) =>
match set_instr {
LocalSet(target_local) if target_local != source_local &&
!hot_lower_impl_body_has_later_local_get(
nested,
i + 5,
target_local,
) =>
match get_instr {
LocalGet(found_local) if found_local == target_local =>
match (source_local, target_local) {
(LocalIdx(raw_source), LocalIdx(raw_target)) => {
let source_local_id = raw_source.reinterpret_as_int()
let target_local_id = raw_target.reinterpret_as_int()
match
(
hot_local_type(func, source_local_id),
hot_local_type(func, target_local_id),
) {
(Some(source_ty), Some(target_ty)) if source_ty ==
target_ty => {
if physical_equal(rewritten, nested) {
rewritten = nested.copy()
}
rewritten[i] = lhs
rewritten[i + 1] = set_instr
rewritten[i + 2] = get_instr
rewritten[i + 3] = rhs
rewritten[i + 4] = op_instr
i = i + 5
continue
}
_ => ()
}
}
}
_ => ()
}
_ => ()
}
_ => ()
}
}
}
i = i + 1
}
rewritten
}
///|
fn hot_lower_body_with_extras_with_options(
func : HotFunc,
options : HotLowerOptions,
) -> HotLowerBodyResult {
hot_verify_all_or_abort(func)
hot_lower_body_with_extras_unverified_with_options(func, options)
}
///|
fn hot_lower_body_with_extras(func : HotFunc) -> HotLowerBodyResult {
hot_lower_body_with_extras_with_options(func, HotLowerOptions::new())
}
///|
fn hot_lower_body_with_extras_unverified_with_options(
func : HotFunc,
options : HotLowerOptions,
) -> HotLowerBodyResult {
// Lowering only needs exact multiplicities, not use-site or local overlays.
let use_def = hot_node_use_counts_build(func)
let label_frames = hot_lower_impl_new_label_frames(func)
let out : Array[Instruction] = []
let value_stack : HotLowerValueStack = hot_lower_value_stack_new()
let extra_locals : Array[ValType] = []
let root_ids = hot_root_span(func)
let preserve_region_results = hot_lower_impl_preserve_region_result_roots(
func,
use_def,
root_ids,
hot_type_result_arity(func, hot_body_result_type(func)),
)
let future_dependency_suffix_minima = hot_lower_impl_future_dependencies(
func, root_ids, label_frames,
)
for i = 0; i < hot_root_count(func); i = i + 1 {
let root_id = root_ids[i]
hot_lower_impl_emit_preceding_future_dependencies(
func,
root_id,
root_ids,
i + 1,
future_dependency_suffix_minima,
use_def,
label_frames,
-1,
value_stack,
out,
extra_locals,
options,
)
hot_lower_impl_emit_root(
func, root_id, use_def, label_frames, -1, value_stack, out, extra_locals, options,
)
hot_lower_impl_drop_dead_nonfinal_root_results(
func,
root_id,
use_def,
preserve_region_results[i],
i + 1 == hot_root_count(func),
value_stack,
out,
)
}
let canonical = hot_lower_impl_canonicalize_body(out, options)
let stackified = if options.preserve_flat_ir_spills {
canonical
} else {
hot_lower_impl_stackify_nested_body(func, canonical, extra_locals, options)
}
let stackified = if options.preserve_flat_ir_spills {
stackified
} else {
hot_lower_impl_fix_split_payload_wrappers_in_body(stackified)
}
let stackified = if options.preserve_flat_ir_spills {
stackified
} else {
hot_lower_impl_rewrite_binary_const_result_spills_in_body(func, stackified)
}
let stackified = hot_lower_impl_prune_dead_local_tees(
func, stackified, extra_locals, options,
)
{
body: hot_lower_impl_trim_redundant_unreachable_tail(
hot_lower_impl_canonicalize_body(stackified, options),
),
extra_locals,
final_stack: value_stack,
}
}
///|
fn hot_lower_body_with_extras_unverified(func : HotFunc) -> HotLowerBodyResult {
hot_lower_body_with_extras_unverified_with_options(
func,
HotLowerOptions::new(),
)
}
///|
pub fn hot_lower_body(func : HotFunc) -> Expr {
Expr::new(hot_lower_body_with_extras(func).body)
}
///|
pub fn hot_lower_body_unverified(func : HotFunc) -> Expr {
Expr::new(hot_lower_body_with_extras_unverified(func).body)
}
///|
pub fn hot_lower_func_with_options(
func : HotFunc,
options : HotLowerOptions,
) -> @lib.Func {
let lowered = hot_lower_body_with_extras_with_options(func, options)
let body = lowered.body
hot_lower_impl_finalize_region_stack(
func,
hot_type_result_arity(func, hot_body_result_type(func)),
lowered.final_stack,
body,
lowered.extra_locals,
)
@lib.Func::new(
@lib.Locals::from_types(
hot_lower_impl_declared_locals(func, lowered.extra_locals),
),
Expr::new(body),
)
}
///|
pub fn hot_lower_func(func : HotFunc) -> @lib.Func {
hot_lower_func_with_options(func, HotLowerOptions::new())
}
///|
pub fn hot_lower_func_unverified_with_options(
func : HotFunc,
options : HotLowerOptions,
) -> @lib.Func {
let lowered = hot_lower_body_with_extras_unverified_with_options(
func, options,
)
let body = lowered.body
hot_lower_impl_finalize_region_stack(
func,
hot_type_result_arity(func, hot_body_result_type(func)),
lowered.final_stack,
body,
lowered.extra_locals,
)
@lib.Func::new(
@lib.Locals::from_types(
hot_lower_impl_declared_locals(func, lowered.extra_locals),
),
Expr::new(body),
)
}
///|
pub fn hot_lower_func_unverified(func : HotFunc) -> @lib.Func {
hot_lower_func_unverified_with_options(func, HotLowerOptions::new())
}
///|
test "lower private nested stackify retains unchanged owned rows" {
let body = [Instruction::i32_const(I32(7)), Instruction::drop()]
let output = hot_lower_impl_stackify_nested_body(
HotFunc::new(),
body,
[],
HotLowerOptions::new(),
)
assert_eq(output, body)
assert_true(physical_equal(output, body))
}
///|
test "lower private struct prefix stackify retains unchanged owned rows" {
let body = [Instruction::i32_const(I32(7)), Instruction::drop()]
let output = hot_lower_impl_stackify_wrapped_struct_set_prefixes(
HotFunc::new(),
body,
[],
HotLowerOptions::new(),
)
assert_eq(output, body)
assert_true(physical_equal(output, body))
}
///|
test "lower private split payload fix retains unchanged owned rows" {
let body = [Instruction::i32_const(I32(7)), Instruction::drop()]
let output = hot_lower_impl_fix_split_payload_wrappers_in_body(body)
assert_eq(output, body)
assert_true(physical_equal(output, body))
}
///|
test "lower private dead tee cleanup retains unchanged owned rows" {
let local = LocalIdx::new(0U)
let body = [
Instruction::i32_const(I32(7)),
Instruction::local_tee(local),
Instruction::drop(),
Instruction::local_get(local),
Instruction::drop(),
]
let output = hot_lower_impl_prune_dead_local_tees_in_body(
body,
[1],
HotLowerOptions::new(),
)
assert_eq(output, body)
assert_true(physical_equal(output, body))
}