///|
pub enum HotLiftError {
InvalidLabelDepth(Int32)
InvalidFunctionIndex(FuncIdx)
MissingFunctionBody(FuncIdx)
MissingFunctionType(FuncIdx)
InvalidBodyState(String)
TypecheckFailure(String, String)
UnsupportedInstruction(String)
} derive(Eq, Debug)
///|
/// Source-order local accesses from an unmodified lift. A missing node (-1)
/// denotes a stack-polymorphic instruction with no concrete operand. Synthetic
/// lifetime captures have no source entry. Raw rewriters must use this map,
/// rather than assuming that HOT node order equals raw instruction order.
pub struct HotLiftLocalAccesses {
local_count : Int32
gets : Array[(LocalId, NodeId)]
writes : Array[(LocalId, NodeId)]
// Opt-in creator journal: (fresh capture read, its preceding capture set).
capture_sources : Array[(NodeId, NodeId)]?
revision : Int32
}
///|
pub fn hot_lift_local_accesses(func : HotFunc) -> HotLiftLocalAccesses {
match func.lifted_local_accesses {
Some(accesses) => {
if accesses.revision != func.revision {
abort("raw local access map requires the original, unmodified HOT lift")
}
accesses
}
None =>
abort(
"raw local access map requires a HOT lift with source recording enabled",
)
}
}
///|
/// Release optional source and capture provenance after its final consumer.
/// Existing journal aliases remain valid; this drops the HotFunc owner only.
pub fn hot_lift_release_local_accesses(func : HotFunc) -> Unit {
func.lifted_local_accesses = None
}
///|
// Only completed stack values are queried while lifting. Their operand trees
// remain immutable as later nodes are appended. A byte extent shares the old
// readiness row: 0 cold, 1 no locals, 2..129 single band 0..127,
// 130..254 mixed bands with maximum 0..124, and 255 unknown.
// Saturated or colliding masks retain the exact recursive conflict check.
priv struct HotLiftLocalAccessMasks {
reads : Array[UInt64]
writes : Array[UInt64]
extent : Array[Byte]
// Exact fallback memoization is needed only for mixed/saturated mask hits.
// A slot packs the full local id above four access/result flags.
mut exact_conflicts : Array[UInt64]?
}
///|
fn HotLiftLocalAccessMasks::new() -> HotLiftLocalAccessMasks {
{ reads: [], writes: [], extent: [], exact_conflicts: None, }
}
///|
priv struct HotLiftLabelFrames {
active : Array[LabelId]
mut legacy_catch_depth : Int32
local_count : Int32
record_local_accesses : Bool
gets : Array[(LocalId, NodeId)]
writes : Array[(LocalId, NodeId)]
capture_sources : Array[(NodeId, NodeId)]?
capture_stack_locals : Bool
materialize_multi_values : Bool
pending_effects : Array[Int32]
first_effect_orders : Array[Int32]
mut local_access_masks : HotLiftLocalAccessMasks?
mut scalar_result_scratch : Array[ValType]?
}
///|
fn hot_lift_impl_new_label_frames(
func : HotFunc,
capture_stack_locals~ : Bool = false,
materialize_multi_values~ : Bool = false,
record_local_accesses~ : Bool = true,
record_capture_sources~ : Bool = false,
) -> HotLiftLabelFrames {
{
active: [],
legacy_catch_depth: 0,
local_count: hot_local_count(func),
record_local_accesses,
gets: [],
writes: [],
capture_sources: if record_capture_sources && record_local_accesses {
Some([])
} else {
None
},
capture_stack_locals,
materialize_multi_values,
pending_effects: [],
first_effect_orders: [],
local_access_masks: None,
scalar_result_scratch: None,
}
}
///|
// This function-owned row is borrowed only until the current data instruction
// has been built. Empty/scalar types are copied by their consumers; multi-result
// type interning retains an owned row and must never receive this scratch.
fn hot_lift_impl_scalar_result_scratch(
frames : HotLiftLabelFrames,
) -> Array[ValType] {
match frames.scalar_result_scratch {
Some(row) => row
None => {
let row : Array[ValType] = []
frames.scalar_result_scratch = Some(row)
row
}
}
}
///|
fn hot_lift_impl_seed_function_label(frames : HotLiftLabelFrames) -> Int32 {
frames.active.push(HOT_IMPLICIT_FUNCTION_LABEL)
frames.active.length() - 1
}
///|
fn hot_lift_impl_push_label(
frames : HotLiftLabelFrames,
label_stack_head : Int32,
label_id : LabelId,
) -> Int32 {
ignore(label_stack_head)
frames.active.push(label_id)
frames.active.length() - 1
}
///|
fn hot_lift_impl_pop_label(frames : HotLiftLabelFrames) -> Unit {
match frames.active.pop() {
Some(_) => ()
None => abort("hot lift label pop requires a non-empty active label stack")
}
}
///|
fn hot_lift_impl_result_type(
func : HotFunc,
result_types : Array[ValType],
) -> TypeId {
match result_types.length() {
0 => hot_type_void(func)
1 => hot_type_intern_value(func, result_types[0])
_ => hot_type_intern_results(func, result_types)
}
}
///|
fn hot_lift_impl_fail_typecheck(
instr : Instruction,
err : String,
) -> HotLiftError {
HotLiftError::TypecheckFailure(instr.to_string(), err)
}
///|
fn hot_lift_impl_all_locals(locals : HotLocals) -> @lib.Locals {
let out : Array[ValType] = []
if locals.params.length() > 0 {
for param in locals.params {
out.push(param)
}
} else {
for _ in 0.. Array[ValType] {
let out : Array[ValType] = []
for vt in locals.iter() {
out.push(vt)
}
out
}
///|
fn hot_lift_impl_validation_locals(
params : Array[ValType],
body_locals : @lib.Locals,
) -> @lib.Locals {
if params.length() == 0 {
return body_locals
}
let runs : Array[@lib.LocalRun] = []
for param in params {
runs.push(@lib.LocalRun::new(1U, param))
}
for run in body_locals.runs() {
runs.push(run)
}
@lib.Locals::new(runs)
}
///|
fn hot_lift_impl_env_from_context(
ctx : HotModuleContext,
locals : HotLocals,
return_types : Array[ValType],
validation_locals~ : @lib.Locals? = None,
) -> Env {
let all_locals = match validation_locals {
Some(decls) => decls
None => hot_lift_impl_all_locals(locals)
}
let env = hot_module_context_validation_env(ctx)
.with_locals(all_locals)
.with_label(return_types)
.with_return_type(Some(return_types))
env
}
///|
fn hot_lift_impl_env_without_context(
locals : HotLocals,
return_types : Array[ValType],
) -> Env {
empty_env()
.with_locals(hot_lift_impl_all_locals(locals))
.with_label(return_types)
.with_return_type(Some(return_types))
}
///|
fn hot_lift_impl_block_results(
env : Env,
block_type : BlockType,
) -> Result[(Array[ValType], Array[ValType]), HotLiftError] {
match env.expand_blocktype(block_type) {
Ok(types) => Ok(types)
Err(err) =>
Err(
HotLiftError::InvalidBodyState(
"invalid block type \{block_type}: \{err}",
),
)
}
}
///|
fn hot_lift_impl_pop_concrete_type(concrete_values : Array[ValType]) -> ValType {
match concrete_values.pop() {
Some(ty) => ty
None => abort("hot lift concrete stack underflow")
}
}
///|
fn hot_lift_impl_pop_value(
values : Array[NodeId],
concrete_values : Array[ValType],
) -> NodeId {
match values.pop() {
Some(id) => {
ignore(hot_lift_impl_pop_concrete_type(concrete_values))
id
}
None => abort("hot lift stack underflow")
}
}
///|
fn hot_lift_impl_pop_values(
values : Array[NodeId],
concrete_values : Array[ValType],
count : Int32,
) -> Array[NodeId] {
let out = Array::make(count, HOT_NONE_NODE)
for idx = count - 1; idx >= 0; idx = idx - 1 {
out[idx] = hot_lift_impl_pop_value(values, concrete_values)
}
out
}
///|
fn hot_lift_impl_pop_typed_values(
values : Array[NodeId],
concrete_values : Array[ValType],
count : Int32,
) -> (Array[NodeId], Array[ValType]) {
let out_values = Array::make(count, HOT_NONE_NODE)
let out_types = Array::make(count, @lib.ValType::i32())
for idx = count - 1; idx >= 0; idx = idx - 1 {
match values.pop() {
Some(id) => {
out_values[idx] = id
out_types[idx] = hot_lift_impl_pop_concrete_type(concrete_values)
}
None => abort("hot lift stack underflow")
}
}
(out_values, out_types)
}
///|
fn hot_lift_impl_peek_values_with_extra(
values : Array[NodeId],
count : Int32,
extra : NodeId,
) -> Array[NodeId] {
let out = Array::make(count + 1, extra)
let start = values.length() - count
for i = 0; i < count; i = i + 1 {
out[i] = values[start + i]
}
out
}
///|
fn hot_lift_impl_require_value_count(
instr : Instruction,
values : Array[NodeId],
count : Int32,
context : String,
) -> Result[Unit, HotLiftError] {
if count <= values.length() {
return Ok(())
}
Err(
HotLiftError::InvalidBodyState(
"hot lift stack underflow in \{context} for \{instr}: need \{count}, have \{values.length()}",
),
)
}
///|
fn hot_lift_impl_pop_value_for_instr(
instr : Instruction,
values : Array[NodeId],
concrete_values : Array[ValType],
context : String,
) -> Result[NodeId, HotLiftError] {
match hot_lift_impl_require_value_count(instr, values, 1, context) {
Ok(_) => Ok(hot_lift_impl_pop_value(values, concrete_values))
Err(err) => Err(err)
}
}
///|
fn hot_lift_impl_pop_values_for_instr(
instr : Instruction,
values : Array[NodeId],
concrete_values : Array[ValType],
count : Int32,
context : String,
) -> Result[Array[NodeId], HotLiftError] {
match hot_lift_impl_require_value_count(instr, values, count, context) {
Ok(_) => Ok(hot_lift_impl_pop_values(values, concrete_values, count))
Err(err) => Err(err)
}
}
///|
fn hot_lift_impl_pop_typed_values_for_instr(
instr : Instruction,
values : Array[NodeId],
concrete_values : Array[ValType],
count : Int32,
context : String,
) -> Result[(Array[NodeId], Array[ValType]), HotLiftError] {
match hot_lift_impl_require_value_count(instr, values, count, context) {
Ok(_) => Ok(hot_lift_impl_pop_typed_values(values, concrete_values, count))
Err(err) => Err(err)
}
}
///|
fn hot_lift_impl_resolve_label(
frames : HotLiftLabelFrames,
label_stack_head : Int32,
label_idx : LabelIdx,
) -> Result[LabelId, HotLiftError] {
ignore(label_stack_head)
let LabelIdx(raw) = label_idx
let depth = raw.reinterpret_as_int()
let idx = frames.active.length() - 1 - depth
if idx >= 0 && idx < frames.active.length() {
return Ok(frames.active[idx])
}
Err(HotLiftError::InvalidLabelDepth(raw.reinterpret_as_int()))
}
///|
fn hot_lift_impl_branch_arity(
func : HotFunc,
label_id : LabelId,
return_type : Array[ValType]?,
) -> Result[Int32, HotLiftError] {
if label_id == HOT_IMPLICIT_FUNCTION_LABEL {
return match return_type {
Some(types) => Ok(types.length())
None => Err(HotLiftError::InvalidLabelDepth(-1))
}
}
Ok(hot_label_branch_arity(func, label_id))
}
///|
fn hot_lift_impl_resolve_catch_label(
frames : HotLiftLabelFrames,
label_stack_head : Int32,
label_idx : LabelIdx,
return_type : Array[ValType]?,
) -> Result[LabelId, HotLiftError] {
ignore(label_stack_head)
let LabelIdx(raw) = label_idx
let depth = raw.reinterpret_as_int()
let idx = frames.active.length() - 1 - depth
if idx >= 0 && idx < frames.active.length() {
return Ok(frames.active[idx])
}
if depth == frames.active.length() {
match return_type {
Some(_) => return Ok(HOT_IMPLICIT_FUNCTION_LABEL)
None => ()
}
}
Err(HotLiftError::InvalidLabelDepth(raw.reinterpret_as_int()))
}
///|
fn hot_lift_impl_try_catch_arm(
frames : HotLiftLabelFrames,
label_stack_head : Int32,
catch_ : Catch,
return_type : Array[ValType]?,
) -> Result[HotCatchArm, HotLiftError] {
match catch_ {
Catch::Catch(tag_idx, label_idx) =>
match
hot_lift_impl_resolve_catch_label(
frames, label_stack_head, label_idx, return_type,
) {
Ok(label_id) => Ok(HotCatchArm::catch_(tag_idx, label_id))
Err(err) => Err(err)
}
Catch::CatchRef(tag_idx, label_idx) =>
match
hot_lift_impl_resolve_catch_label(
frames, label_stack_head, label_idx, return_type,
) {
Ok(label_id) => Ok(HotCatchArm::catch_ref(tag_idx, label_id))
Err(err) => Err(err)
}
Catch::CatchAll(label_idx) =>
match
hot_lift_impl_resolve_catch_label(
frames, label_stack_head, label_idx, return_type,
) {
Ok(label_id) => Ok(HotCatchArm::all(label_id))
Err(err) => Err(err)
}
Catch::CatchAllRef(label_idx) =>
match
hot_lift_impl_resolve_catch_label(
frames, label_stack_head, label_idx, return_type,
) {
Ok(label_id) => Ok(HotCatchArm::all_ref(label_id))
Err(err) => Err(err)
}
}
}
///|
fn hot_lift_impl_make_region(
func : HotFunc,
body : Array[NodeId],
result_arity : Int32,
) -> NodeId {
let id = hot_build_node(
func,
HotOp::Block,
hot_type_void(func),
children=body,
)
let label_id = hot_label_alloc(
func,
HotLabelKind::RegionLabel,
hot_type_void(func),
id,
result_arity,
)
hot_node_set(func, id, { ..hot_node_get(func, id), imm0: label_id, })
id
}
///|
fn hot_lift_impl_intern_control_result_type(
func : HotFunc,
block_type : BlockType,
params : Array[ValType],
results : Array[ValType],
) -> TypeId {
match block_type {
TypeIdxBlockType(_) =>
hot_type_intern_resolved_block_result(func, block_type, params, results)
_ => hot_type_intern_block_result(func, block_type)
}
}
///|
fn hot_lift_impl_stack_suffix(
types : Array[ValType],
suffix_len : Int32,
) -> Array[ValType] {
let out : Array[ValType] = []
let start = types.length() - suffix_len
for i = start; i < types.length(); i = i + 1 {
out.push(types[i])
}
out
}
///|
fn hot_lift_impl_matches_candidate_after(
before_types : Array[ValType],
after_state : TcState,
pop_count : Int32,
candidate_after : TcState,
) -> Bool {
if candidate_after.polymorphic != after_state.polymorphic ||
candidate_after.reachable != after_state.reachable ||
candidate_after.escape != after_state.escape {
return false
}
let prefix_len = before_types.length() - pop_count
if prefix_len + candidate_after.stack.length() != after_state.stack.length() {
return false
}
for i = 0; i < prefix_len; i = i + 1 {
if before_types[i] != after_state.stack[i] {
return false
}
}
for i = 0; i < candidate_after.stack.length(); i = i + 1 {
if candidate_after.stack[i] != after_state.stack[prefix_len + i] {
return false
}
}
true
}
///|
fn hot_lift_impl_generic_pop_count(
instr : Instruction,
candidate_before_types : Array[ValType],
before_state : TcState,
after_state : TcState,
min_count : Int32,
) -> Result[(Int32, Array[ValType]), HotLiftError] {
for count = min_count
count <= candidate_before_types.length()
count = count + 1 {
let candidate_state = tc_state_with_owned_stack(
before_state,
hot_lift_impl_stack_suffix(candidate_before_types, count),
)
match Typecheck::typecheck(instr, candidate_state) {
Ok(candidate_after) =>
if hot_lift_impl_matches_candidate_after(
candidate_before_types, after_state, count, candidate_after,
) {
return Ok((count, candidate_after.stack))
}
Err(_) => ()
}
}
Err(
HotLiftError::InvalidBodyState(
"unable to derive concrete operand count for \{instr}: before=\{candidate_before_types} after=\{after_state.stack} reachable=\{before_state.reachable}",
),
)
}
///|
fn hot_lift_impl_resolve_generic_pop_count(
instr : Instruction,
before_state : TcState,
after_state : TcState,
concrete_before_types : Array[ValType],
) -> Result[(Int32, Array[ValType]), HotLiftError] {
match instr {
Instruction::RefCastDescEq(_, _, _) =>
match
hot_lift_impl_generic_pop_count(
instr,
before_state.stack,
before_state,
after_state,
2,
) {
Ok(two_operand) => return Ok(two_operand)
Err(_) =>
match
hot_lift_impl_generic_pop_count(
instr, concrete_before_types, before_state, after_state, 2,
) {
Ok(two_operand) => return Ok(two_operand)
Err(_) => ()
}
}
_ => ()
}
let primary = hot_lift_impl_generic_pop_count(
instr,
before_state.stack,
before_state,
after_state,
0,
)
if concrete_before_types.length() == 0 {
return primary
}
let can_promote = match instr {
Instruction::Nop
| Instruction::Unreachable
| Instruction::I32Const(_)
| Instruction::I64Const(_)
| Instruction::F32Const(_)
| Instruction::F64Const(_)
| Instruction::RefNull(_)
| Instruction::StringConst(_)
| Instruction::RefFunc(_)
| Instruction::LocalGet(_)
| Instruction::GlobalGet(_)
| Instruction::MemorySize(_)
| Instruction::TableSize(_)
| Instruction::DataDrop(_)
| Instruction::ElemDrop(_) => false
_ => true
}
if !can_promote {
return primary
}
if before_state.reachable {
let has_concrete_prefix = concrete_before_types.length() >
before_state.stack.length()
return match primary {
Ok((count, _)) if count > 0 => primary
Ok(_) if has_concrete_prefix =>
match
hot_lift_impl_generic_pop_count(
instr, concrete_before_types, before_state, after_state, 1,
) {
Ok(promoted) => Ok(promoted)
Err(_) => primary
}
Err(_) if has_concrete_prefix =>
hot_lift_impl_generic_pop_count(
instr, concrete_before_types, before_state, after_state, 1,
)
_ => primary
}
}
let has_concrete_prefix = concrete_before_types.length() >
before_state.stack.length()
if has_concrete_prefix {
match
hot_lift_impl_generic_pop_count(
instr, concrete_before_types, before_state, after_state, 1,
) {
Ok(promoted) => return Ok(promoted)
Err(_) => ()
}
}
match primary {
Ok((count, _)) if count > 0 => primary
Ok(_) =>
match
hot_lift_impl_generic_pop_count(
instr, concrete_before_types, before_state, after_state, 1,
) {
Ok(promoted) => Ok(promoted)
Err(_) => primary
}
Err(_) =>
hot_lift_impl_generic_pop_count(
instr, concrete_before_types, before_state, after_state, 1,
)
}
}
///|
fn hot_lift_impl_call_param_count(env : Env, instr : Instruction) -> Int32? {
match instr {
Instruction::Call(func_idx) | Instruction::ReturnCall(func_idx) =>
match env.get_functype_by_funcidx(func_idx) {
Some(FuncType(params, _)) => Some(params.length())
None => None
}
Instruction::CallIndirect(type_idx, _)
| Instruction::ReturnCallIndirect(type_idx, _) =>
match env.resolve_functype(type_idx) {
Some(FuncType(params, _)) => Some(params.length() + 1)
None => None
}
Instruction::CallRef(type_idx) | Instruction::ReturnCallRef(type_idx) =>
match env.resolve_functype(type_idx) {
Some(FuncType(params, _)) => Some(params.length() + 1)
None => None
}
_ => None
}
}
///|
fn hot_lift_impl_known_pop_count(instr : Instruction, env : Env) -> Int32? {
// SIMD constants are nullary even on a polymorphic stack. Generic arity
// inference must not promote them to consume a preceding scalar operand.
match instr {
Instruction::V128Const(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _) =>
return Some(0)
_ => ()
}
match hot_lift_impl_direct_family(instr) {
Some(HotOp::Nop)
| Some(HotOp::Unreachable)
| Some(HotOp::Const)
| Some(HotOp::LocalGet)
| Some(HotOp::GlobalGet)
| Some(HotOp::TableSize)
| Some(HotOp::MemorySize)
| Some(HotOp::ElemDrop)
| Some(HotOp::DataDrop)
| Some(HotOp::RefNull)
| Some(HotOp::RefFunc) => return Some(0)
Some(HotOp::LocalSet)
| Some(HotOp::LocalTee)
| Some(HotOp::GlobalSet)
| Some(HotOp::TableGet)
| Some(HotOp::MemoryGrow)
| Some(HotOp::Drop)
| Some(HotOp::RefIsNull) => return Some(1)
Some(HotOp::TableSet) | Some(HotOp::TableGrow) => return Some(2)
Some(HotOp::TableFill)
| Some(HotOp::TableCopy)
| Some(HotOp::TableInit)
| Some(HotOp::MemoryCopy)
| Some(HotOp::MemoryFill)
| Some(HotOp::MemoryInit) => return Some(3)
Some(HotOp::Call) =>
match hot_lift_impl_call_param_count(env, instr) {
Some(count) => return Some(count)
None => ()
}
_ => ()
}
match hot_lift_impl_exact_family(instr) {
Some(HotOp::Select) => Some(3)
Some(HotOp::Load)
| Some(HotOp::RefTest)
| Some(HotOp::RefCast)
| Some(HotOp::RefTestDesc)
| Some(HotOp::Unary)
| Some(HotOp::Convert) => Some(1)
Some(HotOp::RefCastDescEq) => None
Some(HotOp::Store) | Some(HotOp::Binary) | Some(HotOp::Compare) => Some(2)
Some(HotOp::Continuation) =>
match instr {
Instruction::Resume(type_idx, _) =>
match env.resolve_cont_functype(type_idx) {
Some(FuncType(params, _)) => Some(params.length() + 1)
None => None
}
Instruction::ResumeThrow(_, tag, _) =>
match env.resolve_tag_functype(tag) {
Some(FuncType(params, _)) => Some(params.length() + 1)
None => None
}
Instruction::ResumeThrowRef(_, _) => Some(2)
Instruction::Suspend(tag) =>
match env.resolve_tag_functype(tag) {
Some(FuncType(params, _)) => Some(params.length())
None => None
}
Instruction::StackSwitch(type_idx, _) =>
match env.resolve_cont_functype(type_idx) {
Some(FuncType(params, _)) => Some(params.length())
None => None
}
_ => None
}
Some(HotOp::Heap) =>
match instr {
Instruction::WaitqueueNew => Some(0)
Instruction::StructGet(_, _)
| Instruction::StructGetS(_, _)
| Instruction::StructGetU(_, _)
| Instruction::ArrayLen
| Instruction::RefAsNonNull
| Instruction::AnyConvertExtern
| Instruction::ExternConvertAny
| Instruction::RefI31
| Instruction::RefI31Shared
| Instruction::I31GetS
| Instruction::I31GetU
| Instruction::ContNew(_) => Some(1)
Instruction::ContBind(source, target) =>
match
(
env.resolve_cont_functype(source),
env.resolve_cont_functype(target),
) {
(Some(FuncType(source_params, _)), Some(FuncType(target_params, _))) =>
Some(source_params.length() - target_params.length() + 1)
_ => None
}
Instruction::StructSet(_, _)
| Instruction::ArrayGet(_)
| Instruction::ArrayGetS(_)
| Instruction::ArrayGetU(_)
| Instruction::ArrayLoad(_, _, _)
| Instruction::RefEq => Some(2)
Instruction::ArraySet(_) | Instruction::ArrayStore(_, _, _) => Some(3)
_ => None
}
Some(HotOp::Atomic) =>
match instr {
Instruction::AtomicFence(_) => Some(0)
Instruction::StructAtomicSet(_, _, _) => Some(2)
Instruction::ArrayAtomicSet(_, _) => Some(3)
Instruction::WaitqueueNotify => Some(2)
Instruction::StructWait(_, _) => Some(4)
Instruction::I32AtomicLoad(_, _)
| Instruction::I64AtomicLoad(_, _)
| Instruction::I32AtomicLoad8U(_, _)
| Instruction::I32AtomicLoad16U(_, _)
| Instruction::I64AtomicLoad8U(_, _)
| Instruction::I64AtomicLoad16U(_, _)
| Instruction::I64AtomicLoad32U(_, _)
| Instruction::StructAtomicGet(_, _, _)
| Instruction::StructAtomicGetS(_, _, _)
| Instruction::StructAtomicGetU(_, _, _) => Some(1)
Instruction::MemoryAtomicNotify(_)
| Instruction::AtomicRmw(_, _, _)
| Instruction::StructAtomicRmw(_, _, _, _)
| Instruction::I32AtomicStore(_, _)
| Instruction::I64AtomicStore(_, _)
| Instruction::I32AtomicStore8(_, _)
| Instruction::I32AtomicStore16(_, _)
| Instruction::I64AtomicStore8(_, _)
| Instruction::I64AtomicStore16(_, _)
| Instruction::I64AtomicStore32(_, _) => Some(2)
Instruction::MemoryAtomicWait32(_)
| Instruction::MemoryAtomicWait64(_)
| Instruction::AtomicCmpxchg(_, _, _)
| Instruction::StructAtomicCmpxchg(_, _, _)
| Instruction::ArrayAtomicRmw(_, _, _) => Some(3)
Instruction::ArrayAtomicGet(_, _)
| Instruction::ArrayAtomicGetS(_, _)
| Instruction::ArrayAtomicGetU(_, _) => Some(2)
Instruction::ArrayAtomicCmpxchg(_, _) => Some(4)
_ => None
}
_ => None
}
}
///|
fn hot_lift_impl_can_materialize_unreachable_instr(
instr : Instruction,
tc_state : TcState,
concrete_values : Array[ValType],
child_count : Int32,
) -> Bool {
if child_count > concrete_values.length() {
return false
}
if tc_state.reachable {
return true
}
match
Typecheck::typecheck(
instr,
make_state_owned(
tc_state.env,
hot_lift_impl_stack_suffix(concrete_values, child_count),
),
) {
Ok(_) => true
Err(_) => false
}
}
///|
fn hot_lift_impl_direct_family(instr : Instruction) -> HotOp? {
match instr {
Instruction::Nop => Some(HotOp::Nop)
Instruction::Unreachable => Some(HotOp::Unreachable)
Instruction::I32Const(_)
| Instruction::I64Const(_)
| Instruction::F32Const(_)
| Instruction::F64Const(_)
| Instruction::StringConst(_) => Some(HotOp::Const)
Instruction::LocalGet(_) => Some(HotOp::LocalGet)
Instruction::LocalSet(_) => Some(HotOp::LocalSet)
Instruction::LocalTee(_) => Some(HotOp::LocalTee)
Instruction::GlobalGet(_) => Some(HotOp::GlobalGet)
Instruction::GlobalSet(_) => Some(HotOp::GlobalSet)
Instruction::TableGet(_) => Some(HotOp::TableGet)
Instruction::TableSet(_) => Some(HotOp::TableSet)
Instruction::TableSize(_) => Some(HotOp::TableSize)
Instruction::TableGrow(_) => Some(HotOp::TableGrow)
Instruction::TableFill(_) => Some(HotOp::TableFill)
Instruction::TableCopy(_, _) => Some(HotOp::TableCopy)
Instruction::TableInit(_, _) => Some(HotOp::TableInit)
Instruction::ElemDrop(_) => Some(HotOp::ElemDrop)
Instruction::MemorySize(_) => Some(HotOp::MemorySize)
Instruction::MemoryGrow(_) => Some(HotOp::MemoryGrow)
Instruction::MemoryCopy(_, _) => Some(HotOp::MemoryCopy)
Instruction::MemoryFill(_) => Some(HotOp::MemoryFill)
Instruction::MemoryInit(_, _) => Some(HotOp::MemoryInit)
Instruction::DataDrop(_) => Some(HotOp::DataDrop)
Instruction::Drop => Some(HotOp::Drop)
Instruction::Call(_) => Some(HotOp::Call)
Instruction::CallIndirect(_, _) => Some(HotOp::CallIndirect)
Instruction::CallRef(_) => Some(HotOp::CallRef)
Instruction::ReturnCall(_) => Some(HotOp::ReturnCall)
Instruction::ReturnCallIndirect(_, _) => Some(HotOp::ReturnCallIndirect)
Instruction::ReturnCallRef(_) => Some(HotOp::ReturnCallRef)
Instruction::RefNull(_) => Some(HotOp::RefNull)
Instruction::RefIsNull => Some(HotOp::RefIsNull)
Instruction::RefFunc(_) => Some(HotOp::RefFunc)
Instruction::Throw(_) => Some(HotOp::Throw)
Instruction::ThrowRef => Some(HotOp::ThrowRef)
_ => None
}
}
///|
fn hot_lift_impl_exact_family(instr : Instruction) -> HotOp? {
match instr {
Instruction::Resume(_, _)
| Instruction::ResumeThrow(_, _, _)
| Instruction::ResumeThrowRef(_, _)
| Instruction::Suspend(_)
| Instruction::StackSwitch(_, _) => Some(HotOp::Continuation)
Instruction::Select(_) => Some(HotOp::Select)
Instruction::I32Load(_)
| Instruction::I64Load(_)
| Instruction::F32Load(_)
| Instruction::F64Load(_)
| Instruction::I32Load8S(_)
| Instruction::I32Load8U(_)
| Instruction::I32Load16S(_)
| Instruction::I32Load16U(_)
| Instruction::I64Load8S(_)
| Instruction::I64Load8U(_)
| Instruction::I64Load16S(_)
| Instruction::I64Load16U(_)
| Instruction::I64Load32S(_)
| Instruction::I64Load32U(_) => Some(HotOp::Load)
Instruction::I32Store(_)
| Instruction::I64Store(_)
| Instruction::F32Store(_)
| Instruction::F64Store(_)
| Instruction::I32Store8(_)
| Instruction::I32Store16(_)
| Instruction::I64Store8(_)
| Instruction::I64Store16(_)
| Instruction::I64Store32(_) => Some(HotOp::Store)
Instruction::RefTest(_, _, _) => Some(HotOp::RefTest)
Instruction::RefCast(_, _, _) => Some(HotOp::RefCast)
Instruction::RefTestDesc(_, _) => Some(HotOp::RefTestDesc)
Instruction::RefCastDescEq(_, _, _) => Some(HotOp::RefCastDescEq)
Instruction::BrOnCast(_, _, _, _) | Instruction::BrOnCastDescEq(_, _, _) =>
Some(HotOp::BrOnCast)
Instruction::BrOnCastFail(_, _, _, _)
| Instruction::BrOnCastDescEqFail(_, _, _) => Some(HotOp::BrOnCastFail)
Instruction::StringMeasureWtf16
| Instruction::StringConcat
| Instruction::StringEq
| Instruction::StringAsWtf16
| Instruction::StringViewWtf16GetCodeunit
| Instruction::StringViewWtf16Slice
| Instruction::StringNewUtf8Array
| Instruction::StringNewWtf16Array
| Instruction::StringEncodeUtf8Array
| Instruction::StringEncodeWtf16Array
| Instruction::StringNewLossyUtf8Array
| Instruction::StringNewWtf8Array
| Instruction::StringEncodeLossyUtf8Array
| Instruction::StringEncodeWtf8Array => Some(HotOp::StringOp)
Instruction::AtomicFence(_)
| Instruction::WaitqueueNotify
| Instruction::StructWait(_, _)
| Instruction::MemoryAtomicNotify(_)
| Instruction::MemoryAtomicWait32(_)
| Instruction::MemoryAtomicWait64(_)
| Instruction::I32AtomicLoad(_, _)
| Instruction::I64AtomicLoad(_, _)
| Instruction::I32AtomicLoad8U(_, _)
| Instruction::I32AtomicLoad16U(_, _)
| Instruction::I64AtomicLoad8U(_, _)
| Instruction::I64AtomicLoad16U(_, _)
| Instruction::I64AtomicLoad32U(_, _)
| Instruction::StructAtomicSet(_, _, _)
| Instruction::ArrayAtomicSet(_, _)
| Instruction::StructAtomicGet(_, _, _)
| Instruction::StructAtomicGetS(_, _, _)
| Instruction::StructAtomicGetU(_, _, _)
| Instruction::StructAtomicRmw(_, _, _, _)
| Instruction::StructAtomicCmpxchg(_, _, _)
| Instruction::ArrayAtomicGet(_, _)
| Instruction::ArrayAtomicGetS(_, _)
| Instruction::ArrayAtomicGetU(_, _)
| Instruction::ArrayAtomicRmw(_, _, _)
| Instruction::ArrayAtomicCmpxchg(_, _)
| Instruction::I32AtomicStore(_, _)
| Instruction::I64AtomicStore(_, _)
| Instruction::I32AtomicStore8(_, _)
| Instruction::I32AtomicStore16(_, _)
| Instruction::I64AtomicStore8(_, _)
| Instruction::I64AtomicStore16(_, _)
| Instruction::I64AtomicStore32(_, _)
| Instruction::AtomicRmw(_, _, _)
| Instruction::AtomicCmpxchg(_, _, _) => Some(HotOp::Atomic)
Instruction::WaitqueueNew
| Instruction::StructNew(_)
| Instruction::ContNew(_)
| Instruction::ContBind(_, _)
| Instruction::StructNewDefault(_)
| Instruction::StructNewDesc(_)
| Instruction::StructNewDefaultDesc(_)
| Instruction::StructGet(_, _)
| Instruction::StructGetS(_, _)
| Instruction::StructGetU(_, _)
| Instruction::StructSet(_, _)
| Instruction::ArrayNew(_)
| Instruction::ArrayNewDefault(_)
| Instruction::ArrayNewFixed(_, _)
| Instruction::ArrayNewData(_, _)
| Instruction::ArrayNewElem(_, _)
| Instruction::ArrayLoad(_, _, _)
| Instruction::ArrayStore(_, _, _)
| Instruction::ArrayGet(_)
| Instruction::ArrayGetS(_)
| Instruction::ArrayGetU(_)
| Instruction::ArraySet(_)
| Instruction::ArrayLen
| Instruction::ArrayFill(_)
| Instruction::ArrayCopy(_, _)
| Instruction::ArrayInitData(_, _)
| Instruction::ArrayInitElem(_, _)
| Instruction::RefEq
| Instruction::RefAsNonNull
| Instruction::RefGetDesc(_)
| Instruction::AnyConvertExtern
| Instruction::ExternConvertAny
| Instruction::RefI31
| Instruction::RefI31Shared
| Instruction::I31GetS
| Instruction::I31GetU => Some(HotOp::Heap)
Instruction::I32Eqz
| Instruction::I64Eqz
| Instruction::I32Clz
| Instruction::I32Ctz
| Instruction::I32Popcnt
| Instruction::I64Clz
| Instruction::I64Ctz
| Instruction::I64Popcnt
| Instruction::F32Abs
| Instruction::F32Neg
| Instruction::F32Ceil
| Instruction::F32Floor
| Instruction::F32Trunc
| Instruction::F32Nearest
| Instruction::F32Sqrt
| Instruction::F64Abs
| Instruction::F64Neg
| Instruction::F64Ceil
| Instruction::F64Floor
| Instruction::F64Trunc
| Instruction::F64Nearest
| Instruction::F64Sqrt
| Instruction::I32Extend8S
| Instruction::I32Extend16S
| Instruction::I64Extend8S
| Instruction::I64Extend16S
| Instruction::I64Extend32S => Some(HotOp::Unary)
Instruction::I32Eq
| Instruction::I32Ne
| Instruction::I32LtS
| Instruction::I32LtU
| Instruction::I32GtS
| Instruction::I32GtU
| Instruction::I32LeS
| Instruction::I32LeU
| Instruction::I32GeS
| Instruction::I32GeU
| Instruction::I64Eq
| Instruction::I64Ne
| Instruction::I64LtS
| Instruction::I64LtU
| Instruction::I64GtS
| Instruction::I64GtU
| Instruction::I64LeS
| Instruction::I64LeU
| Instruction::I64GeS
| Instruction::I64GeU
| Instruction::F32Eq
| Instruction::F32Ne
| Instruction::F32Lt
| Instruction::F32Gt
| Instruction::F32Le
| Instruction::F32Ge
| Instruction::F64Eq
| Instruction::F64Ne
| Instruction::F64Lt
| Instruction::F64Gt
| Instruction::F64Le
| Instruction::F64Ge => Some(HotOp::Compare)
Instruction::I32Add
| Instruction::I32Sub
| Instruction::I32Mul
| Instruction::I32DivS
| Instruction::I32DivU
| Instruction::I32RemS
| Instruction::I32RemU
| Instruction::I32And
| Instruction::I32Or
| Instruction::I32Xor
| Instruction::I32Shl
| Instruction::I32ShrS
| Instruction::I32ShrU
| Instruction::I32Rotl
| Instruction::I32Rotr
| Instruction::I64Add
| Instruction::I64Sub
| Instruction::I64Mul
| Instruction::I64DivS
| Instruction::I64DivU
| Instruction::I64RemS
| Instruction::I64RemU
| Instruction::I64And
| Instruction::I64Or
| Instruction::I64Xor
| Instruction::I64Shl
| Instruction::I64ShrS
| Instruction::I64ShrU
| Instruction::I64Rotl
| Instruction::I64Rotr
| Instruction::F32Add
| Instruction::F32Sub
| Instruction::F32Mul
| Instruction::F32Div
| Instruction::F32Min
| Instruction::F32Max
| Instruction::F32Copysign
| Instruction::F64Add
| Instruction::F64Sub
| Instruction::F64Mul
| Instruction::F64Div
| Instruction::F64Min
| Instruction::F64Max
| Instruction::F64Copysign => Some(HotOp::Binary)
Instruction::I32WrapI64
| Instruction::I32TruncF32S
| Instruction::I32TruncF32U
| Instruction::I32TruncF64S
| Instruction::I32TruncF64U
| Instruction::I64ExtendI32S
| Instruction::I64ExtendI32U
| Instruction::I64TruncF32S
| Instruction::I64TruncF32U
| Instruction::I64TruncF64S
| Instruction::I64TruncF64U
| Instruction::F32ConvertI32S
| Instruction::F32ConvertI32U
| Instruction::F32ConvertI64S
| Instruction::F32ConvertI64U
| Instruction::F32DemoteF64
| Instruction::F64ConvertI32S
| Instruction::F64ConvertI32U
| Instruction::F64ConvertI64S
| Instruction::F64ConvertI64U
| Instruction::F64PromoteF32
| Instruction::I32ReinterpretF32
| Instruction::I64ReinterpretF64
| Instruction::F32ReinterpretI32
| Instruction::F64ReinterpretI64
| Instruction::I32TruncSatF32S
| Instruction::I32TruncSatF32U
| Instruction::I32TruncSatF64S
| Instruction::I32TruncSatF64U
| Instruction::I64TruncSatF32S
| Instruction::I64TruncSatF32U
| Instruction::I64TruncSatF64S
| Instruction::I64TruncSatF64U => Some(HotOp::Convert)
Instruction::V128Load(_)
| Instruction::V128Load8x8S(_)
| Instruction::V128Load8x8U(_)
| Instruction::V128Load16x4S(_)
| Instruction::V128Load16x4U(_)
| Instruction::V128Load32x2S(_)
| Instruction::V128Load32x2U(_)
| Instruction::V128Load8Splat(_)
| Instruction::V128Load16Splat(_)
| Instruction::V128Load32Splat(_)
| Instruction::V128Load64Splat(_)
| Instruction::V128Store(_)
| Instruction::V128Const(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)
| Instruction::I8x16Shuffle(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _)
| Instruction::I8x16Swizzle
| Instruction::I8x16Splat
| Instruction::I16x8Splat
| Instruction::I32x4Splat
| Instruction::I64x2Splat
| Instruction::F32x4Splat
| Instruction::F64x2Splat
| Instruction::I8x16ExtractLaneS(_)
| Instruction::I8x16ExtractLaneU(_)
| Instruction::I8x16ReplaceLane(_)
| Instruction::I16x8ExtractLaneS(_)
| Instruction::I16x8ExtractLaneU(_)
| Instruction::I16x8ReplaceLane(_)
| Instruction::I32x4ExtractLane(_)
| Instruction::I32x4ReplaceLane(_)
| Instruction::I64x2ExtractLane(_)
| Instruction::I64x2ReplaceLane(_)
| Instruction::F32x4ExtractLane(_)
| Instruction::F32x4ReplaceLane(_)
| Instruction::F64x2ExtractLane(_)
| Instruction::F64x2ReplaceLane(_)
| Instruction::I8x16Eq
| Instruction::I8x16Ne
| Instruction::I8x16LtS
| Instruction::I8x16LtU
| Instruction::I8x16GtS
| Instruction::I8x16GtU
| Instruction::I8x16LeS
| Instruction::I8x16LeU
| Instruction::I8x16GeS
| Instruction::I8x16GeU
| Instruction::I16x8Eq
| Instruction::I16x8Ne
| Instruction::I16x8LtS
| Instruction::I16x8LtU
| Instruction::I16x8GtS
| Instruction::I16x8GtU
| Instruction::I16x8LeS
| Instruction::I16x8LeU
| Instruction::I16x8GeS
| Instruction::I16x8GeU
| Instruction::I32x4Eq
| Instruction::I32x4Ne
| Instruction::I32x4LtS
| Instruction::I32x4LtU
| Instruction::I32x4GtS
| Instruction::I32x4GtU
| Instruction::I32x4LeS
| Instruction::I32x4LeU
| Instruction::I32x4GeS
| Instruction::I32x4GeU
| Instruction::F32x4Eq
| Instruction::F32x4Ne
| Instruction::F32x4Lt
| Instruction::F32x4Gt
| Instruction::F32x4Le
| Instruction::F32x4Ge
| Instruction::F64x2Eq
| Instruction::F64x2Ne
| Instruction::F64x2Lt
| Instruction::F64x2Gt
| Instruction::F64x2Le
| Instruction::F64x2Ge
| Instruction::I8x16Abs
| Instruction::I8x16Neg
| Instruction::I8x16Popcnt
| Instruction::I8x16AllTrue
| Instruction::I8x16Bitmask
| Instruction::I8x16NarrowI16x8S
| Instruction::I8x16NarrowI16x8U
| Instruction::F32x4Ceil
| Instruction::F32x4Floor
| Instruction::F32x4Trunc
| Instruction::F32x4Nearest
| Instruction::I8x16Shl
| Instruction::I8x16ShrS
| Instruction::I8x16ShrU
| Instruction::I8x16Add
| Instruction::I8x16AddSatS
| Instruction::I8x16AddSatU
| Instruction::I8x16Sub
| Instruction::I8x16SubSatS
| Instruction::I8x16SubSatU
| Instruction::F64x2Ceil
| Instruction::F64x2Floor
| Instruction::I8x16MinS
| Instruction::I8x16MinU
| Instruction::I8x16MaxS
| Instruction::I8x16MaxU
| Instruction::F64x2Trunc
| Instruction::I8x16AvgrU
| Instruction::I16x8ExtaddPairwiseI8x16S
| Instruction::I16x8ExtaddPairwiseI8x16U
| Instruction::I32x4ExtaddPairwiseI16x8S
| Instruction::I32x4ExtaddPairwiseI16x8U
| Instruction::I16x8Abs
| Instruction::I16x8Neg
| Instruction::I16x8Q15mulrSatS
| Instruction::I16x8AllTrue
| Instruction::I16x8Bitmask
| Instruction::I16x8NarrowI32x4S
| Instruction::I16x8NarrowI32x4U
| Instruction::I16x8ExtendLowI8x16S
| Instruction::I16x8ExtendHighI8x16S
| Instruction::I16x8ExtendLowI8x16U
| Instruction::I16x8ExtendHighI8x16U
| Instruction::I16x8Shl
| Instruction::I16x8ShrS
| Instruction::I16x8ShrU
| Instruction::I16x8Add
| Instruction::I16x8AddSatS
| Instruction::I16x8AddSatU
| Instruction::I16x8Sub
| Instruction::I16x8SubSatS
| Instruction::I16x8SubSatU
| Instruction::F64x2Nearest
| Instruction::I16x8Mul
| Instruction::I16x8MinS
| Instruction::I16x8MinU
| Instruction::I16x8MaxS
| Instruction::I16x8MaxU
| Instruction::I16x8AvgrU
| Instruction::I16x8ExtmulLowI8x16S
| Instruction::I16x8ExtmulHighI8x16S
| Instruction::I16x8ExtmulLowI8x16U
| Instruction::I16x8ExtmulHighI8x16U
| Instruction::I32x4Abs
| Instruction::I32x4Neg
| Instruction::I32x4AllTrue
| Instruction::I32x4Bitmask
| Instruction::I32x4ExtendLowI16x8S
| Instruction::I32x4ExtendHighI16x8S
| Instruction::I32x4ExtendLowI16x8U
| Instruction::I32x4ExtendHighI16x8U
| Instruction::I32x4Shl
| Instruction::I32x4ShrS
| Instruction::I32x4ShrU
| Instruction::I32x4Add
| Instruction::I32x4Sub
| Instruction::I32x4Mul
| Instruction::I32x4MinS
| Instruction::I32x4MinU
| Instruction::I32x4MaxS
| Instruction::I32x4MaxU
| Instruction::I32x4DotI16x8S
| Instruction::I32x4ExtmulLowI16x8S
| Instruction::I32x4ExtmulHighI16x8S
| Instruction::I32x4ExtmulLowI16x8U
| Instruction::I32x4ExtmulHighI16x8U
| Instruction::I64x2Abs
| Instruction::I64x2Neg
| Instruction::I64x2AllTrue
| Instruction::I64x2Bitmask
| Instruction::I64x2ExtendLowI32x4S
| Instruction::I64x2ExtendHighI32x4S
| Instruction::I64x2ExtendLowI32x4U
| Instruction::I64x2ExtendHighI32x4U
| Instruction::I64x2Shl
| Instruction::I64x2ShrS
| Instruction::I64x2ShrU
| Instruction::I64x2Add
| Instruction::I64x2Sub
| Instruction::I64x2Mul
| Instruction::I64x2Eq
| Instruction::I64x2Ne
| Instruction::I64x2LtS
| Instruction::I64x2GtS
| Instruction::I64x2LeS
| Instruction::I64x2GeS
| Instruction::I64x2ExtmulLowI32x4S
| Instruction::I64x2ExtmulHighI32x4S
| Instruction::I64x2ExtmulLowI32x4U
| Instruction::I64x2ExtmulHighI32x4U
| Instruction::F32x4Abs
| Instruction::F32x4Neg
| Instruction::F32x4Sqrt
| Instruction::F32x4Add
| Instruction::F32x4Sub
| Instruction::F32x4Mul
| Instruction::F32x4Div
| Instruction::F32x4Min
| Instruction::F32x4Max
| Instruction::F32x4Pmin
| Instruction::F32x4Pmax
| Instruction::F64x2Abs
| Instruction::F64x2Neg
| Instruction::F64x2Sqrt
| Instruction::F64x2Add
| Instruction::F64x2Sub
| Instruction::F64x2Mul
| Instruction::F64x2Div
| Instruction::F64x2Min
| Instruction::F64x2Max
| Instruction::F64x2Pmin
| Instruction::F64x2Pmax
| Instruction::I32x4TruncSatF32x4S
| Instruction::I32x4TruncSatF32x4U
| Instruction::F32x4ConvertI32x4S
| Instruction::F32x4ConvertI32x4U
| Instruction::I32x4TruncSatF64x2SZero
| Instruction::I32x4TruncSatF64x2UZero
| Instruction::F64x2ConvertLowI32x4S
| Instruction::F64x2ConvertLowI32x4U
| Instruction::I8x16RelaxedSwizzle
| Instruction::I32x4RelaxedTruncF32x4S
| Instruction::I32x4RelaxedTruncF32x4U
| Instruction::I32x4RelaxedTruncZeroF64x2S
| Instruction::I32x4RelaxedTruncZeroF64x2U
| Instruction::F32x4RelaxedMadd
| Instruction::F32x4RelaxedNmadd
| Instruction::F64x2RelaxedMadd
| Instruction::F64x2RelaxedNmadd
| Instruction::I8x16RelaxedLaneselect
| Instruction::I16x8RelaxedLaneselect
| Instruction::I32x4RelaxedLaneselect
| Instruction::I64x2RelaxedLaneselect
| Instruction::F32x4RelaxedMin
| Instruction::F32x4RelaxedMax
| Instruction::F64x2RelaxedMin
| Instruction::F64x2RelaxedMax
| Instruction::I16x8RelaxedQ15mulrS
| Instruction::I16x8RelaxedDotI8x16I7x16S
| Instruction::I32x4RelaxedDotI8x16I7x16AddS
| Instruction::V128Not
| Instruction::V128And
| Instruction::V128Andnot
| Instruction::V128Or
| Instruction::V128Xor
| Instruction::V128Bitselect
| Instruction::V128AnyTrue
| Instruction::V128Load8Lane(_, _)
| Instruction::V128Load16Lane(_, _)
| Instruction::V128Load32Lane(_, _)
| Instruction::V128Load64Lane(_, _)
| Instruction::V128Store8Lane(_, _)
| Instruction::V128Store16Lane(_, _)
| Instruction::V128Store32Lane(_, _)
| Instruction::V128Store64Lane(_, _)
| Instruction::V128Load32Zero(_)
| Instruction::V128Load64Zero(_)
| Instruction::F32x4DemoteF64x2Zero
| Instruction::F64x2PromoteLowF32x4 => Some(HotOp::Simd)
_ => None
}
}
///|
// Immediate build metadata is four scalar fields; it never needs heap identity.
#valtype
priv struct HotLiftNodeShape {
op : HotOp
ty : TypeId
imm0 : Int32
imm1 : Int32
}
///|
fn HotLiftNodeShape::new(
op : HotOp,
ty : TypeId,
imm0 : Int32,
imm1 : Int32,
) -> HotLiftNodeShape {
{ op, ty, imm0, imm1, }
}
///|
fn hot_lift_impl_direct_node_shape(
func : HotFunc,
op : HotOp,
instr : Instruction,
result_types : Array[ValType],
) -> HotLiftNodeShape {
let ty = hot_lift_impl_result_type(func, result_types)
match instr {
Instruction::LocalGet(LocalIdx(raw))
| Instruction::LocalSet(LocalIdx(raw))
| Instruction::LocalTee(LocalIdx(raw)) =>
HotLiftNodeShape::new(op, ty, raw.reinterpret_as_int(), 0)
Instruction::GlobalGet(GlobalIdx(raw))
| Instruction::GlobalSet(GlobalIdx(raw)) =>
HotLiftNodeShape::new(op, ty, raw.reinterpret_as_int(), 0)
Instruction::TableGet(TableIdx(raw))
| Instruction::TableSet(TableIdx(raw))
| Instruction::TableSize(TableIdx(raw))
| Instruction::TableGrow(TableIdx(raw))
| Instruction::TableFill(TableIdx(raw)) =>
HotLiftNodeShape::new(op, ty, raw.reinterpret_as_int(), 0)
Instruction::TableCopy(TableIdx(dst), TableIdx(src)) =>
HotLiftNodeShape::new(
op,
ty,
dst.reinterpret_as_int(),
src.reinterpret_as_int(),
)
Instruction::TableInit(ElemIdx(elem), TableIdx(table)) =>
HotLiftNodeShape::new(
op,
ty,
elem.reinterpret_as_int(),
table.reinterpret_as_int(),
)
Instruction::ElemDrop(ElemIdx(raw)) =>
HotLiftNodeShape::new(op, ty, raw.reinterpret_as_int(), 0)
Instruction::MemorySize(MemIdx(raw))
| Instruction::MemoryGrow(MemIdx(raw))
| Instruction::MemoryFill(MemIdx(raw)) =>
HotLiftNodeShape::new(op, ty, raw.reinterpret_as_int(), 0)
Instruction::MemoryCopy(MemIdx(dst), MemIdx(src)) =>
HotLiftNodeShape::new(
op,
ty,
dst.reinterpret_as_int(),
src.reinterpret_as_int(),
)
Instruction::MemoryInit(DataIdx(data), MemIdx(mem)) =>
HotLiftNodeShape::new(
op,
ty,
data.reinterpret_as_int(),
mem.reinterpret_as_int(),
)
Instruction::DataDrop(DataIdx(raw)) =>
HotLiftNodeShape::new(op, ty, raw.reinterpret_as_int(), 0)
Instruction::Call(FuncIdx(raw)) | Instruction::ReturnCall(FuncIdx(raw)) =>
HotLiftNodeShape::new(op, ty, raw.reinterpret_as_int(), 0)
Instruction::CallIndirect(type_idx, table_idx)
| Instruction::ReturnCallIndirect(type_idx, table_idx) => {
let sig_id = hot_alloc_call_sig(
func,
HotCallSig::new(type_idx, table_idx=Some(table_idx)),
)
HotLiftNodeShape::new(op, ty, sig_id, 0)
}
Instruction::CallRef(type_idx) | Instruction::ReturnCallRef(type_idx) => {
let sig_id = hot_alloc_call_sig(func, HotCallSig::new(type_idx))
HotLiftNodeShape::new(op, ty, sig_id, 0)
}
Instruction::I32Const(v) => {
let const_id = hot_alloc_const(func, HotConstPayload::i32_const(v))
HotLiftNodeShape::new(HotOp::Const, ty, const_id, 0)
}
Instruction::I64Const(v) => {
let const_id = hot_alloc_const(func, HotConstPayload::i64_const(v))
HotLiftNodeShape::new(HotOp::Const, ty, const_id, 0)
}
Instruction::F32Const(v) => {
let const_id = hot_alloc_const(func, HotConstPayload::f32_const(v))
HotLiftNodeShape::new(HotOp::Const, ty, const_id, 0)
}
Instruction::F64Const(v) => {
let const_id = hot_alloc_const(func, HotConstPayload::f64_const(v))
HotLiftNodeShape::new(HotOp::Const, ty, const_id, 0)
}
Instruction::RefNull(rt) => {
let const_id = hot_alloc_const(func, HotConstPayload::ref_null_const(rt))
HotLiftNodeShape::new(HotOp::RefNull, ty, const_id, 0)
}
Instruction::StringConst(bytes) => {
let const_id = hot_alloc_const(func, HotConstPayload::string_const(bytes))
HotLiftNodeShape::new(HotOp::Const, ty, const_id, 0)
}
Instruction::RefFunc(FuncIdx(raw)) =>
HotLiftNodeShape::new(op, ty, raw.reinterpret_as_int(), 0)
Instruction::Throw(TagIdx(raw)) =>
HotLiftNodeShape::new(op, ty, raw.reinterpret_as_int(), 0)
_ => HotLiftNodeShape::new(op, ty, 0, 0)
}
}
///|
fn hot_lift_impl_build_direct_node(
func : HotFunc,
op : HotOp,
instr : Instruction,
result_types : Array[ValType],
children : Array[NodeId],
) -> NodeId {
let { op: build_op, ty, imm0, imm1, } = hot_lift_impl_direct_node_shape(
func, op, instr, result_types,
)
hot_build_node(func, build_op, ty, imm0~, imm1~, children~)
}
///|
fn hot_lift_impl_build_direct_node1(
func : HotFunc,
op : HotOp,
instr : Instruction,
result_types : Array[ValType],
child0 : NodeId,
) -> NodeId {
let { op: build_op, ty, imm0, imm1, } = hot_lift_impl_direct_node_shape(
func, op, instr, result_types,
)
hot_build_node1(func, build_op, ty, child0, imm0~, imm1~)
}
///|
fn hot_lift_impl_build_direct_node2(
func : HotFunc,
op : HotOp,
instr : Instruction,
result_types : Array[ValType],
child0 : NodeId,
child1 : NodeId,
) -> NodeId {
let { op: build_op, ty, imm0, imm1, } = hot_lift_impl_direct_node_shape(
func, op, instr, result_types,
)
hot_build_node2(func, build_op, ty, child0, child1, imm0~, imm1~)
}
///|
fn hot_lift_impl_build_direct_node3(
func : HotFunc,
op : HotOp,
instr : Instruction,
result_types : Array[ValType],
child0 : NodeId,
child1 : NodeId,
child2 : NodeId,
) -> NodeId {
let { op: build_op, ty, imm0, imm1, } = hot_lift_impl_direct_node_shape(
func, op, instr, result_types,
)
hot_build_node3(func, build_op, ty, child0, child1, child2, imm0~, imm1~)
}
///|
fn hot_lift_impl_exact_node_shape(
func : HotFunc,
op : HotOp,
instr : Instruction,
result_types : Array[ValType],
) -> HotLiftNodeShape {
let ty = hot_lift_impl_result_type(func, result_types)
let payload_id = hot_alloc_instr_payload(func, HotInstrPayload::new(instr))
match instr {
Instruction::I32Load(memarg)
| Instruction::I64Load(memarg)
| Instruction::F32Load(memarg)
| Instruction::F64Load(memarg)
| Instruction::I32Load8S(memarg)
| Instruction::I32Load8U(memarg)
| Instruction::I32Load16S(memarg)
| Instruction::I32Load16U(memarg)
| Instruction::I64Load8S(memarg)
| Instruction::I64Load8U(memarg)
| Instruction::I64Load16S(memarg)
| Instruction::I64Load16U(memarg)
| Instruction::I64Load32S(memarg)
| Instruction::I64Load32U(memarg)
| Instruction::I32Store(memarg)
| Instruction::I64Store(memarg)
| Instruction::F32Store(memarg)
| Instruction::F64Store(memarg)
| Instruction::I32Store8(memarg)
| Instruction::I32Store16(memarg)
| Instruction::I64Store8(memarg)
| Instruction::I64Store16(memarg)
| Instruction::I64Store32(memarg) => {
let memarg_id = hot_alloc_memarg(func, HotMemArg::new(memarg))
HotLiftNodeShape::new(op, ty, memarg_id, payload_id)
}
Instruction::AtomicFence(_)
| Instruction::WaitqueueNotify
| Instruction::StructWait(_, _)
| Instruction::StructAtomicSet(_, _, _)
| Instruction::ArrayAtomicSet(_, _)
| Instruction::StructAtomicGet(_, _, _)
| Instruction::StructAtomicGetS(_, _, _)
| Instruction::StructAtomicGetU(_, _, _)
| Instruction::StructAtomicRmw(_, _, _, _)
| Instruction::StructAtomicCmpxchg(_, _, _)
| Instruction::ArrayAtomicGet(_, _)
| Instruction::ArrayAtomicGetS(_, _)
| Instruction::ArrayAtomicGetU(_, _)
| Instruction::ArrayAtomicRmw(_, _, _)
| Instruction::ArrayAtomicCmpxchg(_, _) =>
HotLiftNodeShape::new(op, ty, 0, payload_id)
Instruction::MemoryAtomicNotify(memarg)
| Instruction::MemoryAtomicWait32(memarg)
| Instruction::MemoryAtomicWait64(memarg)
| Instruction::I32AtomicLoad(_, memarg)
| Instruction::I64AtomicLoad(_, memarg)
| Instruction::I32AtomicLoad8U(_, memarg)
| Instruction::I32AtomicLoad16U(_, memarg)
| Instruction::I64AtomicLoad8U(_, memarg)
| Instruction::I64AtomicLoad16U(_, memarg)
| Instruction::I64AtomicLoad32U(_, memarg)
| Instruction::I32AtomicStore(_, memarg)
| Instruction::I64AtomicStore(_, memarg)
| Instruction::I32AtomicStore8(_, memarg)
| Instruction::I32AtomicStore16(_, memarg)
| Instruction::I64AtomicStore8(_, memarg)
| Instruction::I64AtomicStore16(_, memarg)
| Instruction::I64AtomicStore32(_, memarg)
| Instruction::AtomicRmw(_, _, memarg)
| Instruction::AtomicCmpxchg(_, _, memarg) => {
let memarg_id = hot_alloc_memarg(func, HotMemArg::new(memarg))
HotLiftNodeShape::new(op, ty, memarg_id, payload_id)
}
Instruction::RefTest(_, _, _)
| Instruction::RefCast(_, _, _)
| Instruction::RefTestDesc(_, _)
| Instruction::RefCastDescEq(_, _, _)
| Instruction::BrOnCast(_, _, _, _)
| Instruction::BrOnCastFail(_, _, _, _)
| Instruction::BrOnCastDescEq(_, _, _)
| Instruction::BrOnCastDescEqFail(_, _, _) =>
HotLiftNodeShape::new(op, ty, 0, payload_id)
_ => HotLiftNodeShape::new(op, ty, payload_id, 0)
}
}
///|
fn hot_lift_impl_build_exact_node(
func : HotFunc,
op : HotOp,
instr : Instruction,
result_types : Array[ValType],
children : Array[NodeId],
) -> NodeId {
let { op: build_op, ty, imm0, imm1, } = hot_lift_impl_exact_node_shape(
func, op, instr, result_types,
)
hot_build_node(func, build_op, ty, imm0~, imm1~, children~)
}
///|
fn hot_lift_impl_build_exact_node1(
func : HotFunc,
op : HotOp,
instr : Instruction,
result_types : Array[ValType],
child0 : NodeId,
) -> NodeId {
let { op: build_op, ty, imm0, imm1, } = hot_lift_impl_exact_node_shape(
func, op, instr, result_types,
)
hot_build_node1(func, build_op, ty, child0, imm0~, imm1~)
}
///|
fn hot_lift_impl_build_exact_node2(
func : HotFunc,
op : HotOp,
instr : Instruction,
result_types : Array[ValType],
child0 : NodeId,
child1 : NodeId,
) -> NodeId {
let { op: build_op, ty, imm0, imm1, } = hot_lift_impl_exact_node_shape(
func, op, instr, result_types,
)
hot_build_node2(func, build_op, ty, child0, child1, imm0~, imm1~)
}
///|
fn hot_lift_impl_build_exact_node3(
func : HotFunc,
op : HotOp,
instr : Instruction,
result_types : Array[ValType],
child0 : NodeId,
child1 : NodeId,
child2 : NodeId,
) -> NodeId {
let { op: build_op, ty, imm0, imm1, } = hot_lift_impl_exact_node_shape(
func, op, instr, result_types,
)
hot_build_node3(func, build_op, ty, child0, child1, child2, imm0~, imm1~)
}
///|
#inline
fn hot_lift_impl_merge_local_extent(left : Byte, right : Byte) -> Byte {
if left == (1).to_byte() {
return right
}
if right == (1).to_byte() || left == right {
return left
}
let lhs = left.to_int()
let rhs = right.to_int()
if lhs == 255 || rhs == 255 {
return (255).to_byte()
}
let left_band = if lhs < 130 { lhs - 2 } else { lhs - 130 }
let right_band = if rhs < 130 { rhs - 2 } else { rhs - 130 }
let max_band = if left_band > right_band { left_band } else { right_band }
if max_band <= 124 {
(max_band + 130).to_byte()
} else {
(255).to_byte()
}
}
///|
fn hot_lift_impl_collect_local_access_masks(
func : HotFunc,
node_id : NodeId,
masks : HotLiftLocalAccessMasks,
node_visits : Array[Int32]?,
) -> Unit {
while masks.extent.length() <= node_id {
masks.reads.push(0UL)
masks.writes.push(0UL)
masks.extent.push((0).to_byte())
}
if masks.extent[node_id] != (0).to_byte() {
return
}
match node_visits {
Some(count) => count[0] += 1
None => ()
}
// Preserve checked admission while projecting the existing arena record
// here, without a complete-node return boundary or a new view allocation.
hot_node_require_live_read(func, node_id)
let node = func.nodes[node_id]
let mut reads = 0UL
let mut writes = 0UL
let mut extent = (1).to_byte()
match node.op {
HotOp::LocalGet => reads = 1UL << (node.imm0 & 63)
HotOp::LocalSet | HotOp::LocalTee => writes = 1UL << (node.imm0 & 63)
_ => ()
}
match node.op {
HotOp::LocalGet | HotOp::LocalSet | HotOp::LocalTee => {
let band = node.imm0 >> 6
extent = if node.imm0 >= 0 && band <= 127 {
(band + 2).to_byte()
} else {
(255).to_byte()
}
}
_ => ()
}
for slot = 0; slot < node.child_count; slot = slot + 1 {
let child = hot_child_get(func, node_id, slot)
if child >= 0 && hot_node_is_live(func, child) {
hot_lift_impl_collect_local_access_masks(func, child, masks, node_visits)
reads = reads | masks.reads[child]
writes = writes | masks.writes[child]
extent = hot_lift_impl_merge_local_extent(extent, masks.extent[child])
}
}
masks.reads[node_id] = reads
masks.writes[node_id] = writes
masks.extent[node_id] = extent
}
///|
// Preserve both directions of a pending stack value's local dependency. A
// later write must not change its reads; a later read must see its writes.
// Explicit captures make these edges visible to every tree transform.
fn hot_lift_impl_value_conflicts_with_local_access(
func : HotFunc,
node_id : NodeId,
local_id : Int32,
is_write : Bool,
masks~ : HotLiftLocalAccessMasks? = None,
node_visits~ : Array[Int32]? = None,
) -> Bool {
if node_id < 0 || !hot_node_is_live(func, node_id) {
return false
}
match node_visits {
Some(count) => count[0] += 1
None => ()
}
// Preserve checked admission while projecting the existing arena record
// here, without a complete-node return boundary or a new view allocation.
hot_node_require_live_read(func, node_id)
let node = func.nodes[node_id]
if node.imm0 == local_id {
match node.op {
HotOp::LocalSet | HotOp::LocalTee => return true
HotOp::LocalGet if is_write => return true
_ => ()
}
}
if node.child_count == 0 {
return false
}
match masks {
Some(masks) => {
hot_lift_impl_collect_local_access_masks(
func, node_id, masks, node_visits,
)
let extent = masks.extent[node_id].to_int()
if extent < 130 {
if extent == 1 || local_id < 0 || local_id >> 6 != extent - 2 {
return false
}
} else if extent < 255 && local_id >> 6 > extent - 130 {
return false
}
let mask = if is_write {
masks.reads[node_id] | masks.writes[node_id]
} else {
masks.writes[node_id]
}
if (mask & (1UL << (local_id & 63))) == 0UL {
return false
}
// All accesses in one band have unique low-six-bit local identities.
if extent < 130 {
return true
}
}
None => ()
}
// A mixed-band mask can collide through a shared operand DAG. Summaries
// already belong to immutable completed values; memoize the exact answer for
// each node/local/direction rather than expanding that DAG once per path.
let local_key = local_id.reinterpret_as_uint().to_uint64()
let query_bit = if is_write { 4UL } else { 1UL }
if masks is Some(masks) {
if masks.exact_conflicts is Some(rows) && node_id < rows.length() {
let row = rows[node_id]
if row >> 8 == local_key && (row & query_bit) != 0UL {
return (row & (query_bit << 1)) != 0UL
}
}
}
let mut conflict = false
for slot = 0; slot < node.child_count; slot = slot + 1 {
if hot_lift_impl_value_conflicts_with_local_access(
func,
hot_child_get(func, node_id, slot),
local_id,
is_write,
masks~,
node_visits~,
) {
conflict = true
break
}
}
if masks is Some(masks) {
let rows = match masks.exact_conflicts {
Some(rows) => rows
None => {
let rows = Array::make(masks.extent.length(), 0UL)
masks.exact_conflicts = Some(rows)
rows
}
}
while rows.length() <= node_id {
rows.push(0UL)
}
let previous = rows[node_id]
let flags = if previous >> 8 == local_key { previous & 15UL } else { 0UL }
rows[node_id] = (local_key << 8) |
flags |
query_bit |
(if conflict { query_bit << 1 } else { 0UL })
}
conflict
}
///|
fn hot_lift_impl_value_contains_node(
func : HotFunc,
node_id : NodeId,
target : NodeId,
) -> Bool {
if node_id == target {
return true
}
// During lifting, operands are created before their consumers.
if node_id < target {
return false
}
for slot = 0; slot < hot_child_count(func, node_id); slot = slot + 1 {
if hot_lift_impl_value_contains_node(
func,
hot_child_get(func, node_id, slot),
target,
) {
return true
}
}
false
}
///|
fn hot_lift_impl_capture_before_local_access(
func : HotFunc,
roots : Array[NodeId],
values : Array[NodeId],
concrete_values : Array[ValType],
local_id : Int32,
is_write : Bool,
operand_count : Int32,
label_frames : HotLiftLabelFrames,
) -> Unit {
let pending_count = values.length() - operand_count
let mut last_conflict = -1
for idx = 0; idx < pending_count; idx = idx + 1 {
let value_id = values[idx]
if label_frames.local_access_masks is None &&
value_id >= 0 &&
hot_node_is_live(func, value_id) &&
hot_child_count(func, value_id) > 0 {
label_frames.local_access_masks = Some(HotLiftLocalAccessMasks::new())
}
if hot_lift_impl_value_conflicts_with_local_access(
func,
value_id,
local_id,
is_write,
masks=label_frames.local_access_masks,
) &&
!roots.any(fn(root) {
hot_lift_impl_value_contains_node(func, root, value_id)
}) {
last_conflict = idx
}
}
if last_conflict < 0 {
return
}
let mut prefix_end = last_conflict + 1
while prefix_end < values.length() &&
values[prefix_end] == values[last_conflict] {
prefix_end += 1
}
// Capture the complete preceding stack prefix in evaluation order. An
// earlier call can throw into a handler that observes this local, even if
// that call's own operands do not mention the local being overwritten.
let mut idx = 0
while idx < prefix_end {
let value_id = values[idx]
let mut end = idx + 1
while end < prefix_end && values[end] == value_id {
end += 1
}
if !hot_lift_impl_is_stable_capture(
func,
value_id,
label_frames.local_count,
) &&
!roots.any(fn(root) {
hot_lift_impl_value_contains_node(func, root, value_id)
}) {
// A consumed tuple lane already anchors its producer. Capture remaining
// producers once, consuming each producer's lanes in stack-pop order.
for slot = end - 1; slot >= idx; slot = slot - 1 {
let original_type = concrete_values[slot]
let storage_type = match original_type {
@lib.RefTypeValType(ref_type) =>
@lib.ValType::ref_type(ref_type.make_nullable())
_ => original_type
}
let capture_local = hot_append_body_local(func, storage_type)
let write = hot_build_local_set(func, capture_local, value_id)
roots.push(write)
let read = hot_build_local_get(func, capture_local)
match label_frames.capture_sources {
Some(pairs) => pairs.push((read, write))
None => ()
}
values[slot] = match original_type {
@lib.RefTypeValType(ref_type) if ref_type.is_non_nullable() => {
let checked = hot_build_unary(func, original_type, read)
hot_node_exact_instr_set(
func,
checked,
@lib.Instruction::ref_as_non_null(),
)
checked
}
_ => read
}
}
}
idx = end
}
}
///|
// Original instructions cannot access body locals appended by this lift. Once
// captured, their reads remain stable across all later source instructions.
fn hot_lift_impl_is_stable_capture(
func : HotFunc,
node_id : NodeId,
source_local_count : Int32,
) -> Bool {
let node = hot_node_get(func, node_id)
if node.op == HotOp::LocalGet {
return node.imm0 >= source_local_count
}
if node.op == HotOp::Unary &&
node.child_count == 1 &&
hot_node_exact_instr(func, node_id) is Some(@lib.RefAsNonNull) {
let read = hot_node_get(func, hot_child_get(func, node_id, 0))
return read.op == HotOp::LocalGet && read.imm0 >= source_local_count
}
false
}
///|
fn hot_lift_impl_push_node(
roots : Array[NodeId],
values : Array[NodeId],
concrete_values : Array[ValType],
node_id : NodeId,
result_types : Array[ValType],
) -> Unit {
if result_types.length() == 0 {
roots.push(node_id)
} else {
for ty in result_types {
values.push(node_id)
concrete_values.push(ty)
}
}
}
///|
#valtype
// Immediate state/error handoff for both typechecking and instruction lifting.
// Only failure needs an optional error; successful steps do not allocate a Result.
priv struct HotLiftTypecheck {
state : TcState
error : HotLiftError?
}
///|
fn HotLiftTypecheck::success(state : TcState) -> HotLiftTypecheck {
{ state, error: None, }
}
///|
fn HotLiftTypecheck::failure(
state : TcState,
error : HotLiftError,
) -> HotLiftTypecheck {
{ state, error: Some(error), }
}
///|
// Immediate private result; success does not re-box the validator's state.
fn hot_lift_impl_check_instruction(
instr : Instruction,
tc_state : TcState,
) -> HotLiftTypecheck {
match Typecheck::typecheck(instr, tc_state) {
Ok(next) => HotLiftTypecheck::success(next)
Err(err) =>
HotLiftTypecheck::failure(
tc_state,
hot_lift_impl_fail_typecheck(instr, err),
)
}
}
///|
fn hot_lift_impl_invalid_body_state(err : String) -> HotLiftError {
HotLiftError::InvalidBodyState(err)
}
///|
fn hot_lift_impl_region_state(
entry_state : TcState,
region_env : Env,
entry_types : Array[ValType],
) -> TcState {
@validate.tc_state_fork_body(entry_state, region_env, entry_types)
}
///|
fn hot_lift_impl_copy_node_ids(values : Array[NodeId]) -> Array[NodeId] {
if values.length() == 0 {
return []
}
values.copy()
}
///|
fn hot_lift_impl_copy_val_types(types : Array[ValType]) -> Array[ValType] {
if types.length() == 0 {
return []
}
types.copy()
}
///|
fn hot_lift_impl_pushed_types(
before_types : Array[ValType],
after_types : Array[ValType],
pop_count : Int32,
) -> Array[ValType] {
let prefix_len = before_types.length() - pop_count
let out : Array[ValType] = []
for i = prefix_len; i < after_types.length(); i = i + 1 {
out.push(after_types[i])
}
out
}
///|
// Private immediate result: retain owned state/types while avoiding a tuple and
// outer success box for every data instruction. Error details remain unchanged.
#valtype
priv struct HotLiftDataCheck {
state : TcState
pop_count : Int32
pushed : Array[ValType]
error : HotLiftError?
}
///|
fn HotLiftDataCheck::success(
state : TcState,
pop_count : Int32,
pushed : Array[ValType],
) -> HotLiftDataCheck {
{ state, pop_count, pushed, error: None, }
}
///|
fn HotLiftDataCheck::failure(
state : TcState,
error : HotLiftError,
) -> HotLiftDataCheck {
{ state, pop_count: 0, pushed: [], error: Some(error), }
}
///|
fn hot_lift_impl_typecheck_data_instruction(
instr : Instruction,
tc_state : TcState,
concrete_before_types~ : Array[ValType] = [],
scalar_results~ : Array[ValType]? = None,
) -> HotLiftDataCheck {
// The optional row is disjoint from state/signature storage. Clear it even
// on failure; a borrowed result must be consumed before the next call.
if scalar_results is Some(row) {
row.clear()
}
// The region owns this stack. Known arity only needs its old length, so the
// ordinary typechecker can consume it directly. Generic inference below
// retains its snapshot because it replays suffix candidates.
let before_count = tc_state.stack.length()
match hot_lift_impl_known_pop_count(instr, tc_state.env) {
Some(count) if count >= 0 && count <= before_count => {
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftDataCheck::failure(tc_state, err)
}
let after_state = checked.state
let pushed : Array[ValType] = match scalar_results {
Some(row) if after_state.stack.length() - (before_count - count) <= 1 =>
row
_ => []
}
for index = before_count - count
index < after_state.stack.length()
index = index + 1 {
pushed.push(after_state.stack[index])
}
return HotLiftDataCheck::success(after_state, count, pushed)
}
_ => ()
}
let before_types = tc_state.stack
let before_state = tc_state
let typecheck_state = tc_state_with_stack(tc_state, before_types)
let checked = hot_lift_impl_check_instruction(instr, typecheck_state)
if checked.error is Some(err) {
return HotLiftDataCheck::failure(tc_state, err)
}
let after_state = checked.state
match hot_lift_impl_known_pop_count(instr, tc_state.env) {
Some(count) if count <= before_types.length() =>
HotLiftDataCheck::success(
after_state,
count,
hot_lift_impl_pushed_types(before_types, after_state.stack, count),
)
_ =>
match
hot_lift_impl_resolve_generic_pop_count(
instr, before_state, after_state, concrete_before_types,
) {
Ok((child_count, pushed_types)) =>
HotLiftDataCheck::success(after_state, child_count, pushed_types)
Err(err) => HotLiftDataCheck::failure(tc_state, err)
}
}
}
///|
fn hot_lift_impl_lift_region(
func : HotFunc,
expr : Expr,
tc_state : TcState,
label_frames : HotLiftLabelFrames,
label_stack_head : Int32,
initial_values~ : Array[NodeId] = [],
initial_value_types~ : Array[ValType] = [],
) -> Result[(Array[NodeId], Array[NodeId], TcState), HotLiftError] {
let Expr(instrs) = expr
// Entry stacks can share a tag signature or an arm's saved entry types.
// Take ownership once per region before consuming data instructions in place.
let mut tc_state = tc_state_with_stack(tc_state, tc_state.stack)
let roots : Array[NodeId] = []
let values : Array[NodeId] = hot_lift_impl_copy_node_ids(initial_values)
let concrete_values : Array[ValType] = hot_lift_impl_copy_val_types(
initial_value_types,
)
let mut materialized_prefix = 0
for instr in instrs {
let old_root_count = roots.length()
let old_node_count = hot_node_count(func)
let lifted = hot_lift_impl_lift_instruction(
func, instr, tc_state, label_frames, label_stack_head, roots, values, concrete_values,
)
match lifted.error {
Some(err) => return Err(err)
None => tc_state = lifted.state
}
// Local operations already capture their conflicting prefix. Moving a
// newly created scratch read ahead of those captures would read its default.
let local_capture = instr is Instruction::LocalGet(_) ||
instr is Instruction::LocalSet(_) ||
instr is Instruction::LocalTee(_)
if label_frames.capture_stack_locals &&
!local_capture &&
roots.length() > old_root_count {
let mut last = -1
for slot = 0; slot < values.length(); slot = slot + 1 {
if values[slot] < old_node_count &&
hot_lift_impl_pending_observable(
func,
values[slot],
label_frames.pending_effects,
0,
) {
last = slot
}
}
if last >= 0 {
while last + 1 < values.length() && values[last + 1] == values[last] {
last += 1
}
let following : Array[NodeId] = []
for slot = old_root_count; slot < roots.length(); slot = slot + 1 {
following.push(roots[slot])
}
while roots.length() > old_root_count {
ignore(roots.pop())
}
let mut slot = 0
while slot <= last {
let producer = values[slot]
let mut end = slot + 1
while end <= last && values[end] == producer {
end += 1
}
// Generated capture reads already have a source-positioned store.
// Keep those stores and capture only the remaining pending producer.
if !hot_lift_impl_is_stable_capture(
func,
producer,
label_frames.local_count,
) {
for lane = end - 1; lane >= slot; lane = lane - 1 {
let scratch = hot_append_body_local(func, concrete_values[lane])
let write = hot_build_local_set(func, scratch, producer)
roots.push(write)
let read = hot_build_local_get(func, scratch)
match label_frames.capture_sources {
Some(pairs) => pairs.push((read, write))
None => ()
}
values[lane] = read
}
}
slot = end
}
roots.append(following)
}
}
if label_frames.materialize_multi_values {
// Instructions pop and replace a stack suffix. Repair only the boundary
// of the known stable prefix; do not rescan all earlier captured lanes.
materialized_prefix = materialized_prefix.min(values.length())
while materialized_prefix > 0 &&
!hot_lift_impl_is_stable_capture(
func,
values[materialized_prefix - 1],
label_frames.local_count,
) {
materialized_prefix -= 1
}
if values.length() > 1 &&
values[values.length() - 1] == values[values.length() - 2] {
let mut slot = materialized_prefix
while slot < values.length() {
let producer = values[slot]
let mut end = slot + 1
while end < values.length() && values[end] == producer {
end += 1
}
if !hot_lift_impl_is_stable_capture(
func,
producer,
label_frames.local_count,
) {
for lane = end - 1; lane >= slot; lane = lane - 1 {
let original_type = concrete_values[lane]
let storage_type = match original_type {
@lib.RefTypeValType(ref_type) =>
@lib.ValType::ref_type(ref_type.make_nullable())
_ => original_type
}
let scratch = hot_append_body_local(func, storage_type)
let write = hot_build_local_set(func, scratch, producer)
roots.push(write)
let read = hot_build_local_get(func, scratch)
match label_frames.capture_sources {
Some(pairs) => pairs.push((read, write))
None => ()
}
values[lane] = match original_type {
@lib.RefTypeValType(ref_type) if ref_type.is_non_nullable() => {
let checked = hot_build_unary(func, original_type, read)
hot_node_exact_instr_set(
func,
checked,
@lib.Instruction::ref_as_non_null(),
)
checked
}
_ => read
}
}
}
slot = end
}
materialized_prefix = values.length()
}
}
}
// A region result can remain on the operand stack across later effects.
// It has no later consumer to anchor that lifetime during lowering, so
// retain its source position explicitly among the region roots.
let mut maximum_root = -1
for root in roots {
maximum_root = maximum_root.max(root)
}
let needs_effect_orders = tc_state.reachable &&
values.any(fn(value) {
value < maximum_root &&
!hot_lift_impl_is_stable_capture(func, value, label_frames.local_count)
})
for value in values {
let mut insert_at = roots.length()
if needs_effect_orders &&
!hot_lift_impl_is_stable_capture(func, value, label_frames.local_count) &&
hot_node_result_arity(func, value) > 0 &&
!roots.any(fn(root) {
hot_lift_impl_value_contains_node(func, root, value)
}) {
for idx = 0; idx < roots.length(); idx = idx + 1 {
let root = roots[idx]
let order = hot_lift_impl_first_effect_order(
func,
root,
label_frames.first_effect_orders,
)
if order < hot_node_count(func) &&
order > hot_node_get(func, value).order {
insert_at = idx
break
}
}
}
roots.insert(insert_at, value)
}
Ok((roots, values, tc_state))
}
///|
// Fully validated dead bodies may have virtual inputs and no HOT region. Keep
// source-local slots in source order even when none of their nodes materialize.
fn hot_lift_impl_reserve_elided_instruction_accesses(
instr : Instruction,
frames : HotLiftLabelFrames,
) -> Unit {
match instr {
Instruction::LocalGet(LocalIdx(index)) =>
frames.gets.push((index.reinterpret_as_int(), -1))
Instruction::LocalSet(LocalIdx(index))
| Instruction::LocalTee(LocalIdx(index)) =>
frames.writes.push((index.reinterpret_as_int(), -1))
Instruction::Block(_, Expr(nested))
| Instruction::Loop(_, Expr(nested))
| Instruction::TryTable(_, _, Expr(nested)) =>
hot_lift_impl_reserve_elided_accesses(nested, frames)
Instruction::If(_, yes, no) => {
hot_lift_impl_reserve_elided_accesses(yes, frames)
match no {
Some(nested) => hot_lift_impl_reserve_elided_accesses(nested, frames)
None => ()
}
}
Instruction::Try(_, Expr(nested), catches, _) => {
hot_lift_impl_reserve_elided_accesses(nested, frames)
for catch_ in catches {
match catch_ {
@lib.LegacyCatch(_, Expr(nested))
| @lib.LegacyCatchAll(Expr(nested)) =>
hot_lift_impl_reserve_elided_accesses(nested, frames)
}
}
}
_ => ()
}
}
///|
fn hot_lift_impl_reserve_elided_accesses(
body : Array[Instruction],
frames : HotLiftLabelFrames,
) -> Unit {
for instr in body {
hot_lift_impl_reserve_elided_instruction_accesses(instr, frames)
}
}
///|
fn hot_lift_impl_has_deep_void_block_chain(body : Expr) -> Bool {
let mut current = body
let mut depth = 1
while depth < 32 {
match current {
Expr([Instruction::Block(block_type, nested)]) if block_type ==
BlockType::void_() => {
depth += 1
current = nested
}
_ => return false
}
}
true
}
///|
// Consecutive void blocks carry no values between regions. Lift their labels
// and body with an explicit stack so deeply nested valid input does not consume
// one host call frame per block on the WasmGC backend.
fn hot_lift_impl_lift_void_block_chain(
func : HotFunc,
body : Expr,
tc_state : TcState,
label_frames : HotLiftLabelFrames,
label_stack_head : Int32,
roots : Array[NodeId],
values : Array[NodeId],
concrete_values : Array[ValType],
) -> Result[TcState, HotLiftError] {
let states : Array[TcState] = []
let entry_states : Array[TcState] = []
let ids : Array[NodeId] = []
let mut current_body = body
let mut current_state = tc_state
let mut current_stack_head = label_stack_head
while true {
let entry_state = match tc_state_pop_types(current_state, []) {
Ok(next) => next
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let ty = hot_lift_impl_intern_control_result_type(
func,
BlockType::void_(),
[],
[],
)
let id = hot_build_node(func, HotOp::Block, ty)
let label_id = hot_label_alloc(func, HotLabelKind::BlockLabel, ty, id, 0)
hot_node_set(func, id, { ..hot_node_get(func, id), imm0: label_id, })
let next_stack_head = hot_lift_impl_push_label(
label_frames, current_stack_head, label_id,
)
let region_env = entry_state.env.with_label([])
let region_state = hot_lift_impl_region_state(entry_state, region_env, [])
states.push(current_state)
entry_states.push(entry_state)
ids.push(id)
current_state = region_state
current_stack_head = next_stack_head
match current_body {
Expr([Instruction::Block(block_type, nested_body)]) if block_type ==
BlockType::void_() => current_body = nested_body
_ => break
}
}
let (body_roots, _, body_exit_state) = match
hot_lift_impl_lift_region(
func, current_body, current_state, label_frames, current_stack_head,
) {
Ok(state) => state
Err(err) => return Err(err)
}
let mut nested_roots = body_roots
let mut exit_state = body_exit_state
let mut idx = ids.length() - 1
while idx >= 0 {
hot_lift_impl_pop_label(label_frames)
exit_state = match
@validate.tc_state_finish_block(
states[idx],
entry_states[idx],
exit_state,
[],
) {
Ok(state) => state
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
ignore(hot_replace_child_span(func, ids[idx], nested_roots))
nested_roots = [ids[idx]]
idx -= 1
}
hot_lift_impl_push_node(roots, values, concrete_values, ids[0], [])
Ok(exit_state)
}
///|
fn hot_lift_impl_lift_control(
func : HotFunc,
instr : Instruction,
tc_state : TcState,
label_frames : HotLiftLabelFrames,
label_stack_head : Int32,
roots : Array[NodeId],
values : Array[NodeId],
concrete_values : Array[ValType],
) -> Result[TcState, HotLiftError] {
if tc_state.polymorphic {
let (block_type, condition_count) = match instr {
Instruction::Block(block_type, _)
| Instruction::Loop(block_type, _)
| Instruction::Try(block_type, _, _, _)
| Instruction::TryTable(block_type, _, _) => (block_type, 0)
Instruction::If(block_type, _, _) => (block_type, 1)
_ => return Err(HotLiftError::UnsupportedInstruction(instr.to_string()))
}
let (params, _) = match
hot_lift_impl_block_results(tc_state.env, block_type) {
Ok(types) => types
Err(err) => return Err(err)
}
if values.length() < params.length() + condition_count {
// Virtual inputs have no HOT producers. Validate every arm and catch
// before omitting the dead control; concrete operands still need checks.
let after_state = match Typecheck::typecheck(instr, tc_state) {
Ok(next) => next
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
if label_frames.record_local_accesses {
hot_lift_impl_reserve_elided_instruction_accesses(instr, label_frames)
}
match
hot_lift_impl_pop_typed_values_for_instr(
instr,
values,
concrete_values,
values.length(),
"concrete control entry values",
) {
Ok(_) => ()
Err(err) => return Err(err)
}
return Ok(after_state)
}
}
match instr {
Instruction::Block(block_type, body) => {
if block_type == BlockType::void_() &&
hot_lift_impl_has_deep_void_block_chain(body) {
return hot_lift_impl_lift_void_block_chain(
func, body, tc_state, label_frames, label_stack_head, roots, values, concrete_values,
)
}
let (params, results) = match
hot_lift_impl_block_results(tc_state.env, block_type) {
Ok(types) => types
Err(err) => return Err(err)
}
let entry_state = match tc_state_pop_types(tc_state, params) {
Ok(next) => next
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let (entry_values, entry_types) = match
hot_lift_impl_pop_typed_values_for_instr(
instr,
values,
concrete_values,
params.length(),
"block entry values",
) {
Ok(entry_values) => entry_values
Err(err) => return Err(err)
}
let ty = hot_lift_impl_intern_control_result_type(
func, block_type, params, results,
)
let id = hot_build_node(func, HotOp::Block, ty)
let label_id = hot_label_alloc(
func,
HotLabelKind::BlockLabel,
ty,
id,
results.length(),
)
let next_stack = hot_lift_impl_push_label(
label_frames, label_stack_head, label_id,
)
let region_env = entry_state.env.with_label(results)
let region_state = hot_lift_impl_region_state(
entry_state, region_env, entry_types,
)
let (body_roots, _, body_exit_state) = match
hot_lift_impl_lift_region(
func,
body,
region_state,
label_frames,
next_stack,
initial_values=entry_values,
initial_value_types=entry_types,
) {
Ok(state) => state
Err(err) => return Err(err)
}
hot_lift_impl_pop_label(label_frames)
let after_state = match
@validate.tc_state_finish_block(
tc_state, entry_state, body_exit_state, results,
) {
Ok(state) => state
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
// Multi-parameter entries need an explicit prefix for lower's body stack.
// Keep the established single-parameter carrier/spill lowering shape.
let scalar_entry_has_outer_prefix = params.length() == 1 &&
entry_values.length() == 1 &&
hot_node_result_arity(func, entry_values[0]) == 1 &&
values.any(fn(value) { value != entry_values[0] })
let entry_count = if params.length() > 1 || scalar_entry_has_outer_prefix {
entry_values.length()
} else {
0
}
let block_children = if entry_count == 0 {
body_roots
} else {
let children = entry_values.copy()
for root in body_roots {
children.push(root)
}
children
}
ignore(hot_replace_child_span(func, id, block_children))
hot_node_set(func, id, {
..hot_node_get(func, id),
imm0: label_id,
imm1: entry_count,
})
hot_lift_impl_push_node(roots, values, concrete_values, id, results)
Ok(after_state)
}
Instruction::Loop(block_type, body) => {
let (params, results) = match
hot_lift_impl_block_results(tc_state.env, block_type) {
Ok(types) => types
Err(err) => return Err(err)
}
let entry_state = match tc_state_pop_types(tc_state, params) {
Ok(next) => next
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let (entry_values, entry_types) = match
hot_lift_impl_pop_typed_values_for_instr(
instr,
values,
concrete_values,
params.length(),
"loop entry values",
) {
Ok(entry_values) => entry_values
Err(err) => return Err(err)
}
let ty = hot_lift_impl_intern_control_result_type(
func, block_type, params, results,
)
let id = hot_build_node(func, HotOp::Loop, ty)
let label_id = hot_label_alloc(
func,
HotLabelKind::LoopLabel,
ty,
id,
params.length(),
)
let next_stack = hot_lift_impl_push_label(
label_frames, label_stack_head, label_id,
)
let region_env = entry_state.env.with_label(params)
let region_state = hot_lift_impl_region_state(
entry_state, region_env, entry_types,
)
let (body_roots, _, body_exit_state) = match
hot_lift_impl_lift_region(
func,
body,
region_state,
label_frames,
next_stack,
initial_values=entry_values,
initial_value_types=entry_types,
) {
Ok(state) => state
Err(err) => return Err(err)
}
hot_lift_impl_pop_label(label_frames)
let after_state = match
@validate.tc_state_finish_loop(
tc_state, entry_state, body_exit_state, results,
) {
Ok(state) => state
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let loop_children : Array[NodeId] = []
for entry_value in entry_values {
loop_children.push(entry_value)
}
for body_root in body_roots {
loop_children.push(body_root)
}
ignore(hot_replace_child_span(func, id, loop_children))
hot_node_set(func, id, { ..hot_node_get(func, id), imm0: label_id, })
hot_lift_impl_push_node(roots, values, concrete_values, id, results)
Ok(after_state)
}
Instruction::If(block_type, then_instrs, else_instrs) => {
let (params, results) = match
hot_lift_impl_block_results(tc_state.env, block_type) {
Ok(types) => types
Err(err) => return Err(err)
}
let st0 = match tc_state_pop_expect(tc_state, ValType::i32()) {
Ok(next) => next
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let entry_state = match tc_state_pop_types(st0, params) {
Ok(next) => next
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let condition = match
hot_lift_impl_pop_value_for_instr(
instr, values, concrete_values, "if condition",
) {
Ok(condition) => condition
Err(err) => return Err(err)
}
let (entry_values, entry_types) = match
hot_lift_impl_pop_typed_values_for_instr(
instr,
values,
concrete_values,
params.length(),
"if entry values",
) {
Ok(entry_values) => entry_values
Err(err) => return Err(err)
}
let ty = hot_lift_impl_intern_control_result_type(
func, block_type, params, results,
)
let result_arity = results.length()
let id = hot_build_node(func, HotOp::If, ty)
let label_id = hot_label_alloc(
func,
HotLabelKind::IfLabel,
ty,
id,
result_arity,
)
let next_stack = hot_lift_impl_push_label(
label_frames, label_stack_head, label_id,
)
let region_env = entry_state.env.with_label(results)
let then_entry_types = hot_lift_impl_copy_val_types(entry_types)
let then_state = hot_lift_impl_region_state(
entry_state, region_env, then_entry_types,
)
let (then_roots, _, then_exit_state) = match
hot_lift_impl_lift_region(
func,
Expr::new(then_instrs),
then_state,
label_frames,
next_stack,
initial_values=entry_values,
initial_value_types=then_entry_types,
) {
Ok(state) => state
Err(err) => return Err(err)
}
let then_region = hot_lift_impl_make_region(
func, then_roots, result_arity,
)
let mut checked_else_state = hot_lift_impl_region_state(
entry_state, region_env, entry_types,
)
let else_region = match else_instrs {
Some(instrs) => {
let else_state = hot_lift_impl_region_state(
entry_state, region_env, entry_types,
)
match
hot_lift_impl_lift_region(
func,
Expr::new(instrs),
else_state,
label_frames,
next_stack,
initial_values=entry_values,
initial_value_types=entry_types,
) {
Ok((else_roots, _, else_exit_state)) => {
checked_else_state = else_exit_state
hot_lift_impl_make_region(func, else_roots, result_arity)
}
Err(err) => return Err(err)
}
}
None => HOT_NONE_NODE
}
hot_lift_impl_pop_label(label_frames)
let after_state = match
@validate.tc_state_finish_if(
tc_state, entry_state, then_exit_state, checked_else_state, results,
) {
Ok(state) => state
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
// Keep the established condition/region slots. Entry operands are
// evaluated before the condition and shared by the two arm stacks.
let children = [condition, then_region, else_region]
for entry_value in entry_values {
children.push(entry_value)
}
ignore(hot_replace_child_span(func, id, children))
hot_node_set(func, id, { ..hot_node_get(func, id), imm0: label_id, })
hot_lift_impl_push_node(roots, values, concrete_values, id, results)
Ok(after_state)
}
Instruction::Try(block_type, body, catches, delegate) => {
if catches.is_empty() && delegate is None {
// A handlerless legacy try has precisely a block's label and escape
// behavior. It does not install a handler or a rethrow scope.
return hot_lift_impl_lift_control(
func,
Instruction::block(block_type, body),
tc_state,
label_frames,
label_stack_head,
roots,
values,
concrete_values,
)
}
if catches.length() > 1 || (catches.length() > 0 && delegate is Some(_)) {
return Err(HotLiftError::UnsupportedInstruction(instr.to_string()))
}
let checked = hot_lift_impl_check_instruction(
instr,
tc_state_clone(tc_state),
)
if checked.error is Some(err) {
return Err(err)
}
let checked_state = checked.state
let (params, results) = match
hot_lift_impl_block_results(tc_state.env, block_type) {
Ok(types) => types
Err(err) => return Err(err)
}
let entry_state = match tc_state_pop_types(tc_state, params) {
Ok(next) => next
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let (entry_values, entry_types) = match
hot_lift_impl_pop_typed_values_for_instr(
instr,
values,
concrete_values,
params.length(),
"legacy try entry values",
) {
Ok(entry_values) => entry_values
Err(err) => return Err(err)
}
let ty = hot_lift_impl_intern_control_result_type(
func, block_type, params, results,
)
let result_arity = results.length()
let id = hot_build_node(func, HotOp::Try, ty)
let label_id = hot_label_alloc(
func,
HotLabelKind::TryLabel,
ty,
id,
result_arity,
)
let next_stack = hot_lift_impl_push_label(
label_frames, label_stack_head, label_id,
)
let region_env = entry_state.env.with_label(results)
let body_state = hot_lift_impl_region_state(
entry_state, region_env, entry_types,
)
let (body_roots, _, body_exit_state) = match
hot_lift_impl_lift_region(
func,
body,
body_state,
label_frames,
next_stack,
initial_values=entry_values,
initial_value_types=entry_types,
) {
Ok(state) => state
Err(err) => return Err(err)
}
match tc_state_validate_end_stack(body_exit_state, results) {
Ok(_) => ()
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let catch_roots : Array[NodeId] = []
let mut catch_kind = 0
match delegate {
Some(target_depth) => {
catch_kind = -2
let @lib.LabelIdx(raw_depth) = target_depth
let target = match
hot_lift_impl_resolve_label(
label_frames,
label_stack_head,
@lib.LabelIdx::new(raw_depth + 1U),
) {
Ok(target) => target
Err(err) => return Err(err)
}
catch_roots.push(hot_build_delegate(func, target))
}
None =>
match catches {
[catch_] => {
let (payload_types, catch_body) = match catch_ {
@lib.LegacyCatch(tag_idx, catch_body) => {
let @lib.FuncType(payload_types, _) = match
entry_state.env.resolve_tag_functype(tag_idx) {
Some(type_) => type_
None =>
return Err(
hot_lift_impl_invalid_body_state(
"legacy catch tag must resolve to a function type",
),
)
}
let raw_tag = tag_idx.0.reinterpret_as_int()
if raw_tag == 0x7FFFFFFF {
return Err(
hot_lift_impl_invalid_body_state(
"legacy catch tag index cannot be encoded in HOT",
),
)
}
catch_kind = raw_tag + 1
(payload_types, catch_body)
}
@lib.LegacyCatchAll(catch_body) => {
catch_kind = -1
([], catch_body)
}
}
let payload_values : Array[NodeId] = []
for payload_type in payload_types {
let payload = hot_build_catch_payload(
func,
match catch_ {
@lib.LegacyCatch(tag_idx, _) => tag_idx
@lib.LegacyCatchAll(_) => @lib.TagIdx::new(0U)
},
payload_type,
)
payload_values.push(payload)
catch_roots.push(payload)
}
label_frames.legacy_catch_depth += 1
let catch_env = region_env.with_legacy_catch_depth(
label_frames.legacy_catch_depth,
)
let catch_state = hot_lift_impl_region_state(
entry_state, catch_env, payload_types,
)
let lifted_catch = hot_lift_impl_lift_region(
func,
catch_body,
catch_state,
label_frames,
next_stack,
initial_values=payload_values,
initial_value_types=payload_types,
)
label_frames.legacy_catch_depth -= 1
let (lifted_roots, _, catch_exit_state) = match lifted_catch {
Ok(state) => state
Err(err) => return Err(err)
}
match tc_state_validate_end_stack(catch_exit_state, results) {
Ok(_) => ()
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
catch_roots.append(lifted_roots)
}
[] =>
return Err(
HotLiftError::UnsupportedInstruction(instr.to_string()),
)
_ => abort("legacy try catch count checked above")
}
}
hot_lift_impl_pop_label(label_frames)
ignore(
hot_replace_child_span(func, id, [
hot_lift_impl_make_region(func, body_roots, result_arity),
hot_lift_impl_make_region(func, catch_roots, result_arity),
]),
)
hot_node_set(func, id, {
..hot_node_get(func, id),
imm0: label_id,
imm1: catch_kind,
})
hot_lift_impl_push_node(roots, values, concrete_values, id, results)
Ok(checked_state)
}
Instruction::TryTable(block_type, catches, body) => {
let (params, results) = match
hot_lift_impl_block_results(tc_state.env, block_type) {
Ok(types) => types
Err(err) => return Err(err)
}
let entry_state = match tc_state_pop_types(tc_state, params) {
Ok(next) => next
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let (entry_values, entry_types) = match
hot_lift_impl_pop_typed_values_for_instr(
instr,
values,
concrete_values,
params.length(),
"try_table entry values",
) {
Ok(entry_values) => entry_values
Err(err) => return Err(err)
}
let ty = hot_lift_impl_intern_control_result_type(
func, block_type, params, results,
)
let result_arity = results.length()
let id = hot_build_node(func, HotOp::TryTable, ty)
let label_id = hot_label_alloc(
func,
HotLabelKind::TryTableLabel,
ty,
id,
result_arity,
)
let next_stack = hot_lift_impl_push_label(
label_frames, label_stack_head, label_id,
)
let region_env = entry_state.env.with_label(results)
let body_state = hot_lift_impl_region_state(
entry_state, region_env, entry_types,
)
let (body_roots, _, body_exit_state) = match
hot_lift_impl_lift_region(
func,
body,
body_state,
label_frames,
next_stack,
initial_values=entry_values,
initial_value_types=entry_types,
) {
Ok(state) => state
Err(err) => return Err(err)
}
hot_lift_impl_pop_label(label_frames)
for catch_ in catches {
match tc_state_typecheck_catch_clause(entry_state, catch_) {
Ok(_) => ()
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
}
match tc_state_validate_end_stack(body_exit_state, results) {
Ok(_) => ()
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let catch_arms : Array[HotCatchArm] = []
for catch_ in catches {
match
hot_lift_impl_try_catch_arm(
label_frames,
label_stack_head,
catch_,
tc_state.env.return_type,
) {
Ok(arm) => catch_arms.push(arm)
Err(err) => return Err(err)
}
}
let after_state = match
@validate.tc_state_finish_try_table(
tc_state, entry_state, body_exit_state, results, catches,
) {
Ok(state) => state
Err(err) => return Err(hot_lift_impl_invalid_body_state(err))
}
let catch_region = hot_lift_impl_make_region(func, [], result_arity)
let catch_info_id = hot_alloc_catch_info(
func,
HotCatchInfo::new(catch_arms),
)
ignore(
hot_replace_child_span(func, id, [
hot_lift_impl_make_region(func, body_roots, result_arity),
catch_region,
]),
)
hot_node_set(func, id, {
..hot_node_get(func, id),
imm0: label_id,
imm1: catch_info_id,
})
hot_lift_impl_push_node(roots, values, concrete_values, id, results)
Ok(after_state)
}
_ => Err(HotLiftError::UnsupportedInstruction(instr.to_string()))
}
}
///|
fn hot_lift_impl_lift_instruction(
func : HotFunc,
instr : Instruction,
tc_state : TcState,
label_frames : HotLiftLabelFrames,
label_stack_head : Int32,
roots : Array[NodeId],
values : Array[NodeId],
concrete_values : Array[ValType],
) -> HotLiftTypecheck {
// Reserve an entry even when unreachable stack polymorphism prevents a node.
// The reservation is consumed only by this invocation; keep its index and
// kind in scalar locals instead of allocating a temporary tuple per access.
let mut source_access_index = -1
let source_access_is_write = match instr {
Instruction::LocalGet(LocalIdx(raw)) => {
if label_frames.record_local_accesses {
source_access_index = label_frames.gets.length()
label_frames.gets.push((raw.reinterpret_as_int(), -1))
}
false
}
Instruction::LocalSet(LocalIdx(raw))
| Instruction::LocalTee(LocalIdx(raw)) => {
if label_frames.record_local_accesses {
source_access_index = label_frames.writes.length()
label_frames.writes.push((raw.reinterpret_as_int(), -1))
}
true
}
_ => false
}
match instr {
Instruction::Block(_, _)
| Instruction::Loop(_, _)
| Instruction::If(_, _, _)
| Instruction::Try(_, _, _, _)
| Instruction::TryTable(_, _, _) =>
return match
hot_lift_impl_lift_control(
func, instr, tc_state, label_frames, label_stack_head, roots, values, concrete_values,
) {
Ok(next) => HotLiftTypecheck::success(next)
Err(err) => HotLiftTypecheck::failure(tc_state, err)
}
Instruction::Br(label_idx) => {
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
let target = match
hot_lift_impl_resolve_label(label_frames, label_stack_head, label_idx) {
Ok(label_id) => label_id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let payload_count = match
hot_lift_impl_branch_arity(func, target, tc_state.env.return_type) {
Ok(count) => count
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
if !hot_lift_impl_can_materialize_unreachable_instr(
instr, tc_state, concrete_values, payload_count,
) {
return HotLiftTypecheck::success(after_state)
}
let payload = match
hot_lift_impl_pop_values_for_instr(
instr, values, concrete_values, payload_count, "br payload",
) {
Ok(payload) => payload
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let node_id = hot_build_node(
func,
HotOp::Br,
hot_type_void(func),
imm0=target,
children=payload,
)
roots.push(node_id)
values.clear()
concrete_values.clear()
return HotLiftTypecheck::success(after_state)
}
Instruction::BrIf(label_idx) => {
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
let target = match
hot_lift_impl_resolve_label(label_frames, label_stack_head, label_idx) {
Ok(label_id) => label_id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let payload_count = match
hot_lift_impl_branch_arity(func, target, tc_state.env.return_type) {
Ok(count) => count
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
if !hot_lift_impl_can_materialize_unreachable_instr(
instr,
tc_state,
concrete_values,
payload_count + 1,
) {
return HotLiftTypecheck::success(after_state)
}
let condition = match
hot_lift_impl_pop_value_for_instr(
instr, values, concrete_values, "br_if condition",
) {
Ok(condition) => condition
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let children = hot_lift_impl_peek_values_with_extra(
values, payload_count, condition,
)
roots.push(
hot_build_node(
func,
HotOp::BrIf,
hot_type_void(func),
imm0=target,
children~,
),
)
return HotLiftTypecheck::success(after_state)
}
Instruction::BrTable(targets, default_target) => {
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
let default_label = match
hot_lift_impl_resolve_label(
label_frames, label_stack_head, default_target,
) {
Ok(label_id) => label_id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let table_targets : Array[LabelId] = []
for target in targets {
match
hot_lift_impl_resolve_label(label_frames, label_stack_head, target) {
Ok(label_id) => table_targets.push(label_id)
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
}
let payload_count = match
hot_lift_impl_branch_arity(
func,
default_label,
tc_state.env.return_type,
) {
Ok(count) => count
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
if !hot_lift_impl_can_materialize_unreachable_instr(
instr,
tc_state,
concrete_values,
payload_count + 1,
) {
return HotLiftTypecheck::success(after_state)
}
let selector = match
hot_lift_impl_pop_value_for_instr(
instr, values, concrete_values, "br_table selector",
) {
Ok(selector) => selector
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let table_id = hot_alloc_branch_table(
func,
HotBranchTable::new(table_targets, default_label),
)
let children = hot_lift_impl_peek_values_with_extra(
values, payload_count, selector,
)
roots.push(
hot_build_node(
func,
HotOp::BrTable,
hot_type_void(func),
imm0=table_id,
children~,
),
)
return HotLiftTypecheck::success(after_state)
}
Instruction::Return => {
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
let return_types = match tc_state.env.return_type {
Some(types) => types
None =>
return HotLiftTypecheck::failure(
tc_state,
HotLiftError::InvalidBodyState(
"return requires a function return type in the lift context",
),
)
}
if !hot_lift_impl_can_materialize_unreachable_instr(
instr,
tc_state,
concrete_values,
return_types.length(),
) {
return HotLiftTypecheck::success(after_state)
}
let payload = match
hot_lift_impl_pop_values_for_instr(
instr,
values,
concrete_values,
return_types.length(),
"return payload",
) {
Ok(payload) => payload
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
roots.push(
hot_build_node(
func,
HotOp::Return,
hot_type_void(func),
children=payload,
),
)
values.clear()
concrete_values.clear()
return HotLiftTypecheck::success(after_state)
}
Instruction::Rethrow(@lib.LabelIdx(raw_depth)) => {
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
roots.push(
hot_build_node(
func,
HotOp::Rethrow,
hot_type_void(func),
imm0=tc_state.env.legacy_rethrow_catch_ordinal(raw_depth).unwrap(),
),
)
values.clear()
concrete_values.clear()
return HotLiftTypecheck::success(after_state)
}
Instruction::Throw(tag_idx) => {
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
let payload_types = match tc_state.env.resolve_tag_functype(tag_idx) {
Some(@lib.FuncType(params, _)) => params
None =>
return HotLiftTypecheck::failure(
tc_state,
HotLiftError::InvalidBodyState(
"throw requires a resolved tag function type",
),
)
}
if !hot_lift_impl_can_materialize_unreachable_instr(
instr,
tc_state,
concrete_values,
payload_types.length(),
) {
return HotLiftTypecheck::success(after_state)
}
let payload = match
hot_lift_impl_pop_values_for_instr(
instr,
values,
concrete_values,
payload_types.length(),
"throw payload",
) {
Ok(payload) => payload
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let @lib.TagIdx(raw) = tag_idx
roots.push(
hot_build_node(
func,
HotOp::Throw,
hot_type_void(func),
imm0=raw.reinterpret_as_int(),
children=payload,
),
)
values.clear()
concrete_values.clear()
return HotLiftTypecheck::success(after_state)
}
Instruction::ThrowRef => {
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
if !hot_lift_impl_can_materialize_unreachable_instr(
instr, tc_state, concrete_values, 1,
) {
return HotLiftTypecheck::success(after_state)
}
let exnref = match
hot_lift_impl_pop_value_for_instr(
instr, values, concrete_values, "throw_ref payload",
) {
Ok(exnref) => exnref
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
roots.push(
hot_build_node1(func, HotOp::ThrowRef, hot_type_void(func), exnref),
)
values.clear()
concrete_values.clear()
return HotLiftTypecheck::success(after_state)
}
Instruction::BrOnNull(label_idx) => {
let before_types = tc_state.stack
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
let target = match
hot_lift_impl_resolve_label(label_frames, label_stack_head, label_idx) {
Ok(label_id) => label_id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let payload_count = match
hot_lift_impl_branch_arity(func, target, tc_state.env.return_type) {
Ok(count) => count
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
if !hot_lift_impl_can_materialize_unreachable_instr(
instr,
tc_state,
concrete_values,
payload_count + 1,
) {
return HotLiftTypecheck::success(after_state)
}
let guard_id = match
hot_lift_impl_pop_value_for_instr(
instr, values, concrete_values, "br_on_null guard",
) {
Ok(guard_id) => guard_id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let children = hot_lift_impl_peek_values_with_extra(
values, payload_count, guard_id,
)
let pushed_types = hot_lift_impl_pushed_types(
before_types,
after_state.stack,
payload_count + 1,
)
let node_id = hot_build_node(
func,
HotOp::BrOnNull,
hot_lift_impl_result_type(func, pushed_types),
imm0=target,
children~,
)
hot_lift_impl_push_node(
roots, values, concrete_values, node_id, pushed_types,
)
return HotLiftTypecheck::success(after_state)
}
Instruction::BrOnNonNull(label_idx) => {
let before_types = tc_state.stack
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
let target = match
hot_lift_impl_resolve_label(label_frames, label_stack_head, label_idx) {
Ok(label_id) => label_id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let payload_count = match
hot_lift_impl_branch_arity(func, target, tc_state.env.return_type) {
Ok(count) => if count <= 0 { 0 } else { count - 1 }
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
if !hot_lift_impl_can_materialize_unreachable_instr(
instr,
tc_state,
concrete_values,
payload_count + 1,
) {
return HotLiftTypecheck::success(after_state)
}
let guard_id = match
hot_lift_impl_pop_value_for_instr(
instr, values, concrete_values, "br_on_non_null guard",
) {
Ok(guard_id) => guard_id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let children = hot_lift_impl_peek_values_with_extra(
values, payload_count, guard_id,
)
let pushed_types = if before_types.length() >= payload_count + 1 &&
before_types.length() - (payload_count + 1) <=
after_state.stack.length() {
hot_lift_impl_pushed_types(
before_types,
after_state.stack,
payload_count + 1,
)
} else {
[]
}
let node_id = hot_build_node(
func,
HotOp::BrOnNonNull,
hot_lift_impl_result_type(func, pushed_types),
imm0=target,
children~,
)
hot_lift_impl_push_node(
roots, values, concrete_values, node_id, pushed_types,
)
return HotLiftTypecheck::success(after_state)
}
Instruction::BrOnCastDescEq(label_idx, _, _)
| Instruction::BrOnCastDescEqFail(label_idx, _, _) => {
let before_types = hot_lift_impl_copy_val_types(tc_state.stack)
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
let target = match
hot_lift_impl_resolve_label(label_frames, label_stack_head, label_idx) {
Ok(label_id) => label_id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let payload_count = match
hot_lift_impl_branch_arity(func, target, tc_state.env.return_type) {
Ok(count) => count - 1
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
if !hot_lift_impl_can_materialize_unreachable_instr(
instr,
tc_state,
concrete_values,
payload_count + 2,
) {
return HotLiftTypecheck::success(after_state)
}
let descriptor_id = match
hot_lift_impl_pop_value_for_instr(
instr, values, concrete_values, "br_on_cast_desc_eq descriptor",
) {
Ok(id) => id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let source_id = match
hot_lift_impl_pop_value_for_instr(
instr, values, concrete_values, "br_on_cast_desc_eq source",
) {
Ok(id) => id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let children = hot_lift_impl_peek_values_with_extra(
values, payload_count, source_id,
)
children.push(descriptor_id)
let pushed_types = hot_lift_impl_pushed_types(
before_types,
after_state.stack,
payload_count + 2,
)
let op = match instr {
Instruction::BrOnCastDescEq(_, _, _) => HotOp::BrOnCast
_ => HotOp::BrOnCastFail
}
let payload_id = hot_alloc_instr_payload(
func,
HotInstrPayload::new(instr),
)
let node_id = hot_build_node(
func,
op,
hot_lift_impl_result_type(func, pushed_types),
imm0=target,
imm1=payload_id,
children~,
)
hot_lift_impl_push_node(
roots, values, concrete_values, node_id, pushed_types,
)
return HotLiftTypecheck::success(after_state)
}
Instruction::BrOnCast(label_idx, _, _, _)
| Instruction::BrOnCastFail(label_idx, _, _, _) => {
let before_types = tc_state.stack
let checked = hot_lift_impl_check_instruction(instr, tc_state)
if checked.error is Some(err) {
return HotLiftTypecheck::failure(tc_state, err)
}
let after_state = checked.state
let target = match
hot_lift_impl_resolve_label(label_frames, label_stack_head, label_idx) {
Ok(label_id) => label_id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let payload_count = match
hot_lift_impl_branch_arity(func, target, tc_state.env.return_type) {
Ok(count) => count - 1
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
if !hot_lift_impl_can_materialize_unreachable_instr(
instr,
tc_state,
concrete_values,
payload_count + 1,
) {
return HotLiftTypecheck::success(after_state)
}
let guard_id = match
hot_lift_impl_pop_value_for_instr(
instr, values, concrete_values, "br_on_cast guard",
) {
Ok(guard_id) => guard_id
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
let children = hot_lift_impl_peek_values_with_extra(
values, payload_count, guard_id,
)
let pushed_types = hot_lift_impl_pushed_types(
before_types,
after_state.stack,
payload_count + 1,
)
let op = match instr {
Instruction::BrOnCast(_, _, _, _) => HotOp::BrOnCast
_ => HotOp::BrOnCastFail
}
let payload_id = hot_alloc_instr_payload(
func,
HotInstrPayload::new(instr),
)
let node_id = hot_build_node(
func,
op,
hot_lift_impl_result_type(func, pushed_types),
imm0=target,
imm1=payload_id,
children~,
)
hot_lift_impl_push_node(
roots, values, concrete_values, node_id, pushed_types,
)
return HotLiftTypecheck::success(after_state)
}
_ => ()
}
let checked = hot_lift_impl_typecheck_data_instruction(
instr,
tc_state,
concrete_before_types=concrete_values,
scalar_results=Some(hot_lift_impl_scalar_result_scratch(label_frames)),
)
match checked.error {
Some(err) => return HotLiftTypecheck::failure(tc_state, err)
None => ()
}
let after_state = checked.state
let child_count = checked.pop_count
let pushed_types = checked.pushed
if !hot_lift_impl_can_materialize_unreachable_instr(
instr, tc_state, concrete_values, child_count,
) {
return HotLiftTypecheck::success(after_state)
}
match instr {
Instruction::LocalSet(LocalIdx(raw))
| Instruction::LocalTee(LocalIdx(raw)) =>
hot_lift_impl_capture_before_local_access(
func,
roots,
values,
concrete_values,
raw.reinterpret_as_int(),
true,
child_count,
label_frames,
)
Instruction::LocalGet(LocalIdx(raw)) =>
hot_lift_impl_capture_before_local_access(
func,
roots,
values,
concrete_values,
raw.reinterpret_as_int(),
false,
child_count,
label_frames,
)
_ => ()
}
match hot_lift_impl_direct_family(instr) {
Some(op) => {
let node_id = match child_count {
0 => {
let { op: build_op, ty, imm0, imm1, } = hot_lift_impl_direct_node_shape(
func, op, instr, pushed_types,
)
hot_build_node0(func, build_op, ty, imm0~, imm1~)
}
1 => {
let child0 = hot_lift_impl_pop_value(values, concrete_values)
hot_lift_impl_build_direct_node1(
func, op, instr, pushed_types, child0,
)
}
2 => {
let child1 = hot_lift_impl_pop_value(values, concrete_values)
let child0 = hot_lift_impl_pop_value(values, concrete_values)
hot_lift_impl_build_direct_node2(
func, op, instr, pushed_types, child0, child1,
)
}
3 => {
let child2 = hot_lift_impl_pop_value(values, concrete_values)
let child1 = hot_lift_impl_pop_value(values, concrete_values)
let child0 = hot_lift_impl_pop_value(values, concrete_values)
hot_lift_impl_build_direct_node3(
func, op, instr, pushed_types, child0, child1, child2,
)
}
_ => {
let children = match
hot_lift_impl_pop_values_for_instr(
instr, values, concrete_values, child_count, "generic operands",
) {
Ok(children) => children
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
hot_lift_impl_build_direct_node(
func, op, instr, pushed_types, children,
)
}
}
if source_access_index >= 0 {
let accesses = if source_access_is_write {
label_frames.writes
} else {
label_frames.gets
}
accesses[source_access_index] = (
accesses[source_access_index].0,
node_id,
)
}
hot_lift_impl_push_node(
roots, values, concrete_values, node_id, pushed_types,
)
HotLiftTypecheck::success(after_state)
}
None =>
match hot_lift_impl_exact_family(instr) {
Some(op) => {
let node_id = match child_count {
0 => {
let { op: build_op, ty, imm0, imm1, } = hot_lift_impl_exact_node_shape(
func, op, instr, pushed_types,
)
hot_build_node(func, build_op, ty, imm0~, imm1~)
}
1 => {
let child0 = hot_lift_impl_pop_value(values, concrete_values)
hot_lift_impl_build_exact_node1(
func, op, instr, pushed_types, child0,
)
}
2 => {
let child1 = hot_lift_impl_pop_value(values, concrete_values)
let child0 = hot_lift_impl_pop_value(values, concrete_values)
hot_lift_impl_build_exact_node2(
func, op, instr, pushed_types, child0, child1,
)
}
3 => {
let child2 = hot_lift_impl_pop_value(values, concrete_values)
let child1 = hot_lift_impl_pop_value(values, concrete_values)
let child0 = hot_lift_impl_pop_value(values, concrete_values)
hot_lift_impl_build_exact_node3(
func, op, instr, pushed_types, child0, child1, child2,
)
}
_ => {
let children = match
hot_lift_impl_pop_values_for_instr(
instr, values, concrete_values, child_count, "generic operands",
) {
Ok(children) => children
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
hot_lift_impl_build_exact_node(
func, op, instr, pushed_types, children,
)
}
}
if op == HotOp::Continuation {
let targets : Array[LabelId] = []
for handler in hot_continuation_handlers(instr) {
match handler {
@lib.ResumeOnLabel(_, label) =>
match
hot_lift_impl_resolve_label(
label_frames, label_stack_head, label,
) {
Ok(target) => targets.push(target)
Err(err) => return HotLiftTypecheck::failure(tc_state, err)
}
@lib.ResumeOnSwitch(_) => ()
}
}
hot_continuation_targets_set(func, node_id, targets)
}
hot_lift_impl_push_node(
roots, values, concrete_values, node_id, pushed_types,
)
HotLiftTypecheck::success(after_state)
}
None =>
HotLiftTypecheck::failure(
tc_state,
HotLiftError::UnsupportedInstruction(instr.to_string()),
)
}
}
}
///|
fn hot_lift_impl_finish(
func : HotFunc,
roots : Array[NodeId],
label_frames : HotLiftLabelFrames,
verify~ : Bool = true,
) -> HotFunc {
for root in roots {
hot_root_append(func, root)
}
if verify {
hot_verify_all_or_abort(func)
}
func.lifted_local_accesses = if label_frames.record_local_accesses {
Some({
local_count: label_frames.local_count,
gets: label_frames.gets,
writes: label_frames.writes,
capture_sources: label_frames.capture_sources,
revision: func.revision,
})
} else {
None
}
func
}
///|
pub fn hot_lift_body_result(
body : Expr,
locals : HotLocals,
body_result_type~ : BlockType = BlockType::void_(),
) -> Result[HotFunc, HotLiftError] {
let env = hot_lift_impl_env_without_context(
locals,
match hot_lift_impl_block_results(empty_env(), body_result_type) {
Ok((_, results)) => results
Err(err) => return Err(err)
},
)
let func = HotFunc::new(locals~)
func.validation_env = Some(env)
hot_body_result_type_set(
func,
hot_type_intern_block_result(func, body_result_type),
)
let state = make_state(env, [])
let label_frames = hot_lift_impl_new_label_frames(func)
let label_stack_head = hot_lift_impl_seed_function_label(label_frames)
match
hot_lift_impl_lift_region(func, body, state, label_frames, label_stack_head) {
Ok((roots, _, _)) => Ok(hot_lift_impl_finish(func, roots, label_frames))
Err(err) => Err(err)
}
}
///|
pub fn hot_lift_body_with_context(
body : Expr,
locals : HotLocals,
ctx : HotModuleContext,
body_result_type~ : BlockType = BlockType::void_(),
) -> HotFunc {
let func = hot_lift_body_with_context_unverified(
body,
locals,
ctx,
body_result_type~,
)
hot_verify_all_or_abort(func)
func
}
///|
fn hot_lift_impl_body_with_context_result(
body : Expr,
locals : HotLocals,
ctx : HotModuleContext,
body_result_type : BlockType,
validation_locals~ : @lib.Locals? = None,
known_return_types~ : Array[ValType]? = None,
env_build_visits~ : Array[Int32]? = None,
capture_stack_locals~ : Bool = false,
materialize_multi_values~ : Bool = false,
record_local_accesses~ : Bool = true,
record_capture_sources~ : Bool = false,
) -> Result[HotFunc, HotLiftError] {
let return_types = match known_return_types {
Some(types) => types
None => {
match env_build_visits {
Some(visits) => visits[0] += 1
None => ()
}
let return_env = hot_lift_impl_env_from_context(
ctx,
locals,
[],
validation_locals~,
)
match hot_lift_impl_block_results(return_env, body_result_type) {
Ok((_, results)) => results
Err(err) => return Err(err)
}
}
}
match env_build_visits {
Some(visits) => visits[0] += 1
None => ()
}
let env = hot_lift_impl_env_from_context(
ctx,
locals,
return_types,
validation_locals~,
)
let func = HotFunc::new(locals~)
func.validation_env = Some(env)
hot_body_result_type_set(
func,
hot_lift_impl_intern_control_result_type(
func,
body_result_type,
[],
return_types,
),
)
let label_frames = hot_lift_impl_new_label_frames(
func,
capture_stack_locals~,
materialize_multi_values~,
record_local_accesses~,
record_capture_sources~,
)
let label_stack_head = hot_lift_impl_seed_function_label(label_frames)
match
hot_lift_impl_lift_region(
func,
body,
make_state(env, []),
label_frames,
label_stack_head,
) {
Ok((roots, _, _)) =>
Ok(hot_lift_impl_finish(func, roots, label_frames, verify=false))
Err(err) => Err(err)
}
}
///|
pub fn hot_lift_body_with_context_result(
body : Expr,
locals : HotLocals,
ctx : HotModuleContext,
body_result_type~ : BlockType = BlockType::void_(),
) -> Result[HotFunc, HotLiftError] {
hot_lift_impl_body_with_context_result(body, locals, ctx, body_result_type)
}
///|
pub fn hot_lift_body_with_context_unverified(
body : Expr,
locals : HotLocals,
ctx : HotModuleContext,
body_result_type~ : BlockType = BlockType::void_(),
) -> HotFunc {
match
hot_lift_body_with_context_result(body, locals, ctx, body_result_type~) {
Ok(func) => func
Err(err) => abort(err.to_string())
}
}
///|
fn hot_lift_impl_defined_code_index(
total_func_count : Int32,
defined_func_count : Int32,
func_idx : FuncIdx,
) -> Int32? {
let FuncIdx(raw) = func_idx
let code_idx = raw.reinterpret_as_int() -
(total_func_count - defined_func_count)
if code_idx < 0 || code_idx >= defined_func_count {
None
} else {
Some(code_idx)
}
}
///|
pub fn hot_lift_func_with_context(
mod_ : Module,
func_idx : FuncIdx,
ctx : HotModuleContext,
) -> HotFunc {
let func = hot_lift_func_with_context_unverified(mod_, func_idx, ctx)
hot_verify_all_or_abort(func)
func
}
///|
/// Keep original local-access provenance by default. Callers that never query
/// hot_lift_local_accesses may omit it explicitly; capture/conflict analysis,
/// typechecking, result materialization and the lifted graph are unchanged.
/// The source-map getter rejects a lift made with recording disabled.
pub fn hot_lift_func_with_context_result(
mod_ : Module,
func_idx : FuncIdx,
ctx : HotModuleContext,
capture_stack_locals~ : Bool = false,
materialize_multi_values~ : Bool = false,
record_local_accesses~ : Bool = true,
record_capture_sources~ : Bool = false,
) -> Result[HotFunc, HotLiftError] {
let func_type = match hot_module_resolve_func_type(ctx, func_idx) {
Some(type_) => type_
None => return Err(HotLiftError::InvalidFunctionIndex(func_idx))
}
let defs = match mod_.code_sec {
Some(@lib.CodeSec(defs)) => defs
None => return Err(HotLiftError::MissingFunctionBody(func_idx))
}
let code_idx = match
hot_lift_impl_defined_code_index(
ctx.func_types.length(),
defs.length(),
func_idx,
) {
Some(idx) => idx
None => return Err(HotLiftError::MissingFunctionBody(func_idx))
}
let def = defs[code_idx]
let @lib.Func(body_locals, body) = def
let @lib.FuncType(params, results) = func_type
let locals = HotLocals::new(
params~,
locals=hot_lift_impl_declared_locals(body_locals),
)
let func_type_idx = hot_module_resolve_func_type_idx(ctx, func_idx)
let body_result_type = match func_type_idx {
Some(type_idx) => BlockType::type_idx(type_idx)
None =>
match results {
[] => BlockType::void_()
[result] => BlockType::val_type(result)
_ => return Err(HotLiftError::MissingFunctionType(func_idx))
}
}
let hot = match
hot_lift_impl_body_with_context_result(
body,
locals,
ctx,
body_result_type,
validation_locals=Some(
hot_lift_impl_validation_locals(params, body_locals),
),
known_return_types=Some(results),
capture_stack_locals~,
materialize_multi_values~,
record_local_accesses~,
record_capture_sources~,
) {
Ok(hot) => hot
Err(err) => return Err(err)
}
hot.func_type_idx = func_type_idx
Ok(hot)
}
///|
pub fn hot_lift_func_with_context_unverified(
mod_ : Module,
func_idx : FuncIdx,
ctx : HotModuleContext,
) -> HotFunc {
match hot_lift_func_with_context_result(mod_, func_idx, ctx) {
Ok(func) => func
Err(err) => abort(err.to_string())
}
}
///|
pub fn hot_lift_func(mod_ : Module, func_idx : FuncIdx) -> HotFunc {
hot_lift_func_with_context(
mod_,
func_idx,
hot_module_context_from_module(mod_),
)
}
///|
fn hot_lift_impl_pending_observable(
func : HotFunc,
node : NodeId,
cache : Array[Int32],
depth : Int32,
) -> Bool {
if node < 0 {
return false
}
if depth >= 512 {
return true
}
while cache.length() <= node {
cache.push(-1)
}
if cache[node] >= 0 {
return cache[node] == 1
}
let own = effects_mask_for_flags(hot_node_flags(func, node)) |
effects_mask_for_node_exact(func, node)
let mut observable = (own & (0xffffffffU ^ EFFECT_MASK_LOCAL_STATE)) != 0U
if !observable {
for child in hot_child_span(func, node) {
if hot_lift_impl_pending_observable(func, child, cache, depth + 1) {
observable = true
break
}
}
}
cache[node] = if observable { 1 } else { 0 }
observable
}