///|
pub enum HotUseSlotKind {
ChildUseSlot
RootUseSlot
} derive(Eq, Debug)
///|
pub impl Show for HotUseSlotKind with fn output(self, logger) {
logger.write_string(@debug.to_string(self))
}
///|
pub struct HotUseSite {
user : NodeId
slot : Int32
kind : HotUseSlotKind
} derive(Eq, Debug)
///|
pub impl Show for HotUseSite with fn output(self, logger) {
logger.write_string(@debug.to_string(self))
}
///|
pub struct HotUseDef {
node_uses : Array[Array[HotUseSite]]
block_defs : Array[Bitset]
block_uses : Array[Bitset]
local_reads : Array[Array[NodeId]]
local_writes : Array[Array[NodeId]]
local_write_blocks : Array[Array[BlockId]]
} derive(Eq, Debug)
///|
/// Lightweight exact node-use counts without use-site or local-use allocation.
pub struct HotNodeUseCounts {
counts : Array[Int32]
} derive(Eq, Debug)
///|
fn use_def_bitset_clone(src : Bitset) -> Bitset {
let out = bitset_new(src.size)
ignore(bitset_union_into(out, src))
out
}
///|
fn use_def_require_node_range(use_def : HotUseDef, node_id : NodeId) -> Unit {
if node_id < 0 || node_id >= use_def.node_uses.length() {
abort("use-def node query requires a valid node id")
}
}
///|
fn use_def_require_block_range(use_def : HotUseDef, block_id : BlockId) -> Unit {
if block_id < 0 || block_id >= use_def.block_defs.length() {
abort("use-def block query requires a valid block id")
}
}
///|
fn use_def_require_local_range(use_def : HotUseDef, local_id : LocalId) -> Unit {
if local_id < 0 || local_id >= use_def.local_reads.length() {
abort("use-def local query requires a valid local id")
}
}
///|
fn use_def_region_slot(
node : HotNode,
slot : Int32,
func~ : HotFunc? = None,
) -> Bool {
match node.op {
HotOp::Block => slot >= node.imm1
HotOp::Loop =>
match func {
Some(func) => slot >= hot_label_branch_arity(func, node.imm0)
None => true
}
HotOp::If => slot == 1 || slot == 2
HotOp::Try | HotOp::TryTable => slot == 0 || slot == 1
_ => false
}
}
///|
fn use_def_add_site(
node_uses : Array[Array[HotUseSite]],
node_id : NodeId,
site : HotUseSite,
) -> Unit {
node_uses[node_id].push(site)
}
///|
fn use_def_track_local_node(
func : HotFunc,
node_id : NodeId,
local_reads : Array[Array[NodeId]],
local_writes : Array[Array[NodeId]],
) -> Unit {
let node = hot_node_get(func, node_id)
match node.op {
// use_def_visit_node only visits each live node once, so these per-local
// lists do not need an extra linear de-duplication scan.
HotOp::LocalGet => local_reads[node.imm0].push(node_id)
HotOp::LocalSet | HotOp::LocalTee => local_writes[node.imm0].push(node_id)
_ => ()
}
}
///|
fn use_def_visit_region(
func : HotFunc,
region_ref : HotRegionRef,
node_uses : Array[Array[HotUseSite]],
local_reads : Array[Array[NodeId]],
local_writes : Array[Array[NodeId]],
seen : @bitset.BitSet,
) -> Unit {
for slot = 0; slot < hot_region_root_count(func, region_ref); slot = slot + 1 {
let root_id = hot_region_get(func, region_ref, slot)
if root_id == HOT_NONE_NODE || !hot_node_is_live(func, root_id) {
continue
}
use_def_add_site(node_uses, root_id, {
user: region_ref.holder,
slot,
kind: HotUseSlotKind::RootUseSlot,
})
use_def_visit_node(
func, root_id, node_uses, local_reads, local_writes, seen,
)
}
}
///|
fn use_def_visit_node(
func : HotFunc,
node_id : NodeId,
node_uses : Array[Array[HotUseSite]],
local_reads : Array[Array[NodeId]],
local_writes : Array[Array[NodeId]],
seen : @bitset.BitSet,
) -> Unit {
if seen[node_id] {
return
}
seen[node_id] = true
use_def_track_local_node(func, node_id, local_reads, local_writes)
let node = hot_node_get(func, node_id)
match node.op {
HotOp::Block | HotOp::Loop => {
let entry_count = hot_region_body_base_live(func, node_id)
if entry_count > 0 {
for slot in 0.. ()
}
for index = 0; index < node.child_count; index = index + 1 {
let slot = hot_child_evaluation_slot(node, index)
let child_id = func.children[node.child_base + slot]
if child_id == HOT_NONE_NODE || !hot_node_is_live(func, child_id) {
continue
}
use_def_add_site(node_uses, child_id, {
user: node_id,
slot,
kind: HotUseSlotKind::ChildUseSlot,
})
use_def_visit_node(
func, child_id, node_uses, local_reads, local_writes, seen,
)
}
}
///|
fn use_def_block_mark_use(
local_count : Int32,
defs : Bitset,
uses : Bitset,
local_id : LocalId,
) -> Unit {
if local_id < 0 || local_id >= local_count {
abort("use-def local use must reference an existing local")
}
let word_idx = bitset_word_index(local_id)
let mask = bitset_word_bit_mask(local_id)
if (defs.words[word_idx] & mask) == 0UL {
uses.words[word_idx] = uses.words[word_idx] | mask
}
}
///|
fn use_def_block_mark_def(
local_count : Int32,
defs : Bitset,
block_id : BlockId,
local_id : LocalId,
local_write_blocks : Array[Array[BlockId]],
) -> Unit {
if local_id < 0 || local_id >= local_count {
abort("use-def local def must reference an existing local")
}
let word_idx = bitset_word_index(local_id)
let mask = bitset_word_bit_mask(local_id)
let before = defs.words[word_idx]
let after = before | mask
if after != before {
defs.words[word_idx] = after
local_write_blocks[local_id].push(block_id)
}
}
///|
fn use_def_scan_block_node(
func : HotFunc,
cfg : HotCfg,
node_id : NodeId,
block_id : BlockId,
local_count : Int32,
defs : Bitset,
uses : Bitset,
local_write_blocks : Array[Array[BlockId]],
seen_stamps : Array[Int32],
) -> Unit {
// Expanded CFGs give control operands their own blocks. A consumer uses the
// produced value; it must not re-execute the producer's local accesses.
match cfg_block_for_node(cfg, node_id) {
Some(owner) if owner != block_id => return
_ => ()
}
let node = hot_node_get(func, node_id)
if cfg.operands_expanded {
match node.op {
HotOp::LocalGet =>
use_def_block_mark_use(local_count, defs, uses, node.imm0)
HotOp::LocalSet | HotOp::LocalTee =>
use_def_block_mark_def(
local_count,
defs,
block_id,
node.imm0,
local_write_blocks,
)
_ => ()
}
return
}
if seen_stamps[node_id] == block_id {
return
}
seen_stamps[node_id] = block_id
for index = 0; index < node.child_count; index = index + 1 {
let slot = hot_child_evaluation_slot(node, index)
if use_def_region_slot(node, slot, func=Some(func)) {
continue
}
let child_id = func.children[node.child_base + slot]
if child_id == HOT_NONE_NODE || !hot_node_is_live(func, child_id) {
continue
}
if ssa_simple_value_block_operand_allowed(func, child_id) {
let body = hot_region_body_ref(func, child_id)
for body_slot = 0
body_slot < hot_region_root_count(func, body)
body_slot = body_slot + 1 {
let root_id = hot_region_get(func, body, body_slot)
if root_id == HOT_NONE_NODE || !hot_node_is_live(func, root_id) {
continue
}
use_def_scan_block_node(
func, cfg, root_id, block_id, local_count, defs, uses, local_write_blocks,
seen_stamps,
)
}
} else {
use_def_scan_block_node(
func, cfg, child_id, block_id, local_count, defs, uses, local_write_blocks,
seen_stamps,
)
}
}
match node.op {
HotOp::LocalGet =>
use_def_block_mark_use(local_count, defs, uses, node.imm0)
HotOp::LocalSet | HotOp::LocalTee =>
use_def_block_mark_def(
local_count,
defs,
block_id,
node.imm0,
local_write_blocks,
)
_ => ()
}
}
///|
fn use_def_build_block_sets(
func : HotFunc,
cfg : HotCfg,
) -> (Array[Bitset], Array[Bitset], Array[Array[BlockId]]) {
let local_count = hot_local_count(func)
let seen_stamps = Array::make(hot_node_count(func), -1)
let block_defs : Array[Bitset] = []
let block_uses : Array[Bitset] = []
let local_write_blocks : Array[Array[BlockId]] = []
for _ in 0.. (Array[Array[HotUseSite]], Array[Array[NodeId]], Array[Array[NodeId]]) {
let node_uses : Array[Array[HotUseSite]] = []
let seen = @bitset.BitSet::new(hot_node_count(func))
for _ in 0.. Array[Array[BlockId]] {
let local_write_blocks : Array[Array[BlockId]] = []
for _ in 0.. HotUseDef {
let (node_uses, local_reads, local_writes) = use_def_build_node_and_local_uses(
func,
)
{
node_uses,
block_defs: [],
block_uses: [],
local_reads,
local_writes,
local_write_blocks: use_def_empty_local_write_blocks(hot_local_count(func)),
}
}
///|
fn hot_node_use_counts_visit(
func : HotFunc,
node_id : NodeId,
counts : Array[Int32],
seen : @bitset.BitSet,
) -> Unit {
if seen[node_id] {
return
}
seen[node_id] = true
let node = hot_node_get(func, node_id)
for index = 0; index < node.child_count; index = index + 1 {
let slot = hot_child_evaluation_slot(node, index)
let child_id = func.children[node.child_base + slot]
if child_id == HOT_NONE_NODE || !hot_node_is_live(func, child_id) {
continue
}
counts[child_id] = counts[child_id] + 1
hot_node_use_counts_visit(func, child_id, counts, seen)
}
}
///|
/// Count every reachable live root and child reference in the current HOT snapshot.
pub fn hot_node_use_counts_build(func : HotFunc) -> HotNodeUseCounts {
let counts = Array::make(hot_node_count(func), 0)
let seen = @bitset.BitSet::new(hot_node_count(func))
for root_id in func.roots {
if root_id == HOT_NONE_NODE || !hot_node_is_live(func, root_id) {
continue
}
counts[root_id] = counts[root_id] + 1
hot_node_use_counts_visit(func, root_id, counts, seen)
}
{ counts, }
}
///|
pub fn use_def_build(func : HotFunc, cfg~ : HotCfg? = None) -> HotUseDef {
let (node_uses, local_reads, local_writes) = use_def_build_node_and_local_uses(
func,
)
let (block_defs, block_uses, local_write_blocks) = match cfg {
Some(existing) => use_def_build_block_sets(func, existing)
None =>
match hot_verify_control(func) {
Ok(_) => use_def_build_block_sets(func, cfg_build(func))
Err(_) => ([], [], [])
}
}
{
node_uses,
block_defs,
block_uses,
local_reads,
local_writes,
local_write_blocks,
}
}
///|
pub fn node_use_sites(
use_def : HotUseDef,
node_id : NodeId,
) -> Array[HotUseSite] {
use_def_require_node_range(use_def, node_id)
use_def.node_uses[node_id].copy()
}
///|
/// Return direct internal node-use site storage for read-only callers.
///
/// This avoids allocating a full copy of the use sites vector.
/// Callers must not mutate the returned array.
pub fn node_use_sites_borrowed(
use_def : HotUseDef,
node_id : NodeId,
) -> Array[HotUseSite] {
use_def_require_node_range(use_def, node_id)
use_def.node_uses[node_id]
}
///|
pub fn node_use_count(use_def : HotUseDef, node_id : NodeId) -> Int32 {
use_def_require_node_range(use_def, node_id)
use_def.node_uses[node_id].length()
}
///|
pub fn hot_node_use_count(
use_counts : HotNodeUseCounts,
node_id : NodeId,
) -> Int32 {
if node_id < 0 || node_id >= use_counts.counts.length() {
abort("HOT node-use count query requires a valid node id")
}
use_counts.counts[node_id]
}
///|
pub fn block_local_defs(use_def : HotUseDef, block_id : BlockId) -> Bitset {
use_def_require_block_range(use_def, block_id)
use_def_bitset_clone(use_def.block_defs[block_id])
}
///|
pub fn block_local_uses(use_def : HotUseDef, block_id : BlockId) -> Bitset {
use_def_require_block_range(use_def, block_id)
use_def_bitset_clone(use_def.block_uses[block_id])
}
///|
pub fn local_read_nodes(
use_def : HotUseDef,
local_id : LocalId,
) -> Array[NodeId] {
use_def_require_local_range(use_def, local_id)
use_def.local_reads[local_id].copy()
}
///|
/// Return direct internal local-read node storage for read-only callers.
///
/// This avoids allocating a full copy of the local-read list.
/// Callers must not mutate the returned array.
pub fn local_read_nodes_borrowed(
use_def : HotUseDef,
local_id : LocalId,
) -> Array[NodeId] {
use_def_require_local_range(use_def, local_id)
use_def.local_reads[local_id]
}
///|
pub fn local_write_nodes(
use_def : HotUseDef,
local_id : LocalId,
) -> Array[NodeId] {
use_def_require_local_range(use_def, local_id)
use_def.local_writes[local_id].copy()
}
///|
/// Return direct internal local-write node storage for read-only callers.
///
/// This avoids allocating a full copy of the local-write list.
/// Callers must not mutate the returned array.
pub fn local_write_nodes_borrowed(
use_def : HotUseDef,
local_id : LocalId,
) -> Array[NodeId] {
use_def_require_local_range(use_def, local_id)
use_def.local_writes[local_id]
}
///|
pub fn local_write_block_ids(
use_def : HotUseDef,
local_id : LocalId,
) -> Array[BlockId] {
use_def_require_local_range(use_def, local_id)
use_def.local_write_blocks[local_id].copy()
}