// Licensing scope: see ../../LICENSE and this directory's LICENSE and NOTICE.
// Copyright 2023 WebAssembly Community Group participants.
// Retained historical upstream notice; current derivation remains unconfirmed.
// Conservative attribution from pinned Binaryen version_130 counterpart:
// Copyright 2023 WebAssembly Community Group participants
// Counterpart: src/passes/TupleOptimization.cpp (5d704ad5); origin remains unconfirmed.
// Potentially incorporated Binaryen expression retains Apache-2.0 terms.
// Original Starshine contributions use the scoped MIT grant in ../../LICENSE.
// Modified for Starshine through 2026: implementation and validation use
// Starshine's representations and APIs; this is not an unchanged upstream file.
// Conservative notice retention does not establish copying or independence.
///|
struct HotTupleOptimizationGroup {
producer : Int32
lane_locals : Array[Int32]
defining_nodes : Array[Int32]
mut host_lane : Int32?
mut copy_source : Int32?
mut copy_payload_locals : Array[Int32]?
has_copy_payload_candidate : Bool
mut uses : Int32
mut valid_uses : Int32
neighbors : Array[Int32]
mut bad : Bool
} derive(Eq, Debug)
///|
struct HotTupleOptimizationQuerySummary {
node_has_direct_non_scalar_use : @bitset.BitSet
node_has_direct_non_drop_use : @bitset.BitSet
node_has_direct_drop_use : @bitset.BitSet
scalar_forward_present : @bitset.BitSet
scalar_forward_locals : Array[Int32]
scalar_forward_defs : Array[Int32]
scalar_forward_is_tee : @bitset.BitSet
local_single_write_nodes : Array[Int32]
local_is_locally_tee_defined : @bitset.BitSet
has_local_tee_defs : Bool
has_scalarized_tuple_local_cleanup_candidate : Bool
has_single_result_block_copy_candidate : Bool
has_scalar_forward_copy_candidate : Bool
forwarded_non_scalar_use : Array[Int32]
forwarded_non_drop_use : Array[Int32]
forwarded_drop_use : Array[Int32]
forwarded_any_use : Array[Int32]
forwarded_host_tee_use : Array[Int32]
visit_marks : Array[Int32]
mut next_visit_mark : Int32
resolve_visit_marks : Array[Int32]
mut next_resolve_visit_mark : Int32
} derive(Eq, Debug)
///|
struct HotTupleOptimizationAnalysis {
groups : Array[HotTupleOptimizationGroup]
local_group_ids : Array[Int32]
copy_read_nodes : @bitset.BitSet
query_summary : HotTupleOptimizationQuerySummary
} derive(Eq, Debug)
///|
struct HotTupleOptimizationRootSlot {
region_ref : @ir.HotRegionRef
lane_idx : Int32
slot : Int32
} derive(Eq, Debug)
///|
priv struct HotTupleCopyRegionPublications {
region : @ir.HotRegionRef
prefixes : Map[Int32, Array[Int32]]
}
///|
priv struct HotTupleCopyPublicationBatch {
stable_loops : @bitset.BitSet
regions : Map[Int32, HotTupleCopyRegionPublications]
}
///|
fn HotTupleCopyPublicationBatch::new(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
groups : Array[HotTupleOptimizationGroup],
) -> HotTupleCopyPublicationBatch {
let stable_loops = @bitset.BitSet::new(@ir.hot_node_count(func))
for root in func.roots {
if @ir.hot_node_get(func, root).op is @ir.HotOp::Loop {
let sites = @ir.node_use_sites_borrowed(use_def, root)
if sites.length() == 1 &&
sites[0].kind is @ir.HotUseSlotKind::RootUseSlot &&
sites[0].user == @ir.HotRegionRef::root().holder {
stable_loops[root] = true
}
}
}
// Only function-root loops outside every tuple group can be queued. Nested
// holders could be cleared or moved by another selected producer's rewrite.
for group in groups {
if group.producer >= 0 && group.producer < stable_loops.length() {
stable_loops[group.producer] = false
}
}
{ stable_loops, regions: Map([]), }
}
///|
fn tuple_optimization_queue_copy_publication(
batch : HotTupleCopyPublicationBatch,
region : @ir.HotRegionRef,
anchor : Int32,
prefix : Array[Int32],
) -> Bool {
if region != @ir.HotRegionRef::root() &&
(
region.holder < 0 ||
region.holder >= batch.stable_loops.length() ||
!batch.stable_loops[region.holder]
) {
return false
}
let publications = match batch.regions.get(region.holder) {
Some(publications) => publications
None => {
let publications : HotTupleCopyRegionPublications = {
region,
prefixes: Map([]),
}
batch.regions[region.holder] = publications
publications
}
}
if publications.prefixes.contains(anchor) {
abort("tuple-optimization copy publication anchor must be unique")
}
publications.prefixes[anchor] = prefix
true
}
///|
fn tuple_optimization_flush_copy_publications(
ctx : HotPassContext,
func : @ir.HotFunc,
batch : HotTupleCopyPublicationBatch,
) -> Unit {
if batch.regions.is_empty() {
return
}
let seen = @bitset.BitSet::new(@ir.hot_node_count(func))
for _, publications in batch.regions {
let region = publications.region
if region != @ir.HotRegionRef::root() &&
(
!@ir.hot_node_is_live(func, region.holder) ||
!(@ir.hot_node_get(func, region.holder).op is @ir.HotOp::Loop)
) {
abort("tuple-optimization queued loop holder must survive writeback")
}
let roots : Array[Int32] = []
let mut anchors = 0
for slot in 0..<@ir.hot_region_root_count(func, region) {
let root = @ir.hot_region_get(func, region, slot)
match publications.prefixes.get(root) {
Some(prefix) => {
if seen[root] {
abort(
"tuple-optimization copy publication anchor must have one root",
)
}
seen[root] = true
anchors += 1
for node in prefix {
roots.push(node)
}
}
None => ()
}
roots.push(root)
}
if anchors != publications.prefixes.size() {
abort("tuple-optimization copy publication anchor must remain live")
}
// The region API preserves loop entry operands before the body span.
@ir.hot_region_replace_body(func, region, roots)
pass_mark_mutated(ctx, func)
}
}
///|
struct HotTupleOptimizationRewriteCleanupPlan {
removable_root_defs : @bitset.BitSet
removable_root_regions : Array[@ir.HotRegionRef]
removable_root_region_seen : @bitset.BitSet
mut removable_root_region_root_seen : Bool
mut generated_nops : Bool
mut had_preexisting_nops : Bool
skip_drop_local_tee_cleanup : Bool
skip_scalarized_tuple_local_cleanup : Bool
} derive(Eq, Debug)
///|
struct HotTupleOptimizationSeedCollection {
groups : Array[HotTupleOptimizationGroup]
simple_root_cleanup_plan : HotTupleOptimizationRewriteCleanupPlan?
} derive(Eq, Debug)
///|
struct HotTupleOptimizationDropOnlyElisionFacts {
mask : @bitset.BitSet
payload_lanes_by_group : Array[Array[Int32]]
} derive(Eq, Debug)
///|
struct HotTupleOptimizationCopyConsumers {
children_by_source : Array[Array[Int32]]
has_host_consumer : @bitset.BitSet
}
///|
fn tuple_optimization_build_copy_consumers(
groups : Array[HotTupleOptimizationGroup],
) -> HotTupleOptimizationCopyConsumers {
let children_by_source : Array[Array[Int32]] = []
for _ in groups {
children_by_source.push([])
}
let has_host_consumer = @bitset.BitSet::new(groups.length())
for child_id = 0; child_id < groups.length(); child_id = child_id + 1 {
match groups[child_id].copy_source {
Some(source_id) if source_id >= 0 && source_id < groups.length() => {
children_by_source[source_id].push(child_id)
if child_id != source_id &&
tuple_optimization_group_host_lane(groups[child_id]) is Some(_) {
has_host_consumer[source_id] = true
}
}
_ => ()
}
}
{ children_by_source, has_host_consumer, }
}
///|
pub impl Show for HotTupleOptimizationGroup with fn output(self, logger) {
logger.write_string("HotTupleOptimizationGroup(")
logger.write_string("producer=")
self.producer.output(logger)
logger.write_string(", ")
logger.write_string("lane_locals=")
self.lane_locals.output(logger)
logger.write_string(", ")
logger.write_string("defining_nodes=")
self.defining_nodes.output(logger)
logger.write_string(", ")
logger.write_string("host_lane=")
match self.host_lane {
Some(host_lane) => host_lane.output(logger)
None => logger.write_string("None")
}
logger.write_string(", ")
logger.write_string("copy_source=")
match self.copy_source {
Some(copy_source) => copy_source.output(logger)
None => logger.write_string("None")
}
logger.write_string(", ")
logger.write_string("copy_payload_locals=")
match self.copy_payload_locals {
Some(copy_payload_locals) => copy_payload_locals.output(logger)
None => logger.write_string("None")
}
logger.write_string(", ")
logger.write_string("has_copy_payload_candidate=")
self.has_copy_payload_candidate.output(logger)
logger.write_string(", ")
logger.write_string("uses=")
self.uses.output(logger)
logger.write_string(", ")
logger.write_string("valid_uses=")
self.valid_uses.output(logger)
logger.write_string(", ")
logger.write_string("neighbors=")
self.neighbors.output(logger)
logger.write_string(", ")
logger.write_string("bad=")
self.bad.output(logger)
logger.write_char(')')
}
///|
pub fn tuple_optimization_descriptor() -> @ir.HotPassDescriptor {
@ir.HotPassDescriptor::new(
"tuple-optimization",
requires=[@ir.HotAnalysis::use_def()],
invalidates=[
@ir.HotAnalysis::cfg(),
@ir.HotAnalysis::dominance(),
@ir.HotAnalysis::liveness(),
@ir.HotAnalysis::use_def(),
@ir.HotAnalysis::effects(),
@ir.HotAnalysis::loop_info(),
@ir.HotAnalysis::ssa(),
],
)
}
///|
pub fn tuple_optimization_summary() -> String {
"Split exact HOT multivalue spill and copy bridges into fresh scalar locals before later local cleanup."
}
///|
fn tuple_optimization_count_tee_defs(
func : @ir.HotFunc,
defining_nodes : Array[Int32],
) -> Int32 {
let mut count = 0
for node_id in defining_nodes {
match @ir.hot_node_get(func, node_id).op {
@ir.HotOp::LocalTee => count = count + 1
_ => ()
}
}
count
}
///|
fn tuple_optimization_collect_seed_group(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
producer : Int32,
local_seen_marks : Array[Int32],
seen_mark : Int32,
) -> HotTupleOptimizationGroup? {
let result_arity = @ir.hot_node_result_arity(func, producer)
if result_arity <= 1 {
return None
}
let use_sites = @ir.node_use_sites_borrowed(use_def, producer)
if use_sites.length() != result_arity {
return None
}
let lane_locals = Array::make(result_arity, -1)
let defining_nodes = Array::make(result_arity, -1)
let mut host_lane : Int32? = None
let mut tee_defs = 0
for site_idx = 0; site_idx < use_sites.length(); site_idx = site_idx + 1 {
let site = use_sites[site_idx]
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot => if site.slot != 0 { return None }
_ => return None
}
let user = @ir.hot_node_get(func, site.user)
match user.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee => ()
_ => return None
}
let local_id = user.imm0
if local_id < 0 ||
local_id >= local_seen_marks.length() ||
local_seen_marks[local_id] == seen_mark {
return None
}
local_seen_marks[local_id] = seen_mark
let lane_idx = result_arity - 1 - site_idx
lane_locals[lane_idx] = local_id
defining_nodes[lane_idx] = site.user
match user.op {
@ir.HotOp::LocalTee => {
tee_defs = tee_defs + 1
match host_lane {
Some(_) => ()
None => host_lane = Some(lane_idx)
}
}
_ => ()
}
}
Some({
producer,
lane_locals,
defining_nodes,
host_lane,
copy_source: None,
copy_payload_locals: None,
has_copy_payload_candidate: tuple_optimization_block_has_copy_payload_candidate(
func, producer, result_arity,
),
uses: 1 + tee_defs,
valid_uses: 1 + tee_defs,
neighbors: [],
bad: false,
})
}
///|
fn tuple_optimization_collect_seed_groups(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
) -> Array[HotTupleOptimizationGroup] {
let groups : Array[HotTupleOptimizationGroup] = []
let local_seen_marks = Array::make(@ir.hot_local_count(func), 0)
let mut next_seen_mark = 1
for node_id = 0; node_id < @ir.hot_node_count(func); node_id = node_id + 1 {
if !@ir.hot_node_is_live(func, node_id) {
continue
}
if @ir.hot_node_result_arity(func, node_id) <= 1 {
continue
}
let seen_mark = next_seen_mark
next_seen_mark = next_seen_mark + 1
match
tuple_optimization_collect_seed_group(
func, use_def, node_id, local_seen_marks, seen_mark,
) {
Some(group) => groups.push(group)
None => ()
}
}
groups
}
///|
fn tuple_optimization_classify_direct_simple_drop_only_root_elision_node(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
producer : Int32,
cleanup_plan : HotTupleOptimizationRewriteCleanupPlan,
local_seen_marks : Array[Int32],
seen_mark : Int32,
) -> Int32 {
let result_arity = @ir.hot_node_result_arity(func, producer)
if result_arity <= 1 {
return 0
}
let use_sites = @ir.node_use_sites_borrowed(use_def, producer)
if use_sites.length() != result_arity {
return 0
}
let mut lane0_local = -1
let mut lane1_local = -1
let mut lane0_def = -1
let mut lane1_def = -1
let mut has_tee_def = false
for site_idx = 0; site_idx < use_sites.length(); site_idx = site_idx + 1 {
let site = use_sites[site_idx]
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot => if site.slot != 0 { return 0 }
_ => return 0
}
let user = @ir.hot_node_get(func, site.user)
match user.op {
@ir.HotOp::LocalSet => ()
@ir.HotOp::LocalTee => has_tee_def = true
_ => return 0
}
let local_id = user.imm0
if local_id < 0 ||
local_id >= local_seen_marks.length() ||
local_seen_marks[local_id] == seen_mark {
return 0
}
local_seen_marks[local_id] = seen_mark
if result_arity == 2 {
let lane_idx = result_arity - 1 - site_idx
if lane_idx == 0 {
lane0_local = local_id
lane0_def = site.user
} else {
lane1_local = local_id
lane1_def = site.user
}
}
}
if result_arity != 2 || has_tee_def {
return 2
}
if lane0_local < 0 || lane1_local < 0 || lane0_def < 0 || lane1_def < 0 {
return 2
}
if !tuple_optimization_node_has_typeidx_control_type(func, producer) {
return 2
}
if tuple_optimization_require_local_type(func, lane0_local) !=
@lib.ValType::i32() ||
tuple_optimization_require_local_type(func, lane1_local) !=
@lib.ValType::i64() {
return 2
}
if !tuple_optimization_simple_block_has_exact_simple_payload_lanes(
func, producer, 2,
) {
return 2
}
let lane_locals = [lane0_local, lane1_local]
let lane_defs = [lane0_def, lane1_def]
for lane_idx = 0; lane_idx < 2; lane_idx = lane_idx + 1 {
let defining_node = lane_defs[lane_idx]
if defining_node < 0 ||
defining_node >= cleanup_plan.removable_root_defs.length() ||
!@ir.hot_node_is_live(func, defining_node) {
return 2
}
match @ir.hot_node_get(func, defining_node).op {
@ir.HotOp::LocalSet => ()
_ => return 2
}
if !tuple_optimization_has_single_function_root_use(use_def, defining_node) {
return 2
}
let local_id = lane_locals[lane_idx]
let writes = @ir.local_write_nodes_borrowed(use_def, local_id)
if writes.length() != 1 || writes[0] != defining_node {
return 2
}
if !tuple_optimization_local_reads_are_direct_drops_without_summary(
func, use_def, local_id,
) {
return 2
}
tuple_optimization_cleanup_plan_add_removable_root(
cleanup_plan,
defining_node,
@ir.HotRegionRef::root(),
)
}
1
}
///|
fn tuple_optimization_try_direct_simple_drop_only_root_elision_fast_path(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
) -> HotTupleOptimizationRewriteCleanupPlan? {
let direct_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:direct-simple-drop-only-root-elision-fast-path",
)
let validate_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path:validate-function-root-groups",
)
let single_root_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path:single-root-region-fast-path",
)
let cleanup_plan : HotTupleOptimizationRewriteCleanupPlan = {
removable_root_defs: @bitset.BitSet::new(@ir.hot_node_count(func)),
removable_root_regions: [],
removable_root_region_seen: @bitset.BitSet::new(@ir.hot_node_count(func)),
removable_root_region_root_seen: false,
generated_nops: false,
had_preexisting_nops: false,
skip_drop_local_tee_cleanup: true,
skip_scalarized_tuple_local_cleanup: true,
}
let local_seen_marks = Array::make(@ir.hot_local_count(func), 0)
let mut next_seen_mark = 1
let mut saw_seed_group = false
for node_id = 0; node_id < @ir.hot_node_count(func); node_id = node_id + 1 {
if !@ir.hot_node_is_live(func, node_id) {
continue
}
match @ir.hot_node_get(func, node_id).op {
@ir.HotOp::Nop => cleanup_plan.had_preexisting_nops = true
_ => ()
}
let seen_mark = next_seen_mark
next_seen_mark = next_seen_mark + 1
match
tuple_optimization_classify_direct_simple_drop_only_root_elision_node(
func, use_def, node_id, cleanup_plan, local_seen_marks, seen_mark,
) {
0 => ()
1 => saw_seed_group = true
_ => {
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path:single-root-region-fast-path",
single_root_start_us,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path:validate-function-root-groups",
validate_start_us,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:direct-simple-drop-only-root-elision-fast-path",
direct_start_us,
)
return None
}
}
}
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path:single-root-region-fast-path",
single_root_start_us,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path:validate-function-root-groups",
validate_start_us,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:direct-simple-drop-only-root-elision-fast-path",
direct_start_us,
)
if saw_seed_group {
Some(cleanup_plan)
} else {
None
}
}
///|
fn tuple_optimization_collect_seed_groups_with_simple_root_elision(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
) -> HotTupleOptimizationSeedCollection {
let collect_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:pre-collect-simple-drop-only-root-elision-fast-path",
)
let validate_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path:validate-function-root-groups",
)
let single_root_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path:single-root-region-fast-path",
)
let groups : Array[HotTupleOptimizationGroup] = []
let cleanup_plan : HotTupleOptimizationRewriteCleanupPlan = {
removable_root_defs: @bitset.BitSet::new(@ir.hot_node_count(func)),
removable_root_regions: [],
removable_root_region_seen: @bitset.BitSet::new(@ir.hot_node_count(func)),
removable_root_region_root_seen: false,
generated_nops: false,
had_preexisting_nops: tuple_optimization_has_live_nop(func),
skip_drop_local_tee_cleanup: true,
skip_scalarized_tuple_local_cleanup: true,
}
let local_seen_marks = Array::make(@ir.hot_local_count(func), 0)
let mut next_seen_mark = 1
let mut simple_root_possible = true
for node_id = 0; node_id < @ir.hot_node_count(func); node_id = node_id + 1 {
if !@ir.hot_node_is_live(func, node_id) {
continue
}
if @ir.hot_node_result_arity(func, node_id) <= 1 {
continue
}
let seen_mark = next_seen_mark
next_seen_mark = next_seen_mark + 1
match
tuple_optimization_collect_seed_group(
func, use_def, node_id, local_seen_marks, seen_mark,
) {
Some(group) => {
if simple_root_possible &&
!tuple_optimization_try_record_simple_drop_only_function_root_elision_group(
func, use_def, group, cleanup_plan,
) {
simple_root_possible = false
}
groups.push(group)
}
None => ()
}
}
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path:single-root-region-fast-path",
single_root_start_us,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path:validate-function-root-groups",
validate_start_us,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:pre-collect-simple-drop-only-root-elision-fast-path",
collect_start_us,
)
{
groups,
simple_root_cleanup_plan: if simple_root_possible && !groups.is_empty() {
Some(cleanup_plan)
} else {
None
},
}
}
///|
fn tuple_optimization_block_has_copy_payload_candidate(
func : @ir.HotFunc,
producer : Int32,
lane_count : Int32,
) -> Bool {
match @ir.hot_node_get(func, producer).op {
@ir.HotOp::Block => ()
_ => return false
}
let body_ref = @ir.hot_region_body_ref(func, producer)
let mut remaining_lanes = lane_count
let mut slot = @ir.hot_region_root_count(func, body_ref) - 1
while remaining_lanes > 0 {
if slot < 0 {
return false
}
let payload_root = @ir.hot_region_get(func, body_ref, slot)
let root_arity = @ir.hot_node_result_arity(func, payload_root)
if root_arity <= 0 || root_arity > remaining_lanes {
return false
}
match tuple_optimization_try_payload_source_read(func, payload_root) {
Some(_) => ()
None => return false
}
remaining_lanes = remaining_lanes - root_arity
slot = slot - 1
}
true
}
///|
fn tuple_optimization_groups_have_copy_payload_candidate(
groups : Array[HotTupleOptimizationGroup],
) -> Bool {
for group in groups {
if group.has_copy_payload_candidate {
return true
}
}
false
}
///|
fn tuple_optimization_build_local_group_ids(
func : @ir.HotFunc,
groups : Array[HotTupleOptimizationGroup],
) -> Array[Int32] {
let local_group_ids = Array::make(@ir.hot_local_count(func), -1)
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
for local_id in groups[group_id].lane_locals {
if local_id < 0 || local_id >= local_group_ids.length() {
abort("tuple-optimization groups must reference existing locals")
}
local_group_ids[local_id] = group_id
}
}
local_group_ids
}
///|
fn tuple_optimization_try_payload_source_read(
func : @ir.HotFunc,
node_id : Int32,
) -> (Int32, Int32)? {
let node = @ir.hot_node_get(func, node_id)
match node.op {
@ir.HotOp::LocalGet => Some((node.imm0, node_id))
@ir.HotOp::LocalTee => None
_ => None
}
}
///|
fn tuple_optimization_resolve_copy_payload_local(
func : @ir.HotFunc,
summary : HotTupleOptimizationQuerySummary,
local_group_ids : Array[Int32],
local_id : Int32,
visit_mark : Int32,
) -> (Int32, Array[Int32])? {
let alias_read_nodes : Array[Int32] = []
let mut current_local = local_id
while true {
if current_local < 0 || current_local >= local_group_ids.length() {
return None
}
if local_group_ids[current_local] >= 0 {
return Some((current_local, alias_read_nodes))
}
if current_local >= summary.resolve_visit_marks.length() ||
summary.resolve_visit_marks[current_local] == visit_mark {
return None
}
summary.resolve_visit_marks[current_local] = visit_mark
if current_local >= summary.local_single_write_nodes.length() {
return None
}
let write_id = summary.local_single_write_nodes[current_local]
if write_id < 0 {
return None
}
let write = @ir.hot_node_get(func, write_id)
match write.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee => ()
_ => return None
}
if write.imm0 != current_local || @ir.hot_child_count(func, write_id) != 1 {
return None
}
let child = @ir.hot_child_get(func, write_id, 0)
let child_node = @ir.hot_node_get(func, child)
let next_local = match child_node.op {
@ir.HotOp::LocalGet => child_node.imm0
@ir.HotOp::LocalTee => child_node.imm0
_ => return None
}
alias_read_nodes.push(child)
current_local = next_local
}
None
}
///|
fn tuple_optimization_collect_copy_payload_lane_nodes(
func : @ir.HotFunc,
payload_roots : Array[Int32],
) -> Array[Int32]? {
let payload_lane_nodes : Array[Int32] = []
for root_id in payload_roots {
let root = @ir.hot_node_get(func, root_id)
let root_arity = @ir.hot_node_result_arity(func, root_id)
if root_arity <= 0 {
return None
}
if root_arity == 1 {
match root.op {
@ir.HotOp::Block => ()
_ => {
payload_lane_nodes.push(root_id)
continue
}
}
}
match root.op {
@ir.HotOp::Block =>
match
@ir.hot_region_split_trailing_payload_roots(
func,
@ir.hot_region_body_ref(func, root_id),
root_arity,
) {
Some((prefix, nested_payload)) if prefix.is_empty() =>
for lane_node in nested_payload {
payload_lane_nodes.push(lane_node)
}
_ => return None
}
_ => return None
}
}
Some(payload_lane_nodes)
}
///|
fn tuple_optimization_pick_copy_lane_source_node(
func : @ir.HotFunc,
source_value : Int32,
lane_idx : Int32,
lane_count : Int32,
) -> Int32? {
match
tuple_optimization_collect_copy_payload_lane_nodes(func, [source_value]) {
Some(source_lane_nodes) =>
if source_lane_nodes.length() == 1 {
Some(source_lane_nodes[0])
} else if lane_idx >= 0 &&
lane_idx < source_lane_nodes.length() &&
source_lane_nodes.length() == lane_count {
Some(source_lane_nodes[lane_idx])
} else {
None
}
None => None
}
}
///|
fn tuple_optimization_add_neighbor(
neighbors : Array[Int32],
neighbor : Int32,
) -> Unit {
if !neighbors.is_empty() {
let last = neighbors[neighbors.length() - 1]
if neighbor == last {
return
}
if neighbor > last {
neighbors.push(neighbor)
return
}
}
for existing in neighbors {
if existing == neighbor {
return
}
}
neighbors.push(neighbor)
}
///|
fn tuple_optimization_link_copy_groups(
func : @ir.HotFunc,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
local_group_ids : Array[Int32],
copy_read_nodes : @bitset.BitSet,
copy_write_nodes : @bitset.BitSet,
) -> Unit {
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
let producer = groups[group_id].producer
match @ir.hot_node_get(func, producer).op {
@ir.HotOp::Block => ()
_ => continue
}
let body_ref = @ir.hot_region_body_ref(func, producer)
// Copy forwarding retains lane reads, not preceding roots. Keep a prefix
// inside its opaque producer so calls, writes, and traps still execute.
let payload = match
@ir.hot_region_split_trailing_payload_roots(
func,
body_ref,
groups[group_id].lane_locals.length(),
) {
Some((prefix, payload)) if prefix.is_empty() => payload
_ => continue
}
if payload.length() != groups[group_id].lane_locals.length() {
continue
}
let mut source_group = -1
let read_nodes : Array[Int32] = []
let payload_locals : Array[Int32] = []
let resolved_source_locals : Array[Int32] = []
let mut exact_copy = true
for lane_idx = 0; lane_idx < payload.length(); lane_idx = lane_idx + 1 {
let (local_id, read_node) = match
tuple_optimization_try_payload_source_read(func, payload[lane_idx]) {
Some(found) => found
None => {
exact_copy = false
break
}
}
payload_locals.push(local_id)
let (resolved_local, alias_read_nodes) = match
tuple_optimization_resolve_copy_payload_local(
func,
summary,
local_group_ids,
local_id,
summary.next_resolve_visit_mark,
) {
Some(found) => found
None => {
exact_copy = false
break
}
}
summary.next_resolve_visit_mark = summary.next_resolve_visit_mark + 1
if resolved_local < 0 || resolved_local >= local_group_ids.length() {
exact_copy = false
break
}
let mut current_source = local_group_ids[resolved_local]
if current_source == group_id &&
source_group >= 0 &&
source_group < groups.length() &&
groups[group_id].lane_locals[lane_idx] == resolved_local &&
lane_idx < groups[source_group].lane_locals.length() &&
groups[source_group].lane_locals[lane_idx] == resolved_local {
current_source = source_group
}
if current_source < 0 {
exact_copy = false
break
}
if source_group < 0 {
source_group = current_source
} else if source_group != current_source {
exact_copy = false
break
}
if groups[current_source].lane_locals.length() != payload.length() ||
groups[current_source].lane_locals[lane_idx] != resolved_local {
exact_copy = false
break
}
read_nodes.push(read_node)
for alias_read_node in alias_read_nodes {
read_nodes.push(alias_read_node)
}
resolved_source_locals.push(resolved_local)
}
if !exact_copy || source_group < 0 || source_group == group_id {
continue
}
groups[group_id].copy_source = Some(source_group)
groups[group_id].copy_payload_locals = Some(payload_locals)
tuple_optimization_add_neighbor(groups[group_id].neighbors, source_group)
tuple_optimization_add_neighbor(groups[source_group].neighbors, group_id)
groups[source_group].uses = groups[source_group].uses + 1
groups[source_group].valid_uses = groups[source_group].valid_uses + 1
for read_node in read_nodes {
copy_read_nodes[read_node] = true
}
for lane_idx = 0
lane_idx < resolved_source_locals.length()
lane_idx = lane_idx + 1 {
if groups[group_id].lane_locals[lane_idx] !=
resolved_source_locals[lane_idx] {
continue
}
let defining_node = groups[group_id].defining_nodes[lane_idx]
if defining_node >= 0 && defining_node < copy_write_nodes.length() {
copy_write_nodes[defining_node] = true
}
}
}
}
///|
fn tuple_optimization_collect_result_block_copy_group(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
local_group_ids : Array[Int32],
carrier_id : Int32,
local_seen_marks : Array[Int32],
seen_mark : Int32,
forward_stores~ : Bool = false,
) -> (HotTupleOptimizationGroup, Array[Int32])? {
let carrier = @ir.hot_node_get(func, carrier_id)
match carrier.op {
@ir.HotOp::Block => ()
_ => return None
}
if @ir.hot_node_result_arity(func, carrier_id) != 1 {
return None
}
let outer_sites = @ir.node_use_sites_borrowed(use_def, carrier_id)
let mut has_root_site = false
for site in outer_sites {
if site.kind is @ir.HotUseSlotKind::RootUseSlot {
has_root_site = true
break
}
}
// Analysis has not mutated the HOT graph since building use-def. A child-only
// block cannot acquire a root slot here, so avoid the whole-function fallback.
let root_slot = if has_root_site {
tuple_optimization_find_root_slot_with_use_def(func, use_def, carrier_id, 0)
} else {
None
}
let (outer_def, outer_local) = match root_slot {
Some(root_slot) =>
if root_slot.slot + 1 >=
@ir.hot_region_root_count(func, root_slot.region_ref) {
return None
} else {
let next_root = @ir.hot_region_get(
func,
root_slot.region_ref,
root_slot.slot + 1,
)
let next_node = @ir.hot_node_get(func, next_root)
match next_node.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee =>
(next_root, next_node.imm0)
_ => return None
}
}
None => {
if outer_sites.length() != 1 {
return None
}
let outer_site = outer_sites[0]
match outer_site.kind {
@ir.HotUseSlotKind::ChildUseSlot =>
if outer_site.slot != 0 {
return None
}
_ => return None
}
let outer_user = @ir.hot_node_get(func, outer_site.user)
match outer_user.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee =>
(outer_site.user, outer_user.imm0)
_ => return None
}
}
}
let body_ref = @ir.hot_region_body_ref(func, carrier_id)
let body_root_count = @ir.hot_region_root_count(func, body_ref)
if body_root_count < 2 || (forward_stores && body_root_count < 3) {
return None
}
let host_slot = if forward_stores &&
@ir.hot_node_get(func, @ir.hot_region_get(func, body_ref, 0)).op
is @ir.HotOp::LocalGet {
0
} else {
body_root_count - 1
}
let prefix_defs : Array[Int32] = []
let prefix_locals_rev : Array[Int32] = []
let mut prefix_layout = true
for idx = 0; idx < body_root_count; idx = idx + 1 {
if idx == host_slot {
continue
}
let root_id = @ir.hot_region_get(func, body_ref, idx)
let root = @ir.hot_node_get(func, root_id)
match root.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee =>
if @ir.hot_child_count(func, root_id) == 1 {
prefix_defs.push(root_id)
prefix_locals_rev.push(root.imm0)
} else {
prefix_layout = false
break
}
_ => {
prefix_layout = false
break
}
}
}
if prefix_layout {
let lane_count = prefix_defs.length() + 1
let host_value = @ir.hot_region_get(func, body_ref, host_slot)
if lane_count == body_root_count &&
@ir.hot_node_result_arity(func, host_value) > 0 {
let mut source_group = -1
let read_nodes : Array[Int32] = []
let payload_locals : Array[Int32] = []
let mut ok = true
for lane_idx = 0; lane_idx < lane_count; lane_idx = lane_idx + 1 {
let source_value = if lane_idx == 0 {
host_value
} else {
@ir.hot_child_get(
func,
prefix_defs[if forward_stores {
lane_idx - 1
} else {
prefix_defs.length() - lane_idx
}],
0,
)
}
let source_lane = match
tuple_optimization_pick_copy_lane_source_node(
func, source_value, lane_idx, lane_count,
) {
Some(source_lane) => source_lane
None => {
ok = false
break
}
}
let (local_id, read_node) = match
tuple_optimization_try_payload_source_read(func, source_lane) {
Some(found) => found
None => {
ok = false
break
}
}
let (resolved_local, alias_read_nodes) = match
tuple_optimization_resolve_copy_payload_local(
func,
summary,
local_group_ids,
local_id,
summary.next_resolve_visit_mark,
) {
Some(found) => found
None => {
ok = false
break
}
}
summary.next_resolve_visit_mark = summary.next_resolve_visit_mark + 1
if resolved_local < 0 || resolved_local >= local_group_ids.length() {
ok = false
break
}
let current_source = local_group_ids[resolved_local]
if current_source < 0 || current_source >= groups.length() {
ok = false
break
}
if source_group < 0 {
source_group = current_source
} else if source_group != current_source {
ok = false
break
}
if groups[current_source].lane_locals.length() != lane_count ||
groups[current_source].lane_locals[lane_idx] != resolved_local {
ok = false
break
}
read_nodes.push(read_node)
for alias_read_node in alias_read_nodes {
read_nodes.push(alias_read_node)
}
// Alias resolution identifies the source group. The copy still reads
// the captured local, which can outlive a later producer execution.
payload_locals.push(local_id)
}
if ok && source_group >= 0 {
let lane_locals = [outer_local]
let defining_nodes = [outer_def]
if outer_local < 0 || outer_local >= local_seen_marks.length() {
return None
}
local_seen_marks[outer_local] = seen_mark
for lane = 1; lane < lane_count; lane = lane + 1 {
let idx = if forward_stores {
lane - 1
} else {
prefix_defs.length() - lane
}
let local_id = prefix_locals_rev[idx]
if local_id < 0 ||
local_id >= local_seen_marks.length() ||
local_seen_marks[local_id] == seen_mark {
return None
}
local_seen_marks[local_id] = seen_mark
lane_locals.push(local_id)
defining_nodes.push(prefix_defs[idx])
}
let group : HotTupleOptimizationGroup = {
producer: carrier_id,
lane_locals,
defining_nodes,
host_lane: None,
copy_source: Some(source_group),
copy_payload_locals: Some(payload_locals),
has_copy_payload_candidate: false,
uses: tuple_optimization_count_tee_defs(func, defining_nodes),
valid_uses: tuple_optimization_count_tee_defs(func, defining_nodes),
neighbors: [source_group],
bad: false,
}
// Ascending stores can commute only for owned plain reads/stores with
// disjoint source and destination locals. Keep the general path narrow.
if forward_stores &&
tuple_optimization_direct_scalar_copy_payload_nodes(
func, use_def, group,
)
is None {
return None
}
return Some((group, read_nodes))
}
}
}
if forward_stores {
return None
}
let trailing_defs : Array[Int32] = []
let trailing_locals : Array[Int32] = []
let mut scan = body_root_count - 1
while scan >= 0 {
let root_id = @ir.hot_region_get(func, body_ref, scan)
let root = @ir.hot_node_get(func, root_id)
match root.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee =>
if @ir.hot_child_count(func, root_id) == 1 {
trailing_defs.push(root_id)
trailing_locals.push(root.imm0)
scan = scan - 1
continue
}
_ => ()
}
break
}
if trailing_defs.is_empty() || scan < 0 {
return None
}
let lane_count = trailing_defs.length() + 1
let payload_roots : Array[Int32] = []
for idx = 0; idx <= scan; idx = idx + 1 {
payload_roots.push(@ir.hot_region_get(func, body_ref, idx))
}
let payload = match
@ir.hot_split_trailing_payload_roots(func, payload_roots, lane_count) {
Some((prefix, payload)) if prefix.is_empty() => payload
_ => return None
}
let payload_lane_nodes = match
tuple_optimization_collect_copy_payload_lane_nodes(func, payload) {
Some(payload_lane_nodes) => payload_lane_nodes
None => return None
}
if payload_lane_nodes.length() != lane_count {
return None
}
let mut source_group = -1
let read_nodes : Array[Int32] = []
let payload_locals : Array[Int32] = []
for lane_idx = 0
lane_idx < payload_lane_nodes.length()
lane_idx = lane_idx + 1 {
let (local_id, read_node) = match
tuple_optimization_try_payload_source_read(
func,
payload_lane_nodes[lane_idx],
) {
Some(found) => found
None => return None
}
let (resolved_local, alias_read_nodes) = match
tuple_optimization_resolve_copy_payload_local(
func,
summary,
local_group_ids,
local_id,
summary.next_resolve_visit_mark,
) {
Some(found) => found
None => return None
}
summary.next_resolve_visit_mark = summary.next_resolve_visit_mark + 1
if resolved_local < 0 || resolved_local >= local_group_ids.length() {
return None
}
let current_source = local_group_ids[resolved_local]
if current_source < 0 || current_source >= groups.length() {
return None
}
if source_group < 0 {
source_group = current_source
} else if source_group != current_source {
return None
}
if groups[current_source].lane_locals.length() != lane_count ||
groups[current_source].lane_locals[lane_idx] != resolved_local {
return None
}
read_nodes.push(read_node)
for alias_read_node in alias_read_nodes {
read_nodes.push(alias_read_node)
}
// Keep the local read by this lane; resolved_local is provenance only.
payload_locals.push(local_id)
}
if source_group < 0 {
return None
}
let lane_locals = [outer_local]
let defining_nodes = [outer_def]
if outer_local < 0 || outer_local >= local_seen_marks.length() {
return None
}
local_seen_marks[outer_local] = seen_mark
for idx = 0; idx < trailing_defs.length(); idx = idx + 1 {
let local_id = trailing_locals[idx]
if local_id < 0 ||
local_id >= local_seen_marks.length() ||
local_seen_marks[local_id] == seen_mark {
return None
}
local_seen_marks[local_id] = seen_mark
lane_locals.push(local_id)
defining_nodes.push(trailing_defs[idx])
}
Some(
(
{
producer: carrier_id,
lane_locals,
defining_nodes,
host_lane: None,
copy_source: Some(source_group),
copy_payload_locals: Some(payload_locals),
has_copy_payload_candidate: false,
uses: tuple_optimization_count_tee_defs(func, defining_nodes),
valid_uses: tuple_optimization_count_tee_defs(func, defining_nodes),
neighbors: [source_group],
bad: false,
},
read_nodes,
),
)
}
///|
fn tuple_optimization_link_result_block_copy_groups(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
local_group_ids : Array[Int32],
copy_read_nodes : @bitset.BitSet,
) -> Unit {
let local_seen_marks = Array::make(@ir.hot_local_count(func), 0)
let mut next_seen_mark = 1
for carrier_id = 0
carrier_id < @ir.hot_node_count(func)
carrier_id = carrier_id + 1 {
if !@ir.hot_node_is_live(func, carrier_id) {
continue
}
let seen_mark = next_seen_mark
next_seen_mark = next_seen_mark + 2
let collected = tuple_optimization_collect_result_block_copy_group(
func, use_def, summary, groups, local_group_ids, carrier_id, local_seen_marks,
seen_mark,
)
let collected = match collected {
Some(_) => collected
None =>
tuple_optimization_collect_result_block_copy_group(
func,
use_def,
summary,
groups,
local_group_ids,
carrier_id,
local_seen_marks,
seen_mark + 1,
forward_stores=true,
)
}
match collected {
Some((group, read_nodes)) => {
let new_group_id = groups.length()
groups.push(group)
for read_node in read_nodes {
if read_node >= 0 && read_node < copy_read_nodes.length() {
copy_read_nodes[read_node] = true
}
}
let source_group = match group.copy_source {
Some(source_group) => source_group
None => continue
}
if source_group >= 0 && source_group < groups.length() {
tuple_optimization_add_neighbor(
groups[source_group].neighbors,
new_group_id,
)
groups[source_group].uses = groups[source_group].uses + 1
groups[source_group].valid_uses = groups[source_group].valid_uses + 1
}
}
None => ()
}
}
}
///|
fn tuple_optimization_collect_scalar_forward_copy_group(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
local_group_ids : Array[Int32],
copy_read_nodes : @bitset.BitSet,
source_group_id : Int32,
local_seen_marks : Array[Int32],
seen_mark : Int32,
) -> (HotTupleOptimizationGroup, Array[Int32], Int32)? {
if source_group_id < 0 || source_group_id >= groups.length() {
return None
}
let source = groups[source_group_id]
let source_payload_locals = match source.copy_payload_locals {
Some(copy_payload_locals) => copy_payload_locals
None => source.lane_locals
}
if source_payload_locals.length() != source.lane_locals.length() {
return None
}
let lane_locals : Array[Int32] = []
let defining_nodes : Array[Int32] = []
let read_nodes : Array[Int32] = []
let mut consumed_source_tees = 0
let mut has_forwarded_lane = false
for lane_idx = 0
lane_idx < source.lane_locals.length()
lane_idx = lane_idx + 1 {
let local_id = source.lane_locals[lane_idx]
let mut forwarded_local = -1
let mut forwarding_def = -1
let mut forwarding_read = -1
let mut found = false
let mut direct_non_forward_reads = 0
for read_id in @ir.local_read_nodes_borrowed(use_def, local_id) {
if read_id >= 0 &&
read_id < copy_read_nodes.length() &&
copy_read_nodes[read_id] {
continue
}
match tuple_optimization_scalar_forward_local(summary, read_id) {
Some((next_local, next_def, _)) => {
if found ||
next_local < 0 ||
next_local >= local_group_ids.length() ||
local_group_ids[next_local] >= 0 ||
next_local >= local_seen_marks.length() ||
local_seen_marks[next_local] == seen_mark {
return None
}
found = true
forwarded_local = next_local
forwarding_def = next_def
forwarding_read = read_id
}
None => direct_non_forward_reads = direct_non_forward_reads + 1
}
}
if !found {
if direct_non_forward_reads == 1 {
if local_id < 0 ||
local_id >= local_seen_marks.length() ||
local_seen_marks[local_id] == seen_mark {
return None
}
found = true
forwarded_local = local_id
forwarding_def = source.defining_nodes[lane_idx]
} else {
let defining_node = source.defining_nodes[lane_idx]
match @ir.hot_node_get(func, defining_node).op {
@ir.HotOp::LocalTee => {
let sites = @ir.node_use_sites_borrowed(use_def, defining_node)
if sites.length() != 1 {
return None
}
let site = sites[0]
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot => ()
_ => return None
}
let user = @ir.hot_node_get(func, site.user)
match user.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee => {
let next_local = user.imm0
if next_local < 0 ||
next_local >= local_group_ids.length() ||
local_group_ids[next_local] >= 0 ||
next_local >= local_seen_marks.length() ||
local_seen_marks[next_local] == seen_mark ||
next_local >= summary.local_single_write_nodes.length() ||
summary.local_single_write_nodes[next_local] < 0 {
return None
}
found = true
forwarded_local = next_local
forwarding_def = site.user
consumed_source_tees = consumed_source_tees + 1
}
_ => return None
}
}
_ => ()
}
}
}
if !found {
return None
}
local_seen_marks[forwarded_local] = seen_mark
lane_locals.push(forwarded_local)
defining_nodes.push(forwarding_def)
if forwarding_read >= 0 {
read_nodes.push(forwarding_read)
if forwarded_local != local_id {
has_forwarded_lane = true
}
}
}
if !has_forwarded_lane {
return None
}
Some(
(
{
producer: -1,
lane_locals,
defining_nodes,
host_lane: source.host_lane,
copy_source: Some(source_group_id),
copy_payload_locals: Some(source_payload_locals),
has_copy_payload_candidate: false,
uses: tuple_optimization_count_tee_defs(func, defining_nodes),
valid_uses: tuple_optimization_count_tee_defs(func, defining_nodes),
neighbors: [source_group_id],
bad: false,
},
read_nodes,
consumed_source_tees,
),
)
}
///|
fn tuple_optimization_link_scalar_forward_copy_groups(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
local_group_ids : Array[Int32],
copy_read_nodes : @bitset.BitSet,
) -> Unit {
let local_seen_marks = Array::make(@ir.hot_local_count(func), 0)
let mut next_seen_mark = 1
let mut group_id = 0
while group_id < groups.length() {
let group = groups[group_id]
let mut can_link = false
match group.copy_source {
Some(source_group) =>
if group.producer < 0 &&
source_group >= 0 &&
source_group < groups.length() {
for lane_idx = 0
lane_idx < group.lane_locals.length()
lane_idx = lane_idx + 1 {
if lane_idx < groups[source_group].lane_locals.length() &&
group.lane_locals[lane_idx] ==
groups[source_group].lane_locals[lane_idx] {
can_link = true
break
}
}
}
None =>
if group.producer >= 0 {
match @ir.hot_node_get(func, group.producer).op {
@ir.HotOp::Block => ()
_ => can_link = true
}
}
}
if !can_link {
group_id = group_id + 1
continue
}
let seen_mark = next_seen_mark
next_seen_mark = next_seen_mark + 1
match
tuple_optimization_collect_scalar_forward_copy_group(
func, use_def, summary, groups, local_group_ids, copy_read_nodes, group_id,
local_seen_marks, seen_mark,
) {
Some((group, read_nodes, consumed_source_tees)) => {
let new_group_id = groups.length()
groups.push(group)
for local_id in group.lane_locals {
if local_id < 0 || local_id >= local_group_ids.length() {
abort("tuple-optimization scalar copy groups must stay in range")
}
local_group_ids[local_id] = new_group_id
}
tuple_optimization_add_neighbor(
groups[group_id].neighbors,
new_group_id,
)
groups[group_id].uses = groups[group_id].uses + 1 - consumed_source_tees
groups[group_id].valid_uses = groups[group_id].valid_uses +
1 -
consumed_source_tees
for read_node in read_nodes {
if read_node >= 0 && read_node < copy_read_nodes.length() {
copy_read_nodes[read_node] = true
}
}
}
None => ()
}
group_id = group_id + 1
}
}
///|
fn tuple_optimization_build_query_summary(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
) -> HotTupleOptimizationQuerySummary {
let node_count = @ir.hot_node_count(func)
let local_count = @ir.hot_local_count(func)
let node_has_direct_non_scalar_use = @bitset.BitSet::new(node_count)
let node_has_direct_non_drop_use = @bitset.BitSet::new(node_count)
let node_has_direct_drop_use = @bitset.BitSet::new(node_count)
let scalar_forward_present = @bitset.BitSet::new(node_count)
let scalar_forward_locals = Array::make(node_count, -1)
let scalar_forward_defs = Array::make(node_count, -1)
let scalar_forward_is_tee = @bitset.BitSet::new(node_count)
let local_single_write_nodes = Array::make(local_count, -1)
let local_is_locally_tee_defined = @bitset.BitSet::new(local_count)
let mut has_local_tee_defs = false
let mut has_scalarized_tuple_local_cleanup_candidate = false
let mut has_single_result_block_copy_candidate = false
let mut has_scalar_forward_copy_candidate = false
let forwarded_non_scalar_use = Array::make(local_count, -1)
let forwarded_non_drop_use = Array::make(local_count, -1)
let forwarded_drop_use = Array::make(local_count, -1)
let forwarded_any_use = Array::make(local_count, -1)
let forwarded_host_tee_use = Array::make(local_count, -1)
let visit_marks = Array::make(local_count, 0)
let resolve_visit_marks = Array::make(local_count, 0)
for local_id = 0; local_id < local_count; local_id = local_id + 1 {
let writes = @ir.local_write_nodes_borrowed(use_def, local_id)
if writes.length() == 1 {
local_single_write_nodes[local_id] = writes[0]
}
for write_id in writes {
if !@ir.hot_node_is_live(func, write_id) {
continue
}
let write = @ir.hot_node_get(func, write_id)
match write.op {
@ir.HotOp::LocalTee => {
local_is_locally_tee_defined[local_id] = true
has_local_tee_defs = true
}
@ir.HotOp::LocalSet =>
if @ir.hot_child_count(func, write_id) == 1 {
let child_id = @ir.hot_child_get(func, write_id, 0)
if @ir.hot_node_is_live(func, child_id) {
match @ir.hot_node_get(func, child_id).op {
@ir.HotOp::TupleMake =>
if @ir.hot_child_count(func, child_id) > 1 {
has_scalarized_tuple_local_cleanup_candidate = true
}
_ => ()
}
}
}
_ => ()
}
}
}
for node_id = 0; node_id < node_count; node_id = node_id + 1 {
if !@ir.hot_node_is_live(func, node_id) {
continue
}
if !has_single_result_block_copy_candidate &&
@ir.hot_node_result_arity(func, node_id) == 1 {
match @ir.hot_node_get(func, node_id).op {
@ir.HotOp::Block => has_single_result_block_copy_candidate = true
_ => ()
}
}
let sites = @ir.node_use_sites_borrowed(use_def, node_id)
let mut has_non_scalar = false
let mut has_non_drop = false
let mut has_drop = false
for site in sites {
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot =>
match @ir.hot_node_get(func, site.user).op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee => ()
@ir.HotOp::Drop => {
has_non_scalar = true
has_drop = true
}
_ => {
has_non_scalar = true
has_non_drop = true
}
}
_ => {
has_non_scalar = true
has_non_drop = true
}
}
}
node_has_direct_non_scalar_use[node_id] = has_non_scalar
node_has_direct_non_drop_use[node_id] = has_non_drop
node_has_direct_drop_use[node_id] = has_drop
if sites.length() != 1 {
continue
}
let site = sites[0]
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot => ()
_ => continue
}
let user = @ir.hot_node_get(func, site.user)
match user.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee => {
let forwarded_local = user.imm0
if forwarded_local < 0 ||
forwarded_local >= local_count ||
local_single_write_nodes[forwarded_local] < 0 {
continue
}
scalar_forward_present[node_id] = true
has_scalar_forward_copy_candidate = true
scalar_forward_locals[node_id] = forwarded_local
scalar_forward_defs[node_id] = site.user
scalar_forward_is_tee[node_id] = match user.op {
@ir.HotOp::LocalTee => true
_ => false
}
}
_ => ()
}
}
{
node_has_direct_non_scalar_use,
node_has_direct_non_drop_use,
node_has_direct_drop_use,
scalar_forward_present,
scalar_forward_locals,
scalar_forward_defs,
scalar_forward_is_tee,
local_single_write_nodes,
local_is_locally_tee_defined,
has_local_tee_defs,
has_scalarized_tuple_local_cleanup_candidate,
has_single_result_block_copy_candidate,
has_scalar_forward_copy_candidate,
forwarded_non_scalar_use,
forwarded_non_drop_use,
forwarded_drop_use,
forwarded_any_use,
forwarded_host_tee_use,
visit_marks,
next_visit_mark: 1,
resolve_visit_marks,
next_resolve_visit_mark: 1,
}
}
///|
fn tuple_optimization_empty_query_summary() -> HotTupleOptimizationQuerySummary {
{
node_has_direct_non_scalar_use: @bitset.BitSet::new(0),
node_has_direct_non_drop_use: @bitset.BitSet::new(0),
node_has_direct_drop_use: @bitset.BitSet::new(0),
scalar_forward_present: @bitset.BitSet::new(0),
scalar_forward_locals: Array::make(0, -1),
scalar_forward_defs: Array::make(0, -1),
scalar_forward_is_tee: @bitset.BitSet::new(0),
local_single_write_nodes: Array::make(0, -1),
local_is_locally_tee_defined: @bitset.BitSet::new(0),
has_local_tee_defs: false,
has_scalarized_tuple_local_cleanup_candidate: false,
has_single_result_block_copy_candidate: false,
has_scalar_forward_copy_candidate: false,
forwarded_non_scalar_use: Array::make(0, -1),
forwarded_non_drop_use: Array::make(0, -1),
forwarded_drop_use: Array::make(0, -1),
forwarded_any_use: Array::make(0, -1),
forwarded_host_tee_use: Array::make(0, -1),
visit_marks: Array::make(0, 0),
next_visit_mark: 1,
resolve_visit_marks: Array::make(0, 0),
next_resolve_visit_mark: 1,
}
}
///|
fn tuple_optimization_node_has_direct_non_scalar_use(
summary : HotTupleOptimizationQuerySummary,
node_id : Int32,
) -> Bool {
if node_id < 0 || node_id >= summary.node_has_direct_non_scalar_use.length() {
return false
}
summary.node_has_direct_non_scalar_use[node_id]
}
///|
fn tuple_optimization_node_has_direct_non_drop_use(
summary : HotTupleOptimizationQuerySummary,
node_id : Int32,
) -> Bool {
if node_id < 0 || node_id >= summary.node_has_direct_non_drop_use.length() {
return false
}
summary.node_has_direct_non_drop_use[node_id]
}
///|
fn tuple_optimization_scalar_forward_local(
summary : HotTupleOptimizationQuerySummary,
read_id : Int32,
) -> (Int32, Int32, Bool)? {
if read_id < 0 ||
read_id >= summary.scalar_forward_present.length() ||
!summary.scalar_forward_present[read_id] {
return None
}
Some(
(
summary.scalar_forward_locals[read_id],
summary.scalar_forward_defs[read_id],
summary.scalar_forward_is_tee[read_id],
),
)
}
///|
fn tuple_optimization_visit_local_once(
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
visit_mark : Int32,
) -> Bool {
if local_id < 0 ||
local_id >= summary.visit_marks.length() ||
summary.visit_marks[local_id] == visit_mark {
return false
}
summary.visit_marks[local_id] = visit_mark
true
}
///|
fn tuple_optimization_query_cache_get(
cache : Array[Int32],
local_id : Int32,
) -> Bool? {
if local_id < 0 || local_id >= cache.length() {
return Some(false)
}
match cache[local_id] {
0 => Some(false)
1 => Some(true)
_ => None
}
}
///|
fn tuple_optimization_query_cache_set(
cache : Array[Int32],
local_id : Int32,
value : Bool,
) -> Bool {
if local_id >= 0 && local_id < cache.length() {
cache[local_id] = if value { 1 } else { 0 }
}
value
}
///|
fn tuple_optimization_take_visit_mark(
summary : HotTupleOptimizationQuerySummary,
) -> Int32 {
let visit_mark = summary.next_visit_mark
summary.next_visit_mark = summary.next_visit_mark + 1
visit_mark
}
///|
fn tuple_optimization_local_has_forwarded_non_scalar_use_with_mark(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
visit_mark : Int32,
) -> Bool {
match
tuple_optimization_query_cache_get(
summary.forwarded_non_scalar_use,
local_id,
) {
Some(result) => return result
None => ()
}
if !tuple_optimization_visit_local_once(summary, local_id, visit_mark) {
return false
}
let mut result = false
for read_id in @ir.local_read_nodes_borrowed(use_def, local_id) {
match tuple_optimization_scalar_forward_local(summary, read_id) {
Some((next_local, next_def, next_is_tee)) => {
if next_is_tee &&
tuple_optimization_node_has_direct_non_scalar_use(summary, next_def) {
result = true
break
}
if tuple_optimization_local_has_forwarded_non_scalar_use_with_mark(
use_def, summary, next_local, visit_mark,
) {
result = true
break
}
}
None => {
result = true
break
}
}
}
tuple_optimization_query_cache_set(
summary.forwarded_non_scalar_use,
local_id,
result,
)
}
///|
fn tuple_optimization_local_has_forwarded_non_scalar_use(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
) -> Bool {
tuple_optimization_local_has_forwarded_non_scalar_use_with_mark(
use_def,
summary,
local_id,
tuple_optimization_take_visit_mark(summary),
)
}
///|
fn tuple_optimization_node_has_direct_drop_use(
summary : HotTupleOptimizationQuerySummary,
node_id : Int32,
) -> Bool {
if node_id < 0 || node_id >= summary.node_has_direct_drop_use.length() {
return false
}
summary.node_has_direct_drop_use[node_id]
}
///|
fn tuple_optimization_find_direct_drop_use(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
node_id : Int32,
) -> Int32? {
for site in @ir.node_use_sites_borrowed(use_def, node_id) {
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot =>
match @ir.hot_node_get(func, site.user).op {
@ir.HotOp::Drop => return Some(site.user)
_ => ()
}
_ => ()
}
}
None
}
///|
fn tuple_optimization_local_has_forwarded_non_drop_use_with_mark(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
visit_mark : Int32,
) -> Bool {
match
tuple_optimization_query_cache_get(summary.forwarded_non_drop_use, local_id) {
Some(result) => return result
None => ()
}
if !tuple_optimization_visit_local_once(summary, local_id, visit_mark) {
return false
}
let mut result = false
for read_id in @ir.local_read_nodes_borrowed(use_def, local_id) {
match tuple_optimization_scalar_forward_local(summary, read_id) {
Some((next_local, next_def, next_is_tee)) => {
if next_is_tee &&
tuple_optimization_node_has_direct_non_drop_use(summary, next_def) {
result = true
break
}
if tuple_optimization_local_has_forwarded_non_drop_use_with_mark(
use_def, summary, next_local, visit_mark,
) {
result = true
break
}
}
None =>
if tuple_optimization_node_has_direct_non_drop_use(summary, read_id) {
result = true
break
}
}
}
tuple_optimization_query_cache_set(
summary.forwarded_non_drop_use,
local_id,
result,
)
}
///|
fn tuple_optimization_local_has_forwarded_non_drop_use(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
) -> Bool {
tuple_optimization_local_has_forwarded_non_drop_use_with_mark(
use_def,
summary,
local_id,
tuple_optimization_take_visit_mark(summary),
)
}
///|
fn tuple_optimization_local_has_forwarded_drop_use_with_mark(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
visit_mark : Int32,
) -> Bool {
match
tuple_optimization_query_cache_get(summary.forwarded_drop_use, local_id) {
Some(result) => return result
None => ()
}
if !tuple_optimization_visit_local_once(summary, local_id, visit_mark) {
return false
}
let mut result = false
for read_id in @ir.local_read_nodes_borrowed(use_def, local_id) {
match tuple_optimization_scalar_forward_local(summary, read_id) {
Some((next_local, next_def, next_is_tee)) => {
if next_is_tee &&
tuple_optimization_node_has_direct_drop_use(summary, next_def) {
result = true
break
}
if tuple_optimization_local_has_forwarded_drop_use_with_mark(
use_def, summary, next_local, visit_mark,
) {
result = true
break
}
}
None =>
if tuple_optimization_node_has_direct_drop_use(summary, read_id) {
result = true
break
}
}
}
tuple_optimization_query_cache_set(
summary.forwarded_drop_use,
local_id,
result,
)
}
///|
fn tuple_optimization_local_has_forwarded_drop_use(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
) -> Bool {
tuple_optimization_local_has_forwarded_drop_use_with_mark(
use_def,
summary,
local_id,
tuple_optimization_take_visit_mark(summary),
)
}
///|
fn tuple_optimization_local_has_forwarded_any_use_with_mark(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
visit_mark : Int32,
) -> Bool {
match
tuple_optimization_query_cache_get(summary.forwarded_any_use, local_id) {
Some(result) => return result
None => ()
}
if !tuple_optimization_visit_local_once(summary, local_id, visit_mark) {
return false
}
let mut result = false
for read_id in @ir.local_read_nodes_borrowed(use_def, local_id) {
match tuple_optimization_scalar_forward_local(summary, read_id) {
Some((next_local, _, _)) =>
if tuple_optimization_local_has_forwarded_any_use_with_mark(
use_def, summary, next_local, visit_mark,
) {
result = true
break
}
None => {
result = true
break
}
}
}
tuple_optimization_query_cache_set(
summary.forwarded_any_use,
local_id,
result,
)
}
///|
fn tuple_optimization_local_has_forwarded_any_use(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
) -> Bool {
tuple_optimization_local_has_forwarded_any_use_with_mark(
use_def,
summary,
local_id,
tuple_optimization_take_visit_mark(summary),
)
}
///|
fn tuple_optimization_local_has_forwarded_host_tee_use_with_mark(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
visit_mark : Int32,
) -> Bool {
match
tuple_optimization_query_cache_get(summary.forwarded_host_tee_use, local_id) {
Some(result) => return result
None => ()
}
if !tuple_optimization_visit_local_once(summary, local_id, visit_mark) {
return false
}
let locally_tee_defined = local_id >= 0 &&
local_id < summary.local_is_locally_tee_defined.length() &&
summary.local_is_locally_tee_defined[local_id]
let mut result = false
for read_id in @ir.local_read_nodes_borrowed(use_def, local_id) {
match tuple_optimization_scalar_forward_local(summary, read_id) {
Some((next_local, next_def, next_is_tee)) => {
if next_is_tee &&
tuple_optimization_node_has_direct_non_scalar_use(summary, next_def) {
result = true
break
}
if tuple_optimization_local_has_forwarded_host_tee_use_with_mark(
use_def, summary, next_local, visit_mark,
) {
result = true
break
}
}
None =>
if locally_tee_defined &&
tuple_optimization_node_has_direct_non_scalar_use(summary, read_id) {
result = true
break
}
}
}
tuple_optimization_query_cache_set(
summary.forwarded_host_tee_use,
local_id,
result,
)
}
///|
fn tuple_optimization_local_has_forwarded_host_tee_use(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
) -> Bool {
tuple_optimization_local_has_forwarded_host_tee_use_with_mark(
use_def,
summary,
local_id,
tuple_optimization_take_visit_mark(summary),
)
}
///|
fn tuple_optimization_read_is_invalid_scalar_copy_through(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
read_id : Int32,
) -> Bool {
match tuple_optimization_scalar_forward_local(summary, read_id) {
Some((forwarded_local, forwarding_def, is_tee)) => {
if is_tee &&
tuple_optimization_node_has_direct_non_scalar_use(
summary, forwarding_def,
) {
return false
}
!tuple_optimization_local_has_forwarded_non_scalar_use(
use_def, summary, forwarded_local,
)
}
None => false
}
}
///|
fn tuple_optimization_finalize_host_lanes(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
) -> Unit {
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
if groups[group_id].host_lane is Some(_) {
continue
}
let mut candidate : Int32? = None
let mut multiple = false
for lane_idx = 0
lane_idx < groups[group_id].lane_locals.length()
lane_idx = lane_idx + 1 {
let defining_node = groups[group_id].defining_nodes[lane_idx]
let has_direct_host_tee_use = match
@ir.hot_node_get(func, defining_node).op {
@ir.HotOp::LocalTee =>
tuple_optimization_node_has_direct_non_scalar_use(
summary, defining_node,
)
_ => false
}
if !has_direct_host_tee_use &&
!tuple_optimization_local_has_forwarded_host_tee_use(
use_def,
summary,
groups[group_id].lane_locals[lane_idx],
) {
continue
}
match candidate {
Some(_) => {
multiple = true
break
}
None => candidate = Some(lane_idx)
}
}
if !multiple {
groups[group_id].host_lane = candidate
}
}
}
///|
fn tuple_optimization_find_overlap_lane_between_groups(
groups : Array[HotTupleOptimizationGroup],
source_group_id : Int32,
copy_group_id : Int32,
local_id : Int32,
) -> Int32? {
if source_group_id < 0 ||
source_group_id >= groups.length() ||
copy_group_id < 0 ||
copy_group_id >= groups.length() {
return None
}
for lane_idx = 0
lane_idx < groups[source_group_id].lane_locals.length()
lane_idx = lane_idx + 1 {
if lane_idx >= groups[copy_group_id].lane_locals.length() {
return None
}
if groups[source_group_id].lane_locals[lane_idx] == local_id &&
groups[copy_group_id].lane_locals[lane_idx] == local_id {
return Some(lane_idx)
}
}
None
}
///|
fn tuple_optimization_count_lane_traffic(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
local_group_ids : Array[Int32],
copy_read_nodes : @bitset.BitSet,
copy_write_nodes : @bitset.BitSet,
) -> Unit {
// Lifting may materialize every lane before an overlapping destination
// write. A single-definition captured lane copied back to its exact source
// lane preserves that value; it is not an unrelated overwrite. Other writes
// still make the source group bad and propagate to its connected copies.
for group in groups {
let source = match group.copy_source {
Some(index) if index >= 0 && index < groups.length() => groups[index]
_ => continue
}
for lane in 0..= source.lane_locals.length() {
continue
}
let payload = match group.copy_payload_locals {
Some(values) if lane < values.length() => values[lane]
_ => continue
}
if payload != source.lane_locals[lane] {
continue
}
let local_id = group.lane_locals[lane]
let writes = @ir.local_write_nodes_borrowed(use_def, local_id)
if writes.length() != 1 || writes[0] != group.defining_nodes[lane] {
continue
}
// Lane-traffic analysis has not changed the HOT graph since use-def was
// built. A captured write with only child uses has no root position.
if !tuple_optimization_has_root_site_in_fresh_use_def(
func,
use_def,
writes[0],
) {
continue
}
let mut first_root_slot : HotTupleOptimizationRootSlot? = None
for read in @ir.local_read_nodes_borrowed(use_def, local_id) {
let sites = @ir.node_use_sites_borrowed(use_def, read)
if sites.length() != 1 {
continue
}
let site = sites[0]
if site.kind is @ir.HotUseSlotKind::ChildUseSlot {
let write = @ir.hot_node_get(func, site.user)
if write.op is @ir.HotOp::LocalSet &&
write.imm0 == source.lane_locals[lane] {
let first = match first_root_slot {
Some(slot) => slot
None =>
match
tuple_optimization_find_root_slot_with_use_def(
func,
use_def,
writes[0],
lane,
) {
Some(slot) => {
first_root_slot = Some(slot)
slot
}
None => continue
}
}
let last = match
tuple_optimization_find_root_slot_with_use_def(
func,
use_def,
site.user,
lane,
) {
Some(slot) => slot
None => continue
}
if first.region_ref != last.region_ref ||
first.slot >= last.slot ||
last.slot - first.slot > 2 * group.lane_locals.length() {
continue
}
let mut unchanged = true
for index = first.slot + 1; index < last.slot; index = index + 1 {
let root = @ir.hot_region_get(func, first.region_ref, index)
let node = @ir.hot_node_get(func, root)
if node.op is @ir.HotOp::LocalSet &&
node.imm0 != write.imm0 &&
(
group.defining_nodes.contains(root) ||
(
node.child_count == 1 &&
@ir.hot_node_get(func, @ir.hot_child_get(func, root, 0)).op
is @ir.HotOp::LocalGet
)
) {
continue
}
unchanged = false
break
}
if unchanged {
copy_write_nodes[site.user] = true
}
}
}
}
}
}
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
for lane_idx = 0
lane_idx < groups[group_id].lane_locals.length()
lane_idx = lane_idx + 1 {
let local_id = groups[group_id].lane_locals[lane_idx]
let defining_node = groups[group_id].defining_nodes[lane_idx]
let mut overlap_owner_lane : Int32? = None
if local_id >= 0 && local_id < local_group_ids.length() {
let owner_group = local_group_ids[local_id]
if owner_group >= 0 &&
owner_group != group_id &&
owner_group < groups.length() &&
groups[owner_group].copy_source == Some(group_id) {
overlap_owner_lane = tuple_optimization_find_overlap_lane_between_groups(
groups, group_id, owner_group, local_id,
)
}
}
let mut overlap_source_lane : Int32? = None
match groups[group_id].copy_source {
Some(source_group) =>
overlap_source_lane = tuple_optimization_find_overlap_lane_between_groups(
groups, source_group, group_id, local_id,
)
None => ()
}
for write_id in @ir.local_write_nodes_borrowed(use_def, local_id) {
if write_id == defining_node ||
(
write_id >= 0 &&
write_id < copy_write_nodes.length() &&
copy_write_nodes[write_id]
) {
continue
}
let mut skip_overlap_write = false
match overlap_owner_lane {
Some(_) => skip_overlap_write = true
None => ()
}
match overlap_source_lane {
Some(source_lane_idx) =>
match groups[group_id].copy_source {
Some(source_group) =>
if write_id ==
groups[source_group].defining_nodes[source_lane_idx] {
skip_overlap_write = true
}
_ => ()
}
None => ()
}
if skip_overlap_write {
continue
}
groups[group_id].uses = groups[group_id].uses + 1
}
for read_id in @ir.local_read_nodes_borrowed(use_def, local_id) {
match overlap_owner_lane {
Some(_) => continue
None => ()
}
if read_id >= 0 &&
read_id < copy_read_nodes.length() &&
copy_read_nodes[read_id] {
continue
}
groups[group_id].uses = groups[group_id].uses + 1
if tuple_optimization_read_is_invalid_scalar_copy_through(
use_def, summary, read_id,
) {
continue
}
groups[group_id].valid_uses = groups[group_id].valid_uses + 1
}
}
}
}
///|
fn tuple_optimization_propagate_badness(
groups : Array[HotTupleOptimizationGroup],
) -> Unit {
let queue : Array[Int32] = []
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
if groups[group_id].uses > groups[group_id].valid_uses {
groups[group_id].bad = true
queue.push(group_id)
}
}
let mut queue_head = 0
while queue_head < queue.length() {
let group_id = queue[queue_head]
queue_head = queue_head + 1
for neighbor in groups[group_id].neighbors {
if neighbor < 0 || neighbor >= groups.length() || groups[neighbor].bad {
continue
}
groups[neighbor].bad = true
queue.push(neighbor)
}
}
}
///|
fn tuple_optimization_empty_analysis(
func : @ir.HotFunc,
) -> HotTupleOptimizationAnalysis {
{
groups: [],
local_group_ids: Array::make(@ir.hot_local_count(func), -1),
copy_read_nodes: @bitset.BitSet::new(@ir.hot_node_count(func)),
query_summary: tuple_optimization_empty_query_summary(),
}
}
///|
fn tuple_optimization_analyze_with_groups(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
groups : Array[HotTupleOptimizationGroup],
perf~ : HotPerfSession? = None,
) -> HotTupleOptimizationAnalysis {
if groups.is_empty() {
return tuple_optimization_empty_analysis(func)
}
let query_summary_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:build-query-summary",
)
let query_summary = tuple_optimization_build_query_summary(func, use_def)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:build-query-summary", query_summary_start_us,
)
let local_group_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:build-local-group-ids",
)
let mut local_group_ids = tuple_optimization_build_local_group_ids(
func, groups,
)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:build-local-group-ids", local_group_start_us,
)
let copy_read_nodes = @bitset.BitSet::new(@ir.hot_node_count(func))
let copy_write_nodes = @bitset.BitSet::new(@ir.hot_node_count(func))
if tuple_optimization_groups_have_copy_payload_candidate(groups) {
let link_copy_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:link-copy-groups",
)
tuple_optimization_link_copy_groups(
func, query_summary, groups, local_group_ids, copy_read_nodes, copy_write_nodes,
)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:link-copy-groups", link_copy_start_us,
)
} else {
let link_copy_fast_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:link-copy-groups:no-copy-payload-fast-path",
)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:link-copy-groups:no-copy-payload-fast-path",
link_copy_fast_start_us,
)
}
if query_summary.has_single_result_block_copy_candidate {
let link_result_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:link-result-block-copy-groups",
)
tuple_optimization_link_result_block_copy_groups(
func, use_def, query_summary, groups, local_group_ids, copy_read_nodes,
)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:link-result-block-copy-groups", link_result_start_us,
)
let rebuild_after_result_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:rebuild-local-groups-after-result-links",
)
local_group_ids = tuple_optimization_build_local_group_ids(func, groups)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:rebuild-local-groups-after-result-links",
rebuild_after_result_start_us,
)
} else {
let link_result_fast_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:link-result-block-copy-groups:no-single-result-block-fast-path",
)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:link-result-block-copy-groups:no-single-result-block-fast-path",
link_result_fast_start_us,
)
}
if query_summary.has_scalar_forward_copy_candidate {
let link_scalar_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:link-scalar-forward-copy-groups",
)
tuple_optimization_link_scalar_forward_copy_groups(
func, use_def, query_summary, groups, local_group_ids, copy_read_nodes,
)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:link-scalar-forward-copy-groups", link_scalar_start_us,
)
let rebuild_after_scalar_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:rebuild-local-groups-after-scalar-links",
)
local_group_ids = tuple_optimization_build_local_group_ids(func, groups)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:rebuild-local-groups-after-scalar-links",
rebuild_after_scalar_start_us,
)
} else {
let link_scalar_fast_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:link-scalar-forward-copy-groups:no-scalar-forward-fast-path",
)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:link-scalar-forward-copy-groups:no-scalar-forward-fast-path",
link_scalar_fast_start_us,
)
}
let count_traffic_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:count-lane-traffic",
)
tuple_optimization_count_lane_traffic(
func, use_def, query_summary, groups, local_group_ids, copy_read_nodes, copy_write_nodes,
)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:count-lane-traffic", count_traffic_start_us,
)
let finalize_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:finalize-host-lanes",
)
tuple_optimization_finalize_host_lanes(func, use_def, query_summary, groups)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:finalize-host-lanes", finalize_start_us,
)
let propagate_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:propagate-badness",
)
tuple_optimization_propagate_badness(groups)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:propagate-badness", propagate_start_us,
)
{ groups, local_group_ids, copy_read_nodes, query_summary, }
}
///|
fn tuple_optimization_analyze(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
perf~ : HotPerfSession? = None,
) -> HotTupleOptimizationAnalysis {
let collect_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:collect-seed-groups",
)
let groups = tuple_optimization_collect_seed_groups(func, use_def)
perf_stop_timer_opt(
perf, "detail:tuple-optimization:collect-seed-groups", collect_start_us,
)
tuple_optimization_analyze_with_groups(func, use_def, groups, perf~)
}
///|
fn tuple_optimization_require_local_type(
func : @ir.HotFunc,
local_id : Int32,
) -> @lib.ValType {
match @ir.hot_local_type(func, local_id) {
Some(ty) => ty
None => abort("tuple-optimization rewrite requires valid local ids")
}
}
///|
fn tuple_optimization_group_should_rewrite(
group : HotTupleOptimizationGroup,
) -> Bool {
let exact_copy_group = group.producer >= 0 && group.copy_source is Some(_)
!group.bad &&
(group.valid_uses > 1 || exact_copy_group) &&
(group.producer >= 0 || tuple_optimization_group_host_lane(group) is Some(_))
}
///|
fn tuple_optimization_copy_group_has_mixed_direct_and_forwarded_lanes(
groups : Array[HotTupleOptimizationGroup],
group_id : Int32,
) -> Bool {
if group_id < 0 || group_id >= groups.length() {
return false
}
let group = groups[group_id]
let source_group_id = match group.copy_source {
Some(source_group_id) => source_group_id
None => return false
}
if source_group_id < 0 || source_group_id >= groups.length() {
return false
}
let source = groups[source_group_id]
if group.lane_locals.length() != source.lane_locals.length() {
return false
}
let mut has_direct_lane = false
let mut has_forwarded_lane = false
for lane_idx = 0
lane_idx < group.lane_locals.length()
lane_idx = lane_idx + 1 {
if group.lane_locals[lane_idx] == source.lane_locals[lane_idx] {
has_direct_lane = true
} else {
has_forwarded_lane = true
}
}
has_direct_lane && has_forwarded_lane
}
///|
fn tuple_optimization_copy_group_redundant_with_source_host_chain(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
groups : Array[HotTupleOptimizationGroup],
copy_consumers : HotTupleOptimizationCopyConsumers,
group_id : Int32,
) -> Bool {
if group_id < 0 || group_id >= groups.length() {
return false
}
let group = groups[group_id]
let source_group_id = match group.copy_source {
Some(source_group_id) => source_group_id
None => return false
}
if source_group_id < 0 || source_group_id >= groups.length() {
return false
}
if groups[source_group_id].copy_source is None &&
copy_consumers.has_host_consumer[source_group_id] {
return false
}
let source_host_lane = match
tuple_optimization_group_host_lane(groups[source_group_id]) {
Some(source_host_lane) => source_host_lane
None => return false
}
let host_lane = match tuple_optimization_group_host_lane(group) {
Some(host_lane) => host_lane
None => return false
}
let chain = match
tuple_optimization_find_passthrough_root_chain(
func,
use_def,
groups[source_group_id].defining_nodes[source_host_lane],
) {
Some(chain) if !chain.is_empty() => chain
_ => return false
}
chain[chain.length() - 1] == group.defining_nodes[host_lane]
}
///|
fn tuple_optimization_group_has_nested_rootslot_host_copy_wrapper(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
) -> Bool {
if group.copy_source is Some(_) || group.lane_locals.is_empty() {
return false
}
let mut inner_block = -1
for lane_idx = 0
lane_idx < group.lane_locals.length()
lane_idx = lane_idx + 1 {
let local_id = group.lane_locals[lane_idx]
let reads = @ir.local_read_nodes_borrowed(use_def, local_id)
let mut matched = false
for read_id in reads {
let sites = @ir.node_use_sites_borrowed(use_def, read_id)
if sites.length() != 1 {
continue
}
let site = sites[0]
match site.kind {
@ir.HotUseSlotKind::RootUseSlot =>
if site.slot == lane_idx {
match @ir.hot_node_get(func, site.user).op {
@ir.HotOp::Block => {
match inner_block {
-1 => inner_block = site.user
_ => if inner_block != site.user { return false }
}
matched = true
break
}
_ => ()
}
}
_ => ()
}
}
if !matched {
return false
}
}
if inner_block < 0 {
return false
}
let inner_sites = @ir.node_use_sites_borrowed(use_def, inner_block)
if inner_sites.length() != group.lane_locals.length() {
return false
}
let mut outer_block = -1
for lane_idx = 0; lane_idx < inner_sites.length(); lane_idx = lane_idx + 1 {
let site = inner_sites[lane_idx]
match site.kind {
@ir.HotUseSlotKind::RootUseSlot =>
if site.slot == lane_idx {
match @ir.hot_node_get(func, site.user).op {
@ir.HotOp::Block =>
match outer_block {
-1 => outer_block = site.user
_ => if outer_block != site.user { return false }
}
_ => return false
}
} else {
return false
}
_ => return false
}
}
if outer_block < 0 {
return false
}
let outer_sites = @ir.node_use_sites_borrowed(use_def, outer_block)
if outer_sites.length() != 1 {
return false
}
match outer_sites[0].kind {
@ir.HotUseSlotKind::ChildUseSlot =>
match @ir.hot_node_get(func, outer_sites[0].user).op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee => true
_ => false
}
_ => false
}
}
///|
fn tuple_optimization_group_should_rewrite_in_func_indexed(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
copy_consumers : HotTupleOptimizationCopyConsumers,
group_id : Int32,
) -> Bool {
if group_id < 0 || group_id >= groups.length() {
return false
}
let group = groups[group_id]
let suppress_mixed_scalar_forward_host_copy = group.producer < 0 &&
group.copy_source is Some(_) &&
tuple_optimization_group_host_lane(group) is Some(_) &&
tuple_optimization_copy_group_has_mixed_direct_and_forwarded_lanes(
groups, group_id,
)
let mut copy_group_non_drop_lane_count = 0
let mut copy_group_only_uses_host_non_drop = false
match (group.copy_source, tuple_optimization_group_host_lane(group)) {
(Some(_), host_lane) => {
let mut only_host = true
for lane_idx = 0
lane_idx < group.lane_locals.length()
lane_idx = lane_idx + 1 {
let has_non_drop_use = tuple_optimization_local_has_forwarded_non_drop_use(
use_def,
summary,
group.lane_locals[lane_idx],
)
if has_non_drop_use {
copy_group_non_drop_lane_count += 1
match host_lane {
Some(host_lane) if lane_idx == host_lane => ()
_ => only_host = false
}
continue
}
if tuple_optimization_local_has_forwarded_any_use(
use_def,
summary,
group.lane_locals[lane_idx],
) {
()
}
}
copy_group_only_uses_host_non_drop = only_host &&
copy_group_non_drop_lane_count > 0
}
_ => ()
}
let mut suppressed_child_copy_count = 0
for child_group_id in copy_consumers.children_by_source[group_id] {
let mut child_non_drop_lane_count = 0
let mut child_has_drop_only_lane_use = false
for lane_idx = 0
lane_idx < groups[child_group_id].lane_locals.length()
lane_idx = lane_idx + 1 {
let child_local = groups[child_group_id].lane_locals[lane_idx]
if tuple_optimization_local_has_forwarded_non_drop_use(
use_def, summary, child_local,
) {
child_non_drop_lane_count += 1
continue
}
if tuple_optimization_local_has_forwarded_any_use(
use_def, summary, child_local,
) {
child_has_drop_only_lane_use = true
}
}
if groups[child_group_id].producer < 0 &&
child_non_drop_lane_count <= 1 &&
(
child_has_drop_only_lane_use ||
child_non_drop_lane_count < groups[child_group_id].lane_locals.length()
) {
suppressed_child_copy_count += 1
continue
}
suppressed_child_copy_count = -1
break
}
tuple_optimization_group_should_rewrite(groups[group_id]) &&
!suppress_mixed_scalar_forward_host_copy &&
!(group.copy_source is Some(_) &&
group.producer < 0 &&
copy_group_non_drop_lane_count <= 1 &&
copy_group_non_drop_lane_count < group.lane_locals.length()) &&
!(group.copy_source is Some(_) &&
group.producer < 0 &&
suppressed_child_copy_count > 0 &&
groups[group_id].valid_uses ==
1 +
tuple_optimization_count_tee_defs(func, groups[group_id].defining_nodes) +
suppressed_child_copy_count) &&
!(group.copy_source is Some(_) &&
group.producer < 0 &&
copy_group_only_uses_host_non_drop) &&
!tuple_optimization_copy_group_redundant_with_source_host_chain(
func, use_def, groups, copy_consumers, group_id,
)
}
///|
fn tuple_optimization_group_should_rewrite_in_func(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
group_id : Int32,
) -> Bool {
tuple_optimization_group_should_rewrite_in_func_indexed(
func,
use_def,
summary,
groups,
tuple_optimization_build_copy_consumers(groups),
group_id,
)
}
///|
fn tuple_optimization_has_nested_rootslot_host_copy_wrapper_group(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
groups : Array[HotTupleOptimizationGroup],
) -> Bool {
for group in groups {
if tuple_optimization_group_has_nested_rootslot_host_copy_wrapper(
func, use_def, group,
) {
return true
}
}
false
}
///|
fn tuple_optimization_replace_with_built(
ctx : HotPassContext,
func : @ir.HotFunc,
target_id : Int32,
replacement_id : Int32,
) -> Unit {
let replacement = @ir.hot_node_get(func, replacement_id)
match replacement.op {
@ir.HotOp::Block
| @ir.HotOp::Loop
| @ir.HotOp::If
| @ir.HotOp::Try
| @ir.HotOp::TryTable =>
@ir.hot_label_owner_set(func, replacement.imm0, target_id)
_ => ()
}
@ir.hot_delete_detached_node(func, replacement_id)
pass_replace_node(ctx, func, target_id, replacement)
}
///|
fn tuple_optimization_groups_have_copy_source(
groups : Array[HotTupleOptimizationGroup],
) -> Bool {
for group in groups {
if group.copy_source is Some(_) {
return true
}
}
false
}
///|
fn tuple_optimization_build_no_copy_group_rewrite_mask(
groups : Array[HotTupleOptimizationGroup],
perf~ : HotPerfSession? = None,
) -> @bitset.BitSet {
let fast_path_start_us = perf_start_timer_opt(
perf, "detail:tuple-optimization:build-rewrite-mask:no-copy-fast-path",
)
let rewrite_mask = @bitset.BitSet::new(groups.length())
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
rewrite_mask[group_id] = tuple_optimization_group_should_rewrite(
groups[group_id],
)
}
perf_stop_timer_opt(
perf, "detail:tuple-optimization:build-rewrite-mask:no-copy-fast-path", fast_path_start_us,
)
rewrite_mask
}
///|
fn tuple_optimization_build_group_rewrite_mask(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
perf~ : HotPerfSession? = None,
) -> @bitset.BitSet {
if tuple_optimization_has_nested_rootslot_host_copy_wrapper_group(
func, use_def, groups,
) {
return @bitset.BitSet::new(groups.length())
}
if !tuple_optimization_groups_have_copy_source(groups) {
return tuple_optimization_build_no_copy_group_rewrite_mask(groups, perf~)
}
let copy_consumers = tuple_optimization_build_copy_consumers(groups)
let rewrite_mask = @bitset.BitSet::new(groups.length())
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
rewrite_mask[group_id] = tuple_optimization_group_should_rewrite_in_func_indexed(
func, use_def, summary, groups, copy_consumers, group_id,
)
}
rewrite_mask
}
///|
// Moving one defining node moves all its graph occurrences. The current
// publication planners describe one execution site, so shared publications
// must keep their whole connected component at its original sites.
fn tuple_optimization_retain_shared_publication_components(
use_def : @ir.HotUseDef,
groups : Array[HotTupleOptimizationGroup],
rewrite_mask : @bitset.BitSet,
) -> Unit {
let mut shared_groups : Array[Int32]? = None
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
for definition in groups[group_id].defining_nodes {
if @ir.node_use_sites_borrowed(use_def, definition).length() > 1 {
match shared_groups {
Some(queue) => queue.push(group_id)
None => shared_groups = Some([group_id])
}
break
}
}
}
let queue = match shared_groups {
Some(queue) => queue
None => return
}
let retained = @bitset.BitSet::new(groups.length())
for group_id in queue {
retained[group_id] = true
}
let mut head = 0
while head < queue.length() {
let group_id = queue[head]
head += 1
rewrite_mask[group_id] = false
for neighbor in groups[group_id].neighbors {
if neighbor >= 0 && neighbor < groups.length() && !retained[neighbor] {
retained[neighbor] = true
queue.push(neighbor)
}
}
}
}
///|
fn tuple_optimization_bitset_debug_string(values : @bitset.BitSet) -> String {
let out = StringBuilder::new()
out.write_string("[")
let mut first = true
for idx = 0; idx < values.length(); idx = idx + 1 {
if !values[idx] {
continue
}
if !first {
out.write_string(", ")
}
first = false
out.write_string(idx.to_string())
}
out.write_string("]")
out.to_string()
}
///|
fn tuple_optimization_ensure_split_locals(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
copy_read_nodes : @bitset.BitSet,
rewrite_mask : @bitset.BitSet,
rewritten_host_copy_consumers : @bitset.BitSet,
all_copy_consumers : @bitset.BitSet,
retained_opaque_sources : @bitset.BitSet,
drop_only_elision_facts : HotTupleOptimizationDropOnlyElisionFacts,
split_locals_by_group : Array[Array[Int32]],
split_ready : @bitset.BitSet,
split_visiting : @bitset.BitSet,
group_id : Int32,
) -> Array[Int32] {
if group_id < 0 || group_id >= groups.length() {
abort("tuple-optimization rewrite group ids must stay in range")
}
if !rewrite_mask[group_id] {
if groups[group_id].copy_source is Some(_) ||
(retained_opaque_sources[group_id] && !groups[group_id].bad) {
let split_locals = groups[group_id].lane_locals
split_locals_by_group[group_id] = split_locals
split_ready[group_id] = true
split_visiting[group_id] = false
return split_locals
}
abort(
"tuple-optimization rewrite requires a selected or proved retained source",
)
}
if split_ready[group_id] {
return split_locals_by_group[group_id]
}
if split_visiting[group_id] {
abort("tuple-optimization rewrite copy graph must stay acyclic")
}
split_visiting[group_id] = true
if group_id < drop_only_elision_facts.mask.length() &&
drop_only_elision_facts.mask[group_id] {
let split_locals = groups[group_id].lane_locals
split_locals_by_group[group_id] = split_locals
split_ready[group_id] = true
split_visiting[group_id] = false
return split_locals
}
if !tuple_optimization_group_needs_dedicated_split_locals(
func,
use_def,
summary,
groups,
copy_read_nodes,
rewritten_host_copy_consumers,
all_copy_consumers,
group_id,
groups[group_id],
) {
let split_locals = groups[group_id].lane_locals
split_locals_by_group[group_id] = split_locals
split_ready[group_id] = true
split_visiting[group_id] = false
return split_locals
}
let source_lane_locals = match groups[group_id].copy_source {
Some(source_group) => {
if source_group < 0 || source_group >= groups.length() {
abort(
"tuple-optimization rewrite copy sources must stay within good groups",
)
}
tuple_optimization_ensure_split_locals(
func, use_def, summary, groups, copy_read_nodes, rewrite_mask, rewritten_host_copy_consumers,
all_copy_consumers, retained_opaque_sources, drop_only_elision_facts, split_locals_by_group,
split_ready, split_visiting, source_group,
)
}
None => groups[group_id].lane_locals
}
let split_locals : Array[Int32] = []
for local_id in source_lane_locals {
split_locals.push(
@ir.hot_append_body_local(
func,
tuple_optimization_require_local_type(func, local_id),
),
)
}
split_locals_by_group[group_id] = split_locals
split_ready[group_id] = true
split_visiting[group_id] = false
split_locals
}
///|
fn tuple_optimization_collect_rewrite_order_visit(
groups : Array[HotTupleOptimizationGroup],
rewrite_mask : @bitset.BitSet,
seen : @bitset.BitSet,
out : Array[Int32],
group_id : Int32,
) -> Unit {
let path : Array[Int32] = []
let mut current = group_id
while current >= 0 && current < groups.length() && !seen[current] {
seen[current] = true
path.push(current)
match groups[current].copy_source {
Some(source_group) => current = source_group
None => break
}
}
for path_idx = path.length() - 1; path_idx >= 0; path_idx = path_idx - 1 {
let current = path[path_idx]
if rewrite_mask[current] {
out.push(current)
}
}
}
///|
fn tuple_optimization_collect_rewrite_order(
groups : Array[HotTupleOptimizationGroup],
rewrite_mask : @bitset.BitSet,
) -> Array[Int32] {
let order : Array[Int32] = []
let seen = @bitset.BitSet::new(groups.length())
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
if rewrite_mask[group_id] {
tuple_optimization_collect_rewrite_order_visit(
groups, rewrite_mask, seen, order, group_id,
)
}
}
order
}
///|
fn tuple_optimization_precompute_drop_only_elision_facts(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
rewrite_order : Array[Int32],
perf~ : HotPerfSession? = None,
) -> HotTupleOptimizationDropOnlyElisionFacts {
let mask = @bitset.BitSet::new(groups.length())
let payload_lanes_by_group : Array[Array[Int32]] = []
for _ in 0..= groups.length() {
continue
}
match
tuple_optimization_simple_drop_only_source_payload_lanes(
func,
use_def,
summary,
groups[group_id],
) {
Some(payload) => {
mask[group_id] = true
payload_lanes_by_group[group_id] = payload
}
None => ()
}
}
perf_stop_timer_opt(
perf, "detail:tuple-optimization:precompute-drop-only-elision-mask:simple-block-payload-fast-path",
simple_payload_fast_path_start_us,
)
{ mask, payload_lanes_by_group, }
}
///|
fn tuple_optimization_rewrite_order_is_source_only_elide_fast_path(
groups : Array[HotTupleOptimizationGroup],
rewrite_order : Array[Int32],
drop_only_elision_facts : HotTupleOptimizationDropOnlyElisionFacts,
) -> Bool {
if rewrite_order.is_empty() {
return false
}
for group_id in rewrite_order {
if group_id < 0 || group_id >= groups.length() {
return false
}
if groups[group_id].copy_source is Some(_) {
return false
}
if group_id >= drop_only_elision_facts.mask.length() ||
!drop_only_elision_facts.mask[group_id] {
return false
}
}
true
}
///|
fn tuple_optimization_group_host_lane(
group : HotTupleOptimizationGroup,
) -> Int32? {
group.host_lane
}
///|
fn tuple_optimization_build_rewritten_consumer_facts(
groups : Array[HotTupleOptimizationGroup],
rewrite_mask : @bitset.BitSet,
) -> (@bitset.BitSet, @bitset.BitSet, @bitset.BitSet) {
let all_consumers = @bitset.BitSet::new(groups.length())
let consumers = @bitset.BitSet::new(groups.length())
let host_consumers = @bitset.BitSet::new(groups.length())
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
match groups[group_id].copy_source {
Some(source) if source >= 0 &&
source < groups.length() &&
source != group_id => all_consumers[source] = true
_ => ()
}
if group_id >= rewrite_mask.length() || !rewrite_mask[group_id] {
continue
}
match groups[group_id].copy_source {
Some(source) if source >= 0 &&
source < groups.length() &&
source != group_id => {
consumers[source] = true
if tuple_optimization_group_host_lane(groups[group_id]) is Some(_) {
host_consumers[source] = true
}
}
_ => ()
}
}
(consumers, host_consumers, all_consumers)
}
///|
fn tuple_optimization_source_group_host_copy_consumer_needs_preserved_lanes(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
copy_read_nodes : @bitset.BitSet,
source_group_id : Int32,
) -> Bool {
if source_group_id < 0 || source_group_id >= groups.length() {
return false
}
let group = groups[source_group_id]
for lane_idx = 0
lane_idx < group.lane_locals.length()
lane_idx = lane_idx + 1 {
let defining_node = group.defining_nodes[lane_idx]
match @ir.hot_node_get(func, defining_node).op {
@ir.HotOp::LocalTee =>
if tuple_optimization_node_has_direct_non_scalar_use(
summary, defining_node,
) {
return true
}
_ => ()
}
let local_id = group.lane_locals[lane_idx]
for read_id in @ir.local_read_nodes_borrowed(use_def, local_id) {
if read_id >= 0 &&
read_id < copy_read_nodes.length() &&
copy_read_nodes[read_id] {
continue
}
return true
}
}
false
}
///|
fn tuple_optimization_lane_locals_are_strictly_increasing(
lane_locals : Array[Int32],
) -> Bool {
if lane_locals.length() <= 1 {
return true
}
for lane_idx = 1; lane_idx < lane_locals.length(); lane_idx = lane_idx + 1 {
if lane_locals[lane_idx - 1] >= lane_locals[lane_idx] {
return false
}
}
true
}
///|
fn tuple_optimization_copy_group_has_local_overlap(
lane_locals : Array[Int32],
payload_locals : Array[Int32],
) -> Bool {
for local_id in lane_locals {
for payload_local in payload_locals {
if local_id == payload_local {
return true
}
}
}
false
}
///|
// Result-block copies own one outer store and plain inner stores. Moving their
// already captured scalar reads out needs no scratch lanes when stores cannot
// overwrite any payload. Return the owned reads, retaining their source order.
fn tuple_optimization_direct_scalar_copy_payload_nodes(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
) -> Array[Int32]? {
let arity = group.lane_locals.length()
if group.copy_source is None ||
group.host_lane is Some(_) ||
arity < 2 ||
group.defining_nodes.length() != arity ||
group.producer < 0 ||
!@ir.hot_node_is_live(func, group.producer) {
return None
}
let producer = @ir.hot_node_get(func, group.producer)
if !(producer.op is @ir.HotOp::Block) ||
producer.imm1 != 0 ||
@ir.hot_node_result_arity(func, group.producer) != 1 {
return None
}
let payload = match group.copy_payload_locals {
Some(payload) if payload.length() == arity => payload
_ => return None
}
if !tuple_optimization_lane_locals_are_strictly_increasing(payload) ||
!tuple_optimization_lane_locals_are_strictly_increasing(group.lane_locals) ||
tuple_optimization_copy_group_has_local_overlap(group.lane_locals, payload) {
return None
}
let outer_id = group.defining_nodes[0]
let outer = @ir.hot_node_get(func, outer_id)
if !(outer.op is @ir.HotOp::LocalSet) ||
outer.imm0 != group.lane_locals[0] ||
@ir.hot_child_count(func, outer_id) != 1 ||
@ir.hot_child_get(func, outer_id, 0) != group.producer {
return None
}
let outer_sites = @ir.node_use_sites_borrowed(use_def, outer_id)
if outer_sites.length() != 1 ||
!(outer_sites[0].kind is @ir.HotUseSlotKind::RootUseSlot) ||
tuple_optimization_find_root_slot_with_use_def(func, use_def, outer_id, 0)
is None {
return None
}
let producer_sites = @ir.node_use_sites_borrowed(use_def, group.producer)
if producer_sites.length() != 1 ||
!(producer_sites[0].kind is @ir.HotUseSlotKind::ChildUseSlot) ||
producer_sites[0].user != outer_id ||
producer_sites[0].slot != 0 {
return None
}
let body = @ir.hot_region_body_ref(func, group.producer)
if @ir.hot_region_root_count(func, body) != arity {
return None
}
let last = @ir.hot_region_get(func, body, arity - 1)
let prefix_layout = @ir.hot_node_get(func, last).op is @ir.HotOp::LocalGet
let host_slot = if prefix_layout { arity - 1 } else { 0 }
let host_read = @ir.hot_region_get(func, body, host_slot)
let host = @ir.hot_node_get(func, host_read)
if !(host.op is @ir.HotOp::LocalGet) || host.imm0 != payload[0] {
return None
}
let host_sites = @ir.node_use_sites_borrowed(use_def, host_read)
if host_sites.length() != 1 ||
!(host_sites[0].kind is @ir.HotUseSlotKind::RootUseSlot) ||
host_sites[0].user != group.producer ||
host_sites[0].slot != host_slot {
return None
}
let reads = [host_read]
for lane = 1; lane < arity; lane = lane + 1 {
let definition = group.defining_nodes[lane]
let node = @ir.hot_node_get(func, definition)
if !(node.op is @ir.HotOp::LocalSet) ||
node.imm0 != group.lane_locals[lane] ||
@ir.hot_child_count(func, definition) != 1 {
return None
}
let definition_sites = @ir.node_use_sites_borrowed(use_def, definition)
if definition_sites.length() != 1 ||
!(definition_sites[0].kind is @ir.HotUseSlotKind::RootUseSlot) ||
definition_sites[0].user != group.producer {
return None
}
let slot = definition_sites[0].slot
if slot < 0 ||
slot >= arity ||
slot == host_slot ||
@ir.hot_region_get(func, body, slot) != definition {
return None
}
let read = @ir.hot_child_get(func, definition, 0)
let read_node = @ir.hot_node_get(func, read)
if !(read_node.op is @ir.HotOp::LocalGet) || read_node.imm0 != payload[lane] {
return None
}
let read_sites = @ir.node_use_sites_borrowed(use_def, read)
if read_sites.length() != 1 ||
!(read_sites[0].kind is @ir.HotUseSlotKind::ChildUseSlot) ||
read_sites[0].user != definition ||
read_sites[0].slot != 0 {
return None
}
reads.push(read)
}
Some(reads)
}
///|
fn tuple_optimization_plan_retained_opaque_sources(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
groups : Array[HotTupleOptimizationGroup],
rewrite_mask : @bitset.BitSet,
all_copy_consumers : @bitset.BitSet,
) -> @bitset.BitSet {
let retained = @bitset.BitSet::new(groups.length())
let incompatible_consumers = @bitset.BitSet::new(groups.length())
for child = 0; child < groups.length(); child = child + 1 {
if !rewrite_mask[child] {
continue
}
match groups[child].copy_source {
Some(source) if source >= 0 && source < groups.length() =>
if tuple_optimization_direct_scalar_copy_payload_nodes(
func,
use_def,
groups[child],
)
is None {
incompatible_consumers[source] = true
}
_ => ()
}
}
for group_id = 0; group_id < groups.length(); group_id = group_id + 1 {
let group = groups[group_id]
if !rewrite_mask[group_id] ||
group.bad ||
group.copy_source is Some(_) ||
group.host_lane is Some(_) ||
incompatible_consumers[group_id] ||
group.producer < 0 ||
!(@ir.hot_node_get(func, group.producer).op is @ir.HotOp::Block) {
continue
}
// Preserve supported source scalarization and explicit drop-only elision.
// Generic opaque staging may omit stores but add a larger scratch carrier;
// retaining those original stores avoids that growth. A pure arithmetic
// source with copy consumers cannot take the source-only path.
if (
!all_copy_consumers[group_id] &&
tuple_optimization_pure_adjacent_source_lanes(func, use_def, group)
is Some(_)
) ||
tuple_optimization_simple_block_payload_lanes(
func,
group.producer,
group.lane_locals.length(),
)
is Some(_) {
continue
}
let mut region : @ir.HotRegionRef? = None
let mut plain_stores = true
for definition in group.defining_nodes {
if !(@ir.hot_node_get(func, definition).op is @ir.HotOp::LocalSet) ||
@ir.hot_child_count(func, definition) != 1 ||
@ir.hot_child_get(func, definition, 0) != group.producer {
plain_stores = false
break
}
match
tuple_optimization_find_root_slot_with_use_def(
func, use_def, definition, 0,
) {
Some(slot) =>
match region {
Some(previous) if previous != slot.region_ref => {
plain_stores = false
break
}
_ => region = Some(slot.region_ref)
}
None => {
plain_stores = false
break
}
}
}
if plain_stores && !group.defining_nodes.is_empty() {
// No producer or publication is moved. Only compatible scalar copies
// remain selected, and they retain their actual captured payload locals.
retained[group_id] = true
rewrite_mask[group_id] = false
}
}
retained
}
///|
fn tuple_optimization_node_has_typeidx_control_type(
func : @ir.HotFunc,
node_id : Int32,
) -> Bool {
match @ir.hot_node_get(func, node_id).op {
@ir.HotOp::Block
| @ir.HotOp::Loop
| @ir.HotOp::If
| @ir.HotOp::Try
| @ir.HotOp::TryTable => ()
_ => return false
}
match @ir.hot_type_get(func, @ir.hot_node_type(func, node_id)) {
@ir.HotTypeInfo::BlockResult(@lib.TypeIdxBlockType(_)) => true
@ir.HotTypeInfo::BlockResultResolved(@lib.TypeIdxBlockType(_), _, _) => true
_ => false
}
}
///|
fn tuple_optimization_is_simple_i32_i64_typeidx_group(
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
) -> Bool {
group.lane_locals.length() == 2 &&
tuple_optimization_node_has_typeidx_control_type(func, group.producer) &&
tuple_optimization_require_local_type(func, group.lane_locals[0]) ==
@lib.ValType::i32() &&
tuple_optimization_require_local_type(func, group.lane_locals[1]) ==
@lib.ValType::i64()
}
///|
fn tuple_optimization_pure_adjacent_source_lanes(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
) -> Array[Int32]? {
let arity = group.lane_locals.length()
if group.copy_source is Some(_) ||
group.host_lane is Some(_) ||
arity < 2 ||
group.producer < 0 ||
!@ir.hot_node_is_live(func, group.producer) ||
!(@ir.hot_node_get(func, group.producer).op is @ir.HotOp::Block) {
return None
}
// Entry operands precede body roots and must remain in the original block.
if @ir.hot_node_get(func, group.producer).imm1 != 0 {
return None
}
let body = @ir.hot_region_body_ref(func, group.producer)
if @ir.hot_region_root_count(func, body) != arity {
return None
}
// One total budget bounds repeated visits through shared scalar DAGs.
let mut remaining = 128
fn prove_lane(node_id : Int32, depth : Int32) -> Bool {
if depth <= 0 || remaining <= 0 || !@ir.hot_node_is_live(func, node_id) {
return false
}
remaining -= 1
let node = @ir.hot_node_get(func, node_id)
match node.op {
@ir.HotOp::Const =>
@ir.hot_const_get(func, node.imm0) is @ir.HotConstPayload::I32Const(_)
@ir.HotOp::LocalGet =>
node.imm0 >= 0 &&
node.imm0 < @ir.hot_local_param_count(func) &&
@ir.local_write_nodes_borrowed(use_def, node.imm0).is_empty()
@ir.HotOp::Binary =>
@ir.hot_node_exact_instr(func, node_id) is Some(@lib.I32Add) &&
@ir.hot_child_count(func, node_id) == 2 &&
prove_lane(@ir.hot_child_get(func, node_id, 0), depth - 1) &&
prove_lane(@ir.hot_child_get(func, node_id, 1), depth - 1)
_ => false
}
}
let lanes : Array[Int32] = []
let mut has_add = false
for lane_idx = 0; lane_idx < arity; lane_idx = lane_idx + 1 {
let value = @ir.hot_region_get(func, body, lane_idx)
if tuple_optimization_require_local_type(func, group.lane_locals[lane_idx]) !=
@lib.ValType::i32() ||
!prove_lane(value, 32) {
return None
}
if @ir.hot_node_exact_instr(func, value) is Some(@lib.I32Add) {
has_add = true
}
lanes.push(value)
}
if !has_add || group.defining_nodes.length() != arity {
return None
}
let first = match
tuple_optimization_find_root_slot_with_use_def(
func,
use_def,
group.defining_nodes[arity - 1],
arity - 1,
) {
Some(slot) => slot
None => return None
}
for lane_idx = 0; lane_idx < arity; lane_idx = lane_idx + 1 {
let definition = group.defining_nodes[lane_idx]
let node = @ir.hot_node_get(func, definition)
if !(node.op is @ir.HotOp::LocalSet) ||
node.imm0 != group.lane_locals[lane_idx] ||
@ir.hot_child_count(func, definition) != 1 ||
@ir.hot_child_get(func, definition, 0) != group.producer ||
first.slot + arity - 1 - lane_idx >=
@ir.hot_region_root_count(func, first.region_ref) ||
@ir.hot_region_get(
func,
first.region_ref,
first.slot + arity - 1 - lane_idx,
) !=
definition {
return None
}
}
// This is also the deletion proof. Original use-def sites can be stale after
// an earlier group rewrite, so inspect current incoming edges before moving
// the body roots and retiring their old control owner.
for root in @ir.hot_root_span(func) {
if root == group.producer {
return None
}
}
for owner = 0; owner < @ir.hot_node_count(func); owner = owner + 1 {
if !@ir.hot_node_is_live(func, owner) {
continue
}
for slot = 0; slot < @ir.hot_child_count(func, owner); slot = slot + 1 {
if @ir.hot_child_get(func, owner, slot) == group.producer &&
!group.defining_nodes.contains(owner) {
return None
}
}
}
Some(lanes)
}
///|
fn tuple_optimization_group_needs_dedicated_split_locals(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
copy_read_nodes : @bitset.BitSet,
rewritten_host_copy_consumers : @bitset.BitSet,
all_copy_consumers : @bitset.BitSet,
group_id : Int32,
group : HotTupleOptimizationGroup,
) -> Bool {
if tuple_optimization_direct_scalar_copy_payload_nodes(func, use_def, group)
is Some(_) {
return false
}
if !all_copy_consumers[group_id] &&
tuple_optimization_pure_adjacent_source_lanes(func, use_def, group)
is Some(_) {
return false
}
if group.copy_source is None &&
rewritten_host_copy_consumers[group_id] &&
tuple_optimization_source_group_host_copy_consumer_needs_preserved_lanes(
func, use_def, summary, groups, copy_read_nodes, group_id,
) {
return true
}
if group.copy_source is Some(_) &&
!tuple_optimization_lane_locals_are_strictly_increasing(group.lane_locals) {
return true
}
match tuple_optimization_group_host_lane(group) {
Some(host_lane) =>
match
tuple_optimization_has_root_site_in_fresh_use_def(
func,
use_def,
group.defining_nodes[host_lane],
) {
true => false
false =>
match
tuple_optimization_find_passthrough_root_chain(
func,
use_def,
group.defining_nodes[host_lane],
) {
Some(chain) if !chain.is_empty() => false
_ =>
match
tuple_optimization_pick_root_anchor_slot(
func, use_def, group, host_lane,
) {
Some(_) => true
None => false
}
}
}
None =>
match
tuple_optimization_pick_no_host_root_carrier_slot(func, use_def, group) {
Some(slot) =>
if group.copy_source is None &&
slot.region_ref == @ir.HotRegionRef::root() &&
tuple_optimization_node_has_typeidx_control_type(
func,
group.producer,
) {
true
} else {
slot.region_ref != @ir.HotRegionRef::root() &&
(match @ir.hot_node_get(func, group.producer).op {
@ir.HotOp::Block
| @ir.HotOp::If
| @ir.HotOp::Loop
| @ir.HotOp::Try
| @ir.HotOp::TryTable => true
_ => false
})
}
None => false
}
}
}
///|
fn tuple_optimization_has_root_site_in_fresh_use_def(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
target_id : Int32,
) -> Bool {
if target_id < 0 || target_id >= @ir.hot_node_count(func) {
return false
}
// Read-only analysis and split-local planning have not changed the HOT graph.
// Later rewriting still uses the root-slot finder's stale-site fallback.
for site in @ir.node_use_sites_borrowed(use_def, target_id) {
if site.kind is @ir.HotUseSlotKind::RootUseSlot {
return true
}
}
false
}
///|
fn tuple_optimization_split_reuses_group_lanes(
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
) -> Bool {
split_locals == group.lane_locals
}
///|
fn tuple_optimization_find_root_slot_in_region(
func : @ir.HotFunc,
region_ref : @ir.HotRegionRef,
target_id : Int32,
) -> (@ir.HotRegionRef, Int32)? {
for slot = 0
slot < @ir.hot_region_root_count(func, region_ref)
slot = slot + 1 {
let root_id = @ir.hot_region_get(func, region_ref, slot)
if root_id == target_id {
return Some((region_ref, slot))
}
match @ir.hot_node_get(func, root_id).op {
@ir.HotOp::Block | @ir.HotOp::Loop =>
match
tuple_optimization_find_root_slot_in_region(
func,
@ir.hot_region_body_ref(func, root_id),
target_id,
) {
Some(found) => return Some(found)
None => ()
}
@ir.HotOp::If => {
match
tuple_optimization_find_root_slot_in_region(
func,
@ir.hot_region_then_ref(func, root_id),
target_id,
) {
Some(found) => return Some(found)
None => ()
}
match @ir.hot_region_else_ref(func, root_id) {
Some(else_ref) =>
match
tuple_optimization_find_root_slot_in_region(
func, else_ref, target_id,
) {
Some(found) => return Some(found)
None => ()
}
None => ()
}
}
@ir.HotOp::Try =>
match
tuple_optimization_find_root_slot_in_region(
func,
@ir.hot_region_body_ref(func, root_id),
target_id,
) {
Some(found) => return Some(found)
None =>
match @ir.hot_region_catch_ref(func, root_id) {
Some(catch_ref) =>
match
tuple_optimization_find_root_slot_in_region(
func, catch_ref, target_id,
) {
Some(found) => return Some(found)
None => ()
}
None => ()
}
}
@ir.HotOp::TryTable =>
match
tuple_optimization_find_root_slot_in_region(
func,
@ir.hot_region_body_ref(func, root_id),
target_id,
) {
Some(found) => return Some(found)
None =>
match
tuple_optimization_find_root_slot_in_region(
func,
@ir.hot_region_catch_list_ref(func, root_id),
target_id,
) {
Some(found) => return Some(found)
None => ()
}
}
_ => ()
}
}
None
}
///|
fn tuple_optimization_find_root_slot(
func : @ir.HotFunc,
target_id : Int32,
lane_idx : Int32,
) -> HotTupleOptimizationRootSlot? {
match
tuple_optimization_find_root_slot_in_region(
func,
@ir.HotRegionRef::root(),
target_id,
) {
Some((region_ref, slot)) => Some({ region_ref, lane_idx, slot, })
None => None
}
}
///|
fn tuple_optimization_find_root_slot_in_known_region(
func : @ir.HotFunc,
region_ref : @ir.HotRegionRef,
target_id : Int32,
lane_idx : Int32,
start_slot : Int32,
) -> HotTupleOptimizationRootSlot? {
let root_count = @ir.hot_region_root_count(func, region_ref)
let first_slot = if start_slot >= 0 && start_slot < root_count {
start_slot
} else {
0
}
for slot = first_slot; slot < root_count; slot = slot + 1 {
if @ir.hot_region_get(func, region_ref, slot) == target_id {
return Some({ region_ref, lane_idx, slot, })
}
}
for slot = 0; slot < first_slot; slot = slot + 1 {
if @ir.hot_region_get(func, region_ref, slot) == target_id {
return Some({ region_ref, lane_idx, slot, })
}
}
None
}
///|
fn tuple_optimization_find_root_slot_with_use_def(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
target_id : Int32,
lane_idx : Int32,
) -> HotTupleOptimizationRootSlot? {
if target_id >= 0 && target_id < @ir.hot_node_count(func) {
for site in @ir.node_use_sites_borrowed(use_def, target_id) {
match site.kind {
@ir.HotUseSlotKind::RootUseSlot => {
if site.user == -1 {
match
tuple_optimization_find_root_slot_in_known_region(
func,
@ir.HotRegionRef::root(),
target_id,
lane_idx,
site.slot,
) {
Some(slot) => return Some(slot)
None => continue
}
}
let user = @ir.hot_node_get(func, site.user)
for info in @ir.hot_control_region_slot_info(user.op) {
let region_ref = match info.role {
@ir.HotControlRegionRole::BodyRegion =>
Some(@ir.hot_region_body_ref(func, site.user))
@ir.HotControlRegionRole::ThenRegion =>
Some(@ir.hot_region_then_ref(func, site.user))
@ir.HotControlRegionRole::ElseRegion =>
@ir.hot_region_else_ref(func, site.user)
@ir.HotControlRegionRole::CatchRegion =>
@ir.hot_region_catch_ref(func, site.user)
@ir.HotControlRegionRole::CatchListRegion =>
Some(@ir.hot_region_catch_list_ref(func, site.user))
}
match region_ref {
Some(region_ref) =>
match
tuple_optimization_find_root_slot_in_known_region(
func,
region_ref,
target_id,
lane_idx,
site.slot,
) {
Some(slot) => return Some(slot)
None => ()
}
None => ()
}
}
}
_ => ()
}
}
}
tuple_optimization_find_root_slot(func, target_id, lane_idx)
}
///|
fn tuple_optimization_pick_root_anchor_slot(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
host_lane : Int32,
) -> HotTupleOptimizationRootSlot? {
let mut anchor : HotTupleOptimizationRootSlot? = None
for lane_idx = 0
lane_idx < group.defining_nodes.length()
lane_idx = lane_idx + 1 {
if lane_idx == host_lane {
continue
}
let slot = match
tuple_optimization_find_root_slot_with_use_def(
func,
use_def,
group.defining_nodes[lane_idx],
lane_idx,
) {
Some(slot) => slot
None => return None
}
// Copy producers are emptied after their replacement is built. An inner
// definition cannot anchor writes that must survive that cleanup. Keep
// those writes in the existing value-subtree carrier at the outer use.
if group.copy_source is Some(_) &&
group.producer >= 0 &&
slot.region_ref == @ir.hot_region_body_ref(func, group.producer) {
return None
}
match anchor {
Some(current) => {
if current.region_ref != slot.region_ref {
return None
}
if slot.slot < current.slot {
anchor = Some(slot)
}
}
None => anchor = Some(slot)
}
}
anchor
}
///|
fn tuple_optimization_find_passthrough_root_chain(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
target_id : Int32,
) -> Array[Int32]? {
let chain : Array[Int32] = []
let mut current = target_id
while true {
let sites = @ir.node_use_sites_borrowed(use_def, current)
if sites.length() != 1 {
return None
}
let site = sites[0]
match site.kind {
@ir.HotUseSlotKind::RootUseSlot => return Some(chain)
@ir.HotUseSlotKind::ChildUseSlot => {
if site.slot != 0 {
return None
}
let user = @ir.hot_node_get(func, site.user)
match user.op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee => {
if @ir.hot_child_count(func, site.user) != 1 {
return None
}
chain.push(site.user)
current = site.user
}
_ => return None
}
}
}
}
None
}
///|
fn tuple_optimization_wrap_passthrough_root_chain(
func : @ir.HotFunc,
chain : Array[Int32],
inner : Int32,
) -> Int32 {
let mut wrapped = inner
for node_id in chain {
let node = @ir.hot_node_get(func, node_id)
wrapped = match node.op {
@ir.HotOp::LocalSet => @ir.hot_build_local_set(func, node.imm0, wrapped)
@ir.HotOp::LocalTee => @ir.hot_build_local_tee(func, node.imm0, wrapped)
_ =>
abort(
"tuple-optimization passthrough root chains must stay within local.set/local.tee",
)
}
}
wrapped
}
///|
fn tuple_optimization_wrap_passthrough_root_chain_with_terminal_drop(
func : @ir.HotFunc,
chain : Array[Int32],
inner : Int32,
) -> Int32 {
let mut wrapped = inner
for idx = 0; idx < chain.length(); idx = idx + 1 {
let node_id = chain[idx]
let node = @ir.hot_node_get(func, node_id)
let is_root = idx == chain.length() - 1
wrapped = match node.op {
@ir.HotOp::LocalSet =>
if is_root {
@ir.hot_build_drop(
func,
@ir.hot_build_local_tee(func, node.imm0, wrapped),
)
} else {
@ir.hot_build_local_set(func, node.imm0, wrapped)
}
@ir.HotOp::LocalTee =>
if is_root {
@ir.hot_build_drop(
func,
@ir.hot_build_local_tee(func, node.imm0, wrapped),
)
} else {
@ir.hot_build_local_tee(func, node.imm0, wrapped)
}
_ =>
abort(
"tuple-optimization passthrough root chains must stay within local.set/local.tee",
)
}
}
wrapped
}
///|
fn tuple_optimization_build_source_host_replacement(
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
) -> Int32 {
let host_lane = tuple_optimization_group_host_lane(group)
if tuple_optimization_split_reuses_group_lanes(group, split_locals) {
let body : Array[Int32] = []
match host_lane {
Some(host_lane) => {
for lane_idx = group.lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
body.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
group.producer,
),
)
}
return @ir.hot_build_local_tee(
func,
group.lane_locals[host_lane],
@ir.hot_build_block(
func,
@lib.BlockType::val_type(
tuple_optimization_require_local_type(
func,
group.lane_locals[host_lane],
),
),
body + [group.producer],
),
)
}
None => {
for lane_idx = group.lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
body.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
group.producer,
),
)
}
return @ir.hot_build_block(func, @lib.BlockType::void_(), body)
}
}
}
let body : Array[Int32] = []
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
body.push(
@ir.hot_build_local_set(func, split_locals[lane_idx], group.producer),
)
}
match host_lane {
Some(host_lane) => {
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
body.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, split_locals[lane_idx]),
),
)
}
@ir.hot_build_local_tee(
func,
group.lane_locals[host_lane],
@ir.hot_build_block(
func,
@lib.BlockType::val_type(
tuple_optimization_require_local_type(
func,
group.lane_locals[host_lane],
),
),
body + [@ir.hot_build_local_get(func, split_locals[host_lane])],
),
)
}
None => {
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
body.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, split_locals[lane_idx]),
),
)
}
@ir.hot_build_block(func, @lib.BlockType::void_(), body)
}
}
}
///|
fn tuple_optimization_build_source_passthrough_init_root(
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
host_lane : Int32,
carrier_local : Int32,
) -> Int32 {
if host_lane < 0 || host_lane >= split_locals.length() {
abort("tuple-optimization passthrough init host lane must stay in range")
}
let init_body : Array[Int32] = []
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
init_body.push(
@ir.hot_build_local_set(func, split_locals[lane_idx], group.producer),
)
}
let body = [
@ir.hot_build_local_set(
func,
carrier_local,
@ir.hot_build_local_tee(
func,
split_locals[host_lane],
@ir.hot_build_block(
func,
@lib.BlockType::val_type(
tuple_optimization_require_local_type(func, split_locals[host_lane]),
),
init_body + [group.producer],
),
),
),
]
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
body.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, split_locals[lane_idx]),
),
)
}
@ir.hot_build_block(func, @lib.BlockType::void_(), body)
}
///|
fn tuple_optimization_build_source_root_anchor_replacement(
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
host_lane : Int32,
) -> Int32 {
let body : Array[Int32] = []
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
body.push(
@ir.hot_build_local_set(func, split_locals[lane_idx], group.producer),
)
}
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
body.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, split_locals[lane_idx]),
),
)
}
@ir.hot_build_block(func, @lib.BlockType::void_(), body)
}
///|
fn tuple_optimization_build_tuple_make_from_locals(
func : @ir.HotFunc,
lane_locals : Array[Int32],
) -> Int32 {
let values : Array[Int32] = []
let result_types : Array[@lib.ValType] = []
for local_id in lane_locals {
values.push(@ir.hot_build_local_get(func, local_id))
result_types.push(tuple_optimization_require_local_type(func, local_id))
}
@ir.hot_build_tuple_make(func, result_types, values)
}
///|
fn tuple_optimization_build_host_carrier_replacement_from_producer(
func : @ir.HotFunc,
lane_locals : Array[Int32],
host_lane : Int32,
producer : Int32,
) -> Int32 {
if host_lane < 0 || host_lane >= lane_locals.length() {
abort("tuple-optimization synthetic host carrier lane must stay in range")
}
let body : Array[Int32] = []
for lane_idx = lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
body.push(@ir.hot_build_local_set(func, lane_locals[lane_idx], producer))
}
@ir.hot_build_local_tee(
func,
lane_locals[host_lane],
@ir.hot_build_block(
func,
@lib.BlockType::val_type(
tuple_optimization_require_local_type(func, lane_locals[host_lane]),
),
body + [producer],
),
)
}
///|
fn tuple_optimization_build_host_result_from_producer(
func : @ir.HotFunc,
lane_locals : Array[Int32],
host_lane : Int32,
producer : Int32,
) -> Int32 {
if host_lane < 0 || host_lane >= lane_locals.length() {
abort("tuple-optimization synthetic host result lane must stay in range")
}
let body : Array[Int32] = []
for lane_idx = lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
body.push(@ir.hot_build_local_set(func, lane_locals[lane_idx], producer))
}
@ir.hot_build_block(
func,
@lib.BlockType::val_type(
tuple_optimization_require_local_type(func, lane_locals[host_lane]),
),
body + [producer],
)
}
///|
fn tuple_optimization_lane_copy_needs_tuple_carrier(
lane_locals : Array[Int32],
source_lane_locals : Array[Int32],
) -> Bool {
if lane_locals.length() != source_lane_locals.length() {
return true
}
for source_lane = 0
source_lane < source_lane_locals.length()
source_lane = source_lane + 1 {
let source_local = source_lane_locals[source_lane]
for dest_lane = 0
dest_lane < lane_locals.length()
dest_lane = dest_lane + 1 {
if source_local == lane_locals[dest_lane] && source_lane != dest_lane {
return true
}
}
}
false
}
///|
fn tuple_optimization_copy_rewrite_needs_tuple_carrier(
lane_locals : Array[Int32],
source_lane_locals : Array[Int32],
) -> Bool {
tuple_optimization_lane_copy_needs_tuple_carrier(
lane_locals, source_lane_locals,
) ||
!tuple_optimization_lane_locals_are_strictly_increasing(lane_locals) ||
!tuple_optimization_lane_locals_are_strictly_increasing(source_lane_locals)
}
///|
fn tuple_optimization_build_host_carrier_replacement_from_locals(
func : @ir.HotFunc,
lane_locals : Array[Int32],
host_lane : Int32,
source_lane_locals : Array[Int32],
use_tuple_carrier : Bool,
) -> Int32 {
if lane_locals.length() != source_lane_locals.length() {
abort("tuple-optimization host copy carriers must preserve arity")
}
if use_tuple_carrier {
return tuple_optimization_build_host_carrier_replacement_from_producer(
func,
lane_locals,
host_lane,
tuple_optimization_build_tuple_make_from_locals(func, source_lane_locals),
)
}
let body : Array[Int32] = []
for lane_idx = lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
body.push(
@ir.hot_build_local_set(
func,
lane_locals[lane_idx],
@ir.hot_build_local_get(func, source_lane_locals[lane_idx]),
),
)
}
@ir.hot_build_local_tee(
func,
lane_locals[host_lane],
@ir.hot_build_block(
func,
@lib.BlockType::val_type(
tuple_optimization_require_local_type(func, lane_locals[host_lane]),
),
body + [@ir.hot_build_local_get(func, source_lane_locals[host_lane])],
),
)
}
///|
fn tuple_optimization_build_copy_host_replacement(
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
source_split_locals : Array[Int32],
use_tuple_carrier : Bool,
) -> Int32 {
if split_locals.length() != source_split_locals.length() {
abort("tuple-optimization rewrite copy groups must preserve lane arity")
}
let host_lane = tuple_optimization_group_host_lane(group)
match host_lane {
Some(host_lane) => {
let producer_lane_locals = if tuple_optimization_split_reuses_group_lanes(
group, split_locals,
) {
source_split_locals
} else {
split_locals
}
let host_carrier = tuple_optimization_build_host_carrier_replacement_from_locals(
func,
group.lane_locals,
host_lane,
producer_lane_locals,
use_tuple_carrier,
)
if tuple_optimization_split_reuses_group_lanes(group, split_locals) {
return host_carrier
}
let prefix : Array[Int32] = []
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
prefix.push(
@ir.hot_build_local_set(
func,
split_locals[lane_idx],
@ir.hot_build_local_get(func, source_split_locals[lane_idx]),
),
)
}
return @ir.hot_build_block(
func,
@lib.BlockType::val_type(
tuple_optimization_require_local_type(
func,
group.lane_locals[host_lane],
),
),
prefix + [host_carrier],
)
}
None => ()
}
if tuple_optimization_split_reuses_group_lanes(group, split_locals) {
let body : Array[Int32] = []
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
body.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, source_split_locals[lane_idx]),
),
)
}
return @ir.hot_build_block(func, @lib.BlockType::void_(), body)
}
let body : Array[Int32] = []
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
body.push(
@ir.hot_build_local_set(
func,
split_locals[lane_idx],
@ir.hot_build_local_get(func, source_split_locals[lane_idx]),
),
)
}
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
body.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, split_locals[lane_idx]),
),
)
}
@ir.hot_build_block(func, @lib.BlockType::void_(), body)
}
///|
fn tuple_optimization_build_copy_root_anchor_replacement(
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
source_split_locals : Array[Int32],
host_lane : Int32,
) -> Int32 {
let body : Array[Int32] = []
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
body.push(
@ir.hot_build_local_set(
func,
split_locals[lane_idx],
@ir.hot_build_local_get(func, source_split_locals[lane_idx]),
),
)
}
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
body.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, split_locals[lane_idx]),
),
)
}
@ir.hot_build_block(func, @lib.BlockType::void_(), body)
}
///|
fn tuple_optimization_build_copy_root_anchor_tuple_carrier_replacement(
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
source_split_locals : Array[Int32],
host_lane : Int32,
) -> Int32 {
ignore(split_locals)
@ir.hot_build_block(func, @lib.BlockType::void_(), [
@ir.hot_build_drop(
func,
tuple_optimization_build_host_carrier_replacement_from_producer(
func,
group.lane_locals,
host_lane,
tuple_optimization_build_tuple_make_from_locals(
func, source_split_locals,
),
),
),
])
}
///|
fn tuple_optimization_pick_no_host_root_carrier_slot(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
) -> HotTupleOptimizationRootSlot? {
if group.producer < 0 || group.lane_locals.is_empty() {
return None
}
match
tuple_optimization_find_root_slot_with_use_def(
func,
use_def,
group.defining_nodes[0],
0,
) {
Some(slot) => Some(slot)
None => None
}
}
///|
fn tuple_optimization_build_source_root_carrier_replacement(
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
carrier_lane : Int32,
) -> Int32 {
if carrier_lane < 0 || carrier_lane >= group.lane_locals.length() {
abort("tuple-optimization synthetic source carrier lane must stay in range")
}
tuple_optimization_build_root_carrier_replacement_from_producer(
func,
group.producer,
group.lane_locals,
carrier_lane,
)
}
///|
fn tuple_optimization_build_source_nested_root_tuple_carrier_replacement(
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
carrier_lane : Int32,
) -> Int32 {
if carrier_lane < 0 || carrier_lane >= group.lane_locals.length() {
abort(
"tuple-optimization synthetic nested source carrier lane must stay in range",
)
}
if split_locals.length() != group.lane_locals.length() {
abort(
"tuple-optimization nested source root tuple carriers must preserve arity",
)
}
let prefix : Array[Int32] = []
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
prefix.push(
@ir.hot_build_local_set(func, split_locals[lane_idx], group.producer),
)
}
let values : Array[Int32] = []
let result_types : Array[@lib.ValType] = []
for local_id in split_locals {
values.push(@ir.hot_build_local_get(func, local_id))
result_types.push(tuple_optimization_require_local_type(func, local_id))
}
@ir.hot_build_block(
func,
@lib.BlockType::void_(),
prefix +
[
tuple_optimization_build_root_carrier_replacement_from_producer(
func,
@ir.hot_build_tuple_make(func, result_types, values),
group.lane_locals,
carrier_lane,
),
],
)
}
///|
fn tuple_optimization_build_root_carrier_replacement_from_producer(
func : @ir.HotFunc,
producer : Int32,
lane_locals : Array[Int32],
carrier_lane : Int32,
) -> Int32 {
if carrier_lane < 0 || carrier_lane >= lane_locals.length() {
abort("tuple-optimization synthetic root carrier lane must stay in range")
}
let body : Array[Int32] = []
for lane_idx = lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == carrier_lane {
continue
}
body.push(@ir.hot_build_local_set(func, lane_locals[lane_idx], producer))
}
@ir.hot_build_local_set(
func,
lane_locals[carrier_lane],
@ir.hot_build_block(
func,
@lib.BlockType::val_type(
tuple_optimization_require_local_type(func, lane_locals[carrier_lane]),
),
body + [producer],
),
)
}
///|
fn tuple_optimization_build_copy_root_carrier_replacement(
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
source_split_locals : Array[Int32],
carrier_lane : Int32,
use_tuple_carrier~ : Bool = false,
) -> Int32 {
if carrier_lane < 0 || carrier_lane >= group.lane_locals.length() {
abort("tuple-optimization synthetic copy carrier lane must stay in range")
}
if group.lane_locals.length() != source_split_locals.length() {
abort("tuple-optimization synthetic copy carriers must preserve arity")
}
if use_tuple_carrier ||
!tuple_optimization_lane_locals_are_strictly_increasing(group.lane_locals) ||
!tuple_optimization_lane_locals_are_strictly_increasing(source_split_locals) {
let values : Array[Int32] = []
let result_types : Array[@lib.ValType] = []
for lane_idx = 0
lane_idx < source_split_locals.length()
lane_idx = lane_idx + 1 {
let local_id = source_split_locals[lane_idx]
values.push(@ir.hot_build_local_get(func, local_id))
result_types.push(tuple_optimization_require_local_type(func, local_id))
}
return tuple_optimization_build_root_carrier_replacement_from_producer(
func,
@ir.hot_build_tuple_make(func, result_types, values),
group.lane_locals,
carrier_lane,
)
}
let body : Array[Int32] = []
for lane_idx = group.lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == carrier_lane {
continue
}
body.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, source_split_locals[lane_idx]),
),
)
}
@ir.hot_build_local_set(
func,
group.lane_locals[carrier_lane],
@ir.hot_build_block(
func,
@lib.BlockType::val_type(
tuple_optimization_require_local_type(
func,
group.lane_locals[carrier_lane],
),
),
body + [@ir.hot_build_local_get(func, source_split_locals[carrier_lane])],
),
)
}
///|
fn tuple_optimization_try_rewrite_root_host_copy_group_defs(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
source_split_locals : Array[Int32],
host_lane : Int32,
) -> Bool {
if host_lane < 0 || host_lane >= group.lane_locals.length() {
return false
}
if tuple_optimization_copy_rewrite_needs_tuple_carrier(
group.lane_locals,
source_split_locals,
) {
return false
}
let root_slot = match
tuple_optimization_find_root_slot_with_use_def(
func,
use_def,
group.defining_nodes[host_lane],
host_lane,
) {
Some(root_slot) => root_slot
None => return false
}
let source_lane_locals = if tuple_optimization_split_reuses_group_lanes(
group, split_locals,
) {
source_split_locals
} else {
split_locals
}
let prefix : Array[Int32] = []
if !tuple_optimization_split_reuses_group_lanes(group, split_locals) {
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
prefix.push(
@ir.hot_build_local_set(
func,
split_locals[lane_idx],
@ir.hot_build_local_get(func, source_split_locals[lane_idx]),
),
)
}
}
for lane_idx = group.lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
prefix.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, source_lane_locals[lane_idx]),
),
)
}
if !prefix.is_empty() {
@ir.hot_region_insert(func, root_slot.region_ref, root_slot.slot, prefix)
pass_mark_mutated(ctx, func)
}
tuple_optimization_replace_with_built(
ctx,
func,
group.defining_nodes[host_lane],
@ir.hot_build_local_tee(
func,
group.lane_locals[host_lane],
@ir.hot_build_local_get(func, source_lane_locals[host_lane]),
),
)
for lane_idx = 0
lane_idx < group.defining_nodes.length()
lane_idx = lane_idx + 1 {
if lane_idx == host_lane ||
!@ir.hot_node_is_live(func, group.defining_nodes[lane_idx]) {
continue
}
let replacement = match
@ir.hot_node_get(func, group.defining_nodes[lane_idx]).op {
@ir.HotOp::LocalTee =>
@ir.hot_build_local_get(func, source_lane_locals[lane_idx])
_ => @ir.hot_build_nop(func)
}
tuple_optimization_replace_with_built(
ctx,
func,
group.defining_nodes[lane_idx],
replacement,
)
}
true
}
///|
fn tuple_optimization_try_rewrite_root_no_host_copy_group_defs(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
source_split_locals : Array[Int32],
carrier_lane : Int32,
owned_payload_nodes~ : Array[Int32]? = None,
pending_publications~ : HotTupleCopyPublicationBatch? = None,
) -> Bool {
if carrier_lane < 0 || carrier_lane >= group.lane_locals.length() {
return false
}
if tuple_optimization_copy_rewrite_needs_tuple_carrier(
group.lane_locals,
source_split_locals,
) {
return false
}
let root_slot = match
tuple_optimization_find_root_slot_with_use_def(
func,
use_def,
group.defining_nodes[carrier_lane],
carrier_lane,
) {
Some(root_slot) => root_slot
None => return false
}
let source_lane_locals = if tuple_optimization_split_reuses_group_lanes(
group, split_locals,
) {
source_split_locals
} else {
split_locals
}
if owned_payload_nodes is Some(_) &&
!tuple_optimization_split_reuses_group_lanes(group, split_locals) {
return false
}
fn payload_node(lane : Int32) -> Int32 {
match owned_payload_nodes {
Some(nodes) => nodes[lane]
None => @ir.hot_build_local_get(func, source_lane_locals[lane])
}
}
let prefix : Array[Int32] = []
if !tuple_optimization_split_reuses_group_lanes(group, split_locals) {
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
prefix.push(
@ir.hot_build_local_set(
func,
split_locals[lane_idx],
@ir.hot_build_local_get(func, source_split_locals[lane_idx]),
),
)
}
}
for lane_idx = group.lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == carrier_lane {
continue
}
let publication = @ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
payload_node(lane_idx),
)
if owned_payload_nodes is Some(_) {
@ir.hot_preserve_replacement_position(
func,
publication,
group.defining_nodes[lane_idx],
)
}
prefix.push(publication)
}
if !prefix.is_empty() {
let queued = match pending_publications {
Some(batch) if owned_payload_nodes is Some(_) =>
tuple_optimization_queue_copy_publication(
batch,
root_slot.region_ref,
group.defining_nodes[carrier_lane],
prefix,
)
_ => false
}
if !queued {
@ir.hot_region_insert(func, root_slot.region_ref, root_slot.slot, prefix)
}
pass_mark_mutated(ctx, func)
}
let carrier_replacement = match
@ir.hot_node_get(func, group.defining_nodes[carrier_lane]).op {
@ir.HotOp::LocalTee =>
@ir.hot_build_local_tee(
func,
group.lane_locals[carrier_lane],
payload_node(carrier_lane),
)
@ir.HotOp::LocalSet =>
@ir.hot_build_local_set(
func,
group.lane_locals[carrier_lane],
payload_node(carrier_lane),
)
_ => return false
}
if owned_payload_nodes is Some(_) {
@ir.hot_preserve_replacement_position(
func,
carrier_replacement,
group.defining_nodes[carrier_lane],
)
}
tuple_optimization_replace_with_built(
ctx,
func,
group.defining_nodes[carrier_lane],
carrier_replacement,
)
for lane_idx = 0
lane_idx < group.defining_nodes.length()
lane_idx = lane_idx + 1 {
if lane_idx == carrier_lane ||
!@ir.hot_node_is_live(func, group.defining_nodes[lane_idx]) {
continue
}
let replacement = match
@ir.hot_node_get(func, group.defining_nodes[lane_idx]).op {
@ir.HotOp::LocalTee =>
@ir.hot_build_local_get(func, source_lane_locals[lane_idx])
_ => @ir.hot_build_nop(func)
}
tuple_optimization_replace_with_built(
ctx,
func,
group.defining_nodes[lane_idx],
replacement,
)
}
true
}
///|
fn tuple_optimization_clone_simple_lane_value(
func : @ir.HotFunc,
node_id : Int32,
) -> Int32? {
let node = @ir.hot_node_get(func, node_id)
match node.op {
@ir.HotOp::Const =>
match @ir.hot_const_get(func, node.imm0) {
@ir.HotConstPayload::I32Const(value) =>
Some(@ir.hot_build_const_i32(func, value))
@ir.HotConstPayload::I64Const(value) =>
Some(@ir.hot_build_const_i64(func, value))
@ir.HotConstPayload::F32Const(value) =>
Some(@ir.hot_build_const_f32(func, value))
@ir.HotConstPayload::F64Const(value) =>
Some(@ir.hot_build_const_f64(func, value))
_ => None
}
@ir.HotOp::LocalGet => Some(@ir.hot_build_local_get(func, node.imm0))
_ => None
}
}
///|
fn tuple_optimization_is_simple_lane_value(
func : @ir.HotFunc,
node_id : Int32,
) -> Bool {
let node = @ir.hot_node_get(func, node_id)
match node.op {
@ir.HotOp::Const =>
match @ir.hot_const_get(func, node.imm0) {
@ir.HotConstPayload::I32Const(_)
| @ir.HotConstPayload::I64Const(_)
| @ir.HotConstPayload::F32Const(_)
| @ir.HotConstPayload::F64Const(_) => true
_ => false
}
@ir.HotOp::LocalGet => true
_ => false
}
}
///|
fn tuple_optimization_simple_block_payload_lanes_exact_roots(
func : @ir.HotFunc,
producer : Int32,
lane_count : Int32,
) -> Array[Int32]? {
if lane_count <= 0 {
return None
}
match @ir.hot_node_get(func, producer).op {
@ir.HotOp::Block => ()
_ => return None
}
let body_ref = @ir.hot_region_body_ref(func, producer)
if @ir.hot_region_root_count(func, body_ref) != lane_count {
return None
}
let payload : Array[Int32] = []
for slot = 0; slot < lane_count; slot = slot + 1 {
let lane = @ir.hot_region_get(func, body_ref, slot)
if @ir.hot_node_result_arity(func, lane) != 1 ||
!tuple_optimization_is_simple_lane_value(func, lane) {
return None
}
payload.push(lane)
}
Some(payload)
}
///|
fn tuple_optimization_simple_block_has_exact_simple_payload_lanes(
func : @ir.HotFunc,
producer : Int32,
lane_count : Int32,
) -> Bool {
if lane_count <= 0 {
return false
}
match @ir.hot_node_get(func, producer).op {
@ir.HotOp::Block => ()
_ => return false
}
let body_ref = @ir.hot_region_body_ref(func, producer)
if @ir.hot_region_root_count(func, body_ref) != lane_count {
return false
}
for slot = 0; slot < lane_count; slot = slot + 1 {
let lane = @ir.hot_region_get(func, body_ref, slot)
if @ir.hot_node_result_arity(func, lane) != 1 ||
!tuple_optimization_is_simple_lane_value(func, lane) {
return false
}
}
true
}
///|
fn tuple_optimization_simple_block_payload_lanes(
func : @ir.HotFunc,
producer : Int32,
lane_count : Int32,
) -> Array[Int32]? {
match
tuple_optimization_simple_block_payload_lanes_exact_roots(
func, producer, lane_count,
) {
Some(payload) => return Some(payload)
None => ()
}
if lane_count <= 0 {
return None
}
match @ir.hot_node_get(func, producer).op {
@ir.HotOp::Block => ()
_ => return None
}
let body_ref = @ir.hot_region_body_ref(func, producer)
let (prefix, payload) = match
@ir.hot_region_split_trailing_payload_roots(func, body_ref, lane_count) {
Some(found) => found
None => return None
}
if !prefix.is_empty() || payload.length() != lane_count {
return None
}
for lane in payload {
if !tuple_optimization_is_simple_lane_value(func, lane) {
return None
}
}
Some(payload)
}
///|
fn tuple_optimization_clone_simple_block_payload_lanes(
func : @ir.HotFunc,
producer : Int32,
lane_count : Int32,
) -> Array[Int32]? {
let payload = match
tuple_optimization_simple_block_payload_lanes(func, producer, lane_count) {
Some(payload) => payload
None => return None
}
let clones : Array[Int32] = []
for lane in payload {
match tuple_optimization_clone_simple_lane_value(func, lane) {
Some(clone) => clones.push(clone)
None => return None
}
}
Some(clones)
}
///|
fn tuple_optimization_local_has_only_direct_drop_uses(
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
local_id : Int32,
) -> Bool {
if local_id < 0 {
return false
}
for read_id in @ir.local_read_nodes_borrowed(use_def, local_id) {
if read_id >= 0 &&
read_id < summary.scalar_forward_present.length() &&
summary.scalar_forward_present[read_id] {
return false
}
if tuple_optimization_node_has_direct_non_drop_use(summary, read_id) {
return false
}
if !tuple_optimization_node_has_direct_drop_use(summary, read_id) {
return false
}
}
true
}
///|
fn tuple_optimization_simple_drop_only_source_payload_lanes(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
group : HotTupleOptimizationGroup,
) -> Array[Int32]? {
if group.copy_source is Some(_) {
return None
}
if !tuple_optimization_is_simple_i32_i64_typeidx_group(func, group) {
return None
}
let payload = match
tuple_optimization_simple_block_payload_lanes(
func,
group.producer,
group.lane_locals.length(),
) {
Some(payload) => payload
None => return None
}
for lane_idx = 0
lane_idx < group.lane_locals.length()
lane_idx = lane_idx + 1 {
if !tuple_optimization_local_has_only_direct_drop_uses(
use_def,
summary,
group.lane_locals[lane_idx],
) {
return None
}
let defining_node = group.defining_nodes[lane_idx]
match @ir.hot_node_get(func, defining_node).op {
@ir.HotOp::LocalSet => ()
@ir.HotOp::LocalTee => {
if tuple_optimization_node_has_direct_non_drop_use(
summary, defining_node,
) {
return None
}
if !tuple_optimization_node_has_direct_drop_use(summary, defining_node) {
return None
}
}
_ => return None
}
}
Some(payload)
}
///|
fn tuple_optimization_replace_elided_drop_only_def(
func : @ir.HotFunc,
target_id : Int32,
replacement_id : Int32,
) -> Unit {
let replacement = @ir.hot_node_get(func, replacement_id)
@ir.hot_delete_detached_node(func, replacement_id)
@ir.hot_replace_node(func, target_id, replacement)
}
///|
fn tuple_optimization_root_regions_from_use_def(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
target_id : Int32,
) -> Array[@ir.HotRegionRef]? {
if target_id < 0 || target_id >= @ir.hot_node_count(func) {
return None
}
for site in @ir.node_use_sites_borrowed(use_def, target_id) {
match site.kind {
@ir.HotUseSlotKind::RootUseSlot => {
let regions : Array[@ir.HotRegionRef] = []
if site.user == -1 {
regions.push(@ir.HotRegionRef::root())
return Some(regions)
}
let user = @ir.hot_node_get(func, site.user)
for info in @ir.hot_control_region_slot_info(user.op) {
match info.role {
@ir.HotControlRegionRole::BodyRegion =>
regions.push(@ir.hot_region_body_ref(func, site.user))
@ir.HotControlRegionRole::ThenRegion =>
regions.push(@ir.hot_region_then_ref(func, site.user))
@ir.HotControlRegionRole::ElseRegion =>
match @ir.hot_region_else_ref(func, site.user) {
Some(region_ref) => regions.push(region_ref)
None => ()
}
@ir.HotControlRegionRole::CatchRegion =>
match @ir.hot_region_catch_ref(func, site.user) {
Some(region_ref) => regions.push(region_ref)
None => ()
}
@ir.HotControlRegionRole::CatchListRegion =>
regions.push(@ir.hot_region_catch_list_ref(func, site.user))
}
}
if regions.is_empty() {
return None
}
return Some(regions)
}
_ => ()
}
}
None
}
///|
fn tuple_optimization_has_single_function_root_use(
use_def : @ir.HotUseDef,
target_id : Int32,
) -> Bool {
let mut saw_root = false
for site in @ir.node_use_sites_borrowed(use_def, target_id) {
match site.kind {
@ir.HotUseSlotKind::RootUseSlot =>
if site.user == -1 {
if saw_root {
return false
}
saw_root = true
} else {
return false
}
_ => ()
}
}
saw_root
}
///|
fn tuple_optimization_try_elide_simple_drop_only_source_local_set_roots_fast_path(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
cleanup_plan : HotTupleOptimizationRewriteCleanupPlan,
) -> Bool {
let fast_path_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:local-set-root-fast-path",
)
if group.copy_source is Some(_) {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:local-set-root-fast-path",
fast_path_start_us,
)
return false
}
let roots : Array[Int32] = []
let regions_by_root : Array[Array[@ir.HotRegionRef]] = []
for defining_node in group.defining_nodes {
if defining_node < 0 ||
defining_node >= cleanup_plan.removable_root_defs.length() ||
!@ir.hot_node_is_live(func, defining_node) {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:local-set-root-fast-path",
fast_path_start_us,
)
return false
}
let node = @ir.hot_node_get(func, defining_node)
match node.op {
@ir.HotOp::LocalSet =>
if @ir.hot_child_count(func, defining_node) != 1 {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:local-set-root-fast-path",
fast_path_start_us,
)
return false
}
_ => {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:local-set-root-fast-path",
fast_path_start_us,
)
return false
}
}
let regions = match
tuple_optimization_root_regions_from_use_def(func, use_def, defining_node) {
Some(regions) => regions
None => {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:local-set-root-fast-path",
fast_path_start_us,
)
return false
}
}
roots.push(defining_node)
regions_by_root.push(regions)
}
for idx = 0; idx < roots.length(); idx = idx + 1 {
for region_ref in regions_by_root[idx] {
tuple_optimization_cleanup_plan_add_removable_root(
cleanup_plan,
roots[idx],
region_ref,
)
}
}
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:local-set-root-fast-path",
fast_path_start_us,
)
true
}
///|
fn tuple_optimization_try_elide_simple_drop_only_source_group_defs(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
cleanup_plan : HotTupleOptimizationRewriteCleanupPlan,
precomputed_payload_lanes~ : Array[Int32]? = None,
) -> Bool {
let payload = match precomputed_payload_lanes {
Some(payload) => payload
None => {
let payload_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:payload",
)
let payload = match
tuple_optimization_simple_block_payload_lanes(
func,
group.producer,
group.lane_locals.length(),
) {
Some(payload) => payload
None => {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:payload",
payload_start_us,
)
return false
}
}
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:payload",
payload_start_us,
)
payload
}
}
let replace_defs_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:replace-defs",
)
let nop_replacement_id = @ir.hot_build_nop(func)
let nop_replacement = @ir.hot_node_get(func, nop_replacement_id)
@ir.hot_delete_detached_node(func, nop_replacement_id)
let mut immediate_changed = false
for lane_idx = 0
lane_idx < group.defining_nodes.length()
lane_idx = lane_idx + 1 {
let defining_node = group.defining_nodes[lane_idx]
if !@ir.hot_node_is_live(func, defining_node) {
continue
}
match @ir.hot_node_get(func, defining_node).op {
@ir.HotOp::LocalSet =>
match
tuple_optimization_find_root_slot_with_use_def(
func, use_def, defining_node, lane_idx,
) {
Some(root_slot) =>
tuple_optimization_cleanup_plan_add_removable_root(
cleanup_plan,
defining_node,
root_slot.region_ref,
)
None => {
@ir.hot_replace_node(func, defining_node, nop_replacement)
cleanup_plan.generated_nops = true
immediate_changed = true
}
}
@ir.HotOp::LocalTee => {
let replacement = match
tuple_optimization_clone_simple_lane_value(func, payload[lane_idx]) {
Some(clone) => clone
None => {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:replace-defs",
replace_defs_start_us,
)
return false
}
}
tuple_optimization_replace_elided_drop_only_def(
func, defining_node, replacement,
)
immediate_changed = true
}
_ => {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:replace-defs",
replace_defs_start_us,
)
return false
}
}
}
if immediate_changed {
pass_mark_mutated(ctx, func)
}
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:replace-defs",
replace_defs_start_us,
)
true
}
///|
fn tuple_optimization_find_definition_root_slot(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
defining_node : Int32,
lane_idx : Int32,
) -> HotTupleOptimizationRootSlot? {
let mut current = defining_node
let mut remaining = @ir.hot_node_count(func)
while remaining > 0 {
remaining -= 1
match
tuple_optimization_find_root_slot_with_use_def(
func, use_def, current, lane_idx,
) {
Some(slot) => return Some(slot)
None => ()
}
let sites = @ir.node_use_sites_borrowed(use_def, current)
if sites.length() != 1 {
return None
}
let site = sites[0]
if !(site.kind is @ir.HotUseSlotKind::ChildUseSlot) || site.slot != 0 {
return None
}
// Local/drop wrappers evaluate their sole operand first. A direct call
// evaluates operand zero before its later operands and its call effect.
match @ir.hot_node_get(func, site.user).op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee | @ir.HotOp::Drop =>
if @ir.hot_child_count(func, site.user) != 1 {
return None
}
@ir.HotOp::Call => ()
_ => return None
}
current = site.user
}
None
}
///|
fn tuple_optimization_try_rewrite_root_source_group_defs(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
) -> Bool {
if group.defining_nodes.is_empty() {
return false
}
if tuple_optimization_split_reuses_group_lanes(group, split_locals) {
return false
}
let root_slot = match
tuple_optimization_find_definition_root_slot(
func,
use_def,
group.defining_nodes[0],
0,
) {
Some(root_slot) => root_slot
None => return false
}
let mut stage_slot = root_slot.slot
for lane_idx = 0
lane_idx < group.defining_nodes.length()
lane_idx = lane_idx + 1 {
let defining_node = group.defining_nodes[lane_idx]
if !@ir.hot_node_is_live(func, defining_node) {
return false
}
match @ir.hot_node_get(func, defining_node).op {
@ir.HotOp::LocalSet | @ir.HotOp::LocalTee => ()
_ => return false
}
match
tuple_optimization_find_definition_root_slot(
func, use_def, defining_node, lane_idx,
) {
Some(slot) if slot.region_ref == root_slot.region_ref =>
stage_slot = stage_slot.min(slot.slot)
_ => return false
}
}
let prefix : Array[Int32] = []
match
tuple_optimization_clone_simple_block_payload_lanes(
func,
group.producer,
split_locals.length(),
) {
Some(payload_lanes) =>
for lane_idx = 0
lane_idx < split_locals.length()
lane_idx = lane_idx + 1 {
prefix.push(
@ir.hot_build_local_set(
func,
split_locals[lane_idx],
payload_lanes[lane_idx],
),
)
}
None =>
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
prefix.push(
@ir.hot_build_local_set(func, split_locals[lane_idx], group.producer),
)
}
}
let omit_drop_only_original_lanes = tuple_optimization_is_simple_i32_i64_typeidx_group(
func, group,
)
// These roots extract the existing producer, rather than add a computation
// at the insertion point. Retain its order so an earlier pending operand
// (including a call or trap) is evaluated before the staged producer.
for root in prefix {
@ir.hot_preserve_replacement_position(func, root, group.producer)
}
// A previously generated scratch lane can be read between these writes.
// Stage before the first definition, then publish each lane at its old site.
@ir.hot_region_insert(func, root_slot.region_ref, stage_slot, prefix)
pass_mark_mutated(ctx, func)
for lane_idx = 0
lane_idx < group.defining_nodes.length()
lane_idx = lane_idx + 1 {
let defining_node = group.defining_nodes[lane_idx]
let has_non_drop_use = !omit_drop_only_original_lanes ||
tuple_optimization_local_has_forwarded_non_drop_use(
use_def,
summary,
group.lane_locals[lane_idx],
)
let value = @ir.hot_build_local_get(func, split_locals[lane_idx])
let replacement = match @ir.hot_node_get(func, defining_node).op {
@ir.HotOp::LocalTee =>
if has_non_drop_use {
@ir.hot_build_local_tee(func, group.lane_locals[lane_idx], value)
} else {
value
}
@ir.HotOp::LocalSet =>
if has_non_drop_use {
@ir.hot_build_local_set(func, group.lane_locals[lane_idx], value)
} else {
@ir.hot_build_drop(func, value)
}
_ => return false
}
@ir.hot_preserve_replacement_position(func, replacement, defining_node)
tuple_optimization_replace_with_built(ctx, func, defining_node, replacement)
}
true
}
///|
fn tuple_optimization_try_rewrite_anchor_host_source_group_defs(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
host_lane : Int32,
anchor_slot : HotTupleOptimizationRootSlot,
) -> Bool {
if host_lane < 0 || host_lane >= group.lane_locals.length() {
return false
}
if tuple_optimization_split_reuses_group_lanes(group, split_locals) {
return false
}
let payload_lanes = match
tuple_optimization_clone_simple_block_payload_lanes(
func,
group.producer,
split_locals.length(),
) {
Some(payload_lanes) => payload_lanes
None => return false
}
let prefix : Array[Int32] = []
for lane_idx = 0; lane_idx < split_locals.length(); lane_idx = lane_idx + 1 {
prefix.push(
@ir.hot_build_local_set(
func,
split_locals[lane_idx],
payload_lanes[lane_idx],
),
)
}
let omit_drop_only_original_lanes = tuple_optimization_is_simple_i32_i64_typeidx_group(
func, group,
)
for lane_idx = group.lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
if omit_drop_only_original_lanes &&
!tuple_optimization_local_has_forwarded_non_drop_use(
use_def,
summary,
group.lane_locals[lane_idx],
) {
continue
}
prefix.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, split_locals[lane_idx]),
),
)
}
@ir.hot_region_insert(func, anchor_slot.region_ref, anchor_slot.slot, prefix)
pass_mark_mutated(ctx, func)
for lane_idx = 0
lane_idx < group.defining_nodes.length()
lane_idx = lane_idx + 1 {
if !@ir.hot_node_is_live(func, group.defining_nodes[lane_idx]) {
continue
}
let lane_has_non_drop_use = !omit_drop_only_original_lanes ||
tuple_optimization_local_has_forwarded_non_drop_use(
use_def,
summary,
group.lane_locals[lane_idx],
)
let replacement = match
@ir.hot_node_get(func, group.defining_nodes[lane_idx]).op {
@ir.HotOp::LocalTee =>
if lane_has_non_drop_use {
@ir.hot_build_local_tee(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, split_locals[lane_idx]),
)
} else {
@ir.hot_build_local_get(func, split_locals[lane_idx])
}
@ir.HotOp::LocalSet =>
if lane_idx == host_lane && lane_has_non_drop_use {
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, split_locals[lane_idx]),
)
} else if lane_idx == host_lane {
@ir.hot_build_drop(
func,
@ir.hot_build_local_get(func, split_locals[lane_idx]),
)
} else {
@ir.hot_build_nop(func)
}
_ => @ir.hot_build_nop(func)
}
tuple_optimization_replace_with_built(
ctx,
func,
group.defining_nodes[lane_idx],
replacement,
)
}
true
}
///|
fn tuple_optimization_try_rewrite_anchor_host_copy_group_defs(
ctx : HotPassContext,
func : @ir.HotFunc,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
source_split_locals : Array[Int32],
host_lane : Int32,
anchor_slot : HotTupleOptimizationRootSlot,
use_tuple_carrier : Bool,
preserve_host_write_at_def~ : Bool = false,
) -> Bool {
if host_lane < 0 || host_lane >= group.lane_locals.length() {
return false
}
if !use_tuple_carrier &&
tuple_optimization_copy_rewrite_needs_tuple_carrier(
group.lane_locals,
source_split_locals,
) {
return false
}
let source_lane_locals = if use_tuple_carrier ||
tuple_optimization_split_reuses_group_lanes(group, split_locals) {
source_split_locals
} else {
split_locals
}
let split_host_carrier_local : Int32? = if tuple_optimization_split_reuses_group_lanes(
group, split_locals,
) {
None
} else if use_tuple_carrier {
None
} else {
Some(
@ir.hot_append_body_local(
func,
tuple_optimization_require_local_type(
func,
group.lane_locals[host_lane],
),
),
)
}
let carrier_local = if preserve_host_write_at_def {
match split_host_carrier_local {
Some(split_host_carrier_local) => split_host_carrier_local
None =>
@ir.hot_append_body_local(
func,
tuple_optimization_require_local_type(
func,
group.lane_locals[host_lane],
),
)
}
} else {
@ir.hot_append_body_local(
func,
tuple_optimization_require_local_type(func, group.lane_locals[host_lane]),
)
}
let host_source_value = if use_tuple_carrier {
tuple_optimization_build_host_result_from_producer(
func,
group.lane_locals,
host_lane,
tuple_optimization_build_tuple_make_from_locals(func, source_lane_locals),
)
} else {
@ir.hot_build_local_get(
func,
match split_host_carrier_local {
Some(split_host_carrier_local) => split_host_carrier_local
None => source_lane_locals[host_lane]
},
)
}
let prefix : Array[Int32] = []
if !use_tuple_carrier &&
!tuple_optimization_split_reuses_group_lanes(group, split_locals) {
for lane_idx = split_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
match split_host_carrier_local {
Some(split_host_carrier_local) =>
prefix.push(
@ir.hot_build_local_set(
func,
split_host_carrier_local,
@ir.hot_build_local_tee(
func,
split_locals[lane_idx],
@ir.hot_build_local_get(func, source_split_locals[lane_idx]),
),
),
)
None =>
prefix.push(
@ir.hot_build_local_set(
func,
split_locals[lane_idx],
@ir.hot_build_local_get(func, source_split_locals[lane_idx]),
),
)
}
} else {
prefix.push(
@ir.hot_build_local_set(
func,
split_locals[lane_idx],
@ir.hot_build_local_get(func, source_split_locals[lane_idx]),
),
)
}
}
}
if !use_tuple_carrier {
if preserve_host_write_at_def && split_host_carrier_local is Some(_) {
let mut highest_non_host_lane = -1
for lane_idx = group.lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx != host_lane {
highest_non_host_lane = lane_idx
break
}
}
let carried_non_host_locals : Array[Int32] = Array::make(
group.lane_locals.length(),
-1,
)
for lane_idx = 0
lane_idx < group.lane_locals.length()
lane_idx = lane_idx + 1 {
if lane_idx == host_lane || lane_idx == highest_non_host_lane {
continue
}
let carried_local = @ir.hot_append_body_local(
func,
tuple_optimization_require_local_type(
func,
group.lane_locals[lane_idx],
),
)
carried_non_host_locals[lane_idx] = carried_local
prefix.push(
@ir.hot_build_local_set(
func,
carried_local,
@ir.hot_build_local_get(func, source_lane_locals[lane_idx]),
),
)
}
if highest_non_host_lane >= 0 {
prefix.push(
@ir.hot_build_local_set(
func,
group.lane_locals[highest_non_host_lane],
@ir.hot_build_local_get(
func,
source_lane_locals[highest_non_host_lane],
),
),
)
}
for lane_idx = group.lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane || lane_idx == highest_non_host_lane {
continue
}
let carried_local = carried_non_host_locals[lane_idx]
if carried_local < 0 {
continue
}
prefix.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, carried_local),
),
)
}
} else {
for lane_idx = group.lane_locals.length() - 1
lane_idx >= 0
lane_idx = lane_idx - 1 {
if lane_idx == host_lane {
continue
}
prefix.push(
@ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
@ir.hot_build_local_get(func, source_lane_locals[lane_idx]),
),
)
}
}
}
if !preserve_host_write_at_def || split_host_carrier_local is None {
prefix.push(
@ir.hot_build_local_set(
func,
carrier_local,
if preserve_host_write_at_def {
host_source_value
} else {
@ir.hot_build_local_tee(
func,
group.lane_locals[host_lane],
host_source_value,
)
},
),
)
}
@ir.hot_region_insert(func, anchor_slot.region_ref, anchor_slot.slot, prefix)
pass_mark_mutated(ctx, func)
for lane_idx = 0
lane_idx < group.defining_nodes.length()
lane_idx = lane_idx + 1 {
if !@ir.hot_node_is_live(func, group.defining_nodes[lane_idx]) {
continue
}
let replacement = match
@ir.hot_node_get(func, group.defining_nodes[lane_idx]).op {
@ir.HotOp::LocalTee =>
if lane_idx == host_lane {
if preserve_host_write_at_def {
@ir.hot_build_local_tee(
func,
group.lane_locals[host_lane],
@ir.hot_build_local_get(func, carrier_local),
)
} else {
@ir.hot_build_local_get(func, carrier_local)
}
} else {
@ir.hot_build_local_get(func, group.lane_locals[lane_idx])
}
@ir.HotOp::LocalSet =>
if lane_idx == host_lane && preserve_host_write_at_def {
@ir.hot_build_drop(
func,
@ir.hot_build_local_tee(
func,
group.lane_locals[host_lane],
@ir.hot_build_local_get(func, carrier_local),
),
)
} else {
@ir.hot_build_nop(func)
}
_ => @ir.hot_build_nop(func)
}
tuple_optimization_replace_with_built(
ctx,
func,
group.defining_nodes[lane_idx],
replacement,
)
}
true
}
///|
fn tuple_optimization_retire_empty_control(
func : @ir.HotFunc,
producer : Int32,
pending_controls : Array[Int32]?,
) -> Unit {
match pending_controls {
Some(pending) => pending.push(producer)
None =>
if !@ir.hot_delete_detached_control_subtree(func, producer) {
abort("tuple-optimization moved control must have one detached owner")
}
}
}
///|
fn tuple_optimization_try_rewrite_direct_scalar_copy_group_defs(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
pending_controls~ : Array[Int32]? = None,
pending_publications~ : HotTupleCopyPublicationBatch? = None,
) -> Bool {
let reads = match
tuple_optimization_direct_scalar_copy_payload_nodes(func, use_def, group) {
Some(reads) => reads
None => return false
}
let payload = group.copy_payload_locals.unwrap()
if !tuple_optimization_try_rewrite_root_no_host_copy_group_defs(
ctx,
func,
use_def,
group,
group.lane_locals,
payload,
0,
owned_payload_nodes=Some(reads),
pending_publications~,
) {
return false
}
// Reads now belong to the outer stores. The driver checks all empty owners
// together before cleanup, including detachment and unused-label retirement.
@ir.hot_region_replace_body(
func,
@ir.hot_region_body_ref(func, group.producer),
[],
)
tuple_optimization_retire_empty_control(
func,
group.producer,
pending_controls,
)
pass_mark_mutated(ctx, func)
true
}
///|
fn tuple_optimization_rewrite_group_defs(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
group : HotTupleOptimizationGroup,
split_locals : Array[Int32],
cleanup_plan : HotTupleOptimizationRewriteCleanupPlan,
has_rewritten_copy_consumer~ : Bool = false,
source_host_copy_consumer~ : Bool = false,
source_group_has_host_lane~ : Bool = false,
source_split_locals~ : Array[Int32]? = None,
elide_simple_drop_only_source~ : Bool = false,
pending_controls~ : Array[Int32]? = None,
pending_publications~ : HotTupleCopyPublicationBatch? = None,
) -> Unit {
let actual_host_lane = tuple_optimization_group_host_lane(group)
let original_def_orders = group.defining_nodes.map(fn(node) {
@ir.hot_node_get(func, node).order
})
if elide_simple_drop_only_source {
let elide_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source",
)
if tuple_optimization_try_elide_simple_drop_only_source_group_defs(
ctx, func, use_def, group, cleanup_plan,
) {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source",
elide_start_us,
)
return
}
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source",
elide_start_us,
)
}
if !has_rewritten_copy_consumer &&
tuple_optimization_split_reuses_group_lanes(group, split_locals) {
match tuple_optimization_pure_adjacent_source_lanes(func, use_def, group) {
Some(lanes) => {
for lane_idx = 0; lane_idx < lanes.length(); lane_idx = lane_idx + 1 {
let definition = group.defining_nodes[lane_idx]
let replacement = @ir.hot_build_local_set(
func,
group.lane_locals[lane_idx],
lanes[lane_idx],
)
@ir.hot_preserve_replacement_position(func, replacement, definition)
tuple_optimization_replace_with_built(
ctx, func, definition, replacement,
)
}
// Move the scalar roots, then retain only the empty detached owner
// until the driver's checked retirement batch.
@ir.hot_region_replace_body(
func,
@ir.hot_region_body_ref(func, group.producer),
[],
)
tuple_optimization_retire_empty_control(
func,
group.producer,
pending_controls,
)
pass_mark_mutated(ctx, func)
return
}
None => ()
}
}
cleanup_plan.generated_nops = true
match actual_host_lane {
None =>
if tuple_optimization_try_rewrite_direct_scalar_copy_group_defs(
ctx,
func,
use_def,
group,
pending_controls~,
pending_publications~,
) {
return
}
_ => ()
}
if group.copy_source is None &&
tuple_optimization_try_rewrite_root_source_group_defs(
ctx, func, use_def, summary, group, split_locals,
) {
return
}
let no_host_root_carrier_slot = match actual_host_lane {
None =>
tuple_optimization_pick_no_host_root_carrier_slot(func, use_def, group)
Some(_) => None
}
let synthetic_carrier_lane = match actual_host_lane {
None =>
match no_host_root_carrier_slot {
Some(slot) => Some(slot.lane_idx)
None => None
}
Some(_) => None
}
let synthetic_carrier_needs_tuple = match no_host_root_carrier_slot {
Some(slot) => slot.region_ref != @ir.HotRegionRef::root()
None => false
}
let primary_lane = match actual_host_lane {
Some(host_lane) => host_lane
None =>
match synthetic_carrier_lane {
Some(carrier_lane) => carrier_lane
None => group.defining_nodes.length() - 1
}
}
let host_root_slot = match actual_host_lane {
Some(host_lane) =>
tuple_optimization_find_root_slot_with_use_def(
func,
use_def,
group.defining_nodes[host_lane],
host_lane,
)
None => None
}
let host_is_root = match host_root_slot {
Some(_) => true
None => false
}
let host_is_function_root = match host_root_slot {
Some(root_slot) => root_slot.region_ref == @ir.HotRegionRef::root()
None => false
}
let host_passthrough_chain = match actual_host_lane {
Some(host_lane) if !host_is_root =>
match
tuple_optimization_find_passthrough_root_chain(
func,
use_def,
group.defining_nodes[host_lane],
) {
Some(chain) if !chain.is_empty() => Some(chain)
_ => None
}
_ => None
}
if source_split_locals is None &&
source_host_copy_consumer &&
!tuple_optimization_split_reuses_group_lanes(group, split_locals) {
match (actual_host_lane, host_passthrough_chain) {
(Some(host_lane), Some(chain)) => {
let root_id = chain[chain.length() - 1]
let root_slot = match
tuple_optimization_find_root_slot_with_use_def(
func, use_def, root_id, host_lane,
) {
Some(slot) => slot
None =>
abort(
"tuple-optimization source passthrough staging requires a root slot",
)
}
let carrier_local = @ir.hot_append_body_local(
func,
tuple_optimization_require_local_type(
func,
group.lane_locals[host_lane],
),
)
@ir.hot_region_insert(func, root_slot.region_ref, root_slot.slot, [
tuple_optimization_build_source_passthrough_init_root(
func, group, split_locals, host_lane, carrier_local,
),
])
pass_mark_mutated(ctx, func)
tuple_optimization_replace_with_built(
ctx,
func,
group.defining_nodes[host_lane],
@ir.hot_build_local_tee(
func,
group.lane_locals[host_lane],
@ir.hot_build_local_get(func, carrier_local),
),
)
for lane_idx = 0
lane_idx < group.defining_nodes.length()
lane_idx = lane_idx + 1 {
if lane_idx == host_lane ||
!@ir.hot_node_is_live(func, group.defining_nodes[lane_idx]) {
continue
}
tuple_optimization_replace_with_built(
ctx,
func,
group.defining_nodes[lane_idx],
@ir.hot_build_nop(func),
)
}
return
}
_ => ()
}
}
let copy_has_non_host_drop_tail = match actual_host_lane {
Some(host_lane) => {
let mut has_drop_tail = false
for lane_idx = 0
lane_idx < group.lane_locals.length()
lane_idx = lane_idx + 1 {
if lane_idx == host_lane {
continue
}
if tuple_optimization_local_has_forwarded_drop_use(
use_def,
summary,
group.lane_locals[lane_idx],
) {
has_drop_tail = true
break
}
}
has_drop_tail
}
None => false
}
let copy_host_has_drop_tail = match actual_host_lane {
Some(host_lane) => {
let host_def = group.defining_nodes[host_lane]
let direct_drop_use = match @ir.hot_node_get(func, host_def).op {
@ir.HotOp::LocalTee =>
tuple_optimization_node_has_direct_drop_use(summary, host_def)
_ => false
}
direct_drop_use ||
tuple_optimization_local_has_forwarded_drop_use(
use_def,
summary,
group.lane_locals[host_lane],
)
}
None => false
}
let copy_host_needs_tuple_carrier = match source_split_locals {
Some(source_split_locals) =>
tuple_optimization_lane_copy_needs_tuple_carrier(
group.lane_locals,
source_split_locals,
) ||
!tuple_optimization_lane_locals_are_strictly_increasing(
source_split_locals,
) ||
(match host_root_slot {
Some(root_slot) => root_slot.region_ref != @ir.HotRegionRef::root()
None => false
})
None => false
}
let anchor_slot = match actual_host_lane {
Some(host_lane) if host_passthrough_chain is None &&
(
!host_is_root ||
(
host_is_function_root &&
copy_has_non_host_drop_tail &&
!source_group_has_host_lane
)
) =>
tuple_optimization_pick_root_anchor_slot(func, use_def, group, host_lane)
_ => None
}
let has_anchor_slot = match anchor_slot {
Some(_) => true
None => false
}
let anchor_host_preserves_host_write_at_def = !has_rewritten_copy_consumer &&
copy_host_has_drop_tail
match (actual_host_lane, source_split_locals, anchor_slot) {
(Some(host_lane), Some(source_split_locals), Some(anchor_slot)) if !has_rewritten_copy_consumer ||
copy_host_has_drop_tail =>
if tuple_optimization_try_rewrite_anchor_host_copy_group_defs(
ctx,
func,
group,
split_locals,
source_split_locals,
host_lane,
anchor_slot,
copy_host_needs_tuple_carrier,
preserve_host_write_at_def=anchor_host_preserves_host_write_at_def,
) {
return
}
_ => ()
}
match (actual_host_lane, source_split_locals) {
(Some(host_lane), Some(source_split_locals)) if host_is_function_root &&
!copy_host_needs_tuple_carrier =>
if tuple_optimization_try_rewrite_root_host_copy_group_defs(
ctx, func, use_def, group, split_locals, source_split_locals, host_lane,
) {
return
}
_ => ()
}
match (actual_host_lane, source_split_locals, anchor_slot) {
(Some(host_lane), None, Some(anchor_slot)) =>
if tuple_optimization_try_rewrite_anchor_host_source_group_defs(
ctx, func, use_def, summary, group, split_locals, host_lane, anchor_slot,
) {
return
}
_ => ()
}
match (actual_host_lane, source_split_locals, synthetic_carrier_lane) {
(None, Some(source_split_locals), Some(carrier_lane)) if !synthetic_carrier_needs_tuple =>
if tuple_optimization_try_rewrite_root_no_host_copy_group_defs(
ctx, func, use_def, group, split_locals, source_split_locals, carrier_lane,
) {
return
}
_ => ()
}
let copy_anchor_uses_tuple_carrier = match source_split_locals {
Some(_) => has_anchor_slot && copy_host_needs_tuple_carrier
None => false
}
let host_read_replacement = match actual_host_lane {
Some(host_lane) if has_anchor_slot =>
if copy_anchor_uses_tuple_carrier {
match @ir.hot_node_get(func, group.defining_nodes[host_lane]).op {
@ir.HotOp::LocalTee =>
Some(@ir.hot_build_local_get(func, group.lane_locals[host_lane]))
_ => Some(@ir.hot_build_nop(func))
}
} else {
Some(
@ir.hot_build_local_tee(
func,
group.lane_locals[host_lane],
@ir.hot_build_local_get(func, split_locals[host_lane]),
),
)
}
_ => None
}
let raw_host_replacement = match source_split_locals {
Some(source_split_locals) =>
match actual_host_lane {
Some(host_lane) if has_anchor_slot =>
if copy_anchor_uses_tuple_carrier {
tuple_optimization_build_copy_root_anchor_tuple_carrier_replacement(
func, group, split_locals, source_split_locals, host_lane,
)
} else {
tuple_optimization_build_copy_root_anchor_replacement(
func, group, split_locals, source_split_locals, host_lane,
)
}
None =>
match synthetic_carrier_lane {
Some(carrier_lane) =>
tuple_optimization_build_copy_root_carrier_replacement(
func,
group,
source_split_locals,
carrier_lane,
use_tuple_carrier=synthetic_carrier_needs_tuple,
)
None =>
tuple_optimization_build_copy_host_replacement(
func, group, split_locals, source_split_locals, copy_host_needs_tuple_carrier,
)
}
_ =>
tuple_optimization_build_copy_host_replacement(
func, group, split_locals, source_split_locals, copy_host_needs_tuple_carrier,
)
}
None =>
match actual_host_lane {
Some(host_lane) if has_anchor_slot =>
tuple_optimization_build_source_root_anchor_replacement(
func, group, split_locals, host_lane,
)
None =>
match synthetic_carrier_lane {
Some(carrier_lane) =>
if synthetic_carrier_needs_tuple &&
!tuple_optimization_split_reuses_group_lanes(
group, split_locals,
) {
tuple_optimization_build_source_nested_root_tuple_carrier_replacement(
func, group, split_locals, carrier_lane,
)
} else {
tuple_optimization_build_source_root_carrier_replacement(
func, group, carrier_lane,
)
}
None =>
tuple_optimization_build_source_host_replacement(
func, group, split_locals,
)
}
_ =>
tuple_optimization_build_source_host_replacement(
func, group, split_locals,
)
}
}
let host_replacement = match host_passthrough_chain {
Some(chain) => {
let terminal_passthrough_uses_root_local_set = match
@ir.hot_node_get(func, chain[chain.length() - 1]).op {
@ir.HotOp::LocalSet => true
_ => false
}
if source_split_locals is Some(_) &&
!has_rewritten_copy_consumer &&
source_group_has_host_lane &&
terminal_passthrough_uses_root_local_set {
tuple_optimization_wrap_passthrough_root_chain_with_terminal_drop(
func, chain, raw_host_replacement,
)
} else {
tuple_optimization_wrap_passthrough_root_chain(
func, chain, raw_host_replacement,
)
}
}
None => raw_host_replacement
}
let original_direct_host_drop = match actual_host_lane {
Some(host_lane) if source_split_locals is Some(_) &&
source_group_has_host_lane &&
!has_rewritten_copy_consumer =>
tuple_optimization_find_direct_drop_use(
func,
use_def,
group.defining_nodes[host_lane],
)
_ => None
}
let anchor_lane = match anchor_slot {
Some(anchor_slot) => anchor_slot.lane_idx
None => -1
}
tuple_optimization_replace_with_built(
ctx,
func,
match host_passthrough_chain {
Some(chain) => chain[chain.length() - 1]
None =>
match anchor_slot {
Some(anchor_slot) => group.defining_nodes[anchor_slot.lane_idx]
None => group.defining_nodes[primary_lane]
}
},
host_replacement,
)
for lane_idx = 0
lane_idx < group.defining_nodes.length()
lane_idx = lane_idx + 1 {
if lane_idx == primary_lane ||
lane_idx == anchor_lane ||
!@ir.hot_node_is_live(func, group.defining_nodes[lane_idx]) {
continue
}
let replacement = match
@ir.hot_node_get(func, group.defining_nodes[lane_idx]).op {
@ir.HotOp::LocalTee =>
if copy_anchor_uses_tuple_carrier {
@ir.hot_build_local_get(func, group.lane_locals[lane_idx])
} else {
@ir.hot_build_local_get(func, split_locals[lane_idx])
}
_ => @ir.hot_build_nop(func)
}
tuple_optimization_replace_with_built(
ctx,
func,
group.defining_nodes[lane_idx],
replacement,
)
}
match host_read_replacement {
Some(host_read_replacement) =>
tuple_optimization_replace_with_built(
ctx,
func,
group.defining_nodes[primary_lane],
host_read_replacement,
)
None => ()
}
match (actual_host_lane, original_direct_host_drop) {
(Some(host_lane), Some(drop_id)) => {
let drop_node = @ir.hot_node_get(func, drop_id)
let drop_node_is_local_set = match drop_node.op {
@ir.HotOp::LocalSet => true
_ => false
}
if drop_node_is_local_set &&
drop_node.imm0 == group.lane_locals[host_lane] &&
@ir.hot_child_count(func, drop_id) == 1 {
tuple_optimization_replace_with_built(
ctx,
func,
drop_id,
@ir.hot_build_drop(
func,
@ir.hot_build_local_tee(
func,
group.lane_locals[host_lane],
@ir.hot_child_get(func, drop_id, 0),
),
),
)
}
}
_ => ()
}
match actual_host_lane {
Some(host_lane) if source_split_locals is Some(_) &&
source_group_has_host_lane &&
!has_rewritten_copy_consumer &&
host_is_function_root => {
let root_ref = @ir.HotRegionRef::root()
let root_count = @ir.hot_region_root_count(func, root_ref)
let mut terminal_host_set = -1
for slot = 0; slot < root_count; slot = slot + 1 {
let root_id = @ir.hot_region_get(func, root_ref, slot)
let root_node = @ir.hot_node_get(func, root_id)
let is_host_local_set = match root_node.op {
@ir.HotOp::LocalSet =>
root_node.imm0 == group.lane_locals[host_lane] &&
@ir.hot_child_count(func, root_id) == 1
_ => false
}
if !is_host_local_set {
continue
}
let mut has_later_host_read = false
for later = slot + 1; later < root_count; later = later + 1 {
let later_id = @ir.hot_region_get(func, root_ref, later)
let later_node = @ir.hot_node_get(func, later_id)
match later_node.op {
@ir.HotOp::LocalGet =>
if later_node.imm0 == group.lane_locals[host_lane] {
has_later_host_read = true
break
}
_ => ()
}
}
if has_later_host_read {
terminal_host_set = root_id
}
}
if terminal_host_set >= 0 {
tuple_optimization_replace_with_built(
ctx,
func,
terminal_host_set,
@ir.hot_build_drop(
func,
@ir.hot_build_local_tee(
func,
group.lane_locals[host_lane],
@ir.hot_child_get(func, terminal_host_set, 0),
),
),
)
}
}
_ => ()
}
// These reads now depend on the replacement carrier, whose definitions
// were built after the original reads. Refresh the read evaluation orders
// only for single-definition lanes copied by later roots in the same region.
if actual_host_lane is None && group.copy_source is Some(_) {
match no_host_root_carrier_slot {
Some(carrier) =>
for lane in 0.. carrier.slot =>
tuple_optimization_replace_with_built(
ctx,
func,
read,
@ir.hot_build_local_get(func, local_id),
)
_ => ()
}
}
}
None => ()
}
}
}
///|
fn tuple_optimization_rewrite_source_only_elide_fast_path(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
summary : HotTupleOptimizationQuerySummary,
groups : Array[HotTupleOptimizationGroup],
rewrite_order : Array[Int32],
split_locals_by_group : Array[Array[Int32]],
cleanup_plan : HotTupleOptimizationRewriteCleanupPlan,
drop_only_elision_facts : HotTupleOptimizationDropOnlyElisionFacts,
pending_controls : Array[Int32],
) -> Unit {
let fast_path_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:source-only-elide-fast-path",
)
let source_rewrite_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:source",
)
let mut elide_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source",
)
let mut elide_timer_active = true
for group_id in rewrite_order {
let group = groups[group_id]
if !tuple_optimization_try_elide_simple_drop_only_source_local_set_roots_fast_path(
ctx, func, use_def, group, cleanup_plan,
) &&
!tuple_optimization_try_elide_simple_drop_only_source_group_defs(
ctx,
func,
use_def,
group,
cleanup_plan,
precomputed_payload_lanes=Some(
drop_only_elision_facts.payload_lanes_by_group[group_id],
),
) {
if elide_timer_active {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source",
elide_start_us,
)
elide_timer_active = false
}
tuple_optimization_rewrite_group_defs(
ctx,
func,
use_def,
summary,
group,
split_locals_by_group[group_id],
cleanup_plan,
pending_controls=Some(pending_controls),
)
elide_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source",
)
elide_timer_active = true
}
}
if elide_timer_active {
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source",
elide_start_us,
)
}
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:source",
source_rewrite_start_us,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:source-only-elide-fast-path",
fast_path_start_us,
)
}
///|
fn tuple_optimization_rewrite_good_components(
ctx : HotPassContext,
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
analysis : HotTupleOptimizationAnalysis,
) -> Bool {
let rewrite_mask_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:build-rewrite-mask",
)
let rewrite_mask = tuple_optimization_build_group_rewrite_mask(
func,
use_def,
analysis.query_summary,
analysis.groups,
perf=ctx.perf,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:build-rewrite-mask",
rewrite_mask_start_us,
)
tuple_optimization_retain_shared_publication_components(
use_def,
analysis.groups,
rewrite_mask,
)
let has_rewrites_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:has-rewrites-scan",
)
let mut has_rewrites = false
for group_id = 0; group_id < rewrite_mask.length(); group_id = group_id + 1 {
if rewrite_mask[group_id] {
has_rewrites = true
break
}
}
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:has-rewrites-scan",
has_rewrites_start_us,
)
if !has_rewrites {
return false
}
let (
rewritten_copy_consumers,
rewritten_host_copy_consumers,
all_copy_consumers,
) = tuple_optimization_build_rewritten_consumer_facts(
analysis.groups,
rewrite_mask,
)
let retained_opaque_sources = tuple_optimization_plan_retained_opaque_sources(
func,
use_def,
analysis.groups,
rewrite_mask,
all_copy_consumers,
)
// Only original sources (copy_source=None) were unselected. Consumer facts
// above therefore still describe exactly the same rewritten copy groups.
let mut remaining_rewrites = false
for group_id = 0; group_id < rewrite_mask.length(); group_id = group_id + 1 {
if rewrite_mask[group_id] {
remaining_rewrites = true
break
}
}
if !remaining_rewrites {
return false
}
let initial_local_count = @ir.hot_local_count(func)
let cleanup_plan : HotTupleOptimizationRewriteCleanupPlan = {
removable_root_defs: @bitset.BitSet::new(@ir.hot_node_count(func)),
removable_root_regions: [],
removable_root_region_seen: @bitset.BitSet::new(@ir.hot_node_count(func)),
removable_root_region_root_seen: false,
generated_nops: false,
had_preexisting_nops: tuple_optimization_has_live_nop(func),
skip_drop_local_tee_cleanup: !analysis.query_summary.has_local_tee_defs,
skip_scalarized_tuple_local_cleanup: !analysis.query_summary.has_scalarized_tuple_local_cleanup_candidate,
}
let split_locals_by_group : Array[Array[Int32]] = []
for _ in 0.. {
let copy_rewrite_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:copy",
)
tuple_optimization_rewrite_group_defs(
ctx,
func,
use_def,
analysis.query_summary,
group,
split_locals,
cleanup_plan,
has_rewritten_copy_consumer=rewritten_copy_consumers[group_id],
source_group_has_host_lane=tuple_optimization_group_host_lane(
analysis.groups[source_group],
)
is Some(_),
source_split_locals=Some(
match group.copy_payload_locals {
Some(copy_payload_locals) => copy_payload_locals
None => split_locals_by_group[source_group]
},
),
elide_simple_drop_only_source=group_id <
drop_only_elision_facts.mask.length() &&
drop_only_elision_facts.mask[group_id],
pending_controls=Some(pending_controls),
pending_publications=Some(pending_publications),
)
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:copy",
copy_rewrite_start_us,
)
}
None => {
let source_rewrite_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:source",
)
tuple_optimization_rewrite_group_defs(
ctx,
func,
use_def,
analysis.query_summary,
group,
split_locals,
cleanup_plan,
has_rewritten_copy_consumer=rewritten_copy_consumers[group_id],
source_host_copy_consumer=rewritten_host_copy_consumers[group_id],
elide_simple_drop_only_source=group_id <
drop_only_elision_facts.mask.length() &&
drop_only_elision_facts.mask[group_id],
pending_controls=Some(pending_controls),
)
perf_stop_timer_accumulate_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:source",
source_rewrite_start_us,
)
}
}
match group.copy_source {
Some(_) =>
if pending_controls.length() == pending_before &&
group.producer >= 0 &&
@ir.hot_node_is_live(func, group.producer) {
@ir.hot_replace_child_spans_and_control_result_type(
func,
group.producer,
@ir.hot_type_void(func),
[@ir.HotChildSpanRewrite::new(group.producer, [])],
)
pass_mark_mutated(ctx, func)
}
None => ()
}
}
}
// Publish moved reads before checked control retirement or fresh analyses.
tuple_optimization_flush_copy_publications(ctx, func, pending_publications)
// Flush before cleanup can rebuild use-def or any verifier inspects labels.
// A single checked forest retains the same ownership/branch-target guards
// without allocating and scanning the whole function once per empty owner.
if !pending_controls.is_empty() {
if !@ir.hot_delete_detached_control_forest(func, pending_controls) {
abort(
"tuple-optimization moved controls must have detached unique owners",
)
}
pass_mark_mutated(ctx, func)
}
perf_emit_timer_total_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:copy",
)
perf_emit_timer_total_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:source",
)
perf_emit_timer_total_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source",
)
perf_emit_timer_total_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:can-elide",
)
perf_emit_timer_total_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:payload",
)
perf_emit_timer_total_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:local-set-root-fast-path",
)
perf_emit_timer_total_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs:elide-simple-drop-only-source:replace-defs",
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:rewrite-group-defs",
rewrite_defs_start_us,
)
let cleanup_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite",
)
tuple_optimization_cleanup_post_rewrite(
ctx, func, initial_local_count, cleanup_plan,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite",
cleanup_start_us,
)
true
}
///|
fn tuple_optimization_cleanup_drop_local_tees(
ctx : HotPassContext,
func : @ir.HotFunc,
) -> Unit {
let mut has_cleanup_candidate = false
for node_id = 0; node_id < @ir.hot_node_count(func); node_id = node_id + 1 {
if !@ir.hot_node_is_live(func, node_id) {
continue
}
let node = @ir.hot_node_get(func, node_id)
match node.op {
@ir.HotOp::Drop => ()
_ => continue
}
if @ir.hot_child_count(func, node_id) != 1 {
continue
}
let child = @ir.hot_child_get(func, node_id, 0)
if !@ir.hot_node_is_live(func, child) {
continue
}
let child_node = @ir.hot_node_get(func, child)
match child_node.op {
@ir.HotOp::LocalTee =>
if @ir.hot_child_count(func, child) == 1 {
has_cleanup_candidate = true
break
}
_ => ()
}
}
if !has_cleanup_candidate {
return
}
let use_def = @ir.use_def_build(func)
let query_summary = tuple_optimization_build_query_summary(func, use_def)
let node_count = @ir.hot_node_count(func)
for node_id = 0; node_id < node_count; node_id = node_id + 1 {
if !@ir.hot_node_is_live(func, node_id) {
continue
}
let node = @ir.hot_node_get(func, node_id)
match node.op {
@ir.HotOp::Drop => ()
_ => continue
}
if @ir.hot_child_count(func, node_id) != 1 {
continue
}
let child = @ir.hot_child_get(func, node_id, 0)
if !@ir.hot_node_is_live(func, child) {
continue
}
let child_node = @ir.hot_node_get(func, child)
match child_node.op {
@ir.HotOp::LocalTee => ()
_ => continue
}
if @ir.hot_child_count(func, child) != 1 {
continue
}
if tuple_optimization_local_has_forwarded_non_drop_use(
use_def,
query_summary,
child_node.imm0,
) {
continue
}
tuple_optimization_replace_with_built(
ctx,
func,
node_id,
@ir.hot_build_local_set(
func,
child_node.imm0,
@ir.hot_child_get(func, child, 0),
),
)
}
}
///|
fn tuple_optimization_may_have_scalarized_tuple_local_cleanup_candidate(
func : @ir.HotFunc,
) -> Bool {
for node_id = 0; node_id < @ir.hot_node_count(func); node_id = node_id + 1 {
if !@ir.hot_node_is_live(func, node_id) {
continue
}
match @ir.hot_node_get(func, node_id).op {
@ir.HotOp::LocalSet => ()
_ => continue
}
if @ir.hot_child_count(func, node_id) != 1 {
continue
}
let child_id = @ir.hot_child_get(func, node_id, 0)
if !@ir.hot_node_is_live(func, child_id) {
continue
}
match @ir.hot_node_get(func, child_id).op {
@ir.HotOp::TupleMake =>
if @ir.hot_child_count(func, child_id) > 1 {
return true
}
_ => ()
}
}
false
}
///|
fn tuple_optimization_cleanup_scalarized_tuple_locals(
ctx : HotPassContext,
func : @ir.HotFunc,
) -> Unit {
let pre_scan_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:scalarized-tuple-locals:pre-scan",
)
let has_cleanup_candidate = tuple_optimization_may_have_scalarized_tuple_local_cleanup_candidate(
func,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:scalarized-tuple-locals:pre-scan",
pre_scan_start_us,
)
if !has_cleanup_candidate {
return
}
let use_def_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:scalarized-tuple-locals:use-def-build",
)
let use_def = @ir.use_def_build(func)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:scalarized-tuple-locals:use-def-build",
use_def_start_us,
)
for local_id = 0
local_id < @ir.hot_local_count(func)
local_id = local_id + 1 {
let writes = @ir.local_write_nodes_borrowed(use_def, local_id)
if writes.length() != 1 {
continue
}
let writer_id = writes[0]
if !@ir.hot_node_is_live(func, writer_id) {
continue
}
let writer = @ir.hot_node_get(func, writer_id)
match writer.op {
@ir.HotOp::LocalSet => ()
_ => continue
}
if @ir.hot_child_count(func, writer_id) != 1 {
continue
}
let tuple_make_id = @ir.hot_child_get(func, writer_id, 0)
if !@ir.hot_node_is_live(func, tuple_make_id) {
continue
}
match @ir.hot_node_get(func, tuple_make_id).op {
@ir.HotOp::TupleMake => ()
_ => continue
}
let arity = @ir.hot_child_count(func, tuple_make_id)
if arity <= 1 {
continue
}
let source_locals : Array[Int32] = []
let mut can_inline = true
for lane_idx = 0; lane_idx < arity; lane_idx = lane_idx + 1 {
let child_id = @ir.hot_child_get(func, tuple_make_id, lane_idx)
if !@ir.hot_node_is_live(func, child_id) {
can_inline = false
break
}
let child = @ir.hot_node_get(func, child_id)
match child.op {
@ir.HotOp::LocalGet => ()
_ => {
can_inline = false
break
}
}
source_locals.push(child.imm0)
}
if !can_inline {
continue
}
let reads = @ir.local_read_nodes_borrowed(use_def, local_id)
if reads.is_empty() {
continue
}
for read_id in reads {
if !@ir.hot_node_is_live(func, read_id) {
can_inline = false
break
}
match @ir.hot_node_get(func, read_id).op {
@ir.HotOp::LocalGet => ()
_ => {
can_inline = false
break
}
}
let use_sites = @ir.node_use_sites_borrowed(use_def, read_id)
if use_sites.is_empty() {
can_inline = false
break
}
for use_site in use_sites {
match use_site.kind {
@ir.HotUseSlotKind::ChildUseSlot => ()
_ => {
can_inline = false
break
}
}
if use_site.slot != 0 || !@ir.hot_node_is_live(func, use_site.user) {
can_inline = false
break
}
let extract = @ir.hot_node_get(func, use_site.user)
match extract.op {
@ir.HotOp::TupleExtract => ()
_ => {
can_inline = false
break
}
}
if extract.imm0 < 0 || extract.imm0 >= source_locals.length() {
can_inline = false
break
}
}
if !can_inline {
break
}
}
if !can_inline {
continue
}
for read_id in reads {
for use_site in @ir.node_use_sites_borrowed(use_def, read_id) {
let extract_id = use_site.user
let lane_idx = @ir.hot_node_get(func, extract_id).imm0
tuple_optimization_replace_with_built(
ctx,
func,
extract_id,
@ir.hot_build_local_get(func, source_locals[lane_idx]),
)
}
}
tuple_optimization_replace_with_built(
ctx,
func,
writer_id,
@ir.hot_build_nop(func),
)
}
}
///|
fn tuple_optimization_cleanup_unused_body_locals(
ctx : HotPassContext,
func : @ir.HotFunc,
initial_local_count : Int32,
) -> Unit {
let param_count = @ir.hot_param_count(func)
let local_count = @ir.hot_local_count(func)
if initial_local_count < param_count || initial_local_count > local_count {
abort(
"tuple-optimization appended-local cleanup baseline must stay within local bounds",
)
}
let body_local_count = local_count - param_count
if body_local_count <= 0 {
return
}
let body_local_used = @bitset.BitSet::new(body_local_count)
for body_local_id = 0
body_local_id < body_local_count
body_local_id = body_local_id + 1 {
let old_local_id = param_count + body_local_id
if old_local_id < initial_local_count {
body_local_used[body_local_id] = true
}
}
for node_id = 0; node_id < @ir.hot_node_count(func); node_id = node_id + 1 {
if !@ir.hot_node_is_live(func, node_id) {
continue
}
let node = @ir.hot_node_get(func, node_id)
match node.op {
@ir.HotOp::LocalGet | @ir.HotOp::LocalSet | @ir.HotOp::LocalTee => {
let local_id = node.imm0
if local_id >= param_count {
let body_local_id = local_id - param_count
if body_local_id >= 0 && body_local_id < body_local_used.length() {
body_local_used[body_local_id] = true
}
}
}
_ => ()
}
}
let mut has_unused_body_local = false
for body_local_id = 0
body_local_id < body_local_used.length()
body_local_id = body_local_id + 1 {
let old_local_id = param_count + body_local_id
if old_local_id >= initial_local_count && !body_local_used[body_local_id] {
has_unused_body_local = true
break
}
}
if !has_unused_body_local {
return
}
let old_to_new = Array::make(@ir.hot_local_count(func), -1)
for local_id = 0; local_id < param_count; local_id = local_id + 1 {
old_to_new[local_id] = local_id
}
let next_body_locals : Array[@lib.ValType] = []
let mut next_local_id = param_count
for body_local_id = 0
body_local_id < body_local_count
body_local_id = body_local_id + 1 {
let old_local_id = param_count + body_local_id
if body_local_used[body_local_id] {
old_to_new[old_local_id] = next_local_id
next_body_locals.push(func.locals.locals[body_local_id])
next_local_id += 1
}
}
for node_id = 0; node_id < @ir.hot_node_count(func); node_id = node_id + 1 {
if !@ir.hot_node_is_live(func, node_id) {
continue
}
let node = @ir.hot_node_get(func, node_id)
match node.op {
@ir.HotOp::LocalGet | @ir.HotOp::LocalSet | @ir.HotOp::LocalTee => {
let local_id = node.imm0
if local_id < 0 || local_id >= old_to_new.length() {
abort("tuple-optimization local compaction remap must stay in range")
}
let mapped_local_id = old_to_new[local_id]
if mapped_local_id < 0 {
abort(
"tuple-optimization local compaction cannot drop referenced locals",
)
}
if mapped_local_id != local_id {
@ir.ssa_rewrite_local_use(func, node_id, mapped_local_id)
}
}
_ => ()
}
}
func.locals.locals.clear()
for ty in next_body_locals {
func.locals.locals.push(ty)
}
pass_mark_mutated(ctx, func)
}
///|
fn tuple_optimization_has_live_nop(func : @ir.HotFunc) -> Bool {
for node_id = 0; node_id < @ir.hot_node_count(func); node_id = node_id + 1 {
if !@ir.hot_node_is_live(func, node_id) {
continue
}
match @ir.hot_node_get(func, node_id).op {
@ir.HotOp::Nop => return true
_ => ()
}
}
false
}
///|
fn tuple_optimization_cleanup_plan_add_removable_root(
cleanup_plan : HotTupleOptimizationRewriteCleanupPlan,
root_id : Int32,
region_ref : @ir.HotRegionRef,
) -> Unit {
cleanup_plan.removable_root_defs[root_id] = true
let holder = region_ref.holder
if holder < 0 {
if cleanup_plan.removable_root_region_root_seen {
return
}
cleanup_plan.removable_root_region_root_seen = true
} else {
let seen = cleanup_plan.removable_root_region_seen
if holder >= seen.length() {
seen.grow((holder + 1) * 2)
}
if seen[holder] {
return
}
seen[holder] = true
}
cleanup_plan.removable_root_regions.push(region_ref)
}
///|
fn tuple_optimization_prune_nops_in_region(
ctx : HotPassContext,
func : @ir.HotFunc,
region_ref : @ir.HotRegionRef,
) -> Unit {
let next : Array[Int32] = []
let mut changed = false
for slot = 0
slot < @ir.hot_region_root_count(func, region_ref)
slot = slot + 1 {
let root_id = @ir.hot_region_get(func, region_ref, slot)
match @ir.hot_node_get(func, root_id).op {
@ir.HotOp::Block | @ir.HotOp::Loop =>
tuple_optimization_prune_nops_in_region(
ctx,
func,
@ir.hot_region_body_ref(func, root_id),
)
@ir.HotOp::If => {
tuple_optimization_prune_nops_in_region(
ctx,
func,
@ir.hot_region_then_ref(func, root_id),
)
match @ir.hot_region_else_ref(func, root_id) {
Some(else_ref) =>
tuple_optimization_prune_nops_in_region(ctx, func, else_ref)
None => ()
}
}
@ir.HotOp::Try => {
tuple_optimization_prune_nops_in_region(
ctx,
func,
@ir.hot_region_body_ref(func, root_id),
)
match @ir.hot_region_catch_ref(func, root_id) {
Some(catch_ref) =>
tuple_optimization_prune_nops_in_region(ctx, func, catch_ref)
None => ()
}
}
@ir.HotOp::TryTable => {
tuple_optimization_prune_nops_in_region(
ctx,
func,
@ir.hot_region_body_ref(func, root_id),
)
tuple_optimization_prune_nops_in_region(
ctx,
func,
@ir.hot_region_catch_list_ref(func, root_id),
)
}
_ => ()
}
match @ir.hot_node_get(func, root_id).op {
@ir.HotOp::Nop => {
changed = true
continue
}
_ => ()
}
next.push(root_id)
}
if changed {
@ir.hot_region_replace_body(func, region_ref, next)
pass_mark_mutated(ctx, func)
}
}
///|
fn tuple_optimization_remove_elided_drop_only_roots_in_one_region(
ctx : HotPassContext,
func : @ir.HotFunc,
region_ref : @ir.HotRegionRef,
removable_root_defs : @bitset.BitSet,
) -> Unit {
let next : Array[Int32] = []
let removed : Array[Int32] = []
let mut changed = false
for slot = 0
slot < @ir.hot_region_root_count(func, region_ref)
slot = slot + 1 {
let root_id = @ir.hot_region_get(func, region_ref, slot)
let remove_root = root_id >= 0 &&
root_id < removable_root_defs.length() &&
removable_root_defs[root_id]
if remove_root {
changed = true
removed.push(root_id)
continue
}
next.push(root_id)
}
if changed {
@ir.hot_region_replace_body(func, region_ref, next)
for root_id in removed {
if @ir.hot_node_is_live(func, root_id) {
@ir.hot_delete_detached_node(func, root_id)
}
}
pass_mark_mutated(ctx, func)
}
}
///|
fn tuple_optimization_remove_elided_drop_only_roots_in_targeted_regions(
ctx : HotPassContext,
func : @ir.HotFunc,
cleanup_plan : HotTupleOptimizationRewriteCleanupPlan,
) -> Unit {
for region_ref in cleanup_plan.removable_root_regions {
tuple_optimization_remove_elided_drop_only_roots_in_one_region(
ctx,
func,
region_ref,
cleanup_plan.removable_root_defs,
)
}
}
///|
fn tuple_optimization_remove_elided_drop_only_roots_in_region(
ctx : HotPassContext,
func : @ir.HotFunc,
region_ref : @ir.HotRegionRef,
removable_root_defs : @bitset.BitSet,
prune_nops~ : Bool,
) -> Unit {
let next : Array[Int32] = []
let removed : Array[Int32] = []
let mut changed = false
for slot = 0
slot < @ir.hot_region_root_count(func, region_ref)
slot = slot + 1 {
let root_id = @ir.hot_region_get(func, region_ref, slot)
match @ir.hot_node_get(func, root_id).op {
@ir.HotOp::Block | @ir.HotOp::Loop =>
tuple_optimization_remove_elided_drop_only_roots_in_region(
ctx,
func,
@ir.hot_region_body_ref(func, root_id),
removable_root_defs,
prune_nops~,
)
@ir.HotOp::If => {
tuple_optimization_remove_elided_drop_only_roots_in_region(
ctx,
func,
@ir.hot_region_then_ref(func, root_id),
removable_root_defs,
prune_nops~,
)
match @ir.hot_region_else_ref(func, root_id) {
Some(else_ref) =>
tuple_optimization_remove_elided_drop_only_roots_in_region(
ctx,
func,
else_ref,
removable_root_defs,
prune_nops~,
)
None => ()
}
}
@ir.HotOp::Try => {
tuple_optimization_remove_elided_drop_only_roots_in_region(
ctx,
func,
@ir.hot_region_body_ref(func, root_id),
removable_root_defs,
prune_nops~,
)
match @ir.hot_region_catch_ref(func, root_id) {
Some(catch_ref) =>
tuple_optimization_remove_elided_drop_only_roots_in_region(
ctx,
func,
catch_ref,
removable_root_defs,
prune_nops~,
)
None => ()
}
}
@ir.HotOp::TryTable => {
tuple_optimization_remove_elided_drop_only_roots_in_region(
ctx,
func,
@ir.hot_region_body_ref(func, root_id),
removable_root_defs,
prune_nops~,
)
tuple_optimization_remove_elided_drop_only_roots_in_region(
ctx,
func,
@ir.hot_region_catch_list_ref(func, root_id),
removable_root_defs,
prune_nops~,
)
}
_ => ()
}
let remove_root = root_id >= 0 &&
root_id < removable_root_defs.length() &&
removable_root_defs[root_id]
let is_nop_root = match @ir.hot_node_get(func, root_id).op {
@ir.HotOp::Nop => true
_ => false
}
let prune_root_nop = prune_nops && is_nop_root
if remove_root || prune_root_nop {
changed = true
if remove_root {
removed.push(root_id)
}
continue
}
next.push(root_id)
}
if changed {
@ir.hot_region_replace_body(func, region_ref, next)
for root_id in removed {
if @ir.hot_node_is_live(func, root_id) {
@ir.hot_delete_detached_node(func, root_id)
}
}
pass_mark_mutated(ctx, func)
}
}
///|
fn tuple_optimization_local_reads_are_direct_drops_without_summary(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
local_id : Int32,
) -> Bool {
if local_id < 0 || local_id >= @ir.hot_local_count(func) {
return false
}
let reads = @ir.local_read_nodes_borrowed(use_def, local_id)
if reads.is_empty() {
return false
}
for read_id in reads {
if !@ir.hot_node_is_live(func, read_id) {
return false
}
match @ir.hot_node_get(func, read_id).op {
@ir.HotOp::LocalGet => ()
_ => return false
}
let use_sites = @ir.node_use_sites_borrowed(use_def, read_id)
if use_sites.is_empty() {
return false
}
for site in use_sites {
match site.kind {
@ir.HotUseSlotKind::ChildUseSlot => ()
_ => return false
}
if site.slot != 0 || !@ir.hot_node_is_live(func, site.user) {
return false
}
match @ir.hot_node_get(func, site.user).op {
@ir.HotOp::Drop => ()
_ => return false
}
}
}
true
}
///|
fn tuple_optimization_try_record_simple_drop_only_function_root_elision_group(
func : @ir.HotFunc,
use_def : @ir.HotUseDef,
group : HotTupleOptimizationGroup,
cleanup_plan : HotTupleOptimizationRewriteCleanupPlan,
) -> Bool {
if group.copy_source is Some(_) || group.host_lane is Some(_) {
return false
}
if !tuple_optimization_is_simple_i32_i64_typeidx_group(func, group) {
return false
}
if !tuple_optimization_simple_block_has_exact_simple_payload_lanes(
func,
group.producer,
group.lane_locals.length(),
) {
return false
}
for lane_idx = 0
lane_idx < group.lane_locals.length()
lane_idx = lane_idx + 1 {
let defining_node = group.defining_nodes[lane_idx]
if defining_node < 0 ||
defining_node >= cleanup_plan.removable_root_defs.length() ||
!@ir.hot_node_is_live(func, defining_node) {
return false
}
match @ir.hot_node_get(func, defining_node).op {
@ir.HotOp::LocalSet => ()
_ => return false
}
if !tuple_optimization_has_single_function_root_use(use_def, defining_node) {
return false
}
let local_id = group.lane_locals[lane_idx]
let writes = @ir.local_write_nodes_borrowed(use_def, local_id)
if writes.length() != 1 || writes[0] != defining_node {
return false
}
if !tuple_optimization_local_reads_are_direct_drops_without_summary(
func, use_def, local_id,
) {
return false
}
tuple_optimization_cleanup_plan_add_removable_root(
cleanup_plan,
defining_node,
@ir.HotRegionRef::root(),
)
}
true
}
///|
fn tuple_optimization_cleanup_post_rewrite(
ctx : HotPassContext,
func : @ir.HotFunc,
initial_local_count : Int32,
cleanup_plan : HotTupleOptimizationRewriteCleanupPlan,
) -> Unit {
if cleanup_plan.skip_drop_local_tee_cleanup {
let no_tee_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:drop-local-tees:no-local-tee-fast-path",
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:drop-local-tees:no-local-tee-fast-path",
no_tee_start_us,
)
} else {
let drop_tees_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:drop-local-tees",
)
tuple_optimization_cleanup_drop_local_tees(ctx, func)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:drop-local-tees",
drop_tees_start_us,
)
}
if cleanup_plan.skip_scalarized_tuple_local_cleanup {
let no_tuple_make_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:scalarized-tuple-locals:no-tuple-make-local-set-fast-path",
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:scalarized-tuple-locals:no-tuple-make-local-set-fast-path",
no_tuple_make_start_us,
)
} else {
let scalarized_locals_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:scalarized-tuple-locals",
)
tuple_optimization_cleanup_scalarized_tuple_locals(ctx, func)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:scalarized-tuple-locals",
scalarized_locals_start_us,
)
}
let remove_roots_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:remove-elided-drop-only-roots",
)
if cleanup_plan.generated_nops || cleanup_plan.had_preexisting_nops {
let scan_all_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:remove-elided-drop-only-roots:scan-all-regions",
)
tuple_optimization_remove_elided_drop_only_roots_in_region(
ctx,
func,
@ir.HotRegionRef::root(),
cleanup_plan.removable_root_defs,
prune_nops=!cleanup_plan.generated_nops,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:remove-elided-drop-only-roots:scan-all-regions",
scan_all_start_us,
)
} else {
let targeted_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:remove-elided-drop-only-roots:targeted-regions",
)
tuple_optimization_remove_elided_drop_only_roots_in_targeted_regions(
ctx, func, cleanup_plan,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:remove-elided-drop-only-roots:targeted-regions",
targeted_start_us,
)
}
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:remove-elided-drop-only-roots",
remove_roots_start_us,
)
if cleanup_plan.generated_nops {
let prune_nops_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:prune-nops",
)
tuple_optimization_prune_nops_in_region(ctx, func, @ir.HotRegionRef::root())
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:prune-nops",
prune_nops_start_us,
)
}
if @ir.hot_local_count(func) == initial_local_count {
let no_new_local_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:unused-body-locals:no-new-local-fast-path",
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:unused-body-locals:no-new-local-fast-path",
no_new_local_start_us,
)
} else {
let unused_locals_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:unused-body-locals",
)
tuple_optimization_cleanup_unused_body_locals(
ctx, func, initial_local_count,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:cleanup-post-rewrite:unused-body-locals",
unused_locals_start_us,
)
}
}
///|
fn tuple_optimization_run(
ctx : HotPassContext,
func : @ir.HotFunc,
) -> HotPassResult {
let use_def = pass_require_use_def(ctx, func)
match
tuple_optimization_try_direct_simple_drop_only_root_elision_fast_path(
ctx, func, use_def,
) {
Some(cleanup_plan) => {
let fast_path_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path",
)
tuple_optimization_cleanup_post_rewrite(
ctx,
func,
@ir.hot_local_count(func),
cleanup_plan,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path",
fast_path_start_us,
)
return HotPassResult::changed()
}
None => ()
}
let seed_collection = tuple_optimization_collect_seed_groups_with_simple_root_elision(
ctx, func, use_def,
)
let groups = seed_collection.groups
if groups.is_empty() {
return HotPassResult::unchanged()
}
match seed_collection.simple_root_cleanup_plan {
Some(cleanup_plan) => {
let fast_path_start_us = perf_start_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path",
)
tuple_optimization_cleanup_post_rewrite(
ctx,
func,
@ir.hot_local_count(func),
cleanup_plan,
)
perf_stop_timer_opt(
ctx.perf,
"detail:tuple-optimization:simple-drop-only-root-elision-fast-path",
fast_path_start_us,
)
return HotPassResult::changed()
}
None => ()
}
let analysis = tuple_optimization_analyze_with_groups(
func,
use_def,
groups,
perf=ctx.perf,
)
if tuple_optimization_rewrite_good_components(ctx, func, use_def, analysis) {
HotPassResult::changed()
} else {
HotPassResult::unchanged()
}
}