///|
/// Minimal provenance graph for data-deidentification workflows. Nodes carry
/// metadata and checksums only; values stay in the source system while the
/// graph explains where an output came from and which policy transformed it.
pub(all) enum LineageNodeKind {
LineageSource
LineageDocument
LineageFindingSet
LineagePolicy
LineageOutput
LineageReview
LineageExport
} derive(Debug, Eq)
///|
pub(all) enum LineageRelation {
LineageDerivedFrom
LineageScannedBy
LineageRedactedBy
LineageReviewedBy
LineageExportedAs
LineageSupersedes
} derive(Debug, Eq)
///|
pub(all) struct LineageNode {
node_id : String
kind : LineageNodeKind
label : String
created_at : String
checksum : String
attributes : Map[String, String]
} derive(Debug, Eq)
///|
pub(all) struct LineageEdge {
edge_id : String
from_id : String
to_id : String
relation : LineageRelation
actor : String
created_at : String
policy_checksum : String
} derive(Debug, Eq)
///|
pub(all) struct LineageGraph {
graph_id : String
mut nodes : Array[LineageNode]
mut edges : Array[LineageEdge]
mut checksum : String
} derive(Debug)
///|
pub fn lineage_node_kind_name(kind : LineageNodeKind) -> String {
match kind {
LineageSource => "source"
LineageDocument => "document"
LineageFindingSet => "finding_set"
LineagePolicy => "policy"
LineageOutput => "output"
LineageReview => "review"
LineageExport => "export"
}
}
///|
pub fn lineage_relation_name(relation : LineageRelation) -> String {
match relation {
LineageDerivedFrom => "derived_from"
LineageScannedBy => "scanned_by"
LineageRedactedBy => "redacted_by"
LineageReviewedBy => "reviewed_by"
LineageExportedAs => "exported_as"
LineageSupersedes => "supersedes"
}
}
///|
pub fn lineage_node(
kind : LineageNodeKind,
label : String,
created_at : String,
checksum : String,
) -> LineageNode {
{
node_id: stable_hash(
lineage_node_kind_name(kind) + ":" + label + ":" + created_at,
),
kind,
label,
created_at,
checksum,
attributes: Map([]),
}
}
///|
pub fn LineageNode::with_attribute(
node : LineageNode,
key : String,
value : String,
) -> LineageNode {
node.attributes[key] = value
node
}
///|
pub fn LineageNode::is_valid(self : LineageNode) -> Bool {
self.node_id.length() > 0 &&
self.label.length() > 0 &&
self.created_at.length() > 0 &&
self.checksum.length() > 0
}
///|
pub fn lineage_edge(
from_id : String,
to_id : String,
relation : LineageRelation,
actor : String,
created_at : String,
policy_checksum : String,
) -> LineageEdge {
{
edge_id: stable_hash(
from_id + ":" + to_id + ":" + lineage_relation_name(relation),
),
from_id,
to_id,
relation,
actor,
created_at,
policy_checksum,
}
}
///|
pub fn LineageEdge::is_valid(self : LineageEdge) -> Bool {
self.edge_id.length() > 0 &&
self.from_id.length() > 0 &&
self.to_id.length() > 0 &&
self.from_id != self.to_id &&
self.actor.length() > 0 &&
self.created_at.length() > 0
}
///|
pub fn lineage_graph(graph_id : String) -> LineageGraph {
{ graph_id, nodes: [], edges: [], checksum: stable_hash(graph_id) }
}
///|
pub fn LineageGraph::add_node(self : LineageGraph, node : LineageNode) -> Bool {
if !node.is_valid() ||
self.nodes.any(fn(item) { item.node_id == node.node_id }) {
false
} else {
self.nodes.push(node)
self.checksum = stable_hash(
self.nodes.map(fn(item) { item.node_id }).join("\n"),
)
true
}
}
///|
pub fn LineageGraph::add_edge(self : LineageGraph, edge : LineageEdge) -> Bool {
let endpoints_exist = self.nodes.any(fn(node) { node.node_id == edge.from_id }) &&
self.nodes.any(fn(node) { node.node_id == edge.to_id })
if !edge.is_valid() ||
!endpoints_exist ||
self.edges.any(fn(item) { item.edge_id == edge.edge_id }) {
false
} else {
self.edges.push(edge)
self.checksum = stable_hash(
self.edges.map(fn(item) { item.edge_id }).join("\n"),
)
true
}
}
///|
pub fn LineageGraph::find_node(
self : LineageGraph,
node_id : String,
) -> LineageNode? {
let mut found : LineageNode? = None
for node in self.nodes {
if node.node_id == node_id {
found = Some(node)
}
}
found
}
///|
pub fn LineageGraph::outgoing(
self : LineageGraph,
node_id : String,
) -> Array[LineageEdge] {
self.edges.filter(fn(edge) { edge.from_id == node_id })
}
///|
pub fn LineageGraph::incoming(
self : LineageGraph,
node_id : String,
) -> Array[LineageEdge] {
self.edges.filter(fn(edge) { edge.to_id == node_id })
}
///|
pub fn LineageGraph::has_source_path(
self : LineageGraph,
source_id : String,
target_id : String,
) -> Bool {
let mut frontier = [source_id]
let visited : Map[String, Bool] = Map([])
let mut found = false
while !frontier.is_empty() && !found {
let current = frontier[0]
frontier = frontier[1:].to_owned()
if current == target_id {
found = true
} else if !visited.contains(current) {
visited[current] = true
for edge in self.outgoing(current) {
frontier.push(edge.to_id)
}
}
}
found
}
///|
pub fn LineageGraph::node_count(self : LineageGraph) -> Int {
self.nodes.length()
}
///|
pub fn LineageGraph::edge_count(self : LineageGraph) -> Int {
self.edges.length()
}
///|
pub fn LineageGraph::is_valid(self : LineageGraph) -> Bool {
self.graph_id.length() > 0 &&
self.nodes.all(LineageNode::is_valid) &&
self.edges.all(LineageEdge::is_valid) &&
self.edges.all(fn(edge) {
self.nodes.any(fn(node) { node.node_id == edge.from_id }) &&
self.nodes.any(fn(node) { node.node_id == edge.to_id })
})
}
///|
pub fn LineageGraph::kind_counts(self : LineageGraph) -> Map[String, Int] {
let counts : Map[String, Int] = Map([])
for node in self.nodes {
let key = lineage_node_kind_name(node.kind)
counts[key] = counts.get_or_default(key, 0) + 1
}
counts
}
///|
pub fn LineageGraph::relation_counts(self : LineageGraph) -> Map[String, Int] {
let counts : Map[String, Int] = Map([])
for edge in self.edges {
let key = lineage_relation_name(edge.relation)
counts[key] = counts.get_or_default(key, 0) + 1
}
counts
}
///|
pub fn LineageGraph::summary(self : LineageGraph) -> String {
[
"graph_id=" + self.graph_id,
"nodes=" + self.nodes.length().to_string(),
"edges=" + self.edges.length().to_string(),
"valid=" + self.is_valid().to_string(),
"checksum=" + self.checksum,
].join("\n")
}
///|
pub fn LineageGraph::to_json(self : LineageGraph) -> String {
"{" +
"\"graph_id\":\"" +
json_escape(self.graph_id) +
"\"," +
"\"nodes\":[" +
self.nodes
.map(fn(node) {
"{\"id\":\"" +
json_escape(node.node_id) +
"\",\"kind\":\"" +
lineage_node_kind_name(node.kind) +
"\",\"checksum\":\"" +
json_escape(node.checksum) +
"\"}"
})
.join(",") +
"]," +
"\"edges\":[" +
self.edges
.map(fn(edge) {
"{\"id\":\"" +
json_escape(edge.edge_id) +
"\",\"from\":\"" +
json_escape(edge.from_id) +
"\",\"to\":\"" +
json_escape(edge.to_id) +
"\",\"relation\":\"" +
lineage_relation_name(edge.relation) +
"\"}"
})
.join(",") +
"]," +
"\"checksum\":\"" +
json_escape(self.checksum) +
"\"}"
}
///|
pub fn lineage_graph_export_lines(graph : LineageGraph) -> Array[String] {
graph.edges.map(fn(edge) {
[
edge.edge_id,
edge.from_id,
edge.to_id,
lineage_relation_name(edge.relation),
edge.actor,
edge.created_at,
edge.policy_checksum,
]
.map(json_escape)
.join("\t")
})
}
///|
pub fn lineage_graph_export_checksum(graph : LineageGraph) -> String {
stable_hash(lineage_graph_export_lines(graph).join("\n"))
}
///|
pub fn lineage_graph_is_reproducible(
first : LineageGraph,
second : LineageGraph,
) -> Bool {
first.graph_id == second.graph_id && first.checksum == second.checksum
}
///|
pub fn lineage_graph_contains_raw_label(graph : LineageGraph) -> Bool {
graph.nodes.any(fn(node) {
node.label.contains("raw_text=") || node.label.contains("original=")
})
}
///|
pub fn lineage_graph_is_safe(graph : LineageGraph) -> Bool {
graph.is_valid() &&
!lineage_graph_contains_raw_label(graph) &&
graph.nodes.all(fn(node) { node.checksum.length() > 0 })
}