///|
pub struct RawEntityRow {
id : @step.EntityId
name : String
args : String
line : Int32
col : Int32
}
///|
pub struct StatementSpan {
stmt : String
line : Int32
col : Int32
}
///|
// Remove STEP block comments (/* ... */), preserving newlines.
//
// Comments are removed only when they occur outside of STEP string literals.
// We insert a single space where the comment started to avoid accidentally
// merging adjacent tokens.
fn strip_step_comments(input : String) -> String raise StepParseError {
let builder = StringBuilder()
let n = input.length()
let mut in_str = false
let mut in_comment = false
let mut line = 1
let mut col = 1
let mut comment_line = 1
let mut comment_col = 1
let mut i = 0
while i < n {
let ci = cu_at(input, i)
if in_comment {
if ci == Char::to_int('*') &&
i + 1 < n &&
cu_at(input, i + 1) == Char::to_int('/') {
in_comment = false
i = i + 2
col = col + 2
continue
}
if ci == Char::to_int('\n') {
builder.write_char('\n')
line = line + 1
col = 1
i = i + 1
continue
}
col = col + 1
i = i + 1
continue
}
if in_str {
if ci == Char::to_int('\'') {
if i + 1 < n && cu_at(input, i + 1) == Char::to_int('\'') {
// STEP string escaping: doubled quotes '' represent a literal '.
builder.write_char('\'')
builder.write_char('\'')
i = i + 2
col = col + 2
continue
} else {
in_str = false
builder.write_char('\'')
i = i + 1
col = col + 1
continue
}
}
builder.write_char(ci.unsafe_to_char())
i = i + 1
if ci == Char::to_int('\n') {
line = line + 1
col = 1
} else {
col = col + 1
}
continue
}
// Outside strings.
if ci == Char::to_int('\'') {
in_str = true
builder.write_char('\'')
i = i + 1
col = col + 1
continue
}
if ci == Char::to_int('/') &&
i + 1 < n &&
cu_at(input, i + 1) == Char::to_int('*') {
// Start comment.
builder.write_char(' ')
in_comment = true
comment_line = line
comment_col = col
i = i + 2
col = col + 2
continue
}
builder.write_char(ci.unsafe_to_char())
i = i + 1
if ci == Char::to_int('\n') {
line = line + 1
col = 1
} else {
col = col + 1
}
}
if in_comment {
fail_parse_at(
"unterminated STEP block comment (missing '*/')", comment_line, comment_col,
"/*",
)
}
builder.to_string()
}
///|
fn substr(s : String, start : Int32, end : Int32) -> String {
String::unsafe_substring(s, start~, end~)
}
///|
fn cu_at(s : String, i : Int32) -> Int32 {
UInt16::to_int(String::code_unit_at(s, i))
}
///|
fn parse_int_dec(s : String) -> Int32 {
let t = s.trim().to_owned()
let mut sign = 1
let start_i = if t.length() > 0 && cu_at(t, 0) == Char::to_int('-') {
sign = -1
1
} else {
0
}
let mut acc = 0
for i in start_i..= Char::to_int('0') && ci <= Char::to_int('9') {
acc = acc * 10 + (ci - Char::to_int('0'))
}
}
acc * sign
}
///|
// Split the DATA section into individual entity statements.
//
// This is a minimal tokenizer that:
// - splits on `;` only when not inside parentheses and not inside `'...'`.
// - ignores empty statements.
pub fn split_statements(data_section : String) -> Array[String] {
let out : Array[String] = []
let mut in_str = false
let mut builder = StringBuilder()
let n = data_section.length()
let mut i = 0
while i < n {
let ci = cu_at(data_section, i)
if ci == Char::to_int('\'') {
if in_str {
// STEP string escaping: doubled quotes '' represent a literal '.
if i + 1 < n && cu_at(data_section, i + 1) == Char::to_int('\'') {
builder.write_char('\'')
builder.write_char('\'')
i = i + 2
continue
}
in_str = false
builder.write_char('\'')
i = i + 1
continue
}
in_str = true
builder.write_char('\'')
i = i + 1
continue
}
if !in_str && ci == Char::to_int(';') {
let stmt = builder.to_string().trim().to_owned()
if stmt.length() > 0 {
out.push(stmt)
}
builder = StringBuilder()
i = i + 1
continue
}
builder.write_char(ci.unsafe_to_char())
i = i + 1
}
// trailing (if no ending ';')
let trailing = builder.to_string().trim().to_owned()
if trailing.length() > 0 {
out.push(trailing)
}
out
}
///|
// Like split_statements, but also returns a best-effort (line,col) for where each
// statement begins (1-based). Intended for better error messages.
pub fn split_statements_with_pos(
data_section : String,
) -> Array[StatementSpan] raise StepParseError {
let out : Array[StatementSpan] = []
let mut in_str = false
let mut builder = StringBuilder()
let mut line = 1
let mut col = 1
let mut stmt_line = 1
let mut stmt_col = 1
let mut str_line = 1
let mut str_col = 1
let mut seen_non_ws = false
let n = data_section.length()
let mut i = 0
while i < n {
let ci = cu_at(data_section, i)
// Skip leading whitespace outside strings.
if !in_str && !seen_non_ws {
if ci == Char::to_int(' ') ||
ci == Char::to_int('\t') ||
ci == Char::to_int('\n') ||
ci == Char::to_int('\r') {
if ci == Char::to_int('\n') {
line = line + 1
col = 1
} else {
col = col + 1
}
i = i + 1
continue
}
// First non-whitespace character starts a statement.
seen_non_ws = true
stmt_line = line
stmt_col = col
}
if ci == Char::to_int('\'') {
if in_str {
if i + 1 < n && cu_at(data_section, i + 1) == Char::to_int('\'') {
builder.write_char('\'')
builder.write_char('\'')
i = i + 2
col = col + 2
continue
}
in_str = false
builder.write_char('\'')
i = i + 1
col = col + 1
continue
}
in_str = true
str_line = line
str_col = col
builder.write_char('\'')
i = i + 1
col = col + 1
continue
}
if !in_str && ci == Char::to_int(';') {
let stmt = builder.to_string().trim().to_owned()
if stmt.length() > 0 {
out.push(StatementSpan::{ stmt, line: stmt_line, col: stmt_col, })
}
builder = StringBuilder()
seen_non_ws = false
i = i + 1
col = col + 1
continue
}
builder.write_char(ci.unsafe_to_char())
if ci == Char::to_int('\n') {
line = line + 1
col = 1
} else {
col = col + 1
}
i = i + 1
}
let trailing = builder.to_string().trim().to_owned()
if trailing.length() > 0 {
out.push(StatementSpan::{ stmt: trailing, line: stmt_line, col: stmt_col, })
}
if in_str {
fail_parse_at(
"unterminated STEP string literal (missing closing '\'')",
str_line,
str_col,
builder.to_string(),
)
}
out
}
///|
fn apply_base(
line : Int32,
col : Int32,
base_line : Int32,
base_col : Int32,
) -> (Int32, Int32) {
if line == 1 {
(base_line, base_col + col - 1)
} else {
(base_line + line - 1, col)
}
}
///|
fn parse_row(span : StatementSpan) -> RawEntityRow raise StepParseError {
let stmt = span.stmt
// Expected form: #=()
let eq_idx = match String::find(stmt, "=") {
Some(i) => i
None =>
fail_parse_at(
"invalid DATA statement (missing '=')",
span.line,
span.col,
stmt,
)
}
let id_part = substr(stmt, 0, eq_idx).trim().to_owned()
if id_part.length() < 2 || cu_at(id_part, 0) != Char::to_int('#') {
fail_parse_at(
"invalid entity id (expected '#')",
span.line,
span.col,
stmt,
)
}
let id_digits = substr(id_part, 1, id_part.length())
let id_val = parse_int_dec(id_digits)
let id = @step.EntityId::{ value: id_val, }
let rhs = substr(stmt, eq_idx + 1, stmt.length()).trim().to_owned()
// Complex entity instance form: #=( ... );
// This has no leading NAME before the opening paren.
if rhs.length() > 0 && cu_at(rhs, 0) == Char::to_int('(') {
return RawEntityRow::{
id,
name: "__COMPLEX__",
args: rhs,
line: span.line,
col: span.col,
}
}
let lp_idx = match String::find(rhs, "(") {
Some(i) => i
None =>
fail_parse_at(
"invalid DATA statement (missing '(')",
span.line,
span.col,
stmt,
)
}
let name = substr(rhs, 0, lp_idx).trim().to_owned()
let args = substr(rhs, lp_idx, rhs.length()).trim().to_owned()
RawEntityRow::{ id, name, args, line: span.line, col: span.col, }
}
///|
pub fn tokenize_step_data_section(
data_section : String,
) -> Array[RawEntityRow] raise StepParseError {
tokenize_step_data_section_with_base(data_section, 1, 1)
}
///|
// Tokenize the DATA section but report positions relative to the full file.
// `base_line`/`base_col` are the 1-based position of the first character in
// `data_section` within the full (comment-stripped) file.
pub fn tokenize_step_data_section_with_base(
data_section : String,
base_line : Int32,
base_col : Int32,
) -> Array[RawEntityRow] raise StepParseError {
let stmts = split_statements_with_pos(data_section) catch {
StepParseError(info) => {
let (l, c) = match (info.line, info.col) {
(Some(il), Some(ic)) => apply_base(il, ic, base_line, base_col)
_ => (base_line, base_col)
}
raise StepParseError({
message: info.message,
line: Some(l),
col: Some(c),
stmt: info.stmt,
})
}
}
let rows = stmts.map(span => {
let (abs_line, abs_col) = apply_base(
span.line,
span.col,
base_line,
base_col,
)
parse_row(StatementSpan::{ stmt: span.stmt, line: abs_line, col: abs_col, })
})
rows
}
///|
// Split a comma-separated arg list, respecting parentheses and strings.
// Input may include surrounding parentheses.
pub fn split_args(args : String) -> Array[String] {
let s = args.trim().to_owned()
let mut inner_start = 0
let mut inner_end = s.length()
if inner_end >= 2 &&
cu_at(s, 0) == Char::to_int('(') &&
cu_at(s, inner_end - 1) == Char::to_int(')') {
inner_start = 1
inner_end = inner_end - 1
}
let out : Array[String] = []
let mut depth = 0
let mut in_str = false
let mut skip_quote = false
let mut start = inner_start
for i in inner_start.. 0 {
depth = depth - 1
}
} else if ci == Char::to_int(',') && depth == 0 {
let part = substr(s, start, i).trim().to_owned()
out.push(part)
start = i + 1
}
}
}
if start <= inner_end {
let part = substr(s, start, inner_end).trim().to_owned()
if part.length() > 0 {
out.push(part)
}
}
out
}