// similarity.mbt — zero-dependency semantic similarity (mirrors Python `similarity.py`)
//
// Tokenizes text into latin word-pieces and individual CJK characters, builds
// a bag-of-tokens embedding, and scores similarity with cosine distance.
// No regex or ML dependency is used.

///|
let _STOPWORDS : Array[String] = [
  "的", "了", "是", "在", "和", "与", "及", "对", "把", "被", "我", "你",
  "他", "她", "它", "们", "这", "那", "有", "个", "不", "也", "都", "就",
  "而", "等", "以", "为", "其", "此",
]

///|
fn in_stopwords(t : String) -> Bool {
  for w in _STOPWORDS {
    if w == t {
      return true
    }
  }
  false
}

///|
/// Tokenize: lowercase, split latin/alphanumeric runs, keep each CJK char.
pub fn tokenize(text : String) -> Array[String] {
  let lower = text.to_lower()
  let toks : Array[String] = []
  let mut buf = ""
  for c in lower.iter() {
    if c.is_ascii_alphabetic() || c.is_ascii_digit() {
      buf = buf + c.to_string()
    } else {
      if buf != "" {
        if !in_stopwords(buf) {
          toks.push(buf)
        }
        buf = ""
      }
      let cp = c.to_int()
      if cp >= 0x4E00 && cp <= 0x9FFF {
        toks.push(c.to_string())
      }
    }
  }
  if buf != "" && !in_stopwords(buf) {
    toks.push(buf)
  }
  toks
}

///|
/// Bag-of-tokens embedding: token -> count.
pub fn embed(text : String) -> Map[String, Int] {
  let m : Map[String, Int] = Map([], capacity=16)
  for t in tokenize(text) {
    m.set(t, m.get_or_default(t, 0) + 1)
  }
  m
}

///|
/// Cosine similarity between two token-count maps.
pub fn cosine(a : Map[String, Int], b : Map[String, Int]) -> Double {
  if a.length() == 0 || b.length() == 0 {
    return 0.0
  }
  let common = a.keys().filter(fn(k) { b.contains(k) }).to_array()
  if common.length() == 0 {
    return 0.0
  }
  let mut dot = 0.0
  for k in common {
    dot = dot + a.get(k).unwrap().to_double() * b.get(k).unwrap().to_double()
  }
  let na = sqrt_of(a.values().map(fn(v) { v.to_double() * v.to_double() }))
  let nb = sqrt_of(b.values().map(fn(v) { v.to_double() * v.to_double() }))
  if na == 0.0 || nb == 0.0 {
    return 0.0
  }
  dot / (na * nb)
}

///|
fn sqrt_of(xs : Iter[Double]) -> Double {
  let mut s = 0.0
  for x in xs {
    s = s + x
  }
  @math.pow(s, 0.5)
}

///|
/// End-to-end similarity between two raw text strings.
pub fn similarity(text_a : String, text_b : String) -> Double {
  cosine(embed(text_a), embed(text_b))
}