// `spell` — a command-line front end for the spell-checking library.
//
// Contract (see conformance/README.md):
//
// moon run cmd/main -- check --aff --dic --words -
//
// Words are read one per line from stdin (`--words -`) or from a file, and one
// line `1` (correct) / `0` (incorrect) is printed per input word, in input
// order. Empty input lines produce no output at all, so the output lines line up
// one-to-one with the non-empty lines of the input. The dictionary is loaded
// once; a failure to read or parse either file exits non-zero with a message on
// stderr instead of printing verdicts.
//
// The `.aff` file's `SET` directive decides how the bytes of all three inputs
// are decoded. UTF-8 is decoded as UTF-8; anything else (ISO8859-*, KOI8-*, ...)
// is decoded byte-for-byte into the matching code point, which keeps the `.aff`,
// the `.dic` and the input words consistent with one another even for the
// legacy encodings that the Hunspell test corpus uses.
///|
/// How the bytes of the `.aff`, `.dic` and word input are decoded.
priv enum FileEncoding {
Utf8
/// One code point per byte (`ISO8859-1` and a safe fallback for the other
/// single-byte encodings).
Latin1
/// `ISO8859-15`, which differs from Latin-1 in eight code points (notably the
/// euro sign and `œ`/`Œ`).
Latin15
}
///|
/// Run the command line and terminate with its status.
fn main {
let status = run(command_args())
if status != 0 {
@sys.exit(status)
}
}
///|
/// The command line with the program name first, as `run` expects.
///
/// The wasm, wasm-gc and native runtimes already report it that way. Node's
/// `process.argv` (the js backend) starts with the interpreter path instead, so
/// that one entry is dropped and the script path takes the program-name slot.
/// Without this, `--words -` on the js backend could not even reach the
/// word-reading code: the "subcommand" slot held the script path.
#cfg(target="js")
fn command_args() -> Array[String] {
let args = @env.args()
let tail : Array[String] = []
let mut index = 1
while index < args.length() {
tail.push(args[index])
index = index + 1
}
tail
}
///|
#cfg(not(target="js"))
fn command_args() -> Array[String] {
@env.args()
}
///|
/// Parse the arguments, run the `check` subcommand, and return the exit status.
fn run(args : Array[String]) -> Int {
if args.length() < 2 {
print_usage()
return 2
}
if args[1] == "suggest" {
return run_suggest(args)
}
if args[1] != "check" {
eprintln("spell: unknown subcommand \"\{args[1]}\"")
print_usage()
return 2
}
let options = match parse_options(args) {
Ok(options) => options
Err(message) => {
eprintln("spell: \{message}")
print_usage()
return 2
}
}
let aff_path = match options.aff {
Some(path) => path
None => {
eprintln("spell: --aff is required")
print_usage()
return 2
}
}
let dic_path = match options.dic {
Some(path) => path
None => {
eprintln("spell: --dic is required")
print_usage()
return 2
}
}
let aff_bytes = match read_bytes(aff_path) {
Ok(bytes) => bytes
Err(message) => {
eprintln("spell: \{message}")
return 1
}
}
// The `.aff` decides the encoding for all three inputs.
let encoding = sniff_encoding(aff_bytes)
let aff_text = strip_bom(decode_bytes(aff_bytes, encoding))
let dic_bytes = match read_bytes(dic_path) {
Ok(bytes) => bytes
Err(message) => {
eprintln("spell: \{message}")
return 1
}
}
let dic_text = strip_bom(decode_bytes(dic_bytes, encoding))
let dictionary = match load_dictionary(aff_text, dic_text) {
Ok(dictionary) => dictionary
Err(message) => {
eprintln("spell: \{message}")
return 1
}
}
let words_source = match options.words {
Some(source) => source
None => "-"
}
let words_bytes = match read_words_bytes(words_source) {
Ok(bytes) => bytes
Err(message) => {
eprintln("spell: \{message}")
return 1
}
}
let words_text = strip_bom(decode_words(words_bytes, encoding))
let verdicts : Array[String] = []
for raw in words_text.split("\n") {
let line = match raw.strip_suffix("\r") {
Some(view) => view.to_owned()
None => raw.to_owned()
}
// An input line that is empty produces no output, so the output lines line
// up with the non-empty input lines. A line that has content but only
// whitespace still produces a verdict, keeping that alignment exact.
if !line.is_empty() {
let word = line.trim()
let correct = !word.is_empty() && dictionary.check(word.to_owned())
verdicts.push(if correct { "1" } else { "0" })
}
}
if !verdicts.is_empty() {
println(verdicts.join("\n"))
}
0
}
///|
/// Run the `suggest` subcommand.
///
/// The dictionary is loaded exactly as the `check` subcommand loads it, then
/// each non-empty input line is answered with its suggestions joined by `, ` on
/// one output line. A word with no suggestion still produces a line, and that
/// line is empty. The input word itself is never among its own suggestions.
fn run_suggest(args : Array[String]) -> Int {
let options = match parse_options(args) {
Ok(options) => options
Err(message) => {
eprintln("spell: \{message}")
print_usage()
return 2
}
}
let aff_path = match options.aff {
Some(path) => path
None => {
eprintln("spell: --aff is required")
print_usage()
return 2
}
}
let dic_path = match options.dic {
Some(path) => path
None => {
eprintln("spell: --dic is required")
print_usage()
return 2
}
}
let aff_bytes = match read_bytes(aff_path) {
Ok(bytes) => bytes
Err(message) => {
eprintln("spell: \{message}")
return 1
}
}
let encoding = sniff_encoding(aff_bytes)
let aff_text = strip_bom(decode_bytes(aff_bytes, encoding))
let dic_bytes = match read_bytes(dic_path) {
Ok(bytes) => bytes
Err(message) => {
eprintln("spell: \{message}")
return 1
}
}
let dic_text = strip_bom(decode_bytes(dic_bytes, encoding))
let dictionary = match load_dictionary(aff_text, dic_text) {
Ok(dictionary) => dictionary
Err(message) => {
eprintln("spell: \{message}")
return 1
}
}
let words_source = match options.words {
Some(source) => source
None => "-"
}
let words_bytes = match read_words_bytes(words_source) {
Ok(bytes) => bytes
Err(message) => {
eprintln("spell: \{message}")
return 1
}
}
let words_text = strip_bom(decode_words(words_bytes, encoding))
let lines : Array[String] = []
for raw in words_text.split("\n") {
let line = match raw.strip_suffix("\r") {
Some(view) => view.to_owned()
None => raw.to_owned()
}
if !line.is_empty() {
let word = line.trim()
let suggestions = @api.suggest(dictionary, word.to_owned(), 15)
lines.push(suggestions.join(", "))
}
}
if !lines.is_empty() {
println(lines.join("\n"))
}
0
}
///|
/// The accepted `check` options; each is `None` until its flag is seen.
struct Options {
aff : String?
dic : String?
words : String?
}
///|
/// Read the `--aff` / `--dic` / `--words` flags, rejecting anything else.
fn parse_options(args : Array[String]) -> Result[Options, String] {
let mut aff : String? = None
let mut dic : String? = None
let mut words : String? = None
let mut index = 2
while index < args.length() {
let option = args[index]
if index + 1 >= args.length() {
return Err("\{option} needs a value")
}
let value = args[index + 1]
if option == "--aff" {
aff = Some(value)
} else if option == "--dic" {
dic = Some(value)
} else if option == "--words" {
words = Some(value)
} else {
return Err("unknown option \"\{option}\"")
}
index = index + 2
}
Ok({ aff, dic, words, })
}
///|
/// Read a whole file (or, for anything that is not `-`, the word list) raw.
fn read_bytes(path : String) -> Result[Bytes, String] {
Ok(@fs.read_file_to_bytes(path)) catch {
@fs.IOError(message) => Err("cannot read \"\{path}\": \{message}")
}
}
///|
/// Read the `--words` input: `-` means stdin, anything else is a path.
///
/// Stdin is handled separately from a path because the native backend cannot
/// size a stream by seeking (see `read_stdin_bytes`).
fn read_words_bytes(source : String) -> Result[Bytes, String] {
if source == "-" {
read_stdin_bytes()
} else {
read_bytes(source)
}
}
///|
/// Read all of stdin raw.
///
/// The native (and llvm) backend must not use `read_bytes("/dev/stdin")`: that
/// helper seeks to the end of the stream to learn how many bytes to allocate,
/// and a **pipe** is not seekable, so `printf 'hello\n' | spell ... --words -`
/// failed with `Illegal seek` (docs/MOONBIT_GOTCHAS.md #33). Native instead
/// pulls the stream in fixed-size chunks until EOF, which needs no seek.
///
/// wasm, wasm-gc and js expose no fd/stream layer to MoonBit, but their host
/// already resolves `/dev/stdin` for `read_file_to_bytes` and reads a pipe
/// correctly, so those backends keep the whole-file read.
#cfg(any(target="native", target="llvm"))
fn read_stdin_bytes() -> Result[Bytes, String] {
read_all_chunks(native_stdin_chunks())
}
///|
#cfg(not(any(target="native", target="llvm")))
fn read_stdin_bytes() -> Result[Bytes, String] {
read_bytes("/dev/stdin")
}
///|
/// How many bytes `read_all_chunks` asks for at a time.
#cfg(any(target="native", target="llvm"))
let stdin_chunk_size : Int = 65536
///|
/// Read a byte source to the end by appending one chunk after another.
///
/// `source` returns the next slice of input, or an empty view at end of input,
/// or `Err` if the read failed. This is the shape of a `fread` loop, and it is
/// what makes a non-seekable pipe readable: the length of the input never has to
/// be known in advance.
#cfg(any(target="native", target="llvm"))
fn read_all_chunks(
source : () -> Result[BytesView, String],
) -> Result[Bytes, String] {
let buffer = Buffer()
while true {
match source() {
Err(message) => return Err(message)
Ok(chunk) => {
if chunk.is_empty() {
break
}
buffer.write_bytes(chunk)
}
}
}
Ok(buffer.to_bytes())
}
///|
/// Hand `read_all_chunks` one chunk of stdin at a time, with a plain `fread` on
/// a buffer that is reused, so no seek is involved and no data is copied twice.
/// `-1` from the C stub means the read failed.
#cfg(any(target="native", target="llvm"))
fn native_stdin_chunks() -> () -> Result[BytesView, String] {
let scratch = Bytes::make(stdin_chunk_size, 0)
fn() {
let count = read_stdin_chunk(scratch, scratch.length())
if count < 0 {
Err(stdin_read_error())
} else {
Ok(scratch.exact_view(end=count))
}
}
}
///|
/// Read up to `capacity` bytes of stdin into `buffer`. Returns the number of
/// bytes read, `0` at end of input, or `-1` on error (with `errno` set).
#cfg(any(target="native", target="llvm"))
#borrow(buffer)
extern "C" fn read_stdin_chunk(buffer : Bytes, capacity : Int) -> Int = "moonbit_spell_read_stdin_chunk"
///|
/// The reason the last `read_stdin_chunk` failed, as bytes.
#cfg(any(target="native", target="llvm"))
extern "C" fn read_stdin_error_ffi() -> Bytes = "moonbit_spell_stdin_error"
///|
/// A clear, actionable message for a failed stdin read. The seek limitation
/// this reader was written to avoid is gone, so this only fires on a genuine
/// I/O error (a closed descriptor, a hard read failure, ...); it still names the
/// backend and points at the two ways to avoid stdin.
#cfg(any(target="native", target="llvm"))
fn stdin_read_error() -> String {
let reason = @unicode.to_utf8_string(read_stdin_error_ffi())
"cannot read \"-\" (stdin) on the native backend: \{reason}; pass --words or redirect stdin from a regular file (\"< file\")"
}
///|
/// Read the `SET` directive without needing to know the encoding yet: the
/// directive itself is ASCII, so a byte-preserving decode is enough to find it.
/// A leading UTF-8 byte-order mark is skipped so it cannot hide the directive.
///
/// `FLAG UTF-8` is treated as declaring the file encoding when there is no
/// `SET` line: one flag is one Unicode character, so the flag text can only be
/// meant as UTF-8. (The corpus' `flagutf8` suite is exactly this shape, and
/// without this rule a multi-byte flag character is read as two Latin-1
/// characters in the `.aff` but as two one-character flags in the `.dic`, so
/// the two never compare equal.)
fn sniff_encoding(bytes : Bytes) -> FileEncoding {
let text = decode_bytes(bytes, Latin1)
let text = match text.strip_prefix("\u{00EF}\u{00BB}\u{00BF}") {
Some(rest) => rest.to_owned()
None => text
}
let mut flag_utf8 = false
for line in text.split("\n") {
let trimmed = line.trim()
if trimmed.is_empty() {
continue
}
match trimmed.strip_prefix("SET") {
Some(rest) =>
if !rest.is_empty() && first_char_is_space(rest) {
let name = rest.trim()
if name.contains("UTF") || name.contains("utf") {
return Utf8
} else if name.contains("8859-15") {
return Latin15
} else {
return Latin1
}
}
None => ()
}
match trimmed.strip_prefix("FLAG") {
Some(rest) =>
if !rest.is_empty() && first_char_is_space(rest) {
let name = rest.trim()
if name == "UTF-8" || name == "utf-8" {
flag_utf8 = true
}
}
None => ()
}
}
if flag_utf8 {
return Utf8
}
// No `SET` line: Hunspell's documented default is ISO8859-1.
Latin1
}
///|
/// Does `text` begin with a whitespace character?
fn first_char_is_space(text : StringView) -> Bool {
for c in text {
return c.is_whitespace()
}
false
}
///|
/// Decode raw bytes with the file's encoding.
fn decode_bytes(bytes : Bytes, encoding : FileEncoding) -> String {
match encoding {
Utf8 => @unicode.to_utf8_string(bytes)
Latin1 => decode_single_byte(bytes, false)
Latin15 => decode_single_byte(bytes, true)
}
}
///|
/// Decode one code point per byte, optionally applying the ISO8859-15
/// differences from Latin-1.
fn decode_single_byte(bytes : Bytes, latin15 : Bool) -> String {
let buffer = StringBuilder()
let mut index = 0
while index < bytes.length() {
let code = bytes[index].to_int()
let mapped = if latin15 { latin15_code_point(code) } else { code }
buffer.write_char(mapped.unsafe_to_char())
index = index + 1
}
buffer.to_string()
}
///|
/// The eight code points where ISO8859-15 differs from ISO8859-1.
fn latin15_code_point(code : Int) -> Int {
match code {
0xA4 => 0x20AC // €
0xA6 => 0x0160 // Š
0xA8 => 0x0161 // š
0xB4 => 0x017D // Ž
0xB8 => 0x017E // ž
0xBC => 0x0152 // Œ
0xBD => 0x0153 // œ
0xBE => 0x0178 // Ÿ
_ => code
}
}
///|
/// Decode the word list.
///
/// The `.aff`/`.dic` pair and the word list are not guaranteed to share an
/// encoding: the Hunspell test corpus keeps legacy-encoded dictionaries next to
/// UTF-8 word lists. So a word input that is valid UTF-8 is decoded as UTF-8,
/// and only genuinely non-UTF-8 input falls back to the dictionary's
/// single-byte encoding.
fn decode_words(bytes : Bytes, encoding : FileEncoding) -> String {
match encoding {
Utf8 => @unicode.to_utf8_string(bytes)
_ =>
if is_valid_utf8(bytes) {
@unicode.to_utf8_string(bytes)
} else {
decode_bytes(bytes, encoding)
}
}
}
///|
/// Strictly validate UTF-8 (overlong forms, stray continuation bytes and
/// truncated sequences all count as invalid), so the fallback above only
/// triggers when the input really is not UTF-8.
fn is_valid_utf8(bytes : Bytes) -> Bool {
let length = bytes.length()
let mut index = 0
while index < length {
let first = bytes[index].to_int()
if first < 0x80 {
index = index + 1
} else if first >= 0xC2 && first <= 0xDF {
if index + 1 >= length || !is_continuation(bytes[index + 1]) {
return false
}
index = index + 2
} else if first >= 0xE0 && first <= 0xEF {
if index + 2 >= length ||
!is_continuation(bytes[index + 1]) ||
!is_continuation(bytes[index + 2]) {
return false
}
index = index + 3
} else if first >= 0xF0 && first <= 0xF4 {
if index + 3 >= length ||
!is_continuation(bytes[index + 1]) ||
!is_continuation(bytes[index + 2]) ||
!is_continuation(bytes[index + 3]) {
return false
}
index = index + 4
} else {
return false
}
}
true
}
///|
/// Is this byte a UTF-8 continuation byte (`10xxxxxx`)?
fn is_continuation(byte : Byte) -> Bool {
(byte.to_int() & 0xC0) == 0x80
}
///|
/// Drop a leading UTF-8 byte-order mark, which some dictionaries carry.
fn strip_bom(text : String) -> String {
match text.strip_prefix("\u{FEFF}") {
Some(rest) => rest.to_owned()
None => text
}
}
///|
/// Build the dictionary, turning a parse error into a printable message.
fn load_dictionary(
aff_text : String,
dic_text : String,
) -> Result[@api.Dictionary, String] {
Ok(@api.load(aff_text, dic_text)) catch {
error => Err("cannot load dictionary: \{error.to_string()}")
}
}
///|
/// Print the accepted command shape to stderr.
fn print_usage() -> Unit {
eprintln(
"usage: spell check --aff --dic [--words |-]",
)
eprintln(" --words - read one word per line from stdin (the default)")
eprintln(" prints one line per input word: 1 = correct, 0 = incorrect")
eprintln(
"usage: spell suggest --aff --dic [--words |-]",
)
eprintln(" prints one line per input word: its suggestions joined by \", \"")
}