///|
struct Options {
urls : Array[String]
input_files : Array[String]
mut help : Bool
mut quiet : Bool
mut output_document : String?
mut log_file : String?
mut append_log : Bool
mut continue_download : Bool
mut timestamping : Bool
mut tries : Int32
mut retry_connection_refused : Bool
retry_http_statuses : Array[Int32]
mut retry_delay_ms : Int32
mut max_redirects : Int32
mut content_disposition : Bool
headers : Array[@netops.HttpHeader]
mut request_method : @netops.HttpMethod
mut method_explicit : Bool
mut body : @netops.RequestBody
mut body_options_conflict : Bool
mut no_check_certificate : Bool
mut no_proxy : Bool
mut utf8_hosts : Bool
mut connect_timeout_ms : Int32
mut read_timeout_ms : Int32
mut user : String?
mut password : String?
mut auth_no_challenge : Bool
mut cookies : Bool
mut load_cookies : String?
mut save_cookies : String?
mut keep_session_cookies : Bool
}
///|
suberror UsageError {
UsageError(String)
}
///|
fn usage() -> String {
let text =
#|Usage: wget [OPTION]... [URL]...
#| --help display this help and exit
#| --version display version and exit
#| -q, --quiet quiet (no output)
#| -O, --output-document=FILE write documents to FILE
#| -o, --output-file=FILE log messages to FILE
#| -a, --append-output=FILE append messages to FILE
#| -i, --input-file=FILE download URLs found in FILE
#| -c, --continue resume after the already written prefix
#| -N, --timestamping do not re-retrieve unchanged files
#| -t, --tries=NUMBER set number of retries (0 means unlimited)
#| --retry-connrefused retry even if the connection is refused
#| --retry-on-http-error=CODES retry comma-separated HTTP status codes
#| --waitretry=SECONDS wait between retries
#| partial bytes survive retries and failures
#| --max-redirect=NUMBER maximum redirects (default 20)
#| --content-disposition honor Content-Disposition filenames
#| --header=STRING set a request header; last same-name value wins
#| --user=USER HTTP Basic user name
#| --password=PASSWORD HTTP Basic password
#| --http-user=USER HTTP-specific user, before --user fallback
#| --http-password=PASSWORD HTTP-specific password, before --password fallback
#| --auth-no-challenge[=on|off] send Basic credentials before a challenge when enabled
#| --load-cookies=FILE load a Netscape cookie jar
#| --header='Cookie: ...' override jar cookies, including on redirects
#| --save-cookies=FILE save persistent cookies
#| --keep-session-cookies also save session cookies
#| --no-cookies disable cookie storage
#| --method=HTTPMethod use a supported HTTP method
#| --body-data=STRING send STRING with an explicit --method
#| --body-file=FILE send FILE with an explicit --method
#| --no-check-certificate disable TLS certificate verification
#| --no-proxy do not use proxy environment variables
#| --local-encoding=ENCODING URL host input: UTF-8 or ASCII (default ASCII)
#| -T, --timeout=SECONDS set connect and read timeouts
#| --connect-timeout=SECONDS set connection timeout
#| --read-timeout=SECONDS set response inactivity timeout
#| --idle-timeout=SECONDS extension: response inactivity timeout
#|
#|Compatibility limits: HTTP/HTTPS only; --method accepts GET, HEAD, POST,
#|PUT, DELETE, CONNECT, OPTIONS, TRACE, and PATCH. Timer values are bounded
#|by the runtime millisecond timer range.
#|Content-coding bytes remain encoded; Accept-Encoding defaults to identity.
#|Raw space/TAB/CR/LF/DEL in URL paths or queries uses uppercase percent escapes.
#|Redirect spaces use %20 and TAB uses %09; existing escapes remain encoded.
#|Set-Cookie without '=' or a valid Max-Age numeric prefix is ignored.
#|Wget expiry accepts decimal/hex floating prefixes; exactly -1 rejects an update.
#|Zero requests deletion; nonzero negative values retain stale domain-chain entries.
#|Overflow follows the fixed Linux/amd64 profile; saved expiry digits round via binary64.
#|Expires uses four C-locale date formats, UTC normalization and the 69 year pivot.
#|Missing/GMT/signed-digit timezone tails are accepted without applying offsets.
#|GNU chunk payload/trailer EOF rules apply; missing size lines or CRLF fail.
#|HTTP 416 succeeds without opening transfer output; -O is initialized first.
#|Unsolicited 304 exits 8. Continued GET with --content-disposition selects
#|the filename with HEAD before choosing that file's resume offset.
#|Multiple URL failures retain the lowest nonzero transfer exit code.
#|Basic challenges are recognized across repeated fields and quoted comma lists.
#|URL credentials take field-wise precedence; omitted and empty passwords differ.
#|URL credentials wait for each hop's challenge; global credentials cache by hostname.
#|HTTP 401 returns 6, including missing or unsupported challenges; other HTTP errors return 8.
#|ASCII response cookie domains use the complete pinned ICANN/PRIVATE suffix list.
#|Non-ASCII domains/hosts use the fixed non-NLS Wget label/byte-length heuristic.
#|Admission strips leading dots separately from stored Domain identity.
#|Rejected domains discard a matching stored cookie after request-path validation.
#|Cookie request paths use raw prefix matching; missing Path uses the URL directory.
#|Imported cookie files retain Wget's separate admission policy.
#|Host-only cookies isolate nondefault ports and preserve them in saved jars.
#|Cookie parameters accept quoted values; Secure ignores its value, HttpOnly is ignored.
#|Jar lookup skips single-label parent domains and hosts with more than 32 labels.
#|UTF-8 URL hosts use pinned Unicode 15 IDNA with whole-host transitional fallback.
#|Host brackets mapped colons and omits the scheme's default port.
#|Failed conversion retains the raw host; other input encodings are unsupported.
text + "\n"
}
///|
fn default_options() -> Options {
{
urls: [],
input_files: [],
help: false,
quiet: false,
output_document: None,
log_file: None,
append_log: false,
continue_download: false,
timestamping: false,
tries: 20,
retry_connection_refused: false,
retry_http_statuses: [],
retry_delay_ms: 0,
max_redirects: 20,
content_disposition: false,
headers: [],
request_method: @netops.Get,
method_explicit: false,
body: @netops.EmptyBody,
body_options_conflict: false,
no_check_certificate: false,
no_proxy: false,
utf8_hosts: false,
connect_timeout_ms: 900_000,
read_timeout_ms: 900_000,
user: None,
password: None,
auth_no_challenge: false,
cookies: true,
load_cookies: None,
save_cookies: None,
keep_session_cookies: false,
}
}
///|
fn parse_nonnegative(value : String, option : String) -> Int32 raise UsageError {
let result = @string.parse_int(value) catch {
_ => raise UsageError("\{option}: invalid number '\{value}'")
}
if result < 0 {
raise UsageError("\{option}: invalid number '\{value}'")
}
result
}
///|
fn parse_seconds(value : String, option : String) -> Int32 raise UsageError {
@cli.parse_idle_timeout(value) catch {
_ => raise UsageError("\{option}: invalid time period '\{value}'")
}
}
///|
fn parse_header(value : String) -> @netops.HttpHeader raise UsageError {
guard value.find(":") is Some(index) && index > 0 else {
raise UsageError("--header: invalid header '\{value}'")
}
@netops.http_header(
value[:index].to_owned().trim().to_owned(),
value[index + 1:].to_owned().trim().to_owned(),
)
}
///|
fn parse_statuses(value : String) -> Array[Int32] raise UsageError {
let statuses : Array[Int32] = []
for item in value.split(",") {
let status = parse_nonnegative(
item.trim().to_owned(),
"--retry-on-http-error",
)
if status < 100 || status > 599 {
raise UsageError("--retry-on-http-error: invalid HTTP status")
}
statuses.push(status)
}
statuses
}
///|
fn parse_options(args : ArrayView[String]) -> Options raise UsageError {
let specs = [
@cli.flag("quiet", short='q'),
@cli.option("output-document", short='O'),
@cli.option("output-file", short='o'),
@cli.option("append-output", short='a'),
@cli.option("input-file", short='i'),
@cli.flag("continue", short='c'),
@cli.flag("timestamping", short='N'),
@cli.option("tries", short='t'),
@cli.flag("retry-connrefused"),
@cli.option("retry-on-http-error"),
@cli.option("waitretry"),
@cli.option("max-redirect"),
@cli.flag("content-disposition"),
@cli.option("header"),
@cli.option("user"),
@cli.option("password"),
@cli.option("http-user"),
@cli.option("http-password"),
@cli.option("auth-no-challenge"),
@cli.option("load-cookies"),
@cli.option("save-cookies"),
@cli.flag("keep-session-cookies"),
@cli.flag("no-cookies"),
@cli.option("method"),
@cli.option("body-data"),
@cli.option("body-file"),
@cli.flag("no-check-certificate"),
@cli.flag("no-proxy"),
@cli.option("local-encoding"),
@cli.option("timeout", short='T'),
@cli.option("connect-timeout"),
@cli.option("read-timeout"),
@cli.option("idle-timeout"),
@cli.flag("help"),
]
let parsed = @cli.parse(auth_arguments(args, specs), specs) catch {
@cli.CliError(option~, message~, ..) =>
raise UsageError("\{message}: '\{option}'")
}
let options = default_options()
options.urls.append(parsed.operands)
options.input_files.append(parsed.values("input-file"))
options.help = parsed.contains("help")
options.quiet = parsed.contains("quiet")
options.output_document = parsed.last_value("output-document")
options.continue_download = parsed.contains("continue")
options.timestamping = parsed.contains("timestamping")
options.retry_connection_refused = parsed.contains("retry-connrefused")
options.content_disposition = parsed.contains("content-disposition")
options.no_check_certificate = parsed.contains("no-check-certificate")
options.no_proxy = parsed.contains("no-proxy")
if parsed.last_value("local-encoding") is Some(value) {
options.utf8_hosts = match value.to_lower().replace_all(old="-", new="") {
"utf8" => true
"ascii" | "usascii" | "ansi_x3.41968" => false
_ =>
raise UsageError(
"--local-encoding: unsupported encoding '" + value + "'",
)
}
}
options.user = match parsed.last_value("http-user") {
Some(value) => Some(value)
None => parsed.last_value("user")
}
options.password = match parsed.last_value("http-password") {
Some(value) => Some(value)
None => parsed.last_value("password")
}
if parsed.last_value("auth-no-challenge") is Some(value) {
options.auth_no_challenge = match value.to_lower() {
"on" | "yes" | "1" => true
"off" | "no" | "0" => false
_ =>
raise UsageError(
"--auth-no-challenge: Invalid boolean '" +
value +
"'; use `on' or `off'.",
)
}
}
options.cookies = !parsed.contains("no-cookies")
options.load_cookies = parsed.last_value("load-cookies")
options.save_cookies = parsed.last_value("save-cookies")
options.keep_session_cookies = parsed.contains("keep-session-cookies")
match parsed.last_occurrence(["output-file", "append-output"]) {
Some("output-file") => {
options.log_file = parsed.last_value("output-file")
options.append_log = false
}
Some("append-output") => {
options.log_file = parsed.last_value("append-output")
options.append_log = true
}
_ => ()
}
if parsed.last_value("tries") is Some(value) {
options.tries = parse_nonnegative(value, "--tries")
}
if parsed.last_value("waitretry") is Some(value) {
options.retry_delay_ms = parse_seconds(value, "--waitretry")
}
if parsed.last_value("max-redirect") is Some(value) {
options.max_redirects = parse_nonnegative(value, "--max-redirect")
}
for value in parsed.values("retry-on-http-error") {
options.retry_http_statuses.append(parse_statuses(value))
}
for value in parsed.values("header") {
options.headers.push(parse_header(value))
}
if parsed.last_value("method") is Some(value) {
options.request_method = @netops.parse_http_method(value) catch {
_ => raise UsageError("--method: unsupported HTTP method '\{value}'")
}
options.method_explicit = true
}
options.body_options_conflict = parsed.contains("body-data") &&
parsed.contains("body-file")
match parsed.last_occurrence(["body-data", "body-file"]) {
Some("body-data") => {
let value = parsed.last_value("body-data").unwrap()
options.body = @netops.BytesBody(@utf8.encode(value))
}
Some("body-file") => {
let value = parsed.last_value("body-file").unwrap()
options.body = if value == "-" {
@netops.StdinBody
} else {
@netops.FileBody(value)
}
}
_ => ()
}
if parsed.last_value("timeout") is Some(value) {
let timeout = parse_seconds(value, "--timeout")
options.connect_timeout_ms = timeout
options.read_timeout_ms = timeout
}
if parsed.last_value("connect-timeout") is Some(value) {
options.connect_timeout_ms = parse_seconds(value, "--connect-timeout")
}
match parsed.last_occurrence(["read-timeout", "idle-timeout"]) {
Some(name) => {
let value = parsed.last_value(name).unwrap()
options.read_timeout_ms = parse_seconds(value, "--read-timeout")
}
None => ()
}
options
}