///|
fn has_header(headers : ArrayView[@netops.HttpHeader], name : String) -> Bool {
let normalized = name.to_lower()
for header in headers {
if header.name.to_lower() == normalized {
return true
}
}
false
}
///|
fn without_line_endings(data : Bytes) -> Bytes {
let output = @buffer.Buffer(size_hint=data.length())
for byte in data {
if byte != b'\r' && byte != b'\n' && byte != b'\x00' {
output.write_byte(byte)
}
}
output.to_bytes()
}
///|
fn url_encode(data : Bytes) -> Bytes {
let output = @buffer.Buffer(size_hint=data.length())
let digits = b"0123456789ABCDEF"
for byte in data {
let value = byte.to_int()
if (value >= 0x41 && value <= 0x5A) ||
(value >= 0x61 && value <= 0x7A) ||
(value >= 0x30 && value <= 0x39) ||
byte == b'-' ||
byte == b'.' ||
byte == b'_' ||
byte == b'~' {
output.write_byte(byte)
} else {
output.write_byte(b'%')
output.write_byte(digits[value >> 4])
output.write_byte(digits[value & 0x0F])
}
}
output.to_bytes()
}
///|
async fn read_data(spec : DataSpec) -> Bytes {
match spec.kind {
RawData => @utf8.encode(spec.value)
FormData =>
if spec.value.has_prefix("@") {
without_line_endings(@fs.read_file(spec.value[1:]).binary())
} else {
@utf8.encode(spec.value)
}
BinaryData =>
if spec.value.has_prefix("@") {
@fs.read_file(spec.value[1:]).binary()
} else {
@utf8.encode(spec.value)
}
UrlEncodedData => {
let value = spec.value
if value.has_prefix("@") {
url_encode(@fs.read_file(value[1:]).binary())
} else if value.find("@") is Some(index) && index > 0 {
let name = value[:index].to_owned()
@utf8.encode(name + "=") +
url_encode(@fs.read_file(value[index + 1:]).binary())
} else if value.find("=") is Some(index) {
@utf8.encode(value[:index + 1].to_owned()) +
url_encode(@utf8.encode(value[index + 1:].to_owned()))
} else {
url_encode(@utf8.encode(value))
}
}
}
}
///|
async fn request_body(options : Options) -> @netops.RequestBody {
if options.data_specs.length() > 0 {
let output = @buffer.Buffer()
for index, spec in options.data_specs {
if index > 0 {
output.write_byte(b'&')
}
output.write_bytes(read_data(spec))
}
return @netops.BytesBody(output.to_bytes())
}
match options.upload_file {
Some("-") => @netops.StdinBody
Some(path) => @netops.FileBody(path)
None => @netops.EmptyBody
}
}
///|
fn request_method(options : Options) -> @netops.HttpMethod {
match options.request_method {
Some(request) => request
None if options.head => @netops.Head
None if options.upload_file is Some(_) => @netops.Put
None if options.data_specs.length() > 0 => @netops.Post
None => @netops.Get
}
}
///|
fn proxy_for(options : Options, url : String) -> String? {
let no_proxy = match options.no_proxy {
Some(value) => Some(value)
None =>
match @env.get_env_var("NO_PROXY") {
Some(value) => Some(value)
None => @env.get_env_var("no_proxy")
}
}
if @download.proxy_bypassed(url, no_proxy, utf8_hosts=utf8_hosts()) {
return None
}
if options.proxy_url is Some(value) {
return if value == "" { None } else { Some(value) }
}
let name = if url.to_lower().has_prefix("https://") {
"https_proxy"
} else {
"http_proxy"
}
match @env.get_env_var(name) {
Some(value) => Some(value)
None if name == "https_proxy" =>
match @env.get_env_var("HTTPS_PROXY") {
Some(value) => Some(value)
None =>
match @env.get_env_var("ALL_PROXY") {
Some(value) => Some(value)
None => @env.get_env_var("all_proxy")
}
}
None =>
match @env.get_env_var("ALL_PROXY") {
Some(value) => Some(value)
None => @env.get_env_var("all_proxy")
}
}
}
///|
fn output_for(
options : Options,
url : String,
index : Int32,
) -> @download.TransferOutput raise @netops.TransferError {
if index < options.output_specs.length() {
let path = options.output_specs[index]
return if path == "-" {
@download.StandardOutput
} else {
@download.FileOutput(
path~,
mode=@download.Truncate,
content_disposition=false,
remove_on_error=options.remove_on_error,
)
}
}
if options.remote_name {
let normalized = match url.find("#") {
Some(index) => url[:index].to_owned()
None => url
}
let normalized = match normalized.find("?") {
Some(index) => normalized[:index].to_owned()
None => normalized
}
if normalized.has_suffix("/") {
raise @netops.TransferError(
kind=@netops.OutputFailure,
message="Remote filename has no length",
status=None,
context=None,
)
}
let path = @download.remote_name(url)
return @download.FileOutput(
path~,
mode=@download.Truncate,
content_disposition=options.remote_header_name,
remove_on_error=options.remove_on_error,
)
}
@download.StandardOutput
}
///|
fn exit_code(kind : @netops.TransferErrorKind) -> Int32 {
match kind {
@netops.UnsupportedProtocol => 1
@netops.InvalidMethod | @netops.ProtocolFailure => 2
@netops.InvalidUrl => 3
@netops.ProxyFailure => 5
@netops.ConnectionFailure => 7
@netops.MalformedReplyFailure => 8
@netops.PartialBodyFailure => 18
@netops.HttpStatusFailure | @netops.AuthenticationFailure => 22
@netops.OutputFailure => 23
@netops.InputFailure => 26
@netops.RewindFailure => 65
@netops.TimeoutFailure => 28
@netops.RedirectFailure => 47
@netops.ReceiveFailure => 56
@netops.EmptyReplyFailure => 52
@netops.TlsFailure => 60
}
}
///|
fn progress_text(
received : Int64,
content_length : Int64?,
interactive : Bool,
completed : Bool,
) -> String {
let percent = match content_length {
Some(total) if total > 0L =>
(received.to_double() * 100.0 / total.to_double())
.clamp(min=0.0, max=100.0)
.to_int()
_ => 0
}
if interactive {
"\r\{percent}% \{received}" + (if completed { "\n" } else { "\u{1b}[K" })
} else if completed {
"\{percent}% \{received}\n"
} else {
""
}
}
///|
async fn observe_transfer(
show_headers : Bool,
show_progress : Bool,
interactive : Bool,
event : @download.TransferEvent,
) -> Unit {
match event {
@download.ResponseHeader(bytes) if show_headers =>
@stdio.stdout.write(bytes)
@download.ResponseStarted(..) =>
if show_progress {
@stdio.stderr.write(
" % Total % Received Average Speed Time Time Time Current\n",
)
}
@download.Progress(received~, content_length~) if show_progress => {
let text = progress_text(received, content_length, interactive, false)
if text != "" {
@stdio.stderr.write(text)
}
}
@download.Completed(received~, content_length~) if show_progress =>
@stdio.stderr.write(
progress_text(received, content_length, interactive, true),
)
_ => ()
}
}
///|
fn retryable_http_status(status : Int32) -> Bool {
match status {
408 | 429 | 500 | 502 | 503 | 504 | 522 | 524 => true
_ => false
}
}
///|
fn can_retry(
options : Options,
kind : @netops.TransferErrorKind,
status : Int32?,
) -> Bool {
if options.retry_all_errors {
return kind != @netops.InputFailure && kind != @netops.OutputFailure
}
match kind {
@netops.HttpStatusFailure =>
status.map(retryable_http_status).unwrap_or(false)
@netops.ConnectionFailure => options.retry_connection_refused
@netops.TimeoutFailure => true
_ => false
}
}
///|
fn reports_retry_error(
options : Options,
kind : @netops.TransferErrorKind,
) -> Bool {
(!options.silent || options.show_error) &&
(
![@netops.HttpStatusFailure, @netops.AuthenticationFailure].contains(kind) ||
options.fail
)
}
///|
fn error_message(
kind : @netops.TransferErrorKind,
message : String,
context : @netops.TransferFailureContext?,
) -> String {
if kind == @netops.PartialBodyFailure &&
context is Some(context) &&
context.framing is Some(@netops.FixedLength(expected)) &&
context.received_bytes < expected {
return "end of response with \{expected - context.received_bytes} bytes missing"
}
message
}
///|
fn retry_delay(options : Options, attempt : Int32) -> Int32 {
if options.retry_delay_ms > 0 {
return options.retry_delay_ms
}
let exponent = (attempt - 1).clamp(min=0, max=10)
(1000 << exponent).min(600_000)
}
///|
priv struct TransferOutput {
original : @download.TransferOutput
mut opened_path : String?
}
///|
fn TransferOutput::for_attempt(
self : TransferOutput,
) -> @download.TransferOutput {
match (self.original, self.opened_path) {
(@download.FileOutput(..), Some(path)) =>
@download.FileOutput(
path~,
mode=@download.Append,
content_disposition=false,
remove_on_error=false,
)
(@download.FileOutput(path~, mode~, content_disposition~, ..), None) =>
@download.FileOutput(
path~,
mode~,
content_disposition~,
remove_on_error=false,
)
(output, _) => output
}
}
///|
fn TransferOutput::observe(
self : TransferOutput,
event : @download.TransferEvent,
) -> Unit {
if event is @download.OutputStarted(path=Some(path), ..) {
self.opened_path = Some(path)
}
}
///|
async fn TransferOutput::reset_for_retry(self : TransferOutput) -> Unit {
if self.opened_path is Some(path) {
let path = @platform.normalize_input_separators(path)
// Released fs APIs reset by pathname; this does not pin the prior handle's
// identity across replacement or rename races.
if @fs.kind(path, follow_symlink=true) == Regular {
let file = @fs.open(path, mode=WriteOnly, create_mode=TruncateExisting)
file.close()
}
}
}
///|
async fn TransferOutput::cleanup_after_failure(self : TransferOutput) -> Unit {
if self.original is @download.FileOutput(remove_on_error=true, ..) &&
self.opened_path is Some(path) {
let path = @platform.normalize_input_separators(path)
try {
if @fs.kind(path, follow_symlink=true) == Regular {
@fs.remove(path)
}
} catch {
_ => ()
}
}
}
///|
async fn transfer_one(
options : Options,
url : String,
index : Int32,
jar : @download.CookieJar?,
) -> Int32 {
let body = request_body(options) catch {
err => {
if !options.silent || options.show_error {
@stdio.stderr.write("curl: (26) \{err}\n")
}
return 26
}
}
let headers : Array[@netops.HttpHeader] = []
if !has_header(options.headers, "user-agent") &&
!options.suppressed_headers.contains("user-agent") {
headers.push(@netops.http_header("User-Agent", "curl/8.22.0"))
}
headers.append(options.headers)
let explicit_cookies = options.cookies.filter(value => value.contains("="))
let cookie_data = if explicit_cookies.is_empty() {
None
} else {
Some(explicit_cookies.join("; "))
}
if !has_header(headers, "accept") &&
!options.suppressed_headers.contains("accept") {
headers.push(@netops.http_header("Accept", "*/*"))
}
if options.data_specs.length() > 0 &&
!has_header(headers, "content-type") &&
!options.suppressed_headers.contains("content-type") {
headers.push(
@netops.http_header("Content-Type", "application/x-www-form-urlencoded"),
)
}
let output = try {
if options.head {
@download.DiscardOutput
} else {
output_for(options, url, index)
}
} catch {
@netops.TransferError(kind~, message~, ..) => {
let code = exit_code(kind)
if !options.silent || options.show_error {
@stdio.stderr.write("curl: (\{code}) \{message}\n")
}
return code
}
}
let transfer_output : TransferOutput = {
original: output,
opened_path: None,
}
let stdout_terminal = @platform.terminal_output_enabled(
@platform.IfTerminal,
@platform.Stdout,
)
let stderr_terminal = @platform.terminal_output_enabled(
@platform.IfTerminal,
@platform.Stderr,
)
let writes_stdout = match output {
@download.StandardOutput => true
_ => false
}
let show_progress = !options.silent && !(writes_stdout && stdout_terminal)
let mut attempt = 0
let retry_started = @async.now()
let (basic_user, basic_password) = match options.user {
Some(value) =>
match value.split_once(":") {
Some((user, password)) =>
(Some(user.to_owned()), Some(password.to_owned()))
None => (Some(value), None)
}
None => (None, None)
}
for ;; {
if jar is Some(jar) {
for source in options.cookies {
if source == "" || source.contains("=") {
continue
}
let complete = if source == "-" {
jar.load_bytes(@stdio.stdin.read_all().binary())
} else {
jar.load(source) catch {
_ => true // Curl treats unreadable cookie inputs as an empty source.
}
}
if !complete {
if !options.silent || options.show_error {
@stdio.stderr.write(
"curl: (100) A value or data field grew larger than allowed\n",
)
}
return 100
}
}
}
let proxy_url = proxy_for(options, url)
let result = try
@download.transfer(
url,
transfer_output.for_attempt(),
@download.options(
utf8_hosts=utf8_hosts(),
request_method=request_method(options),
headers~,
suppressed_headers=options.suppressed_headers,
body~,
redirects=if options.location {
@download.FollowRedirects(
if options.max_redirects == -1 {
2147483647
} else {
options.max_redirects
},
)
} else {
@download.NoRedirects
},
preserve_method_on_redirect=options.request_method is Some(_),
verify_tls=!options.insecure,
proxy_url?,
connect_timeout_ms=options.connect_timeout_ms,
idle_timeout_ms=options.idle_timeout_ms,
max_time_ms?=options.max_time_ms,
fail_on_http_error=options.fail,
cookie_jar?=jar,
cookie_data?,
basic_user?,
basic_password?,
),
observer=event => {
transfer_output.observe(event)
observe_transfer(options.head, show_progress, stderr_terminal, event)
},
)
catch {
@netops.TransferError(kind~, message~, status~, context~) => {
let diagnostic = error_message(kind, message, context)
if attempt < options.retry_count && can_retry(options, kind, status) {
let next_attempt = attempt + 1
let retry_delay = retry_delay(options, next_attempt)
let elapsed = @async.now() - retry_started
if options.retry_max_time_ms is None ||
elapsed + retry_delay.to_int64() <
options.retry_max_time_ms.unwrap().to_int64() {
attempt = next_attempt
if reports_retry_error(options, kind) {
let code = exit_code(kind)
@stdio.stderr.write("curl: (\{code}) \{diagnostic}\n")
}
if !options.silent {
@stdio.stderr.write(
"Warning: transient problem: \{message}. Retrying...\n",
)
}
transfer_output.reset_for_retry() catch {
error => {
transfer_output.cleanup_after_failure()
if !options.silent || options.show_error {
@stdio.stderr.write(
"curl: (23) cannot reset output for retry: \{error}\n",
)
}
return 23
}
}
@async.sleep(retry_delay)
continue
}
}
transfer_output.cleanup_after_failure()
let code = exit_code(kind)
if !options.silent || options.show_error {
@stdio.stderr.write("curl: (\{code}) \{diagnostic}\n")
}
return code
}
err => {
transfer_output.cleanup_after_failure()
if !options.silent || options.show_error {
@stdio.stderr.write("curl: (1) \{err}\n")
}
return 1
}
} noraise {
value => value
}
if attempt < options.retry_count && retryable_http_status(result.status) {
let next_attempt = attempt + 1
let delay = retry_delay(options, next_attempt)
let elapsed = @async.now() - retry_started
if options.retry_max_time_ms is None ||
elapsed + delay.to_int64() <
options.retry_max_time_ms.unwrap().to_int64() {
attempt = next_attempt
if !options.silent {
@stdio.stderr.write(
"Warning: transient problem: server returned HTTP status \{result.status} \{result.reason}. Retrying...\n",
)
}
transfer_output.reset_for_retry() catch {
error => {
transfer_output.cleanup_after_failure()
if !options.silent || options.show_error {
@stdio.stderr.write(
"curl: (23) cannot reset output for retry: \{error}\n",
)
}
return 23
}
}
@async.sleep(delay)
continue
}
}
return 0
}
}
///|
async fn run(options : Options) -> Int32 {
if options.help {
@stdio.stdout.write(usage())
return 0
}
if options.urls.is_empty() {
@stdio.stderr.write("curl: try 'curl --help' for more information\n")
return 2
}
let jar = if options.cookies.is_empty() && options.cookie_jar is None {
None
} else {
Some(@download.CookieJar::new())
}
let mut status = 0
for index, url in options.urls {
status = transfer_one(options, url, index, jar)
if options.cookie_jar is Some(path) && jar is Some(jar) {
jar.save(path) catch {
// Like curl, cookie-jar persistence does not change transfer status.
_ => ()
}
}
}
status
}
///|
async fn main {
let metadata_args = @env.args()[1:]
if metadata_args.length() == 1 && metadata_args[0] == "--version" {
@stdio.stdout.write("curl \{MODULE_VERSION}\n")
return
}
if metadata_args == ["--help"] {
@stdio.stdout.write(usage())
return
}
let file_headers : Set[Int32] = Set([])
let options = parse_options(
configured_arguments(@env.args()[1:], file_headers~),
file_headers~,
) catch {
ConfigOptionError(option, reason) => {
@stdio.stderr.write(
"curl: option " +
option +
": " +
reason +
"\ncurl: try 'curl --help' or 'curl --manual' for more information\n",
)
@sys.exit(
if reason == "error encountered when reading a file" {
26
} else {
2
},
)
return
}
OptionConflict(option) => {
@stdio.stderr.write(
"curl: option " +
option +
": is badly used here\ncurl: try 'curl --help' or 'curl --manual' for more information\n",
)
@sys.exit(2)
return
}
UsageError(message) => {
@stdio.stderr.write("curl: option error: \{message}\n")
@sys.exit(2)
return
}
error => {
@stdio.stderr.write("curl: option error: " + error.to_string() + "\n")
@sys.exit(2)
return
}
}
let status = run(options)
if status != 0 {
@sys.exit(status)
}
}