diff --git a/.gitignore b/.gitignore index 22e5fd0389..3404dbb1f3 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,5 @@ tarpaulin-report.html # Splashboard .splashboard.toml + +.jarvis/ diff --git a/crates/nu-command/src/network/http/client.rs b/crates/nu-command/src/network/http/client.rs index bc93881972..722d6bbdab 100644 --- a/crates/nu-command/src/network/http/client.rs +++ b/crates/nu-command/src/network/http/client.rs @@ -41,7 +41,6 @@ use ureq::{ typestate::{WithBody, WithoutBody}, unversioned::transport::{ConnectProxyConnector, Connector, SocksConnector}, }; -use url::Url; #[cfg(feature = "native-tls")] use ureq::unversioned::transport::NativeTlsConnector; @@ -255,7 +254,7 @@ pub fn http_parse_url( call: &Call, span: Span, raw_url: Value, -) -> Result, ShellError> { +) -> Result, ShellError> { let url_span = raw_url.span(); let mut requested_url = raw_url.coerce_into_string()?; if requested_url.starts_with(':') { @@ -264,19 +263,62 @@ pub fn http_parse_url( requested_url = format!("http://{requested_url}"); } - let url = match url::Url::parse(&requested_url) { - Ok(u) => u, - Err(_e) => { - return Err(ShellError::UnsupportedInput { - msg: "Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com".to_string(), - input: format!("value: '{requested_url:?}'"), - msg_span: call.head, - input_span: span, - }); - } + let invalid_url = || ShellError::UnsupportedInput { + msg: "Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com" + .to_string(), + input: format!("value: '{requested_url:?}'"), + msg_span: call.head, + input_span: span, }; - Ok((requested_url, url).into_spanned(url_span)) + // Validate with `http::Uri`, the same parser ureq uses to send the request. + // Unlike `url::Url`, it accepts IPv6 zone IDs (e.g. `http://[fe80::13%eth0]`), + // so nushell and ureq agree on which URLs are valid. + match requested_url.parse::() { + Ok(uri) if authority_is_well_formed(&uri) => {} + _ => return Err(invalid_url()), + } + + Ok(requested_url.into_spanned(url_span)) +} + +/// Check that the authority of a parsed URI is well formed. +/// +/// `http::Uri` is lenient: it silently drops trailing garbage after an IPv6 +/// literal's closing bracket (e.g. `[fe80::13%eth0]foo`) and non-numeric or +/// out-of-range ports (e.g. `host:abc`, `host:99999`). Accepting those would +/// silently send the request to a different host or port than the user typed, +/// so they are rejected instead. +fn authority_is_well_formed(uri: &http::Uri) -> bool { + let Some(authority) = uri.authority() else { + return true; + }; + // Ignore any userinfo (`user:pass@`). + let host_port = authority + .as_str() + .rsplit_once('@') + .map_or(authority.as_str(), |(_, host_port)| host_port); + // Everything after the host must be empty or `:`. + let rest = if let Some(after_bracket) = host_port.strip_prefix('[') { + match after_bracket.split_once(']') { + Some((_, rest)) => rest, + None => return false, + } + } else { + match host_port.find(':') { + Some(idx) => &host_port[idx..], + None => return true, + } + }; + if rest.is_empty() { + return true; + } + match rest.strip_prefix(':') { + None => false, + // `host:` has an empty port, which `url::Url` accepts. + Some("") => true, + Some(port) => port.parse::().is_ok(), + } } pub fn http_parse_redirect_mode(mode: Option>) -> Result { @@ -886,6 +928,54 @@ pub fn request_add_custom_headers( Ok(request) } +/// Set the `Host` header for a request whose URL contains an IPv6 zone ID. +/// +/// ureq derives the `Host` header from the URI authority, which includes the +/// zone ID (e.g. `[fe80::13%eth0]`). The zone ID is only meaningful on the +/// client machine, so sending it would make strict servers reject the request. +/// curl strips it, and so do we. A `Host` header set by the user is left alone. +pub fn request_set_host_header( + requested_url: &str, + request: RequestBuilder, +) -> RequestBuilder { + // Respect a `Host` header the user set explicitly. + if request + .headers_ref() + .is_some_and(|headers| headers.contains_key("host")) + { + return request; + } + let Ok(uri) = requested_url.parse::() else { + return request; + }; + let Some(authority) = uri.authority() else { + return request; + }; + let host = authority.host(); + if !host.contains('%') { + return request; + } + let host = host + .strip_prefix('[') + .and_then(|inner| inner.strip_suffix(']')) + .unwrap_or(host); + let host = match host.split_once('%') { + Some((addr, _)) => addr, + None => host, + }; + // Match ureq: omit the port when it is the scheme's default. + let default_port = match uri.scheme_str() { + Some("https") => Some(443), + Some("http") => Some(80), + _ => None, + }; + let value = match authority.port() { + Some(port) if Some(port.as_u16()) != default_port => format!("[{host}]:{port}"), + _ => format!("[{host}]"), + }; + request.header("Host", &value) +} + fn handle_status_error(span: Span, requested_url: &str, response: &mut Response) -> ShellError { let msg = if response.header("content-type") == Some("application/json") { // We use a json response as a heuristic to mean the body will contain a relevant error message. @@ -999,15 +1089,19 @@ fn transform_response_using_content_type( .expect("Failed to parse content type, and failed to default to text/plain"); let ext = match (content_type.type_(), content_type.subtype()) { - (mime::TEXT, mime::PLAIN) => url::Url::parse(requested_url) + // Use `http::Uri` (the parser ureq uses) so that URLs `url::Url` cannot + // parse, such as IPv6 zone IDs, still resolve to the right extension. + (mime::TEXT, mime::PLAIN) => requested_url + .parse::() .map_err(|err| { LabeledError::new(err.to_string()) .with_help("cannot parse") .with_label(format!("Cannot parse URL: {requested_url}"), span) })? - .path_segments() - .and_then(|mut segments| segments.next_back()) - .filter(|&name| !name.is_empty()) + .path() + .rsplit('/') + .next() + .filter(|name| !name.is_empty()) .and_then(|name| { PathBuf::from(name) .extension() @@ -1405,4 +1499,110 @@ mod test { assert!(expanded.is_absolute()); assert!(!expanded.to_string_lossy().contains('~')); } + + #[test] + fn test_http_parse_url_with_ipv6_zone_id() { + let call = Call::new(Span::test_data()); + let span = Span::test_data(); + let raw_url = Value::test_string("http://[fe80::13%eth0]"); + + let parsed = http_parse_url(&call, span, raw_url).unwrap(); + // The requested URL is preserved verbatim, including the zone ID, because + // it is what ureq uses to send the request. + assert_eq!(parsed.item, "http://[fe80::13%eth0]"); + } + + #[test] + fn test_http_parse_url_rejects_invalid_ipv6_with_zone_id() { + let call = Call::new(Span::test_data()); + let span = Span::test_data(); + // A URL with a zone ID but invalid characters after the bracket must be + // rejected, not silently rewritten (e.g. to localhost). + for bad in [ + "http://[fe80::13%eth0]foo", + "http://[fe80::13%eth0]:abc", + "http://[fe80::13%eth0", + ] { + let raw_url = Value::test_string(bad); + assert!( + http_parse_url(&call, span, raw_url).is_err(), + "expected {bad:?} to be rejected" + ); + } + } + + #[test] + fn test_authority_is_well_formed() { + let valid = [ + "http://example.com", + "http://example.com:8080", + "http://example.com:080", + "http://example.com:", + "http://example.com:0", + "http://example.com:65535", + "http://user:pass@example.com:8080/", + "http://[fe80::13%eth0]", + "http://[fe80::13%eth0]:8080", + "http://[fe80::13%eth0]:080", + "http://[fe80::13%eth0]:", + "http://[fe80::13%eth0]/a.txt", + "http://user:pass@[fe80::13%eth0]:8080/", + ]; + for url in valid { + let uri = url.parse::().unwrap(); + assert!( + authority_is_well_formed(&uri), + "expected {url:?} to be well formed" + ); + } + + let invalid = [ + "http://example.com:abc", + "http://example.com:99999", + "http://example.com:65536", + "http://[fe80::13%eth0]foo", + "http://[fe80::13%eth0]:abc", + "http://[fe80::13%eth0]:99999", + ]; + for url in invalid { + let uri = url.parse::().unwrap(); + assert!( + !authority_is_well_formed(&uri), + "expected {url:?} to be rejected" + ); + } + } + + #[test] + fn test_request_set_host_header_strips_zone_id() { + let request = ureq::Agent::config_builder() + .build() + .new_agent() + .get("http://[fe80::13%eth0]:8080/"); + let request = request_set_host_header("http://[fe80::13%eth0]:8080/", request); + + assert_eq!( + request + .headers_ref() + .and_then(|headers| headers.get("host")) + .map(|v| v.to_str().unwrap()), + Some("[fe80::13]:8080") + ); + } + + #[test] + fn test_request_set_host_header_leaves_other_urls_alone() { + let request = ureq::Agent::config_builder() + .build() + .new_agent() + .get("http://example.com:8080/"); + let request = request_set_host_header("http://example.com:8080/", request); + + assert!( + request + .headers_ref() + .and_then(|headers| headers.get("host")) + .is_none() + ); + } } diff --git a/crates/nu-command/src/network/http/delete.rs b/crates/nu-command/src/network/http/delete.rs index 7c9000505d..68abdf2917 100644 --- a/crates/nu-command/src/network/http/delete.rs +++ b/crates/nu-command/src/network/http/delete.rs @@ -2,8 +2,8 @@ use crate::network::http::client::{ HttpBody, RedirectMode, RequestFlags, RequestMetadata, add_unix_socket_flag, check_response_redirection, expand_unix_socket_path, http_client, http_client_pool, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, - request_add_custom_headers, request_handle_response, request_set_timeout, send_request, - send_request_no_body, + request_add_custom_headers, request_handle_response, request_set_host_header, + request_set_timeout, send_request, send_request_no_body, }; use nu_engine::command_prelude::*; @@ -222,7 +222,7 @@ fn helper( ) -> Result { let span = args.url.span(); let Spanned { - item: (requested_url, _), + item: requested_url, span: request_span, } = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -245,6 +245,7 @@ fn helper( request = request_set_timeout(args.timeout, request)?; request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; + request = request_set_host_header(&requested_url, request); let (response, request_headers) = match args.data { None => send_request_no_body(request, request_span, call.head, engine_state.signals()), diff --git a/crates/nu-command/src/network/http/get.rs b/crates/nu-command/src/network/http/get.rs index 6ecede1b0a..b40c1f7e9d 100644 --- a/crates/nu-command/src/network/http/get.rs +++ b/crates/nu-command/src/network/http/get.rs @@ -2,7 +2,7 @@ use crate::network::http::client::{ RequestFlags, RequestMetadata, add_unix_socket_flag, check_response_redirection, expand_unix_socket_path, http_client, http_client_pool, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, request_add_custom_headers, - request_handle_response, request_set_timeout, send_request_no_body, + request_handle_response, request_set_host_header, request_set_timeout, send_request_no_body, }; use nu_engine::command_prelude::*; @@ -199,7 +199,7 @@ fn helper( ) -> Result { let span = args.url.span(); let Spanned { - item: (requested_url, _), + item: requested_url, span: request_span, } = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -223,6 +223,7 @@ fn helper( request = request_set_timeout(args.timeout, request)?; request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; + request = request_set_host_header(&requested_url, request); let (response, request_headers) = send_request_no_body(request, request_span, call.head, engine_state.signals()); diff --git a/crates/nu-command/src/network/http/head.rs b/crates/nu-command/src/network/http/head.rs index fe502c6e04..2fcfe00d88 100644 --- a/crates/nu-command/src/network/http/head.rs +++ b/crates/nu-command/src/network/http/head.rs @@ -3,7 +3,7 @@ use crate::network::http::client::{ expand_unix_socket_path, extract_response_headers, handle_response_status, headers_to_nu, http_client, http_client_pool, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, request_add_custom_headers, request_handle_response, - request_set_timeout, send_request_no_body, + request_set_host_header, request_set_timeout, send_request_no_body, }; use nu_engine::command_prelude::*; use nu_protocol::Signals; @@ -176,7 +176,7 @@ fn helper( ) -> Result { let span = args.url.span(); let Spanned { - item: (requested_url, _), + item: requested_url, span: request_span, } = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -200,6 +200,7 @@ fn helper( request = request_set_timeout(args.timeout, request)?; request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; + request = request_set_host_header(&requested_url, request); let (response, request_headers) = send_request_no_body(request, request_span, call.head, signals); diff --git a/crates/nu-command/src/network/http/options.rs b/crates/nu-command/src/network/http/options.rs index 36f84488fb..e78d75eec6 100644 --- a/crates/nu-command/src/network/http/options.rs +++ b/crates/nu-command/src/network/http/options.rs @@ -3,7 +3,7 @@ use crate::network::http::client::{ discard_response_body, expand_unix_socket_path, extract_response_headers, handle_response_status, headers_to_nu, http_client, http_client_pool, http_parse_url, request_add_authorization_header, request_add_custom_headers, request_handle_response, - request_set_timeout, send_request_no_body, + request_set_host_header, request_set_timeout, send_request_no_body, }; use nu_engine::command_prelude::*; @@ -172,7 +172,7 @@ fn helper( ) -> Result { let span = args.url.span(); let Spanned { - item: (requested_url, _), + item: requested_url, span: request_span, } = http_parse_url(call, span, args.url)?; let redirect_mode = RedirectMode::Follow; @@ -195,6 +195,7 @@ fn helper( request = request_set_timeout(args.timeout, request)?; request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; + request = request_set_host_header(&requested_url, request); let (response, request_headers) = send_request_no_body(request, request_span, call.head, engine_state.signals()); diff --git a/crates/nu-command/src/network/http/patch.rs b/crates/nu-command/src/network/http/patch.rs index 47b3abd4bc..5b44158065 100644 --- a/crates/nu-command/src/network/http/patch.rs +++ b/crates/nu-command/src/network/http/patch.rs @@ -2,7 +2,7 @@ use crate::network::http::client::{ HttpBody, RequestFlags, RequestMetadata, add_unix_socket_flag, check_response_redirection, expand_unix_socket_path, http_client, http_client_pool, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, request_add_custom_headers, - request_handle_response, request_set_timeout, send_request, + request_handle_response, request_set_host_header, request_set_timeout, send_request, }; use nu_engine::command_prelude::*; use nu_protocol::shell_error::generic::GenericError; @@ -223,7 +223,7 @@ fn helper( ) -> Result { let span = args.url.span(); let Spanned { - item: (requested_url, _), + item: requested_url, span: request_span, } = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -247,6 +247,7 @@ fn helper( request = request_set_timeout(args.timeout, request)?; request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; + request = request_set_host_header(&requested_url, request); let (response, request_headers) = send_request( engine_state, diff --git a/crates/nu-command/src/network/http/post.rs b/crates/nu-command/src/network/http/post.rs index 872b6571d4..6e8896d6f5 100644 --- a/crates/nu-command/src/network/http/post.rs +++ b/crates/nu-command/src/network/http/post.rs @@ -2,7 +2,7 @@ use crate::network::http::client::{ HttpBody, RequestFlags, RequestMetadata, add_unix_socket_flag, check_response_redirection, expand_unix_socket_path, http_client, http_client_pool, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, request_add_custom_headers, - request_handle_response, request_set_timeout, send_request, + request_handle_response, request_set_host_header, request_set_timeout, send_request, }; use nu_engine::command_prelude::*; use nu_protocol::shell_error::generic::GenericError; @@ -247,7 +247,7 @@ fn helper( ) -> Result { let span = args.url.span(); let Spanned { - item: (requested_url, _), + item: requested_url, span: request_span, } = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -271,6 +271,7 @@ fn helper( request = request_set_timeout(args.timeout, request)?; request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; + request = request_set_host_header(&requested_url, request); let (response, request_headers) = send_request( engine_state, diff --git a/crates/nu-command/src/network/http/put.rs b/crates/nu-command/src/network/http/put.rs index 23dc702548..f3cc18458f 100644 --- a/crates/nu-command/src/network/http/put.rs +++ b/crates/nu-command/src/network/http/put.rs @@ -2,7 +2,7 @@ use crate::network::http::client::{ HttpBody, RequestFlags, RequestMetadata, add_unix_socket_flag, check_response_redirection, expand_unix_socket_path, http_client, http_client_pool, http_parse_redirect_mode, http_parse_url, request_add_authorization_header, request_add_custom_headers, - request_handle_response, request_set_timeout, send_request, + request_handle_response, request_set_host_header, request_set_timeout, send_request, }; use nu_engine::command_prelude::*; use nu_protocol::shell_error::generic::GenericError; @@ -228,7 +228,7 @@ fn helper( ) -> Result { let span = args.url.span(); let Spanned { - item: (requested_url, _), + item: requested_url, span: request_span, } = http_parse_url(call, span, args.url)?; let redirect_mode = http_parse_redirect_mode(args.redirect)?; @@ -252,6 +252,7 @@ fn helper( request = request_set_timeout(args.timeout, request)?; request = request_add_authorization_header(args.user, args.password, request); request = request_add_custom_headers(args.headers, request)?; + request = request_set_host_header(&requested_url, request); let (response, request_headers) = send_request( engine_state, diff --git a/crates/nu-command/src/network/http/resolver.rs b/crates/nu-command/src/network/http/resolver.rs index b58dc3e307..fe4fb8dc73 100644 --- a/crates/nu-command/src/network/http/resolver.rs +++ b/crates/nu-command/src/network/http/resolver.rs @@ -20,7 +20,18 @@ impl Resolver for DnsLookupResolver { config: &Config, _timeout: NextTimeout, ) -> Result { - let host = uri.host(); + // `uri.host()` returns IPv6 addresses wrapped in square brackets (e.g. + // `[fe80::13%eth0]`), which `getaddrinfo` cannot resolve. Strip the brackets + // so that zone IDs (e.g. `%eth0`) are handled correctly by the resolver. + // RFC 6874 encodes the zone ID separator as `%25`; `getaddrinfo` expects a + // literal `%`, so decode it as well. + let host = uri.host().map(|h| { + let h = h + .strip_prefix('[') + .and_then(|inner| inner.strip_suffix(']')) + .unwrap_or(h); + h.replacen("%25", "%", 1) + }); // Determine the port: use explicit port if provided, otherwise derive from scheme let port = uri.port_u16().unwrap_or_else(|| match uri.scheme_str() { Some("https") => 443, @@ -30,7 +41,7 @@ impl Resolver for DnsLookupResolver { // Pass None as service to avoid "Service not supported for this socket type" errors // in certain environments (e.g., Docker containers on some Linux distributions). // We'll set the port manually on each resolved address. - let addr_info_iter = dns_lookup::getaddrinfo(host, None, None) + let addr_info_iter = dns_lookup::getaddrinfo(host.as_deref(), None, None) .map_err(|err| ureq::Error::Other(Box::new(LookupError(err))))?; let ip_family = config.ip_family(); diff --git a/crates/nu-command/tests/commands/network/http/get.rs b/crates/nu-command/tests/commands/network/http/get.rs index 3cb5c92392..b0ec5cd816 100644 --- a/crates/nu-command/tests/commands/network/http/get.rs +++ b/crates/nu-command/tests/commands/network/http/get.rs @@ -1,4 +1,4 @@ -use mockito::Server; +use mockito::{Server, ServerOpts}; use nu_protocol::shell_error; use nu_test_support::prelude::*; use std::{thread, time::Duration}; @@ -168,6 +168,27 @@ fn http_get_follows_redirect() -> Result { .expect_value_eq("bar") } +// IPv6 literals are wrapped in brackets in URLs (e.g. `http://[::1]:8080/`). +// `url::Url` handles the brackets, but the resolver used to pass the bracketed +// host to `getaddrinfo`, which cannot resolve it. This exercises the resolver +// and the `text/plain` extension lookup with a real request to an IPv6 literal. +#[test] +fn http_get_ipv6_literal() -> Result { + let mut server = Server::new_with_opts(ServerOpts { + host: "::1", + ..Default::default() + }); + + let _mock = server + .mock("GET", "/a.txt") + .with_header("content-type", "text/plain") + .with_body("hello") + .create(); + + let code = format!("http get {url}/a.txt", url = server.url()); + test().run(code).expect_value_eq("hello") +} + #[test] fn http_get_redirect_mode_manual() -> Result { let mut server = Server::new();