//! CLI argument parsing. use clap::Parser; #[derive(Parser)] #[command( name = "servo-fetch", version, about = "A browser engine in a binary — fetch, render, and extract web content.", after_help = "\ Examples: servo-fetch https://example.com Readable Markdown (default) servo-fetch https://example.com --json Structured JSON servo-fetch URL1 URL2 URL3 Parallel batch fetch servo-fetch URL1 URL2 --json Parallel batch (NDJSON) servo-fetch https://example.com --screenshot page.png servo-fetch https://example.com --js \"document.title\" servo-fetch https://example.com -t 60 Custom timeout (seconds) servo-fetch https://example.com --selector article Extract specific section servo-fetch mcp Start MCP server (stdio)" )] pub(crate) struct Cli { #[command(subcommand)] pub command: Option, /// URLs to fetch (one or more) #[arg(num_args = 1..)] pub urls: Vec, /// Output as structured JSON (NDJSON when multiple URLs) #[arg(long, conflicts_with_all = ["screenshot", "js"])] pub json: bool, /// Save screenshot as PNG (single URL only) #[arg(long, value_name = "FILE", conflicts_with_all = ["json", "js"])] pub screenshot: Option, /// Capture the full scrollable page instead of just the viewport. #[arg(long, requires = "screenshot")] pub full_page: bool, /// Execute JavaScript and print the result (single URL only) #[arg(long, value_name = "EXPR", conflicts_with_all = ["json", "screenshot"])] pub js: Option, /// Timeout in seconds for page load #[arg(short = 't', long, default_value_t = 30, value_parser = clap::value_parser!(u64).range(1..), value_name = "SECS")] pub timeout: u64, /// Extra wait in ms after the `load` event, for SPAs that keep hydrating. #[arg(long, default_value_t = 0, value_parser = clap::value_parser!(u64).range(0..=10_000), value_name = "MS")] pub settle: u64, /// CSS selector to extract a specific section #[arg(long, value_name = "CSS")] pub selector: Option, /// Output raw HTML or plain text instead of Readability extraction #[arg(long, value_name = "MODE", value_enum, conflicts_with_all = ["json", "screenshot", "js", "selector"])] pub raw: Option, } /// Raw output mode. #[derive(Debug, Clone, Copy, clap::ValueEnum)] pub(crate) enum RawMode { /// Raw HTML Html, /// Plain text (document.body.innerText) Text, } /// Available subcommands. #[derive(clap::Subcommand)] pub(crate) enum Command { /// Start MCP server (stdio transport by default, or HTTP with --port) Mcp { /// Port for Streamable HTTP transport. Omit for stdio. #[arg(long, value_name = "PORT")] port: Option, }, /// Crawl a website by following links (BFS). Respects robots.txt. Crawl { /// Starting URL to crawl url: String, /// Maximum number of pages to crawl #[arg(long, default_value_t = 50, value_name = "N")] limit: usize, /// Maximum link depth from the seed URL #[arg(long, default_value_t = 3, value_name = "N")] max_depth: usize, /// URL path glob patterns to include (e.g. "/docs/**") #[arg(long, value_name = "GLOB")] include: Vec, /// URL path glob patterns to exclude (e.g. "/docs/archive/**") #[arg(long, value_name = "GLOB")] exclude: Vec, /// Output as NDJSON #[arg(long)] json: bool, /// CSS selector to extract a specific section per page #[arg(long, value_name = "CSS")] selector: Option, /// Timeout in seconds per page #[arg(short = 't', long, default_value_t = 30, value_parser = clap::value_parser!(u64).range(1..), value_name = "SECS")] timeout: u64, /// Extra wait in ms after load event per page #[arg(long, default_value_t = 0, value_parser = clap::value_parser!(u64).range(0..=10_000), value_name = "MS")] settle: u64, }, } /// Validate and sanitize a URL for fetching. Forwards to [`crate::net::validate_url`]. pub(crate) fn validate_url(input: &str) -> anyhow::Result { crate::net::validate_url(input) } #[cfg(test)] mod tests { use super::*; #[test] fn accepts_https() { assert!(validate_url("https://example.com").is_ok()); } #[test] fn accepts_http() { assert!(validate_url("http://example.com").is_ok()); } #[test] fn rejects_file_scheme() { let err = validate_url("file:///etc/passwd").unwrap_err(); assert!(err.to_string().contains("not allowed")); } #[test] fn rejects_javascript_scheme() { let err = validate_url("javascript:alert(1)").unwrap_err(); assert!(err.to_string().contains("not allowed")); } #[test] fn strips_credentials() { let url = validate_url("https://user:pass@example.com").unwrap(); assert!(url.username().is_empty()); assert!(url.password().is_none()); } #[test] fn rejects_invalid_url() { assert!(validate_url("not a url").is_err()); } #[test] fn rejects_private_host_via_url() { assert!(validate_url("http://127.0.0.1/").is_err()); } #[test] fn rejects_hex_ip() { // url::Url::parse normalizes 0x7f000001 → 127.0.0.1 assert!(validate_url("http://0x7f000001/").is_err()); } #[test] fn rejects_decimal_ip() { assert!(validate_url("http://2130706433/").is_err()); } #[test] fn rejects_data_scheme() { assert!( validate_url("data:text/html,

hi

") .unwrap_err() .to_string() .contains("not allowed") ); } #[test] fn raw_mode_from_str() { use clap::ValueEnum; assert!(RawMode::from_str("html", true).is_ok()); assert!(RawMode::from_str("text", true).is_ok()); assert!(RawMode::from_str("xml", true).is_err()); } } #[cfg(test)] mod cli_tests { use assert_cmd::Command; use predicates::prelude::*; fn servo_fetch() -> Command { Command::cargo_bin("servo-fetch").expect("binary exists") } #[test] fn conflicting_json_and_screenshot() { servo_fetch() .args(["--json", "--screenshot", "out.png", "https://example.com"]) .assert() .failure() .stderr(predicate::str::contains("cannot be used with")); } #[test] fn settle_rejects_out_of_range() { servo_fetch() .args(["--settle", "10001", "https://example.com"]) .assert() .failure() .stderr(predicate::str::contains("invalid value")); } #[test] fn raw_conflicts_with_json() { servo_fetch() .args(["--raw", "html", "--json", "https://example.com"]) .assert() .failure() .stderr(predicate::str::contains("cannot be used with")); } #[test] fn full_page_requires_screenshot() { servo_fetch() .args(["--full-page", "https://example.com"]) .assert() .failure() .stderr(predicate::str::contains("--screenshot")); } }