allaun/Research-Stack
1
0
Fork 0
mirror of https://github.com/allaunthefox/Research-Stack.git synced 2026-08-10 08:30:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
Research-Stack/2-Search-Space/search/stract/crates/robotstxt/src/lib.rs

1653 lines
43 KiB
Rust
Raw Blame History

// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! A robots.txt parser and matcher compliant with [RFC 9309](https://www.rfc-editor.org/rfc/rfc9309.html)
//! and [Google's Robots.txt parser](https://github.com/google/robotstxt) (with some extensions).
const MAX_CHAR_LIMIT_DEFAULT: usize = 512_000; // 512 KB
mod parser;
mod pattern;
use std::time::Duration;
use itertools::Itertools;
use pattern::Pattern;
use url::Url;
use crate::parser::Line;
#[derive(Debug, PartialEq, Eq)]
enum Directive {
Allow,
Disallow,
}
impl Ord for Directive {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
match (self, other) {
(Self::Allow, Self::Disallow) => std::cmp::Ordering::Less,
(Self::Disallow, Self::Allow) => std::cmp::Ordering::Greater,
_ => std::cmp::Ordering::Equal,
}
}
}
impl PartialOrd for Directive {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
#[derive(Debug, PartialEq, Eq)]
struct Rule {
pattern: Pattern,
directive: Directive,
}
impl Ord for Rule {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.pattern
.cmp(&other.pattern)
.then(self.directive.cmp(&other.directive))
}
}
impl PartialOrd for Rule {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
#[derive(Debug, Clone, Copy)]
pub struct Params {
pub char_limit: usize,
}
impl Default for Params {
fn default() -> Self {
Self {
char_limit: MAX_CHAR_LIMIT_DEFAULT,
}
}
}
#[derive(Debug)]
pub struct Robots {
rules: Vec<Rule>,
crawl_delay: Option<f32>,
sitemaps: Vec<String>,
}
impl Robots {
fn is_valid_user_agent(useragent: &str) -> bool {
useragent
.chars()
.all(|c| c.is_ascii_alphabetic() || c == '-' || c == '_')
&& !useragent.is_empty()
}
pub fn parse_with_params(
useragent: &str,
robotstxt: &str,
params: Params,
) -> Result<Self, anyhow::Error> {
if !Self::is_valid_user_agent(useragent) {
return Err(anyhow::anyhow!("Invalid user agent"));
}
let robotstxt = robotstxt
.chars()
.take(params.char_limit)
.collect::<String>();
let robotstxt = robotstxt.replace('\0', "\n");
let (_, lines) = parser::parse(&robotstxt).map_err(|e| anyhow::anyhow!(e.to_string()))?;
let mut useragent = useragent.to_lowercase();
if !lines.iter().any(|line| {
if let Line::UserAgent(agents) = line {
agents.iter().any(|agent| {
agent
.chars()
.zip(useragent.chars())
.all(|(c1, c2)| c1.to_ascii_lowercase() == c2)
})
} else {
false
}
}) {
useragent = "*".to_string();
}
let mut rules = Vec::new();
let mut crawl_delay = None;
let mut sitemaps = Vec::new();
let mut idx = 0;
let mut useragent_lines = 0;
while idx < lines.len() {
let line = &lines[idx];
idx += 1;
if let Line::UserAgent(agents) = &line {
useragent_lines += 1;
if agents.iter().any(|agent| {
agent
.chars()
.zip(useragent.chars())
.all(|(c1, c2)| c1.to_ascii_lowercase() == c2)
}) {
let mut has_captured_directive = false;
while idx < lines.len() {
let line = &lines[idx];
idx += 1;
match line {
Line::Allow(path) => {
has_captured_directive = true;
if !path.is_empty() {
rules.push(Rule {
pattern: Pattern::new(path),
directive: Directive::Allow,
});
}
}
Line::Disallow(path) => {
has_captured_directive = true;
if !path.is_empty() {
rules.push(Rule {
pattern: Pattern::new(path),
directive: Directive::Disallow,
});
}
}
Line::UserAgent(_) if has_captured_directive => {
idx -= 1;
break;
}
Line::Sitemap(sitemap) => {
sitemaps.push(sitemap.to_string());
}
Line::CrawlDelay(Some(delay)) => {
has_captured_directive = true;
crawl_delay = Some(*delay);
}
_ => {}
}
}
}
} else if useragent_lines == 0 {
// add preceding rules as global rules
match line {
Line::Allow(path) => {
rules.push(Rule {
pattern: Pattern::new(path),
directive: Directive::Allow,
});
}
Line::Disallow(path) => {
rules.push(Rule {
pattern: Pattern::new(path),
directive: Directive::Disallow,
});
}
Line::CrawlDelay(Some(delay)) => {
crawl_delay = Some(*delay);
}
_ => {}
}
}
if let Line::Sitemap(url) = line {
sitemaps.push(url.to_string());
}
}
Ok(Self {
rules,
crawl_delay,
sitemaps,
})
}
pub fn parse(useragent: &str, robotstxt: &str) -> Result<Self, anyhow::Error> {
Self::parse_with_params(useragent, robotstxt, Params::default())
}
pub fn is_allowed(&self, url: &Url) -> bool {
let path = &Self::prepare_path(url);
self.is_path_allowed(path)
}
fn prepare_path(url: &Url) -> String {
let path = url.path();
// replace multiple slashes with a single slash
let path = path
.chars()
.coalesce(|a, b| {
if a == '/' && b == '/' {
Ok(a)
} else {
Err((a, b))
}
})
.collect::<String>();
if let Some(query) = url.query() {
format!("{}?{}", path, query)
} else {
path
}
}
fn is_precise_path_allowed(&self, path: &str) -> bool {
let mut path = path.to_string();
if path.is_empty() {
path = "/".to_string();
}
if path == "/robots.txt" {
return true;
}
let mut matches: Vec<_> = self
.rules
.iter()
.filter(|rule| rule.pattern.matches(&path))
.collect();
matches.sort();
matches
.first()
.map(|rule| rule.directive == Directive::Allow)
.unwrap_or(true)
}
pub fn is_path_allowed(&self, path: &str) -> bool {
let res = self.is_precise_path_allowed(path);
if !res && path.ends_with('/') {
self.is_precise_path_allowed(format!("{}index.html", path).as_str())
} else {
res
}
}
pub fn crawl_delay(&self) -> Option<Duration> {
self.crawl_delay.map(Duration::from_secs_f32)
}
pub fn sitemaps(&self) -> &[String] {
&self.sitemaps
}
}
#[cfg(test)]
mod tests {
use super::*;
fn is_user_agent_allowed(robots_txt: &str, user_agent: &str, url: &str) -> bool {
let robots = Robots::parse(user_agent, robots_txt).unwrap();
if let Ok(url) = Url::parse(url) {
robots.is_allowed(&url)
} else {
robots.is_path_allowed(url)
}
}
fn is_valid_user_agent(user_agent: &str) -> bool {
Robots::parse(user_agent, "").is_ok()
}
fn test_path(url: &str, expected: &str) {
let url = Url::parse(url)
.unwrap_or_else(|_| Url::parse(&format!("http://foo.bar/{}", url)).unwrap());
assert_eq!(Robots::prepare_path(&url), expected);
}
fn test_escape(url: &str, expected: &str) {
assert_eq!(pattern::percent_encode(url), expected);
}
// Tese test are based on the tests from
// https://github.com/google/robotstxt/blob/455b1583103d13ad88fe526bc058d6b9f3309215/robots_test.cc#L399
#[test]
fn simple() {
let robotstxt = r#"
user-agent: FooBot
disallow: /
"#;
assert!(is_user_agent_allowed("", "FooBot", ""));
assert!(is_user_agent_allowed(robotstxt, "BarBot", ""));
assert!(!is_user_agent_allowed(robotstxt, "FooBot", ""));
assert!(is_user_agent_allowed("", "FooBot", ""));
}
#[test]
fn test_line_syntax_line() {
let robotstxt_correct = r#"
user-agent: FooBot
disallow: /
"#;
let robotstxt_incorrect = r#"
fooL FooBot
bar: /
"#;
let robotstxt_incorrect_accepted = r#"
user-agent: FooBot
disallow /
"#;
let url = "http://foo.bar/x/y";
assert!(!is_user_agent_allowed(robotstxt_correct, "FooBot", url));
assert!(is_user_agent_allowed(robotstxt_incorrect, "FooBot", url));
assert!(!is_user_agent_allowed(
robotstxt_incorrect_accepted,
"FooBot",
url
));
}
#[test]
fn test_line_syntax_groups() {
let robotstxt = r#"
allow: /foo/bar/
user-agent: FooBot
disallow: /
allow: /x/
user-agent: BarBot
disallow: /
allow: /y/
allow: /w/
user-agent: BazBot
user-agent: FooBot
allow: /z/
disallow: /
"#;
let w = "http://foo.bar/w/a";
let x = "http://foo.bar/x/b";
let y = "http://foo.bar/y/c";
let z = "http://foo.bar/z/d";
assert!(is_user_agent_allowed(robotstxt, "FooBot", x));
assert!(is_user_agent_allowed(robotstxt, "FooBot", z));
assert!(!is_user_agent_allowed(robotstxt, "FooBot", y));
assert!(is_user_agent_allowed(robotstxt, "BarBot", y));
assert!(is_user_agent_allowed(robotstxt, "BarBot", w));
assert!(!is_user_agent_allowed(robotstxt, "BarBot", z));
assert!(is_user_agent_allowed(robotstxt, "BazBot", z));
// // lines with rules outside groups are ignored
// let foo = "http://foo.bar/foo/bar/";
// assert!(!is_user_agent_allowed(robotstxt, "FooBot", foo));
// assert!(!is_user_agent_allowed(robotstxt, "BarBot", foo));
// assert!(!is_user_agent_allowed(robotstxt, "BazBot", foo));
}
#[test]
fn test_line_syntax_groups_other_rules() {
let robotstxt = r#"
User-agent: BarBot
Sitemap: https://foo.bar/sitemap
User-agent: *
Disallow: /
"#;
let url = "http://foo.bar/";
assert!(!is_user_agent_allowed(robotstxt, "FooBot", url));
assert!(!is_user_agent_allowed(robotstxt, "BarBot", url));
}
#[test]
fn test_repl_line_names_case_insensitive() {
let robotstxt_upper = r#"
USER-AGENT: FooBot
ALLOW: /x/
DISALLOW: /
"#;
let robotstxt_lower = r#"
user-agent: FooBot
allow: /x/
disallow: /
"#;
let robotstxt_mixed = r#"
uSeR-aGeNt: FooBot
AlLoW: /x/
dIsAlLoW: /
"#;
let url = "http://foo.bar/x/y";
assert!(is_user_agent_allowed(robotstxt_upper, "FooBot", url));
assert!(is_user_agent_allowed(robotstxt_lower, "FooBot", url));
assert!(is_user_agent_allowed(robotstxt_mixed, "FooBot", url));
let url = "http://foo.bar/a/b";
assert!(!is_user_agent_allowed(robotstxt_upper, "FooBot", url));
assert!(!is_user_agent_allowed(robotstxt_lower, "FooBot", url));
assert!(!is_user_agent_allowed(robotstxt_mixed, "FooBot", url));
}
#[test]
fn test_verify_valid_user_agents() {
assert!(is_valid_user_agent("FooBot"));
assert!(is_valid_user_agent("Foobot-Bar"));
assert!(is_valid_user_agent("Foo_Bar"));
assert!(!is_valid_user_agent(""));
assert!(!is_valid_user_agent(" "));
assert!(!is_valid_user_agent("🦀"));
assert!(!is_valid_user_agent("Foobot*"));
assert!(!is_valid_user_agent(" FooBot "));
assert!(!is_valid_user_agent("FooBot/1.0"));
assert!(!is_valid_user_agent("Foobot Bar"));
}
#[test]
fn test_user_agent_case_insensitive() {
let robotstxt_upper = r#"
User-Agent: FOOBAR
Allow: /x/
Disallow: /
"#;
let robotstxt_lower = r#"
User-Agent: foobar
Allow: /x/
Disallow: /
"#;
let robotstxt_mixed = r#"
User-Agent: fOoBaR
Allow: /x/
Disallow: /
"#;
let url_allowed = "http://foo.bar/x/y";
let url_disallowed = "http://foo.bar/a/b";
assert!(is_user_agent_allowed(
robotstxt_upper,
"FooBar",
url_allowed
));
assert!(is_user_agent_allowed(
robotstxt_lower,
"FooBar",
url_allowed
));
assert!(is_user_agent_allowed(
robotstxt_mixed,
"FooBar",
url_allowed
));
assert!(!is_user_agent_allowed(
robotstxt_upper,
"FooBar",
url_disallowed
));
assert!(!is_user_agent_allowed(
robotstxt_lower,
"FooBar",
url_disallowed
));
assert!(!is_user_agent_allowed(
robotstxt_mixed,
"FooBar",
url_disallowed
));
assert!(is_user_agent_allowed(
robotstxt_upper,
"foobar",
url_allowed
));
assert!(is_user_agent_allowed(
robotstxt_lower,
"foobar",
url_allowed
));
assert!(is_user_agent_allowed(
robotstxt_mixed,
"foobar",
url_allowed
));
assert!(!is_user_agent_allowed(
robotstxt_upper,
"foobar",
url_disallowed
));
assert!(!is_user_agent_allowed(
robotstxt_lower,
"foobar",
url_disallowed
));
assert!(!is_user_agent_allowed(
robotstxt_mixed,
"foobar",
url_disallowed
));
}
#[test]
fn test_specific_user_agent() {
let robotstxt = r#"
User-Agent: FooBot
Allow: /
User-Agent: *
Disallow: /
"#;
let url = "http://foo.bar/x/y";
assert!(is_user_agent_allowed(robotstxt, "FooBot", url));
assert!(!is_user_agent_allowed(robotstxt, "BarBot", url));
}
// this is the test from google
//
// #[test]
// fn test_accept_user_agent_up_to_first_space() {
// let robotstxt = r#"
// User-Agent: *
// Disallow: /
// User-Agent: Foo Bar
// Allow: /x/
// Disallow: /
// "#;
//
// let url = "http://foo.bar/x/y";
//
// assert!(is_user_agent_allowed(robotstxt, "Foo", url));
// assert!(!is_user_agent_allowed(robotstxt, "Foo Bar", url));
// }
//
// yet I think it makes more sense to allow the user agent "Foo" and "Bar" to access the url
#[test]
fn test_accept_user_agent_with_space() {
let robotstxt = r#"
User-Agent: *
Disallow: /
User-Agent: Foo Bar
Allow: /x/
Disallow: /
"#;
let url = "http://foo.bar/x/y";
assert!(is_user_agent_allowed(robotstxt, "Foo", url));
assert!(is_user_agent_allowed(robotstxt, "Bar", url));
assert!(!is_user_agent_allowed(robotstxt, "Baz", url));
}
#[test]
fn test_global_groups_secondary() {
let robotstxt_empty = "";
let robotstxt_global = r#"
user-agent: *
allow: /
user-agent: FooBot
disallow: /
"#;
let robotstxt_only_specific = r#"
user-agent: FooBot
allow: /
user-agent: BarBot
disallow: /
user-agent: BazBot
disallow: /
"#;
let url = "http://foo.bar/x/y";
assert!(is_user_agent_allowed(robotstxt_empty, "FooBot", url));
assert!(!is_user_agent_allowed(robotstxt_global, "FooBot", url));
assert!(is_user_agent_allowed(robotstxt_global, "BarBot", url));
assert!(is_user_agent_allowed(
robotstxt_only_specific,
"QuxBot",
url
));
}
#[test]
fn test_allow_disallow_value_case_sensitive() {
let robotstxt_lower = r#"
user-agent: FooBot
disallow: /x/
"#;
let robotstxt_upper = r#"
user-agent: FooBot
disallow: /X/
"#;
let url = "http://foo.bar/x/y";
assert!(!is_user_agent_allowed(robotstxt_lower, "FooBot", url));
assert!(is_user_agent_allowed(robotstxt_upper, "FooBot", url));
}
#[test]
fn test_longest_match() {
let url = "http://foo.bar/x/page.html";
let robotstxt = r#"
user-agent: FooBot
disallow: /x/page.html
allow: /x/
"#;
assert!(!is_user_agent_allowed(robotstxt, "FooBot", url));
let robotstxt = r#"
user-agent: FooBot
allow: /x/page.html
disallow: /x/
"#;
assert!(is_user_agent_allowed(robotstxt, "FooBot", url));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/x/"
));
let robotstxt = r#"
user-agent: FooBot
disallow:
allow:
"#;
assert!(is_user_agent_allowed(robotstxt, "FooBot", url));
let robotstxt = r#"
user-agent: FooBot
disallow: /
allow: /
"#;
assert!(is_user_agent_allowed(robotstxt, "FooBot", url));
let robotstxt = r#"
user-agent: FooBot
disallow: /x
allow: /x/
"#;
let url_a = "http://foo.bar/x";
let url_b = "http://foo.bar/x/";
assert!(!is_user_agent_allowed(robotstxt, "FooBot", url_a));
assert!(is_user_agent_allowed(robotstxt, "FooBot", url_b));
let robotstxt = r#"
user-agent: FooBot
disallow: /x/page.html
allow: /x/page.html
"#;
assert!(is_user_agent_allowed(robotstxt, "FooBot", url));
let robotstxt = r#"
user-agent: FooBot
allow: /page
disallow: /*.html
"#;
let url_a = "http://foo.bar/page";
let url_b = "http://foo.bar/page.html";
assert!(is_user_agent_allowed(robotstxt, "FooBot", url_a));
assert!(!is_user_agent_allowed(robotstxt, "FooBot", url_b));
let robotstxt = r#"
user-agent: FooBot
allow: /x/page.
disallow: /*.html
"#;
assert!(is_user_agent_allowed(robotstxt, "FooBot", url));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/x/y.html"
));
let robotstxt = r#"
User-agent: *
Disallow: /x/
User-agent: FooBot
Disallow: /y/
"#;
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/x/page"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/y/page"
));
}
#[test]
fn test_encoding() {
let robotstxt = r#"
User-agent: FooBot
Disallow: /
Allow: /foo/bar?qux=taz&baz=http://foo.bar?tar&par
"#;
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar?qux=taz&baz=http://foo.bar?tar&par"
));
let robotstxt = r#"
User-agent: FooBot
Disallow: /
Allow: /foo/bar/ツ
"#;
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar/%E3%83%84"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar/ツ"
));
let robotstxt = r#"
User-agent: FooBot
Disallow: /
Allow: /foo/bar/%E3%83%84
"#;
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar/%E3%83%84"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar/ツ"
));
let robotstxt = r#"
User-agent: FooBot
Disallow: /
Allow: /foo/bar/%62%61%7A
"#;
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar/baz"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar/%62%61%7A"
));
let robotstxt = r#"
User-agent: FooBot
Disallow: /
Allow: /path/file-with-a-%2A
"#;
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/path/file-with-a-%2A"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/path/file-with-a-*"
));
}
#[test]
fn test_special_characters() {
let robotstxt = r#"
User-agent: FooBot
Disallow: /foo/bar/quz
Allow: /foo/*/quz
"#;
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar/quz"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/quz"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo//quz"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bax/quz"
));
let robotstxt = r#"
User-agent: FooBot
Disallow: /foo/bar$
Allow: /foo/bar/qux
"#;
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar/qux"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar/"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar/baz"
));
let robotstxt = r#"
User-agent: FooBot
# Disallow: /
Disallow: /foo/quz#qux
Allow: /
"#;
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/bar"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/foo/quz"
));
}
#[test]
fn test_index_html_is_directory() {
let robotstxt = r#"
User-agent: *
Allow: /allowed-slash/index.html
Disallow: /
"#;
assert!(is_user_agent_allowed(
robotstxt,
"foobot",
"http://foo.bar/allowed-slash/"
));
assert!(!is_user_agent_allowed(
robotstxt,
"foobot",
"http://foo.bar/allowed-slash/index.htm"
));
assert!(is_user_agent_allowed(
robotstxt,
"foobot",
"http://foo.bar/allowed-slash/index.html"
));
assert!(!is_user_agent_allowed(
robotstxt,
"foobot",
"http://foo.bar/another-url/"
));
}
#[test]
fn test_google_documentation() {
let robotstxt = r#"
user-agent: FooBot
disallow: /
allow: /fish
"#;
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/bar"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish.html"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish/salmon.html"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fishheads"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fishheads/yummy.html"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish.html?id=anything"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/Fish.asp"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/catfish"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/?id=fish"
));
let robotstxt = r#"
user-agent: FooBot
disallow: /
allow: /fish*
"#;
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/bar"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish.html"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish/salmon.html"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fishheads"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fishheads/yummy.html"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish.html?id=anything"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/Fish.asp"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/catfish"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/?id=fish"
));
// "/fish/" does not equal "/fish"
let robotstxt = r#"
user-agent: FooBot
disallow: /
allow: /fish/
"#;
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/bar/"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish/"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish/salmon"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish/?salmon"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish/salmon.html"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish/?id=anything"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish.html"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/Fish/Salmon.html"
));
// "*.php"
let robotstxt = r#"
user-agent: FooBot
disallow: /
allow: /*.php
"#;
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/bar"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/filename.php"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/folder/filename.php"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/folder/filename.php?parameters"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar//folder/any.php.file.html"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/filename.php/"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/index?f=filename.php/"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/php/"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/index?php"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/windows.PHP"
));
// "/*.php$"
let robotstxt = r#"
user-agent: FooBot
disallow: /
allow: /*.php$
"#;
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/bar"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/filename.php"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/folder/filename.php"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/filename.php?parameters"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/filename.php/"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/filename.php5"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/php/"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/filename?php"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/aaaphpaaa"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar//windows.PHP"
));
// "/fish*.php"
let robotstxt = r#"
user-agent: FooBot
disallow: /
allow: /fish*.php
"#;
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/bar"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fish.php"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fishheads/catfish.php?parameters"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/fishheads/Fish.PHP"
));
// section "order of precedence for group-member records"
assert!(is_user_agent_allowed(
r#"
user-agent: FooBot
allow: /p
disallow: /
"#,
"FooBot",
"http://foo.bar/page"
));
assert!(is_user_agent_allowed(
r#"
user-agent: FooBot
allow: /folder
disallow: /folder
"#,
"FooBot",
"http://foo.bar/folder/page"
));
assert!(!is_user_agent_allowed(
r#"
user-agent: FooBot
allow: /page
disallow: /*.htm
"#,
"FooBot",
"http://foo.bar/page.htm"
));
let robotstxt = r#"
user-agent: FooBot
allow: /$
disallow: /
"#;
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/page.html"
));
}
#[test]
fn test_comma_separated_user_agents() {
// this is not part of the spec, but it is a reasonable
// assumption made by some websites and should therefore be respected
let robotstxt = r#"
User-Agent: FooBot, BarBot
Disallow: /
User-Agent: BazBot
Allow: /
"#;
let url = "http://foo.bar/x/y";
assert!(!is_user_agent_allowed(robotstxt, "FooBot", url));
assert!(!is_user_agent_allowed(robotstxt, "BarBot", url));
assert!(is_user_agent_allowed(robotstxt, "BazBot", url));
}
#[test]
fn test_non_standard_line_example_sitemap() {
let robotstxt = r#"
user-agent: FooBot
allow: /some/path
user-agent: BarBot
Sitemap: http://foo.bar/sitemap.xml
"#;
let robots = Robots::parse("FooBot", robotstxt).unwrap();
assert_eq!(robots.sitemaps(), &["http://foo.bar/sitemap.xml"]);
let robotstxt = r#"
sitemap: http://foo.bar/sitemap.xml
user-agent: FooBot
allow: /some/path
user-agent: BarBot
"#;
let robots = Robots::parse("FooBot", robotstxt).unwrap();
assert_eq!(robots.sitemaps(), &["http://foo.bar/sitemap.xml"]);
}
#[test]
fn test_path_params_query() {
test_path("", "/");
test_path("http://www.example.com", "/");
test_path("http://www.example.com/", "/");
test_path("http://www.example.com/a", "/a");
test_path("http://www.example.com/a/", "/a/");
test_path(
"http://www.example.com/a/b?c=http://d.e/",
"/a/b?c=http://d.e/",
);
test_path(
"http://www.example.com/a/b?c=d&e=f#fragment",
"/a/b?c=d&e=f",
);
test_path("a", "/a");
test_path("a/", "/a/");
test_path("/a", "/a");
test_path("a/b", "/a/b");
test_path("http://example.com?a", "/?a");
test_path("http://example.com/a;b#c", "/a;b");
test_path("http://example.com///a/b/c", "/a/b/c");
}
#[test]
fn test_maybe_escape_pattern() {
test_escape("http://www.example.com", "http://www.example.com");
test_escape("/a/b/c", "/a/b/c");
test_escape("á", "%C3%A1");
test_escape("%C3%A1", "%C3%A1");
test_escape("aá", "a%C3%A1");
}
#[test]
fn test_params_respected() {
let robotstxt = r#"
user-agent: FooBot
disallow: /*?searchTerm=
"#;
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/?searchTerm=someTerm"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/bar/?searchTerm=someTerm"
));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/bar"
));
}
#[test]
fn test_url_fragments() {
let robotstxt = r#"
user-agent: FooBot
disallow: /#fragment
"#;
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/#fragment"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/#fragment"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"http://foo.bar/#fragment"
));
}
#[test]
fn test_forgiveness_disallow_variations() {
let robotstxt = "user-agent: FooBot
disallow: /a
dissallow: /b
dissalow: /c
disalow: /d
diasllow: /e
disallaw: /f\n";
for path in ["/a", "/b", "/c", "/d", "/e", "/f"] {
assert!(!is_user_agent_allowed(robotstxt, "FooBot", path));
}
}
#[test]
fn test_forgiveness_ensure_not_too_forgiving() {
let robotstxt = "user-agent: FooBot
disallow:/a
dissallow/b
disallow /c\n";
assert!(!is_user_agent_allowed(robotstxt, "FooBot", "/a"));
assert!(is_user_agent_allowed(robotstxt, "FooBot", "/b"));
assert!(!is_user_agent_allowed(robotstxt, "FooBot", "/c"));
}
#[test]
fn test_forgiveness_sitemap_variations() {
let robotstxt = "user-agent: FooBot
site-map: /a
sitemap: /b
site map: /c\n";
let robots = Robots::parse("BarBot", robotstxt).unwrap();
assert_eq!(robots.sitemaps(), &["/a", "/b", "/c"]);
}
#[test]
fn test_forgiveness_crawl_delay_variations() {
let robotstxt = "user-agent: FooBot
crawl-delay: 42
user-agent: BarBot
crawl delay: 420
user-agent: BazBot
crawldelay: 360
\n";
let robots = Robots::parse("FooBot", robotstxt).unwrap();
assert_eq!(robots.crawl_delay(), Some(Duration::from_secs(42)));
let robots = Robots::parse("BarBot", robotstxt).unwrap();
assert_eq!(robots.crawl_delay(), Some(Duration::from_secs(420)));
let robots = Robots::parse("BazBot", robotstxt).unwrap();
assert_eq!(robots.crawl_delay(), Some(Duration::from_secs(360)));
}
#[test]
fn test_forgiveness_user_agent_variations() {
let robotstxt = "user-agent: FooBot
disallow: /a
user agent: BarBot
disallow: /b
useragent: BazBot
disallow: /e\n";
assert!(!is_user_agent_allowed(robotstxt, "FooBot", "/a"));
assert!(!is_user_agent_allowed(robotstxt, "BarBot", "/b"));
assert!(!is_user_agent_allowed(robotstxt, "BazBot", "/e"));
}
#[test]
fn test_no_leading_useragent() {
let robotstxt = "disallow: /a
allow: /b";
assert!(!is_user_agent_allowed(robotstxt, "FooBot", "/a"));
assert!(is_user_agent_allowed(robotstxt, "FooBot", "/b"));
}
#[test]
fn test_leading_wildcard() {
let robotstxt = "user-agent: *
disallow: */a
allow: /b";
assert!(!is_user_agent_allowed(robotstxt, "FooBot", "/a"));
assert!(!is_user_agent_allowed(robotstxt, "FooBot", "/c/a"));
assert!(is_user_agent_allowed(robotstxt, "FooBot", "/b"));
}
#[test]
fn test_robot_crazy_long_regex() {
let robotstxt = "User-agent: *
Disallow: /basket*
# Longest string takes priority. This is necessary due to conflicting Allow rules:
Disallow: /*?************************************************************************************donotindex=1*";
assert!(!is_user_agent_allowed(robotstxt, "FooBot", "/basket"));
assert!(!is_user_agent_allowed(robotstxt, "FooBot", "/basket/ball"));
assert!(is_user_agent_allowed(
robotstxt,
"FooBot",
"/example/file?xyz=42"
));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"/example/file?xyz=42&donotindex=1"
));
}
#[test]
fn test_double_return_newline() {
let robotstxt = "\r
User-agent: *\r\r
Disallow: /en-AU/party\r\r\r\n\n\r\n
User-Agent: FooBot
Disallow: /fi-FI/party\r\r\n
Disallow: /en-US/party\r\r\n
\r\n\r\r\r\n\n
Crawl-Delay: 4";
assert!(!is_user_agent_allowed(robotstxt, "BarBot", "/en-AU/party"));
assert!(is_user_agent_allowed(robotstxt, "FooBot", "/en-AU/party"));
assert!(!is_user_agent_allowed(robotstxt, "FooBot", "/fi-FI/party"));
assert!(!is_user_agent_allowed(robotstxt, "FooBot", "/en-US/party"));
}
#[test]
fn test_robot_parses_crazy_long_lines() {
let mut txt = b"Disallow: /".to_vec();
let ending = b"AAAAAAAAAA".to_vec();
// 10 bytes * 1_000_000 = 10MB
for _ in 0..1_000_000 {
txt.extend(&ending);
}
let robotstxt = String::from_utf8(txt).unwrap();
assert!(Robots::parse("FooBot", &robotstxt).is_ok());
}
#[test]
fn test_robot_doesnt_do_full_regex() {
let robotstxt = "User-agent: *
Disallow: /(Cat|Dog).html";
assert!(is_user_agent_allowed(robotstxt, "FooBot", "/Cat.html"));
assert!(is_user_agent_allowed(robotstxt, "FooBot", "/Dog.html"));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"/(Cat|Dog).html"
));
}
#[test]
fn test_robot_starts_with_crawl_delay() {
let robotstxt = "Crawl-Delay: 42
User-Agent: *
Disallow: /blah
User-Agent: BarBot
Allow: /
Crawl-Delay: 1";
let robots = Robots::parse("FooBot", robotstxt).unwrap();
assert_eq!(robots.crawl_delay(), Some(Duration::from_secs(42)));
let robots = Robots::parse("BarBot", robotstxt).unwrap();
assert_eq!(robots.crawl_delay(), Some(Duration::from_secs(1)));
}
#[test]
fn test_robot_handles_random_nulls() {
let robotstxt = "User-Agent: *
\x00\x00Allow: /family\x00\x00
Disallow: /family/photos\x00\x00\x00";
assert!(is_user_agent_allowed(robotstxt, "FooBot", "/family"));
assert!(!is_user_agent_allowed(
robotstxt,
"FooBot",
"/family/photos"
));
}
#[test]
fn test_robot_crawl_delay_not_integer() {
let robotstxt = b"User-Agent: FooBot
Crawl-Delay: 4.2
User-Agent: BarBot
Crawl-Delay: \x41\xc2\xc3\xb1\x42";
let robotstxt = String::from_utf8_lossy(robotstxt);
let robots = Robots::parse("FooBot", &robotstxt).unwrap();
assert_eq!(robots.crawl_delay(), Some(Duration::from_secs_f32(4.2)));
let robots = Robots::parse("BarBot", &robotstxt).unwrap();
assert_eq!(robots.crawl_delay(), None);
}
#[test]
fn test_empty_disallow() {
let robotstxt = "User-Agent: FooBot
Disallow:
";
assert!(is_user_agent_allowed(robotstxt, "FooBot", "/"));
}
}
Reference in a new issue View git blame Copy permalink