robots.txt matching compared with an independent implementation, 1 October 2026, Node v22.22.3, Windows 11.

robots-parser 3.0.1 (npm) is a separate robots.txt library. It is not a dependency of this kit:
it was installed in a scratch folder only for this comparison. Nine robots.txt files (Google's precedence
examples, the fixture files and pattern, merging and case cases), six crawler tokens and twenty paths.

Result:
  robots-parser 3.0.1 vs src/robots.mjs: 1080 of 1080 decisions agree.

Script (diff.mjs), with the import pointed at this kit's src/robots.mjs:

  import robotsParser from "robots-parser";
  import { isAllowed, parseRobots } from "./machine-readability-audit/src/robots.mjs";
  const files = {
    google: "User-agent: *\nAllow: /p\nDisallow: /\n",
    folder: "User-agent: *\nAllow: /folder\nDisallow: /folder\n",
    htm: "User-agent: *\nAllow: /page\nDisallow: /*.htm\n",
    php: "User-agent: *\nAllow: /page\nDisallow: /*.ph\n",
    root: "User-agent: *\nAllow: /$\nDisallow: /\n",
    fixture: "User-agent: *\nAllow: /\nDisallow: /blocked/\n",
    drift: "User-agent: *\nContent-Signal: search=yes, ai-train=no\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: *\nAllow: /\nDisallow: /blocked/\n",
    patterns: "User-agent: *\nDisallow: /fish*.php\nDisallow: /*.php$\nDisallow: /fish/\nAllow: /fish/ok$\n",
    merged: "User-agent: examplebot\nDisallow: /a/\n\nUser-agent: ExampleBot\nDisallow: /c/\n\nUser-agent: *\nDisallow: /\n",
  };
  const paths = ["/", "/p", "/page", "/page.htm", "/page.php5", "/folder/page", "/blocked/", "/blocked/x", "/good/", "/fish.php", "/fishheads/catfish.php?x=1", "/Fish.PHP", "/filename.php", "/filename.php?x", "/fish/", "/fish/ok", "/fish/okay", "/a/1", "/c/1", "/b/1"];
  const tokens = ["Googlebot", "GPTBot", "ClaudeBot", "Google-Extended", "examplebot", "OAI-SearchBot"];
  let same = 0, diff = [];
  for (const [name, text] of Object.entries(files)) {
    const theirs = robotsParser("https://example.com/robots.txt", text);
    const ours = parseRobots(text);
    for (const token of tokens) for (const path of paths) {
      const a = isAllowed(ours, token, path).allowed;
      const b = theirs.isAllowed("https://example.com" + path, token) !== false;
      if (a === b) same++; else diff.push(`${name} ${token} ${path}: ours ${a}, robots-parser ${b}`);
    }
  }
  console.log(`robots-parser 3.0.1 vs src/robots.mjs: ${same} of ${same + diff.length} decisions agree.`);
  for (const d of diff) console.log("  DIFFERS " + d);
