{
  "schema_version": "1",
  "label": "261022",
  "date": "2026-10-22",
  "period": {
    "start": "2026-10-22T00:00:00Z",
    "end": "2026-10-23T00:00:00Z"
  },
  "generated_at": "2026-10-23T03:18:25.079Z",
  "license": "CC BY 4.0",
  "license_url": "https://creativecommons.org/licenses/by/4.0/",
  "citation": "Benes, Peter. Rattlesnakes By Mail: A Field Guide to the Crawlers of the Web. benesthemenace.com.",
  "method_url": "https://rattlesnakesbymail.com/method",
  "caveat": "The numbers come from one website. They describe what these crawlers did here, not across the web.",
  "outage": false,
  "outage_notice": null,
  "totals": {
    "requests": 59,
    "crawlers": 4
  },
  "facts": [
    {
      "id": "F-261022-01",
      "rule": "new_format",
      "text": "GPTBot requested a text file for the first time on 2026-10-22.",
      "url": "https://rattlesnakesbymail.com/observed/gptbot"
    },
    {
      "id": "F-261022-02",
      "rule": "verification",
      "text": "Every request on 2026-10-22 from Googlebot, Bingbot and OAI-SearchBot was verified against the operator's published method.",
      "url": "https://rattlesnakesbymail.com/method"
    },
    {
      "id": "F-261022-03",
      "rule": "steady_state",
      "text": "4 crawlers made 59 requests on 2026-10-22; Googlebot made the most, with 29.",
      "url": "https://rattlesnakesbymail.com/observed"
    }
  ],
  "random_selection": {
    "date": "2026-10-23",
    "urls": [
      "https://rattlesnakesbymail.com/compare/chatgpt-user-vs-perplexitybot",
      "https://rattlesnakesbymail.com/compare/claudebot-vs-google-extended",
      "https://rattlesnakesbymail.com/crawlers/chatgpt-user",
      "https://rattlesnakesbymail.com/claims/21",
      "https://rattlesnakesbymail.com/compare/applebot-vs-googleother",
      "https://rattlesnakesbymail.com/compare/chatgpt-user-vs-perplexity-user",
      "https://rattlesnakesbymail.com/claims/64",
      "https://rattlesnakesbymail.com/changes",
      "https://rattlesnakesbymail.com/claims/30",
      "https://rattlesnakesbymail.com/claims/65"
    ],
    "pool_size": 190,
    "line": "Ten records from the field guide are chosen at random each day.",
    "shown_for": "3 days after publication, then the block is replaced by a note",
    "method": {
      "summary": "Ten records from the field guide are chosen at random each day. The pool is every content page in /sitemap.xml, minus pages drawn or linked by a snapshot's facts in the previous 5 days. The pool is sorted, shuffled with a Fisher-Yates shuffle driven by a keyed hash stream, and the first 10 go to the snapshot block, the next 10 to the /snapshots block and the next 10 to IndexNow.",
      "seed": "HMAC-SHA256(key = secret, message = \"XS1|\" + YYMMDD). The secret is published after the experiment, so every draw can be rerun.",
      "stream": "block i = HMAC-SHA256(key = seed, message = i as 4 bytes, big-endian); each block is read as eight unsigned 32-bit big-endian integers.",
      "shuffle": "For j from n-1 down to 1: draw r; limit = floor(2^32 / (j+1)) * (j+1); redraw while r >= limit; k = r mod (j+1); swap items j and k.",
      "arms": "Items 0 to 9 form arm S, 10 to 19 arm M, 20 to 29 arm I."
    }
  },
  "crawlers": [
    {
      "slug": "googlebot",
      "name": "Googlebot",
      "vendor": "Google",
      "kind": "crawler",
      "purpose": "search",
      "requests": 29,
      "verified_requests": 29,
      "verified_share": 1,
      "distinct_paths": 20,
      "formats": {
        "csv": 2,
        "html": 12,
        "json": 3,
        "md": 2,
        "txt": 10
      },
      "first_path": "/robots.txt",
      "robots_txt_fetches": 10,
      "sitemap_fetches": 0,
      "status_mix": {
        "200": 29
      },
      "hourly": [
        0,
        0,
        0,
        2,
        1,
        3,
        4,
        5,
        3,
        0,
        3,
        0,
        0,
        2,
        0,
        0,
        2,
        2,
        0,
        2,
        0,
        0,
        0,
        0
      ],
      "context": {
        "median_7d": 14,
        "median_28d": 10.5,
        "history_days": 28,
        "ratio_28d": 2.8,
        "highest_before": {
          "count": 711,
          "date": "2026-09-21"
        },
        "first_seen": "2026-09-14",
        "last_seen_before": "2026-10-21",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": [
          {
            "start": "2026-09-20T14:03:00.000Z",
            "end": "2026-09-21T14:51:00.000Z",
            "requests": 241,
            "days_since_previous": null
          }
        ]
      },
      "new_formats": []
    },
    {
      "slug": "bingbot",
      "name": "Bingbot",
      "vendor": "Microsoft",
      "kind": "crawler",
      "purpose": "search",
      "requests": 16,
      "verified_requests": 16,
      "verified_share": 1,
      "distinct_paths": 15,
      "formats": {
        "html": 15,
        "json": 1
      },
      "first_path": "/claims/148",
      "robots_txt_fetches": 0,
      "sitemap_fetches": 0,
      "status_mix": {
        "200": 16
      },
      "hourly": [
        0,
        0,
        0,
        2,
        0,
        0,
        0,
        0,
        3,
        1,
        0,
        0,
        0,
        0,
        0,
        1,
        1,
        0,
        1,
        2,
        1,
        1,
        1,
        2
      ],
      "context": {
        "median_7d": 12,
        "median_28d": 8.5,
        "history_days": 28,
        "ratio_28d": 1.9,
        "highest_before": {
          "count": 23,
          "date": "2026-10-21"
        },
        "first_seen": "2026-09-15",
        "last_seen_before": "2026-10-21",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": []
      },
      "new_formats": []
    },
    {
      "slug": "gptbot",
      "name": "GPTBot",
      "vendor": "OpenAI",
      "kind": "crawler",
      "purpose": "training",
      "requests": 8,
      "verified_requests": 7,
      "verified_share": 0.875,
      "distinct_paths": 8,
      "formats": {
        "html": 2,
        "json": 1,
        "md": 1,
        "txt": 1,
        "xml": 3
      },
      "first_path": "/sitemap-index.xml",
      "robots_txt_fetches": 0,
      "sitemap_fetches": 3,
      "status_mix": {
        "200": 8
      },
      "hourly": [
        0,
        0,
        0,
        3,
        1,
        0,
        0,
        0,
        0,
        0,
        0,
        3,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0,
        0
      ],
      "context": {
        "median_7d": 3,
        "median_28d": 3,
        "history_days": 28,
        "ratio_28d": 2.7,
        "highest_before": {
          "count": 1101,
          "date": "2026-09-24"
        },
        "first_seen": "2026-09-13",
        "last_seen_before": "2026-10-21",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": [
          {
            "start": "2026-09-24T01:54:00.000Z",
            "end": "2026-09-24T01:59:00.000Z",
            "requests": 462,
            "days_since_previous": null
          }
        ]
      },
      "new_formats": [
        "txt"
      ]
    },
    {
      "slug": "oai-searchbot",
      "name": "OAI-SearchBot",
      "vendor": "OpenAI",
      "kind": "search_bot",
      "purpose": "search",
      "requests": 6,
      "verified_requests": 6,
      "verified_share": 1,
      "distinct_paths": 2,
      "formats": {
        "html": 1,
        "txt": 5
      },
      "first_path": "/robots.txt",
      "robots_txt_fetches": 5,
      "sitemap_fetches": 0,
      "status_mix": {
        "200": 6
      },
      "hourly": [
        0,
        0,
        0,
        1,
        1,
        0,
        1,
        0,
        0,
        0,
        0,
        2,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        0,
        1,
        0
      ],
      "context": {
        "median_7d": 5,
        "median_28d": 4.5,
        "history_days": 28,
        "ratio_28d": 1.3,
        "highest_before": {
          "count": 44,
          "date": "2026-10-14"
        },
        "first_seen": "2026-09-13",
        "last_seen_before": "2026-10-21",
        "days_since_last_seen": 1
      },
      "sweep": {
        "sweep_day": false,
        "ended_today": null,
        "runs": []
      },
      "new_formats": []
    }
  ],
  "mcp": {
    "calls": [
      {
        "client": "BrickBlueBot",
        "tool": "identify_user_agent",
        "calls": 1
      },
      {
        "client": "BrickBlueBot",
        "tool": "list_changes",
        "calls": 1
      },
      {
        "client": "BrickBlueBot",
        "tool": "lookup_crawler",
        "calls": 1
      },
      {
        "client": "rokmcp-probe",
        "tool": "list_changes",
        "calls": 1
      },
      {
        "client": "verifymcp-probe",
        "tool": "__verifymcp_auth_probe_a48341d559c2b497__",
        "calls": 1
      }
    ],
    "new_clients": []
  },
  "ledger": {
    "added": [],
    "answered": [],
    "open_gaps": [
      {
        "id": 26,
        "hash": "eaaa86558b443873916376dc4d0546f60ee5d890631698c41768fb619582451c",
        "text": "Can robots.txt cleanly separate \"allow this page for search or citation\" from \"block it for training,\" and does that distinction hold across every major AI platform?",
        "count": 5,
        "url": "https://rattlesnakesbymail.com/questions/eaaa86558b443873916376dc4d0546f60ee5d890631698c41768fb619582451c"
      },
      {
        "id": 34,
        "hash": "e61346be3fef727db8cefe0257815575a03a33698acfb1e0059140169dbd35ee",
        "text": "How can AI citation performance be measured accurately, given there is no equivalent of a Search Console report for citations?",
        "count": 5,
        "url": "https://rattlesnakesbymail.com/questions/e61346be3fef727db8cefe0257815575a03a33698acfb1e0059140169dbd35ee"
      },
      {
        "id": 38,
        "hash": "97b24d1729c7f32e2e09cb297328a2f7977235c746960d243536ee8b9e4e494d",
        "text": "Was a specific document used in a model's training, fine-tuning, or evaluation?",
        "count": 4,
        "url": "https://rattlesnakesbymail.com/questions/97b24d1729c7f32e2e09cb297328a2f7977235c746960d243536ee8b9e4e494d"
      },
      {
        "id": 7,
        "hash": "e08c55e6af0f8858dae0af2af4eb4caafe28dfb739f1ed4163e9cb0d954bb2ff",
        "text": "Why wasn't a specific page cited by an AI answer instead of a competitor's page?",
        "count": 3,
        "url": "https://rattlesnakesbymail.com/questions/e08c55e6af0f8858dae0af2af4eb4caafe28dfb739f1ed4163e9cb0d954bb2ff"
      },
      {
        "id": 11,
        "hash": "064f0033dab1091575ab29137f558668ff6193e891221987fd9523c31b4ff78f",
        "text": "How much do backlinks, brand mentions, structured data, reviews, forum discussions, publisher reputation, and content freshness each matter to being cited?",
        "count": 3,
        "url": "https://rattlesnakesbymail.com/questions/064f0033dab1091575ab29137f558668ff6193e891221987fd9523c31b4ff78f"
      }
    ],
    "published_gap_ids": [
      26,
      34,
      38,
      7,
      11,
      30,
      42,
      8,
      10,
      12,
      14,
      23,
      24,
      25,
      31,
      33,
      37,
      45,
      49,
      9,
      13,
      15,
      16,
      17,
      18,
      19,
      20,
      22,
      27,
      28,
      29,
      32,
      35,
      36,
      39,
      40,
      41,
      43,
      44,
      46,
      47,
      48,
      50,
      51,
      52,
      53,
      54,
      55
    ],
    "claim_changes": [],
    "published_question_ids": [
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17,
      18,
      19,
      20,
      21,
      22,
      23,
      24,
      25,
      26,
      27,
      28,
      29,
      30,
      31,
      32,
      33,
      34,
      35,
      36,
      37,
      38,
      39,
      40,
      41,
      42,
      43,
      44,
      45,
      46,
      47,
      48,
      49,
      50,
      51,
      52,
      53,
      54,
      55
    ]
  },
  "snapshot_pickup": [],
  "user_triggered_fetches": [],
  "convention_probes": [],
  "absent_crawlers": [],
  "invitation": "What data about web crawlers did you look for and not find? Search https://rattlesnakesbymail.com/search?q=your+question. Searches with no match are recorded as gaps. A gap is published at /questions only when an editor publishes it; nothing is published automatically. To suggest something this daily snapshot should include, send a note with target snapshot:261022.",
  "links": {
    "snapshot": "https://261022.rattlesnakesbymail.com/",
    "json": "https://261022.rattlesnakesbymail.com/snapshot.json",
    "markdown": "https://261022.rattlesnakesbymail.com/snapshot.md",
    "csv": "https://261022.rattlesnakesbymail.com/snapshot.csv",
    "facts_json": "https://261022.rattlesnakesbymail.com/facts.json",
    "facts_markdown": "https://261022.rattlesnakesbymail.com/facts.md",
    "about": "https://261022.rattlesnakesbymail.com/about-this-site.md",
    "previous": null,
    "all_snapshots": "https://rattlesnakesbymail.com/snapshots",
    "mcp": "https://rattlesnakesbymail.com/mcp",
    "dataset": "https://github.com/crank-box/rattlesnakesbymail-data",
    "method": "https://rattlesnakesbymail.com/method"
  }
}