{
  "requestedUrl": "https://seotechlist.com/guides/ai-crawler-access",
  "finalUrl": "https://seotechlist.com/guides/ai-crawler-access",
  "pageStatus": 200,
  "robots": {
    "url": "https://seotechlist.com/robots.txt",
    "status": 200,
    "available": true
  },
  "indexing": {
    "robotsMeta": [],
    "xRobotsTag": null,
    "noindex": false,
    "nofollow": false
  },
  "crawlers": [
    {
      "name": "OpenAI search",
      "userAgent": "OAI-SearchBot",
      "provider": "OpenAI",
      "purpose": "search",
      "sourceUrl": "https://platform.openai.com/docs/bots",
      "robots": {
        "allowed": true,
        "matchedRule": "Allow: /",
        "usedSpecificGroup": false
      },
      "probe": {
        "status": 200,
        "finalUrl": "https://seotechlist.com/guides/ai-crawler-access",
        "reachable": true
      }
    },
    {
      "name": "ChatGPT user requests",
      "userAgent": "ChatGPT-User",
      "provider": "OpenAI",
      "purpose": "user-requested",
      "sourceUrl": "https://platform.openai.com/docs/bots",
      "robots": {
        "allowed": true,
        "matchedRule": "Allow: /",
        "usedSpecificGroup": false
      },
      "probe": {
        "status": 200,
        "finalUrl": "https://seotechlist.com/guides/ai-crawler-access",
        "reachable": true
      }
    },
    {
      "name": "OpenAI training",
      "userAgent": "GPTBot",
      "provider": "OpenAI",
      "purpose": "training",
      "sourceUrl": "https://platform.openai.com/docs/bots",
      "robots": {
        "allowed": true,
        "matchedRule": "Allow: /",
        "usedSpecificGroup": false
      },
      "probe": {
        "status": 200,
        "finalUrl": "https://seotechlist.com/guides/ai-crawler-access",
        "reachable": true
      }
    },
    {
      "name": "Claude search",
      "userAgent": "Claude-SearchBot",
      "provider": "Anthropic",
      "purpose": "search",
      "sourceUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "robots": {
        "allowed": true,
        "matchedRule": "Allow: /",
        "usedSpecificGroup": false
      },
      "probe": {
        "status": 200,
        "finalUrl": "https://seotechlist.com/guides/ai-crawler-access",
        "reachable": true
      }
    },
    {
      "name": "Claude user requests",
      "userAgent": "Claude-User",
      "provider": "Anthropic",
      "purpose": "user-requested",
      "sourceUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "robots": {
        "allowed": true,
        "matchedRule": "Allow: /",
        "usedSpecificGroup": false
      },
      "probe": {
        "status": 200,
        "finalUrl": "https://seotechlist.com/guides/ai-crawler-access",
        "reachable": true
      }
    },
    {
      "name": "Anthropic training",
      "userAgent": "ClaudeBot",
      "provider": "Anthropic",
      "purpose": "training",
      "sourceUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "robots": {
        "allowed": true,
        "matchedRule": "Allow: /",
        "usedSpecificGroup": false
      },
      "probe": {
        "status": 200,
        "finalUrl": "https://seotechlist.com/guides/ai-crawler-access",
        "reachable": true
      }
    }
  ],
  "checkedAt": "2026-09-29T01:43:12.931Z",
  "caveat": "User-agent probes come from your machine, not a verified provider IP range."
}