{"id":"robots_txt_check","name":"Robots.txt Check","alias":"RobotsTxtCheck","aliases":[],"description_short":"Fetch a host's robots.txt and optionally test URLs against a user-agent","description_long":"\n    Fetch robots.txt for a domain and optionally check whether URLs are allowed\n    or disallowed for a given user-agent.\n\n    Matching uses Google's robots.txt parser. A 2xx body (including an empty\n    file) is passed to the matcher. A 4xx response other than 429 is treated as\n    no robots.txt, so every URL is allowed. A 429, a 5xx response, or a transport\n    failure is treated as a temporary full disallow, so every URL is disallowed.\n\n    The matcher does not fold a more specific bot such as Googlebot-Image into\n    Googlebot. Pass the user-agent token you want to test.\n    ","description_tool_prompt":"\n    Fetch robots.txt for a domain and optionally check whether URLs are allowed\n    or disallowed for a given user-agent.\n\n    Matching uses Google's robots.txt parser. A 2xx body (including an empty\n    file) is passed to the matcher. A 4xx response other than 429 is treated as\n    no robots.txt, so every URL is allowed. A 429, a 5xx response, or a transport\n    failure is treated as a temporary full disallow, so every URL is disallowed.\n\n    The matcher does not fold a more specific bot such as Googlebot-Image into\n    Googlebot. Pass the user-agent token you want to test.\n    ","source":"The site's robots.txt, matched with Google's robots.txt parser","conversation_examples":["Fetch robots.txt for example.com","Is https://example.com/admin allowed for Googlebot?","Check /cart and /about against Bingbot on www.example.com"],"tags":["url data"],"available_in_chat":false,"available_in_deepthink":false,"available_in_mcp":true,"mcp_audience":"all","config":{"type":"object","properties":{},"required":[]},"item":{"type":"object","properties":{"domain":{"description":"Hostname (www.example.com) or absolute URL. Reduced to an origin (scheme + host + port). https is assumed when the scheme is omitted.","title":"Domain","type":"string"},"urls":{"default":[],"description":"URLs to test against the fetched robots.txt. Absolute http(s) URLs, or paths (/private) resolved against the origin. Each URL must use the same scheme, host, and port. At most 20.","items":{"type":"string"},"maxItems":20,"title":"URLs","type":"array"},"user_agent":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"User-agent token passed to the matcher. Required when urls is non-empty. Not defaulted; a specific bot such as Googlebot-Image is not folded into Googlebot.","title":"User Agent"}},"required":["domain"]},"response":{"$defs":{"RobotsTxtUrlCheck":{"description":"Whether one URL is allowed or disallowed by robots.txt.","properties":{"url":{"description":"The absolute URL that was matched","title":"URL","type":"string"},"allowed":{"description":"True when the URL passes robots.txt, false when it is disallowed","title":"Allowed","type":"boolean"}},"required":["url","allowed"],"title":"RobotsTxtUrlCheck","type":"object"}},"properties":{"found":{"description":"True when the fetch returned HTTP 2xx, including an empty file","title":"Found","type":"boolean"},"content":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"robots.txt body on 2xx, truncated to 500 KiB. Null when found is false. Empty file is found=true and content=''.","title":"Content"},"status_code":{"anyOf":[{"type":"integer"},{"type":"null"}],"description":"HTTP status of the robots.txt response, or null on transport failure","title":"Status Code"},"url":{"description":"The robots.txt URL that was requested","title":"Robots.txt URL","type":"string"},"checks":{"default":[],"description":"One entry per input URL, in order. Empty when urls was omitted. allowed is true when the URL passes robots.txt.","items":{"$ref":"#/$defs/RobotsTxtUrlCheck"},"title":"Checks","type":"array"}},"required":["found","content","status_code","url"],"title":"RobotsTxtCheckProcessedItem","type":"object"},"has_canvas":false,"open_canvas_automatically":false}