{"id":"sitemaps_detection","name":"Sitemaps Detection","alias":"SitemapsDetection","aliases":[],"description_short":"Find the sitemaps declared in the robots.txt of a site's start URLs","description_long":"\n    Read the robots.txt of each origin of the given start URLs and return the\n    sitemap URLs it declares (Sitemap: lines). Only absolute http(s) URLs are\n    kept. Invalid start URLs and unreachable or failing robots.txt files are\n    skipped (they yield no sitemap); the call errors only when no start URL is\n    usable.\n    The robots.txt is fetched like robots_txt_check does: without the\n    project's user agent or authentication headers.\n    ","description_tool_prompt":"\n    Read the robots.txt of each origin of the given start URLs and return the\n    sitemap URLs it declares (Sitemap: lines). Only absolute http(s) URLs are\n    kept. Invalid start URLs and unreachable or failing robots.txt files are\n    skipped (they yield no sitemap); the call errors only when no start URL is\n    usable.\n    The robots.txt is fetched like robots_txt_check does: without the\n    project's user agent or authentication headers.\n    ","source":null,"conversation_examples":[],"tags":["url data"],"available_in_chat":false,"available_in_deepthink":false,"available_in_mcp":true,"mcp_audience":"all","config":{"type":"object","properties":{},"required":[]},"item":{"type":"object","properties":{"start_urls":{"description":"Start URLs of the crawl. The robots.txt of each distinct origin (scheme, host, port) is read once. An explicit default port (:443 for https, :80 for http) is the same origin.","items":{"type":"string"},"maxItems":20,"minItems":1,"title":"Start Urls","type":"array"}},"required":["start_urls"]},"response":{"$defs":{"RobotsTxtSource":{"description":"The robots.txt read for one origin.","properties":{"url":{"description":"The robots.txt URL that was requested","title":"Url","type":"string"},"found":{"description":"Whether a robots.txt was returned (2xx)","title":"Found","type":"boolean"},"status_code":{"anyOf":[{"type":"integer"},{"type":"null"}],"description":"HTTP status, or null on transport failure","title":"Status Code"}},"required":["url","found","status_code"],"title":"RobotsTxtSource","type":"object"}},"properties":{"sitemaps":{"description":"Sitemap URLs declared in the robots.txt files, in order, without duplicates","items":{"type":"string"},"title":"Sitemaps","type":"array"},"robots_txt":{"description":"One entry per origin, in the order of the start URLs","items":{"$ref":"#/$defs/RobotsTxtSource"},"title":"Robots Txt","type":"array"}},"required":["sitemaps","robots_txt"],"title":"SitemapsDetectionProcessedItem","type":"object"},"has_canvas":false,"open_canvas_automatically":false}