Sitemap: https://qaz.wtf/sitemap/ # Fuck-off with Google already. They were good once, but clearly have gone to shit. # https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers User-agent: Googlebot Disallow: / User-agent: GoogleOther Disallow: / User-agent: AdsBot-Google-Mobile Disallow: / User-agent: AdsBot-Google Disallow: / User-agent: Mediapartners-Google Disallow: / User-agent: FeedFetcher-Google Disallow: / User-agent: Storebot-Google Disallow: / User-agent: Google-Site-Verification Disallow: / # Fuck off OpenAI, too. User-agent: GPTBot Disallow: / # You too, Zuckerberg User-agent: facebookexternalhit Disallow: / User-agent: LinkedInBot Disallow: / User-agent: Pinterestbot Disallow: / User-agent: bingbot Crawl-delay: 60 # 14k attempts on one URL each met with a 404? yeah, fuck you bot User-agent: NaverBot User-agent: Yeti Disallow: / # reportedly for Huawei's search engine in China User-agent: AspiegelBot Disallow: / # All of the following are SEO / backlink bots. User-agent: DotBot Disallow: / User-agent: serpstatbot Disallow: / User-agent: MegaIndex Disallow: / User-agent: MJ12bot Disallow: / User-agent: semrushbot Disallow: / User-agent: ahrefsbot Disallow: / User-agent: mauibot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: NetpeakCheckerBot Disallow: / User-agent: Semanticbot Disallow: / # Borrowed from Wikipedia's anti-site slurp section # Some bots are known to be trouble, particularly those designed to copy # entire sites. Please obey robots.txt. User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / User-agent: WebReaper Disallow: / # Nameprotect User-agent: NPBot Disallow: /