# Disallow GPTBot to prevent excessive crawling with invalid pagination User-agent: GPTBot Disallow: / # Allow Googlebot (Google Search) User-agent: Googlebot Allow: / Disallow: /api/ Disallow: /endpoint/ Disallow: /mypage/ Disallow: /*?page=* Disallow: /*?*page=* # Allow Bingbot (Bing Search) User-agent: Bingbot Allow: / Disallow: /api/ Disallow: /endpoint/ Disallow: /mypage/ Disallow: /*?page=* Disallow: /*?*page=* # Rules for all other bots (Yahoo, DuckDuckGo, Baidu, etc.) # These bots can still crawl public pages, but with some restrictions User-agent: * # Allow crawling of main pages first Allow: / Allow: /products/ Allow: /company Allow: /tomsintroduce Allow: /history Allow: /qna Allow: /terms Allow: /privacy # Disallow pagination pages with query parameters (to prevent invalid page numbers) Disallow: /*?page=* Disallow: /*?*page=* # Disallow API endpoints (not needed for SEO) Disallow: /api/ Disallow: /endpoint/ # Disallow mypage (requires authentication, not public content) Disallow: /mypage/ # Crawl-delay for other bots to reduce server load Crawl-delay: 1