# robots.txt for andreagriffiths.dev # Guide search engine crawlers for optimal SEO # Default rules for all crawlers User-agent: * Allow: / # Disallow crawling of build artifacts and system files Disallow: /_astro/ Disallow: /.git/ Disallow: /.github/ Disallow: /node_modules/ # Allow all major search engines full access User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Bingbot Allow: / User-agent: Slurp Allow: / User-agent: DuckDuckBot Allow: / User-agent: Baiduspider Allow: / User-agent: YandexBot Allow: / # AI Bots and Crawlers - Full access to content including Markdown endpoints User-agent: GPTBot Allow: / Allow: /index.md Allow: /writings.md User-agent: ChatGPT-User Allow: / Allow: /index.md Allow: /writings.md User-agent: CCBot Allow: / Allow: /index.md Allow: /writings.md User-agent: anthropic-ai Allow: / Allow: /index.md Allow: /writings.md User-agent: ClaudeBot Allow: / Allow: /index.md Allow: /writings.md User-agent: Claude-Web Allow: / Allow: /index.md Allow: /writings.md User-agent: PerplexityBot Allow: / Allow: /index.md Allow: /writings.md User-agent: Omgilibot Allow: / Allow: /index.md Allow: /writings.md User-agent: FacebookBot Allow: / Allow: /index.md Allow: /writings.md User-agent: Applebot Allow: / Allow: /index.md Allow: /writings.md User-agent: ImagesiftBot Allow: / Allow: /index.md Allow: /writings.md User-agent: cohere-ai Allow: / Allow: /index.md Allow: /writings.md # Crawl-delay for aggressive bots (in seconds) # This helps prevent server overload from aggressive crawlers User-agent: * Crawl-delay: 1 # Sitemap location for search engines Sitemap: https://andreagriffiths.dev/sitemap-index.xml Sitemap: https://andreagriffiths.dev/sitemap-0.xml