Add robots and fix up sitemap.

This commit is contained in:
Michael Fabian 'Xaymar' Dirks
2024-09-07 13:55:17 +02:00
parent 1be1a7d451
commit d998a59fda
2 changed files with 16 additions and 4 deletions
+11
View File
@@ -1,4 +1,11 @@
---
layout: null
sitemap:
exclude: true
---
User-agent: *
Sitemap: {{ "/robots.txt" | absolute_url }}
Disallow: /assets/
Disallow: /feed/
Disallow: /feed.xml
@@ -13,12 +20,14 @@ User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: GPT
User-agent: GPTBot
User-agent: Google-Extended
User-agent: GoogleOther
@@ -29,7 +38,9 @@ User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: OpenAI
User-agent: OAI-SearchBot
User-agent: OAI-Bot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy