Add robots and fix up sitemap.
This commit is contained in:
+11
@@ -1,4 +1,11 @@
|
||||
---
|
||||
layout: null
|
||||
sitemap:
|
||||
exclude: true
|
||||
---
|
||||
|
||||
User-agent: *
|
||||
Sitemap: {{ "/robots.txt" | absolute_url }}
|
||||
Disallow: /assets/
|
||||
Disallow: /feed/
|
||||
Disallow: /feed.xml
|
||||
@@ -13,12 +20,14 @@ User-agent: Applebot
|
||||
User-agent: Applebot-Extended
|
||||
User-agent: Bytespider
|
||||
User-agent: CCBot
|
||||
User-agent: ChatGPT
|
||||
User-agent: ChatGPT-User
|
||||
User-agent: Claude-Web
|
||||
User-agent: ClaudeBot
|
||||
User-agent: Diffbot
|
||||
User-agent: FacebookBot
|
||||
User-agent: FriendlyCrawler
|
||||
User-agent: GPT
|
||||
User-agent: GPTBot
|
||||
User-agent: Google-Extended
|
||||
User-agent: GoogleOther
|
||||
@@ -29,7 +38,9 @@ User-agent: ICC-Crawler
|
||||
User-agent: ImagesiftBot
|
||||
User-agent: Meta-ExternalAgent
|
||||
User-agent: Meta-ExternalFetcher
|
||||
User-agent: OpenAI
|
||||
User-agent: OAI-SearchBot
|
||||
User-agent: OAI-Bot
|
||||
User-agent: PerplexityBot
|
||||
User-agent: PetalBot
|
||||
User-agent: Scrapy
|
||||
|
||||
Reference in New Issue
Block a user