From d998a59fdaa3e9b0cf5f1ff3871387c20a20b27b Mon Sep 17 00:00:00 2001
From: Michael Fabian 'Xaymar' Dirks <info@xaymar.com>
Date: Sat, 7 Sep 2024 13:55:17 +0200
Subject: [PATCH] Add robots and fix up sitemap.

---
 robots.txt  | 11 +++++++++++
 sitemap.xml |  9 +++++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/robots.txt b/robots.txt
index 3567238..7d4dfbf 100644
--- a/robots.txt
+++ b/robots.txt
@@ -1,4 +1,11 @@
+---
+layout: null
+sitemap:
+  exclude: true
+---
+
 User-agent: *
+Sitemap: {{ "/robots.txt" | absolute_url }}
 Disallow: /assets/
 Disallow: /feed/
 Disallow: /feed.xml
@@ -13,12 +20,14 @@ User-agent: Applebot
 User-agent: Applebot-Extended
 User-agent: Bytespider
 User-agent: CCBot
+User-agent: ChatGPT
 User-agent: ChatGPT-User
 User-agent: Claude-Web
 User-agent: ClaudeBot
 User-agent: Diffbot
 User-agent: FacebookBot
 User-agent: FriendlyCrawler
+User-agent: GPT
 User-agent: GPTBot
 User-agent: Google-Extended
 User-agent: GoogleOther
@@ -29,7 +38,9 @@ User-agent: ICC-Crawler
 User-agent: ImagesiftBot
 User-agent: Meta-ExternalAgent
 User-agent: Meta-ExternalFetcher
+User-agent: OpenAI
 User-agent: OAI-SearchBot
+User-agent: OAI-Bot
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: Scrapy
diff --git a/sitemap.xml b/sitemap.xml
index a21c296..e7ee701 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -3,13 +3,13 @@ layout: null
 sitemap:
   exclude: true
 ---
-
+{% capture var %}
 <?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 	{% for post in site.posts %}
 		{% unless post.published == false %}
 		<url>
-			<loc>{{ site.url }}{{ post.url }}</loc>
+			<loc>{{ site.url | xml_encode }}{{ post.url | xml_encode }}</loc>
 			{% if post.sitemap.lastmod %}
 				<lastmod>{{ post.sitemap.lastmod | date: "%Y-%m-%d" }}</lastmod>
 			{% elsif post.date %}
@@ -41,7 +41,7 @@ sitemap:
 		{% endfor %}
 		{% unless skip == true or page.sitemap.exclude == true %}
 		<url>
-			<loc>{{ site.url }}{{ page.url | remove: "index.html" }}</loc>
+			<loc>{{ site.url | xml_encode }}{{ page.url | remove: "index.html" | xml_encode }}</loc>
 			{% if page.sitemap.lastmod %}
 				<lastmod>{{ page.sitemap.lastmod | date: "%Y-%m-%d" }}</lastmod>
 			{% elsif page.date %}
@@ -62,4 +62,5 @@ sitemap:
 		</url>
 		{% endunless %}
 	{% endfor %}
-</urlset>
\ No newline at end of file
+</urlset>
+{% endcapture %}{{ var | normalize_whitespace }}