Add robots and fix up sitemap.

robots.txt

@@ -1,4 +1,11 @@
+---
+layout: null
+sitemap:
+  exclude: true
+---
+
 User-agent: *
+Sitemap: {{ "/robots.txt" | absolute_url }}
 Disallow: /assets/
 Disallow: /feed/
 Disallow: /feed.xml
@@ -13,12 +20,14 @@ User-agent: Applebot
 User-agent: Applebot-Extended
 User-agent: Bytespider
 User-agent: CCBot
+User-agent: ChatGPT
 User-agent: ChatGPT-User
 User-agent: Claude-Web
 User-agent: ClaudeBot
 User-agent: Diffbot
 User-agent: FacebookBot
 User-agent: FriendlyCrawler
+User-agent: GPT
 User-agent: GPTBot
 User-agent: Google-Extended
 User-agent: GoogleOther
@@ -29,7 +38,9 @@ User-agent: ICC-Crawler
 User-agent: ImagesiftBot
 User-agent: Meta-ExternalAgent
 User-agent: Meta-ExternalFetcher
+User-agent: OpenAI
 User-agent: OAI-SearchBot
+User-agent: OAI-Bot
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: Scrapy

sitemap.xml

@@ -3,13 +3,13 @@ layout: null
 sitemap:
   exclude: true
 ---
-
+{% capture var %}
 <?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
 	{% for post in site.posts %}
 		{% unless post.published == false %}
 		<url>
-			<loc>{{ site.url }}{{ post.url }}</loc>
+			<loc>{{ site.url | xml_encode }}{{ post.url | xml_encode }}</loc>
 			{% if post.sitemap.lastmod %}
 				<lastmod>{{ post.sitemap.lastmod | date: "%Y-%m-%d" }}</lastmod>
 			{% elsif post.date %}
@@ -41,7 +41,7 @@ sitemap:
 		{% endfor %}
 		{% unless skip == true or page.sitemap.exclude == true %}
 		<url>
-			<loc>{{ site.url }}{{ page.url | remove: "index.html" }}</loc>
+			<loc>{{ site.url | xml_encode }}{{ page.url | remove: "index.html" | xml_encode }}</loc>
 			{% if page.sitemap.lastmod %}
 				<lastmod>{{ page.sitemap.lastmod | date: "%Y-%m-%d" }}</lastmod>
 			{% elsif page.date %}
@@ -62,4 +62,5 @@ sitemap:
 		</url>
 		{% endunless %}
 	{% endfor %}
-</urlset>
+</urlset>
+{% endcapture %}{{ var | normalize_whitespace }}