From fa4fa9ad7dd6cf840285f3bdce572d4e9bae1530 Mon Sep 17 00:00:00 2001
From: Felix Albrigtsen <felixalbrigtsen@gmail.com>
Date: Wed, 14 Jun 2023 23:26:28 +0200
Subject: [PATCH] Stop bots scraping mediawiki special pages/diffs/history
 (#45)

---
 www/robots.txt | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 www/robots.txt

diff --git a/www/robots.txt b/www/robots.txt
new file mode 100644
index 0000000..8c6b117
--- /dev/null
+++ b/www/robots.txt
@@ -0,0 +1,18 @@
+# This file set up by fredj according to proposed Standard of Robot
+# Exclusion at http://web.nexor.co.uk/mak/doc/robots/norobots.html
+#
+# Currently: allow all robots. 
+# (An empty 'Disallow' line, looking like this:)
+# User-agent: *              # Means: All robots.
+# Disallow:                  # Means: Disallow nothing.              
+
+User-agent: *
+Disallow: /usage
+Disallow: /news_archive
+Disallow: /cgi
+Disallow: /finger
+Disallow: /cgi-bin
+Disallow: /~pallo/usage/
+Disallow: /~pallo/historic/
+Disallow: /~meldahl/
+Disallow: /w/index.php?