summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '2.4/docs/robots.txt')
-rw-r--r--2.4/docs/robots.txt11
1 files changed, 11 insertions, 0 deletions
diff --git a/2.4/docs/robots.txt b/2.4/docs/robots.txt
new file mode 100644
index 0000000..60e6ca3
--- /dev/null
+++ b/2.4/docs/robots.txt
@@ -0,0 +1,11 @@
+# exclude help system from robots
+User-agent: *
+Disallow: /manual/
+Disallow: /doc/
+Disallow: /gif/
+# but allow htdig to index our doc-tree
+User-agent: susedig
+Disallow:
+# disallow stress test
+user-agent: stress-agent
+Disallow: /