geminispace.info

commit f5ae878d3dbecb37a6f917e7ff736fa315aaa2d5
parent e103214e870beefeb2e219c0104a685e82945207
Author: René Wagner <rwa@clttr.info>
Date:   Sat, 16 Sep 2023 13:59:40 +0200

allow robots.txt served as any text/* mimetype

Diffstat:
M
gus/lib/gemini.py
 | 
3
+--
M
tests/gus/lib/test_gemini.py
 | 
10
++++++++++

2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/gus/lib/gemini.py b/gus/lib/gemini.py
@@ -13,7 +13,6 @@ from urllib.parse import (
 from urllib.robotparser import RobotFileParser
 
 import gusmobile
-
 from gus import constants
 from gus.lib.domain import is_domain
 
@@ -76,7 +75,7 @@ class GeminiRobotFileParser(RobotFileParser):
         if response is None:
             self.allow_all = True
             return
-        if not response.status.startswith("2") or not response.content_type == "text/plain":
+        if not response.status.startswith("2") or not response.content_type.startswith("text/"):
             self.allow_all = True
         else:
             self.parse(response.content.splitlines())
diff --git a/tests/gus/lib/test_gemini.py b/tests/gus/lib/test_gemini.py
@@ -193,3 +193,13 @@ Allow: /git
 Disallow: /git/dotfiles/
 """)
         self._assert_fetchable(rp, "/git/dotfiles/tree/0b0de929fa98457d22cbbcee65013ec261b660e2/atom/packages/ex-mode/node_modules/space-pen/node_modules/grim/node_modules/emissary/node_modules/es6-weak-map/node_modules/es5-ext/math/atanh/implement.js", False)
+
+    def test_disallow_libraryinured(self):
+        rp = self._get_parser("""
+     User-agent: indexer
+     Disallow: /document/
+     Disallow: /search/
+
+""")
+        self._assert_fetchable(rp, "gemini://library.inu.red/document/saul-newman-war-on-the-state-stirner-and-deleuze-s-anarchism", False)
+        self._assert_fetchable(rp, "gemini://library.inu.red", True)