geminispace.info

commit 0657f5703ef85ab8672b4a800d12ffb8df713d61
parent fe33be1c978c66af467b16787b18e63672120acb
Author: Natalie Pendragon <natpen@natpen.net>
Date:   Sun, 15 Nov 2020 08:30:01 -0500

More exclusions

Diffstat:
M
gus/crawl.py
 | 
8
++++++++

1 file changed, 8 insertions(+), 0 deletions(-)
diff --git a/gus/crawl.py b/gus/crawl.py
@@ -133,7 +133,15 @@ EXCLUDED_URL_PREFIXES = [
     # issues in its content or something, but that's a problem for a
     # different day
     "gemini://gemini.spam.works/users/dvn/archive/",
+    "gemini://gem.denarii.cloud/chaindata.zip",
 
+    # full web proxy
+    "gemini://drewdevault.com/cgi-bin/web.sh?",
+
+    # killing crawl, I think maybe because it's too big
+    "gemini://gem.denarii.cloud/pichaindata.zip",
+
+    "gemini://matrix.kiwifarms.net",
 ]
 
 EXCLUDED_URL_PATHS = [