commit 0657f5703ef85ab8672b4a800d12ffb8df713d61
parent fe33be1c978c66af467b16787b18e63672120acb
Author: Natalie Pendragon <natpen@natpen.net>
Date: Sun, 15 Nov 2020 08:30:01 -0500
More exclusions
Diffstat:
1 file changed, 8 insertions(+), 0 deletions(-)
diff --git a/gus/crawl.py b/gus/crawl.py
@@ -133,7 +133,15 @@ EXCLUDED_URL_PREFIXES = [
# issues in its content or something, but that's a problem for a
# different day
"gemini://gemini.spam.works/users/dvn/archive/",
+ "gemini://gem.denarii.cloud/chaindata.zip",
+ # full web proxy
+ "gemini://drewdevault.com/cgi-bin/web.sh?",
+
+ # killing crawl, I think maybe because it's too big
+ "gemini://gem.denarii.cloud/pichaindata.zip",
+
+ "gemini://matrix.kiwifarms.net",
]
EXCLUDED_URL_PATHS = [