geminispace.info

commit 1a1650dd7e619ef2102e5f445e1d079b06b689de
parent 4f69e396ef8299946043471e7a3ad7fea8e742b3
Author: Natalie Pendragon <natpen@natpen.net>
Date:   Mon, 16 Nov 2020 07:49:19 -0500

More exclusions

Diffstat:
M
gus/crawl.py
 | 
6
++++++

1 file changed, 6 insertions(+), 0 deletions(-)
diff --git a/gus/crawl.py b/gus/crawl.py
@@ -142,6 +142,12 @@ EXCLUDED_URL_PREFIXES = [
     "gemini://gem.denarii.cloud/pichaindata.zip",
 
     "gemini://matrix.kiwifarms.net",
+
+    # these threads seem to expire
+    "gemini://dioskouroi.xyz/thread",
+
+    # french news mirrors, there's just too much
+    "gemini://jpfox.fr/rss/",
 ]
 
 EXCLUDED_URL_PATHS = [