geminispace.info

commit a4c8ddc92b939eb8d644ecc41d61f35e58e213c9
parent 80ad98435233fab6ae7642bb603353d01b577f2b
Author: Natalie Pendragon <natpen@natpen.net>
Date:   Sun, 23 Feb 2020 09:30:52 -0500

Add some new seed sites

Diffstat:
M
gus/crawl.py
 | 
4
+++-

1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/gus/crawl.py b/gus/crawl.py
@@ -23,6 +23,9 @@ SEED_URLS = [
     "gemini://tilde.pink:1965",
     "gemini://vger.cloud:1965",
     "gemini://yam655.com:1965",
+    "gemini://dump.royniang.com",
+    "gemini://konpeito.media",
+    "gemini://gemini.68kmentat.com",
 ]
 
 
@@ -86,7 +89,6 @@ def get_robots_file(url):
         if r is not None and r.status.startswith("2"):
             robot_file_parser = robotparser.RobotFileParser()
             robot_file_parser.parse(r.content)
-            print(robot_file_parser.site_maps())
             robot_file_map[robot_host] = robot_file_parser
         else:
             robot_file_map[robot_host] = None