geminispace.info

commit 6e54e520144aea538eccf830b2985a41878f8899
parent c791a758f2e2804279ef964486b03854bf07b4a7
Author: René Wagner <rwagner@rw-net.de>
Date:   Mon,  8 Mar 2021 18:51:28 +0100

Merge branch 'master' of git://natpen.net/gus

Diffstat:
M
gus/crawl.py
 | 
2
++

1 file changed, 2 insertions(+), 0 deletions(-)
diff --git a/gus/crawl.py b/gus/crawl.py
@@ -495,6 +495,8 @@ def crawl_page(
         # RobotFileParser will return the higher level value (*) if 
         # no indexer section is found
         can_fetch = robots_file.can_fetch("indexer", gr.normalized_url)
+
+        # same approach as above - last value wins
         crawl_delay = robots_file.crawl_delay("indexer")
 
         if not can_fetch: