geminispace.info

commit a4864548ca222495d94aa8a80c7d7778e74f5fed
parent 96731d16d3db6b277b5dacfcee6d4f4b91449a9c
Author: Natalie Pendragon <natpen@natpen.net>
Date:   Fri, 22 May 2020 09:18:24 -0400

[crawl] Optimize the index after crawls

Diffstat:
M
gus/crawl.py
 | 
3
+++

1 file changed, 3 insertions(+), 0 deletions(-)
diff --git a/gus/crawl.py b/gus/crawl.py
@@ -377,6 +377,9 @@ def run_crawl(should_run_destructive=False, seed_urls=[]):
 
     pickle_robot_file_map(robot_file_map, index_dir)
 
+    ix = open_dir(index_dir)
+    ix.optimize()
+
     index_statistics = compute_index_statistics(index_dir)
     print_index_statistics(index_statistics, crawl_statistics)
     if should_run_destructive: