commit b8eb04a22445fa63d716c1aaf03e0ae4bbf8b636
parent f6bd88672ea163d77bc3460ce460afb29b9c5468
Author: René Wagner <rwa@clttr.info>
Date: Sat, 17 Jul 2021 19:40:20 +0200
remove obsolete code
- threads in serve/views.py, serve/models.py and gus/lib/db_model.py
- run_index_statistics in gus/lib/index_statistics.py
Diffstat:
4 files changed, 1 insertion(+), 123 deletions(-)
diff --git a/gus/lib/db_model.py b/gus/lib/db_model.py
@@ -17,7 +17,7 @@ def init_db(filename=":memory:"):
"""
Bind an SQLite database to the Peewee ORM models.
"""
- models = [Link, Page, Thread, ThreadPage]
+ models = [Link, Page]
db = SqliteDatabase(filename)
db.bind(models)
db.create_tables(models)
@@ -59,22 +59,3 @@ class Link(Model):
def get_is_cross_host_like(from_resource, to_resource):
return from_resource.normalized_host_like != to_resource.normalized_host_like
-
-class Thread(Model):
- """
- Thread definitions.
- """
-
- updated_at = DateTimeField()
-
-
-class ThreadPage(Model):
- """
- Mapping table of threads to their member pages.
- """
-
- thread = ForeignKeyField(Thread, backref="pages", on_delete="CASCADE")
- page = ForeignKeyField(Page, backref="threads", on_delete="CASCADE")
- address = TextField()
- friendly_author = TextField()
- friendly_title = TextField()
diff --git a/gus/lib/index_statistics.py b/gus/lib/index_statistics.py
@@ -80,12 +80,6 @@ def log_index_statistics(index_statistics, crawl_statistics=None):
entry['charset'], entry['count'])
-def run_index_statistics():
- index_statistics = compute_index_statistics("index")
- log_index_statistics(index_statistics, None)
- # persist_index_statistics(index_statistics, "index-statistics.csv")
-
-
def persist_statistics(index_statistics, crawl_statistics, was_destructive, filename):
with open(filename, "a") as f:
f.write(serialize_statistics_line(index_statistics, crawl_statistics, was_destructive))
diff --git a/serve/models.py b/serve/models.py
@@ -71,90 +71,6 @@ ORDER BY l.is_cross_host_like, p_from.url ASC""",
external_backlink_urls = [b.url for b in backlinks if b.is_cross_host_like]
return internal_backlink_urls, external_backlink_urls
- def get_threads(self, sort="recency"):
- sort = sort.lower()
- if sort == "recency":
- threads_query = Thread.raw(
- """SELECT t.*
- , tp.address
- , tp.friendly_author
- , tp.friendly_title
- , p.fetchable_url
- , p.url
- , MIN(c.timestamp) AS first_seen
-FROM (
- SELECT *
- FROM thread
- ORDER BY updated_at DESC
- LIMIT 50) AS t
-JOIN threadpage AS tp
-ON tp.thread_id == t.id
-JOIN page AS p
-ON p.id == tp.page_id
-JOIN crawl AS c
-ON c.page_id == p.id
-WHERE c.status == 20
-GROUP BY tp.id
-ORDER BY t.updated_at DESC, t.id ASC, tp.address ASC"""
- )
- elif sort == "length":
- threads_query = Thread.raw(
- """SELECT t.*
- , tp.address
- , tp.friendly_author
- , tp.friendly_title
- , p.fetchable_url
- , p.url
- , MIN(c.timestamp) AS first_seen
-FROM (
- SELECT t.*, COUNT(tp.id) AS thread_length
- FROM thread AS t
- JOIN threadpage AS tp
- ON tp.thread_id == t.id
- GROUP BY t.id
- ORDER BY thread_length DESC
- LIMIT 50) AS t
-JOIN threadpage AS tp
-ON tp.thread_id == t.id
-JOIN page AS p
-ON p.id == tp.page_id
-JOIN crawl AS c
-ON c.page_id == p.id
-WHERE c.status == 20
-GROUP BY tp.id
-ORDER BY t.thread_length DESC, t.updated_at DESC, t.id ASC, tp.address ASC"""
- )
- else:
- threads_query = ""
- threads = []
- last_date = None
- last_id = None
- for thread_member in threads_query.iterator():
- if thread_member.updated_at.date() != last_date:
- threads.append(
- {"threads": [], "date": thread_member.updated_at}
- )
- last_date = thread_member.updated_at.date()
- if thread_member.id != last_id:
- threads[-1]["threads"].append(
- {"members": [], "updated_at": thread_member.updated_at}
- )
- last_id = thread_member.id
- threads[-1]["threads"][-1]["members"].append(
- {
- "url": thread_member.url,
- "fetchable_url": thread_member.fetchable_url,
- "address": thread_member.address,
- "friendly_author": thread_member.friendly_author,
- "friendly_title": thread_member.friendly_title,
- "first_seen": datetime.strptime(
- thread_member.first_seen, "%Y-%m-%d %H:%M:%S.%f"
- ),
- }
- )
- # return sorted(threads, key=lambda x: (x["updated_at"], ), reverse=True)
- return threads
-
def _get_link_text(result):
if result["content_type"] == "input":
prompt_suffix = ": {}".format(result["prompt"])
diff --git a/serve/views.py b/serve/views.py
@@ -273,16 +273,3 @@ def backlinks(request):
return Response(Status.SUCCESS, "text/gemini", body)
else:
return Response(Status.INPUT, "Gemini URL")
-
-
-@app.route("/threads")
-def threads(request):
- sort = request.query or "recency"
- threads = gus.get_threads(sort)
- body = render_template(
- "threads.gmi",
- threads=threads,
- sort=sort,
- index_modification_time=gus.statistics["index_modification_time"],
- )
- return Response(Status.SUCCESS, "text/gemini", body)
