Skip to content

web search done right

use snippet in nsfw script instead of body

Mikkel Denkerd9848b19b287
diff --git a/scripts/generate_nsfw_dataset.py b/scripts/generate_nsfw_dataset.py
index 604b3e5d..f5172ee1 100755
--- a/scripts/generate_nsfw_dataset.py
+++ b/scripts/generate_nsfw_dataset.py
@@ -5084,10 +5084,13 @@ def search_nsfw(q):
def search_sfw(q):
return search({"query": q})
+def snippet_text(snip):
+ return ''.join([frag['text'] for frag in snip['text']['fragments']])
+
def content(search_results):
return [
- {"url": result["url"], "text": result["title"] + " " + result["body"]}
+ {"url": result["url"], "text": result["title"] + " " + snippet_text(result["snippet"])}
for result in search_results["webpages"]
]