Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Remove the unused Batch schema

Daoud Clarke0f30e1604c66
diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.py
index fbd9d3b..f7d22ef 100644
--- a/mwmbl/crawler/batch.py
+++ b/mwmbl/crawler/batch.py
@@ -111,25 +111,6 @@ class Item(Schema):
)
-class Batch(Schema):
- """A batch of crawled pages submitted by a crawler client."""
-
- user_id: str = Field(
- description=(
- "The crawler's private user ID (a 64-character hex string). "
- "This is hashed server-side before storage — the raw ID is never persisted."
- ),
- example="a" * 64,
- )
- items: list[Item] = Field(
- description="List of crawled URLs and their results. Maximum 100 items per batch.",
- )
- device_name: str = Field(
- description="The crawler's device name (platform.uname().node).",
- example="my-device",
- )
-
-
class NewBatchRequest(Schema):
"""Request for a new batch of URLs to crawl."""
diff --git a/mwmbl/settings.py b/mwmbl/settings.py
index 76522ed..19b1b3d 100644
--- a/mwmbl/settings.py
+++ b/mwmbl/settings.py
@@ -14,7 +14,6 @@ APPLICATION_KEY = os.environ.get("MWMBL_APPLICATION_KEY", "SECRETAPPLICATIONKEY"
KEY_ID = os.environ.get("MWMBL_KEY_ID", "SECRETKEYID")
ENDPOINT_URL = os.environ.get("MWMBL_ENDPOINT_URL", "https://s3.eu-central-003.backblazeb2.com")
BUCKET_NAME = os.environ.get("MWMBL_BUCKET_NAME", "mwmbl-eu-crawl")
-MAX_BATCH_SIZE = 100
USER_ID_LENGTH = 36
PUBLIC_USER_ID_LENGTH = 64
VERSION = "v1"