Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Change URLs to POST request

Daoud Clarkedea14b2f0d32
diff --git a/mwmbl/platform/api.py b/mwmbl/platform/api.py
index fd945ff..28aca37 100644
--- a/mwmbl/platform/api.py
+++ b/mwmbl/platform/api.py
@@ -10,7 +10,7 @@ from ninja_jwt.controller import NinjaJWTDefaultController
from mwmbl.models import MwmblUser, DomainSubmission, SearchResultVote
from mwmbl.platform.schemas import (
Registration, ConfirmEmail, DomainSubmissionSchema, UpdateDomainSubmission,
- VoteRequest, VoteRemoveRequest, VoteResponse, VoteStats, UserVoteHistory
+ VoteRequest, VoteRemoveRequest, VoteStatsRequest, VoteResponse, VoteStats, UserVoteHistory
)
api = NinjaExtraAPI(urls_namespace="platform")
@@ -182,28 +182,27 @@ def vote_on_search_result(request, vote_request: VoteRequest):
return {"status": "ok", "message": f"Vote {action} successfully."}
"/search-results/votes",
response=VoteResponse,
auth=JWTAuth(),
summary="Get vote statistics for search results",
- description="Retrieve vote counts (upvotes and downvotes) for one or more URLs in the context of a specific search query. "
+ description="Retrieve vote counts (upvotes and downvotes) for multiple URLs in the context of a specific search query. "
"Also returns the current user's vote on each result if they have voted. "
- "URLs should be provided as a comma-separated list.",
+ "This endpoint uses POST to handle large numbers of URLs that would exceed URL length limits.",
tags=["Search Result Voting"]
)
-def get_vote_counts(request, query: str, urls: str):
+def get_vote_counts(request, vote_stats_request: VoteStatsRequest):
check_email_verified(request)
- url_list = [url.strip() for url in urls.split(',') if url.strip()]
- if not url_list:
+ if not vote_stats_request.urls:
raise InvalidRequest("At least one URL must be provided.", status=400)
# Get vote counts for each URL
vote_data = {}
- for url in url_list:
+ for url in vote_stats_request.urls:
# Get aggregated vote counts
- votes = SearchResultVote.objects.filter(url=url, query=query)
+ votes = SearchResultVote.objects.filter(url=url, query=vote_stats_request.query)
upvotes = votes.filter(vote_type='upvote').count()
downvotes = votes.filter(vote_type='downvote').count()
diff --git a/mwmbl/platform/schemas.py b/mwmbl/platform/schemas.py
index 180cbab..6c1fdeb 100644
--- a/mwmbl/platform/schemas.py
+++ b/mwmbl/platform/schemas.py
@@ -77,6 +77,19 @@ class VoteStats(Schema):
)
+class VoteStatsRequest(Schema):
+ """Request schema for getting vote statistics for multiple URLs."""
+
+ query: str = Field(
+ description="The search query that returned these results",
+ example="python tutorial"
+ )
+ urls: list[str] = Field(
+ description="List of URLs to get vote statistics for",
+ example=["https://example.com/article", "https://another-site.com/page"]
+ )
+
+
class VoteResponse(Schema):
"""Response schema containing vote statistics for multiple URLs."""
diff --git a/test/test_comma_urls.py b/test/test_comma_urls.py
new file mode 100644
index 0000000..ce56a15
--- /dev/null
+++ b/test/test_comma_urls.py
@@ -0,0 +1,227 @@
+import pytest
+import json
+from urllib.parse import quote
+from django.contrib.auth import get_user_model
+from allauth.account.models import EmailAddress
+from ninja_jwt.tokens import RefreshToken
+
+from mwmbl.models import SearchResultVote
+
+
+User = get_user_model()
+
+
+def verified_user():
+ """Create a user with verified email"""
+ user = User.objects.create_user(
+ username='testuser',
+ password='testpass123'
+ )
+ EmailAddress.objects.create(
+ user=user,
+ verified=True,
+ primary=True
+ )
+ return user
+
+
+def access_token(verified_user):
+ """Generate JWT access token for verified user"""
+ refresh = RefreshToken.for_user(verified_user)
+ return str(refresh.access_token)
+
+
+def test_urls_with_commas_single_url(client, verified_user, access_token):
+ """Test handling URLs with commas using single URL parameter"""
+ # URL with comma in the path (like the original issue)
+ url_with_comma = 'https://en.wikipedia.org/wiki/"Hello,_World!"_program'
+ query = "hello world"
+
+ # Create a vote for this URL
+ SearchResultVote.objects.create(
+ user=verified_user,
+ url=url_with_comma,
+ query=query,
+ vote_type='upvote'
+ )
+
+ # Test getting vote counts using POST
+ vote_stats_request = {
+ "query": query,
+ "urls": [url_with_comma]
+ }
+
+ response = client.post(
+ '/api/v1/platform/search-results/votes',
+ data=json.dumps(vote_stats_request),
+ content_type='application/json',
+ HTTP_AUTHORIZATION=f'Bearer {access_token}'
+ )
+
+ assert response.status_code == 200
+
+ response_data = response.json()
+ assert 'votes' in response_data
+ assert url_with_comma in response_data['votes']
+
+ vote_stats = response_data['votes'][url_with_comma]
+ assert vote_stats['upvotes'] == 1
+ assert vote_stats['downvotes'] == 0
+ assert vote_stats['user_vote'] == 'upvote'
+
+
+def test_urls_with_commas_multiple_urls(client, verified_user, access_token):
+ """Test handling multiple URLs with commas using POST"""
+ # Multiple URLs with commas (from the original issue)
+ url1 = 'https://en.wikipedia.org/wiki/"Hello,_World!"_program'
+ url2 = 'https://en.wikipedia.org/wiki/Hello_Twelve,_Hello_Thirteen,_Hello_Love'
+ query = "hello"
+
+ # Create votes for these URLs
+ SearchResultVote.objects.create(
+ user=verified_user,
+ url=url1,
+ query=query,
+ vote_type='upvote'
+ )
+ SearchResultVote.objects.create(
+ user=verified_user,
+ url=url2,
+ query=query,
+ vote_type='downvote'
+ )
+
+ # Test getting vote counts using POST
+ vote_stats_request = {
+ "query": query,
+ "urls": [url1, url2]
+ }
+
+ response = client.post(
+ '/api/v1/platform/search-results/votes',
+ data=json.dumps(vote_stats_request),
+ content_type='application/json',
+ HTTP_AUTHORIZATION=f'Bearer {access_token}'
+ )
+
+ assert response.status_code == 200
+
+ response_data = response.json()
+ assert len(response_data['votes']) == 2
+ assert url1 in response_data['votes']
+ assert url2 in response_data['votes']
+
+ # Verify vote counts are correct
+ assert response_data['votes'][url1]['upvotes'] == 1
+ assert response_data['votes'][url1]['downvotes'] == 0
+ assert response_data['votes'][url1]['user_vote'] == 'upvote'
+
+ assert response_data['votes'][url2]['upvotes'] == 0
+ assert response_data['votes'][url2]['downvotes'] == 1
+ assert response_data['votes'][url2]['user_vote'] == 'downvote'
+
+
+def test_urls_with_various_special_characters(client, verified_user, access_token):
+ """Test URLs with various special characters that could cause parsing issues"""
+ special_urls = [
+ 'https://example.com/path?param=value,with,commas',
+ 'https://example.com/path?param=value%26other=a,b,c', # URL-encoded ampersand
+ 'https://example.com/path/with,comma/in/path',
+ 'https://example.com/path?query=hello%2Cworld', # URL-encoded comma
+ 'https://example.com/path?data={"key":"value,with,comma"}',
+ ]
+ query = "special characters test"
+
+ # Create votes for all URLs
+ for i, url in enumerate(special_urls):
+ vote_type = 'upvote' if i % 2 == 0 else 'downvote'
+ SearchResultVote.objects.create(
+ user=verified_user,
+ url=url,
+ query=query,
+ vote_type=vote_type
+ )
+
+ # Test getting vote counts using POST
+ vote_stats_request = {
+ "query": query,
+ "urls": special_urls
+ }
+
+ response = client.post(
+ '/api/v1/platform/search-results/votes',
+ data=json.dumps(vote_stats_request),
+ content_type='application/json',
+ HTTP_AUTHORIZATION=f'Bearer {access_token}'
+ )
+
+ assert response.status_code == 200
+
+ response_data = response.json()
+ assert len(response_data['votes']) == len(special_urls)
+
+ # Verify all URLs are present and have correct vote counts
+ for i, url in enumerate(special_urls):
+ assert url in response_data['votes']
+ expected_vote_type = 'upvote' if i % 2 == 0 else 'downvote'
+ assert response_data['votes'][url]['user_vote'] == expected_vote_type
+
+
+def test_voting_on_url_with_commas(client, verified_user, access_token):
+ """Test voting on URLs with commas works correctly"""
+ url_with_comma = 'https://en.wikipedia.org/wiki/"Hello,_World!"_program'
+ query = "hello world"
+
+ vote_data = {
+ "url": url_with_comma,
+ "query": query,
+ "vote_type": "upvote"
+ }
+
+ # Test voting
+ response = client.post(
+ '/api/v1/platform/search-results/vote',
+ data=json.dumps(vote_data),
+ content_type='application/json',
+ HTTP_AUTHORIZATION=f'Bearer {access_token}'
+ )
+
+ assert response.status_code == 200
+
+ response_data = response.json()
+ assert response_data['status'] == 'ok'
+ assert 'Vote created successfully' in response_data['message']
+
+ # Verify vote was created in database with exact URL
+ vote = SearchResultVote.objects.get(
+ user=verified_user,
+ url=url_with_comma,
+ query=query
+ )
+ assert vote.vote_type == 'upvote'
+
+ # Test retrieving the vote using POST
+ vote_stats_request = {
+ "query": query,
+ "urls": [url_with_comma]
+ }
+
+ response = client.post(
+ '/api/v1/platform/search-results/votes',
+ data=json.dumps(vote_stats_request),
+ content_type='application/json',
+ HTTP_AUTHORIZATION=f'Bearer {access_token}'
+ )
+
+ assert response.status_code == 200
+ response_data = response.json()
+ assert url_with_comma in response_data['votes']
+ assert response_data['votes'][url_with_comma]['user_vote'] == 'upvote'
diff --git a/test/test_large_url_list.py b/test/test_large_url_list.py
new file mode 100644
index 0000000..23bf0b6
--- /dev/null
+++ b/test/test_large_url_list.py
@@ -0,0 +1,198 @@
+import pytest
+import json
+from django.contrib.auth import get_user_model
+from allauth.account.models import EmailAddress
+from ninja_jwt.tokens import RefreshToken
+
+from mwmbl.models import SearchResultVote
+
+
+User = get_user_model()
+
+
+def verified_user():
+ """Create a user with verified email"""
+ user = User.objects.create_user(
+ username='testuser',
+ password='testpass123'
+ )
+ EmailAddress.objects.create(
+ user=user,
+ verified=True,
+ primary=True
+ )
+ return user
+
+
+def access_token(verified_user):
+ """Generate JWT access token for verified user"""
+ refresh = RefreshToken.for_user(verified_user)
+ return str(refresh.access_token)
+
+
+def test_large_url_list_post_method(client, verified_user, access_token):
+ """Test handling a large number of URLs (more than 160) that would fail with GET"""
+ # Generate 200 URLs to test well beyond the previous threshold of ~160
+ urls = []
+ query = "test query"
+
+ for i in range(200):
+ # Generate URLs with varying lengths and some special characters
+ url = f'https://example{i}.com/path/to/resource?param=value&id={i}&data={"some,data,with,commas"}'
+ urls.append(url)
+
+ # Create some votes for variety
+ vote_type = 'upvote' if i % 3 == 0 else 'downvote'
+ if i % 2 == 0: # Create votes for every other URL
+ SearchResultVote.objects.create(
+ user=verified_user,
+ url=url,
+ query=query,
+ vote_type=vote_type
+ )
+
+ # This would have failed with the old GET method due to URL length limits
+ vote_stats_request = {
+ "query": query,
+ "urls": urls
+ }
+
+ response = client.post(
+ '/api/v1/platform/search-results/votes',
+ data=json.dumps(vote_stats_request),
+ content_type='application/json',
+ HTTP_AUTHORIZATION=f'Bearer {access_token}'
+ )
+
+ assert response.status_code == 200
+
+ response_data = response.json()
+ assert 'votes' in response_data
+ assert len(response_data['votes']) == 200
+
+ # Verify all URLs are present and have correct vote counts
+ votes_with_data = 0
+ for i, url in enumerate(urls):
+ assert url in response_data['votes']
+ if i % 2 == 0: # URLs that have votes
+ votes_with_data += 1
+ expected_vote_type = 'upvote' if i % 3 == 0 else 'downvote'
+ assert response_data['votes'][url]['user_vote'] == expected_vote_type
+ if expected_vote_type == 'upvote':
+ assert response_data['votes'][url]['upvotes'] == 1
+ assert response_data['votes'][url]['downvotes'] == 0
+ else:
+ assert response_data['votes'][url]['upvotes'] == 0
+ assert response_data['votes'][url]['downvotes'] == 1
+ else: # URLs without votes
+ assert response_data['votes'][url]['upvotes'] == 0
+ assert response_data['votes'][url]['downvotes'] == 0
+ assert response_data['votes'][url]['user_vote'] is None
+
+ assert votes_with_data == 100 # Half of 200 URLs should have votes
+
+
+def test_very_long_urls_in_large_list(client, verified_user, access_token):
+ """Test handling URLs with very long paths that would be problematic in query strings"""
+ base_urls = []
+ query = "complex query with spaces and special chars !@#$%"
+
+ # Create some extremely long URLs that would definitely exceed URL limits in GET
+ for i in range(50):
+ long_path = "/very/long/path/" + "/segment" * 20 + f"/file{i}"
+ long_query_params = "&".join([f"param{j}=very_long_value_with_special_chars_{j}" for j in range(10)])
+ url = f'https://verylongdomainname{i}.example.com{long_path}?{long_query_params}'
+ base_urls.append(url)
+
+ # Create votes for some URLs
+ if i % 3 == 0:
+ SearchResultVote.objects.create(
+ user=verified_user,
+ url=url,
+ query=query,
+ vote_type='upvote'
+ )
+
+ # Add the original problematic URLs from the bug report
+ problematic_urls = [
+ 'http://keybase.io/hello',
+ 'http://keybase.io/hello_',
+ 'https://github.com/hello'
+ ]
+ base_urls.extend(problematic_urls)
+
+ vote_stats_request = {
+ "query": query,
+ "urls": base_urls
+ }
+
+ response = client.post(
+ '/api/v1/platform/search-results/votes',
+ data=json.dumps(vote_stats_request),
+ content_type='application/json',
+ HTTP_AUTHORIZATION=f'Bearer {access_token}'
+ )
+
+ assert response.status_code == 200
+
+ response_data = response.json()
+ assert len(response_data['votes']) == len(base_urls)
+
+ # Verify the original problematic URLs are handled correctly
+ for url in problematic_urls:
+ assert url in response_data['votes']
+ assert response_data['votes'][url]['upvotes'] == 0
+ assert response_data['votes'][url]['downvotes'] == 0
+ assert response_data['votes'][url]['user_vote'] is None
+
+
+def test_edge_case_empty_and_malformed_urls(client, verified_user, access_token):
+ """Test edge cases with empty strings and malformed URLs in the list"""
+ urls = [
+ 'https://valid-url.com/path',
+ '', # Empty string
+ 'https://another-valid.com/path',
+ 'not-a-valid-url', # Malformed URL
+ 'https://third-valid.com/path',
+ ]
+ query = "edge case test"
+
+ # Create vote for the first valid URL
+ SearchResultVote.objects.create(
+ user=verified_user,
+ url='https://valid-url.com/path',
+ query=query,
+ vote_type='upvote'
+ )
+
+ vote_stats_request = {
+ "query": query,
+ "urls": urls
+ }
+
+ response = client.post(
+ '/api/v1/platform/search-results/votes',
+ data=json.dumps(vote_stats_request),
+ content_type='application/json',
+ HTTP_AUTHORIZATION=f'Bearer {access_token}'
+ )
+
+ assert response.status_code == 200
+
+ response_data = response.json()
+ # Should include all URLs (even empty and malformed ones)
+ assert len(response_data['votes']) == len(urls)
+
+ # Valid URL should have vote data
+ assert response_data['votes']['https://valid-url.com/path']['user_vote'] == 'upvote'
+
+ # Empty and malformed URLs should have no votes
+ assert response_data['votes']['']['upvotes'] == 0
+ assert response_data['votes']['not-a-valid-url']['upvotes'] == 0
diff --git a/test/test_voting_api.py b/test/test_voting_api.py
index 7b0e8b2..c897e0e 100644
--- a/test/test_voting_api.py
+++ b/test/test_voting_api.py
@@ -231,8 +231,15 @@ def test_get_vote_counts_single_url(client, verified_user, verified_user2, acces
vote_type='downvote'
)
- response = client.get(
- f'/api/v1/platform/search-results/votes?query={sample_vote_data["query"]}&urls={sample_vote_data["url"]}',
+ vote_stats_request = {
+ "query": sample_vote_data["query"],
+ "urls": [sample_vote_data["url"]]
+ }
+
+ response = client.post(
+ '/api/v1/platform/search-results/votes',
+ data=json.dumps(vote_stats_request),
+ content_type='application/json',
HTTP_AUTHORIZATION=f'Bearer {access_token}'
)
@@ -269,9 +276,15 @@ def test_get_vote_counts_multiple_urls(client, verified_user, access_token):
vote_type='downvote'
)
- urls_param = f"{url1},{url2}"
- response = client.get(
- f'/api/v1/platform/search-results/votes?query={query}&urls={urls_param}',
+ vote_stats_request = {
+ "query": query,
+ "urls": [url1, url2]
+ }
+
+ response = client.post(
+ '/api/v1/platform/search-results/votes',
+ data=json.dumps(vote_stats_request),
+ content_type='application/json',
HTTP_AUTHORIZATION=f'Bearer {access_token}'
)
@@ -289,8 +302,15 @@ def test_get_vote_counts_multiple_urls(client, verified_user, access_token):
@pytest.mark.django_db
def test_get_vote_counts_no_urls(client, access_token):
"""Test getting vote counts without providing URLs"""
- response = client.get(
- '/api/v1/platform/search-results/votes?query=test&urls=',
+ vote_stats_request = {
+ "query": "test",
+ "urls": []
+ }
+
+ response = client.post(
+ '/api/v1/platform/search-results/votes',
+ data=json.dumps(vote_stats_request),
+ content_type='application/json',
HTTP_AUTHORIZATION=f'Bearer {access_token}'
)