An open source, non-profit web search engine
Change URLs to POST request
Daoud Clarkedea14b2f0d32
diff --git a/mwmbl/platform/api.py b/mwmbl/platform/api.pyindex fd945ff..28aca37 100644--- a/mwmbl/platform/api.py+++ b/mwmbl/platform/api.py@@ -10,7 +10,7 @@ from ninja_jwt.controller import NinjaJWTDefaultControllerfrom mwmbl.models import MwmblUser, DomainSubmission, SearchResultVotefrom mwmbl.platform.schemas import (Registration, ConfirmEmail, DomainSubmissionSchema, UpdateDomainSubmission,- VoteRequest, VoteRemoveRequest, VoteResponse, VoteStats, UserVoteHistory+ VoteRequest, VoteRemoveRequest, VoteStatsRequest, VoteResponse, VoteStats, UserVoteHistory)api = NinjaExtraAPI(urls_namespace="platform")@@ -182,28 +182,27 @@ def vote_on_search_result(request, vote_request: VoteRequest):return {"status": "ok", "message": f"Vote {action} successfully."}"/search-results/votes",response=VoteResponse,auth=JWTAuth(),summary="Get vote statistics for search results",- description="Retrieve vote counts (upvotes and downvotes) for one or more URLs in the context of a specific search query. "+ description="Retrieve vote counts (upvotes and downvotes) for multiple URLs in the context of a specific search query. ""Also returns the current user's vote on each result if they have voted. "- "URLs should be provided as a comma-separated list.",+ "This endpoint uses POST to handle large numbers of URLs that would exceed URL length limits.",tags=["Search Result Voting"])-def get_vote_counts(request, query: str, urls: str):+def get_vote_counts(request, vote_stats_request: VoteStatsRequest):check_email_verified(request)- url_list = [url.strip() for url in urls.split(',') if url.strip()]- if not url_list:+ if not vote_stats_request.urls:raise InvalidRequest("At least one URL must be provided.", status=400)# Get vote counts for each URLvote_data = {}- for url in url_list:+ for url in vote_stats_request.urls:# Get aggregated vote counts- votes = SearchResultVote.objects.filter(url=url, query=query)+ votes = SearchResultVote.objects.filter(url=url, query=vote_stats_request.query)upvotes = votes.filter(vote_type='upvote').count()downvotes = votes.filter(vote_type='downvote').count()diff --git a/mwmbl/platform/schemas.py b/mwmbl/platform/schemas.pyindex 180cbab..6c1fdeb 100644--- a/mwmbl/platform/schemas.py+++ b/mwmbl/platform/schemas.py@@ -77,6 +77,19 @@ class VoteStats(Schema):)+class VoteStatsRequest(Schema):+ """Request schema for getting vote statistics for multiple URLs."""++ query: str = Field(+ description="The search query that returned these results",+ example="python tutorial"+ )+ urls: list[str] = Field(+ description="List of URLs to get vote statistics for",+ example=["https://example.com/article", "https://another-site.com/page"]+ )++class VoteResponse(Schema):"""Response schema containing vote statistics for multiple URLs."""diff --git a/test/test_comma_urls.py b/test/test_comma_urls.pynew file mode 100644index 0000000..ce56a15--- /dev/null+++ b/test/test_comma_urls.py@@ -0,0 +1,227 @@+import pytest+import json+from urllib.parse import quote+from django.contrib.auth import get_user_model+from allauth.account.models import EmailAddress+from ninja_jwt.tokens import RefreshToken++from mwmbl.models import SearchResultVote+++User = get_user_model()+++def verified_user():+ """Create a user with verified email"""+ user = User.objects.create_user(+ username='testuser',+ email='[email protected]',+ password='testpass123'+ )+ EmailAddress.objects.create(+ user=user,+ email='[email protected]',+ verified=True,+ primary=True+ )+ return user+++def access_token(verified_user):+ """Generate JWT access token for verified user"""+ refresh = RefreshToken.for_user(verified_user)+ return str(refresh.access_token)+++def test_urls_with_commas_single_url(client, verified_user, access_token):+ """Test handling URLs with commas using single URL parameter"""+ # URL with comma in the path (like the original issue)+ url_with_comma = 'https://en.wikipedia.org/wiki/"Hello,_World!"_program'+ query = "hello world"++ # Create a vote for this URL+ SearchResultVote.objects.create(+ user=verified_user,+ url=url_with_comma,+ query=query,+ vote_type='upvote'+ )++ # Test getting vote counts using POST+ vote_stats_request = {+ "query": query,+ "urls": [url_with_comma]+ }++ response = client.post(+ '/api/v1/platform/search-results/votes',+ data=json.dumps(vote_stats_request),+ content_type='application/json',+ HTTP_AUTHORIZATION=f'Bearer {access_token}'+ )++ assert response.status_code == 200++ response_data = response.json()+ assert 'votes' in response_data+ assert url_with_comma in response_data['votes']++ vote_stats = response_data['votes'][url_with_comma]+ assert vote_stats['upvotes'] == 1+ assert vote_stats['downvotes'] == 0+ assert vote_stats['user_vote'] == 'upvote'+++def test_urls_with_commas_multiple_urls(client, verified_user, access_token):+ """Test handling multiple URLs with commas using POST"""+ # Multiple URLs with commas (from the original issue)+ url1 = 'https://en.wikipedia.org/wiki/"Hello,_World!"_program'+ url2 = 'https://en.wikipedia.org/wiki/Hello_Twelve,_Hello_Thirteen,_Hello_Love'+ query = "hello"++ # Create votes for these URLs+ SearchResultVote.objects.create(+ user=verified_user,+ url=url1,+ query=query,+ vote_type='upvote'+ )+ SearchResultVote.objects.create(+ user=verified_user,+ url=url2,+ query=query,+ vote_type='downvote'+ )++ # Test getting vote counts using POST+ vote_stats_request = {+ "query": query,+ "urls": [url1, url2]+ }++ response = client.post(+ '/api/v1/platform/search-results/votes',+ data=json.dumps(vote_stats_request),+ content_type='application/json',+ HTTP_AUTHORIZATION=f'Bearer {access_token}'+ )++ assert response.status_code == 200++ response_data = response.json()+ assert len(response_data['votes']) == 2+ assert url1 in response_data['votes']+ assert url2 in response_data['votes']++ # Verify vote counts are correct+ assert response_data['votes'][url1]['upvotes'] == 1+ assert response_data['votes'][url1]['downvotes'] == 0+ assert response_data['votes'][url1]['user_vote'] == 'upvote'++ assert response_data['votes'][url2]['upvotes'] == 0+ assert response_data['votes'][url2]['downvotes'] == 1+ assert response_data['votes'][url2]['user_vote'] == 'downvote'+++def test_urls_with_various_special_characters(client, verified_user, access_token):+ """Test URLs with various special characters that could cause parsing issues"""+ special_urls = [+ 'https://example.com/path?param=value,with,commas',+ 'https://example.com/path?param=value%26other=a,b,c', # URL-encoded ampersand+ 'https://example.com/path/with,comma/in/path',+ 'https://example.com/path?query=hello%2Cworld', # URL-encoded comma+ 'https://example.com/path?data={"key":"value,with,comma"}',+ ]+ query = "special characters test"++ # Create votes for all URLs+ for i, url in enumerate(special_urls):+ vote_type = 'upvote' if i % 2 == 0 else 'downvote'+ SearchResultVote.objects.create(+ user=verified_user,+ url=url,+ query=query,+ vote_type=vote_type+ )++ # Test getting vote counts using POST+ vote_stats_request = {+ "query": query,+ "urls": special_urls+ }++ response = client.post(+ '/api/v1/platform/search-results/votes',+ data=json.dumps(vote_stats_request),+ content_type='application/json',+ HTTP_AUTHORIZATION=f'Bearer {access_token}'+ )++ assert response.status_code == 200++ response_data = response.json()+ assert len(response_data['votes']) == len(special_urls)++ # Verify all URLs are present and have correct vote counts+ for i, url in enumerate(special_urls):+ assert url in response_data['votes']+ expected_vote_type = 'upvote' if i % 2 == 0 else 'downvote'+ assert response_data['votes'][url]['user_vote'] == expected_vote_type+++def test_voting_on_url_with_commas(client, verified_user, access_token):+ """Test voting on URLs with commas works correctly"""+ url_with_comma = 'https://en.wikipedia.org/wiki/"Hello,_World!"_program'+ query = "hello world"++ vote_data = {+ "url": url_with_comma,+ "query": query,+ "vote_type": "upvote"+ }++ # Test voting+ response = client.post(+ '/api/v1/platform/search-results/vote',+ data=json.dumps(vote_data),+ content_type='application/json',+ HTTP_AUTHORIZATION=f'Bearer {access_token}'+ )++ assert response.status_code == 200++ response_data = response.json()+ assert response_data['status'] == 'ok'+ assert 'Vote created successfully' in response_data['message']++ # Verify vote was created in database with exact URL+ vote = SearchResultVote.objects.get(+ user=verified_user,+ url=url_with_comma,+ query=query+ )+ assert vote.vote_type == 'upvote'++ # Test retrieving the vote using POST+ vote_stats_request = {+ "query": query,+ "urls": [url_with_comma]+ }++ response = client.post(+ '/api/v1/platform/search-results/votes',+ data=json.dumps(vote_stats_request),+ content_type='application/json',+ HTTP_AUTHORIZATION=f'Bearer {access_token}'+ )++ assert response.status_code == 200+ response_data = response.json()+ assert url_with_comma in response_data['votes']+ assert response_data['votes'][url_with_comma]['user_vote'] == 'upvote'diff --git a/test/test_large_url_list.py b/test/test_large_url_list.pynew file mode 100644index 0000000..23bf0b6--- /dev/null+++ b/test/test_large_url_list.py@@ -0,0 +1,198 @@+import pytest+import json+from django.contrib.auth import get_user_model+from allauth.account.models import EmailAddress+from ninja_jwt.tokens import RefreshToken++from mwmbl.models import SearchResultVote+++User = get_user_model()+++def verified_user():+ """Create a user with verified email"""+ user = User.objects.create_user(+ username='testuser',+ email='[email protected]',+ password='testpass123'+ )+ EmailAddress.objects.create(+ user=user,+ email='[email protected]',+ verified=True,+ primary=True+ )+ return user+++def access_token(verified_user):+ """Generate JWT access token for verified user"""+ refresh = RefreshToken.for_user(verified_user)+ return str(refresh.access_token)+++def test_large_url_list_post_method(client, verified_user, access_token):+ """Test handling a large number of URLs (more than 160) that would fail with GET"""+ # Generate 200 URLs to test well beyond the previous threshold of ~160+ urls = []+ query = "test query"++ for i in range(200):+ # Generate URLs with varying lengths and some special characters+ url = f'https://example{i}.com/path/to/resource?param=value&id={i}&data={"some,data,with,commas"}'+ urls.append(url)++ # Create some votes for variety+ vote_type = 'upvote' if i % 3 == 0 else 'downvote'+ if i % 2 == 0: # Create votes for every other URL+ SearchResultVote.objects.create(+ user=verified_user,+ url=url,+ query=query,+ vote_type=vote_type+ )++ # This would have failed with the old GET method due to URL length limits+ vote_stats_request = {+ "query": query,+ "urls": urls+ }++ response = client.post(+ '/api/v1/platform/search-results/votes',+ data=json.dumps(vote_stats_request),+ content_type='application/json',+ HTTP_AUTHORIZATION=f'Bearer {access_token}'+ )++ assert response.status_code == 200++ response_data = response.json()+ assert 'votes' in response_data+ assert len(response_data['votes']) == 200++ # Verify all URLs are present and have correct vote counts+ votes_with_data = 0+ for i, url in enumerate(urls):+ assert url in response_data['votes']+ if i % 2 == 0: # URLs that have votes+ votes_with_data += 1+ expected_vote_type = 'upvote' if i % 3 == 0 else 'downvote'+ assert response_data['votes'][url]['user_vote'] == expected_vote_type+ if expected_vote_type == 'upvote':+ assert response_data['votes'][url]['upvotes'] == 1+ assert response_data['votes'][url]['downvotes'] == 0+ else:+ assert response_data['votes'][url]['upvotes'] == 0+ assert response_data['votes'][url]['downvotes'] == 1+ else: # URLs without votes+ assert response_data['votes'][url]['upvotes'] == 0+ assert response_data['votes'][url]['downvotes'] == 0+ assert response_data['votes'][url]['user_vote'] is None++ assert votes_with_data == 100 # Half of 200 URLs should have votes+++def test_very_long_urls_in_large_list(client, verified_user, access_token):+ """Test handling URLs with very long paths that would be problematic in query strings"""+ base_urls = []+ query = "complex query with spaces and special chars !@#$%"++ # Create some extremely long URLs that would definitely exceed URL limits in GET+ for i in range(50):+ long_path = "/very/long/path/" + "/segment" * 20 + f"/file{i}"+ long_query_params = "&".join([f"param{j}=very_long_value_with_special_chars_{j}" for j in range(10)])+ url = f'https://verylongdomainname{i}.example.com{long_path}?{long_query_params}'+ base_urls.append(url)++ # Create votes for some URLs+ if i % 3 == 0:+ SearchResultVote.objects.create(+ user=verified_user,+ url=url,+ query=query,+ vote_type='upvote'+ )++ # Add the original problematic URLs from the bug report+ problematic_urls = [+ 'http://keybase.io/hello',+ 'http://keybase.io/hello_',+ 'https://github.com/hello'+ ]+ base_urls.extend(problematic_urls)++ vote_stats_request = {+ "query": query,+ "urls": base_urls+ }++ response = client.post(+ '/api/v1/platform/search-results/votes',+ data=json.dumps(vote_stats_request),+ content_type='application/json',+ HTTP_AUTHORIZATION=f'Bearer {access_token}'+ )++ assert response.status_code == 200++ response_data = response.json()+ assert len(response_data['votes']) == len(base_urls)++ # Verify the original problematic URLs are handled correctly+ for url in problematic_urls:+ assert url in response_data['votes']+ assert response_data['votes'][url]['upvotes'] == 0+ assert response_data['votes'][url]['downvotes'] == 0+ assert response_data['votes'][url]['user_vote'] is None+++def test_edge_case_empty_and_malformed_urls(client, verified_user, access_token):+ """Test edge cases with empty strings and malformed URLs in the list"""+ urls = [+ 'https://valid-url.com/path',+ '', # Empty string+ 'https://another-valid.com/path',+ 'not-a-valid-url', # Malformed URL+ 'https://third-valid.com/path',+ ]+ query = "edge case test"++ # Create vote for the first valid URL+ SearchResultVote.objects.create(+ user=verified_user,+ url='https://valid-url.com/path',+ query=query,+ vote_type='upvote'+ )++ vote_stats_request = {+ "query": query,+ "urls": urls+ }++ response = client.post(+ '/api/v1/platform/search-results/votes',+ data=json.dumps(vote_stats_request),+ content_type='application/json',+ HTTP_AUTHORIZATION=f'Bearer {access_token}'+ )++ assert response.status_code == 200++ response_data = response.json()+ # Should include all URLs (even empty and malformed ones)+ assert len(response_data['votes']) == len(urls)++ # Valid URL should have vote data+ assert response_data['votes']['https://valid-url.com/path']['user_vote'] == 'upvote'++ # Empty and malformed URLs should have no votes+ assert response_data['votes']['']['upvotes'] == 0+ assert response_data['votes']['not-a-valid-url']['upvotes'] == 0diff --git a/test/test_voting_api.py b/test/test_voting_api.pyindex 7b0e8b2..c897e0e 100644--- a/test/test_voting_api.py+++ b/test/test_voting_api.py@@ -231,8 +231,15 @@ def test_get_vote_counts_single_url(client, verified_user, verified_user2, accesvote_type='downvote')- response = client.get(- f'/api/v1/platform/search-results/votes?query={sample_vote_data["query"]}&urls={sample_vote_data["url"]}',+ vote_stats_request = {+ "query": sample_vote_data["query"],+ "urls": [sample_vote_data["url"]]+ }++ response = client.post(+ '/api/v1/platform/search-results/votes',+ data=json.dumps(vote_stats_request),+ content_type='application/json',HTTP_AUTHORIZATION=f'Bearer {access_token}')@@ -269,9 +276,15 @@ def test_get_vote_counts_multiple_urls(client, verified_user, access_token):vote_type='downvote')- urls_param = f"{url1},{url2}"- response = client.get(- f'/api/v1/platform/search-results/votes?query={query}&urls={urls_param}',+ vote_stats_request = {+ "query": query,+ "urls": [url1, url2]+ }++ response = client.post(+ '/api/v1/platform/search-results/votes',+ data=json.dumps(vote_stats_request),+ content_type='application/json',HTTP_AUTHORIZATION=f'Bearer {access_token}')@@ -289,8 +302,15 @@ def test_get_vote_counts_multiple_urls(client, verified_user, access_token):@pytest.mark.django_dbdef test_get_vote_counts_no_urls(client, access_token):"""Test getting vote counts without providing URLs"""- response = client.get(- '/api/v1/platform/search-results/votes?query=test&urls=',+ vote_stats_request = {+ "query": "test",+ "urls": []+ }++ response = client.post(+ '/api/v1/platform/search-results/votes',+ data=json.dumps(vote_stats_request),+ content_type='application/json',HTTP_AUTHORIZATION=f'Bearer {access_token}')