A platform for community discussion. Free, open, simple.
FIX: Patreon API pagination bugs causing missing pledges (#39058)
Rafael dos Santos Silvabbc2fed22b7a
diff --git a/plugins/discourse-patreon/lib/api.rb b/plugins/discourse-patreon/lib/api.rbindex ed5abec2d01..7747cda3757 100644--- a/plugins/discourse-patreon/lib/api.rb+++ b/plugins/discourse-patreon/lib/api.rb@@ -25,7 +25,13 @@ module Patreonlimiter_day.performed! unless limiter_day.can_perform?- full_url = "#{base_url}#{uri}"+ if uri.start_with?("http")+ full_url = uri+ base_url = nil+ else+ full_url = "#{base_url}#{uri}"+ end+Rails.logger.warn("Patreon API request: GET #{full_url}") if SiteSetting.patreon_verbose_logresponse =@@ -35,7 +41,7 @@ module Patreon"Authorization" => "Bearer #{SiteSetting.patreon_creator_access_token}","User-Agent" => "Discourse Patreon Plugin/#{Discourse::VERSION::STRING}",},- ).get(uri)+ ).get(full_url)limiter_hr.performed!limiter_day.performed!diff --git a/plugins/discourse-patreon/lib/api_version/v1.rb b/plugins/discourse-patreon/lib/api_version/v1.rbindex 0b7c9c56005..733d16a2862 100644--- a/plugins/discourse-patreon/lib/api_version/v1.rb+++ b/plugins/discourse-patreon/lib/api_version/v1.rb@@ -60,7 +60,13 @@ module Patreondef self.pull_pledges!(campaign_data)uris = campaign_data[:pledge_uris].dup- pledges_data = []++ if uris.blank?+ Patreon::Pledge.save!([], false, adapter: self)+ return+ end++ is_first_page = trueuris.each do |uri|pledge_data = Patreon::Api.get(uri)@@ -70,10 +76,11 @@ module Patreonuris << next_page_uri if next_page_uri.present?end- pledges_data << pledge_data if pledge_data.present?+ if pledge_data.present?+ Patreon::Pledge.save!([pledge_data], !is_first_page, adapter: self)+ is_first_page = false+ endend-- Patreon::Pledge.save!(pledges_data, adapter: self)enddef self.extract(pledge_data)@@ -84,7 +91,9 @@ module Patreonpledge_data["data"].each do |entry|if entry["type"] == "pledge"- patron_id = entry["relationships"]["patron"]["data"]["id"]+ patron_id = entry.dig("relationships", "patron", "data", "id")+ next if patron_id.nil?+attrs = entry["attributes"]unless entry["relationships"]["reward"]["data"].nil?@@ -93,7 +102,9 @@ module Patreonpledges[patron_id] = attrs["amount_cents"]declines[patron_id] = attrs["declined_since"] if attrs["declined_since"].present?elsif entry["type"] == "member"- patron_id = entry["relationships"]["user"]["data"]["id"]+ patron_id = entry.dig("relationships", "user", "data", "id")+ next if patron_id.nil?+attrs = entry["attributes"]currently_entitled_tiers = entry["relationships"]["currently_entitled_tiers"] || {}@@ -120,12 +131,14 @@ module Patreonrel = entry["relationships"]if entry["type"] == "pledge"- patron_id = rel["patron"]["data"]["id"]- reward_id = rel["reward"]["data"]["id"] if rel["reward"]["data"].present?+ patron_id = rel.dig("patron", "data", "id")+ return if patron_id.nil?+ reward_id = rel.dig("reward", "data", "id")reward_users[reward_id].reject! { |i| i == patron_id } if reward_id.present?elsif entry["type"] == "member"- patron_id = rel["user"]["data"]["id"]+ patron_id = rel.dig("user", "data", "id")+ return if patron_id.nil?(rel.dig("currently_entitled_tiers", "data") || []).each do |tier|(reward_users[tier["id"]] || []).reject! { |i| i == patron_id }@@ -138,7 +151,7 @@ module Patreondef self.get_patreon_id(data)entry = data["data"]key = entry["type"] == "member" ? "user" : "patron"- entry["relationships"][key]["data"]["id"]+ entry.dig("relationships", key, "data", "id")enddef self.webhook_triggersdiff --git a/plugins/discourse-patreon/lib/api_version/v2.rb b/plugins/discourse-patreon/lib/api_version/v2.rbindex e1ca4c551bb..fe202dade99 100644--- a/plugins/discourse-patreon/lib/api_version/v2.rb+++ b/plugins/discourse-patreon/lib/api_version/v2.rb@@ -67,22 +67,25 @@ module Patreondef self.pull_pledges!(campaign_data)campaign_ids = campaign_data[:campaign_ids]- members_data = []+ is_first_page = truecampaign_ids.each do |campaign_id|cursor = nilloop doresponse = Patreon::Api.get(members_data_url(campaign_id, cursor))- break if response.blank? || response["data"].blank?- members_data << response+ if response.blank? || response["data"].blank?+ Patreon::Pledge.save!([], false, adapter: self) if is_first_page+ break+ end++ Patreon::Pledge.save!([response], !is_first_page, adapter: self)+ is_first_page = falsecursor = response.dig("meta", "pagination", "cursors", "next")break if cursor.blank?endend-- Patreon::Pledge.save!(members_data, adapter: self)enddef self.extract(member_data)diff --git a/plugins/discourse-patreon/spec/lib/api_spec.rb b/plugins/discourse-patreon/spec/lib/api_spec.rbindex cd97910d8b6..b726b0f72c8 100644--- a/plugins/discourse-patreon/spec/lib/api_spec.rb+++ b/plugins/discourse-patreon/spec/lib/api_spec.rb@@ -26,6 +26,16 @@ RSpec.describe Patreon::Api doDiscourse.expects(:warn_exception).oncedescribed_class.campaign_dataend++ it "does not double the base URL when uri is already absolute" do+ absolute_url =+ "https://api.patreon.com/api/oauth2/api/campaigns/123/pledges?page%5Bcount%5D=100&sort=created"+ stub_url(200, absolute_url)++ result = described_class.get(absolute_url)+ expect(result).to eq({})+ expect(WebMock).to have_requested(:get, absolute_url)+ endendcontext "with API v2" do