Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Route all GitHub API requests through one rate-limited client (#40637)

Régis Hanol36a8a51ef007
diff --git a/Gemfile b/Gemfile
index 2c04756f3cb..2df98e77f04 100644
--- a/Gemfile
+++ b/Gemfile
@@ -298,7 +298,7 @@ gem "zendesk_api", require: false
# for discourse-subscriptions
gem "stripe", require: false
-# for discourse-github
+# for discourse-code-review
gem "sawyer", require: false
gem "octokit", require: false
diff --git a/app/models/problem_check.rb b/app/models/problem_check.rb
index 4492c095395..78a00e350f9 100644
--- a/app/models/problem_check.rb
+++ b/app/models/problem_check.rb
@@ -74,6 +74,7 @@ class ProblemCheck
ProblemCheck::FailingEmails,
ProblemCheck::ForceHttps,
ProblemCheck::GithubConfig,
+ ProblemCheck::GithubOneboxBackoff,
ProblemCheck::GoogleAnalyticsVersion,
ProblemCheck::GoogleOauth2Config,
ProblemCheck::GroupEmailCredentials,
@@ -217,12 +218,14 @@ class ProblemCheck
target_identifier = target.kind_of?(ActiveRecord::Base) ? target.id : target
Problem.new(
- I18n.t(
- override_key || translation_key,
- base_path: Discourse.base_path,
- **override_data.merge(
- target.present? ? translation_data(target) : translation_data,
- ).symbolize_keys,
+ SiteSettings::LabelFormatter.expand_setting_links(
+ I18n.t(
+ override_key || translation_key,
+ base_path: Discourse.base_path,
+ **override_data.merge(
+ target.present? ? translation_data(target) : translation_data,
+ ).symbolize_keys,
+ ),
),
priority: config.priority,
identifier:,
diff --git a/app/services/problem_check/github_onebox_backoff.rb b/app/services/problem_check/github_onebox_backoff.rb
new file mode 100644
index 00000000000..ca66dcd337f
--- /dev/null
+++ b/app/services/problem_check/github_onebox_backoff.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class ProblemCheck::GithubOneboxBackoff < ProblemCheck
+ self.priority = "low"
+ self.perform_every = 10.minutes
+
+ def call
+ backing_off = Onebox::GithubAccess.tokens.any? { |token| GithubRateLimit.backing_off?(token) }
+ backing_off ? problem : no_problem
+ end
+end
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index 30a74fff4a3..c9fa9a4c69a 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -1814,6 +1814,7 @@ en:
facebook_config: 'The server is configured to allow signup and login with Facebook (enable_facebook_logins), but the app id and app secret values are not set. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/configuring-facebook-login-for-discourse/13394" target="_blank">See this guide to learn more</a>.'
twitter_config: 'The server is configured to allow signup and login with Twitter (enable_twitter_logins), but the key and secret values are not set. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/configuring-twitter-login-for-discourse/13395" target="_blank">See this guide to learn more</a>.'
github_config: 'The server is configured to allow signup and login with GitHub (enable_github_logins), but the client id and secret values are not set. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/configuring-github-login-for-discourse/13745" target="_blank">See this guide to learn more</a>.'
+ github_onebox_backoff: "GitHub is rate-limiting the server, so GitHub links may not preview until the limit resets. Configuring {{setting:github_onebox_access_tokens}} raises the limit significantly."
s3_upload_config: 'The server is configured to upload files to S3, but at least one the following setting is not set: s3_access_key_id, s3_secret_access_key, s3_use_iam_profile, or s3_upload_bucket. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/how-to-set-up-image-uploads-to-s3/7229" target="_blank">See "How to set up image uploads to S3?" to learn more</a>.'
s3_backup_config: 'The server is configured to upload backups to S3, but at least one the following setting is not set: s3_access_key_id, s3_secret_access_key, s3_use_iam_profile, or s3_backup_bucket. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/how-to-set-up-image-uploads-to-s3/7229" target="_blank">See "How to set up image uploads to S3?" to learn more</a>.'
s3_cdn: 'The server is configured to upload files to S3, but there is no S3 CDN configured. This can lead to expensive S3 costs and slower site performance. <a href="https://meta.discourse.org/t/-/148916" target="_blank">See "Using Object Storage for Uploads" to learn more</a>.'
diff --git a/lib/discourse/github_api.rb b/lib/discourse/github_api.rb
new file mode 100644
index 00000000000..40dc1c45a8b
--- /dev/null
+++ b/lib/discourse/github_api.rb
@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "faraday"
+
+module Discourse
+ class GithubApi
+ API_ROOT = "https://api.github.com"
+ ALLOWED_HOSTS = %w[api.github.com raw.githubusercontent.com].freeze
+ DEFAULT_ACCEPT = "application/vnd.github+json"
+ TIMEOUT = 5
+ ETAG_MAX_BYTES = 512.kilobytes
+
+ class Error < StandardError
+ attr_reader :status
+
+ def initialize(message = nil, status: nil)
+ @status = status
+ super(message || "GitHub API error (status #{status})")
+ end
+ end
+
+ class NotFound < Error
+ end
+
+ class Unauthorized < Error
+ end
+
+ class RateLimited < Error
+ end
+
+ def self.for(token:)
+ (@clients ||= {})[token.to_s] ||= new(token)
+ end
+
+ def self.reset_clients!
+ @clients = {}
+ end
+
+ def initialize(token)
+ @token = token.presence
+ end
+
+ def backing_off?
+ GithubRateLimit.backing_off?(@token)
+ end
+
+ def get(path, accept: DEFAULT_ACCEPT, **params)
+ body = request(:get, path, params:, accept:)
+ body.present? ? ::MultiJson.load(body) : nil
+ end
+
+ def post(path, body = nil, accept: DEFAULT_ACCEPT)
+ response_body = request(:post, path, body:, accept:)
+ response_body.present? ? ::MultiJson.load(response_body) : nil
+ end
+
+ def raw_get(path, accept: DEFAULT_ACCEPT)
+ request(:get, path, accept:, cache_etag: false)
+ end
+
+ private
+
+ def request(method, path, params: {}, body: nil, accept:, cache_etag: true)
+ raise RateLimited, "GitHub rate limit backoff active" if backing_off?
+
+ url = path.start_with?("http") ? path : "#{API_ROOT}#{path}"
+ host = URI.parse(url).host
+ if ALLOWED_HOSTS.exclude?(host)
+ raise ArgumentError, "Refusing to call non-GitHub host: #{host.inspect}"
+ end
+
+ etag_key = (etag_cache_key(url, params, accept) if cache_etag && method == :get)
+ cached = (Discourse.cache.read(etag_key) if etag_key)
+
+ response = perform(method, url, params:, body:, accept:, etag: cached&.dig(:etag))
+ backed_off = GithubRateLimit.note_response_headers(response.headers, token: @token)
+
+ case response.status
+ when 200..299
+ store_etag(etag_key, response) if etag_key
+ response.body
+ when 304
+ cached&.dig(:body)
+ when 401
+ raise Unauthorized.new(status: 401)
+ when 404
+ raise NotFound.new(status: 404)
+ else
+ raise RateLimited.new("GitHub rate limited", status: response.status) if backed_off
+ raise Error.new(status: response.status)
+ end
+ end
+
+ def perform(method, url, params:, body:, accept:, etag:)
+ headers = { "Accept" => accept, "User-Agent" => Discourse.user_agent }
+ headers["Authorization"] = "Bearer #{@token}" if @token
+ headers["If-None-Match"] = etag if etag.present?
+ headers["Content-Type"] = "application/json" if body
+
+ connection.run_request(method, url, body && ::MultiJson.dump(body), headers) do |req|
+ req.params.update(params) if params.present?
+ req.options.timeout = TIMEOUT
+ req.options.open_timeout = TIMEOUT
+ end
+ rescue Faraday::Error => e
+ raise Error.new("GitHub API request failed: #{e.message}")
+ end
+
+ def connection
+ @connection ||= Faraday.new(nil) { |f| f.adapter FinalDestination::FaradayAdapter }
+ end
+
+ def etag_cache_key(url, params, accept)
+ identity = @token ? Digest::SHA1.hexdigest(@token) : "anon"
+ query = params.present? ? "?#{params.to_query}" : ""
+ "github_etag:#{identity}:#{accept}:#{url}#{query}"
+ end
+
+ def store_etag(key, response)
+ etag = response.headers["etag"]
+ return if etag.blank? || response.body.to_s.bytesize > ETAG_MAX_BYTES
+
+ Discourse.cache.write(key, { etag:, body: response.body }, expires_in: 1.day)
+ end
+ end
+end
diff --git a/lib/github_rate_limit.rb b/lib/github_rate_limit.rb
new file mode 100644
index 00000000000..a30cd9540db
--- /dev/null
+++ b/lib/github_rate_limit.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "digest"
+
+class GithubRateLimit
+ KEY_PREFIX = "onebox_github_backoff_"
+ MAX_SECONDS = 1.hour.to_i
+
+ class << self
+ def backing_off?(token = nil)
+ redis.get(key(token)).present?
+ end
+
+ def active?
+ redis.scan_each(match: "#{KEY_PREFIX}*").first.present?
+ end
+
+ def note_rate_limit(token: nil, retry_after: nil, remaining: nil, reset_at: nil)
+ seconds =
+ if retry_after.present?
+ retry_after.to_i
+ elsif remaining.to_s == "0" && reset_at.present?
+ reset_at.to_i - Time.now.to_i
+ end
+ return if seconds.nil?
+
+ seconds = seconds.clamp(1, MAX_SECONDS)
+ redis.setex(key(token), seconds, "1")
+ Rails.logger.warn("GitHub API rate limited; backing off for #{seconds}s")
+ seconds
+ end
+
+ def note_response_headers(headers, token: nil)
+ headers = headers.to_h.transform_keys { |k| k.to_s.downcase }
+ note_rate_limit(
+ token:,
+ retry_after: headers["retry-after"],
+ remaining: headers["x-ratelimit-remaining"],
+ reset_at: headers["x-ratelimit-reset"],
+ )
+ end
+
+ def key(token)
+ identity = token.present? ? Digest::SHA1.hexdigest(token) : "unauthenticated"
+ "#{KEY_PREFIX}#{identity}"
+ end
+
+ private
+
+ def redis
+ Discourse.redis.without_namespace
+ end
+ end
+end
diff --git a/lib/inline_oneboxer.rb b/lib/inline_oneboxer.rb
index 6d78c3776f2..b147056f56c 100644
--- a/lib/inline_oneboxer.rb
+++ b/lib/inline_oneboxer.rb
@@ -134,7 +134,10 @@ class InlineOneboxer
onebox = { url: url, title: title }
onebox[:css_class] = css_class if css_class.present?
- Discourse.cache.write(cache_key(url), onebox, expires_in: 1.day) if !opts[:skip_cache]
+ if !opts[:skip_cache]
+ ttl = onebox[:title].blank? ? 1.minute : 1.day
+ Discourse.cache.write(cache_key(url), onebox, expires_in: ttl)
+ end
onebox
end
diff --git a/lib/onebox/engine/github_actions_onebox.rb b/lib/onebox/engine/github_actions_onebox.rb
index 20f25a1ddcf..0ca0d26482b 100644
--- a/lib/onebox/engine/github_actions_onebox.rb
+++ b/lib/onebox/engine/github_actions_onebox.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
require_relative "../mixins/github_body"
-require_relative "../mixins/github_auth_header"
+require_relative "../mixins/github_api"
module Onebox
module Engine
@@ -9,7 +9,7 @@ module Onebox
include Engine
include LayoutSupport
include JSON
- include Onebox::Mixins::GithubAuthHeader
+ include Onebox::Mixins::GithubApi
matches_regexp(
%r{^https?://(?:www\.)?(?:(?:\w)+\.)?github\.com/(?<org>.+)/(?<repo>.+)/(actions/runs/[[:digit:]]+|pull/[[:digit:]]*/checks\?check_run_id=[[:digit:]]+)},
@@ -65,7 +65,7 @@ module Onebox
end
def data
- result = raw(github_auth_header(match[:org])).clone
+ result = raw.clone
status = "unknown"
if result["status"] == "completed"
@@ -84,7 +84,7 @@ module Onebox
elsif type == :pr_run
pr_url =
"https://api.github.com/repos/#{match[:org]}/#{match[:repo]}/pulls/#{match[:pr_id]}"
- ::MultiJson.load(URI.parse(pr_url).open(read_timeout: timeout))["title"]
+ load_json(pr_url)["title"]
end
{
diff --git a/lib/onebox/engine/github_blob_onebox.rb b/lib/onebox/engine/github_blob_onebox.rb
index 9ae691aa13a..e876d11d920 100644
--- a/lib/onebox/engine/github_blob_onebox.rb
+++ b/lib/onebox/engine/github_blob_onebox.rb
@@ -1,13 +1,11 @@
# frozen_string_literal: true
require_relative "../mixins/git_blob_onebox"
-require_relative "../mixins/github_auth_header"
+require_relative "../github_access"
module Onebox
module Engine
class GithubBlobOnebox
- include Onebox::Mixins::GithubAuthHeader
-
def self.git_regexp
%r{^https?://(www\.)?github\.com.*/blob/}
end
@@ -35,8 +33,14 @@ module Onebox
"https://raw.githubusercontent.com/#{match[:org]}/#{match[:repo]}/#{match[:sha1]}/#{match[:file]}"
end
- def auth_headers(match)
- github_auth_header(match[:org])
+ def auth_headers(_match)
+ {}
+ end
+
+ def fetch_raw(url, _headers)
+ Onebox::GithubAccess.client(match[:org]).raw_get(url)
+ rescue ::Discourse::GithubApi::Error => e
+ raise OpenURI::HTTPError.new(e.message, nil)
end
private
diff --git a/lib/onebox/engine/github_commit_onebox.rb b/lib/onebox/engine/github_commit_onebox.rb
index cccbd6d438f..02a0d2e9df9 100644
--- a/lib/onebox/engine/github_commit_onebox.rb
+++ b/lib/onebox/engine/github_commit_onebox.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
require_relative "../mixins/github_body"
-require_relative "../mixins/github_auth_header"
+require_relative "../mixins/github_api"
module Onebox
module Engine
@@ -10,7 +10,7 @@ module Onebox
include LayoutSupport
include JSON
include Onebox::Mixins::GithubBody
- include Onebox::Mixins::GithubAuthHeader
+ include Onebox::Mixins::GithubApi
matches_domain("github.com", "www.github.com")
always_https
@@ -24,9 +24,9 @@ module Onebox
end
def inline_data
- return if github_auth_header(match[:org]).blank?
+ return unless github_token?
- result = raw(github_auth_header(match[:org]))
+ result = raw
message = result["commit"]["message"].split("\n").first
{
title:
@@ -52,7 +52,7 @@ module Onebox
end
def data
- result = raw(github_auth_header(match[:org])).clone
+ result = raw.clone
lines = result["commit"]["message"].split("\n")
result["title"] = lines.first
diff --git a/lib/onebox/engine/github_gist_onebox.rb b/lib/onebox/engine/github_gist_onebox.rb
index 8e46c5ae954..65af85b2e30 100644
--- a/lib/onebox/engine/github_gist_onebox.rb
+++ b/lib/onebox/engine/github_gist_onebox.rb
@@ -1,11 +1,14 @@
# frozen_string_literal: true
+require_relative "../mixins/github_api"
+
module Onebox
module Engine
class GithubGistOnebox
include Engine
include LayoutSupport
include JSON
+ include Onebox::Mixins::GithubApi
MAX_FILES = 3
diff --git a/lib/onebox/engine/github_issue_onebox.rb b/lib/onebox/engine/github_issue_onebox.rb
index e0df2d0be75..23472e352c5 100644
--- a/lib/onebox/engine/github_issue_onebox.rb
+++ b/lib/onebox/engine/github_issue_onebox.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
require_relative "../mixins/github_body"
-require_relative "../mixins/github_auth_header"
+require_relative "../mixins/github_api"
module Onebox
module Engine
@@ -11,7 +11,7 @@ module Onebox
include LayoutSupport
include JSON
include Onebox::Mixins::GithubBody
- include Onebox::Mixins::GithubAuthHeader
+ include Onebox::Mixins::GithubApi
matches_regexp(
%r{^https?://(?:www\.)?(?:(?:\w)+\.)?github\.com/(?<org>.+)/(?<repo>.+)/issues/([[:digit:]]+)},
@@ -24,9 +24,9 @@ module Onebox
end
def inline_data
- return if github_auth_header(match["org"]).blank?
+ return unless github_token?
- result = raw(github_auth_header(match["org"]))
+ result = raw
{
title:
"#{result["title"]} - Issue ##{match["item_id"]} - #{match["org"]}/#{match["repo"]} - GitHub",
@@ -50,7 +50,7 @@ module Onebox
end
def data
- result = raw(github_auth_header(match[:org])).clone
+ result = raw.clone
repo = load_repo
created_at = Time.parse(result["created_at"])
closed_at = Time.parse(result["closed_at"]) if result["closed_at"]
@@ -86,12 +86,6 @@ module Onebox
def load_repo
load_json("https://api.github.com/repos/#{match[:org]}/#{match[:repo]}")
end
-
- def load_json(url)
- ::MultiJson.load(
- URI.parse(url).open({ read_timeout: timeout }.merge(github_auth_header(match[:org]))),
- )
- end
end
end
end
diff --git a/lib/onebox/engine/github_pull_request_onebox.rb b/lib/onebox/engine/github_pull_request_onebox.rb
index 1df4df1d660..18ef1aa4f20 100644
--- a/lib/onebox/engine/github_pull_request_onebox.rb
+++ b/lib/onebox/engine/github_pull_request_onebox.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
require_relative "../mixins/github_body"
-require_relative "../mixins/github_auth_header"
+require_relative "../mixins/github_api"
module Onebox
module Engine
@@ -10,7 +10,7 @@ module Onebox
include LayoutSupport
include JSON
include Onebox::Mixins::GithubBody
- include Onebox::Mixins::GithubAuthHeader
+ include Onebox::Mixins::GithubApi
matches_domain("github.com", "www.github.com")
always_https
@@ -24,7 +24,7 @@ module Onebox
end
def inline_data
- return if github_auth_header(match[:org]).blank? && !SiteSetting.github_pr_status_enabled
+ return if !github_token? && !SiteSetting.github_pr_status_enabled
if commit_sha = @url[%r{/commits?/(\h+)}, 1]
commit =
@@ -40,7 +40,7 @@ module Onebox
)
end
- pr_data = raw(github_auth_header(match[:org]))
+ pr_data = raw
result = {
title:
"#{pr_data["title"]} - Pull Request ##{match[:number]} - #{match[:org]}/#{match[:repository]} - GitHub",
@@ -65,7 +65,7 @@ module Onebox
end
def data
- result = raw(github_auth_header(match[:org])).clone
+ result = raw.clone
result["link"] = link
status_data = fetch_pr_status(result)
@@ -158,15 +158,6 @@ module Onebox
end
end
- def load_json(url)
- ::MultiJson.load(
- URI.parse(url).open({ read_timeout: timeout }.merge(github_auth_header(match[:org]))),
- )
- rescue OpenURI::HTTPError => e
- Rails.logger.warn("GitHub API error: #{e.io.status[0]} fetching #{url}")
- raise
- end
-
def fetch_pr_status(pr_data)
return unless SiteSetting.github_pr_status_enabled
diff --git a/lib/onebox/engine/github_repo_onebox.rb b/lib/onebox/engine/github_repo_onebox.rb
index 591b0b00333..db81ca7be51 100644
--- a/lib/onebox/engine/github_repo_onebox.rb
+++ b/lib/onebox/engine/github_repo_onebox.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
require_relative "../mixins/github_body"
-require_relative "../mixins/github_auth_header"
+require_relative "../mixins/github_api"
module Onebox
module Engine
@@ -9,7 +9,7 @@ module Onebox
include Engine
include LayoutSupport
include JSON
- include Onebox::Mixins::GithubAuthHeader
+ include Onebox::Mixins::GithubApi
matches_domain("github.com", "www.github.com")
always_https
@@ -23,9 +23,9 @@ module Onebox
end
def inline_data
- return if github_auth_header(match[:org]).blank?
+ return unless github_token?
- result = raw(github_auth_header(match[:org]))
+ result = raw
title = "GitHub - #{result["full_name"]}"
title += " - #{Onebox::Helpers.truncate(result["description"])}" if result[
"description"
@@ -43,7 +43,7 @@ module Onebox
end
def data
- result = raw(github_auth_header(match[:org])).clone
+ result = raw.clone
result["link"] = link
description = result["description"]
title = "GitHub - #{result["full_name"]}"
diff --git a/lib/onebox/github_access.rb b/lib/onebox/github_access.rb
new file mode 100644
index 00000000000..b7fc549524c
--- /dev/null
+++ b/lib/onebox/github_access.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Onebox
+ module GithubAccess
+ def self.token(github_org)
+ org_tokens = org_token_map
+ (org_tokens[github_org] || org_tokens["default"]).presence
+ end
+
+ def self.tokens
+ (org_token_map.values.filter_map(&:presence) << nil).uniq
+ end
+
+ def self.client(github_org)
+ ::Discourse::GithubApi.for(token: token(github_org))
+ end
+
+ def self.org_token_map
+ tokens = SiteSetting.github_onebox_access_tokens
+ return {} if tokens.blank?
+
+ tokens.split("\n").to_h { it.split("|") }
+ end
+ private_class_method :org_token_map
+ end
+end
diff --git a/lib/onebox/mixins/git_blob_onebox.rb b/lib/onebox/mixins/git_blob_onebox.rb
index 5f51a062ca5..97d9d0a3b57 100644
--- a/lib/onebox/mixins/git_blob_onebox.rb
+++ b/lib/onebox/mixins/git_blob_onebox.rb
@@ -190,18 +190,17 @@ module Onebox
@model_file = @lang.dup
@raw = "https://render.githubusercontent.com/view/solid?url=" + raw_template(m)
else
- contents =
- URI
- .parse(raw_template(m))
- .open({ read_timeout: timeout }.merge(auth_headers(m)))
- .read
+ contents = fetch_raw(raw_template(m), auth_headers(m)).to_s
+ utf8_contents = contents.dup.force_encoding(Encoding::UTF_8)
- if contents.encoding == Encoding::BINARY || contents.bytes.include?(0)
+ if contents.bytes.include?(0) || !utf8_contents.valid_encoding?
@raw = nil
@binary = true
return
end
+ contents = utf8_contents
+
contents_lines = contents.lines #get contents lines
contents_lines_size = contents_lines.size #get number of lines
@@ -240,6 +239,10 @@ module Onebox
end
end
+ def fetch_raw(url, headers)
+ URI.parse(url).open({ read_timeout: timeout }.merge(headers)).read
+ end
+
def data
@data ||= {
title: title,
diff --git a/lib/onebox/mixins/github_api.rb b/lib/onebox/mixins/github_api.rb
new file mode 100644
index 00000000000..fd350a5dc88
--- /dev/null
+++ b/lib/onebox/mixins/github_api.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "open-uri"
+require_relative "../github_access"
+
+module Onebox
+ module Mixins
+ module GithubApi
+ class GithubRateLimited < OpenURI::HTTPError
+ def initialize(message = "GitHub API rate limit backoff is active")
+ super(message, nil)
+ end
+ end
+
+ def load_json(url, github_org = match_org)
+ github_fetch(url, Onebox::GithubAccess.client(github_org))
+ end
+
+ def github_token?(github_org = match_org)
+ Onebox::GithubAccess.token(github_org).present?
+ end
+
+ private
+
+ def raw
+ @raw ||= load_json(url)
+ end
+
+ def match_org
+ match.names.include?("org") ? match[:org] : nil
+ end
+
+ def github_fetch(url, client)
+ client.get(url)
+ rescue ::Discourse::GithubApi::RateLimited
+ raise GithubRateLimited
+ rescue ::Discourse::GithubApi::Error => e
+ raise OpenURI::HTTPError.new(e.message, nil)
+ end
+ end
+ end
+end
diff --git a/lib/onebox/mixins/github_auth_header.rb b/lib/onebox/mixins/github_auth_header.rb
deleted file mode 100644
index 8754b3c84d5..00000000000
--- a/lib/onebox/mixins/github_auth_header.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Onebox
- module Mixins
- module GithubAuthHeader
- def github_auth_header(github_org)
- return {} if SiteSetting.github_onebox_access_tokens.blank?
-
- org_tokens = SiteSetting.github_onebox_access_tokens.split("\n").to_h { it.split("|") }
-
- token = org_tokens[github_org] || org_tokens["default"]
-
- return {} if token.blank?
-
- { "Authorization" => "Bearer #{token}" }
- end
- end
- end
-end
diff --git a/lib/oneboxer.rb b/lib/oneboxer.rb
index 31cd26195da..79aa175ad63 100644
--- a/lib/oneboxer.rb
+++ b/lib/oneboxer.rb
@@ -520,100 +520,107 @@ module Oneboxer
end
def self.external_onebox(url, available_strategies = nil)
- Discourse
- .cache
- .fetch(onebox_cache_key(url), expires_in: 1.day) do
- uri = URI(url)
- available_strategies ||= Oneboxer.ordered_strategies(uri.hostname)
- strategy = available_strategies.shift
-
- max_redirects = 0 if SiteSetting.block_onebox_on_redirect
- fd =
- FinalDestination.new(
- url,
- get_final_destination_options(url, strategy).merge(
- stop_at_blocked_pages: true,
- max_redirects: max_redirects,
- initial_https_redirect_ignore_limit: SiteSetting.block_onebox_on_redirect,
- ),
- )
- uri = fd.resolve
-
- return blank_onebox if fd.status == :blocked_page
-
- if fd.status != :resolved
- args = { link: url }
- if fd.status == :invalid_address
- args[:error_message] = I18n.t("errors.onebox.invalid_address", hostname: fd.hostname)
- elsif (fd.status_code || uri.nil?) && available_strategies.present?
- # Try a different oneboxing strategy, if we have any options left:
- return external_onebox(url, available_strategies)
- elsif fd.status_code
- args[:error_message] = I18n.t(
- "errors.onebox.error_response",
- status_code: fd.status_code,
- )
- end
+ key = onebox_cache_key(url)
+ cached = Discourse.cache.read(key)
+ return cached if cached
- error_box = blank_onebox
- error_box[:preview] = preview_error_onebox(args)
- return error_box
- end
+ result = compute_external_onebox(url, available_strategies)
+ Discourse.cache.write(key, result, expires_in: external_onebox_ttl(result))
+ result
+ end
+
+ def self.external_onebox_ttl(result)
+ result[:onebox].blank? ? 1.minute : 1.day
+ end
+
+ def self.compute_external_onebox(url, available_strategies = nil)
+ uri = URI(url)
+ available_strategies ||= Oneboxer.ordered_strategies(uri.hostname)
+ strategy = available_strategies.shift
+
+ max_redirects = 0 if SiteSetting.block_onebox_on_redirect
+ fd =
+ FinalDestination.new(
+ url,
+ get_final_destination_options(url, strategy).merge(
+ stop_at_blocked_pages: true,
+ max_redirects: max_redirects,
+ initial_https_redirect_ignore_limit: SiteSetting.block_onebox_on_redirect,
+ ),
+ )
+ uri = fd.resolve
+
+ return blank_onebox if fd.status == :blocked_page
+
+ if fd.status != :resolved
+ args = { link: url }
+ if fd.status == :invalid_address
+ args[:error_message] = I18n.t("errors.onebox.invalid_address", hostname: fd.hostname)
+ elsif (fd.status_code || uri.nil?) && available_strategies.present?
+ # Try a different oneboxing strategy, if we have any options left:
+ return compute_external_onebox(url, available_strategies)
+ elsif fd.status_code
+ args[:error_message] = I18n.t("errors.onebox.error_response", status_code: fd.status_code)
+ end
+
+ error_box = blank_onebox
+ error_box[:preview] = preview_error_onebox(args)
+ return error_box
+ end
- return blank_onebox if uri.blank?
-
- onebox_options = {
- max_width: 695,
- sanitize_config: Onebox::SanitizeConfig::DISCOURSE_ONEBOX,
- allowed_iframe_origins: allowed_iframe_origins,
- hostname: GlobalSetting.hostname,
- facebook_app_access_token: SiteSetting.facebook_app_access_token,
- disable_media_download_controls: SiteSetting.disable_onebox_media_download_controls,
- content_type: fd.content_type,
- }
-
- onebox_options[:cookie] = fd.cookie if fd.cookie
-
- preview_result = Onebox.preview(uri.to_s, onebox_options)
- result = {
- onebox: WordWatcher.censor(preview_result.to_s),
- preview: WordWatcher.censor(preview_result.placeholder_html.to_s),
- }
-
- # NOTE: Call preview_result.errors after calling placeholder_html
- if preview_result.errors.any?
- error_keys = preview_result.errors.keys
- skip_if_only_error = [:image]
- unless error_keys.length == 1 && skip_if_only_error.include?(error_keys.first)
- missing_attributes = error_keys.map(&:to_s).sort.join(I18n.t("word_connector.comma"))
- error_message =
- I18n.t(
- "errors.onebox.missing_data",
- missing_attributes: missing_attributes,
- count: error_keys.size,
- )
- args = preview_result.verified_data.merge(error_message: error_message)
-
- if result[:preview].blank?
- result[:preview] = preview_error_onebox(args)
- else
- doc = Nokogiri::HTML5.fragment(result[:preview])
- aside = doc.at("aside")
-
- if aside
- # Add an error message to the preview that was returned
- error_fragment = preview_error_onebox_fragment(args)
- aside.add_child(error_fragment)
- result[:preview] = doc.to_html
- end
- end
+ return blank_onebox if uri.blank?
+
+ onebox_options = {
+ max_width: 695,
+ sanitize_config: Onebox::SanitizeConfig::DISCOURSE_ONEBOX,
+ allowed_iframe_origins: allowed_iframe_origins,
+ hostname: GlobalSetting.hostname,
+ facebook_app_access_token: SiteSetting.facebook_app_access_token,
+ disable_media_download_controls: SiteSetting.disable_onebox_media_download_controls,
+ content_type: fd.content_type,
+ }
+
+ onebox_options[:cookie] = fd.cookie if fd.cookie
+
+ preview_result = Onebox.preview(uri.to_s, onebox_options)
+ result = {
+ onebox: WordWatcher.censor(preview_result.to_s),
+ preview: WordWatcher.censor(preview_result.placeholder_html.to_s),
+ }
+
+ # NOTE: Call preview_result.errors after calling placeholder_html
+ if preview_result.errors.any?
+ error_keys = preview_result.errors.keys
+ skip_if_only_error = [:image]
+ unless error_keys.length == 1 && skip_if_only_error.include?(error_keys.first)
+ missing_attributes = error_keys.map(&:to_s).sort.join(I18n.t("word_connector.comma"))
+ error_message =
+ I18n.t(
+ "errors.onebox.missing_data",
+ missing_attributes: missing_attributes,
+ count: error_keys.size,
+ )
+ args = preview_result.verified_data.merge(error_message: error_message)
+
+ if result[:preview].blank?
+ result[:preview] = preview_error_onebox(args)
+ else
+ doc = Nokogiri::HTML5.fragment(result[:preview])
+ aside = doc.at("aside")
+
+ if aside
+ # Add an error message to the preview that was returned
+ error_fragment = preview_error_onebox_fragment(args)
+ aside.add_child(error_fragment)
+ result[:preview] = doc.to_html
end
end
+ end
+ end
- Oneboxer.cache_preferred_strategy(uri.hostname, strategy)
+ Oneboxer.cache_preferred_strategy(uri.hostname, strategy)
- result
- end
+ result
end
def self.preview_error_onebox(args, is_fragment = false)
diff --git a/plugins/discourse-ai/lib/agents/tools/github_diff.rb b/plugins/discourse-ai/lib/agents/tools/github_diff.rb
index cb67f5fd34f..069286be082 100644
--- a/plugins/discourse-ai/lib/agents/tools/github_diff.rb
+++ b/plugins/discourse-ai/lib/agents/tools/github_diff.rb
@@ -167,41 +167,14 @@ module DiscourseAi
end
def fetch_diff(api_url, type)
- info = nil
- diff_body = nil
- response_code = "unknown error"
-
- send_http_request(
- api_url,
- headers: {
- "Accept" => "application/json",
- },
- authenticate_github: true,
- ) do |response|
- response_code = response.code
- info = JSON.parse(read_response_body(response)) if response_code == "200"
- end
-
- if response_code == "200"
- send_http_request(
- api_url,
- headers: {
- "Accept" => "application/vnd.github.v3.diff",
- },
- authenticate_github: true,
- ) do |response|
- response_code = response.code
- diff_body = read_response_body(response)
- end
- end
-
- if response_code == "200" && info && diff_body
- diff = self.class.sort_and_shorten_diff(diff_body)
- diff = truncate(diff, max_length: 20_000, percent_length: 0.3, llm: llm)
- yield(info, diff)
- else
- { error: "Failed to retrieve the #{type} information. Status code: #{response_code}" }
- end
+ info = github_client.get(api_url)
+ diff_body = github_client.raw_get(api_url, accept: "application/vnd.github.v3.diff")
+
+ diff = self.class.sort_and_shorten_diff(diff_body)
+ diff = truncate(diff, max_length: 20_000, percent_length: 0.3, llm: llm)
+ yield(info, diff)
+ rescue Discourse::GithubApi::Error => e
+ { error: "Failed to retrieve the #{type} information. #{e.message}" }
end
end
end
diff --git a/plugins/discourse-ai/lib/agents/tools/github_file_content.rb b/plugins/discourse-ai/lib/agents/tools/github_file_content.rb
index 232420cdd1b..0936796269b 100644
--- a/plugins/discourse-ai/lib/agents/tools/github_file_content.rb
+++ b/plugins/discourse-ai/lib/agents/tools/github_file_content.rb
@@ -74,22 +74,8 @@ module DiscourseAi
api_url =
"https://api.github.com/repos/#{owner}/#{repo}/contents/#{file_path}?ref=#{ref}"
- response_code = "-1 unknown"
- body = nil
-
- send_http_request(
- api_url,
- headers: {
- "Accept" => "application/vnd.github.v3+json",
- },
- authenticate_github: true,
- ) do |response|
- response_code = response.code
- body = read_response_body(response)
- end
-
- if response_code == "200"
- file_data = JSON.parse(body)
+ begin
+ file_data = github_client.get(api_url)
content = ensure_utf8(Base64.decode64(file_data["content"].to_s))
snippet =
extract_requested_content(
@@ -103,7 +89,7 @@ module DiscourseAi
start_line: file_request[:start_line],
end_line: file_request[:end_line],
}
- else
+ rescue Discourse::GithubApi::Error
missing_files << file_request[:raw]
end
end
diff --git a/plugins/discourse-ai/lib/agents/tools/github_search_code.rb b/plugins/discourse-ai/lib/agents/tools/github_search_code.rb
index e05cd559452..eff560431ce 100644
--- a/plugins/discourse-ai/lib/agents/tools/github_search_code.rb
+++ b/plugins/discourse-ai/lib/agents/tools/github_search_code.rb
@@ -75,53 +75,36 @@ module DiscourseAi
def invoke
api_url = build_url
- response_code = "unknown error"
- search_data = nil
-
- send_http_request(
- api_url,
- headers: {
- "Accept" => "application/vnd.github.v3.text-match+json",
- },
- authenticate_github: true,
- ) do |response|
- response_code = response.code
- if response_code == "200"
- begin
- search_data = JSON.parse(read_response_body(response))
- rescue JSON::ParserError
- response_code = "500 - JSON parse error"
- end
- end
+ begin
+ search_data =
+ github_client.get(api_url, accept: "application/vnd.github.v3.text-match+json")
+ rescue Discourse::GithubApi::Error => e
+ return { error: "Failed to perform code search. #{e.message}" }
end
- if response_code == "200"
- formatted_results = format_results(search_data["items"])
- grouped = group_by_file(formatted_results)
- trimmed = trim_results(grouped)
-
- total_count = search_data["total_count"].to_i
- total_pages =
- if total_count <= 0
- 1
- else
- [(total_count.to_f / PER_PAGE).ceil, MAX_ALLOWED_PAGE].min
- end
+ formatted_results = format_results(search_data["items"])
+ grouped = group_by_file(formatted_results)
+ trimmed = trim_results(grouped)
- result = { search_results: trimmed }
+ total_count = search_data["total_count"].to_i
+ total_pages =
+ if total_count <= 0
+ 1
+ else
+ [(total_count.to_f / PER_PAGE).ceil, MAX_ALLOWED_PAGE].min
+ end
- result[:page] = page if page > 1
- result[:total_results] = total_count
- result[:next_page] = page + 1 if page < total_pages
+ result = { search_results: trimmed }
- if search_data["incomplete_results"]
- result[:notes] = "GitHub marked the search results as incomplete."
- end
+ result[:page] = page if page > 1
+ result[:total_results] = total_count
+ result[:next_page] = page + 1 if page < total_pages
- result
- else
- { error: "Failed to perform code search. Status code: #{response_code}" }
+ if search_data["incomplete_results"]
+ result[:notes] = "GitHub marked the search results as incomplete."
end
+
+ result
end
private
@@ -283,29 +266,11 @@ module DiscourseAi
"https://api.github.com/repos/#{owner}/#{repo_name}/contents/#{path}?ref=#{actual_ref}"
end
- response_code = nil
- body = nil
-
- send_http_request(
- url,
- headers: {
- "Accept" => "application/vnd.github.v3+json",
- },
- authenticate_github: true,
- ) do |response|
- response_code = response.code
- body = read_response_body(response)
- end
-
- if response_code == "200"
- begin
- data = JSON.parse(body)
- decoded = ensure_utf8(Base64.decode64(data["content"].to_s))
- cache[cache_key] = { content: decoded, total_lines: count_lines(decoded) }
- rescue JSON::ParserError
- cache[cache_key] = nil
- end
- else
+ begin
+ data = github_client.get(url)
+ decoded = ensure_utf8(Base64.decode64(data["content"].to_s))
+ cache[cache_key] = { content: decoded, total_lines: count_lines(decoded) }
+ rescue Discourse::GithubApi::Error
cache[cache_key] = nil
end
diff --git a/plugins/discourse-ai/lib/agents/tools/github_search_files.rb b/plugins/discourse-ai/lib/agents/tools/github_search_files.rb
index 935fc676ee8..64b83c435b1 100644
--- a/plugins/discourse-ai/lib/agents/tools/github_search_files.rb
+++ b/plugins/discourse-ai/lib/agents/tools/github_search_files.rb
@@ -63,46 +63,27 @@ module DiscourseAi
api_url = "https://api.github.com/repos/#{repo}/git/trees/#{branch_name}?recursive=1"
- response_code = "unknown error"
- tree_data = nil
-
- send_http_request(
- api_url,
- headers: {
- "Accept" => "application/vnd.github.v3+json",
- },
- authenticate_github: true,
- ) do |response|
- response_code = response.code
- if response_code == "200"
- begin
- tree_data = JSON.parse(read_response_body(response))
- rescue JSON::ParserError
- response_code = "500 - JSON parse error"
- end
- end
+ begin
+ tree_data = github_client.get(api_url)
+ rescue Discourse::GithubApi::Error => e
+ return { error: "Failed to perform file search. #{e.message}" }
end
- if response_code == "200"
- matching_files =
- tree_data["tree"]
- .select do |item|
- item["type"] == "blob" &&
- keywords.any? { |keyword| item["path"].include?(keyword) }
- end
- .map { |item| { path: item["path"], size: item["size"] } }
- .take(MAX_FILE_SEARCH_RESULTS)
+ matching_files =
+ tree_data["tree"]
+ .select do |item|
+ item["type"] == "blob" && keywords.any? { |keyword| item["path"].include?(keyword) }
+ end
+ .map { |item| { path: item["path"], size: item["size"] } }
+ .take(MAX_FILE_SEARCH_RESULTS)
- result = { matching_files: matching_files, branch: branch_name }
- if matching_files.length == MAX_FILE_SEARCH_RESULTS
- result[
- :note
- ] = "Result limit reached (#{MAX_FILE_SEARCH_RESULTS} files). There may be more matching files."
- end
- result
- else
- { error: "Failed to perform file search. Status code: #{response_code}" }
+ result = { matching_files: matching_files, branch: branch_name }
+ if matching_files.length == MAX_FILE_SEARCH_RESULTS
+ result[
+ :note
+ ] = "Result limit reached (#{MAX_FILE_SEARCH_RESULTS} files). There may be more matching files."
end
+ result
end
end
end
diff --git a/plugins/discourse-ai/lib/agents/tools/tool.rb b/plugins/discourse-ai/lib/agents/tools/tool.rb
index b61b1abbf1a..46ac396c380 100644
--- a/plugins/discourse-ai/lib/agents/tools/tool.rb
+++ b/plugins/discourse-ai/lib/agents/tools/tool.rb
@@ -131,35 +131,14 @@ module DiscourseAi
protected
def fetch_default_branch(repo)
- api_url = "https://api.github.com/repos/#{repo}"
-
- response_code = "unknown error"
- repo_data = nil
-
- send_http_request(
- api_url,
- headers: {
- "Accept" => "application/vnd.github.v3+json",
- },
- authenticate_github: true,
- ) do |response|
- response_code = response.code
- if response_code == "200"
- begin
- repo_data = JSON.parse(read_response_body(response))
- rescue JSON::ParserError
- response_code = "500 - JSON parse error"
- end
- end
- end
-
- response_code == "200" ? repo_data["default_branch"] : "main"
+ github_client.get("https://api.github.com/repos/#{repo}")["default_branch"]
+ rescue Discourse::GithubApi::Error
+ "main"
end
def send_http_request(
url,
headers: {},
- authenticate_github: false,
follow_redirects: false,
method: :get,
body: nil,
@@ -168,7 +147,6 @@ module DiscourseAi
self.class.send_http_request(
url,
headers: headers,
- authenticate_github: authenticate_github,
follow_redirects: follow_redirects,
method: method,
body: body,
@@ -176,10 +154,13 @@ module DiscourseAi
)
end
+ def github_client
+ ::Discourse::GithubApi.for(token: SiteSetting.ai_bot_github_access_token)
+ end
+
def self.send_http_request(
url,
headers: {},
- authenticate_github: false,
follow_redirects: false,
method: :get,
body: nil
@@ -230,9 +211,6 @@ module DiscourseAi
request["User-Agent"] = DiscourseAi::AiBot::USER_AGENT
headers.each { |k, v| request[k] = v }
- if authenticate_github && SiteSetting.ai_bot_github_access_token.present?
- request["Authorization"] = "Bearer #{SiteSetting.ai_bot_github_access_token}"
- end
FinalDestination::HTTP.start(uri.hostname, uri.port, use_ssl: uri.port != 80) do |http|
http.request(request) { |response| yield response, uri }
diff --git a/plugins/discourse-ai/spec/lib/agents/tools/github_diff_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/github_diff_spec.rb
index 3bb71750687..755c5fc3a30 100644
--- a/plugins/discourse-ai/spec/lib/agents/tools/github_diff_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/tools/github_diff_spec.rb
@@ -57,7 +57,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff do
stub_request(:get, "https://api.github.com/repos/#{repo}/commits/abc123").with(
headers: {
- "Accept" => "application/json",
+ "Accept" => "application/vnd.github+json",
},
).to_return(status: 404)
@@ -115,15 +115,13 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff do
it "retrieves both PR info and diff" do
stub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(
headers: {
- "Accept" => "application/json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
+ "Accept" => "application/vnd.github+json",
},
).to_return(status: 200, body: pr_info.to_json)
stub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(
headers: {
"Accept" => "application/vnd.github.v3.diff",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
},
).to_return(status: 200, body: diff)
@@ -139,8 +137,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff do
stub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(
headers: {
- "Accept" => "application/json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
+ "Accept" => "application/vnd.github+json",
"Authorization" => "Bearer ABC",
},
).to_return(status: 200, body: pr_info.to_json)
@@ -148,7 +145,6 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff do
stub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(
headers: {
"Accept" => "application/vnd.github.v3.diff",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
"Authorization" => "Bearer ABC",
},
).to_return(status: 200, body: diff)
@@ -161,8 +157,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff do
it "returns an error for invalid pull request" do
stub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(
headers: {
- "Accept" => "application/json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
+ "Accept" => "application/vnd.github+json",
},
).to_return(status: 404)
@@ -191,15 +186,13 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff do
stub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(
headers: {
- "Accept" => "application/json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
+ "Accept" => "application/vnd.github+json",
},
).to_return(status: 200, body: pr_info_deleted_fork.to_json)
stub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(
headers: {
"Accept" => "application/vnd.github.v3.diff",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
},
).to_return(status: 200, body: diff)
@@ -252,15 +245,13 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff do
it "retrieves commit info and diff" do
stub_request(:get, "https://api.github.com/repos/#{repo}/commits/#{sha}").with(
headers: {
- "Accept" => "application/json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
+ "Accept" => "application/vnd.github+json",
},
).to_return(status: 200, body: commit_info.to_json)
stub_request(:get, "https://api.github.com/repos/#{repo}/commits/#{sha}").with(
headers: {
"Accept" => "application/vnd.github.v3.diff",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
},
).to_return(status: 200, body: diff)
@@ -283,8 +274,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff do
stub_request(:get, "https://api.github.com/repos/#{repo}/commits/#{sha}").with(
headers: {
- "Accept" => "application/json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
+ "Accept" => "application/vnd.github+json",
"Authorization" => "Bearer ABC",
},
).to_return(status: 200, body: commit_info.to_json)
@@ -292,7 +282,6 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff do
stub_request(:get, "https://api.github.com/repos/#{repo}/commits/#{sha}").with(
headers: {
"Accept" => "application/vnd.github.v3.diff",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
"Authorization" => "Bearer ABC",
},
).to_return(status: 200, body: diff)
@@ -305,8 +294,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff do
it "returns an error for invalid commit" do
stub_request(:get, "https://api.github.com/repos/#{repo}/commits/#{sha}").with(
headers: {
- "Accept" => "application/json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
+ "Accept" => "application/vnd.github+json",
},
).to_return(status: 404)
diff --git a/plugins/discourse-ai/spec/lib/agents/tools/github_search_code_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/github_search_code_spec.rb
index 8527787b78c..03f118bf7fa 100644
--- a/plugins/discourse-ai/spec/lib/agents/tools/github_search_code_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/tools/github_search_code_spec.rb
@@ -17,12 +17,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode do
stub_request(
:get,
"https://api.github.com/search/code?page=1&per_page=30&q=def%20hello%20repo:discourse/discourse",
- ).with(
- headers: {
- "Accept" => "application/vnd.github.v3.text-match+json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
- },
- ).to_return(
+ ).with(headers: { "Accept" => "application/vnd.github.v3.text-match+json" }).to_return(
status: 200,
body: {
total_count: 1,
@@ -44,8 +39,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode do
)
stub_request(:get, "https://api.github.com/repos/discourse/discourse/git/blobs/abc123").with(
headers: {
- "Accept" => "application/vnd.github.v3+json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
+ "Accept" => "application/vnd.github+json",
},
).to_return(status: 200, body: { content: Base64.encode64(file_body) }.to_json)
@@ -195,12 +189,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode do
stub_request(
:get,
"https://api.github.com/search/code?page=1&per_page=30&q=A_CONSTANT%20repo:discourse/discourse",
- ).with(
- headers: {
- "Accept" => "application/vnd.github.v3.text-match+json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
- },
- ).to_return(
+ ).with(headers: { "Accept" => "application/vnd.github.v3.text-match+json" }).to_return(
status: 200,
body: {
total_count: 1,
@@ -221,8 +210,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode do
stub_request(:get, "https://api.github.com/repos/discourse/discourse/git/blobs/def456").with(
headers: {
- "Accept" => "application/vnd.github.v3+json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
+ "Accept" => "application/vnd.github+json",
},
).to_return(status: 404)
@@ -252,7 +240,6 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode do
).with(
headers: {
"Accept" => "application/vnd.github.v3.text-match+json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
"Authorization" => "Bearer ABC",
},
).to_return(status: 200, body: { total_count: 0, items: [] }.to_json)
@@ -271,12 +258,9 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode do
stub_request(
:get,
"https://api.github.com/search/code?page=1&per_page=30&q=def%20hello%20repo:discourse/discourse",
- ).with(
- headers: {
- "Accept" => "application/vnd.github.v3.text-match+json",
- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,
- },
- ).to_return(status: 403)
+ ).with(headers: { "Accept" => "application/vnd.github.v3.text-match+json" }).to_return(
+ status: 403,
+ )
result = tool.invoke
expect(result[:search_results]).to be_nil
diff --git a/plugins/discourse-ai/spec/lib/agents/tools/github_search_files_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/github_search_files_spec.rb
index 7c5e997f0bd..4d774ba9bc6 100644
--- a/plugins/discourse-ai/spec/lib/agents/tools/github_search_files_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/tools/github_search_files_spec.rb
@@ -73,7 +73,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchFiles do
result = tool_with_invalid_branch.invoke
expect(result[:matching_files]).to be_nil
- expect(result[:error]).to eq("Failed to perform file search. Status code: 404")
+ expect(result[:error]).to include("Failed to perform file search")
end
it "fetches the default branch if none is specified" do
diff --git a/plugins/discourse-github/app/jobs/regular/replace_github_non_permalinks.rb b/plugins/discourse-github/app/jobs/regular/replace_github_non_permalinks.rb
index 70a82f81908..1c07f4ffaff 100644
--- a/plugins/discourse-github/app/jobs/regular/replace_github_non_permalinks.rb
+++ b/plugins/discourse-github/app/jobs/regular/replace_github_non_permalinks.rb
@@ -1,9 +1,5 @@
# frozen_string_literal: true
-require "net/http"
-require "json"
-require "uri"
-
module Jobs
class ReplaceGithubNonPermalinks < ::Jobs::Base
sidekiq_options queue: "low"
@@ -18,6 +14,9 @@ module Jobs
post = Post.find_by(id: post_id)
return if post.blank?
+ client = Discourse::GithubApi.for(token: SiteSetting.github_linkback_access_token)
+ return if client.backing_off?
+
raw = post.raw.dup
start_raw = raw.dup
@@ -29,14 +28,15 @@ module Jobs
next if excluded?(user, repo, file)
begin
- api_url = "https://api.github.com/repos/#{user}/#{repo}/commits/#{sha1}"
- json = api_request(api_url)
- if json && (json["sha"] != sha1)
- new_sha = json["sha"]
+ commit = client.get("/repos/#{user}/#{repo}/commits/#{sha1}")
+ if commit && commit["sha"] != sha1
+ new_sha = commit["sha"]
old_url = "github.com/#{user}/#{repo}/blob/#{sha1}/#{file}#{from_to}"
new_url = "github.com/#{user}/#{repo}/blob/#{new_sha}/#{file}#{from_to}"
raw.sub!(old_url, new_url)
end
+ rescue Discourse::GithubApi::NotFound
+ next
rescue => e
log(
:error,
@@ -76,13 +76,6 @@ module Jobs
private
- def api_request(url)
- uri = URI(url)
- response = Net::HTTP.get_response(uri)
-
- JSON.parse(response.body) if response.kind_of? Net::HTTPSuccess
- end
-
def log(log_level, message)
Rails.logger.public_send(
log_level,
diff --git a/plugins/discourse-github/app/lib/commits_populator.rb b/plugins/discourse-github/app/lib/commits_populator.rb
index d4c67e61345..b3656fa693b 100644
--- a/plugins/discourse-github/app/lib/commits_populator.rb
+++ b/plugins/discourse-github/app/lib/commits_populator.rb
@@ -10,8 +10,8 @@ module DiscourseGithubPlugin
ROLES = { committer: 0, contributor: 1 }
class PaginatedCommits
- def initialize(octokit, repo, cursor: nil, page_size: 100)
- @client = octokit
+ def initialize(client, repo, cursor: nil, page_size: 100)
+ @client = client
@repo = repo
@cursor = cursor
@page_size = page_size
@@ -30,17 +30,21 @@ module DiscourseGithubPlugin
end
def commits
- @data.repository.defaultBranchRef.target.history.nodes
+ history["nodes"]
end
def next_cursor
- info = @data.repository.defaultBranchRef.target.history.pageInfo
- return unless info.hasNextPage
- info.endCursor
+ info = history["pageInfo"]
+ return unless info["hasNextPage"]
+ info["endCursor"]
end
private
+ def history
+ @data.dig("repository", "defaultBranchRef", "target", "history")
+ end
+
def fetch_commits
owner, name = @repo.name.split("/", 2)
history_args = "first: #{@page_size}"
@@ -82,22 +86,23 @@ module DiscourseGithubPlugin
}
}
QUERY
- response = @client.post("/graphql", { query: query }.to_json)
- raise GraphQLError, response.errors.inspect if response.errors
- raise GraphQLError, response.message if !response.data
- @data = response.data
+ response = @client.post("/graphql", { query: query })
+ raise GraphQLError, "Empty GraphQL response" if response.nil?
+ raise GraphQLError, response["errors"].inspect if response["errors"]
+ raise GraphQLError, response.inspect if !response["data"]
+ @data = response["data"]
end
end
def initialize(repo)
@repo = repo
- @client =
- Octokit::Client.new(access_token: SiteSetting.github_linkback_access_token, per_page: 100)
+ @client = Discourse::GithubApi.for(token: SiteSetting.github_linkback_access_token)
end
def populate!
return unless SiteSetting.github_badges_enabled?
- return if @client.branches(@repo.name).empty?
+ return if @client.backing_off?
+ return if @client.get("/repos/#{@repo.name}/branches").blank?
if @repo.commits.size == 0
build_history!
@@ -130,56 +135,41 @@ module DiscourseGithubPlugin
Discourse.redis.del(front_commit_redis_key)
end
end
- rescue Octokit::Error => err
- case err
- when Octokit::NotFound
- disable_github_badges_and_inform_admin(
- title: I18n.t("github_commits_populator.errors.repository_not_found_pm_title"),
- raw:
- I18n.t(
- "github_commits_populator.errors.repository_not_found_pm",
- repo_name: @repo.name,
- base_path: Discourse.base_path,
- ),
- )
- Rails.logger.warn(
- "Disabled github_badges_enabled site setting due to repository Not Found error ",
- )
- when Octokit::Unauthorized
- disable_github_badges_and_inform_admin(
- title: I18n.t("github_commits_populator.errors.invalid_octokit_credentials_pm_title"),
- raw:
- I18n.t(
- "github_commits_populator.errors.invalid_octokit_credentials_pm",
- base_path: Discourse.base_path,
- ),
- )
- Rails.logger.warn(
- "Disabled github_badges_enabled site setting due to invalid GitHub authentication credentials via github_linkback_access_token.",
- )
- else
- Rails.logger.warn("#{err.class}: #{err.message}")
- end
- rescue Octokit::InvalidRepository => err
+ rescue Discourse::GithubApi::NotFound
disable_github_badges_and_inform_admin(
- title: I18n.t("github_commits_populator.errors.repository_identifier_invalid_pm_title"),
+ title: I18n.t("github_commits_populator.errors.repository_not_found_pm_title"),
raw:
I18n.t(
- "github_commits_populator.errors.repository_identifier_invalid_pm",
+ "github_commits_populator.errors.repository_not_found_pm",
repo_name: @repo.name,
base_path: Discourse.base_path,
),
)
Rails.logger.warn(
- "Disabled github_badges_enabled site setting due to invalid repository identifier",
+ "Disabled github_badges_enabled site setting due to repository Not Found error ",
+ )
+ rescue Discourse::GithubApi::Unauthorized
+ disable_github_badges_and_inform_admin(
+ title: I18n.t("github_commits_populator.errors.invalid_octokit_credentials_pm_title"),
+ raw:
+ I18n.t(
+ "github_commits_populator.errors.invalid_octokit_credentials_pm",
+ base_path: Discourse.base_path,
+ ),
+ )
+ Rails.logger.warn(
+ "Disabled github_badges_enabled site setting due to invalid GitHub authentication credentials via github_linkback_access_token.",
)
+ rescue Discourse::GithubApi::Error => err
+ Rails.logger.warn("#{err.class}: #{err.message}")
end
private
def is_contribution?(commit)
- pr = commit.associatedPullRequests.nodes.first
- pr && pr.author && pr.mergedBy && pr.author.login != pr.mergedBy.login
+ pr = commit.dig("associatedPullRequests", "nodes")&.first
+ pr && pr["author"] && pr["mergedBy"] &&
+ pr.dig("author", "login") != pr.dig("mergedBy", "login")
end
def fetch_new_commits!(stop_at)
@@ -191,7 +181,7 @@ module DiscourseGithubPlugin
stop_at.present? ? [] : @repo.commits.order("committed_at DESC").first(100).pluck(:sha)
while !done
batch.each do |c|
- if c.oid == stop_at || recent_commits.include?(c.oid)
+ if c["oid"] == stop_at || recent_commits.include?(c["oid"])
done = true
break
end
@@ -204,25 +194,32 @@ module DiscourseGithubPlugin
end
return if commits.size == 0
existing_shas = @repo.commits.pluck(:sha)
- commits.reject! { |c| existing_shas.include?(c.oid) }
+ commits.reject! { |c| existing_shas.include?(c["oid"]) }
batch_to_db(commits)
- set_front_commit(commits.first.oid)
+ set_front_commit(commits.first["oid"])
end
# detect if a force push happened and commit is lost
def removed?(sha)
- commit = @client.commit(@repo.name, sha)
- return true if commit.commit.nil?
+ commit = @client.get("/repos/#{@repo.name}/commits/#{sha}")
+ return false if commit.nil?
+ return true if commit["commit"].nil?
found =
- @client.commits(@repo.name, until: commit.commit.committer.date, page: 1, per_page: 1).first
- commit.sha != found.sha
+ @client.get(
+ "/repos/#{@repo.name}/commits",
+ until: commit.dig("commit", "committer", "date"),
+ page: 1,
+ per_page: 1,
+ ).first
+ return false if found.nil?
+ commit["sha"] != found["sha"]
end
def build_history!(cursor: nil)
paginator = PaginatedCommits.new(@client, @repo, cursor: cursor, page_size: 100)
batch = paginator.commits
return if batch.empty?
- set_front_commit(batch.first.oid) if cursor.blank?
+ set_front_commit(batch.first["oid"]) if cursor.blank?
while batch.size > 0
batch_to_db(batch)
@@ -251,11 +248,11 @@ module DiscourseGithubPlugin
def commit_to_hash(commit)
{
- sha: commit.oid,
- email: commit.author.email,
+ sha: commit["oid"],
+ email: commit.dig("author", "email"),
repo_id: @repo.id,
- committed_at: commit.committedDate,
- merge_commit: commit.message.match?(MERGE_COMMIT_REGEX),
+ committed_at: commit["committedDate"],
+ merge_commit: commit["message"].match?(MERGE_COMMIT_REGEX),
role_id: is_contribution?(commit) ? ROLES[:contributor] : ROLES[:committer],
}
end
diff --git a/plugins/discourse-github/app/lib/github_linkback.rb b/plugins/discourse-github/app/lib/github_linkback.rb
index b3d022112d6..0bc57cd0085 100644
--- a/plugins/discourse-github/app/lib/github_linkback.rb
+++ b/plugins/discourse-github/app/lib/github_linkback.rb
@@ -93,6 +93,7 @@ class GithubLinkback
def create
return [] if SiteSetting.github_linkback_access_token.blank?
+ return [] if client.backing_off?
links = []
@@ -101,14 +102,19 @@ class GithubLinkback
return [] if links.length() > SiteSetting.github_linkback_maximum_links
links.each do |link|
- case link.type
- when :commit
- post_commit(link)
- when :pr
- post_pr_or_issue(link, :pr)
- when :issue
- post_pr_or_issue(link, :issue)
- else
+ begin
+ case link.type
+ when :commit
+ post_commit(link)
+ when :pr
+ post_pr_or_issue(link, :pr)
+ when :issue
+ post_pr_or_issue(link, :issue)
+ else
+ next
+ end
+ rescue Discourse::GithubApi::Error => e
+ Rails.logger.warn("Failed to post GitHub linkback for #{link.url}: #{e.message}")
next
end
@@ -138,8 +144,6 @@ class GithubLinkback
return if topic_already_linked_on_github?(link)
- github_url =
- "https://api.github.com/repos/#{link.project}/issues/#{pr_or_issue_number}/comments"
comment =
I18n.t(
type == :pr ? "github_linkback.pr_template" : "github_linkback.issue_template",
@@ -147,14 +151,12 @@ class GithubLinkback
post_url: "#{Discourse.base_url}#{@post.url}",
)
- Excon.post(github_url, body: { body: comment }.to_json, headers: headers)
+ client.post("/repos/#{link.project}/issues/#{pr_or_issue_number}/comments", { body: comment })
end
def post_commit(link)
return if topic_already_linked_on_github?(link)
- github_url = "https://api.github.com/repos/#{link.project}/commits/#{link.sha}/comments"
-
comment =
I18n.t(
"github_linkback.commit_template",
@@ -162,7 +164,7 @@ class GithubLinkback
post_url: "#{Discourse.base_url}#{@post.url}",
)
- Excon.post(github_url, body: { body: comment }.to_json, headers: headers)
+ client.post("/repos/#{link.project}/commits/#{link.sha}/comments", { body: comment })
end
def topic_already_linked_on_github?(link)
@@ -174,38 +176,22 @@ class GithubLinkback
end
texts.any? { |text| text_links_to_topic?(text) }
- rescue Excon::Error
+ rescue Discourse::GithubApi::Error
false
end
def fetch_commit_comment_texts(project, sha)
- response =
- Excon.get(
- "https://api.github.com/repos/#{project}/commits/#{sha}/comments?per_page=100",
- headers:,
- expects: [200],
- )
- JSON.parse(response.body).map { |comment| comment["body"].to_s }
+ client
+ .get("/repos/#{project}/commits/#{sha}/comments", per_page: 100)
+ .map { |comment| comment["body"].to_s }
end
def fetch_pr_or_issue_texts(project, number)
- issue_response =
- Excon.get(
- "https://api.github.com/repos/#{project}/issues/#{number}",
- headers:,
- expects: [200],
- )
-
- comments_response =
- Excon.get(
- "https://api.github.com/repos/#{project}/issues/#{number}/comments?per_page=100",
- headers:,
- expects: [200],
- )
-
[
- JSON.parse(issue_response.body)["body"].to_s,
- *JSON.parse(comments_response.body).map { |comment| comment["body"].to_s },
+ client.get("/repos/#{project}/issues/#{number}")["body"].to_s,
+ *client
+ .get("/repos/#{project}/issues/#{number}/comments", per_page: 100)
+ .map { |comment| comment["body"].to_s },
]
end
@@ -220,11 +206,7 @@ class GithubLinkback
end
end
- def headers
- {
- "Content-Type" => "application/json",
- "Authorization" => "token #{SiteSetting.github_linkback_access_token}",
- "User-Agent" => "Discourse-Github-Linkback",
- }
+ def client
+ @client ||= Discourse::GithubApi.for(token: SiteSetting.github_linkback_access_token)
end
end
diff --git a/plugins/discourse-github/app/lib/github_linkback_access_token_setting_validator.rb b/plugins/discourse-github/app/lib/github_linkback_access_token_setting_validator.rb
index 44834dd2164..471c1778a43 100644
--- a/plugins/discourse-github/app/lib/github_linkback_access_token_setting_validator.rb
+++ b/plugins/discourse-github/app/lib/github_linkback_access_token_setting_validator.rb
@@ -7,10 +7,12 @@ class GithubLinkbackAccessTokenSettingValidator
def valid_value?(val)
return true if val.blank?
- client = Octokit::Client.new(access_token: val, per_page: 1)
- DiscourseGithubPlugin::GithubRepo.repos.each { |repo| client.branches(repo.name) }
+ client = Discourse::GithubApi.new(val)
+ DiscourseGithubPlugin::GithubRepo.repos.each do |repo|
+ client.get("/repos/#{repo.name}/branches")
+ end
true
- rescue Octokit::Unauthorized
+ rescue Discourse::GithubApi::Unauthorized
false
end
diff --git a/plugins/discourse-github/app/models/github_repo.rb b/plugins/discourse-github/app/models/github_repo.rb
index d16804188fa..b6c0d4cd35b 100644
--- a/plugins/discourse-github/app/models/github_repo.rb
+++ b/plugins/discourse-github/app/models/github_repo.rb
@@ -3,7 +3,7 @@
module DiscourseGithubPlugin
class GithubRepo < ActiveRecord::Base
VALID_URL_BASED_REPO_REGEX = %r{https?://github.com/(.+)}
- VALID_USER_BASED_REPO_REGEX = Octokit::Repository::NAME_WITH_OWNER_PATTERN
+ VALID_USER_BASED_REPO_REGEX = %r{\A[\w.-]+/[\w.-]+\z}i
has_many :commits, foreign_key: :repo_id, class_name: :GithubCommit, dependent: :destroy
diff --git a/plugins/discourse-github/plugin.rb b/plugins/discourse-github/plugin.rb
index cc7f97ae425..4a7f024a19b 100644
--- a/plugins/discourse-github/plugin.rb
+++ b/plugins/discourse-github/plugin.rb
@@ -7,9 +7,6 @@
# authors: Robin Ward, Sam Saffron
# url: https://github.com/discourse/discourse/tree/main/plugins/discourse-github
-require "sawyer"
-require "octokit"
-
# Site setting validators must be loaded before initialize
require_relative "app/lib/github_badges_repo_setting_validator"
require_relative "app/lib/github_linkback_access_token_setting_validator"
diff --git a/plugins/discourse-github/spec/jobs/replace_github_non_permalinks_spec.rb b/plugins/discourse-github/spec/jobs/replace_github_non_permalinks_spec.rb
index d428835d023..6677a63fad8 100644
--- a/plugins/discourse-github/spec/jobs/replace_github_non_permalinks_spec.rb
+++ b/plugins/discourse-github/spec/jobs/replace_github_non_permalinks_spec.rb
@@ -25,12 +25,19 @@ describe Jobs::ReplaceGithubNonPermalinks do
status: 200,
body: github_response_body.to_json,
headers: {
+ "Content-Type" => "application/json",
},
)
stub_request(
:get,
"https://api.github.com/repos/test/onebox/commits/815ea9c0a8ffebe7bd7fcd34c10ff28c7a6b6974",
- ).to_return(status: 200, body: github_response_body.to_json, headers: {})
+ ).to_return(
+ status: 200,
+ body: github_response_body.to_json,
+ headers: {
+ "Content-Type" => "application/json",
+ },
+ )
stub_request(:get, "https://api.github.com/repos/test/oneblob/commits/master").to_return(
status: 404,
)
@@ -38,6 +45,7 @@ describe Jobs::ReplaceGithubNonPermalinks do
status: 200,
body: github_response_body2.to_json,
headers: {
+ "Content-Type" => "application/json",
},
)
end
diff --git a/plugins/discourse-github/spec/lib/commits_populator_spec.rb b/plugins/discourse-github/spec/lib/commits_populator_spec.rb
index c976e4a486f..9ed14b6f4ab 100644
--- a/plugins/discourse-github/spec/lib/commits_populator_spec.rb
+++ b/plugins/discourse-github/spec/lib/commits_populator_spec.rb
@@ -6,30 +6,32 @@ describe DiscourseGithubPlugin::CommitsPopulator do
let(:repo) { DiscourseGithubPlugin::GithubRepo.new(name: "discourse/discourse") }
let!(:site_admin1) { Fabricate(:admin) }
let!(:site_admin2) { Fabricate(:admin) }
+ let(:branches_url) { "https://api.github.com/repos/discourse/discourse/branches" }
before do
enable_current_plugin
SiteSetting.github_badges_enabled = true
end
- context "when invalid credentials have been provided for octokit" do
- before { Octokit::Client.any_instance.expects(:branches).raises(Octokit::Unauthorized) }
+ def last_pm
+ Post
+ .joins(:topic)
+ .includes(:topic)
+ .where("topics.archetype = ?", Archetype.private_message)
+ .last
+ end
+
+ context "when invalid credentials have been provided (401)" do
+ before { stub_request(:get, branches_url).to_return(status: 401) }
it "disables github badges and sends a PM to the admin of the site to inform them" do
populator.populate!
expect(SiteSetting.github_badges_enabled).to eq(false)
- sent_pm =
- Post
- .joins(:topic)
- .includes(:topic)
- .where("topics.archetype = ?", Archetype.private_message)
- .last
- expect(sent_pm.topic.allowed_users.include?(site_admin1)).to eq(true)
- expect(sent_pm.topic.allowed_users.include?(site_admin2)).to eq(true)
- expect(sent_pm.topic.title).to eq(
+ expect(last_pm.topic.allowed_users).to include(site_admin1, site_admin2)
+ expect(last_pm.topic.title).to eq(
I18n.t("github_commits_populator.errors.invalid_octokit_credentials_pm_title"),
)
- expect(sent_pm.raw).to eq(
+ expect(last_pm.raw).to eq(
I18n.t(
"github_commits_populator.errors.invalid_octokit_credentials_pm",
base_path: Discourse.base_path,
@@ -38,24 +40,17 @@ describe DiscourseGithubPlugin::CommitsPopulator do
end
end
- context "when the repository is not found" do
- before { Octokit::Client.any_instance.expects(:branches).raises(Octokit::NotFound) }
+ context "when the repository is not found (404)" do
+ before { stub_request(:get, branches_url).to_return(status: 404) }
it "disables github badges and sends a PM to the admin of the site to inform them" do
populator.populate!
expect(SiteSetting.github_badges_enabled).to eq(false)
- sent_pm =
- Post
- .joins(:topic)
- .includes(:topic)
- .where("topics.archetype = ?", Archetype.private_message)
- .last
- expect(sent_pm.topic.allowed_users.include?(site_admin1)).to eq(true)
- expect(sent_pm.topic.allowed_users.include?(site_admin2)).to eq(true)
- expect(sent_pm.topic.title).to eq(
+ expect(last_pm.topic.allowed_users).to include(site_admin1, site_admin2)
+ expect(last_pm.topic.title).to eq(
I18n.t("github_commits_populator.errors.repository_not_found_pm_title"),
)
- expect(sent_pm.raw).to eq(
+ expect(last_pm.raw).to eq(
I18n.t(
"github_commits_populator.errors.repository_not_found_pm",
repo_name: repo.name,
@@ -65,35 +60,8 @@ describe DiscourseGithubPlugin::CommitsPopulator do
end
end
- context "when the repository identifier is invalid" do
- before { Octokit::Client.any_instance.expects(:branches).raises(Octokit::InvalidRepository) }
-
- it "disables github badges and sends a PM to the admin of the site to inform them" do
- populator.populate!
- expect(SiteSetting.github_badges_enabled).to eq(false)
- sent_pm =
- Post
- .joins(:topic)
- .includes(:topic)
- .where("topics.archetype = ?", Archetype.private_message)
- .last
- expect(sent_pm.topic.allowed_users.include?(site_admin1)).to eq(true)
- expect(sent_pm.topic.allowed_users.include?(site_admin2)).to eq(true)
- expect(sent_pm.topic.title).to eq(
- I18n.t("github_commits_populator.errors.repository_identifier_invalid_pm_title"),
- )
- expect(sent_pm.raw).to eq(
- I18n.t(
- "github_commits_populator.errors.repository_identifier_invalid_pm",
- repo_name: repo.name,
- base_path: Discourse.base_path,
- ).strip,
- )
- end
- end
-
- context "if some other octokit error is raised" do
- before { Octokit::Client.any_instance.expects(:branches).raises(Octokit::Error) }
+ context "if some other GitHub error is raised (500)" do
+ before { stub_request(:get, branches_url).to_return(status: 500) }
it "simply logs the error and does nothing else" do
populator.populate!
@@ -103,40 +71,23 @@ describe DiscourseGithubPlugin::CommitsPopulator do
context "if GraphQL returns no data" do
before do
- branches_body = <<~JSON
- [
- {
- "name": "add-group-css-properties",
- "commit": {
- "sha": "0d67b6307042803c351599de715023841cfa9356",
- "url": "https://api.github.com/repos/discourse/discourse/commits/0d67b6307042803c351599de715023841cfa9356"
- },
- "protected": false
- }
- ]
- JSON
-
- graphql_response = <<~JSON
- {
- "message": "Bad credentials",
- "documentation_url": "https://docs.github.com/graphql"
- }
- JSON
-
- stub_request(
- :get,
- "https://api.github.com/repos/discourse/discourse/branches?per_page=100",
- ).to_return(status: 200, body: branches_body)
+ stub_request(:get, branches_url).to_return(
+ status: 200,
+ body: [{ "name" => "main", "commit" => { "sha" => "abc" } }].to_json,
+ headers: {
+ "Content-Type" => "application/json",
+ },
+ )
stub_request(:post, "https://api.github.com/graphql").to_return(
status: 200,
- body: graphql_response,
+ body: { "message" => "Bad credentials" }.to_json,
headers: {
- "content-type": "application/json",
+ "Content-Type" => "application/json",
},
)
end
- it "simply logs the error and does nothing else" do
+ it "raises a GraphQLError" do
expect { populator.populate! }.to raise_error(described_class::GraphQLError)
end
end
@@ -144,9 +95,9 @@ describe DiscourseGithubPlugin::CommitsPopulator do
context "if github_badges_enabled is false" do
before { SiteSetting.github_badges_enabled = false }
- it "early returns before attempting to execute any of the commit fetching, because the plugin likely disabled itself" do
- Octokit::Client.any_instance.expects(:branches).never
+ it "early returns before making any GitHub request" do
populator.populate!
+ expect(a_request(:get, branches_url)).not_to have_been_made
end
end
end
diff --git a/plugins/discourse-github/spec/lib/github_linkback_access_token_setting_validator_spec.rb b/plugins/discourse-github/spec/lib/github_linkback_access_token_setting_validator_spec.rb
index 79cf37b7a04..30a28b331b7 100644
--- a/plugins/discourse-github/spec/lib/github_linkback_access_token_setting_validator_spec.rb
+++ b/plugins/discourse-github/spec/lib/github_linkback_access_token_setting_validator_spec.rb
@@ -8,10 +8,12 @@ describe GithubLinkbackAccessTokenSettingValidator do
before { enable_current_plugin }
describe "#valid_value?" do
- context "when an Octokit::Unauthorized error is raised, meaning the access token cannot access a repo" do
+ context "when the token cannot access a repo (401)" do
before do
setup_repos
- Octokit::Client.any_instance.expects(:branches).raises(Octokit::Unauthorized)
+ stub_request(:get, "https://api.github.com/repos/discourse/discourse/branches").to_return(
+ status: 401,
+ )
end
it "should fail" do
@@ -19,7 +21,7 @@ describe GithubLinkbackAccessTokenSettingValidator do
end
end
- context "when no Octokit::Unauthorized error is raised" do
+ context "when the token is accepted" do
it "should pass, without repos defined" do
expect(validator.valid_value?(value)).to eq(true)
end
@@ -27,7 +29,10 @@ describe GithubLinkbackAccessTokenSettingValidator do
context "when there are repos defined" do
before do
setup_repos
- Octokit::Client.any_instance.expects(:branches).returns([])
+ stub_request(:get, "https://api.github.com/repos/discourse/discourse/branches").to_return(
+ status: 200,
+ body: "[]",
+ )
end
it "should pass if all the repos are accessible" do
diff --git a/plugins/discourse-github/spec/lib/github_linkback_spec.rb b/plugins/discourse-github/spec/lib/github_linkback_spec.rb
index 108220e8cf9..783dd46ce25 100644
--- a/plugins/discourse-github/spec/lib/github_linkback_spec.rb
+++ b/plugins/discourse-github/spec/lib/github_linkback_spec.rb
@@ -200,14 +200,7 @@ describe GithubLinkback do
end
describe "#create" do
- let(:github_api_headers) do
- {
- "Authorization" => "token abcdef",
- "Content-Type" => "application/json",
- "Host" => "api.github.com",
- "User-Agent" => "Discourse-Github-Linkback",
- }
- end
+ let(:github_api_headers) { { "Authorization" => "Bearer abcdef" } }
before { SiteSetting.github_linkback_projects = "discourse/discourse|discourse/*" }
diff --git a/spec/lib/discourse/github_api_spec.rb b/spec/lib/discourse/github_api_spec.rb
new file mode 100644
index 00000000000..f845f2c91de
--- /dev/null
+++ b/spec/lib/discourse/github_api_spec.rb
@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+RSpec.describe Discourse::GithubApi do
+ subject(:client) { described_class.for(token:) }
+
+ let(:token) { "gh_token_abc" }
+
+ before { described_class.reset_clients! }
+
+ def stub_gh(path, status: 200, body: "{}", headers: { "Content-Type" => "application/json" })
+ stub_request(:get, "https://api.github.com#{path}").to_return(status:, body:, headers:)
+ end
+
+ describe "#get" do
+ it "returns a plain string-keyed Hash (not Sawyer)" do
+ stub_gh(
+ "/repos/discourse/discourse/pulls/1",
+ body: { "title" => "Hi", "base" => { "ref" => "main" } }.to_json,
+ )
+
+ result = client.get("/repos/discourse/discourse/pulls/1")
+
+ expect(result).to be_a(Hash)
+ expect(result["base"]["ref"]).to eq("main")
+ expect(result.dig("base", "ref")).to eq("main")
+ end
+
+ it "returns an Array for list endpoints" do
+ stub_gh(
+ "/repos/discourse/discourse/pulls/1/reviews",
+ body: [{ "state" => "APPROVED" }].to_json,
+ )
+ expect(client.get("/repos/discourse/discourse/pulls/1/reviews")).to eq(
+ [{ "state" => "APPROVED" }],
+ )
+ end
+
+ it "sends a Bearer Authorization header when a token is configured" do
+ stub =
+ stub_request(:get, "https://api.github.com/x").with(
+ headers: {
+ "Authorization" => "Bearer #{token}",
+ },
+ ).to_return(status: 200, body: "{}")
+ client.get("/x")
+ expect(stub).to have_been_requested
+ end
+
+ it "sends no Authorization header when unauthenticated" do
+ anon = described_class.for(token: nil)
+ stub = stub_request(:get, "https://api.github.com/x").to_return(status: 200, body: "{}")
+ anon.get("/x")
+ expect(stub).to have_been_requested
+ expect(WebMock).not_to have_requested(:get, "https://api.github.com/x").with(
+ headers: {
+ "Authorization" => /.+/,
+ },
+ )
+ end
+ end
+
+ describe "rate limiting" do
+ it "short-circuits with RateLimited (no HTTP) while backing off" do
+ GithubRateLimit.note_rate_limit(token:, retry_after: 60)
+ expect { client.get("/x") }.to raise_error(described_class::RateLimited)
+ expect(a_request(:get, "https://api.github.com/x")).not_to have_been_made
+ end
+
+ it "records a backoff and raises on a 403 rate-limit response" do
+ stub_gh(
+ "/x",
+ status: 403,
+ headers: {
+ "x-ratelimit-remaining" => "0",
+ "x-ratelimit-reset" => 10.minutes.from_now.to_i.to_s,
+ },
+ )
+ expect { client.get("/x") }.to raise_error(described_class::RateLimited)
+ expect(GithubRateLimit.backing_off?(token)).to eq(true)
+ end
+
+ it "proactively backs off when a 200 reports the budget is exhausted" do
+ stub_gh(
+ "/x",
+ headers: {
+ "Content-Type" => "application/json",
+ "x-ratelimit-remaining" => "0",
+ "x-ratelimit-reset" => 10.minutes.from_now.to_i.to_s,
+ },
+ )
+ client.get("/x")
+ expect(GithubRateLimit.backing_off?(token)).to eq(true)
+ end
+ end
+
+ describe "error mapping" do
+ it "raises NotFound on 404" do
+ stub_gh("/x", status: 404)
+ expect { client.get("/x") }.to raise_error(described_class::NotFound)
+ end
+
+ it "raises Unauthorized on 401" do
+ stub_gh("/x", status: 401)
+ expect { client.get("/x") }.to raise_error(described_class::Unauthorized)
+ end
+
+ it "raises a non-rate-limit error on a 403 with budget remaining (no backoff)" do
+ stub_gh("/x", status: 403, headers: { "x-ratelimit-remaining" => "57" })
+ expect { client.get("/x") }.to raise_error(described_class::Error) do |e|
+ expect(e).not_to be_a(described_class::RateLimited)
+ end
+ expect(GithubRateLimit.backing_off?(token)).to eq(false)
+ end
+
+ it "maps Faraday timeouts/connection failures to its own Error" do
+ stub_request(:get, "https://api.github.com/x").to_timeout
+ expect { client.get("/x") }.to raise_error(described_class::Error)
+ end
+ end
+
+ describe "ETag conditional requests" do
+ it "stores the ETag and serves the cached body on a 304 (no rate-limit cost)" do
+ stub_request(:get, "https://api.github.com/x").to_return(
+ status: 200,
+ body: { "n" => 1 }.to_json,
+ headers: {
+ "Content-Type" => "application/json",
+ "ETag" => 'W/"abc"',
+ },
+ )
+ expect(client.get("/x")).to eq({ "n" => 1 })
+
+ stub_request(:get, "https://api.github.com/x").with(
+ headers: {
+ "If-None-Match" => 'W/"abc"',
+ },
+ ).to_return(status: 304, body: "")
+ expect(client.get("/x")).to eq({ "n" => 1 })
+ end
+ end
+
+ describe "#raw_get" do
+ it "returns the raw body as a String" do
+ stub_request(:get, "https://raw.githubusercontent.com/d/d/main/x.rb").to_return(
+ status: 200,
+ body: "puts 1",
+ )
+ expect(client.raw_get("https://raw.githubusercontent.com/d/d/main/x.rb")).to eq("puts 1")
+ end
+ end
+
+ describe "host allowlist" do
+ it "refuses to call (and leak the token to) a non-GitHub absolute URL" do
+ expect { client.get("https://evil.example.com/steal") }.to raise_error(ArgumentError)
+ expect { client.raw_get("https://evil.example.com/steal") }.to raise_error(ArgumentError)
+ expect(a_request(:get, "https://evil.example.com/steal")).not_to have_been_made
+ end
+ end
+
+ describe "#post" do
+ it "sends a JSON body" do
+ stub =
+ stub_request(:post, "https://api.github.com/repos/d/d/issues/1/comments").with(
+ body: { body: "hi" }.to_json,
+ ).to_return(status: 201, body: "{}")
+ client.post("/repos/d/d/issues/1/comments", { body: "hi" })
+ expect(stub).to have_been_requested
+ end
+ end
+end
diff --git a/spec/lib/github_rate_limit_spec.rb b/spec/lib/github_rate_limit_spec.rb
new file mode 100644
index 00000000000..6f6d44924a5
--- /dev/null
+++ b/spec/lib/github_rate_limit_spec.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+RSpec.describe GithubRateLimit do
+ let(:token) { "gh_token_123" }
+
+ def ttl(token = nil)
+ Discourse.redis.without_namespace.ttl(described_class.key(token))
+ end
+
+ describe ".key" do
+ it "is per-token, and a shared bucket for unauthenticated requests" do
+ expect(described_class.key(token)).to eq(
+ "onebox_github_backoff_#{Digest::SHA1.hexdigest(token)}",
+ )
+ expect(described_class.key(nil)).to eq("onebox_github_backoff_unauthenticated")
+ expect(described_class.key("")).to eq("onebox_github_backoff_unauthenticated")
+ end
+ end
+
+ describe ".note_rate_limit" do
+ it "backs off for the Retry-After duration" do
+ described_class.note_rate_limit(token:, retry_after: 90)
+ expect(described_class.backing_off?(token)).to eq(true)
+ expect(ttl(token)).to be_between(1, 90)
+ end
+
+ it "backs off until x-ratelimit-reset when the remaining budget is 0" do
+ described_class.note_rate_limit(token:, remaining: "0", reset_at: 10.minutes.from_now.to_i)
+ expect(ttl(token)).to be_between(1, 600)
+ end
+
+ it "does nothing when there is budget left and no Retry-After" do
+ described_class.note_rate_limit(token:, remaining: "57")
+ expect(described_class.backing_off?(token)).to eq(false)
+ end
+
+ it "clamps the backoff to one hour" do
+ described_class.note_rate_limit(token:, remaining: "0", reset_at: 10.days.from_now.to_i)
+ expect(ttl(token)).to be <= 1.hour.to_i
+ end
+
+ it "is scoped per token" do
+ described_class.note_rate_limit(token:, retry_after: 90)
+ expect(described_class.backing_off?("other_token")).to eq(false)
+ expect(described_class.backing_off?(nil)).to eq(false)
+ end
+ end
+
+ describe ".note_response_headers" do
+ it "reads the rate-limit headers regardless of casing" do
+ described_class.note_response_headers(
+ { "Retry-After" => "120", "X-RateLimit-Remaining" => "0" },
+ token:,
+ )
+ expect(ttl(token)).to be_between(1, 120)
+ end
+ end
+
+ describe ".active?" do
+ it "is true while any backoff key exists" do
+ expect(described_class.active?).to eq(false)
+ described_class.note_rate_limit(token:, retry_after: 30)
+ expect(described_class.active?).to eq(true)
+ end
+ end
+end
diff --git a/spec/lib/onebox/engine/github_blob_onebox_spec.rb b/spec/lib/onebox/engine/github_blob_onebox_spec.rb
index 6f99bb8c2fc..3147d76de41 100644
--- a/spec/lib/onebox/engine/github_blob_onebox_spec.rb
+++ b/spec/lib/onebox/engine/github_blob_onebox_spec.rb
@@ -30,20 +30,12 @@ RSpec.describe Onebox::Engine::GithubBlobOnebox do
end
it "does not include blob contents if it is binary" do
- # stub_request if the response must be binary (ASCII-8BIT)
- uri_mock = mock("object")
- uri_mock.stubs(:open).returns(
- File.open("#{Rails.root.join("spec/fixtures/pdf/small.pdf")}", "rb"),
+ stub_request(:get, raw_uri).to_return(
+ status: 200,
+ body: File.binread("#{Rails.root.join("spec/fixtures/images/smallest.png")}"),
)
- URI.stubs(:parse).with(link).returns(uri)
- URI
- .stubs(:parse)
- .with(
- "https://raw.githubusercontent.com/discourse/onebox/master/lib/onebox/engine/github_blob_onebox.rb",
- )
- .returns(uri_mock)
- expect(html).not_to include("/Pages")
+ expect(html).not_to include("module Oneboxer")
expect(html).to include("This file is binary.")
end
diff --git a/spec/lib/onebox/github_access_spec.rb b/spec/lib/onebox/github_access_spec.rb
new file mode 100644
index 00000000000..c9624bfed8e
--- /dev/null
+++ b/spec/lib/onebox/github_access_spec.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+RSpec.describe Onebox::GithubAccess do
+ before { Discourse::GithubApi.reset_clients! }
+
+ describe ".token" do
+ it "returns nil when no onebox tokens are configured" do
+ SiteSetting.github_onebox_access_tokens = ""
+ expect(described_class.token("discourse")).to be_nil
+ end
+
+ it "resolves the per-org token, falling back to the default mapping" do
+ SiteSetting.github_onebox_access_tokens = "discourse|org_tok\ndefault|def_tok"
+ expect(described_class.token("discourse")).to eq("org_tok")
+ expect(described_class.token("someoneelse")).to eq("def_tok")
+ end
+
+ it "returns nil for an unmapped org when no default is configured" do
+ SiteSetting.github_onebox_access_tokens = "discourse|org_tok"
+ expect(described_class.token("someoneelse")).to be_nil
+ end
+ end
+
+ describe ".tokens" do
+ it "returns just the unauthenticated identity when nothing is configured" do
+ SiteSetting.github_onebox_access_tokens = ""
+ expect(described_class.tokens).to eq([nil])
+ end
+
+ it "returns every configured token plus the unauthenticated identity" do
+ SiteSetting.github_onebox_access_tokens = "discourse|org_tok\ndefault|def_tok"
+ expect(described_class.tokens).to contain_exactly("org_tok", "def_tok", nil)
+ end
+ end
+
+ describe ".client" do
+ it "builds a client authenticated with the org's onebox token" do
+ SiteSetting.github_onebox_access_tokens = "discourse|org_tok\ndefault|def_tok"
+
+ org_stub =
+ stub_request(:get, "https://api.github.com/x").with(
+ headers: {
+ "Authorization" => "Bearer org_tok",
+ },
+ ).to_return(status: 200, body: "{}")
+ described_class.client("discourse").get("/x")
+ expect(org_stub).to have_been_requested
+
+ default_stub =
+ stub_request(:get, "https://api.github.com/y").with(
+ headers: {
+ "Authorization" => "Bearer def_tok",
+ },
+ ).to_return(status: 200, body: "{}")
+ described_class.client("someoneelse").get("/y")
+ expect(default_stub).to have_been_requested
+ end
+
+ it "builds an unauthenticated client when no onebox token is configured" do
+ SiteSetting.github_onebox_access_tokens = ""
+
+ stub =
+ stub_request(:get, "https://api.github.com/z")
+ .with { |request| !request.headers.key?("Authorization") }
+ .to_return(status: 200, body: "{}")
+ described_class.client("discourse").get("/z")
+ expect(stub).to have_been_requested
+ end
+ end
+end
diff --git a/spec/lib/onebox/mixins/github_api_spec.rb b/spec/lib/onebox/mixins/github_api_spec.rb
new file mode 100644
index 00000000000..d473ed0aef5
--- /dev/null
+++ b/spec/lib/onebox/mixins/github_api_spec.rb
@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+RSpec.describe Onebox::Mixins::GithubApi do
+ let(:pr_link) { "https://github.com/discourse/discourse/pull/1253" }
+ let(:pr_api) { "https://api.github.com/repos/discourse/discourse/pulls/1253" }
+ let(:reset_at) { 30.minutes.from_now.to_i }
+
+ def rate_limit_headers(remaining: "0", reset: reset_at, retry_after: nil)
+ headers = {
+ "x-ratelimit-limit" => "60",
+ "x-ratelimit-remaining" => remaining,
+ "x-ratelimit-reset" => reset.to_s,
+ }
+ headers["retry-after"] = retry_after.to_s if retry_after
+ headers
+ end
+
+ def render_pr
+ Onebox::Engine::GithubPullRequestOnebox.new(pr_link).to_html
+ rescue OpenURI::HTTPError
+ nil
+ end
+
+ context "when GitHub returns a primary rate-limit 403" do
+ before do
+ stub_request(:get, pr_api).to_return(
+ status: [403, "Forbidden"],
+ headers: rate_limit_headers,
+ body: '{"message":"API rate limit exceeded for 1.2.3.4"}',
+ )
+ end
+
+ it "records a backoff (until x-ratelimit-reset) and skips further GitHub requests" do
+ render_pr
+ expect(a_request(:get, pr_api)).to have_been_made.once
+
+ ttl = Discourse.redis.without_namespace.ttl("onebox_github_backoff_unauthenticated")
+ expect(ttl).to be_between(1, 1800)
+
+ render_pr
+ expect(a_request(:get, pr_api)).to have_been_made.once
+ end
+
+ it "short-circuits a different GitHub URL/engine sharing the same identity" do
+ render_pr
+
+ issue_link = "https://github.com/discourse/discourse/issues/999"
+ issue_api = "https://api.github.com/repos/discourse/discourse/issues/999"
+ stub_request(:get, issue_api).to_return(status: 200, body: "{}")
+
+ begin
+ Onebox::Engine::GithubIssueOnebox.new(issue_link).to_html
+ rescue StandardError
+ nil
+ end
+ expect(a_request(:get, issue_api)).not_to have_been_made
+ end
+ end
+
+ context "when GitHub sends a Retry-After header (secondary rate limit)" do
+ before do
+ stub_request(:get, pr_api).to_return(
+ status: [429, "Too Many Requests"],
+ headers: rate_limit_headers(remaining: "42", retry_after: 120),
+ body: "{}",
+ )
+ end
+
+ it "backs off for the Retry-After duration" do
+ render_pr
+ ttl = Discourse.redis.without_namespace.ttl("onebox_github_backoff_unauthenticated")
+ expect(ttl).to be_between(1, 120)
+ end
+ end
+
+ context "when GitHub returns a 403 that is NOT a rate limit (e.g. private repo)" do
+ before do
+ stub_request(:get, pr_api).to_return(
+ status: [403, "Forbidden"],
+ headers: rate_limit_headers(remaining: "57"),
+ body: '{"message":"Must have admin rights to Repository."}',
+ )
+ end
+
+ it "does not back off, so subsequent requests still reach GitHub" do
+ render_pr
+ expect(
+ Discourse.redis.without_namespace.get("onebox_github_backoff_unauthenticated"),
+ ).to be_nil
+
+ render_pr
+ expect(a_request(:get, pr_api)).to have_been_made.twice
+ end
+ end
+
+ context "when an org access token is configured" do
+ before { SiteSetting.github_onebox_access_tokens = "discourse|gh_token_xyz" }
+
+ it "scopes the backoff per token, not globally" do
+ stub_request(:get, pr_api).with(
+ headers: {
+ "Authorization" => "Bearer gh_token_xyz",
+ },
+ ).to_return(status: [403, "Forbidden"], headers: rate_limit_headers, body: "{}")
+
+ render_pr
+
+ token_key = "onebox_github_backoff_#{Digest::SHA1.hexdigest("gh_token_xyz")}"
+ expect(Discourse.redis.without_namespace.get(token_key)).to be_present
+ expect(
+ Discourse.redis.without_namespace.get("onebox_github_backoff_unauthenticated"),
+ ).to be_nil
+ end
+ end
+
+ context "when a successful response reports the rate-limit budget is exhausted" do
+ before do
+ SiteSetting.stubs(:github_pr_status_enabled).returns(false)
+ stub_request(:get, pr_api).to_return(
+ status: 200,
+ headers: rate_limit_headers(remaining: "0"),
+ body: onebox_response("githubpullrequest"),
+ )
+ end
+
+ it "proactively backs off until reset, without waiting for a 403" do
+ render_pr
+ ttl = Discourse.redis.without_namespace.ttl("onebox_github_backoff_unauthenticated")
+ expect(ttl).to be_between(1, 1800)
+
+ WebMock::RequestRegistry.instance.reset!
+ render_pr
+ expect(a_request(:get, pr_api)).not_to have_been_made
+ end
+ end
+
+ context "when any GitHub API engine hits the rate limit on its primary fetch" do
+ before do
+ stub_request(:get, /api\.github\.com/).to_return(
+ status: [403, "Forbidden"],
+ headers: rate_limit_headers,
+ body: "{}",
+ )
+ end
+
+ [
+ ["pull request", Onebox::Engine::GithubPullRequestOnebox, "https://github.com/d/d/pull/1"],
+ ["commit", Onebox::Engine::GithubCommitOnebox, "https://github.com/d/d/commit/#{"a" * 40}"],
+ ["issue", Onebox::Engine::GithubIssueOnebox, "https://github.com/d/d/issues/1"],
+ ["repo", Onebox::Engine::GithubRepoOnebox, "https://github.com/d/d"],
+ ["actions", Onebox::Engine::GithubActionsOnebox, "https://github.com/d/d/actions/runs/1"],
+ ["gist", Onebox::Engine::GithubGistOnebox, "https://gist.github.com/d/abc123def456"],
+ ].each do |name, klass, link|
+ it "records a backoff when the #{name} engine is rate-limited" do
+ begin
+ klass.new(link).to_html
+ rescue StandardError
+ nil
+ end
+ expect(
+ Discourse.redis.without_namespace.get("onebox_github_backoff_unauthenticated"),
+ ).to be_present
+ end
+ end
+ end
+
+ context "when a default access token is configured" do
+ it "authenticates gist API calls with the default token" do
+ SiteSetting.github_onebox_access_tokens = "default|gh_default_token"
+ stub =
+ stub_request(:get, "https://api.github.com/gists/abc123def456").with(
+ headers: {
+ "Authorization" => "Bearer gh_default_token",
+ },
+ ).to_return(status: 200, body: '{"files":{}}')
+
+ Onebox::Engine::GithubGistOnebox.new("https://gist.github.com/d/abc123def456").to_html
+
+ expect(stub).to have_been_requested
+ end
+ end
+end
diff --git a/spec/lib/oneboxer_github_backoff_spec.rb b/spec/lib/oneboxer_github_backoff_spec.rb
new file mode 100644
index 00000000000..c109bbcddc9
--- /dev/null
+++ b/spec/lib/oneboxer_github_backoff_spec.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+RSpec.describe Oneboxer do
+ describe ".external_onebox_ttl" do
+ it "uses a short TTL for a blank/failed result and 1 day for a successful one" do
+ expect(Oneboxer.external_onebox_ttl({ onebox: "", preview: "" })).to eq(1.minute)
+ expect(Oneboxer.external_onebox_ttl({ onebox: "<aside>onebox</aside>", preview: "x" })).to eq(
+ 1.day,
+ )
+ end
+ end
+
+ describe "InlineOneboxer caching" do
+ it "uses a short TTL for a failed lookup (blank title)" do
+ Discourse.cache.expects(:write).with(anything, anything, expires_in: 1.minute)
+ InlineOneboxer.send(:onebox_for, "https://example.com/x", nil, {})
+ end
+
+ it "uses the normal 1 day TTL when a title is found" do
+ Discourse.cache.expects(:write).with(anything, anything, expires_in: 1.day)
+ InlineOneboxer.send(:onebox_for, "https://example.com/x", "Some title", {})
+ end
+ end
+end
diff --git a/spec/services/problem_check/github_onebox_backoff_spec.rb b/spec/services/problem_check/github_onebox_backoff_spec.rb
new file mode 100644
index 00000000000..ba61c35e375
--- /dev/null
+++ b/spec/services/problem_check/github_onebox_backoff_spec.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+RSpec.describe ProblemCheck::GithubOneboxBackoff do
+ subject(:check) { described_class.new }
+
+ describe "#call" do
+ context "when no GitHub backoff is active" do
+ it { expect(check).to be_chill_about_it }
+ end
+
+ context "when the unauthenticated onebox client is backing off" do
+ before { GithubRateLimit.note_rate_limit(remaining: "0", reset_at: 10.minutes.from_now.to_i) }
+
+ it { expect(check).to have_a_problem.with_priority("low") }
+ end
+
+ context "when a configured onebox token is backing off" do
+ before do
+ SiteSetting.github_onebox_access_tokens = "default|onebox_token"
+ GithubRateLimit.note_rate_limit(
+ token: "onebox_token",
+ remaining: "0",
+ reset_at: 10.minutes.from_now.to_i,
+ )
+ end
+
+ it { expect(check).to have_a_problem.with_priority("low") }
+ end
+
+ context "when only an unrelated GitHub token is backing off" do
+ before do
+ SiteSetting.github_onebox_access_tokens = "default|onebox_token"
+ GithubRateLimit.note_rate_limit(
+ token: "ai_bot_token",
+ remaining: "0",
+ reset_at: 10.minutes.from_now.to_i,
+ )
+ end
+
+ it { expect(check).to be_chill_about_it }
+ end
+ end
+end