A platform for community discussion. Free, open, simple.
FEATURE: Route all GitHub API requests through one rate-limited client (#40637)
Régis Hanol36a8a51ef007
diff --git a/Gemfile b/Gemfileindex 2c04756f3cb..2df98e77f04 100644--- a/Gemfile+++ b/Gemfile@@ -298,7 +298,7 @@ gem "zendesk_api", require: false# for discourse-subscriptionsgem "stripe", require: false-# for discourse-github+# for discourse-code-reviewgem "sawyer", require: falsegem "octokit", require: falsediff --git a/app/models/problem_check.rb b/app/models/problem_check.rbindex 4492c095395..78a00e350f9 100644--- a/app/models/problem_check.rb+++ b/app/models/problem_check.rb@@ -74,6 +74,7 @@ class ProblemCheckProblemCheck::FailingEmails,ProblemCheck::ForceHttps,ProblemCheck::GithubConfig,+ ProblemCheck::GithubOneboxBackoff,ProblemCheck::GoogleAnalyticsVersion,ProblemCheck::GoogleOauth2Config,ProblemCheck::GroupEmailCredentials,@@ -217,12 +218,14 @@ class ProblemChecktarget_identifier = target.kind_of?(ActiveRecord::Base) ? target.id : targetProblem.new(- I18n.t(- override_key || translation_key,- base_path: Discourse.base_path,- **override_data.merge(- target.present? ? translation_data(target) : translation_data,- ).symbolize_keys,+ SiteSettings::LabelFormatter.expand_setting_links(+ I18n.t(+ override_key || translation_key,+ base_path: Discourse.base_path,+ **override_data.merge(+ target.present? ? translation_data(target) : translation_data,+ ).symbolize_keys,+ ),),priority: config.priority,identifier:,diff --git a/app/services/problem_check/github_onebox_backoff.rb b/app/services/problem_check/github_onebox_backoff.rbnew file mode 100644index 00000000000..ca66dcd337f--- /dev/null+++ b/app/services/problem_check/github_onebox_backoff.rb@@ -0,0 +1,11 @@+# frozen_string_literal: true++class ProblemCheck::GithubOneboxBackoff < ProblemCheck+ self.priority = "low"+ self.perform_every = 10.minutes++ def call+ backing_off = Onebox::GithubAccess.tokens.any? { |token| GithubRateLimit.backing_off?(token) }+ backing_off ? problem : no_problem+ end+enddiff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex 30a74fff4a3..c9fa9a4c69a 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -1814,6 +1814,7 @@ en:facebook_config: 'The server is configured to allow signup and login with Facebook (enable_facebook_logins), but the app id and app secret values are not set. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/configuring-facebook-login-for-discourse/13394" target="_blank">See this guide to learn more</a>.'twitter_config: 'The server is configured to allow signup and login with Twitter (enable_twitter_logins), but the key and secret values are not set. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/configuring-twitter-login-for-discourse/13395" target="_blank">See this guide to learn more</a>.'github_config: 'The server is configured to allow signup and login with GitHub (enable_github_logins), but the client id and secret values are not set. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/configuring-github-login-for-discourse/13745" target="_blank">See this guide to learn more</a>.'+ github_onebox_backoff: "GitHub is rate-limiting the server, so GitHub links may not preview until the limit resets. Configuring {{setting:github_onebox_access_tokens}} raises the limit significantly."s3_upload_config: 'The server is configured to upload files to S3, but at least one the following setting is not set: s3_access_key_id, s3_secret_access_key, s3_use_iam_profile, or s3_upload_bucket. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/how-to-set-up-image-uploads-to-s3/7229" target="_blank">See "How to set up image uploads to S3?" to learn more</a>.'s3_backup_config: 'The server is configured to upload backups to S3, but at least one the following setting is not set: s3_access_key_id, s3_secret_access_key, s3_use_iam_profile, or s3_backup_bucket. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/how-to-set-up-image-uploads-to-s3/7229" target="_blank">See "How to set up image uploads to S3?" to learn more</a>.'s3_cdn: 'The server is configured to upload files to S3, but there is no S3 CDN configured. This can lead to expensive S3 costs and slower site performance. <a href="https://meta.discourse.org/t/-/148916" target="_blank">See "Using Object Storage for Uploads" to learn more</a>.'diff --git a/lib/discourse/github_api.rb b/lib/discourse/github_api.rbnew file mode 100644index 00000000000..40dc1c45a8b--- /dev/null+++ b/lib/discourse/github_api.rb@@ -0,0 +1,126 @@+# frozen_string_literal: true++require "faraday"++module Discourse+ class GithubApi+ API_ROOT = "https://api.github.com"+ ALLOWED_HOSTS = %w[api.github.com raw.githubusercontent.com].freeze+ DEFAULT_ACCEPT = "application/vnd.github+json"+ TIMEOUT = 5+ ETAG_MAX_BYTES = 512.kilobytes++ class Error < StandardError+ attr_reader :status++ def initialize(message = nil, status: nil)+ @status = status+ super(message || "GitHub API error (status #{status})")+ end+ end++ class NotFound < Error+ end++ class Unauthorized < Error+ end++ class RateLimited < Error+ end++ def self.for(token:)+ (@clients ||= {})[token.to_s] ||= new(token)+ end++ def self.reset_clients!+ @clients = {}+ end++ def initialize(token)+ @token = token.presence+ end++ def backing_off?+ GithubRateLimit.backing_off?(@token)+ end++ def get(path, accept: DEFAULT_ACCEPT, **params)+ body = request(:get, path, params:, accept:)+ body.present? ? ::MultiJson.load(body) : nil+ end++ def post(path, body = nil, accept: DEFAULT_ACCEPT)+ response_body = request(:post, path, body:, accept:)+ response_body.present? ? ::MultiJson.load(response_body) : nil+ end++ def raw_get(path, accept: DEFAULT_ACCEPT)+ request(:get, path, accept:, cache_etag: false)+ end++ private++ def request(method, path, params: {}, body: nil, accept:, cache_etag: true)+ raise RateLimited, "GitHub rate limit backoff active" if backing_off?++ url = path.start_with?("http") ? path : "#{API_ROOT}#{path}"+ host = URI.parse(url).host+ if ALLOWED_HOSTS.exclude?(host)+ raise ArgumentError, "Refusing to call non-GitHub host: #{host.inspect}"+ end++ etag_key = (etag_cache_key(url, params, accept) if cache_etag && method == :get)+ cached = (Discourse.cache.read(etag_key) if etag_key)++ response = perform(method, url, params:, body:, accept:, etag: cached&.dig(:etag))+ backed_off = GithubRateLimit.note_response_headers(response.headers, token: @token)++ case response.status+ when 200..299+ store_etag(etag_key, response) if etag_key+ response.body+ when 304+ cached&.dig(:body)+ when 401+ raise Unauthorized.new(status: 401)+ when 404+ raise NotFound.new(status: 404)+ else+ raise RateLimited.new("GitHub rate limited", status: response.status) if backed_off+ raise Error.new(status: response.status)+ end+ end++ def perform(method, url, params:, body:, accept:, etag:)+ headers = { "Accept" => accept, "User-Agent" => Discourse.user_agent }+ headers["Authorization"] = "Bearer #{@token}" if @token+ headers["If-None-Match"] = etag if etag.present?+ headers["Content-Type"] = "application/json" if body++ connection.run_request(method, url, body && ::MultiJson.dump(body), headers) do |req|+ req.params.update(params) if params.present?+ req.options.timeout = TIMEOUT+ req.options.open_timeout = TIMEOUT+ end+ rescue Faraday::Error => e+ raise Error.new("GitHub API request failed: #{e.message}")+ end++ def connection+ @connection ||= Faraday.new(nil) { |f| f.adapter FinalDestination::FaradayAdapter }+ end++ def etag_cache_key(url, params, accept)+ identity = @token ? Digest::SHA1.hexdigest(@token) : "anon"+ query = params.present? ? "?#{params.to_query}" : ""+ "github_etag:#{identity}:#{accept}:#{url}#{query}"+ end++ def store_etag(key, response)+ etag = response.headers["etag"]+ return if etag.blank? || response.body.to_s.bytesize > ETAG_MAX_BYTES++ Discourse.cache.write(key, { etag:, body: response.body }, expires_in: 1.day)+ end+ end+enddiff --git a/lib/github_rate_limit.rb b/lib/github_rate_limit.rbnew file mode 100644index 00000000000..a30cd9540db--- /dev/null+++ b/lib/github_rate_limit.rb@@ -0,0 +1,54 @@+# frozen_string_literal: true++require "digest"++class GithubRateLimit+ KEY_PREFIX = "onebox_github_backoff_"+ MAX_SECONDS = 1.hour.to_i++ class << self+ def backing_off?(token = nil)+ redis.get(key(token)).present?+ end++ def active?+ redis.scan_each(match: "#{KEY_PREFIX}*").first.present?+ end++ def note_rate_limit(token: nil, retry_after: nil, remaining: nil, reset_at: nil)+ seconds =+ if retry_after.present?+ retry_after.to_i+ elsif remaining.to_s == "0" && reset_at.present?+ reset_at.to_i - Time.now.to_i+ end+ return if seconds.nil?++ seconds = seconds.clamp(1, MAX_SECONDS)+ redis.setex(key(token), seconds, "1")+ Rails.logger.warn("GitHub API rate limited; backing off for #{seconds}s")+ seconds+ end++ def note_response_headers(headers, token: nil)+ headers = headers.to_h.transform_keys { |k| k.to_s.downcase }+ note_rate_limit(+ token:,+ retry_after: headers["retry-after"],+ remaining: headers["x-ratelimit-remaining"],+ reset_at: headers["x-ratelimit-reset"],+ )+ end++ def key(token)+ identity = token.present? ? Digest::SHA1.hexdigest(token) : "unauthenticated"+ "#{KEY_PREFIX}#{identity}"+ end++ private++ def redis+ Discourse.redis.without_namespace+ end+ end+enddiff --git a/lib/inline_oneboxer.rb b/lib/inline_oneboxer.rbindex 6d78c3776f2..b147056f56c 100644--- a/lib/inline_oneboxer.rb+++ b/lib/inline_oneboxer.rb@@ -134,7 +134,10 @@ class InlineOneboxeronebox = { url: url, title: title }onebox[:css_class] = css_class if css_class.present?- Discourse.cache.write(cache_key(url), onebox, expires_in: 1.day) if !opts[:skip_cache]+ if !opts[:skip_cache]+ ttl = onebox[:title].blank? ? 1.minute : 1.day+ Discourse.cache.write(cache_key(url), onebox, expires_in: ttl)+ endoneboxenddiff --git a/lib/onebox/engine/github_actions_onebox.rb b/lib/onebox/engine/github_actions_onebox.rbindex 20f25a1ddcf..0ca0d26482b 100644--- a/lib/onebox/engine/github_actions_onebox.rb+++ b/lib/onebox/engine/github_actions_onebox.rb@@ -1,7 +1,7 @@# frozen_string_literal: truerequire_relative "../mixins/github_body"-require_relative "../mixins/github_auth_header"+require_relative "../mixins/github_api"module Oneboxmodule Engine@@ -9,7 +9,7 @@ module Oneboxinclude Engineinclude LayoutSupportinclude JSON- include Onebox::Mixins::GithubAuthHeader+ include Onebox::Mixins::GithubApimatches_regexp(%r{^https?://(?:www\.)?(?:(?:\w)+\.)?github\.com/(?<org>.+)/(?<repo>.+)/(actions/runs/[[:digit:]]+|pull/[[:digit:]]*/checks\?check_run_id=[[:digit:]]+)},@@ -65,7 +65,7 @@ module Oneboxenddef data- result = raw(github_auth_header(match[:org])).clone+ result = raw.clonestatus = "unknown"if result["status"] == "completed"@@ -84,7 +84,7 @@ module Oneboxelsif type == :pr_runpr_url ="https://api.github.com/repos/#{match[:org]}/#{match[:repo]}/pulls/#{match[:pr_id]}"- ::MultiJson.load(URI.parse(pr_url).open(read_timeout: timeout))["title"]+ load_json(pr_url)["title"]end{diff --git a/lib/onebox/engine/github_blob_onebox.rb b/lib/onebox/engine/github_blob_onebox.rbindex 9ae691aa13a..e876d11d920 100644--- a/lib/onebox/engine/github_blob_onebox.rb+++ b/lib/onebox/engine/github_blob_onebox.rb@@ -1,13 +1,11 @@# frozen_string_literal: truerequire_relative "../mixins/git_blob_onebox"-require_relative "../mixins/github_auth_header"+require_relative "../github_access"module Oneboxmodule Engineclass GithubBlobOnebox- include Onebox::Mixins::GithubAuthHeader-def self.git_regexp%r{^https?://(www\.)?github\.com.*/blob/}end@@ -35,8 +33,14 @@ module Onebox"https://raw.githubusercontent.com/#{match[:org]}/#{match[:repo]}/#{match[:sha1]}/#{match[:file]}"end- def auth_headers(match)- github_auth_header(match[:org])+ def auth_headers(_match)+ {}+ end++ def fetch_raw(url, _headers)+ Onebox::GithubAccess.client(match[:org]).raw_get(url)+ rescue ::Discourse::GithubApi::Error => e+ raise OpenURI::HTTPError.new(e.message, nil)endprivatediff --git a/lib/onebox/engine/github_commit_onebox.rb b/lib/onebox/engine/github_commit_onebox.rbindex cccbd6d438f..02a0d2e9df9 100644--- a/lib/onebox/engine/github_commit_onebox.rb+++ b/lib/onebox/engine/github_commit_onebox.rb@@ -1,7 +1,7 @@# frozen_string_literal: truerequire_relative "../mixins/github_body"-require_relative "../mixins/github_auth_header"+require_relative "../mixins/github_api"module Oneboxmodule Engine@@ -10,7 +10,7 @@ module Oneboxinclude LayoutSupportinclude JSONinclude Onebox::Mixins::GithubBody- include Onebox::Mixins::GithubAuthHeader+ include Onebox::Mixins::GithubApimatches_domain("github.com", "www.github.com")always_https@@ -24,9 +24,9 @@ module Oneboxenddef inline_data- return if github_auth_header(match[:org]).blank?+ return unless github_token?- result = raw(github_auth_header(match[:org]))+ result = rawmessage = result["commit"]["message"].split("\n").first{title:@@ -52,7 +52,7 @@ module Oneboxenddef data- result = raw(github_auth_header(match[:org])).clone+ result = raw.clonelines = result["commit"]["message"].split("\n")result["title"] = lines.firstdiff --git a/lib/onebox/engine/github_gist_onebox.rb b/lib/onebox/engine/github_gist_onebox.rbindex 8e46c5ae954..65af85b2e30 100644--- a/lib/onebox/engine/github_gist_onebox.rb+++ b/lib/onebox/engine/github_gist_onebox.rb@@ -1,11 +1,14 @@# frozen_string_literal: true+require_relative "../mixins/github_api"+module Oneboxmodule Engineclass GithubGistOneboxinclude Engineinclude LayoutSupportinclude JSON+ include Onebox::Mixins::GithubApiMAX_FILES = 3diff --git a/lib/onebox/engine/github_issue_onebox.rb b/lib/onebox/engine/github_issue_onebox.rbindex e0df2d0be75..23472e352c5 100644--- a/lib/onebox/engine/github_issue_onebox.rb+++ b/lib/onebox/engine/github_issue_onebox.rb@@ -1,7 +1,7 @@# frozen_string_literal: truerequire_relative "../mixins/github_body"-require_relative "../mixins/github_auth_header"+require_relative "../mixins/github_api"module Oneboxmodule Engine@@ -11,7 +11,7 @@ module Oneboxinclude LayoutSupportinclude JSONinclude Onebox::Mixins::GithubBody- include Onebox::Mixins::GithubAuthHeader+ include Onebox::Mixins::GithubApimatches_regexp(%r{^https?://(?:www\.)?(?:(?:\w)+\.)?github\.com/(?<org>.+)/(?<repo>.+)/issues/([[:digit:]]+)},@@ -24,9 +24,9 @@ module Oneboxenddef inline_data- return if github_auth_header(match["org"]).blank?+ return unless github_token?- result = raw(github_auth_header(match["org"]))+ result = raw{title:"#{result["title"]} - Issue ##{match["item_id"]} - #{match["org"]}/#{match["repo"]} - GitHub",@@ -50,7 +50,7 @@ module Oneboxenddef data- result = raw(github_auth_header(match[:org])).clone+ result = raw.clonerepo = load_repocreated_at = Time.parse(result["created_at"])closed_at = Time.parse(result["closed_at"]) if result["closed_at"]@@ -86,12 +86,6 @@ module Oneboxdef load_repoload_json("https://api.github.com/repos/#{match[:org]}/#{match[:repo]}")end-- def load_json(url)- ::MultiJson.load(- URI.parse(url).open({ read_timeout: timeout }.merge(github_auth_header(match[:org]))),- )- endendendenddiff --git a/lib/onebox/engine/github_pull_request_onebox.rb b/lib/onebox/engine/github_pull_request_onebox.rbindex 1df4df1d660..18ef1aa4f20 100644--- a/lib/onebox/engine/github_pull_request_onebox.rb+++ b/lib/onebox/engine/github_pull_request_onebox.rb@@ -1,7 +1,7 @@# frozen_string_literal: truerequire_relative "../mixins/github_body"-require_relative "../mixins/github_auth_header"+require_relative "../mixins/github_api"module Oneboxmodule Engine@@ -10,7 +10,7 @@ module Oneboxinclude LayoutSupportinclude JSONinclude Onebox::Mixins::GithubBody- include Onebox::Mixins::GithubAuthHeader+ include Onebox::Mixins::GithubApimatches_domain("github.com", "www.github.com")always_https@@ -24,7 +24,7 @@ module Oneboxenddef inline_data- return if github_auth_header(match[:org]).blank? && !SiteSetting.github_pr_status_enabled+ return if !github_token? && !SiteSetting.github_pr_status_enabledif commit_sha = @url[%r{/commits?/(\h+)}, 1]commit =@@ -40,7 +40,7 @@ module Onebox)end- pr_data = raw(github_auth_header(match[:org]))+ pr_data = rawresult = {title:"#{pr_data["title"]} - Pull Request ##{match[:number]} - #{match[:org]}/#{match[:repository]} - GitHub",@@ -65,7 +65,7 @@ module Oneboxenddef data- result = raw(github_auth_header(match[:org])).clone+ result = raw.cloneresult["link"] = linkstatus_data = fetch_pr_status(result)@@ -158,15 +158,6 @@ module Oneboxendend- def load_json(url)- ::MultiJson.load(- URI.parse(url).open({ read_timeout: timeout }.merge(github_auth_header(match[:org]))),- )- rescue OpenURI::HTTPError => e- Rails.logger.warn("GitHub API error: #{e.io.status[0]} fetching #{url}")- raise- end-def fetch_pr_status(pr_data)return unless SiteSetting.github_pr_status_enableddiff --git a/lib/onebox/engine/github_repo_onebox.rb b/lib/onebox/engine/github_repo_onebox.rbindex 591b0b00333..db81ca7be51 100644--- a/lib/onebox/engine/github_repo_onebox.rb+++ b/lib/onebox/engine/github_repo_onebox.rb@@ -1,7 +1,7 @@# frozen_string_literal: truerequire_relative "../mixins/github_body"-require_relative "../mixins/github_auth_header"+require_relative "../mixins/github_api"module Oneboxmodule Engine@@ -9,7 +9,7 @@ module Oneboxinclude Engineinclude LayoutSupportinclude JSON- include Onebox::Mixins::GithubAuthHeader+ include Onebox::Mixins::GithubApimatches_domain("github.com", "www.github.com")always_https@@ -23,9 +23,9 @@ module Oneboxenddef inline_data- return if github_auth_header(match[:org]).blank?+ return unless github_token?- result = raw(github_auth_header(match[:org]))+ result = rawtitle = "GitHub - #{result["full_name"]}"title += " - #{Onebox::Helpers.truncate(result["description"])}" if result["description"@@ -43,7 +43,7 @@ module Oneboxenddef data- result = raw(github_auth_header(match[:org])).clone+ result = raw.cloneresult["link"] = linkdescription = result["description"]title = "GitHub - #{result["full_name"]}"diff --git a/lib/onebox/github_access.rb b/lib/onebox/github_access.rbnew file mode 100644index 00000000000..b7fc549524c--- /dev/null+++ b/lib/onebox/github_access.rb@@ -0,0 +1,26 @@+# frozen_string_literal: true++module Onebox+ module GithubAccess+ def self.token(github_org)+ org_tokens = org_token_map+ (org_tokens[github_org] || org_tokens["default"]).presence+ end++ def self.tokens+ (org_token_map.values.filter_map(&:presence) << nil).uniq+ end++ def self.client(github_org)+ ::Discourse::GithubApi.for(token: token(github_org))+ end++ def self.org_token_map+ tokens = SiteSetting.github_onebox_access_tokens+ return {} if tokens.blank?++ tokens.split("\n").to_h { it.split("|") }+ end+ private_class_method :org_token_map+ end+enddiff --git a/lib/onebox/mixins/git_blob_onebox.rb b/lib/onebox/mixins/git_blob_onebox.rbindex 5f51a062ca5..97d9d0a3b57 100644--- a/lib/onebox/mixins/git_blob_onebox.rb+++ b/lib/onebox/mixins/git_blob_onebox.rb@@ -190,18 +190,17 @@ module Onebox@model_file = @lang.dup@raw = "https://render.githubusercontent.com/view/solid?url=" + raw_template(m)else- contents =- URI- .parse(raw_template(m))- .open({ read_timeout: timeout }.merge(auth_headers(m)))- .read+ contents = fetch_raw(raw_template(m), auth_headers(m)).to_s+ utf8_contents = contents.dup.force_encoding(Encoding::UTF_8)- if contents.encoding == Encoding::BINARY || contents.bytes.include?(0)+ if contents.bytes.include?(0) || !utf8_contents.valid_encoding?@raw = nil@binary = truereturnend+ contents = utf8_contents+contents_lines = contents.lines #get contents linescontents_lines_size = contents_lines.size #get number of lines@@ -240,6 +239,10 @@ module Oneboxendend+ def fetch_raw(url, headers)+ URI.parse(url).open({ read_timeout: timeout }.merge(headers)).read+ end+def data@data ||= {title: title,diff --git a/lib/onebox/mixins/github_api.rb b/lib/onebox/mixins/github_api.rbnew file mode 100644index 00000000000..fd350a5dc88--- /dev/null+++ b/lib/onebox/mixins/github_api.rb@@ -0,0 +1,42 @@+# frozen_string_literal: true++require "open-uri"+require_relative "../github_access"++module Onebox+ module Mixins+ module GithubApi+ class GithubRateLimited < OpenURI::HTTPError+ def initialize(message = "GitHub API rate limit backoff is active")+ super(message, nil)+ end+ end++ def load_json(url, github_org = match_org)+ github_fetch(url, Onebox::GithubAccess.client(github_org))+ end++ def github_token?(github_org = match_org)+ Onebox::GithubAccess.token(github_org).present?+ end++ private++ def raw+ @raw ||= load_json(url)+ end++ def match_org+ match.names.include?("org") ? match[:org] : nil+ end++ def github_fetch(url, client)+ client.get(url)+ rescue ::Discourse::GithubApi::RateLimited+ raise GithubRateLimited+ rescue ::Discourse::GithubApi::Error => e+ raise OpenURI::HTTPError.new(e.message, nil)+ end+ end+ end+enddiff --git a/lib/onebox/mixins/github_auth_header.rb b/lib/onebox/mixins/github_auth_header.rbdeleted file mode 100644index 8754b3c84d5..00000000000--- a/lib/onebox/mixins/github_auth_header.rb+++ /dev/null@@ -1,19 +0,0 @@-# frozen_string_literal: true--module Onebox- module Mixins- module GithubAuthHeader- def github_auth_header(github_org)- return {} if SiteSetting.github_onebox_access_tokens.blank?-- org_tokens = SiteSetting.github_onebox_access_tokens.split("\n").to_h { it.split("|") }-- token = org_tokens[github_org] || org_tokens["default"]-- return {} if token.blank?-- { "Authorization" => "Bearer #{token}" }- end- end- end-enddiff --git a/lib/oneboxer.rb b/lib/oneboxer.rbindex 31cd26195da..79aa175ad63 100644--- a/lib/oneboxer.rb+++ b/lib/oneboxer.rb@@ -520,100 +520,107 @@ module Oneboxerenddef self.external_onebox(url, available_strategies = nil)- Discourse- .cache- .fetch(onebox_cache_key(url), expires_in: 1.day) do- uri = URI(url)- available_strategies ||= Oneboxer.ordered_strategies(uri.hostname)- strategy = available_strategies.shift-- max_redirects = 0 if SiteSetting.block_onebox_on_redirect- fd =- FinalDestination.new(- url,- get_final_destination_options(url, strategy).merge(- stop_at_blocked_pages: true,- max_redirects: max_redirects,- initial_https_redirect_ignore_limit: SiteSetting.block_onebox_on_redirect,- ),- )- uri = fd.resolve-- return blank_onebox if fd.status == :blocked_page-- if fd.status != :resolved- args = { link: url }- if fd.status == :invalid_address- args[:error_message] = I18n.t("errors.onebox.invalid_address", hostname: fd.hostname)- elsif (fd.status_code || uri.nil?) && available_strategies.present?- # Try a different oneboxing strategy, if we have any options left:- return external_onebox(url, available_strategies)- elsif fd.status_code- args[:error_message] = I18n.t(- "errors.onebox.error_response",- status_code: fd.status_code,- )- end+ key = onebox_cache_key(url)+ cached = Discourse.cache.read(key)+ return cached if cached- error_box = blank_onebox- error_box[:preview] = preview_error_onebox(args)- return error_box- end+ result = compute_external_onebox(url, available_strategies)+ Discourse.cache.write(key, result, expires_in: external_onebox_ttl(result))+ result+ end++ def self.external_onebox_ttl(result)+ result[:onebox].blank? ? 1.minute : 1.day+ end++ def self.compute_external_onebox(url, available_strategies = nil)+ uri = URI(url)+ available_strategies ||= Oneboxer.ordered_strategies(uri.hostname)+ strategy = available_strategies.shift++ max_redirects = 0 if SiteSetting.block_onebox_on_redirect+ fd =+ FinalDestination.new(+ url,+ get_final_destination_options(url, strategy).merge(+ stop_at_blocked_pages: true,+ max_redirects: max_redirects,+ initial_https_redirect_ignore_limit: SiteSetting.block_onebox_on_redirect,+ ),+ )+ uri = fd.resolve++ return blank_onebox if fd.status == :blocked_page++ if fd.status != :resolved+ args = { link: url }+ if fd.status == :invalid_address+ args[:error_message] = I18n.t("errors.onebox.invalid_address", hostname: fd.hostname)+ elsif (fd.status_code || uri.nil?) && available_strategies.present?+ # Try a different oneboxing strategy, if we have any options left:+ return compute_external_onebox(url, available_strategies)+ elsif fd.status_code+ args[:error_message] = I18n.t("errors.onebox.error_response", status_code: fd.status_code)+ end++ error_box = blank_onebox+ error_box[:preview] = preview_error_onebox(args)+ return error_box+ end- return blank_onebox if uri.blank?-- onebox_options = {- max_width: 695,- sanitize_config: Onebox::SanitizeConfig::DISCOURSE_ONEBOX,- allowed_iframe_origins: allowed_iframe_origins,- hostname: GlobalSetting.hostname,- facebook_app_access_token: SiteSetting.facebook_app_access_token,- disable_media_download_controls: SiteSetting.disable_onebox_media_download_controls,- content_type: fd.content_type,- }-- onebox_options[:cookie] = fd.cookie if fd.cookie-- preview_result = Onebox.preview(uri.to_s, onebox_options)- result = {- onebox: WordWatcher.censor(preview_result.to_s),- preview: WordWatcher.censor(preview_result.placeholder_html.to_s),- }-- # NOTE: Call preview_result.errors after calling placeholder_html- if preview_result.errors.any?- error_keys = preview_result.errors.keys- skip_if_only_error = [:image]- unless error_keys.length == 1 && skip_if_only_error.include?(error_keys.first)- missing_attributes = error_keys.map(&:to_s).sort.join(I18n.t("word_connector.comma"))- error_message =- I18n.t(- "errors.onebox.missing_data",- missing_attributes: missing_attributes,- count: error_keys.size,- )- args = preview_result.verified_data.merge(error_message: error_message)-- if result[:preview].blank?- result[:preview] = preview_error_onebox(args)- else- doc = Nokogiri::HTML5.fragment(result[:preview])- aside = doc.at("aside")-- if aside- # Add an error message to the preview that was returned- error_fragment = preview_error_onebox_fragment(args)- aside.add_child(error_fragment)- result[:preview] = doc.to_html- end- end+ return blank_onebox if uri.blank?++ onebox_options = {+ max_width: 695,+ sanitize_config: Onebox::SanitizeConfig::DISCOURSE_ONEBOX,+ allowed_iframe_origins: allowed_iframe_origins,+ hostname: GlobalSetting.hostname,+ facebook_app_access_token: SiteSetting.facebook_app_access_token,+ disable_media_download_controls: SiteSetting.disable_onebox_media_download_controls,+ content_type: fd.content_type,+ }++ onebox_options[:cookie] = fd.cookie if fd.cookie++ preview_result = Onebox.preview(uri.to_s, onebox_options)+ result = {+ onebox: WordWatcher.censor(preview_result.to_s),+ preview: WordWatcher.censor(preview_result.placeholder_html.to_s),+ }++ # NOTE: Call preview_result.errors after calling placeholder_html+ if preview_result.errors.any?+ error_keys = preview_result.errors.keys+ skip_if_only_error = [:image]+ unless error_keys.length == 1 && skip_if_only_error.include?(error_keys.first)+ missing_attributes = error_keys.map(&:to_s).sort.join(I18n.t("word_connector.comma"))+ error_message =+ I18n.t(+ "errors.onebox.missing_data",+ missing_attributes: missing_attributes,+ count: error_keys.size,+ )+ args = preview_result.verified_data.merge(error_message: error_message)++ if result[:preview].blank?+ result[:preview] = preview_error_onebox(args)+ else+ doc = Nokogiri::HTML5.fragment(result[:preview])+ aside = doc.at("aside")++ if aside+ # Add an error message to the preview that was returned+ error_fragment = preview_error_onebox_fragment(args)+ aside.add_child(error_fragment)+ result[:preview] = doc.to_htmlendend+ end+ end- Oneboxer.cache_preferred_strategy(uri.hostname, strategy)+ Oneboxer.cache_preferred_strategy(uri.hostname, strategy)- result- end+ resultenddef self.preview_error_onebox(args, is_fragment = false)diff --git a/plugins/discourse-ai/lib/agents/tools/github_diff.rb b/plugins/discourse-ai/lib/agents/tools/github_diff.rbindex cb67f5fd34f..069286be082 100644--- a/plugins/discourse-ai/lib/agents/tools/github_diff.rb+++ b/plugins/discourse-ai/lib/agents/tools/github_diff.rb@@ -167,41 +167,14 @@ module DiscourseAienddef fetch_diff(api_url, type)- info = nil- diff_body = nil- response_code = "unknown error"-- send_http_request(- api_url,- headers: {- "Accept" => "application/json",- },- authenticate_github: true,- ) do |response|- response_code = response.code- info = JSON.parse(read_response_body(response)) if response_code == "200"- end-- if response_code == "200"- send_http_request(- api_url,- headers: {- "Accept" => "application/vnd.github.v3.diff",- },- authenticate_github: true,- ) do |response|- response_code = response.code- diff_body = read_response_body(response)- end- end-- if response_code == "200" && info && diff_body- diff = self.class.sort_and_shorten_diff(diff_body)- diff = truncate(diff, max_length: 20_000, percent_length: 0.3, llm: llm)- yield(info, diff)- else- { error: "Failed to retrieve the #{type} information. Status code: #{response_code}" }- end+ info = github_client.get(api_url)+ diff_body = github_client.raw_get(api_url, accept: "application/vnd.github.v3.diff")++ diff = self.class.sort_and_shorten_diff(diff_body)+ diff = truncate(diff, max_length: 20_000, percent_length: 0.3, llm: llm)+ yield(info, diff)+ rescue Discourse::GithubApi::Error => e+ { error: "Failed to retrieve the #{type} information. #{e.message}" }endendenddiff --git a/plugins/discourse-ai/lib/agents/tools/github_file_content.rb b/plugins/discourse-ai/lib/agents/tools/github_file_content.rbindex 232420cdd1b..0936796269b 100644--- a/plugins/discourse-ai/lib/agents/tools/github_file_content.rb+++ b/plugins/discourse-ai/lib/agents/tools/github_file_content.rb@@ -74,22 +74,8 @@ module DiscourseAiapi_url ="https://api.github.com/repos/#{owner}/#{repo}/contents/#{file_path}?ref=#{ref}"- response_code = "-1 unknown"- body = nil-- send_http_request(- api_url,- headers: {- "Accept" => "application/vnd.github.v3+json",- },- authenticate_github: true,- ) do |response|- response_code = response.code- body = read_response_body(response)- end-- if response_code == "200"- file_data = JSON.parse(body)+ begin+ file_data = github_client.get(api_url)content = ensure_utf8(Base64.decode64(file_data["content"].to_s))snippet =extract_requested_content(@@ -103,7 +89,7 @@ module DiscourseAistart_line: file_request[:start_line],end_line: file_request[:end_line],}- else+ rescue Discourse::GithubApi::Errormissing_files << file_request[:raw]endenddiff --git a/plugins/discourse-ai/lib/agents/tools/github_search_code.rb b/plugins/discourse-ai/lib/agents/tools/github_search_code.rbindex e05cd559452..eff560431ce 100644--- a/plugins/discourse-ai/lib/agents/tools/github_search_code.rb+++ b/plugins/discourse-ai/lib/agents/tools/github_search_code.rb@@ -75,53 +75,36 @@ module DiscourseAidef invokeapi_url = build_url- response_code = "unknown error"- search_data = nil-- send_http_request(- api_url,- headers: {- "Accept" => "application/vnd.github.v3.text-match+json",- },- authenticate_github: true,- ) do |response|- response_code = response.code- if response_code == "200"- begin- search_data = JSON.parse(read_response_body(response))- rescue JSON::ParserError- response_code = "500 - JSON parse error"- end- end+ begin+ search_data =+ github_client.get(api_url, accept: "application/vnd.github.v3.text-match+json")+ rescue Discourse::GithubApi::Error => e+ return { error: "Failed to perform code search. #{e.message}" }end- if response_code == "200"- formatted_results = format_results(search_data["items"])- grouped = group_by_file(formatted_results)- trimmed = trim_results(grouped)-- total_count = search_data["total_count"].to_i- total_pages =- if total_count <= 0- 1- else- [(total_count.to_f / PER_PAGE).ceil, MAX_ALLOWED_PAGE].min- end+ formatted_results = format_results(search_data["items"])+ grouped = group_by_file(formatted_results)+ trimmed = trim_results(grouped)- result = { search_results: trimmed }+ total_count = search_data["total_count"].to_i+ total_pages =+ if total_count <= 0+ 1+ else+ [(total_count.to_f / PER_PAGE).ceil, MAX_ALLOWED_PAGE].min+ end- result[:page] = page if page > 1- result[:total_results] = total_count- result[:next_page] = page + 1 if page < total_pages+ result = { search_results: trimmed }- if search_data["incomplete_results"]- result[:notes] = "GitHub marked the search results as incomplete."- end+ result[:page] = page if page > 1+ result[:total_results] = total_count+ result[:next_page] = page + 1 if page < total_pages- result- else- { error: "Failed to perform code search. Status code: #{response_code}" }+ if search_data["incomplete_results"]+ result[:notes] = "GitHub marked the search results as incomplete."end++ resultendprivate@@ -283,29 +266,11 @@ module DiscourseAi"https://api.github.com/repos/#{owner}/#{repo_name}/contents/#{path}?ref=#{actual_ref}"end- response_code = nil- body = nil-- send_http_request(- url,- headers: {- "Accept" => "application/vnd.github.v3+json",- },- authenticate_github: true,- ) do |response|- response_code = response.code- body = read_response_body(response)- end-- if response_code == "200"- begin- data = JSON.parse(body)- decoded = ensure_utf8(Base64.decode64(data["content"].to_s))- cache[cache_key] = { content: decoded, total_lines: count_lines(decoded) }- rescue JSON::ParserError- cache[cache_key] = nil- end- else+ begin+ data = github_client.get(url)+ decoded = ensure_utf8(Base64.decode64(data["content"].to_s))+ cache[cache_key] = { content: decoded, total_lines: count_lines(decoded) }+ rescue Discourse::GithubApi::Errorcache[cache_key] = nilenddiff --git a/plugins/discourse-ai/lib/agents/tools/github_search_files.rb b/plugins/discourse-ai/lib/agents/tools/github_search_files.rbindex 935fc676ee8..64b83c435b1 100644--- a/plugins/discourse-ai/lib/agents/tools/github_search_files.rb+++ b/plugins/discourse-ai/lib/agents/tools/github_search_files.rb@@ -63,46 +63,27 @@ module DiscourseAiapi_url = "https://api.github.com/repos/#{repo}/git/trees/#{branch_name}?recursive=1"- response_code = "unknown error"- tree_data = nil-- send_http_request(- api_url,- headers: {- "Accept" => "application/vnd.github.v3+json",- },- authenticate_github: true,- ) do |response|- response_code = response.code- if response_code == "200"- begin- tree_data = JSON.parse(read_response_body(response))- rescue JSON::ParserError- response_code = "500 - JSON parse error"- end- end+ begin+ tree_data = github_client.get(api_url)+ rescue Discourse::GithubApi::Error => e+ return { error: "Failed to perform file search. #{e.message}" }end- if response_code == "200"- matching_files =- tree_data["tree"]- .select do |item|- item["type"] == "blob" &&- keywords.any? { |keyword| item["path"].include?(keyword) }- end- .map { |item| { path: item["path"], size: item["size"] } }- .take(MAX_FILE_SEARCH_RESULTS)+ matching_files =+ tree_data["tree"]+ .select do |item|+ item["type"] == "blob" && keywords.any? { |keyword| item["path"].include?(keyword) }+ end+ .map { |item| { path: item["path"], size: item["size"] } }+ .take(MAX_FILE_SEARCH_RESULTS)- result = { matching_files: matching_files, branch: branch_name }- if matching_files.length == MAX_FILE_SEARCH_RESULTS- result[- :note- ] = "Result limit reached (#{MAX_FILE_SEARCH_RESULTS} files). There may be more matching files."- end- result- else- { error: "Failed to perform file search. Status code: #{response_code}" }+ result = { matching_files: matching_files, branch: branch_name }+ if matching_files.length == MAX_FILE_SEARCH_RESULTS+ result[+ :note+ ] = "Result limit reached (#{MAX_FILE_SEARCH_RESULTS} files). There may be more matching files."end+ resultendendenddiff --git a/plugins/discourse-ai/lib/agents/tools/tool.rb b/plugins/discourse-ai/lib/agents/tools/tool.rbindex b61b1abbf1a..46ac396c380 100644--- a/plugins/discourse-ai/lib/agents/tools/tool.rb+++ b/plugins/discourse-ai/lib/agents/tools/tool.rb@@ -131,35 +131,14 @@ module DiscourseAiprotecteddef fetch_default_branch(repo)- api_url = "https://api.github.com/repos/#{repo}"-- response_code = "unknown error"- repo_data = nil-- send_http_request(- api_url,- headers: {- "Accept" => "application/vnd.github.v3+json",- },- authenticate_github: true,- ) do |response|- response_code = response.code- if response_code == "200"- begin- repo_data = JSON.parse(read_response_body(response))- rescue JSON::ParserError- response_code = "500 - JSON parse error"- end- end- end-- response_code == "200" ? repo_data["default_branch"] : "main"+ github_client.get("https://api.github.com/repos/#{repo}")["default_branch"]+ rescue Discourse::GithubApi::Error+ "main"enddef send_http_request(url,headers: {},- authenticate_github: false,follow_redirects: false,method: :get,body: nil,@@ -168,7 +147,6 @@ module DiscourseAiself.class.send_http_request(url,headers: headers,- authenticate_github: authenticate_github,follow_redirects: follow_redirects,method: method,body: body,@@ -176,10 +154,13 @@ module DiscourseAi)end+ def github_client+ ::Discourse::GithubApi.for(token: SiteSetting.ai_bot_github_access_token)+ end+def self.send_http_request(url,headers: {},- authenticate_github: false,follow_redirects: false,method: :get,body: nil@@ -230,9 +211,6 @@ module DiscourseAirequest["User-Agent"] = DiscourseAi::AiBot::USER_AGENTheaders.each { |k, v| request[k] = v }- if authenticate_github && SiteSetting.ai_bot_github_access_token.present?- request["Authorization"] = "Bearer #{SiteSetting.ai_bot_github_access_token}"- endFinalDestination::HTTP.start(uri.hostname, uri.port, use_ssl: uri.port != 80) do |http|http.request(request) { |response| yield response, uri }diff --git a/plugins/discourse-ai/spec/lib/agents/tools/github_diff_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/github_diff_spec.rbindex 3bb71750687..755c5fc3a30 100644--- a/plugins/discourse-ai/spec/lib/agents/tools/github_diff_spec.rb+++ b/plugins/discourse-ai/spec/lib/agents/tools/github_diff_spec.rb@@ -57,7 +57,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff dostub_request(:get, "https://api.github.com/repos/#{repo}/commits/abc123").with(headers: {- "Accept" => "application/json",+ "Accept" => "application/vnd.github+json",},).to_return(status: 404)@@ -115,15 +115,13 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff doit "retrieves both PR info and diff" dostub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(headers: {- "Accept" => "application/json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,+ "Accept" => "application/vnd.github+json",},).to_return(status: 200, body: pr_info.to_json)stub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(headers: {"Accept" => "application/vnd.github.v3.diff",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,},).to_return(status: 200, body: diff)@@ -139,8 +137,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff dostub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(headers: {- "Accept" => "application/json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,+ "Accept" => "application/vnd.github+json","Authorization" => "Bearer ABC",},).to_return(status: 200, body: pr_info.to_json)@@ -148,7 +145,6 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff dostub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(headers: {"Accept" => "application/vnd.github.v3.diff",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,"Authorization" => "Bearer ABC",},).to_return(status: 200, body: diff)@@ -161,8 +157,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff doit "returns an error for invalid pull request" dostub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(headers: {- "Accept" => "application/json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,+ "Accept" => "application/vnd.github+json",},).to_return(status: 404)@@ -191,15 +186,13 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff dostub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(headers: {- "Accept" => "application/json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,+ "Accept" => "application/vnd.github+json",},).to_return(status: 200, body: pr_info_deleted_fork.to_json)stub_request(:get, "https://api.github.com/repos/#{repo}/pulls/#{pull_id}").with(headers: {"Accept" => "application/vnd.github.v3.diff",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,},).to_return(status: 200, body: diff)@@ -252,15 +245,13 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff doit "retrieves commit info and diff" dostub_request(:get, "https://api.github.com/repos/#{repo}/commits/#{sha}").with(headers: {- "Accept" => "application/json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,+ "Accept" => "application/vnd.github+json",},).to_return(status: 200, body: commit_info.to_json)stub_request(:get, "https://api.github.com/repos/#{repo}/commits/#{sha}").with(headers: {"Accept" => "application/vnd.github.v3.diff",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,},).to_return(status: 200, body: diff)@@ -283,8 +274,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff dostub_request(:get, "https://api.github.com/repos/#{repo}/commits/#{sha}").with(headers: {- "Accept" => "application/json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,+ "Accept" => "application/vnd.github+json","Authorization" => "Bearer ABC",},).to_return(status: 200, body: commit_info.to_json)@@ -292,7 +282,6 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff dostub_request(:get, "https://api.github.com/repos/#{repo}/commits/#{sha}").with(headers: {"Accept" => "application/vnd.github.v3.diff",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,"Authorization" => "Bearer ABC",},).to_return(status: 200, body: diff)@@ -305,8 +294,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubDiff doit "returns an error for invalid commit" dostub_request(:get, "https://api.github.com/repos/#{repo}/commits/#{sha}").with(headers: {- "Accept" => "application/json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,+ "Accept" => "application/vnd.github+json",},).to_return(status: 404)diff --git a/plugins/discourse-ai/spec/lib/agents/tools/github_search_code_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/github_search_code_spec.rbindex 8527787b78c..03f118bf7fa 100644--- a/plugins/discourse-ai/spec/lib/agents/tools/github_search_code_spec.rb+++ b/plugins/discourse-ai/spec/lib/agents/tools/github_search_code_spec.rb@@ -17,12 +17,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode dostub_request(:get,"https://api.github.com/search/code?page=1&per_page=30&q=def%20hello%20repo:discourse/discourse",- ).with(- headers: {- "Accept" => "application/vnd.github.v3.text-match+json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,- },- ).to_return(+ ).with(headers: { "Accept" => "application/vnd.github.v3.text-match+json" }).to_return(status: 200,body: {total_count: 1,@@ -44,8 +39,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode do)stub_request(:get, "https://api.github.com/repos/discourse/discourse/git/blobs/abc123").with(headers: {- "Accept" => "application/vnd.github.v3+json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,+ "Accept" => "application/vnd.github+json",},).to_return(status: 200, body: { content: Base64.encode64(file_body) }.to_json)@@ -195,12 +189,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode dostub_request(:get,"https://api.github.com/search/code?page=1&per_page=30&q=A_CONSTANT%20repo:discourse/discourse",- ).with(- headers: {- "Accept" => "application/vnd.github.v3.text-match+json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,- },- ).to_return(+ ).with(headers: { "Accept" => "application/vnd.github.v3.text-match+json" }).to_return(status: 200,body: {total_count: 1,@@ -221,8 +210,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode dostub_request(:get, "https://api.github.com/repos/discourse/discourse/git/blobs/def456").with(headers: {- "Accept" => "application/vnd.github.v3+json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,+ "Accept" => "application/vnd.github+json",},).to_return(status: 404)@@ -252,7 +240,6 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode do).with(headers: {"Accept" => "application/vnd.github.v3.text-match+json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,"Authorization" => "Bearer ABC",},).to_return(status: 200, body: { total_count: 0, items: [] }.to_json)@@ -271,12 +258,9 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchCode dostub_request(:get,"https://api.github.com/search/code?page=1&per_page=30&q=def%20hello%20repo:discourse/discourse",- ).with(- headers: {- "Accept" => "application/vnd.github.v3.text-match+json",- "User-Agent" => DiscourseAi::AiBot::USER_AGENT,- },- ).to_return(status: 403)+ ).with(headers: { "Accept" => "application/vnd.github.v3.text-match+json" }).to_return(+ status: 403,+ )result = tool.invokeexpect(result[:search_results]).to be_nildiff --git a/plugins/discourse-ai/spec/lib/agents/tools/github_search_files_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/github_search_files_spec.rbindex 7c5e997f0bd..4d774ba9bc6 100644--- a/plugins/discourse-ai/spec/lib/agents/tools/github_search_files_spec.rb+++ b/plugins/discourse-ai/spec/lib/agents/tools/github_search_files_spec.rb@@ -73,7 +73,7 @@ RSpec.describe DiscourseAi::Agents::Tools::GithubSearchFiles doresult = tool_with_invalid_branch.invokeexpect(result[:matching_files]).to be_nil- expect(result[:error]).to eq("Failed to perform file search. Status code: 404")+ expect(result[:error]).to include("Failed to perform file search")endit "fetches the default branch if none is specified" dodiff --git a/plugins/discourse-github/app/jobs/regular/replace_github_non_permalinks.rb b/plugins/discourse-github/app/jobs/regular/replace_github_non_permalinks.rbindex 70a82f81908..1c07f4ffaff 100644--- a/plugins/discourse-github/app/jobs/regular/replace_github_non_permalinks.rb+++ b/plugins/discourse-github/app/jobs/regular/replace_github_non_permalinks.rb@@ -1,9 +1,5 @@# frozen_string_literal: true-require "net/http"-require "json"-require "uri"-module Jobsclass ReplaceGithubNonPermalinks < ::Jobs::Basesidekiq_options queue: "low"@@ -18,6 +14,9 @@ module Jobspost = Post.find_by(id: post_id)return if post.blank?+ client = Discourse::GithubApi.for(token: SiteSetting.github_linkback_access_token)+ return if client.backing_off?+raw = post.raw.dupstart_raw = raw.dup@@ -29,14 +28,15 @@ module Jobsnext if excluded?(user, repo, file)begin- api_url = "https://api.github.com/repos/#{user}/#{repo}/commits/#{sha1}"- json = api_request(api_url)- if json && (json["sha"] != sha1)- new_sha = json["sha"]+ commit = client.get("/repos/#{user}/#{repo}/commits/#{sha1}")+ if commit && commit["sha"] != sha1+ new_sha = commit["sha"]old_url = "github.com/#{user}/#{repo}/blob/#{sha1}/#{file}#{from_to}"new_url = "github.com/#{user}/#{repo}/blob/#{new_sha}/#{file}#{from_to}"raw.sub!(old_url, new_url)end+ rescue Discourse::GithubApi::NotFound+ nextrescue => elog(:error,@@ -76,13 +76,6 @@ module Jobsprivate- def api_request(url)- uri = URI(url)- response = Net::HTTP.get_response(uri)-- JSON.parse(response.body) if response.kind_of? Net::HTTPSuccess- end-def log(log_level, message)Rails.logger.public_send(log_level,diff --git a/plugins/discourse-github/app/lib/commits_populator.rb b/plugins/discourse-github/app/lib/commits_populator.rbindex d4c67e61345..b3656fa693b 100644--- a/plugins/discourse-github/app/lib/commits_populator.rb+++ b/plugins/discourse-github/app/lib/commits_populator.rb@@ -10,8 +10,8 @@ module DiscourseGithubPluginROLES = { committer: 0, contributor: 1 }class PaginatedCommits- def initialize(octokit, repo, cursor: nil, page_size: 100)- @client = octokit+ def initialize(client, repo, cursor: nil, page_size: 100)+ @client = client@repo = repo@cursor = cursor@page_size = page_size@@ -30,17 +30,21 @@ module DiscourseGithubPluginenddef commits- @data.repository.defaultBranchRef.target.history.nodes+ history["nodes"]enddef next_cursor- info = @data.repository.defaultBranchRef.target.history.pageInfo- return unless info.hasNextPage- info.endCursor+ info = history["pageInfo"]+ return unless info["hasNextPage"]+ info["endCursor"]endprivate+ def history+ @data.dig("repository", "defaultBranchRef", "target", "history")+ end+def fetch_commitsowner, name = @repo.name.split("/", 2)history_args = "first: #{@page_size}"@@ -82,22 +86,23 @@ module DiscourseGithubPlugin}}QUERY- response = @client.post("/graphql", { query: query }.to_json)- raise GraphQLError, response.errors.inspect if response.errors- raise GraphQLError, response.message if !response.data- @data = response.data+ response = @client.post("/graphql", { query: query })+ raise GraphQLError, "Empty GraphQL response" if response.nil?+ raise GraphQLError, response["errors"].inspect if response["errors"]+ raise GraphQLError, response.inspect if !response["data"]+ @data = response["data"]endenddef initialize(repo)@repo = repo- @client =- Octokit::Client.new(access_token: SiteSetting.github_linkback_access_token, per_page: 100)+ @client = Discourse::GithubApi.for(token: SiteSetting.github_linkback_access_token)enddef populate!return unless SiteSetting.github_badges_enabled?- return if @client.branches(@repo.name).empty?+ return if @client.backing_off?+ return if @client.get("/repos/#{@repo.name}/branches").blank?if @repo.commits.size == 0build_history!@@ -130,56 +135,41 @@ module DiscourseGithubPluginDiscourse.redis.del(front_commit_redis_key)endend- rescue Octokit::Error => err- case err- when Octokit::NotFound- disable_github_badges_and_inform_admin(- title: I18n.t("github_commits_populator.errors.repository_not_found_pm_title"),- raw:- I18n.t(- "github_commits_populator.errors.repository_not_found_pm",- repo_name: @repo.name,- base_path: Discourse.base_path,- ),- )- Rails.logger.warn(- "Disabled github_badges_enabled site setting due to repository Not Found error ",- )- when Octokit::Unauthorized- disable_github_badges_and_inform_admin(- title: I18n.t("github_commits_populator.errors.invalid_octokit_credentials_pm_title"),- raw:- I18n.t(- "github_commits_populator.errors.invalid_octokit_credentials_pm",- base_path: Discourse.base_path,- ),- )- Rails.logger.warn(- "Disabled github_badges_enabled site setting due to invalid GitHub authentication credentials via github_linkback_access_token.",- )- else- Rails.logger.warn("#{err.class}: #{err.message}")- end- rescue Octokit::InvalidRepository => err+ rescue Discourse::GithubApi::NotFounddisable_github_badges_and_inform_admin(- title: I18n.t("github_commits_populator.errors.repository_identifier_invalid_pm_title"),+ title: I18n.t("github_commits_populator.errors.repository_not_found_pm_title"),raw:I18n.t(- "github_commits_populator.errors.repository_identifier_invalid_pm",+ "github_commits_populator.errors.repository_not_found_pm",repo_name: @repo.name,base_path: Discourse.base_path,),)Rails.logger.warn(- "Disabled github_badges_enabled site setting due to invalid repository identifier",+ "Disabled github_badges_enabled site setting due to repository Not Found error ",+ )+ rescue Discourse::GithubApi::Unauthorized+ disable_github_badges_and_inform_admin(+ title: I18n.t("github_commits_populator.errors.invalid_octokit_credentials_pm_title"),+ raw:+ I18n.t(+ "github_commits_populator.errors.invalid_octokit_credentials_pm",+ base_path: Discourse.base_path,+ ),+ )+ Rails.logger.warn(+ "Disabled github_badges_enabled site setting due to invalid GitHub authentication credentials via github_linkback_access_token.",)+ rescue Discourse::GithubApi::Error => err+ Rails.logger.warn("#{err.class}: #{err.message}")endprivatedef is_contribution?(commit)- pr = commit.associatedPullRequests.nodes.first- pr && pr.author && pr.mergedBy && pr.author.login != pr.mergedBy.login+ pr = commit.dig("associatedPullRequests", "nodes")&.first+ pr && pr["author"] && pr["mergedBy"] &&+ pr.dig("author", "login") != pr.dig("mergedBy", "login")enddef fetch_new_commits!(stop_at)@@ -191,7 +181,7 @@ module DiscourseGithubPluginstop_at.present? ? [] : @repo.commits.order("committed_at DESC").first(100).pluck(:sha)while !donebatch.each do |c|- if c.oid == stop_at || recent_commits.include?(c.oid)+ if c["oid"] == stop_at || recent_commits.include?(c["oid"])done = truebreakend@@ -204,25 +194,32 @@ module DiscourseGithubPluginendreturn if commits.size == 0existing_shas = @repo.commits.pluck(:sha)- commits.reject! { |c| existing_shas.include?(c.oid) }+ commits.reject! { |c| existing_shas.include?(c["oid"]) }batch_to_db(commits)- set_front_commit(commits.first.oid)+ set_front_commit(commits.first["oid"])end# detect if a force push happened and commit is lostdef removed?(sha)- commit = @client.commit(@repo.name, sha)- return true if commit.commit.nil?+ commit = @client.get("/repos/#{@repo.name}/commits/#{sha}")+ return false if commit.nil?+ return true if commit["commit"].nil?found =- @client.commits(@repo.name, until: commit.commit.committer.date, page: 1, per_page: 1).first- commit.sha != found.sha+ @client.get(+ "/repos/#{@repo.name}/commits",+ until: commit.dig("commit", "committer", "date"),+ page: 1,+ per_page: 1,+ ).first+ return false if found.nil?+ commit["sha"] != found["sha"]enddef build_history!(cursor: nil)paginator = PaginatedCommits.new(@client, @repo, cursor: cursor, page_size: 100)batch = paginator.commitsreturn if batch.empty?- set_front_commit(batch.first.oid) if cursor.blank?+ set_front_commit(batch.first["oid"]) if cursor.blank?while batch.size > 0batch_to_db(batch)@@ -251,11 +248,11 @@ module DiscourseGithubPlugindef commit_to_hash(commit){- sha: commit.oid,- email: commit.author.email,+ sha: commit["oid"],+ email: commit.dig("author", "email"),repo_id: @repo.id,- committed_at: commit.committedDate,- merge_commit: commit.message.match?(MERGE_COMMIT_REGEX),+ committed_at: commit["committedDate"],+ merge_commit: commit["message"].match?(MERGE_COMMIT_REGEX),role_id: is_contribution?(commit) ? ROLES[:contributor] : ROLES[:committer],}enddiff --git a/plugins/discourse-github/app/lib/github_linkback.rb b/plugins/discourse-github/app/lib/github_linkback.rbindex b3d022112d6..0bc57cd0085 100644--- a/plugins/discourse-github/app/lib/github_linkback.rb+++ b/plugins/discourse-github/app/lib/github_linkback.rb@@ -93,6 +93,7 @@ class GithubLinkbackdef createreturn [] if SiteSetting.github_linkback_access_token.blank?+ return [] if client.backing_off?links = []@@ -101,14 +102,19 @@ class GithubLinkbackreturn [] if links.length() > SiteSetting.github_linkback_maximum_linkslinks.each do |link|- case link.type- when :commit- post_commit(link)- when :pr- post_pr_or_issue(link, :pr)- when :issue- post_pr_or_issue(link, :issue)- else+ begin+ case link.type+ when :commit+ post_commit(link)+ when :pr+ post_pr_or_issue(link, :pr)+ when :issue+ post_pr_or_issue(link, :issue)+ else+ next+ end+ rescue Discourse::GithubApi::Error => e+ Rails.logger.warn("Failed to post GitHub linkback for #{link.url}: #{e.message}")nextend@@ -138,8 +144,6 @@ class GithubLinkbackreturn if topic_already_linked_on_github?(link)- github_url =- "https://api.github.com/repos/#{link.project}/issues/#{pr_or_issue_number}/comments"comment =I18n.t(type == :pr ? "github_linkback.pr_template" : "github_linkback.issue_template",@@ -147,14 +151,12 @@ class GithubLinkbackpost_url: "#{Discourse.base_url}#{@post.url}",)- Excon.post(github_url, body: { body: comment }.to_json, headers: headers)+ client.post("/repos/#{link.project}/issues/#{pr_or_issue_number}/comments", { body: comment })enddef post_commit(link)return if topic_already_linked_on_github?(link)- github_url = "https://api.github.com/repos/#{link.project}/commits/#{link.sha}/comments"-comment =I18n.t("github_linkback.commit_template",@@ -162,7 +164,7 @@ class GithubLinkbackpost_url: "#{Discourse.base_url}#{@post.url}",)- Excon.post(github_url, body: { body: comment }.to_json, headers: headers)+ client.post("/repos/#{link.project}/commits/#{link.sha}/comments", { body: comment })enddef topic_already_linked_on_github?(link)@@ -174,38 +176,22 @@ class GithubLinkbackendtexts.any? { |text| text_links_to_topic?(text) }- rescue Excon::Error+ rescue Discourse::GithubApi::Errorfalseenddef fetch_commit_comment_texts(project, sha)- response =- Excon.get(- "https://api.github.com/repos/#{project}/commits/#{sha}/comments?per_page=100",- headers:,- expects: [200],- )- JSON.parse(response.body).map { |comment| comment["body"].to_s }+ client+ .get("/repos/#{project}/commits/#{sha}/comments", per_page: 100)+ .map { |comment| comment["body"].to_s }enddef fetch_pr_or_issue_texts(project, number)- issue_response =- Excon.get(- "https://api.github.com/repos/#{project}/issues/#{number}",- headers:,- expects: [200],- )-- comments_response =- Excon.get(- "https://api.github.com/repos/#{project}/issues/#{number}/comments?per_page=100",- headers:,- expects: [200],- )-[- JSON.parse(issue_response.body)["body"].to_s,- *JSON.parse(comments_response.body).map { |comment| comment["body"].to_s },+ client.get("/repos/#{project}/issues/#{number}")["body"].to_s,+ *client+ .get("/repos/#{project}/issues/#{number}/comments", per_page: 100)+ .map { |comment| comment["body"].to_s },]end@@ -220,11 +206,7 @@ class GithubLinkbackendend- def headers- {- "Content-Type" => "application/json",- "Authorization" => "token #{SiteSetting.github_linkback_access_token}",- "User-Agent" => "Discourse-Github-Linkback",- }+ def client+ @client ||= Discourse::GithubApi.for(token: SiteSetting.github_linkback_access_token)endenddiff --git a/plugins/discourse-github/app/lib/github_linkback_access_token_setting_validator.rb b/plugins/discourse-github/app/lib/github_linkback_access_token_setting_validator.rbindex 44834dd2164..471c1778a43 100644--- a/plugins/discourse-github/app/lib/github_linkback_access_token_setting_validator.rb+++ b/plugins/discourse-github/app/lib/github_linkback_access_token_setting_validator.rb@@ -7,10 +7,12 @@ class GithubLinkbackAccessTokenSettingValidatordef valid_value?(val)return true if val.blank?- client = Octokit::Client.new(access_token: val, per_page: 1)- DiscourseGithubPlugin::GithubRepo.repos.each { |repo| client.branches(repo.name) }+ client = Discourse::GithubApi.new(val)+ DiscourseGithubPlugin::GithubRepo.repos.each do |repo|+ client.get("/repos/#{repo.name}/branches")+ endtrue- rescue Octokit::Unauthorized+ rescue Discourse::GithubApi::Unauthorizedfalseenddiff --git a/plugins/discourse-github/app/models/github_repo.rb b/plugins/discourse-github/app/models/github_repo.rbindex d16804188fa..b6c0d4cd35b 100644--- a/plugins/discourse-github/app/models/github_repo.rb+++ b/plugins/discourse-github/app/models/github_repo.rb@@ -3,7 +3,7 @@module DiscourseGithubPluginclass GithubRepo < ActiveRecord::BaseVALID_URL_BASED_REPO_REGEX = %r{https?://github.com/(.+)}- VALID_USER_BASED_REPO_REGEX = Octokit::Repository::NAME_WITH_OWNER_PATTERN+ VALID_USER_BASED_REPO_REGEX = %r{\A[\w.-]+/[\w.-]+\z}ihas_many :commits, foreign_key: :repo_id, class_name: :GithubCommit, dependent: :destroydiff --git a/plugins/discourse-github/plugin.rb b/plugins/discourse-github/plugin.rbindex cc7f97ae425..4a7f024a19b 100644--- a/plugins/discourse-github/plugin.rb+++ b/plugins/discourse-github/plugin.rb@@ -7,9 +7,6 @@# authors: Robin Ward, Sam Saffron# url: https://github.com/discourse/discourse/tree/main/plugins/discourse-github-require "sawyer"-require "octokit"-# Site setting validators must be loaded before initializerequire_relative "app/lib/github_badges_repo_setting_validator"require_relative "app/lib/github_linkback_access_token_setting_validator"diff --git a/plugins/discourse-github/spec/jobs/replace_github_non_permalinks_spec.rb b/plugins/discourse-github/spec/jobs/replace_github_non_permalinks_spec.rbindex d428835d023..6677a63fad8 100644--- a/plugins/discourse-github/spec/jobs/replace_github_non_permalinks_spec.rb+++ b/plugins/discourse-github/spec/jobs/replace_github_non_permalinks_spec.rb@@ -25,12 +25,19 @@ describe Jobs::ReplaceGithubNonPermalinks dostatus: 200,body: github_response_body.to_json,headers: {+ "Content-Type" => "application/json",},)stub_request(:get,"https://api.github.com/repos/test/onebox/commits/815ea9c0a8ffebe7bd7fcd34c10ff28c7a6b6974",- ).to_return(status: 200, body: github_response_body.to_json, headers: {})+ ).to_return(+ status: 200,+ body: github_response_body.to_json,+ headers: {+ "Content-Type" => "application/json",+ },+ )stub_request(:get, "https://api.github.com/repos/test/oneblob/commits/master").to_return(status: 404,)@@ -38,6 +45,7 @@ describe Jobs::ReplaceGithubNonPermalinks dostatus: 200,body: github_response_body2.to_json,headers: {+ "Content-Type" => "application/json",},)enddiff --git a/plugins/discourse-github/spec/lib/commits_populator_spec.rb b/plugins/discourse-github/spec/lib/commits_populator_spec.rbindex c976e4a486f..9ed14b6f4ab 100644--- a/plugins/discourse-github/spec/lib/commits_populator_spec.rb+++ b/plugins/discourse-github/spec/lib/commits_populator_spec.rb@@ -6,30 +6,32 @@ describe DiscourseGithubPlugin::CommitsPopulator dolet(:repo) { DiscourseGithubPlugin::GithubRepo.new(name: "discourse/discourse") }let!(:site_admin1) { Fabricate(:admin) }let!(:site_admin2) { Fabricate(:admin) }+ let(:branches_url) { "https://api.github.com/repos/discourse/discourse/branches" }before doenable_current_pluginSiteSetting.github_badges_enabled = trueend- context "when invalid credentials have been provided for octokit" do- before { Octokit::Client.any_instance.expects(:branches).raises(Octokit::Unauthorized) }+ def last_pm+ Post+ .joins(:topic)+ .includes(:topic)+ .where("topics.archetype = ?", Archetype.private_message)+ .last+ end++ context "when invalid credentials have been provided (401)" do+ before { stub_request(:get, branches_url).to_return(status: 401) }it "disables github badges and sends a PM to the admin of the site to inform them" dopopulator.populate!expect(SiteSetting.github_badges_enabled).to eq(false)- sent_pm =- Post- .joins(:topic)- .includes(:topic)- .where("topics.archetype = ?", Archetype.private_message)- .last- expect(sent_pm.topic.allowed_users.include?(site_admin1)).to eq(true)- expect(sent_pm.topic.allowed_users.include?(site_admin2)).to eq(true)- expect(sent_pm.topic.title).to eq(+ expect(last_pm.topic.allowed_users).to include(site_admin1, site_admin2)+ expect(last_pm.topic.title).to eq(I18n.t("github_commits_populator.errors.invalid_octokit_credentials_pm_title"),)- expect(sent_pm.raw).to eq(+ expect(last_pm.raw).to eq(I18n.t("github_commits_populator.errors.invalid_octokit_credentials_pm",base_path: Discourse.base_path,@@ -38,24 +40,17 @@ describe DiscourseGithubPlugin::CommitsPopulator doendend- context "when the repository is not found" do- before { Octokit::Client.any_instance.expects(:branches).raises(Octokit::NotFound) }+ context "when the repository is not found (404)" do+ before { stub_request(:get, branches_url).to_return(status: 404) }it "disables github badges and sends a PM to the admin of the site to inform them" dopopulator.populate!expect(SiteSetting.github_badges_enabled).to eq(false)- sent_pm =- Post- .joins(:topic)- .includes(:topic)- .where("topics.archetype = ?", Archetype.private_message)- .last- expect(sent_pm.topic.allowed_users.include?(site_admin1)).to eq(true)- expect(sent_pm.topic.allowed_users.include?(site_admin2)).to eq(true)- expect(sent_pm.topic.title).to eq(+ expect(last_pm.topic.allowed_users).to include(site_admin1, site_admin2)+ expect(last_pm.topic.title).to eq(I18n.t("github_commits_populator.errors.repository_not_found_pm_title"),)- expect(sent_pm.raw).to eq(+ expect(last_pm.raw).to eq(I18n.t("github_commits_populator.errors.repository_not_found_pm",repo_name: repo.name,@@ -65,35 +60,8 @@ describe DiscourseGithubPlugin::CommitsPopulator doendend- context "when the repository identifier is invalid" do- before { Octokit::Client.any_instance.expects(:branches).raises(Octokit::InvalidRepository) }-- it "disables github badges and sends a PM to the admin of the site to inform them" do- populator.populate!- expect(SiteSetting.github_badges_enabled).to eq(false)- sent_pm =- Post- .joins(:topic)- .includes(:topic)- .where("topics.archetype = ?", Archetype.private_message)- .last- expect(sent_pm.topic.allowed_users.include?(site_admin1)).to eq(true)- expect(sent_pm.topic.allowed_users.include?(site_admin2)).to eq(true)- expect(sent_pm.topic.title).to eq(- I18n.t("github_commits_populator.errors.repository_identifier_invalid_pm_title"),- )- expect(sent_pm.raw).to eq(- I18n.t(- "github_commits_populator.errors.repository_identifier_invalid_pm",- repo_name: repo.name,- base_path: Discourse.base_path,- ).strip,- )- end- end-- context "if some other octokit error is raised" do- before { Octokit::Client.any_instance.expects(:branches).raises(Octokit::Error) }+ context "if some other GitHub error is raised (500)" do+ before { stub_request(:get, branches_url).to_return(status: 500) }it "simply logs the error and does nothing else" dopopulator.populate!@@ -103,40 +71,23 @@ describe DiscourseGithubPlugin::CommitsPopulator docontext "if GraphQL returns no data" dobefore do- branches_body = <<~JSON- [- {- "name": "add-group-css-properties",- "commit": {- "sha": "0d67b6307042803c351599de715023841cfa9356",- "url": "https://api.github.com/repos/discourse/discourse/commits/0d67b6307042803c351599de715023841cfa9356"- },- "protected": false- }- ]- JSON-- graphql_response = <<~JSON- {- "message": "Bad credentials",- "documentation_url": "https://docs.github.com/graphql"- }- JSON-- stub_request(- :get,- "https://api.github.com/repos/discourse/discourse/branches?per_page=100",- ).to_return(status: 200, body: branches_body)+ stub_request(:get, branches_url).to_return(+ status: 200,+ body: [{ "name" => "main", "commit" => { "sha" => "abc" } }].to_json,+ headers: {+ "Content-Type" => "application/json",+ },+ )stub_request(:post, "https://api.github.com/graphql").to_return(status: 200,- body: graphql_response,+ body: { "message" => "Bad credentials" }.to_json,headers: {- "content-type": "application/json",+ "Content-Type" => "application/json",},)end- it "simply logs the error and does nothing else" do+ it "raises a GraphQLError" doexpect { populator.populate! }.to raise_error(described_class::GraphQLError)endend@@ -144,9 +95,9 @@ describe DiscourseGithubPlugin::CommitsPopulator docontext "if github_badges_enabled is false" dobefore { SiteSetting.github_badges_enabled = false }- it "early returns before attempting to execute any of the commit fetching, because the plugin likely disabled itself" do- Octokit::Client.any_instance.expects(:branches).never+ it "early returns before making any GitHub request" dopopulator.populate!+ expect(a_request(:get, branches_url)).not_to have_been_madeendendenddiff --git a/plugins/discourse-github/spec/lib/github_linkback_access_token_setting_validator_spec.rb b/plugins/discourse-github/spec/lib/github_linkback_access_token_setting_validator_spec.rbindex 79cf37b7a04..30a28b331b7 100644--- a/plugins/discourse-github/spec/lib/github_linkback_access_token_setting_validator_spec.rb+++ b/plugins/discourse-github/spec/lib/github_linkback_access_token_setting_validator_spec.rb@@ -8,10 +8,12 @@ describe GithubLinkbackAccessTokenSettingValidator dobefore { enable_current_plugin }describe "#valid_value?" do- context "when an Octokit::Unauthorized error is raised, meaning the access token cannot access a repo" do+ context "when the token cannot access a repo (401)" dobefore dosetup_repos- Octokit::Client.any_instance.expects(:branches).raises(Octokit::Unauthorized)+ stub_request(:get, "https://api.github.com/repos/discourse/discourse/branches").to_return(+ status: 401,+ )endit "should fail" do@@ -19,7 +21,7 @@ describe GithubLinkbackAccessTokenSettingValidator doendend- context "when no Octokit::Unauthorized error is raised" do+ context "when the token is accepted" doit "should pass, without repos defined" doexpect(validator.valid_value?(value)).to eq(true)end@@ -27,7 +29,10 @@ describe GithubLinkbackAccessTokenSettingValidator docontext "when there are repos defined" dobefore dosetup_repos- Octokit::Client.any_instance.expects(:branches).returns([])+ stub_request(:get, "https://api.github.com/repos/discourse/discourse/branches").to_return(+ status: 200,+ body: "[]",+ )endit "should pass if all the repos are accessible" dodiff --git a/plugins/discourse-github/spec/lib/github_linkback_spec.rb b/plugins/discourse-github/spec/lib/github_linkback_spec.rbindex 108220e8cf9..783dd46ce25 100644--- a/plugins/discourse-github/spec/lib/github_linkback_spec.rb+++ b/plugins/discourse-github/spec/lib/github_linkback_spec.rb@@ -200,14 +200,7 @@ describe GithubLinkback doenddescribe "#create" do- let(:github_api_headers) do- {- "Authorization" => "token abcdef",- "Content-Type" => "application/json",- "Host" => "api.github.com",- "User-Agent" => "Discourse-Github-Linkback",- }- end+ let(:github_api_headers) { { "Authorization" => "Bearer abcdef" } }before { SiteSetting.github_linkback_projects = "discourse/discourse|discourse/*" }diff --git a/spec/lib/discourse/github_api_spec.rb b/spec/lib/discourse/github_api_spec.rbnew file mode 100644index 00000000000..f845f2c91de--- /dev/null+++ b/spec/lib/discourse/github_api_spec.rb@@ -0,0 +1,170 @@+# frozen_string_literal: true++RSpec.describe Discourse::GithubApi do+ subject(:client) { described_class.for(token:) }++ let(:token) { "gh_token_abc" }++ before { described_class.reset_clients! }++ def stub_gh(path, status: 200, body: "{}", headers: { "Content-Type" => "application/json" })+ stub_request(:get, "https://api.github.com#{path}").to_return(status:, body:, headers:)+ end++ describe "#get" do+ it "returns a plain string-keyed Hash (not Sawyer)" do+ stub_gh(+ "/repos/discourse/discourse/pulls/1",+ body: { "title" => "Hi", "base" => { "ref" => "main" } }.to_json,+ )++ result = client.get("/repos/discourse/discourse/pulls/1")++ expect(result).to be_a(Hash)+ expect(result["base"]["ref"]).to eq("main")+ expect(result.dig("base", "ref")).to eq("main")+ end++ it "returns an Array for list endpoints" do+ stub_gh(+ "/repos/discourse/discourse/pulls/1/reviews",+ body: [{ "state" => "APPROVED" }].to_json,+ )+ expect(client.get("/repos/discourse/discourse/pulls/1/reviews")).to eq(+ [{ "state" => "APPROVED" }],+ )+ end++ it "sends a Bearer Authorization header when a token is configured" do+ stub =+ stub_request(:get, "https://api.github.com/x").with(+ headers: {+ "Authorization" => "Bearer #{token}",+ },+ ).to_return(status: 200, body: "{}")+ client.get("/x")+ expect(stub).to have_been_requested+ end++ it "sends no Authorization header when unauthenticated" do+ anon = described_class.for(token: nil)+ stub = stub_request(:get, "https://api.github.com/x").to_return(status: 200, body: "{}")+ anon.get("/x")+ expect(stub).to have_been_requested+ expect(WebMock).not_to have_requested(:get, "https://api.github.com/x").with(+ headers: {+ "Authorization" => /.+/,+ },+ )+ end+ end++ describe "rate limiting" do+ it "short-circuits with RateLimited (no HTTP) while backing off" do+ GithubRateLimit.note_rate_limit(token:, retry_after: 60)+ expect { client.get("/x") }.to raise_error(described_class::RateLimited)+ expect(a_request(:get, "https://api.github.com/x")).not_to have_been_made+ end++ it "records a backoff and raises on a 403 rate-limit response" do+ stub_gh(+ "/x",+ status: 403,+ headers: {+ "x-ratelimit-remaining" => "0",+ "x-ratelimit-reset" => 10.minutes.from_now.to_i.to_s,+ },+ )+ expect { client.get("/x") }.to raise_error(described_class::RateLimited)+ expect(GithubRateLimit.backing_off?(token)).to eq(true)+ end++ it "proactively backs off when a 200 reports the budget is exhausted" do+ stub_gh(+ "/x",+ headers: {+ "Content-Type" => "application/json",+ "x-ratelimit-remaining" => "0",+ "x-ratelimit-reset" => 10.minutes.from_now.to_i.to_s,+ },+ )+ client.get("/x")+ expect(GithubRateLimit.backing_off?(token)).to eq(true)+ end+ end++ describe "error mapping" do+ it "raises NotFound on 404" do+ stub_gh("/x", status: 404)+ expect { client.get("/x") }.to raise_error(described_class::NotFound)+ end++ it "raises Unauthorized on 401" do+ stub_gh("/x", status: 401)+ expect { client.get("/x") }.to raise_error(described_class::Unauthorized)+ end++ it "raises a non-rate-limit error on a 403 with budget remaining (no backoff)" do+ stub_gh("/x", status: 403, headers: { "x-ratelimit-remaining" => "57" })+ expect { client.get("/x") }.to raise_error(described_class::Error) do |e|+ expect(e).not_to be_a(described_class::RateLimited)+ end+ expect(GithubRateLimit.backing_off?(token)).to eq(false)+ end++ it "maps Faraday timeouts/connection failures to its own Error" do+ stub_request(:get, "https://api.github.com/x").to_timeout+ expect { client.get("/x") }.to raise_error(described_class::Error)+ end+ end++ describe "ETag conditional requests" do+ it "stores the ETag and serves the cached body on a 304 (no rate-limit cost)" do+ stub_request(:get, "https://api.github.com/x").to_return(+ status: 200,+ body: { "n" => 1 }.to_json,+ headers: {+ "Content-Type" => "application/json",+ "ETag" => 'W/"abc"',+ },+ )+ expect(client.get("/x")).to eq({ "n" => 1 })++ stub_request(:get, "https://api.github.com/x").with(+ headers: {+ "If-None-Match" => 'W/"abc"',+ },+ ).to_return(status: 304, body: "")+ expect(client.get("/x")).to eq({ "n" => 1 })+ end+ end++ describe "#raw_get" do+ it "returns the raw body as a String" do+ stub_request(:get, "https://raw.githubusercontent.com/d/d/main/x.rb").to_return(+ status: 200,+ body: "puts 1",+ )+ expect(client.raw_get("https://raw.githubusercontent.com/d/d/main/x.rb")).to eq("puts 1")+ end+ end++ describe "host allowlist" do+ it "refuses to call (and leak the token to) a non-GitHub absolute URL" do+ expect { client.get("https://evil.example.com/steal") }.to raise_error(ArgumentError)+ expect { client.raw_get("https://evil.example.com/steal") }.to raise_error(ArgumentError)+ expect(a_request(:get, "https://evil.example.com/steal")).not_to have_been_made+ end+ end++ describe "#post" do+ it "sends a JSON body" do+ stub =+ stub_request(:post, "https://api.github.com/repos/d/d/issues/1/comments").with(+ body: { body: "hi" }.to_json,+ ).to_return(status: 201, body: "{}")+ client.post("/repos/d/d/issues/1/comments", { body: "hi" })+ expect(stub).to have_been_requested+ end+ end+enddiff --git a/spec/lib/github_rate_limit_spec.rb b/spec/lib/github_rate_limit_spec.rbnew file mode 100644index 00000000000..6f6d44924a5--- /dev/null+++ b/spec/lib/github_rate_limit_spec.rb@@ -0,0 +1,66 @@+# frozen_string_literal: true++RSpec.describe GithubRateLimit do+ let(:token) { "gh_token_123" }++ def ttl(token = nil)+ Discourse.redis.without_namespace.ttl(described_class.key(token))+ end++ describe ".key" do+ it "is per-token, and a shared bucket for unauthenticated requests" do+ expect(described_class.key(token)).to eq(+ "onebox_github_backoff_#{Digest::SHA1.hexdigest(token)}",+ )+ expect(described_class.key(nil)).to eq("onebox_github_backoff_unauthenticated")+ expect(described_class.key("")).to eq("onebox_github_backoff_unauthenticated")+ end+ end++ describe ".note_rate_limit" do+ it "backs off for the Retry-After duration" do+ described_class.note_rate_limit(token:, retry_after: 90)+ expect(described_class.backing_off?(token)).to eq(true)+ expect(ttl(token)).to be_between(1, 90)+ end++ it "backs off until x-ratelimit-reset when the remaining budget is 0" do+ described_class.note_rate_limit(token:, remaining: "0", reset_at: 10.minutes.from_now.to_i)+ expect(ttl(token)).to be_between(1, 600)+ end++ it "does nothing when there is budget left and no Retry-After" do+ described_class.note_rate_limit(token:, remaining: "57")+ expect(described_class.backing_off?(token)).to eq(false)+ end++ it "clamps the backoff to one hour" do+ described_class.note_rate_limit(token:, remaining: "0", reset_at: 10.days.from_now.to_i)+ expect(ttl(token)).to be <= 1.hour.to_i+ end++ it "is scoped per token" do+ described_class.note_rate_limit(token:, retry_after: 90)+ expect(described_class.backing_off?("other_token")).to eq(false)+ expect(described_class.backing_off?(nil)).to eq(false)+ end+ end++ describe ".note_response_headers" do+ it "reads the rate-limit headers regardless of casing" do+ described_class.note_response_headers(+ { "Retry-After" => "120", "X-RateLimit-Remaining" => "0" },+ token:,+ )+ expect(ttl(token)).to be_between(1, 120)+ end+ end++ describe ".active?" do+ it "is true while any backoff key exists" do+ expect(described_class.active?).to eq(false)+ described_class.note_rate_limit(token:, retry_after: 30)+ expect(described_class.active?).to eq(true)+ end+ end+enddiff --git a/spec/lib/onebox/engine/github_blob_onebox_spec.rb b/spec/lib/onebox/engine/github_blob_onebox_spec.rbindex 6f99bb8c2fc..3147d76de41 100644--- a/spec/lib/onebox/engine/github_blob_onebox_spec.rb+++ b/spec/lib/onebox/engine/github_blob_onebox_spec.rb@@ -30,20 +30,12 @@ RSpec.describe Onebox::Engine::GithubBlobOnebox doendit "does not include blob contents if it is binary" do- # stub_request if the response must be binary (ASCII-8BIT)- uri_mock = mock("object")- uri_mock.stubs(:open).returns(- File.open("#{Rails.root.join("spec/fixtures/pdf/small.pdf")}", "rb"),+ stub_request(:get, raw_uri).to_return(+ status: 200,+ body: File.binread("#{Rails.root.join("spec/fixtures/images/smallest.png")}"),)- URI.stubs(:parse).with(link).returns(uri)- URI- .stubs(:parse)- .with(- "https://raw.githubusercontent.com/discourse/onebox/master/lib/onebox/engine/github_blob_onebox.rb",- )- .returns(uri_mock)- expect(html).not_to include("/Pages")+ expect(html).not_to include("module Oneboxer")expect(html).to include("This file is binary.")enddiff --git a/spec/lib/onebox/github_access_spec.rb b/spec/lib/onebox/github_access_spec.rbnew file mode 100644index 00000000000..c9624bfed8e--- /dev/null+++ b/spec/lib/onebox/github_access_spec.rb@@ -0,0 +1,70 @@+# frozen_string_literal: true++RSpec.describe Onebox::GithubAccess do+ before { Discourse::GithubApi.reset_clients! }++ describe ".token" do+ it "returns nil when no onebox tokens are configured" do+ SiteSetting.github_onebox_access_tokens = ""+ expect(described_class.token("discourse")).to be_nil+ end++ it "resolves the per-org token, falling back to the default mapping" do+ SiteSetting.github_onebox_access_tokens = "discourse|org_tok\ndefault|def_tok"+ expect(described_class.token("discourse")).to eq("org_tok")+ expect(described_class.token("someoneelse")).to eq("def_tok")+ end++ it "returns nil for an unmapped org when no default is configured" do+ SiteSetting.github_onebox_access_tokens = "discourse|org_tok"+ expect(described_class.token("someoneelse")).to be_nil+ end+ end++ describe ".tokens" do+ it "returns just the unauthenticated identity when nothing is configured" do+ SiteSetting.github_onebox_access_tokens = ""+ expect(described_class.tokens).to eq([nil])+ end++ it "returns every configured token plus the unauthenticated identity" do+ SiteSetting.github_onebox_access_tokens = "discourse|org_tok\ndefault|def_tok"+ expect(described_class.tokens).to contain_exactly("org_tok", "def_tok", nil)+ end+ end++ describe ".client" do+ it "builds a client authenticated with the org's onebox token" do+ SiteSetting.github_onebox_access_tokens = "discourse|org_tok\ndefault|def_tok"++ org_stub =+ stub_request(:get, "https://api.github.com/x").with(+ headers: {+ "Authorization" => "Bearer org_tok",+ },+ ).to_return(status: 200, body: "{}")+ described_class.client("discourse").get("/x")+ expect(org_stub).to have_been_requested++ default_stub =+ stub_request(:get, "https://api.github.com/y").with(+ headers: {+ "Authorization" => "Bearer def_tok",+ },+ ).to_return(status: 200, body: "{}")+ described_class.client("someoneelse").get("/y")+ expect(default_stub).to have_been_requested+ end++ it "builds an unauthenticated client when no onebox token is configured" do+ SiteSetting.github_onebox_access_tokens = ""++ stub =+ stub_request(:get, "https://api.github.com/z")+ .with { |request| !request.headers.key?("Authorization") }+ .to_return(status: 200, body: "{}")+ described_class.client("discourse").get("/z")+ expect(stub).to have_been_requested+ end+ end+enddiff --git a/spec/lib/onebox/mixins/github_api_spec.rb b/spec/lib/onebox/mixins/github_api_spec.rbnew file mode 100644index 00000000000..d473ed0aef5--- /dev/null+++ b/spec/lib/onebox/mixins/github_api_spec.rb@@ -0,0 +1,182 @@+# frozen_string_literal: true++RSpec.describe Onebox::Mixins::GithubApi do+ let(:pr_link) { "https://github.com/discourse/discourse/pull/1253" }+ let(:pr_api) { "https://api.github.com/repos/discourse/discourse/pulls/1253" }+ let(:reset_at) { 30.minutes.from_now.to_i }++ def rate_limit_headers(remaining: "0", reset: reset_at, retry_after: nil)+ headers = {+ "x-ratelimit-limit" => "60",+ "x-ratelimit-remaining" => remaining,+ "x-ratelimit-reset" => reset.to_s,+ }+ headers["retry-after"] = retry_after.to_s if retry_after+ headers+ end++ def render_pr+ Onebox::Engine::GithubPullRequestOnebox.new(pr_link).to_html+ rescue OpenURI::HTTPError+ nil+ end++ context "when GitHub returns a primary rate-limit 403" do+ before do+ stub_request(:get, pr_api).to_return(+ status: [403, "Forbidden"],+ headers: rate_limit_headers,+ body: '{"message":"API rate limit exceeded for 1.2.3.4"}',+ )+ end++ it "records a backoff (until x-ratelimit-reset) and skips further GitHub requests" do+ render_pr+ expect(a_request(:get, pr_api)).to have_been_made.once++ ttl = Discourse.redis.without_namespace.ttl("onebox_github_backoff_unauthenticated")+ expect(ttl).to be_between(1, 1800)++ render_pr+ expect(a_request(:get, pr_api)).to have_been_made.once+ end++ it "short-circuits a different GitHub URL/engine sharing the same identity" do+ render_pr++ issue_link = "https://github.com/discourse/discourse/issues/999"+ issue_api = "https://api.github.com/repos/discourse/discourse/issues/999"+ stub_request(:get, issue_api).to_return(status: 200, body: "{}")++ begin+ Onebox::Engine::GithubIssueOnebox.new(issue_link).to_html+ rescue StandardError+ nil+ end+ expect(a_request(:get, issue_api)).not_to have_been_made+ end+ end++ context "when GitHub sends a Retry-After header (secondary rate limit)" do+ before do+ stub_request(:get, pr_api).to_return(+ status: [429, "Too Many Requests"],+ headers: rate_limit_headers(remaining: "42", retry_after: 120),+ body: "{}",+ )+ end++ it "backs off for the Retry-After duration" do+ render_pr+ ttl = Discourse.redis.without_namespace.ttl("onebox_github_backoff_unauthenticated")+ expect(ttl).to be_between(1, 120)+ end+ end++ context "when GitHub returns a 403 that is NOT a rate limit (e.g. private repo)" do+ before do+ stub_request(:get, pr_api).to_return(+ status: [403, "Forbidden"],+ headers: rate_limit_headers(remaining: "57"),+ body: '{"message":"Must have admin rights to Repository."}',+ )+ end++ it "does not back off, so subsequent requests still reach GitHub" do+ render_pr+ expect(+ Discourse.redis.without_namespace.get("onebox_github_backoff_unauthenticated"),+ ).to be_nil++ render_pr+ expect(a_request(:get, pr_api)).to have_been_made.twice+ end+ end++ context "when an org access token is configured" do+ before { SiteSetting.github_onebox_access_tokens = "discourse|gh_token_xyz" }++ it "scopes the backoff per token, not globally" do+ stub_request(:get, pr_api).with(+ headers: {+ "Authorization" => "Bearer gh_token_xyz",+ },+ ).to_return(status: [403, "Forbidden"], headers: rate_limit_headers, body: "{}")++ render_pr++ token_key = "onebox_github_backoff_#{Digest::SHA1.hexdigest("gh_token_xyz")}"+ expect(Discourse.redis.without_namespace.get(token_key)).to be_present+ expect(+ Discourse.redis.without_namespace.get("onebox_github_backoff_unauthenticated"),+ ).to be_nil+ end+ end++ context "when a successful response reports the rate-limit budget is exhausted" do+ before do+ SiteSetting.stubs(:github_pr_status_enabled).returns(false)+ stub_request(:get, pr_api).to_return(+ status: 200,+ headers: rate_limit_headers(remaining: "0"),+ body: onebox_response("githubpullrequest"),+ )+ end++ it "proactively backs off until reset, without waiting for a 403" do+ render_pr+ ttl = Discourse.redis.without_namespace.ttl("onebox_github_backoff_unauthenticated")+ expect(ttl).to be_between(1, 1800)++ WebMock::RequestRegistry.instance.reset!+ render_pr+ expect(a_request(:get, pr_api)).not_to have_been_made+ end+ end++ context "when any GitHub API engine hits the rate limit on its primary fetch" do+ before do+ stub_request(:get, /api\.github\.com/).to_return(+ status: [403, "Forbidden"],+ headers: rate_limit_headers,+ body: "{}",+ )+ end++ [+ ["pull request", Onebox::Engine::GithubPullRequestOnebox, "https://github.com/d/d/pull/1"],+ ["commit", Onebox::Engine::GithubCommitOnebox, "https://github.com/d/d/commit/#{"a" * 40}"],+ ["issue", Onebox::Engine::GithubIssueOnebox, "https://github.com/d/d/issues/1"],+ ["repo", Onebox::Engine::GithubRepoOnebox, "https://github.com/d/d"],+ ["actions", Onebox::Engine::GithubActionsOnebox, "https://github.com/d/d/actions/runs/1"],+ ["gist", Onebox::Engine::GithubGistOnebox, "https://gist.github.com/d/abc123def456"],+ ].each do |name, klass, link|+ it "records a backoff when the #{name} engine is rate-limited" do+ begin+ klass.new(link).to_html+ rescue StandardError+ nil+ end+ expect(+ Discourse.redis.without_namespace.get("onebox_github_backoff_unauthenticated"),+ ).to be_present+ end+ end+ end++ context "when a default access token is configured" do+ it "authenticates gist API calls with the default token" do+ SiteSetting.github_onebox_access_tokens = "default|gh_default_token"+ stub =+ stub_request(:get, "https://api.github.com/gists/abc123def456").with(+ headers: {+ "Authorization" => "Bearer gh_default_token",+ },+ ).to_return(status: 200, body: '{"files":{}}')++ Onebox::Engine::GithubGistOnebox.new("https://gist.github.com/d/abc123def456").to_html++ expect(stub).to have_been_requested+ end+ end+enddiff --git a/spec/lib/oneboxer_github_backoff_spec.rb b/spec/lib/oneboxer_github_backoff_spec.rbnew file mode 100644index 00000000000..c109bbcddc9--- /dev/null+++ b/spec/lib/oneboxer_github_backoff_spec.rb@@ -0,0 +1,24 @@+# frozen_string_literal: true++RSpec.describe Oneboxer do+ describe ".external_onebox_ttl" do+ it "uses a short TTL for a blank/failed result and 1 day for a successful one" do+ expect(Oneboxer.external_onebox_ttl({ onebox: "", preview: "" })).to eq(1.minute)+ expect(Oneboxer.external_onebox_ttl({ onebox: "<aside>onebox</aside>", preview: "x" })).to eq(+ 1.day,+ )+ end+ end++ describe "InlineOneboxer caching" do+ it "uses a short TTL for a failed lookup (blank title)" do+ Discourse.cache.expects(:write).with(anything, anything, expires_in: 1.minute)+ InlineOneboxer.send(:onebox_for, "https://example.com/x", nil, {})+ end++ it "uses the normal 1 day TTL when a title is found" do+ Discourse.cache.expects(:write).with(anything, anything, expires_in: 1.day)+ InlineOneboxer.send(:onebox_for, "https://example.com/x", "Some title", {})+ end+ end+enddiff --git a/spec/services/problem_check/github_onebox_backoff_spec.rb b/spec/services/problem_check/github_onebox_backoff_spec.rbnew file mode 100644index 00000000000..ba61c35e375--- /dev/null+++ b/spec/services/problem_check/github_onebox_backoff_spec.rb@@ -0,0 +1,43 @@+# frozen_string_literal: true++RSpec.describe ProblemCheck::GithubOneboxBackoff do+ subject(:check) { described_class.new }++ describe "#call" do+ context "when no GitHub backoff is active" do+ it { expect(check).to be_chill_about_it }+ end++ context "when the unauthenticated onebox client is backing off" do+ before { GithubRateLimit.note_rate_limit(remaining: "0", reset_at: 10.minutes.from_now.to_i) }++ it { expect(check).to have_a_problem.with_priority("low") }+ end++ context "when a configured onebox token is backing off" do+ before do+ SiteSetting.github_onebox_access_tokens = "default|onebox_token"+ GithubRateLimit.note_rate_limit(+ token: "onebox_token",+ remaining: "0",+ reset_at: 10.minutes.from_now.to_i,+ )+ end++ it { expect(check).to have_a_problem.with_priority("low") }+ end++ context "when only an unrelated GitHub token is backing off" do+ before do+ SiteSetting.github_onebox_access_tokens = "default|onebox_token"+ GithubRateLimit.note_rate_limit(+ token: "ai_bot_token",+ remaining: "0",+ reset_at: 10.minutes.from_now.to_i,+ )+ end++ it { expect(check).to be_chill_about_it }+ end+ end+end