Skip to content

A platform for community discussion. Free, open, simple.

FIX: Stop new-feature checks reaching GitHub during requests (#43143)

Rafael dos Santos Silvadea480002bf1
diff --git a/app/jobs/regular/refresh_latest_new_feature.rb b/app/jobs/regular/refresh_latest_new_feature.rb
new file mode 100644
index 00000000000..6e970937a38
--- /dev/null
+++ b/app/jobs/regular/refresh_latest_new_feature.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Jobs
+ # Deriving the newest new-feature timestamp filters the feed against the
+ # installed version, which can shell out to git, so it never happens inline
+ # during a request.
+ class RefreshLatestNewFeature < ::Jobs::Base
+ def execute(args)
+ DiscourseUpdates.refresh_latest_new_feature_created_at!
+ end
+ end
+end
diff --git a/lib/discourse_updates.rb b/lib/discourse_updates.rb
index 234c43f0350..c9c48d215a6 100644
--- a/lib/discourse_updates.rb
+++ b/lib/discourse_updates.rb
@@ -168,7 +168,10 @@ module DiscourseUpdates
def update_new_features(response_json = nil)
response_json ||= new_features_response_json
Discourse.redis.set(new_features_key, response_json)
- Discourse.redis.del(latest_new_feature_created_at_key)
+
+ # Derived on write rather than on read: computing it filters the feed,
+ # which can shell out to git once per entry.
+ refresh_latest_new_feature_created_at!
end
def new_features_response_json
@@ -176,6 +179,8 @@ module DiscourseUpdates
Excon.new(new_features_full_endpoint_url).request(
expects: [200],
method: :Get,
+ connect_timeout: 5,
+ write_timeout: 5,
read_timeout: 5,
)
response.body
@@ -240,7 +245,10 @@ module DiscourseUpdates
def has_unseen_features?(user_id)
latest_ts = latest_new_feature_created_at
- return false if latest_ts.nil?
+ if latest_ts.nil?
+ enqueue_latest_new_feature_created_at_refresh
+ return false
+ end
last_seen = new_features_last_seen(user_id)
return true if last_seen.nil?
@@ -248,15 +256,21 @@ module DiscourseUpdates
latest_ts.to_i > last_seen.to_i
end
+ # Read-only on purpose. Deriving this value filters the feed, which can shell
+ # out to git once per entry, so it is only ever computed in the background by
+ # `refresh_latest_new_feature_created_at!`. A missing value means "nothing to
+ # show yet", not "compute it now".
def latest_new_feature_created_at
cached = Discourse.redis.get(latest_new_feature_created_at_key)
- return Time.zone.parse(cached) if cached.present?
+ cached.present? ? Time.zone.parse(cached) : nil
+ end
+ # Must only be called from a background job, never while serving a request.
+ def refresh_latest_new_feature_created_at!
entries =
merge_new_features_with_upcoming_changes(
new_features&.map { |item| item.symbolize_keys } || [],
)
- return nil if entries.blank?
max_entry =
entries.max_by do |item|
@@ -264,6 +278,11 @@ module DiscourseUpdates
val.is_a?(String) ? Time.zone.parse(val).to_i : val.to_i
end
+ if max_entry.blank?
+ clear_latest_new_feature_created_at_cache
+ return nil
+ end
+
max_created_at =
if max_entry[:created_at].is_a?(String)
Time.zone.parse(max_entry[:created_at])
@@ -280,6 +299,15 @@ module DiscourseUpdates
rescue Redis::CannotConnectError
end
+ # Nothing recomputes the timestamp between the daily job runs, so anything
+ # that invalidates it (a deploy, an upcoming change changing status) queues a
+ # refresh. Throttled because callers can be hit by every request.
+ def enqueue_latest_new_feature_created_at_refresh
+ return if !Discourse.redis.set(refresh_latest_new_feature_lock_key, 1, ex: 1.minute, nx: true)
+ Jobs.enqueue(:refresh_latest_new_feature)
+ rescue Redis::CannotConnectError
+ end
+
def new_features_last_seen(user_id)
last_seen = Discourse.redis.get new_features_last_seen_key(user_id)
return nil if last_seen.blank?
@@ -326,6 +354,7 @@ module DiscourseUpdates
new_features_key,
last_viewed_feature_dates_for_users_key,
latest_new_feature_created_at_key,
+ refresh_latest_new_feature_lock_key,
*Discourse.redis.keys("#{missing_versions_key_prefix}*"),
*Discourse.redis.keys(new_features_last_seen_key("*")),
)
@@ -392,6 +421,10 @@ module DiscourseUpdates
"latest_new_feature_created_at"
end
+ def refresh_latest_new_feature_lock_key
+ "refresh_latest_new_feature_lock"
+ end
+
def last_viewed_feature_dates_for_users_key
"last_viewed_feature_dates_for_users_hash"
end
diff --git a/lib/git_utils.rb b/lib/git_utils.rb
index ccd38a3c57b..fa04a2b0fd2 100644
--- a/lib/git_utils.rb
+++ b/lib/git_utils.rb
@@ -1,6 +1,13 @@
# frozen_string_literal: true
+require "open3"
+
class GitUtils
+ # Shelling out to git can block indefinitely (a wedged git process, a stalled
+ # filesystem under `.git`, fork pressure on a loaded box). Callers get the
+ # fallback value instead of hanging.
+ COMMAND_TIMEOUT_SECONDS = 5
+
def self.git_version
filesystem_overrides["git_version"] || try_git("git rev-parse HEAD", "unknown")
end
@@ -18,7 +25,27 @@ class GitUtils
def self.has_commit?(hash)
return false if !hash.match?(/\A[a-f0-9]{40}\Z/)
- try_git("git merge-base --is-ancestor #{hash} HEAD 2> /dev/null; echo $?", "1") == "0"
+ @has_commit ||= {}
+ return @has_commit[hash] if @has_commit.key?(hash)
+
+ # `echo $?` means the output is git's own exit status: 0 for an ancestor, 1 for
+ # not, 128 when the commit isn't in the repo at all. `nil` means git never ran,
+ # which is not an answer, so it isn't memoized.
+ result =
+ try_git(
+ "git merge-base --is-ancestor #{hash} HEAD 2> /dev/null; echo $?",
+ nil,
+ # We are deployed from a partial clone, where a commit missing from the
+ # local graph is otherwise lazily fetched from the remote: a blocking
+ # network call, with no timeout of its own, to answer what has to stay a
+ # local question.
+ env: {
+ "GIT_NO_LAZY_FETCH" => "1",
+ },
+ )
+ return false if result.nil?
+
+ @has_commit[hash] = result == "0"
end
def self.last_commit_date
@@ -27,15 +54,44 @@ class GitUtils
seconds.nil? ? nil : DateTime.strptime(seconds, "%s")
end
- def self.try_git(git_cmd, default_value)
+ def self.try_git(git_cmd, default_value, timeout: COMMAND_TIMEOUT_SECONDS, env: {})
value =
begin
- `#{git_cmd}`.strip
+ capture_stdout(git_cmd, timeout, env)&.strip
rescue StandardError
- default_value
+ nil
end
- (!value.empty? ? value : nil) || default_value
+ # Can't use `presence` here, ActiveSupport may not be loaded yet
+ value.nil? || value.empty? ? default_value : value
+ end
+
+ # Only stdout is returned, matching the behaviour of a backtick call. stderr is
+ # read in its own thread so a chatty command can't fill the pipe and deadlock,
+ # then passed through to ours, where it used to go directly.
+ private_class_method def self.capture_stdout(git_cmd, timeout, env)
+ Open3.popen3(env, git_cmd, pgroup: true) do |stdin, stdout, stderr, wait_thr|
+ stdin.close
+ out_reader = Thread.new { stdout.read }
+ err_reader = Thread.new { stderr.read }
+
+ if wait_thr.join(timeout).nil?
+ kill_process_group(wait_thr.pid)
+ [out_reader, err_reader].each(&:kill)
+ STDERR.puts("GitUtils: `#{git_cmd}` timed out after #{timeout}s")
+ return nil
+ end
+
+ errors = err_reader.value
+ STDERR.write(errors) if !errors.nil? && !errors.empty?
+ out_reader.value
+ end
+ end
+
+ private_class_method def self.kill_process_group(pid)
+ Process.kill("KILL", -pid)
+ rescue Errno::ESRCH, Errno::EPERM
+ # already gone
end
# The `config/git-utils-override.json` file can be used by hosting providers
diff --git a/spec/jobs/refresh_latest_new_feature_spec.rb b/spec/jobs/refresh_latest_new_feature_spec.rb
new file mode 100644
index 00000000000..9b222dffa02
--- /dev/null
+++ b/spec/jobs/refresh_latest_new_feature_spec.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+RSpec.describe Jobs::RefreshLatestNewFeature do
+ let(:newest_date) { 5.minutes.ago }
+
+ before { UpcomingChanges.stubs(:permanent_upcoming_changes).returns([]) }
+
+ after { DiscourseUpdates.clean_state }
+
+ it "derives the latest new feature timestamp from the stored feed" do
+ Discourse.redis.set(
+ "new_features",
+ MultiJson.dump(
+ [
+ { "emoji" => "🤾", "title" => "Old Feature", "created_at" => 40.minutes.ago },
+ { "emoji" => "🙈", "title" => "Newest Feature", "created_at" => newest_date },
+ ],
+ ),
+ )
+
+ described_class.new.execute({})
+
+ expect(DiscourseUpdates.latest_new_feature_created_at).to be_within(1.second).of(newest_date)
+ end
+
+ it "clears the timestamp when there is nothing to show" do
+ Discourse.redis.set("latest_new_feature_created_at", 1.hour.ago.iso8601)
+ Discourse.redis.set("new_features", MultiJson.dump([]))
+
+ described_class.new.execute({})
+
+ expect(DiscourseUpdates.latest_new_feature_created_at).to be_nil
+ end
+end
diff --git a/spec/lib/discourse_updates_spec.rb b/spec/lib/discourse_updates_spec.rb
index f1e5addf984..3b4aaad5819 100644
--- a/spec/lib/discourse_updates_spec.rb
+++ b/spec/lib/discourse_updates_spec.rb
@@ -187,7 +187,7 @@ RSpec.describe DiscourseUpdates do
Discourse.redis.del "new_features_last_seen_user_#{admin.id}"
Discourse.redis.del "new_features_last_seen_user_#{admin2.id}"
- Discourse.redis.set("new_features", MultiJson.dump(sample_features))
+ DiscourseUpdates.update_new_features(MultiJson.dump(sample_features))
end
after { DiscourseUpdates.clean_state }
@@ -379,6 +379,8 @@ RSpec.describe DiscourseUpdates do
height: 66,
},
)
+
+ DiscourseUpdates.refresh_latest_new_feature_created_at!
end
it "marks the injected item as seen" do
@@ -400,44 +402,57 @@ RSpec.describe DiscourseUpdates do
]
end
- before do
- stub_permanent_upcoming_changes!([])
- Discourse.redis.set("new_features", MultiJson.dump(sample_features))
- end
+ before { stub_permanent_upcoming_changes!([]) }
after { DiscourseUpdates.clean_state }
- it "returns the max created_at from merged features" do
- result = DiscourseUpdates.latest_new_feature_created_at
- expect(result).to be_within(1.second).of(newest_date)
+ it "returns the timestamp derived when the feed was stored" do
+ DiscourseUpdates.update_new_features(MultiJson.dump(sample_features))
+
+ expect(DiscourseUpdates.latest_new_feature_created_at).to be_within(1.second).of(newest_date)
end
- it "caches the result in Redis on subsequent calls" do
- DiscourseUpdates.latest_new_feature_created_at
+ it "never derives the timestamp itself" do
+ Discourse.redis.set("new_features", MultiJson.dump(sample_features))
+ GitUtils.expects(:has_commit?).never
+ DiscourseUpdates.expects(:merge_new_features_with_upcoming_changes).never
- Discourse.redis.set("new_features", MultiJson.dump([]))
- result = DiscourseUpdates.latest_new_feature_created_at
- expect(result).to be_within(1.second).of(newest_date)
+ expect(DiscourseUpdates.latest_new_feature_created_at).to be_nil
+ end
+
+ it "returns the cached value even when the stored feed is unreadable" do
+ DiscourseUpdates.update_new_features(MultiJson.dump(sample_features))
+
+ Discourse.redis.set("new_features", "invalid json")
+ expect(DiscourseUpdates.latest_new_feature_created_at).to be_within(1.second).of(newest_date)
end
it "returns nil when no features exist" do
- Discourse.redis.set("new_features", MultiJson.dump([]))
+ DiscourseUpdates.update_new_features(MultiJson.dump([]))
+
expect(DiscourseUpdates.latest_new_feature_created_at).to be_nil
end
- it "is invalidated by update_new_features" do
- DiscourseUpdates.latest_new_feature_created_at
+ it "is refreshed by update_new_features" do
+ DiscourseUpdates.update_new_features(MultiJson.dump(sample_features))
new_date = 1.minute.ago
new_features = [{ "emoji" => "🤾", "title" => "Brand New", "created_at" => new_date }]
DiscourseUpdates.update_new_features(MultiJson.dump(new_features))
- result = DiscourseUpdates.latest_new_feature_created_at
- expect(result).to be_within(1.second).of(new_date)
+ expect(DiscourseUpdates.latest_new_feature_created_at).to be_within(1.second).of(new_date)
+ end
+
+ it "is cleared when the refreshed feed has no features left" do
+ DiscourseUpdates.update_new_features(MultiJson.dump(sample_features))
+ expect(Discourse.redis.get("latest_new_feature_created_at")).to be_present
+
+ DiscourseUpdates.update_new_features(MultiJson.dump([]))
+ expect(Discourse.redis.get("latest_new_feature_created_at")).to be_nil
end
it "is invalidated by clean_state" do
- DiscourseUpdates.latest_new_feature_created_at
+ DiscourseUpdates.update_new_features(MultiJson.dump(sample_features))
expect(Discourse.redis.get("latest_new_feature_created_at")).to be_present
DiscourseUpdates.clean_state
@@ -445,7 +460,7 @@ RSpec.describe DiscourseUpdates do
end
end
- describe "has_unseen_features? caching" do
+ describe "has_unseen_features?" do
fab!(:admin)
let!(:feature_date) { 5.minutes.ago }
let!(:sample_features) do
@@ -454,24 +469,40 @@ RSpec.describe DiscourseUpdates do
before do
stub_permanent_upcoming_changes!([])
- Discourse.redis.set("new_features", MultiJson.dump(sample_features))
+ DiscourseUpdates.clean_state
end
after { DiscourseUpdates.clean_state }
- it "uses the cached timestamp instead of recomputing on repeated calls" do
- DiscourseUpdates.latest_new_feature_created_at
+ it "uses the cached timestamp instead of recomputing" do
+ DiscourseUpdates.update_new_features(MultiJson.dump(sample_features))
Discourse.redis.set("new_features", "invalid json")
expect(DiscourseUpdates.has_unseen_features?(admin.id)).to eq(true)
end
it "returns false when the latest feature timestamp equals last_seen" do
+ DiscourseUpdates.update_new_features(MultiJson.dump(sample_features))
+
freeze_time do
Discourse.redis.set("new_features_last_seen_user_#{admin.id}", feature_date.iso8601)
expect(DiscourseUpdates.has_unseen_features?(admin.id)).to eq(false)
end
end
+
+ it "queues a refresh, instead of deriving inline, when the timestamp is missing" do
+ Discourse.redis.set("new_features", MultiJson.dump(sample_features))
+ GitUtils.expects(:has_commit?).never
+
+ expect(DiscourseUpdates.has_unseen_features?(admin.id)).to eq(false)
+ expect(Jobs::RefreshLatestNewFeature.jobs.size).to eq(1)
+ end
+
+ it "only queues one refresh per throttle window" do
+ 3.times { DiscourseUpdates.has_unseen_features?(admin.id) }
+
+ expect(Jobs::RefreshLatestNewFeature.jobs.size).to eq(1)
+ end
end
describe ".merge_new_features_with_upcoming_changes" do
diff --git a/spec/lib/git_utils_spec.rb b/spec/lib/git_utils_spec.rb
index bbe17deb331..5d955ae7363 100644
--- a/spec/lib/git_utils_spec.rb
+++ b/spec/lib/git_utils_spec.rb
@@ -71,6 +71,8 @@ RSpec.describe GitUtils do
end
describe ".has_commit?" do
+ before { GitUtils.instance_variable_set(:@has_commit, nil) }
+
it "validates commit existence and format" do
within_temp_git_repo do
create_commit
@@ -82,6 +84,66 @@ RSpec.describe GitUtils do
expect(GitUtils.has_commit?("invalid")).to eq(false)
end
end
+
+ it "only shells out once per hash" do
+ within_temp_git_repo do
+ create_commit
+ sha = `git rev-parse HEAD`.strip
+
+ GitUtils.expects(:try_git).once.returns("0")
+
+ expect(GitUtils.has_commit?(sha)).to eq(true)
+ expect(GitUtils.has_commit?(sha)).to eq(true)
+ end
+ end
+
+ it "does not lazily fetch a commit missing from a partial clone" do
+ clone_dir = File.join(temp_dir, "partial")
+
+ within_temp_git_repo do
+ create_commit
+ run("git config uploadpack.allowfilter true")
+ run("git clone --quiet --filter=tree:0 file://#{temp_dir} #{clone_dir}")
+ end
+
+ Dir.chdir(clone_dir) do
+ # An unroutable promisor remote: reaching for it would stall until the
+ # command timeout rather than fail fast.
+ run("git remote set-url origin https://10.255.255.1/unreachable.git")
+
+ elapsed = Benchmark.realtime { expect(GitUtils.has_commit?("a" * 40)).to eq(false) }
+ expect(elapsed).to be < GitUtils::COMMAND_TIMEOUT_SECONDS
+ end
+ end
+
+ it "does not memoize a result when git is unavailable" do
+ GitUtils.expects(:try_git).twice.returns(nil)
+
+ expect(GitUtils.has_commit?("a" * 40)).to eq(false)
+ expect(GitUtils.has_commit?("a" * 40)).to eq(false)
+ end
+ end
+
+ describe ".try_git" do
+ it "returns the fallback value and kills the process when it times out" do
+ expect(GitUtils.try_git("sleep 5; echo nope", "fallback", timeout: 0.1)).to eq("fallback")
+ end
+
+ it "does not include stderr in the returned value" do
+ expect(GitUtils.try_git("echo oops 1>&2; echo out", "fallback")).to eq("out")
+ end
+
+ it "passes the given environment to the command" do
+ expect(GitUtils.try_git("echo $SOME_VAR", "fallback", env: { "SOME_VAR" => "set" })).to eq(
+ "set",
+ )
+ end
+
+ it "returns the fallback value when the command does not exist" do
+ expect(GitUtils.try_git("definitely-not-a-real-command-xyz 2> /dev/null", "fallback")).to eq(
+ "fallback",
+ )
+ end
end
describe ".filesystem_overrides" do
diff --git a/spec/requests/admin/dashboard_controller_spec.rb b/spec/requests/admin/dashboard_controller_spec.rb
index f998eae16af..c5d691c6602 100644
--- a/spec/requests/admin/dashboard_controller_spec.rb
+++ b/spec/requests/admin/dashboard_controller_spec.rb
@@ -36,7 +36,7 @@ RSpec.describe Admin::DashboardController do
},
]
- Discourse.redis.set("new_features", MultiJson.dump(sample_features))
+ DiscourseUpdates.update_new_features(MultiJson.dump(sample_features))
end
describe "#index" do