A platform for community discussion. Free, open, simple.
FIX: move klipy gifs request to backend (#41540)
David Battersby32e170b3fd4e
diff --git a/app/controllers/gifs_controller.rb b/app/controllers/gifs_controller.rbnew file mode 100644index 00000000000..f86165b3cd9--- /dev/null+++ b/app/controllers/gifs_controller.rb@@ -0,0 +1,88 @@+# frozen_string_literal: true++class GifsController < ApplicationController+ requires_login++ KLIPY_CATEGORIES_URL = "https://api.klipy.com/v2/categories"+ KLIPY_SEARCH_URL = "https://api.klipy.com/v2/search"+ PAGE_SIZE = 24+ MAX_QUERY_LENGTH = 100+ MAX_REQUESTS_PER_10_SECONDS = 20++ before_action :ensure_gifs_enabled+ before_action :ensure_klipy_api_key+ before_action :rate_limit++ def categories+ proxy_klipy_request(+ KLIPY_CATEGORIES_URL,+ {+ "type" => "featured",+ "country" => SiteSetting.klipy_country,+ "locale" => SiteSetting.klipy_locale,+ "contentfilter" => SiteSetting.klipy_content_filter,+ },+ )+ end++ def search+ query = params.require(:q)+ if !query.is_a?(String) || query.length > MAX_QUERY_LENGTH+ raise Discourse::InvalidParameters.new(:q)+ end++ proxy_klipy_request(+ KLIPY_SEARCH_URL,+ {+ "q" => query,+ "country" => SiteSetting.klipy_country,+ "locale" => SiteSetting.klipy_locale,+ "contentfilter" => SiteSetting.klipy_content_filter,+ "media_filter" => SiteSetting.klipy_file_detail,+ "limit" => PAGE_SIZE,+ "pos" => params[:pos].presence || "0",+ },+ )+ end++ private++ def ensure_gifs_enabled+ raise Discourse::NotFound if !SiteSetting.enable_gifs?+ end++ def ensure_klipy_api_key+ head :forbidden if SiteSetting.klipy_api_key.blank?+ end++ def rate_limit+ RateLimiter.new(current_user, "gif-search", MAX_REQUESTS_PER_10_SECONDS, 10.seconds).performed!+ end++ def proxy_klipy_request(url, query)+ response =+ Excon.get(+ url,+ headers: {+ "Accept" => "application/json",+ },+ query: query.merge("key" => SiteSetting.klipy_api_key),+ connect_timeout: 5,+ read_timeout: 5,+ )++ render plain: redact_api_key(response.body.to_s),+ status: response.status,+ content_type: "application/json"+ rescue Excon::Error+ head :bad_gateway+ end++ def redact_api_key(body)+ api_key = SiteSetting.klipy_api_key+ return body if api_key.blank?++ [api_key, CGI.escape(api_key)].uniq+ .reduce(body) { |redacted_body, value| redacted_body.gsub(value, "[FILTERED]") }+ end+enddiff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex 12855715002..b68ed6ed94f 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -2714,7 +2714,7 @@ en:enable_inline_emoji_translation: "Enables translation for inline emojis (without any space or punctuation before)"emoji_deny_list: "These emoji will not be available to use in menus or shortcodes."enable_gifs: "Adds a button to easily search and insert GIFs in posts."- klipy_api_key: "Klipy API key. Required to power GIF search. Note: this key is exposed to anyone viewing your site."+ klipy_api_key: "Klipy API key. Required to power GIF search."klipy_file_detail: "Image format to use. WebP has smaller files that load quicker, while GIF provides compatibility with older browsers."klipy_content_filter: "Content safety level for GIF search results."klipy_country: "Two-letter country code for GIF search results."diff --git a/config/routes.rb b/config/routes.rbindex 29fc8dc1da9..90dacb42f5e 100644--- a/config/routes.rb+++ b/config/routes.rb@@ -593,6 +593,8 @@ Discourse::Application.routes.draw dopost "session/passkey/auth" => "session#passkey_login"get "session/scopes" => "session#scopes"get "composer/mentions" => "composer#mentions"+ get "gifs/categories" => "gifs#categories"+ get "gifs/search" => "gifs#search"get "composer_messages" => "composer_messages#index"get "composer_messages/user_not_seen_in_a_while" => "composer_messages#user_not_seen_in_a_while"diff --git a/config/site_settings.yml b/config/site_settings.ymlindex dacfe49c654..2dc43b56b52 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -1691,7 +1691,7 @@ posting:learn_more_url: "https://meta.discourse.org/t/-/404829"klipy_api_key:default: ""- client: true+ secret: truearea: "gifs"klipy_file_detail:default: "webp"diff --git a/frontend/discourse/app/components/modal/gifs.gjs b/frontend/discourse/app/components/modal/gifs.gjsindex 8bce0472cbd..8a6a69ee4da 100644--- a/frontend/discourse/app/components/modal/gifs.gjs+++ b/frontend/discourse/app/components/modal/gifs.gjs@@ -13,9 +13,8 @@ import DModal from "discourse/ui-kit/d-modal";import dLoadingSpinner from "discourse/ui-kit/helpers/d-loading-spinner";import { i18n } from "discourse-i18n";-const KLIPY_SEARCH_URL = "https://api.klipy.com/v2/search";-const KLIPY_CATEGORIES_URL = "https://api.klipy.com/v2/categories";-const PAGE_SIZE = 24;+const GIFS_SEARCH_URL = "/gifs/search.json";+const GIFS_CATEGORIES_URL = "/gifs/categories.json";const MIN_QUERY_LENGTH = 3;export default class GifsModal extends Component {@@ -74,24 +73,10 @@ export default class GifsModal extends Component {}async fetchCategories() {- if (this.siteSettings.klipy_api_key === "") {- return;- }-this.loadingCategories = true;try {- const params = {- key: this.siteSettings.klipy_api_key,- country: this.siteSettings.klipy_country,- locale: this.siteSettings.klipy_locale,- type: "featured",- contentfilter: this.siteSettings.klipy_content_filter,- };-- const response = await fetch(- `${KLIPY_CATEGORIES_URL}?${new URLSearchParams(params)}`- );+ const response = await fetch(getURL(GIFS_CATEGORIES_URL));if (this.isDestroying || this.isDestroyed) {return;@@ -166,10 +151,6 @@ export default class GifsModal extends Component {this.loading = true;try {- if (this.siteSettings.klipy_api_key === "") {- throw new Error(i18n("gifs.bad_api_key"));- }-const response = await fetch(this.getEndpoint(this.query, this.offset));if (this.isDestroying || this.isDestroyed) {@@ -236,7 +217,7 @@ export default class GifsModal extends Component {return i18n("gifs.bad_api_key");}- return `Klipy status ${response.status}: ${this.redactApiKey(message)}`;+ return `Klipy status ${response.status}: ${message}`;}extractErrorMessage(parsed) {@@ -257,26 +238,12 @@ export default class GifsModal extends Component {);}- redactApiKey(message) {- const apiKey = this.siteSettings.klipy_api_key;- if (!apiKey || typeof message !== "string") {- return message;- }- return message.replaceAll(apiKey, "[redacted]");- }-getEndpoint(query, offset) {const params = {- key: this.siteSettings.klipy_api_key,q: query,- country: this.siteSettings.klipy_country,- locale: this.siteSettings.klipy_locale,- contentfilter: this.siteSettings.klipy_content_filter,- media_filter: this.siteSettings.klipy_file_detail,- limit: PAGE_SIZE,pos: offset,};- return `${KLIPY_SEARCH_URL}?${new URLSearchParams(params)}`;+ return getURL(`${GIFS_SEARCH_URL}?${new URLSearchParams(params)}`);}<template>diff --git a/frontend/discourse/tests/acceptance/modal/gifs-test.gjs b/frontend/discourse/tests/acceptance/modal/gifs-test.gjsindex 073defa4e1f..5e5bfe58ff4 100644--- a/frontend/discourse/tests/acceptance/modal/gifs-test.gjs+++ b/frontend/discourse/tests/acceptance/modal/gifs-test.gjs@@ -28,7 +28,6 @@ acceptance("Modal - GIFs", function (needs) {needs.user();needs.settings({enable_gifs: true,- klipy_api_key: "test-key",klipy_country: "US",klipy_locale: "en_US",klipy_content_filter: "high",@@ -58,21 +57,21 @@ acceptance("Modal - GIFs", function (needs) {const categoriesCall = fetchStub.getCalls()- .find((c) => c.args[0].includes("/v2/categories"));+ .find((call) => call.args[0].includes("/gifs/categories.json"));assert.notStrictEqual(categoriesCall,undefined,- "called the Klipy categories endpoint"+ "called the GIF categories proxy");- assert.true(- categoriesCall.args[0].includes("key=test-key"),- "passes api key in query"+ assert.false(+ categoriesCall.args[0].includes("key="),+ "does not pass an API key in the query");});test("search triggers a Klipy fetch with the query", async function (assert) {fetchStub.callsFake(async (url) => {- if (url.includes("/v2/categories")) {+ if (url.includes("/gifs/categories.json")) {return categoriesResponse();}return searchResponse({ results: [gif("a hello gif")] });@@ -87,14 +86,14 @@ acceptance("Modal - GIFs", function (needs) {const searchCall = fetchStub.getCalls()- .find((c) => c.args[0].includes("/v2/search"));+ .find((call) => call.args[0].includes("/gifs/search.json"));- assert.notStrictEqual(- searchCall,- undefined,- "called the Klipy search endpoint"- );+ assert.notStrictEqual(searchCall, undefined, "called the GIF search proxy");assert.true(searchCall.args[0].includes("q=hello"), "passes the query");+ assert.false(+ searchCall.args[0].includes("key="),+ "does not pass an API key in the query"+ );assert.dom(".gifs-modal .gifs-result").exists({ count: 1 }, "renders the result returned by Klipy");@@ -102,7 +101,7 @@ acceptance("Modal - GIFs", function (needs) {test("picking a result invokes customPickHandler with markup", async function (assert) {fetchStub.callsFake(async (url) => {- if (url.includes("/v2/categories")) {+ if (url.includes("/gifs/categories.json")) {return categoriesResponse();}return searchResponse({@@ -138,7 +137,7 @@ acceptance("Modal - GIFs", function (needs) {test("renders a no results state when Klipy returns nothing", async function (assert) {fetchStub.callsFake(async (url) => {- if (url.includes("/v2/categories")) {+ if (url.includes("/gifs/categories.json")) {return categoriesResponse();}return searchResponse({ results: [], next: "" });diff --git a/spec/requests/gifs_controller_spec.rb b/spec/requests/gifs_controller_spec.rbnew file mode 100644index 00000000000..b2f23715ea1--- /dev/null+++ b/spec/requests/gifs_controller_spec.rb@@ -0,0 +1,144 @@+# frozen_string_literal: true++RSpec.describe GifsController do+ fab!(:user)++ before do+ SiteSetting.enable_gifs = true+ SiteSetting.klipy_api_key = "super secret/klipy+key"+ end++ describe "#search" do+ it "proxies Klipy requests with the configured API key without returning it" do+ sign_in(user)++ stub_request(:get, "https://api.klipy.com/v2/search").with(+ query:+ hash_including(+ "key" => SiteSetting.klipy_api_key,+ "q" => "hello",+ "country" => SiteSetting.klipy_country,+ "locale" => SiteSetting.klipy_locale,+ "contentfilter" => SiteSetting.klipy_content_filter,+ "media_filter" => SiteSetting.klipy_file_detail,+ "limit" => "24",+ "pos" => "0",+ ),+ ).to_return(status: 200, body: { results: [], next: "" }.to_json)++ get "/gifs/search.json", params: { q: "hello", pos: "0" }++ expect(response.status).to eq(200)+ expect(response.parsed_body).to eq("results" => [], "next" => "")+ expect(response.body).not_to include(SiteSetting.klipy_api_key)+ end++ it "requires a logged-in user" do+ get "/gifs/search.json", params: { q: "hello" }++ expect(response.status).to eq(403)+ end++ it "returns forbidden when no API key is configured" do+ SiteSetting.klipy_api_key = ""+ sign_in(user)++ get "/gifs/search.json", params: { q: "hello" }++ expect(response.status).to eq(403)+ end++ it "returns not found when GIF search is disabled" do+ SiteSetting.enable_gifs = false+ sign_in(user)++ get "/gifs/search.json", params: { q: "hello" }++ expect(response.status).to eq(404)+ end++ it "rejects queries longer than the maximum length" do+ sign_in(user)++ get "/gifs/search.json", params: { q: "a" * (GifsController::MAX_QUERY_LENGTH + 1) }++ expect(response.status).to eq(400)+ end++ it "returns bad gateway when Klipy cannot be reached" do+ sign_in(user)++ stub_request(:get, "https://api.klipy.com/v2/search").with(+ query: hash_including("q" => "hello"),+ ).to_timeout++ get "/gifs/search.json", params: { q: "hello" }++ expect(response.status).to eq(502)+ end++ it "redacts the API key from Klipy error responses" do+ sign_in(user)++ stub_request(:get, "https://api.klipy.com/v2/search").with(+ query: hash_including("key" => SiteSetting.klipy_api_key),+ ).to_return(status: 500, body: "upstream error #{CGI.escape(SiteSetting.klipy_api_key)}")++ get "/gifs/search.json", params: { q: "hello" }++ expect(response.status).to eq(500)+ expect(response.body).to include("[FILTERED]")+ expect(response.body).not_to include(SiteSetting.klipy_api_key)+ expect(response.body).not_to include(CGI.escape(SiteSetting.klipy_api_key))+ end++ context "when rate limiting is enabled" do+ before { RateLimiter.enable }++ it "returns 429 once the per-user limit is exceeded" do+ sign_in(user)++ stub_request(:get, "https://api.klipy.com/v2/search").with(+ query: hash_including("q" => "hello"),+ ).to_return(status: 200, body: { results: [], next: "" }.to_json)++ GifsController::MAX_REQUESTS_PER_10_SECONDS.times do+ get "/gifs/search.json", params: { q: "hello" }+ expect(response.status).to eq(200)+ end++ get "/gifs/search.json", params: { q: "hello" }+ expect(response.status).to eq(429)+ end+ end+ end++ describe "#categories" do+ it "proxies featured categories with the configured API key without returning it" do+ sign_in(user)++ stub_request(:get, "https://api.klipy.com/v2/categories").with(+ query:+ hash_including(+ "key" => SiteSetting.klipy_api_key,+ "type" => "featured",+ "country" => SiteSetting.klipy_country,+ "locale" => SiteSetting.klipy_locale,+ "contentfilter" => SiteSetting.klipy_content_filter,+ ),+ ).to_return(status: 200, body: { tags: [] }.to_json)++ get "/gifs/categories.json"++ expect(response.status).to eq(200)+ expect(response.parsed_body).to eq("tags" => [])+ expect(response.body).not_to include(SiteSetting.klipy_api_key)+ end++ it "requires a logged-in user" do+ get "/gifs/categories.json"++ expect(response.status).to eq(403)+ end+ end+enddiff --git a/spec/requests/list_controller_spec.rb b/spec/requests/list_controller_spec.rbindex 9c6add11f8d..9ec9f52989d 100644--- a/spec/requests/list_controller_spec.rb+++ b/spec/requests/list_controller_spec.rb@@ -12,6 +12,21 @@ RSpec.describe ListController doenddescribe "#index" do+ it "does not expose the Klipy API key in anonymous preloaded site settings" do+ SiteSetting.klipy_api_key = "super-secret-klipy-key"++ get "/latest"++ expect(response.status).to eq(200)+ expect(response.body).not_to include(SiteSetting.klipy_api_key)+ expect(response.body).to have_tag("div#data-preloaded") do |element|+ data_preloaded = JSON.parse(element.current_scope.attribute("data-preloaded").value)+ site_settings = JSON.parse(data_preloaded["siteSettings"])++ expect(site_settings).not_to have_key("klipy_api_key")+ end+ end+context "when params are invalid" doit "should return a 400 response when `page` param is a string that represent a negative integer" doget "/latest?page=-1"