A platform for community discussion. Free, open, simple.
FEATURE: provide crawler view for chat (#44666)
Kris0d2e57325b59
diff --git a/app/views/layouts/_noscript_footer.html.erb b/app/views/layouts/_noscript_footer.html.erbindex c6d4efbe0f1..afafdc688c3 100644--- a/app/views/layouts/_noscript_footer.html.erb+++ b/app/views/layouts/_noscript_footer.html.erb@@ -11,6 +11,7 @@<a href='<%= path "/categories" %>' itemprop="url"><%= t 'js.filters.categories.title' %> </a></span></li>+ <%= server_plugin_outlet "noscript_footer_nav" %><li itemscope itemtype='http://schema.org/SiteNavigationElement'><span itemprop='name'><a href='<%= path "/guidelines" %>' itemprop="url"><%= t 'guidelines_topic.guidelines_title' %> </a>diff --git a/plugins/chat/app/controllers/chat/chat_controller.rb b/plugins/chat/app/controllers/chat/chat_controller.rbindex e60b6b96a85..5be1897e165 100644--- a/plugins/chat/app/controllers/chat/chat_controller.rb+++ b/plugins/chat/app/controllers/chat/chat_controller.rb@@ -2,11 +2,23 @@module Chatclass ChatController < ::Chat::BaseController+ skip_before_action :check_xhr, if: -> { action_name == "respond" && use_crawler_layout? }before_action :set_channel_and_chatable_with_access_check,except: %i[respond set_user_chat_status dismiss_retention_reminder rebake]+ DIRECTORY_PAGE_SIZE = Chat::ChannelFetcher::MAX_PUBLIC_CHANNEL_RESULTS+def respond- render+ raise ApplicationController::RenderEmpty if !use_crawler_layout? || current_user++ # Channel routes 404 when anonymous access is off, so already-indexed pages drop out of search.+ if params[:channel_id].present?+ render_crawler_channel+ elsif params[:crawler_directory] && Chat.anonymous_public_channel_access_allowed?+ render_crawler_directory+ else+ raise ApplicationController::RenderEmpty+ endenddef react@@ -98,5 +110,73 @@ module Chat).generate_markdownrender json: success_json.merge(markdown: markdown)end++ private++ # Crawler requests skip check_xhr, so respond itself decides what anonymous visitors may see.+ def allow_anonymous_public_chat_access?+ action_name == "respond" && use_crawler_layout?+ end++ def render_crawler_channel+ channel = Chat::Channel.includes(:chatable).find_by_id_or_slug(params[:channel_id])+ raise Discourse::NotFound if !guardian.can_preview_anonymous_public_chat_channel?(channel)++ if params[:thread_id].present?+ thread = channel.threads.find_by(id: params[:thread_id])+ raise Discourse::NotFound if !thread || (!channel.threading_enabled? && !thread.force)+ elsif params[:message_id].present?+ # Channel permalinks to thread replies (e.g. from transcripts) live on the thread page.+ message = channel.chat_messages.find_by(id: params[:message_id])+ return redirect_to(message.url, status: :moved_permanently) if message&.thread_reply?+ end++ @crawler_view =+ Chat::CrawlerView.new(+ channel:,+ thread:,+ date: params[:date],+ page: params[:page],+ target_message_id: params[:message_id],+ )+ raise Discourse::NotFound if !@crawler_view.valid?++ canonical_url(@crawler_view.canonical_url)+ render :crawler_channel+ end++ def render_crawler_directory+ channels =+ Chat::CategoryChannel.joins(+ "INNER JOIN categories ON categories.id = chat_channels.chatable_id",+ ).where(chatable_id: Category.secured(Guardian.new).select(:id))++ page = [params[:page].to_s.to_i, 1].max+ total_pages = [(channels.count.to_f / DIRECTORY_PAGE_SIZE).ceil, 1].max+ raise Discourse::NotFound if page > total_pages++ @crawler_directory_offset = (page - 1) * DIRECTORY_PAGE_SIZE+ @crawler_channels =+ channels+ .includes(:chatable, :last_message)+ .order(+ Arel.sql(+ "LOWER(COALESCE(NULLIF(chat_channels.name, ''), categories.name)), chat_channels.id",+ ),+ )+ .offset(@crawler_directory_offset)+ .limit(DIRECTORY_PAGE_SIZE)++ @crawler_prev_page_url = crawler_directory_url(page - 1) if page > 1+ @crawler_next_page_url = crawler_directory_url(page + 1) if page < total_pages++ canonical_url(crawler_directory_url(page))+ render :crawler_directory+ end++ def crawler_directory_url(page)+ url = "#{Discourse.base_url}/chat"+ page > 1 ? "#{url}?page=#{page}" : url+ endendenddiff --git a/plugins/chat/app/views/chat/chat/_crawler_message.html.erb b/plugins/chat/app/views/chat/chat/_crawler_message.html.erbnew file mode 100644index 00000000000..05de04b7547--- /dev/null+++ b/plugins/chat/app/views/chat/chat/_crawler_message.html.erb@@ -0,0 +1,56 @@+<%# schema: :posting (standalone DiscussionForumPosting), :comment (reply nested in a thread posting), or :original (a thread's first message, whose other properties live on the enclosing posting) %>+<% u = message.user %>+<% emit_md = schema != :original %>+<%+ item_attributes =+ case schema+ when :posting+ { itemscope: true, itemtype: "http://schema.org/DiscussionForumPosting" }+ when :comment+ { itemprop: "comment", itemscope: true, itemtype: "http://schema.org/Comment" }+ else+ {}+ end+%>+<div id="chat-message-<%= message.id %>" class="chat-crawler__message" <%= tag.attributes(item_attributes) %>>+ <% if emit_md %>+ <link itemprop="url" href="<%= view.posting_url(message) %>">+ <% end %>+ <div class="chat-crawler__message-meta">+ <span class="chat-crawler__message-author" <%= tag.attributes(itemprop: "author", itemscope: true, itemtype: "http://schema.org/Person") if emit_md %>>+ <a rel="nofollow" href="<%= Discourse.base_url %>/u/<%= u.username %>"><span <%= tag.attributes(itemprop: "name") if emit_md %>><%= u.username %></span></a>+ <% if emit_md %>+ <meta itemprop="url" content="<%= Discourse.base_url %>/u/<%= u.username %>">+ <% end %>+ <%= "(#{u.name})" if SiteSetting.display_name_on_posts && SiteSetting.enable_names? && u.name.present? %>+ </span>+ <time <%= tag.attributes(itemprop: "datePublished") if emit_md %> datetime="<%= message.created_at.to_formatted_s(:iso8601) %>" class="chat-crawler__message-time">+ <%= l message.created_at, format: :long %>+ </time>+ <% if emit_md && (revision = message.revisions.max_by(&:created_at)) %>+ <meta itemprop="dateModified" content="<%= revision.created_at.to_formatted_s(:iso8601) %>">+ <% end %>+ </div>+ <div class="chat-crawler__message-text post" itemprop="text">+ <%= message.cooked.html_safe %>+ </div>+ <% if (reaction_counts = view.reaction_counts(message)).present? %>+ <div class="chat-crawler__reactions">+ <% reaction_counts.each do |emoji, count| %>+ <span class="chat-crawler__reaction"><%= Emoji.lookup_unicode(emoji) || ":#{emoji}:" %> <%= count %></span>+ <% end %>+ </div>+ <% end %>+ <% if emit_md %>+ <div itemprop="interactionStatistic" itemscope itemtype="http://schema.org/InteractionCounter">+ <meta itemprop="interactionType" content="http://schema.org/LikeAction">+ <meta itemprop="userInteractionCount" content="<%= view.like_count(message) %>">+ </div>+ <% end %>+ <% if view.show_thread_link?(message) %>+ <meta itemprop="commentCount" content="<%= message.thread.replies_count %>">+ <div class="chat-crawler__thread-link">+ <a href="<%= message.thread.url %>"><%= t("chat.crawler.thread_replies", count: message.thread.replies_count) %></a>+ </div>+ <% end %>+</div>diff --git a/plugins/chat/app/views/chat/chat/_crawler_retention_notice.html.erb b/plugins/chat/app/views/chat/chat/_crawler_retention_notice.html.erbnew file mode 100644index 00000000000..d8a02d41f72--- /dev/null+++ b/plugins/chat/app/views/chat/chat/_crawler_retention_notice.html.erb@@ -0,0 +1,3 @@+<% if (retention_days = SiteSetting.chat_channel_retention_days) > 0 %>+ <p class="chat-crawler__retention"><%= t("chat.crawler.retention_notice", count: retention_days) %></p>+<% end %>diff --git a/plugins/chat/app/views/chat/chat/crawler_channel.html.erb b/plugins/chat/app/views/chat/chat/crawler_channel.html.erbnew file mode 100644index 00000000000..60df16ff17d--- /dev/null+++ b/plugins/chat/app/views/chat/chat/crawler_channel.html.erb@@ -0,0 +1,86 @@+<% view = @crawler_view %>+<% category = view.channel.chatable %>++<div class="chat-crawler">+ <h1><%= view.title %></h1>++ <div class="chat-crawler__context">+ <% if view.thread %>+ <a href="<%= view.channel.url %>"><%= view.channel.title %></a>+ <% else %>+ <a href="<%= category.url %>"><%= category.name %></a>+ <% end %>+ </div>++ <%= render "chat/chat/crawler_retention_notice" %>+ <% if include_crawler_content? %>+ <% original_message = view.original_message %>+ <% original_author = original_message&.user %>+ <div class="chat-crawler__messages" <%= tag.attributes(itemscope: true, itemtype: "http://schema.org/DiscussionForumPosting") if original_message %>>+ <% if original_message %>+ <meta itemprop="headline" content="<%= view.title %>">+ <link itemprop="url" href="<%= view.url %>">+ <meta itemprop="datePublished" content="<%= original_message.created_at.to_formatted_s(:iso8601) %>">+ <meta itemprop="commentCount" content="<%= view.thread.replies_count %>">+ <div itemprop="interactionStatistic" itemscope itemtype="http://schema.org/InteractionCounter">+ <meta itemprop="interactionType" content="http://schema.org/LikeAction">+ <meta itemprop="userInteractionCount" content="<%= view.like_count(original_message) %>">+ </div>+ <% if original_author %>+ <span itemprop="author" itemscope itemtype="http://schema.org/Person">+ <meta itemprop="name" content="<%= original_author.username %>">+ <link itemprop="url" href="<%= Discourse.base_url %>/u/<%= original_author.username %>">+ </span>+ <% end %>+ <% end %>++ <% view.messages.each do |message| %>+ <% next if message.user.blank? || message.cooked.blank? %>+ <%+ schema =+ if !original_message+ :posting+ elsif message.id == original_message.id+ :original+ else+ :comment+ end+ %>+ <%= render "chat/chat/crawler_message", message:, view:, schema: %>+ <% end %>+ </div>++ <% if view.prev_page_url || view.next_page_url %>+ <div role="navigation" class="chat-crawler__pagination crawler-post">+ <% if view.prev_page_url %>+ <span><%= link_to t(:prev_page), view.prev_page_url, rel: "prev" %></span>+ <% end %>+ <% if view.next_page_url %>+ <span><b><%= link_to t(:next_page), view.next_page_url, rel: "next" %></b></span>+ <% end %>+ </div>+ <% end %>+ <% else %>+ <%= t(:crawler_content_hidden) %>+ <% end %>+</div>++<% content_for :head do %>+ <% if (unavailable_after = view.unavailable_after) %>+ <meta name="robots" content="unavailable_after: <%= unavailable_after.iso8601 %>">+ <% end %>+ <%= raw crawlable_meta_data(+ title: view.title,+ description: view.description,+ url: view.canonical_url,+ ignore_canonical: true,+ ) %>+ <% if view.prev_page_url %>+ <link rel="prev" href="<%= view.prev_page_url %>">+ <% end %>+ <% if view.next_page_url %>+ <link rel="next" href="<%= view.next_page_url %>">+ <% end %>+<% end %>++<% content_for(:title) { "#{gsub_emoji_to_unicode(view.title)} - #{SiteSetting.title}" } %>diff --git a/plugins/chat/app/views/chat/chat/crawler_directory.html.erb b/plugins/chat/app/views/chat/chat/crawler_directory.html.erbnew file mode 100644index 00000000000..a02ef80cc77--- /dev/null+++ b/plugins/chat/app/views/chat/chat/crawler_directory.html.erb@@ -0,0 +1,54 @@+<div class="chat-crawler">+ <h1><%= t("chat.crawler.directory_title") %></h1>++ <%= render "chat/chat/crawler_retention_notice" %>++ <% if include_crawler_content? %>+ <ul class="chat-crawler__channels" itemscope itemtype="http://schema.org/ItemList">+ <% @crawler_channels.each_with_index do |channel, index| %>+ <li class="chat-crawler__channel" itemprop="itemListElement" itemscope itemtype="http://schema.org/ListItem">+ <meta itemprop="position" content="<%= @crawler_directory_offset + index + 1 %>">+ <h2><a href="<%= channel.url %>" itemprop="url"><span itemprop="name"><%= channel.title %></span></a></h2>+ <% if channel.description.present? %>+ <p><%= channel.description %></p>+ <% end %>+ <div class="chat-crawler__channel-meta">+ <a href="<%= channel.chatable.url %>"><%= channel.chatable.name %></a>+ <span><%= t("chat.crawler.messages_count", count: channel.messages_count) %></span>+ <span><%= t("chat.crawler.members_count", count: channel.user_count) %></span>+ <% if (last_message = channel.last_message).id.present? %>+ <span>+ <%= t("chat.crawler.last_activity") %>+ <time datetime="<%= last_message.created_at.to_formatted_s(:iso8601) %>"><%= l last_message.created_at, format: :long %></time>+ </span>+ <% end %>+ </div>+ </li>+ <% end %>+ </ul>+ <% if @crawler_prev_page_url || @crawler_next_page_url %>+ <div role="navigation" class="chat-crawler__pagination crawler-post">+ <% if @crawler_prev_page_url %>+ <span><%= link_to t(:prev_page), @crawler_prev_page_url, rel: "prev" %></span>+ <% end %>+ <% if @crawler_next_page_url %>+ <span><b><%= link_to t(:next_page), @crawler_next_page_url, rel: "next" %></b></span>+ <% end %>+ </div>+ <% end %>+ <% else %>+ <%= t(:crawler_content_hidden) %>+ <% end %>+</div>++<% content_for :head do %>+ <%= raw crawlable_meta_data(title: t("chat.crawler.directory_title")) %>+ <% if @crawler_prev_page_url %>+ <link rel="prev" href="<%= @crawler_prev_page_url %>">+ <% end %>+ <% if @crawler_next_page_url %>+ <link rel="next" href="<%= @crawler_next_page_url %>">+ <% end %>+<% end %>++<% content_for(:title) { "#{t("chat.crawler.directory_title")} - #{SiteSetting.title}" } %>diff --git a/plugins/chat/app/views/connectors/noscript_footer_nav/chat.html.erb b/plugins/chat/app/views/connectors/noscript_footer_nav/chat.html.erbnew file mode 100644index 00000000000..c53d6fec233--- /dev/null+++ b/plugins/chat/app/views/connectors/noscript_footer_nav/chat.html.erb@@ -0,0 +1,7 @@+<% if defined?(::Chat) && SiteSetting.chat_enabled && Chat.anonymous_public_channel_access_allowed? %>+ <li itemscope itemtype='http://schema.org/SiteNavigationElement'>+ <span itemprop='name'>+ <a href='<%= path "/chat" %>' itemprop="url"><%= t "chat.crawler.nav_title" %> </a>+ </span>+ </li>+<% end %>diff --git a/plugins/chat/assets/stylesheets/common/chat-crawler.scss b/plugins/chat/assets/stylesheets/common/chat-crawler.scssnew file mode 100644index 00000000000..cd5ee35af56--- /dev/null+++ b/plugins/chat/assets/stylesheets/common/chat-crawler.scss@@ -0,0 +1,80 @@+.chat-crawler {+ &__channels {+ list-style: none;+ margin: 1.5em 0;+ padding: 0;+ }++ &__channel {+ padding-block: 1em;+ border-top: 1px solid var(--content-border-color);++ h2 {+ margin: 0;+ font-size: var(--font-up-1);+ overflow-wrap: anywhere;+ }++ p {+ margin: 0.25em 0 0;+ }+ }++ &__channel-meta {+ display: flex;+ flex-wrap: wrap;+ gap: 0.25em 1em;+ margin-top: 0.5em;+ font-size: var(--font-down-1);+ color: var(--primary-medium);+ }++ &__retention {+ font-size: var(--font-down-1);+ color: var(--primary-medium);+ }++ &__message {+ padding-block: 0.75em;+ border-top: 1px solid var(--content-border-color);+ }++ &__message-meta {+ display: flex;+ flex-wrap: wrap;+ align-items: baseline;+ gap: 0.25em 0.75em;+ margin-bottom: 0.25em;+ }++ &__message-author {+ font-weight: bold;+ overflow-wrap: anywhere;+ }++ &__message-time {+ font-size: var(--font-down-1);+ color: var(--primary-medium);+ }++ &__message-text {+ > :first-child {+ margin-top: 0;+ }++ > :last-child {+ margin-bottom: 0;+ }+ }++ &__reactions,+ &__thread-link {+ margin-top: 0.5em;+ }++ &__reactions {+ display: flex;+ flex-wrap: wrap;+ gap: 0.5em;+ }+}diff --git a/plugins/chat/assets/stylesheets/common/index.scss b/plugins/chat/assets/stylesheets/common/index.scssindex c089f892296..f5ef281c5ea 100644--- a/plugins/chat/assets/stylesheets/common/index.scss+++ b/plugins/chat/assets/stylesheets/common/index.scss@@ -80,3 +80,4 @@@import "edit-category-chat";@import "chat-pinned-messages";@import "chat-pinned-bar";+@import "chat-crawler";diff --git a/plugins/chat/config/locales/server.en.yml b/plugins/chat/config/locales/server.en.ymlindex 820b7c7b93a..3c40ed27bea 100644--- a/plugins/chat/config/locales/server.en.yml+++ b/plugins/chat/config/locales/server.en.yml@@ -4,8 +4,8 @@ en:chat_enabled: "Enable chat."chat_pinned_messages: "Enable pinned messages in chat channels."enable_public_channels: "Enable public channels based on categories."- chat_allowed_groups: "Users in these groups can chat, except anonymous users who can only view chat when added to this setting. Note that admins and moderators can always access chat."- chat_channel_retention_days: "Chat messages in regular channels will be retained for this many days. Set to '0' to retain messages forever."+ chat_allowed_groups: "Users in these groups can chat, except anonymous users who can only view chat when added to this setting. Adding anonymous users also lets search engines index public channels. Note that admins and moderators can always access chat."+ chat_channel_retention_days: "Chat messages in regular channels will be retained for this many days. Set to '0' to retain messages forever. When anonymous users can view chat, deleted messages also drop out of search engine results over time, so short retention limits how much chat history can be found through search."chat_dm_retention_days: "Chat messages in personal chat channels will be retained for this many days. Set to '0' to retain messages forever."chat_auto_silence_duration: "Number of minutes that users will be silenced for when they exceed the chat message creation rate limit. Set to '0' to disable auto-silencing."chat_allowed_messages_for_trust_level_0: "Number of messages that trust level 0 users is allowed to send in 30 seconds. Set to '0' to disable limit."@@ -59,6 +59,24 @@ en:chat:deleted_chat_username: deleted+ crawler:+ directory_title: "Chat channels"+ nav_title: "Chat"+ last_activity: "Last activity"+ retention_notice:+ one: "Messages older than %{count} day are deleted, except pinned messages."+ other: "Messages older than %{count} days are deleted, except pinned messages."+ messages_count:+ one: "%{count} message"+ other: "%{count} messages"+ members_count:+ one: "%{count} member"+ other: "%{count} members"+ thread_title: "Thread in %{channel}"+ channel_day_title: "%{channel} – %{date}"+ thread_replies:+ one: "%{count} reply"+ other: "%{count} replies"errors:users_cant_be_added_to_channel: "Users can't be added to this channel."user_cant_be_removed_from_channel: "User can't be removed from this channel"diff --git a/plugins/chat/config/routes.rb b/plugins/chat/config/routes.rbindex 470f32779e5..d00262c73fa 100644--- a/plugins/chat/config/routes.rb+++ b/plugins/chat/config/routes.rb@@ -93,7 +93,7 @@ Chat::Engine.routes.draw do}# chat_controller routes- get "/" => "chat#respond"+ get "/" => "chat#respond", :defaults => { crawler_directory: true }get "/disabled" => "chat#respond"get "/search" => "chat#respond"get "/new-message" => "chat#respond"@@ -101,7 +101,7 @@ Chat::Engine.routes.draw doget "/channels" => "chat#respond"get "/starred-channels" => "chat#respond"get "/threads" => "chat#respond"- get "/browse" => "chat#respond"+ get "/browse" => "chat#respond", :defaults => { crawler_directory: true }get "/browse/all" => "chat#respond"get "/browse/closed" => "chat#respond"get "/browse/open" => "chat#respond"diff --git a/plugins/chat/lib/chat/crawler_view.rb b/plugins/chat/lib/chat/crawler_view.rbnew file mode 100644index 00000000000..f715432bbd7--- /dev/null+++ b/plugins/chat/lib/chat/crawler_view.rb@@ -0,0 +1,227 @@+# frozen_string_literal: true++module Chat+ # Server-rendered, paginated view of a channel or thread for crawlers.+ # Channel pages are bucketed by UTC day so a page URL keeps the same messages until+ # retention purges them, rather than every page shifting as the oldest messages are deleted.+ class CrawlerView+ PAGE_SIZE = 50++ attr_reader :channel, :thread, :date, :page++ def initialize(channel:, thread: nil, date: nil, page: nil, target_message_id: nil)+ @channel = channel+ @thread = thread+ @page = [page.to_s.to_i, 1].max+ @valid = true++ @dated = target_message_id.present? || date.present?++ if target_message_id.present?+ locate_message(target_message_id)+ elsif !thread+ @date = date.present? ? parse_date(date) : latest_date+ end+ end++ def valid?+ return false if !@valid+ return true if page == 1 && (thread || date.nil?)++ page <= total_pages && page_scope.exists?+ end++ def messages+ @messages ||=+ page_scope+ .includes(+ :user,+ :revisions,+ :reactions,+ :chat_channel,+ thread: [:channel, { original_message: :user }],+ )+ .order(:created_at, :id)+ .offset((page - 1) * PAGE_SIZE)+ .limit(PAGE_SIZE)+ .to_a+ end++ def total_pages+ @total_pages ||= pages_in(page_scope)+ end++ def title+ if thread+ thread_title+ elsif date+ I18n.t("chat.crawler.channel_day_title", channel: channel.title, date: formatted_date)+ else+ channel.title+ end+ end++ def description+ text = thread ? original_message_excerpt : channel.description+ text.presence || channel.chatable.description_text+ end++ def url+ thread ? thread.url : channel.url+ end++ def canonical_url+ page_url(date, page)+ end++ # When retention will have purged everything on this page, so search engines can drop it on time.+ # Pinned messages are never purged, and they keep the first pages of their day alive.+ def unavailable_after+ retention_days = SiteSetting.chat_channel_retention_days+ return if thread || !@dated || date.nil? || retention_days <= 0++ pinned_count = page_scope.where(id: Chat::PinnedMessage.select(:chat_message_id)).count+ return if pinned_count > (page - 1) * PAGE_SIZE++ day_range(date).end + retention_days.days+ end++ def formatted_date+ I18n.l(date, format: :date_only) if date+ end++ def prev_page_url+ return page_url(date, page - 1) if page > 1+ return if thread || date.nil?++ previous = adjacent_date(:before)+ page_url(previous, pages_in(scope.where(created_at: day_range(previous)))) if previous+ end++ def next_page_url+ return page_url(date, page + 1) if page < total_pages+ return if thread || date.nil?++ following = adjacent_date(:after)+ page_url(following, 1) if following+ end++ def original_message+ thread&.original_message+ end++ def posting_url(message)+ if message.thread&.original_message_id == message.id+ message.thread.url+ else+ "#{Discourse.base_url_no_prefix}#{message.url}"+ end+ end++ # Like the reactions plugin does for posts, each person who reacted counts as one like.+ def like_count(message)+ message.reactions.map(&:user_id).uniq.size+ end++ def reaction_counts(message)+ message.reactions.sort_by(&:created_at).group_by(&:emoji).transform_values(&:size)+ end++ def show_thread_link?(message)+ message_thread = message.thread+ !thread && message_thread&.original_message_id == message.id &&+ (channel.threading_enabled? || message_thread.force) && message_thread.replies_count > 0+ end++ private++ def scope+ @scope ||=+ if thread+ Chat::Message.where(chat_channel_id: channel.id, thread_id: thread.id)+ elsif channel.threading_enabled?+ Chat::Message.where(chat_channel_id: channel.id).where(<<~SQL, channel_id: channel.id)+ chat_messages.thread_id IS NULL OR chat_messages.id IN (+ SELECT original_message_id+ FROM chat_threads+ WHERE chat_threads.channel_id = :channel_id+ )+ SQL+ else+ # Mirrors the channel UI, which only hides replies of forced threads when threading is off.+ Chat::Message+ .where(chat_channel_id: channel.id)+ .left_joins(:thread)+ .where(+ "chat_threads.id IS NULL OR chat_threads.force = false OR chat_messages.id = chat_threads.original_message_id",+ )+ end+ end++ def page_scope+ date ? scope.where(created_at: day_range(date)) : scope+ end++ def pages_in(relation)+ [(relation.count.to_f / PAGE_SIZE).ceil, 1].max+ end++ def day_range(day)+ start = Time.utc(day.year, day.month, day.day)+ start...(start + 1.day)+ end++ def latest_date+ scope.maximum(:created_at)&.utc&.to_date+ end++ def adjacent_date(direction)+ range = day_range(date)+ time =+ if direction == :before+ scope.where("chat_messages.created_at < ?", range.begin).maximum(:created_at)+ else+ scope.where("chat_messages.created_at >= ?", range.end).minimum(:created_at)+ end+ time&.utc&.to_date+ end++ def parse_date(value)+ Date.iso8601(value.to_s)+ rescue ArgumentError+ @valid = false+ nil+ end++ def locate_message(message_id)+ message = scope.find_by(id: message_id)+ return @valid = false if !message++ @date = message.created_at.utc.to_date if !thread+ preceding =+ page_scope.where(+ "(chat_messages.created_at, chat_messages.id) < (?, ?)",+ message.created_at,+ message.id,+ ).count+ @page = (preceding / PAGE_SIZE) + 1+ end++ def page_url(day, number)+ params = {}+ params[:date] = day.iso8601 if day+ params[:page] = number if number > 1+ params.empty? ? url : "#{url}?#{params.to_query}"+ end++ def original_message_excerpt+ excerpt = original_message&.excerpt_for_display+ CGI.unescapeHTML(excerpt) if excerpt.present?+ end++ def thread_title+ thread.title.presence || original_message_excerpt.presence ||+ I18n.t("chat.crawler.thread_title", channel: channel.title)+ end+ end+enddiff --git a/plugins/chat/spec/requests/chat_controller_spec.rb b/plugins/chat/spec/requests/chat_controller_spec.rbindex 12c35bf1add..809bf39f160 100644--- a/plugins/chat/spec/requests/chat_controller_spec.rb+++ b/plugins/chat/spec/requests/chat_controller_spec.rb@@ -489,4 +489,353 @@ RSpec.describe Chat::ChatController doEXPECTEDendend++ describe "#respond" do+ let(:crawler_headers) { { "HTTP_USER_AGENT" => "Googlebot" } }+ let(:day) { message_0.created_at.utc.to_date.iso8601 }++ before do+ SiteSetting.chat_allowed_groups =+ "#{Group::AUTO_GROUPS[:everyone]}|#{Group::AUTO_GROUPS[:anonymous_users]}"+ end++ it "renders the latest day of channel messages for crawlers" do+ get chat_channel.relative_url, headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.body).to include(message_0.cooked, message_49.cooked)+ expect(response.body).not_to include(message_50.cooked)+ expect(response.body).to include(+ %(<link rel="canonical" href="#{chat_channel.url}?date=#{day}" />),+ )+ expect(response.body).to include(+ %(<link rel="next" href="#{chat_channel.url}?date=#{day}&page=2">),+ )++ postings =+ Nokogiri.HTML5(response.body).css('[itemtype="http://schema.org/DiscussionForumPosting"]')+ expect(postings.size).to eq(50)+ expect(postings.first.at_css('link[itemprop="url"]')["href"]).to eq(+ "#{Discourse.base_url}#{message_0.url}",+ )+ expect(postings.first.at_css('[itemprop="author"] [itemprop="name"]').text).to eq(+ other_user.username,+ )+ expect(postings.first.at_css('[itemprop="text"]').inner_html.strip).to eq(message_0.cooked)+ end++ it "explains message retention when old messages are deleted" do+ SiteSetting.chat_channel_retention_days = 30+ get chat_channel.relative_url, headers: crawler_headers++ expect(response.body).to include(I18n.t("chat.crawler.retention_notice", count: 30))++ SiteSetting.chat_channel_retention_days = 0+ get chat_channel.relative_url, headers: crawler_headers++ expect(response.body).not_to include("chat-crawler__retention")+ end++ it "tells search engines when retention will remove a day page" do+ SiteSetting.chat_channel_retention_days = 30+ expected_expiry = (message_0.created_at.utc.to_date + 31).to_time(:utc).iso8601+ robots_meta = %(<meta name="robots" content="unavailable_after: #{expected_expiry}">)++ get "#{chat_channel.relative_url}?date=#{day}&page=2", headers: crawler_headers+ expect(response.body).to include(robots_meta)++ get chat_channel.relative_url, headers: crawler_headers+ expect(response.body).not_to include("unavailable_after")++ Fabricate(:chat_pinned_message, chat_message: message_0)+ get "#{chat_channel.relative_url}?date=#{day}", headers: crawler_headers+ expect(response.body).not_to include("unavailable_after")++ get "#{chat_channel.relative_url}?date=#{day}&page=2", headers: crawler_headers+ expect(response.body).to include(robots_meta)+ end++ it "counts each user who reacted to a message as one like" do+ Fabricate(:chat_message_reaction, chat_message: message_0, user:, emoji: "+1")+ Fabricate(:chat_message_reaction, chat_message: message_0, user:, emoji: "heart")+ Fabricate(:chat_message_reaction, chat_message: message_0, user: admin, emoji: "+1")++ get chat_channel.relative_url, headers: crawler_headers++ posting = Nokogiri.HTML5(response.body).at_css("#chat-message-#{message_0.id}")+ expect(+ posting.at_css('[itemprop="interactionStatistic"] meta[itemprop="userInteractionCount"]')[+ "content"+ ],+ ).to eq("2")+ expect(posting.css(".chat-crawler__reaction").map { |reaction| reaction.text.squish }).to eq(+ ["#{Emoji.lookup_unicode("+1")} 2", "#{Emoji.lookup_unicode("heart")} 1"],+ )+ end++ it "renders later pages of a day and 404s past the last one" do+ get "#{chat_channel.relative_url}?date=#{day}&page=2", headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.body).to include(message_50.cooked, message_69.cooked)+ expect(response.body).not_to include(message_49.cooked)+ expect(response.body).to include(%(<link rel="prev" href="#{chat_channel.url}?date=#{day}">))++ get "#{chat_channel.relative_url}?date=#{day}&page=3", headers: crawler_headers++ expect(response.status).to eq(404)+ end++ it "links between days that have messages" do+ old_message =+ Fabricate(:chat_message, chat_channel:, user: other_user, created_at: 3.days.ago)+ old_day = old_message.created_at.utc.to_date++ get "#{chat_channel.relative_url}?date=#{old_day.iso8601}", headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.body).to include(old_message.cooked)+ expect(response.body).not_to include(message_0.cooked)+ expect(response.body).to include(I18n.l(old_day, format: :date_only))+ expect(response.body).to include(%(<link rel="next" href="#{chat_channel.url}?date=#{day}">))++ get chat_channel.relative_url, headers: crawler_headers++ expect(response.body).not_to include(old_message.cooked)+ expect(response.body).to include(+ %(<link rel="prev" href="#{chat_channel.url}?date=#{old_day.iso8601}">),+ )+ end++ it "404s for invalid or empty days and missing messages" do+ get "#{chat_channel.relative_url}?date=not-a-date", headers: crawler_headers+ expect(response.status).to eq(404)++ get "#{chat_channel.relative_url}?date=2001-01-01", headers: crawler_headers+ expect(response.status).to eq(404)++ message_1.trash!+ get "#{chat_channel.relative_url}/#{message_1.id}", headers: crawler_headers+ expect(response.status).to eq(404)+ end++ it "renders the page containing a linked message with that page as canonical" do+ get "#{chat_channel.relative_url}/#{message_60.id}", headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.body).to include(message_60.cooked)+ expect(response.body).to include(+ %(<link rel="canonical" href="#{chat_channel.url}?date=#{day}&page=2" />),+ )+ end++ it "links thread replies from the channel and renders them on the thread page" do+ chat_channel.update!(threading_enabled: true)+ thread = Fabricate(:chat_thread, channel: chat_channel, original_message: message_0)+ reply = Fabricate(:chat_message, chat_channel: chat_channel, thread:, message: "a reply")+ thread.update!(replies_count: 1)++ get chat_channel.relative_url, headers: crawler_headers++ expect(response.body).to include(%(href="#{thread.url}"))+ expect(response.body).not_to include(reply.cooked)++ channel_posting = Nokogiri.HTML5(response.body).at_css("#chat-message-#{message_0.id}")+ expect(channel_posting.at_css('link[itemprop="url"]')["href"]).to eq(thread.url)+ expect(channel_posting.at_css('meta[itemprop="commentCount"]')["content"]).to eq("1")++ get thread.relative_url, headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.body).to include(message_0.cooked, reply.cooked)+ expect(response.body).not_to include(message_1.cooked)++ thread_posting =+ Nokogiri.HTML5(response.body).css('[itemtype="http://schema.org/DiscussionForumPosting"]')+ expect(thread_posting.size).to eq(1)+ expect(thread_posting.first.at_css('link[itemprop="url"]')["href"]).to eq(thread.url)+ expect(+ thread_posting.first.at_css('[itemprop="author"] meta[itemprop="name"]')["content"],+ ).to eq(other_user.username)+ expect(+ thread_posting.first.css('[itemprop="comment"]').map { |comment| comment["id"] },+ ).to eq(["chat-message-#{reply.id}"])+ end++ it "redirects channel permalinks of thread replies to the thread" do+ chat_channel.update!(threading_enabled: true)+ thread = Fabricate(:chat_thread, channel: chat_channel, original_message: message_0)+ reply = Fabricate(:chat_message, chat_channel: chat_channel, thread:)++ get "#{chat_channel.relative_url}/#{reply.id}", headers: crawler_headers++ expect(response).to redirect_to(reply.url)+ expect(response.status).to eq(301)++ get reply.url, headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.body).to include(reply.cooked)+ end++ it "links forced threads when threading is off and 404s other threads" do+ forced_thread =+ Fabricate(:chat_thread, channel: chat_channel, original_message: message_0, force: true)+ Fabricate(:chat_message, chat_channel: chat_channel, thread: forced_thread)+ forced_thread.update!(replies_count: 1)+ thread = Fabricate(:chat_thread, channel: chat_channel, original_message: message_1)++ get chat_channel.relative_url, headers: crawler_headers++ expect(response.body).to include(%(href="#{forced_thread.url}"))++ get forced_thread.relative_url, headers: crawler_headers+ expect(response.status).to eq(200)++ get thread.relative_url, headers: crawler_headers+ expect(response.status).to eq(404)+ end++ it "handles malformed pagination params without erroring" do+ get "#{chat_channel.relative_url}?date=#{"1" * 200}", headers: crawler_headers+ expect(response.status).to eq(404)++ get "#{chat_channel.relative_url}?page[]=2", headers: crawler_headers+ expect(response.status).to eq(200)+ expect(response.body).to include(message_0.cooked)+ end++ it "renders the app shell for signed-in users on the crawler layout" do+ restricted_channel =+ Fabricate(+ :category_channel,+ chatable: Fabricate(:private_category, group: Fabricate(:group)),+ )+ restricted_message = Fabricate(:chat_message, chat_channel: restricted_channel)+ sign_in(admin)++ get "#{restricted_channel.relative_url}?print=true"++ expect(response.status).to eq(200)+ expect(response.body).not_to include(restricted_message.cooked)+ end++ it "404s for channels anonymous users cannot see" do+ restricted_channel =+ Fabricate(+ :category_channel,+ chatable: Fabricate(:private_category, group: Fabricate(:group)),+ )+ restricted_message = Fabricate(:chat_message, chat_channel: restricted_channel)+ dm_message = Fabricate(:chat_message, chat_channel: dm_chat_channel, user:)++ get restricted_channel.relative_url, headers: crawler_headers++ expect(response.status).to eq(404)+ expect(response.body).not_to include(restricted_message.cooked)++ get dm_chat_channel.relative_url, headers: crawler_headers++ expect(response.status).to eq(404)+ expect(response.body).not_to include(dm_message.cooked)+ end++ it "404s channels and renders an empty index when anonymous users are not allowed to chat" do+ SiteSetting.chat_allowed_groups = Group::AUTO_GROUPS[:everyone]++ get chat_channel.relative_url, headers: crawler_headers++ expect(response.status).to eq(404)+ expect(response.body).not_to include(message_0.cooked)++ get "/chat", headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.body).not_to include(chat_channel.url)+ end++ it "renders no messages for regular browsers" do+ get chat_channel.relative_url++ expect(response.status).to eq(200)+ expect(response.body).not_to include(message_0.cooked)+ end++ it "lists public channels in the directory for crawlers" do+ restricted_channel =+ Fabricate(+ :category_channel,+ name: "Secret plans",+ chatable: Fabricate(:private_category, group: Fabricate(:group)),+ )++ chat_channel.update!(last_message: message_69, messages_count: 70)++ get "/chat/browse", headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.body).to include(+ %(<link rel="canonical" href="#{Discourse.base_url}/chat" />),+ )++ listed_channels =+ Nokogiri.HTML5(response.body).css(+ '[itemtype="http://schema.org/ItemList"] [itemprop="itemListElement"]',+ )+ expect(listed_channels.map { |item| item.at_css('[itemprop="url"]')["href"] }).to eq(+ [chat_channel.url],+ )+ expect(listed_channels.first.to_html).to include(+ category.name,+ I18n.t("chat.crawler.messages_count", count: 70),+ message_69.created_at.to_formatted_s(:iso8601),+ )+ expect(response.body).not_to include(restricted_channel.url)+ end++ it "paginates the directory" do+ stub_const(Chat::ChatController, :DIRECTORY_PAGE_SIZE, 1) do+ other_channel =+ Fabricate(:category_channel, name: "Zebra talk", chatable: Fabricate(:category))+ chat_channel.update!(name: "Aardvark talk")++ get "/chat", headers: crawler_headers++ expect(response.body).to include(chat_channel.url)+ expect(response.body).not_to include(other_channel.url)+ expect(response.body).to include(+ %(<link rel="next" href="#{Discourse.base_url}/chat?page=2">),+ )++ get "/chat?page=2", headers: crawler_headers++ expect(response.body).to include(other_channel.url)+ expect(response.body).to include(%(<link rel="prev" href="#{Discourse.base_url}/chat">))+ expect(response.body).to include(+ %(<link rel="canonical" href="#{Discourse.base_url}/chat?page=2" />),+ )++ get "/chat?page=3", headers: crawler_headers+ expect(response.status).to eq(404)+ end+ end+ end++ describe "crawler footer" do+ before do+ SiteSetting.chat_allowed_groups =+ "#{Group::AUTO_GROUPS[:everyone]}|#{Group::AUTO_GROUPS[:anonymous_users]}"+ end++ it "links to chat when anonymous users can view chat" do+ get "/latest", headers: { "HTTP_USER_AGENT" => "Googlebot" }++ expect(response.body).to include(%(href='/chat' itemprop="url"))++ SiteSetting.chat_allowed_groups = Group::AUTO_GROUPS[:everyone]+ get "/latest", headers: { "HTTP_USER_AGENT" => "Googlebot" }++ expect(response.body).not_to include(%(href='/chat' itemprop="url"))+ end+ endend