Zulip server and web application. Open-source team chat that helps teams stay productive and focused.
message_fetch_raw_content: Fetch raw content without UserMessage rows.
Apoorva Pendsefecf7f421be1
diff --git a/web/src/compose_reply.ts b/web/src/compose_reply.tsindex 0162fed05d3..549e185b5a0 100644--- a/web/src/compose_reply.ts+++ b/web/src/compose_reply.ts@@ -692,8 +692,8 @@ export function build_and_process_quote_assets_for_messages(message_ids,on_success(raw_content_arr) {for (const [i, message] of messages.entries()) {- const raw_content = raw_content_arr[i];- assert(raw_content !== undefined);+ const raw_content =+ raw_content_arr[i] ?? compose_paste.paste_handler_converter(message.content);quote_assets.push({message, quote_content: raw_content});}callback(quote_assets);diff --git a/web/src/message_fetch_raw_content.ts b/web/src/message_fetch_raw_content.tsindex 2aa046a9b35..4092dd8c16d 100644--- a/web/src/message_fetch_raw_content.ts+++ b/web/src/message_fetch_raw_content.ts@@ -1,23 +1,77 @@import assert from "minimalistic-assert";+import * as z from "zod/mini";import * as channel from "./channel.ts";import * as message_fetch from "./message_fetch.ts";import * as message_store from "./message_store.ts";+import type {Message} from "./message_store.ts";+import * as narrow_state from "./narrow_state.ts";++// NOTE: Keep this and is_history_enabling_term up to date with+// all the future terms that do include historical messages, like the upcoming+// `channels:all` from #33534 and the eventually possible `channels:subscribed`+// #13052.+const history_enabling_narrow_term_schema = z.object({+ operator: z.string(),+ operand: z.union([z.number(), z.string(), z.array(z.number())]),+ negated: z.optional(z.boolean()),+});++function is_history_enabling_term(+ term: z.infer<typeof history_enabling_narrow_term_schema>,+): boolean {+ if (term.negated) {+ return false;+ }+ if (term.operator === "channel" || term.operator === "stream") {+ return true;+ }+ return (+ term.operator === "channels" && (term.operand === "public" || term.operand === "web-public")+ );+}++// Returns a JSON narrow for GET /messages that enables shared history+// (channel/stream, or channels:public|web-public). Returns undefined+// when no such terms are present so the caller can omit the parameter.+function get_history_enabling_narrow_for_raw_content_fetch(): string | undefined {+ const filter = narrow_state.filter();+ if (filter === undefined) {+ return undefined;+ }++ const encoded = message_fetch.get_narrow_for_message_fetch(filter);+ if (encoded === "") {+ return undefined;+ }++ const history_terms = z+ .array(history_enabling_narrow_term_schema)+ .parse(JSON.parse(encoded))+ .filter((term) => is_history_enabling_term(term));++ if (history_terms.length === 0) {+ return undefined;+ }+ return JSON.stringify(history_terms);+}export function get_raw_content_for_messages(info: {message_ids: number[];- on_success: (raw_content_arr: string[]) => void;+ on_success: (raw_content_arr: (string | undefined)[]) => void;on_error: () => void;timeout_ms?: number | undefined;}): void {const {message_ids, on_success, on_error, timeout_ms} = info;const message_ids_that_require_fetching: number[] = [];- const raw_content_arr: string[] = Array.from({length: message_ids.length});+ const raw_content_arr: (string | undefined)[] = Array.from({length: message_ids.length});+ const messages: Message[] = [];- // We fill what we can from the store+ // We fill what we can from message_store.for (const [i, id] of message_ids.entries()) {const message = message_store.get(id);assert(message !== undefined);+ messages.push(message);if (message.raw_content) {raw_content_arr[i] = message.raw_content;} else {@@ -30,12 +84,18 @@ export function get_raw_content_for_messages(info: {return;}+ // GET /messages with message_ids uses personal history unless a+ // history-enabling narrow is passed+ // (https://zulip.com/api/get-messages#parameter-narrow).+ const narrow = get_history_enabling_narrow_for_raw_content_fetch();+channel.get({url: "/json/messages",data: {allow_empty_topic_name: true,apply_markdown: false,message_ids: JSON.stringify(message_ids_that_require_fetching),+ ...(narrow !== undefined && {narrow}),},success(raw_data) {const data = message_fetch.message_ids_response_schema.parse(raw_data);@@ -47,9 +107,22 @@ export function get_raw_content_for_messages(info: {fetched_raw_content_map.set(raw_message.id, parsed_message.content);}- // Fill the remaining holes in the final array+ // We fill remaining holes from the fetch first, then from any+ // raw_content cached on the message mid-flight. Anything still+ // missing stays unset for the caller.for (const [i, id] of message_ids.entries()) {- raw_content_arr[i] ??= fetched_raw_content_map.get(id)!;+ if (raw_content_arr[i] !== undefined) {+ continue;+ }+ const fetched = fetched_raw_content_map.get(id);+ if (fetched !== undefined) {+ raw_content_arr[i] = fetched;+ continue;+ }+ const message = messages[i]!;+ if (message.raw_content !== undefined) {+ raw_content_arr[i] = message.raw_content;+ }}on_success(raw_content_arr);diff --git a/web/tests/compose_reply.test.cjs b/web/tests/compose_reply.test.cjsindex 4e156667b83..cfc27b5ee9f 100644--- a/web/tests/compose_reply.test.cjs+++ b/web/tests/compose_reply.test.cjs@@ -389,6 +389,88 @@ run_test("build_and_process_quote_assets_for_messages", ({override}) => {{message: msg_unhydrated, quote_content: "converted_by_turndown: <p>unhydrated</p>"},"Fallback to using paste_handler_converter",);++ // Case: Partial success. Undefined slots fall back to HTML→markdown.+ override(+ compose_paste,+ "paste_handler_converter",+ (content) => `converted_by_turndown: ${content}`,+ );++ const msg_from_message_store = {+ id: 3,+ raw_content: "From message_store",+ content: "<p>message_store only</p>",+ };+ const msg_html_only = {id: 4, content: "<p>html only body</p>"};+ add_messages_to_message_store([+ msg_hydrated,+ msg_unhydrated,+ msg_from_message_store,+ msg_html_only,+ ]);++ override(+ message_fetch_raw_content,+ "get_raw_content_for_messages",+ ({_message_ids, on_success, _on_error}) => {+ // Explicit undefined for ids the batch did not return.+ on_success(["batch raw", undefined, undefined, undefined]);+ },+ );++ compose_reply.build_and_process_quote_assets_for_messages([1, 2, 3, 4], (assets) => {+ result_assets = assets;+ });++ assert.equal(result_assets.length, 4);+ assert.deepEqual(result_assets[0], {+ message: msg_hydrated,+ quote_content: "batch raw",+ });+ assert.deepEqual(+ result_assets[1],+ {+ message: msg_unhydrated,+ quote_content: "converted_by_turndown: <p>unhydrated</p>",+ },+ "Undefined batch slot uses HTML conversion.",+ );+ assert.deepEqual(+ result_assets[2],+ {+ message: msg_from_message_store,+ quote_content: "converted_by_turndown: <p>message_store only</p>",+ },+ "Undefined batch slot ignores message_store raw_content on success path.",+ );+ assert.deepEqual(+ result_assets[3],+ {+ message: msg_html_only,+ quote_content: "converted_by_turndown: <p>html only body</p>",+ },+ "Undefined batch slot with neither uses HTML conversion.",+ );++ // Case: All slots are undefined. Success path uses HTML conversion only.+ override(+ message_fetch_raw_content,+ "get_raw_content_for_messages",+ ({_message_ids, on_success, _on_error}) => {+ on_success([undefined, undefined]);+ },+ );+ compose_reply.build_and_process_quote_assets_for_messages([3, 4], (assets) => {+ result_assets = assets;+ });+ assert.deepEqual(+ result_assets.map((a) => a.quote_content),+ [+ "converted_by_turndown: <p>message_store only</p>",+ "converted_by_turndown: <p>html only body</p>",+ ],+ );});run_test("get_quote_menu_selection", ({override_rewire}) => {diff --git a/web/tests/message_fetch_raw_content.test.cjs b/web/tests/message_fetch_raw_content.test.cjsindex d484297611c..54600bbd381 100644--- a/web/tests/message_fetch_raw_content.test.cjs+++ b/web/tests/message_fetch_raw_content.test.cjs@@ -2,14 +2,17 @@const assert = require("node:assert/strict");+const {make_user} = require("./lib/example_user.cjs");const {zrequire, mock_esm} = require("./lib/namespace.cjs");const {run_test} = require("./lib/test.cjs");const stream_data = zrequire("stream_data");const message_store = zrequire("message_store");-const message_fetch_raw_content = zrequire("message_fetch_raw_content");+const {Filter} = zrequire("filter");const message_fetch = mock_esm("../src/message_fetch");+const narrow_state = mock_esm("../src/narrow_state");const channel = mock_esm("../src/channel");+const message_fetch_raw_content = zrequire("message_fetch_raw_content");function add_messages_to_message_store(messages) {message_store.clear_for_testing();@@ -18,7 +21,7 @@ function add_messages_to_message_store(messages) {}}-// We only rely on message_fetch for type validation.+// We only rely on message_fetch for type validation and narrow encoding.message_fetch.message_ids_response_schema = {parse: (data) => data,};@@ -75,10 +78,23 @@ run_test("get_raw_content_for_messages", ({override}) => {assert.ok(success_called, "Should call on_success immediately if all messages are hydrated");- // Case: Fetching missing raw_content successfully+ // Case: Fetching missing raw_content successfully. Only+ // history-enabling channel terms are passed for include_history.let channel_get_args;success_called = false;+ const fake_filter = {};+ const full_encoded_narrow = JSON.stringify([+ {operator: "channel", operand: social.stream_id},+ {operator: "topic", operand: "some-topic"},+ {operator: "is", operand: "unread"},+ ]);+ const history_only_narrow = JSON.stringify([{operator: "channel", operand: social.stream_id}]);+ override(narrow_state, "filter", () => fake_filter);+ override(message_fetch, "get_narrow_for_message_fetch", (filter) => {+ assert.equal(filter, fake_filter);+ return full_encoded_narrow;+ });override(channel, "get", (args) => {channel_get_args = args;args.success({@@ -102,12 +118,310 @@ run_test("get_raw_content_for_messages", ({override}) => {JSON.stringify([2]),"Should only request hydration for messages missing raw_content",);+ assert.equal(+ channel_get_args.data.narrow,+ history_only_narrow,+ "Should strip topic/is: and keep only channel for shared history",+ );// It is safe to update raw_content for messages from channels// the user is subscribed to.assert.equal(msg_2.raw_content, "Fetched markdown content");assert.ok(success_called, "Should call on_success after successfully hydrating");assert.deepEqual(success_call_args, [msg_1.raw_content, msg_2.raw_content]);+ // Case: channels:public is kept as history-enabling. Other terms are stripped.+ success_called = false;+ delete msg_2.raw_content;+ override(narrow_state, "filter", () => fake_filter);+ override(message_fetch, "get_narrow_for_message_fetch", () =>+ JSON.stringify([+ {operator: "channels", operand: "public"},+ {operator: "search", operand: "keyword"},+ ]),+ );+ override(channel, "get", (args) => {+ channel_get_args = args;+ args.success({+ messages: [+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},+ ],+ });+ });++ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [1, 2],+ on_success() {+ success_called = true;+ },+ });++ assert.equal(+ channel_get_args.data.narrow,+ JSON.stringify([{operator: "channels", operand: "public"}]),+ );+ assert.ok(success_called);++ // Case: channels:web-public is kept as history-enabling.+ success_called = false;+ delete msg_2.raw_content;+ override(narrow_state, "filter", () => fake_filter);+ override(message_fetch, "get_narrow_for_message_fetch", () =>+ JSON.stringify([{operator: "channels", operand: "web-public"}]),+ );+ override(channel, "get", (args) => {+ channel_get_args = args;+ args.success({+ messages: [+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},+ ],+ });+ });++ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [1, 2],+ on_success() {+ success_called = true;+ },+ });++ assert.equal(+ channel_get_args.data.narrow,+ JSON.stringify([{operator: "channels", operand: "web-public"}]),+ );+ assert.ok(success_called);++ // Case: For an is: operator, no history terms are present, so we omit narrow.+ success_called = false;+ delete msg_2.raw_content;+ override(narrow_state, "filter", () => fake_filter);+ override(message_fetch, "get_narrow_for_message_fetch", () =>+ JSON.stringify([{operator: "is", operand: "starred"}]),+ );+ override(channel, "get", (args) => {+ channel_get_args = args;+ args.success({+ messages: [+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},+ ],+ });+ });++ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [1, 2],+ on_success() {+ success_called = true;+ },+ });++ assert.equal(channel_get_args.data.narrow, undefined);+ assert.ok(success_called);++ // Case: Negated channel terms are not history-enabling and are dropped.+ success_called = false;+ delete msg_2.raw_content;+ override(narrow_state, "filter", () => fake_filter);+ override(message_fetch, "get_narrow_for_message_fetch", () =>+ JSON.stringify([+ {operator: "channel", operand: social.stream_id, negated: true},+ {operator: "channel", operand: denmark.stream_id},+ ]),+ );+ override(channel, "get", (args) => {+ channel_get_args = args;+ args.success({+ messages: [+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},+ ],+ });+ });++ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [1, 2],+ on_success() {+ success_called = true;+ },+ });++ assert.equal(+ channel_get_args.data.narrow,+ JSON.stringify([{operator: "channel", operand: denmark.stream_id}]),+ );+ assert.ok(success_called);++ // Case: The encoded narrow is empty, so omit the narrow parameter.+ success_called = false;+ delete msg_2.raw_content;+ override(narrow_state, "filter", () => fake_filter);+ override(message_fetch, "get_narrow_for_message_fetch", () => "");+ override(channel, "get", (args) => {+ channel_get_args = args;+ args.success({+ messages: [+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},+ ],+ });+ });++ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [1, 2],+ on_success(args) {+ success_called = true;+ success_call_args = args;+ },+ });++ assert.equal(channel_get_args.data.narrow, undefined);+ assert.ok(success_called);++ // Case: There is no current filter (e.g. recent conversations), so no narrow param.+ success_called = false;+ delete msg_2.raw_content;+ override(narrow_state, "filter", () => undefined);+ override(channel, "get", (args) => {+ channel_get_args = args;+ args.success({+ messages: [+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},+ ],+ });+ });++ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [1, 2],+ on_success(args) {+ success_called = true;+ success_call_args = args;+ },+ });++ assert.equal(channel_get_args.data.narrow, undefined);+ assert.ok(success_called);++ // Case: A partial batch response leaves explicit undefined for missing ids.+ success_called = false;+ delete msg_2.raw_content;+ override(narrow_state, "filter", () => undefined);+ override(channel, "get", (args) => {+ channel_get_args = args;+ args.success({+ messages: [],+ });+ });++ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [1, 2],+ on_success(args) {+ success_called = true;+ success_call_args = args;+ },+ });++ assert.ok(success_called);+ assert.equal(success_call_args.length, 2);+ assert.equal(success_call_args[0], msg_1.raw_content);+ // The slot is present as undefined.+ assert.equal(success_call_args[1], undefined);++ // Case: A partial batch where message_store gains raw_content during the request.+ success_called = false;+ delete msg_2.raw_content;+ override(narrow_state, "filter", () => undefined);+ override(channel, "get", (args) => {+ // Simulate another path hydrating message_store before success runs.+ msg_2.raw_content = "Hydrated in message_store mid-flight";+ args.success({messages: []});+ });++ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [1, 2],+ on_success(args) {+ success_called = true;+ success_call_args = args;+ },+ });++ assert.ok(success_called);+ assert.deepEqual(success_call_args, [+ msg_1.raw_content,+ "Hydrated in message_store mid-flight",+ ]);++ // Case: Mixed partial. One id is returned and one is missing (undefined slot).+ success_called = false;+ delete msg_2.raw_content;+ delete msg_3.raw_content;+ override(narrow_state, "filter", () => undefined);+ override(channel, "get", (args) => {+ args.success({+ messages: [{id: 2, content_type: "text/x-markdown", content: "only two"}],+ });+ });++ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [2, 3],+ on_success(args) {+ success_called = true;+ success_call_args = args;+ },+ });++ assert.ok(success_called);+ assert.equal(success_call_args.length, 2);+ assert.equal(success_call_args[0], "only two");+ assert.equal(success_call_args[1], undefined);++ // Case: The server returns messages out of request order. The result+ // still matches message_ids order, with undefined for the missing middle id.+ success_called = false;+ delete msg_1.raw_content;+ delete msg_2.raw_content;+ delete msg_3.raw_content;+ // msg_1 is Denmark (subscribed); msg_3 is social (unsubscribed).+ add_messages_to_message_store([+ {+ id: 1,+ content: "<p>one html</p>",+ type: "stream",+ stream_id: denmark.stream_id,+ },+ {+ id: 2,+ content: "<p>two html</p>",+ type: "stream",+ stream_id: denmark.stream_id,+ },+ {+ id: 3,+ content: "<p>three html</p>",+ type: "stream",+ stream_id: social.stream_id,+ },+ ]);+ override(narrow_state, "filter", () => undefined);+ override(channel, "get", (args) => {+ assert.equal(args.data.message_ids, JSON.stringify([1, 2, 3]));+ // Deliberately reverse order and omit id 2.+ args.success({+ messages: [+ {id: 3, content_type: "text/x-markdown", content: "raw three"},+ {id: 1, content_type: "text/x-markdown", content: "raw one"},+ ],+ });+ });++ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [1, 2, 3],+ on_success(args) {+ success_called = true;+ success_call_args = args;+ },+ });++ assert.ok(success_called);+ assert.deepEqual(success_call_args, ["raw one", undefined, "raw three"]);+ assert.equal(success_call_args.length, 3);+ assert.equal(success_call_args[1], undefined);+// Case: Network error during hydrationsuccess_called = false;let error_called = false;@@ -131,6 +445,91 @@ run_test("get_raw_content_for_messages", ({override}) => {assert.ok(error_called, "Should call on_error if the network request fails");});+run_test("get_raw_content_for_messages encodes real Filter narrows", ({override}) => {+ const dm_user_a = make_user();+ const dm_user_b = make_user();+ const msg = {+ id: 2,+ content: "<p>HTML content</p>",+ type: "stream",+ stream_id: social.stream_id,+ };+ add_messages_to_message_store([msg]);++ let channel_get_args;+ override(channel, "get", (args) => {+ channel_get_args = args;+ args.success({+ messages: [+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},+ ],+ });+ });+ override(message_fetch, "get_narrow_for_message_fetch", (filter) => {+ const terms = filter.public_terms().map((term) => {+ if (term.operator === "channel" && typeof term.operand === "string") {+ const stream = stream_data.get_sub_by_id_string(term.operand);+ if (stream !== undefined) {+ return {...term, operand: stream.stream_id};+ }+ }+ return term;+ });+ return JSON.stringify(terms);+ });++ function fetch_narrow_for_filter(filter) {+ delete msg.raw_content;+ override(narrow_state, "filter", () => filter);+ let success_called = false;+ message_fetch_raw_content.get_raw_content_for_messages({+ message_ids: [2],+ on_success() {+ success_called = true;+ },+ });+ assert.ok(success_called);+ return channel_get_args.data.narrow;+ }++ // Channel + topic + is:unread keeps only channel; operand is coerced to int.+ assert.equal(+ fetch_narrow_for_filter(+ new Filter([+ {operator: "channel", operand: social.stream_id.toString()},+ {operator: "topic", operand: "some-topic"},+ {operator: "is", operand: "unread"},+ ]),+ ),+ JSON.stringify([{operator: "channel", operand: social.stream_id, negated: false}]),+ );++ // channels:public + search keeps only channels:public.+ assert.equal(+ fetch_narrow_for_filter(+ new Filter([+ {operator: "channels", operand: "public"},+ {operator: "search", operand: "keyword"},+ ]),+ ),+ JSON.stringify([{operator: "channels", operand: "public", negated: false}]),+ );++ // DM array operand must parse; it is not history-enabling, so omit narrow.+ assert.equal(+ fetch_narrow_for_filter(+ new Filter([{operator: "dm", operand: [dm_user_a.user_id, dm_user_b.user_id]}]),+ ),+ undefined,+ );++ // is:starred alone omits the narrow parameter.+ assert.equal(+ fetch_narrow_for_filter(new Filter([{operator: "is", operand: "starred"}])),+ undefined,+ );+});+run_test("get_raw_content_for_single_message", ({override}) => {const msg_1 = {id: 1,diff --git a/zerver/lib/narrow.py b/zerver/lib/narrow.pyindex c06d3ab44de..1f0d5e9588a 100644--- a/zerver/lib/narrow.py+++ b/zerver/lib/narrow.py@@ -782,6 +782,10 @@ def ok_to_include_history(include_history = Falseif narrow is not None:+ # NOTE: This should be kept in sync with+ # message_fetch_raw_content.is_history_enabling_term+ # so we don't end up stripping new history enabling terms,+ # when fetching raw content for multiple messages.for term in narrow:if term.operator in channel_operators and not term.negated:operand: str | int = term.operand