Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

message_fetch_raw_content: Fetch raw content without UserMessage rows.

Apoorva Pendsefecf7f421be1
diff --git a/web/src/compose_reply.ts b/web/src/compose_reply.ts
index 0162fed05d3..549e185b5a0 100644
--- a/web/src/compose_reply.ts
+++ b/web/src/compose_reply.ts
@@ -692,8 +692,8 @@ export function build_and_process_quote_assets_for_messages(
message_ids,
on_success(raw_content_arr) {
for (const [i, message] of messages.entries()) {
- const raw_content = raw_content_arr[i];
- assert(raw_content !== undefined);
+ const raw_content =
+ raw_content_arr[i] ?? compose_paste.paste_handler_converter(message.content);
quote_assets.push({message, quote_content: raw_content});
}
callback(quote_assets);
diff --git a/web/src/message_fetch_raw_content.ts b/web/src/message_fetch_raw_content.ts
index 2aa046a9b35..4092dd8c16d 100644
--- a/web/src/message_fetch_raw_content.ts
+++ b/web/src/message_fetch_raw_content.ts
@@ -1,23 +1,77 @@
import assert from "minimalistic-assert";
+import * as z from "zod/mini";
import * as channel from "./channel.ts";
import * as message_fetch from "./message_fetch.ts";
import * as message_store from "./message_store.ts";
+import type {Message} from "./message_store.ts";
+import * as narrow_state from "./narrow_state.ts";
+
+// NOTE: Keep this and is_history_enabling_term up to date with
+// all the future terms that do include historical messages, like the upcoming
+// `channels:all` from #33534 and the eventually possible `channels:subscribed`
+// #13052.
+const history_enabling_narrow_term_schema = z.object({
+ operator: z.string(),
+ operand: z.union([z.number(), z.string(), z.array(z.number())]),
+ negated: z.optional(z.boolean()),
+});
+
+function is_history_enabling_term(
+ term: z.infer<typeof history_enabling_narrow_term_schema>,
+): boolean {
+ if (term.negated) {
+ return false;
+ }
+ if (term.operator === "channel" || term.operator === "stream") {
+ return true;
+ }
+ return (
+ term.operator === "channels" && (term.operand === "public" || term.operand === "web-public")
+ );
+}
+
+// Returns a JSON narrow for GET /messages that enables shared history
+// (channel/stream, or channels:public|web-public). Returns undefined
+// when no such terms are present so the caller can omit the parameter.
+function get_history_enabling_narrow_for_raw_content_fetch(): string | undefined {
+ const filter = narrow_state.filter();
+ if (filter === undefined) {
+ return undefined;
+ }
+
+ const encoded = message_fetch.get_narrow_for_message_fetch(filter);
+ if (encoded === "") {
+ return undefined;
+ }
+
+ const history_terms = z
+ .array(history_enabling_narrow_term_schema)
+ .parse(JSON.parse(encoded))
+ .filter((term) => is_history_enabling_term(term));
+
+ if (history_terms.length === 0) {
+ return undefined;
+ }
+ return JSON.stringify(history_terms);
+}
export function get_raw_content_for_messages(info: {
message_ids: number[];
- on_success: (raw_content_arr: string[]) => void;
+ on_success: (raw_content_arr: (string | undefined)[]) => void;
on_error: () => void;
timeout_ms?: number | undefined;
}): void {
const {message_ids, on_success, on_error, timeout_ms} = info;
const message_ids_that_require_fetching: number[] = [];
- const raw_content_arr: string[] = Array.from({length: message_ids.length});
+ const raw_content_arr: (string | undefined)[] = Array.from({length: message_ids.length});
+ const messages: Message[] = [];
- // We fill what we can from the store
+ // We fill what we can from message_store.
for (const [i, id] of message_ids.entries()) {
const message = message_store.get(id);
assert(message !== undefined);
+ messages.push(message);
if (message.raw_content) {
raw_content_arr[i] = message.raw_content;
} else {
@@ -30,12 +84,18 @@ export function get_raw_content_for_messages(info: {
return;
}
+ // GET /messages with message_ids uses personal history unless a
+ // history-enabling narrow is passed
+ // (https://zulip.com/api/get-messages#parameter-narrow).
+ const narrow = get_history_enabling_narrow_for_raw_content_fetch();
+
channel.get({
url: "/json/messages",
data: {
allow_empty_topic_name: true,
apply_markdown: false,
message_ids: JSON.stringify(message_ids_that_require_fetching),
+ ...(narrow !== undefined && {narrow}),
},
success(raw_data) {
const data = message_fetch.message_ids_response_schema.parse(raw_data);
@@ -47,9 +107,22 @@ export function get_raw_content_for_messages(info: {
fetched_raw_content_map.set(raw_message.id, parsed_message.content);
}
- // Fill the remaining holes in the final array
+ // We fill remaining holes from the fetch first, then from any
+ // raw_content cached on the message mid-flight. Anything still
+ // missing stays unset for the caller.
for (const [i, id] of message_ids.entries()) {
- raw_content_arr[i] ??= fetched_raw_content_map.get(id)!;
+ if (raw_content_arr[i] !== undefined) {
+ continue;
+ }
+ const fetched = fetched_raw_content_map.get(id);
+ if (fetched !== undefined) {
+ raw_content_arr[i] = fetched;
+ continue;
+ }
+ const message = messages[i]!;
+ if (message.raw_content !== undefined) {
+ raw_content_arr[i] = message.raw_content;
+ }
}
on_success(raw_content_arr);
diff --git a/web/tests/compose_reply.test.cjs b/web/tests/compose_reply.test.cjs
index 4e156667b83..cfc27b5ee9f 100644
--- a/web/tests/compose_reply.test.cjs
+++ b/web/tests/compose_reply.test.cjs
@@ -389,6 +389,88 @@ run_test("build_and_process_quote_assets_for_messages", ({override}) => {
{message: msg_unhydrated, quote_content: "converted_by_turndown: <p>unhydrated</p>"},
"Fallback to using paste_handler_converter",
);
+
+ // Case: Partial success. Undefined slots fall back to HTML→markdown.
+ override(
+ compose_paste,
+ "paste_handler_converter",
+ (content) => `converted_by_turndown: ${content}`,
+ );
+
+ const msg_from_message_store = {
+ id: 3,
+ raw_content: "From message_store",
+ content: "<p>message_store only</p>",
+ };
+ const msg_html_only = {id: 4, content: "<p>html only body</p>"};
+ add_messages_to_message_store([
+ msg_hydrated,
+ msg_unhydrated,
+ msg_from_message_store,
+ msg_html_only,
+ ]);
+
+ override(
+ message_fetch_raw_content,
+ "get_raw_content_for_messages",
+ ({_message_ids, on_success, _on_error}) => {
+ // Explicit undefined for ids the batch did not return.
+ on_success(["batch raw", undefined, undefined, undefined]);
+ },
+ );
+
+ compose_reply.build_and_process_quote_assets_for_messages([1, 2, 3, 4], (assets) => {
+ result_assets = assets;
+ });
+
+ assert.equal(result_assets.length, 4);
+ assert.deepEqual(result_assets[0], {
+ message: msg_hydrated,
+ quote_content: "batch raw",
+ });
+ assert.deepEqual(
+ result_assets[1],
+ {
+ message: msg_unhydrated,
+ quote_content: "converted_by_turndown: <p>unhydrated</p>",
+ },
+ "Undefined batch slot uses HTML conversion.",
+ );
+ assert.deepEqual(
+ result_assets[2],
+ {
+ message: msg_from_message_store,
+ quote_content: "converted_by_turndown: <p>message_store only</p>",
+ },
+ "Undefined batch slot ignores message_store raw_content on success path.",
+ );
+ assert.deepEqual(
+ result_assets[3],
+ {
+ message: msg_html_only,
+ quote_content: "converted_by_turndown: <p>html only body</p>",
+ },
+ "Undefined batch slot with neither uses HTML conversion.",
+ );
+
+ // Case: All slots are undefined. Success path uses HTML conversion only.
+ override(
+ message_fetch_raw_content,
+ "get_raw_content_for_messages",
+ ({_message_ids, on_success, _on_error}) => {
+ on_success([undefined, undefined]);
+ },
+ );
+ compose_reply.build_and_process_quote_assets_for_messages([3, 4], (assets) => {
+ result_assets = assets;
+ });
+ assert.deepEqual(
+ result_assets.map((a) => a.quote_content),
+ [
+ "converted_by_turndown: <p>message_store only</p>",
+ "converted_by_turndown: <p>html only body</p>",
+ ],
+ );
});
run_test("get_quote_menu_selection", ({override_rewire}) => {
diff --git a/web/tests/message_fetch_raw_content.test.cjs b/web/tests/message_fetch_raw_content.test.cjs
index d484297611c..54600bbd381 100644
--- a/web/tests/message_fetch_raw_content.test.cjs
+++ b/web/tests/message_fetch_raw_content.test.cjs
@@ -2,14 +2,17 @@
const assert = require("node:assert/strict");
+const {make_user} = require("./lib/example_user.cjs");
const {zrequire, mock_esm} = require("./lib/namespace.cjs");
const {run_test} = require("./lib/test.cjs");
const stream_data = zrequire("stream_data");
const message_store = zrequire("message_store");
-const message_fetch_raw_content = zrequire("message_fetch_raw_content");
+const {Filter} = zrequire("filter");
const message_fetch = mock_esm("../src/message_fetch");
+const narrow_state = mock_esm("../src/narrow_state");
const channel = mock_esm("../src/channel");
+const message_fetch_raw_content = zrequire("message_fetch_raw_content");
function add_messages_to_message_store(messages) {
message_store.clear_for_testing();
@@ -18,7 +21,7 @@ function add_messages_to_message_store(messages) {
}
}
-// We only rely on message_fetch for type validation.
+// We only rely on message_fetch for type validation and narrow encoding.
message_fetch.message_ids_response_schema = {
parse: (data) => data,
};
@@ -75,10 +78,23 @@ run_test("get_raw_content_for_messages", ({override}) => {
assert.ok(success_called, "Should call on_success immediately if all messages are hydrated");
- // Case: Fetching missing raw_content successfully
+ // Case: Fetching missing raw_content successfully. Only
+ // history-enabling channel terms are passed for include_history.
let channel_get_args;
success_called = false;
+ const fake_filter = {};
+ const full_encoded_narrow = JSON.stringify([
+ {operator: "channel", operand: social.stream_id},
+ {operator: "topic", operand: "some-topic"},
+ {operator: "is", operand: "unread"},
+ ]);
+ const history_only_narrow = JSON.stringify([{operator: "channel", operand: social.stream_id}]);
+ override(narrow_state, "filter", () => fake_filter);
+ override(message_fetch, "get_narrow_for_message_fetch", (filter) => {
+ assert.equal(filter, fake_filter);
+ return full_encoded_narrow;
+ });
override(channel, "get", (args) => {
channel_get_args = args;
args.success({
@@ -102,12 +118,310 @@ run_test("get_raw_content_for_messages", ({override}) => {
JSON.stringify([2]),
"Should only request hydration for messages missing raw_content",
);
+ assert.equal(
+ channel_get_args.data.narrow,
+ history_only_narrow,
+ "Should strip topic/is: and keep only channel for shared history",
+ );
// It is safe to update raw_content for messages from channels
// the user is subscribed to.
assert.equal(msg_2.raw_content, "Fetched markdown content");
assert.ok(success_called, "Should call on_success after successfully hydrating");
assert.deepEqual(success_call_args, [msg_1.raw_content, msg_2.raw_content]);
+ // Case: channels:public is kept as history-enabling. Other terms are stripped.
+ success_called = false;
+ delete msg_2.raw_content;
+ override(narrow_state, "filter", () => fake_filter);
+ override(message_fetch, "get_narrow_for_message_fetch", () =>
+ JSON.stringify([
+ {operator: "channels", operand: "public"},
+ {operator: "search", operand: "keyword"},
+ ]),
+ );
+ override(channel, "get", (args) => {
+ channel_get_args = args;
+ args.success({
+ messages: [
+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},
+ ],
+ });
+ });
+
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [1, 2],
+ on_success() {
+ success_called = true;
+ },
+ });
+
+ assert.equal(
+ channel_get_args.data.narrow,
+ JSON.stringify([{operator: "channels", operand: "public"}]),
+ );
+ assert.ok(success_called);
+
+ // Case: channels:web-public is kept as history-enabling.
+ success_called = false;
+ delete msg_2.raw_content;
+ override(narrow_state, "filter", () => fake_filter);
+ override(message_fetch, "get_narrow_for_message_fetch", () =>
+ JSON.stringify([{operator: "channels", operand: "web-public"}]),
+ );
+ override(channel, "get", (args) => {
+ channel_get_args = args;
+ args.success({
+ messages: [
+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},
+ ],
+ });
+ });
+
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [1, 2],
+ on_success() {
+ success_called = true;
+ },
+ });
+
+ assert.equal(
+ channel_get_args.data.narrow,
+ JSON.stringify([{operator: "channels", operand: "web-public"}]),
+ );
+ assert.ok(success_called);
+
+ // Case: For an is: operator, no history terms are present, so we omit narrow.
+ success_called = false;
+ delete msg_2.raw_content;
+ override(narrow_state, "filter", () => fake_filter);
+ override(message_fetch, "get_narrow_for_message_fetch", () =>
+ JSON.stringify([{operator: "is", operand: "starred"}]),
+ );
+ override(channel, "get", (args) => {
+ channel_get_args = args;
+ args.success({
+ messages: [
+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},
+ ],
+ });
+ });
+
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [1, 2],
+ on_success() {
+ success_called = true;
+ },
+ });
+
+ assert.equal(channel_get_args.data.narrow, undefined);
+ assert.ok(success_called);
+
+ // Case: Negated channel terms are not history-enabling and are dropped.
+ success_called = false;
+ delete msg_2.raw_content;
+ override(narrow_state, "filter", () => fake_filter);
+ override(message_fetch, "get_narrow_for_message_fetch", () =>
+ JSON.stringify([
+ {operator: "channel", operand: social.stream_id, negated: true},
+ {operator: "channel", operand: denmark.stream_id},
+ ]),
+ );
+ override(channel, "get", (args) => {
+ channel_get_args = args;
+ args.success({
+ messages: [
+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},
+ ],
+ });
+ });
+
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [1, 2],
+ on_success() {
+ success_called = true;
+ },
+ });
+
+ assert.equal(
+ channel_get_args.data.narrow,
+ JSON.stringify([{operator: "channel", operand: denmark.stream_id}]),
+ );
+ assert.ok(success_called);
+
+ // Case: The encoded narrow is empty, so omit the narrow parameter.
+ success_called = false;
+ delete msg_2.raw_content;
+ override(narrow_state, "filter", () => fake_filter);
+ override(message_fetch, "get_narrow_for_message_fetch", () => "");
+ override(channel, "get", (args) => {
+ channel_get_args = args;
+ args.success({
+ messages: [
+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},
+ ],
+ });
+ });
+
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [1, 2],
+ on_success(args) {
+ success_called = true;
+ success_call_args = args;
+ },
+ });
+
+ assert.equal(channel_get_args.data.narrow, undefined);
+ assert.ok(success_called);
+
+ // Case: There is no current filter (e.g. recent conversations), so no narrow param.
+ success_called = false;
+ delete msg_2.raw_content;
+ override(narrow_state, "filter", () => undefined);
+ override(channel, "get", (args) => {
+ channel_get_args = args;
+ args.success({
+ messages: [
+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},
+ ],
+ });
+ });
+
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [1, 2],
+ on_success(args) {
+ success_called = true;
+ success_call_args = args;
+ },
+ });
+
+ assert.equal(channel_get_args.data.narrow, undefined);
+ assert.ok(success_called);
+
+ // Case: A partial batch response leaves explicit undefined for missing ids.
+ success_called = false;
+ delete msg_2.raw_content;
+ override(narrow_state, "filter", () => undefined);
+ override(channel, "get", (args) => {
+ channel_get_args = args;
+ args.success({
+ messages: [],
+ });
+ });
+
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [1, 2],
+ on_success(args) {
+ success_called = true;
+ success_call_args = args;
+ },
+ });
+
+ assert.ok(success_called);
+ assert.equal(success_call_args.length, 2);
+ assert.equal(success_call_args[0], msg_1.raw_content);
+ // The slot is present as undefined.
+ assert.equal(success_call_args[1], undefined);
+
+ // Case: A partial batch where message_store gains raw_content during the request.
+ success_called = false;
+ delete msg_2.raw_content;
+ override(narrow_state, "filter", () => undefined);
+ override(channel, "get", (args) => {
+ // Simulate another path hydrating message_store before success runs.
+ msg_2.raw_content = "Hydrated in message_store mid-flight";
+ args.success({messages: []});
+ });
+
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [1, 2],
+ on_success(args) {
+ success_called = true;
+ success_call_args = args;
+ },
+ });
+
+ assert.ok(success_called);
+ assert.deepEqual(success_call_args, [
+ msg_1.raw_content,
+ "Hydrated in message_store mid-flight",
+ ]);
+
+ // Case: Mixed partial. One id is returned and one is missing (undefined slot).
+ success_called = false;
+ delete msg_2.raw_content;
+ delete msg_3.raw_content;
+ override(narrow_state, "filter", () => undefined);
+ override(channel, "get", (args) => {
+ args.success({
+ messages: [{id: 2, content_type: "text/x-markdown", content: "only two"}],
+ });
+ });
+
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [2, 3],
+ on_success(args) {
+ success_called = true;
+ success_call_args = args;
+ },
+ });
+
+ assert.ok(success_called);
+ assert.equal(success_call_args.length, 2);
+ assert.equal(success_call_args[0], "only two");
+ assert.equal(success_call_args[1], undefined);
+
+ // Case: The server returns messages out of request order. The result
+ // still matches message_ids order, with undefined for the missing middle id.
+ success_called = false;
+ delete msg_1.raw_content;
+ delete msg_2.raw_content;
+ delete msg_3.raw_content;
+ // msg_1 is Denmark (subscribed); msg_3 is social (unsubscribed).
+ add_messages_to_message_store([
+ {
+ id: 1,
+ content: "<p>one html</p>",
+ type: "stream",
+ stream_id: denmark.stream_id,
+ },
+ {
+ id: 2,
+ content: "<p>two html</p>",
+ type: "stream",
+ stream_id: denmark.stream_id,
+ },
+ {
+ id: 3,
+ content: "<p>three html</p>",
+ type: "stream",
+ stream_id: social.stream_id,
+ },
+ ]);
+ override(narrow_state, "filter", () => undefined);
+ override(channel, "get", (args) => {
+ assert.equal(args.data.message_ids, JSON.stringify([1, 2, 3]));
+ // Deliberately reverse order and omit id 2.
+ args.success({
+ messages: [
+ {id: 3, content_type: "text/x-markdown", content: "raw three"},
+ {id: 1, content_type: "text/x-markdown", content: "raw one"},
+ ],
+ });
+ });
+
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [1, 2, 3],
+ on_success(args) {
+ success_called = true;
+ success_call_args = args;
+ },
+ });
+
+ assert.ok(success_called);
+ assert.deepEqual(success_call_args, ["raw one", undefined, "raw three"]);
+ assert.equal(success_call_args.length, 3);
+ assert.equal(success_call_args[1], undefined);
+
// Case: Network error during hydration
success_called = false;
let error_called = false;
@@ -131,6 +445,91 @@ run_test("get_raw_content_for_messages", ({override}) => {
assert.ok(error_called, "Should call on_error if the network request fails");
});
+run_test("get_raw_content_for_messages encodes real Filter narrows", ({override}) => {
+ const dm_user_a = make_user();
+ const dm_user_b = make_user();
+ const msg = {
+ id: 2,
+ content: "<p>HTML content</p>",
+ type: "stream",
+ stream_id: social.stream_id,
+ };
+ add_messages_to_message_store([msg]);
+
+ let channel_get_args;
+ override(channel, "get", (args) => {
+ channel_get_args = args;
+ args.success({
+ messages: [
+ {id: 2, content_type: "text/x-markdown", content: "Fetched markdown content"},
+ ],
+ });
+ });
+ override(message_fetch, "get_narrow_for_message_fetch", (filter) => {
+ const terms = filter.public_terms().map((term) => {
+ if (term.operator === "channel" && typeof term.operand === "string") {
+ const stream = stream_data.get_sub_by_id_string(term.operand);
+ if (stream !== undefined) {
+ return {...term, operand: stream.stream_id};
+ }
+ }
+ return term;
+ });
+ return JSON.stringify(terms);
+ });
+
+ function fetch_narrow_for_filter(filter) {
+ delete msg.raw_content;
+ override(narrow_state, "filter", () => filter);
+ let success_called = false;
+ message_fetch_raw_content.get_raw_content_for_messages({
+ message_ids: [2],
+ on_success() {
+ success_called = true;
+ },
+ });
+ assert.ok(success_called);
+ return channel_get_args.data.narrow;
+ }
+
+ // Channel + topic + is:unread keeps only channel; operand is coerced to int.
+ assert.equal(
+ fetch_narrow_for_filter(
+ new Filter([
+ {operator: "channel", operand: social.stream_id.toString()},
+ {operator: "topic", operand: "some-topic"},
+ {operator: "is", operand: "unread"},
+ ]),
+ ),
+ JSON.stringify([{operator: "channel", operand: social.stream_id, negated: false}]),
+ );
+
+ // channels:public + search keeps only channels:public.
+ assert.equal(
+ fetch_narrow_for_filter(
+ new Filter([
+ {operator: "channels", operand: "public"},
+ {operator: "search", operand: "keyword"},
+ ]),
+ ),
+ JSON.stringify([{operator: "channels", operand: "public", negated: false}]),
+ );
+
+ // DM array operand must parse; it is not history-enabling, so omit narrow.
+ assert.equal(
+ fetch_narrow_for_filter(
+ new Filter([{operator: "dm", operand: [dm_user_a.user_id, dm_user_b.user_id]}]),
+ ),
+ undefined,
+ );
+
+ // is:starred alone omits the narrow parameter.
+ assert.equal(
+ fetch_narrow_for_filter(new Filter([{operator: "is", operand: "starred"}])),
+ undefined,
+ );
+});
+
run_test("get_raw_content_for_single_message", ({override}) => {
const msg_1 = {
id: 1,
diff --git a/zerver/lib/narrow.py b/zerver/lib/narrow.py
index c06d3ab44de..1f0d5e9588a 100644
--- a/zerver/lib/narrow.py
+++ b/zerver/lib/narrow.py
@@ -782,6 +782,10 @@ def ok_to_include_history(
include_history = False
if narrow is not None:
+ # NOTE: This should be kept in sync with
+ # message_fetch_raw_content.is_history_enabling_term
+ # so we don't end up stripping new history enabling terms,
+ # when fetching raw content for multiple messages.
for term in narrow:
if term.operator in channel_operators and not term.negated:
operand: str | int = term.operand