Skip to content

Open-source live-chat, email support, omni-channel desk. An alternative to Intercom, Zendesk, Salesforce Service Cloud etc. 🔥💬

fix: skip IMAP emails that another fetch job already saved (#15978)

Shivam Mishrafbb90f753cb0
diff --git a/app/mailboxes/imap/imap_mailbox.rb b/app/mailboxes/imap/imap_mailbox.rb
index 843df425226..d531ed7aa09 100644
--- a/app/mailboxes/imap/imap_mailbox.rb
+++ b/app/mailboxes/imap/imap_mailbox.rb
@@ -18,6 +18,8 @@ class Imap::ImapMailbox
return unless incoming_email_from_valid_email?
ActiveRecord::Base.transaction do
+ next if duplicate_email?
+
find_or_create_contact
find_or_create_conversation
create_message
@@ -39,6 +41,19 @@ class Imap::ImapMailbox
@processed_mail = MailPresenter.new(@inbound_mail, @account)
end
+ # Fetch jobs for an inbox can overlap and hold the same email, long after their own duplicate check ran.
+ # The advisory lock makes a job wait while another job saves the same email. Postgres releases it when the transaction ends.
+ # Waiting, instead of skipping, means a reply later in the batch can find this email's conversation.
+ # The check after the lock sees the copy the other job committed. It bypasses the query cache,
+ # which can still hold the result of the same query from the fetch-stage check.
+ def duplicate_email?
+ source_id = sanitize_mailbox_value(@processed_mail.message_id)
+ lock_key = Digest::SHA256.digest(source_id).unpack1('l>')
+ ActiveRecord::Base.connection.select_value("SELECT pg_advisory_xact_lock(#{@inbox.id}, #{lock_key})")
+
+ Message.uncached { @inbox.messages.exists?(source_id: source_id) }
+ end
+
# Replies to a forwarded email stay in the forward recipient's own conversation
def find_conversation_by_in_reply_to
return if in_reply_to.blank?
diff --git a/spec/fixtures/files/malformed_reply_to.eml b/spec/fixtures/files/malformed_reply_to.eml
index 5a5a4dff77e..91ba2b326d6 100644
--- a/spec/fixtures/files/malformed_reply_to.eml
+++ b/spec/fixtures/files/malformed_reply_to.eml
@@ -2,6 +2,7 @@ From: Sender <[email protected]>
Reply-To: <"">
Subject: Empty Reply-To address
+Message-ID: <[email protected]>
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
diff --git a/spec/mailboxes/imap/imap_mailbox_spec.rb b/spec/mailboxes/imap/imap_mailbox_spec.rb
index 0ebca57de81..a8842c91d7e 100644
--- a/spec/mailboxes/imap/imap_mailbox_spec.rb
+++ b/spec/mailboxes/imap/imap_mailbox_spec.rb
@@ -115,6 +115,36 @@ RSpec.describe Imap::ImapMailbox do
end
end
+ context 'when another fetch job has already saved the email' do
+ let(:inbound_mail) { create_inbound_email_from_mail(from: '[email protected]', to: '[email protected]', subject: 'Hello!') }
+
+ it 'does not create a second conversation or message' do
+ create(:message, account: account, inbox: inbox, source_id: inbound_mail.mail.message_id)
+
+ expect do
+ class_instance.process(inbound_mail.mail, channel)
+ end.to not_change(Conversation, :count).and not_change(Message, :count)
+ end
+ end
+
+ context 'when another fetch job saves the email after this job cached its fetch-stage duplicate check' do
+ let(:inbound_mail) { create_inbound_email_from_mail(from: '[email protected]', to: '[email protected]', subject: 'Hello!') }
+ let(:message_id) { inbound_mail.mail.message_id }
+
+ it 'does not create a second conversation or message' do
+ ActiveRecord::Base.cache do
+ inbox.messages.exists?(source_id: message_id)
+ # The other job writes on its own connection, so this job's query cache is not cleared
+ ActiveRecord::Base.uncached(dirties: false) { create(:message, account: account, inbox: inbox, source_id: message_id) }
+
+ class_instance.process(inbound_mail.mail, channel)
+ end
+
+ expect(inbox.messages.where(source_id: message_id).count).to eq(1)
+ expect(inbox.conversations.count).to eq(1)
+ end
+ end
+
context 'when a new email contains null bytes' do
let(:inbound_mail) do
Mail.new.tap do |mail|