Skip to content

armgit / spree

Public Imported from GitHub
0

Open Source Platform for DTC, B2B Commerce, Marketplaces & Omnichannel. REST APIs, TypeScript SDKs, and production-ready Next.js storefront. Self-host it. Own your data. No vendor lock-in. Zero platform fees.

Add ability to skip publishing events in Import (#14369)

Damian Legawiec403610457c64
diff --git a/spree/core/app/jobs/spree/imports/create_rows_job.rb b/spree/core/app/jobs/spree/imports/create_rows_job.rb
index c89e07a7ad..c1c3bd853f 100644
--- a/spree/core/app/jobs/spree/imports/create_rows_job.rb
+++ b/spree/core/app/jobs/spree/imports/create_rows_job.rb
@@ -9,12 +9,8 @@ module Spree
BATCH_SIZE = 1000
- # `inline` isn't persisted, so it has to be threaded through the chain —
- # otherwise a caller that asked for a synchronous run would enqueue from
- # here onwards.
- def perform(import_id, inline: false)
+ def perform(import_id)
import = Spree::Import.find(import_id)
- import.inline = inline
import.start_processing! if import.status != 'processing'
create_rows_sequentially(import)
diff --git a/spree/core/app/jobs/spree/imports/process_group_job.rb b/spree/core/app/jobs/spree/imports/process_group_job.rb
index 37ac556fad..52e653ceb5 100644
--- a/spree/core/app/jobs/spree/imports/process_group_job.rb
+++ b/spree/core/app/jobs/spree/imports/process_group_job.rb
@@ -21,11 +21,16 @@ module Spree
# (and its raw CSV data) in memory for the whole job.
ROWS_BATCH_SIZE = 100
- def perform(import_id, row_ids, inline: false)
+ def perform(import_id, row_ids)
import = Spree::Import.find(import_id)
- import.inline = inline
Spree::Current.store = import.store
+ process_group(import, row_ids)
+ end
+
+ private
+
+ def process_group(import, row_ids)
with_store_content_locale(import.store) do
mappings = import.mappings.mapped.to_a
schema_fields = import.schema_fields
@@ -34,40 +39,53 @@ module Spree
started_at = Time.current
processed_rows = []
- row_ids.each_slice(ROWS_BATCH_SIZE) do |ids|
- # Skip rows already completed on a prior attempt so retries don't double-process them.
- rows = import.rows.where(id: ids).pending_and_failed.order(:row_number).to_a
- # Share the already-loaded import across rows: each row's processor reads
- # `row.import` (store, ability, lookup cache), and without this every row
- # lazily loads its own Import instance and rebuilds all of that per row.
- rows.each { |row| row.association(:import).target = import }
-
- if large
- Spree::Events.disable do
- rows.each { |row| row.bulk_process!(mappings: mappings, schema_fields: schema_fields) }
- end
- elsif grouped
- # A group is one product plus its variants: per-record lifecycle
- # events (variant.created, price.created, product.updated per
- # touch) are noise to subscribers — one product event is published
- # for the whole group below. import_row.* events still flow.
- Spree::Events.disable_lifecycle do
- rows.each { |row| row.process!(mappings: mappings, schema_fields: schema_fields) }
- end
- else
- rows.each do |row|
- row.process!(mappings: mappings, schema_fields: schema_fields)
+ skipping_row_events(import) do
+ row_ids.each_slice(ROWS_BATCH_SIZE) do |ids|
+ # Skip rows already completed on a prior attempt so retries don't double-process them.
+ rows = import.rows.where(id: ids).pending_and_failed.order(:row_number).to_a
+ # Share the already-loaded import across rows: each row's processor reads
+ # `row.import` (store, ability, lookup cache), and without this every row
+ # lazily loads its own Import instance and rebuilds all of that per row.
+ rows.each { |row| row.association(:import).target = import }
+
+ if large
+ Spree::Events.disable do
+ rows.each { |row| row.bulk_process!(mappings: mappings, schema_fields: schema_fields) }
+ end
+ elsif grouped
+ # A group is one product plus its variants: per-record lifecycle
+ # events (variant.created, price.created, product.updated per
+ # touch) are noise to subscribers — one product event is published
+ # for the whole group below. import_row.* events still flow.
+ Spree::Events.disable_lifecycle do
+ rows.each { |row| row.process!(mappings: mappings, schema_fields: schema_fields) }
+ end
+ else
+ rows.each do |row|
+ row.process!(mappings: mappings, schema_fields: schema_fields)
+ end
end
+ processed_rows.concat(rows)
end
- processed_rows.concat(rows)
+
+ publish_group_events(processed_rows, started_at) if grouped
end
- publish_group_events(processed_rows, started_at) if grouped
+ # Outside `skipping_row_events` on purpose: the import's own
+ # lifecycle (`import.progress`, `import.completed`) is a handful of
+ # events per run and stays observable even for seeded data.
check_import_completion(import, large)
end
end
- private
+ # Seeded catalogs (sample/demo data) shouldn't reach webhooks or
+ # analytics. Scoped to the rows: the import's own lifecycle events stay
+ # observable.
+ def skipping_row_events(import, &block)
+ return Spree::Events.disable(&block) if import.preferred_skip_events
+
+ block.call
+ end
# One event per product touched by this group: `product.created` when the
# product came into existence during this run, `product.updated` otherwise
diff --git a/spree/core/app/jobs/spree/imports/process_rows_job.rb b/spree/core/app/jobs/spree/imports/process_rows_job.rb
index 3125ba391d..47b8e0c359 100644
--- a/spree/core/app/jobs/spree/imports/process_rows_job.rb
+++ b/spree/core/app/jobs/spree/imports/process_rows_job.rb
@@ -4,9 +4,8 @@ module Spree
BATCH_SIZE = 100
UNGROUPED_KEY = '__ungrouped__'.freeze
- def perform(import_id, inline: false)
+ def perform(import_id)
import = Spree::Import.find(import_id)
- import.inline = inline
dispatch_groups(import)
end
@@ -27,7 +26,7 @@ module Spree
# Inline callers need the group finished before `perform` returns; the
# group job's completion bookkeeping works either way.
def dispatch_group(import, row_ids)
- return ProcessGroupJob.perform_now(import.id, row_ids, inline: true) if import.inline?
+ return ProcessGroupJob.perform_now(import.id, row_ids) if import.preferred_inline
ProcessGroupJob.perform_later(import.id, row_ids)
end
diff --git a/spree/core/app/models/spree/import.rb b/spree/core/app/models/spree/import.rb
index 4c1ff3e596..0294802dc4 100644
--- a/spree/core/app/models/spree/import.rb
+++ b/spree/core/app/models/spree/import.rb
@@ -93,19 +93,30 @@ module Spree
after_transition on: :retry_failed_rows, do: :process_rows_async
end
- # Run the pipeline in-process instead of enqueuing it. Virtual and not
- # persisted — it describes how *this* caller wants the import driven, not
- # anything about the record.
- #
- # For rake tasks, seeds and the console, where the caller needs the data to
- # exist when the call returns and there may be no worker attached. Never set
- # it from a request: a large CSV blocks for minutes.
- attribute :inline, :boolean, default: false
-
#
# Preferences
#
preference :delimiter, :string, default: ','
+ # Run the pipeline in-process instead of enqueuing it. For rake tasks, seeds
+ # and the console, where the caller needs the data to exist when the call
+ # returns and there may be no worker attached. Never set it from a request:
+ # a large CSV blocks for minutes.
+ #
+ # A preference, not a virtual attribute, so it survives the `Import.find`
+ # each processing job does — otherwise every job would have to take (and
+ # forward) an `inline:` argument.
+ preference :inline, :boolean, default: false
+ # Suppress the events the *rows* publish — a `product.created` per imported
+ # product, plus `import_row.*`. The import's own `import.*` events still
+ # fire: a handful per import is useful, thousands from a seeded catalog are
+ # not.
+ #
+ # For seeding demo or sample data, where subscribers would otherwise deliver
+ # webhooks and analytics for a catalog nobody actually created. A preference
+ # rather than a virtual attribute because the processing jobs each reload
+ # the record, so it has to outlive the enqueue. Independent of `inline`: a
+ # background import can still be silent, and an inline one observed.
+ preference :skip_events, :boolean, default: false
# Absolute URL of the dashboard's imports view, captured at create and
# validated against the store's allowed origins — the import-done email
# links back to it. Blank for legacy-admin or app-created imports.
@@ -336,7 +347,7 @@ module Spree
def create_rows_async
# The 2s delay lets the attachment settle before the job reads it; running
# inline there is nothing to wait for.
- return Spree::Imports::CreateRowsJob.perform_now(id, inline: true) if inline?
+ return Spree::Imports::CreateRowsJob.perform_now(id) if preferred_inline
Spree::Imports::CreateRowsJob.set(wait: 2.seconds).perform_later(id)
end
@@ -344,7 +355,7 @@ module Spree
# Processes rows asynchronously
# @return [void]
def process_rows_async
- return Spree::Imports::ProcessRowsJob.perform_now(id, inline: true) if inline?
+ return Spree::Imports::ProcessRowsJob.perform_now(id) if preferred_inline
Spree::Imports::ProcessRowsJob.perform_later(id)
end
diff --git a/spree/core/app/services/spree/imports/row_processors/product_variant.rb b/spree/core/app/services/spree/imports/row_processors/product_variant.rb
index 18e75854f5..a514060e57 100644
--- a/spree/core/app/services/spree/imports/row_processors/product_variant.rb
+++ b/spree/core/app/services/spree/imports/row_processors/product_variant.rb
@@ -295,14 +295,14 @@ module Spree
end
def handle_tags(product)
- return Spree::Imports::AssignTagsJob.perform_now(product.id, attributes['tags']) if import.inline?
+ return Spree::Imports::AssignTagsJob.perform_now(product.id, attributes['tags']) if import.preferred_inline
Spree::Imports::AssignTagsJob.perform_later(product.id, attributes['tags'])
end
def handle_categories(product)
names = prepare_taxon_pretty_names
- return Spree::Imports::CreateCategoriesJob.perform_now(product.id, store.id, names) if import.inline?
+ return Spree::Imports::CreateCategoriesJob.perform_now(product.id, store.id, names) if import.preferred_inline
Spree::Imports::CreateCategoriesJob.perform_later(product.id, store.id, names)
end
diff --git a/spree/core/app/services/spree/sample_data/import_builder.rb b/spree/core/app/services/spree/sample_data/import_builder.rb
index 975d58a63b..85591c5441 100644
--- a/spree/core/app/services/spree/sample_data/import_builder.rb
+++ b/spree/core/app/services/spree/sample_data/import_builder.rb
@@ -4,7 +4,7 @@ module Spree
# but processing nothing — the two runners differ only in what they do next.
class ImportBuilder
# @return [Spree::Import] persisted, unprocessed
- def self.call(csv_path:, import_class:, store: nil, user: nil)
+ def self.call(csv_path:, import_class:, store: nil, user: nil, inline: false, skip_events: false)
store ||= Spree::Store.default
# An admin of the target store, not `admin_user_class.first` — in a
# multi-store or multi-tenant app that global first can belong to a
@@ -14,6 +14,10 @@ module Spree
raise 'No admin user found. Please run seeds first.' unless user
import = import_class.new(owner: store, user: user)
+ # Persisted before save: the processing jobs reload the record, so these
+ # have to travel with it rather than live on the instance.
+ import.preferred_inline = inline
+ import.preferred_skip_events = skip_events
import.number = import.generate_permalink(import_class)
import.attachment.attach(
io: File.open(csv_path),
diff --git a/spree/core/app/services/spree/sample_data/import_runner.rb b/spree/core/app/services/spree/sample_data/import_runner.rb
index 027dac0c32..5917bec897 100644
--- a/spree/core/app/services/spree/sample_data/import_runner.rb
+++ b/spree/core/app/services/spree/sample_data/import_runner.rb
@@ -14,13 +14,16 @@ module Spree
class ImportRunner
prepend Spree::ServiceModule::Base
+ # Pass `skip_events: true` when seeding demo data — subscribers would
+ # otherwise deliver webhooks and analytics for a catalog nobody created.
+ #
# @return [Spree::ServiceModule::Result] wrapping the import — completed
# when inline, queued otherwise
- def call(csv_path:, import_class:, store: nil, user: nil, inline: false)
+ def call(csv_path:, import_class:, store: nil, user: nil, inline: false, skip_events: false)
import = Spree::SampleData::ImportBuilder.call(
- csv_path: csv_path, import_class: import_class, store: store, user: user
+ csv_path: csv_path, import_class: import_class, store: store, user: user,
+ inline: inline, skip_events: skip_events
)
- import.inline = inline
import.start_mapping
import.complete_mapping
diff --git a/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb b/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb
index 2e7b9a077d..b1335e3fed 100644
--- a/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb
+++ b/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb
@@ -64,6 +64,35 @@ RSpec.describe Spree::Imports::ProcessGroupJob, type: :job do
expect(published['product.updated']).to eq(0)
end
+ context 'when the import was created with skip_events' do
+ before { import.update!(preferred_skip_events: true) }
+
+ it 'suppresses the row-level product events' do
+ published = Hash.new(0)
+ %w[product.created product.updated].each do |name|
+ Spree::Events.subscribe(name, async: false) { published[name] += 1 }
+ end
+ Spree::Events.activate!
+
+ described_class.perform_now(import.id, [row.id])
+
+ expect(published['product.created']).to eq(0)
+ expect(published['product.updated']).to eq(0)
+ # The rows still process — only their events are silenced.
+ expect(row.reload.status).to eq('completed')
+ end
+
+ it 'still publishes the import lifecycle events' do
+ published = Hash.new(0)
+ Spree::Events.subscribe('import.completed', async: false) { published['import.completed'] += 1 }
+ Spree::Events.activate!
+
+ described_class.perform_now(import.id, [row.id])
+
+ expect(published['import.completed']).to eq(1)
+ end
+ end
+
it 'publishes product.updated when the product existed before this run' do
described_class.perform_now(import.id, [row.id])
row.reload.update_columns(status: 'pending')
diff --git a/spree/core/spec/services/spree/sample_data/import_runner_spec.rb b/spree/core/spec/services/spree/sample_data/import_runner_spec.rb
index d1c930b174..ddf7b77e65 100644
--- a/spree/core/spec/services/spree/sample_data/import_runner_spec.rb
+++ b/spree/core/spec/services/spree/sample_data/import_runner_spec.rb
@@ -50,6 +50,42 @@ RSpec.describe Spree::SampleData::ImportRunner, type: :service do
end
end
+ # Seeding a demo catalog shouldn't fan out a `product.created` per product to
+ # webhooks and analytics. The import's own `import.*` events still fire.
+ describe 'skip_events', :events do
+ let(:csv_path) { Spree::Core::Engine.root.join('db', 'sample_data', 'products.csv') }
+
+ # The suppression itself is asserted where the boundary lives — see
+ # ProcessGroupJob's specs. Here we only check that the flag survives the
+ # trip to the jobs and that the import's own lifecycle keeps flowing.
+
+ # Only the rows go quiet — the import's own lifecycle stays observable, so
+ # a dashboard or subscriber can still tell when seeding finished.
+ it 'still publishes the import lifecycle events' do
+ names = []
+ allow(Spree::Events).to receive(:publish).and_wrap_original do |original, name, *rest|
+ names << name
+ original.call(name, *rest)
+ end
+
+ described_class.call(csv_path: csv_path, import_class: Spree::Imports::Products,
+ skip_events: true)
+ perform_enqueued_jobs(only: [Spree::Imports::CreateRowsJob, Spree::Imports::ProcessRowsJob,
+ Spree::Imports::ProcessGroupJob])
+
+ # The import's own lifecycle is untouched by the row suppression.
+ expect(names).to include('import.created')
+ expect(names.select { |name| name.to_s.start_with?('import.') }).to be_present
+ end
+
+ it 'travels with the import so the processing jobs see it' do
+ import = described_class.call(csv_path: csv_path, import_class: Spree::Imports::Products,
+ skip_events: true).value
+
+ expect(Spree::Import.find(import.id).preferred_skip_events).to be true
+ end
+ end
+
it 'runs as an admin of the target store' do
expect(result.value.user).to eq(admin)
end