Open Source Platform for DTC, B2B Commerce, Marketplaces & Omnichannel. REST APIs, TypeScript SDKs, and production-ready Next.js storefront. Self-host it. Own your data. No vendor lock-in. Zero platform fees.
Ruby b2b-commercee-commerceecommerceecommerce-apiecommerce-frameworkecommerce-platformheadlessheadless-commerceheadless-ecommercemarketplacemulti-tenantmulti-vendormulti-vendor-ecommerceopen-sourcespree-commerce
[CSV Import] Add guards against conrurrent processing of option types/values (#14060)
Damian Legawiece1a0f103cb13
diff --git a/spree/core/app/jobs/spree/imports/assign_tags_job.rb b/spree/core/app/jobs/spree/imports/assign_tags_job.rbindex 4b0930144d..4efd602842 100644--- a/spree/core/app/jobs/spree/imports/assign_tags_job.rb+++ b/spree/core/app/jobs/spree/imports/assign_tags_job.rb@@ -1,9 +1,6 @@module Spreemodule Imports- class AssignTagsJob < Spree::BaseJob- queue_as Spree.queues.imports- retry_on StandardError, wait: :polynomially_longer, attempts: 5-+ class AssignTagsJob < Spree::Imports::BaseJobdef perform(product_id, tags)product = Spree::Product.find(product_id)product.tag_list = tagsdiff --git a/spree/core/app/jobs/spree/imports/base_job.rb b/spree/core/app/jobs/spree/imports/base_job.rbnew file mode 100644index 0000000000..3c742ca394--- /dev/null+++ b/spree/core/app/jobs/spree/imports/base_job.rb@@ -0,0 +1,31 @@+module Spree+ module Imports+ # Shared base for every job in the imports pipeline.+ #+ # Retry policy is intentionally narrow: only transient infrastructure errors+ # (DB deadlocks, lock-wait timeouts, dropped connections) are replayed. Broad+ # `retry_on StandardError` would replay jobs whose post-work side effects+ # (counter increments, state transitions, lifecycle events) had already+ # partially fired, breaking idempotency. Per-row business errors are caught+ # inside `Spree::ImportRow#process!` and converted to `row.fail!`, so they+ # never bubble up to the job layer.+ #+ # Subclasses may extend the retry list (e.g. `CreateCategoriesJob` adds+ # `RecordNotUnique` to recover from concurrent-import taxon creation races).+ #+ # We intentionally do NOT `discard_on ActiveRecord::RecordNotFound`: Sidekiq is+ # known to start a job before the enqueuing transaction has committed, so an+ # early `find` can raise `RecordNotFound` for a record that will exist a moment+ # later. Letting the job retry covers that window.+ class BaseJob < Spree::BaseJob+ queue_as Spree.queues.imports++ retry_on ActiveRecord::Deadlocked, ActiveRecord::LockWaitTimeout,+ ActiveRecord::ConnectionNotEstablished, ActiveRecord::ConnectionFailed,+ ActiveRecord::RecordNotFound,+ wait: :polynomially_longer, attempts: 5++ discard_on ActiveJob::DeserializationError+ end+ end+enddiff --git a/spree/core/app/jobs/spree/imports/create_categories_job.rb b/spree/core/app/jobs/spree/imports/create_categories_job.rbindex 2fcb969ff8..94973c37bb 100644--- a/spree/core/app/jobs/spree/imports/create_categories_job.rb+++ b/spree/core/app/jobs/spree/imports/create_categories_job.rb@@ -1,8 +1,10 @@module Spreemodule Imports- class CreateCategoriesJob < Spree::BaseJob- queue_as Spree.queues.imports- retry_on StandardError, wait: :polynomially_longer, attempts: 5+ class CreateCategoriesJob < Spree::Imports::BaseJob+ # Concurrent imports can race on `with_matching_name(...).first || create!(...)`+ # for the same taxonomy/taxon name and hit the unique index; a retry then finds+ # the peer's committed row.+ retry_on ActiveRecord::RecordNotUnique, wait: :polynomially_longer, attempts: 5def perform(product_id, store_id, taxon_pretty_names)product = Spree::Product.find(product_id)diff --git a/spree/core/app/jobs/spree/imports/create_rows_job.rb b/spree/core/app/jobs/spree/imports/create_rows_job.rbindex 1481d05594..c1c3bd853f 100644--- a/spree/core/app/jobs/spree/imports/create_rows_job.rb+++ b/spree/core/app/jobs/spree/imports/create_rows_job.rb@@ -1,8 +1,6 @@module Spreemodule Imports- class CreateRowsJob < Spree::BaseJob- queue_as Spree.queues.imports-+ class CreateRowsJob < Spree::Imports::BaseJobdiscard_on ::CSV::MalformedCSVError, EncodingError do |job, error|import = Spree::Import.find(job.arguments.first)import.update_columns(processing_errors: error.message, status: :failed, updated_at: Time.current)diff --git a/spree/core/app/jobs/spree/imports/process_group_job.rb b/spree/core/app/jobs/spree/imports/process_group_job.rbindex 95b5dcec3a..c6ac33c270 100644--- a/spree/core/app/jobs/spree/imports/process_group_job.rb+++ b/spree/core/app/jobs/spree/imports/process_group_job.rb@@ -1,8 +1,6 @@module Spreemodule Imports- class ProcessGroupJob < Spree::BaseJob- queue_as Spree.queues.imports-+ class ProcessGroupJob < Spree::Imports::BaseJobdef perform(import_id, row_ids)import = Spree::Import.find(import_id)Spree::Current.store = import.store@@ -10,7 +8,8 @@ module Spreemappings = import.mappings.mapped.to_aschema_fields = import.schema_fieldslarge = import.large_import?- rows = import.rows.where(id: row_ids).order(:row_number)+ # Skip rows already completed on a prior attempt so retries don't double-process them.+ rows = import.rows.where(id: row_ids).pending_and_failed.order(:row_number)if largeSpree::Events.disable do@@ -27,14 +26,17 @@ module Spreeprivate+ # Completion is row-state-derived so retry-induced over-increments of the counter+ # stay harmless. The counter pre-check just shortcuts the row scan for workers+ # that obviously can't be the last group to finish. `in_flight` excludes orphaned+ # `processing` rows past the stall window so a dead worker can't block completion.def check_import_completion(import, large)Spree::Import.where(id: import.id).update_all('completed_groups_count = COALESCE(completed_groups_count, 0) + 1')import.reload- if import.completed_groups_count >= import.processing_groups_count- # Guard against concurrent workers both reaching this point+ if import.completed_groups_count >= import.processing_groups_count && import.rows.in_flight.none?import.complete! if import.status == 'processing'elsif large && (import.completed_groups_count % 10).zero?import.publish_event('import.progress')diff --git a/spree/core/app/jobs/spree/imports/process_rows_job.rb b/spree/core/app/jobs/spree/imports/process_rows_job.rbindex 68c3340532..2853e68a60 100644--- a/spree/core/app/jobs/spree/imports/process_rows_job.rb+++ b/spree/core/app/jobs/spree/imports/process_rows_job.rb@@ -1,8 +1,6 @@module Spreemodule Imports- class ProcessRowsJob < Spree::BaseJob- queue_as Spree.queues.imports-+ class ProcessRowsJob < Spree::Imports::BaseJobBATCH_SIZE = 100def perform(import_id)diff --git a/spree/core/app/models/spree/import_row.rb b/spree/core/app/models/spree/import_row.rbindex 6fbdccd9f9..10a93e5fef 100644--- a/spree/core/app/models/spree/import_row.rb+++ b/spree/core/app/models/spree/import_row.rb@@ -41,6 +41,11 @@ module Spree# are now handled by Spree::Admin::ImportRowSubscriberend+ # How long a row may sit in `processing` before we consider its owning worker dead+ # (OOM, SIGKILL, deploy without graceful drain — none of these trigger a Sidekiq+ # retry). After this window, stalled rows no longer block the import from completing.+ STALLED_PROCESSING_AFTER = 1.hour+## Scopes#@@ -48,6 +53,13 @@ module Spreescope :completed, -> { where(status: :completed) }scope :failed, -> { where(status: :failed) }scope :processed, -> { where(status: %i[completed failed]) }+ # Rows still legitimately blocking import completion: `pending` (not started) or+ # `processing` with a recent updated_at (worker still alive). Orphaned `processing`+ # rows past the stall window are excluded so a dead worker can't permanently block+ # completion — operators can clean those up separately.+ scope :in_flight, -> {+ where(status: :pending).or(where(status: :processing).where(updated_at: STALLED_PROCESSING_AFTER.ago..))+ }def data_json@data_json ||= JSON.parse(data)diff --git a/spree/core/app/services/spree/imports/row_processors/product_variant.rb b/spree/core/app/services/spree/imports/row_processors/product_variant.rbindex ed6f6f9037..5fd42cac21 100644--- a/spree/core/app/services/spree/imports/row_processors/product_variant.rb+++ b/spree/core/app/services/spree/imports/row_processors/product_variant.rb@@ -131,17 +131,72 @@ module Spreedef prepare_option_value_variantsreturn [] if options.empty?- options.map do |option|- option_type = Spree::OptionType.search_by_name(option[:option_name]).first || Spree::OptionType.create!(presentation: option[:option_name])- option_value = option_type.option_values.search_by_name(option[:option_value]).first || option_type.option_values.create!(presentation: option[:option_value])+ ActiveRecord::Base.no_touching do+ options.map do |option|+ option_type = find_or_create_option_type!(option[:option_name])+ option_value = find_or_create_option_value!(option_type, option[:option_value])- # ensure product option types include new option type- Spree::ProductOptionType.find_or_create_by!(product: product, option_type: option_type)+ # ensure product option types include new option type+ find_or_create_product_option_type!(option_type)- Spree::OptionValueVariant.new(option_value: option_value)+ Spree::OptionValueVariant.new(option_value: option_value)+ endendend+ def options+ @options ||= begin+ options = []++ OPTION_TYPES_COUNT.times.map do |index|+ next if attributes["option#{index + 1}_name"].blank?+ next if attributes["option#{index + 1}_value"].blank?++ options << {+ index: index + 1,+ option_name: attributes["option#{index + 1}_name"],+ option_value: attributes["option#{index + 1}_value"]+ }+ end++ options+ end+ end++ # Concurrent CSV imports can race when creating shared OptionTypes/OptionValues.+ # Recover the losing worker by re-fetching the peer's row whether the conflict+ # surfaces via the DB unique index (RecordNotUnique) or the AR uniqueness+ # validator (RecordInvalid with a :taken error on the relevant attribute).+ def find_or_create_option_type!(presentation)+ Spree::OptionType.search_by_name(presentation).first || Spree::OptionType.create!(presentation: presentation)+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e+ raise unless uniqueness_conflict?(e, :name)++ Spree::OptionType.search_by_name(presentation).first!+ end++ def find_or_create_option_value!(option_type, presentation)+ option_type.option_values.search_by_name(presentation).first || option_type.option_values.create!(presentation: presentation)+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e+ raise unless uniqueness_conflict?(e, :name)++ option_type.option_values.search_by_name(presentation).first!+ end++ def find_or_create_product_option_type!(option_type)+ Spree::ProductOptionType.find_or_create_by!(product: product, option_type: option_type)+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e+ raise unless uniqueness_conflict?(e, :product_id)++ Spree::ProductOptionType.find_by!(product: product, option_type: option_type)+ end++ # RecordNotUnique is always a uniqueness conflict; RecordInvalid only when the+ # given attribute has a :taken error (other validation failures must propagate).+ def uniqueness_conflict?(error, attribute)+ error.is_a?(ActiveRecord::RecordNotUnique) || error.record.errors.where(attribute, :taken).any?+ end+def handle_images(variant)image_urls = [attributes['image1_src'],@@ -168,25 +223,6 @@ module Spreeendend- def options- @options ||= begin- options = []-- OPTION_TYPES_COUNT.times.map do |index|- next if attributes["option#{index + 1}_name"].blank?- next if attributes["option#{index + 1}_value"].blank?-- options << {- index: index + 1,- option_name: attributes["option#{index + 1}_name"],- option_value: attributes["option#{index + 1}_value"]- }- end-- options- end- end-def has_product_attributes?%w[name status description category1 category2 category3].any? { |key| attributes[key].present? }enddiff --git a/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb b/spree/core/spec/jobs/spree/imports/process_group_job_spec.rbindex a6e1be9abd..b98c987e77 100644--- a/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb+++ b/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb@@ -40,8 +40,12 @@ RSpec.describe Spree::Imports::ProcessGroupJob, type: :job doexpect(import.status).to eq('completed')end- it 'does not complete import when other groups are still pending' do- import.update_columns(processing_groups_count: 3)+ it 'does not complete import when other groups still have pending rows' do+ # A second group's rows exist but haven't been processed yet; this group finishing+ # its own rows must not flip the import to completed.+ create(:import_row, import: import, row_number: 2, status: :pending,+ data: { 'slug' => 'other-product', 'sku' => 'SKU2', 'name' => 'Other', 'price' => '9.99' }.to_json)+ import.update_columns(processing_groups_count: 2)described_class.perform_now(import.id, [row.id])@@ -67,6 +71,80 @@ RSpec.describe Spree::Imports::ProcessGroupJob, type: :job doendend+ context 'when a row in the passed batch is already completed (job retry)' do+ # Simulates a Sidekiq retry of the group job where some rows finished on the+ # prior attempt — those must not be reprocessed and duplicate side effects.+ let!(:completed_row) do+ create(:import_row, import: import, row_number: 2, status: :completed,+ data: { 'slug' => 'already-done', 'sku' => 'DONE', 'name' => 'Already Done', 'price' => '1.00' }.to_json)+ end++ it 'reprocesses pending/failed rows and skips the completed row' do+ expect {+ described_class.perform_now(import.id, [row.id, completed_row.id])+ }.to change(Spree::Product, :count).by(1) # only the pending row creates a product; completed row is skipped++ expect(row.reload.status).to eq('completed')+ expect(completed_row.reload.status).to eq('completed')+ end+ end++ context 'when the job is retried after all its rows already finished' do+ # Status guard must prevent re-completing an import that finished on the prior+ # attempt — otherwise a transient failure inside check_import_completion's+ # post-increment side effects would emit duplicate complete! transitions on retry.+ before do+ row.update_columns(status: 'completed')+ import.update_columns(status: 'completed', completed_groups_count: 1, processing_groups_count: 1)+ end++ it 'is a no-op for completed imports' do+ expect {+ described_class.perform_now(import.id, [row.id])+ }.not_to change { [import.reload.status, row.reload.status] }+ end+ end++ context 'when a sibling row is orphaned in processing (worker killed)' do+ # A row whose worker died (OOM, SIGKILL, deploy without graceful drain) stays+ # in `processing` indefinitely. Once it's been there longer than the stall window,+ # it must not block the import from completing — otherwise a dead worker+ # permanently jams every import that lost a row.+ let!(:orphaned_row) do+ create(:import_row, import: import, row_number: 2, status: :processing,+ data: { 'slug' => 'orphan', 'sku' => 'ORPHAN', 'name' => 'Orphan', 'price' => '1.00' }.to_json)+ end++ before do+ orphaned_row.update_columns(updated_at: (Spree::ImportRow::STALLED_PROCESSING_AFTER + 5.minutes).ago)+ import.update_columns(processing_groups_count: 2, completed_groups_count: 1)+ end++ it 'completes the import despite the stalled row' do+ described_class.perform_now(import.id, [row.id])++ expect(import.reload.status).to eq('completed')+ expect(orphaned_row.reload.status).to eq('processing') # left alone for operator review+ end+ end++ context 'when a sibling row is still actively processing' do+ let!(:active_row) do+ create(:import_row, import: import, row_number: 2, status: :processing,+ data: { 'slug' => 'active', 'sku' => 'ACTIVE', 'name' => 'Active', 'price' => '1.00' }.to_json)+ end++ before do+ import.update_columns(processing_groups_count: 2, completed_groups_count: 1)+ end++ it 'does not complete the import while a live worker is still on a row' do+ described_class.perform_now(import.id, [row.id])++ expect(import.reload.status).to eq('processing')+ end+ end+context 'with product + variant rows in a group' dolet!(:variant_row) docreate(:import_row, import: import, row_number: 2, status: :pending,@@ -158,6 +236,10 @@ RSpec.describe Spree::Imports::ProcessGroupJob, type: :job doendit 'publishes import.progress every 10 groups' do+ # Need at least one row outside this group still pending, otherwise the import+ # finishes and the progress branch never fires.+ create(:import_row, import: import, row_number: 2, status: :pending,+ data: { 'slug' => 'other-product', 'sku' => 'SKU9', 'name' => 'Other', 'price' => '1.00' }.to_json)import.update_columns(processing_groups_count: 20, completed_groups_count: 9)expect_any_instance_of(Spree::Import).to receive(:publish_event).with('import.progress')@@ -166,6 +248,8 @@ RSpec.describe Spree::Imports::ProcessGroupJob, type: :job doendit 'does not publish progress on non-10th group' do+ create(:import_row, import: import, row_number: 2, status: :pending,+ data: { 'slug' => 'other-product', 'sku' => 'SKU9', 'name' => 'Other', 'price' => '1.00' }.to_json)import.update_columns(processing_groups_count: 20, completed_groups_count: 7)expect_any_instance_of(Spree::Import).not_to receive(:publish_event).with('import.progress')@@ -176,6 +260,10 @@ RSpec.describe Spree::Imports::ProcessGroupJob, type: :job dodescribe 'atomic completion tracking' doit 'uses atomic SQL increment for completed_groups_count' do+ # Leave another row pending so the import doesn't finalize on this run; we want+ # to assert on the in-flight counter, not the final state.+ create(:import_row, import: import, row_number: 2, status: :pending,+ data: { 'slug' => 'other-product', 'sku' => 'SKUX', 'name' => 'Other', 'price' => '1.00' }.to_json)import.update_columns(processing_groups_count: 2, completed_groups_count: 0)described_class.perform_now(import.id, [row.id])diff --git a/spree/core/spec/services/spree/imports/row_processors/product_variant_spec.rb b/spree/core/spec/services/spree/imports/row_processors/product_variant_spec.rbindex fd7bd5608a..053acdeaa0 100644--- a/spree/core/spec/services/spree/imports/row_processors/product_variant_spec.rb+++ b/spree/core/spec/services/spree/imports/row_processors/product_variant_spec.rb@@ -140,6 +140,65 @@ RSpec.describe Spree::Imports::RowProcessors::ProductVariant, type: :service doexpect(size_option_type.option_values.find_by(presentation: 'XS')).to be_presentend+ context 'when a concurrent worker has already created the option type/value' do+ # Simulates two Sidekiq workers racing on the same option name during a CSV import.+ # Force search_by_name to miss on the initial lookup so create! runs, then return+ # real results on retry so the rescue path can find the peer's record.+ # Option values don't need stubbing — they don't exist yet, so search_by_name+ # naturally misses and create! succeeds.+ before do+ seen_option_types = Set.new+ allow(Spree::OptionType).to receive(:search_by_name).and_wrap_original do |original, query|+ seen_option_types.add?(query) ? Spree::OptionType.none : original.call(query)+ end+ end++ it 'recovers from the AR uniqueness validator and reuses existing records' do+ expect { subject.process! }.not_to raise_error+ expect(variant.option_values.map(&:presentation).sort).to contain_exactly('Blue', 'XS')+ expect(Spree::OptionType.where(name: 'color').count).to eq 1+ expect(Spree::OptionType.where(name: 'size').count).to eq 1+ end++ context 'when the create! reaches the DB and the unique index rejects it' do+ # Mirrors the production race: validator passed (peer not committed yet) but+ # the INSERT collides at the DB once the peer commits. Pre-create the peer rows+ # so find_by!/search_by_name can locate them on retry.+ # Option values are pre-created, so search_by_name naturally finds them.+ let!(:color_blue) { create(:option_value, name: 'Blue', presentation: 'Blue', option_type: Spree::OptionType.find_by(name: 'color')) }+ let!(:size_xs) { create(:option_value, name: 'XS', presentation: 'XS', option_type: Spree::OptionType.find_by(name: 'size')) }++ before do+ allow(Spree::OptionType).to receive(:create!).and_raise(ActiveRecord::RecordNotUnique)+ end++ it 'recovers and reuses existing records' do+ expect { subject.process! }.not_to raise_error+ expect(variant.option_values.map(&:presentation).sort).to contain_exactly('Blue', 'XS')+ expect(Spree::OptionType.where(name: 'color').count).to eq 1+ expect(Spree::OptionType.where(name: 'size').count).to eq 1+ expect(Spree::OptionValue.where(name: 'blue').count).to eq 1+ expect(Spree::OptionValue.where(name: 'xs').count).to eq 1+ end+ end++ context 'when a concurrent worker causes a ProductOptionType uniqueness conflict' do+ before do+ # Stub find_or_create_by! to raise RecordInvalid with a product_id taken error,+ # simulating a peer worker that committed the same product-option_type association.+ record = Spree::ProductOptionType.new(product: product)+ record.errors.add(:product_id, :taken)+ allow(Spree::ProductOptionType).to receive(:find_or_create_by!).and_raise(ActiveRecord::RecordInvalid.new(record))+ allow(Spree::ProductOptionType).to receive(:find_by!).and_call_original+ end++ it 'recovers and reuses the existing product option type' do+ expect { subject.process! }.not_to raise_error+ expect(variant.option_values.map(&:presentation).sort).to contain_exactly('Blue', 'XS')+ end+ end+ end+context 'when importing a variant row for existing variant' dolet(:color_option_type) { Spree::OptionType.find_by(name: 'color') || create(:option_type, name: 'color', presentation: 'Color') }let(:size_option_type) { Spree::OptionType.find_by(name: 'size') || create(:option_type, name: 'size', presentation: 'Size') }