Open Source Platform for DTC, B2B Commerce, Marketplaces & Omnichannel. REST APIs, TypeScript SDKs, and production-ready Next.js storefront. Self-host it. Own your data. No vendor lock-in. Zero platform fees.
Ruby b2b-commercee-commerceecommerceecommerce-apiecommerce-frameworkecommerce-platformheadlessheadless-commerceheadless-ecommercemarketplacemulti-tenantmulti-vendormulti-vendor-ecommerceopen-sourcespree-commerce
Parallel CSV import processing for large files (#13944)
Damian Legawiec6b6e781b5312
diff --git a/spree/admin/app/controllers/spree/admin/imports_controller.rb b/spree/admin/app/controllers/spree/admin/imports_controller.rbindex de3b8e1f7d..8bb2c0542e 100644--- a/spree/admin/app/controllers/spree/admin/imports_controller.rb+++ b/spree/admin/app/controllers/spree/admin/imports_controller.rb@@ -14,7 +14,7 @@ module Spreeif @object.status == 'mapping'@mappings = @object.mappings@mappings_options = @object.unmapped_file_columns.map { |file_column| [file_column, file_column] }- else+ elsif [email protected]_import?@rows = @object.rows.processed.includes(:item)endenddiff --git a/spree/admin/app/subscribers/spree/admin/import_row_subscriber.rb b/spree/admin/app/subscribers/spree/admin/import_row_subscriber.rbindex e729f81aad..e5b6b4c3af 100644--- a/spree/admin/app/subscribers/spree/admin/import_row_subscriber.rb+++ b/spree/admin/app/subscribers/spree/admin/import_row_subscriber.rb@@ -23,6 +23,7 @@ module Spreeimport_row = Spree::ImportRow.find_by_prefix_id(import_row_id)return unless import_row+ return if import_row.import.large_import?add_row_to_import_view(import_row)update_footer_in_import_view(import_row)diff --git a/spree/admin/app/subscribers/spree/admin/import_subscriber.rb b/spree/admin/app/subscribers/spree/admin/import_subscriber.rbindex f88720d24e..86c5ad9ec6 100644--- a/spree/admin/app/subscribers/spree/admin/import_subscriber.rb+++ b/spree/admin/app/subscribers/spree/admin/import_subscriber.rb@@ -4,24 +4,17 @@ module Spreemodule Admin# Handles Import events for the admin interface.#- # This subscriber listens to import.completed event and handles:- # - Updating the loader in the import view (Turbo Streams)- #- # We use async: false because the UI update should happen immediately- # after the import completes.+ # We use async: false because the UI updates should happen immediately.#class ImportSubscriber < Spree::Subscriber- subscribes_to 'import.completed', async: false+ subscribes_to 'import.completed', 'import.progress', async: falseon 'import.completed', :update_loader_in_import_view+ on 'import.progress', :update_footer_in_import_viewdef update_loader_in_import_view(event)- import_id = event.payload['id']- return unless import_id-- import = Spree::Import.find_by_prefix_id(import_id)+ import = find_import(event)return unless import- return unless import.respond_to?(:broadcast_update_to)import.broadcast_update_to("import_#{import.id}_loader",@@ -30,6 +23,27 @@ module Spreelocals: { import: import })end++ def update_footer_in_import_view(event)+ import = find_import(event)+ return unless import++ import.broadcast_replace_to(+ "import_#{import.id}_footer",+ target: 'footer',+ partial: 'spree/admin/imports/footer',+ locals: { import: import }+ )+ end++ private++ def find_import(event)+ import_id = event.payload['id']+ return unless import_id++ Spree::Import.find_by_prefix_id(import_id)+ endendendenddiff --git a/spree/admin/app/views/spree/admin/imports/_loader.html.erb b/spree/admin/app/views/spree/admin/imports/_loader.html.erbindex 6c980eae29..b289fd9920 100644--- a/spree/admin/app/views/spree/admin/imports/_loader.html.erb+++ b/spree/admin/app/views/spree/admin/imports/_loader.html.erb@@ -2,6 +2,11 @@<div class="flex justify-center items-center my-12 flex-col pb-12"><%= icon('check', class: 'text-green-700', style: 'font-size: 4rem;') %><p class="text-green-700 mb-3"><%= Spree.t('admin.imports.import_done') %>!</p>+ <% if import.large_import? %>+ <p class="text-gray-600 mb-3">+ <%= Spree.t('admin.imports.large_import_summary', completed: import.rows.completed.count, failed: import.rows.failed.count) %>+ </p>+ <% end %><%= link_to_with_icon 'arrow-back-up', Spree.t('admin.back_to_dashboard'), spree.admin_path, class: 'btn btn-light mb-12' %></div><% else %>diff --git a/spree/admin/app/views/spree/admin/imports/show.html.erb b/spree/admin/app/views/spree/admin/imports/show.html.erbindex 01b652760a..3739a46424 100644--- a/spree/admin/app/views/spree/admin/imports/show.html.erb+++ b/spree/admin/app/views/spree/admin/imports/show.html.erb@@ -25,6 +25,20 @@</div><%= render 'spree/admin/imports/mapping_footer', import: @import %>+<% elsif @import.large_import? %>+ <%= turbo_stream_from "import_#{@import.id}_footer" %>+ <%= turbo_stream_from "import_#{@import.id}_loader" %>++ <div class="p-6 text-center text-gray-600 my-8">+ <p class="mb-2"><%= Spree.t('admin.imports.large_import_message', count: @import.rows_count) %></p>+ <p><%= Spree.t('admin.imports.large_import_background_note') %></p>+ </div>++ <div id="loader" data-controller="auto-scroll">+ <%= render 'spree/admin/imports/loader', import: @import %>+ </div>++ <%= render 'spree/admin/imports/footer', import: @import %><% else %><%= turbo_stream_from "import_#{@import.id}_rows" %><%= turbo_stream_from "import_#{@import.id}_footer" %>diff --git a/spree/admin/config/locales/en.yml b/spree/admin/config/locales/en.ymlindex 7d1c7b91f8..b682cb07f1 100644--- a/spree/admin/config/locales/en.yml+++ b/spree/admin/config/locales/en.yml@@ -136,6 +136,9 @@ en:all_required_columns_mapped: Good job, all required columns are mapped!field_required: Field requiredimport_done: All done!+ large_import_background_note: You can close this page — the import will continue in the background.+ large_import_message: This import contains %{count} rows and is too large to display individually.+ large_import_summary: "%{completed} completed, %{failed} failed"mapping_required: This field is requiredplease_map_all_required_fields: Please map all the required fields to continue.waiting_for_file_to_be_processed: Waiting for file to be processed...diff --git a/spree/core/app/jobs/spree/imports/process_group_job.rb b/spree/core/app/jobs/spree/imports/process_group_job.rbnew file mode 100644index 0000000000..95b5dcec3a--- /dev/null+++ b/spree/core/app/jobs/spree/imports/process_group_job.rb@@ -0,0 +1,45 @@+module Spree+ module Imports+ class ProcessGroupJob < Spree::BaseJob+ queue_as Spree.queues.imports++ def perform(import_id, row_ids)+ import = Spree::Import.find(import_id)+ Spree::Current.store = import.store++ mappings = import.mappings.mapped.to_a+ schema_fields = import.schema_fields+ large = import.large_import?+ rows = import.rows.where(id: row_ids).order(:row_number)++ if large+ Spree::Events.disable do+ rows.each { |row| row.bulk_process!(mappings: mappings, schema_fields: schema_fields) }+ end+ else+ rows.each do |row|+ row.process!(mappings: mappings, schema_fields: schema_fields)+ end+ end++ check_import_completion(import, large)+ end++ private++ def check_import_completion(import, large)+ Spree::Import.where(id: import.id).update_all(+ 'completed_groups_count = COALESCE(completed_groups_count, 0) + 1'+ )+ import.reload++ if import.completed_groups_count >= import.processing_groups_count+ # Guard against concurrent workers both reaching this point+ import.complete! if import.status == 'processing'+ elsif large && (import.completed_groups_count % 10).zero?+ import.publish_event('import.progress')+ end+ end+ end+ end+enddiff --git a/spree/core/app/jobs/spree/imports/process_rows_job.rb b/spree/core/app/jobs/spree/imports/process_rows_job.rbindex dc9566ba06..68c3340532 100644--- a/spree/core/app/jobs/spree/imports/process_rows_job.rb+++ b/spree/core/app/jobs/spree/imports/process_rows_job.rb@@ -3,16 +3,62 @@ module Spreeclass ProcessRowsJob < Spree::BaseJobqueue_as Spree.queues.imports+ BATCH_SIZE = 100+def perform(import_id)import = Spree::Import.find(import_id)+ dispatch_groups(import)+ end++ private++ def dispatch_groups(import)+ group_field = import.group_column+ group_mapping = group_field && import.mappings.mapped.find_by(schema_field: group_field)+ file_column = group_mapping&.file_column- # process all rows in sequential order- import.rows.pending_and_failed.find_each(batch_size: 100) do |row|- row.process!+ if file_column+ dispatch_grouped(import, file_column)+ else+ dispatch_batched(import)end+ end++ def dispatch_grouped(import, file_column)+ groups = Hash.new { |h, k| h[k] = [] }++ import.rows.pending_and_failed.order(:row_number).pluck(:id, :data).each do |id, data|+ parsed = JSON.parse(data)+ key = parsed[file_column].to_s.strip.downcase.presence || '__ungrouped__'+ groups[key] << id+ rescue JSON::ParserError+ groups['__ungrouped__'] << id+ end++ # Set count before enqueuing so workers can't complete prematurely+ import.update_columns(+ processing_groups_count: groups.size,+ completed_groups_count: 0,+ updated_at: Time.current+ )++ groups.each_value { |row_ids| ProcessGroupJob.perform_later(import.id, row_ids) }+ end++ def dispatch_batched(import)+ # Count first, then enqueue — prevents premature completion+ row_id_batches = import.rows.pending_and_failed.order(:row_number)+ .pluck(:id)+ .each_slice(BATCH_SIZE)+ .to_a++ import.update_columns(+ processing_groups_count: row_id_batches.size,+ completed_groups_count: 0,+ updated_at: Time.current+ )- # mark as complete- import.complete!+ row_id_batches.each { |row_ids| ProcessGroupJob.perform_later(import.id, row_ids) }endendenddiff --git a/spree/core/app/models/spree/import.rb b/spree/core/app/models/spree/import.rbindex d68aa3157c..9956739a6c 100644--- a/spree/core/app/models/spree/import.rb+++ b/spree/core/app/models/spree/import.rb@@ -76,6 +76,21 @@ module Spree#preference :delimiter, :string, default: ','+ # Returns true if the import has more rows than the large import threshold.+ # Large imports skip per-row UI broadcasts and use bulk processing.+ # @return [Boolean]+ def large_import?+ rows_count >= Spree::Config[:large_import_threshold]+ end++ # Returns the schema field name used to group rows for parallel processing.+ # Rows sharing the same value in this field are processed together in one job.+ # Returns nil for imports where rows are independent (default — batched in chunks).+ # @return [String, nil]+ def group_column+ nil+ end+# Returns true if the import is in mapping state# @return [Boolean]def mapping?diff --git a/spree/core/app/models/spree/import_row.rb b/spree/core/app/models/spree/import_row.rbindex 65784485ac..6fbdccd9f9 100644--- a/spree/core/app/models/spree/import_row.rb+++ b/spree/core/app/models/spree/import_row.rb@@ -77,9 +77,9 @@ module Spreedata_json[mapping.file_column]end- def process!+ def process!(mappings: nil, schema_fields: nil)start_processing!- self.item = import.row_processor_class.new(self).process!+ self.item = import.row_processor_class.new(self, mappings: mappings, schema_fields: schema_fields).process!complete!rescue StandardError => eRails.error.report(e, handled: true, context: { import_row_id: id }, source: 'spree.core')@@ -87,6 +87,18 @@ module Spreefail!end+ # Bulk processing mode for large imports.+ # Uses update_columns to skip callbacks, validations, and event publishing.+ def bulk_process!(mappings:, schema_fields:)+ update_columns(status: 'processing', updated_at: Time.current)+ processor = import.row_processor_class.new(self, mappings: mappings, schema_fields: schema_fields)+ self.item = processor.process!+ update_columns(status: 'completed', item_type: item.class.name, item_id: item.id, updated_at: Time.current)+ rescue StandardError => e+ Rails.error.report(e, handled: true, context: { import_row_id: id }, source: 'spree.core')+ update_columns(status: 'failed', validation_errors: e.message, updated_at: Time.current)+ end+def publish_import_row_completed_eventpublish_event('import_row.completed')enddiff --git a/spree/core/app/models/spree/imports/product_translations.rb b/spree/core/app/models/spree/imports/product_translations.rbindex e188baf078..1845e2a091 100644--- a/spree/core/app/models/spree/imports/product_translations.rb+++ b/spree/core/app/models/spree/imports/product_translations.rb@@ -5,6 +5,10 @@ module SpreeSpree::Imports::RowProcessors::ProductTranslationend+ def group_column+ 'slug'+ end+def model_classSpree::Productenddiff --git a/spree/core/app/models/spree/imports/products.rb b/spree/core/app/models/spree/imports/products.rbindex e6dd5eb6b7..489a895555 100644--- a/spree/core/app/models/spree/imports/products.rb+++ b/spree/core/app/models/spree/imports/products.rb@@ -4,6 +4,11 @@ module Spreedef row_processor_classSpree::Imports::RowProcessors::ProductVariantend++ # Group by slug: product row + its variant rows must be processed together+ def group_column+ 'slug'+ endendendenddiff --git a/spree/core/app/services/spree/imports/row_processors/base.rb b/spree/core/app/services/spree/imports/row_processors/base.rbindex 1e109d3ad0..862ed8bd9c 100644--- a/spree/core/app/services/spree/imports/row_processors/base.rb+++ b/spree/core/app/services/spree/imports/row_processors/base.rb@@ -2,10 +2,14 @@ module Spreemodule Importsmodule RowProcessorsclass Base- def initialize(row)+ def initialize(row, mappings: nil, schema_fields: nil)@row = row@import = row.import- @attributes = row.to_schema_hash+ @attributes = if mappings && schema_fields+ build_schema_hash(row, mappings, schema_fields)+ else+ row.to_schema_hash+ endendattr_reader :row, :import, :attributes@@ -13,6 +17,19 @@ module Spreedef process!raise NotImplementedError, 'Subclasses must implement the process! method'end++ private++ def build_schema_hash(row, mappings, schema_fields)+ attributes = {}+ schema_fields.each do |field|+ mapping = mappings.find { |m| m.schema_field == field[:name] }+ next unless mapping&.mapped?++ attributes[field[:name]] = row.data_json[mapping.file_column]+ end+ attributes+ endendendenddiff --git a/spree/core/app/services/spree/imports/row_processors/product_translation.rb b/spree/core/app/services/spree/imports/row_processors/product_translation.rbindex cf53f5b42b..9361b9a29f 100644--- a/spree/core/app/services/spree/imports/row_processors/product_translation.rb+++ b/spree/core/app/services/spree/imports/row_processors/product_translation.rb@@ -4,7 +4,7 @@ module Spreeclass ProductTranslation < BaseTRANSLATABLE_FIELDS = %w[name description meta_title meta_description].freeze- def initialize(row)+ def initialize(row, **)super@store = row.storeenddiff --git a/spree/core/app/services/spree/imports/row_processors/product_variant.rb b/spree/core/app/services/spree/imports/row_processors/product_variant.rbindex a1b27a88ee..b39dcc00c7 100644--- a/spree/core/app/services/spree/imports/row_processors/product_variant.rb+++ b/spree/core/app/services/spree/imports/row_processors/product_variant.rb@@ -4,7 +4,7 @@ module Spreeclass ProductVariant < BaseOPTION_TYPES_COUNT = 3- def initialize(row)+ def initialize(row, **)super@store = row.store@product = ensure_product_existsdiff --git a/spree/core/db/migrate/20260424100000_add_processing_groups_to_spree_imports.rb b/spree/core/db/migrate/20260424100000_add_processing_groups_to_spree_imports.rbnew file mode 100644index 0000000000..29fa221a80--- /dev/null+++ b/spree/core/db/migrate/20260424100000_add_processing_groups_to_spree_imports.rb@@ -0,0 +1,6 @@+class AddProcessingGroupsToSpreeImports < ActiveRecord::Migration[7.2]+ def change+ add_column :spree_imports, :processing_groups_count, :integer, default: 0+ add_column :spree_imports, :completed_groups_count, :integer, default: 0+ end+enddiff --git a/spree/core/lib/spree/core/configuration.rb b/spree/core/lib/spree/core/configuration.rbindex 07689fae83..0251add6b0 100644--- a/spree/core/lib/spree/core/configuration.rb+++ b/spree/core/lib/spree/core/configuration.rb@@ -117,6 +117,9 @@ module Spreepreference :gift_card_batch_web_limit, :integer, default: 500 # number of gift card codes to be generated in the web process, more than this will be generated in a background jobpreference :gift_card_batch_limit, :integer, default: 50_000+ # imports+ preference :large_import_threshold, :integer, default: 500 # imports with more rows than this skip per-row UI broadcasts and use bulk processing+endendenddiff --git a/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb b/spree/core/spec/jobs/spree/imports/process_group_job_spec.rbnew file mode 100644index 0000000000..a6e1be9abd--- /dev/null+++ b/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb@@ -0,0 +1,187 @@+require 'spec_helper'++RSpec.describe Spree::Imports::ProcessGroupJob, type: :job do+ let(:store) { @default_store }++ let!(:import) do+ create(:import, owner: store, type: 'Spree::Imports::Products', status: :processing,+ processing_groups_count: 1, completed_groups_count: 0)+ end++ let!(:row) do+ create(:import_row, import: import, row_number: 1, status: :pending,+ data: { 'slug' => 'test-product', 'sku' => 'SKU1', 'name' => 'Test Product', 'price' => '9.99' }.to_json)+ end++ before do+ allow_any_instance_of(Spree::Import).to receive(:csv_headers).and_return(['slug', 'sku', 'name', 'price'])+ import.create_mappings+ end++ it 'sets Spree::Current.store from the import' do+ described_class.perform_now(import.id, [row.id])++ expect(Spree::Current.store).to eq(store)+ end++ it 'creates the product and completes the import' do+ expect {+ described_class.perform_now(import.id, [row.id])+ }.to change(Spree::Product, :count).by(1)++ row.reload+ expect(row.status).to eq('completed')+ expect(row.item).to be_a(Spree::Variant)+ expect(row.item.product.name).to eq('Test Product')+ expect(row.item.sku).to eq('SKU1')++ import.reload+ expect(import.completed_groups_count).to eq(1)+ expect(import.status).to eq('completed')+ end++ it 'does not complete import when other groups are still pending' do+ import.update_columns(processing_groups_count: 3)++ described_class.perform_now(import.id, [row.id])++ import.reload+ expect(import.completed_groups_count).to eq(1)+ expect(import.status).to eq('processing')+ end++ context 'when a row fails' do+ before do+ allow_any_instance_of(Spree::Imports::RowProcessors::ProductVariant).to receive(:process!).and_raise(StandardError, 'something went wrong')+ end++ it 'marks the row as failed and continues' do+ described_class.perform_now(import.id, [row.id])++ row.reload+ expect(row.status).to eq('failed')+ expect(row.validation_errors).to eq('something went wrong')++ # Import still completes (single group)+ expect(import.reload.status).to eq('completed')+ end+ end++ context 'with product + variant rows in a group' do+ let!(:variant_row) do+ create(:import_row, import: import, row_number: 2, status: :pending,+ data: { 'slug' => 'test-product', 'sku' => 'SKU2', 'price' => '19.99', 'option1_name' => 'Color', 'option1_value' => 'Red' }.to_json)+ end++ before do+ allow_any_instance_of(Spree::Import).to receive(:csv_headers).and_return(['slug', 'sku', 'name', 'price', 'option1_name', 'option1_value'])+ import.mappings.destroy_all+ import.create_mappings+ end++ it 'creates product and variant in row_number order' do+ expect {+ described_class.perform_now(import.id, [variant_row.id, row.id])+ }.to change(Spree::Product, :count).by(1)++ expect(row.reload.status).to eq('completed')+ expect(variant_row.reload.status).to eq('completed')++ product = row.item.product+ expect(product.name).to eq('Test Product')+ expect(variant_row.item).to be_a(Spree::Variant)+ expect(variant_row.item.product).to eq(product)+ expect(variant_row.item.sku).to eq('SKU2')+ end+ end++ describe 'large import (bulk_process!)' do+ before do+ allow_any_instance_of(Spree::Import).to receive(:large_import?).and_return(true)+ end++ it 'creates the product via bulk_process!' do+ expect {+ described_class.perform_now(import.id, [row.id])+ }.to change(Spree::Product, :count).by(1)++ row.reload+ expect(row.status).to eq('completed')+ expect(row.item).to be_a(Spree::Variant)+ expect(row.item.product.name).to eq('Test Product')+ expect(row.item.sku).to eq('SKU1')+ end++ context 'with product + variant rows' do+ let!(:variant_row) do+ create(:import_row, import: import, row_number: 2, status: :pending,+ data: { 'slug' => 'test-product', 'sku' => 'SKU2', 'price' => '19.99', 'option1_name' => 'Color', 'option1_value' => 'Red' }.to_json)+ end++ before do+ allow_any_instance_of(Spree::Import).to receive(:csv_headers).and_return(['slug', 'sku', 'name', 'price', 'option1_name', 'option1_value'])+ import.mappings.destroy_all+ import.create_mappings+ end++ it 'creates product and variant via bulk_process!' do+ expect {+ described_class.perform_now(import.id, [row.id, variant_row.id])+ }.to change(Spree::Product, :count).by(1)+ .and change(Spree::Variant, :count).by(2) # master + variant++ row.reload+ variant_row.reload+ expect(row.status).to eq('completed')+ expect(variant_row.status).to eq('completed')++ product = row.item.product+ expect(product.name).to eq('Test Product')+ expect(variant_row.item.product).to eq(product)+ expect(variant_row.item.sku).to eq('SKU2')+ end+ end++ it 'disables events during processing' do+ events_were_disabled = false++ original_bulk_process = Spree::ImportRow.instance_method(:bulk_process!)+ allow_any_instance_of(Spree::ImportRow).to receive(:bulk_process!) do |row_instance, **kwargs|+ events_were_disabled = !Spree::Events.enabled?+ original_bulk_process.bind_call(row_instance, **kwargs)+ end++ described_class.perform_now(import.id, [row.id])++ expect(events_were_disabled).to be true+ expect(row.reload.status).to eq('completed')+ end++ it 'publishes import.progress every 10 groups' do+ import.update_columns(processing_groups_count: 20, completed_groups_count: 9)++ expect_any_instance_of(Spree::Import).to receive(:publish_event).with('import.progress')++ described_class.perform_now(import.id, [row.id])+ end++ it 'does not publish progress on non-10th group' do+ import.update_columns(processing_groups_count: 20, completed_groups_count: 7)++ expect_any_instance_of(Spree::Import).not_to receive(:publish_event).with('import.progress')++ described_class.perform_now(import.id, [row.id])+ end+ end++ describe 'atomic completion tracking' do+ it 'uses atomic SQL increment for completed_groups_count' do+ import.update_columns(processing_groups_count: 2, completed_groups_count: 0)++ described_class.perform_now(import.id, [row.id])++ expect(import.reload.completed_groups_count).to eq(1)+ expect(import.status).to eq('processing')+ end+ end+enddiff --git a/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb b/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rbindex bec0e617ed..eec4368ff8 100644--- a/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb+++ b/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb@@ -3,59 +3,102 @@ require 'spec_helper'RSpec.describe Spree::Imports::ProcessRowsJob, type: :job dolet(:store) { @default_store }- let!(:import) do- create(- :import,- owner: store,- type: 'Spree::Imports::Products',- status: :processing- )- end+ describe 'grouped import (Products)' do+ let!(:import) do+ create(:import, owner: store, type: 'Spree::Imports::Products', status: :processing)+ end- let!(:pending_row) do- create(- :import_row,- import: import,- row_number: 1,- status: :pending,- data: { 'slug' => 'test-product', 'sku' => 'SKU1', 'name' => 'Test Product', 'price' => '9.99' }.to_json- )- end+ before do+ allow_any_instance_of(Spree::Import).to receive(:csv_headers).and_return(['slug', 'sku', 'name', 'price', 'option1_name', 'option1_value'])+ import.create_mappings+ end- let!(:failed_row) do- create(- :import_row,- import: import,- row_number: 2,- status: :failed,- data: { 'slug' => 'failed-product', 'sku' => 'SKU2', 'name' => 'Failed Product', 'price' => '19.99' }.to_json- )- end+ let!(:product_row) do+ create(:import_row, import: import, row_number: 1, status: :pending,+ data: { 'slug' => 'denim-shirt', 'sku' => 'SKU1', 'name' => 'Denim Shirt', 'price' => '9.99' }.to_json)+ end- let!(:processed_row) do- create(- :import_row,- import: import,- row_number: 3,- status: :completed,- data: { 'slug' => 'processed-product', 'sku' => 'SKU3', 'name' => 'Processed Product', 'price' => '29.99' }.to_json- )- end+ let!(:variant_row) do+ create(:import_row, import: import, row_number: 2, status: :pending,+ data: { 'slug' => 'denim-shirt', 'sku' => 'SKU2', 'price' => '19.99', 'option1_name' => 'Color', 'option1_value' => 'Red' }.to_json)+ end- before do- allow(import).to receive(:csv_headers).and_return(['slug', 'sku', 'name', 'price'])- import.create_mappings- end+ let!(:other_product_row) do+ create(:import_row, import: import, row_number: 3, status: :pending,+ data: { 'slug' => 'cotton-tee', 'sku' => 'SKU3', 'name' => 'Cotton Tee', 'price' => '14.99' }.to_json)+ end++ it 'groups rows by slug and dispatches one job per product' do+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, [product_row.id, variant_row.id])+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, [other_product_row.id])- it 'processes pending and failed rows' do- expect {described_class.perform_now(import.id)- }.to change { pending_row.reload.status }.from('pending').to('completed').and change { failed_row.reload.status }.from('failed').to('completed')++ import.reload+ expect(import.processing_groups_count).to eq(2)+ expect(import.completed_groups_count).to eq(0)+ end++ it 'skips already completed rows' do+ product_row.update_columns(status: 'completed')++ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, [variant_row.id])+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, [other_product_row.id])++ described_class.perform_now(import.id)++ expect(import.reload.processing_groups_count).to eq(2)+ end++ context 'with failed rows' do+ before { variant_row.update_columns(status: 'failed') }++ it 'includes failed rows for reprocessing' do+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, [product_row.id, variant_row.id])+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, [other_product_row.id])++ described_class.perform_now(import.id)+ end+ endend- it 'marks import as complete after processing all rows' do- expect {+ describe 'batched import (Customers — no group_column)' do+ let!(:import) do+ create(:import, owner: store, type: 'Spree::Imports::Customers', status: :processing)+ end++ let!(:rows) do+ 5.times.map do |i|+ create(:import_row, import: import, row_number: i + 1, status: :pending,+ data: { 'email' => "user#{i}@example.com" }.to_json)+ end+ end++ it 'batches all rows into chunks and dispatches' do+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, rows.map(&:id))+described_class.perform_now(import.id)- }.to change { import.reload.status }.from('processing').to('completed')++ expect(import.reload.processing_groups_count).to eq(1)+ end++ context 'with more rows than BATCH_SIZE' do+ before { stub_const('Spree::Imports::ProcessRowsJob::BATCH_SIZE', 2) }++ it 'dispatches multiple batch jobs' do+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later).exactly(3).times++ described_class.perform_now(import.id)++ expect(import.reload.processing_groups_count).to eq(3)+ end+ endendend