Skip to content

armgit / spree

Public Imported from GitHub
0

Open Source Platform for DTC, B2B Commerce, Marketplaces & Omnichannel. REST APIs, TypeScript SDKs, and production-ready Next.js storefront. Self-host it. Own your data. No vendor lock-in. Zero platform fees.

Parallel CSV import processing for large files (#13944)

Damian Legawiec6b6e781b5312
diff --git a/spree/admin/app/controllers/spree/admin/imports_controller.rb b/spree/admin/app/controllers/spree/admin/imports_controller.rb
index de3b8e1f7d..8bb2c0542e 100644
--- a/spree/admin/app/controllers/spree/admin/imports_controller.rb
+++ b/spree/admin/app/controllers/spree/admin/imports_controller.rb
@@ -14,7 +14,7 @@ module Spree
if @object.status == 'mapping'
@mappings = @object.mappings
@mappings_options = @object.unmapped_file_columns.map { |file_column| [file_column, file_column] }
- else
+ elsif [email protected]_import?
@rows = @object.rows.processed.includes(:item)
end
end
diff --git a/spree/admin/app/subscribers/spree/admin/import_row_subscriber.rb b/spree/admin/app/subscribers/spree/admin/import_row_subscriber.rb
index e729f81aad..e5b6b4c3af 100644
--- a/spree/admin/app/subscribers/spree/admin/import_row_subscriber.rb
+++ b/spree/admin/app/subscribers/spree/admin/import_row_subscriber.rb
@@ -23,6 +23,7 @@ module Spree
import_row = Spree::ImportRow.find_by_prefix_id(import_row_id)
return unless import_row
+ return if import_row.import.large_import?
add_row_to_import_view(import_row)
update_footer_in_import_view(import_row)
diff --git a/spree/admin/app/subscribers/spree/admin/import_subscriber.rb b/spree/admin/app/subscribers/spree/admin/import_subscriber.rb
index f88720d24e..86c5ad9ec6 100644
--- a/spree/admin/app/subscribers/spree/admin/import_subscriber.rb
+++ b/spree/admin/app/subscribers/spree/admin/import_subscriber.rb
@@ -4,24 +4,17 @@ module Spree
module Admin
# Handles Import events for the admin interface.
#
- # This subscriber listens to import.completed event and handles:
- # - Updating the loader in the import view (Turbo Streams)
- #
- # We use async: false because the UI update should happen immediately
- # after the import completes.
+ # We use async: false because the UI updates should happen immediately.
#
class ImportSubscriber < Spree::Subscriber
- subscribes_to 'import.completed', async: false
+ subscribes_to 'import.completed', 'import.progress', async: false
on 'import.completed', :update_loader_in_import_view
+ on 'import.progress', :update_footer_in_import_view
def update_loader_in_import_view(event)
- import_id = event.payload['id']
- return unless import_id
-
- import = Spree::Import.find_by_prefix_id(import_id)
+ import = find_import(event)
return unless import
- return unless import.respond_to?(:broadcast_update_to)
import.broadcast_update_to(
"import_#{import.id}_loader",
@@ -30,6 +23,27 @@ module Spree
locals: { import: import }
)
end
+
+ def update_footer_in_import_view(event)
+ import = find_import(event)
+ return unless import
+
+ import.broadcast_replace_to(
+ "import_#{import.id}_footer",
+ target: 'footer',
+ partial: 'spree/admin/imports/footer',
+ locals: { import: import }
+ )
+ end
+
+ private
+
+ def find_import(event)
+ import_id = event.payload['id']
+ return unless import_id
+
+ Spree::Import.find_by_prefix_id(import_id)
+ end
end
end
end
diff --git a/spree/admin/app/views/spree/admin/imports/_loader.html.erb b/spree/admin/app/views/spree/admin/imports/_loader.html.erb
index 6c980eae29..b289fd9920 100644
--- a/spree/admin/app/views/spree/admin/imports/_loader.html.erb
+++ b/spree/admin/app/views/spree/admin/imports/_loader.html.erb
@@ -2,6 +2,11 @@
<div class="flex justify-center items-center my-12 flex-col pb-12">
<%= icon('check', class: 'text-green-700', style: 'font-size: 4rem;') %>
<p class="text-green-700 mb-3"><%= Spree.t('admin.imports.import_done') %>!</p>
+ <% if import.large_import? %>
+ <p class="text-gray-600 mb-3">
+ <%= Spree.t('admin.imports.large_import_summary', completed: import.rows.completed.count, failed: import.rows.failed.count) %>
+ </p>
+ <% end %>
<%= link_to_with_icon 'arrow-back-up', Spree.t('admin.back_to_dashboard'), spree.admin_path, class: 'btn btn-light mb-12' %>
</div>
<% else %>
diff --git a/spree/admin/app/views/spree/admin/imports/show.html.erb b/spree/admin/app/views/spree/admin/imports/show.html.erb
index 01b652760a..3739a46424 100644
--- a/spree/admin/app/views/spree/admin/imports/show.html.erb
+++ b/spree/admin/app/views/spree/admin/imports/show.html.erb
@@ -25,6 +25,20 @@
</div>
<%= render 'spree/admin/imports/mapping_footer', import: @import %>
+<% elsif @import.large_import? %>
+ <%= turbo_stream_from "import_#{@import.id}_footer" %>
+ <%= turbo_stream_from "import_#{@import.id}_loader" %>
+
+ <div class="p-6 text-center text-gray-600 my-8">
+ <p class="mb-2"><%= Spree.t('admin.imports.large_import_message', count: @import.rows_count) %></p>
+ <p><%= Spree.t('admin.imports.large_import_background_note') %></p>
+ </div>
+
+ <div id="loader" data-controller="auto-scroll">
+ <%= render 'spree/admin/imports/loader', import: @import %>
+ </div>
+
+ <%= render 'spree/admin/imports/footer', import: @import %>
<% else %>
<%= turbo_stream_from "import_#{@import.id}_rows" %>
<%= turbo_stream_from "import_#{@import.id}_footer" %>
diff --git a/spree/admin/config/locales/en.yml b/spree/admin/config/locales/en.yml
index 7d1c7b91f8..b682cb07f1 100644
--- a/spree/admin/config/locales/en.yml
+++ b/spree/admin/config/locales/en.yml
@@ -136,6 +136,9 @@ en:
all_required_columns_mapped: Good job, all required columns are mapped!
field_required: Field required
import_done: All done!
+ large_import_background_note: You can close this page — the import will continue in the background.
+ large_import_message: This import contains %{count} rows and is too large to display individually.
+ large_import_summary: "%{completed} completed, %{failed} failed"
mapping_required: This field is required
please_map_all_required_fields: Please map all the required fields to continue.
waiting_for_file_to_be_processed: Waiting for file to be processed...
diff --git a/spree/core/app/jobs/spree/imports/process_group_job.rb b/spree/core/app/jobs/spree/imports/process_group_job.rb
new file mode 100644
index 0000000000..95b5dcec3a
--- /dev/null
+++ b/spree/core/app/jobs/spree/imports/process_group_job.rb
@@ -0,0 +1,45 @@
+module Spree
+ module Imports
+ class ProcessGroupJob < Spree::BaseJob
+ queue_as Spree.queues.imports
+
+ def perform(import_id, row_ids)
+ import = Spree::Import.find(import_id)
+ Spree::Current.store = import.store
+
+ mappings = import.mappings.mapped.to_a
+ schema_fields = import.schema_fields
+ large = import.large_import?
+ rows = import.rows.where(id: row_ids).order(:row_number)
+
+ if large
+ Spree::Events.disable do
+ rows.each { |row| row.bulk_process!(mappings: mappings, schema_fields: schema_fields) }
+ end
+ else
+ rows.each do |row|
+ row.process!(mappings: mappings, schema_fields: schema_fields)
+ end
+ end
+
+ check_import_completion(import, large)
+ end
+
+ private
+
+ def check_import_completion(import, large)
+ Spree::Import.where(id: import.id).update_all(
+ 'completed_groups_count = COALESCE(completed_groups_count, 0) + 1'
+ )
+ import.reload
+
+ if import.completed_groups_count >= import.processing_groups_count
+ # Guard against concurrent workers both reaching this point
+ import.complete! if import.status == 'processing'
+ elsif large && (import.completed_groups_count % 10).zero?
+ import.publish_event('import.progress')
+ end
+ end
+ end
+ end
+end
diff --git a/spree/core/app/jobs/spree/imports/process_rows_job.rb b/spree/core/app/jobs/spree/imports/process_rows_job.rb
index dc9566ba06..68c3340532 100644
--- a/spree/core/app/jobs/spree/imports/process_rows_job.rb
+++ b/spree/core/app/jobs/spree/imports/process_rows_job.rb
@@ -3,16 +3,62 @@ module Spree
class ProcessRowsJob < Spree::BaseJob
queue_as Spree.queues.imports
+ BATCH_SIZE = 100
+
def perform(import_id)
import = Spree::Import.find(import_id)
+ dispatch_groups(import)
+ end
+
+ private
+
+ def dispatch_groups(import)
+ group_field = import.group_column
+ group_mapping = group_field && import.mappings.mapped.find_by(schema_field: group_field)
+ file_column = group_mapping&.file_column
- # process all rows in sequential order
- import.rows.pending_and_failed.find_each(batch_size: 100) do |row|
- row.process!
+ if file_column
+ dispatch_grouped(import, file_column)
+ else
+ dispatch_batched(import)
end
+ end
+
+ def dispatch_grouped(import, file_column)
+ groups = Hash.new { |h, k| h[k] = [] }
+
+ import.rows.pending_and_failed.order(:row_number).pluck(:id, :data).each do |id, data|
+ parsed = JSON.parse(data)
+ key = parsed[file_column].to_s.strip.downcase.presence || '__ungrouped__'
+ groups[key] << id
+ rescue JSON::ParserError
+ groups['__ungrouped__'] << id
+ end
+
+ # Set count before enqueuing so workers can't complete prematurely
+ import.update_columns(
+ processing_groups_count: groups.size,
+ completed_groups_count: 0,
+ updated_at: Time.current
+ )
+
+ groups.each_value { |row_ids| ProcessGroupJob.perform_later(import.id, row_ids) }
+ end
+
+ def dispatch_batched(import)
+ # Count first, then enqueue — prevents premature completion
+ row_id_batches = import.rows.pending_and_failed.order(:row_number)
+ .pluck(:id)
+ .each_slice(BATCH_SIZE)
+ .to_a
+
+ import.update_columns(
+ processing_groups_count: row_id_batches.size,
+ completed_groups_count: 0,
+ updated_at: Time.current
+ )
- # mark as complete
- import.complete!
+ row_id_batches.each { |row_ids| ProcessGroupJob.perform_later(import.id, row_ids) }
end
end
end
diff --git a/spree/core/app/models/spree/import.rb b/spree/core/app/models/spree/import.rb
index d68aa3157c..9956739a6c 100644
--- a/spree/core/app/models/spree/import.rb
+++ b/spree/core/app/models/spree/import.rb
@@ -76,6 +76,21 @@ module Spree
#
preference :delimiter, :string, default: ','
+ # Returns true if the import has more rows than the large import threshold.
+ # Large imports skip per-row UI broadcasts and use bulk processing.
+ # @return [Boolean]
+ def large_import?
+ rows_count >= Spree::Config[:large_import_threshold]
+ end
+
+ # Returns the schema field name used to group rows for parallel processing.
+ # Rows sharing the same value in this field are processed together in one job.
+ # Returns nil for imports where rows are independent (default — batched in chunks).
+ # @return [String, nil]
+ def group_column
+ nil
+ end
+
# Returns true if the import is in mapping state
# @return [Boolean]
def mapping?
diff --git a/spree/core/app/models/spree/import_row.rb b/spree/core/app/models/spree/import_row.rb
index 65784485ac..6fbdccd9f9 100644
--- a/spree/core/app/models/spree/import_row.rb
+++ b/spree/core/app/models/spree/import_row.rb
@@ -77,9 +77,9 @@ module Spree
data_json[mapping.file_column]
end
- def process!
+ def process!(mappings: nil, schema_fields: nil)
start_processing!
- self.item = import.row_processor_class.new(self).process!
+ self.item = import.row_processor_class.new(self, mappings: mappings, schema_fields: schema_fields).process!
complete!
rescue StandardError => e
Rails.error.report(e, handled: true, context: { import_row_id: id }, source: 'spree.core')
@@ -87,6 +87,18 @@ module Spree
fail!
end
+ # Bulk processing mode for large imports.
+ # Uses update_columns to skip callbacks, validations, and event publishing.
+ def bulk_process!(mappings:, schema_fields:)
+ update_columns(status: 'processing', updated_at: Time.current)
+ processor = import.row_processor_class.new(self, mappings: mappings, schema_fields: schema_fields)
+ self.item = processor.process!
+ update_columns(status: 'completed', item_type: item.class.name, item_id: item.id, updated_at: Time.current)
+ rescue StandardError => e
+ Rails.error.report(e, handled: true, context: { import_row_id: id }, source: 'spree.core')
+ update_columns(status: 'failed', validation_errors: e.message, updated_at: Time.current)
+ end
+
def publish_import_row_completed_event
publish_event('import_row.completed')
end
diff --git a/spree/core/app/models/spree/imports/product_translations.rb b/spree/core/app/models/spree/imports/product_translations.rb
index e188baf078..1845e2a091 100644
--- a/spree/core/app/models/spree/imports/product_translations.rb
+++ b/spree/core/app/models/spree/imports/product_translations.rb
@@ -5,6 +5,10 @@ module Spree
Spree::Imports::RowProcessors::ProductTranslation
end
+ def group_column
+ 'slug'
+ end
+
def model_class
Spree::Product
end
diff --git a/spree/core/app/models/spree/imports/products.rb b/spree/core/app/models/spree/imports/products.rb
index e6dd5eb6b7..489a895555 100644
--- a/spree/core/app/models/spree/imports/products.rb
+++ b/spree/core/app/models/spree/imports/products.rb
@@ -4,6 +4,11 @@ module Spree
def row_processor_class
Spree::Imports::RowProcessors::ProductVariant
end
+
+ # Group by slug: product row + its variant rows must be processed together
+ def group_column
+ 'slug'
+ end
end
end
end
diff --git a/spree/core/app/services/spree/imports/row_processors/base.rb b/spree/core/app/services/spree/imports/row_processors/base.rb
index 1e109d3ad0..862ed8bd9c 100644
--- a/spree/core/app/services/spree/imports/row_processors/base.rb
+++ b/spree/core/app/services/spree/imports/row_processors/base.rb
@@ -2,10 +2,14 @@ module Spree
module Imports
module RowProcessors
class Base
- def initialize(row)
+ def initialize(row, mappings: nil, schema_fields: nil)
@row = row
@import = row.import
- @attributes = row.to_schema_hash
+ @attributes = if mappings && schema_fields
+ build_schema_hash(row, mappings, schema_fields)
+ else
+ row.to_schema_hash
+ end
end
attr_reader :row, :import, :attributes
@@ -13,6 +17,19 @@ module Spree
def process!
raise NotImplementedError, 'Subclasses must implement the process! method'
end
+
+ private
+
+ def build_schema_hash(row, mappings, schema_fields)
+ attributes = {}
+ schema_fields.each do |field|
+ mapping = mappings.find { |m| m.schema_field == field[:name] }
+ next unless mapping&.mapped?
+
+ attributes[field[:name]] = row.data_json[mapping.file_column]
+ end
+ attributes
+ end
end
end
end
diff --git a/spree/core/app/services/spree/imports/row_processors/product_translation.rb b/spree/core/app/services/spree/imports/row_processors/product_translation.rb
index cf53f5b42b..9361b9a29f 100644
--- a/spree/core/app/services/spree/imports/row_processors/product_translation.rb
+++ b/spree/core/app/services/spree/imports/row_processors/product_translation.rb
@@ -4,7 +4,7 @@ module Spree
class ProductTranslation < Base
TRANSLATABLE_FIELDS = %w[name description meta_title meta_description].freeze
- def initialize(row)
+ def initialize(row, **)
super
@store = row.store
end
diff --git a/spree/core/app/services/spree/imports/row_processors/product_variant.rb b/spree/core/app/services/spree/imports/row_processors/product_variant.rb
index a1b27a88ee..b39dcc00c7 100644
--- a/spree/core/app/services/spree/imports/row_processors/product_variant.rb
+++ b/spree/core/app/services/spree/imports/row_processors/product_variant.rb
@@ -4,7 +4,7 @@ module Spree
class ProductVariant < Base
OPTION_TYPES_COUNT = 3
- def initialize(row)
+ def initialize(row, **)
super
@store = row.store
@product = ensure_product_exists
diff --git a/spree/core/db/migrate/20260424100000_add_processing_groups_to_spree_imports.rb b/spree/core/db/migrate/20260424100000_add_processing_groups_to_spree_imports.rb
new file mode 100644
index 0000000000..29fa221a80
--- /dev/null
+++ b/spree/core/db/migrate/20260424100000_add_processing_groups_to_spree_imports.rb
@@ -0,0 +1,6 @@
+class AddProcessingGroupsToSpreeImports < ActiveRecord::Migration[7.2]
+ def change
+ add_column :spree_imports, :processing_groups_count, :integer, default: 0
+ add_column :spree_imports, :completed_groups_count, :integer, default: 0
+ end
+end
diff --git a/spree/core/lib/spree/core/configuration.rb b/spree/core/lib/spree/core/configuration.rb
index 07689fae83..0251add6b0 100644
--- a/spree/core/lib/spree/core/configuration.rb
+++ b/spree/core/lib/spree/core/configuration.rb
@@ -117,6 +117,9 @@ module Spree
preference :gift_card_batch_web_limit, :integer, default: 500 # number of gift card codes to be generated in the web process, more than this will be generated in a background job
preference :gift_card_batch_limit, :integer, default: 50_000
+ # imports
+ preference :large_import_threshold, :integer, default: 500 # imports with more rows than this skip per-row UI broadcasts and use bulk processing
+
end
end
end
diff --git a/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb b/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb
new file mode 100644
index 0000000000..a6e1be9abd
--- /dev/null
+++ b/spree/core/spec/jobs/spree/imports/process_group_job_spec.rb
@@ -0,0 +1,187 @@
+require 'spec_helper'
+
+RSpec.describe Spree::Imports::ProcessGroupJob, type: :job do
+ let(:store) { @default_store }
+
+ let!(:import) do
+ create(:import, owner: store, type: 'Spree::Imports::Products', status: :processing,
+ processing_groups_count: 1, completed_groups_count: 0)
+ end
+
+ let!(:row) do
+ create(:import_row, import: import, row_number: 1, status: :pending,
+ data: { 'slug' => 'test-product', 'sku' => 'SKU1', 'name' => 'Test Product', 'price' => '9.99' }.to_json)
+ end
+
+ before do
+ allow_any_instance_of(Spree::Import).to receive(:csv_headers).and_return(['slug', 'sku', 'name', 'price'])
+ import.create_mappings
+ end
+
+ it 'sets Spree::Current.store from the import' do
+ described_class.perform_now(import.id, [row.id])
+
+ expect(Spree::Current.store).to eq(store)
+ end
+
+ it 'creates the product and completes the import' do
+ expect {
+ described_class.perform_now(import.id, [row.id])
+ }.to change(Spree::Product, :count).by(1)
+
+ row.reload
+ expect(row.status).to eq('completed')
+ expect(row.item).to be_a(Spree::Variant)
+ expect(row.item.product.name).to eq('Test Product')
+ expect(row.item.sku).to eq('SKU1')
+
+ import.reload
+ expect(import.completed_groups_count).to eq(1)
+ expect(import.status).to eq('completed')
+ end
+
+ it 'does not complete import when other groups are still pending' do
+ import.update_columns(processing_groups_count: 3)
+
+ described_class.perform_now(import.id, [row.id])
+
+ import.reload
+ expect(import.completed_groups_count).to eq(1)
+ expect(import.status).to eq('processing')
+ end
+
+ context 'when a row fails' do
+ before do
+ allow_any_instance_of(Spree::Imports::RowProcessors::ProductVariant).to receive(:process!).and_raise(StandardError, 'something went wrong')
+ end
+
+ it 'marks the row as failed and continues' do
+ described_class.perform_now(import.id, [row.id])
+
+ row.reload
+ expect(row.status).to eq('failed')
+ expect(row.validation_errors).to eq('something went wrong')
+
+ # Import still completes (single group)
+ expect(import.reload.status).to eq('completed')
+ end
+ end
+
+ context 'with product + variant rows in a group' do
+ let!(:variant_row) do
+ create(:import_row, import: import, row_number: 2, status: :pending,
+ data: { 'slug' => 'test-product', 'sku' => 'SKU2', 'price' => '19.99', 'option1_name' => 'Color', 'option1_value' => 'Red' }.to_json)
+ end
+
+ before do
+ allow_any_instance_of(Spree::Import).to receive(:csv_headers).and_return(['slug', 'sku', 'name', 'price', 'option1_name', 'option1_value'])
+ import.mappings.destroy_all
+ import.create_mappings
+ end
+
+ it 'creates product and variant in row_number order' do
+ expect {
+ described_class.perform_now(import.id, [variant_row.id, row.id])
+ }.to change(Spree::Product, :count).by(1)
+
+ expect(row.reload.status).to eq('completed')
+ expect(variant_row.reload.status).to eq('completed')
+
+ product = row.item.product
+ expect(product.name).to eq('Test Product')
+ expect(variant_row.item).to be_a(Spree::Variant)
+ expect(variant_row.item.product).to eq(product)
+ expect(variant_row.item.sku).to eq('SKU2')
+ end
+ end
+
+ describe 'large import (bulk_process!)' do
+ before do
+ allow_any_instance_of(Spree::Import).to receive(:large_import?).and_return(true)
+ end
+
+ it 'creates the product via bulk_process!' do
+ expect {
+ described_class.perform_now(import.id, [row.id])
+ }.to change(Spree::Product, :count).by(1)
+
+ row.reload
+ expect(row.status).to eq('completed')
+ expect(row.item).to be_a(Spree::Variant)
+ expect(row.item.product.name).to eq('Test Product')
+ expect(row.item.sku).to eq('SKU1')
+ end
+
+ context 'with product + variant rows' do
+ let!(:variant_row) do
+ create(:import_row, import: import, row_number: 2, status: :pending,
+ data: { 'slug' => 'test-product', 'sku' => 'SKU2', 'price' => '19.99', 'option1_name' => 'Color', 'option1_value' => 'Red' }.to_json)
+ end
+
+ before do
+ allow_any_instance_of(Spree::Import).to receive(:csv_headers).and_return(['slug', 'sku', 'name', 'price', 'option1_name', 'option1_value'])
+ import.mappings.destroy_all
+ import.create_mappings
+ end
+
+ it 'creates product and variant via bulk_process!' do
+ expect {
+ described_class.perform_now(import.id, [row.id, variant_row.id])
+ }.to change(Spree::Product, :count).by(1)
+ .and change(Spree::Variant, :count).by(2) # master + variant
+
+ row.reload
+ variant_row.reload
+ expect(row.status).to eq('completed')
+ expect(variant_row.status).to eq('completed')
+
+ product = row.item.product
+ expect(product.name).to eq('Test Product')
+ expect(variant_row.item.product).to eq(product)
+ expect(variant_row.item.sku).to eq('SKU2')
+ end
+ end
+
+ it 'disables events during processing' do
+ events_were_disabled = false
+
+ original_bulk_process = Spree::ImportRow.instance_method(:bulk_process!)
+ allow_any_instance_of(Spree::ImportRow).to receive(:bulk_process!) do |row_instance, **kwargs|
+ events_were_disabled = !Spree::Events.enabled?
+ original_bulk_process.bind_call(row_instance, **kwargs)
+ end
+
+ described_class.perform_now(import.id, [row.id])
+
+ expect(events_were_disabled).to be true
+ expect(row.reload.status).to eq('completed')
+ end
+
+ it 'publishes import.progress every 10 groups' do
+ import.update_columns(processing_groups_count: 20, completed_groups_count: 9)
+
+ expect_any_instance_of(Spree::Import).to receive(:publish_event).with('import.progress')
+
+ described_class.perform_now(import.id, [row.id])
+ end
+
+ it 'does not publish progress on non-10th group' do
+ import.update_columns(processing_groups_count: 20, completed_groups_count: 7)
+
+ expect_any_instance_of(Spree::Import).not_to receive(:publish_event).with('import.progress')
+
+ described_class.perform_now(import.id, [row.id])
+ end
+ end
+
+ describe 'atomic completion tracking' do
+ it 'uses atomic SQL increment for completed_groups_count' do
+ import.update_columns(processing_groups_count: 2, completed_groups_count: 0)
+
+ described_class.perform_now(import.id, [row.id])
+
+ expect(import.reload.completed_groups_count).to eq(1)
+ expect(import.status).to eq('processing')
+ end
+ end
+end
diff --git a/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb b/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb
index bec0e617ed..eec4368ff8 100644
--- a/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb
+++ b/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb
@@ -3,59 +3,102 @@ require 'spec_helper'
RSpec.describe Spree::Imports::ProcessRowsJob, type: :job do
let(:store) { @default_store }
- let!(:import) do
- create(
- :import,
- owner: store,
- type: 'Spree::Imports::Products',
- status: :processing
- )
- end
+ describe 'grouped import (Products)' do
+ let!(:import) do
+ create(:import, owner: store, type: 'Spree::Imports::Products', status: :processing)
+ end
- let!(:pending_row) do
- create(
- :import_row,
- import: import,
- row_number: 1,
- status: :pending,
- data: { 'slug' => 'test-product', 'sku' => 'SKU1', 'name' => 'Test Product', 'price' => '9.99' }.to_json
- )
- end
+ before do
+ allow_any_instance_of(Spree::Import).to receive(:csv_headers).and_return(['slug', 'sku', 'name', 'price', 'option1_name', 'option1_value'])
+ import.create_mappings
+ end
- let!(:failed_row) do
- create(
- :import_row,
- import: import,
- row_number: 2,
- status: :failed,
- data: { 'slug' => 'failed-product', 'sku' => 'SKU2', 'name' => 'Failed Product', 'price' => '19.99' }.to_json
- )
- end
+ let!(:product_row) do
+ create(:import_row, import: import, row_number: 1, status: :pending,
+ data: { 'slug' => 'denim-shirt', 'sku' => 'SKU1', 'name' => 'Denim Shirt', 'price' => '9.99' }.to_json)
+ end
- let!(:processed_row) do
- create(
- :import_row,
- import: import,
- row_number: 3,
- status: :completed,
- data: { 'slug' => 'processed-product', 'sku' => 'SKU3', 'name' => 'Processed Product', 'price' => '29.99' }.to_json
- )
- end
+ let!(:variant_row) do
+ create(:import_row, import: import, row_number: 2, status: :pending,
+ data: { 'slug' => 'denim-shirt', 'sku' => 'SKU2', 'price' => '19.99', 'option1_name' => 'Color', 'option1_value' => 'Red' }.to_json)
+ end
- before do
- allow(import).to receive(:csv_headers).and_return(['slug', 'sku', 'name', 'price'])
- import.create_mappings
- end
+ let!(:other_product_row) do
+ create(:import_row, import: import, row_number: 3, status: :pending,
+ data: { 'slug' => 'cotton-tee', 'sku' => 'SKU3', 'name' => 'Cotton Tee', 'price' => '14.99' }.to_json)
+ end
+
+ it 'groups rows by slug and dispatches one job per product' do
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, [product_row.id, variant_row.id])
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, [other_product_row.id])
- it 'processes pending and failed rows' do
- expect {
described_class.perform_now(import.id)
- }.to change { pending_row.reload.status }.from('pending').to('completed').and change { failed_row.reload.status }.from('failed').to('completed')
+
+ import.reload
+ expect(import.processing_groups_count).to eq(2)
+ expect(import.completed_groups_count).to eq(0)
+ end
+
+ it 'skips already completed rows' do
+ product_row.update_columns(status: 'completed')
+
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, [variant_row.id])
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, [other_product_row.id])
+
+ described_class.perform_now(import.id)
+
+ expect(import.reload.processing_groups_count).to eq(2)
+ end
+
+ context 'with failed rows' do
+ before { variant_row.update_columns(status: 'failed') }
+
+ it 'includes failed rows for reprocessing' do
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, [product_row.id, variant_row.id])
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, [other_product_row.id])
+
+ described_class.perform_now(import.id)
+ end
+ end
end
- it 'marks import as complete after processing all rows' do
- expect {
+ describe 'batched import (Customers — no group_column)' do
+ let!(:import) do
+ create(:import, owner: store, type: 'Spree::Imports::Customers', status: :processing)
+ end
+
+ let!(:rows) do
+ 5.times.map do |i|
+ create(:import_row, import: import, row_number: i + 1, status: :pending,
+ data: { 'email' => "user#{i}@example.com" }.to_json)
+ end
+ end
+
+ it 'batches all rows into chunks and dispatches' do
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, rows.map(&:id))
+
described_class.perform_now(import.id)
- }.to change { import.reload.status }.from('processing').to('completed')
+
+ expect(import.reload.processing_groups_count).to eq(1)
+ end
+
+ context 'with more rows than BATCH_SIZE' do
+ before { stub_const('Spree::Imports::ProcessRowsJob::BATCH_SIZE', 2) }
+
+ it 'dispatches multiple batch jobs' do
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later).exactly(3).times
+
+ described_class.perform_now(import.id)
+
+ expect(import.reload.processing_groups_count).to eq(3)
+ end
+ end
end
end