Skip to content

armgit / spree

Public Imported from GitHub
0

Open Source Platform for DTC, B2B Commerce, Marketplaces & Omnichannel. REST APIs, TypeScript SDKs, and production-ready Next.js storefront. Self-host it. Own your data. No vendor lock-in. Zero platform fees.

Fix Import group performance issues (#14280)

Damian Legawiec1b466ec92890
diff --git a/.claude/settings.local.json b/.claude/settings.local.json
index fb752a110f..961422fb25 100644
--- a/.claude/settings.local.json
+++ b/.claude/settings.local.json
@@ -15,7 +15,15 @@
"Bash(rm -rf server/.git server/.gitignore)",
"Skill(update-config)",
"Skill(simplify)",
- "mcp__claude_ai_Claude_Code_Remote__list_triggers"
+ "mcp__claude_ai_Claude_Code_Remote__list_triggers",
+ "mcp__scout-apm__list_apps",
+ "mcp__scout-apm__get_app_jobs",
+ "mcp__scout-apm__get_app_job_traces",
+ "mcp__scout-apm__get_job_metrics",
+ "mcp__scout-apm__get_app_trace",
+ "mcp__scout-apm__get_app_insights",
+ "mcp__scout-apm__get_app_error_groups",
+ "mcp__scout-apm__get_app_anomaly_events"
],
"deny": [
"Bash(sudo *)",
diff --git a/spree/core/app/jobs/spree/imports/process_group_job.rb b/spree/core/app/jobs/spree/imports/process_group_job.rb
index c6ac33c270..4bae14c7c3 100644
--- a/spree/core/app/jobs/spree/imports/process_group_job.rb
+++ b/spree/core/app/jobs/spree/imports/process_group_job.rb
@@ -1,6 +1,10 @@
module Spree
module Imports
class ProcessGroupJob < Spree::Imports::BaseJob
+ # Rows are loaded in slices so a large group never holds every ImportRow
+ # (and its raw CSV data) in memory for the whole job.
+ ROWS_BATCH_SIZE = 100
+
def perform(import_id, row_ids)
import = Spree::Import.find(import_id)
Spree::Current.store = import.store
@@ -8,16 +12,23 @@ module Spree
mappings = import.mappings.mapped.to_a
schema_fields = import.schema_fields
large = import.large_import?
- # Skip rows already completed on a prior attempt so retries don't double-process them.
- rows = import.rows.where(id: row_ids).pending_and_failed.order(:row_number)
- if large
- Spree::Events.disable do
- rows.each { |row| row.bulk_process!(mappings: mappings, schema_fields: schema_fields) }
- end
- else
- rows.each do |row|
- row.process!(mappings: mappings, schema_fields: schema_fields)
+ row_ids.each_slice(ROWS_BATCH_SIZE) do |ids|
+ # Skip rows already completed on a prior attempt so retries don't double-process them.
+ rows = import.rows.where(id: ids).pending_and_failed.order(:row_number).to_a
+ # Share the already-loaded import across rows: each row's processor reads
+ # `row.import` (store, ability, lookup cache), and without this every row
+ # lazily loads its own Import instance and rebuilds all of that per row.
+ rows.each { |row| row.association(:import).target = import }
+
+ if large
+ Spree::Events.disable do
+ rows.each { |row| row.bulk_process!(mappings: mappings, schema_fields: schema_fields) }
+ end
+ else
+ rows.each do |row|
+ row.process!(mappings: mappings, schema_fields: schema_fields)
+ end
end
end
diff --git a/spree/core/app/jobs/spree/imports/process_rows_job.rb b/spree/core/app/jobs/spree/imports/process_rows_job.rb
index 2853e68a60..0363feb658 100644
--- a/spree/core/app/jobs/spree/imports/process_rows_job.rb
+++ b/spree/core/app/jobs/spree/imports/process_rows_job.rb
@@ -2,6 +2,7 @@ module Spree
module Imports
class ProcessRowsJob < Spree::Imports::BaseJob
BATCH_SIZE = 100
+ UNGROUPED_KEY = '__ungrouped__'.freeze
def perform(import_id)
import = Spree::Import.find(import_id)
@@ -27,20 +28,27 @@ module Spree
import.rows.pending_and_failed.order(:row_number).pluck(:id, :data).each do |id, data|
parsed = JSON.parse(data)
- key = parsed[file_column].to_s.strip.downcase.presence || '__ungrouped__'
+ key = parsed[file_column].to_s.strip.downcase.presence || UNGROUPED_KEY
groups[key] << id
rescue JSON::ParserError
- groups['__ungrouped__'] << id
+ groups[UNGROUPED_KEY] << id
end
+ # Rows without a group value don't depend on each other, so they don't have
+ # to share a single job — split them into bounded batches. Real groups stay
+ # intact: their rows must run sequentially (product row before variant rows).
+ ungrouped = groups.delete(UNGROUPED_KEY)
+ batches = groups.values
+ ungrouped&.each_slice(BATCH_SIZE) { |row_ids| batches << row_ids }
+
# Set count before enqueuing so workers can't complete prematurely
import.update_columns(
- processing_groups_count: groups.size,
+ processing_groups_count: batches.size,
completed_groups_count: 0,
updated_at: Time.current
)
- groups.each_value { |row_ids| ProcessGroupJob.perform_later(import.id, row_ids) }
+ batches.each { |row_ids| ProcessGroupJob.perform_later(import.id, row_ids) }
end
def dispatch_batched(import)
diff --git a/spree/core/app/models/spree/import.rb b/spree/core/app/models/spree/import.rb
index be46f95e3e..9933b6fc8e 100644
--- a/spree/core/app/models/spree/import.rb
+++ b/spree/core/app/models/spree/import.rb
@@ -245,6 +245,15 @@ module Spree
@current_ability ||= Spree.ability_class.new(user, { store: store })
end
+ # Per-instance cache shared by row processors within a single processing job.
+ # Group jobs funnel every row through the same Import instance, so lookups of
+ # shared records (tax/shipping categories, option types, metafield definitions)
+ # resolve once per job instead of once per row.
+ # @return [Hash]
+ def row_lookup_cache
+ @row_lookup_cache ||= {}
+ end
+
def event_serializer_class
'Spree::Api::V3::ImportSerializer'.safe_constantize
end
diff --git a/spree/core/app/services/spree/imports/row_processors/base.rb b/spree/core/app/services/spree/imports/row_processors/base.rb
index 862ed8bd9c..0d805ec2f2 100644
--- a/spree/core/app/services/spree/imports/row_processors/base.rb
+++ b/spree/core/app/services/spree/imports/row_processors/base.rb
@@ -20,6 +20,15 @@ module Spree
private
+ # Memoizes a shared-record lookup (including nil misses) in the import's
+ # per-job cache so repeated rows don't re-run the same query.
+ def cached_lookup(*key)
+ cache = import.row_lookup_cache
+ return cache[key] if cache.key?(key)
+
+ cache[key] = yield
+ end
+
def build_schema_hash(row, mappings, schema_fields)
attributes = {}
schema_fields.each do |field|
diff --git a/spree/core/app/services/spree/imports/row_processors/product_variant.rb b/spree/core/app/services/spree/imports/row_processors/product_variant.rb
index 97c61ef245..8b2ae36a79 100644
--- a/spree/core/app/services/spree/imports/row_processors/product_variant.rb
+++ b/spree/core/app/services/spree/imports/row_processors/product_variant.rb
@@ -123,12 +123,16 @@ module Spree
def prepare_shipping_category
shipping_category_name = attributes['shipping_category'].strip
- Spree::ShippingCategory.find_by(name: shipping_category_name)
+ cached_lookup(:shipping_category, shipping_category_name) do
+ Spree::ShippingCategory.find_by(name: shipping_category_name)
+ end
end
def prepare_tax_category
tax_category_name = attributes['tax_category'].strip
- Spree::TaxCategory.find_by(name: tax_category_name)
+ cached_lookup(:tax_category, tax_category_name) do
+ Spree::TaxCategory.find_by(name: tax_category_name)
+ end
end
def prepare_option_value_variants
@@ -171,27 +175,39 @@ module Spree
# surfaces via the DB unique index (RecordNotUnique) or the AR uniqueness
# validator (RecordInvalid with a :taken error on the relevant attribute).
def find_or_create_option_type!(presentation)
- Spree::OptionType.search_by_name(presentation).first || Spree::OptionType.create!(presentation: presentation)
- rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e
- raise unless uniqueness_conflict?(e, :name)
+ cached_lookup(:option_type, presentation) do
+ begin
+ Spree::OptionType.search_by_name(presentation).first || Spree::OptionType.create!(presentation: presentation)
+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e
+ raise unless uniqueness_conflict?(e, :name)
- Spree::OptionType.search_by_name(presentation).first!
+ Spree::OptionType.search_by_name(presentation).first!
+ end
+ end
end
def find_or_create_option_value!(option_type, presentation)
- option_type.option_values.search_by_name(presentation).first || option_type.option_values.create!(presentation: presentation)
- rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e
- raise unless uniqueness_conflict?(e, :name)
+ cached_lookup(:option_value, option_type.id, presentation) do
+ begin
+ option_type.option_values.search_by_name(presentation).first || option_type.option_values.create!(presentation: presentation)
+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e
+ raise unless uniqueness_conflict?(e, :name)
- option_type.option_values.search_by_name(presentation).first!
+ option_type.option_values.search_by_name(presentation).first!
+ end
+ end
end
def find_or_create_product_option_type!(option_type)
- Spree::ProductOptionType.find_or_create_by!(product: product, option_type: option_type)
- rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e
- raise unless uniqueness_conflict?(e, :product_id)
+ cached_lookup(:product_option_type, product.id, option_type.id) do
+ begin
+ Spree::ProductOptionType.find_or_create_by!(product: product, option_type: option_type)
+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e
+ raise unless uniqueness_conflict?(e, :product_id)
- Spree::ProductOptionType.find_by!(product: product, option_type: option_type)
+ Spree::ProductOptionType.find_by!(product: product, option_type: option_type)
+ end
+ end
end
# RecordNotUnique is always a uniqueness conflict; RecordInvalid only when the
@@ -245,11 +261,13 @@ module Spree
namespace, key = product.extract_namespace_and_key(full_key)
# Find or initialize metafield definition
- metafield_definition = Spree::MetafieldDefinition.find_by(
- namespace: namespace,
- key: key,
- resource_type: product.class.name
- )
+ metafield_definition = cached_lookup(:metafield_definition, namespace, key, product.class.name) do
+ Spree::MetafieldDefinition.find_by(
+ namespace: namespace,
+ key: key,
+ resource_type: product.class.name
+ )
+ end
next unless metafield_definition
diff --git a/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb b/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb
index eec4368ff8..c9b66ae7a0 100644
--- a/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb
+++ b/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb
@@ -66,6 +66,32 @@ RSpec.describe Spree::Imports::ProcessRowsJob, type: :job do
described_class.perform_now(import.id)
end
end
+
+ context 'with rows missing the group value' do
+ before { stub_const('Spree::Imports::ProcessRowsJob::BATCH_SIZE', 2) }
+
+ let!(:ungrouped_rows) do
+ 3.times.map do |i|
+ create(:import_row, import: import, row_number: 4 + i, status: :pending,
+ data: { 'sku' => "LONE#{i}", 'name' => "Lone #{i}", 'price' => '5.00' }.to_json)
+ end
+ end
+
+ it 'splits ungrouped rows into bounded batches instead of one unbounded job' do
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, [product_row.id, variant_row.id])
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, [other_product_row.id])
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, ungrouped_rows.first(2).map(&:id))
+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)
+ .with(import.id, [ungrouped_rows.last.id])
+
+ described_class.perform_now(import.id)
+
+ expect(import.reload.processing_groups_count).to eq(4)
+ end
+ end
end
describe 'batched import (Customers — no group_column)' do