Open Source Platform for DTC, B2B Commerce, Marketplaces & Omnichannel. REST APIs, TypeScript SDKs, and production-ready Next.js storefront. Self-host it. Own your data. No vendor lock-in. Zero platform fees.
Ruby b2b-commercee-commerceecommerceecommerce-apiecommerce-frameworkecommerce-platformheadlessheadless-commerceheadless-ecommercemarketplacemulti-tenantmulti-vendormulti-vendor-ecommerceopen-sourcespree-commerce
Fix Import group performance issues (#14280)
Damian Legawiec1b466ec92890
diff --git a/.claude/settings.local.json b/.claude/settings.local.jsonindex fb752a110f..961422fb25 100644--- a/.claude/settings.local.json+++ b/.claude/settings.local.json@@ -15,7 +15,15 @@"Bash(rm -rf server/.git server/.gitignore)","Skill(update-config)","Skill(simplify)",- "mcp__claude_ai_Claude_Code_Remote__list_triggers"+ "mcp__claude_ai_Claude_Code_Remote__list_triggers",+ "mcp__scout-apm__list_apps",+ "mcp__scout-apm__get_app_jobs",+ "mcp__scout-apm__get_app_job_traces",+ "mcp__scout-apm__get_job_metrics",+ "mcp__scout-apm__get_app_trace",+ "mcp__scout-apm__get_app_insights",+ "mcp__scout-apm__get_app_error_groups",+ "mcp__scout-apm__get_app_anomaly_events"],"deny": ["Bash(sudo *)",diff --git a/spree/core/app/jobs/spree/imports/process_group_job.rb b/spree/core/app/jobs/spree/imports/process_group_job.rbindex c6ac33c270..4bae14c7c3 100644--- a/spree/core/app/jobs/spree/imports/process_group_job.rb+++ b/spree/core/app/jobs/spree/imports/process_group_job.rb@@ -1,6 +1,10 @@module Spreemodule Importsclass ProcessGroupJob < Spree::Imports::BaseJob+ # Rows are loaded in slices so a large group never holds every ImportRow+ # (and its raw CSV data) in memory for the whole job.+ ROWS_BATCH_SIZE = 100+def perform(import_id, row_ids)import = Spree::Import.find(import_id)Spree::Current.store = import.store@@ -8,16 +12,23 @@ module Spreemappings = import.mappings.mapped.to_aschema_fields = import.schema_fieldslarge = import.large_import?- # Skip rows already completed on a prior attempt so retries don't double-process them.- rows = import.rows.where(id: row_ids).pending_and_failed.order(:row_number)- if large- Spree::Events.disable do- rows.each { |row| row.bulk_process!(mappings: mappings, schema_fields: schema_fields) }- end- else- rows.each do |row|- row.process!(mappings: mappings, schema_fields: schema_fields)+ row_ids.each_slice(ROWS_BATCH_SIZE) do |ids|+ # Skip rows already completed on a prior attempt so retries don't double-process them.+ rows = import.rows.where(id: ids).pending_and_failed.order(:row_number).to_a+ # Share the already-loaded import across rows: each row's processor reads+ # `row.import` (store, ability, lookup cache), and without this every row+ # lazily loads its own Import instance and rebuilds all of that per row.+ rows.each { |row| row.association(:import).target = import }++ if large+ Spree::Events.disable do+ rows.each { |row| row.bulk_process!(mappings: mappings, schema_fields: schema_fields) }+ end+ else+ rows.each do |row|+ row.process!(mappings: mappings, schema_fields: schema_fields)+ endendenddiff --git a/spree/core/app/jobs/spree/imports/process_rows_job.rb b/spree/core/app/jobs/spree/imports/process_rows_job.rbindex 2853e68a60..0363feb658 100644--- a/spree/core/app/jobs/spree/imports/process_rows_job.rb+++ b/spree/core/app/jobs/spree/imports/process_rows_job.rb@@ -2,6 +2,7 @@ module Spreemodule Importsclass ProcessRowsJob < Spree::Imports::BaseJobBATCH_SIZE = 100+ UNGROUPED_KEY = '__ungrouped__'.freezedef perform(import_id)import = Spree::Import.find(import_id)@@ -27,20 +28,27 @@ module Spreeimport.rows.pending_and_failed.order(:row_number).pluck(:id, :data).each do |id, data|parsed = JSON.parse(data)- key = parsed[file_column].to_s.strip.downcase.presence || '__ungrouped__'+ key = parsed[file_column].to_s.strip.downcase.presence || UNGROUPED_KEYgroups[key] << idrescue JSON::ParserError- groups['__ungrouped__'] << id+ groups[UNGROUPED_KEY] << idend+ # Rows without a group value don't depend on each other, so they don't have+ # to share a single job — split them into bounded batches. Real groups stay+ # intact: their rows must run sequentially (product row before variant rows).+ ungrouped = groups.delete(UNGROUPED_KEY)+ batches = groups.values+ ungrouped&.each_slice(BATCH_SIZE) { |row_ids| batches << row_ids }+# Set count before enqueuing so workers can't complete prematurelyimport.update_columns(- processing_groups_count: groups.size,+ processing_groups_count: batches.size,completed_groups_count: 0,updated_at: Time.current)- groups.each_value { |row_ids| ProcessGroupJob.perform_later(import.id, row_ids) }+ batches.each { |row_ids| ProcessGroupJob.perform_later(import.id, row_ids) }enddef dispatch_batched(import)diff --git a/spree/core/app/models/spree/import.rb b/spree/core/app/models/spree/import.rbindex be46f95e3e..9933b6fc8e 100644--- a/spree/core/app/models/spree/import.rb+++ b/spree/core/app/models/spree/import.rb@@ -245,6 +245,15 @@ module Spree@current_ability ||= Spree.ability_class.new(user, { store: store })end+ # Per-instance cache shared by row processors within a single processing job.+ # Group jobs funnel every row through the same Import instance, so lookups of+ # shared records (tax/shipping categories, option types, metafield definitions)+ # resolve once per job instead of once per row.+ # @return [Hash]+ def row_lookup_cache+ @row_lookup_cache ||= {}+ end+def event_serializer_class'Spree::Api::V3::ImportSerializer'.safe_constantizeenddiff --git a/spree/core/app/services/spree/imports/row_processors/base.rb b/spree/core/app/services/spree/imports/row_processors/base.rbindex 862ed8bd9c..0d805ec2f2 100644--- a/spree/core/app/services/spree/imports/row_processors/base.rb+++ b/spree/core/app/services/spree/imports/row_processors/base.rb@@ -20,6 +20,15 @@ module Spreeprivate+ # Memoizes a shared-record lookup (including nil misses) in the import's+ # per-job cache so repeated rows don't re-run the same query.+ def cached_lookup(*key)+ cache = import.row_lookup_cache+ return cache[key] if cache.key?(key)++ cache[key] = yield+ end+def build_schema_hash(row, mappings, schema_fields)attributes = {}schema_fields.each do |field|diff --git a/spree/core/app/services/spree/imports/row_processors/product_variant.rb b/spree/core/app/services/spree/imports/row_processors/product_variant.rbindex 97c61ef245..8b2ae36a79 100644--- a/spree/core/app/services/spree/imports/row_processors/product_variant.rb+++ b/spree/core/app/services/spree/imports/row_processors/product_variant.rb@@ -123,12 +123,16 @@ module Spreedef prepare_shipping_categoryshipping_category_name = attributes['shipping_category'].strip- Spree::ShippingCategory.find_by(name: shipping_category_name)+ cached_lookup(:shipping_category, shipping_category_name) do+ Spree::ShippingCategory.find_by(name: shipping_category_name)+ endenddef prepare_tax_categorytax_category_name = attributes['tax_category'].strip- Spree::TaxCategory.find_by(name: tax_category_name)+ cached_lookup(:tax_category, tax_category_name) do+ Spree::TaxCategory.find_by(name: tax_category_name)+ endenddef prepare_option_value_variants@@ -171,27 +175,39 @@ module Spree# surfaces via the DB unique index (RecordNotUnique) or the AR uniqueness# validator (RecordInvalid with a :taken error on the relevant attribute).def find_or_create_option_type!(presentation)- Spree::OptionType.search_by_name(presentation).first || Spree::OptionType.create!(presentation: presentation)- rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e- raise unless uniqueness_conflict?(e, :name)+ cached_lookup(:option_type, presentation) do+ begin+ Spree::OptionType.search_by_name(presentation).first || Spree::OptionType.create!(presentation: presentation)+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e+ raise unless uniqueness_conflict?(e, :name)- Spree::OptionType.search_by_name(presentation).first!+ Spree::OptionType.search_by_name(presentation).first!+ end+ endenddef find_or_create_option_value!(option_type, presentation)- option_type.option_values.search_by_name(presentation).first || option_type.option_values.create!(presentation: presentation)- rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e- raise unless uniqueness_conflict?(e, :name)+ cached_lookup(:option_value, option_type.id, presentation) do+ begin+ option_type.option_values.search_by_name(presentation).first || option_type.option_values.create!(presentation: presentation)+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e+ raise unless uniqueness_conflict?(e, :name)- option_type.option_values.search_by_name(presentation).first!+ option_type.option_values.search_by_name(presentation).first!+ end+ endenddef find_or_create_product_option_type!(option_type)- Spree::ProductOptionType.find_or_create_by!(product: product, option_type: option_type)- rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e- raise unless uniqueness_conflict?(e, :product_id)+ cached_lookup(:product_option_type, product.id, option_type.id) do+ begin+ Spree::ProductOptionType.find_or_create_by!(product: product, option_type: option_type)+ rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid => e+ raise unless uniqueness_conflict?(e, :product_id)- Spree::ProductOptionType.find_by!(product: product, option_type: option_type)+ Spree::ProductOptionType.find_by!(product: product, option_type: option_type)+ end+ endend# RecordNotUnique is always a uniqueness conflict; RecordInvalid only when the@@ -245,11 +261,13 @@ module Spreenamespace, key = product.extract_namespace_and_key(full_key)# Find or initialize metafield definition- metafield_definition = Spree::MetafieldDefinition.find_by(- namespace: namespace,- key: key,- resource_type: product.class.name- )+ metafield_definition = cached_lookup(:metafield_definition, namespace, key, product.class.name) do+ Spree::MetafieldDefinition.find_by(+ namespace: namespace,+ key: key,+ resource_type: product.class.name+ )+ endnext unless metafield_definitiondiff --git a/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb b/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rbindex eec4368ff8..c9b66ae7a0 100644--- a/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb+++ b/spree/core/spec/jobs/spree/imports/process_rows_job_spec.rb@@ -66,6 +66,32 @@ RSpec.describe Spree::Imports::ProcessRowsJob, type: :job dodescribed_class.perform_now(import.id)endend++ context 'with rows missing the group value' do+ before { stub_const('Spree::Imports::ProcessRowsJob::BATCH_SIZE', 2) }++ let!(:ungrouped_rows) do+ 3.times.map do |i|+ create(:import_row, import: import, row_number: 4 + i, status: :pending,+ data: { 'sku' => "LONE#{i}", 'name' => "Lone #{i}", 'price' => '5.00' }.to_json)+ end+ end++ it 'splits ungrouped rows into bounded batches instead of one unbounded job' do+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, [product_row.id, variant_row.id])+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, [other_product_row.id])+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, ungrouped_rows.first(2).map(&:id))+ expect(Spree::Imports::ProcessGroupJob).to receive(:perform_later)+ .with(import.id, [ungrouped_rows.last.id])++ described_class.perform_now(import.id)++ expect(import.reload.processing_groups_count).to eq(4)+ end+ endenddescribe 'batched import (Customers — no group_column)' do