Open Source Platform for DTC, B2B Commerce, Marketplaces & Omnichannel. REST APIs, TypeScript SDKs, and production-ready Next.js storefront. Self-host it. Own your data. No vendor lock-in. Zero platform fees.
Ruby b2b-commercee-commerceecommerceecommerce-apiecommerce-frameworkecommerce-platformheadlessheadless-commerceheadless-ecommercemarketplacemulti-tenantmulti-vendormulti-vendor-ecommerceopen-sourcespree-commerce
Refactor `ImportRunner` also added option to run imports inline
Damian Legawiec56ade3bc333e
diff --git a/spree/core/app/jobs/spree/imports/create_rows_job.rb b/spree/core/app/jobs/spree/imports/create_rows_job.rbindex c1c3bd853f..c89e07a7ad 100644--- a/spree/core/app/jobs/spree/imports/create_rows_job.rb+++ b/spree/core/app/jobs/spree/imports/create_rows_job.rb@@ -9,8 +9,12 @@ module SpreeBATCH_SIZE = 1000- def perform(import_id)+ # `inline` isn't persisted, so it has to be threaded through the chain —+ # otherwise a caller that asked for a synchronous run would enqueue from+ # here onwards.+ def perform(import_id, inline: false)import = Spree::Import.find(import_id)+ import.inline = inlineimport.start_processing! if import.status != 'processing'create_rows_sequentially(import)diff --git a/spree/core/app/jobs/spree/imports/process_group_job.rb b/spree/core/app/jobs/spree/imports/process_group_job.rbindex d57413c9da..37ac556fad 100644--- a/spree/core/app/jobs/spree/imports/process_group_job.rb+++ b/spree/core/app/jobs/spree/imports/process_group_job.rb@@ -21,8 +21,9 @@ module Spree# (and its raw CSV data) in memory for the whole job.ROWS_BATCH_SIZE = 100- def perform(import_id, row_ids)+ def perform(import_id, row_ids, inline: false)import = Spree::Import.find(import_id)+ import.inline = inlineSpree::Current.store = import.storewith_store_content_locale(import.store) dodiff --git a/spree/core/app/jobs/spree/imports/process_rows_job.rb b/spree/core/app/jobs/spree/imports/process_rows_job.rbindex 0363feb658..3125ba391d 100644--- a/spree/core/app/jobs/spree/imports/process_rows_job.rb+++ b/spree/core/app/jobs/spree/imports/process_rows_job.rb@@ -4,8 +4,9 @@ module SpreeBATCH_SIZE = 100UNGROUPED_KEY = '__ungrouped__'.freeze- def perform(import_id)+ def perform(import_id, inline: false)import = Spree::Import.find(import_id)+ import.inline = inlinedispatch_groups(import)end@@ -23,6 +24,14 @@ module Spreeendend+ # Inline callers need the group finished before `perform` returns; the+ # group job's completion bookkeeping works either way.+ def dispatch_group(import, row_ids)+ return ProcessGroupJob.perform_now(import.id, row_ids, inline: true) if import.inline?++ ProcessGroupJob.perform_later(import.id, row_ids)+ end+def dispatch_grouped(import, file_column)groups = Hash.new { |h, k| h[k] = [] }@@ -48,7 +57,7 @@ module Spreeupdated_at: Time.current)- batches.each { |row_ids| ProcessGroupJob.perform_later(import.id, row_ids) }+ batches.each { |row_ids| dispatch_group(import, row_ids) }enddef dispatch_batched(import)@@ -64,7 +73,7 @@ module Spreeupdated_at: Time.current)- row_id_batches.each { |row_ids| ProcessGroupJob.perform_later(import.id, row_ids) }+ row_id_batches.each { |row_ids| dispatch_group(import, row_ids) }endendenddiff --git a/spree/core/app/models/spree/import.rb b/spree/core/app/models/spree/import.rbindex 4e6071f392..4c1ff3e596 100644--- a/spree/core/app/models/spree/import.rb+++ b/spree/core/app/models/spree/import.rb@@ -93,6 +93,15 @@ module Spreeafter_transition on: :retry_failed_rows, do: :process_rows_asyncend+ # Run the pipeline in-process instead of enqueuing it. Virtual and not+ # persisted — it describes how *this* caller wants the import driven, not+ # anything about the record.+ #+ # For rake tasks, seeds and the console, where the caller needs the data to+ # exist when the call returns and there may be no worker attached. Never set+ # it from a request: a large CSV blocks for minutes.+ attribute :inline, :boolean, default: false+## Preferences#@@ -325,12 +334,18 @@ module Spree# Creates rows asynchronously# @return [void]def create_rows_async+ # The 2s delay lets the attachment settle before the job reads it; running+ # inline there is nothing to wait for.+ return Spree::Imports::CreateRowsJob.perform_now(id, inline: true) if inline?+Spree::Imports::CreateRowsJob.set(wait: 2.seconds).perform_later(id)end# Processes rows asynchronously# @return [void]def process_rows_async+ return Spree::Imports::ProcessRowsJob.perform_now(id, inline: true) if inline?+Spree::Imports::ProcessRowsJob.perform_later(id)enddiff --git a/spree/core/app/services/spree/imports/row_processors/product_variant.rb b/spree/core/app/services/spree/imports/row_processors/product_variant.rbindex 8b2ae36a79..18e75854f5 100644--- a/spree/core/app/services/spree/imports/row_processors/product_variant.rb+++ b/spree/core/app/services/spree/imports/row_processors/product_variant.rb@@ -295,11 +295,16 @@ module Spreeenddef handle_tags(product)+ return Spree::Imports::AssignTagsJob.perform_now(product.id, attributes['tags']) if import.inline?+Spree::Imports::AssignTagsJob.perform_later(product.id, attributes['tags'])enddef handle_categories(product)- Spree::Imports::CreateCategoriesJob.perform_later(product.id, store.id, prepare_taxon_pretty_names)+ names = prepare_taxon_pretty_names+ return Spree::Imports::CreateCategoriesJob.perform_now(product.id, store.id, names) if import.inline?++ Spree::Imports::CreateCategoriesJob.perform_later(product.id, store.id, names)enddef prepare_taxon_pretty_namesdiff --git a/spree/core/app/services/spree/sample_data/helper.rb b/spree/core/app/services/spree/sample_data/helper.rbnew file mode 100644index 0000000000..b400c1411a--- /dev/null+++ b/spree/core/app/services/spree/sample_data/helper.rb@@ -0,0 +1,55 @@+module Spree+ module SampleData+ # Shared by the sample-data drivers: `Spree::SampleData::Loader` (the+ # synchronous, rake-facing one) and any app that loads a subset itself —+ # e.g. running the configuration files inline and handing the CSVs to the+ # import pipeline. Keeping the file locations and catalog-publishing rules+ # here means they don't have to be kept in sync by hand.+ module Helper+ private++ def sample_data_path+ @sample_data_path ||= Spree::Core::Engine.root.join('db', 'sample_data')+ end++ def load_ruby_file(name)+ file = sample_data_path.join("#{name}.rb")+ load file.to_s if file.exist?+ end++ # Publishes the catalog to every channel the store has. The import only+ # adds a product to the default channel, and only when it has none, so+ # anything else the store defines (wholesale, a locale-specific storefront,+ # whatever an app has seeded) would otherwise be left empty. Catalog parity+ # is the point — channels differentiate through their own gates and price+ # lists, not through a narrower catalog. `add_products` upserts in bulk and+ # skips rows that already exist.+ def publish_sample_products(store)+ product_ids = store.product_ids+ return if product_ids.empty?++ store.channels.each { |channel| channel.add_products(product_ids) }+ end++ # Cheap proxy for "the base seeds have run": the US states are seeded+ # alongside everything else, and products/addresses depend on them.+ def seeds_loaded?+ Spree::State.joins(:country).where(spree_countries: { iso: 'US' }).exists?+ end++ def ensure_seeds_loaded+ return if seeds_loaded?++ Spree::Seeds::All.call+ end++ def without_geocoding+ previous = Spree::Config[:geocode_addresses]+ Spree::Config[:geocode_addresses] = false+ yield+ ensure+ Spree::Config[:geocode_addresses] = previous+ end+ end+ end+enddiff --git a/spree/core/app/services/spree/sample_data/import_builder.rb b/spree/core/app/services/spree/sample_data/import_builder.rbnew file mode 100644index 0000000000..975d58a63b--- /dev/null+++ b/spree/core/app/services/spree/sample_data/import_builder.rb@@ -0,0 +1,28 @@+module Spree+ module SampleData+ # Creates the `Spree::Import` record for a sample CSV, attaching the file+ # but processing nothing — the two runners differ only in what they do next.+ class ImportBuilder+ # @return [Spree::Import] persisted, unprocessed+ def self.call(csv_path:, import_class:, store: nil, user: nil)+ store ||= Spree::Store.default+ # An admin of the target store, not `admin_user_class.first` — in a+ # multi-store or multi-tenant app that global first can belong to a+ # different store entirely.+ user ||= store.users.first++ raise 'No admin user found. Please run seeds first.' unless user++ import = import_class.new(owner: store, user: user)+ import.number = import.generate_permalink(import_class)+ import.attachment.attach(+ io: File.open(csv_path),+ filename: File.basename(csv_path),+ content_type: 'text/csv'+ )+ import.save!(validate: false)+ import+ end+ end+ end+enddiff --git a/spree/core/app/services/spree/sample_data/import_runner.rb b/spree/core/app/services/spree/sample_data/import_runner.rbindex 63bb5f96c8..027dac0c32 100644--- a/spree/core/app/services/spree/sample_data/import_runner.rb+++ b/spree/core/app/services/spree/sample_data/import_runner.rb@@ -1,53 +1,31 @@-require 'csv'-module Spreemodule SampleData+ # Runs a sample CSV through the regular import pipeline: `start_mapping`+ # auto-assigns the column mappings from the CSV headers (the sample files+ # use the canonical schema names, so every column maps) and+ # `complete_mapping` kicks off row creation and processing.+ #+ # Enqueues by default, so row creation, batching and the per-import+ # concurrency cap behave exactly as they do for a merchant-uploaded CSV.+ # Pass `inline: true` for rake tasks, seeds and the console — where the data+ # has to exist when the call returns and there may be no worker attached.+ # Never inline from a request: `products.csv` alone is 211 rows, each+ # downloading a remote image.class ImportRunnerprepend Spree::ServiceModule::Base- def call(csv_path:, import_class:)- store = Spree::Store.default- admin = Spree.admin_user_class.first-- raise 'No admin user found. Please run seeds first.' unless admin-- import = import_class.new(- owner: store,- user: admin- )- import.number = import.generate_permalink(import_class)- import.attachment.attach(- io: File.open(csv_path),- filename: File.basename(csv_path),- content_type: 'text/csv'+ # @return [Spree::ServiceModule::Result] wrapping the import — completed+ # when inline, queued otherwise+ def call(csv_path:, import_class:, store: nil, user: nil, inline: false)+ import = Spree::SampleData::ImportBuilder.call(+ csv_path: csv_path, import_class: import_class, store: store, user: user)- import.save!(validate: false)- import.update_columns(status: 'processing')- import.create_mappings-- row_number = 0- failed = 0-- ::CSV.foreach(csv_path, headers: true) do |csv_row|- row_number += 1- import_row = import.rows.create!(- row_number: row_number,- data: csv_row.to_h.to_json,- status: 'pending'- )-- begin- import_row.process!- rescue StandardError => e- failed += 1- puts "\n Warning: Row #{row_number} failed: #{e.message}"- end+ import.inline = inline- print '.' if (row_number % 10).zero?- end+ import.start_mapping+ import.complete_mapping- import.update!(status: 'completed')- puts "\n Processed #{row_number} rows (#{failed} failed)"+ success(import.reload)endendenddiff --git a/spree/core/app/services/spree/sample_data/loader.rb b/spree/core/app/services/spree/sample_data/loader.rbindex 9a6c9a66ad..60c6dbfb5f 100644--- a/spree/core/app/services/spree/sample_data/loader.rb+++ b/spree/core/app/services/spree/sample_data/loader.rb@@ -2,11 +2,13 @@ module Spreemodule SampleDataclass Loaderprepend Spree::ServiceModule::Base+ include Spree::SampleData::Helperdef callSpree::Events.disable dowithout_geocoding doActiveRecord::Base.no_touching do+ puts 'Running seeds first...' unless seeds_loaded?ensure_seeds_loadedputs 'Loading sample configuration data...'@@ -28,10 +30,7 @@ module Spreeload_productsputs 'Publishing sample products on the default channel...'- publish_sample_products-- puts 'Loading sample categories...'- load_categories+ publish_sample_products(Spree::Store.default)puts 'Loading sample product translations...'load_product_translations@@ -56,18 +55,6 @@ module Spreeprivate- def ensure_seeds_loaded- us = Spree::Country.find_by(iso: 'US')- return if us&.states&.any? && Spree::Store.default&.persisted?-- puts 'Running seeds first...'- Spree::Seeds::All.call- end-- def sample_data_path- @sample_data_path ||= Spree::Core::Engine.root.join('db', 'sample_data')- end-def load_configuration_dataload_ruby_file('shipping_methods')load_ruby_file('payment_methods')@@ -76,57 +63,19 @@ module Spreedef load_productscsv_path = sample_data_path.join('products.csv')- Spree::SampleData::ImportRunner.call(csv_path: csv_path, import_class: Spree::Imports::Products)- end-- # Publishes the catalog to both surfaces: the public default channel and- # the gated wholesale channel (catalog parity; wholesale differentiation- # comes from the price list in +wholesale.rb+ and the channel's gate).- def publish_sample_products- store = Spree::Store.default- store.default_channel.add_products(store.product_ids)- store.channels.find_by(code: Spree::Seeds::Channels::WHOLESALE_CODE)&.add_products(store.product_ids)- end-- def load_categories- store = Spree::Store.default- csv_path = sample_data_path.join('products.csv')-- require 'csv'- ::CSV.foreach(csv_path, headers: true) do |row|- product = store.products.find_by(slug: row['slug'])- next unless product-- categories = [row['category1'], row['category2'], row['category3']].compact_blank- next if categories.empty?-- Spree::Imports::CreateCategoriesJob.perform_now(product.id, store.id, categories)- end+ Spree::SampleData::ImportRunner.call(csv_path: csv_path, import_class: Spree::Imports::Products, inline: true)enddef load_product_translationscsv_path = sample_data_path.join('product_translations.csv')return unless csv_path.exist?- Spree::SampleData::ImportRunner.call(csv_path: csv_path, import_class: Spree::Imports::ProductTranslations)+ Spree::SampleData::ImportRunner.call(csv_path: csv_path, import_class: Spree::Imports::ProductTranslations, inline: true)enddef load_customerscsv_path = sample_data_path.join('customers.csv')- Spree::SampleData::ImportRunner.call(csv_path: csv_path, import_class: Spree::Imports::Customers)- end-- def load_ruby_file(name)- file = sample_data_path.join("#{name}.rb")- load file.to_s if file.exist?- end-- def without_geocoding- previous = Spree::Config[:geocode_addresses]- Spree::Config[:geocode_addresses] = false- yield- ensure- Spree::Config[:geocode_addresses] = previous+ Spree::SampleData::ImportRunner.call(csv_path: csv_path, import_class: Spree::Imports::Customers, inline: true)endendenddiff --git a/spree/core/spec/services/spree/sample_data/import_runner_spec.rb b/spree/core/spec/services/spree/sample_data/import_runner_spec.rbnew file mode 100644index 0000000000..d1c930b174--- /dev/null+++ b/spree/core/spec/services/spree/sample_data/import_runner_spec.rb@@ -0,0 +1,64 @@+require 'spec_helper'++RSpec.describe Spree::SampleData::ImportRunner, type: :service do+ let(:store) { @default_store }+ let!(:admin) { create(:admin_user) }+ let(:csv_path) { Spree::Core::Engine.root.join('db', 'sample_data', 'customers.csv') }++ before { store.add_user(admin) }++ subject(:result) { described_class.call(csv_path: csv_path, import_class: Spree::Imports::Customers) }++ it 'returns the queued import' do+ expect(result).to be_success+ expect(result.value).to be_a(Spree::Imports::Customers)+ expect(result.value.owner).to eq(store)+ end++ # The whole point of this runner: hand the CSV to the pipeline and return, so+ # a request never pays for the rows.+ it 'creates no rows of its own' do+ expect(result.value.rows).to be_empty+ expect(result.value.status).to eq('completed_mapping')+ end++ it 'enqueues row creation' do+ expect { result }.to have_enqueued_job(Spree::Imports::CreateRowsJob)+ end++ it 'auto-maps the sample CSV columns' do+ expect(result.value.mappings.mapped.pluck(:schema_field)).to include('email')+ end++ # The rake/seed path: same pipeline, performed in-process so the data exists+ # when the call returns. Uses product translations — a self-contained CSV that+ # doesn't pull in the address/newsletter side effects customers do.+ context 'inline' do+ let(:csv_path) { Spree::Core::Engine.root.join('db', 'sample_data', 'product_translations.csv') }++ subject(:result) do+ described_class.call(csv_path: csv_path, import_class: Spree::Imports::ProductTranslations, inline: true)+ end++ it 'completes the import before returning' do+ expect(result.value.status).to eq('completed')+ expect(result.value.rows.count).to be_positive+ end++ it 'enqueues nothing' do+ expect { result }.not_to have_enqueued_job(Spree::Imports::CreateRowsJob)+ end+ end++ it 'runs as an admin of the target store' do+ expect(result.value.user).to eq(admin)+ end++ context 'when the store has no admin' do+ before { store.users.each { |user| store.remove_user(user) } }++ it 'raises rather than attributing the import to an unrelated admin' do+ expect { result }.to raise_error(/No admin user found/)+ end+ end+end