Skip to content

armgit / spree

Public Imported from GitHub
0

Open Source Platform for DTC, B2B Commerce, Marketplaces & Omnichannel. REST APIs, TypeScript SDKs, and production-ready Next.js storefront. Self-host it. Own your data. No vendor lock-in. Zero platform fees.

Let buyers and sellers attach documents without the file command (#14542)

Damian Legawiec6f92367b372f
diff --git a/spree/core/app/models/concerns/spree/purchase/purchase_order.rb b/spree/core/app/models/concerns/spree/purchase/purchase_order.rb
index 8876392331..b9d7ee9e54 100644
--- a/spree/core/app/models/concerns/spree/purchase/purchase_order.rb
+++ b/spree/core/app/models/concerns/spree/purchase/purchase_order.rb
@@ -37,14 +37,18 @@ module Spree
# download action.
has_one_attached :po_document, service: Spree.private_storage_service_name
- # Buyers are the lower-trust writers here — the storefront uploads this
- # — so the bytes decide what the file is, not the header the uploader
- # sent. Without spoofing_protection a script named `po.pdf` passes.
validates :po_document,
- content_type: { in: PO_DOCUMENT_CONTENT_TYPES, spoofing_protection: true },
size: { less_than_or_equal_to: MAX_PO_DOCUMENT_SIZE },
if: -> { po_document.attached? }
+ # Buyers are the lower-trust writers here — the storefront uploads this
+ # — so the bytes decide what the file is, not the header the uploader
+ # sent. Without that check a script named `po.pdf` passes.
+ validates_with Spree::AttachmentContentTypeValidator,
+ attributes: [:po_document],
+ in: PO_DOCUMENT_CONTENT_TYPES,
+ if: -> { po_document.attached? }
+
validate :po_document_within_size_on_disk, if: -> { po_document.attached? }
end
diff --git a/spree/core/app/models/spree/seller_requirement_submission.rb b/spree/core/app/models/spree/seller_requirement_submission.rb
index c763eaf369..07e06bd401 100644
--- a/spree/core/app/models/spree/seller_requirement_submission.rb
+++ b/spree/core/app/models/spree/seller_requirement_submission.rb
@@ -43,20 +43,21 @@ module Spree
#
validate :requirement_belongs_to_sellers_store
+ validates :file,
+ size: { less_than_or_equal_to: ->(_record) { Spree::Config.max_seller_document_upload_size } },
+ if: -> { file.attached? }
+
# Sellers are the marketplace's lower-trust writers and an operator opens
# these files on their own machine, so the bytes decide what a file is,
- # not the header the uploader sent. `spoofing_protection` is what makes
- # that true: without it a script named `certificate.pdf` passes.
+ # not the header the uploader sent: without that check a script named
+ # `certificate.pdf` passes.
#
# The accepted list comes from the requirement's own kind, the same value
# the seller API serializes so the panel's picker can hint it.
- validates :file,
- content_type: {
- in: ->(record) { record.requirement&.accepted_content_types || [] },
- spoofing_protection: true
- },
- size: { less_than_or_equal_to: ->(_record) { Spree::Config.max_seller_document_upload_size } },
- if: -> { file.attached? }
+ validates_with Spree::AttachmentContentTypeValidator,
+ attributes: [:file],
+ in: ->(record) { record.requirement&.accepted_content_types || [] },
+ if: -> { file.attached? }
#
# Scopes
diff --git a/spree/core/app/validators/spree/attachment_content_type_validator.rb b/spree/core/app/validators/spree/attachment_content_type_validator.rb
new file mode 100644
index 0000000000..e55e8e1c36
--- /dev/null
+++ b/spree/core/app/validators/spree/attachment_content_type_validator.rb
@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module Spree
+ # Checks an Active Storage attachment against an allowlist of content types,
+ # deciding what a file is from its bytes rather than from the header the
+ # uploader sent.
+ #
+ # This is the low-trust upload path: buyers attach purchase orders and
+ # sellers attach registration documents, and an operator opens both on their
+ # own machine. Without reading the bytes a shell script or an HTML page named
+ # `po.pdf` passes, so the declared content type is discarded outright and the
+ # filename is used only as described below.
+ #
+ # Reached through `validates_with` rather than a `validates` option, because
+ # Rails resolves that shorthand against the model's own ancestry and would
+ # not find a constant living in this namespace.
+ #
+ # validates_with Spree::AttachmentContentTypeValidator,
+ # attributes: [:po_document], in: CONTENT_TYPES
+ # validates_with Spree::AttachmentContentTypeValidator,
+ # attributes: [:file], in: ->(record) { record.accepted }
+ #
+ # Marcel does the reading, which is deliberate: it ships with Active Storage
+ # and identifies every type these allowlists carry without the Unix `file`
+ # command, a binary absent from most slim container images and whose absence
+ # otherwise surfaces to a buyer mid-checkout.
+ #
+ # @see Spree::Purchase::PurchaseOrder
+ # @see Spree::SellerRequirementSubmission
+ class AttachmentContentTypeValidator < ActiveModel::EachValidator
+ # Office files are containers — a zip for the modern formats, an OLE
+ # compound file for the older ones — and which office format a given
+ # container holds is recorded in a part that can sit anywhere inside it.
+ # Magic bytes alone therefore identify some of them only as the container,
+ # however much of the file is read. For those types, and only those, the
+ # extension breaks the tie.
+ #
+ # This is narrower than it looks. The container itself is still proven from
+ # the bytes, so no script, document or executable can borrow the name: the
+ # slack is that a plain zip renamed `.docx` is accepted as one. That is the
+ # same answer a merchant's own machine gives when they double-click it.
+ #
+ # A container whose own marker happens to fall within the bytes read is
+ # identified outright and never reaches the tie-break at all.
+ CONTAINER_CONTENT_TYPES = %w[application/zip application/x-ole-storage].freeze
+
+ # A misspelled or forgotten option would otherwise leave this validator
+ # silently accepting everything, on the one path whose whole purpose is
+ # refusing hostile files. An allowlist that resolves to empty at runtime is
+ # still allowed — a submission whose requirement names no types accepts any.
+ def check_validity!
+ return if options.key?(:in) || options.key?(:with)
+
+ raise ArgumentError, 'You must pass either :in or :with to the validator'
+ end
+
+ def validate_each(record, attribute, _value)
+ attachment = record.public_send(attribute)
+ return unless attachment.attached?
+
+ permitted = permitted_content_types(record)
+ return if permitted.empty?
+
+ pending = pending_attachables(record, attribute)
+
+ blobs_for(attachment).each_with_index do |blob, index|
+ validate_blob(record, attribute, blob, permitted, pending[index])
+ end
+ end
+
+ private
+
+ def permitted_content_types(record)
+ value = options[:in] || options[:with]
+ value = value.call(record) if value.respond_to?(:call)
+
+ Array.wrap(value).map(&:to_s)
+ end
+
+ # Serves both `has_one_attached` and `has_many_attached` without asking the
+ # proxy to behave like an array, which the single-attachment one does not.
+ def blobs_for(attachment)
+ attachment.respond_to?(:blobs) ? attachment.blobs : [attachment.blob]
+ end
+
+ # Where the bytes live depends on how the file arrived. A direct upload — the
+ # storefront and dashboard path — has already stored them and sends back a
+ # signed id. An attachment handed raw IO, as seeds and imports do, has not
+ # been uploaded yet: Active Storage holds it until the record saves, so it is
+ # read from the pending attachable instead. Reading storage in that case
+ # would find nothing and reject a file that is perfectly good.
+ def pending_attachables(record, attribute)
+ change = record.attachment_changes[attribute.to_s]
+ return [] if change.nil?
+
+ change.respond_to?(:attachables) ? Array(change.attachables) : [change.attachable]
+ end
+
+ def validate_blob(record, attribute, blob, permitted, attachable)
+ return if blob.nil?
+
+ return if permitted_bytes?(blob, attachable, permitted)
+
+ record.errors.add(attribute, :spree_content_type_not_allowed)
+ end
+
+ def permitted_bytes?(blob, attachable, permitted)
+ head = leading_bytes(blob, attachable)
+
+ # Bytes that could not be read at all are somebody else's problem to
+ # report — see {#leading_bytes} — and are not evidence of a bad file type.
+ return true if head.nil?
+ return false if head.empty?
+
+ # No declared type is passed: Marcel falls back to it for bytes it cannot
+ # read, which would hand the decision back to whoever uploaded the file.
+ detected = Marcel::MimeType.for(StringIO.new(head))
+ return true if permitted.include?(detected)
+ return false unless CONTAINER_CONTENT_TYPES.include?(detected)
+
+ permitted.include?(office_type_from_extension(blob))
+ end
+
+ # Asks the extension alone, rather than re-reading the bytes with a filename
+ # attached. Marcel would only let the name win where it records the office
+ # type as a descendant of the container, which it does for the zip-based
+ # formats but not the OLE-based ones — so going through it would quietly
+ # reject every legacy Word document whose internal marker sits beyond the
+ # bytes we read.
+ #
+ # @return [String, nil]
+ def office_type_from_extension(blob)
+ filename = blob.filename.to_s
+ return if File.extname(filename).blank?
+
+ Marcel::MimeType.for(name: filename)
+ end
+
+ # Only the leading bytes are read: that is all a magic-number match needs,
+ # and it keeps a 10 MB purchase order from being pulled into memory to
+ # identify its first few bytes.
+ #
+ # @return [String, nil] the bytes, or nil when there are none to read
+ def leading_bytes(blob, attachable)
+ pending = pending_bytes(attachable)
+ return pending if pending
+
+ blob.service.download_chunk(blob.key, 0...MAGIC_BYTES_LENGTH)
+ rescue ActiveStorage::FileNotFoundError
+ # An abandoned direct upload leaves the blob row without its bytes. Saying
+ # the type is wrong would misdescribe that, so this stays quiet and lets
+ # the callers report the upload itself as incomplete.
+ nil
+ end
+
+ # @return [String, nil] the head of a file not yet uploaded, leaving the
+ # caller's IO rewound so the upload itself still sees the whole thing
+ def pending_bytes(attachable)
+ case attachable
+ when Pathname then attachable.open('rb') { |file| file.read(MAGIC_BYTES_LENGTH) }
+ when Hash then read_head(attachable[:io])
+ else read_head(attachable)
+ end
+ end
+
+ # An uploaded file and a plain IO are both read in place and rewound: the
+ # upload still has to send the whole file afterwards.
+ def read_head(io)
+ io = io.open if io.is_a?(ActionDispatch::Http::UploadedFile)
+ return unless io.respond_to?(:read) && io.respond_to?(:rewind)
+
+ begin
+ io.rewind
+ io.read(MAGIC_BYTES_LENGTH)
+ ensure
+ io.rewind
+ end
+ end
+
+ MAGIC_BYTES_LENGTH = 4.kilobytes
+ private_constant :MAGIC_BYTES_LENGTH
+ end
+end
diff --git a/spree/core/config/locales/en.yml b/spree/core/config/locales/en.yml
index b2c5515aad..d310cfe0e1 100644
--- a/spree/core/config/locales/en.yml
+++ b/spree/core/config/locales/en.yml
@@ -248,6 +248,7 @@ en:
blank: can't be blank
must_apply_to_products: must be a custom field definition for products
not_registered: is not a registered provider
+ spree_content_type_not_allowed: is not a file type we accept
models:
spree/allowed_origin:
attributes:
diff --git a/spree/core/spec/validators/spree/attachment_content_type_validator_spec.rb b/spree/core/spec/validators/spree/attachment_content_type_validator_spec.rb
new file mode 100644
index 0000000000..a9007afc0e
--- /dev/null
+++ b/spree/core/spec/validators/spree/attachment_content_type_validator_spec.rb
@@ -0,0 +1,286 @@
+require 'spec_helper'
+
+RSpec.describe Spree::AttachmentContentTypeValidator do
+ # A throwaway model so the validator is exercised on its own rather than
+ # through whichever allowlist a real one happens to carry.
+ let(:model_class) do
+ Class.new(Spree::Cart) do
+ def self.name = 'Spree::Cart'
+
+ has_one_attached :document
+ validates_with Spree::AttachmentContentTypeValidator,
+ attributes: [:document],
+ in: %w[application/pdf image/png application/msword application/vnd.openxmlformats-officedocument.wordprocessingml.document]
+ end
+ end
+
+ let(:record) { model_class.new(store: @default_store) }
+
+ def attach(bytes, filename:, content_type:)
+ record.document.attach(io: StringIO.new(bytes.b), filename: filename, content_type: content_type)
+ record
+ end
+
+ # Minimal but genuine signatures — the validator reads these, so a
+ # placeholder string would prove nothing.
+ let(:pdf_bytes) { "%PDF-1.7\n1 0 obj\n<<>>\nendobj\ntrailer\n" }
+ let(:png_bytes) { "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR#{"\x00" * 64}" }
+
+ describe 'a file whose bytes match the allowlist' do
+ it 'accepts a PDF' do
+ attach(pdf_bytes, filename: 'po.pdf', content_type: 'application/pdf').validate
+
+ expect(record.errors[:document]).to be_empty
+ end
+
+ it 'accepts a PNG' do
+ attach(png_bytes, filename: 'scan.png', content_type: 'image/png').validate
+
+ expect(record.errors[:document]).to be_empty
+ end
+ end
+
+ describe 'a file whose bytes contradict what the uploader declared' do
+ # The whole point of reading the bytes: an uploader controls the filename
+ # and the content type header, and neither is evidence.
+ it 'rejects a shell script presented as a PDF' do
+ attach("#!/bin/sh\nrm -rf /\n", filename: 'po.pdf', content_type: 'application/pdf').validate
+
+ expect(record.errors[:document]).to include('is not a file type we accept')
+ end
+
+ it 'rejects an HTML page presented as a PDF' do
+ attach('<html><script>alert(1)</script></html>', filename: 'po.pdf', content_type: 'application/pdf').validate
+
+ expect(record.errors[:document]).to be_present
+ end
+
+ it 'rejects a Windows executable presented as a PNG' do
+ attach("MZ\x90\x00\x03\x00\x00\x00#{"\x00" * 200}", filename: 'logo.png', content_type: 'image/png').validate
+
+ expect(record.errors[:document]).to be_present
+ end
+
+ # A type that is real, but not one this attachment accepts.
+ it 'rejects a GIF presented as a PNG' do
+ attach("GIF89a#{"\x00" * 64}", filename: 'scan.png', content_type: 'image/png').validate
+
+ expect(record.errors[:document]).to be_present
+ end
+ end
+
+ # Office formats are zip archives whose flavour is recorded in a part that
+ # can sit anywhere inside, so the extension settles which one it is — but
+ # only once the bytes have proven it is an archive at all.
+ describe 'an office document' do
+ # A real zip archive, built by hand so the spec does not depend on a zip
+ # library being in the bundle. One stored (uncompressed) entry is enough
+ # for the archive to be recognised as one.
+ let(:docx_bytes) do
+ name = 'word/document.xml'
+ body = '<?xml version="1.0"?><w:document/>'
+ crc = Zlib.crc32(body)
+ local = [0x04034b50, 20, 0, 0, 0, 0, crc, body.bytesize, body.bytesize, name.bytesize, 0].pack('Vv5V3v2')
+ local += name + body
+ central = [0x02014b50, 20, 20, 0, 0, 0, 0, crc, body.bytesize, body.bytesize,
+ name.bytesize, 0, 0, 0, 0, 0, 0].pack('Vv6V3v5V2')
+ central += name
+ ending = [0x06054b50, 0, 0, 1, 1, central.bytesize, local.bytesize, 0].pack('Vv4V2v')
+
+ local + central + ending
+ end
+
+ it 'accepts one named as the office type it is' do
+ attach(docx_bytes, filename: 'po.docx',
+ content_type: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document').validate
+
+ expect(record.errors[:document]).to be_empty
+ end
+
+ # The extension only ever narrows an archive — it cannot rescue bytes that
+ # are not one.
+ it 'rejects a script named with an office extension' do
+ attach("#!/bin/sh\necho pwned\n", filename: 'po.docx',
+ content_type: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document').validate
+
+ expect(record.errors[:document]).to be_present
+ end
+
+ # Where a legacy Word document records what it is depends on its internal
+ # layout, and in a real one it routinely sits past the bytes read here. The
+ # container plus the extension has to be enough, or every such file is
+ # refused — which is what happens if the tie-break asks Marcel to weigh the
+ # name against the bytes, since it does not record msword as a kind of OLE.
+ it 'accepts a legacy Word document whose marker sits beyond the bytes read' do
+ ole = +"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1".b
+ ole << ("\x00" * 20_000).b
+ ole[8000, 24] = 'WordDocument'.encode('utf-16le').b
+
+ attach(ole, filename: 'po.doc', content_type: 'application/msword').validate
+
+ expect(record.errors[:document]).to be_empty
+ end
+
+ # An office extension on a container is a tie-break, never a passport: the
+ # bytes still have to be that container.
+ it 'rejects a script named with a legacy Word extension' do
+ attach("#!/bin/sh\necho pwned\n", filename: 'po.doc', content_type: 'application/msword').validate
+
+ expect(record.errors[:document]).to be_present
+ end
+
+ # A Word document identifies itself from inside its container, so it is
+ # accepted on its bytes rather than on its name.
+ it 'accepts a legacy Word document on its own marker' do
+ ole = +"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1".b
+ ole << ("\x00" * 5000).b
+ ole[1536, 24] = 'WordDocument'.encode('utf-16le').b
+
+ attach(ole, filename: 'po.doc', content_type: 'application/msword').validate
+
+ expect(record.errors[:document]).to be_empty
+ end
+ end
+
+ describe 'when the allowlist is empty' do
+ let(:model_class) do
+ Class.new(Spree::Cart) do
+ def self.name = 'Spree::Cart'
+
+ has_one_attached :document
+ validates_with Spree::AttachmentContentTypeValidator, attributes: [:document], in: []
+ end
+ end
+
+ it 'accepts anything, since nothing was asked for' do
+ attach("#!/bin/sh\n", filename: 'anything.sh', content_type: 'text/plain').validate
+
+ expect(record.errors[:document]).to be_empty
+ end
+ end
+
+ describe 'when the allowlist is a callable' do
+ let(:model_class) do
+ Class.new(Spree::Cart) do
+ def self.name = 'Spree::Cart'
+
+ has_one_attached :document
+ validates_with Spree::AttachmentContentTypeValidator,
+ attributes: [:document], in: ->(_record) { %w[image/png] }
+ end
+ end
+
+ it 'resolves it against the record' do
+ attach(png_bytes, filename: 'scan.png', content_type: 'image/png').validate
+ expect(record.errors[:document]).to be_empty
+
+ other = model_class.new(store: @default_store)
+ other.document.attach(io: StringIO.new(pdf_bytes.b), filename: 'po.pdf', content_type: 'application/pdf')
+ other.validate
+
+ expect(other.errors[:document]).to be_present
+ end
+ end
+
+ # Files reach Active Storage in several shapes, and the bytes have to be
+ # found in all of them — a shape the validator cannot read would either
+ # reject good files or, worse, wave bad ones through.
+ describe 'however the file was handed over' do
+ it 'reads an uploaded file from a form' do
+ file = Tempfile.new(['po', '.pdf'], binmode: true)
+ file.write(pdf_bytes.b)
+ file.rewind
+ upload = ActionDispatch::Http::UploadedFile.new(
+ tempfile: file, filename: 'po.pdf', type: 'application/pdf'
+ )
+
+ record.document.attach(upload)
+ record.validate
+
+ expect(record.errors[:document]).to be_empty
+ end
+
+ it 'reads a path on disk' do
+ path = Rails.root.join('tmp', "validator_#{SecureRandom.hex(4)}.pdf")
+ FileUtils.mkdir_p(File.dirname(path))
+ File.binwrite(path, pdf_bytes.b)
+
+ record.document.attach(io: File.open(path, 'rb'), filename: 'po.pdf', content_type: 'application/pdf')
+ record.validate
+
+ expect(record.errors[:document]).to be_empty
+ ensure
+ FileUtils.rm_f(path)
+ end
+
+ # Reading the head must not consume the file: whatever runs next still has
+ # to send every byte.
+ it 'leaves the caller io rewound for the upload that follows' do
+ io = StringIO.new(pdf_bytes.b)
+ record.document.attach(io: io, filename: 'po.pdf', content_type: 'application/pdf')
+ record.validate
+
+ expect(io.pos).to eq(0)
+ expect(io.read).to eq(pdf_bytes.b)
+ end
+ end
+
+ # An allowlist option that never arrives would leave this accepting anything
+ # on an upload path whose whole purpose is refusing hostile files, so it is
+ # refused at boot rather than silently at runtime.
+ describe 'when the allowlist option is missing' do
+ it 'refuses to define the validation' do
+ expect do
+ Class.new(Spree::Cart) do
+ def self.name = 'Spree::Cart'
+
+ has_one_attached :document
+ validates_with Spree::AttachmentContentTypeValidator,
+ attributes: [:document], types: %w[application/pdf]
+ end
+ end.to raise_error(ArgumentError, /:in or :with/)
+ end
+ end
+
+ describe 'when no file is attached' do
+ it 'says nothing' do
+ record.validate
+
+ expect(record.errors[:document]).to be_empty
+ end
+ end
+
+ # An abandoned direct upload leaves the blob row without its bytes, as does a
+ # file deleted from storage behind the record's back. The callers report the
+ # former as an incomplete upload, which is more use to a buyer than being
+ # told the type is wrong, so this validator stays quiet rather than blaming
+ # the file type for bytes that were never delivered.
+ describe 'when the stored bytes are missing' do
+ it 'adds no content type error' do
+ record.save!
+ record.document.attach(io: StringIO.new(pdf_bytes.b), filename: 'po.pdf', content_type: 'application/pdf')
+ record.save!
+ record.document.blob.service.delete(record.document.blob.key)
+ record.reload
+
+ # Guards against this passing for the wrong reason: the attachment has to
+ # be genuinely present for the validator to look at it at all.
+ expect(record.document).to be_attached
+ expect(record.errors[:document]).to be_empty
+
+ expect { record.validate }.not_to raise_error
+ expect(record.errors[:document]).to be_empty
+ end
+ end
+
+ # The bug this validator replaces: the previous implementation shelled out to
+ # the Unix `file` command and, on an image without it, raised its own error
+ # whose internal text reached the buyer as the validation message.
+ it 'does not shell out to the file command' do
+ expect(Open3).not_to receive(:capture2)
+
+ attach(pdf_bytes, filename: 'po.pdf', content_type: 'application/pdf').validate
+
+ expect(record.errors[:document]).to be_empty
+ end
+end
diff --git a/spree/core/spec/workflows/spree/seller_requirement_submissions/lifecycle_spec.rb b/spree/core/spec/workflows/spree/seller_requirement_submissions/lifecycle_spec.rb
index de689a021e..af99b34bda 100644
--- a/spree/core/spec/workflows/spree/seller_requirement_submissions/lifecycle_spec.rb
+++ b/spree/core/spec/workflows/spree/seller_requirement_submissions/lifecycle_spec.rb
@@ -56,7 +56,7 @@ RSpec.describe 'seller requirement submission workflows' do
)
expect(result).to be_failure
- expect(result.error.value.full_messages.join).to match(/invalid content type/i)
+ expect(result.error.value.full_messages.join).to match(/not a file type we accept/i)
end
# The header is the uploader's word for what a file is. A seller is the
@@ -76,7 +76,7 @@ RSpec.describe 'seller requirement submission workflows' do
)
expect(result).to be_failure
- expect(result.error.value.full_messages.join).to match(/content type/i)
+ expect(result.error.value.full_messages.join).to match(/not a file type we accept/i)
end
it 'refuses a document larger than the configured limit' do