Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
CSV imports (no UI) (#3895)
ruslandoga279e89c69311
diff --git a/config/.env.dev b/config/.env.devindex 32ab90a455..58e4ca827d 100644--- a/config/.env.dev+++ b/config/.env.dev@@ -27,3 +27,4 @@ S3_SECRET_ACCESS_KEY=minioadminS3_REGION=us-east-1S3_ENDPOINT=http://localhost:10000S3_EXPORTS_BUCKET=dev-exports+S3_IMPORTS_BUCKET=dev-importsdiff --git a/config/.env.test b/config/.env.testindex d41cde0ebf..8fe3862a76 100644--- a/config/.env.test+++ b/config/.env.test@@ -22,3 +22,4 @@ S3_SECRET_ACCESS_KEY=minioadminS3_REGION=us-east-1S3_ENDPOINT=http://localhost:10000S3_EXPORTS_BUCKET=test-exports+S3_IMPORTS_BUCKET=test-importsdiff --git a/config/runtime.exs b/config/runtime.exsindex cd1fbbb75a..ad64d357fa 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -736,6 +736,10 @@ unless s3_disabled? do%{name: "S3_EXPORTS_BUCKET",example: "my-csv-exports-bucket"+ },+ %{+ name: "S3_IMPORTS_BUCKET",+ example: "my-csv-imports-bucket"}]@@ -771,5 +775,7 @@ unless s3_disabled? dohost: s3_host,port: s3_port- config :plausible, Plausible.S3, exports_bucket: s3_env_value.("S3_EXPORTS_BUCKET")+ config :plausible, Plausible.S3,+ exports_bucket: s3_env_value.("S3_EXPORTS_BUCKET"),+ imports_bucket: s3_env_value.("S3_IMPORTS_BUCKET")enddiff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.exindex d2d02e5426..2d10ca32e9 100644--- a/lib/plausible/imported/csv_importer.ex+++ b/lib/plausible/imported/csv_importer.ex@@ -20,7 +20,13 @@ defmodule Plausible.Imported.CSVImporter do@impl truedef import_data(site_import, opts) do- %{id: import_id, site_id: site_id} = site_import+ %{+ id: import_id,+ site_id: site_id,+ start_date: start_date,+ end_date: end_date+ } = site_import+uploads = Keyword.fetch!(opts, :uploads)%{access_key_id: s3_access_key_id, secret_access_key: s3_secret_access_key} =@@ -31,52 +37,36 @@ defmodule Plausible.Imported.CSVImporter do|> Keyword.replace!(:pool_size, 1)|> Ch.start_link()- ranges =- Enum.map(uploads, fn upload ->- %{"filename" => filename, "s3_url" => s3_url} = upload-- ".csv" = Path.extname(filename)- table = Path.rootname(filename)- ensure_importable_table!(table)-- s3_structure = input_structure!(table)-- statement =- """- INSERT INTO {table:Identifier} \- SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \- FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String})\- """-- params =- %{- "table" => table,- "site_id" => site_id,- "import_id" => import_id,- "s3_url" => s3_url,- "s3_access_key_id" => s3_access_key_id,- "s3_secret_access_key" => s3_secret_access_key,- "s3_format" => "CSVWithNames",- "s3_structure" => s3_structure- }-- Ch.query!(ch, statement, params, timeout: :infinity)-- %Ch.Result{rows: [[min_date, max_date]]} =- Ch.query!(- ch,- "SELECT min(date), max(date) FROM {table:Identifier} WHERE site_id = {site_id:UInt64} AND import_id = {import_id:UInt64}",- %{"table" => table, "site_id" => site_id, "import_id" => import_id}- )-- Date.range(min_date, max_date)- end)-- {:ok,- %{- start_date: Enum.min_by(ranges, & &1.first, Date).first,- end_date: Enum.max_by(ranges, & &1.last, Date).last- }}+ Enum.each(uploads, fn upload ->+ %{"filename" => filename, "s3_url" => s3_url} = upload++ {table, _, _} = parse_filename!(filename)+ s3_structure = input_structure!(table)++ statement =+ """+ INSERT INTO {table:Identifier} \+ SELECT {site_id:UInt64} AS site_id, *, {import_id:UInt64} AS import_id \+ FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \+ WHERE date >= {start_date:Date} AND date <= {end_date:Date}\+ """++ params =+ %{+ "table" => table,+ "site_id" => site_id,+ "import_id" => import_id,+ "s3_url" => s3_url,+ "s3_access_key_id" => s3_access_key_id,+ "s3_secret_access_key" => s3_secret_access_key,+ "s3_format" => "CSVWithNames",+ "s3_structure" => s3_structure,+ "start_date" => start_date,+ "end_date" => end_date+ }++ Ch.query!(ch, statement, params, timeout: :infinity)+ end)rescue# we are cancelling on any argument or ClickHouse errorse in [ArgumentError, Ch.Error] ->@@ -103,12 +93,85 @@ defmodule Plausible.Imported.CSVImporter do"date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64"}+ @doc """+ Extracts min/max date range from a list of uploads.++ Examples:++ iex> date_range([+ ...> %{"filename" => "imported_devices_20190101_20210101.csv"},+ ...> "imported_pages_20200101_20220101.csv"+ ...> ])+ Date.range(~D[2019-01-01], ~D[2022-01-01])++ iex> date_range([])+ ** (ArgumentError) empty uploads++ """+ @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t()+ def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)+ def date_range([]), do: raise(ArgumentError, "empty uploads")++ defp date_range([upload | uploads], prev_start_date, prev_end_date) do+ filename =+ case upload do+ %{"filename" => filename} -> filename+ filename when is_binary(filename) -> filename+ end++ {_table, start_date, end_date} = parse_filename!(filename)++ start_date =+ if prev_start_date do+ Enum.min([start_date, prev_start_date], Date)+ else+ start_date+ end++ end_date =+ if prev_end_date do+ Enum.max([end_date, prev_end_date], Date)+ else+ end_date+ end++ date_range(uploads, start_date, end_date)+ end++ defp date_range([], first, last), do: Date.range(first, last)++ @spec parse_date!(String.t()) :: Date.t()+ defp parse_date!(date) do+ date |> Timex.parse!("{YYYY}{0M}{0D}") |> NaiveDateTime.to_date()+ end++ @doc """+ Extracts table name and min/max dates from the filename.++ Examples:++ iex> parse_filename!("my_data.csv")+ ** (ArgumentError) invalid filename++ iex> parse_filename!("imported_devices_00010101_20250101.csv")+ {"imported_devices", ~D[0001-01-01], ~D[2025-01-01]}++ """+ @spec parse_filename!(String.t()) ::+ {table :: String.t(), start_date :: Date.t(), end_date :: Date.t()}+ def parse_filename!(filename)+for {table, input_structure} <- input_structures dodefp input_structure!(unquote(table)), do: unquote(input_structure)- defp ensure_importable_table!(unquote(table)), do: :ok++ def parse_filename!(+ <<unquote(table)::bytes, ?_, start_date::8-bytes, ?_, end_date::8-bytes, ".csv">>+ ) do+ {unquote(table), parse_date!(start_date), parse_date!(end_date)}+ endend- defp ensure_importable_table!(table) do- raise ArgumentError, "table #{table} is not supported for data import"+ def parse_filename!(_filename) do+ raise ArgumentError, "invalid filename"endenddiff --git a/lib/plausible/s3.ex b/lib/plausible/s3.exindex 9086e002e1..2768ce9eff 100644--- a/lib/plausible/s3.ex+++ b/lib/plausible/s3.ex@@ -18,9 +18,64 @@ defmodule Plausible.S3 do@spec exports_bucket :: String.t()def exports_bucket, do: config(:exports_bucket)+ @doc """+ Returns the pre-configured S3 bucket for CSV imports.++ config :plausible, Plausible.S3,+ imports_bucket: System.fetch_env!("S3_IMPORTS_BUCKET")++ Example:++ iex> imports_bucket()+ "test-imports"++ """+ @spec imports_bucket :: String.t()+ def imports_bucket, do: config(:imports_bucket)+defp config, do: Application.fetch_env!(:plausible, __MODULE__)defp config(key), do: Keyword.fetch!(config(), key)+ @doc """+ Presigns an upload for an imported file.++ In the current implementation the bucket always goes into the path component.++ Example:++ iex> %{+ ...> s3_url: "http://localhost:10000/test-imports/123/imported_browsers.csv",+ ...> presigned_url: "http://localhost:10000/test-imports/123/imported_browsers.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin" <> _+ ...> } = import_presign_upload(_site_id = 123, _filename = "imported_browsers.csv")++ """+ def import_presign_upload(site_id, filename) do+ config = ExAws.Config.new(:s3)+ s3_path = Path.join(to_string(site_id), filename)+ bucket = imports_bucket()+ {:ok, presigned_url} = ExAws.S3.presigned_url(config, :put, bucket, s3_path)+ %{s3_url: extract_s3_url(presigned_url), presigned_url: presigned_url}+ end++ # to make ClickHouse see MinIO in dev and test envs we replace+ # the host in the S3 URL with whatever's set in S3_CLICKHOUSE_HOST env var+ if Mix.env() in [:dev, :test, :small_dev, :small_test] do+ defp extract_s3_url(presigned_url) do+ [s3_url, _] = String.split(presigned_url, "?")++ if ch_host = System.get_env("S3_CLICKHOUSE_HOST") do+ URI.to_string(%URI{URI.parse(s3_url) | host: ch_host})+ else+ s3_url+ end+ end+ else+ defp extract_s3_url(presigned_url) do+ [s3_url, _] = String.split(presigned_url, "?")+ s3_url+ end+ end+@doc """Chunks and uploads Zip archive to the provided S3 destination.@@ -77,6 +132,12 @@ defmodule Plausible.S3 do@doc """Returns `access_key_id` and `secret_access_key` to be used by ClickHouse during imports from S3.++ Example:++ iex> import_clickhouse_credentials()+ %{access_key_id: "minioadmin", secret_access_key: "minioadmin"}+"""@spec import_clickhouse_credentials ::%{access_key_id: String.t(), secret_access_key: String.t()}diff --git a/lib/workers/export_csv.ex b/lib/workers/export_csv.exindex 1eb3d42e1e..5b46d4fd57 100644--- a/lib/workers/export_csv.ex+++ b/lib/workers/export_csv.ex@@ -25,43 +25,56 @@ defmodule Plausible.Workers.ExportCSV do|> Keyword.replace!(:pool_size, 1)|> Ch.start_link()- # NOTE: should we use site.timezone?- # %Ch.Result{rows: [[min_date, max_date]]} =- # Ch.query!(- # ch,- # "SELECT toDate(min(timestamp)), toDate(max(timestamp)) FROM events_v2 WHERE site_id={site_id:UInt64}",- # %{"site_id" => site_id}- # )-- download_url =- DBConnection.run(+ %Ch.Result{rows: [[%Date{} = min_date, %Date{} = max_date]]} =+ Ch.query!(ch,- fn conn ->- conn- |> Plausible.Exports.stream_archive(- # date_range: Date.range(min_date, max_date)- Plausible.Exports.export_queries(site_id, extname: ".csv"),- format: "CSVWithNames"- )- |> Plausible.S3.export_upload_multipart(s3_bucket, s3_path, s3_config_overrides(args))- end,- timeout: :infinity+ "SELECT toDate(min(timestamp)), toDate(max(timestamp)) FROM events_v2 WHERE site_id={site_id:UInt64}",+ %{"site_id" => site_id}+ )++ if max_date == ~D[1970-01-01] do+ # NOTE: replace with proper Plausible.Email template+ Plausible.Mailer.deliver_now!(+ Bamboo.Email.new_email(+ from: PlausibleWeb.Email.mailer_email_from(),+ to: email,+ subject: "EXPORT FAILURE",+ text_body: "there is nothing to export"+ ))+ else+ download_url =+ DBConnection.run(+ ch,+ fn conn ->+ conn+ |> Plausible.Exports.stream_archive(+ Plausible.Exports.export_queries(site_id,+ date_range: Date.range(min_date, max_date),+ extname: ".csv"+ ),+ format: "CSVWithNames"+ )+ |> Plausible.S3.export_upload_multipart(s3_bucket, s3_path, s3_config_overrides(args))+ end,+ timeout: :infinity+ )- # NOTE: replace with proper Plausible.Email template- Plausible.Mailer.deliver_now!(- Bamboo.Email.new_email(- from: "[email protected]",- to: email,- subject: "EXPORT SUCCESS",- text_body: """- download it from #{download_url}! hurry up! you have 24 hours!"- """,- html_body: """- download it from <a href="#{download_url}">here</a>! hurry up! you have 24 hours!- """+ # NOTE: replace with proper Plausible.Email template+ Plausible.Mailer.deliver_now!(+ Bamboo.Email.new_email(+ from: PlausibleWeb.Email.mailer_email_from(),+ to: email,+ subject: "EXPORT SUCCESS",+ text_body: """+ download it from #{download_url}! hurry up! you have 24 hours!"+ """,+ html_body: """+ download it from <a href="#{download_url}">here</a>! hurry up! you have 24 hours!+ """+ ))- )+ end:okenddiff --git a/test/plausible/config_test.exs b/test/plausible/config_test.exsindex 1adb30a6d2..ce2829fa6c 100644--- a/test/plausible/config_test.exs+++ b/test/plausible/config_test.exs@@ -198,7 +198,8 @@ defmodule Plausible.ConfigTest do{"S3_SECRET_ACCESS_KEY", nil},{"S3_REGION", nil},{"S3_ENDPOINT", nil},- {"S3_EXPORTS_BUCKET", nil}+ {"S3_EXPORTS_BUCKET", nil},+ {"S3_IMPORTS_BUCKET", nil}]result =@@ -211,13 +212,14 @@ defmodule Plausible.ConfigTest doassert %ArgumentError{} = resultassert Exception.message(result) == """- Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION, S3_ENDPOINT, S3_EXPORTS_BUCKET environment variable(s):+ Missing S3 configuration. Please set S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION, S3_ENDPOINT, S3_EXPORTS_BUCKET, S3_IMPORTS_BUCKET environment variable(s):\tS3_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE\tS3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\tS3_REGION=us-east-1\tS3_ENDPOINT=https://<ACCOUNT_ID>.r2.cloudflarestorage.com\tS3_EXPORTS_BUCKET=my-csv-exports-bucket+ \tS3_IMPORTS_BUCKET=my-csv-imports-bucket"""end@@ -227,7 +229,8 @@ defmodule Plausible.ConfigTest do{"S3_SECRET_ACCESS_KEY", nil},{"S3_REGION", "eu-north-1"},{"S3_ENDPOINT", nil},- {"S3_EXPORTS_BUCKET", "my-exports"}+ {"S3_EXPORTS_BUCKET", "my-exports"},+ {"S3_IMPORTS_BUCKET", nil}]result =@@ -240,10 +243,11 @@ defmodule Plausible.ConfigTest doassert %ArgumentError{} = resultassert Exception.message(result) == """- Missing S3 configuration. Please set S3_SECRET_ACCESS_KEY, S3_ENDPOINT environment variable(s):+ Missing S3 configuration. Please set S3_SECRET_ACCESS_KEY, S3_ENDPOINT, S3_IMPORTS_BUCKET environment variable(s):\tS3_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\tS3_ENDPOINT=https://<ACCOUNT_ID>.r2.cloudflarestorage.com+ \tS3_IMPORTS_BUCKET=my-csv-imports-bucket"""end@@ -253,7 +257,8 @@ defmodule Plausible.ConfigTest do{"S3_SECRET_ACCESS_KEY", "minioadmin"},{"S3_REGION", "us-east-1"},{"S3_ENDPOINT", "http://localhost:6000"},- {"S3_EXPORTS_BUCKET", "my-exports"}+ {"S3_EXPORTS_BUCKET", "my-exports"},+ {"S3_IMPORTS_BUCKET", "my-imports"}]config = runtime_config(env)@@ -266,8 +271,9 @@ defmodule Plausible.ConfigTest dos3: [scheme: "http://", host: "localhost", port: 6000]]- assert get_in(runtime_config(env), [:plausible, Plausible.S3]) == [- exports_bucket: "my-exports"+ assert get_in(config, [:plausible, Plausible.S3]) == [+ exports_bucket: "my-exports",+ imports_bucket: "my-imports"]endenddiff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex 60322a3852..5b988483cd 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -4,6 +4,8 @@ defmodule Plausible.Imported.CSVImporterTest doalias Testcontainers.MinioContainerrequire SiteImport+ doctest CSVImporter, import: true+@moduletag :miniosetup_all do@@ -55,9 +57,12 @@ defmodule Plausible.Imported.CSVImporterTest do"imported_visitors"]+ start_date = "20231001"+ end_date = "20240102"+uploads =Enum.map(tables, fn table ->- filename = "#{table}.csv"+ filename = "#{table}_#{start_date}_#{end_date}.csv"%{"filename" => filename,@@ -65,11 +70,12 @@ defmodule Plausible.Imported.CSVImporterTest do}end)+ date_range = CSVImporter.date_range(uploads)+assert {:ok, job} =CSVImporter.new_import(site, user,- # to satisfy the non null constraints on the table I'm providing "0" dates (according to ClickHouse)- start_date: ~D[1970-01-01],- end_date: ~D[1970-01-01],+ start_date: date_range.first,+ end_date: date_range.last,uploads: uploads)@@ -80,8 +86,8 @@ defmodule Plausible.Imported.CSVImporterTest do%{id: ^import_id,source: :csv,- start_date: ~D[1970-01-01],- end_date: ~D[1970-01-01],+ start_date: ~D[2023-10-01],+ end_date: ~D[2024-01-02],status: SiteImport.pending()}] = Plausible.Imported.list_all_imports(site)@@ -97,7 +103,7 @@ defmodule Plausible.Imported.CSVImporterTest dotest "imports tables from S3", %{site: site, user: user, s3: s3, container: minio} docsvs = [%{- name: "imported_browsers.csv",+ name: "imported_browsers_20211230_20211231.csv",body: """"date","browser","visitors","visits","visit_duration","bounces""2021-12-30","Amazon Silk",2,2,0,2@@ -122,7 +128,7 @@ defmodule Plausible.Imported.CSVImporterTest do"""},%{- name: "imported_devices.csv",+ name: "imported_devices_20211230_20220102.csv",body: """"date","device","visitors","visits","visit_duration","bounces""2021-12-30","Desktop",25,28,75,27@@ -140,7 +146,7 @@ defmodule Plausible.Imported.CSVImporterTest do"""},%{- name: "imported_entry_pages.csv",+ name: "imported_entry_pages_20211230_20211231.csv",body: """"date","visitors","entrances","visit_duration","bounces","entry_page""2021-12-30",6,6,0,6,"/14776416252794997127"@@ -173,7 +179,7 @@ defmodule Plausible.Imported.CSVImporterTest do"""},%{- name: "imported_exit_pages.csv",+ name: "imported_exit_pages_20211230_20211231.csv",body: """"date","visitors","exits","exit_page""2021-12-30",6,6,"/14776416252794997127"@@ -198,7 +204,7 @@ defmodule Plausible.Imported.CSVImporterTest do"""},%{- name: "imported_locations.csv",+ name: "imported_locations_20211230_20211231.csv",body: """"date","country","region","city","visitors","visits","visit_duration","bounces""2021-12-30","AU","",0,1,1,43,0@@ -235,7 +241,7 @@ defmodule Plausible.Imported.CSVImporterTest do"""},%{- name: "imported_operating_systems.csv",+ name: "imported_operating_systems_20211230_20220101.csv",body: """"date","operating_system","visitors","visits","visit_duration","bounces""2021-12-30","Android",25,26,254,24@@ -255,7 +261,7 @@ defmodule Plausible.Imported.CSVImporterTest do"""},%{- name: "imported_pages.csv",+ name: "imported_pages_20211230_20220101.csv",body: """"date","visitors","pageviews","exits","time_on_page","hostname","page""2021-12-30",1,1,0,43,"lucky.numbers.com","/14776416252794997127"@@ -277,7 +283,7 @@ defmodule Plausible.Imported.CSVImporterTest do"""},%{- name: "imported_sources.csv",+ name: "imported_sources_20211230_20220106.csv",body: """"date","source","utm_medium","utm_campaign","utm_content","utm_term","visitors","visits","visit_duration","bounces""2021-12-30","","","","","",25,26,254,24@@ -307,7 +313,7 @@ defmodule Plausible.Imported.CSVImporterTest do"""},%{- name: "imported_visitors.csv",+ name: "imported_visitors_20111225_20111230.csv",body: """"date","visitors","pageviews","bounces","visits","visit_duration""2011-12-25",5,50,2,7,8640@@ -327,13 +333,12 @@ defmodule Plausible.Imported.CSVImporterTest do%{"filename" => name, "s3_url" => minio_url(minio, "imports", key)}end+ date_range = CSVImporter.date_range(uploads)+{:ok, job} =- CSVImporter.new_import(- site,- user,- # to satisfy the non null constraints on the table I'm providing "0" dates (according to ClickHouse)- start_date: ~D[1970-01-01],- end_date: ~D[1970-01-01],+ CSVImporter.new_import(site, user,+ start_date: date_range.first,+ end_date: date_range.last,uploads: uploads)@@ -341,7 +346,6 @@ defmodule Plausible.Imported.CSVImporterTest doassert :ok = Plausible.Workers.ImportAnalytics.perform(job)- # on successfull import the start and end dates are updatedassert %SiteImport{start_date: ~D[2011-12-25],end_date: ~D[2022-01-06],@@ -355,7 +359,7 @@ defmodule Plausible.Imported.CSVImporterTest dotest "fails on invalid CSV", %{site: site, user: user, s3: s3, container: minio} docsvs = [%{- name: "imported_browsers.csv",+ name: "imported_browsers_20211230_20211231.csv",body: """"date","browser","visitors","visits","visit_duration","bounces""2021-12-30","Amazon Silk",2,2,0,2@@ -368,7 +372,7 @@ defmodule Plausible.Imported.CSVImporterTest do"""},%{- name: "imported_devices.csv",+ name: "imported_devices_20211230_20211231.csv",body: """"date","device","visitors","visit_duration","bounces""2021-12-30","Desktop",28,ehhhh....@@ -383,12 +387,12 @@ defmodule Plausible.Imported.CSVImporterTest do%{"filename" => name, "s3_url" => minio_url(minio, "imports", key)}end+ date_range = CSVImporter.date_range(uploads)+{:ok, job} =- CSVImporter.new_import(- site,- user,- start_date: ~D[1970-01-01],- end_date: ~D[1970-01-01],+ CSVImporter.new_import(site, user,+ start_date: date_range.first,+ end_date: date_range.last,uploads: uploads)@@ -508,12 +512,12 @@ defmodule Plausible.Imported.CSVImporterTest doend)# run importer+ date_range = CSVImporter.date_range(uploads)+{:ok, job} =- CSVImporter.new_import(- site,- user,- start_date: ~D[1970-01-01],- end_date: ~D[1970-01-01],+ CSVImporter.new_import(site, user,+ start_date: date_range.first,+ end_date: date_range.last,uploads: uploads)@@ -533,7 +537,13 @@ defmodule Plausible.Imported.CSVImporterTest doenddefp minio_url(minio, bucket, key) do- port = minio |> MinioContainer.connection_opts() |> Keyword.fetch!(:port)- Path.join(["http://172.17.0.1:#{port}", bucket, key])+ arch = to_string(:erlang.system_info(:system_architecture))++ if String.contains?(arch, "darwin") do+ Path.join(["http://#{minio.ip_address}:9000", bucket, key])+ else+ port = minio |> MinioContainer.connection_opts() |> Keyword.fetch!(:port)+ Path.join(["http://172.17.0.1:#{port}", bucket, key])+ endendenddiff --git a/test/plausible/s3_test.exs b/test/plausible/s3_test.exsnew file mode 100644index 0000000000..7a14b403bb--- /dev/null+++ b/test/plausible/s3_test.exs@@ -0,0 +1,4 @@+defmodule Plausible.S3Test do+ use ExUnit.Case, async: true+ doctest Plausible.S3, import: true+end