A platform for community discussion. Free, open, simple.
MT: Import event locations and invitees in generic bulk import
Gerhard Schlageref3c7e400f2f
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rbindex 38c79a37b15..8076fbfcc44 100644--- a/script/bulk_import/base.rb+++ b/script/bulk_import/base.rb@@ -944,6 +944,7 @@ class BulkImport::Baseraw_inviteesnameurl+ locationcustom_fieldsremindersrecurrence@@ -951,6 +952,8 @@ class BulkImport::Baseminimal]+ POST_EVENT_INVITEE_COLUMNS = %i[post_id user_id status notified recurring created_at updated_at]+POST_EVENT_DATES_COLUMNS = %i[event_idstarts_at@@ -1271,6 +1274,10 @@ class BulkImport::Basecreate_records(rows, "discourse_post_event_events", POST_EVENT_COLUMNS, &block)end+ def create_event_invitees(rows, &block)+ create_records(rows, "discourse_post_event_invitee", POST_EVENT_INVITEE_COLUMNS, &block)+ end+def create_post_event_dates(rows, &block)create_records(rows, "discourse_calendar_post_event_dates", POST_EVENT_DATES_COLUMNS, &block)end@@ -2095,6 +2102,15 @@ class BulkImport::Basepost_eventend+ def process_discourse_post_event_invitee(invitee)+ # Imported sign-ups must not trigger invitation notifications.+ invitee[:notified] = true+ invitee[:recurring] = false+ invitee[:created_at] ||= NOW+ invitee[:updated_at] ||= NOW+ invitee+ end+def process_discourse_calendar_post_event_dates(post_event_date)post_event_date[:created_at] ||= NOWpost_event_date[:updated_at] ||= NOWdiff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rbindex 97718b759ff..9851bdc42d6 100644--- a/script/bulk_import/generic_bulk.rb+++ b/script/bulk_import/generic_bulk.rb@@ -25,6 +25,8 @@ class BulkImport::Generic < BulkImport::BaseDELTA_IMPORT = ENV["DELTA_IMPORT"].present?CONTENT_UPLOAD_REFERENCE_TYPES = %w[posts chat_messages]LAST_VIEWED_AT_PLACEHOLDER = "1970-01-01 00:00:00"+ # Same quote pairs as the BBCode parser accepts (QUOTATION_MARKS in discourse-markdown-it).+ BBCODE_QUOTATION_MARKS = ["\"\"", "''", "«»", "“”", "””", "‘’", "„“", "‚’", "‹›"]def initialize(db_path, uploads_db_path = nil)self.class.validate_modes!(merge_import: MERGE_IMPORT, delta_import: DELTA_IMPORT)@@ -681,6 +683,7 @@ class BulkImport::Generic < BulkImport::Baseimport_answersimport_gamification_scoresimport_post_events+ import_event_inviteesimport_badge_groupingsimport_badges@@ -2694,6 +2697,7 @@ class BulkImport::Generic < BulkImport::Base)endurl = event["url"]+ location = event_location(event)custom_fields = event["custom_fields"] ? JSON.parse(event["custom_fields"]) : niltext = +"[event"@@ -2702,9 +2706,10 @@ class BulkImport::Generic < BulkImport::Basetext << %{ end="#{format_event_time(ends_at, timezone)}"} if ends_attext << %{ timezone="#{timezone}"}text << %{ status="#{status}"} if status- text << %{ name="#{name}"} if name- text << %{ url="#{url}"} if url- custom_fields.each { |key, value| text << %{ #{key}="#{value}"} } if custom_fields+ text << bbcode_attribute("name", name) if name+ text << bbcode_attribute("url", url) if url+ text << bbcode_attribute("location", location) if location+ custom_fields&.each { |key, value| text << bbcode_attribute(key, value) }text << "]\n"text << "[/event]\n"text@@ -2719,6 +2724,20 @@ class BulkImport::Generic < BulkImport::Basetime.in_time_zone(timezone).strftime("%Y-%m-%d %H:%M")end+ # Line breaks would end the BBCode tag, and the plugin stores at most 1000 characters.+ def event_location(event)+ event["location"]&.squish.presence&.[](0..999)+ end++ # Picks a quote pair that doesn't occur in the value, so it needs no escaping.+ def bbcode_attribute(name, value)+ value = value.to_s+ marks = BBCODE_QUOTATION_MARKS.find { |pair| pair.chars.none? { |mark| value.include?(mark) } }+ return %{ #{name}="#{value.delete('"')}"} unless marks++ %{ #{name}=#{marks[0]}#{value}#{marks[1]}}+ end+def import_post_custom_fieldsputs "", "Importing post custom fields..."@@ -4364,6 +4383,7 @@ class BulkImport::Generic < BulkImport::Baseoriginal_ends_at: to_datetime(row["ends_at"]),name: row["name"],url: row["url"] ? row["url"][0..999] : nil,+ location: event_location(row),custom_fields: row["custom_fields"] || default_custom_fields,timezone:,raw_invitees:@@ -4426,6 +4446,66 @@ class BulkImport::Generic < BulkImport::Basepost_events.closeend+ def import_event_invitees+ unless defined?(DiscourseEvents::Events)+ puts "", "Skipping event invitees, because the events plugin is not installed."+ return+ end++ unless @source_db.get_first_value(+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'event_invitees'",+ )+ return+ end++ puts "", "Importing event invitees..."++ invitees = query(<<~SQL)+ SELECT event_invitees.*, events.post_id AS event_post_id, events.status AS event_status+ FROM event_invitees+ JOIN events ON events.id = event_invitees.event_id+ ORDER BY event_invitees.event_id, event_invitees.user_id+ SQL++ statuses = DiscourseEvents::Events::Invitee.statuses+ public_status = DiscourseEvents::Events::Event.statuses[:public]+ # The default scope hides suspended, silenced and staged users.+ existing_invitees = DiscourseEvents::Events::Invitee.unscoped.pluck(:post_id, :user_id).to_set++ create_event_invitees(invitees) do |row|+ post_id = post_id_from_imported_id(row["event_post_id"])+ user_id = user_id_from_imported_id(row["user_id"])+ next unless post_id && user_id++ # The plugin removes invitees from standalone events and from private events+ # without invited groups, which is all the importer creates besides public ones.+ if row["event_status"] != public_status+ log_import_issue(+ "event invitee skipped (event isn't public)",+ "user #{row["user_id"]} of event #{row["event_id"]}",+ )+ next+ end++ status = row["status"] || statuses[:going]+ if statuses.values.exclude?(status)+ log_import_issue(+ "event invitee skipped (unknown status)",+ "status #{status} of user #{row["user_id"]} in event #{row["event_id"]}",+ )+ next+ end++ next unless existing_invitees.add?([post_id, user_id])++ created_at = to_datetime(row["created_at"])++ { post_id:, user_id:, status:, created_at:, updated_at: created_at }+ end++ invitees.close+ end+def import_tag_usersputs "", "Importing tag users..."diff --git a/spec/script/bulk_import/generic_bulk_spec.rb b/spec/script/bulk_import/generic_bulk_spec.rbindex 341cb209ccf..6a3d224af7a 100644--- a/spec/script/bulk_import/generic_bulk_spec.rb+++ b/spec/script/bulk_import/generic_bulk_spec.rb@@ -656,12 +656,15 @@ if generic_import_dependencies_availableendend- describe "event timezones" do+ describe "events" dofab!(:vienna_post, :post)fab!(:untimed_post, :post)fab!(:invalid_post, :post)+ fab!(:attendee, :user)+ fab!(:other_attendee, :user)let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }+ let(:import_issue_log_path) { File.join(Dir.mktmpdir, "issues.log") }let(:importer) dodescribed_class.allocate.tap do |instance|instance.instance_variable_set(:@source_db, source_db)@@ -673,10 +676,8 @@ if generic_import_dependencies_available:@topic_id_by_post_id,[vienna_post, untimed_post, invalid_post].to_h { |post| [post.id, post.topic_id] },)- instance.instance_variable_set(- :@import_issue_log_path,- File.join(Dir.mktmpdir, "issues.log"),- )+ instance.instance_variable_set(:@users, { 10 => attendee.id, 11 => other_attendee.id })+ instance.instance_variable_set(:@import_issue_log_path, import_issue_log_path)instance.instance_variable_set(:@raw_connection,ActiveRecord::Base.connection.raw_connection,@@ -684,30 +685,34 @@ if generic_import_dependencies_availableinstance.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)endend+ let(:public_status) { DiscourseEvents::Events::Event.statuses[:public] }+ let(:standalone_status) { DiscourseEvents::Events::Event.statuses[:standalone] }before { skip "requires the events plugin" unless defined?(DiscourseEvents::Events) }after { source_db.close }- def create_events_table(with_timezone: true)+ def create_events_table(new_columns: true)source_db.execute(<<~SQL)CREATE TABLE events (id INTEGER, post_id INTEGER, status INTEGER, starts_at DATETIME, ends_at DATETIME,- name TEXT, url TEXT, custom_fields JSON_TEXT #{", timezone TEXT" if with_timezone}+ name TEXT, url TEXT, custom_fields JSON_TEXT+ #{", location TEXT, timezone TEXT" if new_columns})SQLend- def insert_event(id, timezone = nil)- columns = %w[id post_id status starts_at ends_at]- values = [id, id, 0, "2026-07-01T08:00:00Z", "2026-07-01T09:30:00Z"]- if timezone- columns << "timezone"- values << timezone- end+ def insert_event(id, **attributes)+ row = {+ id:,+ post_id: id,+ status: 0,+ starts_at: "2026-07-01T08:00:00Z",+ ends_at: "2026-07-01T09:30:00Z",+ }.merge(attributes)source_db.execute(- "INSERT INTO events (#{columns.join(", ")}) VALUES (#{(["?"] * values.size).join(", ")})",- values,+ "INSERT INTO events (#{row.keys.join(", ")}) VALUES (#{(["?"] * row.size).join(", ")})",+ row.values,)end@@ -715,10 +720,14 @@ if generic_import_dependencies_availablesource_db.get_first_row("SELECT * FROM events WHERE id = ?", id)end+ def import_issues+ File.exist?(import_issue_log_path) ? File.read(import_issue_log_path) : ""+ end+describe "#event_bbcode" doit "writes start and end as local times in the event's timezone" docreate_events_table- insert_event(1, "Europe/Vienna")+ insert_event(1, timezone: "Europe/Vienna")bbcode = importer.event_bbcode(event_row(1))@@ -733,7 +742,7 @@ if generic_import_dependencies_availableit "falls back to UTC for missing and invalid timezones" docreate_events_tableinsert_event(2)- insert_event(3, "+01:00")+ insert_event(3, timezone: "+01:00")[2, 3].each do |id|expect(importer.event_bbcode(event_row(id))).to include(@@ -742,20 +751,55 @@ if generic_import_dependencies_availableendend- it "uses UTC when the events table has no timezone column" do- create_events_table(with_timezone: false)+ it "writes the location as one line of at most 1000 characters" do+ create_events_table+ insert_event(1, location: "Town Hall\nRoom 1")+ insert_event(2, location: "x" * 1200)++ expect(importer.event_bbcode(event_row(1))).to include(%{ location="Town Hall Room 1"})+ expect(importer.event_bbcode(event_row(2))).to include(%{ location="#{"x" * 1000}"})+ end++ it "quotes attributes so the plugin reads them back unchanged" do+ SiteSetting.discourse_post_event_enabled = true+ SiteSetting.discourse_post_event_allowed_custom_fields = "room"+ create_events_table+ insert_event(+ 1,+ status: public_status,+ name: %{The "Old" Town Hall},+ location: %{The "Old" Town Hall, 'Room' «1»},+ custom_fields: { room: %{"A" 'B'} }.to_json,+ )++ bbcode = importer.event_bbcode(event_row(1))+ post = Fabricate.build(:post, raw: bbcode)++ expect(bbcode).to include(%{ location=“The "Old" Town Hall, 'Room' «1»”})+ expect(DiscourseEvents::Events::Parser.extract_events(post).first).to include(+ name: %{The "Old" Town Hall},+ location: %{The "Old" Town Hall, 'Room' «1»},+ room: CGI.escapeHTML(%{"A" 'B'}),+ )+ end++ it "uses UTC and no location when the events table has neither column" do+ create_events_table(new_columns: false)insert_event(1)- expect(importer.event_bbcode(event_row(1))).to include('timezone="UTC"')+ bbcode = importer.event_bbcode(event_row(1))++ expect(bbcode).to include('timezone="UTC"')+ expect(bbcode).not_to include("location")endenddescribe "#import_post_events" doit "imports valid timezones and logs invalid ones" docreate_events_table- insert_event(1, "Europe/Vienna")+ insert_event(1, timezone: "Europe/Vienna")insert_event(2)- insert_event(3, "+01:00")+ insert_event(3, timezone: "+01:00")importer.import_post_events@@ -768,18 +812,110 @@ if generic_import_dependencies_available[untimed_post.id, "UTC"],[invalid_post.id, "UTC"],)- expect(File.read(importer.instance_variable_get(:@import_issue_log_path))).to eq(- "[invalid event timezone, using UTC] +01:00 (event 3)\n",+ expect(import_issues).to eq("[invalid event timezone, using UTC] +01:00 (event 3)\n")+ end++ it "imports the location" do+ create_events_table+ insert_event(1, location: "Town Hall")+ insert_event(2)+ insert_event(3, location: "x" * 1200)++ importer.import_post_events++ expect(+ DiscourseEvents::Events::Event.where(+ id: [vienna_post.id, untimed_post.id, invalid_post.id],+ ).pluck(:id, :location),+ ).to contain_exactly(+ [vienna_post.id, "Town Hall"],+ [untimed_post.id, nil],+ [invalid_post.id, "x" * 1000],)end- it "imports events from intermediate databases without a timezone column" do- create_events_table(with_timezone: false)+ it "imports events from intermediate databases without the new columns" do+ create_events_table(new_columns: false)insert_event(1)importer.import_post_events- expect(DiscourseEvents::Events::Event.find(vienna_post.id).timezone).to eq("UTC")+ event = DiscourseEvents::Events::Event.find(vienna_post.id)+ expect(event.timezone).to eq("UTC")+ expect(event.location).to be_nil+ end+ end++ describe "#import_event_invitees" do+ def create_invitees_table+ source_db.execute(<<~SQL)+ CREATE TABLE event_invitees (+ event_id INTEGER, user_id INTEGER, status INTEGER, created_at DATETIME+ )+ SQL+ end++ def insert_invitee(event_id, user_id, status: nil, created_at: "2017-03-04T05:06:07Z")+ source_db.execute(+ "INSERT INTO event_invitees VALUES (?, ?, ?, ?)",+ [event_id, user_id, status, created_at],+ )+ end++ def imported_invitees+ DiscourseEvents::Events::Invitee+ .unscoped+ .order(:post_id, :user_id)+ .pluck(:post_id, :user_id, :status, :notified, :created_at)+ end++ it "imports sign-ups of public events as notified attendees" do+ going = DiscourseEvents::Events::Invitee.statuses[:going]+ interested = DiscourseEvents::Events::Invitee.statuses[:interested]+ create_events_table+ create_invitees_table+ # The intermediate event id differs from the post id on purpose.+ insert_event(101, post_id: 1, status: public_status)+ insert_invitee(101, 10)+ insert_invitee(101, 11, status: interested)++ importer.import_event_invitees+ importer.import_event_invitees++ signed_up_at = Time.utc(2017, 3, 4, 5, 6, 7)+ expect(imported_invitees).to eq(+ [+ [vienna_post.id, attendee.id, going, true, signed_up_at],+ [vienna_post.id, other_attendee.id, interested, true, signed_up_at],+ ],+ )+ end++ it "skips invitees that can't be imported" do+ create_events_table+ create_invitees_table+ insert_event(1, status: public_status)+ insert_event(2, status: standalone_status)+ insert_event(99, status: public_status)+ insert_invitee(1, 99)+ insert_invitee(99, 10)+ insert_invitee(2, 10)+ insert_invitee(1, 11, status: 42)++ importer.import_event_invitees++ expect(imported_invitees).to be_empty+ expect(import_issues.lines).to contain_exactly(+ "[event invitee skipped (event isn't public)] user 10 of event 2\n",+ "[event invitee skipped (unknown status)] status 42 of user 11 in event 1\n",+ )+ end++ it "skips intermediate databases without an event_invitees table" do+ create_events_table(new_columns: false)++ expect { importer.import_event_invitees }.not_to raise_error+ expect(imported_invitees).to be_emptyendendend