Skip to content

Self-hosted project management for devs and their clients: tickets, time tracking, docs and meetings, with an AI assistant that cites its sources. AGPL-3.0.

perf(meetings): count, replace and find a mark in bytes, in one pass per line

Guido Boyke4461e9612da4
diff --git a/app/services/meetings/term_replacement.rb b/app/services/meetings/term_replacement.rb
index 6046f0b..db16349 100644
--- a/app/services/meetings/term_replacement.rb
+++ b/app/services/meetings/term_replacement.rb
@@ -31,8 +31,12 @@ module Meetings
WORD_GAP = '[[:blank:]]+'
# One whole-word match: its line, where it starts in the line and how long
- # it is (characters), and whether it starts inside the speaker label.
- Match = Data.define(:line_index, :char_offset, :length, :in_label)
+ # it is (characters, what term_corrections stores), whether it starts inside
+ # the speaker label, and the same place in bytes. The bytes are for the
+ # work: Ruby finds the n-th character of a text with umlauts by counting
+ # from its start, so a long line with many matches would be counted over
+ # and over (a transcript can be one line of a megabyte).
+ Match = Data.define(:line_index, :char_offset, :length, :in_label, :byte_offset, :byte_length)
Replaced = Data.define(:text, :count)
# skip_frontmatter: false for a text that has none by construction (the
@@ -68,11 +72,7 @@ module Meetings
by_line = targets.group_by(&:line_index)
result = lines.each_with_index.map do |line, index|
hits = by_line[index]
- next line unless hits
-
- hits.sort_by(&:char_offset).reverse.each_with_object(line.dup) do |hit, changed|
- changed[hit.char_offset, hit.length] = with
- end
+ hits ? replaced_line(line, hits, with) : line
end
Replaced.new(text: result.join("\n"), count: targets.size)
end
@@ -109,14 +109,35 @@ module Meetings
lines.each_with_index.flat_map do |line, index|
next [] if index < first_content_line
- label_end = labels[index]&.ends_at
- line.to_enum(:scan, regex).map do
- found = Regexp.last_match
- start = found.begin(0)
- Match.new(line_index: index, char_offset: start, length: found[0].length,
- in_label: !label_end.nil? && start < label_end)
- end
+ matches_in(line, index, regex)
+ end
+ end
+
+ # The matches of one line. The character offset is counted on from the
+ # match before, never from the start of the line again.
+ def matches_in(line, index, regex)
+ label_end = labels[index]&.ends_at
+ chars = 0
+ bytes = 0
+ line.to_enum(:scan, regex).map do
+ found = Regexp.last_match
+ byte_start = found.byteoffset(0).first
+ chars += line.byteslice(bytes, byte_start - bytes).length
+ bytes = byte_start
+ Match.new(line_index: index, char_offset: chars, length: found[0].length,
+ in_label: !label_end.nil? && chars < label_end, byte_offset: byte_start, byte_length: found[0].bytesize)
+ end
+ end
+
+ # One line with `with` in place of its hits, put together from the pieces
+ # between them (in bytes, one pass).
+ def replaced_line(line, hits, with)
+ cursor = 0
+ result = hits.sort_by(&:byte_offset).each_with_object(+'') do |hit, changed|
+ changed << line.byteslice(cursor, hit.byte_offset - cursor) << with
+ cursor = hit.byte_offset + hit.byte_length
end
+ result << line.byteslice(cursor, line.bytesize - cursor)
end
def pattern
diff --git a/app/services/meetings/transcript_mark.rb b/app/services/meetings/transcript_mark.rb
index e0fbc68..66e7dfb 100644
--- a/app/services/meetings/transcript_mark.rb
+++ b/app/services/meetings/transcript_mark.rb
@@ -77,7 +77,7 @@ module Meetings
def map(start)
shown = TermReplacement.new(text: visible, heard: @mark, skip_frontmatter: false).scan
line_index, column = line_and_column(start)
- ordinal = shown.index { |match| match.line_index == line_index && match.char_offset == column }
+ ordinal = shown.index { |match| match.line_index == line_index && match.byte_offset == column }
return refuse(:not_a_word) unless ordinal
stored = TermReplacement.new(text: @transcript, heard: @mark, participants: @participants).scan
@@ -96,30 +96,42 @@ module Meetings
@visible ||= Nokogiri::HTML5.fragment(@displayed_html).text
end
+ # Where the mark starts in the text the page shows, in bytes, or nil.
+ # Bytes, because Ruby finds the n-th character of a text with umlauts by
+ # counting from its start: a search over many candidates would count a
+ # long transcript again for each of them.
def locate
- return @offset if @offset && fits?(@offset)
+ if @offset && @offset >= 0
+ start = visible[0, @offset].bytesize
+ return start if fits?(start)
+ end
starts = []
from = 0
- while (found = visible.index(@mark, from))
+ while (found = visible.byteindex(@mark, from))
starts << found if fits?(found)
- from = found + 1
+ break if starts.size > 1
+
+ from = found + @mark[0].bytesize
end
starts.first if starts.one?
end
+ # The mark and its context at this place (in bytes).
def fits?(start)
- finish = start + @mark.length
- return false if start < @before.length || finish + @after.length > visible.length
+ finish = start + @mark.bytesize
+ return false if start < @before.bytesize || finish + @after.bytesize > visible.bytesize
- visible[start, @mark.length] == @mark &&
- visible[start - @before.length, @before.length] == @before &&
- visible[finish, @after.length] == @after
+ visible.byteslice(start, @mark.bytesize) == @mark &&
+ visible.byteslice(start - @before.bytesize, @before.bytesize) == @before &&
+ visible.byteslice(finish, @after.bytesize) == @after
end
+ # The line of a place and where it starts in that line, in bytes, the way
+ # TermReplacement::Match#byte_offset counts.
def line_and_column(start)
- prefix = visible[0, start]
- line_start = prefix.rindex("\n")
+ prefix = visible.byteslice(0, start)
+ line_start = prefix.byterindex("\n")
[prefix.count("\n"), line_start ? start - line_start - 1 : start]
end
diff --git a/spec/services/meetings/term_replacement_spec.rb b/spec/services/meetings/term_replacement_spec.rb
index dd5a33a..155922b 100644
--- a/spec/services/meetings/term_replacement_spec.rb
+++ b/spec/services/meetings/term_replacement_spec.rb
@@ -174,6 +174,23 @@ RSpec.describe Meetings::TermReplacement do
expect(replaced("Lalabeis\r\nLalabeis\n", 'Lalabeis', with: 'Lalabase')).to eq("Lalabase\r\nLalabase\n")
end
+ # A transcript can be one line of half a megabyte. Ruby finds the n-th
+ # character of a text with umlauts by counting from its start; counting
+ # each match from the start of the line again took about nine seconds for
+ # this line (review of the first version), counting on takes milliseconds.
+ it 'counts and replaces many matches in one long line in one pass' do
+ line = (['Grüße an Lalabeis'] * 20_000).join(' und ')
+ subject = replacement(line, 'Lalabeis')
+ result = nil
+
+ elapsed = Benchmark.realtime { result = subject.replace(with: 'Lalabase') }
+
+ expect(elapsed).to be < 2
+ expect(result.count).to eq(20_000)
+ expect(subject.occurrences.last.char_offset).to eq(line.rindex('Lalabeis'))
+ expect(result.text).to eq(line.gsub('Lalabeis', 'Lalabase'))
+ end
+
it 'counts in characters, not bytes' do
occurrence = replacement('Grüße, Lalabeis', 'Lalabeis').occurrences.first
diff --git a/spec/services/meetings/transcript_mark_spec.rb b/spec/services/meetings/transcript_mark_spec.rb
index 6628170..1631280 100644
--- a/spec/services/meetings/transcript_mark_spec.rb
+++ b/spec/services/meetings/transcript_mark_spec.rb
@@ -122,6 +122,22 @@ RSpec.describe Meetings::TranscriptMark do
expect(resolve(text, **browser_mark(text, 'Lalabeis')).place.to_h.values_at(:line_index, :char_offset)).to eq([0, 18])
end
+ # One line of half a megabyte with the form 20,000 times, and an offset that
+ # does not fit: the context search runs over every candidate. In characters
+ # that took about eight seconds; in bytes it takes milliseconds.
+ it 'finds a mark in one long line with many candidates quickly, also by its context alone' do
+ line = "Anna: #{(['Grüße an Lalabeis'] * 20_000).join(' und ')}"
+ shown = visible(line)
+ start = shown.rindex('Lalabeis')
+ mark = { text: 'Lalabeis', before: shown[start - 40, 40], after: shown[start + 8, 40].to_s, offset: 3 }
+ result = nil
+
+ elapsed = Benchmark.realtime { result = resolve(line, **mark) }
+
+ expect(elapsed).to be < 2
+ expect(result.place.to_h).to eq(line_index: 0, char_offset: line.rindex('Lalabeis'), length: 8)
+ end
+
it 'refuses a blank mark' do
expect(resolve(transcript, text: ' ', before: '', after: '', offset: 0).error).to eq(:not_found)
end