A platform for community discussion. Free, open, simple.
# frozen_string_literal: true
module DiscourseAi
module Translation
class ContentSplitter
DEFAULT_CHUNK_SIZE = 8192
BBCODE_PATTERNS = [
%r{\[table.*?\].*?\[/table\]}m,
%r{\[quote.*?\].*?\[/quote\]}m,
%r{\[details.*?\].*?\[/details\]}m,
%r{\<details.*?\>.*?\</details\>}m,
%r{\[spoiler.*?\].*?\[/spoiler\]}m,
%r{\[code.*?\].*?\[/code\]}m,
/```.*?```/m,
].freeze
TEXT_BOUNDARIES = [
/\n\s*\n\s*|\r\n\s*\r\n\s*/, # double newlines with optional spaces
/[.!?]\s+/, # sentence endings
/[,;]\s+/, # clause endings
/\n|\r\n/, # single newlines
/\s+/, # any whitespace
].freeze
def self.split(content:, chunk_size: DEFAULT_CHUNK_SIZE, &measure)
return [] if content.nil?
return [""] if content.empty?
measure ||= :length.to_proc
chunks = []
offset = 0
while offset < content.length
chunk = content[offset..]
loop do
length = fitting_length(chunk, chunk_size, &measure)
break if length == chunk.length
chunk = extract_mixed_chunk(chunk, size: length)
break if measure.call(chunk) <= chunk_size
end
chunks << chunk
offset += chunk.length
end
chunks
end
private
def self.fitting_length(text, size, &measure)
return text.length if measure.call(text) <= size
length = (1..text.length).bsearch { |n| measure.call(text[0...n]) > size } - 1
raise ArgumentError, "Chunk size is too small to fit content" if length == 0
length
end
def self.extract_mixed_chunk(text, size:)
split_point =
find_nearest_bbcode_end_index(text, size) || find_nearest_html_end_index(text, size) ||
find_text_boundary(text, size) || size
split_point += 1 while split_point < size && text[split_point].match?(/\s/)
text[0...split_point]
end
def self.find_nearest_html_end_index(text, target_pos)
return if !text.include?("<")
handler = HtmlBoundaries.new(text, target_pos)
Nokogiri::HTML4::SAX::Parser
.new(handler, Encoding::UTF_8)
.parse_memory(text) { |context| handler.context = context }
handler.element_end || handler.text_end
end
def self.find_nearest_bbcode_end_index(text, target_pos)
BBCODE_PATTERNS
.flat_map do |pattern|
text.to_enum(:scan, pattern).flat_map { Regexp.last_match.offset(0) }
end
.select { |position| position.positive? && position <= target_pos }
.max
end
def self.find_text_boundary(text, target_pos)
head = text[0...target_pos]
TEXT_BOUNDARIES.each { |pattern| return Regexp.last_match.end(0) if head.rindex(pattern) }
nil
end
class HtmlBoundaries < Nokogiri::XML::SAX::Document
attr_accessor :context
attr_reader :element_end, :text_end
def initialize(text, target_pos)
@target_pos = target_pos
@line_offsets = [0]
text.each_line { |line| @line_offsets << @line_offsets.last + line.length }
end
def end_element(_name)
@element_end = source_position || @element_end
end
alias_method :comment, :end_element
def characters(_text)
@text_end = source_position || @text_end
end
alias_method :cdata_block, :characters
private
def source_position
position = @line_offsets[context.line - 1] + context.column - 1
position if position.positive? && position <= @target_pos
end
end
end
end
end