Skip to content

A platform for community discussion. Free, open, simple.

4.7 KB
Raw Blame
# frozen_string_literal: true

class RagDocumentSource < ActiveRecord::Base
  MIN_REFRESH_INTERVAL_HOURS = 1
  MAX_REFRESH_INTERVAL_HOURS = 8_760

  belongs_to :target, polymorphic: true
  belongs_to :upload, optional: true
  belongs_to :pending_upload, class_name: "Upload", optional: true

  validates :target_type, inclusion: { in: %w[AiAgent] }
  validates :url, presence: true, length: { maximum: 2000 }
  validates :url_digest, presence: true, length: { is: 64 }
  validates :url_digest, uniqueness: { scope: %i[target_type target_id] }
  validates :refresh_interval_hours,
            numericality: {
              only_integer: true,
              greater_than_or_equal_to: MIN_REFRESH_INTERVAL_HOURS,
              less_than_or_equal_to: MAX_REFRESH_INTERVAL_HOURS,
            }
  validate :validate_url

  before_validation :normalize_url
  before_validation :set_url_digest
  before_save :expire_changed_source
  before_destroy :remove_rag_content
  after_commit :enqueue_refresh, on: %i[create update]

  scope :due, -> { where(next_refresh_at: nil).or(where("next_refresh_at <= ?", Time.zone.now)) }

  def self.promote_pending_upload(target:, upload:)
    find_by(target:, pending_upload_id: upload.id)&.promote_pending_upload!(upload)
  end

  def self.mark_indexing_failed(target:, upload:, error:)
    find_by(target:, pending_upload_id: upload.id)&.mark_indexing_failed!(error)
  end

  def due?
    next_refresh_at.nil? || next_refresh_at <= Time.zone.now
  end

  def indexing_status
    return "failed" if last_error.present?
    return "indexing" if pending_upload_id.present?
    return "indexed" if upload_id.present?

    "pending"
  end

  def promote_pending_upload!(upload)
    with_lock do
      return if pending_upload_id != upload.id

      old_upload_id = upload_id
      UploadReference.ensure_exist!(upload_ids: [upload.id], target:)
      update_columns(
        upload_id: upload.id,
        pending_upload_id: nil,
        last_error_at: nil,
        last_error: nil,
        updated_at: Time.zone.now,
      )
      remove_rag_content_for(old_upload_id) if old_upload_id.present? && old_upload_id != upload.id
    end
  end

  def mark_indexing_failed!(error)
    update_columns(
      next_refresh_at: 1.hour.from_now,
      last_error_at: Time.zone.now,
      last_error: error.message.to_s.truncate(4000),
      updated_at: Time.zone.now,
    )
  end

  private

  def normalize_url
    uri = URI.parse(url.to_s.strip)
    uri.fragment = nil
    self.url = uri.normalize.to_s
  rescue URI::InvalidURIError
    self.url = url.to_s.strip
  end

  def set_url_digest
    self.url_digest = Digest::SHA256.hexdigest(url.to_s)
  end

  def expire_changed_source
    if url_changed?
      self.etag = nil
      self.last_modified = nil
      self.next_refresh_at = nil
    elsif refresh_interval_hours_changed?
      self.next_refresh_at = nil
    end
  end

  def validate_url
    uri = URI.parse(url.to_s)
    valid = uri.scheme.in?(%w[http https]) && uri.host.present? && uri.userinfo.blank?
    errors.add(:url, :invalid) if !valid
  rescue URI::InvalidURIError
    errors.add(:url, :invalid)
  end

  def enqueue_refresh
    if !previous_changes.key?("id") && !previous_changes.key?("url") &&
         !previous_changes.key?("refresh_interval_hours")
      return
    end

    Jobs.enqueue(:refresh_rag_document_source, rag_document_source_id: id)
  end

  def remove_rag_content
    [upload_id, pending_upload_id].compact.uniq.each { |id| remove_rag_content_for(id) }
  end

  def remove_rag_content_for(id)
    RagDocumentFragment.where(target:, upload_id: id).destroy_all
    UploadReference.where(target:, upload_id: id).destroy_all
  end
end

# == Schema Information
#
# Table name: rag_document_sources
#
#  id                     :bigint           not null, primary key
#  etag                   :string
#  last_error             :text
#  last_error_at          :datetime
#  last_fetched_at        :datetime
#  last_modified          :string
#  managed                :boolean          default(FALSE), not null
#  next_refresh_at        :datetime
#  refresh_interval_hours :integer          default(24), not null
#  target_type            :string(800)      not null
#  url                    :string(2000)     not null
#  url_digest             :string(64)       not null
#  created_at             :datetime         not null
#  updated_at             :datetime         not null
#  pending_upload_id      :integer
#  target_id              :bigint           not null
#  upload_id              :integer
#
# Indexes
#
#  idx_rag_document_sources_target_url                      (target_type,target_id,url_digest) UNIQUE
#  index_rag_document_sources_on_next_refresh_at            (next_refresh_at)
#  index_rag_document_sources_on_target_type_and_target_id  (target_type,target_id)
#