Skip to content

Self-hosted project management for devs and their clients: tickets, time tracking, docs and meetings, with an AI assistant that cites its sources. AGPL-3.0.

Race the checked addresses and keep DNS inside the open timeout

Guido Boyke1094f6491c93
diff --git a/.github/workflows/guards.yml b/.github/workflows/guards.yml
index 6e0b7f8..ba8ab1a 100644
--- a/.github/workflows/guards.yml
+++ b/.github/workflows/guards.yml
@@ -149,6 +149,7 @@ jobs:
spec/services/lalabase/indexer/git_cloner_spec.rb
spec/services/lalabase/public_address_spec.rb
spec/services/lalabase/llm/public_endpoint_adapter_spec.rb
+ spec/services/lalabase/llm/address_race_spec.rb
spec/components/neo_notification_card_component_spec.rb
spec/components/neo_notification_component_spec.rb
spec/components/neo_time_share_bar_component_spec.rb
diff --git a/Gemfile b/Gemfile
index 6981459..30a143f 100644
--- a/Gemfile
+++ b/Gemfile
@@ -104,6 +104,10 @@ gem 'mcp', '~> 0.22' # Official Model Context Protocol SDK (MCP serve
gem 'parser', '~> 3.3' # Ruby AST parser (M6 — Layer-1 code symbol extraction)
gem 'pry-rails' # Enhanced Rails console with Pry
gem 'ruby-openai', '~> 7.1' # OpenAI API wrapper
+# Lalabase::Llm::PublicEndpointAdapter takes over Net::HTTP#timeouted_connect, which
+# 0.6.0 does not have; the adapter refuses to load without it. 0.9.1 is the version
+# Ruby 4.0.7 ships. A newer line is fine once the adapter spec is green on it.
+gem 'net-http', '~> 0.9.1'
gem 'ruby_tree_sitter', '~> 2.1' # TypeScript/JS AST parser (M10 — Layer-1). Precompiled multi-platform runtime; grammar (.so/.dylib) built from vendored C sources.
# Text extraction
diff --git a/Gemfile.lock b/Gemfile.lock
index 68dac6b..53fa31b 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -310,8 +310,8 @@ GEM
neighbor (0.5.2)
activerecord (>= 7)
nenv (0.3.0)
- net-http (0.6.0)
- uri
+ net-http (0.9.1)
+ uri (>= 0.11.1)
net-imap (0.6.6)
date
net-protocol
@@ -702,6 +702,7 @@ DEPENDENCIES
minitest (~> 5.26)
mission_control-jobs
neighbor
+ net-http (~> 0.9.1)
ostruct
pagy (~> 9.0)
parser (~> 3.3)
diff --git a/app/services/actors/save_ai_configuration.rb b/app/services/actors/save_ai_configuration.rb
index ba18d80..8d06aca 100644
--- a/app/services/actors/save_ai_configuration.rb
+++ b/app/services/actors/save_ai_configuration.rb
@@ -67,6 +67,12 @@ module Actors
# column outside the audit is a change nobody can reconstruct afterwards.
WRITABLE = AiConfigurationAudit::COLUMNS
+ # Seconds the endpoint's DNS lookup may take. It runs while #perform holds the
+ # organisation's row lock, so a stalled resolver must not hold that lock for the
+ # system resolver's own limits. A name that does not answer in time is refused
+ # like one that does not resolve.
+ API_BASE_LOOKUP_TIMEOUT = 3
+
def initialize(organisation:, params:, current_user:)
super()
@organisation = organisation
@@ -201,7 +207,7 @@ module Actors
end
def public_host?(hostname)
- Lalabase::PublicAddress.public?(hostname)
+ Lalabase::PublicAddress.public?(hostname, timeout: API_BASE_LOOKUP_TIMEOUT)
end
# The other half of the exfiltration guard. Keeping a stored key (E2-A) is safe only
diff --git a/app/services/lalabase/llm/address_race.rb b/app/services/lalabase/llm/address_race.rb
new file mode 100644
index 0000000..dc26470
--- /dev/null
+++ b/app/services/lalabase/llm/address_race.rb
@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'socket'
+
+# Opens a TCP connection to the first of several addresses that answers, the way Ruby
+# connects to a name with several addresses (Happy Eyeballs, RFC 8305): the next
+# address starts 250 ms after the previous one, or at once when it fails, and the
+# earlier attempts keep running. The first connection wins and the others are closed.
+#
+# Lalabase::Llm::PublicEndpointAdapter needs it because it connects to CHECKED
+# addresses, not to the name, and so cannot leave the choice to Ruby. Without the race
+# an endpoint whose IPv6 address drops packets would spend the whole open timeout on
+# that address and never try IPv4.
+#
+# It knows nothing about which addresses are allowed; the caller passes only checked
+# ones.
+module Lalabase
+ module Llm
+ class AddressRace
+ # RFC 8305 §5, and what Ruby uses for names.
+ CONNECTION_ATTEMPT_DELAY = 0.25
+
+ def self.call(**kwargs)
+ new(**kwargs).call
+ end
+
+ # @param addresses [Array<String>] IP literals, tried in the given order
+ # @param port [Integer]
+ # @param deadline [Float, nil] monotonic clock value; nil waits without a limit
+ def initialize(addresses:, port:, deadline:)
+ @waiting = addresses.map { |address| Addrinfo.tcp(address, port) }
+ @deadline = deadline
+ @pending = []
+ @last_error = nil
+ @next_start = monotonic_now
+ end
+
+ # @return [TCPSocket] connected
+ # @raise [Net::OpenTimeout] when the deadline passes first
+ # @raise [SystemCallError] the last failure, when every address failed
+ def call
+ loop do
+ raise Net::OpenTimeout, 'execution expired' if @deadline && monotonic_now >= @deadline
+
+ start_next_attempt if @waiting.any? && monotonic_now >= @next_start
+ raise @last_error if @pending.empty? && @waiting.empty?
+
+ winner = wait_for_an_attempt
+ return as_tcp_socket(winner) if winner
+ end
+ ensure
+ @pending.each(&:close)
+ end
+
+ private
+
+ def start_next_attempt
+ @pending << open_attempt(@waiting.shift)
+ @next_start = monotonic_now + CONNECTION_ATTEMPT_DELAY
+ rescue SystemCallError => e
+ # Failed at once (refused, no route, no IPv6 on this host): the next address
+ # starts in the next round, without waiting for the delay.
+ @last_error = e
+ end
+
+ # A socket whose connect is under way. One that connected at once is writable
+ # straight away, and the wait picks it up like any other.
+ def open_attempt(addrinfo)
+ socket = Socket.new(addrinfo.afamily, Socket::SOCK_STREAM, 0)
+ socket.connect_nonblock(addrinfo, exception: false)
+ socket
+ rescue SystemCallError
+ socket&.close
+ raise
+ end
+
+ # Waits until an attempt finishes, the next one is due, or the deadline passes.
+ # Returns the connected socket, or nil to go round again.
+ def wait_for_an_attempt
+ _, writable, = IO.select(nil, @pending, nil, wait_seconds)
+ Array(writable).each do |socket|
+ @pending.delete(socket)
+ error = connect_error(socket)
+ return socket if error.nil?
+
+ socket.close
+ @last_error = error
+ @next_start = monotonic_now
+ end
+ nil
+ end
+
+ def wait_seconds
+ until_next = @next_start if @waiting.any?
+ limit = [until_next, @deadline].compact.min
+ limit && [limit - monotonic_now, 0].max
+ end
+
+ def connect_error(socket)
+ code = socket.getsockopt(Socket::SOL_SOCKET, Socket::SO_ERROR).int
+ code.zero? ? nil : SystemCallError.new('connect(2)', code)
+ end
+
+ # Net::HTTP expects the socket TCPSocket.open returns (#peeraddr, among others).
+ # The descriptor moves to the TCPSocket; the Socket must not close it when it is
+ # collected.
+ def as_tcp_socket(socket)
+ socket.autoclose = false
+ TCPSocket.for_fd(socket.fileno)
+ end
+
+ def monotonic_now
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ end
+ end
+ end
+end
diff --git a/app/services/lalabase/llm/public_endpoint_adapter.rb b/app/services/lalabase/llm/public_endpoint_adapter.rb
index bac2971..51c1829 100644
--- a/app/services/lalabase/llm/public_endpoint_adapter.rb
+++ b/app/services/lalabase/llm/public_endpoint_adapter.rb
@@ -8,13 +8,13 @@ require 'faraday/net_http'
# can point at the app server when the decrypted key is sent (DNS rebinding).
#
# So for every connection it resolves the host through Lalabase::PublicAddress, refuses
-# unless every address is public, and connects only to those addresses, in the order
-# the resolver gave them (PinnedAddresses). Net::HTTP keeps the NAME for SNI and the
-# certificate check, so TLS still verifies the host the customer named.
+# unless every address is public, and connects only to those addresses
+# (CheckedAddresses). Net::HTTP keeps the NAME for SNI and the certificate check, so
+# TLS still verifies the host the customer named.
#
-# Through a proxy the proxy resolves the name and the pinned address means nothing, so
-# a proxied request is refused. An operator who needs one configures the endpoint on a
-# system credential, which does not use this adapter.
+# Through a proxy the proxy resolves the name and the checked addresses mean nothing,
+# so a proxied request is refused. An operator who needs one configures the endpoint
+# on a system credential, which does not use this adapter.
#
# Entry point: Faraday calls it (connection.adapter(PublicEndpointAdapter))
module Lalabase
@@ -24,52 +24,34 @@ module Lalabase
# retry_on in the jobs, ConnectionCheck), with no port or reason in its text.
class EndpointNotPublicError < Faraday::ConnectionFailed; end
- # Mixed into the one Net::HTTP object of a connection. Given the name, Net::HTTP
- # let TCPSocket try every address of it; given one pinned address, a dual-stack
- # endpoint with broken IPv6 would fail although its IPv4 address answers. So the
- # TCP connect walks the checked addresses in order and gives up with the last
- # error.
+ # The seam below is private to Net::HTTP. Without it Net::HTTP would resolve and
+ # connect to the name itself, unchecked, so the adapter refuses to load instead.
+ SEAM_MISSING = 'PublicEndpointAdapter needs Net::HTTP#timeouted_connect (net-http >= 0.9, see Gemfile)'
+ raise LoadError, SEAM_MISSING unless Net::HTTP.private_method_defined?(:timeouted_connect)
+
+ # Mixed into the one Net::HTTP object of a connection. It takes over
+ # Net::HTTP#timeouted_connect, which opens the TCP socket and nothing else, and
+ # keeps what Net::HTTP did with a name:
#
- # Only around Net::HTTP#connect, which opens the socket and does the TLS
- # handshake and nothing else: the request is written after it returns, so
- # nothing has been sent when the next address is tried, and a request is never
- # sent twice. On failure #connect closes the socket it opened. #connect is
- # private, but it is the seam both the net-http gem in the bundle (0.6.0) and the
- # one in Ruby 4.0 share; the helper below it differs between the two.
+ # * ONE open timeout for everything up to the open socket, the DNS lookup
+ # included. The lookup runs here, inside it, and not before Net::HTTP starts.
+ # * a race across the addresses (Lalabase::Llm::AddressRace), so an address that
+ # drops packets does not use up the timeout before the next one is tried.
+ # * TLS afterwards, with its own open timeout, exactly as before.
#
- # All attempts share ONE open timeout, as all addresses of a name did before.
- # Restarting it per address would turn the client's budget (120 seconds from
- # ruby-openai) into one per address, and a worker would hang N times as long on
- # an endpoint that drops packets. Each attempt gets what is left; the configured
- # value is restored afterwards for any later reconnect. Since #connect also does
- # the TLS handshake, the handshake now draws on the same budget: stricter than
- # before, never longer.
- module PinnedAddresses
- CONNECT_ERRORS = [SystemCallError, IO::TimeoutError, Net::OpenTimeout].freeze
-
- attr_accessor :pinned_addresses
+ # Nothing has been written when this returns or fails, so a request is never sent
+ # twice.
+ module CheckedAddresses
+ attr_accessor :checked_host
private
- def connect
- budget = open_timeout
- deadline = budget && (monotonic_now + budget)
- pinned_addresses.each_with_index do |address, index|
- self.ipaddr = address
- self.open_timeout = remaining_until(deadline) if deadline
- return super
- rescue *CONNECT_ERRORS
- raise if index == pinned_addresses.size - 1
- end
- ensure
- self.open_timeout = budget
- end
-
- def remaining_until(deadline)
- remaining = deadline - monotonic_now
- raise Net::OpenTimeout, 'the open timeout is spent' unless remaining.positive?
-
- remaining
+ def timeouted_connect(_conn_addr, conn_port)
+ deadline = open_timeout && (monotonic_now + open_timeout)
+ addresses = PublicAddress.resolve!(checked_host, timeout: open_timeout)
+ AddressRace.call(addresses: addresses, port: conn_port, deadline: deadline)
+ rescue PublicAddress::NotPublicError
+ raise EndpointNotPublicError, 'the endpoint is not a public address'
end
def monotonic_now
@@ -80,16 +62,10 @@ module Lalabase
def net_http_connection(env)
raise EndpointNotPublicError, 'the endpoint is not a public address' if env[:request][:proxy]
- addresses = PublicAddress.resolve!(env[:url].hostname)
super.tap do |http|
- http.extend(PinnedAddresses)
- http.pinned_addresses = addresses
- # What Net::HTTP reports as the address it connects to (#ipaddr, and the
- # "Failed to open TCP connection to" message) is the first checked one.
- http.ipaddr = addresses.first
+ http.extend(CheckedAddresses)
+ http.checked_host = env[:url].hostname
end
- rescue PublicAddress::NotPublicError
- raise EndpointNotPublicError, 'the endpoint is not a public address'
end
end
end
diff --git a/app/services/lalabase/public_address.rb b/app/services/lalabase/public_address.rb
index ff701df..8d6d34c 100644
--- a/app/services/lalabase/public_address.rb
+++ b/app/services/lalabase/public_address.rb
@@ -32,8 +32,9 @@ module Lalabase
# service networks.
BLOCKED_RANGES = ['100.64.0.0/10'].map { |cidr| IPAddr.new(cidr) }.freeze
- DEFAULT_RESOLVER = lambda do |hostname|
- Addrinfo.getaddrinfo(hostname, nil, nil, :STREAM).map(&:ip_address).uniq
+ # timeout nil means the system resolver's own limits.
+ DEFAULT_RESOLVER = lambda do |hostname, timeout: nil|
+ Addrinfo.getaddrinfo(hostname, nil, nil, :STREAM, timeout: timeout).map(&:ip_address).uniq
end
class << self
@@ -45,22 +46,24 @@ module Lalabase
@resolver || DEFAULT_RESOLVER
end
+ # @param timeout [Numeric, nil] seconds the lookup may take. A lookup that runs
+ # out of time is a name that does not resolve.
# @return [Array<String>] every address the host resolves to, all of them public
# @raise [NotPublicError] when the host is blank, does not resolve, or any one of
# its addresses is not public
- def resolve!(hostname)
+ def resolve!(hostname, timeout: nil)
host = hostname.to_s.delete_prefix('[').delete_suffix(']')
raise NotPublicError, 'blank host' if host.empty?
- addresses = addresses_of(host)
+ addresses = addresses_of(host, timeout)
raise NotPublicError, "#{host} does not resolve" if addresses.empty?
raise NotPublicError, "#{host} is not a public address" unless addresses.all? { |a| public_ip?(a) }
addresses
end
- def public?(hostname)
- resolve!(hostname)
+ def public?(hostname, timeout: nil)
+ resolve!(hostname, timeout: timeout)
true
rescue NotPublicError
false
@@ -70,15 +73,15 @@ module Lalabase
# A literal is its own answer; everything else, including the numeric notations
# IPAddr does not speak, goes to the resolver.
- def addresses_of(host)
+ def addresses_of(host, timeout)
[IPAddr.new(host).to_s]
rescue IPAddr::InvalidAddressError
- resolved_addresses(host)
+ resolved_addresses(host, timeout)
end
- def resolved_addresses(host)
- Array(resolver.call(host))
- rescue SocketError, SystemCallError
+ def resolved_addresses(host, timeout)
+ Array(resolver.call(host, timeout: timeout))
+ rescue SocketError, SystemCallError, IO::TimeoutError
[]
end
diff --git a/spec/services/actors/save_ai_configuration_spec.rb b/spec/services/actors/save_ai_configuration_spec.rb
index f3668c1..45dc966 100644
--- a/spec/services/actors/save_ai_configuration_spec.rb
+++ b/spec/services/actors/save_ai_configuration_spec.rb
@@ -245,11 +245,24 @@ RSpec.describe Actors::SaveAiConfiguration do
# Same answer as for an inward name: a different one would tell the caller which
# internal names exist.
it 'refuses a name that does not resolve, with the same answer' do
- Lalabase::PublicAddress.resolver = ->(_hostname) { raise SocketError, 'no such name' }
+ Lalabase::PublicAddress.resolver = ->(_hostname, **) { raise SocketError, 'no such name' }
expect(save_endpoint('https://nowhere.example/v1').code).to eq(:api_base_not_public)
end
+ # The lookup runs under the organisation's row lock; a stalled resolver must not
+ # hold it for the system's own limits.
+ it 'bounds the DNS lookup of the endpoint' do
+ timeouts = []
+ Lalabase::PublicAddress.resolver = lambda do |_hostname, timeout:|
+ timeouts << timeout
+ ['52.1.2.3']
+ end
+
+ expect(save_endpoint('https://api.provider.example/v1')).to be_success
+ expect(timeouts).to eq([described_class::API_BASE_LOOKUP_TIMEOUT])
+ end
+
it 'refuses something that is not a URL at all' do
expect(save_endpoint('not a url').code).to eq(:api_base_not_https)
end
diff --git a/spec/services/lalabase/llm/address_race_spec.rb b/spec/services/lalabase/llm/address_race_spec.rb
new file mode 100644
index 0000000..16260fd
--- /dev/null
+++ b/spec/services/lalabase/llm/address_race_spec.rb
@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# What: Lalabase::Llm::AddressRace, the TCP connect over the checked addresses of a
+# customer endpoint.
+# Why: given the name, Ruby races the addresses (Happy Eyeballs) inside one open
+# timeout. Connecting to checked addresses one after the other lost both: an address
+# that drops packets used up the timeout before the next one was tried, and every
+# address got a timeout of its own. Each example pins one half of what Ruby did.
+#
+# A stalled attempt is the write end of a full pipe in place of the socket: it never
+# becomes writable while nobody reads, which is what a connect to an address that
+# drops packets looks like, and it does not depend on how the network of the machine
+# running the spec treats a documentation address.
+
+require 'rails_helper'
+require 'socket'
+
+RSpec.describe Lalabase::Llm::AddressRace, :ci_guard do
+ let(:server) { TCPServer.new('127.0.0.1', 0) }
+ let(:port) { server.addr[1] }
+ let(:stalled) { [] }
+ let(:readers) { [] }
+
+ after do
+ server.close
+ readers.each(&:close)
+ end
+
+ def now
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ end
+
+ # Addresses from 192.0.2.0/24 stall; every other one is really connected.
+ def race(addresses, timeout:)
+ instance = described_class.new(addresses: addresses, port: port, deadline: now + timeout)
+ allow(instance).to receive(:open_attempt).and_wrap_original do |original, addrinfo|
+ next original.call(addrinfo) unless addrinfo.ip_address.start_with?('192.0.2.')
+
+ reader, writer = IO.pipe
+ readers << reader
+ nil while writer.write_nonblock('x' * 65_536, exception: false) != :wait_writable
+ stalled << writer
+ writer
+ end
+ instance.call
+ end
+
+ it 'returns a connected TCPSocket, which Net::HTTP expects' do
+ socket = race(['127.0.0.1'], timeout: 2)
+
+ expect(socket).to be_a(TCPSocket)
+ expect(socket.peeraddr[3]).to eq('127.0.0.1')
+ ensure
+ socket&.close
+ end
+
+ it 'tries the next address after the attempt delay while the first one stalls' do
+ started = now
+ socket = race(['192.0.2.1', '127.0.0.1'], timeout: 2)
+
+ expect(socket.peeraddr[1]).to eq(port)
+ expect(now - started).to be_between(described_class::CONNECTION_ATTEMPT_DELAY - 0.05, 1.0)
+ expect(stalled).to all(be_closed)
+ ensure
+ socket&.close
+ end
+
+ it 'starts the next address at once when one fails' do
+ started = now
+ # Nothing listens on ::1 at this port: refused, or unavailable without IPv6.
+ socket = race(['::1', '127.0.0.1'], timeout: 2)
+
+ expect(socket.peeraddr[3]).to eq('127.0.0.1')
+ expect(now - started).to be < described_class::CONNECTION_ATTEMPT_DELAY
+ ensure
+ socket&.close
+ end
+
+ it 'spends one timeout on all addresses together' do
+ started = now
+
+ expect { race(['192.0.2.1', '192.0.2.2', '192.0.2.3'], timeout: 0.3) }.to raise_error(Net::OpenTimeout)
+ expect(now - started).to be < 0.6
+ expect(stalled).to all(be_closed)
+ end
+
+ it 'raises the last failure when every address fails' do
+ free_port = TCPServer.new('127.0.0.1', 0).then { |s| s.addr[1].tap { s.close } }
+ instance = described_class.new(addresses: ['::1', '127.0.0.1'], port: free_port, deadline: now + 2)
+
+ expect { instance.call }.to raise_error(SystemCallError)
+ end
+end
diff --git a/spec/services/lalabase/llm/public_endpoint_adapter_spec.rb b/spec/services/lalabase/llm/public_endpoint_adapter_spec.rb
index 27488e3..f0836b0 100644
--- a/spec/services/lalabase/llm/public_endpoint_adapter_spec.rb
+++ b/spec/services/lalabase/llm/public_endpoint_adapter_spec.rb
@@ -5,8 +5,8 @@
# Why: a check at save time cannot hold. DNS answers again for every request, and a
# name that pointed outward when it was saved can point at the app server when the
# key is sent (DNS rebinding). So the address is judged again for every connection,
-# and the socket goes to exactly the address that was judged, never to a second
-# lookup of the name.
+# and the socket goes only to the addresses that were judged, never to a second
+# lookup of the name. The race across them has its own spec (address_race_spec.rb).
require 'rails_helper'
require 'socket'
@@ -22,37 +22,77 @@ RSpec.describe Lalabase::Llm::PublicEndpointAdapter, :ci_guard do
end
describe 'the connection it builds' do
- it 'connects to the address it checked and keeps the name for TLS' do
- resolve_names('api.provider.example' => ['52.1.2.3'])
+ let(:server) { TCPServer.new('127.0.0.1', 0) }
+ let(:raced) { [] }
+
+ after { server.close }
+
+ # The race gets only checked addresses and hands back a socket to the local
+ # listener, so these examples need no network.
+ before do
+ allow(Lalabase::Llm::AddressRace).to receive(:call) do |addresses:, port:, deadline:|
+ raced << { addresses: addresses, port: port, deadline: deadline }
+ TCPSocket.new('127.0.0.1', server.addr[1])
+ end
+ end
+
+ def open(url)
+ http = connection_for(url)
+ http.start(&:started?)
+ http
+ end
+
+ it 'connects only to the addresses it checked and keeps the name for TLS' do
+ resolve_names('api.provider.example' => ['52.1.2.3', '2606:4700:4700::1111'])
- http = connection_for('https://api.provider.example/v1/models')
+ http = open("http://api.provider.example:#{server.addr[1]}/v1/models")
- expect(http.ipaddr).to eq('52.1.2.3')
+ expect(raced.last).to include(addresses: ['52.1.2.3', '2606:4700:4700::1111'], port: server.addr[1])
expect(http.address).to eq('api.provider.example')
end
- it 'refuses a name that points inward' do
+ it 'refuses a name that points inward, before any connect' do
resolve_names('localhost.attacker.example' => ['127.0.0.1'])
- expect { connection_for('https://localhost.attacker.example/v1/models') }
+ expect { open('https://localhost.attacker.example/v1/models') }
.to raise_error(described_class::EndpointNotPublicError)
+ expect(raced).to be_empty
end
# Rebinding: the first answer was public (that is what the save saw), the next
# one is not. The adapter asks again and refuses.
it 'judges the address again for every connection' do
answers = [['52.1.2.3'], ['127.0.0.1']]
- Lalabase::PublicAddress.resolver = ->(_hostname) { answers.shift }
+ Lalabase::PublicAddress.resolver = ->(_hostname, **) { answers.shift }
- expect(connection_for('https://rebind.attacker.example/v1').ipaddr).to eq('52.1.2.3')
- expect { connection_for('https://rebind.attacker.example/v1') }
+ open("http://rebind.attacker.example:#{server.addr[1]}/v1")
+ expect { open("http://rebind.attacker.example:#{server.addr[1]}/v1") }
.to raise_error(described_class::EndpointNotPublicError)
+ expect(raced.size).to eq(1)
end
- # Through a proxy the proxy resolves the name, and the address checked here would
+ # Given the name, Net::HTTP spent its open timeout on the lookup AND the connect.
+ # A lookup outside it would let a slow resolver hold a worker past the budget.
+ it 'counts the DNS lookup against the open timeout' do
+ lookup = {}
+ Lalabase::PublicAddress.resolver = lambda do |_hostname, timeout:|
+ lookup[:timeout] = timeout
+ sleep 0.2
+ ['52.1.2.3']
+ end
+ http = connection_for("http://slow-dns.provider.example:#{server.addr[1]}/v1")
+ http.open_timeout = 0.3
+
+ http.start(&:started?)
+
+ expect(lookup[:timeout]).to eq(0.3)
+ left_for_the_connect = raced.last[:deadline] - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ expect(left_for_the_connect).to be < 0.15
+ end
+
+ # Through a proxy the proxy resolves the name, and the addresses checked here would
# mean nothing.
it 'refuses to go through a proxy' do
- resolve_names('api.provider.example' => ['52.1.2.3'])
env = env_for('https://api.provider.example/v1')
env.request.proxy = { uri: URI('http://proxy.example:3128') }
@@ -63,83 +103,27 @@ RSpec.describe Lalabase::Llm::PublicEndpointAdapter, :ci_guard do
it 'is a connection failure to every rescue that already handles one' do
expect(described_class::EndpointNotPublicError.ancestors).to include(Faraday::ConnectionFailed)
end
- end
-
- # With the name, Net::HTTP let TCPSocket try every address of it. Pinned to one
- # address, a dual-stack endpoint with broken IPv6 would fail every request although
- # its IPv4 address answers. So the TCP connect walks the checked addresses in order,
- # and only the connect: nothing has been sent at that point, so trying the next
- # address cannot send a request twice.
- describe 'failover across the checked addresses' do
- let(:server) { TCPServer.new('127.0.0.1', 0) }
- let(:attempts) { [] }
-
- after { server.close }
-
- def connect_through(unreachable:)
- allow(TCPSocket).to receive(:open).and_wrap_original do |original, address, _port, *rest, **options|
- attempts << address
- raise Errno::ENETUNREACH, address if unreachable.include?(address)
-
- original.call('127.0.0.1', server.addr[1], *rest, **options)
- end
-
- http = connection_for("http://dual.provider.example:#{server.addr[1]}/v1")
- http.start(&:started?)
- end
-
- it 'connects to the next checked address when the first is unreachable' do
- resolve_names('dual.provider.example' => ['2001:db8::1', '203.0.113.20'])
-
- expect(connect_through(unreachable: ['2001:db8::1'])).to be(true)
- expect(attempts).to eq(['2001:db8::1', '203.0.113.20'])
- end
-
- it 'tries only the checked addresses and fails with the last error when none answers' do
- resolve_names('dual.provider.example' => ['2001:db8::1', '203.0.113.20'])
-
- expect { connect_through(unreachable: ['2001:db8::1', '203.0.113.20']) }
- .to raise_error(Errno::ENETUNREACH)
- expect(attempts).to eq(['2001:db8::1', '203.0.113.20'])
- end
-
- # With the name, every address shared one open timeout. Restarting it for each
- # checked address would turn the client's budget into one per address, and a
- # worker thread would hang N times as long on an endpoint that drops packets.
- it 'shares one open timeout across all checked addresses' do
- resolve_names('dual.provider.example' => ['203.0.113.20', '203.0.113.21', '203.0.113.22'])
- allow(TCPSocket).to receive(:open) do |address, *|
- attempts << address
- sleep 5
- end
-
- http = connection_for("http://dual.provider.example:#{server.addr[1]}/v1")
- http.open_timeout = 0.3
- started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-
- expect { http.start(&:started?) }.to raise_error(Net::OpenTimeout)
- expect(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).to be < 0.6
- expect(http.open_timeout).to eq(0.3)
- end
# Once connected, the request may have been written. A failure from there on must
- # not go to the next address: a chat request would run, and be billed, twice.
- it 'does not try the next address when the request fails after the connect' do
- resolve_names('dual.provider.example' => ['203.0.113.20', '203.0.113.21'])
+ # not connect again: a chat request would run, and be billed, twice.
+ it 'does not connect again when the request fails after the connect' do
+ resolve_names('api.provider.example' => ['203.0.113.20', '203.0.113.21'])
listener = Thread.new { loop { server.accept.close } }
- allow(TCPSocket).to receive(:open).and_wrap_original do |original, address, _port, *rest, **options|
- attempts << address
- original.call('127.0.0.1', server.addr[1], *rest, **options)
- end
- http = connection_for("http://dual.provider.example:#{server.addr[1]}/v1")
+ http = connection_for("http://api.provider.example:#{server.addr[1]}/v1")
expect { http.request(Net::HTTP::Post.new('/v1/chat/completions')) }.to raise_error(StandardError)
- expect(attempts).to eq(['203.0.113.20'])
+ expect(raced.size).to eq(1)
ensure
listener&.kill
end
end
+ # The seam is private to Net::HTTP; without it, Net::HTTP would resolve and connect
+ # to the name on its own, unchecked.
+ it 'sits on a Net::HTTP that still has the seam it takes over' do
+ expect(Net::HTTP.private_method_defined?(:timeouted_connect)).to be(true)
+ end
+
# End to end, with the real OpenAI client: a listener on loopback stands for a
# service on the app server, and a public-looking name resolves to it.
describe 'Lalabase::Llm::OpenAiCompatibleProvider at request time' do
diff --git a/spec/services/lalabase/public_address_spec.rb b/spec/services/lalabase/public_address_spec.rb
index 5384254..419cc5c 100644
--- a/spec/services/lalabase/public_address_spec.rb
+++ b/spec/services/lalabase/public_address_spec.rb
@@ -55,13 +55,13 @@ RSpec.describe Lalabase::PublicAddress, :ci_guard do
# Unresolvable and inward must read the same: a different answer for a name that
# does not exist would tell the caller which internal names do.
it 'refuses a name that does not resolve' do
- described_class.resolver = ->(_hostname) { raise SocketError, 'getaddrinfo: nodename nor servname provided' }
+ described_class.resolver = ->(_hostname, **) { raise SocketError, 'getaddrinfo: nodename nor servname provided' }
expect { described_class.resolve!('nowhere.example') }.to raise_error(described_class::NotPublicError)
end
it 'refuses a name that resolves to nothing' do
- described_class.resolver = ->(_hostname) { [] }
+ described_class.resolver = ->(_hostname, **) { [] }
expect { described_class.resolve!('empty.example') }.to raise_error(described_class::NotPublicError)
end
diff --git a/spec/support/public_address_resolver.rb b/spec/support/public_address_resolver.rb
index 6802e73..de8ef32 100644
--- a/spec/support/public_address_resolver.rb
+++ b/spec/support/public_address_resolver.rb
@@ -22,7 +22,7 @@ module PublicAddressResolverHelpers
}.freeze
def self.resolver_for(table)
- lambda do |hostname|
+ lambda do |hostname, timeout: nil| # rubocop:disable Lint/UnusedBlockArgument
numeric = numeric_addresses(hostname)
next numeric if numeric