Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

23.5 KB
Raw Blame
use std::sync::{
    atomic::{AtomicBool, AtomicUsize, Ordering},
    Arc, Mutex,
};
use std::time::Duration;

use async_trait::async_trait;
use cymbal::error::{ResolveError, UnhandledError};
use cymbal::frames::{Frame, RawFrame};
use cymbal::langs::native::DebugImage;
use cymbal::modes::resolution::load_monitor::LoadMonitor;
use cymbal::modes::resolution::service::{CymbalResolutionService, ServiceConfig};
use cymbal::symbolication::symbol::SymbolResolver;
use cymbal::symbolication::symbol_store::{chunk_id::OrChunkId, proguard::ProguardRef};
use cymbal::types::operator::TeamId;
use cymbal::types::{Exception, Stacktrace};
use cymbal_proto::cymbal::resolution::v1::cymbal_resolution_client::CymbalResolutionClient;
use cymbal_proto::cymbal::resolution::v1::cymbal_resolution_server::CymbalResolution;
use cymbal_proto::cymbal::resolution::v1::cymbal_resolution_server::CymbalResolutionServer;
use cymbal_proto::cymbal::resolution::v1::{
    resolve_outcome, ErrorKind, ResolveItem, ResolveOutcome, SubscribeRequest,
};
use cymbal_proto::CYMBAL_RESOLUTION_MAX_MESSAGE_SIZE_BYTES;
use futures::StreamExt;
use tokio::sync::Semaphore;
use tonic::transport::{Channel, Server};
use tonic::Request;
use uuid::Uuid;

#[derive(Default)]
struct FakeResolver {
    fail_unhandled: bool,
    resolved_frames: Vec<Frame>,
    release_id: Option<Uuid>,
    release_refs: Arc<Mutex<Vec<String>>>,
}

#[async_trait]
impl SymbolResolver for FakeResolver {
    async fn resolve_raw_frame(
        &self,
        _team_id: TeamId,
        _frame: &RawFrame,
        _debug_images: &[DebugImage],
    ) -> Result<Vec<Frame>, UnhandledError> {
        if self.fail_unhandled {
            return Err(UnhandledError::Other("forced resolver failure".to_string()));
        }
        Ok(self.resolved_frames.clone())
    }

    async fn resolve_java_class(
        &self,
        _team_id: TeamId,
        _symbolset_ref: OrChunkId<ProguardRef>,
        _class: String,
    ) -> Result<String, ResolveError> {
        unreachable!("fake resolver does not need java class resolution for these tests")
    }

    async fn resolve_dart_minified_name(
        &self,
        _team_id: TeamId,
        _symbolset_ref: String,
        _minified_name: &str,
    ) -> Result<String, ResolveError> {
        unreachable!("fake resolver does not need dart name resolution for these tests")
    }

    async fn latest_release_id(
        &self,
        _team_id: TeamId,
        symbol_set_refs: &[String],
    ) -> Result<Option<Uuid>, UnhandledError> {
        *self.release_refs.lock().expect("release refs poisoned") = symbol_set_refs.to_vec();
        Ok(self.release_id)
    }
}

struct SlowResolver {
    delay: Duration,
    first_call_delay: Option<Duration>,
    active: Arc<AtomicUsize>,
    max_active: Arc<AtomicUsize>,
    started: Arc<AtomicUsize>,
}

#[async_trait]
impl SymbolResolver for SlowResolver {
    async fn resolve_raw_frame(
        &self,
        _team_id: TeamId,
        _frame: &RawFrame,
        _debug_images: &[DebugImage],
    ) -> Result<Vec<Frame>, UnhandledError> {
        let call_index = self.started.fetch_add(1, Ordering::AcqRel);
        let active = self.active.fetch_add(1, Ordering::AcqRel) + 1;
        update_max(&self.max_active, active);

        let delay = if call_index == 0 {
            self.first_call_delay.unwrap_or(self.delay)
        } else {
            self.delay
        };
        tokio::time::sleep(delay).await;
        self.active.fetch_sub(1, Ordering::AcqRel);
        Ok(Vec::new())
    }

    async fn resolve_java_class(
        &self,
        _team_id: TeamId,
        _symbolset_ref: OrChunkId<ProguardRef>,
        _class: String,
    ) -> Result<String, ResolveError> {
        unreachable!("slow resolver does not need java class resolution for these tests")
    }

    async fn resolve_dart_minified_name(
        &self,
        _team_id: TeamId,
        _symbolset_ref: String,
        _minified_name: &str,
    ) -> Result<String, ResolveError> {
        unreachable!("slow resolver does not need dart name resolution for these tests")
    }
}

fn update_max(max_active: &AtomicUsize, candidate: usize) {
    let mut current = max_active.load(Ordering::Acquire);
    while candidate > current {
        match max_active.compare_exchange(current, candidate, Ordering::AcqRel, Ordering::Acquire) {
            Ok(_) => return,
            Err(actual) => current = actual,
        }
    }
}

fn fast_service_config() -> ServiceConfig {
    ServiceConfig {
        default_tick_interval: Duration::from_millis(20),
        min_tick_interval: Duration::from_millis(1),
        max_tick_interval: Duration::from_secs(1),
    }
}

fn make_service(resolver: FakeResolver) -> CymbalResolutionService {
    make_service_with_config(
        Arc::new(resolver),
        Arc::new(Semaphore::new(4)),
        4,
        fast_service_config(),
    )
}

fn make_service_with_config(
    resolver: Arc<dyn SymbolResolver>,
    symbol_resolution_limiter: Arc<Semaphore>,
    max_in_flight: u32,
    service_config: ServiceConfig,
) -> CymbalResolutionService {
    CymbalResolutionService::new(
        resolver,
        symbol_resolution_limiter,
        LoadMonitor::new(max_in_flight),
        "test-instance",
        service_config,
        Arc::new(AtomicBool::new(false)),
    )
}

async fn spawn_test_channel(service: CymbalResolutionService) -> Channel {
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
        .await
        .expect("bind test server");
    let addr = listener.local_addr().expect("test server local addr");
    let incoming = futures::stream::unfold(listener, |listener| async {
        Some((listener.accept().await.map(|(stream, _)| stream), listener))
    });

    tokio::spawn(async move {
        Server::builder()
            .add_service(
                CymbalResolutionServer::new(service)
                    .max_decoding_message_size(CYMBAL_RESOLUTION_MAX_MESSAGE_SIZE_BYTES)
                    .max_encoding_message_size(CYMBAL_RESOLUTION_MAX_MESSAGE_SIZE_BYTES),
            )
            .serve_with_incoming(incoming)
            .await
            .expect("test gRPC server exits cleanly");
    });

    Channel::from_shared(format!("http://{addr}"))
        .expect("valid test endpoint")
        .connect()
        .await
        .expect("connect to test server")
}

async fn resolve_items(
    service: CymbalResolutionService,
    items: Vec<ResolveItem>,
) -> Vec<ResolveOutcome> {
    resolve_items_with_accepted(service, items)
        .await
        .into_iter()
        .filter(|outcome| {
            !matches!(
                outcome_result(outcome),
                resolve_outcome::Result::Accepted(_)
            )
        })
        .collect()
}

async fn resolve_items_with_accepted(
    service: CymbalResolutionService,
    items: Vec<ResolveItem>,
) -> Vec<ResolveOutcome> {
    let channel = spawn_test_channel(service).await;
    let mut client = CymbalResolutionClient::new(channel)
        .max_decoding_message_size(CYMBAL_RESOLUTION_MAX_MESSAGE_SIZE_BYTES)
        .max_encoding_message_size(CYMBAL_RESOLUTION_MAX_MESSAGE_SIZE_BYTES);
    let response = client
        .resolve(Request::new(futures::stream::iter(items)))
        .await
        .expect("resolve returns response");
    let mut stream = response.into_inner();
    let mut outcomes = Vec::new();
    while let Some(outcome) = stream.next().await {
        outcomes.push(outcome.expect("outcome must not be status error"));
    }
    outcomes
}

fn raw_exception(exception_type: &str) -> Exception {
    Exception {
        exception_id: None,
        exception_type: exception_type.to_string(),
        exception_message: "boom".to_string(),
        mechanism: None,
        module: None,
        thread_id: None,
        stack: Some(Stacktrace::Raw { frames: vec![] }),
    }
}

fn make_item(id: u64, exc: &Exception) -> ResolveItem {
    ResolveItem {
        id,
        team_id: 7,
        exception_json: serde_json::to_vec(exc).expect("serialize exception"),
        metadata: Vec::new(),
        deadline_ms: 1_000,
    }
}

fn make_item_with_metadata(id: u64, exc: &Exception, metadata: Vec<u8>) -> ResolveItem {
    ResolveItem {
        id,
        metadata,
        ..make_item(id, exc)
    }
}

fn outcome_result(outcome: &ResolveOutcome) -> &resolve_outcome::Result {
    outcome.result.as_ref().expect("outcome result present")
}

fn error_kind(outcome: &ResolveOutcome) -> ErrorKind {
    let resolve_outcome::Result::Error(error) = outcome_result(outcome) else {
        panic!("expected error outcome, got {outcome:?}");
    };
    ErrorKind::try_from(error.kind).expect("known error kind")
}

fn assert_no_retry(outcomes: &[ResolveOutcome]) {
    assert!(
        outcomes
            .iter()
            .all(|outcome| !matches!(outcome_result(outcome), resolve_outcome::Result::Retry(_))),
        "server overload must be an ErrorKind, not a Retry outcome: {outcomes:?}",
    );
}

#[tokio::test]
async fn bidi_resolve_stream_resolves_multiple_items_and_echoes_ids() {
    let service = make_service(FakeResolver::default());
    let exc = raw_exception("RuntimeError");

    let outcomes = resolve_items(service, vec![make_item(41, &exc), make_item(42, &exc)]).await;

    assert_eq!(outcomes.len(), 2);
    assert_eq!(
        outcomes
            .iter()
            .map(|outcome| outcome.id)
            .collect::<Vec<_>>(),
        vec![41, 42]
    );
    for outcome in outcomes {
        let resolve_outcome::Result::Done(done) = outcome_result(&outcome) else {
            panic!("expected Done outcome, got {outcome:?}");
        };
        let resolved: Exception = serde_json::from_slice(&done.resolved_exception_json)
            .expect("valid resolved exception");
        assert_eq!(resolved.exception_type, "RuntimeError");
        assert!(matches!(resolved.stack, Some(Stacktrace::Resolved { .. })));
    }
}

#[tokio::test]
async fn bidi_resolve_stream_accepts_messages_larger_than_tonic_default() {
    let service = make_service(FakeResolver::default());
    let mut exc = raw_exception("RuntimeError");
    exc.exception_message = "x".repeat(5 * 1024 * 1024);
    let item = make_item(41, &exc);

    assert!(item.exception_json.len() > 4 * 1024 * 1024);
    assert!(item.exception_json.len() < CYMBAL_RESOLUTION_MAX_MESSAGE_SIZE_BYTES);

    let outcomes = resolve_items(service, vec![item]).await;

    assert_eq!(outcomes.len(), 1);
    let resolve_outcome::Result::Done(done) = outcome_result(&outcomes[0]) else {
        panic!("expected Done outcome, got {:?}", outcomes[0]);
    };
    let resolved: Exception =
        serde_json::from_slice(&done.resolved_exception_json).expect("valid resolved exception");
    assert_eq!(resolved.exception_message.len(), 5 * 1024 * 1024);
}

#[tokio::test]
async fn admitted_items_emit_accepted_before_terminal_outcome() {
    let service = make_service(FakeResolver::default());
    let exc = raw_exception("RuntimeError");

    let outcomes = resolve_items_with_accepted(service, vec![make_item(41, &exc)]).await;

    assert_eq!(outcomes.len(), 2);
    assert_eq!(outcomes[0].id, 41);
    assert!(matches!(
        outcome_result(&outcomes[0]),
        resolve_outcome::Result::Accepted(_)
    ));
    assert_eq!(outcomes[1].id, 41);
    assert!(matches!(
        outcome_result(&outcomes[1]),
        resolve_outcome::Result::Done(_)
    ));
}

#[tokio::test]
async fn raw_frames_are_resolved_into_done_payload() {
    let raw_frame = sample_raw_frame();
    let mut resolver_frame = sample_resolved_frame(&raw_frame);
    resolver_frame.frame_id = raw_frame.frame_id(123, 99, &[]);
    let service = make_service(FakeResolver {
        resolved_frames: vec![resolver_frame],
        ..Default::default()
    });
    let mut exc = raw_exception("RuntimeError");
    exc.stack = Some(Stacktrace::Raw {
        frames: vec![raw_frame.clone()],
    });

    let outcomes = resolve_items(service, vec![make_item(1, &exc)]).await;
    assert_eq!(outcomes.len(), 1);
    let resolve_outcome::Result::Done(done) = outcome_result(&outcomes[0]) else {
        panic!("expected Done outcome, got {:?}", outcomes[0]);
    };
    let resolved: Exception =
        serde_json::from_slice(&done.resolved_exception_json).expect("valid resolved exception");

    let Some(Stacktrace::Resolved { frames }) = resolved.stack else {
        panic!("raw stack must be replaced with resolved frames");
    };
    let expected_wire_frame: Frame = serde_json::from_value(
        serde_json::to_value(sample_resolved_frame(&raw_frame)).expect("serialize expected frame"),
    )
    .expect("deserialize expected wire frame");
    assert_eq!(frames, vec![expected_wire_frame]);
}

#[tokio::test]
async fn the_release_rides_the_outcome_and_is_looked_up_by_distinct_symbol_set_ref() {
    let release_id = Uuid::from_u128(7);
    let release_refs = Arc::new(Mutex::new(Vec::new()));
    let service = make_service(FakeResolver {
        release_id: Some(release_id),
        release_refs: release_refs.clone(),
        ..Default::default()
    });
    let mut exc = raw_exception("RuntimeError");
    // Two frames from one chunk and one from another: the lookup sees each ref once.
    exc.stack = Some(Stacktrace::Raw {
        frames: vec![
            js_raw_frame("chunk-a"),
            js_raw_frame("chunk-a"),
            js_raw_frame("chunk-b"),
        ],
    });

    let outcomes = resolve_items(service, vec![make_item(1, &exc)]).await;
    let resolve_outcome::Result::Done(done) = outcome_result(&outcomes[0]) else {
        panic!("expected Done outcome, got {:?}", outcomes[0]);
    };

    assert_eq!(done.release_id, release_id.to_string());
    assert_eq!(
        *release_refs.lock().expect("release refs poisoned"),
        vec!["chunk-a".to_string(), "chunk-b".to_string()]
    );
}

#[tokio::test]
async fn no_bound_release_leaves_the_outcome_release_id_empty() {
    let service = make_service(FakeResolver::default());
    let mut exc = raw_exception("RuntimeError");
    exc.stack = Some(Stacktrace::Raw {
        frames: vec![sample_raw_frame()],
    });

    let outcomes = resolve_items(service, vec![make_item(1, &exc)]).await;
    let resolve_outcome::Result::Done(done) = outcome_result(&outcomes[0]) else {
        panic!("expected Done outcome, got {:?}", outcomes[0]);
    };

    assert!(done.release_id.is_empty());
}

#[tokio::test]
async fn bidi_resolve_stream_emits_outcomes_as_items_complete_out_of_order() {
    let active = Arc::new(AtomicUsize::new(0));
    let max_active = Arc::new(AtomicUsize::new(0));
    let started = Arc::new(AtomicUsize::new(0));
    let service = make_service_with_config(
        Arc::new(SlowResolver {
            delay: Duration::from_millis(5),
            first_call_delay: Some(Duration::from_millis(80)),
            active,
            max_active: max_active.clone(),
            started: started.clone(),
        }),
        Arc::new(Semaphore::new(8)),
        8,
        fast_service_config(),
    );
    let mut exc = raw_exception("RuntimeError");
    exc.stack = Some(Stacktrace::Raw {
        frames: vec![sample_raw_frame()],
    });
    let items = (1..=4).map(|id| make_item(id, &exc)).collect();

    let outcomes = resolve_items(service, items).await;

    assert_eq!(started.load(Ordering::Acquire), 4);
    assert!(
        max_active.load(Ordering::Acquire) > 1,
        "items should be processed concurrently",
    );
    assert_eq!(outcomes.len(), 4);
    assert_ne!(
        outcomes[0].id, 1,
        "a faster later item should complete before the slower first item",
    );
    assert!(outcomes
        .iter()
        .all(|outcome| matches!(outcome_result(outcome), resolve_outcome::Result::Done(_))));
}

#[tokio::test]
async fn invalid_payload_and_metadata_formats_are_invalid_payload_errors() {
    let service = make_service(FakeResolver::default());
    let exc = raw_exception("RuntimeError");
    let outcomes = resolve_items(
        service,
        vec![
            ResolveItem {
                id: 1,
                team_id: 7,
                exception_json: b"not-json".to_vec(),
                metadata: Vec::new(),
                deadline_ms: 1_000,
            },
            make_item_with_metadata(2, &exc, b"not-json".to_vec()),
            make_item_with_metadata(3, &exc, br#"{"debug_images_json":"not-a-list"}"#.to_vec()),
        ],
    )
    .await;

    assert_eq!(outcomes.len(), 3);
    for outcome in outcomes {
        assert_eq!(error_kind(&outcome), ErrorKind::InvalidPayload);
    }
}

#[tokio::test]
async fn unhandled_resolver_failure_classifies_as_unhandled_error() {
    let service = make_service(FakeResolver {
        fail_unhandled: true,
        ..Default::default()
    });
    let mut exc = raw_exception("RuntimeError");
    exc.stack = Some(Stacktrace::Raw {
        frames: vec![sample_raw_frame()],
    });

    let outcomes = resolve_items(service, vec![make_item(1, &exc)]).await;

    assert_eq!(outcomes.len(), 1);
    assert_eq!(outcomes[0].id, 1);
    assert_eq!(error_kind(&outcomes[0]), ErrorKind::Unhandled);
}

#[tokio::test]
async fn load_monitor_overload_is_an_overloaded_error_outcome_only() {
    let active = Arc::new(AtomicUsize::new(0));
    let max_active = Arc::new(AtomicUsize::new(0));
    let started = Arc::new(AtomicUsize::new(0));
    let service = make_service_with_config(
        Arc::new(SlowResolver {
            delay: Duration::from_millis(80),
            first_call_delay: None,
            active,
            max_active,
            started,
        }),
        Arc::new(Semaphore::new(8)),
        1,
        fast_service_config(),
    );
    let mut exc = raw_exception("RuntimeError");
    exc.stack = Some(Stacktrace::Raw {
        frames: vec![sample_raw_frame()],
    });

    let outcomes = resolve_items(service, vec![make_item(1, &exc), make_item(2, &exc)]).await;

    assert_no_retry(&outcomes);
    let overloaded = outcomes
        .iter()
        .find(|outcome| outcome.id == 2)
        .expect("second item should be rejected by max_in_flight");
    assert_eq!(error_kind(overloaded), ErrorKind::Overloaded);
    let completed = outcomes
        .iter()
        .find(|outcome| outcome.id == 1)
        .expect("first admitted item should complete");
    assert!(matches!(
        outcome_result(completed),
        resolve_outcome::Result::Done(_)
    ));
}

#[tokio::test]
async fn symbol_limiter_overload_is_an_overloaded_error_outcome_only() {
    let limiter = Arc::new(Semaphore::new(1));
    let _permit = limiter.clone().acquire_owned().await.unwrap();
    limiter.close();
    let service = make_service_with_config(
        Arc::new(FakeResolver::default()),
        limiter,
        4,
        fast_service_config(),
    );

    let outcomes = resolve_items(service, vec![make_item(1, &java_exception_for_overload())]).await;

    assert_eq!(outcomes.len(), 1);
    assert_no_retry(&outcomes);
    assert_eq!(error_kind(&outcomes[0]), ErrorKind::Overloaded);
}

fn java_exception_for_overload() -> Exception {
    let frame_json = serde_json::json!({
        "platform": "java",
        "module": "com.example",
        "filename": "A.java",
        "function": "f",
        "in_app": true,
    });
    let frame: RawFrame = serde_json::from_value(frame_json).expect("valid java frame");
    Exception {
        exception_id: None,
        exception_type: "Boom".to_string(),
        exception_message: "msg".to_string(),
        mechanism: None,
        module: Some("com.example".to_string()),
        thread_id: None,
        stack: Some(Stacktrace::Raw {
            frames: vec![frame],
        }),
    }
}

fn js_raw_frame(chunk_id: &str) -> RawFrame {
    let json = serde_json::json!({
        "platform": "web:javascript",
        "filename": "a.js",
        "function": "f",
        "in_app": true,
        "lineno": 1,
        "colno": 1,
        "chunk_id": chunk_id,
    });
    serde_json::from_value(json).expect("valid raw frame")
}

fn sample_raw_frame() -> RawFrame {
    let json = serde_json::json!({
        "platform": "web:javascript",
        "filename": "a.js",
        "function": "f",
        "in_app": true,
        "lineno": 1,
        "colno": 1,
    });
    serde_json::from_value(json).expect("valid raw frame")
}

fn sample_resolved_frame(raw_frame: &RawFrame) -> Frame {
    Frame {
        frame_id: raw_frame.frame_id(7, 0, &[]),
        mangled_name: "f".to_string(),
        line: Some(42),
        column: Some(7),
        source: Some("src/app.ts".to_string()),
        module: Some("app".to_string()),
        in_app: true,
        resolved_name: Some("renderCheckout".to_string()),
        lang: "javascript".to_string(),
        resolved: true,
        resolve_failure: None,
        synthetic: false,
        suspicious: false,
        junk_drawer: None,
        code_variables: None,
        context: None,
    }
}

#[tokio::test]
async fn subscribe_emits_periodic_load_events_with_monotonic_sequence() {
    let service = make_service(FakeResolver::default());
    let response = service
        .subscribe(Request::new(SubscribeRequest {
            subscriber_id: "test-subscriber".to_string(),
            tick_hint_ms: 0,
        }))
        .await
        .expect("subscribe returns response");
    let mut stream = response.into_inner();

    let mut events = Vec::new();
    for _ in 0..3 {
        let event = stream
            .next()
            .await
            .expect("stream open")
            .expect("event without status error");
        events.push(event);
    }
    drop(stream);

    assert_eq!(events.len(), 3);
    assert_eq!(events[0].sequence, 1);
    assert_eq!(events[1].sequence, 2);
    assert_eq!(events[2].sequence, 3);
    assert!(!events[0].draining);
    assert_eq!(events[0].in_flight, 0);
    assert_eq!(events[0].max_in_flight, 4);
    assert_eq!(events[0].service_instance_id, "test-instance");
}

#[tokio::test]
async fn subscribe_reflects_draining_state() {
    let draining = Arc::new(AtomicBool::new(true));
    let service = CymbalResolutionService::new(
        Arc::new(FakeResolver::default()),
        Arc::new(Semaphore::new(4)),
        LoadMonitor::new(4),
        "draining-test",
        fast_service_config(),
        draining,
    );
    let response = service
        .subscribe(Request::new(SubscribeRequest {
            subscriber_id: "draining-watcher".to_string(),
            tick_hint_ms: 0,
        }))
        .await
        .expect("subscribe returns response");
    let mut stream = response.into_inner();

    let event = stream.next().await.expect("stream open").expect("event ok");
    assert!(event.draining);
    assert_eq!(event.service_instance_id, "draining-test");
}

#[tokio::test]
async fn subscribe_clamps_caller_hint_to_server_bounds() {
    let service = make_service(FakeResolver::default());
    let response = service
        .subscribe(Request::new(SubscribeRequest {
            subscriber_id: "fast-caller".to_string(),
            tick_hint_ms: 0,
        }))
        .await
        .expect("subscribe accepts default hints");
    let mut stream = response.into_inner();
    let event = stream.next().await.expect("at least one event").unwrap();
    assert_eq!(event.sequence, 1);
}

#[tokio::test]
async fn subscribe_terminates_when_caller_drops_stream() {
    let service = make_service(FakeResolver::default());
    let response = service
        .subscribe(Request::new(SubscribeRequest::default()))
        .await
        .unwrap();
    let mut stream = response.into_inner();
    let _first = stream.next().await.unwrap();
    drop(stream);

    tokio::time::sleep(Duration::from_millis(30)).await;

    let response = service
        .subscribe(Request::new(SubscribeRequest::default()))
        .await
        .unwrap();
    let mut stream2 = response.into_inner();
    let event = stream2.next().await.unwrap().unwrap();
    assert_eq!(event.sequence, 1, "new subscription restarts sequence at 1");
}