A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: lsp potential race conditions (#8012)
Myles Scolnicka38e82b56e72
diff --git a/frontend/src/core/codemirror/lsp/__tests__/notebook-lsp.test.ts b/frontend/src/core/codemirror/lsp/__tests__/notebook-lsp.test.tsindex e3c55bf05d..e1d17b3c7b 100644--- a/frontend/src/core/codemirror/lsp/__tests__/notebook-lsp.test.ts+++ b/frontend/src/core/codemirror/lsp/__tests__/notebook-lsp.test.ts@@ -25,6 +25,30 @@ const Cells = {};describe("createNotebookLens", () => {+ it("should produce correct lens for same inputs", () => {+ // Use unique content for this test+ const cellIds: CellId[] = [Cells.cell1, Cells.cell2];+ const codes: Record<CellId, string> = {+ [Cells.cell1]: "unique_memo_test_line1",+ [Cells.cell2]: "unique_memo_test_line2",+ };++ const lens1 = createNotebookLens(cellIds, codes);+ const lens2 = createNotebookLens(cellIds, codes);++ // Same inputs should produce equivalent lens (same merged text)+ expect(lens1.mergedText).toBe(lens2.mergedText);+ expect(lens1.cellIds).toEqual(lens2.cellIds);++ // Different content should produce a different lens+ const differentCodes: Record<CellId, string> = {+ [Cells.cell1]: "unique_memo_different",+ [Cells.cell2]: "unique_memo_content",+ };+ const lens3 = createNotebookLens(cellIds, differentCodes);+ expect(lens3.mergedText).not.toBe(lens1.mergedText);+ });+it("should calculate correct line offsets", () => {const cellIds: CellId[] = [Cells.cell1, Cells.cell2, Cells.cell3];const codes: Record<CellId, string> = {@@ -111,6 +135,7 @@ describe("createNotebookLens", () => {),).toBe(true);+ // This triggers the cross-cell diagnostic warning (start in range, end outside)expect(lens.isInRange({@@ -122,6 +147,24 @@ describe("createNotebookLens", () => {).toBe(false);});+ it("should detect cross-cell diagnostics where start is in range but end is outside", () => {+ const cellIds: CellId[] = [Cells.cell1, Cells.cell2];+ const codes: Record<CellId, string> = {+ [Cells.cell1]: "line1\nline2",+ [Cells.cell2]: "line3",+ };+ const lens = createNotebookLens(cellIds, codes);++ // Range that starts in cell1 (line 1) but ends in cell2 (line 2 = first line of cell2)+ // This should return false and log a warning+ const crossCellRange = {+ start: { line: 1, character: 0 }, // line 1 is in cell1+ end: { line: 2, character: 0 }, // line 2 is in cell2+ };++ expect(lens.isInRange(crossCellRange, Cells.cell1)).toBe(false);+ });+it("should join all code into merged text", () => {const cellIds: CellId[] = [Cells.cell1, Cells.cell2, Cells.cell3];const codes: Record<CellId, string> = {@@ -1120,6 +1163,83 @@ describe("NotebookLanguageServerClient", () => {});});+ describe("sync returns lens to prevent race conditions", () => {+ it("should return the lens used for synchronization", async () => {+ mockClient.textDocumentDidChange = vi+ .fn()+ .mockImplementation((params) => params);++ const result = await notebookClient.sync();++ // sync() should return both params and the lens+ expect(result).toHaveProperty("params");+ expect(result).toHaveProperty("lens");+ expect(result.lens.cellIds).toEqual([+ Cells.cell1,+ Cells.cell2,+ Cells.cell3,+ ]);+ expect(result.lens.mergedText).toContain("# this is a comment");+ });++ it("should return consistent lens even when called multiple times rapidly", async () => {+ mockClient.textDocumentDidChange = vi+ .fn()+ .mockImplementation((params) => params);++ // Call sync multiple times rapidly+ const [result1, result2, result3] = await Promise.all([+ notebookClient.sync(),+ notebookClient.sync(),+ notebookClient.sync(),+ ]);++ // All should return the same lens content (memoized)+ expect(result1.lens.mergedText).toBe(result2.lens.mergedText);+ expect(result2.lens.mergedText).toBe(result3.lens.mergedText);+ });+ });++ describe("SEEN_CELL_DOCUMENT_URIS memory management", () => {+ it("should track opened cells in SEEN_CELL_DOCUMENT_URIS", async () => {+ // Clear any existing state+ const seenUris = (NotebookLanguageServerClient as any)+ .SEEN_CELL_DOCUMENT_URIS;+ seenUris.clear();++ // Open some cells to add them to SEEN_CELL_DOCUMENT_URIS+ await notebookClient.textDocumentDidOpen({+ textDocument: {+ uri: CellDocumentUri.of(Cells.cell1),+ languageId: "python",+ version: 1,+ text: "code1",+ },+ });++ await notebookClient.textDocumentDidOpen({+ textDocument: {+ uri: CellDocumentUri.of(Cells.cell2),+ languageId: "python",+ version: 1,+ text: "code2",+ },+ });++ // Verify cells were added+ expect(seenUris.has(CellDocumentUri.of(Cells.cell1))).toBe(true);+ expect(seenUris.has(CellDocumentUri.of(Cells.cell2))).toBe(true);+ expect(seenUris.size).toBe(2);+ });++ it("should have pruneSeenCellUris static method", () => {+ // Verify the method exists and is callable+ expect(+ typeof (NotebookLanguageServerClient as any).pruneSeenCellUris,+ ).toBe("function");+ });+ });+describe("initialization and configuration", () => {it("should send configuration after initialization", async () => {const configNotifications: any[] = [];diff --git a/frontend/src/core/codemirror/lsp/lens.ts b/frontend/src/core/codemirror/lsp/lens.tsindex 9bb12cf80d..c8127ed5c8 100644--- a/frontend/src/core/codemirror/lsp/lens.ts+++ b/frontend/src/core/codemirror/lsp/lens.ts@@ -68,12 +68,28 @@ export function createNotebookLens(reverseRange: (range: LSP.Range, cellId: CellId) =>shiftRange(range, -getCurrentLineOffset(cellId)),+ /**+ * Check if a range falls entirely within the given cell.+ * Returns false for ranges that span multiple cells (cross-cell diagnostics).+ */isInRange: (range: LSP.Range, cellId: CellId) => {const cellLines = codes[cellId].split("\n").length;const offset = cellLineOffsets.get(cellId) || 0;const startLine = range.start.line - offset;const endLine = range.end.line - offset;- return startLine >= 0 && endLine < cellLines;++ const startInRange = startLine >= 0 && startLine < cellLines;+ const endInRange = endLine >= 0 && endLine < cellLines;++ // Warn about cross-cell diagnostics that start in this cell but end outside+ if (startInRange && !endInRange) {+ Logger.warn(+ "[lsp] Cross-cell diagnostic detected: range starts in cell but ends outside",+ { cellId, range, cellLines, offset },+ );+ }++ return startInRange && endInRange;},getEditsForNewText: (newText: string) => {diff --git a/frontend/src/core/codemirror/lsp/notebook-lsp.ts b/frontend/src/core/codemirror/lsp/notebook-lsp.tsindex 6ec52e9133..0ede423403 100644--- a/frontend/src/core/codemirror/lsp/notebook-lsp.ts+++ b/frontend/src/core/codemirror/lsp/notebook-lsp.ts@@ -113,8 +113,28 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {EditorView | null | undefined>;private readonly initialSettings: Record<string, unknown>;+ /**+ * Tracks which cell document URIs have been opened with the LSP server.+ * Used to clear diagnostics for cells that no longer have any.+ *+ * This set is pruned when diagnostics are processed to only include+ * cells that exist in the current notebook snapshot.+ */private static readonly SEEN_CELL_DOCUMENT_URIS = new Set<CellDocumentUri>();+ /**+ * Remove cell URIs that are no longer in the notebook.+ * Called during diagnostic processing to prevent memory leaks.+ */+ private static pruneSeenCellUris(currentCellIds: Set<CellId>): void {+ for (const uri of NotebookLanguageServerClient.SEEN_CELL_DOCUMENT_URIS) {+ const cellId = CellDocumentUri.parse(uri);+ if (!currentCellIds.has(cellId)) {+ NotebookLanguageServerClient.SEEN_CELL_DOCUMENT_URIS.delete(uri);+ }+ }+ }+/*** Cache of completion items to avoid jitter while typing in the same completion item*/@@ -274,26 +294,30 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {};}- async sync(): Promise<LSP.DidChangeTextDocumentParams> {+ /**+ * Synchronize the document with the LSP server and return the lens used.+ * This ensures the caller uses the same lens that was sent to the server,+ * avoiding race conditions if cells change between sync and subsequent operations.+ */+ async sync(): Promise<{+ params: LSP.DidChangeTextDocumentParams;+ lens: NotebookLens;+ }> {const { lens, version, didChange } = this.snapshotter.snapshot();- if (!didChange) {- return {- textDocument: {- uri: this.documentUri,- version: version,- },- contentChanges: [{ text: lens.mergedText }],- };- }-- // Update changes for merged doc, etc.- return this.client.textDocumentDidChange({+ const params: LSP.DidChangeTextDocumentParams = {textDocument: {uri: this.documentUri,version: version,},contentChanges: [{ text: lens.mergedText }],- });+ };++ if (didChange) {+ // Update changes for merged doc+ await this.client.textDocumentDidChange(params);+ }++ return { params, lens };}public async textDocumentDidChange(params: LSP.DidChangeTextDocumentParams) {@@ -301,7 +325,8 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {// We know how to only handle single content changes// But that is all we expect to receiveif (params.contentChanges.length === 1) {- return this.sync();+ const { params: syncParams } = await this.sync();+ return syncParams;}Logger.warn("[lsp] Unhandled textDocumentDidChange", params);@@ -319,9 +344,8 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {return null;}- // This LSP method has no version, so lets sync and then get the latest snapshot- await this.sync();- const { lens } = this.snapshotter.getLatestSnapshot();+ // Sync and use the same lens that was sent to the server+ const { lens } = await this.sync();// Find the lens for this cellconst cellId = CellDocumentUri.parse(cellDocumentUri);@@ -345,9 +369,8 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {return null;}- // This LSP method has no version, so lets sync and then get the latest snapshot- await this.sync();- const { lens } = this.snapshotter.getLatestSnapshot();+ // Sync and use the same lens that was sent to the server+ const { lens } = await this.sync();const cellId = CellDocumentUri.parse(cellDocumentUri);@@ -367,6 +390,23 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {return response;}+ /**+ * Code actions are currently disabled because mapping code action edits+ * back to individual cells is complex and error-prone.+ *+ * The LSP server returns edits in merged document coordinates, but we need+ * to apply them to individual cell editors. Unlike simple position transforms+ * (hover, completion), code actions can include arbitrary text edits that+ * may span multiple cells or change line counts, making the reverse mapping+ * unreliable.+ *+ * To enable this, we would need to:+ * 1. Transform edit ranges from merged doc coordinates to cell coordinates+ * 2. Handle edits that span cell boundaries (split or reject them)+ * 3. Apply edits atomically across multiple cell editors+ *+ * See textDocumentRename for a similar workaround that manually applies edits.+ */textDocumentCodeAction(params: LSP.CodeActionParams,): Promise<(LSP.Command | LSP.CodeAction)[] | null> {@@ -378,12 +418,23 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {}/**- * HACK- * This whole function is a hack to work around the fact that we don't have- * a great way to map the edits back to the appropriate LSP view plugin.+ * Rename implementation with manual edit application.+ *+ * This is a workaround because the standard LSP rename flow doesn't work well+ * with our notebook architecture. The LSP server returns a WorkspaceEdit with+ * edits in merged document coordinates, but CodeMirror's LSP plugin expects+ * edits for individual cell documents.+ *+ * Instead of trying to transform the WorkspaceEdit (which is complex because+ * edits can span cells, change line counts, etc.), we:+ * 1. Request the rename from the LSP server+ * 2. Extract the new merged text from the response+ * 3. Split it back into per-cell text using the lens+ * 4. Manually update each cell's editor*- * Instead, we parse out the new text from the response and then update the- * code in the plugins manually, instead of using the LSP.+ * This approach is simpler and more reliable, though it bypasses the normal+ * LSP edit application flow. The trade-off is that we lose undo grouping+ * across cells.*/async textDocumentRename(params: LSP.RenameParams,@@ -397,9 +448,8 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {const cellId = CellDocumentUri.parse(cellDocumentUri);- // This LSP method has no version, so lets sync and then get the latest snapshot- await this.sync();- const { lens } = this.snapshotter.getLatestSnapshot();+ // Sync and use the same lens that was sent to the server+ const { lens } = await this.sync();const transformedPosition = lens.transformPosition(params.position, cellId);@@ -468,6 +518,9 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {}}+ const failedCells: CellId[] = [];+ let updatedCount = 0;+for (const [currentCellId, ev] of Objects.entries(editors)) {// For private variable renames, only update the originating cellif (isPrivateRename && currentCellId !== cellId) {@@ -476,12 +529,17 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {const newCode = editsToNewCode.get(currentCellId);if (newCode == null) {- Logger.warn("No new code for cell", currentCellId);+ Logger.warn("[lsp] No new code for cell during rename", currentCellId);+ failedCells.push(currentCellId);continue;}if (!ev) {- Logger.warn("No view for plugin", currentCellId);+ Logger.warn(+ "[lsp] No editor view for cell during rename",+ currentCellId,+ );+ failedCells.push(currentCellId);continue;}@@ -489,9 +547,19 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {const code = getEditorCodeAsPython(ev);if (code !== newCode) {updateEditorCodeFromPython(ev, newCode);+ updatedCount++;}}+ if (failedCells.length > 0) {+ Logger.error(+ `[lsp] Rename partially failed: could not update ${failedCells.length} cell(s)`,+ failedCells,+ );+ }++ Logger.debug(`[lsp] Rename completed: updated ${updatedCount} cell(s)`);+return {...response,documentChanges: [@@ -531,9 +599,8 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {return null;}- // This LSP method has no version, so lets sync and then get the latest snapshot- await this.sync();- const { lens } = this.snapshotter.getLatestSnapshot();+ // Sync and use the same lens that was sent to the server+ const { lens } = await this.sync();const cellId = CellDocumentUri.parse(cellDocumentUri);@@ -549,9 +616,8 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {public async textDocumentHover(params: LSP.HoverParams) {Logger.debug("[lsp] textDocumentHover", params);- // This LSP method has no version, so lets sync and then get the latest snapshot- await this.sync();- const { lens } = this.snapshotter.getLatestSnapshot();+ // Sync and use the same lens that was sent to the server+ const { lens } = await this.sync();const cellId = CellDocumentUri.parse(params.textDocument.uri);@@ -655,6 +721,10 @@ export class NotebookLanguageServerClient implements ILanguageServerClient {}}+ // Prune any cell URIs for cells that no longer exist+ const currentCellIds = new Set(lens.cellIds);+ NotebookLanguageServerClient.pruneSeenCellUris(currentCellIds);+const cellsToClear = new Set(NotebookLanguageServerClient.SEEN_CELL_DOCUMENT_URIS,);diff --git a/marimo/_server/lsp.py b/marimo/_server/lsp.pyindex 66752e64dc..5e1f4d4efb 100644--- a/marimo/_server/lsp.py+++ b/marimo/_server/lsp.py@@ -44,10 +44,17 @@ class BaseLspServer(LspServer):self.process: Optional[subprocess.Popen[str]] = Noneself._health_check_task: Optional[asyncio.Task[None]] = Noneself._startup_failed = False+ self._start_lock = asyncio.Lock()self.log_file = _loggers.get_log_directory() / f"{self.id}.log"@server_tracer.start_as_current_span("lsp_server.start")async def start(self) -> Optional[AlertNotification]:+ # Use lock to prevent race conditions when start() is called concurrently+ # (e.g., user rapidly toggles LSP settings)+ async with self._start_lock:+ return await self._start_internal()++ async def _start_internal(self) -> Optional[AlertNotification]:if self.process is not None:LOGGER.debug("LSP server already started")return None@@ -194,9 +201,23 @@ class BaseLspServer(LspServer):self._health_check_task.cancel()if self.process is not None:+ LOGGER.debug("Stopping LSP server at port %s", self.port)self.process.terminate()+ try:+ # Wait for graceful shutdown with timeout+ self.process.wait(timeout=5)+ LOGGER.debug("LSP server stopped gracefully")+ except subprocess.TimeoutExpired:+ # Force kill if process doesn't respond to terminate+ LOGGER.warning(+ "LSP server did not stop gracefully, forcing kill"+ )+ self.process.kill()+ try:+ self.process.wait(timeout=2)+ except subprocess.TimeoutExpired:+ LOGGER.error("Failed to kill LSP server process")self.process = None- LOGGER.debug("Stopped LSP server at port %s", self.port)else:LOGGER.debug("LSP server not running")diff --git a/tests/_server/test_lsp.py b/tests/_server/test_lsp.pyindex 66b00454ed..8cebdcc899 100644--- a/tests/_server/test_lsp.py+++ b/tests/_server/test_lsp.py@@ -92,9 +92,65 @@ async def test_base_lsp_server_start_stop(# Test stopserver.stop()mock_process.terminate.assert_called_once()+ # Verify wait was called with timeout for graceful shutdown+ mock_process.wait.assert_called_once_with(timeout=5)assert server.process is None+async def test_base_lsp_server_stop_force_kill_on_timeout(+ mock_popen: mock.MagicMock,+ mock_process: mock.MagicMock,+):+ """Test that stop() force kills the process if it doesn't respond to terminate."""+ del mock_popen+ import subprocess++ server = MockLspServer(port=8000)+ server.validate_requirements = mock.MagicMock(return_value=True)++ # Start the server+ await server.start()++ # Make wait() raise TimeoutExpired to simulate hung process+ mock_process.wait.side_effect = [+ subprocess.TimeoutExpired("cmd", 5), # First wait (terminate)+ None, # Second wait (after kill)+ ]++ # Test stop with force kill+ server.stop()++ mock_process.terminate.assert_called_once()+ mock_process.kill.assert_called_once()+ # wait should be called twice: once after terminate, once after kill+ assert mock_process.wait.call_count == 2+ assert server.process is None+++async def test_base_lsp_server_start_lock_prevents_concurrent_starts(+ mock_popen: mock.MagicMock,+ mock_process: mock.MagicMock,+):+ """Test that start() uses a lock to prevent race conditions."""+ del mock_process+ import asyncio++ server = MockLspServer(port=8000)+ server.validate_requirements = mock.MagicMock(return_value=True)++ # Call start() multiple times concurrently+ results = await asyncio.gather(+ server.start(),+ server.start(),+ server.start(),+ )++ # All should succeed (return None), but Popen should only be called once+ assert all(r is None for r in results)+ # Only one Popen call due to the lock+ mock_popen.assert_called_once()++async def test_base_lsp_server_missing_binary(mock_which: mock.MagicMock):server = MockLspServer(port=8000)mock_which.return_value = None