Skip to content

🎥 Make videos programmatically with React

Handle Recast whitespace offsets consistently

Jonny Burger93c9fd68227d
diff --git a/packages/studio-codemods/src/recast-loc-to-offset.ts b/packages/studio-codemods/src/recast-loc-to-offset.ts
index 6c7d347e09..b2680fb4f2 100644
--- a/packages/studio-codemods/src/recast-loc-to-offset.ts
+++ b/packages/studio-codemods/src/recast-loc-to-offset.ts
@@ -1,29 +1,66 @@
const RECAST_TAB_WIDTH = 4;
-// Recast removes a byte order mark, normalizes CRLF line endings, and expands
-// tabs before parsing. Absolute indexes stored in Babel's `extra` metadata
-// therefore need to be mapped back to offsets in the original source.
-export const recastIndexToOffset = (input: string, index: number): number => {
- let sourceOffset = input.startsWith('\uFEFF') ? 1 : 0;
+type RecastPosition =
+ | {index: number; type: 'index'}
+ | {column: number; line: number; type: 'location'};
+
+// Recast normalizes every line ending, removes zero-width indentation, and
+// expands tabs in leading whitespace before parsing. Walk both representations
+// together so parser positions can be mapped back to the original source.
+const recastPositionToOffset = (
+ input: string,
+ target: RecastPosition,
+): number => {
+ let sourceOffset = 0;
let recastIndex = 0;
+ let line = 1;
let column = 0;
- while (sourceOffset < input.length && recastIndex < index) {
- if (input[sourceOffset] === '\r' && input[sourceOffset + 1] === '\n') {
- sourceOffset += 2;
- recastIndex++;
- column = 0;
+ let isLeadingWhitespace = true;
+ const isAtTarget = () =>
+ target.type === 'index'
+ ? recastIndex === target.index
+ : line === target.line && column === target.column;
+
+ while (sourceOffset < input.length) {
+ const character = input[sourceOffset];
+ if (
+ isLeadingWhitespace &&
+ (character === '\v' || character === '\f' || character === '\uFEFF')
+ ) {
+ sourceOffset++;
continue;
}
- if (input[sourceOffset] === '\n' || input[sourceOffset] === '\r') {
- sourceOffset++;
+ if (isAtTarget()) {
+ return sourceOffset;
+ }
+
+ const isCrLf = character === '\r' && input[sourceOffset + 1] === '\n';
+ if (
+ isCrLf ||
+ character === '\r' ||
+ character === '\n' ||
+ character === '\u2028' ||
+ character === '\u2029'
+ ) {
+ sourceOffset += isCrLf ? 2 : 1;
recastIndex++;
+ line++;
column = 0;
+ isLeadingWhitespace = true;
continue;
}
- if (input[sourceOffset] === '\t') {
+ if (isLeadingWhitespace && character === '\t') {
const width = RECAST_TAB_WIDTH - (column % RECAST_TAB_WIDTH);
+ const targetIsInsideTab =
+ target.type === 'index'
+ ? target.index < recastIndex + width
+ : target.line === line && target.column < column + width;
+ if (targetIsInsideTab) {
+ return sourceOffset;
+ }
+
sourceOffset++;
recastIndex += width;
column += width;
@@ -33,35 +70,16 @@ export const recastIndexToOffset = (input: string, index: number): number => {
sourceOffset++;
recastIndex++;
column++;
+ isLeadingWhitespace = isLeadingWhitespace && /\s/.test(character);
}
return sourceOffset;
};
-// Recast expands tabs before parsing, so its columns cannot be used as
-// character offsets in the original source.
+export const recastIndexToOffset = (input: string, index: number): number =>
+ recastPositionToOffset(input, {index, type: 'index'});
+
export const recastLocToOffset = (
input: string,
loc: {line: number; column: number},
-): number => {
- const lines = input.split('\n');
- let offset = 0;
- for (let i = 0; i < loc.line - 1; i++) {
- offset += lines[i].length + 1;
- }
-
- const line = lines[loc.line - 1];
- let column = 0;
- const firstSourceCharacter =
- loc.line === 1 && line.startsWith('\uFEFF') ? 1 : 0;
- for (let i = firstSourceCharacter; i < line.length; i++) {
- if (column >= loc.column) {
- return offset + i;
- }
-
- column +=
- line[i] === '\t' ? RECAST_TAB_WIDTH - (column % RECAST_TAB_WIDTH) : 1;
- }
-
- return offset + line.length;
-};
+): number => recastPositionToOffset(input, {...loc, type: 'location'});
diff --git a/packages/studio-codemods/src/test/function-source-edits.test.ts b/packages/studio-codemods/src/test/function-source-edits.test.ts
index e41c0edf98..6f9569aaef 100644
--- a/packages/studio-codemods/src/test/function-source-edits.test.ts
+++ b/packages/studio-codemods/src/test/function-source-edits.test.ts
@@ -5,40 +5,56 @@ import {
captureFunctionSourceSnapshots,
getFunctionSourceEditsForPrependedStatements,
} from '../function-source-edits';
-import {recastIndexToOffset} from '../recast-loc-to-offset';
+import {recastIndexToOffset, recastLocToOffset} from '../recast-loc-to-offset';
import {parseAst} from '../sequence-props/parse-ast';
const b = recast.types.builders;
-test('maps Recast indexes back to source offsets with BOM, tabs, and CRLF', () => {
- const input = '\uFEFF\texport const C = () => (\r\n\t<div />\r\n);\r\n';
- const ast = parseAst(input);
- const declaration = ast.program.body[0];
- if (declaration.type !== 'ExportNamedDeclaration') {
- throw new Error('Expected an export declaration');
- }
+test('maps Recast positions across source whitespace normalization', () => {
+ const inputs = [
+ '\uFEFF\texport const C = () => (\r\n\t<div />\r\n);\r\n',
+ 'export const C =\t() => (<div />);\n',
+ 'export const C = () => (\u2028 <div />\u2029);\r',
+ ];
- const variableDeclaration = declaration.declaration;
- if (variableDeclaration?.type !== 'VariableDeclaration') {
- throw new Error('Expected a variable declaration');
- }
+ for (const input of inputs) {
+ const ast = parseAst(input);
+ const declaration = ast.program.body[0];
+ if (declaration.type !== 'ExportNamedDeclaration') {
+ throw new Error('Expected an export declaration');
+ }
- const initializer = variableDeclaration.declarations[0].init;
- if (initializer?.type !== 'ArrowFunctionExpression') {
- throw new Error('Expected an arrow function');
- }
+ const variableDeclaration = declaration.declaration;
+ if (variableDeclaration?.type !== 'VariableDeclaration') {
+ throw new Error('Expected a variable declaration');
+ }
- const extra = initializer.body.extra as
- | {parenStart?: number}
- | null
- | undefined;
- if (typeof extra?.parenStart !== 'number') {
- throw new Error('Expected a parenthesized expression body');
- }
+ const initializer = variableDeclaration.declarations[0].init;
+ if (
+ initializer?.type !== 'ArrowFunctionExpression' ||
+ !initializer.body.loc
+ ) {
+ throw new Error('Expected a located arrow function');
+ }
- expect(recastIndexToOffset(input, extra.parenStart)).toBe(
- input.indexOf('(', input.indexOf('=>')),
- );
+ const extra = initializer.body.extra as
+ | {parenStart?: number}
+ | null
+ | undefined;
+ if (typeof extra?.parenStart !== 'number') {
+ throw new Error('Expected a parenthesized expression body');
+ }
+
+ expect(recastIndexToOffset(input, extra.parenStart)).toBe(
+ input.indexOf('(', input.indexOf('=>')),
+ );
+ expect(recastLocToOffset(input, initializer.body.loc.start)).toBe(
+ input.indexOf('<div'),
+ );
+ expect(recastLocToOffset(input, initializer.body.loc.end)).toBe(
+ input.indexOf('/>') + 2,
+ );
+ }
});
test('rejects inserted statements outside the leading prefix', () => {