Skip to content

armgit / bento

Public Imported from GitHub
0

Bento, the office suite that fits in a file

Improve the `diff` algorithm to match related `Tokens`. (#555)

Rahul Ravikumarfe19ab9c53a9
diff --git a/scripts/test-codediff.ts b/scripts/test-codediff.ts
index dc45671a..a5215d75 100644
--- a/scripts/test-codediff.ts
+++ b/scripts/test-codediff.ts
@@ -162,5 +162,33 @@ const toksOf = (states: State[] | undefined) =>
check('call -> plain reference keeps identity', !!a && !!b && a.morphId() === b.morphId())
}
+// 10. RELATED TOKENS KEEP THEIR IDENTITY
+{
+ const beforeSnippet = `
+ fun inner() {
+ val x = 10
+ val y = 20
+ println(x + y)
+ }
+ `
+ const afterSnippet = `
+ fun outer() {
+ fun inner() {
+ val x = 10
+ val y = 20
+ println(x + y)
+ }
+ }
+ `
+ const states = new HeckelDiff(codeDoc([beforeSnippet, afterSnippet], 'kotlin')).computeDiffs('g')
+ const before = states.get(0)
+ const after = states.get(1)
+ // Filter out the fallback scope. Every other token should match.
+ const beforeIds = toksOf(before).filter((t => t.scopes.join('') !== 'x')).map(t => t.morphId())
+ const afterIds = new Set(idsOf(after))
+ const matched = beforeIds.every(id => afterIds.has(id))
+ check('related tokens should keep their identity', matched)
+}
+
console.log(FAILS.length ? `\n${FAILS.length} FAILED` : '\nall passed')
process.exit(FAILS.length ? 1 : 0)
diff --git a/slides/src/diff.ts b/slides/src/diff.ts
index 4144f46d..80f25a34 100644
--- a/slides/src/diff.ts
+++ b/slides/src/diff.ts
@@ -12,6 +12,12 @@
import type { BentoDoc, Slide } from "./model.ts"
import { tokenize, type Tok } from "../../kernel/src/tokenize.ts"
+// Tokens that usually occur in pairs are related to each other.
+// This way, if one is matched then the other is also matched automatically.
+const RELATED_TOKENS_BEGIN = new Set<string>(['{', '[', '('])
+const RELATED_TOKENS_END = new Set<string>(['}', ']', ')'])
+const RELATED_SEARCH_LIMIT = 2
+
// The stable token that we can diff between slides.
export class Token {
@@ -25,6 +31,10 @@ export class Token {
readonly scopes: Array<string>
readonly lineNumber: number
readonly offset: number
+ // Related tokens when applicable.
+ // This is only populated when the content is in the `RELATED_TOKENS_*` set.
+ _related: Token|undefined
+ _relatedIndex: number|undefined
constructor(content: string, scopes: Array<string>, lineNumber: number, offset: number) {
this.content = content
@@ -32,6 +42,8 @@ export class Token {
this.lineNumber = lineNumber
this.offset = offset
this._id = undefined
+ this._related = undefined
+ this._relatedIndex = undefined
}
morphId(): string {
@@ -57,12 +69,16 @@ export class Token {
const buckets = this.scopes.map((sc) => (sc === 'f' ? 'x' : sc))
const json = {
content: this.content,
- scopes: buckets,
- depth: this.depth()
+ scopes: buckets
}
return JSON.stringify(json)
}
+ assignRelated(related: Token, relatedIndex: number) {
+ this._related = related
+ this._relatedIndex = relatedIndex
+ }
+
/**
* Adapt one kernel token, splitting multiline tokens at newlines FIRST.
*
@@ -278,6 +294,25 @@ export class HeckelDiff {
statesP[np] = match
statesC[nc] = match
next.push(nc)
+ // Check for related tokens
+ const rp = previous[np]._relatedIndex
+ const rc = current[nc]._relatedIndex
+ if (current[nc]._related && previous[np]._related && rc !== undefined && rp !== undefined) {
+ if (statesP[rp].kind !== 'empty') continue
+ if (statesC[rc].kind !== 'empty') continue
+ const related: Match = {
+ kind: 'match',
+ previousIdx: rp,
+ previous: previous[np]._related,
+ currentIdx: rc,
+ current: current[nc]._related
+ }
+ // Propagate morph id
+ related.current.setMorphId(related.previous.morphId())
+ statesP[rp] = related
+ statesC[rc] = related
+ next.push(rc)
+ }
}
}
frontier = next
@@ -316,12 +351,38 @@ export class HeckelDiff {
* the previous add-then-delete dance re-added a key on its third occurrence,
* so any token repeating an odd number of times — a third `)` is enough —
* became a false "unique" anchor and Phase 1 paired wrong positions.
+ *
+ * We also use this opportunity to relate tokens that ought to occur in matching pairs.
+ * Punctuation tokens like `{`, `(`, and `[` have a matching `}`, `)`, and `]` respectively.
+ * Comments are always prefixed by a comment preamble, so we don't have to worry about scopes.
* Regression: scripts/test-codediff.ts.
*/
private static anchors(tokens: Token[]): Map<string, { frequency: number, index: number }> {
+ // Keep track of related tokens.
+ // Useful to track matching pairs of tokens so we can consume them cleanly.
+ const stack = new Array<{token: Token, index: number}>()
const freq = new Map<string, { frequency: number, index: number }>()
for (let i = 0; i < tokens.length; i += 1) {
- const key = tokens[i].key()
+ const token = tokens[i]
+ const key = token.key()
+ const content = token.content
+ if (RELATED_TOKENS_BEGIN.has(content)) {
+ stack.push({token: token, index: i})
+ }
+ if (RELATED_TOKENS_END.has(content)) {
+ var j = 0
+ while(j < RELATED_SEARCH_LIMIT) {
+ const begin = stack.pop()
+ if (!begin) break
+ if (HeckelDiff.areRelated(begin.token, token)) {
+ // Relate the tokens to each other.
+ begin.token.assignRelated(token, i)
+ token.assignRelated(begin.token, begin.index)
+ break
+ }
+ j += 1
+ }
+ }
const e = freq.get(key)
if (e) e.frequency += 1
else freq.set(key, { frequency: 1, index: i })
@@ -330,6 +391,18 @@ export class HeckelDiff {
return freq
}
+ private static areRelated(first: Token, second: Token): boolean {
+ // It's enough to only check for the content here.
+ // We are intentionally not making too many assumptions around the Token
+ // scopes here, to keep the Tokenizer open-ended. We are also intentionally
+ // skipping over `<` and `>` because without suitable scopes we might confuse
+ // them with comparison operators.
+ if (first.content === '{' && second.content === '}') return true
+ if (first.content === '[' && second.content === ']') return true
+ if (first.content === '(' && second.content === ')') return true
+ return false
+ }
+
private static zip(parsed: Array<Token[]>): Array<number[]> {
const pairs: Array<number[]> = []
let index = 0