Build your own AI office with Univer Office SDK. Bring editing, calculation, collaboration, and file processing into your product for people and AI agents. Built on an extensible open-source core.
TypeScript boardcollaborationdocdocxexceloffce365officepdfpptpptxpresentationsdksheetslidesspreadsheetuniverwordwordprocessorworkspacexlsx
fix(formula): fix CELL_INVERTED_INDEX_CACHE range set (#3807)
wpxp1234561092fcb0cced
diff --git a/packages/engine-formula/package.json b/packages/engine-formula/package.jsonindex 88f580e3a9..4a9af77c3c 100644--- a/packages/engine-formula/package.json+++ b/packages/engine-formula/package.json@@ -62,6 +62,7 @@"rxjs": ">=7.0.0"},"dependencies": {+ "@flatten-js/interval-tree": "^1.1.3","@univerjs/core": "workspace:*","@univerjs/rpc": "workspace:*","decimal.js": "^10.4.3",diff --git a/packages/engine-formula/src/basics/inverted-index-cache.ts b/packages/engine-formula/src/basics/inverted-index-cache.tsindex d0a570350d..49adcec26f 100644--- a/packages/engine-formula/src/basics/inverted-index-cache.ts+++ b/packages/engine-formula/src/basics/inverted-index-cache.ts@@ -14,6 +14,9 @@* limitations under the License.*/+import type { NumericTuple } from '@flatten-js/interval-tree';+import IntervalTree from '@flatten-js/interval-tree';+export class InvertedIndexCache {/*** {@@ -29,8 +32,7 @@ export class InvertedIndexCache {*/private _cache: Map<string, Map<string, Map<number, Map<string | number | boolean | null, Set<number>>>>> = new Map();- private _continueBuildingCache: Map<string, Map<string, Map<number, { startRow: number; endRow: number }>>> =- new Map();+ private _continueBuildingCache: Map<string, Map<string, Map<number, IntervalTree<NumericTuple>>>> = new Map();set(unitId: string, sheetId: string, column: number, value: string | number | boolean | null, row: number) {if (!this.shouldContinueBuildingCache(unitId, sheetId, column, row)) {@@ -68,31 +70,17 @@ export class InvertedIndexCache {return this._cache.get(unitId)?.get(sheetId)?.get(column);}- getCellPositions(unitId: string, sheetId: string, column: number, value: string | number | boolean) {- return this._cache.get(unitId)?.get(sheetId)?.get(column)?.get(value);+ getCellPositions(unitId: string, sheetId: string, column: number, value: string | number | boolean, rowsInCache: NumericTuple[]) {+ const rows = this._cache.get(unitId)?.get(sheetId)?.get(column)?.get(value);+ // return rows?.values().filter((row) => rowsInCache.some(([start, end]) => row >= start && row <= end));+ return rows && [...rows].filter((row) => rowsInCache.some(([start, end]) => row >= start && row <= end));}- getCellPosition(- unitId: string,- sheetId: string,- column: number,- value: string | number | boolean,- startRow: number,- endRow: number- ) {- const rows = this.getCellPositions(unitId, sheetId, column, value);- if (rows == null) {+ setContinueBuildingCache(unitId: string, sheetId: string, column: number, startRow: number, endRow: number) {+ if (column === -1 || startRow === -1 || endRow === -1) {return;}- for (const row of rows) {- if (row >= startRow && row <= endRow) {- return row;- }- }- }-- setContinueBuildingCache(unitId: string, sheetId: string, column: number, startRow: number, endRow: number) {let unitMap = this._continueBuildingCache.get(unitId);if (unitMap == null) {unitMap = new Map();@@ -107,53 +95,117 @@ export class InvertedIndexCache {let columnMap = sheetMap.get(column);if (columnMap == null) {- columnMap = { startRow, endRow };+ columnMap = new IntervalTree<NumericTuple>();+ columnMap.insert([startRow, endRow]);sheetMap.set(column, columnMap);return;}- columnMap.startRow = Math.min(columnMap.startRow, startRow);-- columnMap.endRow = Math.max(columnMap.endRow, endRow);+ this._handleNewInterval(columnMap, startRow, endRow);}shouldContinueBuildingCache(unitId: string, sheetId: string, column: number, row: number) {- const rowRange = this._continueBuildingCache.get(unitId)?.get(sheetId)?.get(column);- if (rowRange == null) {+ const columnMap = this._continueBuildingCache.get(unitId)?.get(sheetId)?.get(column);++ if (!columnMap) {return true;}- const { startRow, endRow } = rowRange;-- if (row >= startRow && row <= endRow) {- return false;- }+ const result = columnMap.search([row, row]);- return true;+ return result.length === 0;}canUseCache(unitId: string, sheetId: string, column: number, rangeStartRow: number, rangeEndRow: number) {- if (column === -1 || rangeStartRow === -1 || rangeEndRow === -1) {- return false;+ const columnMap = this._continueBuildingCache.get(unitId)?.get(sheetId)?.get(column);++ if (column === -1 || rangeStartRow === -1 || rangeEndRow === -1 || !columnMap) {+ return {+ rowsInCache: [],+ rowsNotInCache: [],+ };}- const rowRange = this._continueBuildingCache.get(unitId)?.get(sheetId)?.get(column);- if (rowRange == null) {- return false;++ const result = columnMap.search([rangeStartRow, rangeEndRow]);++ if (result.length === 0) {+ return {+ rowsInCache: [],+ rowsNotInCache: [],+ };}- const { startRow, endRow } = rowRange;+ result.sort((a, b) => a[0] - b[0]);- if (!(rangeStartRow > endRow || rangeEndRow < startRow)) {- return true;+ const rowsInCache: NumericTuple[] = [];+ const rowsNotInCache: NumericTuple[] = [];++ let _rangeStartRow = rangeStartRow;++ for (const interval of result) {+ const [start, end] = interval;++ if (_rangeStartRow >= start) {+ if (rangeEndRow <= end) {+ rowsInCache.push([_rangeStartRow, rangeEndRow]);+ break;+ }++ rowsInCache.push([_rangeStartRow, end]);+ _rangeStartRow = end + 1;+ } else {+ if (rangeEndRow > end) {+ rowsInCache.push([start, end]);+ rowsNotInCache.push([_rangeStartRow, start - 1]);+ _rangeStartRow = end + 1;+ continue;+ }++ rowsInCache.push([start, rangeEndRow]);+ rowsNotInCache.push([_rangeStartRow, start - 1]);+ }}- return false;+ return {+ rowsInCache,+ rowsNotInCache,+ };}clear() {this._cache.clear();this._continueBuildingCache.clear();}++ private _handleNewInterval(columnMap: IntervalTree<NumericTuple>, startRow: number, endRow: number) {+ let result = columnMap.search([startRow, endRow]);++ // the range is not overlapping with any existing range+ if (result.length === 0) {+ // check if the range is adjacent to any existing range+ const adjacentRange: NumericTuple = [startRow - 1 < 0 ? 0 : startRow - 1, endRow + 1];++ result = columnMap.search(adjacentRange);++ // the range is not overlapping or adjacent to any existing range, then insert it+ if (result.length === 0) {+ columnMap.insert([startRow, endRow]);+ return;+ }+ }++ // merge overlapping or adjacent ranges+ let min = startRow;+ let max = endRow;++ for (const interval of result) {+ min = Math.min(min, interval[0]);+ max = Math.max(max, interval[1]);+ columnMap.remove(interval);+ }++ columnMap.insert([min, max]);+ }}export const CELL_INVERTED_INDEX_CACHE = new InvertedIndexCache();diff --git a/packages/engine-formula/src/engine/utils/numfmt-kit.ts b/packages/engine-formula/src/engine/utils/numfmt-kit.tsindex b54e96de19..cf230a33fe 100644--- a/packages/engine-formula/src/engine/utils/numfmt-kit.ts+++ b/packages/engine-formula/src/engine/utils/numfmt-kit.ts@@ -356,46 +356,19 @@ export function stringIsNumberPattern(input: string) {const numberPattern = numfmt.parseNumber(_input);if (numberPattern) {- stringToNumberPatternCache.set(_input, {- value: numberPattern.v as number,- pattern: numberPattern.z as string,- });-- return {- isNumberPattern: true,- value: numberPattern.v as number,- pattern: numberPattern.z as string,- };+ return setNumberPatternCache(_input, numberPattern.v as number, numberPattern.z as string);}const datePattern = numfmt.parseDate(_input);if (datePattern) {- stringToNumberPatternCache.set(_input, {- value: datePattern.v as number,- pattern: datePattern.z as string,- });-- return {- isNumberPattern: true,- value: datePattern.v as number,- pattern: datePattern.z as string,- };+ return setNumberPatternCache(_input, datePattern.v as number, datePattern.z as string);}const timePattern = numfmt.parseTime(_input);if (timePattern) {- stringToNumberPatternCache.set(_input, {- value: timePattern.v as number,- pattern: timePattern.z as string,- });-- return {- isNumberPattern: true,- value: timePattern.v as number,- pattern: timePattern.z as string,- };+ return setNumberPatternCache(_input, timePattern.v as number, timePattern.z as string);}return {@@ -403,6 +376,19 @@ export function stringIsNumberPattern(input: string) {};}+function setNumberPatternCache(input: string, value: number, pattern: string) {+ stringToNumberPatternCache.set(input, {+ value,+ pattern,+ });++ return {+ isNumberPattern: true,+ value,+ pattern,+ };+}+export function clearStringToNumberPatternCache() {stringToNumberPatternCache.clear();}diff --git a/packages/engine-formula/src/engine/value-object/array-value-object.ts b/packages/engine-formula/src/engine/value-object/array-value-object.tsindex 3006077bbd..d6933d256d 100644--- a/packages/engine-formula/src/engine/value-object/array-value-object.ts+++ b/packages/engine-formula/src/engine/value-object/array-value-object.ts@@ -1478,7 +1478,7 @@ export class ArrayValueObject extends BaseValueObject {return newArray;}- // eslint-disable-next-line max-lines-per-function, complexity+ // eslint-disable-next-line max-lines-per-functionprivate _batchOperatorValue(valueObject: BaseValueObject,column: number,@@ -1489,8 +1489,6 @@ export class ArrayValueObject extends BaseValueObject {) {const rowCount = this._rowCount;- let canUseCache = false;-const unitId = this.getUnitId();const sheetId = this.getSheetId();const startRow = this.getCurrentRow();@@ -1500,7 +1498,7 @@ export class ArrayValueObject extends BaseValueObject {* then retrieve the judgment from the inverted index. This enhances performance.*/if (batchOperatorType === BatchOperatorType.COMPARE) {- canUseCache = CELL_INVERTED_INDEX_CACHE.canUseCache(+ const { rowsInCache, rowsNotInCache } = CELL_INVERTED_INDEX_CACHE.canUseCache(unitId,sheetId,column + startColumn,@@ -1508,13 +1506,15 @@ export class ArrayValueObject extends BaseValueObject {startRow + rowCount - 1);- if (canUseCache === true) {+ if (rowsInCache.length > 0) {if (operator === compareToken.EQUALS) {+ // TODO@DR-Univer: When comparing equal with two parameters, one of them is error, and the logic here is wrongconst rowPositions = CELL_INVERTED_INDEX_CACHE.getCellPositions(unitId,sheetId,column + startColumn,- valueObject.getValue()+ valueObject.getValue(),+ rowsInCache);if (rowPositions != null) {@@ -1540,7 +1540,9 @@ export class ArrayValueObject extends BaseValueObject {if (rowValuePositions != null) {rowValuePositions.forEach((rowPositions, rowValue) => {let currentValue: Nullable<BaseValueObject> = NullValueObject.create(); // handle blank cell- if (typeof rowValue === 'string') {+ if (ERROR_TYPE_SET.has(rowValue as ErrorType)) {+ currentValue = ErrorValueObject.create(rowValue as ErrorType);+ } else if (typeof rowValue === 'string') {currentValue = StringValueObject.create(rowValue);} else if (typeof rowValue === 'number') {currentValue = NumberValueObject.create(rowValue);@@ -1548,28 +1550,57 @@ export class ArrayValueObject extends BaseValueObject {currentValue = BooleanValueObject.create(rowValue);}- const matchResult = currentValue.compare(valueObject, operator as compareToken, isCaseSensitive);- if ((matchResult as BooleanValueObject).getValue() === true) {+ let matchResult;+ if (currentValue.isError()) {+ matchResult = currentValue;+ } else if (valueObject.isError()) {+ matchResult = valueObject;+ } else {+ matchResult = currentValue.compare(valueObject, operator as compareToken, isCaseSensitive);+ }++ if (matchResult.isError() || (matchResult as BooleanValueObject).getValue() === true) {rowPositions.forEach((index) => {if (index >= startRow && index <= startRow + rowCount - 1) {if (result[index - startRow] == null) {result[index - startRow] = [];}- result[index - startRow][column] = BooleanValueObject.create(true);+ result[index - startRow][column] = matchResult;}});}});+ }+ }- // for (let r = 0; r < rowCount; r++) {- // if (result[r] == null) {- // result[r] = [];- // }+ // handle the not in cache rows+ if (rowsNotInCache.length > 0) {+ for (const interval of rowsNotInCache) {+ const [start, end] = interval;++ for (let r = start; r <= end; r++) {+ this.__batchOperatorRowValue(+ valueObject,+ column,+ result,+ batchOperatorType,+ r,+ unitId,+ sheetId,+ startRow,+ startColumn,+ operator,+ isCaseSensitive+ );+ }- // if (result[r][column] == null) {- // result[r][column] = BooleanValueObject.create(false);- // }- // }+ CELL_INVERTED_INDEX_CACHE.setContinueBuildingCache(+ unitId,+ sheetId,+ column + startColumn,+ start,+ end+ );}}@@ -1578,90 +1609,19 @@ export class ArrayValueObject extends BaseValueObject {}for (let r = 0; r < rowCount; r++) {- const currentValue = this.getValueOrDefault(r, column);- if (result[r] == null) {- result[r] = [];- }- if (currentValue && valueObject) {- if (currentValue.isError()) {- result[r][column] = currentValue as ErrorValueObject;- } else if (valueObject.isError()) {- // 1 + #NAME? gets #NAME?- result[r][column] = valueObject;- } else {- switch (batchOperatorType) {- case BatchOperatorType.PLUS:- result[r][column] = currentValue.plus(valueObject);- break;- case BatchOperatorType.MINUS:- result[r][column] = currentValue.minus(valueObject);- break;- case BatchOperatorType.MULTIPLY:- result[r][column] = currentValue.multiply(valueObject);- break;- case BatchOperatorType.DIVIDED:- result[r][column] = currentValue.divided(valueObject);- break;- case BatchOperatorType.MOD:- result[r][column] = currentValue.mod(valueObject);- break;- case BatchOperatorType.COMPARE:- if (!operator) {- result[r][column] = ErrorValueObject.create(ErrorType.VALUE);- } else {- result[r][column] = currentValue.compare(valueObject, operator as compareToken, isCaseSensitive);- }- break;- case BatchOperatorType.CONCATENATE_FRONT:- result[r][column] = currentValue.concatenateFront(valueObject);- break;- case BatchOperatorType.CONCATENATE_BACK:- result[r][column] = currentValue.concatenateBack(valueObject);- break;- case BatchOperatorType.POW:- result[r][column] = currentValue.pow(valueObject);- break;- case BatchOperatorType.ROUND:- result[r][column] = currentValue.round(valueObject);- break;- case BatchOperatorType.FLOOR:- result[r][column] = currentValue.floor(valueObject);- break;- case BatchOperatorType.ATAN2:- result[r][column] = currentValue.atan2(valueObject);- break;- case BatchOperatorType.CEIL:- result[r][column] = currentValue.ceil(valueObject);- break;- }- }- } else {- result[r][column] = ErrorValueObject.create(ErrorType.NA);- }-- /**- * Inverted indexing enhances matching performance.- */- if (currentValue == null) {- continue;- }- if (currentValue.isError()) {- CELL_INVERTED_INDEX_CACHE.set(- unitId,- sheetId,- column + startColumn,- (currentValue as ErrorValueObject).getErrorType(),- r + startRow- );- } else {- CELL_INVERTED_INDEX_CACHE.set(- unitId,- sheetId,- column + startColumn,- currentValue.getValue(),- r + startRow- );- }+ this.__batchOperatorRowValue(+ valueObject,+ column,+ result,+ batchOperatorType,+ r,+ unitId,+ sheetId,+ startRow,+ startColumn,+ operator,+ isCaseSensitive+ );}CELL_INVERTED_INDEX_CACHE.setContinueBuildingCache(@@ -1673,6 +1633,109 @@ export class ArrayValueObject extends BaseValueObject {);}+ // eslint-disable-next-line+ private __batchOperatorRowValue(+ valueObject: BaseValueObject,+ column: number,+ result: BaseValueObject[][],+ batchOperatorType: BatchOperatorType,+ r: number,+ unitId: string,+ sheetId: string,+ startRow: number,+ startColumn: number,+ operator?: compareToken,+ isCaseSensitive?: boolean+ ) {+ const currentValue = this.getValueOrDefault(r, column);++ if (result[r] == null) {+ result[r] = [];+ }++ if (currentValue && valueObject) {+ if (currentValue.isError()) {+ result[r][column] = currentValue as ErrorValueObject;+ } else if (valueObject.isError()) {+ // 1 + #NAME? gets #NAME?+ result[r][column] = valueObject;+ } else {+ switch (batchOperatorType) {+ case BatchOperatorType.PLUS:+ result[r][column] = currentValue.plus(valueObject);+ break;+ case BatchOperatorType.MINUS:+ result[r][column] = currentValue.minus(valueObject);+ break;+ case BatchOperatorType.MULTIPLY:+ result[r][column] = currentValue.multiply(valueObject);+ break;+ case BatchOperatorType.DIVIDED:+ result[r][column] = currentValue.divided(valueObject);+ break;+ case BatchOperatorType.MOD:+ result[r][column] = currentValue.mod(valueObject);+ break;+ case BatchOperatorType.COMPARE:+ if (!operator) {+ result[r][column] = ErrorValueObject.create(ErrorType.VALUE);+ } else {+ result[r][column] = currentValue.compare(valueObject, operator as compareToken, isCaseSensitive);+ }+ break;+ case BatchOperatorType.CONCATENATE_FRONT:+ result[r][column] = currentValue.concatenateFront(valueObject);+ break;+ case BatchOperatorType.CONCATENATE_BACK:+ result[r][column] = currentValue.concatenateBack(valueObject);+ break;+ case BatchOperatorType.POW:+ result[r][column] = currentValue.pow(valueObject);+ break;+ case BatchOperatorType.ROUND:+ result[r][column] = currentValue.round(valueObject);+ break;+ case BatchOperatorType.FLOOR:+ result[r][column] = currentValue.floor(valueObject);+ break;+ case BatchOperatorType.ATAN2:+ result[r][column] = currentValue.atan2(valueObject);+ break;+ case BatchOperatorType.CEIL:+ result[r][column] = currentValue.ceil(valueObject);+ break;+ }+ }+ } else {+ result[r][column] = ErrorValueObject.create(ErrorType.NA);+ }++ /**+ * Inverted indexing enhances matching performance.+ */+ if (currentValue == null) {+ return;+ }++ if (currentValue.isError()) {+ CELL_INVERTED_INDEX_CACHE.set(+ unitId,+ sheetId,+ column + startColumn,+ (currentValue as ErrorValueObject).getErrorType(),+ r + startRow+ );+ } else {+ CELL_INVERTED_INDEX_CACHE.set(+ unitId,+ sheetId,+ column + startColumn,+ currentValue.getValue(),+ r + startRow+ );+ }+ }+// eslint-disable-next-line max-lines-per-function, complexityprivate _batchOperatorArray(valueObject: BaseValueObject,diff --git a/packages/engine-formula/src/functions/logical/if/index.ts b/packages/engine-formula/src/functions/logical/if/index.tsindex cc0d921cd1..043c9ec61e 100644--- a/packages/engine-formula/src/functions/logical/if/index.ts+++ b/packages/engine-formula/src/functions/logical/if/index.ts@@ -27,13 +27,13 @@ export class If extends BaseFunction {override maxParams = 3;override calculate(logicalTest: BaseValueObject, valueIfTrue: BaseValueObject, valueIfFalse: BaseValueObject = BooleanValueObject.create(false)) {- if (logicalTest.isError()) {- return logicalTest;- }-// get single value objectconst _logicalTest = this._getSingleValueObject(logicalTest);+ if (_logicalTest.isError()) {+ return _logicalTest;+ }+if (!_logicalTest.isArray()) {return _logicalTest.getValue() ? valueIfTrue : valueIfFalse;}diff --git a/packages/engine-formula/src/services/runtime.service.ts b/packages/engine-formula/src/services/runtime.service.tsindex 49d765c8e9..9be2de3a51 100644--- a/packages/engine-formula/src/services/runtime.service.ts+++ b/packages/engine-formula/src/services/runtime.service.ts@@ -28,7 +28,6 @@ import type { ArrayValueObject } from '../engine/value-object/array-value-objectimport { createIdentifier, Disposable, isNullCell, ObjectMatrix } from '@univerjs/core';import { isInDirtyRange } from '../basics/dirty';import { ErrorType } from '../basics/error-type';-import { CELL_INVERTED_INDEX_CACHE } from '../basics/inverted-index-cache';import { getRuntimeFeatureCell } from '../engine/utils/get-runtime-feature-cell';import { clearNumberFormatTypeCache, clearStringToNumberPatternCache } from '../engine/utils/numfmt-kit';@@ -514,15 +513,6 @@ export class FormulaRuntimeService extends Disposable implements IFormulaRuntimesheetData.setValue(row, column, valueObject);clearArrayUnitData.setValue(row, column, valueObject);- // Formula calculation results are saved to cache- CELL_INVERTED_INDEX_CACHE.set(- unitId,- sheetId,- column,- firstCell.getValue(),- row- );-return;}@@ -545,15 +535,6 @@ export class FormulaRuntimeService extends Disposable implements IFormulaRuntimesheetData.setValue(row, column, errorObject);clearArrayUnitData.setValue(row, column, errorObject);- // Formula calculation results are saved to cache- CELL_INVERTED_INDEX_CACHE.set(- unitId,- sheetId,- column,- ErrorType.SPILL,- row- );-/*** When there are values within the array formula range, the entire formula will result in an error.* In this case, you need to clear the previous range data to prevent other formulas from referencing the old values.@@ -581,15 +562,6 @@ export class FormulaRuntimeService extends Disposable implements IFormulaRuntime} else {const spillError = ErrorValueObject.create(ErrorType.SPILL);objectValueRefOrArray.iterator((valueObject, rowIndex, columnIndex) => {- // Formula calculation results are saved to cache- CELL_INVERTED_INDEX_CACHE.set(- unitId,- sheetId,- column - startColumn + columnIndex,- !valueObject ? 0 : valueObject.getValue(),- row - startRow + rowIndex- );-const value = objectValueToCellValue(valueObject);if (rowIndex === startRow && columnIndex === startColumn) {/**@@ -613,16 +585,6 @@ export class FormulaRuntimeService extends Disposable implements IFormulaRuntime} else {const valueObject = objectValueToCellValue(functionVariant as BaseValueObject);sheetData.setValue(row, column, valueObject);-- // Formula calculation results are saved to cache- CELL_INVERTED_INDEX_CACHE.set(- unitId,- sheetId,- column,- (functionVariant as BaseValueObject).getValue(),- row- );-clearArrayUnitData.setValue(row, column, valueObject);}}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex cd54ecd140..8be1de1e25 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1537,6 +1537,9 @@ importers:packages/engine-formula:dependencies:+ '@flatten-js/interval-tree':+ specifier: ^1.1.3+ version: 1.1.3'@univerjs/core':specifier: workspace:*version: link:../core