Skip to content

Build your own AI office with Univer Office SDK. Bring editing, calculation, collaboration, and file processing into your product for people and AI agents. Built on an extensible open-source core.

fix(formula): update inverted index cache (#6145)

Univer397ff9732b96
diff --git a/packages/engine-formula/src/basics/inverted-index-cache.ts b/packages/engine-formula/src/basics/inverted-index-cache.ts
index 418319278f..759f7b5c1e 100644
--- a/packages/engine-formula/src/basics/inverted-index-cache.ts
+++ b/packages/engine-formula/src/basics/inverted-index-cache.ts
@@ -17,6 +17,9 @@
import type { NumericTuple } from '@flatten-js/interval-tree';
import IntervalTree from '@flatten-js/interval-tree';
+// empty cell, empty string and null value share the same inverted index cache key
+export const DEFAULT_EMPTY_CELL_KEY = Symbol('EMPTY_CELL');
+
export class InvertedIndexCache {
/**
* {
@@ -30,11 +33,11 @@ export class InvertedIndexCache {
* }
* }
*/
- private _cache: Map<string, Map<string, Map<number, Map<string | number | boolean | null, Set<number>>>>> = new Map();
+ private _cache: Map<string, Map<string, Map<number, Map<string | number | boolean | null | symbol, Set<number>>>>> = new Map();
private _continueBuildingCache: Map<string, Map<string, Map<number, IntervalTree<NumericTuple>>>> = new Map();
- set(unitId: string, sheetId: string, column: number, value: string | number | boolean | null, row: number, isForceUpdate: boolean = false) {
+ set(unitId: string, sheetId: string, column: number, value: string | number | boolean | null | symbol, row: number, isForceUpdate: boolean = false) {
if (!this.shouldContinueBuildingCache(unitId, sheetId, column, row) && !isForceUpdate) {
return;
}
@@ -68,7 +71,10 @@ export class InvertedIndexCache {
}
// Because the inverted index cache is used for compare operation, it should be case-insensitive.
- const _value = typeof value === 'string' ? value.toLowerCase() : value;
+ let _value = typeof value === 'string' ? value.toLowerCase() : value;
+ if (_value === '' || _value === null) {
+ _value = DEFAULT_EMPTY_CELL_KEY;
+ }
let cellList = columnMap.get(_value);
if (cellList == null) {
@@ -83,9 +89,12 @@ export class InvertedIndexCache {
return this._cache.get(unitId)?.get(sheetId)?.get(column);
}
- getCellPositions(unitId: string, sheetId: string, column: number, value: string | number | boolean, rowsInCache: NumericTuple[]) {
+ getCellPositions(unitId: string, sheetId: string, column: number, value: string | number | boolean | null | symbol, rowsInCache: NumericTuple[]) {
// Because the inverted index cache is used for compare operation, it should be case-insensitive.
- const _value = typeof value === 'string' ? value.toLowerCase() : value;
+ let _value = typeof value === 'string' ? value.toLowerCase() : value;
+ if (_value === '' || _value === null) {
+ _value = DEFAULT_EMPTY_CELL_KEY;
+ }
const rows = this._cache.get(unitId)?.get(sheetId)?.get(column)?.get(_value);
return rows && [...rows].filter((row) => rowsInCache.some(([start, end]) => row >= start && row <= end));
}
diff --git a/packages/engine-formula/src/engine/value-object/array-value-object.ts b/packages/engine-formula/src/engine/value-object/array-value-object.ts
index 4822fcc8e4..ac5d41df09 100644
--- a/packages/engine-formula/src/engine/value-object/array-value-object.ts
+++ b/packages/engine-formula/src/engine/value-object/array-value-object.ts
@@ -20,7 +20,7 @@ import type { callbackMapFnType, IArrayValueObject } from './base-value-object';
import { isRealNum } from '@univerjs/core';
import { BooleanValue } from '../../basics/common';
import { ERROR_TYPE_SET, ErrorType } from '../../basics/error-type';
-import { CELL_INVERTED_INDEX_CACHE } from '../../basics/inverted-index-cache';
+import { CELL_INVERTED_INDEX_CACHE, DEFAULT_EMPTY_CELL_KEY } from '../../basics/inverted-index-cache';
import { regexTestArrayValue } from '../../basics/regex';
import { compareToken } from '../../basics/token';
import { ArrayBinarySearchType, ArrayOrderSearchType, getCompare } from '../utils/compare';
@@ -1525,7 +1525,7 @@ export class ArrayValueObject extends BaseValueObject {
unitId,
sheetId,
column + startColumn,
- valueObject.getValue(),
+ valueObject.isNull() ? null : valueObject.getValue(),
rowsInCache
);
@@ -1722,27 +1722,12 @@ export class ArrayValueObject extends BaseValueObject {
result[r][column] = ErrorValueObject.create(ErrorType.NA);
}
- /**
- * Inverted indexing enhances matching performance.
- */
- if (currentValue == null) {
- return;
- }
-
- /**
- * The blank cell are not stored in the inverted index, so skip it.
- * If needed store it in the future. ask @DR-Univer
- */
- if (currentValue.isNull()) {
- return;
- }
-
- if (currentValue.isError()) {
+ if (!currentValue || currentValue?.isNull()) {
CELL_INVERTED_INDEX_CACHE.set(
unitId,
sheetId,
column + startColumn,
- (currentValue as ErrorValueObject).getErrorType(),
+ DEFAULT_EMPTY_CELL_KEY,
r + startRow
);
} else {
diff --git a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache.spec.ts b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache.spec.ts
index 628d563a99..d7e55c70f0 100644
--- a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache.spec.ts
+++ b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache.spec.ts
@@ -219,7 +219,7 @@ const getFunctionsTestWorkbookData = (): IWorkbookData => {
};
};
-describe('Test inverted index cache', () => {
+describe('Test inverted index cache 1', () => {
let get: Injector['get'];
let worksheet: Worksheet;
let formulaEngine: FFormula;
diff --git a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache2.spec.ts b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache2.spec.ts
index f3fc77d402..560ec1e10b 100644
--- a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache2.spec.ts
+++ b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache2.spec.ts
@@ -159,7 +159,7 @@ const getFunctionsTestWorkbookData = (): IWorkbookData => {
};
};
-describe('Test inverted index cache', () => {
+describe('Test inverted index cache 2', () => {
let get: Injector['get'];
let lexer: Lexer;
let astTreeBuilder: AstTreeBuilder;
diff --git a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache3.spec.ts b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache3.spec.ts
index 2b331d5997..702af340f4 100644
--- a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache3.spec.ts
+++ b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache3.spec.ts
@@ -178,7 +178,7 @@ const getFunctionsTestWorkbookData = (): IWorkbookData => {
};
};
-describe('Test inverted index cache', () => {
+describe('Test inverted index cache 3', () => {
let get: Injector['get'];
let worksheet: Worksheet;
let formulaEngine: FFormula;
diff --git a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache4.spec.ts b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache4.spec.ts
index 55ece5eb9e..52ab27754d 100644
--- a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache4.spec.ts
+++ b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache4.spec.ts
@@ -123,7 +123,7 @@ const getFunctionsTestWorkbookData = (): IWorkbookData => {
};
};
-describe('Test inverted index cache', () => {
+describe('Test inverted index cache 4', () => {
let get: Injector['get'];
let worksheet: Worksheet;
let formulaEngine: FFormula;
diff --git a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache5.spec.ts b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache5.spec.ts
index ba414d2f6e..4e4fc8a021 100644
--- a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache5.spec.ts
+++ b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache5.spec.ts
@@ -1773,7 +1773,7 @@ const getFunctionsTestWorkbookData = (): IWorkbookData => {
};
};
-describe('Test inverted index cache', () => {
+describe('Test inverted index cache 5', () => {
let get: Injector['get'];
let worksheet: Worksheet;
let formulaEngine: FFormula;
diff --git a/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache6.spec.ts b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache6.spec.ts
new file mode 100644
index 0000000000..811a604a6d
--- /dev/null
+++ b/packages/sheets/src/controllers/__tests__/formula/array-inverted-index-cache6.spec.ts
@@ -0,0 +1,468 @@
+/**
+ * Copyright 2023-present DreamNum Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { CellValue, CellValueType, Ctor, Injector, IWorkbookData, Nullable, Worksheet } from '@univerjs/core';
+import type { BaseFunction, IFunctionNames } from '@univerjs/engine-formula';
+import type { FFormula } from '@univerjs/engine-formula/facade';
+import { ICommandService, LocaleType } from '@univerjs/core';
+import { FormulaDataModel, functionMeta, IFormulaCurrentConfigService, IFormulaRuntimeService, IFunctionService, SetArrayFormulaDataMutation, SetFormulaCalculationNotificationMutation, SetFormulaCalculationResultMutation, SetFormulaCalculationStartMutation, SetFormulaCalculationStopMutation } from '@univerjs/engine-formula';
+import { beforeEach, describe, expect, it } from 'vitest';
+import { SetRangeValuesMutation } from '../../../commands/mutations/set-range-values.mutation';
+import { createFunctionTestBed } from './create-function-test-bed';
+
+import '@univerjs/engine-formula/facade';
+
+const unitId = 'test';
+const subUnitId = 'sheet1';
+
+const getFunctionsTestWorkbookData = (): IWorkbookData => {
+ return {
+ id: unitId,
+ appVersion: '3.0.0-alpha',
+ styles: {
+ R4: {
+ n: {
+ pattern: '[$-409]mmm\\-yy;@',
+ },
+ },
+ R5: {
+ n: {
+ pattern: 'YYYY-M-D',
+ },
+ },
+ },
+ sheets: {
+ sheet1: {
+ id: subUnitId,
+ cellData: {
+ 0: {
+ 0: {
+ v: 'Hire Date',
+ t: 1,
+ },
+ 1: {
+ v: 'Term Date',
+ t: 1,
+ },
+ 3: {
+ v: 'Month 2016',
+ t: 1,
+ },
+ 4: {
+ v: '42370',
+ t: 2,
+ s: 'R4',
+ },
+ 5: {
+ v: '42401',
+ t: 2,
+ s: 'R4',
+ },
+ 6: {
+ v: '42430',
+ t: 2,
+ s: 'R4',
+ },
+ 7: {
+ v: '42461',
+ t: 2,
+ s: 'R4',
+ },
+ 8: {
+ v: '42491',
+ t: 2,
+ s: 'R4',
+ },
+ 9: {
+ v: '42522',
+ t: 2,
+ s: 'R4',
+ },
+ 10: {
+ v: '42552',
+ t: 2,
+ s: 'R4',
+ },
+ 11: {
+ v: '42583',
+ t: 2,
+ s: 'R4',
+ },
+ 12: {
+ v: '42614',
+ t: 2,
+ s: 'R4',
+ },
+ 13: {
+ v: '42644',
+ t: 2,
+ s: 'R4',
+ },
+ 14: {
+ v: '42675',
+ t: 2,
+ s: 'R4',
+ },
+ 15: {
+ v: '42705',
+ t: 2,
+ s: 'R4',
+ },
+ },
+ 1: {
+ 0: {
+ v: '40514',
+ t: 2,
+ s: 'R5',
+ },
+ 3: {
+ v: 'Current',
+ t: 1,
+ },
+ },
+ 2: {
+ 0: {
+ v: '40665',
+ t: 2,
+ s: 'R5',
+ },
+ 3: {
+ v: 'Term/Resign/Trans',
+ t: 1,
+ },
+ },
+ 3: {
+ 0: {
+ v: '40721',
+ t: 2,
+ s: 'R5',
+ },
+ 3: {
+ v: 'Budget',
+ t: 1,
+ },
+ },
+ 4: {
+ 0: {
+ v: '40931',
+ t: 2,
+ s: 'R5',
+ },
+ 3: {
+ v: 'Difference',
+ t: 1,
+ },
+ },
+ 5: {
+ 0: {
+ v: '41110',
+ t: 2,
+ s: 'R5',
+ },
+ 1: {
+ v: '42370',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 6: {
+ 0: {
+ v: '41298',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 7: {
+ 0: {
+ v: '41437',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 8: {
+ 0: {
+ v: '41652',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 9: {
+ 0: {
+ v: '41652',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 10: {
+ 0: {
+ v: '41786',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 11: {
+ 0: {
+ v: '41799',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 12: {
+ 0: {
+ v: '41884',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 13: {
+ 0: {
+ v: '41981',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 14: {
+ 0: {
+ v: '42100',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 15: {
+ 0: {
+ v: '42114',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 16: {
+ 0: {
+ v: '42142',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 17: {
+ 0: {
+ v: '42142',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 18: {
+ 0: {
+ v: '42156',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 19: {
+ 0: {
+ v: '42163',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 20: {
+ 0: {
+ v: '42226',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 21: {
+ 0: {
+ v: '42296',
+ t: 2,
+ s: 'R5',
+ },
+ },
+ 25: {
+ 5: {
+ f: '=$B$2:$B$22>F$1',
+ },
+ 9: {
+ f: '=$B$2:$B$22=""',
+ },
+ },
+ },
+ },
+ },
+ locale: LocaleType.ZH_CN,
+ name: '',
+ sheetOrder: [],
+ };
+};
+
+describe('Test inverted index cache 6', () => {
+ let get: Injector['get'];
+ let worksheet: Worksheet;
+ let formulaEngine: FFormula;
+ let commandService: ICommandService;
+ let formulaDataModel: FormulaDataModel;
+ let getArrayFormulaResult: (row: number, column: number) => Array<Array<{ v: Nullable<CellValue>; t: CellValueType } | null>> | null;
+
+ beforeEach(async () => {
+ const testBed = createFunctionTestBed(getFunctionsTestWorkbookData());
+
+ get = testBed.get;
+ worksheet = testBed.sheet.getSheetBySheetId(subUnitId) as Worksheet;
+ formulaEngine = testBed.api.getFormula() as FFormula;
+ formulaDataModel = get(FormulaDataModel);
+
+ commandService = get(ICommandService);
+
+ commandService.registerCommand(SetFormulaCalculationStartMutation);
+ commandService.registerCommand(SetFormulaCalculationStopMutation);
+ commandService.registerCommand(SetFormulaCalculationResultMutation);
+ commandService.registerCommand(SetFormulaCalculationNotificationMutation);
+ commandService.registerCommand(SetArrayFormulaDataMutation);
+ commandService.registerCommand(SetRangeValuesMutation);
+
+ const functionService = get(IFunctionService);
+
+ const formulaCurrentConfigService = get(IFormulaCurrentConfigService);
+
+ const formulaRuntimeService = get(IFormulaRuntimeService);
+
+ formulaCurrentConfigService.load({
+ formulaData: {},
+ arrayFormulaCellData: {},
+ arrayFormulaRange: {},
+ forceCalculate: false,
+ dirtyRanges: [],
+ dirtyNameMap: {},
+ dirtyDefinedNameMap: {},
+ dirtyUnitFeatureMap: {},
+ dirtyUnitOtherFormulaMap: {},
+ excludedCell: {},
+ allUnitData: {
+ [testBed.unitId]: testBed.sheetData,
+ },
+ });
+
+ const sheetItem = testBed.sheetData[testBed.sheetId];
+
+ formulaRuntimeService.setCurrent(
+ 0,
+ 0,
+ sheetItem.rowCount,
+ sheetItem.columnCount,
+ testBed.sheetId,
+ testBed.unitId
+ );
+
+ const functions = [
+ ...functionMeta,
+ ]
+ .map((registerObject) => {
+ const Func = registerObject[0] as Ctor<BaseFunction>;
+ const name = registerObject[1] as IFunctionNames;
+
+ return new Func(name);
+ });
+
+ functionService.registerExecutors(
+ ...functions
+ );
+
+ formulaEngine.executeCalculation();
+ await formulaEngine.onCalculationEnd();
+
+ getArrayFormulaResult = (row: number, column: number) => {
+ const arrayFormulaRange = formulaDataModel.getArrayFormulaRange();
+ const rangeInfo = arrayFormulaRange?.[unitId]?.[subUnitId]?.[row]?.[column];
+ const arrayFormulaCellData = formulaDataModel.getArrayFormulaCellData();
+ const valueMatrix = arrayFormulaCellData?.[unitId]?.[subUnitId];
+
+ if (!rangeInfo || !valueMatrix) {
+ return null;
+ }
+
+ const { startRow, endRow, startColumn, endColumn } = rangeInfo;
+ const result: Array<Array<{ v: Nullable<CellValue>; t: CellValueType } | null>> = [];
+
+ for (let r = startRow; r <= endRow; r++) {
+ const rowValues: Array<{ v: Nullable<CellValue>; t: CellValueType } | null> = [];
+ for (let c = startColumn; c <= endColumn; c++) {
+ const cellValue = valueMatrix?.[r]?.[c];
+ if (!cellValue) {
+ rowValues.push(null);
+ } else {
+ rowValues.push({
+ v: cellValue?.v ?? null,
+ t: cellValue?.t ?? 2,
+ });
+ }
+ }
+ result.push(rowValues);
+ }
+
+ return result;
+ };
+ });
+
+ describe('Test formula', () => {
+ it('Test blank cell compare', () => {
+ expect(getArrayFormulaResult(25, 5)).toStrictEqual([
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 0, t: 3 }],
+ ]);
+ expect(getArrayFormulaResult(25, 9)).toStrictEqual([
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 0, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ [{ v: 1, t: 3 }],
+ ]);
+ });
+ });
+});