Build your own AI office with Univer Office SDK. Bring editing, calculation, collaboration, and file processing into your product for people and AI agents. Built on an extensible open-source core.
TypeScript boardcollaborationdocdocxexceloffce365officepdfpptpptxpresentationsdksheetslidesspreadsheetuniverwordwordprocessorworkspacexlsx
fix(formula-engine): fix lexer consuming whitespace error (#5011)
Wenzhao Hufe617f3da9cf
diff --git a/.vscode/settings.json b/.vscode/settings.jsonindex 7e2d23ed03..0edfb1a40c 100644--- a/.vscode/settings.json+++ b/.vscode/settings.json@@ -154,6 +154,12 @@// Disable the default formatter, use eslint instead"prettier.enable": false,"search.followSymlinks": false,+ "search.exclude": {+ "**/node_modules": true,+ "**/bower_components": true,+ "**/*.code-search": true,+ "**/lib": true+ },// Auto fix"editor.formatOnSave": false,diff --git a/packages/engine-formula/src/engine/analysis/__tests__/lexer-tree-builder.spec.ts b/packages/engine-formula/src/engine/analysis/__tests__/lexer-tree-builder.spec.tsindex 5b75a3a4e6..75fce94b7f 100644--- a/packages/engine-formula/src/engine/analysis/__tests__/lexer-tree-builder.spec.ts+++ b/packages/engine-formula/src/engine/analysis/__tests__/lexer-tree-builder.spec.ts@@ -30,7 +30,7 @@ describe('lexer nodeMaker test', () => {'=lambda(x,y, x*y*x)(sum(1,(1+2)*3),2)+1-max(100,200)') as LexerNode;expect(JSON.stringify(node.serialize())).toStrictEqual(- '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"lambda","st":0,"ed":5,"children":[{"token":"L_1","st":14,"ed":16,"children":[{"token":"P_1","st":15,"ed":17,"children":[{"token":"sum","st":19,"ed":21,"children":[{"token":"P_1","st":19,"ed":21,"children":["1"]},{"token":"P_1","st":21,"ed":23,"children":["1","2","+","3","*"]}]}]},{"token":"P_1","st":30,"ed":32,"children":["2"]}]},{"token":"P_1","st":3,"ed":5,"children":["x"]},{"token":"P_1","st":5,"ed":7,"children":["y"]},{"token":"P_1","st":7,"ed":9,"children":[" x","y","*","x","*"]}]},"1","+",{"token":"max","st":39,"ed":41,"children":[{"token":"P_1","st":39,"ed":41,"children":["100"]},{"token":"P_1","st":43,"ed":45,"children":["200"]}]},"-"]}'+ '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"lambda","st":0,"ed":5,"children":[{"token":"L_1","st":14,"ed":16,"children":[{"token":"P_1","st":15,"ed":17,"children":[{"token":"sum","st":19,"ed":21,"children":[{"token":"P_1","st":19,"ed":21,"children":["1"]},{"token":"P_1","st":21,"ed":23,"children":["1","2","+","3","*"]}]}]},{"token":"P_1","st":30,"ed":32,"children":["2"]}]},{"token":"P_1","st":3,"ed":5,"children":["x"]},{"token":"P_1","st":5,"ed":7,"children":["y"]},{"token":"P_1","st":7,"ed":9,"children":["x","y","*","x","*"]}]},"1","+",{"token":"max","st":39,"ed":41,"children":[{"token":"P_1","st":39,"ed":41,"children":["100"]},{"token":"P_1","st":43,"ed":45,"children":["200"]}]},"-"]}');});@@ -39,7 +39,7 @@ describe('lexer nodeMaker test', () => {'=(-(1+2)--@A1:B2 + 5)/2 + -sum(indirect(A5):B10# + B6# + A1:offset(C5, 1, 1) , 100) + {1,2,3;4,5,6;7,8,10} + lambda(x,y,z, x*y*z)(sum(1,(1+2)*3),2,lambda(x,y, @offset(A1:B0,x#*y#))(1,2):C20) + sum((1+2%)*30%, 1+2)%') as LexerNode;expect(JSON.stringify(node.serialize())).toStrictEqual(- '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"-","st":1,"ed":1,"children":[{"token":"P_1","st":-1,"ed":1,"children":["1","2","+"]}]},{"token":"-","st":-1,"ed":-1,"children":[{"token":"@","st":-1,"ed":-1,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B2 ","st":-1,"ed":-1,"children":[]}]}]}]}]},"-"," 5","+","2 ","/",{"token":" -sum","st":24,"ed":28,"children":[{"token":"P_1","st":26,"ed":28,"children":[{"token":"#","st":-1,"ed":-1,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"indirect","st":30,"ed":37,"children":[{"token":"P_1","st":35,"ed":37,"children":["A5"]}]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B10","st":-1,"ed":-1,"children":[]}]}]}]},{"token":"#","st":-1,"ed":-1,"children":[" B6"]}," +",{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":" A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"offset","st":59,"ed":64,"children":[{"token":"P_1","st":62,"ed":64,"children":["C5"]},{"token":"P_1","st":65,"ed":67,"children":[" 1"]},{"token":"P_1","st":68,"ed":70,"children":[" 1"]}]}]}]}," +"]},{"token":"P_1","st":74,"ed":76,"children":[" 100"]}]},"+"," {1,2,3;4,5,6;7,8,10}"," +",{"token":" lambda","st":109,"ed":115,"children":[{"token":"L_1","st":126,"ed":128,"children":[{"token":"P_1","st":127,"ed":129,"children":[{"token":"sum","st":131,"ed":133,"children":[{"token":"P_1","st":131,"ed":133,"children":["1"]},{"token":"P_1","st":133,"ed":135,"children":["1","2","+","3","*"]}]}]},{"token":"P_1","st":142,"ed":144,"children":["2"]},{"token":"P_1","st":144,"ed":146,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"lambda","st":148,"ed":153,"children":[{"token":"L_1","st":177,"ed":179,"children":[{"token":"P_1","st":178,"ed":180,"children":["1"]},{"token":"P_1","st":180,"ed":182,"children":["2"]}]},{"token":"P_1","st":151,"ed":153,"children":["x"]},{"token":"P_1","st":153,"ed":155,"children":["y"]},{"token":"P_1","st":155,"ed":157,"children":[{"token":" @offset","st":159,"ed":166,"children":[{"token":"P_1","st":164,"ed":166,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B0","st":-1,"ed":-1,"children":[]}]}]}]},{"token":"P_1","st":170,"ed":172,"children":[{"token":"#","st":-1,"ed":-1,"children":["x"]},{"token":"#","st":-1,"ed":-1,"children":["y"]},"*"]}]}]}]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"C20","st":-1,"ed":-1,"children":[]}]}]}]}]},{"token":"P_1","st":113,"ed":115,"children":["x"]},{"token":"P_1","st":115,"ed":117,"children":["y"]},{"token":"P_1","st":117,"ed":119,"children":["z"]},{"token":"P_1","st":119,"ed":121,"children":[" x","y","*","z","*"]}]}," +",{"token":"%","st":-1,"ed":-1,"children":[{"token":" sum","st":193,"ed":196,"children":[{"token":"P_1","st":194,"ed":196,"children":["1",{"token":"%","st":-1,"ed":-1,"children":["2"]},"+",{"token":"%","st":-1,"ed":-1,"children":["30"]},"*"]},{"token":"P_1","st":205,"ed":207,"children":[" 1","2","+"]}]}]}," +"]}'+ '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"-","st":1,"ed":1,"children":[{"token":"P_1","st":-1,"ed":1,"children":["1","2","+"]}]},{"token":"-","st":-1,"ed":-1,"children":[{"token":"@","st":-1,"ed":-1,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B2 ","st":-1,"ed":-1,"children":[]}]}]}]}]},"-","5","+","2 ","/",{"token":"-sum","st":25,"ed":28,"children":[{"token":"P_1","st":26,"ed":28,"children":[{"token":"#","st":-1,"ed":-1,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"indirect","st":30,"ed":37,"children":[{"token":"P_1","st":35,"ed":37,"children":["A5"]}]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B10","st":-1,"ed":-1,"children":[]}]}]}]},{"token":"#","st":-1,"ed":-1,"children":["B6"]},"+",{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"offset","st":59,"ed":64,"children":[{"token":"P_1","st":62,"ed":64,"children":["C5"]},{"token":"P_1","st":65,"ed":67,"children":["1"]},{"token":"P_1","st":68,"ed":70,"children":["1"]}]}]}]},"+"]},{"token":"P_1","st":74,"ed":76,"children":["100"]}]},"+","{1,2,3;4,5,6;7,8,10}","+",{"token":"lambda","st":110,"ed":115,"children":[{"token":"L_1","st":126,"ed":128,"children":[{"token":"P_1","st":127,"ed":129,"children":[{"token":"sum","st":131,"ed":133,"children":[{"token":"P_1","st":131,"ed":133,"children":["1"]},{"token":"P_1","st":133,"ed":135,"children":["1","2","+","3","*"]}]}]},{"token":"P_1","st":142,"ed":144,"children":["2"]},{"token":"P_1","st":144,"ed":146,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"lambda","st":148,"ed":153,"children":[{"token":"L_1","st":177,"ed":179,"children":[{"token":"P_1","st":178,"ed":180,"children":["1"]},{"token":"P_1","st":180,"ed":182,"children":["2"]}]},{"token":"P_1","st":151,"ed":153,"children":["x"]},{"token":"P_1","st":153,"ed":155,"children":["y"]},{"token":"P_1","st":155,"ed":157,"children":[{"token":"@offset","st":160,"ed":166,"children":[{"token":"P_1","st":164,"ed":166,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B0","st":-1,"ed":-1,"children":[]}]}]}]},{"token":"P_1","st":170,"ed":172,"children":[{"token":"#","st":-1,"ed":-1,"children":["x"]},{"token":"#","st":-1,"ed":-1,"children":["y"]},"*"]}]}]}]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"C20","st":-1,"ed":-1,"children":[]}]}]}]}]},{"token":"P_1","st":113,"ed":115,"children":["x"]},{"token":"P_1","st":115,"ed":117,"children":["y"]},{"token":"P_1","st":117,"ed":119,"children":["z"]},{"token":"P_1","st":119,"ed":121,"children":["x","y","*","z","*"]}]},"+",{"token":"%","st":-1,"ed":-1,"children":[{"token":"sum","st":194,"ed":196,"children":[{"token":"P_1","st":194,"ed":196,"children":["1",{"token":"%","st":-1,"ed":-1,"children":["2"]},"+",{"token":"%","st":-1,"ed":-1,"children":["30"]},"*"]},{"token":"P_1","st":205,"ed":207,"children":["1","2","+"]}]}]},"+"]}');});@@ -57,7 +57,7 @@ describe('lexer nodeMaker test', () => {'=(sum(max(B1:C10,10)*5-100,((1+1)*2+5)/2,10, lambda(x,y, x*y*x)(sum(1,(1+2)*3),2))+lambda(x,y, x*y*x)(sum(1,(1+2)*3),2)+count(B1:C10,10*5-100))*5-100') as LexerNode;expect(JSON.stringify(node.serialize())).toStrictEqual(- '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"sum","st":1,"ed":3,"children":[{"token":"P_1","st":1,"ed":3,"children":[{"token":"max","st":5,"ed":7,"children":[{"token":"P_1","st":5,"ed":7,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"C10","st":-1,"ed":-1,"children":[]}]}]}]},{"token":"P_1","st":12,"ed":14,"children":["10"]}]},"5","*","100","-"]},{"token":"P_1","st":22,"ed":24,"children":["1","1","+","2","*","5","+","2","/"]},{"token":"P_1","st":36,"ed":38,"children":["10"]},{"token":"P_1","st":39,"ed":41,"children":[{"token":" lambda","st":43,"ed":49,"children":[{"token":"L_1","st":58,"ed":60,"children":[{"token":"P_1","st":59,"ed":61,"children":[{"token":"sum","st":63,"ed":65,"children":[{"token":"P_1","st":63,"ed":65,"children":["1"]},{"token":"P_1","st":65,"ed":67,"children":["1","2","+","3","*"]}]}]},{"token":"P_1","st":74,"ed":76,"children":["2"]}]},{"token":"P_1","st":47,"ed":49,"children":["x"]},{"token":"P_1","st":49,"ed":51,"children":["y"]},{"token":"P_1","st":51,"ed":53,"children":[" x","y","*","x","*"]}]}]}]},{"token":"lambda","st":82,"ed":87,"children":[{"token":"L_1","st":96,"ed":98,"children":[{"token":"P_1","st":97,"ed":99,"children":[{"token":"sum","st":101,"ed":103,"children":[{"token":"P_1","st":101,"ed":103,"children":["1"]},{"token":"P_1","st":103,"ed":105,"children":["1","2","+","3","*"]}]}]},{"token":"P_1","st":112,"ed":114,"children":["2"]}]},{"token":"P_1","st":85,"ed":87,"children":["x"]},{"token":"P_1","st":87,"ed":89,"children":["y"]},{"token":"P_1","st":89,"ed":91,"children":[" x","y","*","x","*"]}]},"+",{"token":"count","st":119,"ed":123,"children":[{"token":"P_1","st":121,"ed":123,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"C10","st":-1,"ed":-1,"children":[]}]}]}]},{"token":"P_1","st":128,"ed":130,"children":["10","5","*","100","-"]}]},"+","5","*","100","-"]}'+ '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"sum","st":1,"ed":3,"children":[{"token":"P_1","st":1,"ed":3,"children":[{"token":"max","st":5,"ed":7,"children":[{"token":"P_1","st":5,"ed":7,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"C10","st":-1,"ed":-1,"children":[]}]}]}]},{"token":"P_1","st":12,"ed":14,"children":["10"]}]},"5","*","100","-"]},{"token":"P_1","st":22,"ed":24,"children":["1","1","+","2","*","5","+","2","/"]},{"token":"P_1","st":36,"ed":38,"children":["10"]},{"token":"P_1","st":39,"ed":41,"children":[{"token":"lambda","st":44,"ed":49,"children":[{"token":"L_1","st":58,"ed":60,"children":[{"token":"P_1","st":59,"ed":61,"children":[{"token":"sum","st":63,"ed":65,"children":[{"token":"P_1","st":63,"ed":65,"children":["1"]},{"token":"P_1","st":65,"ed":67,"children":["1","2","+","3","*"]}]}]},{"token":"P_1","st":74,"ed":76,"children":["2"]}]},{"token":"P_1","st":47,"ed":49,"children":["x"]},{"token":"P_1","st":49,"ed":51,"children":["y"]},{"token":"P_1","st":51,"ed":53,"children":["x","y","*","x","*"]}]}]}]},{"token":"lambda","st":82,"ed":87,"children":[{"token":"L_1","st":96,"ed":98,"children":[{"token":"P_1","st":97,"ed":99,"children":[{"token":"sum","st":101,"ed":103,"children":[{"token":"P_1","st":101,"ed":103,"children":["1"]},{"token":"P_1","st":103,"ed":105,"children":["1","2","+","3","*"]}]}]},{"token":"P_1","st":112,"ed":114,"children":["2"]}]},{"token":"P_1","st":85,"ed":87,"children":["x"]},{"token":"P_1","st":87,"ed":89,"children":["y"]},{"token":"P_1","st":89,"ed":91,"children":["x","y","*","x","*"]}]},"+",{"token":"count","st":119,"ed":123,"children":[{"token":"P_1","st":121,"ed":123,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"C10","st":-1,"ed":-1,"children":[]}]}]}]},{"token":"P_1","st":128,"ed":130,"children":["10","5","*","100","-"]}]},"+","5","*","100","-"]}');});@@ -71,14 +71,14 @@ describe('lexer nodeMaker test', () => {it('REDUCE', () => {const node = lexerTreeBuilder.treeBuilder('=REDUCE(1, A1:C2, LAMBDA(a,b,a+b^2))') as LexerNode;expect(JSON.stringify(node.serialize())).toStrictEqual(- '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"REDUCE","st":0,"ed":5,"children":[{"token":"P_1","st":3,"ed":5,"children":["1"]},{"token":"P_1","st":5,"ed":7,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":" A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"C2","st":-1,"ed":-1,"children":[]}]}]}]},{"token":"P_1","st":12,"ed":14,"children":[{"token":" LAMBDA","st":16,"ed":22,"children":[{"token":"P_1","st":20,"ed":22,"children":["a"]},{"token":"P_1","st":22,"ed":24,"children":["b"]},{"token":"P_1","st":24,"ed":26,"children":["a","b","2","^","+"]}]}]}]}]}'+ '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"REDUCE","st":0,"ed":5,"children":[{"token":"P_1","st":3,"ed":5,"children":["1"]},{"token":"P_1","st":5,"ed":7,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"C2","st":-1,"ed":-1,"children":[]}]}]}]},{"token":"P_1","st":12,"ed":14,"children":[{"token":"LAMBDA","st":17,"ed":22,"children":[{"token":"P_1","st":20,"ed":22,"children":["a"]},{"token":"P_1","st":22,"ed":24,"children":["b"]},{"token":"P_1","st":24,"ed":26,"children":["a","b","2","^","+"]}]}]}]}]}');});it('missing default arguments', () => {const node = lexerTreeBuilder.treeBuilder('=SUM(, A1:B1)') as LexerNode;expect(JSON.stringify(node.serialize())).toStrictEqual(- '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"SUM","st":0,"ed":2,"children":[{"token":"P_1","st":0,"ed":2,"children":[]},{"token":"P_1","st":1,"ed":3,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":" A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B1","st":-1,"ed":-1,"children":[]}]}]}]}]}]}'+ '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"SUM","st":0,"ed":2,"children":[{"token":"P_1","st":0,"ed":2,"children":[]},{"token":"P_1","st":1,"ed":3,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B1","st":-1,"ed":-1,"children":[]}]}]}]}]}]}');});@@ -131,14 +131,13 @@ describe('lexer nodeMaker test', () => {// eslint-disable-next-line no-consoleconsole.log(`Elapsed time: ${elapsed} ms`);- const expectedMaxTime = 3000; // 毫秒数 // CI may time out TODO: performance testexpect(elapsed).toBeGreaterThan(0);});it('error include function test', () => {const node = lexerTreeBuilder.treeBuilder('=sum(#REF! + 5 , #REF!)') as LexerNode;expect(JSON.stringify(node.serialize())).toStrictEqual(- '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"sum","st":0,"ed":2,"children":[{"token":"P_1","st":0,"ed":2,"children":["#REF!"," 5 "," +"]},{"token":"P_1","st":11,"ed":13,"children":["#REF!"]}]}]}'+ '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"sum","st":0,"ed":2,"children":[{"token":"P_1","st":0,"ed":2,"children":["#REF!","5 ","+"]},{"token":"P_1","st":11,"ed":13,"children":["#REF!"]}]}]}');});@@ -149,17 +148,17 @@ describe('lexer nodeMaker test', () => {it('minus test complex', () => {const node = lexerTreeBuilder.treeBuilder('= ( 2019-09-09 ) ') as LexerNode;- expect(JSON.stringify(node.serialize())).toStrictEqual('{"token":"R_1","st":-1,"ed":-1,"children":[" 2019","09","-","09 ","-"]}');+ expect(JSON.stringify(node.serialize())).toStrictEqual('{"token":"R_1","st":-1,"ed":-1,"children":["2019","09","-","09 ","-"]}');});it('sheet range', () => {const node = lexerTreeBuilder.treeBuilder('=SUM(SUM(Sheet5:Sheet6!A1:B10) + 100, 1)') as LexerNode;- expect(JSON.stringify(node.serialize())).toStrictEqual('{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"SUM","st":0,"ed":2,"children":[{"token":"P_1","st":0,"ed":2,"children":[{"token":"SUM","st":4,"ed":6,"children":[{"token":"P_1","st":4,"ed":6,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"Sheet5","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"Sheet6!A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B10","st":-1,"ed":-1,"children":[]}]}]}]}]}]}]}," 100"," +"]},{"token":"P_1","st":32,"ed":34,"children":[" 1"]}]}]}');+ expect(JSON.stringify(node.serialize())).toStrictEqual('{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"SUM","st":0,"ed":2,"children":[{"token":"P_1","st":0,"ed":2,"children":[{"token":"SUM","st":4,"ed":6,"children":[{"token":"P_1","st":4,"ed":6,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"Sheet5","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"Sheet6!A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B10","st":-1,"ed":-1,"children":[]}]}]}]}]}]}]},"100","+"]},{"token":"P_1","st":32,"ed":34,"children":["1"]}]}]}');});it('minus ref', () => {const node = lexerTreeBuilder.treeBuilder('= 1--A1 ') as LexerNode;- expect(JSON.stringify(node.serialize())).toStrictEqual('{"token":"R_1","st":-1,"ed":-1,"children":[" 1","-A1 ","-"]}');+ expect(JSON.stringify(node.serialize())).toStrictEqual('{"token":"R_1","st":-1,"ed":-1,"children":["1","-A1 ","-"]}');});it('negative ref', () => {@@ -209,7 +208,7 @@ describe('lexer nodeMaker test', () => {it('error as parameter', () => {const node = lexerTreeBuilder.treeBuilder('=sum(#NUM! + #VALUE!) + #SPILL! - (#CALC!)') as LexerNode;- expect(JSON.stringify(node.serialize())).toStrictEqual('{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"sum","st":0,"ed":2,"children":[{"token":"P_1","st":0,"ed":2,"children":["#NUM!","#VALUE!"," +"]}]},"#SPILL!"," +","#CALC!","-"]}');+ expect(JSON.stringify(node.serialize())).toStrictEqual('{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"sum","st":0,"ed":2,"children":[{"token":"P_1","st":0,"ed":2,"children":["#NUM!","#VALUE!","+"]}]},"#SPILL!","+","#CALC!","-"]}');});it('error type lexer function', () => {@@ -234,7 +233,7 @@ describe('lexer nodeMaker test', () => {it('one minus for formula', () => {const node = lexerTreeBuilder.treeBuilder('= -A1') as LexerNode;- expect(JSON.stringify(node.serialize())).toStrictEqual('{"token":"R_1","st":-1,"ed":-1,"children":[" -A1"]}');+ expect(JSON.stringify(node.serialize())).toStrictEqual('{"token":"R_1","st":-1,"ed":-1,"children":["-A1"]}');});it('operator with concatenate test', () => {@@ -468,11 +467,12 @@ describe('lexer nodeMaker test', () => {token: '"TRUE"',},',',+ ' ',{endIndex: 27,nodeType: 2,- startIndex: 20,- token: ' "FALSE"',+ startIndex: 21,+ token: '"FALSE"',},')',]);@@ -550,11 +550,17 @@ describe('lexer nodeMaker test', () => {it('multi blank to minus formula', () => {expect(lexerTreeBuilder.sequenceNodesBuilder('= -A1')).toStrictEqual([+ ' ',+ ' ',+ ' ',+ ' ',+ ' ',+ ' ',{endIndex: 6,nodeType: 0,- startIndex: 0,- token: ' -',+ startIndex: 6,+ token: '-',},{endIndex: 8,@@ -570,7 +576,12 @@ describe('lexer nodeMaker test', () => {it('one minus sequences error', () => {expect(lexerTreeBuilder.sequenceNodesBuilder('=-\r\n')).toStrictEqual([- '-',+ {+ endIndex: 1,+ nodeType: 3,+ startIndex: 0,+ token: '- ',+ },]);});@@ -653,7 +664,7 @@ describe('lexer nodeMaker test', () => {it('Complex Formula', () => {const result = lexerTreeBuilder.convertRefersToAbsolute('=SUM(A1:B10) + LAMBDA(x, y, x*y*x)(A1:B10, A10) + MAX(A1:B10,SUM(A2))', AbsoluteRefType.ALL, AbsoluteRefType.ALL);- expect(result).toStrictEqual('=SUM($A$1:$B$10) + LAMBDA(x, y, x*y*x)($A$1:$B$10,$A$10) + MAX($A$1:$B$10,SUM($A$2))');+ expect(result).toStrictEqual('=SUM($A$1:$B$10) + LAMBDA(x, y, x*y*x)($A$1:$B$10, $A$10) + MAX($A$1:$B$10,SUM($A$2))');});it('manually set current sheet name', () => {@@ -732,4 +743,18 @@ describe('lexer nodeMaker test', () => {expect(result).toStrictEqual("=SUM( 'dv-test'!E26)");});});++ describe('bad cases from users', () => {+ it('univer-pro #4120', () => {+ // The space before [workbook4] will lead to error.+ const node = lexerTreeBuilder.treeBuilder("'[workbook1]Sheet 01'!A5* [workbook4]aaa!A5") as LexerNode;+ expect(node.serialize()).toStrictEqual(+ { token: 'R_1', st: -1, ed: -1, children: [+ "'[workbook1]Sheet 01'!A5",+ '[workbook4]aaa!A5',+ '*',+ ] }+ );+ });+ });});diff --git a/packages/engine-formula/src/engine/analysis/__tests__/lexer.spec.ts b/packages/engine-formula/src/engine/analysis/__tests__/lexer.spec.tsindex 5531b5a48d..47f5b17b79 100644--- a/packages/engine-formula/src/engine/analysis/__tests__/lexer.spec.ts+++ b/packages/engine-formula/src/engine/analysis/__tests__/lexer.spec.ts@@ -15,14 +15,13 @@*/import type { Injector, IWorkbookData, Univer, Workbook } from '@univerjs/core';+import type { LexerNode } from '../lexer-node';import { LocaleType } from '@univerjs/core';import { afterEach, beforeEach, describe, expect, it } from 'vitest';-+import { IFormulaCurrentConfigService } from '../../../services/current-data.service';import { IDefinedNamesService } from '../../../services/defined-names.service';import { Lexer } from '../lexer';-import type { LexerNode } from '../lexer-node';import { LexerTreeBuilder } from '../lexer-tree-builder';-import { IFormulaCurrentConfigService } from '../../../services/current-data.service';import { createCommandTestBed } from './create-command-test-bed';const TEST_WORKBOOK_DATA: IWorkbookData = {@@ -85,7 +84,11 @@ describe('lexer test', () => {// runtimeService.setCurrent(0, 0, 4, 1, 'sheet1', 'test');- lexer = new Lexer(definedNamesService, lexerTreeBuilder, formulaCurrentConfigService);+ lexer = new Lexer(+ definedNamesService,+ lexerTreeBuilder,+ formulaCurrentConfigService+ );});afterEach(() => {@@ -94,23 +97,161 @@ describe('lexer test', () => {describe('lexer definedName', () => {it('simple', () => {- definedNamesService.registerDefinedName('test', { id: 'test1', name: 'myName', formulaOrRefString: '$A$10:$C$100' });+ definedNamesService.registerDefinedName('test', {+ id: 'test1',+ name: 'myName',+ formulaOrRefString: '$A$10:$C$100',+ });const node = lexer.treeBuilder('=myName') as LexerNode;- expect(JSON.stringify(node.serialize())).toStrictEqual(- '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"$A$10","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"$C$100","st":-1,"ed":-1,"children":[]}]}]}]}'- );+ expect(node.serialize()).toStrictEqual({+ token: 'R_1',+ st: -1,+ ed: -1,+ children: [+ {+ token: ':',+ st: -1,+ ed: -1,+ children: [+ {+ token: 'P_1',+ st: -1,+ ed: -1,+ children: [+ {+ token: '$A$10',+ st: -1,+ ed: -1,+ children: [],+ },+ ],+ },+ {+ token: 'P_1',+ st: -1,+ ed: -1,+ children: [+ {+ token: '$C$100',+ st: -1,+ ed: -1,+ children: [],+ },+ ],+ },+ ],+ },+ ],+ });});it('lambda', () => {- definedNamesService.registerDefinedName('test', { id: 'test2', name: 'myName', formulaOrRefString: 'lambda(x, y , x*x*y)' });+ definedNamesService.registerDefinedName('test', {+ id: 'test2',+ name: 'myName',+ formulaOrRefString: 'lambda(x, y , x*x*y)',+ });- const node = lexer.treeBuilder('=myName(1+sum(A1:B1), 100)') as LexerNode;-- expect(JSON.stringify(node.serialize())).toStrictEqual(- '{"token":"R_1","st":-1,"ed":-1,"children":[{"token":"lambda","st":0,"ed":5,"children":[{"token":"L_1","st":16,"ed":18,"children":[{"token":"P_1","st":17,"ed":19,"children":["1",{"token":"sum","st":23,"ed":25,"children":[{"token":"P_1","st":23,"ed":25,"children":[{"token":":","st":-1,"ed":-1,"children":[{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"A1","st":-1,"ed":-1,"children":[]}]},{"token":"P_1","st":-1,"ed":-1,"children":[{"token":"B1","st":-1,"ed":-1,"children":[]}]}]}]}]},"+"]},{"token":"P_1","st":30,"ed":32,"children":[" 100"]}]},{"token":"P_1","st":3,"ed":5,"children":["x"]},{"token":"P_1","st":5,"ed":7,"children":[" y "]},{"token":"P_1","st":9,"ed":11,"children":[" x","x","*","y","*"]}]}]}'- );+ const node = lexer.treeBuilder(+ '=myName(1+sum(A1:B1), 100)'+ ) as LexerNode;+ expect(node.serialize()).toStrictEqual({+ token: 'R_1',+ st: -1,+ ed: -1,+ children: [+ {+ token: 'lambda',+ st: 0,+ ed: 5,+ children: [+ {+ token: 'L_1',+ st: 16,+ ed: 18,+ children: [+ {+ token: 'P_1',+ st: 17,+ ed: 19,+ children: [+ '1',+ {+ token: 'sum',+ st: 23,+ ed: 25,+ children: [+ {+ token: 'P_1',+ st: 23,+ ed: 25,+ children: [+ {+ token: ':',+ st: -1,+ ed: -1,+ children: [+ {+ token: 'P_1',+ st: -1,+ ed: -1,+ children:+ [+ {+ token: 'A1',+ st: -1,+ ed: -1,+ children:+ [],+ },+ ],+ },+ {+ token: 'P_1',+ st: -1,+ ed: -1,+ children:+ [+ {+ token: 'B1',+ st: -1,+ ed: -1,+ children:+ [],+ },+ ],+ },+ ],+ },+ ],+ },+ ],+ },+ '+',+ ],+ },+ {+ token: 'P_1',+ st: 30,+ ed: 32,+ children: ['100'],+ },+ ],+ },+ { token: 'P_1', st: 3, ed: 5, children: ['x'] },+ { token: 'P_1', st: 5, ed: 7, children: ['y '] },+ {+ token: 'P_1',+ st: 9,+ ed: 11,+ children: ['x', 'x', '*', 'y', '*'],+ },+ ],+ },+ ],+ });});});});diff --git a/packages/engine-formula/src/engine/analysis/lexer-tree-builder.ts b/packages/engine-formula/src/engine/analysis/lexer-tree-builder.tsindex 89c4fab2b7..7793b0f23b 100644--- a/packages/engine-formula/src/engine/analysis/lexer-tree-builder.ts+++ b/packages/engine-formula/src/engine/analysis/lexer-tree-builder.ts@@ -1252,33 +1252,28 @@ export class LexerTreeBuilder extends Disposable {}}+ // NOTE@wzhudev: this method is over complex, need to refactor. Using recursive descend to parse the formula string+ // without a standalone lexer phase// eslint-disable-next-line max-lines-per-function, complexityprivate _nodeMaker(formulaStringRaw: string, sequenceArray?: ISequenceArray[], matchCurrentNodeIndex?: number) {// Editor will add '\r\n' at the end of the formula string- let formulaString = formulaStringRaw.replace(/\r\n$/, '').replace(/\r/g, ' ').replace(/\n/g, ' ');-+ let formulaString = formulaStringRaw.replace(/\r\n$|\r|\n/g, ' ');if (formulaString.substring(0, 1) === operatorToken.EQUALS) {formulaString = formulaString.substring(1);}- // let isZeroAdded = false;- // if (formulaString.substring(0, 1) === operatorToken.MINUS) {- // formulaString = `0${formulaString}`;- // isZeroAdded = true;- // }-- const formulaStringArray = formulaString.split('');- const formulaStringArrayCount = formulaStringArray.length;+ const formulaChars = formulaString.split('');+ const charsCount = formulaChars.length;- let cur = 0;this._resetTemp();- if (this._formulaErrorLastTokenCheck(formulaStringArray, formulaStringArrayCount - 1)) {+ if (this._formulaErrorLastTokenCheck(formulaChars, charsCount - 1)) {this._formalErrorOccurred();}- while (cur < formulaStringArrayCount) {- const currentString = formulaStringArray[cur];+ let cur = 0;+ while (cur < charsCount) {+ const currentString = formulaChars[cur];if (matchCurrentNodeIndex === cur) {return [this._currentLexerNode, currentString];@@ -1289,15 +1284,15 @@ export class LexerTreeBuilder extends Disposable {this.isDoubleQuotationClose() &&this.isBracesClose() &&this.isSquareBracketClose() &&- this._checkIfErrorObject(cur, formulaStringArray)+ this._checkIfErrorObject(cur, formulaChars)) {- const errorType = this._findErrorObject(cur, formulaStringArray);+ const errorType = this._findErrorObject(cur, formulaChars);if (errorType == null) {return ErrorType.VALUE;}this._pushNodeToChildren(errorType);for (let i = 0; i < errorType.length; i++) {- const curStr = formulaStringArray[cur];+ const curStr = formulaChars[cur];this._pushSegment(curStr);this._addSequenceArray(sequenceArray, curStr, cur);cur++;@@ -1320,10 +1315,10 @@ export class LexerTreeBuilder extends Disposable {this._openBracket(bracketType.FUNCTION);this._closeLambda();- const nextCurrentString = formulaStringArray[cur + 1];+ const nextCurrentString = formulaChars[cur + 1];if (nextCurrentString && nextCurrentString === matchToken.CLOSE_BRACKET) {// when function has not parameter, return to parentNode , e.g. row()- if (!this._setParentCurrentLexerNode() && cur !== formulaStringArrayCount - 1) {+ if (!this._setParentCurrentLexerNode() && cur !== charsCount - 1) {return ErrorType.VALUE;}/**@@ -1351,7 +1346,7 @@ export class LexerTreeBuilder extends Disposable {this.isSingleQuotationClose() &&this.isDoubleQuotationClose()) {- if (this._formulaErrorLastTokenCheck(formulaStringArray, cur - 1)) {+ if (this._formulaErrorLastTokenCheck(formulaChars, cur - 1)) {this._formalErrorOccurred();}@@ -1362,17 +1357,17 @@ export class LexerTreeBuilder extends Disposable {this._pushNodeToChildren(currentString);} else if (currentBracket === bracketType.FUNCTION) {// function close- const nextCurrentString = formulaStringArray[cur + 1];+ const nextCurrentString = formulaChars[cur + 1];if (nextCurrentString && nextCurrentString === matchToken.OPEN_BRACKET) {// lambda handler, e.g. =lambda(x,y, x*y*x)(1,2)- if (!this._setParentCurrentLexerNode() && cur !== formulaStringArrayCount - 1) {+ if (!this._setParentCurrentLexerNode() && cur !== charsCount - 1) {return ErrorType.VALUE;}this._newAndPushCurrentLexerNode(DEFAULT_TOKEN_TYPE_LAMBDA_PARAMETER, cur, true);this._openLambda();} else {- if (!this._setAncestorCurrentLexerNode() && cur !== formulaStringArrayCount - 1) {+ if (!this._setAncestorCurrentLexerNode() && cur !== charsCount - 1) {return ErrorType.VALUE;}}@@ -1434,7 +1429,7 @@ export class LexerTreeBuilder extends Disposable {if (this.isDoubleQuotationClose()) {this._openDoubleQuotation();} else {- const nextCurrentString = formulaStringArray[cur + 1];+ const nextCurrentString = formulaChars[cur + 1];if (nextCurrentString && nextCurrentString === matchToken.DOUBLE_QUOTATION) {cur++;} else {@@ -1454,7 +1449,7 @@ export class LexerTreeBuilder extends Disposable {this._resetSegment();}} else {- const nextCurrentString = formulaStringArray[cur + 1];+ const nextCurrentString = formulaChars[cur + 1];if (nextCurrentString && nextCurrentString === matchToken.SINGLE_QUOTATION) {// handle 'Sheet'1'!A1@@ -1481,7 +1476,7 @@ export class LexerTreeBuilder extends Disposable {this.isBracesClose() &&this.isSquareBracketClose()) {- if (this._formulaErrorLastTokenCheck(formulaStringArray, cur - 1)) {+ if (this._formulaErrorLastTokenCheck(formulaChars, cur - 1)) {this._formalErrorOccurred();}@@ -1496,7 +1491,7 @@ export class LexerTreeBuilder extends Disposable {this._resetSegment();if (!this._setParentCurrentLexerNode() &&- cur !== formulaStringArrayCount - 1 &&+ cur !== charsCount - 1 &¤tBracket != null) {return ErrorType.VALUE;@@ -1628,7 +1623,7 @@ export class LexerTreeBuilder extends Disposable {this._openColon(upLevel);} else if (SUFFIX_TOKEN_SET.has(currentString) &&- this._checkSimilarErrorToken(currentString, cur, formulaStringArray) &&+ this._checkSimilarErrorToken(currentString, cur, formulaChars) &&this.isSingleQuotationClose() &&this.isDoubleQuotationClose() &&this.isSquareBracketClose() &&@@ -1664,8 +1659,8 @@ export class LexerTreeBuilder extends Disposable {if (currentString === operatorToken.MINUS && (trimSegment === '')) {// negative number- const prevString = this._findPreviousToken(formulaStringArray, cur - 1) || '';- const nextString = this._findNextToken(formulaStringArray, cur + 1) || '';+ const prevString = this._findPreviousToken(formulaChars, cur - 1) || '';+ const nextString = this._findNextToken(formulaChars, cur + 1) || '';if (this._unexpectedEndingTokenExcludeOperator(prevString) && this._isOperatorToken(nextString)) {this._pushNodeToChildren('0');this._pushNodeToChildren(operatorToken.MINUS);@@ -1688,7 +1683,7 @@ export class LexerTreeBuilder extends Disposable {continue;}}- } else if (this._segment.length > 0 && this._isScientificNotation(formulaStringArray, cur, currentString)) {+ } else if (this._segment.length > 0 && this._isScientificNotation(formulaChars, cur, currentString)) {this._pushSegment(currentString);this._addSequenceArray(sequenceArray, currentString, cur);@@ -1703,7 +1698,7 @@ export class LexerTreeBuilder extends Disposable {}if (currentString === operatorToken.LESS_THAN || currentString === operatorToken.GREATER_THAN) {- const nextCurrentString = formulaStringArray[cur + 1];+ const nextCurrentString = formulaChars[cur + 1];if (nextCurrentString && OPERATOR_TOKEN_SET.has(currentString + nextCurrentString)) {this._pushNodeToChildren(trimSegment + currentString + nextCurrentString);@@ -1721,7 +1716,10 @@ export class LexerTreeBuilder extends Disposable {}this._resetSegment();} else {- this._pushSegment(currentString);+ // Do not push a space when the segment is empty, consuming useless spaces here.+ if (this._segment !== '' || currentString !== ' ') {+ this._pushSegment(currentString);+ }}this._addSequenceArray(sequenceArray, currentString, cur);diff --git a/packages/engine-formula/src/engine/analysis/lexer.ts b/packages/engine-formula/src/engine/analysis/lexer.tsindex dc8620b027..6496051c2e 100644--- a/packages/engine-formula/src/engine/analysis/lexer.ts+++ b/packages/engine-formula/src/engine/analysis/lexer.ts@@ -14,14 +14,14 @@* limitations under the License.*/-import { Disposable, Inject } from '@univerjs/core';--import { IDefinedNamesService } from '../../services/defined-names.service';import type { ISequenceArray } from '../utils/sequence';-import { sequenceNodeType } from '../utils/sequence';++import { Disposable, Inject } from '@univerjs/core';+import { ErrorType } from '../../basics/error-type';import { operatorToken } from '../../basics/token';import { IFormulaCurrentConfigService } from '../../services/current-data.service';-import { ErrorType } from '../../basics/error-type';+import { IDefinedNamesService } from '../../services/defined-names.service';+import { sequenceNodeType } from '../utils/sequence';import { LexerTreeBuilder } from './lexer-tree-builder';export class Lexer extends Disposable {diff --git a/tests/formula-integration/README.md b/tests/formula-integration/README.mdindex 7d7a8e1fb8..0c3410d43e 100644--- a/tests/formula-integration/README.md+++ b/tests/formula-integration/README.md@@ -1,3 +1,3 @@# formula-integration-test-This is a test repository for the formula integration feature. It also exports some util functions that can be used in the advanced formula engine integration test.+This is a test repository for the formula integration feature.diff --git a/tests/formula-integration/src/__tests__/formua-correctness.spec.ts b/tests/formula-integration/src/__tests__/formua-correctness.spec.tsdeleted file mode 100644index dc200d9bfa..0000000000--- a/tests/formula-integration/src/__tests__/formua-correctness.spec.ts+++ /dev/null@@ -1,24 +0,0 @@-/**- * Copyright 2023-present DreamNum Co., Ltd.- *- * Licensed under the Apache License, Version 2.0 (the "License");- * you may not use this file except in compliance with the License.- * You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import { describe, it } from 'vitest';-import { expectCalculationResultMatchesSnapshot } from '../__testing__/util';--describe('Test formula correctness', () => {- it('basic sum', () => {- expectCalculationResultMatchesSnapshot();- });-});