High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(extract): item resolution includes the item node, truncation reports only a dropped non-empty item, and the item resolver is built once per group (PIN-392)
Luigi Agostib50f3fa6e80f
diff --git a/docs/architecture/extract.md b/docs/architecture/extract.mdindex e992df9a..0b52591b 100644--- a/docs/architecture/extract.md+++ b/docs/architecture/extract.md@@ -143,10 +143,11 @@ The items are the container's direct children carrying the dominant roleTwo per-item modes exist:-- **Subtree.** The flat resolver runs over the item's descendants only (the- item node itself excluded), so `price` in item 3 can never match item 1 and a- field absent from one item stays missing there even when every other item- has it.+- **Subtree.** The flat resolver runs over the item's own subtree only (the+ item node included, since a pruned tree often leaves a menu's or result's+ text on the item node itself), so `price` in item 3 can never match item 1+ and a field absent from one item stays missing there even when every other+ item has it.- **Columnar.** When the items are `row`s and the nearest enclosing `table` or`grid` (or the container itself) holds a header row, each field is resolvedonce against the header cells to a column index, and every row reads the cell@@ -155,8 +156,9 @@ Two per-item modes exist:the field missing with `no_match`.Items where every field is missing are dropped. The output is capped at the-smaller of the schema's `maxItems` and `Options.MaxItems` (default `100`); when-unresolved items remain past the cap the field reports `truncated: true`.+smaller of the schema's `maxItems` and `Options.MaxItems` (default `100`); the+cap is tested after empties are dropped, so `truncated: true` is reported only+when a non-empty item was actually left out.Fewer items than `minItems` leaves the property missing with reason`too_few_items`.diff --git a/internal/extract/group.go b/internal/extract/group.goindex 512b86c8..ea2d96c1 100644--- a/internal/extract/group.go+++ b/internal/extract/group.go@@ -21,6 +21,7 @@ type group struct {items []intcolumns []intscore float64+ resolver itemResolver}func resolveArray(prop Property, v view, opts Options) (FieldResult, any, bool) {@@ -28,21 +29,20 @@ func resolveArray(prop Property, v view, opts Options) (FieldResult, any, bool)if !ok {return fr, nil, false}- items := newItemResolver(g, *prop.Items, v, opts)limit := opts.MaxItemsif prop.MaxItems > 0 && prop.MaxItems < limit {limit = prop.MaxItems}data := []map[string]any{}for _, item := range g.items {+ res := g.resolver.resolve(item)+ if len(res.Data) == 0 {+ continue+ }if len(data) == limit {fr.Truncated = truebreak}- res := items.resolve(item)- if len(res.Data) == 0 {- continue- }data = append(data, res.Data)fr.Items = append(fr.Items, ItemResult{Ref: v.nodes[item].Ref, Fields: res.Fields})}@@ -70,7 +70,8 @@ func chooseGroup(prop Property, v view, opts Options) (group, FieldResult, bool)}for i := range candidates {- candidates[i].score = scoreGroup(candidates[i], *prop.Items, v, opts)+ candidates[i].resolver = newItemResolver(candidates[i], *prop.Items, v, opts)+ candidates[i].score = scoreGroup(candidates[i], *prop.Items)}sort.SliceStable(candidates, func(i, j int) bool {a, b := candidates[i], candidates[j]@@ -166,15 +167,14 @@ func headerColumns(v view, container int) []int {return nil}-func scoreGroup(g group, schema Schema, v view, opts Options) float64 {- items := newItemResolver(g, schema, v, opts)+func scoreGroup(g group, schema Schema) float64 {sample := g.itemsif len(sample) > sampleItems {sample = sample[:sampleItems]}resolved := 0for _, item := range sample {- resolved += len(items.resolve(item).Data)+ resolved += len(g.resolver.resolve(item).Data)}return roundScore(float64(resolved) / float64(len(sample)*len(schema.Properties)))}@@ -216,7 +216,7 @@ func newItemResolver(g group, schema Schema, v view, opts Options) itemResolverfunc (r itemResolver) resolve(item int) Result {if r.columns == nil {- return resolveObject(r.schema, r.v.descendants(item), r.opts)+ return resolveObject(r.schema, r.v.subtree(item), r.opts)}cells := r.v.children(item)result := Result{Data: map[string]any{}, Fields: map[string]FieldResult{}}@@ -276,8 +276,8 @@ func (v view) children(i int) []int {return children}-func (v view) descendants(i int) view {- return newView(v.nodes[i+1 : v.subtreeEnd(i)])+func (v view) subtree(i int) view {+ return newView(v.nodes[i:v.subtreeEnd(i)])}func (v view) ancestors(i int) []int {diff --git a/internal/extract/group_test.go b/internal/extract/group_test.goindex cf17c040..d08ab400 100644--- a/internal/extract/group_test.go+++ b/internal/extract/group_test.go@@ -252,8 +252,8 @@ func TestView_ChildrenAndSubtreeFromDepth(t *testing.T) {if !reflect.DeepEqual(refs, []string{"e3", "e6", "e9", "e11"}) {t.Errorf("children of e2 = %v", refs)}- if got := v.descendants(v.index["e6"]).nodes; len(got) != 2 || got[0].Ref != "e7" || got[1].Ref != "e8" {- t.Errorf("descendants of e6 = %v", got)+ if got := v.subtree(v.index["e6"]).nodes; len(got) != 3 || got[0].Ref != "e6" || got[2].Ref != "e8" {+ t.Errorf("subtree of e6 = %v", got)}var anc []stringfor _, a := range v.ancestors(v.index["e12"]) {@@ -276,3 +276,35 @@ func TestResolveArray_GroupResolvingNoFieldIsNoRepeatedGroup(t *testing.T) {t.Errorf("field = %+v", got.Fields["results"])}}++func TestResolveArray_ItemNodeItselfCarriesTheField(t *testing.T) {+ nodes := []observe.A11yNode{+ {Ref: "e1", Role: "list", Depth: 0},+ {Ref: "e2", Role: "link", Name: "Home", Depth: 1},+ {Ref: "e3", Role: "link", Name: "Shop", Depth: 1},+ {Ref: "e4", Role: "link", Name: "About", Depth: 1},+ {Ref: "e5", Role: "link", Name: "Contact", Depth: 1},+ }+ schema := mustSchema(t, `{"type":"object","properties":{+ "menu":{"type":"array","items":{"type":"object","properties":{+ "title":{"type":"string","x-pinchtab-hint":"role:link"}}}}}}`)+ got := Resolve(schema, nodes, Options{})+ want := []map[string]any{{"title": "Home"}, {"title": "Shop"}, {"title": "About"}, {"title": "Contact"}}+ if list := items(t, got, "menu"); !reflect.DeepEqual(list, want) {+ t.Fatalf("menu = %v, want %v (%+v)", list, want, got.Fields["menu"])+ }+ if fr := got.Fields["menu"]; fr.Ref != "e1" || fr.Items[1].Ref != "e3" || fr.Items[1].Fields["title"].Ref != "e3" {+ t.Errorf("refs = %+v", fr)+ }+}++func TestResolveArray_CapReachedExactlyWithTrailingEmptyItemIsNotTruncated(t *testing.T) {+ nodes := append(searchNodes(), observe.A11yNode{Ref: "e14", Role: "listitem", Depth: 2}, observe.A11yNode{Ref: "e15", Role: "text", Name: "Sponsored", Depth: 3})+ schema := mustSchema(t, `{"type":"object","properties":{+ "results":{"type":"array","maxItems":4,"items":{"type":"object","properties":{+ "title":{"type":"string","x-pinchtab-hint":"role:link"}}}}}}`)+ got := Resolve(schema, nodes, Options{})+ if n := len(items(t, got, "results")); n != 4 || got.Fields["results"].Truncated {+ t.Errorf("len=%d truncated=%v, want 4 items and no truncation when only an empty item follows", n, got.Fields["results"].Truncated)+ }+}