Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(extract): item resolution includes the item node, truncation reports only a dropped non-empty item, and the item resolver is built once per group (PIN-392)

Luigi Agostib50f3fa6e80f
diff --git a/docs/architecture/extract.md b/docs/architecture/extract.md
index e992df9a..0b52591b 100644
--- a/docs/architecture/extract.md
+++ b/docs/architecture/extract.md
@@ -143,10 +143,11 @@ The items are the container's direct children carrying the dominant role
Two per-item modes exist:
-- **Subtree.** The flat resolver runs over the item's descendants only (the
- item node itself excluded), so `price` in item 3 can never match item 1 and a
- field absent from one item stays missing there even when every other item
- has it.
+- **Subtree.** The flat resolver runs over the item's own subtree only (the
+ item node included, since a pruned tree often leaves a menu's or result's
+ text on the item node itself), so `price` in item 3 can never match item 1
+ and a field absent from one item stays missing there even when every other
+ item has it.
- **Columnar.** When the items are `row`s and the nearest enclosing `table` or
`grid` (or the container itself) holds a header row, each field is resolved
once against the header cells to a column index, and every row reads the cell
@@ -155,8 +156,9 @@ Two per-item modes exist:
the field missing with `no_match`.
Items where every field is missing are dropped. The output is capped at the
-smaller of the schema's `maxItems` and `Options.MaxItems` (default `100`); when
-unresolved items remain past the cap the field reports `truncated: true`.
+smaller of the schema's `maxItems` and `Options.MaxItems` (default `100`); the
+cap is tested after empties are dropped, so `truncated: true` is reported only
+when a non-empty item was actually left out.
Fewer items than `minItems` leaves the property missing with reason
`too_few_items`.
diff --git a/internal/extract/group.go b/internal/extract/group.go
index 512b86c8..ea2d96c1 100644
--- a/internal/extract/group.go
+++ b/internal/extract/group.go
@@ -21,6 +21,7 @@ type group struct {
items []int
columns []int
score float64
+ resolver itemResolver
}
func resolveArray(prop Property, v view, opts Options) (FieldResult, any, bool) {
@@ -28,21 +29,20 @@ func resolveArray(prop Property, v view, opts Options) (FieldResult, any, bool)
if !ok {
return fr, nil, false
}
- items := newItemResolver(g, *prop.Items, v, opts)
limit := opts.MaxItems
if prop.MaxItems > 0 && prop.MaxItems < limit {
limit = prop.MaxItems
}
data := []map[string]any{}
for _, item := range g.items {
+ res := g.resolver.resolve(item)
+ if len(res.Data) == 0 {
+ continue
+ }
if len(data) == limit {
fr.Truncated = true
break
}
- res := items.resolve(item)
- if len(res.Data) == 0 {
- continue
- }
data = append(data, res.Data)
fr.Items = append(fr.Items, ItemResult{Ref: v.nodes[item].Ref, Fields: res.Fields})
}
@@ -70,7 +70,8 @@ func chooseGroup(prop Property, v view, opts Options) (group, FieldResult, bool)
}
for i := range candidates {
- candidates[i].score = scoreGroup(candidates[i], *prop.Items, v, opts)
+ candidates[i].resolver = newItemResolver(candidates[i], *prop.Items, v, opts)
+ candidates[i].score = scoreGroup(candidates[i], *prop.Items)
}
sort.SliceStable(candidates, func(i, j int) bool {
a, b := candidates[i], candidates[j]
@@ -166,15 +167,14 @@ func headerColumns(v view, container int) []int {
return nil
}
-func scoreGroup(g group, schema Schema, v view, opts Options) float64 {
- items := newItemResolver(g, schema, v, opts)
+func scoreGroup(g group, schema Schema) float64 {
sample := g.items
if len(sample) > sampleItems {
sample = sample[:sampleItems]
}
resolved := 0
for _, item := range sample {
- resolved += len(items.resolve(item).Data)
+ resolved += len(g.resolver.resolve(item).Data)
}
return roundScore(float64(resolved) / float64(len(sample)*len(schema.Properties)))
}
@@ -216,7 +216,7 @@ func newItemResolver(g group, schema Schema, v view, opts Options) itemResolver
func (r itemResolver) resolve(item int) Result {
if r.columns == nil {
- return resolveObject(r.schema, r.v.descendants(item), r.opts)
+ return resolveObject(r.schema, r.v.subtree(item), r.opts)
}
cells := r.v.children(item)
result := Result{Data: map[string]any{}, Fields: map[string]FieldResult{}}
@@ -276,8 +276,8 @@ func (v view) children(i int) []int {
return children
}
-func (v view) descendants(i int) view {
- return newView(v.nodes[i+1 : v.subtreeEnd(i)])
+func (v view) subtree(i int) view {
+ return newView(v.nodes[i:v.subtreeEnd(i)])
}
func (v view) ancestors(i int) []int {
diff --git a/internal/extract/group_test.go b/internal/extract/group_test.go
index cf17c040..d08ab400 100644
--- a/internal/extract/group_test.go
+++ b/internal/extract/group_test.go
@@ -252,8 +252,8 @@ func TestView_ChildrenAndSubtreeFromDepth(t *testing.T) {
if !reflect.DeepEqual(refs, []string{"e3", "e6", "e9", "e11"}) {
t.Errorf("children of e2 = %v", refs)
}
- if got := v.descendants(v.index["e6"]).nodes; len(got) != 2 || got[0].Ref != "e7" || got[1].Ref != "e8" {
- t.Errorf("descendants of e6 = %v", got)
+ if got := v.subtree(v.index["e6"]).nodes; len(got) != 3 || got[0].Ref != "e6" || got[2].Ref != "e8" {
+ t.Errorf("subtree of e6 = %v", got)
}
var anc []string
for _, a := range v.ancestors(v.index["e12"]) {
@@ -276,3 +276,35 @@ func TestResolveArray_GroupResolvingNoFieldIsNoRepeatedGroup(t *testing.T) {
t.Errorf("field = %+v", got.Fields["results"])
}
}
+
+func TestResolveArray_ItemNodeItselfCarriesTheField(t *testing.T) {
+ nodes := []observe.A11yNode{
+ {Ref: "e1", Role: "list", Depth: 0},
+ {Ref: "e2", Role: "link", Name: "Home", Depth: 1},
+ {Ref: "e3", Role: "link", Name: "Shop", Depth: 1},
+ {Ref: "e4", Role: "link", Name: "About", Depth: 1},
+ {Ref: "e5", Role: "link", Name: "Contact", Depth: 1},
+ }
+ schema := mustSchema(t, `{"type":"object","properties":{
+ "menu":{"type":"array","items":{"type":"object","properties":{
+ "title":{"type":"string","x-pinchtab-hint":"role:link"}}}}}}`)
+ got := Resolve(schema, nodes, Options{})
+ want := []map[string]any{{"title": "Home"}, {"title": "Shop"}, {"title": "About"}, {"title": "Contact"}}
+ if list := items(t, got, "menu"); !reflect.DeepEqual(list, want) {
+ t.Fatalf("menu = %v, want %v (%+v)", list, want, got.Fields["menu"])
+ }
+ if fr := got.Fields["menu"]; fr.Ref != "e1" || fr.Items[1].Ref != "e3" || fr.Items[1].Fields["title"].Ref != "e3" {
+ t.Errorf("refs = %+v", fr)
+ }
+}
+
+func TestResolveArray_CapReachedExactlyWithTrailingEmptyItemIsNotTruncated(t *testing.T) {
+ nodes := append(searchNodes(), observe.A11yNode{Ref: "e14", Role: "listitem", Depth: 2}, observe.A11yNode{Ref: "e15", Role: "text", Name: "Sponsored", Depth: 3})
+ schema := mustSchema(t, `{"type":"object","properties":{
+ "results":{"type":"array","maxItems":4,"items":{"type":"object","properties":{
+ "title":{"type":"string","x-pinchtab-hint":"role:link"}}}}}}`)
+ got := Resolve(schema, nodes, Options{})
+ if n := len(items(t, got, "results")); n != 4 || got.Fields["results"].Truncated {
+ t.Errorf("len=%d truncated=%v, want 4 items and no truncation when only an empty item follows", n, got.Fields["results"].Truncated)
+ }
+}