Skip to content

A continuation of HomeBox the inventory and organization system built for the Home User

feat: "ar" scanner functionality (#1421)

Mattbe25efe62a21
diff --git a/frontend/components/App/ScannerModal.vue b/frontend/components/App/ScannerModal.vue
index f15e63d3..de0fc99f 100644
--- a/frontend/components/App/ScannerModal.vue
+++ b/frontend/components/App/ScannerModal.vue
@@ -34,7 +34,7 @@
</div>
<!-- eslint-disable-next-line tailwindcss/no-custom-classname -->
<video ref="video" class="aspect-video w-full rounded-lg bg-muted shadow" poster="data:image/gif,AAAA" />
- <div class="mt-4">
+ <div class="mt-4 flex flex-col gap-3">
<Select v-model="selectedSource">
<SelectTrigger class="w-full">
<SelectValue :placeholder="t('scanner.select_video_source')" />
@@ -45,6 +45,10 @@
</SelectItem>
</SelectContent>
</Select>
+ <Button variant="outline" class="w-full" @click="openArMode">
+ <MdiCameraOutline class="mr-2" />
+ {{ t("scanner_ar.ar_mode") }}
+ </Button>
</div>
</div>
</DialogScrollContent>
@@ -61,6 +65,7 @@
import { Button, ButtonGroup } from "@/components/ui/button";
import MdiBarcode from "~icons/mdi/barcode";
import MdiAlertCircleOutline from "~icons/mdi/alert-circle-outline";
+ import MdiCameraOutline from "~icons/mdi/camera-outline";
import { useDialog } from "@/components/ui/dialog-provider";
const { t } = useI18n();
@@ -87,6 +92,11 @@
openDialog(DialogID.ProductImport, { params: { barcode: detectedBarcode.value } });
};
+ const openArMode = () => {
+ closeDialog(DialogID.Scanner);
+ navigateTo("/scanner-ar");
+ };
+
const startScanner = async () => {
errorMessage.value = null;
if (!(navigator && navigator.mediaDevices && "enumerateDevices" in navigator.mediaDevices)) {
diff --git a/frontend/components/Scanner/ARControls.vue b/frontend/components/Scanner/ARControls.vue
new file mode 100644
index 00000000..ea665013
--- /dev/null
+++ b/frontend/components/Scanner/ARControls.vue
@@ -0,0 +1,42 @@
+<template>
+ <div class="pointer-events-none absolute inset-0 z-20 flex flex-col justify-between">
+ <!-- Top bar -->
+ <div class="pointer-events-auto flex items-center gap-3 bg-background/60 p-3 backdrop-blur-sm">
+ <Button variant="ghost" size="icon" :aria-label="t('scanner_ar.back')" :title="t('scanner_ar.back')" @click="$emit('back')">
+ <MdiArrowLeft class="size-5" />
+ </Button>
+ <span class="text-sm font-semibold text-foreground">{{ t("scanner_ar.title") }}</span>
+ <div class="grow" />
+ <div v-if="isScanning" class="flex items-center gap-1.5">
+ <span class="size-2 animate-pulse rounded-full bg-green-500" />
+ <span class="text-xs text-muted-foreground">{{ t("scanner_ar.scanning") }}</span>
+ </div>
+ </div>
+
+ <!-- Bottom bar -->
+ <div class="pointer-events-auto flex items-center justify-center gap-4 bg-background/60 p-3 backdrop-blur-sm">
+ <Button variant="outline" size="sm" @click="$emit('switchCamera')">
+ <MdiCameraFlip class="mr-1.5 size-4" />
+ {{ t("scanner_ar.switch_camera") }}
+ </Button>
+ </div>
+ </div>
+</template>
+
+<script setup lang="ts">
+ import { useI18n } from "vue-i18n";
+ import { Button } from "@/components/ui/button";
+ import MdiArrowLeft from "~icons/mdi/arrow-left";
+ import MdiCameraFlip from "~icons/mdi/camera-flip";
+
+ defineProps<{
+ isScanning: boolean;
+ }>();
+
+ defineEmits<{
+ back: [];
+ switchCamera: [];
+ }>();
+
+ const { t } = useI18n();
+</script>
diff --git a/frontend/components/Scanner/AROverlayCard.vue b/frontend/components/Scanner/AROverlayCard.vue
new file mode 100644
index 00000000..a01dd3b5
--- /dev/null
+++ b/frontend/components/Scanner/AROverlayCard.vue
@@ -0,0 +1,225 @@
+<template>
+ <div
+ ref="cardRef"
+ class="absolute left-0 top-0 z-10 w-[250px] cursor-pointer rounded-lg border border-border bg-background/85 p-2 shadow-lg backdrop-blur-sm"
+ :style="cardStyle"
+ @click="handleClick"
+ >
+ <!-- Loading state -->
+ <div v-if="loading" class="flex flex-col gap-1.5">
+ <Skeleton class="h-4 w-28" />
+ <Skeleton class="h-3 w-20" />
+ </div>
+
+ <!-- Error state -->
+ <div v-else-if="error || !entity" class="text-xs text-destructive">
+ {{ t("scanner_ar.not_found") }}
+ </div>
+
+ <!-- Item with optional children -->
+ <div v-else-if="entity.item" class="flex flex-col gap-1">
+ <span class="truncate text-sm font-semibold text-foreground">{{ entity.item.name }}</span>
+ <div class="flex flex-wrap items-center gap-1.5">
+ <Badge v-if="entity.item.location" variant="secondary" class="text-xs">
+ {{ entity.item.location.name }}
+ </Badge>
+ <span v-if="entity.item.quantity > 1" class="text-xs text-muted-foreground">
+ {{ t("scanner_ar.qty") }}: {{ entity.item.quantity }}
+ </span>
+ </div>
+ <span v-if="entity.item.purchasePrice" class="text-xs text-muted-foreground">
+ {{ formattedPrice }}
+ </span>
+ <!-- Child items -->
+ <div v-if="entity.childItems && entity.childItems.length > 0" class="mt-1 border-t border-border pt-1">
+ <span class="text-xs font-medium text-muted-foreground">
+ {{ entity.childItems.length }} {{ t("scanner_ar.child_items", { count: entity.childItems.length }) }}:
+ </span>
+ <div
+ v-for="child in entity.childItems.slice(0, 3)"
+ :key="child.id"
+ class="truncate text-xs text-muted-foreground"
+ >
+ &bull; {{ child.name }}
+ </div>
+ <span v-if="entity.childItems.length > 3" class="text-xs text-muted-foreground">
+ +{{ entity.childItems.length - 3 }} {{ t("scanner_ar.more") }}
+ </span>
+ </div>
+ </div>
+
+ <!-- Location with items -->
+ <div v-else-if="entity.location" class="flex flex-col gap-1">
+ <span class="truncate text-sm font-semibold text-foreground">{{ entity.location.name }}</span>
+ <div class="flex flex-wrap items-center gap-1.5">
+ <span v-if="entity.location.children.length > 0" class="text-xs text-muted-foreground">
+ {{ entity.location.children.length }} {{ t("scanner_ar.children", { count: entity.location.children.length }) }}
+ </span>
+ </div>
+ <span v-if="entity.location.totalPrice" class="text-xs text-muted-foreground">
+ {{ formattedTotalPrice }}
+ </span>
+ <!-- Items in location -->
+ <div v-if="entity.childItems && entity.childItems.length > 0" class="mt-1 border-t border-border pt-1">
+ <span class="text-xs font-medium text-muted-foreground">
+ {{ entity.childItems.length }} {{ t("scanner_ar.items_in_location", { count: entity.childItems.length }) }}:
+ </span>
+ <div
+ v-for="child in entity.childItems.slice(0, 3)"
+ :key="child.id"
+ class="truncate text-xs text-muted-foreground"
+ >
+ &bull; {{ child.name }}
+ </div>
+ <span v-if="entity.childItems.length > 3" class="text-xs text-muted-foreground">
+ +{{ entity.childItems.length - 3 }} {{ t("scanner_ar.more") }}
+ </span>
+ </div>
+ </div>
+
+ <!-- Asset with multiple items (no single parent item) -->
+ <div v-else-if="entity.childItems && entity.childItems.length > 0" class="flex flex-col gap-1">
+ <span class="text-sm font-semibold text-foreground">
+ {{ entity.childItems.length }} {{ t("scanner_ar.items", { count: entity.childItems.length }) }}
+ </span>
+ <div
+ v-for="child in entity.childItems.slice(0, 3)"
+ :key="child.id"
+ class="truncate text-xs text-muted-foreground"
+ >
+ &bull; {{ child.name }}
+ </div>
+ <span v-if="entity.childItems.length > 3" class="text-xs text-muted-foreground">
+ +{{ entity.childItems.length - 3 }} {{ t("scanner_ar.more") }}
+ </span>
+ </div>
+ </div>
+</template>
+
+<script setup lang="ts">
+ import { computed, ref, watchEffect } from "vue";
+ import { useElementSize } from "@vueuse/core";
+ import { useI18n } from "vue-i18n";
+ import type { EntityData, Pose3D, Point2D } from "@/composables/use-barcode-detector";
+ import { solveHomography, homographyToMatrix3d } from "@/composables/use-barcode-detector";
+ import { Badge } from "@/components/ui/badge";
+ import { Skeleton } from "@/components/ui/skeleton";
+
+ const props = defineProps<{
+ position: DOMRect;
+ pose: Pose3D;
+ cornerPoints: Point2D[];
+ entity: EntityData | null;
+ entityType: "item" | "location" | "asset";
+ loading: boolean;
+ error: boolean;
+ }>();
+
+ const { t } = useI18n();
+
+ const formattedPrice = ref("");
+ const formattedTotalPrice = ref("");
+
+ const cardRef = ref<HTMLElement>();
+ const { width: cardW, height: cardH } = useElementSize(cardRef);
+
+ function ptDist(a: Point2D, b: Point2D): number {
+ return Math.sqrt((a.x - b.x) ** 2 + (a.y - b.y) ** 2);
+ }
+
+ const cardStyle = computed(() => {
+ const corners = props.cornerPoints;
+ const w = cardW.value;
+ const h = cardH.value;
+
+ // Need 4 corner points and a measured card size to compute homography
+ if (corners.length < 4 || !w || !h) {
+ // Fallback: simple translate positioning
+ const box = props.position;
+ const left = box.x + box.width / 2;
+ const top = box.y + box.height + 8;
+ return {
+ transformOrigin: "0 0",
+ transform: `translate(${left}px, ${top}px) translateX(-50%)`,
+ };
+ }
+
+ const [tl, tr, br, bl] = corners as [Point2D, Point2D, Point2D, Point2D];
+
+ // The QR code's left and right edge vectors define the "down" direction on the QR plane
+ const leftDir = { x: bl.x - tl.x, y: bl.y - tl.y };
+ const rightDir = { x: br.x - tr.x, y: br.y - tr.y };
+ const leftLen = ptDist(tl, bl);
+ const rightLen = ptDist(tr, br);
+
+ // Gap between QR bottom edge and card top edge (as fraction of edge length)
+ const gap = 8;
+ const gapL = leftLen > 0 ? gap / leftLen : 0;
+ const gapR = rightLen > 0 ? gap / rightLen : 0;
+
+ // How far to extend for the card's height, proportional to QR size
+ const avgEdge = (leftLen + rightLen) / 2 || 1;
+ const heightRatio = h / avgEdge;
+
+ // Destination corners: card sits below QR on the same perspective plane
+ // Top edge = QR bottom edge + small gap along perspective lines
+ // Bottom edge = further along the same perspective lines
+ const dstTL = { x: bl.x + leftDir.x * gapL, y: bl.y + leftDir.y * gapL };
+ const dstTR = { x: br.x + rightDir.x * gapR, y: br.y + rightDir.y * gapR };
+ const dstBL = {
+ x: bl.x + leftDir.x * (gapL + heightRatio),
+ y: bl.y + leftDir.y * (gapL + heightRatio),
+ };
+ const dstBR = {
+ x: br.x + rightDir.x * (gapR + heightRatio),
+ y: br.y + rightDir.y * (gapR + heightRatio),
+ };
+
+ // Source: card's natural rect (before transform)
+ const src: [Point2D, Point2D, Point2D, Point2D] = [
+ { x: 0, y: 0 },
+ { x: w, y: 0 },
+ { x: w, y: h },
+ { x: 0, y: h },
+ ];
+ const dst: [Point2D, Point2D, Point2D, Point2D] = [dstTL, dstTR, dstBR, dstBL];
+
+ const H = solveHomography(src, dst);
+ const matrix = homographyToMatrix3d(H);
+
+ return {
+ transformOrigin: "0 0",
+ transform: matrix,
+ };
+ });
+
+ let currencyFormatter: ((value: number | string) => string) | null = null;
+
+ watchEffect(async () => {
+ const item = props.entity?.item;
+ const location = props.entity?.location;
+
+ if (!item?.purchasePrice && !location?.totalPrice) {
+ formattedPrice.value = "";
+ formattedTotalPrice.value = "";
+ return;
+ }
+
+ if (!currencyFormatter) {
+ currencyFormatter = await useFormatCurrency();
+ }
+
+ formattedPrice.value = item?.purchasePrice ? currencyFormatter(item.purchasePrice) : "";
+ formattedTotalPrice.value = location?.totalPrice ? currencyFormatter(location.totalPrice) : "";
+ });
+
+ function handleClick() {
+ if (props.entity?.item?.id) {
+ navigateTo(`/item/${props.entity.item.id}`);
+ } else if (props.entity?.location?.id) {
+ navigateTo(`/location/${props.entity.location.id}`);
+ } else if (props.entity?.childItems && props.entity.childItems.length > 0) {
+ navigateTo(`/assets/${props.entity.childItems[0]!.assetId}`);
+ }
+ }
+</script>
diff --git a/frontend/components/Scanner/ARView.vue b/frontend/components/Scanner/ARView.vue
new file mode 100644
index 00000000..1af85d82
--- /dev/null
+++ b/frontend/components/Scanner/ARView.vue
@@ -0,0 +1,81 @@
+<template>
+ <div ref="container" class="relative h-dvh w-dvw overflow-hidden bg-black">
+ <!-- Camera feed -->
+ <video ref="videoEl" class="size-full object-cover" autoplay muted playsinline poster="data:image/gif,AAAA" />
+
+ <!-- Error state -->
+ <div v-if="error" class="absolute inset-0 flex items-center justify-center bg-background/90 p-6">
+ <div class="flex max-w-sm flex-col items-center gap-4 text-center">
+ <MdiAlertCircleOutline class="size-12 text-destructive" />
+ <p class="text-sm text-foreground">{{ t(error) }}</p>
+ <Button variant="outline" @click="startCamera">
+ {{ t("scanner_ar.retry") }}
+ </Button>
+ </div>
+ </div>
+
+ <!-- Overlay cards -->
+ <ScannerAROverlayCard
+ v-for="detection in detections"
+ :key="`${detection.entityType}:${detection.id}`"
+ :position="detection.boundingBox"
+ :pose="detection.pose"
+ :corner-points="detection.cornerPoints"
+ :entity="detection.data"
+ :entity-type="detection.entityType"
+ :loading="detection.loading"
+ :error="detection.error"
+ />
+
+ <!-- Controls -->
+ <ScannerARControls :is-scanning="isScanning" @back="handleBack" @switch-camera="switchCamera" />
+ </div>
+</template>
+
+<script setup lang="ts">
+ import { ref, onMounted, onBeforeUnmount, watch } from "vue";
+ import { useI18n } from "vue-i18n";
+ import { useBarcodeDetector } from "@/composables/use-barcode-detector";
+ import { Button } from "@/components/ui/button";
+ import ScannerAROverlayCard from "@/components/Scanner/AROverlayCard.vue";
+ import ScannerARControls from "@/components/Scanner/ARControls.vue";
+ import MdiAlertCircleOutline from "~icons/mdi/alert-circle-outline";
+
+ const { t } = useI18n();
+ const videoEl = ref<HTMLVideoElement>();
+ const container = ref<HTMLDivElement>();
+
+ const { isScanning, error, detections, startCamera, stopCamera, switchCamera } = useBarcodeDetector(videoEl);
+
+ let lastDetectionCount = 0;
+ watch(detections, val => {
+ if (val.length !== lastDetectionCount) {
+ console.debug(
+ "[AR:View] detections count changed:",
+ lastDetectionCount,
+ "->",
+ val.length,
+ val.map(d => `${d.entityType}:${d.id} loading=${d.loading} error=${d.error}`)
+ );
+ lastDetectionCount = val.length;
+ }
+ });
+
+ watch(error, val => {
+ if (val) console.error("[AR:View] error state:", val);
+ });
+
+ function handleBack() {
+ const router = useRouter();
+ router.back();
+ }
+
+ onMounted(() => {
+ console.debug("[AR:View] mounted, videoEl:", !!videoEl.value);
+ startCamera();
+ });
+
+ onBeforeUnmount(() => {
+ stopCamera();
+ });
+</script>
diff --git a/frontend/composables/use-barcode-detector.ts b/frontend/composables/use-barcode-detector.ts
new file mode 100644
index 00000000..4fbcc3b9
--- /dev/null
+++ b/frontend/composables/use-barcode-detector.ts
@@ -0,0 +1,665 @@
+import { ref, type Ref, watch, onUnmounted } from "vue";
+import { useDocumentVisibility } from "@vueuse/core";
+import type { ItemOut, ItemSummary, LocationOut } from "~~/lib/api/types/data-contracts";
+import type { WorkerResponse } from "~~/workers/barcode-detector";
+
+export interface EntityData {
+ item?: ItemOut;
+ location?: LocationOut;
+ childItems?: ItemSummary[];
+}
+
+export interface Point2D {
+ x: number;
+ y: number;
+}
+
+/** 3D pose derived from QR code corner points */
+export interface Pose3D {
+ centerX: number;
+ centerY: number;
+ rotateZ: number;
+ rotateX: number;
+ rotateY: number;
+ scale: number;
+}
+
+export interface DetectedEntity {
+ id: string;
+ entityType: "item" | "location" | "asset";
+ rawValue: string;
+ boundingBox: DOMRect;
+ cornerPoints: Point2D[];
+ pose: Pose3D;
+ data: EntityData | null;
+ loading: boolean;
+ error: boolean;
+ lastSeen: number;
+}
+
+interface CachedEntity {
+ data: EntityData;
+ fetchedAt: number;
+}
+
+const CACHE_TTL = 60_000;
+const STALE_TIMEOUT = 1500;
+const MAX_DETECTIONS = 10;
+const SMOOTH = 0.6;
+/** Minimum ms between detection requests sent to worker */
+const DETECT_INTERVAL = 50; // ~20fps detection, rAF loop runs at display rate for smooth lerp
+
+function lerpVal(prev: number, next: number, t: number): number {
+ return prev * t + next * (1 - t);
+}
+
+function lerpPoint(prev: Point2D, next: Point2D, t: number): Point2D {
+ return { x: lerpVal(prev.x, next.x, t), y: lerpVal(prev.y, next.y, t) };
+}
+
+function lerpRect(prev: DOMRect, next: DOMRect, t: number): DOMRect {
+ return new DOMRect(
+ lerpVal(prev.x, next.x, t),
+ lerpVal(prev.y, next.y, t),
+ lerpVal(prev.width, next.width, t),
+ lerpVal(prev.height, next.height, t)
+ );
+}
+
+function lerpCorners(prev: Point2D[], next: Point2D[], t: number): Point2D[] {
+ if (prev.length !== next.length) return next;
+ return prev.map((p, i) => lerpPoint(p, next[i]!, t));
+}
+
+function parseHomeboxUrl(rawValue: string): { entityType: "item" | "location" | "asset"; id: string } | null {
+ try {
+ let pathname: string;
+ try {
+ const url = new URL(rawValue);
+ if (url.origin !== globalThis.location?.origin) {
+ return null;
+ }
+ pathname = url.pathname;
+ } catch {
+ if (rawValue.startsWith("/")) {
+ pathname = rawValue;
+ } else {
+ return null;
+ }
+ }
+
+ const sanitized = pathname.replace(/[^a-zA-Z0-9-_/]/g, "");
+
+ const assetMatch = sanitized.match(/^\/a\/([a-zA-Z0-9-_]+)/);
+ if (assetMatch) return { entityType: "asset", id: assetMatch[1]! };
+
+ const itemMatch = sanitized.match(/^\/item\/([a-zA-Z0-9-_]+)/);
+ if (itemMatch) return { entityType: "item", id: itemMatch[1]! };
+
+ const locationMatch = sanitized.match(/^\/location\/([a-zA-Z0-9-_]+)/);
+ if (locationMatch) return { entityType: "location", id: locationMatch[1]! };
+
+ return null;
+ } catch {
+ return null;
+ }
+}
+
+interface VideoTransform {
+ scaleX: number;
+ scaleY: number;
+ offsetX: number;
+ offsetY: number;
+}
+
+function getVideoTransform(videoW: number, videoH: number, clientW: number, clientH: number): VideoTransform {
+ if (!videoW || !videoH || !clientW || !clientH) {
+ return { scaleX: 1, scaleY: 1, offsetX: 0, offsetY: 0 };
+ }
+
+ const videoAspect = videoW / videoH;
+ const clientAspect = clientW / clientH;
+
+ if (videoAspect > clientAspect) {
+ const s = clientH / videoH;
+ return { scaleX: s, scaleY: s, offsetX: (clientW - videoW * s) / 2, offsetY: 0 };
+ } else {
+ const s = clientW / videoW;
+ return { scaleX: s, scaleY: s, offsetX: 0, offsetY: (clientH - videoH * s) / 2 };
+ }
+}
+
+function mapBox(bbox: { x: number; y: number; width: number; height: number }, t: VideoTransform): DOMRect {
+ return new DOMRect(
+ bbox.x * t.scaleX + t.offsetX,
+ bbox.y * t.scaleY + t.offsetY,
+ bbox.width * t.scaleX,
+ bbox.height * t.scaleY
+ );
+}
+
+function mapCorners(corners: Array<{ x: number; y: number }>, t: VideoTransform): Point2D[] {
+ return corners.map(c => ({
+ x: c.x * t.scaleX + t.offsetX,
+ y: c.y * t.scaleY + t.offsetY,
+ }));
+}
+
+function ptDist(a: Point2D, b: Point2D): number {
+ return Math.sqrt((a.x - b.x) ** 2 + (a.y - b.y) ** 2);
+}
+
+const REFERENCE_QR_SIZE = 200;
+
+function computePose(corners: Point2D[]): Pose3D {
+ if (corners.length < 4) {
+ return { centerX: 0, centerY: 0, rotateZ: 0, rotateX: 0, rotateY: 0, scale: 1 };
+ }
+
+ const [tl, tr, br, bl] = corners as [Point2D, Point2D, Point2D, Point2D];
+
+ const centerX = (tl.x + tr.x + br.x + bl.x) / 4;
+ const centerY = (tl.y + tr.y + br.y + bl.y) / 4;
+ const rotateZ = Math.atan2(tr.y - tl.y, tr.x - tl.x) * (180 / Math.PI);
+
+ const topEdge = ptDist(tl, tr);
+ const bottomEdge = ptDist(bl, br);
+ const xRatio = bottomEdge / (topEdge || 1);
+ const rotateX = Math.max(-60, Math.min(60, (xRatio - 1) * 50));
+
+ const leftEdge = ptDist(tl, bl);
+ const rightEdge = ptDist(tr, br);
+ const yRatio = rightEdge / (leftEdge || 1);
+ const rotateY = Math.max(-60, Math.min(60, (1 - yRatio) * 50));
+
+ const avgEdge = (topEdge + bottomEdge + leftEdge + rightEdge) / 4;
+ const scale = Math.max(0.3, Math.min(2.0, avgEdge / REFERENCE_QR_SIZE));
+
+ return { centerX, centerY, rotateZ, rotateX, rotateY, scale };
+}
+
+export function solveHomography(
+ src: [Point2D, Point2D, Point2D, Point2D],
+ dst: [Point2D, Point2D, Point2D, Point2D]
+): number[] {
+ const aug: number[][] = [];
+ for (let i = 0; i < 4; i++) {
+ const { x, y } = src[i]!;
+ const { x: u, y: v } = dst[i]!;
+ aug.push([x, y, 1, 0, 0, 0, -x * u, -y * u, u]);
+ aug.push([0, 0, 0, x, y, 1, -x * v, -y * v, v]);
+ }
+
+ const n = 8;
+ for (let col = 0; col < n; col++) {
+ let maxRow = col;
+ for (let row = col + 1; row < n; row++) {
+ if (Math.abs(aug[row]![col]!) > Math.abs(aug[maxRow]![col]!)) maxRow = row;
+ }
+ [aug[col], aug[maxRow]] = [aug[maxRow]!, aug[col]!];
+ const pivot = aug[col]![col]!;
+ if (Math.abs(pivot) < 1e-10) return [1, 0, 0, 0, 1, 0, 0, 0, 1];
+ for (let row = col + 1; row < n; row++) {
+ const factor = aug[row]![col]! / pivot;
+ for (let j = col; j <= n; j++) {
+ aug[row]![j]! -= factor * aug[col]![j]!;
+ }
+ }
+ }
+
+ const h = new Array(n).fill(0);
+ for (let i = n - 1; i >= 0; i--) {
+ h[i] = aug[i]![n]!;
+ for (let j = i + 1; j < n; j++) {
+ h[i] -= aug[i]![j]! * h[j];
+ }
+ h[i] /= aug[i]![i]!;
+ }
+
+ return [...h, 1];
+}
+
+export function homographyToMatrix3d(H: number[]): string {
+ return `matrix3d(${H[0]}, ${H[3]}, 0, ${H[6]}, ${H[1]}, ${H[4]}, 0, ${H[7]}, 0, 0, 1, 0, ${H[2]}, ${H[5]}, 0, ${H[8]})`;
+}
+
+export function useBarcodeDetector(videoRef: Ref<HTMLVideoElement | undefined>) {
+ const isSupported = ref(false);
+ const isScanning = ref(false);
+ const error = ref<string | null>(null);
+ const detections = ref<DetectedEntity[]>([]);
+
+ let worker: Worker | null = null;
+ let stream: MediaStream | null = null;
+ let scanning = false;
+ let currentDeviceIndex = 0;
+ let videoDevices: MediaDeviceInfo[] = [];
+ let detectRequestId = 0;
+ let detectInFlight = false;
+ let lastDetectTime = 0;
+ /** Incremented on each new session; stale loops bail out after awaits */
+ let sessionId = 0;
+
+ const entityCache = new Map<string, CachedEntity>();
+ const pendingRequests = new Map<string, Promise<EntityData | null>>();
+ const trackedEntities = new Map<string, DetectedEntity>();
+
+ const visibility = useDocumentVisibility();
+ let wasScanning = false;
+
+ watch(visibility, vis => {
+ if (vis === "hidden" && scanning) {
+ wasScanning = true;
+ pauseDetection();
+ } else if (vis === "visible" && wasScanning) {
+ wasScanning = false;
+ resumeDetection();
+ }
+ });
+
+ function initWorker(): Promise<void> {
+ return new Promise((resolve, reject) => {
+ try {
+ worker = new Worker(new URL("~~/workers/barcode-detector.ts", import.meta.url), { type: "module" });
+
+ worker.onmessage = (event: MessageEvent<WorkerResponse>) => {
+ const msg = event.data;
+
+ if (msg.type === "ready") {
+ isSupported.value = true;
+ console.debug("[AR] Worker ready, BarcodeDetector initialized");
+ resolve();
+ } else if (msg.type === "error") {
+ console.error("[AR] Worker init error:", msg.message);
+ isSupported.value = false;
+ reject(new Error(msg.message));
+ } else if (msg.type === "result") {
+ handleDetectionResult(msg.barcodes);
+ }
+ };
+
+ worker.onerror = e => {
+ console.error("[AR] Worker error:", e);
+ isSupported.value = false;
+ reject(e);
+ };
+ } catch (e) {
+ console.error("[AR] Failed to create worker:", e);
+ reject(e);
+ }
+ });
+ }
+
+ function handleDetectionResult(
+ barcodes: Array<{
+ rawValue: string;
+ format: string;
+ boundingBox: { x: number; y: number; width: number; height: number };
+ cornerPoints: Array<{ x: number; y: number }>;
+ }>
+ ) {
+ detectInFlight = false;
+
+ const video = videoRef.value;
+ if (!video || !scanning) return;
+
+ const vt = getVideoTransform(video.videoWidth, video.videoHeight, video.clientWidth, video.clientHeight);
+ const now = Date.now();
+
+ for (const barcode of barcodes.slice(0, MAX_DETECTIONS)) {
+ const parsed = parseHomeboxUrl(barcode.rawValue);
+ if (!parsed) continue;
+
+ const key = `${parsed.entityType}:${parsed.id}`;
+ const mappedBox = mapBox(barcode.boundingBox, vt);
+ const mappedCorners = mapCorners(barcode.cornerPoints, vt);
+ const pose = computePose(mappedCorners);
+ const existing = trackedEntities.get(key);
+
+ if (existing) {
+ existing.boundingBox = lerpRect(existing.boundingBox, mappedBox, SMOOTH);
+ existing.cornerPoints = lerpCorners(existing.cornerPoints, mappedCorners, SMOOTH);
+ existing.pose = {
+ centerX: lerpVal(existing.pose.centerX, pose.centerX, SMOOTH),
+ centerY: lerpVal(existing.pose.centerY, pose.centerY, SMOOTH),
+ rotateZ: lerpVal(existing.pose.rotateZ, pose.rotateZ, SMOOTH),
+ rotateX: lerpVal(existing.pose.rotateX, pose.rotateX, SMOOTH),
+ rotateY: lerpVal(existing.pose.rotateY, pose.rotateY, SMOOTH),
+ scale: lerpVal(existing.pose.scale, pose.scale, SMOOTH),
+ };
+ existing.lastSeen = now;
+ } else {
+ console.debug("[AR] New entity detected:", key);
+ const entity: DetectedEntity = {
+ id: parsed.id,
+ entityType: parsed.entityType,
+ rawValue: barcode.rawValue,
+ boundingBox: mappedBox,
+ cornerPoints: mappedCorners,
+ pose,
+ data: null,
+ loading: true,
+ error: false,
+ lastSeen: now,
+ };
+ trackedEntities.set(key, entity);
+
+ fetchEntityData(parsed.entityType, parsed.id).then(data => {
+ const tracked = trackedEntities.get(key);
+ if (tracked) {
+ tracked.data = data;
+ tracked.loading = false;
+ tracked.error = data === null;
+ console.debug(
+ "[AR] Data fetched for",
+ key,
+ data
+ ? `item: ${data.item?.name ?? "-"}, location: ${data.location?.name ?? "-"}, children: ${data.childItems?.length ?? 0}`
+ : "not found"
+ );
+ updateDetections();
+ }
+ });
+ }
+ }
+
+ // Remove stale entries
+ for (const [key, entity] of trackedEntities) {
+ if (now - entity.lastSeen > STALE_TIMEOUT) {
+ trackedEntities.delete(key);
+ }
+ }
+
+ updateDetections();
+ }
+
+ async function fetchEntityData(entityType: "item" | "location" | "asset", id: string): Promise<EntityData | null> {
+ const cacheKey = `${entityType}:${id}`;
+
+ const cached = entityCache.get(cacheKey);
+ if (cached && Date.now() - cached.fetchedAt < CACHE_TTL) {
+ return cached.data;
+ }
+
+ const pending = pendingRequests.get(cacheKey);
+ if (pending) return pending;
+
+ const api = useUserApi();
+ const fetcher = async (): Promise<EntityData | null> => {
+ try {
+ if (entityType === "asset") {
+ const { data } = await api.assets.get(id);
+ if (data && data.items.length > 0) {
+ if (data.items.length === 1) {
+ const [itemRes, childRes] = await Promise.all([
+ api.items.get(data.items[0]!.id),
+ api.items.getAll({ parentIds: [data.items[0]!.id] }),
+ ]);
+ const result: EntityData = { item: itemRes.data ?? undefined, childItems: childRes.data?.items ?? [] };
+ entityCache.set(cacheKey, { data: result, fetchedAt: Date.now() });
+ return result;
+ }
+ const result: EntityData = { childItems: data.items };
+ entityCache.set(cacheKey, { data: result, fetchedAt: Date.now() });
+ return result;
+ }
+ return null;
+ }
+
+ if (entityType === "item") {
+ const [itemRes, childRes] = await Promise.all([api.items.get(id), api.items.getAll({ parentIds: [id] })]);
+ if (itemRes.data) {
+ const result: EntityData = { item: itemRes.data, childItems: childRes.data?.items ?? [] };
+ entityCache.set(cacheKey, { data: result, fetchedAt: Date.now() });
+ return result;
+ }
+ return null;
+ }
+
+ const [locRes, itemsRes] = await Promise.all([api.locations.get(id), api.items.getAll({ locations: [id] })]);
+ if (locRes.data) {
+ const result: EntityData = { location: locRes.data, childItems: itemsRes.data?.items ?? [] };
+ entityCache.set(cacheKey, { data: result, fetchedAt: Date.now() });
+ return result;
+ }
+ return null;
+ } catch {
+ return null;
+ } finally {
+ pendingRequests.delete(cacheKey);
+ }
+ };
+ const request = fetcher();
+
+ pendingRequests.set(cacheKey, request);
+ return request;
+ }
+
+ /** Tear down stream, worker, and detection without touching sessionId */
+ function teardown() {
+ stopDetection();
+ if (worker) {
+ worker.terminate();
+ worker = null;
+ }
+ if (stream) {
+ stream.getTracks().forEach(t => t.stop());
+ stream = null;
+ }
+ if (videoRef.value) {
+ videoRef.value.srcObject = null;
+ }
+ trackedEntities.clear();
+ detections.value = [];
+ }
+
+ async function startCamera() {
+ // Invalidate any previous session and tear down
+ const mySession = ++sessionId;
+ teardown();
+
+ error.value = null;
+ console.debug("[AR] startCamera called, session:", mySession);
+
+ if (!navigator?.mediaDevices?.getUserMedia) {
+ error.value = "scanner.unsupported";
+ return;
+ }
+
+ try {
+ stream = await navigator.mediaDevices.getUserMedia({
+ video: {
+ facingMode: { ideal: "environment" },
+ width: { ideal: 1920 },
+ height: { ideal: 1080 },
+ },
+ });
+ if (sessionId !== mySession) return; // superseded
+
+ console.debug("[AR] Camera stream acquired");
+
+ const devices = await navigator.mediaDevices.enumerateDevices();
+ if (sessionId !== mySession) return;
+ videoDevices = devices.filter(d => d.kind === "videoinput");
+
+ const currentTrack = stream.getVideoTracks()[0];
+ if (currentTrack) {
+ const settings = currentTrack.getSettings();
+ currentDeviceIndex = videoDevices.findIndex(d => d.deviceId === settings.deviceId);
+ if (currentDeviceIndex === -1) currentDeviceIndex = 0;
+ }
+
+ if (videoRef.value) {
+ videoRef.value.srcObject = stream;
+ videoRef.value.setAttribute("playsinline", "true");
+ await videoRef.value.play();
+ if (sessionId !== mySession) return;
+ console.debug("[AR] Video playing:", videoRef.value.videoWidth, "x", videoRef.value.videoHeight);
+ }
+
+ await initWorker();
+ if (sessionId !== mySession) return;
+
+ if (isSupported.value) {
+ startDetection(mySession);
+ } else {
+ error.value = "scanner_ar.unsupported";
+ }
+ } catch (err: unknown) {
+ if (sessionId !== mySession) return;
+ console.error("[AR] startCamera error:", err);
+ if (err instanceof Error && err.name === "NotAllowedError") {
+ error.value = "scanner.permission_denied";
+ } else {
+ error.value = "scanner.error";
+ }
+ }
+ }
+
+ function stopCamera() {
+ sessionId++;
+ teardown();
+ }
+
+ async function switchCamera() {
+ if (videoDevices.length < 2) return;
+
+ const mySession = ++sessionId;
+ currentDeviceIndex = (currentDeviceIndex + 1) % videoDevices.length;
+ const nextDevice = videoDevices[currentDeviceIndex];
+ if (!nextDevice) return;
+
+ stopDetection();
+ if (stream) {
+ stream.getTracks().forEach(t => t.stop());
+ }
+
+ try {
+ stream = await navigator.mediaDevices.getUserMedia({
+ video: { deviceId: { exact: nextDevice.deviceId } },
+ });
+ if (sessionId !== mySession) return;
+
+ if (videoRef.value) {
+ videoRef.value.srcObject = stream;
+ await videoRef.value.play();
+ if (sessionId !== mySession) return;
+ }
+
+ startDetection(mySession);
+ } catch {
+ if (sessionId !== mySession) return;
+ error.value = "scanner.error";
+ }
+ }
+
+ function startDetection(session: number) {
+ if (sessionId !== session) return;
+ scanning = true;
+ isScanning.value = true;
+ detectInFlight = false;
+ lastDetectTime = 0;
+ renderLoop(session);
+ }
+
+ function stopDetection() {
+ scanning = false;
+ isScanning.value = false;
+ }
+
+ function pauseDetection() {
+ scanning = false;
+ isScanning.value = false;
+ }
+
+ function resumeDetection() {
+ if (!stream || !worker) return;
+ scanning = true;
+ isScanning.value = true;
+ renderLoop(sessionId);
+ }
+
+ let frameCount = 0;
+ let lastLogTime = 0;
+
+ function renderLoop(session: number) {
+ if (!scanning || !videoRef.value || sessionId !== session) return;
+
+ const video = videoRef.value;
+ const now = Date.now();
+
+ // Log stats periodically
+ frameCount++;
+ if (now - lastLogTime > 3000) {
+ console.debug(
+ `[AR] Stats: ${frameCount} frames, ${trackedEntities.size} tracked, detect in-flight: ${detectInFlight}`
+ );
+ lastLogTime = now;
+ frameCount = 0;
+ }
+
+ // Send frame to worker for detection at throttled rate
+ if (!detectInFlight && worker && video.readyState >= 2 && now - lastDetectTime >= DETECT_INTERVAL) {
+ try {
+ const frame = createImageBitmap(video);
+ frame.then(bitmap => {
+ if (!worker || !scanning || sessionId !== session) {
+ bitmap.close();
+ return;
+ }
+ detectInFlight = true;
+ lastDetectTime = now;
+ detectRequestId++;
+ worker.postMessage({ type: "detect", frame: bitmap, id: detectRequestId }, [bitmap]);
+ });
+ } catch {
+ // createImageBitmap can fail if video not ready
+ }
+ }
+
+ // Remove stale entries every frame (so cards disappear promptly)
+ let removedStale = false;
+ for (const [key, entity] of trackedEntities) {
+ if (now - entity.lastSeen > STALE_TIMEOUT) {
+ trackedEntities.delete(key);
+ removedStale = true;
+ }
+ }
+ if (removedStale) {
+ updateDetections();
+ }
+
+ if (scanning && sessionId === session) {
+ requestAnimationFrame(() => renderLoop(session));
+ }
+ }
+
+ function updateDetections() {
+ detections.value = Array.from(trackedEntities.values()).map(e => ({
+ id: e.id,
+ entityType: e.entityType,
+ rawValue: e.rawValue,
+ boundingBox: new DOMRect(e.boundingBox.x, e.boundingBox.y, e.boundingBox.width, e.boundingBox.height),
+ cornerPoints: e.cornerPoints.map(p => ({ ...p })),
+ pose: { ...e.pose },
+ data: e.data,
+ loading: e.loading,
+ error: e.error,
+ lastSeen: e.lastSeen,
+ }));
+ }
+
+ onUnmounted(() => {
+ stopCamera();
+ });
+
+ return {
+ isSupported,
+ isScanning,
+ error,
+ detections,
+ startCamera,
+ stopCamera,
+ switchCamera,
+ };
+}
diff --git a/frontend/locales/en.json b/frontend/locales/en.json
index 47798c71..c0d18c6b 100644
--- a/frontend/locales/en.json
+++ b/frontend/locales/en.json
@@ -826,6 +826,22 @@
"title": "Scanner",
"unsupported": "Media Stream API is not supported without HTTPS"
},
+ "scanner_ar": {
+ "ar_mode": "AR Mode",
+ "back": "Back",
+ "child_items": "{count, plural, one {child item} other {child items}}",
+ "children": "{count, plural, one {sub-location} other {sub-locations}}",
+ "items": "{count, plural, one {item} other {items}}",
+ "items_in_location": "{count, plural, one {item} other {items}}",
+ "more": "more",
+ "not_found": "Item not found",
+ "qty": "Qty",
+ "retry": "Retry",
+ "scanning": "Scanning",
+ "switch_camera": "Switch Camera",
+ "title": "AR Scanner",
+ "unsupported": "AR scanning is not supported in this browser"
+ },
"tags": {
"no_results": "No Tags Found",
"tag_delete_confirm": "Are you sure you want to delete this tag? This action cannot be undone.",
diff --git a/frontend/package.json b/frontend/package.json
index 9e9c4f86..5da99c8e 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -65,6 +65,7 @@
"@vueuse/core": "^13.9.0",
"@vueuse/nuxt": "^13.9.0",
"@vueuse/router": "^13.9.0",
+ "barcode-detector": "^3.0.2",
"@zxing/library": "^0.21.3",
"autoprefixer": "^10.4.27",
"class-variance-authority": "^0.7.1",
diff --git a/frontend/pages/scanner-ar.vue b/frontend/pages/scanner-ar.vue
new file mode 100644
index 00000000..93cee430
--- /dev/null
+++ b/frontend/pages/scanner-ar.vue
@@ -0,0 +1,16 @@
+<template>
+ <ScannerARView />
+</template>
+
+<script setup lang="ts">
+ import ScannerARView from "@/components/Scanner/ARView.vue";
+
+ definePageMeta({
+ middleware: ["auth"],
+ layout: "empty",
+ });
+
+ useHead({
+ title: "HomeBox | AR Scanner",
+ });
+</script>
diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml
index c0c077ba..b5d876d7 100644
--- a/frontend/pnpm-lock.yaml
+++ b/frontend/pnpm-lock.yaml
@@ -76,7 +76,10 @@ importers:
version: 0.21.3
autoprefixer:
specifier: ^10.4.27
- version: 10.4.27([email protected])
+ version: 10.4.27([email protected])
+ barcode-detector:
+ specifier: ^3.0.2
+ version: 3.1.2(@types/[email protected])
class-variance-authority:
specifier: ^0.7.1
version: 0.7.1
@@ -2306,6 +2309,9 @@ packages:
resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==}
+
resolution: {integrity: sha512-03ruubjWyOHlmljCVoxSuNDdmfZDzsrrz0P2LeJsOXr+ZwFQ+0yQIwNCwt/GYhV7Z31fgtXJTAEs+FYlEL851g==}
@@ -2941,6 +2947,9 @@ packages:
resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==}
engines: {node: 18 || 20 || >=22}
+ resolution: {integrity: sha512-Q5kjXpVH5I3ItykNzbWmfWnNryFN1ZTWp10k9/PKJuS0RnoKR7jTrHEJODR4fn04bRomq7TJwie/Dr9fj/GoGQ==}
+
resolution: {integrity: sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==}
peerDependencies:
@@ -6772,6 +6781,11 @@ packages:
resolution: {integrity: sha512-dpvY17vxYIW3+bNrP0ClUlaiY0CiIRK3tnoLaGoQsQcY9/I/NpzIWQ7tQNhbV7LacQMpCII6wVzuL3tuWOyfuA==}
+ resolution: {integrity: sha512-2YMAriaYHX9wrBY2k7H0epSo+dyCaCZg/vOtt+nEDXM9ul480gkXz/9SkwpOeHcD2H5qqDG8lWDSBwpTcZpa6w==}
+ peerDependencies:
+ '@types/emscripten': '>=1.39.6'
+
snapshots:
'@alloc/[email protected]': {}
@@ -9105,6 +9119,8 @@ snapshots:
'@types/[email protected]': {}
+ '@types/[email protected]': {}
+
dependencies:
'@types/estree': 1.0.8
@@ -9902,6 +9918,12 @@ snapshots:
+ dependencies:
+ zxing-wasm: 3.0.2(@types/[email protected])
+ transitivePeerDependencies:
+ - '@types/emscripten'
+
@@ -14288,3 +14310,8 @@ snapshots:
readable-stream: 4.7.0
+
+ dependencies:
+ '@types/emscripten': 1.41.5
+ type-fest: 5.5.0
diff --git a/frontend/workers/barcode-detector.ts b/frontend/workers/barcode-detector.ts
new file mode 100644
index 00000000..92f3df8c
--- /dev/null
+++ b/frontend/workers/barcode-detector.ts
@@ -0,0 +1,77 @@
+import { BarcodeDetector } from "barcode-detector";
+
+interface DetectRequest {
+ type: "detect";
+ frame: ImageBitmap;
+ id: number;
+}
+
+interface DetectResult {
+ type: "result";
+ id: number;
+ barcodes: Array<{
+ rawValue: string;
+ format: string;
+ boundingBox: { x: number; y: number; width: number; height: number };
+ cornerPoints: Array<{ x: number; y: number }>;
+ }>;
+}
+
+interface ReadyMessage {
+ type: "ready";
+}
+
+interface ErrorMessage {
+ type: "error";
+ message: string;
+}
+
+export type WorkerMessage = DetectRequest;
+export type WorkerResponse = DetectResult | ReadyMessage | ErrorMessage;
+
+let detector: BarcodeDetector | null = null;
+
+async function init() {
+ try {
+ detector = new BarcodeDetector({ formats: ["qr_code"] });
+ self.postMessage({ type: "ready" } satisfies ReadyMessage);
+ } catch (e) {
+ self.postMessage({ type: "error", message: `BarcodeDetector init failed: ${e}` } satisfies ErrorMessage);
+ }
+}
+
+self.onmessage = async (event: MessageEvent<WorkerMessage>) => {
+ const msg = event.data;
+
+ if (msg.type === "detect") {
+ if (!detector) {
+ self.postMessage({ type: "result", id: msg.id, barcodes: [] } satisfies DetectResult);
+ msg.frame.close();
+ return;
+ }
+
+ try {
+ const results = await detector.detect(msg.frame);
+
+ const barcodes = results.map(b => ({
+ rawValue: b.rawValue,
+ format: b.format,
+ boundingBox: {
+ x: b.boundingBox.x,
+ y: b.boundingBox.y,
+ width: b.boundingBox.width,
+ height: b.boundingBox.height,
+ },
+ cornerPoints: Array.from(b.cornerPoints).map(p => ({ x: p.x, y: p.y })),
+ }));
+
+ self.postMessage({ type: "result", id: msg.id, barcodes } satisfies DetectResult);
+ } catch {
+ self.postMessage({ type: "result", id: msg.id, barcodes: [] } satisfies DetectResult);
+ } finally {
+ msg.frame.close();
+ }
+ }
+};
+
+init();