A continuation of HomeBox the inventory and organization system built for the Home User
feat: "ar" scanner functionality (#1421)
Mattbe25efe62a21
diff --git a/frontend/components/App/ScannerModal.vue b/frontend/components/App/ScannerModal.vueindex f15e63d3..de0fc99f 100644--- a/frontend/components/App/ScannerModal.vue+++ b/frontend/components/App/ScannerModal.vue@@ -34,7 +34,7 @@</div><!-- eslint-disable-next-line tailwindcss/no-custom-classname --><video ref="video" class="aspect-video w-full rounded-lg bg-muted shadow" poster="data:image/gif,AAAA" />- <div class="mt-4">+ <div class="mt-4 flex flex-col gap-3"><Select v-model="selectedSource"><SelectTrigger class="w-full"><SelectValue :placeholder="t('scanner.select_video_source')" />@@ -45,6 +45,10 @@</SelectItem></SelectContent></Select>+ <Button variant="outline" class="w-full" @click="openArMode">+ <MdiCameraOutline class="mr-2" />+ {{ t("scanner_ar.ar_mode") }}+ </Button></div></div></DialogScrollContent>@@ -61,6 +65,7 @@import { Button, ButtonGroup } from "@/components/ui/button";import MdiBarcode from "~icons/mdi/barcode";import MdiAlertCircleOutline from "~icons/mdi/alert-circle-outline";+ import MdiCameraOutline from "~icons/mdi/camera-outline";import { useDialog } from "@/components/ui/dialog-provider";const { t } = useI18n();@@ -87,6 +92,11 @@openDialog(DialogID.ProductImport, { params: { barcode: detectedBarcode.value } });};+ const openArMode = () => {+ closeDialog(DialogID.Scanner);+ navigateTo("/scanner-ar");+ };+const startScanner = async () => {errorMessage.value = null;if (!(navigator && navigator.mediaDevices && "enumerateDevices" in navigator.mediaDevices)) {diff --git a/frontend/components/Scanner/ARControls.vue b/frontend/components/Scanner/ARControls.vuenew file mode 100644index 00000000..ea665013--- /dev/null+++ b/frontend/components/Scanner/ARControls.vue@@ -0,0 +1,42 @@+<template>+ <div class="pointer-events-none absolute inset-0 z-20 flex flex-col justify-between">+ <!-- Top bar -->+ <div class="pointer-events-auto flex items-center gap-3 bg-background/60 p-3 backdrop-blur-sm">+ <Button variant="ghost" size="icon" :aria-label="t('scanner_ar.back')" :title="t('scanner_ar.back')" @click="$emit('back')">+ <MdiArrowLeft class="size-5" />+ </Button>+ <span class="text-sm font-semibold text-foreground">{{ t("scanner_ar.title") }}</span>+ <div class="grow" />+ <div v-if="isScanning" class="flex items-center gap-1.5">+ <span class="size-2 animate-pulse rounded-full bg-green-500" />+ <span class="text-xs text-muted-foreground">{{ t("scanner_ar.scanning") }}</span>+ </div>+ </div>++ <!-- Bottom bar -->+ <div class="pointer-events-auto flex items-center justify-center gap-4 bg-background/60 p-3 backdrop-blur-sm">+ <Button variant="outline" size="sm" @click="$emit('switchCamera')">+ <MdiCameraFlip class="mr-1.5 size-4" />+ {{ t("scanner_ar.switch_camera") }}+ </Button>+ </div>+ </div>+</template>++<script setup lang="ts">+ import { useI18n } from "vue-i18n";+ import { Button } from "@/components/ui/button";+ import MdiArrowLeft from "~icons/mdi/arrow-left";+ import MdiCameraFlip from "~icons/mdi/camera-flip";++ defineProps<{+ isScanning: boolean;+ }>();++ defineEmits<{+ back: [];+ switchCamera: [];+ }>();++ const { t } = useI18n();+</script>diff --git a/frontend/components/Scanner/AROverlayCard.vue b/frontend/components/Scanner/AROverlayCard.vuenew file mode 100644index 00000000..a01dd3b5--- /dev/null+++ b/frontend/components/Scanner/AROverlayCard.vue@@ -0,0 +1,225 @@+<template>+ <div+ ref="cardRef"+ class="absolute left-0 top-0 z-10 w-[250px] cursor-pointer rounded-lg border border-border bg-background/85 p-2 shadow-lg backdrop-blur-sm"+ :style="cardStyle"+ @click="handleClick"+ >+ <!-- Loading state -->+ <div v-if="loading" class="flex flex-col gap-1.5">+ <Skeleton class="h-4 w-28" />+ <Skeleton class="h-3 w-20" />+ </div>++ <!-- Error state -->+ <div v-else-if="error || !entity" class="text-xs text-destructive">+ {{ t("scanner_ar.not_found") }}+ </div>++ <!-- Item with optional children -->+ <div v-else-if="entity.item" class="flex flex-col gap-1">+ <span class="truncate text-sm font-semibold text-foreground">{{ entity.item.name }}</span>+ <div class="flex flex-wrap items-center gap-1.5">+ <Badge v-if="entity.item.location" variant="secondary" class="text-xs">+ {{ entity.item.location.name }}+ </Badge>+ <span v-if="entity.item.quantity > 1" class="text-xs text-muted-foreground">+ {{ t("scanner_ar.qty") }}: {{ entity.item.quantity }}+ </span>+ </div>+ <span v-if="entity.item.purchasePrice" class="text-xs text-muted-foreground">+ {{ formattedPrice }}+ </span>+ <!-- Child items -->+ <div v-if="entity.childItems && entity.childItems.length > 0" class="mt-1 border-t border-border pt-1">+ <span class="text-xs font-medium text-muted-foreground">+ {{ entity.childItems.length }} {{ t("scanner_ar.child_items", { count: entity.childItems.length }) }}:+ </span>+ <div+ v-for="child in entity.childItems.slice(0, 3)"+ :key="child.id"+ class="truncate text-xs text-muted-foreground"+ >+ • {{ child.name }}+ </div>+ <span v-if="entity.childItems.length > 3" class="text-xs text-muted-foreground">+ +{{ entity.childItems.length - 3 }} {{ t("scanner_ar.more") }}+ </span>+ </div>+ </div>++ <!-- Location with items -->+ <div v-else-if="entity.location" class="flex flex-col gap-1">+ <span class="truncate text-sm font-semibold text-foreground">{{ entity.location.name }}</span>+ <div class="flex flex-wrap items-center gap-1.5">+ <span v-if="entity.location.children.length > 0" class="text-xs text-muted-foreground">+ {{ entity.location.children.length }} {{ t("scanner_ar.children", { count: entity.location.children.length }) }}+ </span>+ </div>+ <span v-if="entity.location.totalPrice" class="text-xs text-muted-foreground">+ {{ formattedTotalPrice }}+ </span>+ <!-- Items in location -->+ <div v-if="entity.childItems && entity.childItems.length > 0" class="mt-1 border-t border-border pt-1">+ <span class="text-xs font-medium text-muted-foreground">+ {{ entity.childItems.length }} {{ t("scanner_ar.items_in_location", { count: entity.childItems.length }) }}:+ </span>+ <div+ v-for="child in entity.childItems.slice(0, 3)"+ :key="child.id"+ class="truncate text-xs text-muted-foreground"+ >+ • {{ child.name }}+ </div>+ <span v-if="entity.childItems.length > 3" class="text-xs text-muted-foreground">+ +{{ entity.childItems.length - 3 }} {{ t("scanner_ar.more") }}+ </span>+ </div>+ </div>++ <!-- Asset with multiple items (no single parent item) -->+ <div v-else-if="entity.childItems && entity.childItems.length > 0" class="flex flex-col gap-1">+ <span class="text-sm font-semibold text-foreground">+ {{ entity.childItems.length }} {{ t("scanner_ar.items", { count: entity.childItems.length }) }}+ </span>+ <div+ v-for="child in entity.childItems.slice(0, 3)"+ :key="child.id"+ class="truncate text-xs text-muted-foreground"+ >+ • {{ child.name }}+ </div>+ <span v-if="entity.childItems.length > 3" class="text-xs text-muted-foreground">+ +{{ entity.childItems.length - 3 }} {{ t("scanner_ar.more") }}+ </span>+ </div>+ </div>+</template>++<script setup lang="ts">+ import { computed, ref, watchEffect } from "vue";+ import { useElementSize } from "@vueuse/core";+ import { useI18n } from "vue-i18n";+ import type { EntityData, Pose3D, Point2D } from "@/composables/use-barcode-detector";+ import { solveHomography, homographyToMatrix3d } from "@/composables/use-barcode-detector";+ import { Badge } from "@/components/ui/badge";+ import { Skeleton } from "@/components/ui/skeleton";++ const props = defineProps<{+ position: DOMRect;+ pose: Pose3D;+ cornerPoints: Point2D[];+ entity: EntityData | null;+ entityType: "item" | "location" | "asset";+ loading: boolean;+ error: boolean;+ }>();++ const { t } = useI18n();++ const formattedPrice = ref("");+ const formattedTotalPrice = ref("");++ const cardRef = ref<HTMLElement>();+ const { width: cardW, height: cardH } = useElementSize(cardRef);++ function ptDist(a: Point2D, b: Point2D): number {+ return Math.sqrt((a.x - b.x) ** 2 + (a.y - b.y) ** 2);+ }++ const cardStyle = computed(() => {+ const corners = props.cornerPoints;+ const w = cardW.value;+ const h = cardH.value;++ // Need 4 corner points and a measured card size to compute homography+ if (corners.length < 4 || !w || !h) {+ // Fallback: simple translate positioning+ const box = props.position;+ const left = box.x + box.width / 2;+ const top = box.y + box.height + 8;+ return {+ transformOrigin: "0 0",+ transform: `translate(${left}px, ${top}px) translateX(-50%)`,+ };+ }++ const [tl, tr, br, bl] = corners as [Point2D, Point2D, Point2D, Point2D];++ // The QR code's left and right edge vectors define the "down" direction on the QR plane+ const leftDir = { x: bl.x - tl.x, y: bl.y - tl.y };+ const rightDir = { x: br.x - tr.x, y: br.y - tr.y };+ const leftLen = ptDist(tl, bl);+ const rightLen = ptDist(tr, br);++ // Gap between QR bottom edge and card top edge (as fraction of edge length)+ const gap = 8;+ const gapL = leftLen > 0 ? gap / leftLen : 0;+ const gapR = rightLen > 0 ? gap / rightLen : 0;++ // How far to extend for the card's height, proportional to QR size+ const avgEdge = (leftLen + rightLen) / 2 || 1;+ const heightRatio = h / avgEdge;++ // Destination corners: card sits below QR on the same perspective plane+ // Top edge = QR bottom edge + small gap along perspective lines+ // Bottom edge = further along the same perspective lines+ const dstTL = { x: bl.x + leftDir.x * gapL, y: bl.y + leftDir.y * gapL };+ const dstTR = { x: br.x + rightDir.x * gapR, y: br.y + rightDir.y * gapR };+ const dstBL = {+ x: bl.x + leftDir.x * (gapL + heightRatio),+ y: bl.y + leftDir.y * (gapL + heightRatio),+ };+ const dstBR = {+ x: br.x + rightDir.x * (gapR + heightRatio),+ y: br.y + rightDir.y * (gapR + heightRatio),+ };++ // Source: card's natural rect (before transform)+ const src: [Point2D, Point2D, Point2D, Point2D] = [+ { x: 0, y: 0 },+ { x: w, y: 0 },+ { x: w, y: h },+ { x: 0, y: h },+ ];+ const dst: [Point2D, Point2D, Point2D, Point2D] = [dstTL, dstTR, dstBR, dstBL];++ const H = solveHomography(src, dst);+ const matrix = homographyToMatrix3d(H);++ return {+ transformOrigin: "0 0",+ transform: matrix,+ };+ });++ let currencyFormatter: ((value: number | string) => string) | null = null;++ watchEffect(async () => {+ const item = props.entity?.item;+ const location = props.entity?.location;++ if (!item?.purchasePrice && !location?.totalPrice) {+ formattedPrice.value = "";+ formattedTotalPrice.value = "";+ return;+ }++ if (!currencyFormatter) {+ currencyFormatter = await useFormatCurrency();+ }++ formattedPrice.value = item?.purchasePrice ? currencyFormatter(item.purchasePrice) : "";+ formattedTotalPrice.value = location?.totalPrice ? currencyFormatter(location.totalPrice) : "";+ });++ function handleClick() {+ if (props.entity?.item?.id) {+ navigateTo(`/item/${props.entity.item.id}`);+ } else if (props.entity?.location?.id) {+ navigateTo(`/location/${props.entity.location.id}`);+ } else if (props.entity?.childItems && props.entity.childItems.length > 0) {+ navigateTo(`/assets/${props.entity.childItems[0]!.assetId}`);+ }+ }+</script>diff --git a/frontend/components/Scanner/ARView.vue b/frontend/components/Scanner/ARView.vuenew file mode 100644index 00000000..1af85d82--- /dev/null+++ b/frontend/components/Scanner/ARView.vue@@ -0,0 +1,81 @@+<template>+ <div ref="container" class="relative h-dvh w-dvw overflow-hidden bg-black">+ <!-- Camera feed -->+ <video ref="videoEl" class="size-full object-cover" autoplay muted playsinline poster="data:image/gif,AAAA" />++ <!-- Error state -->+ <div v-if="error" class="absolute inset-0 flex items-center justify-center bg-background/90 p-6">+ <div class="flex max-w-sm flex-col items-center gap-4 text-center">+ <MdiAlertCircleOutline class="size-12 text-destructive" />+ <p class="text-sm text-foreground">{{ t(error) }}</p>+ <Button variant="outline" @click="startCamera">+ {{ t("scanner_ar.retry") }}+ </Button>+ </div>+ </div>++ <!-- Overlay cards -->+ <ScannerAROverlayCard+ v-for="detection in detections"+ :key="`${detection.entityType}:${detection.id}`"+ :position="detection.boundingBox"+ :pose="detection.pose"+ :corner-points="detection.cornerPoints"+ :entity="detection.data"+ :entity-type="detection.entityType"+ :loading="detection.loading"+ :error="detection.error"+ />++ <!-- Controls -->+ <ScannerARControls :is-scanning="isScanning" @back="handleBack" @switch-camera="switchCamera" />+ </div>+</template>++<script setup lang="ts">+ import { ref, onMounted, onBeforeUnmount, watch } from "vue";+ import { useI18n } from "vue-i18n";+ import { useBarcodeDetector } from "@/composables/use-barcode-detector";+ import { Button } from "@/components/ui/button";+ import ScannerAROverlayCard from "@/components/Scanner/AROverlayCard.vue";+ import ScannerARControls from "@/components/Scanner/ARControls.vue";+ import MdiAlertCircleOutline from "~icons/mdi/alert-circle-outline";++ const { t } = useI18n();+ const videoEl = ref<HTMLVideoElement>();+ const container = ref<HTMLDivElement>();++ const { isScanning, error, detections, startCamera, stopCamera, switchCamera } = useBarcodeDetector(videoEl);++ let lastDetectionCount = 0;+ watch(detections, val => {+ if (val.length !== lastDetectionCount) {+ console.debug(+ "[AR:View] detections count changed:",+ lastDetectionCount,+ "->",+ val.length,+ val.map(d => `${d.entityType}:${d.id} loading=${d.loading} error=${d.error}`)+ );+ lastDetectionCount = val.length;+ }+ });++ watch(error, val => {+ if (val) console.error("[AR:View] error state:", val);+ });++ function handleBack() {+ const router = useRouter();+ router.back();+ }++ onMounted(() => {+ console.debug("[AR:View] mounted, videoEl:", !!videoEl.value);+ startCamera();+ });++ onBeforeUnmount(() => {+ stopCamera();+ });+</script>diff --git a/frontend/composables/use-barcode-detector.ts b/frontend/composables/use-barcode-detector.tsnew file mode 100644index 00000000..4fbcc3b9--- /dev/null+++ b/frontend/composables/use-barcode-detector.ts@@ -0,0 +1,665 @@+import { ref, type Ref, watch, onUnmounted } from "vue";+import { useDocumentVisibility } from "@vueuse/core";+import type { ItemOut, ItemSummary, LocationOut } from "~~/lib/api/types/data-contracts";+import type { WorkerResponse } from "~~/workers/barcode-detector";++export interface EntityData {+ item?: ItemOut;+ location?: LocationOut;+ childItems?: ItemSummary[];+}++export interface Point2D {+ x: number;+ y: number;+}++/** 3D pose derived from QR code corner points */+export interface Pose3D {+ centerX: number;+ centerY: number;+ rotateZ: number;+ rotateX: number;+ rotateY: number;+ scale: number;+}++export interface DetectedEntity {+ id: string;+ entityType: "item" | "location" | "asset";+ rawValue: string;+ boundingBox: DOMRect;+ cornerPoints: Point2D[];+ pose: Pose3D;+ data: EntityData | null;+ loading: boolean;+ error: boolean;+ lastSeen: number;+}++interface CachedEntity {+ data: EntityData;+ fetchedAt: number;+}++const CACHE_TTL = 60_000;+const STALE_TIMEOUT = 1500;+const MAX_DETECTIONS = 10;+const SMOOTH = 0.6;+/** Minimum ms between detection requests sent to worker */+const DETECT_INTERVAL = 50; // ~20fps detection, rAF loop runs at display rate for smooth lerp++function lerpVal(prev: number, next: number, t: number): number {+ return prev * t + next * (1 - t);+}++function lerpPoint(prev: Point2D, next: Point2D, t: number): Point2D {+ return { x: lerpVal(prev.x, next.x, t), y: lerpVal(prev.y, next.y, t) };+}++function lerpRect(prev: DOMRect, next: DOMRect, t: number): DOMRect {+ return new DOMRect(+ lerpVal(prev.x, next.x, t),+ lerpVal(prev.y, next.y, t),+ lerpVal(prev.width, next.width, t),+ lerpVal(prev.height, next.height, t)+ );+}++function lerpCorners(prev: Point2D[], next: Point2D[], t: number): Point2D[] {+ if (prev.length !== next.length) return next;+ return prev.map((p, i) => lerpPoint(p, next[i]!, t));+}++function parseHomeboxUrl(rawValue: string): { entityType: "item" | "location" | "asset"; id: string } | null {+ try {+ let pathname: string;+ try {+ const url = new URL(rawValue);+ if (url.origin !== globalThis.location?.origin) {+ return null;+ }+ pathname = url.pathname;+ } catch {+ if (rawValue.startsWith("/")) {+ pathname = rawValue;+ } else {+ return null;+ }+ }++ const sanitized = pathname.replace(/[^a-zA-Z0-9-_/]/g, "");++ const assetMatch = sanitized.match(/^\/a\/([a-zA-Z0-9-_]+)/);+ if (assetMatch) return { entityType: "asset", id: assetMatch[1]! };++ const itemMatch = sanitized.match(/^\/item\/([a-zA-Z0-9-_]+)/);+ if (itemMatch) return { entityType: "item", id: itemMatch[1]! };++ const locationMatch = sanitized.match(/^\/location\/([a-zA-Z0-9-_]+)/);+ if (locationMatch) return { entityType: "location", id: locationMatch[1]! };++ return null;+ } catch {+ return null;+ }+}++interface VideoTransform {+ scaleX: number;+ scaleY: number;+ offsetX: number;+ offsetY: number;+}++function getVideoTransform(videoW: number, videoH: number, clientW: number, clientH: number): VideoTransform {+ if (!videoW || !videoH || !clientW || !clientH) {+ return { scaleX: 1, scaleY: 1, offsetX: 0, offsetY: 0 };+ }++ const videoAspect = videoW / videoH;+ const clientAspect = clientW / clientH;++ if (videoAspect > clientAspect) {+ const s = clientH / videoH;+ return { scaleX: s, scaleY: s, offsetX: (clientW - videoW * s) / 2, offsetY: 0 };+ } else {+ const s = clientW / videoW;+ return { scaleX: s, scaleY: s, offsetX: 0, offsetY: (clientH - videoH * s) / 2 };+ }+}++function mapBox(bbox: { x: number; y: number; width: number; height: number }, t: VideoTransform): DOMRect {+ return new DOMRect(+ bbox.x * t.scaleX + t.offsetX,+ bbox.y * t.scaleY + t.offsetY,+ bbox.width * t.scaleX,+ bbox.height * t.scaleY+ );+}++function mapCorners(corners: Array<{ x: number; y: number }>, t: VideoTransform): Point2D[] {+ return corners.map(c => ({+ x: c.x * t.scaleX + t.offsetX,+ y: c.y * t.scaleY + t.offsetY,+ }));+}++function ptDist(a: Point2D, b: Point2D): number {+ return Math.sqrt((a.x - b.x) ** 2 + (a.y - b.y) ** 2);+}++const REFERENCE_QR_SIZE = 200;++function computePose(corners: Point2D[]): Pose3D {+ if (corners.length < 4) {+ return { centerX: 0, centerY: 0, rotateZ: 0, rotateX: 0, rotateY: 0, scale: 1 };+ }++ const [tl, tr, br, bl] = corners as [Point2D, Point2D, Point2D, Point2D];++ const centerX = (tl.x + tr.x + br.x + bl.x) / 4;+ const centerY = (tl.y + tr.y + br.y + bl.y) / 4;+ const rotateZ = Math.atan2(tr.y - tl.y, tr.x - tl.x) * (180 / Math.PI);++ const topEdge = ptDist(tl, tr);+ const bottomEdge = ptDist(bl, br);+ const xRatio = bottomEdge / (topEdge || 1);+ const rotateX = Math.max(-60, Math.min(60, (xRatio - 1) * 50));++ const leftEdge = ptDist(tl, bl);+ const rightEdge = ptDist(tr, br);+ const yRatio = rightEdge / (leftEdge || 1);+ const rotateY = Math.max(-60, Math.min(60, (1 - yRatio) * 50));++ const avgEdge = (topEdge + bottomEdge + leftEdge + rightEdge) / 4;+ const scale = Math.max(0.3, Math.min(2.0, avgEdge / REFERENCE_QR_SIZE));++ return { centerX, centerY, rotateZ, rotateX, rotateY, scale };+}++export function solveHomography(+ src: [Point2D, Point2D, Point2D, Point2D],+ dst: [Point2D, Point2D, Point2D, Point2D]+): number[] {+ const aug: number[][] = [];+ for (let i = 0; i < 4; i++) {+ const { x, y } = src[i]!;+ const { x: u, y: v } = dst[i]!;+ aug.push([x, y, 1, 0, 0, 0, -x * u, -y * u, u]);+ aug.push([0, 0, 0, x, y, 1, -x * v, -y * v, v]);+ }++ const n = 8;+ for (let col = 0; col < n; col++) {+ let maxRow = col;+ for (let row = col + 1; row < n; row++) {+ if (Math.abs(aug[row]![col]!) > Math.abs(aug[maxRow]![col]!)) maxRow = row;+ }+ [aug[col], aug[maxRow]] = [aug[maxRow]!, aug[col]!];+ const pivot = aug[col]![col]!;+ if (Math.abs(pivot) < 1e-10) return [1, 0, 0, 0, 1, 0, 0, 0, 1];+ for (let row = col + 1; row < n; row++) {+ const factor = aug[row]![col]! / pivot;+ for (let j = col; j <= n; j++) {+ aug[row]![j]! -= factor * aug[col]![j]!;+ }+ }+ }++ const h = new Array(n).fill(0);+ for (let i = n - 1; i >= 0; i--) {+ h[i] = aug[i]![n]!;+ for (let j = i + 1; j < n; j++) {+ h[i] -= aug[i]![j]! * h[j];+ }+ h[i] /= aug[i]![i]!;+ }++ return [...h, 1];+}++export function homographyToMatrix3d(H: number[]): string {+ return `matrix3d(${H[0]}, ${H[3]}, 0, ${H[6]}, ${H[1]}, ${H[4]}, 0, ${H[7]}, 0, 0, 1, 0, ${H[2]}, ${H[5]}, 0, ${H[8]})`;+}++export function useBarcodeDetector(videoRef: Ref<HTMLVideoElement | undefined>) {+ const isSupported = ref(false);+ const isScanning = ref(false);+ const error = ref<string | null>(null);+ const detections = ref<DetectedEntity[]>([]);++ let worker: Worker | null = null;+ let stream: MediaStream | null = null;+ let scanning = false;+ let currentDeviceIndex = 0;+ let videoDevices: MediaDeviceInfo[] = [];+ let detectRequestId = 0;+ let detectInFlight = false;+ let lastDetectTime = 0;+ /** Incremented on each new session; stale loops bail out after awaits */+ let sessionId = 0;++ const entityCache = new Map<string, CachedEntity>();+ const pendingRequests = new Map<string, Promise<EntityData | null>>();+ const trackedEntities = new Map<string, DetectedEntity>();++ const visibility = useDocumentVisibility();+ let wasScanning = false;++ watch(visibility, vis => {+ if (vis === "hidden" && scanning) {+ wasScanning = true;+ pauseDetection();+ } else if (vis === "visible" && wasScanning) {+ wasScanning = false;+ resumeDetection();+ }+ });++ function initWorker(): Promise<void> {+ return new Promise((resolve, reject) => {+ try {+ worker = new Worker(new URL("~~/workers/barcode-detector.ts", import.meta.url), { type: "module" });++ worker.onmessage = (event: MessageEvent<WorkerResponse>) => {+ const msg = event.data;++ if (msg.type === "ready") {+ isSupported.value = true;+ console.debug("[AR] Worker ready, BarcodeDetector initialized");+ resolve();+ } else if (msg.type === "error") {+ console.error("[AR] Worker init error:", msg.message);+ isSupported.value = false;+ reject(new Error(msg.message));+ } else if (msg.type === "result") {+ handleDetectionResult(msg.barcodes);+ }+ };++ worker.onerror = e => {+ console.error("[AR] Worker error:", e);+ isSupported.value = false;+ reject(e);+ };+ } catch (e) {+ console.error("[AR] Failed to create worker:", e);+ reject(e);+ }+ });+ }++ function handleDetectionResult(+ barcodes: Array<{+ rawValue: string;+ format: string;+ boundingBox: { x: number; y: number; width: number; height: number };+ cornerPoints: Array<{ x: number; y: number }>;+ }>+ ) {+ detectInFlight = false;++ const video = videoRef.value;+ if (!video || !scanning) return;++ const vt = getVideoTransform(video.videoWidth, video.videoHeight, video.clientWidth, video.clientHeight);+ const now = Date.now();++ for (const barcode of barcodes.slice(0, MAX_DETECTIONS)) {+ const parsed = parseHomeboxUrl(barcode.rawValue);+ if (!parsed) continue;++ const key = `${parsed.entityType}:${parsed.id}`;+ const mappedBox = mapBox(barcode.boundingBox, vt);+ const mappedCorners = mapCorners(barcode.cornerPoints, vt);+ const pose = computePose(mappedCorners);+ const existing = trackedEntities.get(key);++ if (existing) {+ existing.boundingBox = lerpRect(existing.boundingBox, mappedBox, SMOOTH);+ existing.cornerPoints = lerpCorners(existing.cornerPoints, mappedCorners, SMOOTH);+ existing.pose = {+ centerX: lerpVal(existing.pose.centerX, pose.centerX, SMOOTH),+ centerY: lerpVal(existing.pose.centerY, pose.centerY, SMOOTH),+ rotateZ: lerpVal(existing.pose.rotateZ, pose.rotateZ, SMOOTH),+ rotateX: lerpVal(existing.pose.rotateX, pose.rotateX, SMOOTH),+ rotateY: lerpVal(existing.pose.rotateY, pose.rotateY, SMOOTH),+ scale: lerpVal(existing.pose.scale, pose.scale, SMOOTH),+ };+ existing.lastSeen = now;+ } else {+ console.debug("[AR] New entity detected:", key);+ const entity: DetectedEntity = {+ id: parsed.id,+ entityType: parsed.entityType,+ rawValue: barcode.rawValue,+ boundingBox: mappedBox,+ cornerPoints: mappedCorners,+ pose,+ data: null,+ loading: true,+ error: false,+ lastSeen: now,+ };+ trackedEntities.set(key, entity);++ fetchEntityData(parsed.entityType, parsed.id).then(data => {+ const tracked = trackedEntities.get(key);+ if (tracked) {+ tracked.data = data;+ tracked.loading = false;+ tracked.error = data === null;+ console.debug(+ "[AR] Data fetched for",+ key,+ data+ ? `item: ${data.item?.name ?? "-"}, location: ${data.location?.name ?? "-"}, children: ${data.childItems?.length ?? 0}`+ : "not found"+ );+ updateDetections();+ }+ });+ }+ }++ // Remove stale entries+ for (const [key, entity] of trackedEntities) {+ if (now - entity.lastSeen > STALE_TIMEOUT) {+ trackedEntities.delete(key);+ }+ }++ updateDetections();+ }++ async function fetchEntityData(entityType: "item" | "location" | "asset", id: string): Promise<EntityData | null> {+ const cacheKey = `${entityType}:${id}`;++ const cached = entityCache.get(cacheKey);+ if (cached && Date.now() - cached.fetchedAt < CACHE_TTL) {+ return cached.data;+ }++ const pending = pendingRequests.get(cacheKey);+ if (pending) return pending;++ const api = useUserApi();+ const fetcher = async (): Promise<EntityData | null> => {+ try {+ if (entityType === "asset") {+ const { data } = await api.assets.get(id);+ if (data && data.items.length > 0) {+ if (data.items.length === 1) {+ const [itemRes, childRes] = await Promise.all([+ api.items.get(data.items[0]!.id),+ api.items.getAll({ parentIds: [data.items[0]!.id] }),+ ]);+ const result: EntityData = { item: itemRes.data ?? undefined, childItems: childRes.data?.items ?? [] };+ entityCache.set(cacheKey, { data: result, fetchedAt: Date.now() });+ return result;+ }+ const result: EntityData = { childItems: data.items };+ entityCache.set(cacheKey, { data: result, fetchedAt: Date.now() });+ return result;+ }+ return null;+ }++ if (entityType === "item") {+ const [itemRes, childRes] = await Promise.all([api.items.get(id), api.items.getAll({ parentIds: [id] })]);+ if (itemRes.data) {+ const result: EntityData = { item: itemRes.data, childItems: childRes.data?.items ?? [] };+ entityCache.set(cacheKey, { data: result, fetchedAt: Date.now() });+ return result;+ }+ return null;+ }++ const [locRes, itemsRes] = await Promise.all([api.locations.get(id), api.items.getAll({ locations: [id] })]);+ if (locRes.data) {+ const result: EntityData = { location: locRes.data, childItems: itemsRes.data?.items ?? [] };+ entityCache.set(cacheKey, { data: result, fetchedAt: Date.now() });+ return result;+ }+ return null;+ } catch {+ return null;+ } finally {+ pendingRequests.delete(cacheKey);+ }+ };+ const request = fetcher();++ pendingRequests.set(cacheKey, request);+ return request;+ }++ /** Tear down stream, worker, and detection without touching sessionId */+ function teardown() {+ stopDetection();+ if (worker) {+ worker.terminate();+ worker = null;+ }+ if (stream) {+ stream.getTracks().forEach(t => t.stop());+ stream = null;+ }+ if (videoRef.value) {+ videoRef.value.srcObject = null;+ }+ trackedEntities.clear();+ detections.value = [];+ }++ async function startCamera() {+ // Invalidate any previous session and tear down+ const mySession = ++sessionId;+ teardown();++ error.value = null;+ console.debug("[AR] startCamera called, session:", mySession);++ if (!navigator?.mediaDevices?.getUserMedia) {+ error.value = "scanner.unsupported";+ return;+ }++ try {+ stream = await navigator.mediaDevices.getUserMedia({+ video: {+ facingMode: { ideal: "environment" },+ width: { ideal: 1920 },+ height: { ideal: 1080 },+ },+ });+ if (sessionId !== mySession) return; // superseded++ console.debug("[AR] Camera stream acquired");++ const devices = await navigator.mediaDevices.enumerateDevices();+ if (sessionId !== mySession) return;+ videoDevices = devices.filter(d => d.kind === "videoinput");++ const currentTrack = stream.getVideoTracks()[0];+ if (currentTrack) {+ const settings = currentTrack.getSettings();+ currentDeviceIndex = videoDevices.findIndex(d => d.deviceId === settings.deviceId);+ if (currentDeviceIndex === -1) currentDeviceIndex = 0;+ }++ if (videoRef.value) {+ videoRef.value.srcObject = stream;+ videoRef.value.setAttribute("playsinline", "true");+ await videoRef.value.play();+ if (sessionId !== mySession) return;+ console.debug("[AR] Video playing:", videoRef.value.videoWidth, "x", videoRef.value.videoHeight);+ }++ await initWorker();+ if (sessionId !== mySession) return;++ if (isSupported.value) {+ startDetection(mySession);+ } else {+ error.value = "scanner_ar.unsupported";+ }+ } catch (err: unknown) {+ if (sessionId !== mySession) return;+ console.error("[AR] startCamera error:", err);+ if (err instanceof Error && err.name === "NotAllowedError") {+ error.value = "scanner.permission_denied";+ } else {+ error.value = "scanner.error";+ }+ }+ }++ function stopCamera() {+ sessionId++;+ teardown();+ }++ async function switchCamera() {+ if (videoDevices.length < 2) return;++ const mySession = ++sessionId;+ currentDeviceIndex = (currentDeviceIndex + 1) % videoDevices.length;+ const nextDevice = videoDevices[currentDeviceIndex];+ if (!nextDevice) return;++ stopDetection();+ if (stream) {+ stream.getTracks().forEach(t => t.stop());+ }++ try {+ stream = await navigator.mediaDevices.getUserMedia({+ video: { deviceId: { exact: nextDevice.deviceId } },+ });+ if (sessionId !== mySession) return;++ if (videoRef.value) {+ videoRef.value.srcObject = stream;+ await videoRef.value.play();+ if (sessionId !== mySession) return;+ }++ startDetection(mySession);+ } catch {+ if (sessionId !== mySession) return;+ error.value = "scanner.error";+ }+ }++ function startDetection(session: number) {+ if (sessionId !== session) return;+ scanning = true;+ isScanning.value = true;+ detectInFlight = false;+ lastDetectTime = 0;+ renderLoop(session);+ }++ function stopDetection() {+ scanning = false;+ isScanning.value = false;+ }++ function pauseDetection() {+ scanning = false;+ isScanning.value = false;+ }++ function resumeDetection() {+ if (!stream || !worker) return;+ scanning = true;+ isScanning.value = true;+ renderLoop(sessionId);+ }++ let frameCount = 0;+ let lastLogTime = 0;++ function renderLoop(session: number) {+ if (!scanning || !videoRef.value || sessionId !== session) return;++ const video = videoRef.value;+ const now = Date.now();++ // Log stats periodically+ frameCount++;+ if (now - lastLogTime > 3000) {+ console.debug(+ `[AR] Stats: ${frameCount} frames, ${trackedEntities.size} tracked, detect in-flight: ${detectInFlight}`+ );+ lastLogTime = now;+ frameCount = 0;+ }++ // Send frame to worker for detection at throttled rate+ if (!detectInFlight && worker && video.readyState >= 2 && now - lastDetectTime >= DETECT_INTERVAL) {+ try {+ const frame = createImageBitmap(video);+ frame.then(bitmap => {+ if (!worker || !scanning || sessionId !== session) {+ bitmap.close();+ return;+ }+ detectInFlight = true;+ lastDetectTime = now;+ detectRequestId++;+ worker.postMessage({ type: "detect", frame: bitmap, id: detectRequestId }, [bitmap]);+ });+ } catch {+ // createImageBitmap can fail if video not ready+ }+ }++ // Remove stale entries every frame (so cards disappear promptly)+ let removedStale = false;+ for (const [key, entity] of trackedEntities) {+ if (now - entity.lastSeen > STALE_TIMEOUT) {+ trackedEntities.delete(key);+ removedStale = true;+ }+ }+ if (removedStale) {+ updateDetections();+ }++ if (scanning && sessionId === session) {+ requestAnimationFrame(() => renderLoop(session));+ }+ }++ function updateDetections() {+ detections.value = Array.from(trackedEntities.values()).map(e => ({+ id: e.id,+ entityType: e.entityType,+ rawValue: e.rawValue,+ boundingBox: new DOMRect(e.boundingBox.x, e.boundingBox.y, e.boundingBox.width, e.boundingBox.height),+ cornerPoints: e.cornerPoints.map(p => ({ ...p })),+ pose: { ...e.pose },+ data: e.data,+ loading: e.loading,+ error: e.error,+ lastSeen: e.lastSeen,+ }));+ }++ onUnmounted(() => {+ stopCamera();+ });++ return {+ isSupported,+ isScanning,+ error,+ detections,+ startCamera,+ stopCamera,+ switchCamera,+ };+}diff --git a/frontend/locales/en.json b/frontend/locales/en.jsonindex 47798c71..c0d18c6b 100644--- a/frontend/locales/en.json+++ b/frontend/locales/en.json@@ -826,6 +826,22 @@"title": "Scanner","unsupported": "Media Stream API is not supported without HTTPS"},+ "scanner_ar": {+ "ar_mode": "AR Mode",+ "back": "Back",+ "child_items": "{count, plural, one {child item} other {child items}}",+ "children": "{count, plural, one {sub-location} other {sub-locations}}",+ "items": "{count, plural, one {item} other {items}}",+ "items_in_location": "{count, plural, one {item} other {items}}",+ "more": "more",+ "not_found": "Item not found",+ "qty": "Qty",+ "retry": "Retry",+ "scanning": "Scanning",+ "switch_camera": "Switch Camera",+ "title": "AR Scanner",+ "unsupported": "AR scanning is not supported in this browser"+ },"tags": {"no_results": "No Tags Found","tag_delete_confirm": "Are you sure you want to delete this tag? This action cannot be undone.",diff --git a/frontend/package.json b/frontend/package.jsonindex 9e9c4f86..5da99c8e 100644--- a/frontend/package.json+++ b/frontend/package.json@@ -65,6 +65,7 @@"@vueuse/core": "^13.9.0","@vueuse/nuxt": "^13.9.0","@vueuse/router": "^13.9.0",+ "barcode-detector": "^3.0.2","@zxing/library": "^0.21.3","autoprefixer": "^10.4.27","class-variance-authority": "^0.7.1",diff --git a/frontend/pages/scanner-ar.vue b/frontend/pages/scanner-ar.vuenew file mode 100644index 00000000..93cee430--- /dev/null+++ b/frontend/pages/scanner-ar.vue@@ -0,0 +1,16 @@+<template>+ <ScannerARView />+</template>++<script setup lang="ts">+ import ScannerARView from "@/components/Scanner/ARView.vue";++ definePageMeta({+ middleware: ["auth"],+ layout: "empty",+ });++ useHead({+ title: "HomeBox | AR Scanner",+ });+</script>diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yamlindex c0c077ba..b5d876d7 100644--- a/frontend/pnpm-lock.yaml+++ b/frontend/pnpm-lock.yaml@@ -76,7 +76,10 @@ importers:version: 0.21.3autoprefixer:specifier: ^10.4.27- version: 10.4.27([email protected])+ version: 10.4.27([email protected])+ barcode-detector:+ specifier: ^3.0.2+ version: 3.1.2(@types/[email protected])class-variance-authority:specifier: ^0.7.1version: 0.7.1@@ -2306,6 +2309,9 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==}+ '@types/[email protected]':+ resolution: {integrity: sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==}+'@types/[email protected]':resolution: {integrity: sha512-03ruubjWyOHlmljCVoxSuNDdmfZDzsrrz0P2LeJsOXr+ZwFQ+0yQIwNCwt/GYhV7Z31fgtXJTAEs+FYlEL851g==}@@ -2941,6 +2947,9 @@ packages:resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==}engines: {node: 18 || 20 || >=22}+ resolution: {integrity: sha512-Q5kjXpVH5I3ItykNzbWmfWnNryFN1ZTWp10k9/PKJuS0RnoKR7jTrHEJODR4fn04bRomq7TJwie/Dr9fj/GoGQ==}+resolution: {integrity: sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==}peerDependencies:@@ -6772,6 +6781,11 @@ packages:resolution: {integrity: sha512-dpvY17vxYIW3+bNrP0ClUlaiY0CiIRK3tnoLaGoQsQcY9/I/NpzIWQ7tQNhbV7LacQMpCII6wVzuL3tuWOyfuA==}+ resolution: {integrity: sha512-2YMAriaYHX9wrBY2k7H0epSo+dyCaCZg/vOtt+nEDXM9ul480gkXz/9SkwpOeHcD2H5qqDG8lWDSBwpTcZpa6w==}+ peerDependencies:+ '@types/emscripten': '>=1.39.6'+snapshots:'@alloc/[email protected]': {}@@ -9105,6 +9119,8 @@ snapshots:'@types/[email protected]': {}+ '@types/[email protected]': {}+'@types/[email protected]':dependencies:'@types/estree': 1.0.8@@ -9902,6 +9918,12 @@ snapshots:+ [email protected](@types/[email protected]):+ dependencies:+ zxing-wasm: 3.0.2(@types/[email protected])+ transitivePeerDependencies:+ - '@types/emscripten'+@@ -14288,3 +14310,8 @@ snapshots:readable-stream: 4.7.0++ [email protected](@types/[email protected]):+ dependencies:+ '@types/emscripten': 1.41.5+ type-fest: 5.5.0diff --git a/frontend/workers/barcode-detector.ts b/frontend/workers/barcode-detector.tsnew file mode 100644index 00000000..92f3df8c--- /dev/null+++ b/frontend/workers/barcode-detector.ts@@ -0,0 +1,77 @@+import { BarcodeDetector } from "barcode-detector";++interface DetectRequest {+ type: "detect";+ frame: ImageBitmap;+ id: number;+}++interface DetectResult {+ type: "result";+ id: number;+ barcodes: Array<{+ rawValue: string;+ format: string;+ boundingBox: { x: number; y: number; width: number; height: number };+ cornerPoints: Array<{ x: number; y: number }>;+ }>;+}++interface ReadyMessage {+ type: "ready";+}++interface ErrorMessage {+ type: "error";+ message: string;+}++export type WorkerMessage = DetectRequest;+export type WorkerResponse = DetectResult | ReadyMessage | ErrorMessage;++let detector: BarcodeDetector | null = null;++async function init() {+ try {+ detector = new BarcodeDetector({ formats: ["qr_code"] });+ self.postMessage({ type: "ready" } satisfies ReadyMessage);+ } catch (e) {+ self.postMessage({ type: "error", message: `BarcodeDetector init failed: ${e}` } satisfies ErrorMessage);+ }+}++self.onmessage = async (event: MessageEvent<WorkerMessage>) => {+ const msg = event.data;++ if (msg.type === "detect") {+ if (!detector) {+ self.postMessage({ type: "result", id: msg.id, barcodes: [] } satisfies DetectResult);+ msg.frame.close();+ return;+ }++ try {+ const results = await detector.detect(msg.frame);++ const barcodes = results.map(b => ({+ rawValue: b.rawValue,+ format: b.format,+ boundingBox: {+ x: b.boundingBox.x,+ y: b.boundingBox.y,+ width: b.boundingBox.width,+ height: b.boundingBox.height,+ },+ cornerPoints: Array.from(b.cornerPoints).map(p => ({ x: p.x, y: p.y })),+ }));++ self.postMessage({ type: "result", id: msg.id, barcodes } satisfies DetectResult);+ } catch {+ self.postMessage({ type: "result", id: msg.id, barcodes: [] } satisfies DetectResult);+ } finally {+ msg.frame.close();+ }+ }+};++init();