Add pet labeling UI overlay to recording playback

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Aaron D. Lee 2026-04-03 13:30:52 -04:00
parent 32955bc7e4
commit 4274d1373f
3 changed files with 615 additions and 1 deletions

View File

@ -0,0 +1,205 @@
/* Vigilar — Pet labeling overlay for recording playback */
/* Container: sits on top of the video element, sized to match it */
.detection-overlay {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
pointer-events: none; /* pass-through by default; bboxes re-enable */
z-index: 10;
}
/* Individual bounding box */
.detection-bbox {
position: absolute;
pointer-events: auto;
cursor: pointer;
box-sizing: border-box;
border: 2px solid #00d4d4; /* cyan default for animals */
border-radius: 3px;
transition: border-color 0.15s ease, box-shadow 0.15s ease;
}
.detection-bbox:hover {
border-color: #00ffff;
box-shadow: 0 0 0 2px rgba(0, 255, 255, 0.25);
}
.detection-bbox.bbox-person {
border-color: #dc3545;
}
.detection-bbox.bbox-person:hover {
border-color: #ff4d5e;
box-shadow: 0 0 0 2px rgba(220, 53, 69, 0.25);
}
.detection-bbox.bbox-vehicle {
border-color: #0d6efd;
}
.detection-bbox.bbox-vehicle:hover {
border-color: #3d8bfd;
box-shadow: 0 0 0 2px rgba(13, 110, 253, 0.25);
}
.detection-bbox.bbox-labeled {
border-style: solid;
opacity: 0.85;
}
/* Species + confidence tag shown on top-left corner of bbox */
.detection-bbox-label {
position: absolute;
top: -1px;
left: -1px;
padding: 1px 6px;
font-size: 0.7rem;
font-weight: 600;
line-height: 1.4;
background-color: #00d4d4;
color: #0d1117;
border-radius: 2px 0 2px 0;
white-space: nowrap;
pointer-events: none;
user-select: none;
transition: background-color 0.15s ease;
}
.detection-bbox.bbox-person .detection-bbox-label {
background-color: #dc3545;
color: #fff;
}
.detection-bbox.bbox-vehicle .detection-bbox-label {
background-color: #0d6efd;
color: #fff;
}
/* "Who is this?" popup card */
.label-popup {
position: fixed; /* positioned by JS via clientX/clientY */
z-index: 9999;
min-width: 180px;
max-width: 240px;
background-color: #161b22;
border: 1px solid #30363d;
border-radius: 8px;
box-shadow: 0 8px 24px rgba(0, 0, 0, 0.6);
padding: 10px;
animation: popup-appear 0.12s ease-out;
pointer-events: auto;
}
@keyframes popup-appear {
from {
opacity: 0;
transform: scale(0.92) translateY(-4px);
}
to {
opacity: 1;
transform: scale(1) translateY(0);
}
}
.label-popup-title {
font-size: 0.75rem;
font-weight: 600;
color: #8b949e;
text-transform: uppercase;
letter-spacing: 0.05em;
margin-bottom: 8px;
padding-bottom: 6px;
border-bottom: 1px solid #30363d;
}
.label-popup-pets {
display: flex;
flex-direction: column;
gap: 4px;
margin-bottom: 6px;
}
.label-popup-pet-btn {
display: flex;
align-items: center;
gap: 8px;
width: 100%;
text-align: left;
padding: 5px 8px;
border: 1px solid #30363d;
border-radius: 6px;
background: transparent;
color: #e6edf3;
font-size: 0.85rem;
cursor: pointer;
transition: background-color 0.1s ease, border-color 0.1s ease;
}
.label-popup-pet-btn:hover {
background-color: #21262d;
border-color: #58a6ff;
}
.label-popup-pet-btn .pet-species-dot {
width: 8px;
height: 8px;
border-radius: 50%;
background-color: #00d4d4;
flex-shrink: 0;
}
.label-popup-divider {
border: none;
border-top: 1px solid #30363d;
margin: 6px 0;
}
.label-popup-action-btn {
display: flex;
align-items: center;
gap: 6px;
width: 100%;
text-align: left;
padding: 5px 8px;
border: 1px dashed #30363d;
border-radius: 6px;
background: transparent;
color: #8b949e;
font-size: 0.8rem;
cursor: pointer;
transition: color 0.1s ease, border-color 0.1s ease;
}
.label-popup-action-btn:hover {
color: #e6edf3;
border-color: #8b949e;
}
/* Spinner shown while fetching pets */
.label-popup-loading {
text-align: center;
color: #8b949e;
font-size: 0.8rem;
padding: 8px 0;
}
/* Confirmation flash on successful label */
.label-confirm-flash {
position: absolute;
top: 0;
left: 0;
right: 0;
bottom: 0;
border-radius: 3px;
background-color: rgba(0, 212, 212, 0.2);
pointer-events: none;
animation: confirm-flash 0.5s ease-out forwards;
}
@keyframes confirm-flash {
0% { opacity: 1; }
100% { opacity: 0; }
}

View File

@ -0,0 +1,377 @@
/* Vigilar Pet labeling overlay for recording playback
*
* Integration:
* 1. Add to the page:
* <link rel="stylesheet" href="{{ url_for('static', filename='css/pet-labeling.css') }}">
* <script src="{{ url_for('static', filename='js/pet-labeling.js') }}"></script>
*
* 2. Wrap the <video> element in a position:relative container and add
* `data-pet-labeling="true"` to that container, along with a
* `data-detections` attribute holding the JSON array of detections:
*
* <div class="position-relative" data-pet-labeling="true"
* data-detections='[{"id":1,"species":"cat","confidence":0.92,
* "bbox":[0.1,0.15,0.35,0.6]}]'>
* <video id="player-video" class="w-100" controls></video>
* </div>
*
* bbox values are fractional [x1, y1, x2, y2] relative to video dimensions
* (same convention as YOLO normalised coords).
*
* 3. The module self-initialises on DOMContentLoaded and also exposes
* window.PetLabeling for manual calls:
*
* PetLabeling.init(containerEl, detectionsArray);
* PetLabeling.updateDetections(containerEl, detectionsArray);
*/
(function () {
'use strict';
// -------------------------------------------------------------------------
// Constants
// -------------------------------------------------------------------------
const ANIMAL_SPECIES = new Set(['cat', 'dog', 'bird', 'rabbit', 'hamster', 'fish', 'animal']);
// Map species to bbox CSS class (animals get default cyan)
function bboxClass(species) {
if (species === 'person') return 'bbox-person';
if (species === 'vehicle' || species === 'car' || species === 'truck') return 'bbox-vehicle';
return ''; // cyan default (animals / unknown)
}
// -------------------------------------------------------------------------
// Pets cache — fetched once per page load, filtered by species on demand
// -------------------------------------------------------------------------
let _petsCache = null; // null = not fetched yet; [] = fetched, none found
async function fetchPets() {
if (_petsCache !== null) return _petsCache;
try {
const resp = await fetch('/pets/api/status');
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
const data = await resp.json();
_petsCache = Array.isArray(data.pets) ? data.pets : [];
} catch (e) {
_petsCache = [];
}
return _petsCache;
}
function petsForSpecies(pets, species) {
if (!species) return pets;
// For generic "animal" detections show all pets; otherwise filter by species
if (ANIMAL_SPECIES.has(species) && species !== 'animal') {
return pets.filter(p => p.species && p.species.toLowerCase() === species.toLowerCase());
}
return pets;
}
// -------------------------------------------------------------------------
// Active popup — only one at a time
// -------------------------------------------------------------------------
let _activePopup = null;
function dismissActivePopup() {
if (_activePopup) {
_activePopup.remove();
_activePopup = null;
}
}
// -------------------------------------------------------------------------
// Label popup
// -------------------------------------------------------------------------
async function showLabelPopup(bbox, detection, anchorEl) {
dismissActivePopup();
const popup = document.createElement('div');
popup.className = 'label-popup';
popup.innerHTML = `
<div class="label-popup-title">
<i class="bi bi-tag-fill me-1"></i>Who is this?
</div>
<div class="label-popup-loading">
<span class="spinner-border spinner-border-sm me-1" role="status"></span>
Loading
</div>`;
document.body.appendChild(popup);
_activePopup = popup;
// Position near the clicked bbox, keep on screen
positionPopup(popup, anchorEl);
// Fetch pets (cached after first call)
const allPets = await fetchPets();
if (_activePopup !== popup) return; // dismissed while loading
const pets = petsForSpecies(allPets, detection.species);
let petsHtml = '';
if (pets.length > 0) {
petsHtml = pets.map(p => `
<button class="label-popup-pet-btn" data-pet-id="${p.id}" data-pet-name="${escHtml(p.name)}">
<span class="pet-species-dot"></span>
${escHtml(p.name)}
<small class="ms-auto text-muted">${escHtml(p.species || '')}</small>
</button>`).join('');
}
popup.innerHTML = `
<div class="label-popup-title">
<i class="bi bi-tag-fill me-1"></i>Who is this?
<small class="float-end text-muted text-capitalize">${escHtml(detection.species || 'animal')}</small>
</div>
<div class="label-popup-pets">
${petsHtml}
</div>
${pets.length > 0 ? '<hr class="label-popup-divider">' : ''}
<button class="label-popup-action-btn" data-action="unknown">
<i class="bi bi-question-circle"></i>Unknown
</button>
<button class="label-popup-action-btn mt-1" data-action="new-pet">
<i class="bi bi-plus-circle"></i>+ New Pet
</button>`;
// Re-position after content change (height may differ)
positionPopup(popup, anchorEl);
// Pet selection
popup.querySelectorAll('.label-popup-pet-btn').forEach(btn => {
btn.addEventListener('click', (e) => {
e.stopPropagation();
const petId = btn.dataset.petId;
const petName = btn.dataset.petName;
applyLabel(detection, petId, petName, bbox);
dismissActivePopup();
});
});
// Unknown
const unknownBtn = popup.querySelector('[data-action="unknown"]');
if (unknownBtn) {
unknownBtn.addEventListener('click', (e) => {
e.stopPropagation();
applyLabel(detection, 'unknown', 'Unknown', bbox);
dismissActivePopup();
});
}
// New Pet — open the pets dashboard in a new tab
const newPetBtn = popup.querySelector('[data-action="new-pet"]');
if (newPetBtn) {
newPetBtn.addEventListener('click', (e) => {
e.stopPropagation();
window.open('/pets/', '_blank');
dismissActivePopup();
});
}
}
function positionPopup(popup, anchorEl) {
const rect = anchorEl.getBoundingClientRect();
const popW = popup.offsetWidth || 200;
const popH = popup.offsetHeight || 160;
const margin = 8;
// Prefer right of bbox; fall back to left
let left = rect.right + margin;
if (left + popW > window.innerWidth - margin) {
left = rect.left - popW - margin;
}
if (left < margin) left = margin;
// Align top with bbox; clamp to viewport
let top = rect.top;
if (top + popH > window.innerHeight - margin) {
top = window.innerHeight - popH - margin;
}
if (top < margin) top = margin;
popup.style.left = `${left}px`;
popup.style.top = `${top}px`;
}
// -------------------------------------------------------------------------
// Label API call
// -------------------------------------------------------------------------
async function applyLabel(detection, petId, petName, bboxEl) {
// Optimistic UI: update the label tag immediately
const labelTag = bboxEl.querySelector('.detection-bbox-label');
if (labelTag) {
labelTag.textContent = petId === 'unknown' ? 'Unknown' : petName;
}
bboxEl.classList.add('bbox-labeled');
// Confirmation flash
const flash = document.createElement('div');
flash.className = 'label-confirm-flash';
bboxEl.appendChild(flash);
flash.addEventListener('animationend', () => flash.remove());
if (!detection.id || petId === 'unknown') return;
try {
await fetch(`/pets/${detection.id}/label`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ pet_id: petId }),
});
} catch (e) {
// Silently fail — offline mode
}
}
// -------------------------------------------------------------------------
// Overlay rendering
// -------------------------------------------------------------------------
function renderOverlay(container, detections) {
// Remove any existing overlay
const existing = container.querySelector('.detection-overlay');
if (existing) existing.remove();
if (!detections || detections.length === 0) return;
const video = container.querySelector('video');
if (!video) return;
const overlay = document.createElement('div');
overlay.className = 'detection-overlay';
container.appendChild(overlay);
// Draw bboxes sized to the video's rendered dimensions
function drawBboxes() {
overlay.innerHTML = '';
const vRect = video.getBoundingClientRect();
const cRect = container.getBoundingClientRect();
// Offset from container origin
const offX = vRect.left - cRect.left;
const offY = vRect.top - cRect.top;
const vW = vRect.width;
const vH = vRect.height;
if (vW === 0 || vH === 0) return;
detections.forEach(det => {
const [x1f, y1f, x2f, y2f] = det.bbox || [0, 0, 1, 1];
const left = offX + x1f * vW;
const top = offY + y1f * vH;
const width = (x2f - x1f) * vW;
const height = (y2f - y1f) * vH;
const bbox = document.createElement('div');
bbox.className = `detection-bbox ${bboxClass(det.species)}`.trimEnd();
bbox.style.cssText = `
left: ${left}px;
top: ${top}px;
width: ${width}px;
height: ${height}px;
`;
const confidence = det.confidence !== undefined
? ` ${Math.round(det.confidence * 100)}%`
: '';
const labelText = det.pet_name
? det.pet_name
: `${det.species || 'animal'}${confidence}`;
const label = document.createElement('div');
label.className = 'detection-bbox-label';
label.textContent = labelText;
bbox.appendChild(label);
// Only show "Who is this?" popup for animal detections
if (!det.species || ANIMAL_SPECIES.has(det.species.toLowerCase())) {
bbox.addEventListener('click', (e) => {
e.stopPropagation();
showLabelPopup(bbox, det, bbox);
});
}
overlay.appendChild(bbox);
});
}
// Draw immediately and on video resize/metadata
drawBboxes();
video.addEventListener('loadedmetadata', drawBboxes);
const ro = new ResizeObserver(drawBboxes);
ro.observe(video);
// Store cleanup handle
overlay._roCleanup = () => ro.disconnect();
}
// -------------------------------------------------------------------------
// Public API
// -------------------------------------------------------------------------
function init(container, detections) {
if (!container) return;
renderOverlay(container, detections);
}
function updateDetections(container, detections) {
if (!container) return;
const old = container.querySelector('.detection-overlay');
if (old && old._roCleanup) old._roCleanup();
renderOverlay(container, detections);
}
// -------------------------------------------------------------------------
// Auto-init: find all [data-pet-labeling="true"] containers on page load
// -------------------------------------------------------------------------
function autoInit() {
document.querySelectorAll('[data-pet-labeling="true"]').forEach(container => {
let detections = [];
const raw = container.dataset.detections;
if (raw) {
try { detections = JSON.parse(raw); } catch (e) { detections = []; }
}
init(container, detections);
});
}
if (document.readyState === 'loading') {
document.addEventListener('DOMContentLoaded', autoInit);
} else {
autoInit();
}
// Dismiss popup when clicking outside
document.addEventListener('click', (e) => {
if (_activePopup && !_activePopup.contains(e.target)) {
dismissActivePopup();
}
});
document.addEventListener('keydown', (e) => {
if (e.key === 'Escape') dismissActivePopup();
});
// -------------------------------------------------------------------------
// Helpers
// -------------------------------------------------------------------------
function escHtml(str) {
if (!str) return '';
return String(str)
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;');
}
// Expose public API
window.PetLabeling = { init, updateDetections };
})();

View File

@ -1,6 +1,10 @@
{% extends "base.html" %}
{% block title %}Vigilar — Recordings{% endblock %}
{% block head %}
<link rel="stylesheet" href="{{ url_for('static', filename='css/pet-labeling.css') }}">
{% endblock %}
{% block content %}
<div class="d-flex justify-content-between align-items-center mb-3">
<h5 class="mb-0"><i class="bi bi-camera-video me-2"></i>Recordings</h5>
@ -38,8 +42,11 @@
</button>
</div>
<div class="card-body p-0">
<!-- data-pet-labeling and data-detections are updated by JS when a recording is loaded -->
<div class="position-relative" id="player-video-wrap" data-pet-labeling="true" data-detections="[]">
<video id="player-video" class="w-100" controls></video>
</div>
</div>
</div>
<!-- Legend -->
@ -90,6 +97,7 @@
</div>
<script src="{{ url_for('static', filename='js/timeline.js') }}"></script>
<script src="{{ url_for('static', filename='js/pet-labeling.js') }}"></script>
<script>
(function() {
'use strict';
@ -112,11 +120,35 @@
playerVideo.src = '';
});
const playerVideoWrap = document.getElementById('player-video-wrap');
async function loadDetections(recordingId) {
try {
const resp = await fetch(`/recordings/${recordingId}/detections`);
if (!resp.ok) return [];
return await resp.json();
} catch (e) {
return [];
}
}
function handleSegmentClick(seg) {
playerCard.classList.remove('d-none');
playerTitle.textContent = `Recording #${seg.id} (${seg.type || 'motion'})`;
playerVideo.src = `/recordings/${seg.id}/download`;
playerVideo.play().catch(() => {});
// Reset overlay while detections load
if (window.PetLabeling) {
PetLabeling.updateDetections(playerVideoWrap, []);
}
// Fetch detections for this recording and render overlay
loadDetections(seg.id).then(detections => {
if (window.PetLabeling) {
PetLabeling.updateDetections(playerVideoWrap, detections);
}
});
}
function loadTimeline() {