/* BioReveal — Bioscan (face + skin only — audio removed 2026-05-12). * * 25 seconds. Camera on. * - Per-frame face RGB → rPPG vitals (HR, breathing, autonomic activation) * - Final face frame → /api/skin (hydration, redness, texture, spots) * * The user's verbal goal is now captured as text on the Goals step (a * textarea). Audio capture was noisy and not pulling its weight — kill * it pre-Shen so the bioscan is just "hold still for 25 seconds." * * Captured numbers are NOT shown to user (they're estimates, can be wrong). * Just "✓ Captured" — the AI synthesizer turns the raw signals into useful * analysis on the Reveal page. */ const StepBioscan = ({ value, onChange }) => { const videoRef = React.useRef(null); const canvasRef = React.useRef(null); const streamRef = React.useRef(null); const landmarkerRef = React.useRef(null); const captureBufRef = React.useRef({ r: [], g: [], b: [] }); const startTsRef = React.useRef(0); const rafRef = React.useRef(0); const [status, setStatus] = React.useState("idle"); // idle | loading | requesting | capturing | analyzing | done | error | denied const [progress, setProgress] = React.useState(0); const [usingMP, setUsingMP] = React.useState(false); const [errMsg, setErrMsg] = React.useState(""); const [captured, setCaptured] = React.useState(value?.captured || false); const [hadVitals, setHadVitals] = React.useState(false); const [hadSkin, setHadSkin] = React.useState(false); const CAPTURE_SECONDS = 25; const FS = 30; const cleanup = () => { cancelAnimationFrame(rafRef.current); if (streamRef.current) { streamRef.current.getTracks().forEach(t => t.stop()); streamRef.current = null; } }; React.useEffect(() => () => cleanup(), []); const tryLoadFaceLM = async () => { if (landmarkerRef.current) return landmarkerRef.current; try { const mod = await import("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/+esm"); const { FilesetResolver, FaceLandmarker } = mod; const fileset = await FilesetResolver.forVisionTasks("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/wasm"); const lm = await FaceLandmarker.createFromOptions(fileset, { baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task" }, runningMode: "VIDEO", numFaces: 1, }); landmarkerRef.current = lm; return lm; } catch(_) { return null; } }; const start = async () => { setErrMsg(""); setCaptured(false); setStatus("loading"); const lm = await tryLoadFaceLM(); setUsingMP(!!lm); setStatus("requesting"); let stream; try { stream = await navigator.mediaDevices.getUserMedia({ video: { facingMode: "user", width: { ideal: 720 }, height: { ideal: 720 }, frameRate: { ideal: 30 } }, audio: false, }); } catch (err) { setErrMsg(err?.name === "NotAllowedError" ? "Camera permission needed. Click the lock icon in the address bar to allow camera, then retry." : err.message); setStatus(err?.name === "NotAllowedError" ? "denied" : "error"); return; } streamRef.current = stream; const v = videoRef.current; v.srcObject = stream; try { await v.play(); } catch(_){} setStatus("capturing"); startTsRef.current = performance.now(); captureBufRef.current = { r: [], g: [], b: [] }; const canvas = canvasRef.current; const cctx = canvas.getContext("2d", { willReadFrequently: true }); canvas.width = v.videoWidth || 720; canvas.height = v.videoHeight || 720; const tick = async (ts) => { if (!streamRef.current) return; const elapsed = (ts - startTsRef.current) / 1000; if (elapsed >= CAPTURE_SECONDS) { await analyzeAll(canvas); return; } try { cctx.drawImage(v, 0, 0, canvas.width, canvas.height); const W = canvas.width, H = canvas.height; let rsum = 0, gsum = 0, bsum = 0, count = 0; if (lm) { let result; try { result = lm.detectForVideo(v, ts); } catch(_){} const lms = result?.faceLandmarks?.[0]; if (lms && lms.length > 200) { const idxs = [50, 280, 10, 117, 346]; const half = 14; for (const i of idxs) { const p = lms[i]; const cx = Math.round(p.x * W), cy = Math.round(p.y * H); const x0 = Math.max(0, cx - half), y0 = Math.max(0, cy - half); const w = Math.min(W - x0, half * 2), h = Math.min(H - y0, half * 2); const data = cctx.getImageData(x0, y0, w, h).data; for (let j = 0; j < data.length; j += 4) { rsum += data[j]; gsum += data[j+1]; bsum += data[j+2]; count++; } } } } if (count === 0) { const w = Math.round(W * 0.4), h = Math.round(H * 0.4); const x0 = Math.max(0, Math.round(W/2 - w/2)), y0 = Math.max(0, Math.round(H * 0.45 - h/2)); const data = cctx.getImageData(x0, y0, w, h).data; for (let j = 0; j < data.length; j += 64) { rsum += data[j]; gsum += data[j+1]; bsum += data[j+2]; count++; } } if (count > 0) { captureBufRef.current.r.push(rsum / count); captureBufRef.current.g.push(gsum / count); captureBufRef.current.b.push(bsum / count); } setProgress(Math.min(100, Math.round((elapsed / CAPTURE_SECONDS) * 100))); } catch(e){ console.warn(e); } rafRef.current = requestAnimationFrame(tick); }; rafRef.current = requestAnimationFrame(tick); }; const analyzeAll = async (canvas) => { setStatus("analyzing"); cleanup(); const buf = captureBufRef.current; let skinResult = null; try { const blob = await new Promise(res => canvas.toBlob(res, "image/jpeg", 0.85)); if (blob) { const r = await fetch("/api/skin", { method: "POST", headers: { "Content-Type": "image/jpeg" }, body: blob }); const d = await r.json(); if (d.ok) skinResult = d; } } catch(e){ console.warn("skin failed", e); } let vitalsResult = null; if (buf.r.length >= 60) { try { const r = await fetch("/api/vitals", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ r: buf.r, g: buf.g, b: buf.b, fs: FS }), }); const d = await r.json(); if (d.ok) vitalsResult = d; } catch(e){ console.warn("vitals failed", e); } } if (!vitalsResult && !skinResult) { setStatus("error"); setErrMsg("Couldn't extract any signals — try again with better light, facing the camera."); return; } setHadVitals(!!vitalsResult); setHadSkin(!!skinResult); setCaptured(true); onChange({ captured: true, vitals: vitalsResult, skin: skinResult }); setStatus("done"); }; const skip = () => { cleanup(); onChange({ skipped: true }); setStatus("done"); setCaptured(false); }; const retry = () => { setStatus("idle"); setProgress(0); setErrMsg(""); setCaptured(false); }; return ( <>
03 / 04 · The bioscan · Optional

25 seconds. Look at the camera.

Hold still in soft light. We read your face — heart rate, breathing, skin signals. No audio. No raw numbers shown.