From 72151c1f5602931b3c301f1bf54bb70d34904eb3 Mon Sep 17 00:00:00 2001 From: xEsiah Date: Sun, 25 Oct 2026 13:36:09 +0100 Subject: [PATCH] fix: resolve client audio glitch, prevent stream resets on tab switch, and handle process cleanup on unload --- apps/kai-sa-cam/src/App.tsx | 56 +++++++---- apps/kai-sa-cam/src/hooks/useTalk.ts | 138 ++++++++++++++------------- apps/kai-sa-cam/src/server/index.ts | 124 ++++++++++++------------ 3 files changed, 164 insertions(+), 154 deletions(-) diff --git a/apps/kai-sa-cam/src/App.tsx b/apps/kai-sa-cam/src/App.tsx index bcc4849..7900d28 100644 --- a/apps/kai-sa-cam/src/App.tsx +++ b/apps/kai-sa-cam/src/App.tsx @@ -31,6 +31,16 @@ export function App() { const { startTalk, stopTalk, error: talkError } = useTalk({ gain }); + useEffect(() => { + const handleBeforeUnload = () => { + if (isRecording) { + navigator.sendBeacon("/api/recordings/stop"); + } + }; + window.addEventListener("beforeunload", handleBeforeUnload); + return () => window.removeEventListener("beforeunload", handleBeforeUnload); + }, [isRecording]); + const handleRecord = async () => { if (isRecording) { await stopRecording(); @@ -85,28 +95,32 @@ export function App() { - {activeTab === "live" && ( -
- - - {(recError || talkError) && ( -
- {recError || talkError} -
- )} -
- )} +
+ + + {(recError || talkError) && ( +
+ {recError || talkError} +
+ )} +
- {activeTab === "media" && } +
+ +
); diff --git a/apps/kai-sa-cam/src/hooks/useTalk.ts b/apps/kai-sa-cam/src/hooks/useTalk.ts index c5f31ed..330e0ac 100644 --- a/apps/kai-sa-cam/src/hooks/useTalk.ts +++ b/apps/kai-sa-cam/src/hooks/useTalk.ts @@ -1,118 +1,120 @@ -import { useState, useCallback, useRef, useEffect } from 'react' +import { useState, useCallback, useRef, useEffect } from "react"; interface UseTalkOptions { - gain?: number + gain?: number; } export function useTalk({ gain = 1 }: UseTalkOptions = {}) { - const [isTalking, setIsTalking] = useState(false) - const [error, setError] = useState(null) - const wsRef = useRef(null) - const mediaStreamRef = useRef(null) - const audioContextRef = useRef(null) - const workletNodeRef = useRef(null) + const [isTalking, setIsTalking] = useState(false); + const [error, setError] = useState(null); + const wsRef = useRef(null); + const mediaStreamRef = useRef(null); + const audioContextRef = useRef(null); + const workletNodeRef = useRef(null); const startTalk = useCallback(async () => { - setError(null) + setError(null); try { const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, }, - }) - mediaStreamRef.current = stream + }); + mediaStreamRef.current = stream; - const audioContext = new AudioContext() - audioContextRef.current = audioContext - await audioContext.resume() + const audioContext = new AudioContext(); + audioContextRef.current = audioContext; + await audioContext.resume(); - // Load the worklet for audio processing - const workletBlob = new Blob([` + const workletBlob = new Blob( + [ + ` class Processor extends AudioWorkletProcessor { process(inputs) { const input = inputs[0] - if (input.length > 0) { - const channel = input[0] - this.port.postMessage(channel.slice(0)) + if (input && input.length > 0) { + const float32 = input[0] + const int16 = new Int16Array(float32.length) + for (let i = 0; i < float32.length; i++) { + const s = Math.max(-1, Math.min(1, float32[i])) + int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF + } + this.port.postMessage(int16.buffer, [int16.buffer]) } return true } } registerProcessor('pcm-processor', Processor) - `], { type: 'application/javascript' }) - const workletUrl = URL.createObjectURL(workletBlob) - await audioContext.audioWorklet.addModule(workletUrl) + `, + ], + { type: "application/javascript" }, + ); + const workletUrl = URL.createObjectURL(workletBlob); + await audioContext.audioWorklet.addModule(workletUrl); - const workletNode = new AudioWorkletNode(audioContext, 'pcm-processor') - workletNodeRef.current = workletNode + const workletNode = new AudioWorkletNode(audioContext, "pcm-processor"); + workletNodeRef.current = workletNode; - const source = audioContext.createMediaStreamSource(stream) - const gainNode = audioContext.createGain() - gainNode.gain.value = gain + const source = audioContext.createMediaStreamSource(stream); + const gainNode = audioContext.createGain(); + gainNode.gain.value = gain; - source.connect(gainNode) - gainNode.connect(workletNode) - workletNode.connect(audioContext.destination) + source.connect(gainNode); + gainNode.connect(workletNode); + workletNode.connect(audioContext.destination); - // Connect to WebSocket - const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:' - const ws = new WebSocket(`${protocol}//${location.host}/api/talk`) - wsRef.current = ws + const protocol = location.protocol === "https:" ? "wss:" : "ws:"; + const ws = new WebSocket(`${protocol}//${location.host}/api/talk`); + wsRef.current = ws; - ws.binaryType = 'arraybuffer' + ws.binaryType = "arraybuffer"; ws.onopen = () => { - ws.send(JSON.stringify({ rate: audioContext.sampleRate, gain })) - } + ws.send(JSON.stringify({ rate: audioContext.sampleRate, gain })); + }; - ws.onerror = (e) => { - console.error('Talk WS error:', e) - setError('Erreur connexion micro') - stopTalk() - } - - ws.onclose = () => { - if (isTalking) { - setError('Connexion micro perdue, reconnexion...') - // Could auto-reconnect here - } - } + ws.onerror = () => { + setError("Erreur connexion micro"); + stopTalk(); + }; workletNode.port.onmessage = (e) => { if (ws.readyState === WebSocket.OPEN) { - ws.send(e.data) + ws.send(e.data); } - } + }; - setIsTalking(true) + setIsTalking(true); } catch (e) { - setError(e instanceof Error ? e.message : 'Impossible d\'accéder au microphone') + setError( + e instanceof Error ? e.message : "Impossible d'accéder au microphone", + ); } - }, [gain]) + }, [gain]); const stopTalk = useCallback(() => { if (wsRef.current) { - wsRef.current.close() - wsRef.current = null + wsRef.current.close(); + wsRef.current = null; } if (mediaStreamRef.current) { - mediaStreamRef.current.getTracks().forEach(t => t.stop()) - mediaStreamRef.current = null + mediaStreamRef.current.getTracks().forEach((t) => t.stop()); + mediaStreamRef.current = null; } if (audioContextRef.current) { - audioContextRef.current.close() - audioContextRef.current = null + audioContextRef.current.close(); + audioContextRef.current = null; } - workletNodeRef.current = null - setIsTalking(false) - }, []) + workletNodeRef.current = null; + setIsTalking(false); + }, []); useEffect(() => { return () => { - stopTalk() - } - }, [stopTalk]) + stopTalk(); + }; + }, [stopTalk]); - return { isTalking, error, startTalk, stopTalk } -} \ No newline at end of file + return { isTalking, error, startTalk, stopTalk }; +} diff --git a/apps/kai-sa-cam/src/server/index.ts b/apps/kai-sa-cam/src/server/index.ts index 9c8b79a..c83146e 100644 --- a/apps/kai-sa-cam/src/server/index.ts +++ b/apps/kai-sa-cam/src/server/index.ts @@ -1,103 +1,97 @@ -import express from 'express' -import { createProxyMiddleware } from 'http-proxy-middleware' -import { createServer } from 'http' -import { WebSocketServer, WebSocket } from 'ws' -import { spawn, ChildProcessWithoutNullStreams } from 'child_process' -import { join } from 'path' -import { recorderRouter } from './recorder.js' -import { mediaRouter } from './media.js' -import { talkRouter } from './talk.js' -import { healthRouter } from './health.js' +import express from "express"; +import { createProxyMiddleware } from "http-proxy-middleware"; +import { createServer } from "http"; +import { WebSocketServer, WebSocket } from "ws"; +import { spawn, ChildProcessWithoutNullStreams } from "child_process"; +import { join } from "path"; +import { recorderRouter } from "./recorder.js"; +import { mediaRouter } from "./media.js"; +import { talkRouter } from "./talk.js"; +import { healthRouter } from "./health.js"; // Use process.cwd() instead of import.meta.url for bundled compatibility -const __dirname = process.cwd() +const __dirname = process.cwd(); -const app = express() -const PORT = process.env.PORT || 8090 +const app = express(); +const PORT = process.env.PORT || 8090; // Middleware -app.use(express.json({ limit: '10mb' })) -app.use(express.urlencoded({ extended: true })) +app.use(express.json({ limit: "10mb" })); +app.use(express.urlencoded({ extended: true })); // Serve static files (built React app) -app.use(express.static(join(__dirname, 'dist'))) +app.use(express.static(join(__dirname, "dist"))); // Proxy go2rtc requests const go2rtcProxy = createProxyMiddleware({ - target: process.env.GO2RTC_URL || 'http://kai-sa-go2rtc:1984', + target: process.env.GO2RTC_URL || "http://kai-sa-go2rtc:1984", changeOrigin: true, ws: true, pathRewrite: { - '^/go2rtc': '', + "^/go2rtc": "", }, -}) +}); -app.use('/go2rtc', go2rtcProxy) +app.use("/go2rtc", go2rtcProxy); // API routes -app.use('/api/recordings', recorderRouter) -app.use('/api/media', mediaRouter) -app.use('/api/talk', talkRouter) -app.use('/api', healthRouter) +app.use("/api/recordings", recorderRouter); +app.use("/api/media", mediaRouter); +app.use("/api/talk", talkRouter); +app.use("/api", healthRouter); // SPA fallback - serve index.html for non-API routes -app.get('*', (req, res, next) => { - if (req.path.startsWith('/api') || req.path.startsWith('/go2rtc')) { - return next() +app.get("*", (req, res, next) => { + if (req.path.startsWith("/api") || req.path.startsWith("/go2rtc")) { + return next(); } - res.sendFile(join(__dirname, 'dist/index.html')) -}) + res.sendFile(join(__dirname, "dist/index.html")); +}); -const server = createServer(app) +const server = createServer(app); -server.on('upgrade', (req: any, socket: any, head: any) => { - if (req.url?.startsWith('/go2rtc')) { - go2rtcProxy.upgrade(req, socket, head) +server.on("upgrade", (req: any, socket: any, head: any) => { + if (req.url?.startsWith("/go2rtc")) { + go2rtcProxy.upgrade(req, socket, head); } -}) +}); // WebSocket server for talk -const wss = new WebSocketServer({ server, path: '/api/talk' }) +const wss = new WebSocketServer({ server, path: "/api/talk" }); -wss.on('connection', (ws: WebSocket, _req) => { - console.log('Talk WS connection established') - - let paplayProcess: ChildProcessWithoutNullStreams | null = null - - ws.on('message', (data: Buffer | string, isBinary: boolean) => { +wss.on("connection", (ws: WebSocket) => { + let paplayProcess: ChildProcessWithoutNullStreams | null = null; + + ws.on("message", (data: Buffer | string, isBinary: boolean) => { if (!isBinary) { try { - const msg = JSON.parse(data.toString()) + const msg = JSON.parse(data.toString()); if (msg.rate && !paplayProcess) { - // Start paplay - paplayProcess = spawn('paplay', [ - '--raw', + paplayProcess = spawn("paplay", [ + "--raw", `--rate=${msg.rate}`, - '--format=s16le', - '--channels=1', - '--latency-msec=150', - ]) - paplayProcess.on('error', (e: Error) => console.error('paplay error:', e.message)) + "--format=s16le", + "--channels=1", + "--latency-msec=150", + ]); } - } catch (e) { - console.error('Talk WS message parse error:', e) - } - return + } catch (e) {} + return; } - + if (paplayProcess?.stdin) { - paplayProcess.stdin.write(data) + paplayProcess.stdin.write(data); } - }) - - ws.on('close', () => { + }); + + ws.on("close", () => { if (paplayProcess) { - paplayProcess.stdin.end() - paplayProcess = null + paplayProcess.kill("SIGKILL"); + paplayProcess = null; } - }) -}) + }); +}); server.listen(PORT, () => { - console.log(`kai-sa-cam server running on port ${PORT}`) -}) \ No newline at end of file + console.log(`kai-sa-cam server running on port ${PORT}`); +});