fix: resolve client audio glitch, prevent stream resets on tab switch, and handle process cleanup on unload
CD — Kai-Sa-Cam Deployment / deploy (push) Successful in 10s
CD — Kai-Sa-Cam Deployment / deploy (push) Successful in 10s
This commit is contained in:
+35
-21
@@ -31,6 +31,16 @@ export function App() {
|
||||
|
||||
const { startTalk, stopTalk, error: talkError } = useTalk({ gain });
|
||||
|
||||
useEffect(() => {
|
||||
const handleBeforeUnload = () => {
|
||||
if (isRecording) {
|
||||
navigator.sendBeacon("/api/recordings/stop");
|
||||
}
|
||||
};
|
||||
window.addEventListener("beforeunload", handleBeforeUnload);
|
||||
return () => window.removeEventListener("beforeunload", handleBeforeUnload);
|
||||
}, [isRecording]);
|
||||
|
||||
const handleRecord = async () => {
|
||||
if (isRecording) {
|
||||
await stopRecording();
|
||||
@@ -85,28 +95,32 @@ export function App() {
|
||||
</button>
|
||||
</nav>
|
||||
|
||||
{activeTab === "live" && (
|
||||
<div className="flex-1 flex flex-col p-4 gap-4 overflow-hidden">
|
||||
<VideoPlayer streamUrl="https://stream.esiah.dev/stream.html?src=webcam&mode=mse" />
|
||||
<Controls
|
||||
isRecording={isRecording}
|
||||
recordingTime={recordingTime}
|
||||
isTalking={isTalking}
|
||||
gain={gain}
|
||||
onRecord={handleRecord}
|
||||
onSnapshot={handleSnapshot}
|
||||
onTalk={handleTalk}
|
||||
disabled={Boolean(recError || talkError)}
|
||||
/>
|
||||
{(recError || talkError) && (
|
||||
<div className="text-red-400 text-sm text-center px-4">
|
||||
{recError || talkError}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
<div
|
||||
className={`flex-1 flex flex-col p-4 gap-4 overflow-hidden ${activeTab === "live" ? "" : "hidden"}`}
|
||||
>
|
||||
<VideoPlayer streamUrl="https://stream.esiah.dev/stream.html?src=webcam&mode=mse" />
|
||||
<Controls
|
||||
isRecording={isRecording}
|
||||
recordingTime={recordingTime}
|
||||
isTalking={isTalking}
|
||||
gain={gain}
|
||||
onRecord={handleRecord}
|
||||
onSnapshot={handleSnapshot}
|
||||
onTalk={handleTalk}
|
||||
disabled={Boolean(recError || talkError)}
|
||||
/>
|
||||
{(recError || talkError) && (
|
||||
<div className="text-red-400 text-sm text-center px-4">
|
||||
{recError || talkError}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{activeTab === "media" && <MediaGallery />}
|
||||
<div
|
||||
className={`flex-1 flex flex-col ${activeTab === "media" ? "" : "hidden"}`}
|
||||
>
|
||||
<MediaGallery />
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1,118 +1,120 @@
|
||||
import { useState, useCallback, useRef, useEffect } from 'react'
|
||||
import { useState, useCallback, useRef, useEffect } from "react";
|
||||
|
||||
interface UseTalkOptions {
|
||||
gain?: number
|
||||
gain?: number;
|
||||
}
|
||||
|
||||
export function useTalk({ gain = 1 }: UseTalkOptions = {}) {
|
||||
const [isTalking, setIsTalking] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const wsRef = useRef<WebSocket | null>(null)
|
||||
const mediaStreamRef = useRef<MediaStream | null>(null)
|
||||
const audioContextRef = useRef<AudioContext | null>(null)
|
||||
const workletNodeRef = useRef<AudioWorkletNode | null>(null)
|
||||
const [isTalking, setIsTalking] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const mediaStreamRef = useRef<MediaStream | null>(null);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const workletNodeRef = useRef<AudioWorkletNode | null>(null);
|
||||
|
||||
const startTalk = useCallback(async () => {
|
||||
setError(null)
|
||||
setError(null);
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
},
|
||||
})
|
||||
mediaStreamRef.current = stream
|
||||
});
|
||||
mediaStreamRef.current = stream;
|
||||
|
||||
const audioContext = new AudioContext()
|
||||
audioContextRef.current = audioContext
|
||||
await audioContext.resume()
|
||||
const audioContext = new AudioContext();
|
||||
audioContextRef.current = audioContext;
|
||||
await audioContext.resume();
|
||||
|
||||
// Load the worklet for audio processing
|
||||
const workletBlob = new Blob([`
|
||||
const workletBlob = new Blob(
|
||||
[
|
||||
`
|
||||
class Processor extends AudioWorkletProcessor {
|
||||
process(inputs) {
|
||||
const input = inputs[0]
|
||||
if (input.length > 0) {
|
||||
const channel = input[0]
|
||||
this.port.postMessage(channel.slice(0))
|
||||
if (input && input.length > 0) {
|
||||
const float32 = input[0]
|
||||
const int16 = new Int16Array(float32.length)
|
||||
for (let i = 0; i < float32.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, float32[i]))
|
||||
int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF
|
||||
}
|
||||
this.port.postMessage(int16.buffer, [int16.buffer])
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
registerProcessor('pcm-processor', Processor)
|
||||
`], { type: 'application/javascript' })
|
||||
const workletUrl = URL.createObjectURL(workletBlob)
|
||||
await audioContext.audioWorklet.addModule(workletUrl)
|
||||
`,
|
||||
],
|
||||
{ type: "application/javascript" },
|
||||
);
|
||||
const workletUrl = URL.createObjectURL(workletBlob);
|
||||
await audioContext.audioWorklet.addModule(workletUrl);
|
||||
|
||||
const workletNode = new AudioWorkletNode(audioContext, 'pcm-processor')
|
||||
workletNodeRef.current = workletNode
|
||||
const workletNode = new AudioWorkletNode(audioContext, "pcm-processor");
|
||||
workletNodeRef.current = workletNode;
|
||||
|
||||
const source = audioContext.createMediaStreamSource(stream)
|
||||
const gainNode = audioContext.createGain()
|
||||
gainNode.gain.value = gain
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
const gainNode = audioContext.createGain();
|
||||
gainNode.gain.value = gain;
|
||||
|
||||
source.connect(gainNode)
|
||||
gainNode.connect(workletNode)
|
||||
workletNode.connect(audioContext.destination)
|
||||
source.connect(gainNode);
|
||||
gainNode.connect(workletNode);
|
||||
workletNode.connect(audioContext.destination);
|
||||
|
||||
// Connect to WebSocket
|
||||
const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:'
|
||||
const ws = new WebSocket(`${protocol}//${location.host}/api/talk`)
|
||||
wsRef.current = ws
|
||||
const protocol = location.protocol === "https:" ? "wss:" : "ws:";
|
||||
const ws = new WebSocket(`${protocol}//${location.host}/api/talk`);
|
||||
wsRef.current = ws;
|
||||
|
||||
ws.binaryType = 'arraybuffer'
|
||||
ws.binaryType = "arraybuffer";
|
||||
|
||||
ws.onopen = () => {
|
||||
ws.send(JSON.stringify({ rate: audioContext.sampleRate, gain }))
|
||||
}
|
||||
ws.send(JSON.stringify({ rate: audioContext.sampleRate, gain }));
|
||||
};
|
||||
|
||||
ws.onerror = (e) => {
|
||||
console.error('Talk WS error:', e)
|
||||
setError('Erreur connexion micro')
|
||||
stopTalk()
|
||||
}
|
||||
|
||||
ws.onclose = () => {
|
||||
if (isTalking) {
|
||||
setError('Connexion micro perdue, reconnexion...')
|
||||
// Could auto-reconnect here
|
||||
}
|
||||
}
|
||||
ws.onerror = () => {
|
||||
setError("Erreur connexion micro");
|
||||
stopTalk();
|
||||
};
|
||||
|
||||
workletNode.port.onmessage = (e) => {
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(e.data)
|
||||
ws.send(e.data);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
setIsTalking(true)
|
||||
setIsTalking(true);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : 'Impossible d\'accéder au microphone')
|
||||
setError(
|
||||
e instanceof Error ? e.message : "Impossible d'accéder au microphone",
|
||||
);
|
||||
}
|
||||
}, [gain])
|
||||
}, [gain]);
|
||||
|
||||
const stopTalk = useCallback(() => {
|
||||
if (wsRef.current) {
|
||||
wsRef.current.close()
|
||||
wsRef.current = null
|
||||
wsRef.current.close();
|
||||
wsRef.current = null;
|
||||
}
|
||||
if (mediaStreamRef.current) {
|
||||
mediaStreamRef.current.getTracks().forEach(t => t.stop())
|
||||
mediaStreamRef.current = null
|
||||
mediaStreamRef.current.getTracks().forEach((t) => t.stop());
|
||||
mediaStreamRef.current = null;
|
||||
}
|
||||
if (audioContextRef.current) {
|
||||
audioContextRef.current.close()
|
||||
audioContextRef.current = null
|
||||
audioContextRef.current.close();
|
||||
audioContextRef.current = null;
|
||||
}
|
||||
workletNodeRef.current = null
|
||||
setIsTalking(false)
|
||||
}, [])
|
||||
workletNodeRef.current = null;
|
||||
setIsTalking(false);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stopTalk()
|
||||
}
|
||||
}, [stopTalk])
|
||||
stopTalk();
|
||||
};
|
||||
}, [stopTalk]);
|
||||
|
||||
return { isTalking, error, startTalk, stopTalk }
|
||||
}
|
||||
return { isTalking, error, startTalk, stopTalk };
|
||||
}
|
||||
|
||||
@@ -1,103 +1,97 @@
|
||||
import express from 'express'
|
||||
import { createProxyMiddleware } from 'http-proxy-middleware'
|
||||
import { createServer } from 'http'
|
||||
import { WebSocketServer, WebSocket } from 'ws'
|
||||
import { spawn, ChildProcessWithoutNullStreams } from 'child_process'
|
||||
import { join } from 'path'
|
||||
import { recorderRouter } from './recorder.js'
|
||||
import { mediaRouter } from './media.js'
|
||||
import { talkRouter } from './talk.js'
|
||||
import { healthRouter } from './health.js'
|
||||
import express from "express";
|
||||
import { createProxyMiddleware } from "http-proxy-middleware";
|
||||
import { createServer } from "http";
|
||||
import { WebSocketServer, WebSocket } from "ws";
|
||||
import { spawn, ChildProcessWithoutNullStreams } from "child_process";
|
||||
import { join } from "path";
|
||||
import { recorderRouter } from "./recorder.js";
|
||||
import { mediaRouter } from "./media.js";
|
||||
import { talkRouter } from "./talk.js";
|
||||
import { healthRouter } from "./health.js";
|
||||
|
||||
// Use process.cwd() instead of import.meta.url for bundled compatibility
|
||||
const __dirname = process.cwd()
|
||||
const __dirname = process.cwd();
|
||||
|
||||
const app = express()
|
||||
const PORT = process.env.PORT || 8090
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 8090;
|
||||
|
||||
// Middleware
|
||||
app.use(express.json({ limit: '10mb' }))
|
||||
app.use(express.urlencoded({ extended: true }))
|
||||
app.use(express.json({ limit: "10mb" }));
|
||||
app.use(express.urlencoded({ extended: true }));
|
||||
|
||||
// Serve static files (built React app)
|
||||
app.use(express.static(join(__dirname, 'dist')))
|
||||
app.use(express.static(join(__dirname, "dist")));
|
||||
|
||||
// Proxy go2rtc requests
|
||||
const go2rtcProxy = createProxyMiddleware({
|
||||
target: process.env.GO2RTC_URL || 'http://kai-sa-go2rtc:1984',
|
||||
target: process.env.GO2RTC_URL || "http://kai-sa-go2rtc:1984",
|
||||
changeOrigin: true,
|
||||
ws: true,
|
||||
pathRewrite: {
|
||||
'^/go2rtc': '',
|
||||
"^/go2rtc": "",
|
||||
},
|
||||
})
|
||||
});
|
||||
|
||||
app.use('/go2rtc', go2rtcProxy)
|
||||
app.use("/go2rtc", go2rtcProxy);
|
||||
|
||||
// API routes
|
||||
app.use('/api/recordings', recorderRouter)
|
||||
app.use('/api/media', mediaRouter)
|
||||
app.use('/api/talk', talkRouter)
|
||||
app.use('/api', healthRouter)
|
||||
app.use("/api/recordings", recorderRouter);
|
||||
app.use("/api/media", mediaRouter);
|
||||
app.use("/api/talk", talkRouter);
|
||||
app.use("/api", healthRouter);
|
||||
|
||||
// SPA fallback - serve index.html for non-API routes
|
||||
app.get('*', (req, res, next) => {
|
||||
if (req.path.startsWith('/api') || req.path.startsWith('/go2rtc')) {
|
||||
return next()
|
||||
app.get("*", (req, res, next) => {
|
||||
if (req.path.startsWith("/api") || req.path.startsWith("/go2rtc")) {
|
||||
return next();
|
||||
}
|
||||
res.sendFile(join(__dirname, 'dist/index.html'))
|
||||
})
|
||||
res.sendFile(join(__dirname, "dist/index.html"));
|
||||
});
|
||||
|
||||
const server = createServer(app)
|
||||
const server = createServer(app);
|
||||
|
||||
server.on('upgrade', (req: any, socket: any, head: any) => {
|
||||
if (req.url?.startsWith('/go2rtc')) {
|
||||
go2rtcProxy.upgrade(req, socket, head)
|
||||
server.on("upgrade", (req: any, socket: any, head: any) => {
|
||||
if (req.url?.startsWith("/go2rtc")) {
|
||||
go2rtcProxy.upgrade(req, socket, head);
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
// WebSocket server for talk
|
||||
const wss = new WebSocketServer({ server, path: '/api/talk' })
|
||||
const wss = new WebSocketServer({ server, path: "/api/talk" });
|
||||
|
||||
wss.on('connection', (ws: WebSocket, _req) => {
|
||||
console.log('Talk WS connection established')
|
||||
|
||||
let paplayProcess: ChildProcessWithoutNullStreams | null = null
|
||||
|
||||
ws.on('message', (data: Buffer | string, isBinary: boolean) => {
|
||||
wss.on("connection", (ws: WebSocket) => {
|
||||
let paplayProcess: ChildProcessWithoutNullStreams | null = null;
|
||||
|
||||
ws.on("message", (data: Buffer | string, isBinary: boolean) => {
|
||||
if (!isBinary) {
|
||||
try {
|
||||
const msg = JSON.parse(data.toString())
|
||||
const msg = JSON.parse(data.toString());
|
||||
if (msg.rate && !paplayProcess) {
|
||||
// Start paplay
|
||||
paplayProcess = spawn('paplay', [
|
||||
'--raw',
|
||||
paplayProcess = spawn("paplay", [
|
||||
"--raw",
|
||||
`--rate=${msg.rate}`,
|
||||
'--format=s16le',
|
||||
'--channels=1',
|
||||
'--latency-msec=150',
|
||||
])
|
||||
paplayProcess.on('error', (e: Error) => console.error('paplay error:', e.message))
|
||||
"--format=s16le",
|
||||
"--channels=1",
|
||||
"--latency-msec=150",
|
||||
]);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Talk WS message parse error:', e)
|
||||
}
|
||||
return
|
||||
} catch (e) {}
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
if (paplayProcess?.stdin) {
|
||||
paplayProcess.stdin.write(data)
|
||||
paplayProcess.stdin.write(data);
|
||||
}
|
||||
})
|
||||
|
||||
ws.on('close', () => {
|
||||
});
|
||||
|
||||
ws.on("close", () => {
|
||||
if (paplayProcess) {
|
||||
paplayProcess.stdin.end()
|
||||
paplayProcess = null
|
||||
paplayProcess.kill("SIGKILL");
|
||||
paplayProcess = null;
|
||||
}
|
||||
})
|
||||
})
|
||||
});
|
||||
});
|
||||
|
||||
server.listen(PORT, () => {
|
||||
console.log(`kai-sa-cam server running on port ${PORT}`)
|
||||
})
|
||||
console.log(`kai-sa-cam server running on port ${PORT}`);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user