This repository has no description
21 kB
683 lines
1// hermes-voice page: one WebRTC session with gpt-live-1 per "call", client
2// delegation answered by Hermes through the local server (/delegate).
3"use strict";
4
5const PRICE_PER_MINUTE = 0.05;
6// The voice model accepts at most 500 tokens per append; ~4 chars per token.
7const APPEND_CHAR_LIMIT = 1400;
8// Transcript deltas can trail the delegation event; wait for the last words.
9const DELEGATION_SETTLE_MS = 400;
10// Once Hermes ended the call (end_call item), the call closes when the voice
11// has been quiet this long after speaking the reply, at the latest after
12// END_CALL_MAX_MS.
13const END_CALL_QUIET_MS = 800;
14const END_CALL_MAX_MS = 30000;
15const STORAGE_KEY = "hermes-voice";
16
17const talk = document.getElementById("talk");
18const statusLine = document.getElementById("status");
19const costLine = document.getElementById("cost");
20const logList = document.getElementById("log");
21const voice = document.getElementById("voice");
22const approvalBox = document.getElementById("approval");
23const approvalText = document.getElementById("approval-text");
24const voiceSelect = document.getElementById("voice-select");
25const VOICE_KEY = "hermes-voice-voice";
26const TOKEN_KEY = "hermes-voice-token";
27const loginForm = document.getElementById("login");
28const tokenInput = document.getElementById("token");
29
30// Every API route needs the page's access token; a 401 asks for it again.
31async function api(path, options = {}) {
32 const response = await fetch(path, {
33 ...options,
34 headers: {
35 ...options.headers,
36 Authorization: `Bearer ${localStorage.getItem(TOKEN_KEY) || ""}`,
37 },
38 });
39 if (response.status === 401) {
40 showLogin();
41 throw new Error("Sign in with the access token first");
42 }
43 return response;
44}
45
46// Spoken answer to an approval question. Denials win, so "no, don't do it"
47// never approves; anything unclear is asked again.
48function classifyApproval(said) {
49 if (
50 /\b(no|nope|deny|denied|don'?t|do not|stop|cancel|abort|reject|nein|nicht)\b/i.test(
51 said,
52 )
53 )
54 return "deny";
55 if (
56 /\b(yes|yeah|yep|approve|approved|go ahead|do it|okay|ok|sure|confirm|allow|ja)\b/i.test(
57 said,
58 )
59 )
60 return "once";
61 return null;
62}
63
64let settings = { idle_seconds: 90 };
65let call = null;
66let state = loadState();
67
68function newConversation() {
69 const day = new Date().toISOString().slice(0, 10).replaceAll("-", "");
70 const rand = crypto.getRandomValues(new Uint32Array(1))[0].toString(16);
71 return {
72 hermesSession: `voice-${day}-${rand}`,
73 lines: [],
74 delegated: 0,
75 billedSeconds: 0,
76 };
77}
78
79function loadState() {
80 try {
81 const saved = JSON.parse(localStorage.getItem(STORAGE_KEY));
82 if (saved && saved.hermesSession) return saved;
83 } catch (_) {
84 /* fall through */
85 }
86 return newConversation();
87}
88
89function saveState() {
90 localStorage.setItem(STORAGE_KEY, JSON.stringify(state));
91}
92
93function setStatus(text) {
94 statusLine.textContent = text;
95}
96
97function renderCost() {
98 const current = call ? call.usageSeconds : 0;
99 const total = state.billedSeconds + current;
100 const money = (seconds) =>
101 `$${((seconds / 60) * PRICE_PER_MINUTE).toFixed(3)}`;
102 const clock = (seconds) =>
103 `${Math.floor(seconds / 60)}:${String(Math.floor(seconds % 60)).padStart(2, "0")}`;
104 costLine.textContent = call
105 ? `this call ${clock(current)} · ${money(current)} — conversation ${money(total)}`
106 : `conversation ${clock(total)} · ${money(total)}`;
107}
108
109function renderLog() {
110 logList.replaceChildren(
111 ...state.lines.map((line) => {
112 const item = document.createElement("li");
113 item.className = line.speaker;
114 item.textContent = line.text;
115 return item;
116 }),
117 );
118 window.scrollTo(0, document.body.scrollHeight);
119}
120
121function addLine(speaker, text) {
122 state.lines.push({ speaker, text });
123 saveState();
124 renderLog();
125}
126
127// Deltas are appended exactly as received; a speaker change starts a new line.
128function addTranscript(speaker, delta) {
129 const last = state.lines.at(-1);
130 if (
131 last &&
132 last.speaker === speaker &&
133 call &&
134 call.lastSpeaker === speaker
135 ) {
136 last.text += delta;
137 } else {
138 state.lines.push({ speaker, text: delta });
139 }
140 if (call) call.lastSpeaker = speaker;
141 saveState();
142 renderLog();
143}
144
145function chunks(text) {
146 const clean = text.replace(/\s+/g, " ").trim();
147 const out = [];
148 let current = "";
149 for (const sentence of clean.split(/(?<=[.!?])\s+/)) {
150 if (sentence.length > APPEND_CHAR_LIMIT) {
151 if (current) out.push(current);
152 current = "";
153 for (let i = 0; i < sentence.length; i += APPEND_CHAR_LIMIT)
154 out.push(sentence.slice(i, i + APPEND_CHAR_LIMIT));
155 } else if (
156 current &&
157 current.length + 1 + sentence.length > APPEND_CHAR_LIMIT
158 ) {
159 out.push(current);
160 current = sentence;
161 } else {
162 current = current ? `${current} ${sentence}` : sentence;
163 }
164 }
165 if (current) out.push(current);
166 return out;
167}
168
169class Call {
170 constructor() {
171 this.usageSeconds = 0;
172 this.lastActivity = Date.now();
173 this.speaking = false;
174 this.lastSpeaker = null;
175 this.delegation = null;
176 this.closed = false;
177 this.closing = false;
178 this.ending = null;
179 this.eventCounter = 0;
180 }
181
182 async start() {
183 const pc = new RTCPeerConnection();
184 this.pc = pc;
185 pc.addEventListener("track", (event) => {
186 const stream = new MediaStream([event.track]);
187 voice.srcObject = stream;
188 voice.play().catch(() => {});
189 this.probeSpeaking(stream);
190 });
191 pc.addEventListener("connectionstatechange", () => {
192 if (
193 pc.connectionState === "failed" ||
194 pc.connectionState === "disconnected"
195 )
196 this.finish("connection lost");
197 });
198
199 this.mic = await navigator.mediaDevices.getUserMedia({
200 audio: {
201 echoCancellation: true,
202 noiseSuppression: true,
203 autoGainControl: true,
204 },
205 });
206 for (const track of this.mic.getAudioTracks()) pc.addTrack(track, this.mic);
207
208 // The data channel must exist before the offer so its m-line is negotiated.
209 this.events = pc.createDataChannel("oai-events");
210 this.events.addEventListener("message", ({ data }) =>
211 this.onEvent(JSON.parse(data)),
212 );
213 this.events.addEventListener("close", () =>
214 this.finish("connection closed"),
215 );
216
217 await pc.setLocalDescription(await pc.createOffer());
218 await this.iceGathered();
219
220 const response = await api("/session", {
221 method: "POST",
222 headers: { "Content-Type": "application/json" },
223 body: JSON.stringify({
224 sdp: pc.localDescription.sdp,
225 history: state.lines,
226 // Fixed per session: a new pick applies from the next call on.
227 voice: voiceSelect.value,
228 }),
229 });
230 const answer = await response.json();
231 if (!response.ok)
232 throw new Error(answer.error || `session failed (${response.status})`);
233 this.sessionId = answer.session_id;
234 await pc.setRemoteDescription({ type: "answer", sdp: answer.sdp });
235
236 this.idleTimer = setInterval(() => this.checkIdle(), 1000);
237 }
238
239 iceGathered() {
240 if (this.pc.iceGatheringState === "complete") return Promise.resolve();
241 return new Promise((resolve) => {
242 const done = () => {
243 if (this.pc.iceGatheringState !== "complete") return;
244 clearTimeout(timer);
245 resolve();
246 };
247 // Trickle is fine: the vendor answers with the candidates it has.
248 const timer = setTimeout(resolve, 3000);
249 this.pc.addEventListener("icegatheringstatechange", done);
250 });
251 }
252
253 probeSpeaking(stream) {
254 const context = new AudioContext();
255 const analyser = context.createAnalyser();
256 analyser.fftSize = 512;
257 context.createMediaStreamSource(stream).connect(analyser);
258 const buffer = new Uint8Array(analyser.frequencyBinCount);
259 let quiet = 0;
260 this.audioContext = context;
261 this.probe = setInterval(() => {
262 analyser.getByteTimeDomainData(buffer);
263 let peak = 0;
264 for (const sample of buffer)
265 peak = Math.max(peak, Math.abs(sample - 128));
266 quiet = peak > 6 ? 0 : quiet + 1;
267 const speaking = quiet < 4;
268 if (speaking) this.lastActivity = Date.now();
269 if (speaking !== this.speaking) {
270 this.speaking = speaking;
271 talk.classList.toggle("speaking", speaking);
272 }
273 if (this.ending?.spoken && quiet * 100 >= END_CALL_QUIET_MS) this.close();
274 }, 100);
275 }
276
277 send(event) {
278 if (!this.events || this.events.readyState !== "open") return false;
279 this.eventCounter += 1;
280 this.events.send(
281 JSON.stringify({ event_id: `evt_${this.eventCounter}`, ...event }),
282 );
283 return true;
284 }
285
286 onEvent(event) {
287 switch (event.type) {
288 case "session.started":
289 setStatus("Listening");
290 break;
291 case "session.input_transcript.delta":
292 this.lastActivity = Date.now();
293 addTranscript("user", event.delta || "");
294 break;
295 case "session.output_transcript.delta":
296 this.lastActivity = Date.now();
297 addTranscript("assistant", event.delta || "");
298 if (this.ending) this.ending.spoken = true;
299 break;
300 case "session.delegation.created":
301 this.delegate(event.delegation.id);
302 break;
303 case "session.usage.updated":
304 this.usageSeconds = event.usage?.seconds ?? this.usageSeconds;
305 renderCost();
306 break;
307 case "error":
308 addLine("error", event.error?.message || "voice model error");
309 break;
310 case "session.closed":
311 if (event.usage?.seconds != null)
312 this.usageSeconds = event.usage.seconds;
313 this.finish(event.reason || "closed");
314 break;
315 }
316 }
317
318 async delegate(id) {
319 // One Hermes turn at a time: a newer delegation (usually a correction)
320 // supersedes the running one, and the server stops the old run.
321 // While Hermes waits for an approval, the next delegation is the user's
322 // answer, not a new request: the turn must keep running.
323 if (this.approval) {
324 this.answerApproval(id);
325 return;
326 }
327 const replace = Boolean(this.delegation);
328 if (this.delegation) this.delegation.controller.abort();
329 const controller = new AbortController();
330 const delegation = { id, controller };
331 this.delegation = delegation;
332 setStatus("Hermes is working…");
333
334 await new Promise((resolve) => setTimeout(resolve, DELEGATION_SETTLE_MS));
335 // A hang-up meanwhile sent these lines along itself.
336 if (this.delegation !== delegation || this.closed) return;
337 const fresh = state.lines
338 .slice(state.delegated)
339 .filter((line) => line.speaker !== "hermes" && line.speaker !== "error");
340 const lines = fresh.some((line) => line.speaker === "user")
341 ? fresh
342 : state.lines
343 .filter(
344 (line) => line.speaker === "user" || line.speaker === "assistant",
345 )
346 .slice(-4);
347 state.delegated = state.lines.length;
348 saveState();
349
350 let answer = "";
351 let failure = null;
352 let endCall = false;
353 try {
354 const response = await api("/delegate", {
355 method: "POST",
356 headers: { "Content-Type": "application/json" },
357 body: JSON.stringify({
358 hermes_session: state.hermesSession,
359 lines,
360 replace,
361 }),
362 signal: controller.signal,
363 });
364 if (!response.ok)
365 throw new Error(
366 (await response.json()).error ||
367 `delegate failed (${response.status})`,
368 );
369 const reader = response.body
370 .pipeThrough(new TextDecoderStream())
371 .getReader();
372 let buffered = "";
373 for (;;) {
374 const { value, done } = await reader.read();
375 if (done) break;
376 buffered += value;
377 let newline;
378 while ((newline = buffered.indexOf("\n")) >= 0) {
379 const line = buffered.slice(0, newline);
380 buffered = buffered.slice(newline + 1);
381 // Blank lines are the server's keepalives.
382 if (!line.trim()) continue;
383 const item = JSON.parse(line);
384 this.lastActivity = Date.now();
385 if (item.type === "approval") {
386 this.askApproval(id, item);
387 continue;
388 }
389 // Anything after an approval request means Hermes moved on
390 // (answered here, by button, or by its own timeout).
391 if (this.approval) this.clearApproval();
392 if (item.type === "tool" && item.status === "running") {
393 setStatus(`Hermes: ${item.label}`);
394 this.send({
395 type: "session.thinking.append",
396 delegation_id: id,
397 content: `Hermes is working on it: ${item.label}`.slice(
398 0,
399 APPEND_CHAR_LIMIT,
400 ),
401 });
402 } else if (item.type === "end_call") {
403 endCall = true;
404 } else if (item.type === "delta") {
405 answer += item.text;
406 } else if (item.type === "done" && item.error) {
407 failure = item.error;
408 } else if (item.type === "error") {
409 failure = item.message;
410 }
411 }
412 }
413 } catch (error) {
414 if (controller.signal.aborted) return;
415 failure = String(error.message || error);
416 }
417 if (this.delegation !== delegation) return;
418 this.delegation = null;
419 this.clearApproval();
420 this.lastActivity = Date.now();
421
422 if (failure && !answer.trim()) {
423 addLine("error", failure);
424 this.send({
425 type: "session.commentary.append",
426 delegation_id: id,
427 content: `The request to Hermes failed: ${failure}`.slice(
428 0,
429 APPEND_CHAR_LIMIT,
430 ),
431 });
432 } else {
433 const text = answer.trim() || "Hermes finished without an answer.";
434 addLine("hermes", text);
435 for (const chunk of chunks(text))
436 this.send({
437 type: "session.commentary.append",
438 delegation_id: id,
439 content: chunk,
440 });
441 }
442 if (endCall) this.endAfterReply();
443 else setStatus("Listening");
444 }
445
446 // Hermes ended the call: close it once the voice has spoken the reply just
447 // handed over.
448 endAfterReply() {
449 this.ending = { spoken: false };
450 this.endTimer = setTimeout(() => this.close(), END_CALL_MAX_MS);
451 setStatus("Ending the call…");
452 }
453
454 askApproval(id, item) {
455 this.approval = { delegationId: id, code: item.code };
456 const what = item.description || "run a command";
457 setStatus("Hermes needs your approval");
458 addLine(
459 "hermes",
460 `Approval needed: ${what}${item.command ? `\n${item.command}` : ""}`,
461 );
462 approvalText.textContent = item.command ? `${what}: ${item.command}` : what;
463 approvalBox.hidden = false;
464 this.send({
465 type: "session.commentary.append",
466 delegation_id: id,
467 content: (
468 `Hermes needs the user's approval before it continues: ${what}.` +
469 (item.command ? ` The command is: ${item.command}.` : "") +
470 " Ask the user whether to approve or deny it."
471 ).slice(0, APPEND_CHAR_LIMIT),
472 });
473 }
474
475 clearApproval() {
476 this.approval = null;
477 approvalBox.hidden = true;
478 }
479
480 async answerApproval(id) {
481 await new Promise((resolve) => setTimeout(resolve, DELEGATION_SETTLE_MS));
482 const said = state.lines
483 .slice(state.delegated)
484 .filter((line) => line.speaker === "user")
485 .map((line) => line.text)
486 .join(" ");
487 state.delegated = state.lines.length;
488 saveState();
489 const choice = classifyApproval(said);
490 if (!choice) {
491 this.send({
492 type: "session.commentary.append",
493 delegation_id: id,
494 content:
495 "That was not a clear answer. Ask the user again: approve or deny?",
496 });
497 return;
498 }
499 await this.respondApproval(choice, id);
500 }
501
502 async respondApproval(choice, replyId) {
503 const approval = this.approval;
504 if (!approval) return;
505 this.clearApproval();
506 setStatus(choice === "deny" ? "Denying…" : "Approving…");
507 let message;
508 let ok = false;
509 try {
510 const response = await api("/approve", {
511 method: "POST",
512 headers: { "Content-Type": "application/json" },
513 body: JSON.stringify({ code: approval.code, choice }),
514 });
515 const result = await response.json();
516 ok = response.ok;
517 message = ok
518 ? choice === "deny"
519 ? "Denied. Hermes will not run it."
520 : "Approved. Hermes is continuing."
521 : `Could not send the answer to Hermes: ${result.error}`;
522 } catch (error) {
523 message = `Could not send the answer to Hermes: ${error.message || error}`;
524 }
525 addLine(ok ? "hermes" : "error", message);
526 // A fast turn can finish before this reply returns; its result was
527 // already handed to the voice, so an "is continuing" would be stale.
528 if (ok && !this.delegation) return;
529 setStatus(ok ? "Hermes is working…" : "Listening");
530 this.send({
531 type: "session.commentary.append",
532 delegation_id: replyId || approval.delegationId,
533 content: message.slice(0, APPEND_CHAR_LIMIT),
534 });
535 }
536
537 checkIdle() {
538 if (this.closed || this.delegation || this.speaking) return;
539 if (Date.now() - this.lastActivity > settings.idle_seconds * 1000) {
540 setStatus("Hanging up (idle)");
541 this.close();
542 }
543 }
544
545 close() {
546 if (this.closed || this.closing) return;
547 this.closing = true;
548 if (!this.send({ type: "session.close" })) {
549 this.finish("closed");
550 return;
551 }
552 // Wait for session.closed (final usage); give up after a while.
553 this.closeTimer = setTimeout(() => this.finish("close timed out"), 15000);
554 }
555
556 // Tell the server the call is over, with what the user said since the last
557 // delegation: a request spoken right before hanging up still reaches
558 // Hermes. keepalive lets the request outlive the page.
559 hangUp() {
560 const lines = state.lines
561 .slice(state.delegated)
562 .filter(
563 (line) => line.speaker === "user" || line.speaker === "assistant",
564 );
565 state.delegated = state.lines.length;
566 saveState();
567 api("/hangup", {
568 method: "POST",
569 keepalive: true,
570 headers: { "Content-Type": "application/json" },
571 body: JSON.stringify({ hermes_session: state.hermesSession, lines }),
572 }).catch(() => {});
573 }
574
575 finish(reason) {
576 if (this.closed) return;
577 this.closed = true;
578 clearTimeout(this.closeTimer);
579 clearTimeout(this.endTimer);
580 clearInterval(this.idleTimer);
581 clearInterval(this.probe);
582 this.hangUp();
583 // Only the stream is dropped; the server keeps following the Hermes run.
584 if (this.delegation) this.delegation.controller.abort();
585 this.audioContext?.close().catch(() => {});
586 this.mic?.getTracks().forEach((track) => track.stop());
587 this.events?.close();
588 this.pc?.close();
589 voice.srcObject = null;
590 state.billedSeconds += this.usageSeconds;
591 saveState();
592 if (call === this) call = null;
593 talk.classList.remove("live", "speaking");
594 talk.textContent = "Talk";
595 talk.disabled = false;
596 setStatus(`Idle (${reason})`);
597 renderCost();
598 }
599}
600
601talk.addEventListener("click", async () => {
602 if (call) {
603 talk.disabled = true;
604 setStatus("Hanging up…");
605 call.close();
606 return;
607 }
608 call = new Call();
609 talk.disabled = true;
610 talk.classList.add("live");
611 talk.textContent = "Hang up";
612 setStatus("Connecting…");
613 // Must run inside the click handler: mobile browsers only allow audio
614 // playback started by a user gesture.
615 voice.play().catch(() => {});
616 try {
617 await call.start();
618 talk.disabled = false;
619 } catch (error) {
620 addLine("error", String(error.message || error));
621 call.finish("failed to start");
622 }
623});
624
625window.addEventListener("pagehide", () => call?.finish("page closed"));
626
627document.getElementById("new").addEventListener("click", () => {
628 if (call) call.close();
629 state = newConversation();
630 saveState();
631 renderLog();
632 renderCost();
633 setStatus("Idle (new conversation)");
634});
635
636document
637 .getElementById("approve")
638 .addEventListener("click", () => call?.respondApproval("once", null));
639document
640 .getElementById("deny")
641 .addEventListener("click", () => call?.respondApproval("deny", null));
642
643function showLogin() {
644 loginForm.hidden = false;
645 talk.disabled = true;
646 setStatus("Sign in with the access token");
647}
648
649loginForm.addEventListener("submit", (event) => {
650 event.preventDefault();
651 localStorage.setItem(TOKEN_KEY, tokenInput.value.trim());
652 tokenInput.value = "";
653 loadConfig();
654});
655
656voiceSelect.addEventListener("change", () =>
657 localStorage.setItem(VOICE_KEY, voiceSelect.value),
658);
659
660async function loadConfig() {
661 let config;
662 try {
663 const response = await api("/config");
664 config = await response.json();
665 } catch (_) {
666 return; // 401 already showed the login form
667 }
668 settings = config;
669 loginForm.hidden = true;
670 talk.disabled = false;
671 setStatus("Idle");
672 const saved = localStorage.getItem(VOICE_KEY);
673 voiceSelect.replaceChildren(
674 ...Object.entries(config.voices).map(
675 ([name, description]) => new Option(`${name} — ${description}`, name),
676 ),
677 );
678 voiceSelect.value = saved in config.voices ? saved : config.voice;
679}
680
681loadConfig();
682renderLog();
683renderCost();