This repository has no description
20 kB
660 lines
1// hermes-voice page: one WebRTC session with gpt-live-1 per "call", client
2// delegation answered by Hermes through the local server (/delegate).
3"use strict";
4
5const PRICE_PER_MINUTE = 0.05;
6// The voice model accepts at most 500 tokens per append; ~4 chars per token.
7const APPEND_CHAR_LIMIT = 1400;
8// Transcript deltas can trail the delegation event; wait for the last words.
9const DELEGATION_SETTLE_MS = 400;
10const STORAGE_KEY = "hermes-voice";
11
12const talk = document.getElementById("talk");
13const statusLine = document.getElementById("status");
14const costLine = document.getElementById("cost");
15const logList = document.getElementById("log");
16const voice = document.getElementById("voice");
17const approvalBox = document.getElementById("approval");
18const approvalText = document.getElementById("approval-text");
19const voiceSelect = document.getElementById("voice-select");
20const VOICE_KEY = "hermes-voice-voice";
21const TOKEN_KEY = "hermes-voice-token";
22const loginForm = document.getElementById("login");
23const tokenInput = document.getElementById("token");
24
25// Every API route needs the page's access token; a 401 asks for it again.
26async function api(path, options = {}) {
27 const response = await fetch(path, {
28 ...options,
29 headers: {
30 ...options.headers,
31 Authorization: `Bearer ${localStorage.getItem(TOKEN_KEY) || ""}`,
32 },
33 });
34 if (response.status === 401) {
35 showLogin();
36 throw new Error("Sign in with the access token first");
37 }
38 return response;
39}
40
41// Spoken answer to an approval question. Denials win, so "no, don't do it"
42// never approves; anything unclear is asked again.
43function classifyApproval(said) {
44 if (
45 /\b(no|nope|deny|denied|don'?t|do not|stop|cancel|abort|reject|nein|nicht)\b/i.test(
46 said,
47 )
48 )
49 return "deny";
50 if (
51 /\b(yes|yeah|yep|approve|approved|go ahead|do it|okay|ok|sure|confirm|allow|ja)\b/i.test(
52 said,
53 )
54 )
55 return "once";
56 return null;
57}
58
59let settings = { idle_seconds: 90 };
60let call = null;
61let state = loadState();
62
63function newConversation() {
64 const day = new Date().toISOString().slice(0, 10).replaceAll("-", "");
65 const rand = crypto.getRandomValues(new Uint32Array(1))[0].toString(16);
66 return {
67 hermesSession: `voice-${day}-${rand}`,
68 lines: [],
69 delegated: 0,
70 billedSeconds: 0,
71 };
72}
73
74function loadState() {
75 try {
76 const saved = JSON.parse(localStorage.getItem(STORAGE_KEY));
77 if (saved && saved.hermesSession) return saved;
78 } catch (_) {
79 /* fall through */
80 }
81 return newConversation();
82}
83
84function saveState() {
85 localStorage.setItem(STORAGE_KEY, JSON.stringify(state));
86}
87
88function setStatus(text) {
89 statusLine.textContent = text;
90}
91
92function renderCost() {
93 const current = call ? call.usageSeconds : 0;
94 const total = state.billedSeconds + current;
95 const money = (seconds) =>
96 `$${((seconds / 60) * PRICE_PER_MINUTE).toFixed(3)}`;
97 const clock = (seconds) =>
98 `${Math.floor(seconds / 60)}:${String(Math.floor(seconds % 60)).padStart(2, "0")}`;
99 costLine.textContent = call
100 ? `this call ${clock(current)} · ${money(current)} — conversation ${money(total)}`
101 : `conversation ${clock(total)} · ${money(total)}`;
102}
103
104function renderLog() {
105 logList.replaceChildren(
106 ...state.lines.map((line) => {
107 const item = document.createElement("li");
108 item.className = line.speaker;
109 item.textContent = line.text;
110 return item;
111 }),
112 );
113 window.scrollTo(0, document.body.scrollHeight);
114}
115
116function addLine(speaker, text) {
117 state.lines.push({ speaker, text });
118 saveState();
119 renderLog();
120}
121
122// Deltas are appended exactly as received; a speaker change starts a new line.
123function addTranscript(speaker, delta) {
124 const last = state.lines.at(-1);
125 if (
126 last &&
127 last.speaker === speaker &&
128 call &&
129 call.lastSpeaker === speaker
130 ) {
131 last.text += delta;
132 } else {
133 state.lines.push({ speaker, text: delta });
134 }
135 if (call) call.lastSpeaker = speaker;
136 saveState();
137 renderLog();
138}
139
140function chunks(text) {
141 const clean = text.replace(/\s+/g, " ").trim();
142 const out = [];
143 let current = "";
144 for (const sentence of clean.split(/(?<=[.!?])\s+/)) {
145 if (sentence.length > APPEND_CHAR_LIMIT) {
146 if (current) out.push(current);
147 current = "";
148 for (let i = 0; i < sentence.length; i += APPEND_CHAR_LIMIT)
149 out.push(sentence.slice(i, i + APPEND_CHAR_LIMIT));
150 } else if (
151 current &&
152 current.length + 1 + sentence.length > APPEND_CHAR_LIMIT
153 ) {
154 out.push(current);
155 current = sentence;
156 } else {
157 current = current ? `${current} ${sentence}` : sentence;
158 }
159 }
160 if (current) out.push(current);
161 return out;
162}
163
164class Call {
165 constructor() {
166 this.usageSeconds = 0;
167 this.lastActivity = Date.now();
168 this.speaking = false;
169 this.lastSpeaker = null;
170 this.delegation = null;
171 this.closed = false;
172 this.eventCounter = 0;
173 }
174
175 async start() {
176 const pc = new RTCPeerConnection();
177 this.pc = pc;
178 pc.addEventListener("track", (event) => {
179 const stream = new MediaStream([event.track]);
180 voice.srcObject = stream;
181 voice.play().catch(() => {});
182 this.probeSpeaking(stream);
183 });
184 pc.addEventListener("connectionstatechange", () => {
185 if (
186 pc.connectionState === "failed" ||
187 pc.connectionState === "disconnected"
188 )
189 this.finish("connection lost");
190 });
191
192 this.mic = await navigator.mediaDevices.getUserMedia({
193 audio: {
194 echoCancellation: true,
195 noiseSuppression: true,
196 autoGainControl: true,
197 },
198 });
199 for (const track of this.mic.getAudioTracks()) pc.addTrack(track, this.mic);
200
201 // The data channel must exist before the offer so its m-line is negotiated.
202 this.events = pc.createDataChannel("oai-events");
203 this.events.addEventListener("message", ({ data }) =>
204 this.onEvent(JSON.parse(data)),
205 );
206 this.events.addEventListener("close", () =>
207 this.finish("connection closed"),
208 );
209
210 await pc.setLocalDescription(await pc.createOffer());
211 await this.iceGathered();
212
213 const response = await api("/session", {
214 method: "POST",
215 headers: { "Content-Type": "application/json" },
216 body: JSON.stringify({
217 sdp: pc.localDescription.sdp,
218 history: state.lines,
219 // Fixed per session: a new pick applies from the next call on.
220 voice: voiceSelect.value,
221 }),
222 });
223 const answer = await response.json();
224 if (!response.ok)
225 throw new Error(answer.error || `session failed (${response.status})`);
226 this.sessionId = answer.session_id;
227 await pc.setRemoteDescription({ type: "answer", sdp: answer.sdp });
228
229 this.idleTimer = setInterval(() => this.checkIdle(), 1000);
230 }
231
232 iceGathered() {
233 if (this.pc.iceGatheringState === "complete") return Promise.resolve();
234 return new Promise((resolve) => {
235 const done = () => {
236 if (this.pc.iceGatheringState !== "complete") return;
237 clearTimeout(timer);
238 resolve();
239 };
240 // Trickle is fine: the vendor answers with the candidates it has.
241 const timer = setTimeout(resolve, 3000);
242 this.pc.addEventListener("icegatheringstatechange", done);
243 });
244 }
245
246 probeSpeaking(stream) {
247 const context = new AudioContext();
248 const analyser = context.createAnalyser();
249 analyser.fftSize = 512;
250 context.createMediaStreamSource(stream).connect(analyser);
251 const buffer = new Uint8Array(analyser.frequencyBinCount);
252 let quiet = 0;
253 this.audioContext = context;
254 this.probe = setInterval(() => {
255 analyser.getByteTimeDomainData(buffer);
256 let peak = 0;
257 for (const sample of buffer)
258 peak = Math.max(peak, Math.abs(sample - 128));
259 quiet = peak > 6 ? 0 : quiet + 1;
260 const speaking = quiet < 4;
261 if (speaking) this.lastActivity = Date.now();
262 if (speaking !== this.speaking) {
263 this.speaking = speaking;
264 talk.classList.toggle("speaking", speaking);
265 }
266 }, 100);
267 }
268
269 send(event) {
270 if (!this.events || this.events.readyState !== "open") return false;
271 this.eventCounter += 1;
272 this.events.send(
273 JSON.stringify({ event_id: `evt_${this.eventCounter}`, ...event }),
274 );
275 return true;
276 }
277
278 onEvent(event) {
279 switch (event.type) {
280 case "session.started":
281 setStatus("Listening");
282 break;
283 case "session.input_transcript.delta":
284 this.lastActivity = Date.now();
285 addTranscript("user", event.delta || "");
286 break;
287 case "session.output_transcript.delta":
288 this.lastActivity = Date.now();
289 addTranscript("assistant", event.delta || "");
290 break;
291 case "session.delegation.created":
292 this.delegate(event.delegation.id);
293 break;
294 case "session.usage.updated":
295 this.usageSeconds = event.usage?.seconds ?? this.usageSeconds;
296 renderCost();
297 break;
298 case "error":
299 addLine("error", event.error?.message || "voice model error");
300 break;
301 case "session.closed":
302 if (event.usage?.seconds != null)
303 this.usageSeconds = event.usage.seconds;
304 this.finish(event.reason || "closed");
305 break;
306 }
307 }
308
309 async delegate(id) {
310 // One Hermes turn at a time: a newer delegation (usually a correction)
311 // supersedes the running one, and the server stops the old run.
312 // While Hermes waits for an approval, the next delegation is the user's
313 // answer, not a new request: the turn must keep running.
314 if (this.approval) {
315 this.answerApproval(id);
316 return;
317 }
318 const replace = Boolean(this.delegation);
319 if (this.delegation) this.delegation.controller.abort();
320 const controller = new AbortController();
321 const delegation = { id, controller };
322 this.delegation = delegation;
323 setStatus("Hermes is working…");
324
325 await new Promise((resolve) => setTimeout(resolve, DELEGATION_SETTLE_MS));
326 // A hang-up meanwhile sent these lines along itself.
327 if (this.delegation !== delegation || this.closed) return;
328 const fresh = state.lines
329 .slice(state.delegated)
330 .filter((line) => line.speaker !== "hermes" && line.speaker !== "error");
331 const lines = fresh.some((line) => line.speaker === "user")
332 ? fresh
333 : state.lines
334 .filter(
335 (line) => line.speaker === "user" || line.speaker === "assistant",
336 )
337 .slice(-4);
338 state.delegated = state.lines.length;
339 saveState();
340
341 let answer = "";
342 let failure = null;
343 try {
344 const response = await api("/delegate", {
345 method: "POST",
346 headers: { "Content-Type": "application/json" },
347 body: JSON.stringify({
348 hermes_session: state.hermesSession,
349 lines,
350 replace,
351 }),
352 signal: controller.signal,
353 });
354 if (!response.ok)
355 throw new Error(
356 (await response.json()).error ||
357 `delegate failed (${response.status})`,
358 );
359 const reader = response.body
360 .pipeThrough(new TextDecoderStream())
361 .getReader();
362 let buffered = "";
363 for (;;) {
364 const { value, done } = await reader.read();
365 if (done) break;
366 buffered += value;
367 let newline;
368 while ((newline = buffered.indexOf("\n")) >= 0) {
369 const line = buffered.slice(0, newline);
370 buffered = buffered.slice(newline + 1);
371 // Blank lines are the server's keepalives.
372 if (!line.trim()) continue;
373 const item = JSON.parse(line);
374 this.lastActivity = Date.now();
375 if (item.type === "approval") {
376 this.askApproval(id, item);
377 continue;
378 }
379 // Anything after an approval request means Hermes moved on
380 // (answered here, by button, or by its own timeout).
381 if (this.approval) this.clearApproval();
382 if (item.type === "tool" && item.status === "running") {
383 setStatus(`Hermes: ${item.label}`);
384 this.send({
385 type: "session.thinking.append",
386 delegation_id: id,
387 content: `Hermes is working on it: ${item.label}`.slice(
388 0,
389 APPEND_CHAR_LIMIT,
390 ),
391 });
392 } else if (item.type === "delta") {
393 answer += item.text;
394 } else if (item.type === "done" && item.error) {
395 failure = item.error;
396 } else if (item.type === "error") {
397 failure = item.message;
398 }
399 }
400 }
401 } catch (error) {
402 if (controller.signal.aborted) return;
403 failure = String(error.message || error);
404 }
405 if (this.delegation !== delegation) return;
406 this.delegation = null;
407 this.clearApproval();
408 this.lastActivity = Date.now();
409
410 if (failure && !answer.trim()) {
411 addLine("error", failure);
412 this.send({
413 type: "session.commentary.append",
414 delegation_id: id,
415 content: `The request to Hermes failed: ${failure}`.slice(
416 0,
417 APPEND_CHAR_LIMIT,
418 ),
419 });
420 } else {
421 const text = answer.trim() || "Hermes finished without an answer.";
422 addLine("hermes", text);
423 for (const chunk of chunks(text))
424 this.send({
425 type: "session.commentary.append",
426 delegation_id: id,
427 content: chunk,
428 });
429 }
430 setStatus("Listening");
431 }
432
433 askApproval(id, item) {
434 this.approval = { delegationId: id, code: item.code };
435 const what = item.description || "run a command";
436 setStatus("Hermes needs your approval");
437 addLine(
438 "hermes",
439 `Approval needed: ${what}${item.command ? `\n${item.command}` : ""}`,
440 );
441 approvalText.textContent = item.command ? `${what}: ${item.command}` : what;
442 approvalBox.hidden = false;
443 this.send({
444 type: "session.commentary.append",
445 delegation_id: id,
446 content: (
447 `Hermes needs the user's approval before it continues: ${what}.` +
448 (item.command ? ` The command is: ${item.command}.` : "") +
449 " Ask the user whether to approve or deny it."
450 ).slice(0, APPEND_CHAR_LIMIT),
451 });
452 }
453
454 clearApproval() {
455 this.approval = null;
456 approvalBox.hidden = true;
457 }
458
459 async answerApproval(id) {
460 await new Promise((resolve) => setTimeout(resolve, DELEGATION_SETTLE_MS));
461 const said = state.lines
462 .slice(state.delegated)
463 .filter((line) => line.speaker === "user")
464 .map((line) => line.text)
465 .join(" ");
466 state.delegated = state.lines.length;
467 saveState();
468 const choice = classifyApproval(said);
469 if (!choice) {
470 this.send({
471 type: "session.commentary.append",
472 delegation_id: id,
473 content:
474 "That was not a clear answer. Ask the user again: approve or deny?",
475 });
476 return;
477 }
478 await this.respondApproval(choice, id);
479 }
480
481 async respondApproval(choice, replyId) {
482 const approval = this.approval;
483 if (!approval) return;
484 this.clearApproval();
485 setStatus(choice === "deny" ? "Denying…" : "Approving…");
486 let message;
487 let ok = false;
488 try {
489 const response = await api("/approve", {
490 method: "POST",
491 headers: { "Content-Type": "application/json" },
492 body: JSON.stringify({ code: approval.code, choice }),
493 });
494 const result = await response.json();
495 ok = response.ok;
496 message = ok
497 ? choice === "deny"
498 ? "Denied. Hermes will not run it."
499 : "Approved. Hermes is continuing."
500 : `Could not send the answer to Hermes: ${result.error}`;
501 } catch (error) {
502 message = `Could not send the answer to Hermes: ${error.message || error}`;
503 }
504 addLine(ok ? "hermes" : "error", message);
505 // A fast turn can finish before this reply returns; its result was
506 // already handed to the voice, so an "is continuing" would be stale.
507 if (ok && !this.delegation) return;
508 setStatus(ok ? "Hermes is working…" : "Listening");
509 this.send({
510 type: "session.commentary.append",
511 delegation_id: replyId || approval.delegationId,
512 content: message.slice(0, APPEND_CHAR_LIMIT),
513 });
514 }
515
516 checkIdle() {
517 if (this.closed || this.delegation || this.speaking) return;
518 if (Date.now() - this.lastActivity > settings.idle_seconds * 1000) {
519 setStatus("Hanging up (idle)");
520 this.close();
521 }
522 }
523
524 close() {
525 if (this.closed) return;
526 if (!this.send({ type: "session.close" })) {
527 this.finish("closed");
528 return;
529 }
530 // Wait for session.closed (final usage); give up after a while.
531 this.closeTimer = setTimeout(() => this.finish("close timed out"), 15000);
532 }
533
534 // Tell the server the call is over, with what the user said since the last
535 // delegation: a request spoken right before hanging up still reaches
536 // Hermes. keepalive lets the request outlive the page.
537 hangUp() {
538 const lines = state.lines
539 .slice(state.delegated)
540 .filter(
541 (line) => line.speaker === "user" || line.speaker === "assistant",
542 );
543 state.delegated = state.lines.length;
544 saveState();
545 api("/hangup", {
546 method: "POST",
547 keepalive: true,
548 headers: { "Content-Type": "application/json" },
549 body: JSON.stringify({ hermes_session: state.hermesSession, lines }),
550 }).catch(() => {});
551 }
552
553 finish(reason) {
554 if (this.closed) return;
555 this.closed = true;
556 clearTimeout(this.closeTimer);
557 clearInterval(this.idleTimer);
558 clearInterval(this.probe);
559 this.hangUp();
560 // Only the stream is dropped; the server keeps following the Hermes run.
561 if (this.delegation) this.delegation.controller.abort();
562 this.audioContext?.close().catch(() => {});
563 this.mic?.getTracks().forEach((track) => track.stop());
564 this.events?.close();
565 this.pc?.close();
566 voice.srcObject = null;
567 state.billedSeconds += this.usageSeconds;
568 saveState();
569 if (call === this) call = null;
570 talk.classList.remove("live", "speaking");
571 talk.textContent = "Talk";
572 talk.disabled = false;
573 setStatus(`Idle (${reason})`);
574 renderCost();
575 }
576}
577
578talk.addEventListener("click", async () => {
579 if (call) {
580 talk.disabled = true;
581 setStatus("Hanging up…");
582 call.close();
583 return;
584 }
585 call = new Call();
586 talk.disabled = true;
587 talk.classList.add("live");
588 talk.textContent = "Hang up";
589 setStatus("Connecting…");
590 // Must run inside the click handler: mobile browsers only allow audio
591 // playback started by a user gesture.
592 voice.play().catch(() => {});
593 try {
594 await call.start();
595 talk.disabled = false;
596 } catch (error) {
597 addLine("error", String(error.message || error));
598 call.finish("failed to start");
599 }
600});
601
602window.addEventListener("pagehide", () => call?.finish("page closed"));
603
604document.getElementById("new").addEventListener("click", () => {
605 if (call) call.close();
606 state = newConversation();
607 saveState();
608 renderLog();
609 renderCost();
610 setStatus("Idle (new conversation)");
611});
612
613document
614 .getElementById("approve")
615 .addEventListener("click", () => call?.respondApproval("once", null));
616document
617 .getElementById("deny")
618 .addEventListener("click", () => call?.respondApproval("deny", null));
619
620function showLogin() {
621 loginForm.hidden = false;
622 talk.disabled = true;
623 setStatus("Sign in with the access token");
624}
625
626loginForm.addEventListener("submit", (event) => {
627 event.preventDefault();
628 localStorage.setItem(TOKEN_KEY, tokenInput.value.trim());
629 tokenInput.value = "";
630 loadConfig();
631});
632
633voiceSelect.addEventListener("change", () =>
634 localStorage.setItem(VOICE_KEY, voiceSelect.value),
635);
636
637async function loadConfig() {
638 let config;
639 try {
640 const response = await api("/config");
641 config = await response.json();
642 } catch (_) {
643 return; // 401 already showed the login form
644 }
645 settings = config;
646 loginForm.hidden = true;
647 talk.disabled = false;
648 setStatus("Idle");
649 const saved = localStorage.getItem(VOICE_KEY);
650 voiceSelect.replaceChildren(
651 ...Object.entries(config.voices).map(
652 ([name, description]) => new Option(`${name} — ${description}`, name),
653 ),
654 );
655 voiceSelect.value = saved in config.voices ? saved : config.voice;
656}
657
658loadConfig();
659renderLog();
660renderCost();