This repository has no description
19 kB
634 lines
1// hermes-voice page: one WebRTC session with gpt-live-1 per "call", client
2// delegation answered by Hermes through the local server (/delegate).
3"use strict";
4
5const PRICE_PER_MINUTE = 0.05;
6// The voice model accepts at most 500 tokens per append; ~4 chars per token.
7const APPEND_CHAR_LIMIT = 1400;
8// Transcript deltas can trail the delegation event; wait for the last words.
9const DELEGATION_SETTLE_MS = 400;
10const STORAGE_KEY = "hermes-voice";
11
12const talk = document.getElementById("talk");
13const statusLine = document.getElementById("status");
14const costLine = document.getElementById("cost");
15const logList = document.getElementById("log");
16const voice = document.getElementById("voice");
17const approvalBox = document.getElementById("approval");
18const approvalText = document.getElementById("approval-text");
19const voiceSelect = document.getElementById("voice-select");
20const VOICE_KEY = "hermes-voice-voice";
21const TOKEN_KEY = "hermes-voice-token";
22const loginForm = document.getElementById("login");
23const tokenInput = document.getElementById("token");
24
25// Every API route needs the page's access token; a 401 asks for it again.
26async function api(path, options = {}) {
27 const response = await fetch(path, {
28 ...options,
29 headers: {
30 ...options.headers,
31 Authorization: `Bearer ${localStorage.getItem(TOKEN_KEY) || ""}`,
32 },
33 });
34 if (response.status === 401) {
35 showLogin();
36 throw new Error("Sign in with the access token first");
37 }
38 return response;
39}
40
41// Spoken answer to an approval question. Denials win, so "no, don't do it"
42// never approves; anything unclear is asked again.
43function classifyApproval(said) {
44 if (
45 /\b(no|nope|deny|denied|don'?t|do not|stop|cancel|abort|reject|nein|nicht)\b/i.test(
46 said,
47 )
48 )
49 return "deny";
50 if (
51 /\b(yes|yeah|yep|approve|approved|go ahead|do it|okay|ok|sure|confirm|allow|ja)\b/i.test(
52 said,
53 )
54 )
55 return "once";
56 return null;
57}
58
59let settings = { idle_seconds: 90 };
60let call = null;
61let state = loadState();
62
63function newConversation() {
64 const day = new Date().toISOString().slice(0, 10).replaceAll("-", "");
65 const rand = crypto.getRandomValues(new Uint32Array(1))[0].toString(16);
66 return {
67 hermesSession: `voice-${day}-${rand}`,
68 lines: [],
69 delegated: 0,
70 billedSeconds: 0,
71 };
72}
73
74function loadState() {
75 try {
76 const saved = JSON.parse(localStorage.getItem(STORAGE_KEY));
77 if (saved && saved.hermesSession) return saved;
78 } catch (_) {
79 /* fall through */
80 }
81 return newConversation();
82}
83
84function saveState() {
85 localStorage.setItem(STORAGE_KEY, JSON.stringify(state));
86}
87
88function setStatus(text) {
89 statusLine.textContent = text;
90}
91
92function renderCost() {
93 const current = call ? call.usageSeconds : 0;
94 const total = state.billedSeconds + current;
95 const money = (seconds) =>
96 `$${((seconds / 60) * PRICE_PER_MINUTE).toFixed(3)}`;
97 const clock = (seconds) =>
98 `${Math.floor(seconds / 60)}:${String(Math.floor(seconds % 60)).padStart(2, "0")}`;
99 costLine.textContent = call
100 ? `this call ${clock(current)} · ${money(current)} — conversation ${money(total)}`
101 : `conversation ${clock(total)} · ${money(total)}`;
102}
103
104function renderLog() {
105 logList.replaceChildren(
106 ...state.lines.map((line) => {
107 const item = document.createElement("li");
108 item.className = line.speaker;
109 item.textContent = line.text;
110 return item;
111 }),
112 );
113 window.scrollTo(0, document.body.scrollHeight);
114}
115
116function addLine(speaker, text) {
117 state.lines.push({ speaker, text });
118 saveState();
119 renderLog();
120}
121
122// Deltas are appended exactly as received; a speaker change starts a new line.
123function addTranscript(speaker, delta) {
124 const last = state.lines.at(-1);
125 if (
126 last &&
127 last.speaker === speaker &&
128 call &&
129 call.lastSpeaker === speaker
130 ) {
131 last.text += delta;
132 } else {
133 state.lines.push({ speaker, text: delta });
134 }
135 if (call) call.lastSpeaker = speaker;
136 saveState();
137 renderLog();
138}
139
140function chunks(text) {
141 const clean = text.replace(/\s+/g, " ").trim();
142 const out = [];
143 let current = "";
144 for (const sentence of clean.split(/(?<=[.!?])\s+/)) {
145 if (sentence.length > APPEND_CHAR_LIMIT) {
146 if (current) out.push(current);
147 current = "";
148 for (let i = 0; i < sentence.length; i += APPEND_CHAR_LIMIT)
149 out.push(sentence.slice(i, i + APPEND_CHAR_LIMIT));
150 } else if (
151 current &&
152 current.length + 1 + sentence.length > APPEND_CHAR_LIMIT
153 ) {
154 out.push(current);
155 current = sentence;
156 } else {
157 current = current ? `${current} ${sentence}` : sentence;
158 }
159 }
160 if (current) out.push(current);
161 return out;
162}
163
164class Call {
165 constructor() {
166 this.usageSeconds = 0;
167 this.lastActivity = Date.now();
168 this.speaking = false;
169 this.lastSpeaker = null;
170 this.delegation = null;
171 this.closed = false;
172 this.eventCounter = 0;
173 }
174
175 async start() {
176 const pc = new RTCPeerConnection();
177 this.pc = pc;
178 pc.addEventListener("track", (event) => {
179 const stream = new MediaStream([event.track]);
180 voice.srcObject = stream;
181 voice.play().catch(() => {});
182 this.probeSpeaking(stream);
183 });
184 pc.addEventListener("connectionstatechange", () => {
185 if (
186 pc.connectionState === "failed" ||
187 pc.connectionState === "disconnected"
188 )
189 this.finish("connection lost");
190 });
191
192 this.mic = await navigator.mediaDevices.getUserMedia({
193 audio: {
194 echoCancellation: true,
195 noiseSuppression: true,
196 autoGainControl: true,
197 },
198 });
199 for (const track of this.mic.getAudioTracks()) pc.addTrack(track, this.mic);
200
201 // The data channel must exist before the offer so its m-line is negotiated.
202 this.events = pc.createDataChannel("oai-events");
203 this.events.addEventListener("message", ({ data }) =>
204 this.onEvent(JSON.parse(data)),
205 );
206 this.events.addEventListener("close", () =>
207 this.finish("connection closed"),
208 );
209
210 await pc.setLocalDescription(await pc.createOffer());
211 await this.iceGathered();
212
213 const response = await api("/session", {
214 method: "POST",
215 headers: { "Content-Type": "application/json" },
216 body: JSON.stringify({
217 sdp: pc.localDescription.sdp,
218 history: state.lines,
219 // Fixed per session: a new pick applies from the next call on.
220 voice: voiceSelect.value,
221 }),
222 });
223 const answer = await response.json();
224 if (!response.ok)
225 throw new Error(answer.error || `session failed (${response.status})`);
226 this.sessionId = answer.session_id;
227 await pc.setRemoteDescription({ type: "answer", sdp: answer.sdp });
228
229 this.idleTimer = setInterval(() => this.checkIdle(), 1000);
230 }
231
232 iceGathered() {
233 if (this.pc.iceGatheringState === "complete") return Promise.resolve();
234 return new Promise((resolve) => {
235 const done = () => {
236 if (this.pc.iceGatheringState !== "complete") return;
237 clearTimeout(timer);
238 resolve();
239 };
240 // Trickle is fine: the vendor answers with the candidates it has.
241 const timer = setTimeout(resolve, 3000);
242 this.pc.addEventListener("icegatheringstatechange", done);
243 });
244 }
245
246 probeSpeaking(stream) {
247 const context = new AudioContext();
248 const analyser = context.createAnalyser();
249 analyser.fftSize = 512;
250 context.createMediaStreamSource(stream).connect(analyser);
251 const buffer = new Uint8Array(analyser.frequencyBinCount);
252 let quiet = 0;
253 this.audioContext = context;
254 this.probe = setInterval(() => {
255 analyser.getByteTimeDomainData(buffer);
256 let peak = 0;
257 for (const sample of buffer)
258 peak = Math.max(peak, Math.abs(sample - 128));
259 quiet = peak > 6 ? 0 : quiet + 1;
260 const speaking = quiet < 4;
261 if (speaking) this.lastActivity = Date.now();
262 if (speaking !== this.speaking) {
263 this.speaking = speaking;
264 talk.classList.toggle("speaking", speaking);
265 }
266 }, 100);
267 }
268
269 send(event) {
270 if (!this.events || this.events.readyState !== "open") return false;
271 this.eventCounter += 1;
272 this.events.send(
273 JSON.stringify({ event_id: `evt_${this.eventCounter}`, ...event }),
274 );
275 return true;
276 }
277
278 onEvent(event) {
279 switch (event.type) {
280 case "session.started":
281 setStatus("Listening");
282 break;
283 case "session.input_transcript.delta":
284 this.lastActivity = Date.now();
285 addTranscript("user", event.delta || "");
286 break;
287 case "session.output_transcript.delta":
288 this.lastActivity = Date.now();
289 addTranscript("assistant", event.delta || "");
290 break;
291 case "session.delegation.created":
292 this.delegate(event.delegation.id);
293 break;
294 case "session.usage.updated":
295 this.usageSeconds = event.usage?.seconds ?? this.usageSeconds;
296 renderCost();
297 break;
298 case "error":
299 addLine("error", event.error?.message || "voice model error");
300 break;
301 case "session.closed":
302 if (event.usage?.seconds != null)
303 this.usageSeconds = event.usage.seconds;
304 this.finish(event.reason || "closed");
305 break;
306 }
307 }
308
309 async delegate(id) {
310 // One Hermes turn at a time: a newer delegation (usually a correction)
311 // supersedes the running one, and abandoning the request interrupts it.
312 // While Hermes waits for an approval, the next delegation is the user's
313 // answer, not a new request: the turn must keep running.
314 if (this.approval) {
315 this.answerApproval(id);
316 return;
317 }
318 if (this.delegation) this.delegation.controller.abort();
319 const controller = new AbortController();
320 const delegation = { id, controller };
321 this.delegation = delegation;
322 setStatus("Hermes is working…");
323
324 await new Promise((resolve) => setTimeout(resolve, DELEGATION_SETTLE_MS));
325 if (this.delegation !== delegation) return;
326 const fresh = state.lines
327 .slice(state.delegated)
328 .filter((line) => line.speaker !== "hermes" && line.speaker !== "error");
329 const lines = fresh.some((line) => line.speaker === "user")
330 ? fresh
331 : state.lines
332 .filter(
333 (line) => line.speaker === "user" || line.speaker === "assistant",
334 )
335 .slice(-4);
336 state.delegated = state.lines.length;
337 saveState();
338
339 let answer = "";
340 let failure = null;
341 try {
342 const response = await api("/delegate", {
343 method: "POST",
344 headers: { "Content-Type": "application/json" },
345 body: JSON.stringify({ hermes_session: state.hermesSession, lines }),
346 signal: controller.signal,
347 });
348 if (!response.ok)
349 throw new Error(
350 (await response.json()).error ||
351 `delegate failed (${response.status})`,
352 );
353 const reader = response.body
354 .pipeThrough(new TextDecoderStream())
355 .getReader();
356 let buffered = "";
357 for (;;) {
358 const { value, done } = await reader.read();
359 if (done) break;
360 buffered += value;
361 let newline;
362 while ((newline = buffered.indexOf("\n")) >= 0) {
363 const item = JSON.parse(buffered.slice(0, newline));
364 buffered = buffered.slice(newline + 1);
365 this.lastActivity = Date.now();
366 if (item.type === "approval") {
367 this.askApproval(id, item);
368 continue;
369 }
370 // Anything after an approval request means Hermes moved on
371 // (answered here, by button, or by its own timeout).
372 if (this.approval) this.clearApproval();
373 if (item.type === "tool" && item.status === "running") {
374 setStatus(`Hermes: ${item.label}`);
375 this.send({
376 type: "session.thinking.append",
377 delegation_id: id,
378 content: `Hermes is working on it: ${item.label}`.slice(
379 0,
380 APPEND_CHAR_LIMIT,
381 ),
382 });
383 } else if (item.type === "delta") {
384 answer += item.text;
385 } else if (item.type === "done" && item.error) {
386 failure = item.error;
387 } else if (item.type === "error") {
388 failure = item.message;
389 }
390 }
391 }
392 } catch (error) {
393 if (controller.signal.aborted) return;
394 failure = String(error.message || error);
395 }
396 if (this.delegation !== delegation) return;
397 this.delegation = null;
398 this.clearApproval();
399 this.lastActivity = Date.now();
400
401 if (failure && !answer.trim()) {
402 addLine("error", failure);
403 this.send({
404 type: "session.commentary.append",
405 delegation_id: id,
406 content: `The request to Hermes failed: ${failure}`.slice(
407 0,
408 APPEND_CHAR_LIMIT,
409 ),
410 });
411 } else {
412 const text = answer.trim() || "Hermes finished without an answer.";
413 addLine("hermes", text);
414 for (const chunk of chunks(text))
415 this.send({
416 type: "session.commentary.append",
417 delegation_id: id,
418 content: chunk,
419 });
420 }
421 setStatus("Listening");
422 }
423
424 askApproval(id, item) {
425 this.approval = {
426 delegationId: id,
427 runId: item.run_id,
428 requestId: item.request_id,
429 };
430 const what = item.description || "run a command";
431 setStatus("Hermes needs your approval");
432 addLine(
433 "hermes",
434 `Approval needed: ${what}${item.command ? `\n${item.command}` : ""}`,
435 );
436 approvalText.textContent = item.command ? `${what}: ${item.command}` : what;
437 approvalBox.hidden = false;
438 this.send({
439 type: "session.commentary.append",
440 delegation_id: id,
441 content: (
442 `Hermes needs the user's approval before it continues: ${what}.` +
443 (item.command ? ` The command is: ${item.command}.` : "") +
444 " Ask the user whether to approve or deny it."
445 ).slice(0, APPEND_CHAR_LIMIT),
446 });
447 }
448
449 clearApproval() {
450 this.approval = null;
451 approvalBox.hidden = true;
452 }
453
454 async answerApproval(id) {
455 await new Promise((resolve) => setTimeout(resolve, DELEGATION_SETTLE_MS));
456 const said = state.lines
457 .slice(state.delegated)
458 .filter((line) => line.speaker === "user")
459 .map((line) => line.text)
460 .join(" ");
461 state.delegated = state.lines.length;
462 saveState();
463 const choice = classifyApproval(said);
464 if (!choice) {
465 this.send({
466 type: "session.commentary.append",
467 delegation_id: id,
468 content:
469 "That was not a clear answer. Ask the user again: approve or deny?",
470 });
471 return;
472 }
473 await this.respondApproval(choice, id);
474 }
475
476 async respondApproval(choice, replyId) {
477 const approval = this.approval;
478 if (!approval) return;
479 this.clearApproval();
480 setStatus(choice === "deny" ? "Denying…" : "Approving…");
481 let message;
482 let ok = false;
483 try {
484 const response = await api("/approve", {
485 method: "POST",
486 headers: { "Content-Type": "application/json" },
487 body: JSON.stringify({
488 run_id: approval.runId,
489 request_id: approval.requestId,
490 choice,
491 }),
492 });
493 const result = await response.json();
494 ok = response.ok;
495 message = ok
496 ? choice === "deny"
497 ? "Denied. Hermes will not run it."
498 : "Approved. Hermes is continuing."
499 : `Could not send the answer to Hermes: ${result.error}`;
500 } catch (error) {
501 message = `Could not send the answer to Hermes: ${error.message || error}`;
502 }
503 addLine(ok ? "hermes" : "error", message);
504 // A fast turn can finish before this reply returns; its result was
505 // already handed to the voice, so an "is continuing" would be stale.
506 if (ok && !this.delegation) return;
507 setStatus(ok ? "Hermes is working…" : "Listening");
508 this.send({
509 type: "session.commentary.append",
510 delegation_id: replyId || approval.delegationId,
511 content: message.slice(0, APPEND_CHAR_LIMIT),
512 });
513 }
514
515 checkIdle() {
516 if (this.closed || this.delegation || this.speaking) return;
517 if (Date.now() - this.lastActivity > settings.idle_seconds * 1000) {
518 setStatus("Hanging up (idle)");
519 this.close();
520 }
521 }
522
523 close() {
524 if (this.closed) return;
525 if (!this.send({ type: "session.close" })) {
526 this.finish("closed");
527 return;
528 }
529 // Wait for session.closed (final usage); give up after a while.
530 this.closeTimer = setTimeout(() => this.finish("close timed out"), 15000);
531 }
532
533 finish(reason) {
534 if (this.closed) return;
535 this.closed = true;
536 clearTimeout(this.closeTimer);
537 clearInterval(this.idleTimer);
538 clearInterval(this.probe);
539 if (this.delegation) this.delegation.controller.abort();
540 this.audioContext?.close().catch(() => {});
541 this.mic?.getTracks().forEach((track) => track.stop());
542 this.events?.close();
543 this.pc?.close();
544 voice.srcObject = null;
545 state.billedSeconds += this.usageSeconds;
546 saveState();
547 if (call === this) call = null;
548 talk.classList.remove("live", "speaking");
549 talk.textContent = "Talk";
550 talk.disabled = false;
551 setStatus(`Idle (${reason})`);
552 renderCost();
553 }
554}
555
556talk.addEventListener("click", async () => {
557 if (call) {
558 talk.disabled = true;
559 setStatus("Hanging up…");
560 call.close();
561 return;
562 }
563 call = new Call();
564 talk.disabled = true;
565 talk.classList.add("live");
566 talk.textContent = "Hang up";
567 setStatus("Connecting…");
568 // Must run inside the click handler: mobile browsers only allow audio
569 // playback started by a user gesture.
570 voice.play().catch(() => {});
571 try {
572 await call.start();
573 talk.disabled = false;
574 } catch (error) {
575 addLine("error", String(error.message || error));
576 call.finish("failed to start");
577 }
578});
579
580document.getElementById("new").addEventListener("click", () => {
581 if (call) call.close();
582 state = newConversation();
583 saveState();
584 renderLog();
585 renderCost();
586 setStatus("Idle (new conversation)");
587});
588
589document
590 .getElementById("approve")
591 .addEventListener("click", () => call?.respondApproval("once", null));
592document
593 .getElementById("deny")
594 .addEventListener("click", () => call?.respondApproval("deny", null));
595
596function showLogin() {
597 loginForm.hidden = false;
598 talk.disabled = true;
599 setStatus("Sign in with the access token");
600}
601
602loginForm.addEventListener("submit", (event) => {
603 event.preventDefault();
604 localStorage.setItem(TOKEN_KEY, tokenInput.value.trim());
605 tokenInput.value = "";
606 loadConfig();
607});
608
609voiceSelect.addEventListener("change", () =>
610 localStorage.setItem(VOICE_KEY, voiceSelect.value),
611);
612
613async function loadConfig() {
614 let config;
615 try {
616 const response = await api("/config");
617 config = await response.json();
618 } catch (_) {
619 return; // 401 already showed the login form
620 }
621 settings = config;
622 loginForm.hidden = true;
623 talk.disabled = false;
624 setStatus("Idle");
625 const saved = localStorage.getItem(VOICE_KEY);
626 voiceSelect.replaceChildren(
627 ...config.voices.map((name) => new Option(name, name)),
628 );
629 voiceSelect.value = config.voices.includes(saved) ? saved : config.voice;
630}
631
632loadConfig();
633renderLog();
634renderCost();