This repository has no description
22 kB
682 lines
1// hermes-voice page: one WebRTC session with gpt-live-1 per "call", client
2// delegation answered by Hermes through the local server (/delegate).
3"use strict";
4
5const PRICE_PER_MINUTE = 0.05;
6// The voice model accepts at most 500 tokens per append; ~4 chars per token.
7const APPEND_CHAR_LIMIT = 1400;
8// Transcript deltas can trail the delegation event; wait for the last words.
9const DELEGATION_SETTLE_MS = 400;
10// Once Hermes ended the call (end_call item), the call closes when the voice
11// has been quiet this long after speaking the reply, at the latest after
12// END_CALL_MAX_MS.
13const END_CALL_QUIET_MS = 800;
14const END_CALL_MAX_MS = 30000;
15const STORAGE_KEY = "hermes-voice";
16
17const talk = document.getElementById("talk");
18const statusLine = document.getElementById("status");
19const costLine = document.getElementById("cost");
20const logList = document.getElementById("log");
21const voice = document.getElementById("voice");
22const approvalBox = document.getElementById("approval");
23const approvalText = document.getElementById("approval-text");
24const voiceSelect = document.getElementById("voice-select");
25const VOICE_KEY = "hermes-voice-voice";
26
27// The proxy in front adds the access token to logged-in requests. Once the
28// login expired it redirects to the login, which a reload follows.
29async function api(path, options = {}) {
30 const response = await fetch(path, { ...options, redirect: "manual" });
31 if (response.type === "opaqueredirect") {
32 location.reload();
33 throw new Error("Login expired");
34 }
35 return response;
36}
37
38// Spoken answer to an approval question. Denials win, so "no, don't do it"
39// never approves; anything unclear is asked again.
40function classifyApproval(said) {
41 if (
42 /\b(no|nope|deny|denied|don'?t|do not|stop|cancel|abort|reject|nein|nicht)\b/i.test(
43 said,
44 )
45 )
46 return "deny";
47 if (
48 /\b(yes|yeah|yep|approve|approved|go ahead|do it|okay|ok|sure|confirm|allow|ja)\b/i.test(
49 said,
50 )
51 )
52 return "once";
53 return null;
54}
55
56let settings = { idle_seconds: 90 };
57let call = null;
58let state = loadState();
59
60function newConversation() {
61 const day = new Date().toISOString().slice(0, 10).replaceAll("-", "");
62 const rand = crypto.getRandomValues(new Uint32Array(1))[0].toString(16);
63 return {
64 hermesSession: `voice-${day}-${rand}`,
65 lines: [],
66 delegated: 0,
67 billedSeconds: 0,
68 };
69}
70
71function loadState() {
72 try {
73 const saved = JSON.parse(localStorage.getItem(STORAGE_KEY));
74 if (saved && saved.hermesSession) return saved;
75 } catch (_) {
76 /* fall through */
77 }
78 return newConversation();
79}
80
81function saveState() {
82 localStorage.setItem(STORAGE_KEY, JSON.stringify(state));
83}
84
85function setStatus(text) {
86 statusLine.textContent = text;
87}
88
89function renderCost() {
90 const current = call ? call.usageSeconds : 0;
91 const total = state.billedSeconds + current;
92 const money = (seconds) =>
93 `$${((seconds / 60) * PRICE_PER_MINUTE).toFixed(3)}`;
94 const clock = (seconds) =>
95 `${Math.floor(seconds / 60)}:${String(Math.floor(seconds % 60)).padStart(2, "0")}`;
96 costLine.textContent = call
97 ? `this call ${clock(current)} · ${money(current)} — conversation ${money(total)}`
98 : `conversation ${clock(total)} · ${money(total)}`;
99}
100
101// Only the transcript scrolls (style.css); it follows the newest line unless
102// the user scrolled up. The controls above it change height after a render
103// (cost line, approval box), so a resize re-pins it too. A resize leaves
104// scrollTop alone; only moving up (the user) unpins.
105let logPinned = true;
106let logTop = 0;
107logList.addEventListener("scroll", () => {
108 if (logList.scrollTop + logList.clientHeight >= logList.scrollHeight - 4)
109 logPinned = true;
110 else if (logList.scrollTop < logTop) logPinned = false;
111 logTop = logList.scrollTop;
112});
113new ResizeObserver(() => {
114 if (logPinned) logList.scrollTop = logList.scrollHeight;
115}).observe(logList);
116
117function renderLog() {
118 logList.replaceChildren(
119 ...state.lines.map((line) => {
120 const item = document.createElement("li");
121 item.className = line.speaker;
122 item.textContent = line.text;
123 return item;
124 }),
125 );
126 logList.scrollTop = logList.scrollHeight;
127}
128
129function addLine(speaker, text) {
130 state.lines.push({ speaker, text });
131 saveState();
132 renderLog();
133}
134
135// Deltas are appended exactly as received; a speaker change starts a new line.
136function addTranscript(speaker, delta) {
137 const last = state.lines.at(-1);
138 if (
139 last &&
140 last.speaker === speaker &&
141 call &&
142 call.lastSpeaker === speaker
143 ) {
144 last.text += delta;
145 } else {
146 state.lines.push({ speaker, text: delta });
147 }
148 if (call) call.lastSpeaker = speaker;
149 saveState();
150 renderLog();
151}
152
153function chunks(text) {
154 const clean = text.replace(/\s+/g, " ").trim();
155 const out = [];
156 let current = "";
157 for (const sentence of clean.split(/(?<=[.!?])\s+/)) {
158 if (sentence.length > APPEND_CHAR_LIMIT) {
159 if (current) out.push(current);
160 current = "";
161 for (let i = 0; i < sentence.length; i += APPEND_CHAR_LIMIT)
162 out.push(sentence.slice(i, i + APPEND_CHAR_LIMIT));
163 } else if (
164 current &&
165 current.length + 1 + sentence.length > APPEND_CHAR_LIMIT
166 ) {
167 out.push(current);
168 current = sentence;
169 } else {
170 current = current ? `${current} ${sentence}` : sentence;
171 }
172 }
173 if (current) out.push(current);
174 return out;
175}
176
177class Call {
178 constructor() {
179 this.usageSeconds = 0;
180 this.lastActivity = Date.now();
181 this.speaking = false;
182 this.lastSpeaker = null;
183 this.delegation = null;
184 this.closed = false;
185 this.closing = false;
186 this.ending = null;
187 this.eventCounter = 0;
188 }
189
190 async start() {
191 const pc = new RTCPeerConnection();
192 this.pc = pc;
193 pc.addEventListener("track", (event) => {
194 const stream = new MediaStream([event.track]);
195 voice.srcObject = stream;
196 voice.play().catch(() => {});
197 this.probeSpeaking(stream);
198 });
199 pc.addEventListener("connectionstatechange", () => {
200 if (
201 pc.connectionState === "failed" ||
202 pc.connectionState === "disconnected"
203 )
204 this.finish("connection lost");
205 });
206
207 this.mic = await navigator.mediaDevices.getUserMedia({
208 audio: {
209 echoCancellation: true,
210 noiseSuppression: true,
211 autoGainControl: true,
212 },
213 });
214 for (const track of this.mic.getAudioTracks()) pc.addTrack(track, this.mic);
215
216 // The data channel must exist before the offer so its m-line is negotiated.
217 this.events = pc.createDataChannel("oai-events");
218 this.events.addEventListener("message", ({ data }) =>
219 this.onEvent(JSON.parse(data)),
220 );
221 this.events.addEventListener("close", () =>
222 this.finish("connection closed"),
223 );
224
225 await pc.setLocalDescription(await pc.createOffer());
226 await this.iceGathered();
227
228 const response = await api("/session", {
229 method: "POST",
230 headers: { "Content-Type": "application/json" },
231 body: JSON.stringify({
232 sdp: pc.localDescription.sdp,
233 history: state.lines,
234 // Fixed per session: a new pick applies from the next call on.
235 voice: voiceSelect.value,
236 }),
237 });
238 const answer = await response.json();
239 if (!response.ok)
240 throw new Error(answer.error || `session failed (${response.status})`);
241 this.sessionId = answer.session_id;
242 await pc.setRemoteDescription({ type: "answer", sdp: answer.sdp });
243
244 this.idleTimer = setInterval(() => this.checkIdle(), 1000);
245 }
246
247 iceGathered() {
248 if (this.pc.iceGatheringState === "complete") return Promise.resolve();
249 return new Promise((resolve) => {
250 const done = () => {
251 if (this.pc.iceGatheringState !== "complete") return;
252 clearTimeout(timer);
253 resolve();
254 };
255 // Trickle is fine: the vendor answers with the candidates it has.
256 const timer = setTimeout(resolve, 3000);
257 this.pc.addEventListener("icegatheringstatechange", done);
258 });
259 }
260
261 probeSpeaking(stream) {
262 const context = new AudioContext();
263 const analyser = context.createAnalyser();
264 analyser.fftSize = 512;
265 context.createMediaStreamSource(stream).connect(analyser);
266 const buffer = new Uint8Array(analyser.frequencyBinCount);
267 let quiet = 0;
268 this.audioContext = context;
269 this.probe = setInterval(() => {
270 analyser.getByteTimeDomainData(buffer);
271 let peak = 0;
272 for (const sample of buffer)
273 peak = Math.max(peak, Math.abs(sample - 128));
274 quiet = peak > 6 ? 0 : quiet + 1;
275 const speaking = quiet < 4;
276 if (speaking) this.lastActivity = Date.now();
277 if (speaking !== this.speaking) {
278 this.speaking = speaking;
279 talk.classList.toggle("speaking", speaking);
280 }
281 if (this.ending?.spoken && quiet * 100 >= END_CALL_QUIET_MS) this.close();
282 }, 100);
283 }
284
285 send(event) {
286 if (!this.events || this.events.readyState !== "open") return false;
287 this.eventCounter += 1;
288 this.events.send(
289 JSON.stringify({ event_id: `evt_${this.eventCounter}`, ...event }),
290 );
291 return true;
292 }
293
294 onEvent(event) {
295 switch (event.type) {
296 case "session.started":
297 setStatus("Listening");
298 break;
299 case "session.input_transcript.delta":
300 this.lastActivity = Date.now();
301 addTranscript("user", event.delta || "");
302 break;
303 case "session.output_transcript.delta":
304 this.lastActivity = Date.now();
305 addTranscript("assistant", event.delta || "");
306 if (this.ending) this.ending.spoken = true;
307 break;
308 case "session.delegation.created":
309 this.delegate(event.delegation.id);
310 break;
311 case "session.usage.updated":
312 this.usageSeconds = event.usage?.seconds ?? this.usageSeconds;
313 renderCost();
314 break;
315 case "error":
316 addLine("error", event.error?.message || "voice model error");
317 break;
318 case "session.closed":
319 if (event.usage?.seconds != null)
320 this.usageSeconds = event.usage.seconds;
321 this.finish(event.reason || "closed");
322 break;
323 }
324 }
325
326 async delegate(id) {
327 // One Hermes turn at a time: a newer delegation (usually a correction)
328 // supersedes the running one, and the server stops the old run.
329 // While Hermes waits for an approval, the next delegation is the user's
330 // answer, not a new request: the turn must keep running.
331 if (this.approval) {
332 this.answerApproval(id);
333 return;
334 }
335 const replace = Boolean(this.delegation);
336 if (this.delegation) this.delegation.controller.abort();
337 const controller = new AbortController();
338 const delegation = { id, controller };
339 this.delegation = delegation;
340 setStatus("Hermes is working…");
341
342 await new Promise((resolve) => setTimeout(resolve, DELEGATION_SETTLE_MS));
343 // A hang-up meanwhile sent these lines along itself.
344 if (this.delegation !== delegation || this.closed) return;
345 const fresh = state.lines
346 .slice(state.delegated)
347 .filter((line) => line.speaker !== "hermes" && line.speaker !== "error");
348 const lines = fresh.some((line) => line.speaker === "user")
349 ? fresh
350 : state.lines
351 .filter(
352 (line) => line.speaker === "user" || line.speaker === "assistant",
353 )
354 .slice(-4);
355 state.delegated = state.lines.length;
356 saveState();
357
358 let answer = "";
359 let failure = null;
360 let endCall = false;
361 try {
362 const response = await api("/delegate", {
363 method: "POST",
364 headers: { "Content-Type": "application/json" },
365 body: JSON.stringify({
366 hermes_session: state.hermesSession,
367 lines,
368 replace,
369 }),
370 signal: controller.signal,
371 });
372 if (!response.ok)
373 throw new Error(
374 (await response.json()).error ||
375 `delegate failed (${response.status})`,
376 );
377 const reader = response.body
378 .pipeThrough(new TextDecoderStream())
379 .getReader();
380 let buffered = "";
381 for (;;) {
382 const { value, done } = await reader.read();
383 if (done) break;
384 buffered += value;
385 let newline;
386 while ((newline = buffered.indexOf("\n")) >= 0) {
387 const line = buffered.slice(0, newline);
388 buffered = buffered.slice(newline + 1);
389 // Blank lines are the server's keepalives.
390 if (!line.trim()) continue;
391 const item = JSON.parse(line);
392 this.lastActivity = Date.now();
393 if (item.type === "approval") {
394 this.askApproval(id, item);
395 continue;
396 }
397 // Anything after an approval request means Hermes moved on
398 // (answered here, by button, or by its own timeout).
399 if (this.approval) this.clearApproval();
400 if (item.type === "tool" && item.status === "running") {
401 setStatus(`Hermes: ${item.label}`);
402 this.send({
403 type: "session.thinking.append",
404 delegation_id: id,
405 content: `Hermes is working on it: ${item.label}`.slice(
406 0,
407 APPEND_CHAR_LIMIT,
408 ),
409 });
410 } else if (item.type === "end_call") {
411 endCall = true;
412 } else if (item.type === "delta") {
413 answer += item.text;
414 } else if (item.type === "done" && item.error) {
415 failure = item.error;
416 } else if (item.type === "error") {
417 failure = item.message;
418 }
419 }
420 }
421 } catch (error) {
422 if (controller.signal.aborted) return;
423 failure = String(error.message || error);
424 }
425 if (this.delegation !== delegation) return;
426 this.delegation = null;
427 this.clearApproval();
428 this.lastActivity = Date.now();
429
430 if (failure && !answer.trim()) {
431 addLine("error", failure);
432 this.send({
433 type: "session.commentary.append",
434 delegation_id: id,
435 content: `The request to Hermes failed: ${failure}`.slice(
436 0,
437 APPEND_CHAR_LIMIT,
438 ),
439 });
440 } else {
441 const text = answer.trim() || "Hermes finished without an answer.";
442 addLine("hermes", text);
443 for (const chunk of chunks(text))
444 this.send({
445 type: "session.commentary.append",
446 delegation_id: id,
447 content: chunk,
448 });
449 }
450 if (endCall) this.endAfterReply();
451 else setStatus("Listening");
452 }
453
454 // Hermes ended the call: close it once the voice has spoken the reply just
455 // handed over.
456 endAfterReply() {
457 this.ending = { spoken: false };
458 this.endTimer = setTimeout(() => this.close(), END_CALL_MAX_MS);
459 setStatus("Ending the call…");
460 }
461
462 askApproval(id, item) {
463 this.approval = { delegationId: id, code: item.code };
464 const what = item.description || "run a command";
465 setStatus("Hermes needs your approval");
466 addLine(
467 "hermes",
468 `Approval needed: ${what}${item.command ? `\n${item.command}` : ""}`,
469 );
470 approvalText.textContent = item.command ? `${what}: ${item.command}` : what;
471 approvalBox.hidden = false;
472 this.send({
473 type: "session.commentary.append",
474 delegation_id: id,
475 content: (
476 `Hermes needs the user's approval before it continues: ${what}.` +
477 (item.command ? ` The command is: ${item.command}.` : "") +
478 " Ask the user whether to approve or deny it."
479 ).slice(0, APPEND_CHAR_LIMIT),
480 });
481 }
482
483 clearApproval() {
484 this.approval = null;
485 approvalBox.hidden = true;
486 }
487
488 async answerApproval(id) {
489 await new Promise((resolve) => setTimeout(resolve, DELEGATION_SETTLE_MS));
490 const said = state.lines
491 .slice(state.delegated)
492 .filter((line) => line.speaker === "user")
493 .map((line) => line.text)
494 .join(" ");
495 state.delegated = state.lines.length;
496 saveState();
497 const choice = classifyApproval(said);
498 if (!choice) {
499 this.send({
500 type: "session.commentary.append",
501 delegation_id: id,
502 content:
503 "That was not a clear answer. Ask the user again: approve or deny?",
504 });
505 return;
506 }
507 await this.respondApproval(choice, id);
508 }
509
510 async respondApproval(choice, replyId) {
511 const approval = this.approval;
512 if (!approval) return;
513 this.clearApproval();
514 setStatus(choice === "deny" ? "Denying…" : "Approving…");
515 let message;
516 let ok = false;
517 try {
518 const response = await api("/approve", {
519 method: "POST",
520 headers: { "Content-Type": "application/json" },
521 body: JSON.stringify({ code: approval.code, choice }),
522 });
523 const result = await response.json();
524 ok = response.ok;
525 message = ok
526 ? choice === "deny"
527 ? "Denied. Hermes will not run it."
528 : "Approved. Hermes is continuing."
529 : `Could not send the answer to Hermes: ${result.error}`;
530 } catch (error) {
531 message = `Could not send the answer to Hermes: ${error.message || error}`;
532 }
533 addLine(ok ? "hermes" : "error", message);
534 // A fast turn can finish before this reply returns; its result was
535 // already handed to the voice, so an "is continuing" would be stale.
536 if (ok && !this.delegation) return;
537 setStatus(ok ? "Hermes is working…" : "Listening");
538 this.send({
539 type: "session.commentary.append",
540 delegation_id: replyId || approval.delegationId,
541 content: message.slice(0, APPEND_CHAR_LIMIT),
542 });
543 }
544
545 checkIdle() {
546 if (this.closed || this.delegation || this.speaking) return;
547 if (Date.now() - this.lastActivity > settings.idle_seconds * 1000) {
548 setStatus("Hanging up (idle)");
549 this.close();
550 }
551 }
552
553 close() {
554 if (this.closed || this.closing) return;
555 this.closing = true;
556 if (!this.send({ type: "session.close" })) {
557 this.finish("closed");
558 return;
559 }
560 // Wait for session.closed (final usage); give up after a while.
561 this.closeTimer = setTimeout(() => this.finish("close timed out"), 15000);
562 }
563
564 // Tell the server the call is over, with what the user said since the last
565 // delegation: a request spoken right before hanging up still reaches
566 // Hermes. keepalive lets the request outlive the page.
567 hangUp() {
568 const lines = state.lines
569 .slice(state.delegated)
570 .filter(
571 (line) => line.speaker === "user" || line.speaker === "assistant",
572 );
573 state.delegated = state.lines.length;
574 saveState();
575 api("/hangup", {
576 method: "POST",
577 keepalive: true,
578 headers: { "Content-Type": "application/json" },
579 body: JSON.stringify({ hermes_session: state.hermesSession, lines }),
580 }).catch(() => {});
581 }
582
583 finish(reason) {
584 if (this.closed) return;
585 this.closed = true;
586 clearTimeout(this.closeTimer);
587 clearTimeout(this.endTimer);
588 clearInterval(this.idleTimer);
589 clearInterval(this.probe);
590 this.hangUp();
591 // Only the stream is dropped; the server keeps following the Hermes run.
592 if (this.delegation) this.delegation.controller.abort();
593 this.audioContext?.close().catch(() => {});
594 this.mic?.getTracks().forEach((track) => track.stop());
595 this.events?.close();
596 this.pc?.close();
597 voice.srcObject = null;
598 state.billedSeconds += this.usageSeconds;
599 saveState();
600 if (call === this) call = null;
601 talk.classList.remove("live", "speaking");
602 talk.textContent = "Talk";
603 talk.disabled = false;
604 setStatus(`Idle (${reason})`);
605 renderCost();
606 }
607}
608
609async function startCall() {
610 call = new Call();
611 talk.disabled = true;
612 talk.classList.add("live");
613 talk.textContent = "Hang up";
614 setStatus("Connecting…");
615 // Must run inside the click handler: mobile browsers only allow audio
616 // playback started by a user gesture.
617 voice.play().catch(() => {});
618 try {
619 await call.start();
620 talk.disabled = false;
621 } catch (error) {
622 addLine("error", String(error.message || error));
623 call.finish("failed to start");
624 }
625}
626
627talk.addEventListener("click", () => {
628 if (call) {
629 talk.disabled = true;
630 setStatus("Hanging up…");
631 call.close();
632 return;
633 }
634 startCall();
635});
636
637window.addEventListener("pagehide", () => call?.finish("page closed"));
638
639document.getElementById("new").addEventListener("click", () => {
640 if (call) call.close();
641 state = newConversation();
642 saveState();
643 renderLog();
644 renderCost();
645 setStatus("Idle (new conversation)");
646});
647
648document
649 .getElementById("approve")
650 .addEventListener("click", () => call?.respondApproval("once", null));
651document
652 .getElementById("deny")
653 .addEventListener("click", () => call?.respondApproval("deny", null));
654
655voiceSelect.addEventListener("change", () =>
656 localStorage.setItem(VOICE_KEY, voiceSelect.value),
657);
658
659async function loadConfig() {
660 let config;
661 try {
662 const response = await api("/config");
663 config = await response.json();
664 } catch (error) {
665 setStatus(`Settings not loaded: ${error.message}`);
666 return;
667 }
668 settings = config;
669 talk.disabled = false;
670 setStatus("Idle");
671 const saved = localStorage.getItem(VOICE_KEY);
672 voiceSelect.replaceChildren(
673 ...Object.entries(config.voices).map(
674 ([name, description]) => new Option(`${name} — ${description}`, name),
675 ),
676 );
677 voiceSelect.value = saved in config.voices ? saved : config.voice;
678}
679
680loadConfig();
681renderLog();
682renderCost();