This repository has no description
21 kB
662 lines
1// hermes-voice page: one WebRTC session with gpt-live-1 per "call", client
2// delegation answered by Hermes through the local server (/delegate).
3"use strict";
4
5const PRICE_PER_MINUTE = 0.05;
6// The voice model accepts at most 500 tokens per append; ~4 chars per token.
7const APPEND_CHAR_LIMIT = 1400;
8// Transcript deltas can trail the delegation event; wait for the last words.
9const DELEGATION_SETTLE_MS = 400;
10// Once Hermes ended the call (end_call item), the call closes when the voice
11// has been quiet this long after speaking the reply, at the latest after
12// END_CALL_MAX_MS.
13const END_CALL_QUIET_MS = 800;
14const END_CALL_MAX_MS = 30000;
15const STORAGE_KEY = "hermes-voice";
16
17const talk = document.getElementById("talk");
18const statusLine = document.getElementById("status");
19const costLine = document.getElementById("cost");
20const logList = document.getElementById("log");
21const voice = document.getElementById("voice");
22const approvalBox = document.getElementById("approval");
23const approvalText = document.getElementById("approval-text");
24const voiceSelect = document.getElementById("voice-select");
25const VOICE_KEY = "hermes-voice-voice";
26
27// The proxy in front adds the access token to logged-in requests. Once the
28// login expired it redirects to the login, which a reload follows.
29async function api(path, options = {}) {
30 const response = await fetch(path, { ...options, redirect: "manual" });
31 if (response.type === "opaqueredirect") {
32 location.reload();
33 throw new Error("Login expired");
34 }
35 return response;
36}
37
38// Spoken answer to an approval question. Denials win, so "no, don't do it"
39// never approves; anything unclear is asked again.
40function classifyApproval(said) {
41 if (
42 /\b(no|nope|deny|denied|don'?t|do not|stop|cancel|abort|reject|nein|nicht)\b/i.test(
43 said,
44 )
45 )
46 return "deny";
47 if (
48 /\b(yes|yeah|yep|approve|approved|go ahead|do it|okay|ok|sure|confirm|allow|ja)\b/i.test(
49 said,
50 )
51 )
52 return "once";
53 return null;
54}
55
56let settings = { idle_seconds: 90 };
57let call = null;
58let state = loadState();
59
60function newConversation() {
61 const day = new Date().toISOString().slice(0, 10).replaceAll("-", "");
62 const rand = crypto.getRandomValues(new Uint32Array(1))[0].toString(16);
63 return {
64 hermesSession: `voice-${day}-${rand}`,
65 lines: [],
66 delegated: 0,
67 billedSeconds: 0,
68 };
69}
70
71function loadState() {
72 try {
73 const saved = JSON.parse(localStorage.getItem(STORAGE_KEY));
74 if (saved && saved.hermesSession) return saved;
75 } catch (_) {
76 /* fall through */
77 }
78 return newConversation();
79}
80
81function saveState() {
82 localStorage.setItem(STORAGE_KEY, JSON.stringify(state));
83}
84
85function setStatus(text) {
86 statusLine.textContent = text;
87}
88
89function renderCost() {
90 const current = call ? call.usageSeconds : 0;
91 const total = state.billedSeconds + current;
92 const money = (seconds) =>
93 `$${((seconds / 60) * PRICE_PER_MINUTE).toFixed(3)}`;
94 const clock = (seconds) =>
95 `${Math.floor(seconds / 60)}:${String(Math.floor(seconds % 60)).padStart(2, "0")}`;
96 costLine.textContent = call
97 ? `this call ${clock(current)} · ${money(current)} — conversation ${money(total)}`
98 : `conversation ${clock(total)} · ${money(total)}`;
99}
100
101function renderLog() {
102 logList.replaceChildren(
103 ...state.lines.map((line) => {
104 const item = document.createElement("li");
105 item.className = line.speaker;
106 item.textContent = line.text;
107 return item;
108 }),
109 );
110 window.scrollTo(0, document.body.scrollHeight);
111}
112
113function addLine(speaker, text) {
114 state.lines.push({ speaker, text });
115 saveState();
116 renderLog();
117}
118
119// Deltas are appended exactly as received; a speaker change starts a new line.
120function addTranscript(speaker, delta) {
121 const last = state.lines.at(-1);
122 if (
123 last &&
124 last.speaker === speaker &&
125 call &&
126 call.lastSpeaker === speaker
127 ) {
128 last.text += delta;
129 } else {
130 state.lines.push({ speaker, text: delta });
131 }
132 if (call) call.lastSpeaker = speaker;
133 saveState();
134 renderLog();
135}
136
137function chunks(text) {
138 const clean = text.replace(/\s+/g, " ").trim();
139 const out = [];
140 let current = "";
141 for (const sentence of clean.split(/(?<=[.!?])\s+/)) {
142 if (sentence.length > APPEND_CHAR_LIMIT) {
143 if (current) out.push(current);
144 current = "";
145 for (let i = 0; i < sentence.length; i += APPEND_CHAR_LIMIT)
146 out.push(sentence.slice(i, i + APPEND_CHAR_LIMIT));
147 } else if (
148 current &&
149 current.length + 1 + sentence.length > APPEND_CHAR_LIMIT
150 ) {
151 out.push(current);
152 current = sentence;
153 } else {
154 current = current ? `${current} ${sentence}` : sentence;
155 }
156 }
157 if (current) out.push(current);
158 return out;
159}
160
161class Call {
162 constructor() {
163 this.usageSeconds = 0;
164 this.lastActivity = Date.now();
165 this.speaking = false;
166 this.lastSpeaker = null;
167 this.delegation = null;
168 this.closed = false;
169 this.closing = false;
170 this.ending = null;
171 this.eventCounter = 0;
172 }
173
174 async start() {
175 const pc = new RTCPeerConnection();
176 this.pc = pc;
177 pc.addEventListener("track", (event) => {
178 const stream = new MediaStream([event.track]);
179 voice.srcObject = stream;
180 voice.play().catch(() => {});
181 this.probeSpeaking(stream);
182 });
183 pc.addEventListener("connectionstatechange", () => {
184 if (
185 pc.connectionState === "failed" ||
186 pc.connectionState === "disconnected"
187 )
188 this.finish("connection lost");
189 });
190
191 this.mic = await navigator.mediaDevices.getUserMedia({
192 audio: {
193 echoCancellation: true,
194 noiseSuppression: true,
195 autoGainControl: true,
196 },
197 });
198 for (const track of this.mic.getAudioTracks()) pc.addTrack(track, this.mic);
199
200 // The data channel must exist before the offer so its m-line is negotiated.
201 this.events = pc.createDataChannel("oai-events");
202 this.events.addEventListener("message", ({ data }) =>
203 this.onEvent(JSON.parse(data)),
204 );
205 this.events.addEventListener("close", () =>
206 this.finish("connection closed"),
207 );
208
209 await pc.setLocalDescription(await pc.createOffer());
210 await this.iceGathered();
211
212 const response = await api("/session", {
213 method: "POST",
214 headers: { "Content-Type": "application/json" },
215 body: JSON.stringify({
216 sdp: pc.localDescription.sdp,
217 history: state.lines,
218 // Fixed per session: a new pick applies from the next call on.
219 voice: voiceSelect.value,
220 }),
221 });
222 const answer = await response.json();
223 if (!response.ok)
224 throw new Error(answer.error || `session failed (${response.status})`);
225 this.sessionId = answer.session_id;
226 await pc.setRemoteDescription({ type: "answer", sdp: answer.sdp });
227
228 this.idleTimer = setInterval(() => this.checkIdle(), 1000);
229 }
230
231 iceGathered() {
232 if (this.pc.iceGatheringState === "complete") return Promise.resolve();
233 return new Promise((resolve) => {
234 const done = () => {
235 if (this.pc.iceGatheringState !== "complete") return;
236 clearTimeout(timer);
237 resolve();
238 };
239 // Trickle is fine: the vendor answers with the candidates it has.
240 const timer = setTimeout(resolve, 3000);
241 this.pc.addEventListener("icegatheringstatechange", done);
242 });
243 }
244
245 probeSpeaking(stream) {
246 const context = new AudioContext();
247 const analyser = context.createAnalyser();
248 analyser.fftSize = 512;
249 context.createMediaStreamSource(stream).connect(analyser);
250 const buffer = new Uint8Array(analyser.frequencyBinCount);
251 let quiet = 0;
252 this.audioContext = context;
253 this.probe = setInterval(() => {
254 analyser.getByteTimeDomainData(buffer);
255 let peak = 0;
256 for (const sample of buffer)
257 peak = Math.max(peak, Math.abs(sample - 128));
258 quiet = peak > 6 ? 0 : quiet + 1;
259 const speaking = quiet < 4;
260 if (speaking) this.lastActivity = Date.now();
261 if (speaking !== this.speaking) {
262 this.speaking = speaking;
263 talk.classList.toggle("speaking", speaking);
264 }
265 if (this.ending?.spoken && quiet * 100 >= END_CALL_QUIET_MS) this.close();
266 }, 100);
267 }
268
269 send(event) {
270 if (!this.events || this.events.readyState !== "open") return false;
271 this.eventCounter += 1;
272 this.events.send(
273 JSON.stringify({ event_id: `evt_${this.eventCounter}`, ...event }),
274 );
275 return true;
276 }
277
278 onEvent(event) {
279 switch (event.type) {
280 case "session.started":
281 setStatus("Listening");
282 break;
283 case "session.input_transcript.delta":
284 this.lastActivity = Date.now();
285 addTranscript("user", event.delta || "");
286 break;
287 case "session.output_transcript.delta":
288 this.lastActivity = Date.now();
289 addTranscript("assistant", event.delta || "");
290 if (this.ending) this.ending.spoken = true;
291 break;
292 case "session.delegation.created":
293 this.delegate(event.delegation.id);
294 break;
295 case "session.usage.updated":
296 this.usageSeconds = event.usage?.seconds ?? this.usageSeconds;
297 renderCost();
298 break;
299 case "error":
300 addLine("error", event.error?.message || "voice model error");
301 break;
302 case "session.closed":
303 if (event.usage?.seconds != null)
304 this.usageSeconds = event.usage.seconds;
305 this.finish(event.reason || "closed");
306 break;
307 }
308 }
309
310 async delegate(id) {
311 // One Hermes turn at a time: a newer delegation (usually a correction)
312 // supersedes the running one, and the server stops the old run.
313 // While Hermes waits for an approval, the next delegation is the user's
314 // answer, not a new request: the turn must keep running.
315 if (this.approval) {
316 this.answerApproval(id);
317 return;
318 }
319 const replace = Boolean(this.delegation);
320 if (this.delegation) this.delegation.controller.abort();
321 const controller = new AbortController();
322 const delegation = { id, controller };
323 this.delegation = delegation;
324 setStatus("Hermes is working…");
325
326 await new Promise((resolve) => setTimeout(resolve, DELEGATION_SETTLE_MS));
327 // A hang-up meanwhile sent these lines along itself.
328 if (this.delegation !== delegation || this.closed) return;
329 const fresh = state.lines
330 .slice(state.delegated)
331 .filter((line) => line.speaker !== "hermes" && line.speaker !== "error");
332 const lines = fresh.some((line) => line.speaker === "user")
333 ? fresh
334 : state.lines
335 .filter(
336 (line) => line.speaker === "user" || line.speaker === "assistant",
337 )
338 .slice(-4);
339 state.delegated = state.lines.length;
340 saveState();
341
342 let answer = "";
343 let failure = null;
344 let endCall = false;
345 try {
346 const response = await api("/delegate", {
347 method: "POST",
348 headers: { "Content-Type": "application/json" },
349 body: JSON.stringify({
350 hermes_session: state.hermesSession,
351 lines,
352 replace,
353 }),
354 signal: controller.signal,
355 });
356 if (!response.ok)
357 throw new Error(
358 (await response.json()).error ||
359 `delegate failed (${response.status})`,
360 );
361 const reader = response.body
362 .pipeThrough(new TextDecoderStream())
363 .getReader();
364 let buffered = "";
365 for (;;) {
366 const { value, done } = await reader.read();
367 if (done) break;
368 buffered += value;
369 let newline;
370 while ((newline = buffered.indexOf("\n")) >= 0) {
371 const line = buffered.slice(0, newline);
372 buffered = buffered.slice(newline + 1);
373 // Blank lines are the server's keepalives.
374 if (!line.trim()) continue;
375 const item = JSON.parse(line);
376 this.lastActivity = Date.now();
377 if (item.type === "approval") {
378 this.askApproval(id, item);
379 continue;
380 }
381 // Anything after an approval request means Hermes moved on
382 // (answered here, by button, or by its own timeout).
383 if (this.approval) this.clearApproval();
384 if (item.type === "tool" && item.status === "running") {
385 setStatus(`Hermes: ${item.label}`);
386 this.send({
387 type: "session.thinking.append",
388 delegation_id: id,
389 content: `Hermes is working on it: ${item.label}`.slice(
390 0,
391 APPEND_CHAR_LIMIT,
392 ),
393 });
394 } else if (item.type === "end_call") {
395 endCall = true;
396 } else if (item.type === "delta") {
397 answer += item.text;
398 } else if (item.type === "done" && item.error) {
399 failure = item.error;
400 } else if (item.type === "error") {
401 failure = item.message;
402 }
403 }
404 }
405 } catch (error) {
406 if (controller.signal.aborted) return;
407 failure = String(error.message || error);
408 }
409 if (this.delegation !== delegation) return;
410 this.delegation = null;
411 this.clearApproval();
412 this.lastActivity = Date.now();
413
414 if (failure && !answer.trim()) {
415 addLine("error", failure);
416 this.send({
417 type: "session.commentary.append",
418 delegation_id: id,
419 content: `The request to Hermes failed: ${failure}`.slice(
420 0,
421 APPEND_CHAR_LIMIT,
422 ),
423 });
424 } else {
425 const text = answer.trim() || "Hermes finished without an answer.";
426 addLine("hermes", text);
427 for (const chunk of chunks(text))
428 this.send({
429 type: "session.commentary.append",
430 delegation_id: id,
431 content: chunk,
432 });
433 }
434 if (endCall) this.endAfterReply();
435 else setStatus("Listening");
436 }
437
438 // Hermes ended the call: close it once the voice has spoken the reply just
439 // handed over.
440 endAfterReply() {
441 this.ending = { spoken: false };
442 this.endTimer = setTimeout(() => this.close(), END_CALL_MAX_MS);
443 setStatus("Ending the call…");
444 }
445
446 askApproval(id, item) {
447 this.approval = { delegationId: id, code: item.code };
448 const what = item.description || "run a command";
449 setStatus("Hermes needs your approval");
450 addLine(
451 "hermes",
452 `Approval needed: ${what}${item.command ? `\n${item.command}` : ""}`,
453 );
454 approvalText.textContent = item.command ? `${what}: ${item.command}` : what;
455 approvalBox.hidden = false;
456 this.send({
457 type: "session.commentary.append",
458 delegation_id: id,
459 content: (
460 `Hermes needs the user's approval before it continues: ${what}.` +
461 (item.command ? ` The command is: ${item.command}.` : "") +
462 " Ask the user whether to approve or deny it."
463 ).slice(0, APPEND_CHAR_LIMIT),
464 });
465 }
466
467 clearApproval() {
468 this.approval = null;
469 approvalBox.hidden = true;
470 }
471
472 async answerApproval(id) {
473 await new Promise((resolve) => setTimeout(resolve, DELEGATION_SETTLE_MS));
474 const said = state.lines
475 .slice(state.delegated)
476 .filter((line) => line.speaker === "user")
477 .map((line) => line.text)
478 .join(" ");
479 state.delegated = state.lines.length;
480 saveState();
481 const choice = classifyApproval(said);
482 if (!choice) {
483 this.send({
484 type: "session.commentary.append",
485 delegation_id: id,
486 content:
487 "That was not a clear answer. Ask the user again: approve or deny?",
488 });
489 return;
490 }
491 await this.respondApproval(choice, id);
492 }
493
494 async respondApproval(choice, replyId) {
495 const approval = this.approval;
496 if (!approval) return;
497 this.clearApproval();
498 setStatus(choice === "deny" ? "Denying…" : "Approving…");
499 let message;
500 let ok = false;
501 try {
502 const response = await api("/approve", {
503 method: "POST",
504 headers: { "Content-Type": "application/json" },
505 body: JSON.stringify({ code: approval.code, choice }),
506 });
507 const result = await response.json();
508 ok = response.ok;
509 message = ok
510 ? choice === "deny"
511 ? "Denied. Hermes will not run it."
512 : "Approved. Hermes is continuing."
513 : `Could not send the answer to Hermes: ${result.error}`;
514 } catch (error) {
515 message = `Could not send the answer to Hermes: ${error.message || error}`;
516 }
517 addLine(ok ? "hermes" : "error", message);
518 // A fast turn can finish before this reply returns; its result was
519 // already handed to the voice, so an "is continuing" would be stale.
520 if (ok && !this.delegation) return;
521 setStatus(ok ? "Hermes is working…" : "Listening");
522 this.send({
523 type: "session.commentary.append",
524 delegation_id: replyId || approval.delegationId,
525 content: message.slice(0, APPEND_CHAR_LIMIT),
526 });
527 }
528
529 checkIdle() {
530 if (this.closed || this.delegation || this.speaking) return;
531 if (Date.now() - this.lastActivity > settings.idle_seconds * 1000) {
532 setStatus("Hanging up (idle)");
533 this.close();
534 }
535 }
536
537 close() {
538 if (this.closed || this.closing) return;
539 this.closing = true;
540 if (!this.send({ type: "session.close" })) {
541 this.finish("closed");
542 return;
543 }
544 // Wait for session.closed (final usage); give up after a while.
545 this.closeTimer = setTimeout(() => this.finish("close timed out"), 15000);
546 }
547
548 // Tell the server the call is over, with what the user said since the last
549 // delegation: a request spoken right before hanging up still reaches
550 // Hermes. keepalive lets the request outlive the page.
551 hangUp() {
552 const lines = state.lines
553 .slice(state.delegated)
554 .filter(
555 (line) => line.speaker === "user" || line.speaker === "assistant",
556 );
557 state.delegated = state.lines.length;
558 saveState();
559 api("/hangup", {
560 method: "POST",
561 keepalive: true,
562 headers: { "Content-Type": "application/json" },
563 body: JSON.stringify({ hermes_session: state.hermesSession, lines }),
564 }).catch(() => {});
565 }
566
567 finish(reason) {
568 if (this.closed) return;
569 this.closed = true;
570 clearTimeout(this.closeTimer);
571 clearTimeout(this.endTimer);
572 clearInterval(this.idleTimer);
573 clearInterval(this.probe);
574 this.hangUp();
575 // Only the stream is dropped; the server keeps following the Hermes run.
576 if (this.delegation) this.delegation.controller.abort();
577 this.audioContext?.close().catch(() => {});
578 this.mic?.getTracks().forEach((track) => track.stop());
579 this.events?.close();
580 this.pc?.close();
581 voice.srcObject = null;
582 state.billedSeconds += this.usageSeconds;
583 saveState();
584 if (call === this) call = null;
585 talk.classList.remove("live", "speaking");
586 talk.textContent = "Talk";
587 talk.disabled = false;
588 setStatus(`Idle (${reason})`);
589 renderCost();
590 }
591}
592
593talk.addEventListener("click", async () => {
594 if (call) {
595 talk.disabled = true;
596 setStatus("Hanging up…");
597 call.close();
598 return;
599 }
600 call = new Call();
601 talk.disabled = true;
602 talk.classList.add("live");
603 talk.textContent = "Hang up";
604 setStatus("Connecting…");
605 // Must run inside the click handler: mobile browsers only allow audio
606 // playback started by a user gesture.
607 voice.play().catch(() => {});
608 try {
609 await call.start();
610 talk.disabled = false;
611 } catch (error) {
612 addLine("error", String(error.message || error));
613 call.finish("failed to start");
614 }
615});
616
617window.addEventListener("pagehide", () => call?.finish("page closed"));
618
619document.getElementById("new").addEventListener("click", () => {
620 if (call) call.close();
621 state = newConversation();
622 saveState();
623 renderLog();
624 renderCost();
625 setStatus("Idle (new conversation)");
626});
627
628document
629 .getElementById("approve")
630 .addEventListener("click", () => call?.respondApproval("once", null));
631document
632 .getElementById("deny")
633 .addEventListener("click", () => call?.respondApproval("deny", null));
634
635voiceSelect.addEventListener("change", () =>
636 localStorage.setItem(VOICE_KEY, voiceSelect.value),
637);
638
639async function loadConfig() {
640 let config;
641 try {
642 const response = await api("/config");
643 config = await response.json();
644 } catch (error) {
645 setStatus(`Settings not loaded: ${error.message}`);
646 return;
647 }
648 settings = config;
649 talk.disabled = false;
650 setStatus("Idle");
651 const saved = localStorage.getItem(VOICE_KEY);
652 voiceSelect.replaceChildren(
653 ...Object.entries(config.voices).map(
654 ([name, description]) => new Option(`${name} — ${description}`, name),
655 ),
656 );
657 voiceSelect.value = saved in config.voices ? saved : config.voice;
658}
659
660loadConfig();
661renderLog();
662renderCost();