···11+"""hermes-voice: talk to Hermes through OpenAI's full-duplex gpt-live-1.
22+33+The browser holds a WebRTC session with gpt-live-1, which listens and speaks at
44+the same time and chats on its own. Whenever the user asks for real work, the
55+voice model emits a client delegation; the page hands the recent transcript to
66+this server, which runs it as an ordinary Hermes turn on the API server
77+(/v1/chat/completions, one X-Hermes-Session-Id per conversation) and streams
88+the reply back. The page then gives the answer to the voice model, which says
99+it aloud.
1010+1111+This server exists so neither secret reaches the browser: it exchanges the
1212+page's SDP offer for a Live session with the OpenAI key, and it calls Hermes
1313+with the API server's bearer token. That token is root-equivalent for the
1414+Hermes host, so whoever can use this server can drive the agent: every API
1515+route requires the page's own access token (--access-token-file), and serving
1616+beyond loopback without one is refused.
1717+1818+Vendor contract: https://developers.openai.com/api/docs/guides/live-delegation
1919+"""
2020+2121+import argparse
2222+import hmac
2323+import json
2424+import logging
2525+import os
2626+import re
2727+import socket
2828+import time
2929+import urllib.error
3030+import urllib.request
3131+from http import HTTPStatus
3232+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
3333+from pathlib import Path
3434+3535+log = logging.getLogger("hermes-voice")
3636+3737+OPENAI_BASE_URL = "https://api.openai.com/v1"
3838+DEFAULT_MODEL = "gpt-live-1"
3939+DEFAULT_VOICE = "marin"
4040+# gpt-live-1 built-in voices (live-conversations guide voice table, plus the
4141+# realtime defaults marin and cedar).
4242+BUILT_IN_VOICES = (
4343+ "marin", "cedar", "quartz", "ripple", "vesper", "willow", "stone", "gleam",
4444+ "meridian", "bossa", "tempo", "beacon", "delta", "cinder",
4545+)
4646+# gpt-live-1 bills every second a session is open, silence included; the page
4747+# closes the session after this much inactivity.
4848+DEFAULT_IDLE_SECONDS = 90
4949+MAX_BODY = 256 * 1024
5050+# Hermes turns run tools for minutes; the API server streams tool progress, but
5151+# a single long tool call can be silent for a while.
5252+HERMES_TIMEOUT = 900
5353+SESSION_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
5454+# Hermes run ids are "chatcmpl-<hex>", approval request ids hex.
5555+RUN_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,256}$")
5656+5757+# Frontend persona, adapted from Hermes's own (tools/voice_live.py): role,
5858+# style and a labelled delegation policy. The backend carries the real
5959+# instructions, tools and memory.
6060+PERSONA = """You are Hermes, lass's warm, upbeat and genuinely helpful voice assistant. Sound friendly and glad to help: positive and encouraging, never curt, annoyed, bored or sarcastic, even when delivering bad news or when the user is short with you. Talk fast: speak at a rapid, energetic pace, like someone who talks quickly by nature, with no slow drawn-out words and no long pauses between sentences. Keep replies short; a brief warm acknowledgment before the facts is good.
6161+6262+Backchannel policy: Use moderate backchannels. Acknowledge naturally without competing with the main response.
6363+6464+Interruption policy: Stop speaking when the user interrupts. Listen to what they say.
6565+6666+Delegation policy:
6767+Backend tools:
6868+- Hermes agent: a full AI agent with tools. It can run commands on lass's machines, read and edit files, manage the calendar, inspect the coding agents in herdr, browse the web, search, and remember things across sessions. It is the one who actually does work and knows facts.
6969+7070+Delegate to the backend when:
7171+- The user asks a question that needs facts, current information, or careful reasoning.
7272+- The user asks you to do, check, find, make, fix, run or remember anything.
7373+- A correction changes work already requested.
7474+7575+Do not delegate to the backend when:
7676+- The user greets you, makes small talk, or asks you to repeat a result already provided.
7777+- You need a brief clarification to understand the request.
7878+7979+Delegate before giving an answer that depends on backend work. Do not guess the result while waiting; say briefly and cheerfully that you are on it, then wait for the result.
8080+8181+Approval policy:
8282+Hermes sometimes needs the user's approval before it runs a command. When you are told so, explain in plain words what Hermes wants to do and ask whether to approve or deny it. When the user answers, delegate to the backend so the answer reaches Hermes. Never decide an approval yourself."""
8383+8484+# Ephemeral system prompt for every delegated Hermes turn (Hermes layers system
8585+# messages on top of its core prompt without storing them).
8686+BACKEND_NOTE = """This turn is a delegation from a live spoken conversation. The user message is a speech transcript: it may contain mis-hearings, phonetic spellings of technical terms, hesitations and later corrections; use the latest intent. Your reply will be spoken aloud by a friendly voice model that paraphrases it: answer in warm, plain conversational sentences, keep it short (a few sentences unless the user asked for detail), no markdown, no lists, no code blocks, no URLs read out character by character. Do the work with your tools as usual; only the final facts need to be spoken. Report an action as done only after it actually succeeded."""
8787+8888+8989+def glossary_text(glossary):
9090+ """Glossary entries as prompt lines: `term` or `term (sounds like "…")`."""
9191+ lines = []
9292+ for entry in glossary:
9393+ if isinstance(entry, str):
9494+ lines.append(f"- {entry}")
9595+ else:
9696+ term = entry["term"]
9797+ spoken = entry.get("spoken")
9898+ lines.append(f'- {term} (sounds like "{spoken}")' if spoken else f"- {term}")
9999+ return "\n".join(lines)
100100+101101+102102+class Config:
103103+ def __init__(self, raw, openai_key, hermes_url, hermes_key):
104104+ self.model = raw.get("model", DEFAULT_MODEL)
105105+ self.voice = raw.get("voice", DEFAULT_VOICE)
106106+ self.idle_seconds = int(raw.get("idle_seconds", DEFAULT_IDLE_SECONDS))
107107+ self.glossary = raw.get("glossary", [])
108108+ self.extra_instructions = raw.get("instructions", "").strip()
109109+ self.openai_key = openai_key
110110+ self.hermes_url = hermes_url.rstrip("/")
111111+ self.hermes_key = hermes_key
112112+113113+ def live_instructions(self):
114114+ parts = [PERSONA]
115115+ if self.glossary:
116116+ parts.append(
117117+ "Technical terms the user says often. Understand them when spoken and "
118118+ "say them like this:\n" + glossary_text(self.glossary)
119119+ )
120120+ if self.extra_instructions:
121121+ parts.append(self.extra_instructions)
122122+ return "\n\n".join(parts)
123123+124124+ def backend_note(self):
125125+ if not self.glossary:
126126+ return BACKEND_NOTE
127127+ return (
128128+ BACKEND_NOTE
129129+ + "\n\nTerms the transcript may contain as phonetic misspellings:\n"
130130+ + glossary_text(self.glossary)
131131+ )
132132+133133+ def voices(self):
134134+ """Voices the page may pick: the built-ins plus the configured one
135135+ (which may be a custom voice id)."""
136136+ return list(dict.fromkeys([self.voice, *BUILT_IN_VOICES]))
137137+138138+139139+def voice_param(voice):
140140+ # Custom voices are objects ({"id": "voice_…"}); built-ins are names.
141141+ if voice.startswith("voice_"):
142142+ return {"id": voice}
143143+ return voice
144144+145145+146146+def live_history(lines):
147147+ """Seed history for a new Live session from the page's transcript lines
148148+ (spoken dialogue only; Hermes answers were spoken through the voice)."""
149149+ items = []
150150+ dialogue = [line for line in lines if line.get("speaker") in ("user", "assistant")]
151151+ for line in dialogue[-24:]:
152152+ text = str(line.get("text", "")).strip()[:1200]
153153+ if not text:
154154+ continue
155155+ if line["speaker"] == "assistant":
156156+ content = {"type": "output_text", "text": text}
157157+ else:
158158+ content = {"type": "input_text", "text": text}
159159+ items.append({"type": "message", "role": line["speaker"], "content": [content]})
160160+ return items
161161+162162+163163+def create_live_session(config, sdp, history, voice):
164164+ session = {
165165+ "model": config.model,
166166+ "instructions": config.live_instructions(),
167167+ "audio": {"output": {"voice": voice_param(voice)}},
168168+ "delegation": {"type": "client"},
169169+ # The page is an untrusted frontend: it may only answer delegations and
170170+ # hang up, not rewrite the session.
171171+ "client": {
172172+ "data_channel": {
173173+ "allowed_client_events": [
174174+ "session.commentary.append",
175175+ "session.thinking.append",
176176+ "session.close",
177177+ ]
178178+ }
179179+ },
180180+ }
181181+ if history:
182182+ session["input"] = history
183183+ body = json.dumps(
184184+ {"session": session, "transport": {"type": "webrtc", "sdp": sdp}}
185185+ ).encode()
186186+ request = urllib.request.Request(
187187+ f"{OPENAI_BASE_URL}/live/sessions",
188188+ data=body,
189189+ method="POST",
190190+ headers={
191191+ "Authorization": f"Bearer {config.openai_key}",
192192+ "Content-Type": "application/json",
193193+ },
194194+ )
195195+ with urllib.request.urlopen(request, timeout=30) as response:
196196+ answer = json.load(response)
197197+ return {"session_id": answer["session"]["id"], "sdp": answer["transport"]["sdp"]}
198198+199199+200200+def transcript_message(lines):
201201+ rendered = []
202202+ for line in lines:
203203+ text = str(line.get("text", "")).strip()
204204+ if text:
205205+ who = "Assistant" if line.get("speaker") == "assistant" else "User"
206206+ rendered.append(f"{who}: {text}")
207207+ return "Spoken conversation since the last request, newest last:\n" + "\n".join(
208208+ rendered
209209+ )
210210+211211+212212+def iter_sse(response):
213213+ """Yield (event, data) pairs from a text/event-stream response."""
214214+ event, data = None, []
215215+ for raw in response:
216216+ line = raw.decode("utf-8", "replace").rstrip("\r\n")
217217+ if not line:
218218+ if data:
219219+ yield event, "\n".join(data)
220220+ event, data = None, []
221221+ elif line.startswith("event:"):
222222+ event = line[6:].strip()
223223+ elif line.startswith("data:"):
224224+ data.append(line[5:].lstrip())
225225+ if data:
226226+ yield event, "\n".join(data)
227227+228228+229229+def hermes_turn(config, hermes_session, lines):
230230+ """Run one Hermes turn; yield NDJSON-ready dicts (tool, delta, done)."""
231231+ body = json.dumps(
232232+ {
233233+ "messages": [
234234+ {"role": "system", "content": config.backend_note()},
235235+ {"role": "user", "content": transcript_message(lines)},
236236+ ],
237237+ "stream": True,
238238+ }
239239+ ).encode()
240240+ request = urllib.request.Request(
241241+ f"{config.hermes_url}/v1/chat/completions",
242242+ data=body,
243243+ method="POST",
244244+ headers={
245245+ "Authorization": f"Bearer {config.hermes_key}",
246246+ "Content-Type": "application/json",
247247+ "X-Hermes-Session-Id": hermes_session,
248248+ },
249249+ )
250250+ with urllib.request.urlopen(request, timeout=HERMES_TIMEOUT) as response:
251251+ for event, data in iter_sse(response):
252252+ if data == "[DONE]":
253253+ return
254254+ payload = json.loads(data)
255255+ if event == "hermes.tool.progress":
256256+ yield {
257257+ "type": "tool",
258258+ "status": payload.get("status"),
259259+ "label": payload.get("label") or payload.get("tool"),
260260+ }
261261+ continue
262262+ if event == "approval.request":
263263+ # Hermes blocks the turn until POST /v1/runs/{run_id}/approval
264264+ # answers (or its own approval timeout denies).
265265+ yield {
266266+ "type": "approval",
267267+ "run_id": payload.get("run_id"),
268268+ "request_id": payload.get("request_id"),
269269+ "command": payload.get("command"),
270270+ "description": payload.get("description"),
271271+ }
272272+ continue
273273+ if event:
274274+ continue
275275+ for choice in payload.get("choices", []):
276276+ text = choice.get("delta", {}).get("content")
277277+ if text:
278278+ yield {"type": "delta", "text": text}
279279+ if choice.get("finish_reason"):
280280+ done = {"type": "done", "finish_reason": choice["finish_reason"]}
281281+ if "error" in payload:
282282+ done["error"] = payload["error"].get("message")
283283+ yield done
284284+285285+286286+def hermes_approve(config, run_id, request_id, choice):
287287+ """Answer a pending approval of a streaming Hermes turn."""
288288+ answer = {"choice": choice}
289289+ if request_id:
290290+ answer["request_id"] = request_id
291291+ request = urllib.request.Request(
292292+ f"{config.hermes_url}/v1/runs/{run_id}/approval",
293293+ data=json.dumps(answer).encode(),
294294+ method="POST",
295295+ headers={
296296+ "Authorization": f"Bearer {config.hermes_key}",
297297+ "Content-Type": "application/json",
298298+ },
299299+ )
300300+ with urllib.request.urlopen(request, timeout=30) as response:
301301+ return json.load(response)
302302+303303+304304+class Handler(BaseHTTPRequestHandler):
305305+ server_version = "hermes-voice"
306306+ config: Config
307307+ static: Path
308308+ access_token: "str | None" = None
309309+310310+ def authorized(self):
311311+ """Check the page's bearer token; answer 401 when it is wrong."""
312312+ if self.access_token is None:
313313+ return True
314314+ header = self.headers.get("Authorization", "")
315315+ scheme, _, token = header.partition(" ")
316316+ if scheme == "Bearer" and hmac.compare_digest(
317317+ token.encode(), self.access_token.encode()
318318+ ):
319319+ return True
320320+ self.send_json(HTTPStatus.UNAUTHORIZED, {"error": "access token required"})
321321+ return False
322322+323323+ def log_message(self, format, *args):
324324+ log.info("%s %s", self.address_string(), format % args)
325325+326326+ def send_json(self, status, payload):
327327+ body = json.dumps(payload).encode()
328328+ self.send_response(status)
329329+ self.send_header("Content-Type", "application/json")
330330+ self.send_header("Content-Length", str(len(body)))
331331+ self.send_header("Cache-Control", "no-store")
332332+ self.end_headers()
333333+ self.wfile.write(body)
334334+335335+ def read_json(self):
336336+ length = int(self.headers.get("Content-Length") or 0)
337337+ if length <= 0 or length > MAX_BODY:
338338+ raise ValueError("bad body length")
339339+ return json.loads(self.rfile.read(length))
340340+341341+ def do_GET(self):
342342+ if self.path == "/config":
343343+ if not self.authorized():
344344+ return
345345+ self.send_json(
346346+ HTTPStatus.OK,
347347+ {
348348+ "model": self.config.model,
349349+ "voice": self.config.voice,
350350+ "voices": self.config.voices(),
351351+ "idle_seconds": self.config.idle_seconds,
352352+ },
353353+ )
354354+ return
355355+ names = {
356356+ "/": ("index.html", "text/html; charset=utf-8"),
357357+ "/app.js": ("app.js", "text/javascript; charset=utf-8"),
358358+ "/style.css": ("style.css", "text/css; charset=utf-8"),
359359+ }
360360+ if self.path not in names:
361361+ self.send_error(HTTPStatus.NOT_FOUND)
362362+ return
363363+ name, content_type = names[self.path]
364364+ body = (self.static / name).read_bytes()
365365+ self.send_response(HTTPStatus.OK)
366366+ self.send_header("Content-Type", content_type)
367367+ self.send_header("Content-Length", str(len(body)))
368368+ self.send_header("Cache-Control", "no-store")
369369+ self.end_headers()
370370+ self.wfile.write(body)
371371+372372+ def do_POST(self):
373373+ if not self.authorized():
374374+ return
375375+ try:
376376+ request = self.read_json()
377377+ except ValueError:
378378+ self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid JSON body"})
379379+ return
380380+ if self.path == "/session":
381381+ self.handle_session(request)
382382+ elif self.path == "/delegate":
383383+ self.handle_delegate(request)
384384+ elif self.path == "/approve":
385385+ self.handle_approve(request)
386386+ else:
387387+ self.send_error(HTTPStatus.NOT_FOUND)
388388+389389+ def handle_session(self, request):
390390+ # The vendor's SDP parser needs the offer byte-exact (trailing CRLF too).
391391+ sdp = request.get("sdp")
392392+ if not isinstance(sdp, str) or not sdp.strip():
393393+ self.send_json(HTTPStatus.BAD_REQUEST, {"error": "sdp offer required"})
394394+ return
395395+ try:
396396+ voice = request.get("voice") or self.config.voice
397397+ if voice not in self.config.voices():
398398+ self.send_json(HTTPStatus.BAD_REQUEST, {"error": f"unknown voice {voice!r}"})
399399+ return
400400+ result = create_live_session(
401401+ self.config, sdp, live_history(request.get("history") or []), voice
402402+ )
403403+ except urllib.error.HTTPError as error:
404404+ detail = error.read().decode("utf-8", "replace")[:800]
405405+ log.warning("live session creation failed: %s %s", error.code, detail)
406406+ self.send_json(
407407+ HTTPStatus.BAD_GATEWAY,
408408+ {"error": f"OpenAI rejected the session ({error.code}): {detail}"},
409409+ )
410410+ return
411411+ except (urllib.error.URLError, OSError, KeyError, ValueError) as error:
412412+ log.warning("live session creation failed: %s", error)
413413+ self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"OpenAI: {error}"})
414414+ return
415415+ self.send_json(HTTPStatus.OK, result)
416416+417417+ def handle_delegate(self, request):
418418+ hermes_session = request.get("hermes_session")
419419+ lines = request.get("lines")
420420+ if not isinstance(hermes_session, str) or not SESSION_ID_RE.match(hermes_session):
421421+ self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid hermes_session"})
422422+ return
423423+ if not isinstance(lines, list) or not lines:
424424+ self.send_json(HTTPStatus.BAD_REQUEST, {"error": "lines required"})
425425+ return
426426+ self.send_response(HTTPStatus.OK)
427427+ self.send_header("Content-Type", "application/x-ndjson")
428428+ self.send_header("Cache-Control", "no-store")
429429+ self.send_header("X-Accel-Buffering", "no")
430430+ self.end_headers()
431431+ try:
432432+ try:
433433+ started = time.monotonic()
434434+ tools = 0
435435+ for item in hermes_turn(self.config, hermes_session, lines):
436436+ if item["type"] == "tool" and item["status"] == "running":
437437+ tools += 1
438438+ self.write_line(item)
439439+ log.info(
440440+ "hermes turn took %.1fs (%d tool calls)", time.monotonic() - started, tools
441441+ )
442442+ except urllib.error.HTTPError as error:
443443+ detail = error.read().decode("utf-8", "replace")[:400]
444444+ self.write_line({"type": "error", "message": f"Hermes {error.code}: {detail}"})
445445+ except (urllib.error.URLError, OSError, ValueError) as error:
446446+ self.write_line({"type": "error", "message": f"Hermes unreachable: {error}"})
447447+ except (BrokenPipeError, ConnectionResetError):
448448+ # The page abandoned this delegation (a newer one superseded it).
449449+ # Leaving the with-block closed the upstream stream, which makes
450450+ # Hermes interrupt the turn.
451451+ log.info("delegation for %s abandoned by the page", hermes_session)
452452+453453+ def handle_approve(self, request):
454454+ run_id = request.get("run_id")
455455+ request_id = request.get("request_id")
456456+ choice = request.get("choice")
457457+ # No "always": a permanent allowlist entry is not a voice decision.
458458+ if (
459459+ choice not in ("once", "session", "deny")
460460+ or not isinstance(run_id, str)
461461+ or not RUN_ID_RE.match(run_id)
462462+ or not (
463463+ request_id is None
464464+ or (isinstance(request_id, str) and RUN_ID_RE.match(request_id))
465465+ )
466466+ ):
467467+ self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid approval answer"})
468468+ return
469469+ try:
470470+ result = hermes_approve(self.config, run_id, request_id, choice)
471471+ except urllib.error.HTTPError as error:
472472+ detail = error.read().decode("utf-8", "replace")[:400]
473473+ self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
474474+ return
475475+ except (urllib.error.URLError, OSError, ValueError) as error:
476476+ self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
477477+ return
478478+ log.info("approval %s for run %s: %s", choice, run_id, result.get("resolved"))
479479+ self.send_json(HTTPStatus.OK, result)
480480+481481+ def write_line(self, item):
482482+ self.wfile.write(json.dumps(item).encode() + b"\n")
483483+ self.wfile.flush()
484484+485485+486486+def main():
487487+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
488488+ parser.add_argument("--listen", default="127.0.0.1:8790", help="host:port to serve on")
489489+ parser.add_argument("--hermes-url", required=True, help="Hermes API server base URL")
490490+ parser.add_argument("--hermes-key-file", required=True, help="file holding API_SERVER_KEY")
491491+ parser.add_argument("--openai-key-file", required=True, help="file holding the OpenAI API key")
492492+ parser.add_argument("--access-token-file", help="file holding the token the page must present")
493493+ parser.add_argument("--config", help="JSON file: voice, model, idle_seconds, glossary, instructions")
494494+ parser.add_argument(
495495+ "--static",
496496+ default=os.environ.get("HERMES_VOICE_STATIC"),
497497+ help="directory with index.html, app.js, style.css (default: $HERMES_VOICE_STATIC)",
498498+ )
499499+ args = parser.parse_args()
500500+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
501501+502502+ host, _, port = args.listen.rpartition(":")
503503+ host = host.strip("[]")
504504+ if args.access_token_file is None and host not in ("127.0.0.1", "::1", "localhost"):
505505+ parser.error("--access-token-file is required when not listening on loopback")
506506+507507+ raw = json.loads(Path(args.config).read_text()) if args.config else {}
508508+ config = Config(
509509+ raw,
510510+ openai_key=Path(args.openai_key_file).read_text().strip(),
511511+ hermes_url=args.hermes_url,
512512+ hermes_key=Path(args.hermes_key_file).read_text().strip(),
513513+ )
514514+ Handler.config = config
515515+ Handler.static = Path(args.static)
516516+ if args.access_token_file:
517517+ Handler.access_token = Path(args.access_token_file).read_text().strip()
518518+ server_class = ThreadingHTTPServer
519519+ if ":" in host:
520520+ # "::" is dual-stack on Linux (bindv6only=0).
521521+ server_class = type("Server6", (ThreadingHTTPServer,), {"address_family": socket.AF_INET6})
522522+ server = server_class((host, int(port)), Handler)
523523+ server.daemon_threads = True
524524+ log.info("serving on http://%s", args.listen)
525525+ server.serve_forever()
526526+527527+528528+if __name__ == "__main__":
529529+ main()
···11+bin "hermes-voice"
22+about "Serve a page for live voice conversations with Hermes through OpenAI gpt-live-1"
33+44+flag "--listen" help="host:port to serve on; IPv6 as [::]:port (default: 127.0.0.1:8790)" {
55+ arg "<host:port>"
66+}
77+flag "--hermes-url" help="Hermes API server base URL, e.g. https://hermes.lassul.us" required=#true {
88+ arg "<url>"
99+}
1010+flag "--hermes-key-file" help="File holding the Hermes API_SERVER_KEY (clan vars get coaxmetal hermes-api/api_key)" required=#true {
1111+ arg "<file>"
1212+}
1313+flag "--openai-key-file" help="File holding the OpenAI API key" required=#true {
1414+ arg "<file>"
1515+}
1616+flag "--access-token-file" help="File holding the token every API route requires; mandatory unless listening on loopback, since the page drives Hermes" {
1717+ arg "<file>"
1818+}
1919+flag "--config" help="JSON file with voice, model, idle_seconds, glossary and instructions" {
2020+ arg "<file>"
2121+}
2222+flag "--static" help="Page directory (default: $HERMES_VOICE_STATIC, set by the package)" {
2323+ arg "<dir>"
2424+}