This repository has no description
39 kB
859 lines
1"""hermes-voice: talk to Hermes through OpenAI's full-duplex gpt-live-1.
2
3The browser holds a WebRTC session with gpt-live-1, which listens and speaks at
4the same time and chats on its own. Whenever the user asks for real work, the
5voice model emits a client delegation; the page hands the recent transcript to
6this server, which starts it as a Hermes run on the API server (POST /v1/runs,
7one Hermes session id per conversation), follows the run's event stream and
8streams the reply back. The page then gives the answer to the voice model,
9which says it aloud.
10
11A run lives independently of any HTTP connection, and this server follows
12every run it started to its end, whether or not a page is still listening. A
13newer delegation within the same call stops the previous run (the user
14corrected or replaced the request); hanging up just ends the call, and the run
15finishes whatever the user asked for. On hang-up the page sends what the user
16said since the last delegation, so a request spoken right before hanging up
17still reaches Hermes. When a run asks for approval while no page is listening,
18the question goes to the owner's Matrix DM (the gateway's Matrix home room,
19through its webhook platform), and the hermes-voice plugin's
20`/assist approve|deny <code>` command answers it.
21
22Hermes ends a call itself: when the user says goodbye, the voice model
23delegates that like any request, and Hermes calls the plugin's end_call tool.
24This server sees the call in the run's events and tells a listening page to
25close the session once the voice has spoken the reply.
26
27This server exists so neither secret reaches the browser: it exchanges the
28page's SDP offer for a Live session with the OpenAI key, and it calls Hermes
29with the API server's bearer token. That token is root-equivalent for the
30Hermes host, so whoever can use this server can drive the agent: every API
31route requires the page's own access token (--access-token-file), and serving
32beyond loopback without one is refused.
33
34Vendor contract: https://developers.openai.com/api/docs/guides/live-delegation
35"""
36
37import argparse
38import hashlib
39import hmac
40import json
41import logging
42import os
43import re
44import secrets
45import socket
46import threading
47import time
48import urllib.error
49import urllib.request
50from http import HTTPStatus
51from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
52from pathlib import Path
53
54log = logging.getLogger("hermes-voice")
55
56OPENAI_BASE_URL = "https://api.openai.com/v1"
57DEFAULT_MODEL = "gpt-live-1"
58DEFAULT_VOICE = "marin"
59# gpt-live-1 voices as OpenAI's voice picker lists them: language, regional
60# influence, presentation and character, natural voices first, then the
61# generated ones. The docs' voice table and the openai SDK (3.28.0) list only
62# part of them; /v1/live/sessions accepts the German, French, Hindi, Japanese
63# and Korean voices as well, while an unknown name is refused with 403.
64BUILT_IN_VOICES = {
65 "gleam": "English, North American, female: warm and conversational",
66 "ripple": "English, Australian, male: warm and conversational",
67 "vesper": "English, British, male: clear and direct",
68 "willow": "English, Irish, female: expressive and bright",
69 "stone": "English, Irish, male: calm and precise",
70 "meridian": "English, North American, male: grounded and measured",
71 "bossa": "Portuguese, Brazilian, female: clear and direct",
72 "tempo": "Portuguese, Brazilian, male: grounded and measured",
73 "cedar": "English, North American, male: open and upbeat",
74 "marin": "English, North American, female: animated and earnest",
75 "nuri": "German, female: warm and animated",
76 "juni": "German, female: bright and expressive",
77 "flitz": "German, male: rich and playful",
78 "brise": "French, female: clear and conversational",
79 "sillage": "French, male: warm and grounded",
80 "nira": "Hindi, female: soft and thoughtful",
81 "harema": "Japanese, female: gentle and engaging",
82 "shida": "Japanese, male: quiet and grounded",
83 "noeul": "Korean, female: airy and expressive",
84 "quartz": "English, Australian, female, generated: warm and conversational",
85 "alloy": "English, North American, male, generated: balanced and clear",
86 "ash": "English, North American, male, generated: savvy and relaxed",
87 "ballad": "English, North American, male, generated: cheerful and candid",
88 "coral": "English, North American, female, generated: easygoing and versatile",
89 "echo": "English, North American, male, generated: bright and inquisitive",
90 "sage": "English, North American, female, generated: confident and optimistic",
91 "shimmer": "English, North American, female, generated: composed and direct",
92 "verse": "English, North American, male, generated: calm and affirming",
93 "beacon": "English, Filipino, male, generated: clear and personable",
94 "delta": "English, Southern U.S., female, generated: warm and easygoing",
95 "cinder": "English, Southern U.S., male, generated: steady and grounded",
96}
97# gpt-live-1 bills every second a session is open, silence included; the page
98# closes the session after this much inactivity.
99DEFAULT_IDLE_SECONDS = 90
100MAX_BODY = 256 * 1024
101# The run event stream sends a keepalive every 10s, so a read that stalls this
102# long means the API server is gone.
103EVENTS_READ_TIMEOUT = 60
104# Same cadence towards the page: a write is the only way to notice that a page
105# went away while a run is quiet (e.g. waiting for an approval).
106PAGE_KEEPALIVE_SECONDS = 10
107SESSION_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
108# Hermes's tool that ends the call (hermes-plugin/__init__.py).
109END_CALL_TOOL = "end_call"
110
111# Frontend persona, adapted from Hermes's own (tools/voice_live.py): role,
112# style and a labelled delegation policy. The backend carries the real
113# instructions, tools and memory.
114PERSONA = """You are Hermes, the user's warm, upbeat and genuinely helpful voice assistant. Sound friendly and glad to help: positive and encouraging, never curt, annoyed, bored or sarcastic, even when delivering bad news or when the user is short with you. Talk fast: speak at a rapid, energetic pace, like someone who talks quickly by nature, with no slow drawn-out words and no long pauses between sentences. Keep replies short; a brief warm acknowledgment before the facts is good.
115
116Backchannel policy: Use moderate backchannels. Acknowledge naturally without competing with the main response.
117
118Interruption policy: Stop speaking when the user interrupts. Listen to what they say.
119
120Delegation policy:
121Backend tools:
122- Hermes agent: a full AI agent with tools. It can run commands on the user's machines, read and edit files, manage the calendar, write and read messages in the user's Matrix chat, browse the web, search, and remember things across sessions. It is the one who actually does work and knows facts.
123
124Delegate to the backend when:
125- The user asks a question that needs facts, current information, or careful reasoning.
126- The user asks you to do, check, find, make, fix, run or remember anything.
127- A correction changes work already requested.
128
129- The user says goodbye, says that is all, asks you to hang up, or says they are leaving. Hermes ends the call after your reply; you may still say a short goodbye, but never end the call yourself.
130
131Do not delegate to the backend when:
132- The user greets you, makes small talk, or asks you to repeat a result already provided.
133- You need a brief clarification to understand the request.
134
135Delegate before giving an answer that depends on backend work. Do not guess the result while waiting; say briefly and cheerfully that you are on it, then wait for the result.
136
137Approval policy:
138Hermes sometimes needs the user's approval before it runs a command. When you are told so, explain in plain words what Hermes wants to do and ask whether to approve or deny it. When the user answers, delegate to the backend so the answer reaches Hermes. Never decide an approval yourself."""
139
140# Ephemeral system prompt for every delegated Hermes turn (Hermes layers system
141# messages on top of its core prompt without storing them).
142BACKEND_NOTE = """This turn is a delegation from a live spoken conversation. The user message is a speech transcript: it may contain mis-hearings, phonetic spellings of technical terms, hesitations and later corrections; use the latest intent. Your reply will be spoken aloud by a friendly voice model that paraphrases it: answer in warm, plain conversational sentences, keep it short (a few sentences unless the user asked for detail), no markdown, no lists, no code blocks, no URLs read out character by character. Do the work with your tools as usual; only the final facts need to be spoken. Report an action as done only after it actually succeeded. The user may hang up before your reply is spoken: when they asked for a result or a question somewhere else, such as their Matrix chat (matrix_dm_send), send it there yourself. When the user says goodbye, says that is all, asks to hang up or says they are leaving, call end_call: only after any work they asked for has been started or handed off (for example a result promised to their Matrix chat), and never while you still need an answer from them, such as an approval. Your reply is still spoken before the call ends, so make it a short confirmation or goodbye."""
143
144# Instead of BACKEND_NOTE for what the user said right before hanging up, when
145# the voice model had not delegated it yet.
146HUNG_UP_NOTE = """This turn comes from a live spoken conversation the user has already hung up. The user message is a speech transcript of what they said since the last request: it may contain mis-hearings, phonetic spellings of technical terms, hesitations and later corrections; use the latest intent. Nobody hears your reply. If the transcript asks for something, do it with your tools; when the user asked for a result somewhere, such as their Matrix chat (matrix_dm_send), send it there, and if you need something from them, ask in their Matrix chat. If it only says goodbye or thanks, or repeats a request already handled in this conversation, do nothing and reply with a short note saying so. The call is over already: do not call end_call."""
147
148
149def glossary_text(glossary):
150 """Glossary entries as prompt lines: `term` or `term (sounds like "…")`."""
151 lines = []
152 for entry in glossary:
153 if isinstance(entry, str):
154 lines.append(f"- {entry}")
155 else:
156 term = entry["term"]
157 spoken = entry.get("spoken")
158 lines.append(f'- {term} (sounds like "{spoken}")' if spoken else f"- {term}")
159 return "\n".join(lines)
160
161
162class Config:
163 def __init__(self, raw, openai_key, hermes_url, hermes_key, matrix_webhook=None):
164 self.matrix_webhook = matrix_webhook
165 self.model = raw.get("model", DEFAULT_MODEL)
166 self.voice = raw.get("voice", DEFAULT_VOICE)
167 self.idle_seconds = int(raw.get("idle_seconds", DEFAULT_IDLE_SECONDS))
168 self.glossary = raw.get("glossary", [])
169 self.extra_instructions = raw.get("instructions", "").strip()
170 self.openai_key = openai_key
171 self.hermes_url = hermes_url.rstrip("/")
172 self.hermes_key = hermes_key
173
174 def live_instructions(self):
175 parts = [PERSONA]
176 if self.glossary:
177 parts.append(
178 "Technical terms the user says often. Understand them when spoken and "
179 "say them like this:\n" + glossary_text(self.glossary)
180 )
181 if self.extra_instructions:
182 parts.append(self.extra_instructions)
183 return "\n\n".join(parts)
184
185 def backend_note(self, hung_up=False):
186 note = HUNG_UP_NOTE if hung_up else BACKEND_NOTE
187 if not self.glossary:
188 return note
189 return (
190 note
191 + "\n\nTerms the transcript may contain as phonetic misspellings:\n"
192 + glossary_text(self.glossary)
193 )
194
195 def voices(self):
196 """Voices the page may pick, name -> description: the built-ins plus
197 the configured one (which may be a custom voice id)."""
198 voices = dict(BUILT_IN_VOICES)
199 voices.setdefault(self.voice, "custom voice")
200 return voices
201
202
203def voice_param(voice):
204 # Custom voices are objects ({"id": "voice_…"}); built-ins are names.
205 if voice.startswith("voice_"):
206 return {"id": voice}
207 return voice
208
209
210def live_history(lines):
211 """Seed history for a new Live session from the page's transcript lines
212 (spoken dialogue only; Hermes answers were spoken through the voice)."""
213 items = []
214 dialogue = [line for line in lines if line.get("speaker") in ("user", "assistant")]
215 for line in dialogue[-24:]:
216 text = str(line.get("text", "")).strip()[:1200]
217 if not text:
218 continue
219 if line["speaker"] == "assistant":
220 content = {"type": "output_text", "text": text}
221 else:
222 content = {"type": "input_text", "text": text}
223 items.append({"type": "message", "role": line["speaker"], "content": [content]})
224 return items
225
226
227def create_live_session(config, sdp, history, voice):
228 session = {
229 "model": config.model,
230 "instructions": config.live_instructions(),
231 "audio": {"output": {"voice": voice_param(voice)}},
232 "delegation": {"type": "client"},
233 # The page is an untrusted frontend: it may only answer delegations and
234 # hang up, not rewrite the session.
235 "client": {
236 "data_channel": {
237 "allowed_client_events": [
238 "session.commentary.append",
239 "session.thinking.append",
240 "session.close",
241 ]
242 }
243 },
244 }
245 if history:
246 session["input"] = history
247 body = json.dumps(
248 {"session": session, "transport": {"type": "webrtc", "sdp": sdp}}
249 ).encode()
250 request = urllib.request.Request(
251 f"{OPENAI_BASE_URL}/live/sessions",
252 data=body,
253 method="POST",
254 headers={
255 "Authorization": f"Bearer {config.openai_key}",
256 "Content-Type": "application/json",
257 },
258 )
259 with urllib.request.urlopen(request, timeout=30) as response:
260 answer = json.load(response)
261 return {"session_id": answer["session"]["id"], "sdp": answer["transport"]["sdp"]}
262
263
264def transcript_message(lines):
265 rendered = []
266 for line in lines:
267 text = str(line.get("text", "")).strip()
268 if text:
269 who = "Assistant" if line.get("speaker") == "assistant" else "User"
270 rendered.append(f"{who}: {text}")
271 return "Spoken conversation since the last request, newest last:\n" + "\n".join(
272 rendered
273 )
274
275
276def iter_sse(response):
277 """Yield (event, data) pairs from a text/event-stream response."""
278 event, data = None, []
279 for raw in response:
280 line = raw.decode("utf-8", "replace").rstrip("\r\n")
281 if not line:
282 if data:
283 yield event, "\n".join(data)
284 event, data = None, []
285 elif line.startswith("event:"):
286 event = line[6:].strip()
287 elif line.startswith("data:"):
288 data.append(line[5:].lstrip())
289 if data:
290 yield event, "\n".join(data)
291
292
293def hermes_request(config, method, path, payload=None, timeout=30):
294 """Open a request to the Hermes API server (the caller closes it)."""
295 request = urllib.request.Request(
296 f"{config.hermes_url}{path}",
297 data=None if payload is None else json.dumps(payload).encode(),
298 method=method,
299 headers={
300 "Authorization": f"Bearer {config.hermes_key}",
301 "Content-Type": "application/json",
302 },
303 )
304 return urllib.request.urlopen(request, timeout=timeout)
305
306
307def hermes_call(config, method, path, payload=None):
308 with hermes_request(config, method, path, payload) as response:
309 return json.load(response)
310
311
312def hermes_start_run(config, hermes_session, lines, hung_up=False):
313 """Start one Hermes turn as a run; return its run id. Hermes queues it
314 behind a run of the same session that is still going."""
315 # A plain string input: Hermes then loads the history of session_id itself.
316 return hermes_call(
317 config,
318 "POST",
319 "/v1/runs",
320 {
321 "input": transcript_message(lines),
322 "instructions": config.backend_note(hung_up),
323 "session_id": hermes_session,
324 },
325 )["run_id"]
326
327
328def hermes_follow_run(config, run_id):
329 """Follow a run's events; yield NDJSON-ready dicts (tool, delta, approval,
330 end_call, done, error). Leaving early only detaches: the run keeps going."""
331 with hermes_request(
332 config, "GET", f"/v1/runs/{run_id}/events", timeout=EVENTS_READ_TIMEOUT
333 ) as response:
334 for _, data in iter_sse(response):
335 payload = json.loads(data)
336 event = payload.get("event", "")
337 if payload.get("tool") == END_CALL_TOOL:
338 # The plugin's tool fails outside a voice session; the page
339 # closes the call after speaking the reply.
340 if event == "tool.completed" and not payload.get("error"):
341 yield {"type": "end_call"}
342 elif event in ("tool.started", "tool.completed"):
343 status = "running" if event == "tool.started" else "completed"
344 if payload.get("error"):
345 status = "error"
346 yield {"type": "tool", "status": status, "label": payload.get("tool")}
347 elif event == "approval.request":
348 # The run blocks until POST /v1/runs/{run_id}/approval answers.
349 # Under the gateway's session authority the request is named
350 # prompt_id and the answer must carry its execution_generation.
351 yield {
352 "type": "approval",
353 "request_id": payload.get("request_id") or payload.get("prompt_id"),
354 "execution_generation": payload.get("execution_generation"),
355 "command": payload.get("command"),
356 "description": payload.get("description"),
357 "timeout_s": payload.get("timeout_s"),
358 }
359 elif event == "run.completed":
360 # Only the final answer: message.delta also streams commentary.
361 yield {"type": "delta", "text": payload.get("output") or ""}
362 yield {"type": "done", "finish_reason": "stop"}
363 return
364 elif event in ("run.failed", "run.cancelled", "run.interrupted"):
365 status = event.removeprefix("run.")
366 error = payload.get("error") or f"Hermes run {status}"
367 yield {"type": "done", "finish_reason": status, "error": error}
368 return
369 yield {"type": "error", "message": "Hermes event stream ended early"}
370
371
372def hermes_approve(config, run_id, approval, choice):
373 """Answer a pending approval of a Hermes run."""
374 answer = {"choice": choice}
375 if approval.get("request_id"):
376 answer["request_id"] = approval["request_id"]
377 if approval.get("execution_generation") is not None:
378 answer["execution_generation"] = approval["execution_generation"]
379 return hermes_call(config, "POST", f"/v1/runs/{run_id}/approval", answer)
380
381
382def notify_matrix(config, text):
383 """Post text to the owner's Matrix DM (the gateway's Matrix home room)
384 through the gateway's webhook platform (a deliver_only route, signed with
385 the API server key)."""
386 body = json.dumps({"text": text}).encode()
387 stamp = str(int(time.time()))
388 signature = hmac.new(
389 config.hermes_key.encode(), stamp.encode() + b"." + body, hashlib.sha256
390 ).hexdigest()
391 request = urllib.request.Request(
392 config.matrix_webhook,
393 data=body,
394 method="POST",
395 headers={
396 "Content-Type": "application/json",
397 "X-Webhook-Timestamp": stamp,
398 "X-Webhook-Signature-V2": signature,
399 },
400 )
401 with urllib.request.urlopen(request, timeout=30):
402 pass
403
404
405# Live runs by run id, and the latest run per Hermes session, which a
406# correction in the same call replaces.
407RUNS = {}
408LATEST = {}
409RUNS_LOCK = threading.Lock()
410# Pending approvals by short code: the page and the Matrix command answer
411# with the code.
412APPROVALS = {}
413APPROVALS_LOCK = threading.Lock()
414
415
416class Run:
417 """A Hermes run this server follows to its end, page or no page. Pages
418 replay its items; an approval nobody can hear goes to Matrix."""
419
420 def __init__(self, config, hermes_session, run_id, hung_up=False):
421 self.config = config
422 self.hermes_session = hermes_session
423 self.run_id = run_id
424 self.items = []
425 self.finished = False
426 self.pages = 0
427 self.hung_up = hung_up
428 self.cond = threading.Condition()
429 with RUNS_LOCK:
430 RUNS[run_id] = self
431 if not hung_up:
432 LATEST[hermes_session] = self
433 threading.Thread(target=self.follow, daemon=True).start()
434
435 def follow(self):
436 started = time.monotonic()
437 tools = 0
438 try:
439 for item in hermes_follow_run(self.config, self.run_id):
440 if item["type"] == "tool" and item["status"] == "running":
441 tools += 1
442 self.add(item)
443 except urllib.error.HTTPError as error:
444 detail = error.read().decode("utf-8", "replace")[:400]
445 self.add({"type": "error", "message": f"Hermes {error.code}: {detail}"})
446 except (urllib.error.URLError, OSError, ValueError) as error:
447 self.add({"type": "error", "message": f"Hermes unreachable: {error}"})
448 settle_approvals(self)
449 with self.cond:
450 self.finished = True
451 self.cond.notify_all()
452 with RUNS_LOCK:
453 RUNS.pop(self.run_id, None)
454 if LATEST.get(self.hermes_session) is self:
455 del LATEST[self.hermes_session]
456 last = self.items[-1] if self.items else {}
457 log.info(
458 "run %s ended (%s) after %.1fs, %d tool calls",
459 self.run_id,
460 last.get("finish_reason") or last.get("message"),
461 time.monotonic() - started,
462 tools,
463 )
464
465 def add(self, item):
466 # Anything after an approval request means Hermes moved on (answered,
467 # or its own timeout).
468 settle_approvals(self)
469 if item["type"] == "approval":
470 item["code"] = open_approval(self, item)
471 with self.cond:
472 self.items.append(item)
473 self.cond.notify_all()
474 unheard = not self.pages or self.hung_up
475 if item["type"] == "end_call":
476 # Without a page the call has ended already.
477 log.info("run %s ends the call%s", self.run_id, " (no call)" if unheard else "")
478 self.ask_unheard()
479
480 def attach(self):
481 with self.cond:
482 self.pages += 1
483
484 def detach(self):
485 with self.cond:
486 self.pages -= 1
487 self.ask_unheard()
488
489 def hang_up(self):
490 with self.cond:
491 self.hung_up = True
492 self.ask_unheard()
493
494 def ask_unheard(self):
495 """Send a pending approval to Matrix once no call can answer it."""
496 with self.cond:
497 if self.pages and not self.hung_up:
498 return
499 with APPROVALS_LOCK:
500 unasked = [
501 (code, item)
502 for code, (run, item) in APPROVALS.items()
503 if run is self and not item.get("asked")
504 ]
505 for _, item in unasked:
506 item["asked"] = True
507 for code, item in unasked:
508 ask_matrix(self.config, code, item)
509
510
511def open_approval(run, item):
512 with APPROVALS_LOCK:
513 code = secrets.token_hex(2)
514 while code in APPROVALS:
515 code = secrets.token_hex(2)
516 APPROVALS[code] = (run, item)
517 return code
518
519
520def settle_approvals(run):
521 with APPROVALS_LOCK:
522 for code in [code for code, (owner, _) in APPROVALS.items() if owner is run]:
523 del APPROVALS[code]
524
525
526def ask_matrix(config, code, item):
527 if not config.matrix_webhook:
528 log.info("approval %s has no listener and no Matrix webhook", code)
529 return
530 text = (
531 "Hermes needs your approval for work from a voice call: "
532 + (item.get("description") or "run a command")
533 )
534 if item.get("command"):
535 text += f"\n```\n{item['command']}\n```"
536 text += f"\nAnswer with `/assist approve {code}` or `/assist deny {code}`."
537 if item.get("timeout_s"):
538 text += f" Unanswered, it is denied after {round(item['timeout_s'] / 60)} minutes."
539 try:
540 notify_matrix(config, text)
541 log.info("approval %s asked in Matrix", code)
542 except (urllib.error.URLError, OSError) as error:
543 log.warning("asking approval %s in Matrix failed: %s", code, error)
544
545
546def stop_run(config, run_id):
547 """POST /v1/runs/{run_id}/stop; failures are only logged."""
548 try:
549 hermes_call(config, "POST", f"/v1/runs/{run_id}/stop")
550 log.info("stopped run %s", run_id)
551 except (urllib.error.URLError, OSError, ValueError) as error:
552 log.warning("stopping run %s failed: %s", run_id, error)
553
554
555class Handler(BaseHTTPRequestHandler):
556 server_version = "hermes-voice"
557 config: Config
558 static: Path
559 access_token: "str | None" = None
560
561 def authorized(self, also_hermes_key=False):
562 """Check the page's bearer token (or, where allowed, the API server key
563 the hermes-voice plugin in the gateway holds); answer 401 when it is
564 wrong."""
565 if self.access_token is None:
566 return True
567 header = self.headers.get("Authorization", "")
568 scheme, _, token = header.partition(" ")
569 accepted = [self.access_token]
570 if also_hermes_key:
571 accepted.append(self.config.hermes_key)
572 if scheme == "Bearer" and any(
573 hmac.compare_digest(token.encode(), key.encode()) for key in accepted
574 ):
575 return True
576 self.send_json(HTTPStatus.UNAUTHORIZED, {"error": "access token required"})
577 return False
578
579 def log_message(self, format, *args):
580 log.info("%s %s", self.address_string(), format % args)
581
582 def send_json(self, status, payload):
583 body = json.dumps(payload).encode()
584 self.send_response(status)
585 self.send_header("Content-Type", "application/json")
586 self.send_header("Content-Length", str(len(body)))
587 self.send_header("Cache-Control", "no-store")
588 self.end_headers()
589 self.wfile.write(body)
590
591 def read_json(self):
592 length = int(self.headers.get("Content-Length") or 0)
593 if length <= 0 or length > MAX_BODY:
594 raise ValueError("bad body length")
595 return json.loads(self.rfile.read(length))
596
597 def do_GET(self):
598 if self.path == "/config":
599 if not self.authorized():
600 return
601 self.send_json(
602 HTTPStatus.OK,
603 {
604 "model": self.config.model,
605 "voice": self.config.voice,
606 "voices": self.config.voices(),
607 "idle_seconds": self.config.idle_seconds,
608 },
609 )
610 return
611 names = {
612 "/": ("index.html", "text/html; charset=utf-8"),
613 "/app.js": ("app.js", "text/javascript; charset=utf-8"),
614 "/style.css": ("style.css", "text/css; charset=utf-8"),
615 }
616 if self.path not in names:
617 self.send_error(HTTPStatus.NOT_FOUND)
618 return
619 name, content_type = names[self.path]
620 body = (self.static / name).read_bytes()
621 self.send_response(HTTPStatus.OK)
622 self.send_header("Content-Type", content_type)
623 self.send_header("Content-Length", str(len(body)))
624 self.send_header("Cache-Control", "no-store")
625 self.end_headers()
626 self.wfile.write(body)
627
628 def do_POST(self):
629 if not self.authorized(also_hermes_key=self.path == "/approve"):
630 return
631 try:
632 request = self.read_json()
633 except ValueError:
634 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid JSON body"})
635 return
636 if self.path == "/session":
637 self.handle_session(request)
638 elif self.path == "/delegate":
639 self.handle_delegate(request)
640 elif self.path == "/hangup":
641 self.handle_hangup(request)
642 elif self.path == "/approve":
643 self.handle_approve(request)
644 else:
645 self.send_error(HTTPStatus.NOT_FOUND)
646
647 def handle_session(self, request):
648 # The vendor's SDP parser needs the offer byte-exact (trailing CRLF too).
649 sdp = request.get("sdp")
650 if not isinstance(sdp, str) or not sdp.strip():
651 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "sdp offer required"})
652 return
653 try:
654 voice = request.get("voice") or self.config.voice
655 if voice not in self.config.voices():
656 self.send_json(HTTPStatus.BAD_REQUEST, {"error": f"unknown voice {voice!r}"})
657 return
658 result = create_live_session(
659 self.config, sdp, live_history(request.get("history") or []), voice
660 )
661 except urllib.error.HTTPError as error:
662 detail = error.read().decode("utf-8", "replace")[:800]
663 log.warning("live session creation failed: %s %s", error.code, detail)
664 self.send_json(
665 HTTPStatus.BAD_GATEWAY,
666 {"error": f"OpenAI rejected the session ({error.code}): {detail}"},
667 )
668 return
669 except (urllib.error.URLError, OSError, KeyError, ValueError) as error:
670 log.warning("live session creation failed: %s", error)
671 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"OpenAI: {error}"})
672 return
673 self.send_json(HTTPStatus.OK, result)
674
675 def handle_delegate(self, request):
676 hermes_session = request.get("hermes_session")
677 lines = request.get("lines")
678 if not isinstance(hermes_session, str) or not SESSION_ID_RE.match(hermes_session):
679 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid hermes_session"})
680 return
681 if not isinstance(lines, list) or not lines:
682 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "lines required"})
683 return
684 # A correction within the same call (the page says so) supersedes and
685 # stops the run it replaces; runs from earlier calls keep going.
686 if request.get("replace") is True:
687 with RUNS_LOCK:
688 previous = LATEST.pop(hermes_session, None)
689 if previous:
690 stop_run(self.config, previous.run_id)
691 try:
692 run = Run(
693 self.config,
694 hermes_session,
695 hermes_start_run(self.config, hermes_session, lines),
696 )
697 except urllib.error.HTTPError as error:
698 detail = error.read().decode("utf-8", "replace")[:400]
699 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
700 return
701 except (urllib.error.URLError, OSError, ValueError, KeyError) as error:
702 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
703 return
704 self.send_response(HTTPStatus.OK)
705 self.send_header("Content-Type", "application/x-ndjson")
706 self.send_header("Cache-Control", "no-store")
707 self.send_header("X-Accel-Buffering", "no")
708 self.end_headers()
709 run.attach()
710 try:
711 sent = 0
712 while True:
713 with run.cond:
714 if sent == len(run.items) and not run.finished:
715 run.cond.wait(PAGE_KEEPALIVE_SECONDS)
716 new = run.items[sent:]
717 sent += len(new)
718 done = run.finished and sent == len(run.items)
719 for item in new:
720 self.write_line(item)
721 if done:
722 break
723 if not new:
724 self.wfile.write(b"\n")
725 self.wfile.flush()
726 except (BrokenPipeError, ConnectionResetError):
727 # Only the page is gone: this server keeps following the run.
728 log.info("page left run %s", run.run_id)
729 finally:
730 run.detach()
731
732 def handle_hangup(self, request):
733 """The call ended. No page hears this conversation's runs any more, and
734 what the user said since the last delegation becomes its own run."""
735 hermes_session = request.get("hermes_session")
736 lines = request.get("lines") or []
737 if (
738 not isinstance(hermes_session, str)
739 or not SESSION_ID_RE.match(hermes_session)
740 or not isinstance(lines, list)
741 ):
742 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid hang-up"})
743 return
744 with RUNS_LOCK:
745 LATEST.pop(hermes_session, None)
746 runs = [run for run in RUNS.values() if run.hermes_session == hermes_session]
747 for run in runs:
748 run.hang_up()
749 if not any(line.get("speaker") == "user" for line in lines):
750 self.send_json(HTTPStatus.OK, {"run_id": None})
751 return
752 try:
753 run_id = hermes_start_run(self.config, hermes_session, lines, hung_up=True)
754 except urllib.error.HTTPError as error:
755 detail = error.read().decode("utf-8", "replace")[:400]
756 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
757 return
758 except (urllib.error.URLError, OSError, ValueError, KeyError) as error:
759 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
760 return
761 Run(self.config, hermes_session, run_id, hung_up=True)
762 log.info("hang-up of %s left a request: run %s", hermes_session, run_id)
763 self.send_json(HTTPStatus.OK, {"run_id": run_id})
764
765 def handle_approve(self, request):
766 code = request.get("code")
767 choice = request.get("choice")
768 # No "always": a permanent allowlist entry is not a voice decision.
769 if choice not in ("once", "session", "deny") or not (
770 code is None or isinstance(code, str)
771 ):
772 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid approval answer"})
773 return
774 with APPROVALS_LOCK:
775 if code is None and len(APPROVALS) == 1:
776 code = next(iter(APPROVALS))
777 pending = APPROVALS.get(code)
778 if pending is None:
779 self.send_json(
780 HTTPStatus.CONFLICT,
781 {"error": f"no pending approval {code}" if code else "no single pending approval"},
782 )
783 return
784 run, item = pending
785 try:
786 result = hermes_approve(self.config, run.run_id, item, choice)
787 except urllib.error.HTTPError as error:
788 detail = error.read().decode("utf-8", "replace")[:400]
789 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
790 return
791 except (urllib.error.URLError, OSError, ValueError) as error:
792 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
793 return
794 with APPROVALS_LOCK:
795 APPROVALS.pop(code, None)
796 if result.get("status") == "already_resolved":
797 self.send_json(HTTPStatus.CONFLICT, {"error": f"approval {code} was already answered"})
798 return
799 log.info("approval %s (run %s): %s %s", code, run.run_id, choice, result.get("status"))
800 self.send_json(
801 HTTPStatus.OK,
802 {"code": code, "choice": choice, "description": item.get("description")},
803 )
804
805 def write_line(self, item):
806 self.wfile.write(json.dumps(item).encode() + b"\n")
807 self.wfile.flush()
808
809
810def main():
811 parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
812 parser.add_argument("--listen", default="127.0.0.1:8790", help="host:port to serve on")
813 parser.add_argument("--hermes-url", required=True, help="Hermes API server base URL")
814 parser.add_argument("--hermes-key-file", required=True, help="file holding API_SERVER_KEY")
815 parser.add_argument("--openai-key-file", required=True, help="file holding the OpenAI API key")
816 parser.add_argument("--access-token-file", help="file holding the token the page must present")
817 parser.add_argument(
818 "--matrix-webhook",
819 help="Hermes webhook route (deliver_only to Matrix, signed with API_SERVER_KEY) that "
820 "receives approvals no call can answer",
821 )
822 parser.add_argument("--config", help="JSON file: voice, model, idle_seconds, glossary, instructions")
823 parser.add_argument(
824 "--static",
825 default=os.environ.get("HERMES_VOICE_STATIC"),
826 help="directory with index.html, app.js, style.css (default: $HERMES_VOICE_STATIC)",
827 )
828 args = parser.parse_args()
829 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
830
831 host, _, port = args.listen.rpartition(":")
832 host = host.strip("[]")
833 if args.access_token_file is None and host not in ("127.0.0.1", "::1", "localhost"):
834 parser.error("--access-token-file is required when not listening on loopback")
835
836 raw = json.loads(Path(args.config).read_text()) if args.config else {}
837 config = Config(
838 raw,
839 openai_key=Path(args.openai_key_file).read_text().strip(),
840 hermes_url=args.hermes_url,
841 hermes_key=Path(args.hermes_key_file).read_text().strip(),
842 matrix_webhook=args.matrix_webhook,
843 )
844 Handler.config = config
845 Handler.static = Path(args.static)
846 if args.access_token_file:
847 Handler.access_token = Path(args.access_token_file).read_text().strip()
848 server_class = ThreadingHTTPServer
849 if ":" in host:
850 # "::" is dual-stack on Linux (bindv6only=0).
851 server_class = type("Server6", (ThreadingHTTPServer,), {"address_family": socket.AF_INET6})
852 server = server_class((host, int(port)), Handler)
853 server.daemon_threads = True
854 log.info("serving on http://%s", args.listen)
855 server.serve_forever()
856
857
858if __name__ == "__main__":
859 main()