This repository has no description
39 kB
864 lines
1"""hermes-voice: talk to Hermes through OpenAI's full-duplex gpt-live-1.
2
3The browser holds a WebRTC session with gpt-live-1, which listens and speaks at
4the same time and chats on its own. Whenever the user asks for real work, the
5voice model emits a client delegation; the page hands the recent transcript to
6this server, which starts it as a Hermes run on the API server (POST /v1/runs,
7one Hermes session id per conversation), follows the run's event stream and
8streams the reply back. The page then gives the answer to the voice model,
9which says it aloud.
10
11A run lives independently of any HTTP connection, and this server follows
12every run it started to its end, whether or not a page is still listening. A
13newer delegation within the same call stops the previous run (the user
14corrected or replaced the request); hanging up just ends the call, and the run
15finishes whatever the user asked for. On hang-up the page sends what the user
16said since the last delegation, so a request spoken right before hanging up
17still reaches Hermes. When a run asks for approval while no page is listening,
18the question goes to the owner's Matrix DM (the gateway's Matrix home room,
19through its webhook platform), and the hermes-voice plugin's
20`/assist approve|deny <code>` command answers it.
21
22Hermes ends a call itself: when the user says goodbye, the voice model
23delegates that like any request, and Hermes calls the plugin's end_call tool.
24This server sees the call in the run's events and tells a listening page to
25close the session once the voice has spoken the reply.
26
27This server exists so neither secret reaches the browser: it exchanges the
28page's SDP offer for a Live session with the OpenAI key, and it calls Hermes
29with the API server's bearer token. That token is root-equivalent for the
30Hermes host, so whoever can use this server can drive the agent: every API
31route requires the page's own access token (--access-token-file), and serving
32beyond loopback without one is refused.
33
34Vendor contract: https://developers.openai.com/api/docs/guides/live-delegation
35"""
36
37import argparse
38import hashlib
39import hmac
40import json
41import logging
42import os
43import re
44import secrets
45import socket
46import threading
47import time
48import urllib.error
49import urllib.request
50from http import HTTPStatus
51from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
52from pathlib import Path
53
54log = logging.getLogger("hermes-voice")
55
56OPENAI_BASE_URL = "https://api.openai.com/v1"
57DEFAULT_MODEL = "gpt-live-1"
58DEFAULT_VOICE = "marin"
59# gpt-live-1 voices as OpenAI's voice picker lists them: language, regional
60# influence, presentation and character, natural voices first, then the
61# generated ones. The docs' voice table and the openai SDK (3.28.0) list only
62# part of them; /v1/live/sessions accepts the German, French, Hindi, Japanese
63# and Korean voices as well, while an unknown name is refused with 403.
64BUILT_IN_VOICES = {
65 "gleam": "English, North American, female: warm and conversational",
66 "ripple": "English, Australian, male: warm and conversational",
67 "vesper": "English, British, male: clear and direct",
68 "willow": "English, Irish, female: expressive and bright",
69 "stone": "English, Irish, male: calm and precise",
70 "meridian": "English, North American, male: grounded and measured",
71 "bossa": "Portuguese, Brazilian, female: clear and direct",
72 "tempo": "Portuguese, Brazilian, male: grounded and measured",
73 "cedar": "English, North American, male: open and upbeat",
74 "marin": "English, North American, female: animated and earnest",
75 "nuri": "German, female: warm and animated",
76 "juni": "German, female: bright and expressive",
77 "flitz": "German, male: rich and playful",
78 "brise": "French, female: clear and conversational",
79 "sillage": "French, male: warm and grounded",
80 "nira": "Hindi, female: soft and thoughtful",
81 "harema": "Japanese, female: gentle and engaging",
82 "shida": "Japanese, male: quiet and grounded",
83 "noeul": "Korean, female: airy and expressive",
84 "quartz": "English, Australian, female, generated: warm and conversational",
85 "alloy": "English, North American, male, generated: balanced and clear",
86 "ash": "English, North American, male, generated: savvy and relaxed",
87 "ballad": "English, North American, male, generated: cheerful and candid",
88 "coral": "English, North American, female, generated: easygoing and versatile",
89 "echo": "English, North American, male, generated: bright and inquisitive",
90 "sage": "English, North American, female, generated: confident and optimistic",
91 "shimmer": "English, North American, female, generated: composed and direct",
92 "verse": "English, North American, male, generated: calm and affirming",
93 "beacon": "English, Filipino, male, generated: clear and personable",
94 "delta": "English, Southern U.S., female, generated: warm and easygoing",
95 "cinder": "English, Southern U.S., male, generated: steady and grounded",
96}
97# gpt-live-1 bills every second a session is open, silence included; the page
98# closes the session after this much inactivity.
99DEFAULT_IDLE_SECONDS = 90
100MAX_BODY = 256 * 1024
101# The run event stream sends a keepalive every 10s, so a read that stalls this
102# long means the API server is gone.
103EVENTS_READ_TIMEOUT = 60
104# Same cadence towards the page: a write is the only way to notice that a page
105# went away while a run is quiet (e.g. waiting for an approval).
106PAGE_KEEPALIVE_SECONDS = 10
107SESSION_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
108# Hermes's tool that ends the call (hermes-plugin/__init__.py).
109END_CALL_TOOL = "end_call"
110
111# Frontend persona, adapted from Hermes's own (tools/voice_live.py): role,
112# style and a labelled delegation policy. The backend carries the real
113# instructions, tools and memory.
114PERSONA = """You are Hermes, the user's warm, upbeat and genuinely helpful voice assistant. Sound friendly and glad to help: positive and encouraging, never curt, annoyed, bored or sarcastic, even when delivering bad news or when the user is short with you. Talk fast: speak at a rapid, energetic pace, like someone who talks quickly by nature, with no slow drawn-out words and no long pauses between sentences. Keep replies short; a brief warm acknowledgment before the facts is good.
115
116Backchannel policy: Use moderate backchannels. Acknowledge naturally without competing with the main response.
117
118Interruption policy: Stop speaking when the user interrupts. Listen to what they say.
119
120Delegation policy:
121Backend tools:
122- Hermes agent: a full AI agent with tools. It can run commands on the user's machines, read and edit files, manage the calendar, write and read messages in the user's Matrix chat, browse the web, search, and remember things across sessions. It is the one who actually does work and knows facts.
123
124Delegate to the backend when:
125- The user asks a question that needs facts, current information, or careful reasoning.
126- The user asks you to do, check, find, make, fix, run or remember anything.
127- A correction changes work already requested.
128
129- The user says goodbye, says that is all, asks you to hang up, or says they are leaving. Hermes ends the call after your reply; you may still say a short goodbye, but never end the call yourself.
130
131Do not delegate to the backend when:
132- The user greets you, makes small talk, or asks you to repeat a result already provided.
133- You need a brief clarification to understand the request.
134
135Delegate before giving an answer that depends on backend work. Do not guess the result while waiting; say briefly and cheerfully that you are on it, then wait for the result.
136
137Approval policy:
138Hermes sometimes needs the user's approval before it runs a command. When you are told so, explain in plain words what Hermes wants to do and ask whether to approve or deny it. When the user answers, delegate to the backend so the answer reaches Hermes. Never decide an approval yourself."""
139
140# Ephemeral system prompt for every delegated Hermes turn (Hermes layers system
141# messages on top of its core prompt without storing them).
142BACKEND_NOTE = """This turn is a delegation from a live spoken conversation. The user message is a speech transcript: it may contain mis-hearings, phonetic spellings of technical terms, hesitations and later corrections; use the latest intent. Your reply will be spoken aloud by a friendly voice model that paraphrases it: answer in warm, plain conversational sentences, keep it short (a few sentences unless the user asked for detail), no markdown, no lists, no code blocks, no URLs read out character by character. Do the work with your tools as usual; only the final facts need to be spoken. Report an action as done only after it actually succeeded. The user may hang up before your reply is spoken: when they asked for a result or a question somewhere else, such as their Matrix chat (matrix_dm_send), send it there yourself. When the user says goodbye, says that is all, asks to hang up or says they are leaving, call end_call: only after any work they asked for has been started or handed off (for example a result promised to their Matrix chat), and never while you still need an answer from them, such as an approval. Your reply is still spoken before the call ends, so make it a short confirmation or goodbye."""
143
144# Instead of BACKEND_NOTE for what the user said right before hanging up, when
145# the voice model had not delegated it yet.
146HUNG_UP_NOTE = """This turn comes from a live spoken conversation the user has already hung up. The user message is a speech transcript of what they said since the last request: it may contain mis-hearings, phonetic spellings of technical terms, hesitations and later corrections; use the latest intent. Nobody hears your reply. If the transcript asks for something, do it with your tools; when the user asked for a result somewhere, such as their Matrix chat (matrix_dm_send), send it there, and if you need something from them, ask in their Matrix chat. If it only says goodbye or thanks, or repeats a request already handled in this conversation, do nothing and reply with a short note saying so. The call is over already: do not call end_call."""
147
148
149def glossary_text(glossary):
150 """Glossary entries as prompt lines: `term` or `term (sounds like "…")`."""
151 lines = []
152 for entry in glossary:
153 if isinstance(entry, str):
154 lines.append(f"- {entry}")
155 else:
156 term = entry["term"]
157 spoken = entry.get("spoken")
158 lines.append(f'- {term} (sounds like "{spoken}")' if spoken else f"- {term}")
159 return "\n".join(lines)
160
161
162class Config:
163 def __init__(self, raw, openai_key, hermes_url, hermes_key, matrix_webhook=None):
164 self.matrix_webhook = matrix_webhook
165 self.model = raw.get("model", DEFAULT_MODEL)
166 self.voice = raw.get("voice", DEFAULT_VOICE)
167 self.idle_seconds = int(raw.get("idle_seconds", DEFAULT_IDLE_SECONDS))
168 self.glossary = raw.get("glossary", [])
169 self.extra_instructions = raw.get("instructions", "").strip()
170 self.openai_key = openai_key
171 self.hermes_url = hermes_url.rstrip("/")
172 self.hermes_key = hermes_key
173
174 def live_instructions(self):
175 parts = [PERSONA]
176 if self.glossary:
177 parts.append(
178 "Technical terms the user says often. Understand them when spoken and "
179 "say them like this:\n" + glossary_text(self.glossary)
180 )
181 if self.extra_instructions:
182 parts.append(self.extra_instructions)
183 return "\n\n".join(parts)
184
185 def backend_note(self, hung_up=False):
186 note = HUNG_UP_NOTE if hung_up else BACKEND_NOTE
187 if not self.glossary:
188 return note
189 return (
190 note
191 + "\n\nTerms the transcript may contain as phonetic misspellings:\n"
192 + glossary_text(self.glossary)
193 )
194
195 def voices(self):
196 """Voices the page may pick, name -> description: the built-ins plus
197 the configured one (which may be a custom voice id)."""
198 voices = dict(BUILT_IN_VOICES)
199 voices.setdefault(self.voice, "custom voice")
200 return voices
201
202
203def voice_param(voice):
204 # Custom voices are objects ({"id": "voice_…"}); built-ins are names.
205 if voice.startswith("voice_"):
206 return {"id": voice}
207 return voice
208
209
210def live_history(lines):
211 """Seed history for a new Live session from the page's transcript lines
212 (spoken dialogue only; Hermes answers were spoken through the voice)."""
213 items = []
214 dialogue = [line for line in lines if line.get("speaker") in ("user", "assistant")]
215 for line in dialogue[-24:]:
216 text = str(line.get("text", "")).strip()[:1200]
217 if not text:
218 continue
219 if line["speaker"] == "assistant":
220 content = {"type": "output_text", "text": text}
221 else:
222 content = {"type": "input_text", "text": text}
223 items.append({"type": "message", "role": line["speaker"], "content": [content]})
224 return items
225
226
227def create_live_session(config, sdp, history, voice):
228 session = {
229 "model": config.model,
230 "instructions": config.live_instructions(),
231 "audio": {"output": {"voice": voice_param(voice)}},
232 "delegation": {"type": "client"},
233 # The page is an untrusted frontend: it may only answer delegations and
234 # hang up, not rewrite the session.
235 "client": {
236 "data_channel": {
237 "allowed_client_events": [
238 "session.commentary.append",
239 "session.thinking.append",
240 "session.close",
241 ]
242 }
243 },
244 }
245 if history:
246 session["input"] = history
247 body = json.dumps(
248 {"session": session, "transport": {"type": "webrtc", "sdp": sdp}}
249 ).encode()
250 request = urllib.request.Request(
251 f"{OPENAI_BASE_URL}/live/sessions",
252 data=body,
253 method="POST",
254 headers={
255 "Authorization": f"Bearer {config.openai_key}",
256 "Content-Type": "application/json",
257 },
258 )
259 with urllib.request.urlopen(request, timeout=30) as response:
260 answer = json.load(response)
261 return {"session_id": answer["session"]["id"], "sdp": answer["transport"]["sdp"]}
262
263
264def transcript_message(lines):
265 rendered = []
266 for line in lines:
267 text = str(line.get("text", "")).strip()
268 if text:
269 who = "Assistant" if line.get("speaker") == "assistant" else "User"
270 rendered.append(f"{who}: {text}")
271 return "Spoken conversation since the last request, newest last:\n" + "\n".join(
272 rendered
273 )
274
275
276def iter_sse(response):
277 """Yield (event, data) pairs from a text/event-stream response."""
278 event, data = None, []
279 for raw in response:
280 line = raw.decode("utf-8", "replace").rstrip("\r\n")
281 if not line:
282 if data:
283 yield event, "\n".join(data)
284 event, data = None, []
285 elif line.startswith("event:"):
286 event = line[6:].strip()
287 elif line.startswith("data:"):
288 data.append(line[5:].lstrip())
289 if data:
290 yield event, "\n".join(data)
291
292
293def hermes_request(config, method, path, payload=None, timeout=30):
294 """Open a request to the Hermes API server (the caller closes it)."""
295 request = urllib.request.Request(
296 f"{config.hermes_url}{path}",
297 data=None if payload is None else json.dumps(payload).encode(),
298 method=method,
299 headers={
300 "Authorization": f"Bearer {config.hermes_key}",
301 "Content-Type": "application/json",
302 },
303 )
304 return urllib.request.urlopen(request, timeout=timeout)
305
306
307def hermes_call(config, method, path, payload=None):
308 with hermes_request(config, method, path, payload) as response:
309 return json.load(response)
310
311
312def hermes_start_run(config, hermes_session, lines, hung_up=False):
313 """Start one Hermes turn as a run; return its run id. Hermes queues it
314 behind a run of the same session that is still going."""
315 # A plain string input: Hermes then loads the history of session_id itself.
316 return hermes_call(
317 config,
318 "POST",
319 "/v1/runs",
320 {
321 "input": transcript_message(lines),
322 "instructions": config.backend_note(hung_up),
323 "session_id": hermes_session,
324 },
325 )["run_id"]
326
327
328def hermes_follow_run(config, run_id):
329 """Follow a run's events; yield NDJSON-ready dicts (tool, delta, approval,
330 end_call, done, error). Leaving early only detaches: the run keeps going."""
331 with hermes_request(
332 config, "GET", f"/v1/runs/{run_id}/events", timeout=EVENTS_READ_TIMEOUT
333 ) as response:
334 for _, data in iter_sse(response):
335 payload = json.loads(data)
336 event = payload.get("event", "")
337 if payload.get("tool") == END_CALL_TOOL:
338 # The plugin's tool fails outside a voice session; the page
339 # closes the call after speaking the reply.
340 if event == "tool.completed" and not payload.get("error"):
341 yield {"type": "end_call"}
342 elif event in ("tool.started", "tool.completed"):
343 status = "running" if event == "tool.started" else "completed"
344 if payload.get("error"):
345 status = "error"
346 yield {"type": "tool", "status": status, "label": payload.get("tool")}
347 elif event == "approval.request":
348 # The run blocks until POST /v1/runs/{run_id}/approval answers.
349 # Under the gateway's session authority the request is named
350 # prompt_id and the answer must carry its execution_generation.
351 yield {
352 "type": "approval",
353 "request_id": payload.get("request_id") or payload.get("prompt_id"),
354 "execution_generation": payload.get("execution_generation"),
355 "command": payload.get("command"),
356 "description": payload.get("description"),
357 "timeout_s": payload.get("timeout_s"),
358 }
359 elif event == "run.completed":
360 # Only the final answer: message.delta also streams commentary.
361 yield {"type": "delta", "text": payload.get("output") or ""}
362 yield {"type": "done", "finish_reason": "stop"}
363 return
364 elif event in ("run.failed", "run.cancelled", "run.interrupted"):
365 status = event.removeprefix("run.")
366 error = payload.get("error") or f"Hermes run {status}"
367 yield {"type": "done", "finish_reason": status, "error": error}
368 return
369 yield {"type": "error", "message": "Hermes event stream ended early"}
370
371
372def hermes_approve(config, run_id, approval, choice):
373 """Answer a pending approval of a Hermes run."""
374 answer = {"choice": choice}
375 if approval.get("request_id"):
376 answer["request_id"] = approval["request_id"]
377 if approval.get("execution_generation") is not None:
378 answer["execution_generation"] = approval["execution_generation"]
379 return hermes_call(config, "POST", f"/v1/runs/{run_id}/approval", answer)
380
381
382def notify_matrix(config, text):
383 """Post text to the owner's Matrix DM (the gateway's Matrix home room)
384 through the gateway's webhook platform (a deliver_only route, signed with
385 the API server key)."""
386 body = json.dumps({"text": text}).encode()
387 stamp = str(int(time.time()))
388 signature = hmac.new(
389 config.hermes_key.encode(), stamp.encode() + b"." + body, hashlib.sha256
390 ).hexdigest()
391 request = urllib.request.Request(
392 config.matrix_webhook,
393 data=body,
394 method="POST",
395 headers={
396 "Content-Type": "application/json",
397 "X-Webhook-Timestamp": stamp,
398 "X-Webhook-Signature-V2": signature,
399 },
400 )
401 with urllib.request.urlopen(request, timeout=30):
402 pass
403
404
405# Live runs by run id, and the latest run per Hermes session, which a
406# correction in the same call replaces.
407RUNS = {}
408LATEST = {}
409RUNS_LOCK = threading.Lock()
410# Pending approvals by short code: the page and the Matrix command answer
411# with the code.
412APPROVALS = {}
413APPROVALS_LOCK = threading.Lock()
414
415
416class Run:
417 """A Hermes run this server follows to its end, page or no page. Pages
418 replay its items; an approval nobody can hear goes to Matrix."""
419
420 def __init__(self, config, hermes_session, run_id, hung_up=False):
421 self.config = config
422 self.hermes_session = hermes_session
423 self.run_id = run_id
424 self.items = []
425 self.finished = False
426 self.pages = 0
427 self.hung_up = hung_up
428 self.cond = threading.Condition()
429 with RUNS_LOCK:
430 RUNS[run_id] = self
431 if not hung_up:
432 LATEST[hermes_session] = self
433 threading.Thread(target=self.follow, daemon=True).start()
434
435 def follow(self):
436 started = time.monotonic()
437 tools = 0
438 try:
439 for item in hermes_follow_run(self.config, self.run_id):
440 if item["type"] == "tool" and item["status"] == "running":
441 tools += 1
442 self.add(item)
443 except urllib.error.HTTPError as error:
444 detail = error.read().decode("utf-8", "replace")[:400]
445 self.add({"type": "error", "message": f"Hermes {error.code}: {detail}"})
446 except (urllib.error.URLError, OSError, ValueError) as error:
447 self.add({"type": "error", "message": f"Hermes unreachable: {error}"})
448 settle_approvals(self)
449 with self.cond:
450 self.finished = True
451 self.cond.notify_all()
452 with RUNS_LOCK:
453 RUNS.pop(self.run_id, None)
454 if LATEST.get(self.hermes_session) is self:
455 del LATEST[self.hermes_session]
456 last = self.items[-1] if self.items else {}
457 log.info(
458 "run %s ended (%s) after %.1fs, %d tool calls",
459 self.run_id,
460 last.get("finish_reason") or last.get("message"),
461 time.monotonic() - started,
462 tools,
463 )
464
465 def add(self, item):
466 # Anything after an approval request means Hermes moved on (answered,
467 # or its own timeout).
468 settle_approvals(self)
469 if item["type"] == "approval":
470 item["code"] = open_approval(self, item)
471 with self.cond:
472 self.items.append(item)
473 self.cond.notify_all()
474 unheard = not self.pages or self.hung_up
475 if item["type"] == "end_call":
476 # Without a page the call has ended already.
477 log.info("run %s ends the call%s", self.run_id, " (no call)" if unheard else "")
478 self.ask_unheard()
479
480 def attach(self):
481 with self.cond:
482 self.pages += 1
483
484 def detach(self):
485 with self.cond:
486 self.pages -= 1
487 self.ask_unheard()
488
489 def hang_up(self):
490 with self.cond:
491 self.hung_up = True
492 self.ask_unheard()
493
494 def ask_unheard(self):
495 """Send a pending approval to Matrix once no call can answer it."""
496 with self.cond:
497 if self.pages and not self.hung_up:
498 return
499 with APPROVALS_LOCK:
500 unasked = [
501 (code, item)
502 for code, (run, item) in APPROVALS.items()
503 if run is self and not item.get("asked")
504 ]
505 for _, item in unasked:
506 item["asked"] = True
507 for code, item in unasked:
508 ask_matrix(self.config, code, item)
509
510
511def open_approval(run, item):
512 with APPROVALS_LOCK:
513 code = secrets.token_hex(2)
514 while code in APPROVALS:
515 code = secrets.token_hex(2)
516 APPROVALS[code] = (run, item)
517 return code
518
519
520def settle_approvals(run):
521 with APPROVALS_LOCK:
522 for code in [code for code, (owner, _) in APPROVALS.items() if owner is run]:
523 del APPROVALS[code]
524
525
526def ask_matrix(config, code, item):
527 if not config.matrix_webhook:
528 log.info("approval %s has no listener and no Matrix webhook", code)
529 return
530 text = (
531 "Hermes needs your approval for work from a voice call: "
532 + (item.get("description") or "run a command")
533 )
534 if item.get("command"):
535 text += f"\n```\n{item['command']}\n```"
536 text += f"\nAnswer with `/assist approve {code}` or `/assist deny {code}`."
537 if item.get("timeout_s"):
538 text += f" Unanswered, it is denied after {round(item['timeout_s'] / 60)} minutes."
539 try:
540 notify_matrix(config, text)
541 log.info("approval %s asked in Matrix", code)
542 except (urllib.error.URLError, OSError) as error:
543 log.warning("asking approval %s in Matrix failed: %s", code, error)
544
545
546def stop_run(config, run_id):
547 """POST /v1/runs/{run_id}/stop; failures are only logged."""
548 try:
549 hermes_call(config, "POST", f"/v1/runs/{run_id}/stop")
550 log.info("stopped run %s", run_id)
551 except (urllib.error.URLError, OSError, ValueError) as error:
552 log.warning("stopping run %s failed: %s", run_id, error)
553
554
555class Handler(BaseHTTPRequestHandler):
556 server_version = "hermes-voice"
557 config: Config
558 static: Path
559 access_token: "str | None" = None
560
561 def authorized(self, also_hermes_key=False):
562 """Check the page's bearer token (or, where allowed, the API server key
563 the hermes-voice plugin in the gateway holds); answer 401 when it is
564 wrong."""
565 if self.access_token is None:
566 return True
567 header = self.headers.get("Authorization", "")
568 scheme, _, token = header.partition(" ")
569 accepted = [self.access_token]
570 if also_hermes_key:
571 accepted.append(self.config.hermes_key)
572 if scheme == "Bearer" and any(
573 hmac.compare_digest(token.encode(), key.encode()) for key in accepted
574 ):
575 return True
576 self.send_json(HTTPStatus.UNAUTHORIZED, {"error": "access token required"})
577 return False
578
579 def log_message(self, format, *args):
580 log.info("%s %s", self.address_string(), format % args)
581
582 def send_json(self, status, payload):
583 body = json.dumps(payload).encode()
584 self.send_response(status)
585 self.send_header("Content-Type", "application/json")
586 self.send_header("Content-Length", str(len(body)))
587 self.send_header("Cache-Control", "no-store")
588 self.end_headers()
589 self.wfile.write(body)
590
591 def read_json(self):
592 length = int(self.headers.get("Content-Length") or 0)
593 if length <= 0 or length > MAX_BODY:
594 raise ValueError("bad body length")
595 return json.loads(self.rfile.read(length))
596
597 def do_GET(self):
598 if self.path == "/config":
599 if not self.authorized():
600 return
601 self.send_json(
602 HTTPStatus.OK,
603 {
604 "model": self.config.model,
605 "voice": self.config.voice,
606 "voices": self.config.voices(),
607 "idle_seconds": self.config.idle_seconds,
608 },
609 )
610 return
611 names = {
612 "/": ("index.html", "text/html; charset=utf-8"),
613 "/app.js": ("app.js", "text/javascript; charset=utf-8"),
614 "/style.css": ("style.css", "text/css; charset=utf-8"),
615 "/manifest.webmanifest": ("manifest.webmanifest", "application/manifest+json"),
616 "/icon-192.png": ("icon-192.png", "image/png"),
617 "/icon-512.png": ("icon-512.png", "image/png"),
618 # Missing until the Android signing key exists.
619 "/.well-known/assetlinks.json": (".well-known/assetlinks.json", "application/json"),
620 }
621 if self.path not in names or not (self.static / names[self.path][0]).is_file():
622 self.send_error(HTTPStatus.NOT_FOUND)
623 return
624 name, content_type = names[self.path]
625 body = (self.static / name).read_bytes()
626 self.send_response(HTTPStatus.OK)
627 self.send_header("Content-Type", content_type)
628 self.send_header("Content-Length", str(len(body)))
629 self.send_header("Cache-Control", "no-store")
630 self.end_headers()
631 self.wfile.write(body)
632
633 def do_POST(self):
634 if not self.authorized(also_hermes_key=self.path == "/approve"):
635 return
636 try:
637 request = self.read_json()
638 except ValueError:
639 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid JSON body"})
640 return
641 if self.path == "/session":
642 self.handle_session(request)
643 elif self.path == "/delegate":
644 self.handle_delegate(request)
645 elif self.path == "/hangup":
646 self.handle_hangup(request)
647 elif self.path == "/approve":
648 self.handle_approve(request)
649 else:
650 self.send_error(HTTPStatus.NOT_FOUND)
651
652 def handle_session(self, request):
653 # The vendor's SDP parser needs the offer byte-exact (trailing CRLF too).
654 sdp = request.get("sdp")
655 if not isinstance(sdp, str) or not sdp.strip():
656 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "sdp offer required"})
657 return
658 try:
659 voice = request.get("voice") or self.config.voice
660 if voice not in self.config.voices():
661 self.send_json(HTTPStatus.BAD_REQUEST, {"error": f"unknown voice {voice!r}"})
662 return
663 result = create_live_session(
664 self.config, sdp, live_history(request.get("history") or []), voice
665 )
666 except urllib.error.HTTPError as error:
667 detail = error.read().decode("utf-8", "replace")[:800]
668 log.warning("live session creation failed: %s %s", error.code, detail)
669 self.send_json(
670 HTTPStatus.BAD_GATEWAY,
671 {"error": f"OpenAI rejected the session ({error.code}): {detail}"},
672 )
673 return
674 except (urllib.error.URLError, OSError, KeyError, ValueError) as error:
675 log.warning("live session creation failed: %s", error)
676 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"OpenAI: {error}"})
677 return
678 self.send_json(HTTPStatus.OK, result)
679
680 def handle_delegate(self, request):
681 hermes_session = request.get("hermes_session")
682 lines = request.get("lines")
683 if not isinstance(hermes_session, str) or not SESSION_ID_RE.match(hermes_session):
684 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid hermes_session"})
685 return
686 if not isinstance(lines, list) or not lines:
687 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "lines required"})
688 return
689 # A correction within the same call (the page says so) supersedes and
690 # stops the run it replaces; runs from earlier calls keep going.
691 if request.get("replace") is True:
692 with RUNS_LOCK:
693 previous = LATEST.pop(hermes_session, None)
694 if previous:
695 stop_run(self.config, previous.run_id)
696 try:
697 run = Run(
698 self.config,
699 hermes_session,
700 hermes_start_run(self.config, hermes_session, lines),
701 )
702 except urllib.error.HTTPError as error:
703 detail = error.read().decode("utf-8", "replace")[:400]
704 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
705 return
706 except (urllib.error.URLError, OSError, ValueError, KeyError) as error:
707 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
708 return
709 self.send_response(HTTPStatus.OK)
710 self.send_header("Content-Type", "application/x-ndjson")
711 self.send_header("Cache-Control", "no-store")
712 self.send_header("X-Accel-Buffering", "no")
713 self.end_headers()
714 run.attach()
715 try:
716 sent = 0
717 while True:
718 with run.cond:
719 if sent == len(run.items) and not run.finished:
720 run.cond.wait(PAGE_KEEPALIVE_SECONDS)
721 new = run.items[sent:]
722 sent += len(new)
723 done = run.finished and sent == len(run.items)
724 for item in new:
725 self.write_line(item)
726 if done:
727 break
728 if not new:
729 self.wfile.write(b"\n")
730 self.wfile.flush()
731 except (BrokenPipeError, ConnectionResetError):
732 # Only the page is gone: this server keeps following the run.
733 log.info("page left run %s", run.run_id)
734 finally:
735 run.detach()
736
737 def handle_hangup(self, request):
738 """The call ended. No page hears this conversation's runs any more, and
739 what the user said since the last delegation becomes its own run."""
740 hermes_session = request.get("hermes_session")
741 lines = request.get("lines") or []
742 if (
743 not isinstance(hermes_session, str)
744 or not SESSION_ID_RE.match(hermes_session)
745 or not isinstance(lines, list)
746 ):
747 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid hang-up"})
748 return
749 with RUNS_LOCK:
750 LATEST.pop(hermes_session, None)
751 runs = [run for run in RUNS.values() if run.hermes_session == hermes_session]
752 for run in runs:
753 run.hang_up()
754 if not any(line.get("speaker") == "user" for line in lines):
755 self.send_json(HTTPStatus.OK, {"run_id": None})
756 return
757 try:
758 run_id = hermes_start_run(self.config, hermes_session, lines, hung_up=True)
759 except urllib.error.HTTPError as error:
760 detail = error.read().decode("utf-8", "replace")[:400]
761 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
762 return
763 except (urllib.error.URLError, OSError, ValueError, KeyError) as error:
764 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
765 return
766 Run(self.config, hermes_session, run_id, hung_up=True)
767 log.info("hang-up of %s left a request: run %s", hermes_session, run_id)
768 self.send_json(HTTPStatus.OK, {"run_id": run_id})
769
770 def handle_approve(self, request):
771 code = request.get("code")
772 choice = request.get("choice")
773 # No "always": a permanent allowlist entry is not a voice decision.
774 if choice not in ("once", "session", "deny") or not (
775 code is None or isinstance(code, str)
776 ):
777 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid approval answer"})
778 return
779 with APPROVALS_LOCK:
780 if code is None and len(APPROVALS) == 1:
781 code = next(iter(APPROVALS))
782 pending = APPROVALS.get(code)
783 if pending is None:
784 self.send_json(
785 HTTPStatus.CONFLICT,
786 {"error": f"no pending approval {code}" if code else "no single pending approval"},
787 )
788 return
789 run, item = pending
790 try:
791 result = hermes_approve(self.config, run.run_id, item, choice)
792 except urllib.error.HTTPError as error:
793 detail = error.read().decode("utf-8", "replace")[:400]
794 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
795 return
796 except (urllib.error.URLError, OSError, ValueError) as error:
797 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
798 return
799 with APPROVALS_LOCK:
800 APPROVALS.pop(code, None)
801 if result.get("status") == "already_resolved":
802 self.send_json(HTTPStatus.CONFLICT, {"error": f"approval {code} was already answered"})
803 return
804 log.info("approval %s (run %s): %s %s", code, run.run_id, choice, result.get("status"))
805 self.send_json(
806 HTTPStatus.OK,
807 {"code": code, "choice": choice, "description": item.get("description")},
808 )
809
810 def write_line(self, item):
811 self.wfile.write(json.dumps(item).encode() + b"\n")
812 self.wfile.flush()
813
814
815def main():
816 parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
817 parser.add_argument("--listen", default="127.0.0.1:8790", help="host:port to serve on")
818 parser.add_argument("--hermes-url", required=True, help="Hermes API server base URL")
819 parser.add_argument("--hermes-key-file", required=True, help="file holding API_SERVER_KEY")
820 parser.add_argument("--openai-key-file", required=True, help="file holding the OpenAI API key")
821 parser.add_argument("--access-token-file", help="file holding the token the page must present")
822 parser.add_argument(
823 "--matrix-webhook",
824 help="Hermes webhook route (deliver_only to Matrix, signed with API_SERVER_KEY) that "
825 "receives approvals no call can answer",
826 )
827 parser.add_argument("--config", help="JSON file: voice, model, idle_seconds, glossary, instructions")
828 parser.add_argument(
829 "--static",
830 default=os.environ.get("HERMES_VOICE_STATIC"),
831 help="directory with index.html, app.js, style.css (default: $HERMES_VOICE_STATIC)",
832 )
833 args = parser.parse_args()
834 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
835
836 host, _, port = args.listen.rpartition(":")
837 host = host.strip("[]")
838 if args.access_token_file is None and host not in ("127.0.0.1", "::1", "localhost"):
839 parser.error("--access-token-file is required when not listening on loopback")
840
841 raw = json.loads(Path(args.config).read_text()) if args.config else {}
842 config = Config(
843 raw,
844 openai_key=Path(args.openai_key_file).read_text().strip(),
845 hermes_url=args.hermes_url,
846 hermes_key=Path(args.hermes_key_file).read_text().strip(),
847 matrix_webhook=args.matrix_webhook,
848 )
849 Handler.config = config
850 Handler.static = Path(args.static)
851 if args.access_token_file:
852 Handler.access_token = Path(args.access_token_file).read_text().strip()
853 server_class = ThreadingHTTPServer
854 if ":" in host:
855 # "::" is dual-stack on Linux (bindv6only=0).
856 server_class = type("Server6", (ThreadingHTTPServer,), {"address_family": socket.AF_INET6})
857 server = server_class((host, int(port)), Handler)
858 server.daemon_threads = True
859 log.info("serving on http://%s", args.listen)
860 server.serve_forever()
861
862
863if __name__ == "__main__":
864 main()