forked from
lass.pds.lassul.us/superconfig
This repository has no description
39 kB
867 lines
1"""hermes-voice: talk to Hermes through OpenAI's full-duplex gpt-live-1.
2
3The browser holds a WebRTC session with gpt-live-1, which listens and speaks at
4the same time and chats on its own. Whenever the user asks for real work, the
5voice model emits a client delegation; the page hands the recent transcript to
6this server, which starts it as a Hermes run on the API server (POST /v1/runs,
7one Hermes session id per conversation), follows the run's event stream and
8streams the reply back. The page then gives the answer to the voice model,
9which says it aloud.
10
11A run lives independently of any HTTP connection, and this server follows
12every run it started to its end, whether or not a page is still listening. A
13newer delegation within the same call stops the previous run (the user
14corrected or replaced the request); hanging up just ends the call, and the run
15finishes whatever the user asked for. On hang-up the page sends what the user
16said since the last delegation, so a request spoken right before hanging up
17still reaches Hermes. When a run asks for approval while no page is listening,
18the question goes to the owner's Matrix DM (the gateway's Matrix home room,
19through its webhook platform), and the hermes-voice plugin's
20`/assist approve|deny <code>` command answers it.
21
22Hermes ends a call itself: when the user says goodbye, the voice model
23delegates that like any request, and Hermes calls the plugin's end_call tool.
24This server sees the call in the run's events and tells a listening page to
25close the session once the voice has spoken the reply.
26
27This server exists so neither secret reaches the browser: it exchanges the
28page's SDP offer for a Live session with the OpenAI key, and it calls Hermes
29with the API server's bearer token. That token is root-equivalent for the
30Hermes host, so whoever can use this server can drive the agent: every API
31route requires the page's own access token (--access-token-file), and serving
32beyond loopback without one is refused.
33
34Vendor contract: https://developers.openai.com/api/docs/guides/live-delegation
35"""
36
37import argparse
38import hashlib
39import hmac
40import json
41import logging
42import os
43import re
44import secrets
45import socket
46import threading
47import time
48import urllib.error
49import urllib.parse
50import urllib.request
51from http import HTTPStatus
52from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
53from pathlib import Path
54
55log = logging.getLogger("hermes-voice")
56
57OPENAI_BASE_URL = "https://api.openai.com/v1"
58DEFAULT_MODEL = "gpt-live-1"
59DEFAULT_VOICE = "marin"
60# gpt-live-1 voices as OpenAI's voice picker lists them: language, regional
61# influence, presentation and character, natural voices first, then the
62# generated ones. The docs' voice table and the openai SDK (3.28.0) list only
63# part of them; /v1/live/sessions accepts the German, French, Hindi, Japanese
64# and Korean voices as well, while an unknown name is refused with 403.
65BUILT_IN_VOICES = {
66 "gleam": "English, North American, female: warm and conversational",
67 "ripple": "English, Australian, male: warm and conversational",
68 "vesper": "English, British, male: clear and direct",
69 "willow": "English, Irish, female: expressive and bright",
70 "stone": "English, Irish, male: calm and precise",
71 "meridian": "English, North American, male: grounded and measured",
72 "bossa": "Portuguese, Brazilian, female: clear and direct",
73 "tempo": "Portuguese, Brazilian, male: grounded and measured",
74 "cedar": "English, North American, male: open and upbeat",
75 "marin": "English, North American, female: animated and earnest",
76 "nuri": "German, female: warm and animated",
77 "juni": "German, female: bright and expressive",
78 "flitz": "German, male: rich and playful",
79 "brise": "French, female: clear and conversational",
80 "sillage": "French, male: warm and grounded",
81 "nira": "Hindi, female: soft and thoughtful",
82 "harema": "Japanese, female: gentle and engaging",
83 "shida": "Japanese, male: quiet and grounded",
84 "noeul": "Korean, female: airy and expressive",
85 "quartz": "English, Australian, female, generated: warm and conversational",
86 "alloy": "English, North American, male, generated: balanced and clear",
87 "ash": "English, North American, male, generated: savvy and relaxed",
88 "ballad": "English, North American, male, generated: cheerful and candid",
89 "coral": "English, North American, female, generated: easygoing and versatile",
90 "echo": "English, North American, male, generated: bright and inquisitive",
91 "sage": "English, North American, female, generated: confident and optimistic",
92 "shimmer": "English, North American, female, generated: composed and direct",
93 "verse": "English, North American, male, generated: calm and affirming",
94 "beacon": "English, Filipino, male, generated: clear and personable",
95 "delta": "English, Southern U.S., female, generated: warm and easygoing",
96 "cinder": "English, Southern U.S., male, generated: steady and grounded",
97}
98# gpt-live-1 bills every second a session is open, silence included; the page
99# closes the session after this much inactivity.
100DEFAULT_IDLE_SECONDS = 90
101MAX_BODY = 256 * 1024
102# The run event stream sends a keepalive every 10s, so a read that stalls this
103# long means the API server is gone.
104EVENTS_READ_TIMEOUT = 60
105# Same cadence towards the page: a write is the only way to notice that a page
106# went away while a run is quiet (e.g. waiting for an approval).
107PAGE_KEEPALIVE_SECONDS = 10
108SESSION_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
109# Hermes's tool that ends the call (hermes-plugin/__init__.py).
110END_CALL_TOOL = "end_call"
111
112# Frontend persona, adapted from Hermes's own (tools/voice_live.py): role,
113# style and a labelled delegation policy. The backend carries the real
114# instructions, tools and memory.
115PERSONA = """You are Hermes, the user's warm, upbeat and genuinely helpful voice assistant. Sound friendly and glad to help: positive and encouraging, never curt, annoyed, bored or sarcastic, even when delivering bad news or when the user is short with you. Talk fast: speak at a rapid, energetic pace, like someone who talks quickly by nature, with no slow drawn-out words and no long pauses between sentences. Keep replies short; a brief warm acknowledgment before the facts is good.
116
117Backchannel policy: Use moderate backchannels. Acknowledge naturally without competing with the main response.
118
119Interruption policy: Stop speaking when the user interrupts. Listen to what they say.
120
121Delegation policy:
122Backend tools:
123- Hermes agent: a full AI agent with tools. It can run commands on the user's machines, read and edit files, manage the calendar, write and read messages in the user's Matrix chat, browse the web, search, and remember things across sessions. It is the one who actually does work and knows facts.
124
125Delegate to the backend when:
126- The user asks a question that needs facts, current information, or careful reasoning.
127- The user asks you to do, check, find, make, fix, run or remember anything.
128- A correction changes work already requested.
129
130- The user says goodbye, says that is all, asks you to hang up, or says they are leaving. Hermes ends the call after your reply; you may still say a short goodbye, but never end the call yourself.
131
132Do not delegate to the backend when:
133- The user greets you, makes small talk, or asks you to repeat a result already provided.
134- You need a brief clarification to understand the request.
135
136Delegate before giving an answer that depends on backend work. Do not guess the result while waiting; say briefly and cheerfully that you are on it, then wait for the result.
137
138Approval policy:
139Hermes sometimes needs the user's approval before it runs a command. When you are told so, explain in plain words what Hermes wants to do and ask whether to approve or deny it. When the user answers, delegate to the backend so the answer reaches Hermes. Never decide an approval yourself."""
140
141# Ephemeral system prompt for every delegated Hermes turn (Hermes layers system
142# messages on top of its core prompt without storing them).
143BACKEND_NOTE = """This turn is a delegation from a live spoken conversation. The user message is a speech transcript: it may contain mis-hearings, phonetic spellings of technical terms, hesitations and later corrections; use the latest intent. Your reply will be spoken aloud by a friendly voice model that paraphrases it: answer in warm, plain conversational sentences, keep it short (a few sentences unless the user asked for detail), no markdown, no lists, no code blocks, no URLs read out character by character. Do the work with your tools as usual; only the final facts need to be spoken. Report an action as done only after it actually succeeded. The user may hang up before your reply is spoken: when they asked for a result or a question somewhere else, such as their Matrix chat (matrix_dm_send), send it there yourself. When the user says goodbye, says that is all, asks to hang up or says they are leaving, call end_call: only after any work they asked for has been started or handed off (for example a result promised to their Matrix chat), and never while you still need an answer from them, such as an approval. Your reply is still spoken before the call ends, so make it a short confirmation or goodbye."""
144
145# Instead of BACKEND_NOTE for what the user said right before hanging up, when
146# the voice model had not delegated it yet.
147HUNG_UP_NOTE = """This turn comes from a live spoken conversation the user has already hung up. The user message is a speech transcript of what they said since the last request: it may contain mis-hearings, phonetic spellings of technical terms, hesitations and later corrections; use the latest intent. Nobody hears your reply. If the transcript asks for something, do it with your tools; when the user asked for a result somewhere, such as their Matrix chat (matrix_dm_send), send it there, and if you need something from them, ask in their Matrix chat. If it only says goodbye or thanks, or repeats a request already handled in this conversation, do nothing and reply with a short note saying so. The call is over already: do not call end_call."""
148
149
150def glossary_text(glossary):
151 """Glossary entries as prompt lines: `term` or `term (sounds like "…")`."""
152 lines = []
153 for entry in glossary:
154 if isinstance(entry, str):
155 lines.append(f"- {entry}")
156 else:
157 term = entry["term"]
158 spoken = entry.get("spoken")
159 lines.append(f'- {term} (sounds like "{spoken}")' if spoken else f"- {term}")
160 return "\n".join(lines)
161
162
163class Config:
164 def __init__(self, raw, openai_key, hermes_url, hermes_key, matrix_webhook=None):
165 self.matrix_webhook = matrix_webhook
166 self.model = raw.get("model", DEFAULT_MODEL)
167 self.voice = raw.get("voice", DEFAULT_VOICE)
168 self.idle_seconds = int(raw.get("idle_seconds", DEFAULT_IDLE_SECONDS))
169 self.glossary = raw.get("glossary", [])
170 self.extra_instructions = raw.get("instructions", "").strip()
171 self.openai_key = openai_key
172 self.hermes_url = hermes_url.rstrip("/")
173 self.hermes_key = hermes_key
174
175 def live_instructions(self):
176 parts = [PERSONA]
177 if self.glossary:
178 parts.append(
179 "Technical terms the user says often. Understand them when spoken and "
180 "say them like this:\n" + glossary_text(self.glossary)
181 )
182 if self.extra_instructions:
183 parts.append(self.extra_instructions)
184 return "\n\n".join(parts)
185
186 def backend_note(self, hung_up=False):
187 note = HUNG_UP_NOTE if hung_up else BACKEND_NOTE
188 if not self.glossary:
189 return note
190 return (
191 note
192 + "\n\nTerms the transcript may contain as phonetic misspellings:\n"
193 + glossary_text(self.glossary)
194 )
195
196 def voices(self):
197 """Voices the page may pick, name -> description: the built-ins plus
198 the configured one (which may be a custom voice id)."""
199 voices = dict(BUILT_IN_VOICES)
200 voices.setdefault(self.voice, "custom voice")
201 return voices
202
203
204def voice_param(voice):
205 # Custom voices are objects ({"id": "voice_…"}); built-ins are names.
206 if voice.startswith("voice_"):
207 return {"id": voice}
208 return voice
209
210
211def live_history(lines):
212 """Seed history for a new Live session from the page's transcript lines
213 (spoken dialogue only; Hermes answers were spoken through the voice)."""
214 items = []
215 dialogue = [line for line in lines if line.get("speaker") in ("user", "assistant")]
216 for line in dialogue[-24:]:
217 text = str(line.get("text", "")).strip()[:1200]
218 if not text:
219 continue
220 if line["speaker"] == "assistant":
221 content = {"type": "output_text", "text": text}
222 else:
223 content = {"type": "input_text", "text": text}
224 items.append({"type": "message", "role": line["speaker"], "content": [content]})
225 return items
226
227
228def create_live_session(config, sdp, history, voice):
229 session = {
230 "model": config.model,
231 "instructions": config.live_instructions(),
232 "audio": {"output": {"voice": voice_param(voice)}},
233 "delegation": {"type": "client"},
234 # The page is an untrusted frontend: it may only answer delegations and
235 # hang up, not rewrite the session.
236 "client": {
237 "data_channel": {
238 "allowed_client_events": [
239 "session.commentary.append",
240 "session.thinking.append",
241 "session.close",
242 ]
243 }
244 },
245 }
246 if history:
247 session["input"] = history
248 body = json.dumps(
249 {"session": session, "transport": {"type": "webrtc", "sdp": sdp}}
250 ).encode()
251 request = urllib.request.Request(
252 f"{OPENAI_BASE_URL}/live/sessions",
253 data=body,
254 method="POST",
255 headers={
256 "Authorization": f"Bearer {config.openai_key}",
257 "Content-Type": "application/json",
258 },
259 )
260 with urllib.request.urlopen(request, timeout=30) as response:
261 answer = json.load(response)
262 return {"session_id": answer["session"]["id"], "sdp": answer["transport"]["sdp"]}
263
264
265def transcript_message(lines):
266 rendered = []
267 for line in lines:
268 text = str(line.get("text", "")).strip()
269 if text:
270 who = "Assistant" if line.get("speaker") == "assistant" else "User"
271 rendered.append(f"{who}: {text}")
272 return "Spoken conversation since the last request, newest last:\n" + "\n".join(
273 rendered
274 )
275
276
277def iter_sse(response):
278 """Yield (event, data) pairs from a text/event-stream response."""
279 event, data = None, []
280 for raw in response:
281 line = raw.decode("utf-8", "replace").rstrip("\r\n")
282 if not line:
283 if data:
284 yield event, "\n".join(data)
285 event, data = None, []
286 elif line.startswith("event:"):
287 event = line[6:].strip()
288 elif line.startswith("data:"):
289 data.append(line[5:].lstrip())
290 if data:
291 yield event, "\n".join(data)
292
293
294def hermes_request(config, method, path, payload=None, timeout=30):
295 """Open a request to the Hermes API server (the caller closes it)."""
296 request = urllib.request.Request(
297 f"{config.hermes_url}{path}",
298 data=None if payload is None else json.dumps(payload).encode(),
299 method=method,
300 headers={
301 "Authorization": f"Bearer {config.hermes_key}",
302 "Content-Type": "application/json",
303 },
304 )
305 return urllib.request.urlopen(request, timeout=timeout)
306
307
308def hermes_call(config, method, path, payload=None):
309 with hermes_request(config, method, path, payload) as response:
310 return json.load(response)
311
312
313def hermes_start_run(config, hermes_session, lines, hung_up=False):
314 """Start one Hermes turn as a run; return its run id. Hermes queues it
315 behind a run of the same session that is still going."""
316 # A plain string input: Hermes then loads the history of session_id itself.
317 return hermes_call(
318 config,
319 "POST",
320 "/v1/runs",
321 {
322 "input": transcript_message(lines),
323 "instructions": config.backend_note(hung_up),
324 "session_id": hermes_session,
325 },
326 )["run_id"]
327
328
329def hermes_follow_run(config, run_id):
330 """Follow a run's events; yield NDJSON-ready dicts (tool, delta, approval,
331 end_call, done, error). Leaving early only detaches: the run keeps going."""
332 with hermes_request(
333 config, "GET", f"/v1/runs/{run_id}/events", timeout=EVENTS_READ_TIMEOUT
334 ) as response:
335 for _, data in iter_sse(response):
336 payload = json.loads(data)
337 event = payload.get("event", "")
338 if payload.get("tool") == END_CALL_TOOL:
339 # The plugin's tool fails outside a voice session; the page
340 # closes the call after speaking the reply.
341 if event == "tool.completed" and not payload.get("error"):
342 yield {"type": "end_call"}
343 elif event in ("tool.started", "tool.completed"):
344 status = "running" if event == "tool.started" else "completed"
345 if payload.get("error"):
346 status = "error"
347 yield {"type": "tool", "status": status, "label": payload.get("tool")}
348 elif event == "approval.request":
349 # The run blocks until POST /v1/runs/{run_id}/approval answers.
350 # Under the gateway's session authority the request is named
351 # prompt_id and the answer must carry its execution_generation.
352 yield {
353 "type": "approval",
354 "request_id": payload.get("request_id") or payload.get("prompt_id"),
355 "execution_generation": payload.get("execution_generation"),
356 "command": payload.get("command"),
357 "description": payload.get("description"),
358 "timeout_s": payload.get("timeout_s"),
359 }
360 elif event == "run.completed":
361 # Only the final answer: message.delta also streams commentary.
362 yield {"type": "delta", "text": payload.get("output") or ""}
363 yield {"type": "done", "finish_reason": "stop"}
364 return
365 elif event in ("run.failed", "run.cancelled", "run.interrupted"):
366 status = event.removeprefix("run.")
367 error = payload.get("error") or f"Hermes run {status}"
368 yield {"type": "done", "finish_reason": status, "error": error}
369 return
370 yield {"type": "error", "message": "Hermes event stream ended early"}
371
372
373def hermes_approve(config, run_id, approval, choice):
374 """Answer a pending approval of a Hermes run."""
375 answer = {"choice": choice}
376 if approval.get("request_id"):
377 answer["request_id"] = approval["request_id"]
378 if approval.get("execution_generation") is not None:
379 answer["execution_generation"] = approval["execution_generation"]
380 return hermes_call(config, "POST", f"/v1/runs/{run_id}/approval", answer)
381
382
383def notify_matrix(config, text):
384 """Post text to the owner's Matrix DM (the gateway's Matrix home room)
385 through the gateway's webhook platform (a deliver_only route, signed with
386 the API server key)."""
387 body = json.dumps({"text": text}).encode()
388 stamp = str(int(time.time()))
389 signature = hmac.new(
390 config.hermes_key.encode(), stamp.encode() + b"." + body, hashlib.sha256
391 ).hexdigest()
392 request = urllib.request.Request(
393 config.matrix_webhook,
394 data=body,
395 method="POST",
396 headers={
397 "Content-Type": "application/json",
398 "X-Webhook-Timestamp": stamp,
399 "X-Webhook-Signature-V2": signature,
400 },
401 )
402 with urllib.request.urlopen(request, timeout=30):
403 pass
404
405
406# Live runs by run id, and the latest run per Hermes session, which a
407# correction in the same call replaces.
408RUNS = {}
409LATEST = {}
410RUNS_LOCK = threading.Lock()
411# Pending approvals by short code: the page and the Matrix command answer
412# with the code.
413APPROVALS = {}
414APPROVALS_LOCK = threading.Lock()
415
416
417class Run:
418 """A Hermes run this server follows to its end, page or no page. Pages
419 replay its items; an approval nobody can hear goes to Matrix."""
420
421 def __init__(self, config, hermes_session, run_id, hung_up=False):
422 self.config = config
423 self.hermes_session = hermes_session
424 self.run_id = run_id
425 self.items = []
426 self.finished = False
427 self.pages = 0
428 self.hung_up = hung_up
429 self.cond = threading.Condition()
430 with RUNS_LOCK:
431 RUNS[run_id] = self
432 if not hung_up:
433 LATEST[hermes_session] = self
434 threading.Thread(target=self.follow, daemon=True).start()
435
436 def follow(self):
437 started = time.monotonic()
438 tools = 0
439 try:
440 for item in hermes_follow_run(self.config, self.run_id):
441 if item["type"] == "tool" and item["status"] == "running":
442 tools += 1
443 self.add(item)
444 except urllib.error.HTTPError as error:
445 detail = error.read().decode("utf-8", "replace")[:400]
446 self.add({"type": "error", "message": f"Hermes {error.code}: {detail}"})
447 except (urllib.error.URLError, OSError, ValueError) as error:
448 self.add({"type": "error", "message": f"Hermes unreachable: {error}"})
449 settle_approvals(self)
450 with self.cond:
451 self.finished = True
452 self.cond.notify_all()
453 with RUNS_LOCK:
454 RUNS.pop(self.run_id, None)
455 if LATEST.get(self.hermes_session) is self:
456 del LATEST[self.hermes_session]
457 last = self.items[-1] if self.items else {}
458 log.info(
459 "run %s ended (%s) after %.1fs, %d tool calls",
460 self.run_id,
461 last.get("finish_reason") or last.get("message"),
462 time.monotonic() - started,
463 tools,
464 )
465
466 def add(self, item):
467 # Anything after an approval request means Hermes moved on (answered,
468 # or its own timeout).
469 settle_approvals(self)
470 if item["type"] == "approval":
471 item["code"] = open_approval(self, item)
472 with self.cond:
473 self.items.append(item)
474 self.cond.notify_all()
475 unheard = not self.pages or self.hung_up
476 if item["type"] == "end_call":
477 # Without a page the call has ended already.
478 log.info("run %s ends the call%s", self.run_id, " (no call)" if unheard else "")
479 self.ask_unheard()
480
481 def attach(self):
482 with self.cond:
483 self.pages += 1
484
485 def detach(self):
486 with self.cond:
487 self.pages -= 1
488 self.ask_unheard()
489
490 def hang_up(self):
491 with self.cond:
492 self.hung_up = True
493 self.ask_unheard()
494
495 def ask_unheard(self):
496 """Send a pending approval to Matrix once no call can answer it."""
497 with self.cond:
498 if self.pages and not self.hung_up:
499 return
500 with APPROVALS_LOCK:
501 unasked = [
502 (code, item)
503 for code, (run, item) in APPROVALS.items()
504 if run is self and not item.get("asked")
505 ]
506 for _, item in unasked:
507 item["asked"] = True
508 for code, item in unasked:
509 ask_matrix(self.config, code, item)
510
511
512def open_approval(run, item):
513 with APPROVALS_LOCK:
514 code = secrets.token_hex(2)
515 while code in APPROVALS:
516 code = secrets.token_hex(2)
517 APPROVALS[code] = (run, item)
518 return code
519
520
521def settle_approvals(run):
522 with APPROVALS_LOCK:
523 for code in [code for code, (owner, _) in APPROVALS.items() if owner is run]:
524 del APPROVALS[code]
525
526
527def ask_matrix(config, code, item):
528 if not config.matrix_webhook:
529 log.info("approval %s has no listener and no Matrix webhook", code)
530 return
531 text = (
532 "Hermes needs your approval for work from a voice call: "
533 + (item.get("description") or "run a command")
534 )
535 if item.get("command"):
536 text += f"\n```\n{item['command']}\n```"
537 text += f"\nAnswer with `/assist approve {code}` or `/assist deny {code}`."
538 if item.get("timeout_s"):
539 text += f" Unanswered, it is denied after {round(item['timeout_s'] / 60)} minutes."
540 try:
541 notify_matrix(config, text)
542 log.info("approval %s asked in Matrix", code)
543 except (urllib.error.URLError, OSError) as error:
544 log.warning("asking approval %s in Matrix failed: %s", code, error)
545
546
547def stop_run(config, run_id):
548 """POST /v1/runs/{run_id}/stop; failures are only logged."""
549 try:
550 hermes_call(config, "POST", f"/v1/runs/{run_id}/stop")
551 log.info("stopped run %s", run_id)
552 except (urllib.error.URLError, OSError, ValueError) as error:
553 log.warning("stopping run %s failed: %s", run_id, error)
554
555
556class Handler(BaseHTTPRequestHandler):
557 server_version = "hermes-voice"
558 config: Config
559 static: Path
560 access_token: "str | None" = None
561
562 def authorized(self, also_hermes_key=False):
563 """Check the page's bearer token (or, where allowed, the API server key
564 the hermes-voice plugin in the gateway holds); answer 401 when it is
565 wrong."""
566 if self.access_token is None:
567 return True
568 header = self.headers.get("Authorization", "")
569 scheme, _, token = header.partition(" ")
570 accepted = [self.access_token]
571 if also_hermes_key:
572 accepted.append(self.config.hermes_key)
573 if scheme == "Bearer" and any(
574 hmac.compare_digest(token.encode(), key.encode()) for key in accepted
575 ):
576 return True
577 self.send_json(HTTPStatus.UNAUTHORIZED, {"error": "access token required"})
578 return False
579
580 def log_message(self, format, *args):
581 log.info("%s %s", self.address_string(), format % args)
582
583 def send_json(self, status, payload):
584 body = json.dumps(payload).encode()
585 self.send_response(status)
586 self.send_header("Content-Type", "application/json")
587 self.send_header("Content-Length", str(len(body)))
588 self.send_header("Cache-Control", "no-store")
589 self.end_headers()
590 self.wfile.write(body)
591
592 def read_json(self):
593 length = int(self.headers.get("Content-Length") or 0)
594 if length <= 0 or length > MAX_BODY:
595 raise ValueError("bad body length")
596 return json.loads(self.rfile.read(length))
597
598 def do_GET(self):
599 if self.path == "/config":
600 if not self.authorized():
601 return
602 self.send_json(
603 HTTPStatus.OK,
604 {
605 "model": self.config.model,
606 "voice": self.config.voice,
607 "voices": self.config.voices(),
608 "idle_seconds": self.config.idle_seconds,
609 },
610 )
611 return
612 names = {
613 "/": ("index.html", "text/html; charset=utf-8"),
614 "/app.js": ("app.js", "text/javascript; charset=utf-8"),
615 "/style.css": ("style.css", "text/css; charset=utf-8"),
616 "/manifest.webmanifest": ("manifest.webmanifest", "application/manifest+json"),
617 "/icon-192.png": ("icon-192.png", "image/png"),
618 "/icon-512.png": ("icon-512.png", "image/png"),
619 # Missing until the Android signing key exists.
620 "/.well-known/assetlinks.json": (".well-known/assetlinks.json", "application/json"),
621 }
622 # The page takes query parameters (/?listen from the Android app).
623 path = urllib.parse.urlsplit(self.path).path
624 if path not in names or not (self.static / names[path][0]).is_file():
625 self.send_error(HTTPStatus.NOT_FOUND)
626 return
627 name, content_type = names[path]
628 body = (self.static / name).read_bytes()
629 self.send_response(HTTPStatus.OK)
630 self.send_header("Content-Type", content_type)
631 self.send_header("Content-Length", str(len(body)))
632 self.send_header("Cache-Control", "no-store")
633 self.end_headers()
634 self.wfile.write(body)
635
636 def do_POST(self):
637 if not self.authorized(also_hermes_key=self.path == "/approve"):
638 return
639 try:
640 request = self.read_json()
641 except ValueError:
642 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid JSON body"})
643 return
644 if self.path == "/session":
645 self.handle_session(request)
646 elif self.path == "/delegate":
647 self.handle_delegate(request)
648 elif self.path == "/hangup":
649 self.handle_hangup(request)
650 elif self.path == "/approve":
651 self.handle_approve(request)
652 else:
653 self.send_error(HTTPStatus.NOT_FOUND)
654
655 def handle_session(self, request):
656 # The vendor's SDP parser needs the offer byte-exact (trailing CRLF too).
657 sdp = request.get("sdp")
658 if not isinstance(sdp, str) or not sdp.strip():
659 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "sdp offer required"})
660 return
661 try:
662 voice = request.get("voice") or self.config.voice
663 if voice not in self.config.voices():
664 self.send_json(HTTPStatus.BAD_REQUEST, {"error": f"unknown voice {voice!r}"})
665 return
666 result = create_live_session(
667 self.config, sdp, live_history(request.get("history") or []), voice
668 )
669 except urllib.error.HTTPError as error:
670 detail = error.read().decode("utf-8", "replace")[:800]
671 log.warning("live session creation failed: %s %s", error.code, detail)
672 self.send_json(
673 HTTPStatus.BAD_GATEWAY,
674 {"error": f"OpenAI rejected the session ({error.code}): {detail}"},
675 )
676 return
677 except (urllib.error.URLError, OSError, KeyError, ValueError) as error:
678 log.warning("live session creation failed: %s", error)
679 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"OpenAI: {error}"})
680 return
681 self.send_json(HTTPStatus.OK, result)
682
683 def handle_delegate(self, request):
684 hermes_session = request.get("hermes_session")
685 lines = request.get("lines")
686 if not isinstance(hermes_session, str) or not SESSION_ID_RE.match(hermes_session):
687 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid hermes_session"})
688 return
689 if not isinstance(lines, list) or not lines:
690 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "lines required"})
691 return
692 # A correction within the same call (the page says so) supersedes and
693 # stops the run it replaces; runs from earlier calls keep going.
694 if request.get("replace") is True:
695 with RUNS_LOCK:
696 previous = LATEST.pop(hermes_session, None)
697 if previous:
698 stop_run(self.config, previous.run_id)
699 try:
700 run = Run(
701 self.config,
702 hermes_session,
703 hermes_start_run(self.config, hermes_session, lines),
704 )
705 except urllib.error.HTTPError as error:
706 detail = error.read().decode("utf-8", "replace")[:400]
707 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
708 return
709 except (urllib.error.URLError, OSError, ValueError, KeyError) as error:
710 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
711 return
712 self.send_response(HTTPStatus.OK)
713 self.send_header("Content-Type", "application/x-ndjson")
714 self.send_header("Cache-Control", "no-store")
715 self.send_header("X-Accel-Buffering", "no")
716 self.end_headers()
717 run.attach()
718 try:
719 sent = 0
720 while True:
721 with run.cond:
722 if sent == len(run.items) and not run.finished:
723 run.cond.wait(PAGE_KEEPALIVE_SECONDS)
724 new = run.items[sent:]
725 sent += len(new)
726 done = run.finished and sent == len(run.items)
727 for item in new:
728 self.write_line(item)
729 if done:
730 break
731 if not new:
732 self.wfile.write(b"\n")
733 self.wfile.flush()
734 except (BrokenPipeError, ConnectionResetError):
735 # Only the page is gone: this server keeps following the run.
736 log.info("page left run %s", run.run_id)
737 finally:
738 run.detach()
739
740 def handle_hangup(self, request):
741 """The call ended. No page hears this conversation's runs any more, and
742 what the user said since the last delegation becomes its own run."""
743 hermes_session = request.get("hermes_session")
744 lines = request.get("lines") or []
745 if (
746 not isinstance(hermes_session, str)
747 or not SESSION_ID_RE.match(hermes_session)
748 or not isinstance(lines, list)
749 ):
750 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid hang-up"})
751 return
752 with RUNS_LOCK:
753 LATEST.pop(hermes_session, None)
754 runs = [run for run in RUNS.values() if run.hermes_session == hermes_session]
755 for run in runs:
756 run.hang_up()
757 if not any(line.get("speaker") == "user" for line in lines):
758 self.send_json(HTTPStatus.OK, {"run_id": None})
759 return
760 try:
761 run_id = hermes_start_run(self.config, hermes_session, lines, hung_up=True)
762 except urllib.error.HTTPError as error:
763 detail = error.read().decode("utf-8", "replace")[:400]
764 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
765 return
766 except (urllib.error.URLError, OSError, ValueError, KeyError) as error:
767 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
768 return
769 Run(self.config, hermes_session, run_id, hung_up=True)
770 log.info("hang-up of %s left a request: run %s", hermes_session, run_id)
771 self.send_json(HTTPStatus.OK, {"run_id": run_id})
772
773 def handle_approve(self, request):
774 code = request.get("code")
775 choice = request.get("choice")
776 # No "always": a permanent allowlist entry is not a voice decision.
777 if choice not in ("once", "session", "deny") or not (
778 code is None or isinstance(code, str)
779 ):
780 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid approval answer"})
781 return
782 with APPROVALS_LOCK:
783 if code is None and len(APPROVALS) == 1:
784 code = next(iter(APPROVALS))
785 pending = APPROVALS.get(code)
786 if pending is None:
787 self.send_json(
788 HTTPStatus.CONFLICT,
789 {"error": f"no pending approval {code}" if code else "no single pending approval"},
790 )
791 return
792 run, item = pending
793 try:
794 result = hermes_approve(self.config, run.run_id, item, choice)
795 except urllib.error.HTTPError as error:
796 detail = error.read().decode("utf-8", "replace")[:400]
797 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
798 return
799 except (urllib.error.URLError, OSError, ValueError) as error:
800 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
801 return
802 with APPROVALS_LOCK:
803 APPROVALS.pop(code, None)
804 if result.get("status") == "already_resolved":
805 self.send_json(HTTPStatus.CONFLICT, {"error": f"approval {code} was already answered"})
806 return
807 log.info("approval %s (run %s): %s %s", code, run.run_id, choice, result.get("status"))
808 self.send_json(
809 HTTPStatus.OK,
810 {"code": code, "choice": choice, "description": item.get("description")},
811 )
812
813 def write_line(self, item):
814 self.wfile.write(json.dumps(item).encode() + b"\n")
815 self.wfile.flush()
816
817
818def main():
819 parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
820 parser.add_argument("--listen", default="127.0.0.1:8790", help="host:port to serve on")
821 parser.add_argument("--hermes-url", required=True, help="Hermes API server base URL")
822 parser.add_argument("--hermes-key-file", required=True, help="file holding API_SERVER_KEY")
823 parser.add_argument("--openai-key-file", required=True, help="file holding the OpenAI API key")
824 parser.add_argument("--access-token-file", help="file holding the token the page must present")
825 parser.add_argument(
826 "--matrix-webhook",
827 help="Hermes webhook route (deliver_only to Matrix, signed with API_SERVER_KEY) that "
828 "receives approvals no call can answer",
829 )
830 parser.add_argument("--config", help="JSON file: voice, model, idle_seconds, glossary, instructions")
831 parser.add_argument(
832 "--static",
833 default=os.environ.get("HERMES_VOICE_STATIC"),
834 help="directory with index.html, app.js, style.css (default: $HERMES_VOICE_STATIC)",
835 )
836 args = parser.parse_args()
837 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
838
839 host, _, port = args.listen.rpartition(":")
840 host = host.strip("[]")
841 if args.access_token_file is None and host not in ("127.0.0.1", "::1", "localhost"):
842 parser.error("--access-token-file is required when not listening on loopback")
843
844 raw = json.loads(Path(args.config).read_text()) if args.config else {}
845 config = Config(
846 raw,
847 openai_key=Path(args.openai_key_file).read_text().strip(),
848 hermes_url=args.hermes_url,
849 hermes_key=Path(args.hermes_key_file).read_text().strip(),
850 matrix_webhook=args.matrix_webhook,
851 )
852 Handler.config = config
853 Handler.static = Path(args.static)
854 if args.access_token_file:
855 Handler.access_token = Path(args.access_token_file).read_text().strip()
856 server_class = ThreadingHTTPServer
857 if ":" in host:
858 # "::" is dual-stack on Linux (bindv6only=0).
859 server_class = type("Server6", (ThreadingHTTPServer,), {"address_family": socket.AF_INET6})
860 server = server_class((host, int(port)), Handler)
861 server.daemon_threads = True
862 log.info("serving on http://%s", args.listen)
863 server.serve_forever()
864
865
866if __name__ == "__main__":
867 main()