This repository has no description
40 kB
875 lines
1"""hermes-voice: talk to Hermes through OpenAI's full-duplex gpt-live-1.
2
3The browser holds a WebRTC session with gpt-live-1, which listens and speaks at
4the same time and chats on its own. Whenever the user asks for real work, the
5voice model emits a client delegation; the page hands the recent transcript to
6this server, which starts it as a Hermes run on the API server (POST /v1/runs,
7one Hermes session id per conversation), follows the run's event stream and
8streams the reply back. The page then gives the answer to the voice model,
9which says it aloud.
10
11A run lives independently of any HTTP connection, and this server follows
12every run it started to its end, whether or not a page is still listening. A
13newer delegation within the same call stops the previous run (the user
14corrected or replaced the request); hanging up just ends the call, and the run
15finishes whatever the user asked for. On hang-up the page sends what the user
16said since the last delegation, so a request spoken right before hanging up
17still reaches Hermes. When a run asks for approval while no page is listening,
18the question goes to the owner's Matrix DM (the gateway's Matrix home room,
19through its webhook platform), and the hermes-voice plugin's
20`/assist approve|deny <code>` command answers it.
21
22Hermes ends a call itself: when the user says goodbye, the voice model
23delegates that like any request, and Hermes calls the plugin's end_call tool.
24This server sees the call in the run's events and tells a listening page to
25close the session once the voice has spoken the reply.
26
27This server exists so neither secret reaches the browser: it exchanges the
28page's SDP offer for a Live session with the OpenAI key, and it calls Hermes
29with the API server's bearer token. That token is root-equivalent for the
30Hermes host, so whoever can use this server can drive the agent: every API
31route requires the page's own access token (--access-token-file), and serving
32beyond loopback without one is refused.
33
34The Android app (android/) is a second client of the same routes: it holds
35the call itself (WebRTC, no page) and presents its own token
36(--app-token-file), which the proxy passes through without the web login.
37
38Vendor contract: https://developers.openai.com/api/docs/guides/live-delegation
39"""
40
41import argparse
42import hashlib
43import hmac
44import json
45import logging
46import os
47import re
48import secrets
49import socket
50import threading
51import time
52import urllib.error
53import urllib.parse
54import urllib.request
55from http import HTTPStatus
56from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
57from pathlib import Path
58
59log = logging.getLogger("hermes-voice")
60
61OPENAI_BASE_URL = "https://api.openai.com/v1"
62DEFAULT_MODEL = "gpt-live-1"
63DEFAULT_VOICE = "marin"
64# gpt-live-1 voices as OpenAI's voice picker lists them: language, regional
65# influence, presentation and character, natural voices first, then the
66# generated ones. The docs' voice table and the openai SDK (3.28.0) list only
67# part of them; /v1/live/sessions accepts the German, French, Hindi, Japanese
68# and Korean voices as well, while an unknown name is refused with 403.
69BUILT_IN_VOICES = {
70 "gleam": "English, North American, female: warm and conversational",
71 "ripple": "English, Australian, male: warm and conversational",
72 "vesper": "English, British, male: clear and direct",
73 "willow": "English, Irish, female: expressive and bright",
74 "stone": "English, Irish, male: calm and precise",
75 "meridian": "English, North American, male: grounded and measured",
76 "bossa": "Portuguese, Brazilian, female: clear and direct",
77 "tempo": "Portuguese, Brazilian, male: grounded and measured",
78 "cedar": "English, North American, male: open and upbeat",
79 "marin": "English, North American, female: animated and earnest",
80 "nuri": "German, female: warm and animated",
81 "juni": "German, female: bright and expressive",
82 "flitz": "German, male: rich and playful",
83 "brise": "French, female: clear and conversational",
84 "sillage": "French, male: warm and grounded",
85 "nira": "Hindi, female: soft and thoughtful",
86 "harema": "Japanese, female: gentle and engaging",
87 "shida": "Japanese, male: quiet and grounded",
88 "noeul": "Korean, female: airy and expressive",
89 "quartz": "English, Australian, female, generated: warm and conversational",
90 "alloy": "English, North American, male, generated: balanced and clear",
91 "ash": "English, North American, male, generated: savvy and relaxed",
92 "ballad": "English, North American, male, generated: cheerful and candid",
93 "coral": "English, North American, female, generated: easygoing and versatile",
94 "echo": "English, North American, male, generated: bright and inquisitive",
95 "sage": "English, North American, female, generated: confident and optimistic",
96 "shimmer": "English, North American, female, generated: composed and direct",
97 "verse": "English, North American, male, generated: calm and affirming",
98 "beacon": "English, Filipino, male, generated: clear and personable",
99 "delta": "English, Southern U.S., female, generated: warm and easygoing",
100 "cinder": "English, Southern U.S., male, generated: steady and grounded",
101}
102# gpt-live-1 bills every second a session is open, silence included; the page
103# closes the session after this much inactivity.
104DEFAULT_IDLE_SECONDS = 90
105MAX_BODY = 256 * 1024
106# The run event stream sends a keepalive every 10s, so a read that stalls this
107# long means the API server is gone.
108EVENTS_READ_TIMEOUT = 60
109# Same cadence towards the page: a write is the only way to notice that a page
110# went away while a run is quiet (e.g. waiting for an approval).
111PAGE_KEEPALIVE_SECONDS = 10
112SESSION_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
113# Hermes's tool that ends the call (hermes-plugin/__init__.py).
114END_CALL_TOOL = "end_call"
115
116# Frontend persona, adapted from Hermes's own (tools/voice_live.py): role,
117# style and a labelled delegation policy. The backend carries the real
118# instructions, tools and memory.
119PERSONA = """You are Hermes, the user's warm, upbeat and genuinely helpful voice assistant. Sound friendly and glad to help: positive and encouraging, never curt, annoyed, bored or sarcastic, even when delivering bad news or when the user is short with you. Talk fast: speak at a rapid, energetic pace, like someone who talks quickly by nature, with no slow drawn-out words and no long pauses between sentences. Keep replies short; a brief warm acknowledgment before the facts is good.
120
121Backchannel policy: Use moderate backchannels. Acknowledge naturally without competing with the main response.
122
123Interruption policy: Stop speaking when the user interrupts. Listen to what they say.
124
125Delegation policy:
126Backend tools:
127- Hermes agent: a full AI agent with tools. It can run commands on the user's machines, read and edit files, manage the calendar, write and read messages in the user's Matrix chat, browse the web, search, and remember things across sessions. It is the one who actually does work and knows facts.
128
129Delegate to the backend when:
130- The user asks a question that needs facts, current information, or careful reasoning.
131- The user asks you to do, check, find, make, fix, run or remember anything.
132- A correction changes work already requested.
133
134- The user says goodbye, says that is all, asks you to hang up, or says they are leaving. Hermes ends the call after your reply; you may still say a short goodbye, but never end the call yourself.
135
136Do not delegate to the backend when:
137- The user greets you, makes small talk, or asks you to repeat a result already provided.
138- You need a brief clarification to understand the request.
139
140Delegate before giving an answer that depends on backend work. Do not guess the result while waiting; say briefly and cheerfully that you are on it, then wait for the result.
141
142Approval policy:
143Hermes sometimes needs the user's approval before it runs a command. When you are told so, explain in plain words what Hermes wants to do and ask whether to approve or deny it. When the user answers, delegate to the backend so the answer reaches Hermes. Never decide an approval yourself."""
144
145# Ephemeral system prompt for every delegated Hermes turn (Hermes layers system
146# messages on top of its core prompt without storing them).
147BACKEND_NOTE = """This turn is a delegation from a live spoken conversation. The user message is a speech transcript: it may contain mis-hearings, phonetic spellings of technical terms, hesitations and later corrections; use the latest intent. Your reply will be spoken aloud by a friendly voice model that paraphrases it: answer in warm, plain conversational sentences, keep it short (a few sentences unless the user asked for detail), no markdown, no lists, no code blocks, no URLs read out character by character. Do the work with your tools as usual; only the final facts need to be spoken. Report an action as done only after it actually succeeded. The user may hang up before your reply is spoken: when they asked for a result or a question somewhere else, such as their Matrix chat (matrix_dm_send), send it there yourself. When the user says goodbye, says that is all, asks to hang up or says they are leaving, call end_call: only after any work they asked for has been started or handed off (for example a result promised to their Matrix chat), and never while you still need an answer from them, such as an approval. Your reply is still spoken before the call ends, so make it a short confirmation or goodbye."""
148
149# Instead of BACKEND_NOTE for what the user said right before hanging up, when
150# the voice model had not delegated it yet.
151HUNG_UP_NOTE = """This turn comes from a live spoken conversation the user has already hung up. The user message is a speech transcript of what they said since the last request: it may contain mis-hearings, phonetic spellings of technical terms, hesitations and later corrections; use the latest intent. Nobody hears your reply. If the transcript asks for something, do it with your tools; when the user asked for a result somewhere, such as their Matrix chat (matrix_dm_send), send it there, and if you need something from them, ask in their Matrix chat. If it only says goodbye or thanks, or repeats a request already handled in this conversation, do nothing and reply with a short note saying so. The call is over already: do not call end_call."""
152
153
154def glossary_text(glossary):
155 """Glossary entries as prompt lines: `term` or `term (sounds like "…")`."""
156 lines = []
157 for entry in glossary:
158 if isinstance(entry, str):
159 lines.append(f"- {entry}")
160 else:
161 term = entry["term"]
162 spoken = entry.get("spoken")
163 lines.append(f'- {term} (sounds like "{spoken}")' if spoken else f"- {term}")
164 return "\n".join(lines)
165
166
167class Config:
168 def __init__(self, raw, openai_key, hermes_url, hermes_key, matrix_webhook=None):
169 self.matrix_webhook = matrix_webhook
170 self.model = raw.get("model", DEFAULT_MODEL)
171 self.voice = raw.get("voice", DEFAULT_VOICE)
172 self.idle_seconds = int(raw.get("idle_seconds", DEFAULT_IDLE_SECONDS))
173 self.glossary = raw.get("glossary", [])
174 self.extra_instructions = raw.get("instructions", "").strip()
175 self.openai_key = openai_key
176 self.hermes_url = hermes_url.rstrip("/")
177 self.hermes_key = hermes_key
178
179 def live_instructions(self):
180 parts = [PERSONA]
181 if self.glossary:
182 parts.append(
183 "Technical terms the user says often. Understand them when spoken and "
184 "say them like this:\n" + glossary_text(self.glossary)
185 )
186 if self.extra_instructions:
187 parts.append(self.extra_instructions)
188 return "\n\n".join(parts)
189
190 def backend_note(self, hung_up=False):
191 note = HUNG_UP_NOTE if hung_up else BACKEND_NOTE
192 if not self.glossary:
193 return note
194 return (
195 note
196 + "\n\nTerms the transcript may contain as phonetic misspellings:\n"
197 + glossary_text(self.glossary)
198 )
199
200 def voices(self):
201 """Voices the page may pick, name -> description: the built-ins plus
202 the configured one (which may be a custom voice id)."""
203 voices = dict(BUILT_IN_VOICES)
204 voices.setdefault(self.voice, "custom voice")
205 return voices
206
207
208def voice_param(voice):
209 # Custom voices are objects ({"id": "voice_…"}); built-ins are names.
210 if voice.startswith("voice_"):
211 return {"id": voice}
212 return voice
213
214
215def live_history(lines):
216 """Seed history for a new Live session from the page's transcript lines
217 (spoken dialogue only; Hermes answers were spoken through the voice)."""
218 items = []
219 dialogue = [line for line in lines if line.get("speaker") in ("user", "assistant")]
220 for line in dialogue[-24:]:
221 text = str(line.get("text", "")).strip()[:1200]
222 if not text:
223 continue
224 if line["speaker"] == "assistant":
225 content = {"type": "output_text", "text": text}
226 else:
227 content = {"type": "input_text", "text": text}
228 items.append({"type": "message", "role": line["speaker"], "content": [content]})
229 return items
230
231
232def create_live_session(config, sdp, history, voice):
233 session = {
234 "model": config.model,
235 "instructions": config.live_instructions(),
236 "audio": {"output": {"voice": voice_param(voice)}},
237 "delegation": {"type": "client"},
238 # The page is an untrusted frontend: it may only answer delegations and
239 # hang up, not rewrite the session.
240 "client": {
241 "data_channel": {
242 "allowed_client_events": [
243 "session.commentary.append",
244 "session.thinking.append",
245 "session.close",
246 ]
247 }
248 },
249 }
250 if history:
251 session["input"] = history
252 body = json.dumps(
253 {"session": session, "transport": {"type": "webrtc", "sdp": sdp}}
254 ).encode()
255 request = urllib.request.Request(
256 f"{OPENAI_BASE_URL}/live/sessions",
257 data=body,
258 method="POST",
259 headers={
260 "Authorization": f"Bearer {config.openai_key}",
261 "Content-Type": "application/json",
262 },
263 )
264 with urllib.request.urlopen(request, timeout=30) as response:
265 answer = json.load(response)
266 return {"session_id": answer["session"]["id"], "sdp": answer["transport"]["sdp"]}
267
268
269def transcript_message(lines):
270 rendered = []
271 for line in lines:
272 text = str(line.get("text", "")).strip()
273 if text:
274 who = "Assistant" if line.get("speaker") == "assistant" else "User"
275 rendered.append(f"{who}: {text}")
276 return "Spoken conversation since the last request, newest last:\n" + "\n".join(
277 rendered
278 )
279
280
281def iter_sse(response):
282 """Yield (event, data) pairs from a text/event-stream response."""
283 event, data = None, []
284 for raw in response:
285 line = raw.decode("utf-8", "replace").rstrip("\r\n")
286 if not line:
287 if data:
288 yield event, "\n".join(data)
289 event, data = None, []
290 elif line.startswith("event:"):
291 event = line[6:].strip()
292 elif line.startswith("data:"):
293 data.append(line[5:].lstrip())
294 if data:
295 yield event, "\n".join(data)
296
297
298def hermes_request(config, method, path, payload=None, timeout=30):
299 """Open a request to the Hermes API server (the caller closes it)."""
300 request = urllib.request.Request(
301 f"{config.hermes_url}{path}",
302 data=None if payload is None else json.dumps(payload).encode(),
303 method=method,
304 headers={
305 "Authorization": f"Bearer {config.hermes_key}",
306 "Content-Type": "application/json",
307 },
308 )
309 return urllib.request.urlopen(request, timeout=timeout)
310
311
312def hermes_call(config, method, path, payload=None):
313 with hermes_request(config, method, path, payload) as response:
314 return json.load(response)
315
316
317def hermes_start_run(config, hermes_session, lines, hung_up=False):
318 """Start one Hermes turn as a run; return its run id. Hermes queues it
319 behind a run of the same session that is still going."""
320 # A plain string input: Hermes then loads the history of session_id itself.
321 return hermes_call(
322 config,
323 "POST",
324 "/v1/runs",
325 {
326 "input": transcript_message(lines),
327 "instructions": config.backend_note(hung_up),
328 "session_id": hermes_session,
329 },
330 )["run_id"]
331
332
333def hermes_follow_run(config, run_id):
334 """Follow a run's events; yield NDJSON-ready dicts (tool, delta, approval,
335 end_call, done, error). Leaving early only detaches: the run keeps going."""
336 with hermes_request(
337 config, "GET", f"/v1/runs/{run_id}/events", timeout=EVENTS_READ_TIMEOUT
338 ) as response:
339 for _, data in iter_sse(response):
340 payload = json.loads(data)
341 event = payload.get("event", "")
342 if payload.get("tool") == END_CALL_TOOL:
343 # The plugin's tool fails outside a voice session; the page
344 # closes the call after speaking the reply.
345 if event == "tool.completed" and not payload.get("error"):
346 yield {"type": "end_call"}
347 elif event in ("tool.started", "tool.completed"):
348 status = "running" if event == "tool.started" else "completed"
349 if payload.get("error"):
350 status = "error"
351 yield {"type": "tool", "status": status, "label": payload.get("tool")}
352 elif event == "approval.request":
353 # The run blocks until POST /v1/runs/{run_id}/approval answers.
354 # Under the gateway's session authority the request is named
355 # prompt_id and the answer must carry its execution_generation.
356 yield {
357 "type": "approval",
358 "request_id": payload.get("request_id") or payload.get("prompt_id"),
359 "execution_generation": payload.get("execution_generation"),
360 "command": payload.get("command"),
361 "description": payload.get("description"),
362 "timeout_s": payload.get("timeout_s"),
363 }
364 elif event == "run.completed":
365 # Only the final answer: message.delta also streams commentary.
366 yield {"type": "delta", "text": payload.get("output") or ""}
367 yield {"type": "done", "finish_reason": "stop"}
368 return
369 elif event in ("run.failed", "run.cancelled", "run.interrupted"):
370 status = event.removeprefix("run.")
371 error = payload.get("error") or f"Hermes run {status}"
372 yield {"type": "done", "finish_reason": status, "error": error}
373 return
374 yield {"type": "error", "message": "Hermes event stream ended early"}
375
376
377def hermes_approve(config, run_id, approval, choice):
378 """Answer a pending approval of a Hermes run."""
379 answer = {"choice": choice}
380 if approval.get("request_id"):
381 answer["request_id"] = approval["request_id"]
382 if approval.get("execution_generation") is not None:
383 answer["execution_generation"] = approval["execution_generation"]
384 return hermes_call(config, "POST", f"/v1/runs/{run_id}/approval", answer)
385
386
387def notify_matrix(config, text):
388 """Post text to the owner's Matrix DM (the gateway's Matrix home room)
389 through the gateway's webhook platform (a deliver_only route, signed with
390 the API server key)."""
391 body = json.dumps({"text": text}).encode()
392 stamp = str(int(time.time()))
393 signature = hmac.new(
394 config.hermes_key.encode(), stamp.encode() + b"." + body, hashlib.sha256
395 ).hexdigest()
396 request = urllib.request.Request(
397 config.matrix_webhook,
398 data=body,
399 method="POST",
400 headers={
401 "Content-Type": "application/json",
402 "X-Webhook-Timestamp": stamp,
403 "X-Webhook-Signature-V2": signature,
404 },
405 )
406 with urllib.request.urlopen(request, timeout=30):
407 pass
408
409
410# Live runs by run id, and the latest run per Hermes session, which a
411# correction in the same call replaces.
412RUNS = {}
413LATEST = {}
414RUNS_LOCK = threading.Lock()
415# Pending approvals by short code: the page and the Matrix command answer
416# with the code.
417APPROVALS = {}
418APPROVALS_LOCK = threading.Lock()
419
420
421class Run:
422 """A Hermes run this server follows to its end, page or no page. Pages
423 replay its items; an approval nobody can hear goes to Matrix."""
424
425 def __init__(self, config, hermes_session, run_id, hung_up=False):
426 self.config = config
427 self.hermes_session = hermes_session
428 self.run_id = run_id
429 self.items = []
430 self.finished = False
431 self.pages = 0
432 self.hung_up = hung_up
433 self.cond = threading.Condition()
434 with RUNS_LOCK:
435 RUNS[run_id] = self
436 if not hung_up:
437 LATEST[hermes_session] = self
438 threading.Thread(target=self.follow, daemon=True).start()
439
440 def follow(self):
441 started = time.monotonic()
442 tools = 0
443 try:
444 for item in hermes_follow_run(self.config, self.run_id):
445 if item["type"] == "tool" and item["status"] == "running":
446 tools += 1
447 self.add(item)
448 except urllib.error.HTTPError as error:
449 detail = error.read().decode("utf-8", "replace")[:400]
450 self.add({"type": "error", "message": f"Hermes {error.code}: {detail}"})
451 except (urllib.error.URLError, OSError, ValueError) as error:
452 self.add({"type": "error", "message": f"Hermes unreachable: {error}"})
453 settle_approvals(self)
454 with self.cond:
455 self.finished = True
456 self.cond.notify_all()
457 with RUNS_LOCK:
458 RUNS.pop(self.run_id, None)
459 if LATEST.get(self.hermes_session) is self:
460 del LATEST[self.hermes_session]
461 last = self.items[-1] if self.items else {}
462 log.info(
463 "run %s ended (%s) after %.1fs, %d tool calls",
464 self.run_id,
465 last.get("finish_reason") or last.get("message"),
466 time.monotonic() - started,
467 tools,
468 )
469
470 def add(self, item):
471 # Anything after an approval request means Hermes moved on (answered,
472 # or its own timeout).
473 settle_approvals(self)
474 if item["type"] == "approval":
475 item["code"] = open_approval(self, item)
476 with self.cond:
477 self.items.append(item)
478 self.cond.notify_all()
479 unheard = not self.pages or self.hung_up
480 if item["type"] == "end_call":
481 # Without a page the call has ended already.
482 log.info("run %s ends the call%s", self.run_id, " (no call)" if unheard else "")
483 self.ask_unheard()
484
485 def attach(self):
486 with self.cond:
487 self.pages += 1
488
489 def detach(self):
490 with self.cond:
491 self.pages -= 1
492 self.ask_unheard()
493
494 def hang_up(self):
495 with self.cond:
496 self.hung_up = True
497 self.ask_unheard()
498
499 def ask_unheard(self):
500 """Send a pending approval to Matrix once no call can answer it."""
501 with self.cond:
502 if self.pages and not self.hung_up:
503 return
504 with APPROVALS_LOCK:
505 unasked = [
506 (code, item)
507 for code, (run, item) in APPROVALS.items()
508 if run is self and not item.get("asked")
509 ]
510 for _, item in unasked:
511 item["asked"] = True
512 for code, item in unasked:
513 ask_matrix(self.config, code, item)
514
515
516def open_approval(run, item):
517 with APPROVALS_LOCK:
518 code = secrets.token_hex(2)
519 while code in APPROVALS:
520 code = secrets.token_hex(2)
521 APPROVALS[code] = (run, item)
522 return code
523
524
525def settle_approvals(run):
526 with APPROVALS_LOCK:
527 for code in [code for code, (owner, _) in APPROVALS.items() if owner is run]:
528 del APPROVALS[code]
529
530
531def ask_matrix(config, code, item):
532 if not config.matrix_webhook:
533 log.info("approval %s has no listener and no Matrix webhook", code)
534 return
535 text = (
536 "Hermes needs your approval for work from a voice call: "
537 + (item.get("description") or "run a command")
538 )
539 if item.get("command"):
540 text += f"\n```\n{item['command']}\n```"
541 text += f"\nAnswer with `/assist approve {code}` or `/assist deny {code}`."
542 if item.get("timeout_s"):
543 text += f" Unanswered, it is denied after {round(item['timeout_s'] / 60)} minutes."
544 try:
545 notify_matrix(config, text)
546 log.info("approval %s asked in Matrix", code)
547 except (urllib.error.URLError, OSError) as error:
548 log.warning("asking approval %s in Matrix failed: %s", code, error)
549
550
551def stop_run(config, run_id):
552 """POST /v1/runs/{run_id}/stop; failures are only logged."""
553 try:
554 hermes_call(config, "POST", f"/v1/runs/{run_id}/stop")
555 log.info("stopped run %s", run_id)
556 except (urllib.error.URLError, OSError, ValueError) as error:
557 log.warning("stopping run %s failed: %s", run_id, error)
558
559
560class Handler(BaseHTTPRequestHandler):
561 server_version = "hermes-voice"
562 config: Config
563 static: Path
564 access_token: "str | None" = None
565 app_token: "str | None" = None
566
567 def authorized(self, also_hermes_key=False):
568 """Check the page's bearer token (or, where allowed, the API server key
569 the hermes-voice plugin in the gateway holds); answer 401 when it is
570 wrong."""
571 if self.access_token is None:
572 return True
573 header = self.headers.get("Authorization", "")
574 scheme, _, token = header.partition(" ")
575 accepted = [self.access_token]
576 if self.app_token:
577 accepted.append(self.app_token)
578 if also_hermes_key:
579 accepted.append(self.config.hermes_key)
580 if scheme == "Bearer" and any(
581 hmac.compare_digest(token.encode(), key.encode()) for key in accepted
582 ):
583 return True
584 self.send_json(HTTPStatus.UNAUTHORIZED, {"error": "access token required"})
585 return False
586
587 def log_message(self, format, *args):
588 log.info("%s %s", self.address_string(), format % args)
589
590 def send_json(self, status, payload):
591 body = json.dumps(payload).encode()
592 self.send_response(status)
593 self.send_header("Content-Type", "application/json")
594 self.send_header("Content-Length", str(len(body)))
595 self.send_header("Cache-Control", "no-store")
596 self.end_headers()
597 self.wfile.write(body)
598
599 def read_json(self):
600 length = int(self.headers.get("Content-Length") or 0)
601 if length <= 0 or length > MAX_BODY:
602 raise ValueError("bad body length")
603 return json.loads(self.rfile.read(length))
604
605 def do_GET(self):
606 if self.path == "/config":
607 if not self.authorized():
608 return
609 self.send_json(
610 HTTPStatus.OK,
611 {
612 "model": self.config.model,
613 "voice": self.config.voice,
614 "voices": self.config.voices(),
615 "idle_seconds": self.config.idle_seconds,
616 },
617 )
618 return
619 names = {
620 "/": ("index.html", "text/html; charset=utf-8"),
621 "/app.js": ("app.js", "text/javascript; charset=utf-8"),
622 "/style.css": ("style.css", "text/css; charset=utf-8"),
623 "/manifest.webmanifest": ("manifest.webmanifest", "application/manifest+json"),
624 "/icon-192.png": ("icon-192.png", "image/png"),
625 "/icon-512.png": ("icon-512.png", "image/png"),
626 }
627 # Query strings do not pick a different file.
628 path = urllib.parse.urlsplit(self.path).path
629 if path not in names or not (self.static / names[path][0]).is_file():
630 self.send_error(HTTPStatus.NOT_FOUND)
631 return
632 name, content_type = names[path]
633 body = (self.static / name).read_bytes()
634 self.send_response(HTTPStatus.OK)
635 self.send_header("Content-Type", content_type)
636 self.send_header("Content-Length", str(len(body)))
637 self.send_header("Cache-Control", "no-store")
638 self.end_headers()
639 self.wfile.write(body)
640
641 def do_POST(self):
642 if not self.authorized(also_hermes_key=self.path == "/approve"):
643 return
644 try:
645 request = self.read_json()
646 except ValueError:
647 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid JSON body"})
648 return
649 if self.path == "/session":
650 self.handle_session(request)
651 elif self.path == "/delegate":
652 self.handle_delegate(request)
653 elif self.path == "/hangup":
654 self.handle_hangup(request)
655 elif self.path == "/approve":
656 self.handle_approve(request)
657 else:
658 self.send_error(HTTPStatus.NOT_FOUND)
659
660 def handle_session(self, request):
661 # The vendor's SDP parser needs the offer byte-exact (trailing CRLF too).
662 sdp = request.get("sdp")
663 if not isinstance(sdp, str) or not sdp.strip():
664 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "sdp offer required"})
665 return
666 try:
667 voice = request.get("voice") or self.config.voice
668 if voice not in self.config.voices():
669 self.send_json(HTTPStatus.BAD_REQUEST, {"error": f"unknown voice {voice!r}"})
670 return
671 result = create_live_session(
672 self.config, sdp, live_history(request.get("history") or []), voice
673 )
674 except urllib.error.HTTPError as error:
675 detail = error.read().decode("utf-8", "replace")[:800]
676 log.warning("live session creation failed: %s %s", error.code, detail)
677 self.send_json(
678 HTTPStatus.BAD_GATEWAY,
679 {"error": f"OpenAI rejected the session ({error.code}): {detail}"},
680 )
681 return
682 except (urllib.error.URLError, OSError, KeyError, ValueError) as error:
683 log.warning("live session creation failed: %s", error)
684 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"OpenAI: {error}"})
685 return
686 self.send_json(HTTPStatus.OK, result)
687
688 def handle_delegate(self, request):
689 hermes_session = request.get("hermes_session")
690 lines = request.get("lines")
691 if not isinstance(hermes_session, str) or not SESSION_ID_RE.match(hermes_session):
692 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid hermes_session"})
693 return
694 if not isinstance(lines, list) or not lines:
695 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "lines required"})
696 return
697 # A correction within the same call (the page says so) supersedes and
698 # stops the run it replaces; runs from earlier calls keep going.
699 if request.get("replace") is True:
700 with RUNS_LOCK:
701 previous = LATEST.pop(hermes_session, None)
702 if previous:
703 stop_run(self.config, previous.run_id)
704 try:
705 run = Run(
706 self.config,
707 hermes_session,
708 hermes_start_run(self.config, hermes_session, lines),
709 )
710 except urllib.error.HTTPError as error:
711 detail = error.read().decode("utf-8", "replace")[:400]
712 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
713 return
714 except (urllib.error.URLError, OSError, ValueError, KeyError) as error:
715 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
716 return
717 self.send_response(HTTPStatus.OK)
718 self.send_header("Content-Type", "application/x-ndjson")
719 self.send_header("Cache-Control", "no-store")
720 self.send_header("X-Accel-Buffering", "no")
721 self.end_headers()
722 run.attach()
723 try:
724 sent = 0
725 while True:
726 with run.cond:
727 if sent == len(run.items) and not run.finished:
728 run.cond.wait(PAGE_KEEPALIVE_SECONDS)
729 new = run.items[sent:]
730 sent += len(new)
731 done = run.finished and sent == len(run.items)
732 for item in new:
733 self.write_line(item)
734 if done:
735 break
736 if not new:
737 self.wfile.write(b"\n")
738 self.wfile.flush()
739 except (BrokenPipeError, ConnectionResetError):
740 # Only the page is gone: this server keeps following the run.
741 log.info("page left run %s", run.run_id)
742 finally:
743 run.detach()
744
745 def handle_hangup(self, request):
746 """The call ended. No page hears this conversation's runs any more, and
747 what the user said since the last delegation becomes its own run."""
748 hermes_session = request.get("hermes_session")
749 lines = request.get("lines") or []
750 if (
751 not isinstance(hermes_session, str)
752 or not SESSION_ID_RE.match(hermes_session)
753 or not isinstance(lines, list)
754 ):
755 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid hang-up"})
756 return
757 with RUNS_LOCK:
758 LATEST.pop(hermes_session, None)
759 runs = [run for run in RUNS.values() if run.hermes_session == hermes_session]
760 for run in runs:
761 run.hang_up()
762 if not any(line.get("speaker") == "user" for line in lines):
763 self.send_json(HTTPStatus.OK, {"run_id": None})
764 return
765 try:
766 run_id = hermes_start_run(self.config, hermes_session, lines, hung_up=True)
767 except urllib.error.HTTPError as error:
768 detail = error.read().decode("utf-8", "replace")[:400]
769 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
770 return
771 except (urllib.error.URLError, OSError, ValueError, KeyError) as error:
772 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
773 return
774 Run(self.config, hermes_session, run_id, hung_up=True)
775 log.info("hang-up of %s left a request: run %s", hermes_session, run_id)
776 self.send_json(HTTPStatus.OK, {"run_id": run_id})
777
778 def handle_approve(self, request):
779 code = request.get("code")
780 choice = request.get("choice")
781 # No "always": a permanent allowlist entry is not a voice decision.
782 if choice not in ("once", "session", "deny") or not (
783 code is None or isinstance(code, str)
784 ):
785 self.send_json(HTTPStatus.BAD_REQUEST, {"error": "invalid approval answer"})
786 return
787 with APPROVALS_LOCK:
788 if code is None and len(APPROVALS) == 1:
789 code = next(iter(APPROVALS))
790 pending = APPROVALS.get(code)
791 if pending is None:
792 self.send_json(
793 HTTPStatus.CONFLICT,
794 {"error": f"no pending approval {code}" if code else "no single pending approval"},
795 )
796 return
797 run, item = pending
798 try:
799 result = hermes_approve(self.config, run.run_id, item, choice)
800 except urllib.error.HTTPError as error:
801 detail = error.read().decode("utf-8", "replace")[:400]
802 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes {error.code}: {detail}"})
803 return
804 except (urllib.error.URLError, OSError, ValueError) as error:
805 self.send_json(HTTPStatus.BAD_GATEWAY, {"error": f"Hermes unreachable: {error}"})
806 return
807 with APPROVALS_LOCK:
808 APPROVALS.pop(code, None)
809 if result.get("status") == "already_resolved":
810 self.send_json(HTTPStatus.CONFLICT, {"error": f"approval {code} was already answered"})
811 return
812 log.info("approval %s (run %s): %s %s", code, run.run_id, choice, result.get("status"))
813 self.send_json(
814 HTTPStatus.OK,
815 {"code": code, "choice": choice, "description": item.get("description")},
816 )
817
818 def write_line(self, item):
819 self.wfile.write(json.dumps(item).encode() + b"\n")
820 self.wfile.flush()
821
822
823def main():
824 parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
825 parser.add_argument("--listen", default="127.0.0.1:8790", help="host:port to serve on")
826 parser.add_argument("--hermes-url", required=True, help="Hermes API server base URL")
827 parser.add_argument("--hermes-key-file", required=True, help="file holding API_SERVER_KEY")
828 parser.add_argument("--openai-key-file", required=True, help="file holding the OpenAI API key")
829 parser.add_argument("--access-token-file", help="file holding the token the page must present")
830 parser.add_argument("--app-token-file", help="file holding the token the Android app presents")
831 parser.add_argument(
832 "--matrix-webhook",
833 help="Hermes webhook route (deliver_only to Matrix, signed with API_SERVER_KEY) that "
834 "receives approvals no call can answer",
835 )
836 parser.add_argument("--config", help="JSON file: voice, model, idle_seconds, glossary, instructions")
837 parser.add_argument(
838 "--static",
839 default=os.environ.get("HERMES_VOICE_STATIC"),
840 help="directory with index.html, app.js, style.css (default: $HERMES_VOICE_STATIC)",
841 )
842 args = parser.parse_args()
843 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
844
845 host, _, port = args.listen.rpartition(":")
846 host = host.strip("[]")
847 if args.access_token_file is None and host not in ("127.0.0.1", "::1", "localhost"):
848 parser.error("--access-token-file is required when not listening on loopback")
849
850 raw = json.loads(Path(args.config).read_text()) if args.config else {}
851 config = Config(
852 raw,
853 openai_key=Path(args.openai_key_file).read_text().strip(),
854 hermes_url=args.hermes_url,
855 hermes_key=Path(args.hermes_key_file).read_text().strip(),
856 matrix_webhook=args.matrix_webhook,
857 )
858 Handler.config = config
859 Handler.static = Path(args.static)
860 if args.access_token_file:
861 Handler.access_token = Path(args.access_token_file).read_text().strip()
862 if args.app_token_file:
863 Handler.app_token = Path(args.app_token_file).read_text().strip()
864 server_class = ThreadingHTTPServer
865 if ":" in host:
866 # "::" is dual-stack on Linux (bindv6only=0).
867 server_class = type("Server6", (ThreadingHTTPServer,), {"address_family": socket.AF_INET6})
868 server = server_class((host, int(port)), Handler)
869 server.daemon_threads = True
870 log.info("serving on http://%s", args.listen)
871 server.serve_forever()
872
873
874if __name__ == "__main__":
875 main()