+13 -5

configs/hermes-proxy.nix

View file Reviewed

··· 88 88 89 89 # Public door to hermes-voice on coaxmetal (machines/coaxmetal/hermes-voice.nix): 90 90 # the voice page for the phone. Everything but the files Chrome fetches 91 91 - # without cookies to install the page and verify the Android app needs a 92 92 - # login on id.lassul.us (oauth2-proxy below, via auth_request); logged-in 93 93 - # requests get hermes-voice's access token added. /delegate streams NDJSON 94 94 - # for the length of a Hermes turn, so the streaming settings apply. 91 91 + # without cookies to install the page needs a login on id.lassul.us 92 92 + # (oauth2-proxy below, via auth_request); logged-in requests get 93 93 + # hermes-voice's access token added. /app/ is the Android app's door: no 94 94 + # web login, the app's own bearer token goes through to hermes-voice, which 95 95 + # checks it. /delegate streams NDJSON for the length of a Hermes turn, so 96 96 + # the streaming settings apply. 95 97 services.nginx.virtualHosts.${assist} = { 96 98 enableACME = true; 97 99 forceSSL = true; ··· 103 105 locations = 104 106 lib.genAttrs 105 107 [ 106 106 - "= /.well-known/assetlinks.json" 107 108 "= /manifest.webmanifest" 108 109 "= /icon-192.png" 109 110 "= /icon-512.png" ··· 120 121 proxyPass = voiceUpstream; 121 122 extraConfig = streamingProxy + '' 122 123 include ${voiceAuthHeader}; 124 124 + ''; 125 125 + }; 126 126 + "/app/" = { 127 127 + recommendedProxySettings = true; 128 128 + proxyPass = "${voiceUpstream}/"; 129 129 + extraConfig = streamingProxy + '' 130 130 + auth_request off; 123 131 ''; 124 132 }; 125 133 "/oauth2/" = {

+1 -1

.github/workflows/assist-android.yml

View file Reviewed

··· 1 1 name: assist-android 2 2 3 3 - # Builds the Android wrapper of the voice page (packages.hermes-voice-android), 3 3 + # Builds the Android app of hermes-voice (packages.hermes-voice-android), 4 4 # signs it and publishes it as assist.apk of a new release 5 5 # assist-android-v<run number>, so Obtainium sees every build. 6 6 # Stable URL: https://github.com/lassulus/superconfig/releases/latest/download/assist.apk

+25 -5

machines/coaxmetal/hermes-voice.nix

View file Reviewed

··· 20 20 # 21 21 # SECURITY: the page drives Hermes, which has terminal tools and code 22 22 # execution as lass (machines/coaxmetal/hermes.nix). The access token 23 23 - # (configs/hermes-voice-vars.nix) is the gate on every API route; treat it 24 24 - # like the hermes-api key. 23 23 + # (configs/hermes-voice-vars.nix) and the Android app's token 24 24 + # (hermes-voice-app below) are the gates on every API route; treat both 25 25 + # like the hermes-api key. The app takes calls from the headset button with 26 26 + # the phone locked, approvals by voice included. 25 27 systemd.services.hermes-voice = { 26 28 description = "hermes-voice: live voice conversations with Hermes"; 27 29 wantedBy = [ "multi-user.target" ]; ··· 45 47 "%d/openai-key" 46 48 "--access-token-file" 47 49 "%d/access-token" 50 50 + "--app-token-file" 51 51 + "%d/app-token" 48 52 "--matrix-webhook" 49 53 "http://127.0.0.1:${toString webhookPort}/webhooks/voice-approval" 50 54 ]; ··· 52 56 "hermes-key:${config.clan.core.vars.generators.hermes-api.files."api_key".path}" 53 57 "openai-key:${config.clan.core.vars.generators.hermes-voice-openai.files."api-key".path}" 54 58 "access-token:${config.clan.core.vars.generators.hermes-voice.files."access-token".path}" 59 59 + "app-token:${config.clan.core.vars.generators.hermes-voice-app.files.token.path}" 55 60 ]; 56 61 DynamicUser = true; 57 62 Restart = "on-failure"; ··· 107 112 persist = true; 108 113 }; 109 114 110 110 - # Signing key of the Android wrapper (packages.hermes-voice-android); only 111 111 - # read when signing, so nothing is deployed. The fingerprint goes into the 112 112 - # page's assetlinks.json (tools/hermes-voice/flake-module.nix). 115 115 + # Bearer token of the Android app (tools/hermes-voice/android), checked by 116 116 + # hermes-voice next to the page's access token; the proxy passes it through 117 117 + # on assist.lassul.us/app/ without the web login (configs/hermes-proxy.nix). 118 118 + # Paste it into the app once: 119 119 + # clan vars get coaxmetal hermes-voice-app/token 120 120 + clan.core.vars.generators.hermes-voice-app = { 121 121 + files.token = { }; 122 122 + runtimeInputs = [ 123 123 + pkgs.coreutils 124 124 + pkgs.openssl 125 125 + ]; 126 126 + script = '' 127 127 + openssl rand -hex 32 | tr -d '\n' > "$out/token" 128 128 + ''; 129 129 + }; 130 130 + 131 131 + # Signing key of the Android app (packages.hermes-voice-android); only read 132 132 + # when signing, so nothing is deployed. 113 133 clan.core.vars.generators.hermes-voice-android-signing = { 114 134 files.keystore.deploy = false; 115 135 files.password.deploy = false;

+3 -30

tools/hermes-voice/flake-module.nix

View file Reviewed

··· 1 1 - { self, ... }: 2 1 { 3 2 perSystem = 4 4 - { lib, pkgs, ... }: 3 3 + { pkgs, ... }: 5 4 let 6 6 - # The Android wrapper's package name (android/AndroidManifest.xml). 7 7 - androidPackage = "us.lassul.assist"; 8 8 - # Written by `clan vars generate coaxmetal` (machines/coaxmetal/hermes-voice.nix); 9 9 - # until then there is no assetlinks.json and Chrome opens the app as a 10 10 - # Custom Tab. 11 11 - fingerprint = self + "/vars/per-machine/coaxmetal/hermes-voice-android-signing/fingerprint/value"; 12 12 - assetlinks = pkgs.writeText "assetlinks.json" ( 13 13 - builtins.toJSON [ 14 14 - { 15 15 - # use_as_origin: the app's postMessages to the page (android/src/…/Link.java). 16 16 - relation = [ 17 17 - "delegate_permission/common.handle_all_urls" 18 18 - "delegate_permission/common.use_as_origin" 19 19 - ]; 20 20 - target = { 21 21 - namespace = "android_app"; 22 22 - package_name = androidPackage; 23 23 - sha256_cert_fingerprints = [ (lib.removeSuffix "\n" (builtins.readFile fingerprint)) ]; 24 24 - }; 25 25 - } 26 26 - ] 27 27 - ); 28 5 icon = pkgs.runCommand "hermes-voice-icon" { nativeBuildInputs = [ pkgs.imagemagick ]; } '' 29 6 mkdir $out 30 7 for size in 192 512; do ··· 37 14 cp -r ${./static} $out 38 15 chmod u+w $out 39 16 cp ${icon}/*.png $out/ 40 40 - ${lib.optionalString (builtins.pathExists fingerprint) '' 41 41 - mkdir $out/.well-known 42 42 - cp ${assetlinks} $out/.well-known/assetlinks.json 43 43 - ''} 44 17 ''; 45 18 # The SDK is unfree; it only builds the APK and is not part of any system. 46 19 inherit ··· 67 40 }).overrideAttrs 68 41 { passthru.usage = builtins.readFile ./usage.kdl; }; 69 42 70 70 - # The voice page as Android app and digital assistant (Trusted Web 71 71 - # Activity), unsigned. 43 43 + # The Android app: holds calls itself (WebRTC) and answers the headset's 44 44 + # voice button, unsigned. 72 45 packages.hermes-voice-android = android; 73 46 checks.hermes-voice-android = android; 74 47

+11 -3

tools/hermes-voice/hermes_voice.py

View file Reviewed

··· 31 31 route requires the page's own access token (--access-token-file), and serving 32 32 beyond loopback without one is refused. 33 33 34 34 + The Android app (android/) is a second client of the same routes: it holds 35 35 + the call itself (WebRTC, no page) and presents its own token 36 36 + (--app-token-file), which the proxy passes through without the web login. 37 37 + 34 38 Vendor contract: https://developers.openai.com/api/docs/guides/live-delegation 35 39 """ 36 40 ··· 558 562 config: Config 559 563 static: Path 560 564 access_token: "str | None" = None 565 565 + app_token: "str | None" = None 561 566 562 567 def authorized(self, also_hermes_key=False): 563 568 """Check the page's bearer token (or, where allowed, the API server key ··· 568 573 header = self.headers.get("Authorization", "") 569 574 scheme, _, token = header.partition(" ") 570 575 accepted = [self.access_token] 576 576 + if self.app_token: 577 577 + accepted.append(self.app_token) 571 578 if also_hermes_key: 572 579 accepted.append(self.config.hermes_key) 573 580 if scheme == "Bearer" and any( ··· 616 623 "/manifest.webmanifest": ("manifest.webmanifest", "application/manifest+json"), 617 624 "/icon-192.png": ("icon-192.png", "image/png"), 618 625 "/icon-512.png": ("icon-512.png", "image/png"), 619 619 - # Missing until the Android signing key exists. 620 620 - "/.well-known/assetlinks.json": (".well-known/assetlinks.json", "application/json"), 621 626 } 622 622 - # The page takes query parameters (/?listen from the Android app). 627 627 + # Query strings do not pick a different file. 623 628 path = urllib.parse.urlsplit(self.path).path 624 629 if path not in names or not (self.static / names[path][0]).is_file(): 625 630 self.send_error(HTTPStatus.NOT_FOUND) ··· 822 827 parser.add_argument("--hermes-key-file", required=True, help="file holding API_SERVER_KEY") 823 828 parser.add_argument("--openai-key-file", required=True, help="file holding the OpenAI API key") 824 829 parser.add_argument("--access-token-file", help="file holding the token the page must present") 830 830 + parser.add_argument("--app-token-file", help="file holding the token the Android app presents") 825 831 parser.add_argument( 826 832 "--matrix-webhook", 827 833 help="Hermes webhook route (deliver_only to Matrix, signed with API_SERVER_KEY) that " ··· 853 859 Handler.static = Path(args.static) 854 860 if args.access_token_file: 855 861 Handler.access_token = Path(args.access_token_file).read_text().strip() 862 862 + if args.app_token_file: 863 863 + Handler.app_token = Path(args.app_token_file).read_text().strip() 856 864 server_class = ThreadingHTTPServer 857 865 if ":" in host: 858 866 # "::" is dual-stack on Linux (bindv6only=0).

+40 -22

tools/hermes-voice/android/AndroidManifest.xml

View file Reviewed

··· 2 2 <manifest xmlns:android="http://schemas.android.com/apk/res/android" 3 3 package="us.lassul.assist"> 4 4 5 5 - <!-- CallService: each call of the page as a self-managed Telecom call. --> 5 5 + <!-- VoiceCall: WebRTC to gpt-live-1 and the hermes-voice API. --> 6 6 + <uses-permission android:name="android.permission.INTERNET" /> 7 7 + <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" /> 8 8 + <uses-permission android:name="android.permission.RECORD_AUDIO" /> 9 9 + <uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" /> 10 10 + <!-- CallService: the call as a self-managed Telecom call, in the foreground. --> 6 11 <uses-permission android:name="android.permission.MANAGE_OWN_CALLS" /> 7 12 <uses-permission android:name="android.permission.FOREGROUND_SERVICE" /> 8 13 <uses-permission android:name="android.permission.FOREGROUND_SERVICE_PHONE_CALL" /> 9 9 - 10 10 - <queries> 11 11 - <intent> 12 12 - <action android:name="android.support.customtabs.action.CustomTabsService" /> 13 13 - </intent> 14 14 - </queries> 14 14 + <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" /> 15 15 + <uses-permission android:name="android.permission.POST_NOTIFICATIONS" /> 15 16 16 17 <application 17 18 android:label="Assist" 18 19 android:icon="@mipmap/ic_launcher" 19 19 - android:allowBackup="false"> 20 20 - <!-- Over the lock screen and waking the display: a headset button 21 21 - (VOICE_COMMAND) usually arrives while the phone is locked; the 22 22 - activity then asks to unlock before it opens the page. --> 20 20 + android:allowBackup="false" 21 21 + android:extractNativeLibs="true" 22 22 + android:theme="@android:style/Theme.DeviceDefault.DayNight"> 23 23 + <!-- Launcher: the current conversation and Call / Hang up. --> 23 24 <activity 24 24 - android:name=".AssistActivity" 25 25 - android:exported="true" 26 26 - android:showWhenLocked="true" 27 27 - android:turnScreenOn="true" 28 28 - android:theme="@android:style/Theme.Translucent.NoTitleBar"> 25 25 + android:name=".MainActivity" 26 26 + android:exported="true"> 29 27 <intent-filter> 30 28 <action android:name="android.intent.action.MAIN" /> 31 29 <category android:name="android.intent.category.LAUNCHER" /> 32 30 </intent-filter> 31 31 + </activity> 32 32 + <activity 33 33 + android:name=".SettingsActivity" 34 34 + android:label="Settings" 35 35 + android:exported="false" /> 36 36 + <activity 37 37 + android:name=".SessionsActivity" 38 38 + android:label="Sessions" 39 39 + android:exported="false" /> 40 40 + <activity 41 41 + android:name=".TranscriptActivity" 42 42 + android:exported="false" /> 43 43 + <!-- The headset's voice button (HFP AT+BVRA=1 makes HeadsetService 44 44 + send VOICE_COMMAND) and the assist gesture: start a call and 45 45 + vanish. Over the lock screen and waking the display, so the 46 46 + call starts from the foreground without an unlock. --> 47 47 + <activity 48 48 + android:name=".AssistActivity" 49 49 + android:exported="true" 50 50 + android:excludeFromRecents="true" 51 51 + android:noHistory="true" 52 52 + android:taskAffinity="" 53 53 + android:showWhenLocked="true" 54 54 + android:turnScreenOn="true" 55 55 + android:theme="@android:style/Theme.Translucent.NoTitleBar"> 33 56 <intent-filter> 34 57 <action android:name="android.intent.action.ASSIST" /> 35 58 <category android:name="android.intent.category.DEFAULT" /> 36 59 </intent-filter> 37 37 - <!-- Bluetooth headsets: HFP AT+BVRA=1 makes HeadsetService start it. --> 38 60 <intent-filter> 39 61 <action android:name="android.intent.action.VOICE_COMMAND" /> 40 62 <category android:name="android.intent.category.DEFAULT" /> ··· 43 65 <service 44 66 android:name=".CallService" 45 67 android:exported="false" 46 46 - android:foregroundServiceType="phoneCall" /> 47 47 - <!-- The browser binds it to deliver the page's postMessages. --> 48 48 - <service 49 49 - android:name="androidx.browser.customtabs.PostMessageService" 50 50 - android:exported="true" /> 68 68 + android:foregroundServiceType="phoneCall|microphone" /> 51 69 </application> 52 70 </manifest>

+14 -10

tools/hermes-voice/android/default.nix

View file Reviewed

··· 21 21 sdkRoot = "${sdk.androidsdk}/libexec/android-sdk"; 22 22 buildTools = "${sdkRoot}/build-tools/36.0.0"; 23 23 androidJar = "${sdkRoot}/platforms/android-36/android.jar"; 24 24 - browser = fetchurl { 25 25 - url = "https://dl.google.com/android/maven2/androidx/browser/browser/1.10.0/browser-1.10.0.aar"; 26 26 - hash = "sha256-+0DR7l/ZEMnREiAIaKlTQVanqZmrwo+7PGgVouOsUVA="; 24 24 + # WebRTC for Android (LiveKit's build of Google's libwebrtc): the Java API 25 25 + # and the native library, which needs none of the AAR's other files. 26 26 + webrtc = fetchurl { 27 27 + url = "https://repo1.maven.org/maven2/io/github/webrtc-sdk/android/150.7871.01/android-150.7871.01.aar"; 28 28 + hash = "sha256-ChYnsaSMK8F9mkDWL8R71FFm9EoxHpWRfxR8QC3jebA="; 27 29 }; 28 30 in 29 31 # Unsigned release APK; tools/hermes-voice/flake-module.nix has the signer. 30 30 - # Only androidx.browser's classes go in: the parts used here need none of its 31 31 - # dependencies. 32 32 + # arm64 only: the app runs on Pixels. 32 33 stdenvNoCC.mkDerivation { 33 34 pname = "hermes-voice-android"; 34 35 version = versionName; ··· 40 41 ]; 41 42 buildPhase = '' 42 43 runHook preBuild 43 43 - unzip -q ${browser} classes.jar -d browser 44 44 + unzip -q ${webrtc} classes.jar 'jni/arm64-v8a/*' -d webrtc 44 45 45 46 mkdir -p res/mipmap-xxxhdpi 46 47 cp ${icon}/icon-192.png res/mipmap-xxxhdpi/ic_launcher.png ··· 51 52 --version-code ${toString versionCode} --version-name ${versionName} 52 53 53 54 javac -source 8 -target 8 -Xlint:-options -encoding UTF-8 \ 54 54 - -bootclasspath ${androidJar} -cp browser/classes.jar \ 55 55 + -bootclasspath ${androidJar} -cp webrtc/classes.jar \ 55 56 -d classes $(find src -name '*.java') 56 57 ${buildTools}/d8 --release --min-api 34 --lib ${androidJar} --output . \ 57 57 - $(find classes -name '*.class') browser/classes.jar 58 58 + $(find classes -name '*.class') webrtc/classes.jar 58 59 59 59 - touch -d @315532800 classes.dex 60 60 - TZ=UTC zip -X -D unsigned.apk classes.dex 60 60 + mkdir -p lib/arm64-v8a 61 61 + cp webrtc/jni/arm64-v8a/*.so lib/arm64-v8a/ 62 62 + touch -d @315532800 classes*.dex lib/arm64-v8a/*.so 63 63 + TZ=UTC zip -X -D unsigned.apk classes*.dex 64 64 + TZ=UTC zip -X -D -0 unsigned.apk lib/arm64-v8a/*.so 61 65 runHook postBuild 62 66 ''; 63 67 installPhase = ''

+22 -40

tools/hermes-voice/static/app.js

View file Reviewed

··· 98 98 : `conversation ${clock(total)} · ${money(total)}`; 99 99 } 100 100 101 101 + // Only the transcript scrolls (style.css); it follows the newest line unless 102 102 + // the user scrolled up. The controls above it change height after a render 103 103 + // (cost line, approval box), so a resize re-pins it too. A resize leaves 104 104 + // scrollTop alone; only moving up (the user) unpins. 105 105 + let logPinned = true; 106 106 + let logTop = 0; 107 107 + logList.addEventListener("scroll", () => { 108 108 + if (logList.scrollTop + logList.clientHeight >= logList.scrollHeight - 4) 109 109 + logPinned = true; 110 110 + else if (logList.scrollTop < logTop) logPinned = false; 111 111 + logTop = logList.scrollTop; 112 112 + }); 113 113 + new ResizeObserver(() => { 114 114 + if (logPinned) logList.scrollTop = logList.scrollHeight; 115 115 + }).observe(logList); 116 116 + 101 117 function renderLog() { 102 118 logList.replaceChildren( 103 119 ...state.lines.map((line) => { ··· 107 123 return item; 108 124 }), 109 125 ); 110 110 - window.scrollTo(0, document.body.scrollHeight); 126 126 + logList.scrollTop = logList.scrollHeight; 111 127 } 112 128 113 129 function addLine(speaker, text) { ··· 581 597 voice.srcObject = null; 582 598 state.billedSeconds += this.usageSeconds; 583 599 saveState(); 584 584 - if (call === this) { 585 585 - call = null; 586 586 - tellApp("ended"); 587 587 - } 600 600 + if (call === this) call = null; 588 601 talk.classList.remove("live", "speaking"); 589 602 talk.textContent = "Talk"; 590 603 talk.disabled = false; ··· 595 608 596 609 async function startCall() { 597 610 call = new Call(); 598 598 - tellApp("started"); 599 611 talk.disabled = true; 600 612 talk.classList.add("live"); 601 613 talk.textContent = "Hang up"; 602 614 setStatus("Connecting…"); 603 615 // Must run inside the click handler: mobile browsers only allow audio 604 604 - // playback started by a user gesture. Started without one (?listen), the 605 605 - // track handler's play() runs while the mic records, which Chrome allows. 616 616 + // playback started by a user gesture. 606 617 voice.play().catch(() => {}); 607 618 try { 608 619 await call.start(); ··· 613 624 } 614 625 } 615 626 616 616 - function hangUpCall() { 617 617 - talk.disabled = true; 618 618 - setStatus("Hanging up…"); 619 619 - call.close(); 620 620 - } 621 621 - 622 627 talk.addEventListener("click", () => { 623 628 if (call) { 624 624 - hangUpCall(); 629 629 + talk.disabled = true; 630 630 + setStatus("Hanging up…"); 631 631 + call.close(); 625 632 return; 626 633 } 627 634 startCall(); 628 635 }); 629 636 630 637 window.addEventListener("pagehide", () => call?.finish("page closed")); 631 631 - 632 632 - // The Android app holds each call in Android's telephony (Telecom), so a 633 633 - // headset's hang-up button reaches it; it then sends "hangup". Its first 634 634 - // message, as this origin (assetlinks.json: use_as_origin), brings the port. 635 635 - let appPort = null; 636 636 - 637 637 - function tellApp(message) { 638 638 - appPort?.postMessage(message); 639 639 - } 640 640 - 641 641 - window.addEventListener("message", (event) => { 642 642 - if (event.origin !== location.origin || !event.ports[0]) return; 643 643 - appPort = event.ports[0]; 644 644 - appPort.onmessage = ({ data }) => { 645 645 - if (data === "hangup" && call) hangUpCall(); 646 646 - }; 647 647 - // ?listen starts the call before the app's channel is up. 648 648 - if (call) tellApp("started"); 649 649 - }); 650 638 651 639 document.getElementById("new").addEventListener("click", () => { 652 640 if (call) call.close(); ··· 687 675 ), 688 676 ); 689 677 voiceSelect.value = saved in config.voices ? saved : config.voice; 690 690 - // The Android app (and the headset button) opens /?listen: talk right away. 691 691 - // Dropped from the URL once used, so a reload does not call again. 692 692 - if (new URLSearchParams(location.search).has("listen")) { 693 693 - history.replaceState(null, "", location.pathname); 694 694 - startCall(); 695 695 - } 696 678 } 697 679 698 680 loadConfig();

+28 -2

tools/hermes-voice/static/style.css

View file Reviewed

··· 10 10 box-sizing: border-box; 11 11 } 12 12 13 13 + /* One screen high: only the transcript scrolls, so Talk/Hang up (and an 14 14 + approval) always stay in view. 100% of the containing block, not vh/dvh: 15 15 + in Chrome on Android those resolve taller than the visible area, and 16 16 + the document then scrolls to hide the browser bars. */ 17 17 + html, 18 18 + body { 19 19 + height: 100%; 20 20 + overflow: hidden; 21 21 + } 22 22 + 13 23 body { 14 24 margin: 0; 15 25 background: #111; ··· 21 31 22 32 main { 23 33 max-width: 40rem; 34 34 + height: 100%; 24 35 margin: 0 auto; 25 25 - padding: max(1rem, env(safe-area-inset-top)) 1rem 2rem; 36 36 + padding: max(1rem, env(safe-area-inset-top)) 1rem 37 37 + max(1rem, env(safe-area-inset-bottom)); 38 38 + display: flex; 39 39 + flex-direction: column; 40 40 + } 41 41 + 42 42 + main > :not(.log) { 43 43 + flex: none; 26 44 } 27 45 28 46 header { ··· 95 113 } 96 114 97 115 .log { 116 116 + flex: 1 1 0; 117 117 + min-height: 0; 118 118 + overflow-y: auto; 119 119 + /* A swipe past either end does not move the page. */ 120 120 + overscroll-behavior: contain; 98 121 list-style: none; 99 122 padding: 0; 100 100 - margin-top: 1.5rem; 123 123 + margin: 1.5rem 0 0; 101 124 } 102 125 103 126 .log li { ··· 135 158 } 136 159 137 160 .approval p { 161 161 + /* A long command scrolls here instead of pushing the buttons away. */ 162 162 + max-height: 30vh; 163 163 + overflow-y: auto; 138 164 margin: 0 0 0.6rem; 139 165 white-space: pre-wrap; 140 166 word-break: break-word;

+141

tools/hermes-voice/android/src/us/lassul/assist/Api.java

View file Reviewed

··· 1 1 + package us.lassul.assist; 2 2 + 3 3 + import android.content.Context; 4 4 + import android.content.SharedPreferences; 5 5 + import java.io.ByteArrayOutputStream; 6 6 + import java.io.IOException; 7 7 + import java.io.InputStream; 8 8 + import java.io.OutputStream; 9 9 + import java.net.HttpURLConnection; 10 10 + import java.net.URL; 11 11 + import java.nio.charset.StandardCharsets; 12 12 + import org.json.JSONException; 13 13 + import org.json.JSONObject; 14 14 + 15 15 + /** 16 16 + * hermes-voice's API (tools/hermes-voice/hermes_voice.py) behind assist.lassul.us/app/, where the 17 17 + * proxy passes the app's own bearer token through instead of asking for the web login. 18 18 + */ 19 19 + final class Api { 20 20 + private static final String BASE = "https://assist.lassul.us/app"; 21 21 + private static final String PREFS = "api"; 22 22 + 23 23 + /** An answer other than 2xx, with the server's error text. */ 24 24 + static final class Failure extends IOException { 25 25 + Failure(String message) { 26 26 + super(message); 27 27 + } 28 28 + } 29 29 + 30 30 + private final String token; 31 31 + 32 32 + private Api(String token) { 33 33 + this.token = token; 34 34 + } 35 35 + 36 36 + /** Null until the token is set (SetupActivity). */ 37 37 + static Api load(Context context) { 38 38 + String token = prefs(context).getString("token", null); 39 39 + return token == null || token.isEmpty() ? null : new Api(token); 40 40 + } 41 41 + 42 42 + static void saveToken(Context context, String token) { 43 43 + prefs(context).edit().putString("token", token.trim()).apply(); 44 44 + } 45 45 + 46 46 + static boolean hasToken(Context context) { 47 47 + return load(context) != null; 48 48 + } 49 49 + 50 50 + private static SharedPreferences prefs(Context context) { 51 51 + return context.getSharedPreferences(PREFS, Context.MODE_PRIVATE); 52 52 + } 53 53 + 54 54 + JSONObject get(String path) throws IOException, JSONException { 55 55 + HttpURLConnection connection = open(path, "GET"); 56 56 + try { 57 57 + return answer(connection); 58 58 + } finally { 59 59 + connection.disconnect(); 60 60 + } 61 61 + } 62 62 + 63 63 + JSONObject post(String path, JSONObject body) throws IOException, JSONException { 64 64 + HttpURLConnection connection = open(path, "POST"); 65 65 + try { 66 66 + send(connection, body); 67 67 + return answer(connection); 68 68 + } finally { 69 69 + connection.disconnect(); 70 70 + } 71 71 + } 72 72 + 73 73 + /** 74 74 + * A POST whose answer is read as a stream (/delegate's NDJSON); the caller reads and 75 75 + * disconnects. Throws Failure on an error answer. 76 76 + */ 77 77 + HttpURLConnection stream(String path, JSONObject body) throws IOException, JSONException { 78 78 + HttpURLConnection connection = open(path, "POST"); 79 79 + // The server writes a keepalive line every 10 s, so a long silence means it is gone. 80 80 + connection.setReadTimeout(60_000); 81 81 + send(connection, body); 82 82 + int status = connection.getResponseCode(); 83 83 + if (status / 100 != 2) { 84 84 + String error = errorOf(connection, status); 85 85 + connection.disconnect(); 86 86 + throw new Failure(error); 87 87 + } 88 88 + return connection; 89 89 + } 90 90 + 91 91 + private HttpURLConnection open(String path, String method) throws IOException { 92 92 + HttpURLConnection connection = (HttpURLConnection) new URL(BASE + path).openConnection(); 93 93 + connection.setRequestMethod(method); 94 94 + connection.setRequestProperty("Authorization", "Bearer " + token); 95 95 + connection.setConnectTimeout(10_000); 96 96 + connection.setReadTimeout(30_000); 97 97 + return connection; 98 98 + } 99 99 + 100 100 + private static void send(HttpURLConnection connection, JSONObject body) throws IOException { 101 101 + byte[] bytes = body.toString().getBytes(StandardCharsets.UTF_8); 102 102 + connection.setDoOutput(true); 103 103 + connection.setRequestProperty("Content-Type", "application/json"); 104 104 + connection.setFixedLengthStreamingMode(bytes.length); 105 105 + try (OutputStream out = connection.getOutputStream()) { 106 106 + out.write(bytes); 107 107 + } 108 108 + } 109 109 + 110 110 + private static JSONObject answer(HttpURLConnection connection) 111 111 + throws IOException, JSONException { 112 112 + int status = connection.getResponseCode(); 113 113 + if (status / 100 != 2) { 114 114 + throw new Failure(errorOf(connection, status)); 115 115 + } 116 116 + return new JSONObject(read(connection.getInputStream())); 117 117 + } 118 118 + 119 119 + private static String errorOf(HttpURLConnection connection, int status) { 120 120 + try { 121 121 + InputStream stream = connection.getErrorStream(); 122 122 + if (stream != null) { 123 123 + return new JSONObject(read(stream)).optString("error", "HTTP " + status); 124 124 + } 125 125 + } catch (IOException | JSONException ignored) { 126 126 + // The status says enough. 127 127 + } 128 128 + return "HTTP " + status; 129 129 + } 130 130 + 131 131 + private static String read(InputStream stream) throws IOException { 132 132 + ByteArrayOutputStream out = new ByteArrayOutputStream(); 133 133 + byte[] buffer = new byte[8192]; 134 134 + try (InputStream in = stream) { 135 135 + for (int n; (n = in.read(buffer)) > 0; ) { 136 136 + out.write(buffer, 0, n); 137 137 + } 138 138 + } 139 139 + return new String(out.toByteArray(), StandardCharsets.UTF_8); 140 140 + } 141 141 + }

+8 -46

tools/hermes-voice/android/src/us/lassul/assist/AssistActivity.java

View file Reviewed

··· 1 1 package us.lassul.assist; 2 2 3 3 import android.app.Activity; 4 4 - import android.app.KeyguardManager; 5 4 import android.content.Intent; 6 6 - import android.os.Bundle; 7 5 8 6 /** 9 9 - * Entry point (launcher, assist gesture, headset VOICE_COMMAND): unlocks if needed and opens the 10 10 - * voice page, which starts listening at once (?listen). 7 7 + * The headset's voice button (VOICE_COMMAND) and the assist gesture: starts a call and vanishes. 8 8 + * Starting it here, in onResume, is what lets CallService run in the foreground with the 9 9 + * microphone: Android allows that only while the app is in front, which this activity is for a 10 10 + * moment, over the lock screen too (showWhenLocked, turnScreenOn in the manifest). 11 11 */ 12 12 public class AssistActivity extends Activity { 13 13 - private boolean launched; 14 14 - 15 13 @Override 16 16 - protected void onCreate(Bundle state) { 17 17 - super.onCreate(state); 18 18 - KeyguardManager keyguard = getSystemService(KeyguardManager.class); 19 19 - if (!keyguard.isKeyguardLocked()) { 20 20 - open(); 21 21 - return; 22 22 - } 23 23 - // Started over the lock screen (showWhenLocked): the browser cannot show there. 24 24 - keyguard.requestDismissKeyguard(this, new KeyguardManager.KeyguardDismissCallback() { 25 25 - @Override 26 26 - public void onDismissSucceeded() { 27 27 - open(); 28 28 - } 29 29 - 30 30 - @Override 31 31 - public void onDismissCancelled() { 32 32 - finish(); 33 33 - } 34 34 - 35 35 - @Override 36 36 - public void onDismissError() { 37 37 - finish(); 38 38 - } 39 39 - }); 40 40 - } 41 41 - 42 42 - private void open() { 43 43 - launched = true; 44 44 - // Started now, while this activity is in front: from the background Android refuses it. 45 45 - startForegroundService(new Intent(this, CallService.class)); 46 46 - Link.open(this); 47 47 - } 48 48 - 49 49 - // Back from the page: this activity only sits below it in the task. 50 50 - @Override 51 51 - protected void onRestart() { 52 52 - super.onRestart(); 53 53 - if (launched) { 54 54 - finish(); 55 55 - } 14 14 + protected void onResume() { 15 15 + super.onResume(); 16 16 + startForegroundService(new Intent(this, CallService.class).setAction(CallService.ACTION_START)); 17 17 + finish(); 56 18 } 57 19 }

+154 -85

tools/hermes-voice/android/src/us/lassul/assist/CallService.java

View file Reviewed

··· 1 1 package us.lassul.assist; 2 2 3 3 + import android.Manifest; 3 4 import android.app.Notification; 4 5 import android.app.NotificationChannel; 5 6 import android.app.NotificationManager; 7 7 + import android.app.PendingIntent; 8 8 + import android.app.Person; 6 9 import android.app.Service; 7 10 import android.content.ComponentName; 8 11 import android.content.Intent; 12 12 + import android.content.pm.PackageManager; 9 13 import android.content.pm.ServiceInfo; 14 14 + import android.media.AudioManager; 15 15 + import android.media.ToneGenerator; 10 16 import android.net.Uri; 11 17 import android.os.Bundle; 12 18 import android.os.Handler; ··· 28 34 import java.util.function.Consumer; 29 35 30 36 /** 31 31 - * Holds each call of the page as a self-managed call in Telecom, so a Bluetooth headset sees a 32 32 - * call and its hang-up (AT+CHUP) reaches this app, which tells the page. In the foreground, as 33 33 - * otherwise Android freezes the process once the page is in front. Stops when idle: a call the 34 34 - * page starts after that is not held (the app cannot start this service from the background). 37 37 + * One call at a time, without a screen: a VoiceCall held as a self-managed Telecom call, so the 38 38 + * audio goes to the Bluetooth headset and its hang-up button (AT+CHUP) ends the call. In the 39 39 + * foreground with the microphone; stops when the call ends. A tone says the voice listens (or 40 40 + * that the call failed). 35 41 */ 36 42 public class CallService extends Service { 43 43 + static final String TAG = "Assist"; 44 44 + static final String ACTION_START = "us.lassul.assist.START"; 45 45 + static final String ACTION_HANG_UP = "us.lassul.assist.HANG_UP"; 37 46 private static final String CHANNEL = "call"; 38 38 - private static final long IDLE_MS = 2 * 60 * 1000; 47 47 + private static final int NOTIFICATION = 1; 39 48 40 40 - private static CallService running; 41 41 - 42 42 - private final Handler handler = new Handler(Looper.getMainLooper()); 43 43 - private final Runnable idleStop = new Runnable() { 44 44 - @Override 45 45 - public void run() { 46 46 - stopSelf(); 47 47 - } 48 48 - }; 49 49 + private final Handler main = new Handler(Looper.getMainLooper()); 49 50 private PhoneAccountHandle account; 50 50 - private CallControl call; 51 51 - // The page's call runs; false while adding means: disconnect once added. 52 52 - private boolean wanted; 51 51 + private CallControl telecom; 52 52 + private VoiceCall voice; 53 53 + // What the main screen shows (main thread); null without a call. 54 54 + private static String state; 53 55 54 54 - static void pageStarted() { 55 55 - if (running != null) { 56 56 - running.startCall(); 57 57 - } 56 56 + static String state() { 57 57 + return state; 58 58 } 59 59 60 60 - static void pageEnded() { 61 61 - if (running != null) { 62 62 - running.endCall(); 63 63 - } 60 60 + private static void setState(String next) { 61 61 + state = next; 62 62 + Live.changed(); 64 63 } 65 64 66 65 @Override 67 66 public void onCreate() { 68 67 super.onCreate(); 69 69 - running = this; 70 70 - NotificationManager notifications = getSystemService(NotificationManager.class); 71 71 - notifications.createNotificationChannel( 68 68 + getSystemService(NotificationManager.class).createNotificationChannel( 72 69 new NotificationChannel(CHANNEL, "Voice call", NotificationManager.IMPORTANCE_LOW)); 73 73 - Notification notification = new Notification.Builder(this, CHANNEL) 74 74 - .setSmallIcon(android.R.drawable.sym_action_call) 75 75 - .setContentTitle("Assist") 76 76 - .setContentText("The headset button hangs up") 77 77 - .setOngoing(true) 78 78 - .build(); 79 79 - startForeground(1, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_PHONE_CALL); 80 70 account = new PhoneAccountHandle(new ComponentName(this, CallService.class), "assist"); 81 71 getSystemService(TelecomManager.class).registerPhoneAccount( 82 72 PhoneAccount.builder(account, "Assist") ··· 88 78 89 79 @Override 90 80 public int onStartCommand(Intent intent, int flags, int startId) { 91 91 - if (call == null && !wanted) { 92 92 - idle(); 81 81 + String action = intent != null ? intent.getAction() : null; 82 82 + if (ACTION_HANG_UP.equals(action)) { 83 83 + if (voice != null) { 84 84 + voice.close(); 85 85 + } else { 86 86 + stopSelf(); 87 87 + } 88 88 + return START_NOT_STICKY; 93 89 } 90 90 + if (voice != null) { 91 91 + // A call runs already: the button press changes nothing. 92 92 + return START_NOT_STICKY; 93 93 + } 94 94 + Api api = Api.load(this); 95 95 + if (api == null 96 96 + || checkSelfPermission(Manifest.permission.RECORD_AUDIO) 97 97 + != PackageManager.PERMISSION_GRANTED) { 98 98 + // startForegroundService() needs startForeground(); the microphone type would be 99 99 + // refused without the permission. 100 100 + startForeground(NOTIFICATION, setupNotification(), 101 101 + ServiceInfo.FOREGROUND_SERVICE_TYPE_PHONE_CALL); 102 102 + Log.w(TAG, "not set up: open Assist"); 103 103 + tone(ToneGenerator.TONE_PROP_NACK); 104 104 + main.postDelayed(new Runnable() { 105 105 + @Override 106 106 + public void run() { 107 107 + stopSelf(); 108 108 + } 109 109 + }, 1000); 110 110 + return START_NOT_STICKY; 111 111 + } 112 112 + startForeground(NOTIFICATION, callNotification(), 113 113 + ServiceInfo.FOREGROUND_SERVICE_TYPE_PHONE_CALL 114 114 + | ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE); 115 115 + setState("Connecting…"); 116 116 + voice = new VoiceCall(this, api, Conversation.load(this), new VoiceCall.Listener() { 117 117 + @Override 118 118 + public void onListening() { 119 119 + setState("Listening"); 120 120 + tone(ToneGenerator.TONE_PROP_ACK); 121 121 + } 122 122 + 123 123 + @Override 124 124 + public void onFinished(String reason, boolean failed) { 125 125 + voice = null; 126 126 + setState(null); 127 127 + if (failed) { 128 128 + tone(ToneGenerator.TONE_PROP_NACK); 129 129 + } 130 130 + disconnectTelecom(); 131 131 + // Let a failure tone play before the service, and with it the call audio, goes. 132 132 + main.postDelayed(new Runnable() { 133 133 + @Override 134 134 + public void run() { 135 135 + if (voice == null) { 136 136 + stopSelf(); 137 137 + } 138 138 + } 139 139 + }, failed ? 1000 : 0); 140 140 + } 141 141 + }); 142 142 + addTelecomCall(); 94 143 return START_NOT_STICKY; 95 144 } 96 145 97 146 @Override 98 147 public void onDestroy() { 99 99 - handler.removeCallbacksAndMessages(null); 100 100 - wanted = false; 101 101 - disconnect(); 102 102 - Link.close(); 103 103 - running = null; 148 148 + main.removeCallbacksAndMessages(null); 149 149 + if (voice != null) { 150 150 + voice.close(); 151 151 + } 152 152 + disconnectTelecom(); 104 153 super.onDestroy(); 105 154 } 106 155 ··· 109 158 return null; 110 159 } 111 160 112 112 - private void idle() { 113 113 - handler.removeCallbacks(idleStop); 114 114 - handler.postDelayed(idleStop, IDLE_MS); 115 115 - } 116 116 - 117 117 - private void startCall() { 118 118 - handler.removeCallbacks(idleStop); 119 119 - if (wanted) { 120 120 - return; 121 121 - } 122 122 - wanted = true; 123 123 - if (call != null) { 124 124 - return; 125 125 - } 161 161 + /** The voice starts once Telecom has the call (audio routed), or without it if refused. */ 162 162 + private void addTelecomCall() { 126 163 CallAttributes attributes = new CallAttributes.Builder( 127 164 account, 128 165 CallAttributes.DIRECTION_OUTGOING, ··· 136 173 new OutcomeReceiver<CallControl, CallException>() { 137 174 @Override 138 175 public void onResult(CallControl control) { 139 139 - Log.i(Link.TAG, "telecom call added"); 140 140 - call = control; 141 141 - if (wanted) { 142 142 - control.setActive(getMainExecutor(), logged("setActive")); 143 143 - } else { 144 144 - disconnect(); 176 176 + if (voice == null) { 177 177 + control.disconnect(new DisconnectCause(DisconnectCause.LOCAL), 178 178 + getMainExecutor(), logged("disconnect")); 179 179 + return; 145 180 } 181 181 + telecom = control; 182 182 + control.setActive(getMainExecutor(), logged("setActive")); 183 183 + voice.start(); 146 184 } 147 185 148 186 @Override 149 187 public void onError(CallException error) { 150 150 - Log.w(Link.TAG, "telecom call refused", error); 151 151 - wanted = false; 152 152 - idle(); 188 188 + Log.w(TAG, "telecom refused the call; the headset cannot hang up", error); 189 189 + if (voice != null) { 190 190 + voice.start(); 191 191 + } 153 192 } 154 193 }, 155 194 new Control(), 156 195 new Events()); 157 196 } 158 197 159 159 - private void endCall() { 160 160 - wanted = false; 161 161 - disconnect(); 162 162 - idle(); 163 163 - } 164 164 - 165 165 - private void disconnect() { 166 166 - if (call != null) { 167 167 - call.disconnect( 168 168 - new DisconnectCause(DisconnectCause.LOCAL), getMainExecutor(), logged("disconnect")); 169 169 - call = null; 198 198 + private void disconnectTelecom() { 199 199 + if (telecom != null) { 200 200 + telecom.disconnect(new DisconnectCause(DisconnectCause.LOCAL), getMainExecutor(), 201 201 + logged("disconnect")); 202 202 + telecom = null; 170 203 } 171 204 } 172 205 173 173 - private static OutcomeReceiver<Void, CallException> logged(String what) { 206 206 + private void tone(int tone) { 207 207 + final ToneGenerator generator = new ToneGenerator(AudioManager.STREAM_VOICE_CALL, 80); 208 208 + generator.startTone(tone, 300); 209 209 + main.postDelayed(new Runnable() { 210 210 + @Override 211 211 + public void run() { 212 212 + generator.release(); 213 213 + } 214 214 + }, 500); 215 215 + } 216 216 + 217 217 + private Notification callNotification() { 218 218 + PendingIntent hangUp = PendingIntent.getService(this, 0, 219 219 + new Intent(this, CallService.class).setAction(ACTION_HANG_UP), 220 220 + PendingIntent.FLAG_IMMUTABLE); 221 221 + return new Notification.Builder(this, CHANNEL) 222 222 + .setSmallIcon(android.R.drawable.sym_action_call) 223 223 + .setStyle(Notification.CallStyle.forOngoingCall( 224 224 + new Person.Builder().setName("Hermes").build(), hangUp)) 225 225 + .setOngoing(true) 226 226 + .build(); 227 227 + } 228 228 + 229 229 + private Notification setupNotification() { 230 230 + PendingIntent setup = PendingIntent.getActivity(this, 0, 231 231 + new Intent(this, SettingsActivity.class), PendingIntent.FLAG_IMMUTABLE); 232 232 + return new Notification.Builder(this, CHANNEL) 233 233 + .setSmallIcon(android.R.drawable.stat_notify_error) 234 234 + .setContentTitle("Assist is not set up") 235 235 + .setContentText("Open it to enter the token and allow the microphone") 236 236 + .setContentIntent(setup) 237 237 + .build(); 238 238 + } 239 239 + 240 240 + private static OutcomeReceiver<Void, CallException> logged(final String what) { 174 241 return new OutcomeReceiver<Void, CallException>() { 175 242 @Override 176 243 public void onResult(Void result) {} 177 244 178 245 @Override 179 246 public void onError(CallException error) { 180 180 - Log.w(Link.TAG, what + " failed", error); 247 247 + Log.w(TAG, what + " failed", error); 181 248 } 182 249 }; 183 250 } ··· 198 265 done.accept(true); 199 266 } 200 267 201 201 - // Telecom ended the call: the headset hung up (AT+CHUP) or a call took over. 268 268 + // Telecom ended the call: the headset hung up (AT+CHUP) or another call took over. 202 269 @Override 203 270 public void onDisconnect(DisconnectCause cause, Consumer<Boolean> done) { 204 204 - Log.i(Link.TAG, "telecom disconnect: " + cause); 271 271 + Log.i(TAG, "telecom disconnect: " + cause); 205 272 done.accept(true); 206 206 - call = null; 207 207 - wanted = false; 208 208 - Link.hangUp(); 209 209 - idle(); 273 273 + telecom = null; 274 274 + if (voice != null) { 275 275 + voice.close(); 276 276 + } 210 277 } 211 278 212 279 @Override ··· 217 284 218 285 private static final class Events implements CallEventCallback { 219 286 @Override 220 220 - public void onCallEndpointChanged(CallEndpoint endpoint) {} 287 287 + public void onCallEndpointChanged(CallEndpoint endpoint) { 288 288 + Log.i(TAG, "audio to " + endpoint); 289 289 + } 221 290 222 291 @Override 223 292 public void onAvailableCallEndpointsChanged(List<CallEndpoint> endpoints) {}

+270

tools/hermes-voice/android/src/us/lassul/assist/Conversation.java

View file Reviewed

··· 1 1 + package us.lassul.assist; 2 2 + 3 3 + import android.content.Context; 4 4 + import android.content.SharedPreferences; 5 5 + import android.graphics.Color; 6 6 + import android.graphics.Typeface; 7 7 + import android.text.SpannableStringBuilder; 8 8 + import android.text.Spanned; 9 9 + import android.text.style.ForegroundColorSpan; 10 10 + import android.text.style.StyleSpan; 11 11 + import android.util.AtomicFile; 12 12 + import android.util.Log; 13 13 + import java.io.File; 14 14 + import java.io.FileOutputStream; 15 15 + import java.io.IOException; 16 16 + import java.nio.charset.StandardCharsets; 17 17 + import java.security.SecureRandom; 18 18 + import java.text.SimpleDateFormat; 19 19 + import java.util.ArrayList; 20 20 + import java.util.Arrays; 21 21 + import java.util.Collections; 22 22 + import java.util.Comparator; 23 23 + import java.util.Date; 24 24 + import java.util.List; 25 25 + import java.util.Locale; 26 26 + import java.util.TimeZone; 27 27 + import org.json.JSONArray; 28 28 + import org.json.JSONException; 29 29 + import org.json.JSONObject; 30 30 + 31 31 + /** 32 32 + * A conversation with Hermes across calls, like the web page's localStorage state: the 33 33 + * transcript, how much of it went to Hermes already, and Hermes's session id. Every conversation 34 34 + * is a file in conversations/; the current one continues with the next call unless the last call 35 35 + * is over an hour old or "New conversation" was chosen. 36 36 + */ 37 37 + final class Conversation { 38 38 + static final int ACCENT = Color.parseColor("#4fb477"); 39 39 + static final int LIVE = Color.parseColor("#d9534f"); 40 40 + static final int DIM = Color.parseColor("#8a8a8a"); 41 41 + private static final long NEW_AFTER_MS = 60 * 60 * 1000; 42 42 + private static final String PREFS = "conversation"; 43 43 + 44 44 + static final class Line { 45 45 + final String speaker; 46 46 + String text; 47 47 + 48 48 + Line(String speaker, String text) { 49 49 + this.speaker = speaker; 50 50 + this.text = text; 51 51 + } 52 52 + } 53 53 + 54 54 + final String hermesSession; 55 55 + final long started; 56 56 + final List<Line> lines = new ArrayList<>(); 57 57 + // Lines before this index went to Hermes (delegation or hang-up) already. 58 58 + int delegated; 59 59 + double billedSeconds; 60 60 + private final Context context; 61 61 + 62 62 + private Conversation(Context context, String hermesSession, long started) { 63 63 + this.context = context.getApplicationContext(); 64 64 + this.hermesSession = hermesSession; 65 65 + this.started = started; 66 66 + } 67 67 + 68 68 + /** The conversation the next call continues. */ 69 69 + static Conversation load(Context context) { 70 70 + SharedPreferences prefs = prefs(context); 71 71 + String current = prefs.getString("current", null); 72 72 + if (current != null 73 73 + && System.currentTimeMillis() - prefs.getLong("last_used", 0) < NEW_AFTER_MS) { 74 74 + Conversation saved = read(context, current); 75 75 + if (saved != null) { 76 76 + return saved; 77 77 + } 78 78 + } 79 79 + return new Conversation(context, newSessionId(), System.currentTimeMillis()); 80 80 + } 81 81 + 82 82 + /** The conversation shown on the main screen: the current one, however old; null if none. */ 83 83 + static Conversation current(Context context) { 84 84 + String current = prefs(context).getString("current", null); 85 85 + return current == null ? null : read(context, current); 86 86 + } 87 87 + 88 88 + /** The next call starts a new conversation (and Hermes session); the old one stays listed. */ 89 89 + static void startNew(Context context) { 90 90 + prefs(context).edit().remove("current").apply(); 91 91 + Live.changed(); 92 92 + } 93 93 + 94 94 + /** All conversations, newest first. */ 95 95 + static List<Conversation> all(Context context) { 96 96 + File[] files = directory(context).listFiles(); 97 97 + List<Conversation> out = new ArrayList<>(); 98 98 + for (File file : files == null ? new File[0] : files) { 99 99 + String name = file.getName(); 100 100 + if (name.endsWith(".json")) { 101 101 + Conversation conversation = 102 102 + read(context, name.substring(0, name.length() - ".json".length())); 103 103 + if (conversation != null) { 104 104 + out.add(conversation); 105 105 + } 106 106 + } 107 107 + } 108 108 + Collections.sort(out, new Comparator<Conversation>() { 109 109 + @Override 110 110 + public int compare(Conversation a, Conversation b) { 111 111 + return Long.compare(b.started, a.started); 112 112 + } 113 113 + }); 114 114 + return out; 115 115 + } 116 116 + 117 117 + static Conversation read(Context context, String hermesSession) { 118 118 + File file = file(context, hermesSession); 119 119 + if (!file.exists()) { 120 120 + return null; 121 121 + } 122 122 + try { 123 123 + JSONObject state = new JSONObject( 124 124 + new String(new AtomicFile(file).readFully(), StandardCharsets.UTF_8)); 125 125 + Conversation conversation = new Conversation(context, hermesSession, 126 126 + state.optLong("started", file.lastModified())); 127 127 + JSONArray lines = state.getJSONArray("lines"); 128 128 + for (int i = 0; i < lines.length(); i++) { 129 129 + JSONObject line = lines.getJSONObject(i); 130 130 + conversation.lines.add(new Line(line.getString("speaker"), line.getString("text"))); 131 131 + } 132 132 + conversation.delegated = state.optInt("delegated", conversation.lines.size()); 133 133 + conversation.billedSeconds = state.optDouble("billed_seconds", 0); 134 134 + return conversation; 135 135 + } catch (IOException | JSONException error) { 136 136 + Log.w(CallService.TAG, "unreadable conversation " + hermesSession, error); 137 137 + return null; 138 138 + } 139 139 + } 140 140 + 141 141 + /** voice-YYYYMMDD-<hex>, like the web page's. */ 142 142 + private static String newSessionId() { 143 143 + SimpleDateFormat day = new SimpleDateFormat("yyyyMMdd", Locale.ROOT); 144 144 + day.setTimeZone(TimeZone.getTimeZone("UTC")); 145 145 + String random = Integer.toHexString(new SecureRandom().nextInt()); 146 146 + return "voice-" + day.format(new Date()) + "-" + random; 147 147 + } 148 148 + 149 149 + void save() { 150 150 + AtomicFile file = new AtomicFile(file(context, hermesSession)); 151 151 + FileOutputStream out = null; 152 152 + try { 153 153 + JSONObject state = new JSONObject() 154 154 + .put("started", started) 155 155 + .put("lines", toJson(lines)) 156 156 + .put("delegated", delegated) 157 157 + .put("billed_seconds", billedSeconds); 158 158 + directory(context).mkdirs(); 159 159 + out = file.startWrite(); 160 160 + out.write(state.toString().getBytes(StandardCharsets.UTF_8)); 161 161 + file.finishWrite(out); 162 162 + } catch (IOException | JSONException error) { 163 163 + file.failWrite(out); 164 164 + Log.w(CallService.TAG, "conversation not saved", error); 165 165 + return; 166 166 + } 167 167 + prefs(context).edit() 168 168 + .putString("current", hermesSession) 169 169 + .putLong("last_used", System.currentTimeMillis()) 170 170 + .apply(); 171 171 + Live.changed(); 172 172 + } 173 173 + 174 174 + void add(String speaker, String text) { 175 175 + lines.add(new Line(speaker, text)); 176 176 + save(); 177 177 + } 178 178 + 179 179 + /** Lines since the last hand-over to Hermes, of the given speakers. */ 180 180 + List<Line> undelegated(String... speakers) { 181 181 + List<Line> out = new ArrayList<>(); 182 182 + List<String> wanted = Arrays.asList(speakers); 183 183 + for (Line line : lines.subList(Math.min(delegated, lines.size()), lines.size())) { 184 184 + if (wanted.contains(line.speaker)) { 185 185 + out.add(line); 186 186 + } 187 187 + } 188 188 + return out; 189 189 + } 190 190 + 191 191 + /** The first thing the user said, for the session list. */ 192 192 + String title() { 193 193 + for (Line line : lines) { 194 194 + if (line.speaker.equals("user") && !line.text.trim().isEmpty()) { 195 195 + return line.text.trim(); 196 196 + } 197 197 + } 198 198 + return "(nothing said)"; 199 199 + } 200 200 + 201 201 + /** The transcript as the web page shows it: colored speaker labels, Hermes dimmed. */ 202 202 + CharSequence render() { 203 203 + SpannableStringBuilder out = new SpannableStringBuilder(); 204 204 + for (Line line : lines) { 205 205 + String text = line.text.trim(); 206 206 + if (text.isEmpty()) { 207 207 + continue; 208 208 + } 209 209 + if (out.length() > 0) { 210 210 + out.append("\n\n"); 211 211 + } 212 212 + String label; 213 213 + int labelColor; 214 214 + int textColor = 0; 215 215 + switch (line.speaker) { 216 216 + case "user": 217 217 + label = "you: "; 218 218 + labelColor = ACCENT; 219 219 + break; 220 220 + case "assistant": 221 221 + label = "voice: "; 222 222 + labelColor = DIM; 223 223 + break; 224 224 + case "error": 225 225 + label = "error: "; 226 226 + labelColor = LIVE; 227 227 + textColor = LIVE; 228 228 + break; 229 229 + default: 230 230 + label = line.speaker + ": "; 231 231 + labelColor = ACCENT; 232 232 + textColor = DIM; 233 233 + break; 234 234 + } 235 235 + int start = out.length(); 236 236 + out.append(label); 237 237 + out.setSpan(new ForegroundColorSpan(labelColor), start, out.length(), 238 238 + Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); 239 239 + out.setSpan(new StyleSpan(Typeface.BOLD), start, out.length(), 240 240 + Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); 241 241 + int body = out.length(); 242 242 + out.append(text); 243 243 + if (textColor != 0) { 244 244 + out.setSpan(new ForegroundColorSpan(textColor), body, out.length(), 245 245 + Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); 246 246 + } 247 247 + } 248 248 + return out; 249 249 + } 250 250 + 251 251 + static JSONArray toJson(List<Line> lines) throws JSONException { 252 252 + JSONArray out = new JSONArray(); 253 253 + for (Line line : lines) { 254 254 + out.put(new JSONObject().put("speaker", line.speaker).put("text", line.text)); 255 255 + } 256 256 + return out; 257 257 + } 258 258 + 259 259 + private static SharedPreferences prefs(Context context) { 260 260 + return context.getSharedPreferences(PREFS, Context.MODE_PRIVATE); 261 261 + } 262 262 + 263 263 + private static File directory(Context context) { 264 264 + return new File(context.getFilesDir(), "conversations"); 265 265 + } 266 266 + 267 267 + private static File file(Context context, String hermesSession) { 268 268 + return new File(directory(context), hermesSession + ".json"); 269 269 + } 270 270 + }

-157

tools/hermes-voice/android/src/us/lassul/assist/Link.java

Reviewed

··· 1 1 - package us.lassul.assist; 2 2 - 3 3 - import android.app.Activity; 4 4 - import android.content.ComponentName; 5 5 - import android.content.Context; 6 6 - import android.content.Intent; 7 7 - import android.content.pm.PackageManager; 8 8 - import android.content.pm.ResolveInfo; 9 9 - import android.net.Uri; 10 10 - import android.os.Bundle; 11 11 - import android.util.Log; 12 12 - import androidx.browser.customtabs.CustomTabsCallback; 13 13 - import androidx.browser.customtabs.CustomTabsClient; 14 14 - import androidx.browser.customtabs.CustomTabsIntent; 15 15 - import androidx.browser.customtabs.CustomTabsService; 16 16 - import androidx.browser.customtabs.CustomTabsServiceConnection; 17 17 - import androidx.browser.customtabs.CustomTabsSession; 18 18 - import androidx.browser.trusted.TrustedWebActivityIntentBuilder; 19 19 - 20 20 - /** 21 21 - * The voice page, opened as a Trusted Web Activity, else as a Custom Tab, and its postMessage 22 22 - * channel: the page reports "started" and "ended" for each call, the app sends "hangup". One per 23 23 - * process, used on the main thread only (androidx delivers the callbacks there). 24 24 - */ 25 25 - final class Link { 26 26 - static final String TAG = "Assist"; 27 27 - // The app speaks as this origin; assetlinks.json grants it (use_as_origin). 28 28 - private static final Uri ORIGIN = Uri.parse("https://assist.lassul.us"); 29 29 - private static final Uri URL = Uri.parse("https://assist.lassul.us/?listen"); 30 30 - 31 31 - private static Context app; 32 32 - private static CustomTabsServiceConnection connection; 33 33 - private static CustomTabsSession session; 34 34 - private static boolean twa; 35 35 - // Opened before the browser's service connected: launched once it is. 36 36 - private static Activity pending; 37 37 - 38 38 - private static final CustomTabsCallback CALLBACK = new CustomTabsCallback() { 39 39 - @Override 40 40 - public void onRelationshipValidationResult( 41 41 - int relation, Uri origin, boolean result, Bundle extras) { 42 42 - Log.i(TAG, "use_as_origin " + origin + ": " + result); 43 43 - } 44 44 - 45 45 - // Every page load needs its own channel; the page takes the port from the first message. 46 46 - @Override 47 47 - public void onNavigationEvent(int event, Bundle extras) { 48 48 - if (event == NAVIGATION_FINISHED && session != null) { 49 49 - boolean requested = session.requestPostMessageChannel(ORIGIN, ORIGIN, new Bundle()); 50 50 - Log.i(TAG, "postMessage channel requested: " + requested); 51 51 - } 52 52 - } 53 53 - 54 54 - @Override 55 55 - public void onMessageChannelReady(Bundle extras) { 56 56 - Log.i(TAG, "postMessage channel ready: " + session.postMessage("hello", null)); 57 57 - } 58 58 - 59 59 - @Override 60 60 - public void onPostMessage(String message, Bundle extras) { 61 61 - Log.i(TAG, "page: " + message); 62 62 - if ("started".equals(message)) { 63 63 - CallService.pageStarted(); 64 64 - } else if ("ended".equals(message)) { 65 65 - CallService.pageEnded(); 66 66 - } 67 67 - } 68 68 - }; 69 69 - 70 70 - private Link() {} 71 71 - 72 72 - static void open(Activity activity) { 73 73 - if (session != null) { 74 74 - launch(activity); 75 75 - return; 76 76 - } 77 77 - pending = activity; 78 78 - if (connection != null) { 79 79 - return; 80 80 - } 81 81 - String provider = twaProvider(activity); 82 82 - twa = provider != null; 83 83 - if (provider == null) { 84 84 - provider = CustomTabsClient.getPackageName(activity, null); 85 85 - } 86 86 - app = activity.getApplicationContext(); 87 87 - connection = new CustomTabsServiceConnection() { 88 88 - @Override 89 89 - public void onCustomTabsServiceConnected(ComponentName name, CustomTabsClient client) { 90 90 - client.warmup(0); 91 91 - session = client.newSession(CALLBACK); 92 92 - if (session != null) { 93 93 - session.validateRelationship( 94 94 - CustomTabsService.RELATION_USE_AS_ORIGIN, ORIGIN, null); 95 95 - } 96 96 - Activity waiting = pending; 97 97 - pending = null; 98 98 - if (waiting != null && !waiting.isFinishing()) { 99 99 - launch(waiting); 100 100 - } 101 101 - } 102 102 - 103 103 - @Override 104 104 - public void onServiceDisconnected(ComponentName name) { 105 105 - session = null; 106 106 - } 107 107 - }; 108 108 - if (provider == null || !CustomTabsClient.bindCustomTabsService(app, provider, connection)) { 109 109 - connection = null; 110 110 - pending = null; 111 111 - launch(activity); 112 112 - } 113 113 - } 114 114 - 115 115 - /** Sends "hangup" to the page, if it is connected. */ 116 116 - static void hangUp() { 117 117 - if (session != null) { 118 118 - Log.i(TAG, "hangup sent: " + session.postMessage("hangup", null)); 119 119 - } 120 120 - } 121 121 - 122 122 - static void close() { 123 123 - if (connection != null) { 124 124 - app.unbindService(connection); 125 125 - } 126 126 - connection = null; 127 127 - session = null; 128 128 - pending = null; 129 129 - } 130 130 - 131 131 - /** In the session's browser; without one (no Custom Tabs browser) a plain Custom Tab. */ 132 132 - private static void launch(Activity activity) { 133 133 - Intent intent; 134 134 - if (session == null) { 135 135 - intent = new CustomTabsIntent.Builder().build().intent.setData(URL); 136 136 - } else if (twa) { 137 137 - intent = new TrustedWebActivityIntentBuilder(URL).build(session).getIntent(); 138 138 - } else { 139 139 - intent = new CustomTabsIntent.Builder(session).build().intent.setData(URL); 140 140 - } 141 141 - activity.startActivity(intent); 142 142 - } 143 143 - 144 144 - /** The first browser offering Trusted Web Activities, null if none does. */ 145 145 - private static String twaProvider(Context context) { 146 146 - Intent intent = new Intent(CustomTabsService.ACTION_CUSTOM_TABS_CONNECTION); 147 147 - for (ResolveInfo info : 148 148 - context.getPackageManager() 149 149 - .queryIntentServices(intent, PackageManager.GET_RESOLVED_FILTER)) { 150 150 - if (info.filter != null 151 151 - && info.filter.hasCategory(CustomTabsService.TRUSTED_WEB_ACTIVITY_CATEGORY)) { 152 152 - return info.serviceInfo.packageName; 153 153 - } 154 154 - } 155 155 - return null; 156 156 - } 157 157 - }

+28

tools/hermes-voice/android/src/us/lassul/assist/Live.java

View file Reviewed

··· 1 1 + package us.lassul.assist; 2 2 + 3 3 + import java.util.ArrayList; 4 4 + import java.util.List; 5 5 + 6 6 + /** 7 7 + * Tells the screens that the call state or a conversation changed, so they redraw while a call 8 8 + * runs. Main thread only; the app's screens and CallService share the process. 9 9 + */ 10 10 + final class Live { 11 11 + private static final List<Runnable> watchers = new ArrayList<>(); 12 12 + 13 13 + private Live() {} 14 14 + 15 15 + static void watch(Runnable watcher) { 16 16 + watchers.add(watcher); 17 17 + } 18 18 + 19 19 + static void unwatch(Runnable watcher) { 20 20 + watchers.remove(watcher); 21 21 + } 22 22 + 23 23 + static void changed() { 24 24 + for (Runnable watcher : new ArrayList<>(watchers)) { 25 25 + watcher.run(); 26 26 + } 27 27 + } 28 28 + }

+166

tools/hermes-voice/android/src/us/lassul/assist/MainActivity.java

View file Reviewed

··· 1 1 + package us.lassul.assist; 2 2 + 3 3 + import android.Manifest; 4 4 + import android.app.Activity; 5 5 + import android.content.Intent; 6 6 + import android.content.pm.PackageManager; 7 7 + import android.content.res.ColorStateList; 8 8 + import android.graphics.Typeface; 9 9 + import android.graphics.drawable.GradientDrawable; 10 10 + import android.os.Bundle; 11 11 + import android.view.Gravity; 12 12 + import android.view.Menu; 13 13 + import android.view.MenuItem; 14 14 + import android.view.View; 15 15 + import android.widget.Button; 16 16 + import android.widget.LinearLayout; 17 17 + import android.widget.ScrollView; 18 18 + import android.widget.TextView; 19 19 + import android.widget.Toast; 20 20 + 21 21 + /** 22 22 + * The launcher screen: the current conversation, live during a call, and a Call / Hang up 23 23 + * button. New conversation, the session list and the settings are in the menu. 24 24 + */ 25 25 + public class MainActivity extends Activity { 26 26 + private static final int NEW_CONVERSATION = 1; 27 27 + private static final int SESSIONS = 2; 28 28 + private static final int SETTINGS = 3; 29 29 + 30 30 + private TextView transcript; 31 31 + private ColorStateList textColor; 32 32 + private ScrollView scroll; 33 33 + private TextView status; 34 34 + private Button call; 35 35 + private final Runnable redraw = new Runnable() { 36 36 + @Override 37 37 + public void run() { 38 38 + refresh(); 39 39 + } 40 40 + }; 41 41 + 42 42 + @Override 43 43 + protected void onCreate(Bundle state) { 44 44 + super.onCreate(state); 45 45 + float dp = getResources().getDisplayMetrics().density; 46 46 + LinearLayout layout = new LinearLayout(this); 47 47 + layout.setOrientation(LinearLayout.VERTICAL); 48 48 + Ui.fitBars(this, layout, Ui.dp(this, 16)); 49 49 + 50 50 + transcript = new TextView(this); 51 51 + transcript.setTextSize(16); 52 52 + transcript.setLineSpacing(0, 1.15f); 53 53 + transcript.setTextIsSelectable(true); 54 54 + textColor = transcript.getTextColors(); 55 55 + scroll = new ScrollView(this); 56 56 + scroll.addView(transcript); 57 57 + layout.addView(scroll, 58 58 + new LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, 0, 1)); 59 59 + 60 60 + status = new TextView(this); 61 61 + status.setGravity(Gravity.CENTER); 62 62 + status.setTextColor(Conversation.DIM); 63 63 + status.setPadding(0, (int) (12 * dp), 0, (int) (12 * dp)); 64 64 + layout.addView(status); 65 65 + 66 66 + call = new Button(this); 67 67 + call.setTextColor(0xffffffff); 68 68 + call.setTextSize(20); 69 69 + call.setTypeface(Typeface.DEFAULT_BOLD); 70 70 + call.setAllCaps(false); 71 71 + call.setOnClickListener(new View.OnClickListener() { 72 72 + @Override 73 73 + public void onClick(View view) { 74 74 + if (CallService.state() != null) { 75 75 + startService(new Intent(MainActivity.this, CallService.class) 76 76 + .setAction(CallService.ACTION_HANG_UP)); 77 77 + } else if (!ready()) { 78 78 + startActivity(new Intent(MainActivity.this, SettingsActivity.class)); 79 79 + } else { 80 80 + startForegroundService(new Intent(MainActivity.this, CallService.class) 81 81 + .setAction(CallService.ACTION_START)); 82 82 + } 83 83 + } 84 84 + }); 85 85 + layout.addView(call, new LinearLayout.LayoutParams( 86 86 + LinearLayout.LayoutParams.MATCH_PARENT, (int) (64 * dp))); 87 87 + setContentView(layout); 88 88 + } 89 89 + 90 90 + @Override 91 91 + protected void onResume() { 92 92 + super.onResume(); 93 93 + Live.watch(redraw); 94 94 + refresh(); 95 95 + } 96 96 + 97 97 + @Override 98 98 + protected void onPause() { 99 99 + Live.unwatch(redraw); 100 100 + super.onPause(); 101 101 + } 102 102 + 103 103 + @Override 104 104 + public boolean onCreateOptionsMenu(Menu menu) { 105 105 + menu.add(Menu.NONE, NEW_CONVERSATION, 0, "New conversation"); 106 106 + menu.add(Menu.NONE, SESSIONS, 1, "Sessions"); 107 107 + menu.add(Menu.NONE, SETTINGS, 2, "Settings"); 108 108 + return true; 109 109 + } 110 110 + 111 111 + @Override 112 112 + public boolean onOptionsItemSelected(MenuItem item) { 113 113 + switch (item.getItemId()) { 114 114 + case NEW_CONVERSATION: 115 115 + if (CallService.state() != null) { 116 116 + // The running call would save its conversation right back. 117 117 + Toast.makeText(this, "Hang up first", Toast.LENGTH_SHORT).show(); 118 118 + } else { 119 119 + Conversation.startNew(this); 120 120 + } 121 121 + return true; 122 122 + case SESSIONS: 123 123 + startActivity(new Intent(this, SessionsActivity.class)); 124 124 + return true; 125 125 + case SETTINGS: 126 126 + startActivity(new Intent(this, SettingsActivity.class)); 127 127 + return true; 128 128 + default: 129 129 + return super.onOptionsItemSelected(item); 130 130 + } 131 131 + } 132 132 + 133 133 + private boolean ready() { 134 134 + return Api.hasToken(this) 135 135 + && checkSelfPermission(Manifest.permission.RECORD_AUDIO) 136 136 + == PackageManager.PERMISSION_GRANTED; 137 137 + } 138 138 + 139 139 + private void refresh() { 140 140 + String state = CallService.state(); 141 141 + boolean ready = ready(); 142 142 + GradientDrawable background = new GradientDrawable(); 143 143 + background.setCornerRadius(32 * getResources().getDisplayMetrics().density); 144 144 + background.setColor(state != null ? Conversation.LIVE : Conversation.ACCENT); 145 145 + call.setBackground(background); 146 146 + call.setText(state != null ? "Hang up" : ready ? "Call" : "Set up"); 147 147 + status.setText(state != null 148 148 + ? state 149 149 + : ready ? "Hold the headset's voice button, or tap Call" : "Not set up yet"); 150 150 + 151 151 + Conversation current = Conversation.current(this); 152 152 + if (current == null || current.lines.isEmpty()) { 153 153 + transcript.setText("New conversation. What you and Hermes say shows up here."); 154 154 + transcript.setTextColor(Conversation.DIM); 155 155 + } else { 156 156 + transcript.setText(current.render()); 157 157 + transcript.setTextColor(textColor); 158 158 + } 159 159 + scroll.post(new Runnable() { 160 160 + @Override 161 161 + public void run() { 162 162 + scroll.fullScroll(View.FOCUS_DOWN); 163 163 + } 164 164 + }); 165 165 + } 166 166 + }

+95

tools/hermes-voice/android/src/us/lassul/assist/SessionsActivity.java

View file Reviewed

··· 1 1 + package us.lassul.assist; 2 2 + 3 3 + import android.app.Activity; 4 4 + import android.content.Intent; 5 5 + import android.os.Bundle; 6 6 + import android.text.format.DateUtils; 7 7 + import android.view.Gravity; 8 8 + import android.view.View; 9 9 + import android.view.ViewGroup; 10 10 + import android.widget.AdapterView; 11 11 + import android.widget.ArrayAdapter; 12 12 + import android.widget.FrameLayout; 13 13 + import android.widget.ListView; 14 14 + import android.widget.TextView; 15 15 + import java.util.List; 16 16 + 17 17 + /** All conversations, newest first; one opens its transcript. */ 18 18 + public class SessionsActivity extends Activity { 19 19 + private ListView list; 20 20 + private final Runnable redraw = new Runnable() { 21 21 + @Override 22 22 + public void run() { 23 23 + refresh(); 24 24 + } 25 25 + }; 26 26 + 27 27 + @Override 28 28 + protected void onCreate(Bundle state) { 29 29 + super.onCreate(state); 30 30 + getActionBar().setDisplayHomeAsUpEnabled(true); 31 31 + list = new ListView(this); 32 32 + list.setOnItemClickListener(new AdapterView.OnItemClickListener() { 33 33 + @Override 34 34 + public void onItemClick(AdapterView<?> parent, View view, int position, long id) { 35 35 + Conversation conversation = (Conversation) parent.getItemAtPosition(position); 36 36 + startActivity(new Intent(SessionsActivity.this, TranscriptActivity.class) 37 37 + .putExtra(TranscriptActivity.SESSION, conversation.hermesSession)); 38 38 + } 39 39 + }); 40 40 + TextView empty = new TextView(this); 41 41 + empty.setText("No conversations yet."); 42 42 + empty.setTextColor(Conversation.DIM); 43 43 + empty.setGravity(Gravity.CENTER); 44 44 + FrameLayout root = new FrameLayout(this); 45 45 + root.addView(list); 46 46 + root.addView(empty); 47 47 + list.setEmptyView(empty); 48 48 + Ui.fitBars(this, root, 0); 49 49 + setContentView(root); 50 50 + } 51 51 + 52 52 + @Override 53 53 + protected void onResume() { 54 54 + super.onResume(); 55 55 + Live.watch(redraw); 56 56 + refresh(); 57 57 + } 58 58 + 59 59 + @Override 60 60 + protected void onPause() { 61 61 + Live.unwatch(redraw); 62 62 + super.onPause(); 63 63 + } 64 64 + 65 65 + @Override 66 66 + public boolean onNavigateUp() { 67 67 + finish(); 68 68 + return true; 69 69 + } 70 70 + 71 71 + private void refresh() { 72 72 + List<Conversation> conversations = Conversation.all(this); 73 73 + Conversation current = Conversation.current(this); 74 74 + final String currentSession = current == null ? null : current.hermesSession; 75 75 + list.setAdapter(new ArrayAdapter<Conversation>( 76 76 + this, android.R.layout.simple_list_item_2, android.R.id.text1, conversations) { 77 77 + @Override 78 78 + public View getView(int position, View convertView, ViewGroup parent) { 79 79 + View row = super.getView(position, convertView, parent); 80 80 + Conversation conversation = getItem(position); 81 81 + String when = DateUtils.formatDateTime(getContext(), conversation.started, 82 82 + DateUtils.FORMAT_SHOW_DATE | DateUtils.FORMAT_SHOW_TIME 83 83 + | DateUtils.FORMAT_ABBREV_MONTH); 84 84 + ((TextView) row.findViewById(android.R.id.text1)).setText( 85 85 + conversation.hermesSession.equals(currentSession) 86 86 + ? when + " · current" 87 87 + : when); 88 88 + TextView title = row.findViewById(android.R.id.text2); 89 89 + title.setText(conversation.title()); 90 90 + title.setMaxLines(2); 91 91 + return row; 92 92 + } 93 93 + }); 94 94 + } 95 95 + }

+110

tools/hermes-voice/android/src/us/lassul/assist/SettingsActivity.java

View file Reviewed

··· 1 1 + package us.lassul.assist; 2 2 + 3 3 + import android.Manifest; 4 4 + import android.app.Activity; 5 5 + import android.content.pm.PackageManager; 6 6 + import android.os.Bundle; 7 7 + import android.text.InputType; 8 8 + import android.view.View; 9 9 + import android.widget.Button; 10 10 + import android.widget.EditText; 11 11 + import android.widget.LinearLayout; 12 12 + import android.widget.ScrollView; 13 13 + import android.widget.TextView; 14 14 + 15 15 + /** 16 16 + * Setup: the app token (`clan vars get coaxmetal hermes-voice-app/token`) and the microphone and 17 17 + * notification permissions. 18 18 + */ 19 19 + public class SettingsActivity extends Activity { 20 20 + private TextView status; 21 21 + private EditText token; 22 22 + private Button permissions; 23 23 + 24 24 + @Override 25 25 + protected void onCreate(Bundle state) { 26 26 + super.onCreate(state); 27 27 + getActionBar().setDisplayHomeAsUpEnabled(true); 28 28 + int pad = (int) (16 * getResources().getDisplayMetrics().density); 29 29 + LinearLayout layout = new LinearLayout(this); 30 30 + layout.setOrientation(LinearLayout.VERTICAL); 31 31 + 32 32 + status = new TextView(this); 33 33 + status.setPadding(0, 0, 0, pad); 34 34 + layout.addView(status); 35 35 + 36 36 + token = new EditText(this); 37 37 + token.setHint("App token"); 38 38 + token.setInputType(InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_VARIATION_PASSWORD); 39 39 + // A password manager's popup over the field swallows the tap on Save. 40 40 + token.setImportantForAutofill(View.IMPORTANT_FOR_AUTOFILL_NO); 41 41 + layout.addView(token); 42 42 + layout.addView(button("Save token", new View.OnClickListener() { 43 43 + @Override 44 44 + public void onClick(View view) { 45 45 + if (token.getText().toString().trim().isEmpty()) { 46 46 + return; 47 47 + } 48 48 + Api.saveToken(SettingsActivity.this, token.getText().toString()); 49 49 + token.setText(""); 50 50 + refresh(); 51 51 + } 52 52 + })); 53 53 + 54 54 + permissions = button("Allow microphone and notifications", new View.OnClickListener() { 55 55 + @Override 56 56 + public void onClick(View view) { 57 57 + requestPermissions(new String[] { 58 58 + Manifest.permission.RECORD_AUDIO, Manifest.permission.POST_NOTIFICATIONS 59 59 + }, 0); 60 60 + } 61 61 + }); 62 62 + layout.addView(permissions); 63 63 + 64 64 + TextView help = new TextView(this); 65 65 + help.setPadding(0, pad, 0, 0); 66 66 + help.setTextColor(Conversation.DIM); 67 67 + help.setText("The token: clan vars get coaxmetal hermes-voice-app/token\n\n" 68 68 + + "Hold the headset's voice button to talk to Hermes, a short press hangs up;" 69 69 + + " a tone says it listens. Pick Assist when Android asks which app handles voice" 70 70 + + " commands. On GrapheneOS, Assist also needs the Network permission."); 71 71 + layout.addView(help); 72 72 + 73 73 + ScrollView scroll = new ScrollView(this); 74 74 + scroll.addView(layout); 75 75 + Ui.fitBars(this, scroll, pad); 76 76 + setContentView(scroll); 77 77 + } 78 78 + 79 79 + @Override 80 80 + protected void onResume() { 81 81 + super.onResume(); 82 82 + refresh(); 83 83 + } 84 84 + 85 85 + @Override 86 86 + public void onRequestPermissionsResult(int code, String[] permissions, int[] results) { 87 87 + refresh(); 88 88 + } 89 89 + 90 90 + @Override 91 91 + public boolean onNavigateUp() { 92 92 + finish(); 93 93 + return true; 94 94 + } 95 95 + 96 96 + private void refresh() { 97 97 + boolean mic = checkSelfPermission(Manifest.permission.RECORD_AUDIO) 98 98 + == PackageManager.PERMISSION_GRANTED; 99 99 + status.setText("Token: " + (Api.hasToken(this) ? "set" : "missing") 100 100 + + "\nMicrophone: " + (mic ? "allowed" : "not allowed")); 101 101 + permissions.setVisibility(mic ? View.GONE : View.VISIBLE); 102 102 + } 103 103 + 104 104 + private Button button(String label, View.OnClickListener action) { 105 105 + Button button = new Button(this); 106 106 + button.setText(label); 107 107 + button.setOnClickListener(action); 108 108 + return button; 109 109 + } 110 110 + }

+43

tools/hermes-voice/android/src/us/lassul/assist/TranscriptActivity.java

View file Reviewed

··· 1 1 + package us.lassul.assist; 2 2 + 3 3 + import android.app.Activity; 4 4 + import android.os.Bundle; 5 5 + import android.text.format.DateUtils; 6 6 + import android.widget.ScrollView; 7 7 + import android.widget.TextView; 8 8 + 9 9 + /** One conversation's transcript, read-only. */ 10 10 + public class TranscriptActivity extends Activity { 11 11 + static final String SESSION = "session"; 12 12 + 13 13 + @Override 14 14 + protected void onCreate(Bundle state) { 15 15 + super.onCreate(state); 16 16 + getActionBar().setDisplayHomeAsUpEnabled(true); 17 17 + Conversation conversation = Conversation.read(this, getIntent().getStringExtra(SESSION)); 18 18 + if (conversation == null) { 19 19 + finish(); 20 20 + return; 21 21 + } 22 22 + setTitle(DateUtils.formatDateTime(this, conversation.started, 23 23 + DateUtils.FORMAT_SHOW_DATE | DateUtils.FORMAT_SHOW_TIME 24 24 + | DateUtils.FORMAT_ABBREV_MONTH)); 25 25 + int pad = (int) (16 * getResources().getDisplayMetrics().density); 26 26 + TextView transcript = new TextView(this); 27 27 + transcript.setTextSize(16); 28 28 + transcript.setLineSpacing(0, 1.15f); 29 29 + transcript.setTextIsSelectable(true); 30 30 + transcript.setPadding(pad, pad, pad, pad); 31 31 + transcript.setText(conversation.render()); 32 32 + ScrollView scroll = new ScrollView(this); 33 33 + scroll.addView(transcript); 34 34 + Ui.fitBars(this, scroll, 0); 35 35 + setContentView(scroll); 36 36 + } 37 37 + 38 38 + @Override 39 39 + public boolean onNavigateUp() { 40 40 + finish(); 41 41 + return true; 42 42 + } 43 43 + }

+41

tools/hermes-voice/android/src/us/lassul/assist/Ui.java

View file Reviewed

··· 1 1 + package us.lassul.assist; 2 2 + 3 3 + import android.app.Activity; 4 4 + import android.graphics.Insets; 5 5 + import android.util.TypedValue; 6 6 + import android.view.View; 7 7 + import android.view.ViewGroup; 8 8 + import android.view.WindowInsets; 9 9 + 10 10 + /** Layout helpers shared by the screens. */ 11 11 + final class Ui { 12 12 + private Ui() {} 13 13 + 14 14 + static int dp(Activity activity, int value) { 15 15 + return (int) (value * activity.getResources().getDisplayMetrics().density); 16 16 + } 17 17 + 18 18 + /** 19 19 + * Pads a screen's root view clear of the status bar, the action bar, the navigation bar and 20 20 + * the keyboard: apps targeting Android 15+ are drawn edge to edge, under all of them. 21 21 + */ 22 22 + static void fitBars(Activity activity, View root, final int pad) { 23 23 + TypedValue size = new TypedValue(); 24 24 + activity.getTheme().resolveAttribute(android.R.attr.actionBarSize, size, true); 25 25 + final int actionBar = TypedValue.complexToDimensionPixelSize( 26 26 + size.data, activity.getResources().getDisplayMetrics()); 27 27 + if (root instanceof ViewGroup) { 28 28 + ((ViewGroup) root).setClipToPadding(false); 29 29 + } 30 30 + root.setOnApplyWindowInsetsListener(new View.OnApplyWindowInsetsListener() { 31 31 + @Override 32 32 + public WindowInsets onApplyWindowInsets(View view, WindowInsets insets) { 33 33 + Insets bars = insets.getInsets(WindowInsets.Type.systemBars() 34 34 + | WindowInsets.Type.displayCutout() | WindowInsets.Type.ime()); 35 35 + view.setPadding(bars.left + pad, bars.top + actionBar + pad / 2, 36 36 + bars.right + pad, bars.bottom + pad); 37 37 + return WindowInsets.CONSUMED; 38 38 + } 39 39 + }); 40 40 + } 41 41 + }

+908

tools/hermes-voice/android/src/us/lassul/assist/VoiceCall.java

View file Reviewed

··· 1 1 + package us.lassul.assist; 2 2 + 3 3 + import android.content.Context; 4 4 + import android.os.Handler; 5 5 + import android.os.Looper; 6 6 + import android.os.SystemClock; 7 7 + import android.util.Log; 8 8 + import java.io.BufferedReader; 9 9 + import java.io.InputStreamReader; 10 10 + import java.net.HttpURLConnection; 11 11 + import java.nio.ByteBuffer; 12 12 + import java.nio.ByteOrder; 13 13 + import java.nio.ShortBuffer; 14 14 + import java.nio.charset.StandardCharsets; 15 15 + import java.util.ArrayList; 16 16 + import java.util.Collections; 17 17 + import java.util.List; 18 18 + import java.util.concurrent.ExecutorService; 19 19 + import java.util.concurrent.Executors; 20 20 + import java.util.regex.Pattern; 21 21 + import org.json.JSONArray; 22 22 + import org.json.JSONException; 23 23 + import org.json.JSONObject; 24 24 + import org.webrtc.AudioSource; 25 25 + import org.webrtc.AudioTrack; 26 26 + import org.webrtc.AudioTrackSink; 27 27 + import org.webrtc.DataChannel; 28 28 + import org.webrtc.IceCandidate; 29 29 + import org.webrtc.MediaConstraints; 30 30 + import org.webrtc.MediaStream; 31 31 + import org.webrtc.MediaStreamTrack; 32 32 + import org.webrtc.PeerConnection; 33 33 + import org.webrtc.PeerConnectionFactory; 34 34 + import org.webrtc.RtpReceiver; 35 35 + import org.webrtc.SdpObserver; 36 36 + import org.webrtc.SessionDescription; 37 37 + import org.webrtc.audio.JavaAudioDeviceModule; 38 38 + 39 39 + /** 40 40 + * One call with gpt-live-1 over WebRTC, delegating to Hermes through hermes-voice: the web page's 41 41 + * Call (static/app.js) in Java, with the same events, delegation, voice approvals, end_call and 42 42 + * idle rules. Lives on the main thread; WebRTC and network callbacks are posted to it. 43 43 + */ 44 44 + final class VoiceCall { 45 45 + interface Listener { 46 46 + /** The voice model listens (session.started). */ 47 47 + void onListening(); 48 48 + 49 49 + /** The call is over; the reason is for the log. */ 50 50 + void onFinished(String reason, boolean failed); 51 51 + } 52 52 + 53 53 + // The voice model accepts at most 500 tokens per append; ~4 chars per token. 54 54 + private static final int APPEND_CHAR_LIMIT = 1400; 55 55 + // Transcript deltas can trail the delegation event; wait for the last words. 56 56 + private static final long DELEGATION_SETTLE_MS = 400; 57 57 + // Once Hermes ended the call (end_call item), the call closes when the voice has been quiet 58 58 + // this long after speaking the reply, at the latest after END_CALL_MAX_MS. 59 59 + private static final long END_CALL_QUIET_MS = 800; 60 60 + private static final long END_CALL_MAX_MS = 30_000; 61 61 + private static final long CLOSE_TIMEOUT_MS = 15_000; 62 62 + // Trickle is fine: the vendor answers with the candidates it has. 63 63 + private static final long ICE_TIMEOUT_MS = 3_000; 64 64 + private static final long TICK_MS = 100; 65 65 + // The voice counts as speaking while a sample above this came within SPEAKING_HOLD_MS. 66 66 + private static final int SPEAKING_PEAK = 1500; 67 67 + private static final long SPEAKING_HOLD_MS = 400; 68 68 + // Spoken answer to an approval question. Denials win, so "no, don't do it" never approves; 69 69 + // anything unclear is asked again. 70 70 + private static final Pattern DENY = Pattern.compile( 71 71 + "\\b(no|nope|deny|denied|don'?t|do not|stop|cancel|abort|reject|nein|nicht)\\b", 72 72 + Pattern.CASE_INSENSITIVE); 73 73 + private static final Pattern APPROVE = Pattern.compile( 74 74 + "\\b(yes|yeah|yep|approve|approved|go ahead|do it|okay|ok|sure|confirm|allow|ja)\\b", 75 75 + Pattern.CASE_INSENSITIVE); 76 76 + 77 77 + private static boolean webrtcLoaded; 78 78 + 79 79 + private final Context context; 80 80 + private final Api api; 81 81 + private final Conversation conversation; 82 82 + private final Listener listener; 83 83 + private final Handler main = new Handler(Looper.getMainLooper()); 84 84 + private final ExecutorService io = Executors.newCachedThreadPool(); 85 85 + 86 86 + private JavaAudioDeviceModule audio; 87 87 + private PeerConnectionFactory factory; 88 88 + private PeerConnection pc; 89 89 + private AudioSource source; 90 90 + private AudioTrack mic; 91 91 + private DataChannel events; 92 92 + private boolean offerSent; 93 93 + private long idleMs = 90_000; 94 94 + 95 95 + // Written by WebRTC's audio thread, read by tick(). 96 96 + private volatile long lastLoud; 97 97 + private long lastActivity = SystemClock.elapsedRealtime(); 98 98 + private boolean speaking; 99 99 + private String lastSpeaker; 100 100 + private Delegation delegation; 101 101 + private Approval approval; 102 102 + private boolean closed; 103 103 + private boolean closing; 104 104 + private boolean ending; 105 105 + private boolean endingSpoken; 106 106 + private int eventCounter; 107 107 + private double usageSeconds; 108 108 + 109 109 + private final Runnable sendOffer = new Runnable() { 110 110 + @Override 111 111 + public void run() { 112 112 + sendOffer(); 113 113 + } 114 114 + }; 115 115 + private final Runnable tick = new Runnable() { 116 116 + @Override 117 117 + public void run() { 118 118 + tick(); 119 119 + } 120 120 + }; 121 121 + private final Runnable closeTimedOut = new Runnable() { 122 122 + @Override 123 123 + public void run() { 124 124 + finish("close timed out", false); 125 125 + } 126 126 + }; 127 127 + private final Runnable endNow = new Runnable() { 128 128 + @Override 129 129 + public void run() { 130 130 + close(); 131 131 + } 132 132 + }; 133 133 + 134 134 + /** A Hermes turn: the /delegate stream for one delegation of the voice model. */ 135 135 + private static final class Delegation { 136 136 + final String id; 137 137 + final StringBuilder answer = new StringBuilder(); 138 138 + String failure; 139 139 + boolean endCall; 140 140 + volatile boolean aborted; 141 141 + volatile HttpURLConnection connection; 142 142 + 143 143 + Delegation(String id) { 144 144 + this.id = id; 145 145 + } 146 146 + } 147 147 + 148 148 + private static final class Approval { 149 149 + final String delegationId; 150 150 + final String code; 151 151 + 152 152 + Approval(String delegationId, String code) { 153 153 + this.delegationId = delegationId; 154 154 + this.code = code; 155 155 + } 156 156 + } 157 157 + 158 158 + VoiceCall(Context context, Api api, Conversation conversation, Listener listener) { 159 159 + this.context = context; 160 160 + this.api = api; 161 161 + this.conversation = conversation; 162 162 + this.listener = listener; 163 163 + } 164 164 + 165 165 + void start() { 166 166 + io.execute(new Runnable() { 167 167 + @Override 168 168 + public void run() { 169 169 + try { 170 170 + final long idle = api.get("/config").optLong("idle_seconds", 90) * 1000; 171 171 + main.post(new Runnable() { 172 172 + @Override 173 173 + public void run() { 174 174 + idleMs = idle; 175 175 + connect(); 176 176 + } 177 177 + }); 178 178 + } catch (Exception error) { 179 179 + failLater("Settings not loaded: " + error.getMessage()); 180 180 + } 181 181 + } 182 182 + }); 183 183 + } 184 184 + 185 185 + private void connect() { 186 186 + if (closed) { 187 187 + return; 188 188 + } 189 189 + if (!webrtcLoaded) { 190 190 + PeerConnectionFactory.initialize(PeerConnectionFactory.InitializationOptions 191 191 + .builder(context.getApplicationContext()) 192 192 + .createInitializationOptions()); 193 193 + webrtcLoaded = true; 194 194 + } 195 195 + audio = JavaAudioDeviceModule.builder(context) 196 196 + .setUseHardwareAcousticEchoCanceler(true) 197 197 + .setUseHardwareNoiseSuppressor(true) 198 198 + .createAudioDeviceModule(); 199 199 + factory = PeerConnectionFactory.builder() 200 200 + .setAudioDeviceModule(audio) 201 201 + .createPeerConnectionFactory(); 202 202 + PeerConnection.RTCConfiguration config = 203 203 + new PeerConnection.RTCConfiguration(new ArrayList<PeerConnection.IceServer>()); 204 204 + config.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN; 205 205 + pc = factory.createPeerConnection(config, new PeerObserver()); 206 206 + if (pc == null) { 207 207 + fail("WebRTC refused the connection"); 208 208 + return; 209 209 + } 210 210 + source = factory.createAudioSource(new MediaConstraints()); 211 211 + mic = factory.createAudioTrack("mic", source); 212 212 + pc.addTrack(mic, Collections.singletonList("mic")); 213 213 + // The data channel must exist before the offer so its m-line is negotiated. 214 214 + events = pc.createDataChannel("oai-events", new DataChannel.Init()); 215 215 + events.registerObserver(new EventsObserver()); 216 216 + pc.createOffer(new Sdp("offer") { 217 217 + @Override 218 218 + public void onCreateSuccess(final SessionDescription offer) { 219 219 + main.post(new Runnable() { 220 220 + @Override 221 221 + public void run() { 222 222 + if (closed) { 223 223 + return; 224 224 + } 225 225 + pc.setLocalDescription(new Sdp("local description") { 226 226 + @Override 227 227 + public void onSetSuccess() { 228 228 + main.postDelayed(sendOffer, ICE_TIMEOUT_MS); 229 229 + } 230 230 + }, offer); 231 231 + } 232 232 + }); 233 233 + } 234 234 + }, new MediaConstraints()); 235 235 + } 236 236 + 237 237 + private void sendOffer() { 238 238 + if (offerSent || closed) { 239 239 + return; 240 240 + } 241 241 + offerSent = true; 242 242 + main.removeCallbacks(sendOffer); 243 243 + final JSONObject body; 244 244 + try { 245 245 + body = new JSONObject() 246 246 + .put("sdp", pc.getLocalDescription().description) 247 247 + .put("history", Conversation.toJson(conversation.lines)); 248 248 + } catch (JSONException error) { 249 249 + fail(error.toString()); 250 250 + return; 251 251 + } 252 252 + io.execute(new Runnable() { 253 253 + @Override 254 254 + public void run() { 255 255 + try { 256 256 + final String answer = api.post("/session", body).getString("sdp"); 257 257 + main.post(new Runnable() { 258 258 + @Override 259 259 + public void run() { 260 260 + if (closed) { 261 261 + return; 262 262 + } 263 263 + pc.setRemoteDescription(new Sdp("remote description"), 264 264 + new SessionDescription(SessionDescription.Type.ANSWER, answer)); 265 265 + main.postDelayed(tick, TICK_MS); 266 266 + } 267 267 + }); 268 268 + } catch (Exception error) { 269 269 + failLater(error.getMessage()); 270 270 + } 271 271 + } 272 272 + }); 273 273 + } 274 274 + 275 275 + private void tick() { 276 276 + if (closed) { 277 277 + return; 278 278 + } 279 279 + long now = SystemClock.elapsedRealtime(); 280 280 + long quiet = now - lastLoud; 281 281 + speaking = quiet < SPEAKING_HOLD_MS; 282 282 + if (speaking) { 283 283 + lastActivity = now; 284 284 + } 285 285 + if (ending && endingSpoken && quiet >= END_CALL_QUIET_MS) { 286 286 + close(); 287 287 + } else if (delegation == null && !speaking && now - lastActivity > idleMs) { 288 288 + Log.i(CallService.TAG, "hanging up (idle)"); 289 289 + close(); 290 290 + } 291 291 + main.postDelayed(tick, TICK_MS); 292 292 + } 293 293 + 294 294 + private boolean send(JSONObject event) { 295 295 + if (events == null || events.state() != DataChannel.State.OPEN) { 296 296 + return false; 297 297 + } 298 298 + try { 299 299 + eventCounter += 1; 300 300 + event.put("event_id", "evt_" + eventCounter); 301 301 + } catch (JSONException error) { 302 302 + return false; 303 303 + } 304 304 + byte[] bytes = event.toString().getBytes(StandardCharsets.UTF_8); 305 305 + return events.send(new DataChannel.Buffer(ByteBuffer.wrap(bytes), false)); 306 306 + } 307 307 + 308 308 + private void append(String type, String delegationId, String content) { 309 309 + try { 310 310 + send(new JSONObject() 311 311 + .put("type", type) 312 312 + .put("delegation_id", delegationId) 313 313 + .put("content", truncate(content))); 314 314 + } catch (JSONException error) { 315 315 + Log.w(CallService.TAG, "append failed", error); 316 316 + } 317 317 + } 318 318 + 319 319 + private void onEvent(JSONObject event) { 320 320 + switch (event.optString("type")) { 321 321 + case "session.started": 322 322 + listener.onListening(); 323 323 + break; 324 324 + case "session.input_transcript.delta": 325 325 + lastActivity = SystemClock.elapsedRealtime(); 326 326 + addTranscript("user", event.optString("delta")); 327 327 + break; 328 328 + case "session.output_transcript.delta": 329 329 + lastActivity = SystemClock.elapsedRealtime(); 330 330 + addTranscript("assistant", event.optString("delta")); 331 331 + if (ending) { 332 332 + endingSpoken = true; 333 333 + } 334 334 + break; 335 335 + case "session.delegation.created": 336 336 + JSONObject created = event.optJSONObject("delegation"); 337 337 + if (created != null) { 338 338 + delegate(created.optString("id")); 339 339 + } 340 340 + break; 341 341 + case "session.usage.updated": 342 342 + usageSeconds = usageOf(event, usageSeconds); 343 343 + break; 344 344 + case "error": 345 345 + JSONObject error = event.optJSONObject("error"); 346 346 + conversation.add("error", error != null 347 347 + ? error.optString("message", "voice model error") 348 348 + : "voice model error"); 349 349 + break; 350 350 + case "session.closed": 351 351 + usageSeconds = usageOf(event, usageSeconds); 352 352 + finish(event.optString("reason", "closed"), false); 353 353 + break; 354 354 + default: 355 355 + break; 356 356 + } 357 357 + } 358 358 + 359 359 + private static double usageOf(JSONObject event, double fallback) { 360 360 + JSONObject usage = event.optJSONObject("usage"); 361 361 + return usage != null ? usage.optDouble("seconds", fallback) : fallback; 362 362 + } 363 363 + 364 364 + // Deltas are appended exactly as received; a speaker change starts a new line. 365 365 + private void addTranscript(String speaker, String delta) { 366 366 + List<Conversation.Line> lines = conversation.lines; 367 367 + Conversation.Line last = lines.isEmpty() ? null : lines.get(lines.size() - 1); 368 368 + if (last != null && last.speaker.equals(speaker) && speaker.equals(lastSpeaker)) { 369 369 + last.text += delta; 370 370 + conversation.save(); 371 371 + } else { 372 372 + conversation.add(speaker, delta); 373 373 + } 374 374 + lastSpeaker = speaker; 375 375 + } 376 376 + 377 377 + private void delegate(String id) { 378 378 + // One Hermes turn at a time: a newer delegation (usually a correction) supersedes the 379 379 + // running one, and the server stops the old run. While Hermes waits for an approval, 380 380 + // the next delegation is the user's answer, not a new request. 381 381 + if (approval != null) { 382 382 + answerApproval(id); 383 383 + return; 384 384 + } 385 385 + final boolean replace = delegation != null; 386 386 + if (delegation != null) { 387 387 + abort(delegation); 388 388 + } 389 389 + final Delegation current = new Delegation(id); 390 390 + delegation = current; 391 391 + main.postDelayed(new Runnable() { 392 392 + @Override 393 393 + public void run() { 394 394 + runDelegation(current, replace); 395 395 + } 396 396 + }, DELEGATION_SETTLE_MS); 397 397 + } 398 398 + 399 399 + private void runDelegation(final Delegation current, boolean replace) { 400 400 + // A hang-up meanwhile sent these lines along itself. 401 401 + if (delegation != current || closed) { 402 402 + return; 403 403 + } 404 404 + List<Conversation.Line> lines = conversation.undelegated("user", "assistant"); 405 405 + boolean userSpoke = false; 406 406 + for (Conversation.Line line : lines) { 407 407 + userSpoke |= line.speaker.equals("user"); 408 408 + } 409 409 + if (!userSpoke) { 410 410 + List<Conversation.Line> dialogue = new ArrayList<>(); 411 411 + for (Conversation.Line line : conversation.lines) { 412 412 + if (line.speaker.equals("user") || line.speaker.equals("assistant")) { 413 413 + dialogue.add(line); 414 414 + } 415 415 + } 416 416 + lines = dialogue.subList(Math.max(0, dialogue.size() - 4), dialogue.size()); 417 417 + } 418 418 + conversation.delegated = conversation.lines.size(); 419 419 + conversation.save(); 420 420 + final JSONObject body; 421 421 + try { 422 422 + body = new JSONObject() 423 423 + .put("hermes_session", conversation.hermesSession) 424 424 + .put("lines", Conversation.toJson(lines)) 425 425 + .put("replace", replace); 426 426 + } catch (JSONException error) { 427 427 + return; 428 428 + } 429 429 + io.execute(new Runnable() { 430 430 + @Override 431 431 + public void run() { 432 432 + stream(current, body); 433 433 + } 434 434 + }); 435 435 + } 436 436 + 437 437 + /** Network thread: reads /delegate's NDJSON and posts each item to the main thread. */ 438 438 + private void stream(final Delegation current, JSONObject body) { 439 439 + String failure = null; 440 440 + HttpURLConnection connection = null; 441 441 + try { 442 442 + connection = api.stream("/delegate", body); 443 443 + current.connection = connection; 444 444 + if (current.aborted) { 445 445 + return; 446 446 + } 447 447 + BufferedReader reader = new BufferedReader( 448 448 + new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8)); 449 449 + for (String line; (line = reader.readLine()) != null; ) { 450 450 + // Blank lines are the server's keepalives. 451 451 + if (line.trim().isEmpty()) { 452 452 + continue; 453 453 + } 454 454 + final JSONObject item = new JSONObject(line); 455 455 + main.post(new Runnable() { 456 456 + @Override 457 457 + public void run() { 458 458 + onItem(current, item); 459 459 + } 460 460 + }); 461 461 + } 462 462 + } catch (Exception error) { 463 463 + if (current.aborted) { 464 464 + return; 465 465 + } 466 466 + failure = String.valueOf(error.getMessage()); 467 467 + } finally { 468 468 + if (connection != null) { 469 469 + connection.disconnect(); 470 470 + } 471 471 + } 472 472 + final String streamFailure = failure; 473 473 + main.post(new Runnable() { 474 474 + @Override 475 475 + public void run() { 476 476 + onDone(current, streamFailure); 477 477 + } 478 478 + }); 479 479 + } 480 480 + 481 481 + private void onItem(Delegation current, JSONObject item) { 482 482 + if (delegation != current) { 483 483 + return; 484 484 + } 485 485 + lastActivity = SystemClock.elapsedRealtime(); 486 486 + String type = item.optString("type"); 487 487 + if (type.equals("approval")) { 488 488 + askApproval(current.id, item); 489 489 + return; 490 490 + } 491 491 + // Anything after an approval request means Hermes moved on (answered here, from Matrix, 492 492 + // or by its own timeout). 493 493 + approval = null; 494 494 + if (type.equals("tool") && item.optString("status").equals("running")) { 495 495 + append("session.thinking.append", current.id, 496 496 + "Hermes is working on it: " + item.optString("label")); 497 497 + } else if (type.equals("end_call")) { 498 498 + current.endCall = true; 499 499 + } else if (type.equals("delta")) { 500 500 + current.answer.append(item.optString("text")); 501 501 + } else if (type.equals("done") && item.has("error") && !item.isNull("error")) { 502 502 + current.failure = item.optString("error"); 503 503 + } else if (type.equals("error")) { 504 504 + current.failure = item.optString("message"); 505 505 + } 506 506 + } 507 507 + 508 508 + private void onDone(Delegation current, String streamFailure) { 509 509 + if (delegation != current) { 510 510 + return; 511 511 + } 512 512 + delegation = null; 513 513 + approval = null; 514 514 + lastActivity = SystemClock.elapsedRealtime(); 515 515 + String failure = current.failure != null ? current.failure : streamFailure; 516 516 + String answer = current.answer.toString().trim(); 517 517 + if (failure != null && answer.isEmpty()) { 518 518 + conversation.add("error", failure); 519 519 + append("session.commentary.append", current.id, 520 520 + "The request to Hermes failed: " + failure); 521 521 + } else { 522 522 + String text = answer.isEmpty() ? "Hermes finished without an answer." : answer; 523 523 + conversation.add("hermes", text); 524 524 + for (String chunk : chunks(text)) { 525 525 + append("session.commentary.append", current.id, chunk); 526 526 + } 527 527 + } 528 528 + if (current.endCall) { 529 529 + // Hermes ended the call: close it once the voice has spoken the reply just handed over. 530 530 + ending = true; 531 531 + endingSpoken = false; 532 532 + main.postDelayed(endNow, END_CALL_MAX_MS); 533 533 + } 534 534 + } 535 535 + 536 536 + private void abort(final Delegation current) { 537 537 + current.aborted = true; 538 538 + io.execute(new Runnable() { 539 539 + @Override 540 540 + public void run() { 541 541 + HttpURLConnection connection = current.connection; 542 542 + if (connection != null) { 543 543 + connection.disconnect(); 544 544 + } 545 545 + } 546 546 + }); 547 547 + } 548 548 + 549 549 + private void askApproval(String id, JSONObject item) { 550 550 + approval = new Approval(id, item.isNull("code") ? null : item.optString("code", null)); 551 551 + String what = item.optString("description"); 552 552 + if (what.isEmpty()) { 553 553 + what = "run a command"; 554 554 + } 555 555 + String command = item.isNull("command") ? "" : item.optString("command"); 556 556 + conversation.add("hermes", 557 557 + "Approval needed: " + what + (command.isEmpty() ? "" : "\n" + command)); 558 558 + append("session.commentary.append", id, 559 559 + "Hermes needs the user's approval before it continues: " + what + "." 560 560 + + (command.isEmpty() ? "" : " The command is: " + command + ".") 561 561 + + " Ask the user whether to approve or deny it."); 562 562 + } 563 563 + 564 564 + private void answerApproval(final String id) { 565 565 + main.postDelayed(new Runnable() { 566 566 + @Override 567 567 + public void run() { 568 568 + if (closed) { 569 569 + return; 570 570 + } 571 571 + StringBuilder said = new StringBuilder(); 572 572 + for (Conversation.Line line : conversation.undelegated("user")) { 573 573 + said.append(line.text).append(' '); 574 574 + } 575 575 + conversation.delegated = conversation.lines.size(); 576 576 + conversation.save(); 577 577 + String choice = classifyApproval(said.toString()); 578 578 + if (choice == null) { 579 579 + append("session.commentary.append", id, 580 580 + "That was not a clear answer. Ask the user again: approve or deny?"); 581 581 + return; 582 582 + } 583 583 + respondApproval(choice, id); 584 584 + } 585 585 + }, DELEGATION_SETTLE_MS); 586 586 + } 587 587 + 588 588 + static String classifyApproval(String said) { 589 589 + if (DENY.matcher(said).find()) { 590 590 + return "deny"; 591 591 + } 592 592 + if (APPROVE.matcher(said).find()) { 593 593 + return "once"; 594 594 + } 595 595 + return null; 596 596 + } 597 597 + 598 598 + private void respondApproval(final String choice, final String replyId) { 599 599 + final Approval answered = approval; 600 600 + if (answered == null) { 601 601 + return; 602 602 + } 603 603 + approval = null; 604 604 + io.execute(new Runnable() { 605 605 + @Override 606 606 + public void run() { 607 607 + String message; 608 608 + boolean ok = false; 609 609 + try { 610 610 + api.post("/approve", new JSONObject() 611 611 + .put("code", answered.code == null ? JSONObject.NULL : answered.code) 612 612 + .put("choice", choice)); 613 613 + ok = true; 614 614 + message = choice.equals("deny") 615 615 + ? "Denied. Hermes will not run it." 616 616 + : "Approved. Hermes is continuing."; 617 617 + } catch (Exception error) { 618 618 + message = "Could not send the answer to Hermes: " + error.getMessage(); 619 619 + } 620 620 + final String reply = message; 621 621 + final boolean sent = ok; 622 622 + main.post(new Runnable() { 623 623 + @Override 624 624 + public void run() { 625 625 + conversation.add(sent ? "hermes" : "error", reply); 626 626 + // A fast turn can finish before this reply returns; its result was already 627 627 + // handed to the voice, so an "is continuing" would be stale. 628 628 + if (closed || (sent && delegation == null)) { 629 629 + return; 630 630 + } 631 631 + append("session.commentary.append", 632 632 + replyId != null ? replyId : answered.delegationId, reply); 633 633 + } 634 634 + }); 635 635 + } 636 636 + }); 637 637 + } 638 638 + 639 639 + /** Ends the call gracefully: the voice model closes the session and reports final usage. */ 640 640 + void close() { 641 641 + if (closed || closing) { 642 642 + return; 643 643 + } 644 644 + closing = true; 645 645 + try { 646 646 + if (!send(new JSONObject().put("type", "session.close"))) { 647 647 + finish("closed", false); 648 648 + return; 649 649 + } 650 650 + } catch (JSONException error) { 651 651 + finish("closed", false); 652 652 + return; 653 653 + } 654 654 + main.postDelayed(closeTimedOut, CLOSE_TIMEOUT_MS); 655 655 + } 656 656 + 657 657 + private void failLater(final String message) { 658 658 + main.post(new Runnable() { 659 659 + @Override 660 660 + public void run() { 661 661 + fail(message); 662 662 + } 663 663 + }); 664 664 + } 665 665 + 666 666 + private void fail(String message) { 667 667 + if (closed) { 668 668 + return; 669 669 + } 670 670 + Log.w(CallService.TAG, "call failed: " + message); 671 671 + conversation.add("error", String.valueOf(message)); 672 672 + finish("failed to start", true); 673 673 + } 674 674 + 675 675 + private void finish(String reason, boolean failed) { 676 676 + if (closed) { 677 677 + return; 678 678 + } 679 679 + closed = true; 680 680 + main.removeCallbacksAndMessages(null); 681 681 + hangUp(); 682 682 + // Only the stream is dropped; the server keeps following the Hermes run. 683 683 + if (delegation != null) { 684 684 + abort(delegation); 685 685 + delegation = null; 686 686 + } 687 687 + if (events != null) { 688 688 + events.unregisterObserver(); 689 689 + events.close(); 690 690 + events.dispose(); 691 691 + } 692 692 + if (pc != null) { 693 693 + pc.close(); 694 694 + pc.dispose(); 695 695 + } 696 696 + if (mic != null) { 697 697 + mic.dispose(); 698 698 + } 699 699 + if (source != null) { 700 700 + source.dispose(); 701 701 + } 702 702 + if (factory != null) { 703 703 + factory.dispose(); 704 704 + } 705 705 + if (audio != null) { 706 706 + audio.release(); 707 707 + } 708 708 + conversation.billedSeconds += usageSeconds; 709 709 + conversation.save(); 710 710 + io.shutdown(); 711 711 + Log.i(CallService.TAG, "call ended: " + reason); 712 712 + listener.onFinished(reason, failed); 713 713 + } 714 714 + 715 715 + // Tell the server the call is over, with what the user said since the last delegation: a 716 716 + // request spoken right before hanging up still reaches Hermes. 717 717 + private void hangUp() { 718 718 + List<Conversation.Line> lines = conversation.undelegated("user", "assistant"); 719 719 + conversation.delegated = conversation.lines.size(); 720 720 + conversation.save(); 721 721 + final JSONObject body; 722 722 + try { 723 723 + body = new JSONObject() 724 724 + .put("hermes_session", conversation.hermesSession) 725 725 + .put("lines", Conversation.toJson(lines)); 726 726 + } catch (JSONException error) { 727 727 + return; 728 728 + } 729 729 + io.execute(new Runnable() { 730 730 + @Override 731 731 + public void run() { 732 732 + try { 733 733 + api.post("/hangup", body); 734 734 + } catch (Exception error) { 735 735 + Log.w(CallService.TAG, "hang-up not delivered: " + error.getMessage()); 736 736 + } 737 737 + } 738 738 + }); 739 739 + } 740 740 + 741 741 + private static String truncate(String text) { 742 742 + return text.length() > APPEND_CHAR_LIMIT ? text.substring(0, APPEND_CHAR_LIMIT) : text; 743 743 + } 744 744 + 745 745 + /** Sentences packed into appends of at most APPEND_CHAR_LIMIT characters. */ 746 746 + static List<String> chunks(String text) { 747 747 + String clean = text.replaceAll("\\s+", " ").trim(); 748 748 + List<String> out = new ArrayList<>(); 749 749 + String current = ""; 750 750 + for (String sentence : clean.split("(?<=[.!?])\\s+")) { 751 751 + if (sentence.length() > APPEND_CHAR_LIMIT) { 752 752 + if (!current.isEmpty()) { 753 753 + out.add(current); 754 754 + } 755 755 + current = ""; 756 756 + for (int i = 0; i < sentence.length(); i += APPEND_CHAR_LIMIT) { 757 757 + out.add(sentence.substring(i, Math.min(sentence.length(), i + APPEND_CHAR_LIMIT))); 758 758 + } 759 759 + } else if (!current.isEmpty() 760 760 + && current.length() + 1 + sentence.length() > APPEND_CHAR_LIMIT) { 761 761 + out.add(current); 762 762 + current = sentence; 763 763 + } else { 764 764 + current = current.isEmpty() ? sentence : current + " " + sentence; 765 765 + } 766 766 + } 767 767 + if (!current.isEmpty()) { 768 768 + out.add(current); 769 769 + } 770 770 + return out; 771 771 + } 772 772 + 773 773 + /** Logs failures; a failed description ends the call. */ 774 774 + private class Sdp implements SdpObserver { 775 775 + private final String what; 776 776 + 777 777 + Sdp(String what) { 778 778 + this.what = what; 779 779 + } 780 780 + 781 781 + @Override 782 782 + public void onCreateSuccess(SessionDescription description) {} 783 783 + 784 784 + @Override 785 785 + public void onSetSuccess() {} 786 786 + 787 787 + @Override 788 788 + public void onCreateFailure(String error) { 789 789 + failLater(what + ": " + error); 790 790 + } 791 791 + 792 792 + @Override 793 793 + public void onSetFailure(String error) { 794 794 + failLater(what + ": " + error); 795 795 + } 796 796 + } 797 797 + 798 798 + private class PeerObserver implements PeerConnection.Observer { 799 799 + @Override 800 800 + public void onConnectionChange(PeerConnection.PeerConnectionState state) { 801 801 + if (state == PeerConnection.PeerConnectionState.FAILED 802 802 + || state == PeerConnection.PeerConnectionState.DISCONNECTED) { 803 803 + main.post(new Runnable() { 804 804 + @Override 805 805 + public void run() { 806 806 + finish("connection lost", false); 807 807 + } 808 808 + }); 809 809 + } 810 810 + } 811 811 + 812 812 + @Override 813 813 + public void onIceGatheringChange(PeerConnection.IceGatheringState state) { 814 814 + if (state == PeerConnection.IceGatheringState.COMPLETE) { 815 815 + main.post(sendOffer); 816 816 + } 817 817 + } 818 818 + 819 819 + // The voice's audio plays by itself; the sink only measures it, for speaking(). 820 820 + @Override 821 821 + public void onAddTrack(RtpReceiver receiver, MediaStream[] streams) { 822 822 + MediaStreamTrack track = receiver.track(); 823 823 + if (track instanceof AudioTrack) { 824 824 + ((AudioTrack) track).addSink(new AudioTrackSink() { 825 825 + @Override 826 826 + public void onData(ByteBuffer data, int bitsPerSample, int sampleRate, 827 827 + int channels, int frames, long timestamp) { 828 828 + if (bitsPerSample != 16) { 829 829 + return; 830 830 + } 831 831 + ShortBuffer samples = data.duplicate().order(ByteOrder.nativeOrder()) 832 832 + .asShortBuffer(); 833 833 + while (samples.hasRemaining()) { 834 834 + if (Math.abs(samples.get()) > SPEAKING_PEAK) { 835 835 + lastLoud = SystemClock.elapsedRealtime(); 836 836 + return; 837 837 + } 838 838 + } 839 839 + } 840 840 + }); 841 841 + } 842 842 + } 843 843 + 844 844 + @Override 845 845 + public void onSignalingChange(PeerConnection.SignalingState state) {} 846 846 + 847 847 + @Override 848 848 + public void onIceConnectionChange(PeerConnection.IceConnectionState state) {} 849 849 + 850 850 + @Override 851 851 + public void onIceConnectionReceivingChange(boolean receiving) {} 852 852 + 853 853 + @Override 854 854 + public void onIceCandidate(IceCandidate candidate) {} 855 855 + 856 856 + @Override 857 857 + public void onIceCandidatesRemoved(IceCandidate[] candidates) {} 858 858 + 859 859 + @Override 860 860 + public void onAddStream(MediaStream stream) {} 861 861 + 862 862 + @Override 863 863 + public void onRemoveStream(MediaStream stream) {} 864 864 + 865 865 + @Override 866 866 + public void onDataChannel(DataChannel channel) {} 867 867 + 868 868 + @Override 869 869 + public void onRenegotiationNeeded() {} 870 870 + } 871 871 + 872 872 + private class EventsObserver implements DataChannel.Observer { 873 873 + @Override 874 874 + public void onMessage(DataChannel.Buffer buffer) { 875 875 + byte[] bytes = new byte[buffer.data.remaining()]; 876 876 + buffer.data.get(bytes); 877 877 + final String text = new String(bytes, StandardCharsets.UTF_8); 878 878 + main.post(new Runnable() { 879 879 + @Override 880 880 + public void run() { 881 881 + if (closed) { 882 882 + return; 883 883 + } 884 884 + try { 885 885 + onEvent(new JSONObject(text)); 886 886 + } catch (JSONException error) { 887 887 + Log.w(CallService.TAG, "unreadable event", error); 888 888 + } 889 889 + } 890 890 + }); 891 891 + } 892 892 + 893 893 + @Override 894 894 + public void onStateChange() { 895 895 + if (events.state() == DataChannel.State.CLOSED) { 896 896 + main.post(new Runnable() { 897 897 + @Override 898 898 + public void run() { 899 899 + finish("connection closed", false); 900 900 + } 901 901 + }); 902 902 + } 903 903 + } 904 904 + 905 905 + @Override 906 906 + public void onBufferedAmountChange(long previous) {} 907 907 + } 908 908 + }