diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 00000000..78227cf3 --- /dev/null +++ b/SETUP.md @@ -0,0 +1,77 @@ +# Setup: Claude subscription on the glasses + +Goal: "Hey Meta" is answered by your Claude Pro/Max plan through Visor and a small bridge running in Termux. Why it works and what was measured: [findings.md](findings.md). + +You need: an Android phone with this fork of Visor installed and your glasses already set up in it, a Claude Pro or Max account, and about 20 minutes. Claude Code does not list Android as a supported platform, so this is experimental, and your plan's usage limits apply. + +## 1. Termux + +1. Install Termux from F-Droid or GitHub, not the Play Store (that build is obsolete). +2. In Termux: + + ```sh + pkg update + pkg install proot-distro + proot-distro install ubuntu + proot-distro login ubuntu + ``` + + From here on, "in Ubuntu" means that last prompt (`root@localhost`). + +## 2. Node and Claude Code (in Ubuntu) + +Claude Code needs a recent Node (20 or newer). Check with `node -v` after the install; if Ubuntu's package is older, install a newer one (for example with [nvm](https://github.com/nvm-sh/nvm)). + +```sh +apt update && apt install -y nodejs npm curl +npm install -g @anthropic-ai/claude-code +claude +``` + +`claude` asks how to log in: choose the Claude subscription. It prints a link; open it in the phone's browser, sign in, and paste the code it gives back into Termux. Then check it works: + +```sh +claude -p "Reply with OK" +``` + +Log in inside Ubuntu, not in plain Termux: the bridge runs there and uses that login. + +## 3. The bridge (in Ubuntu) + +```sh +cd /root +curl -O https://raw.githubusercontent.com/henriqueleote/visor/main/bridge.js +node bridge.js +``` + +It prints `Claude voice bridge on http://127.0.0.1:8080`. From a second Termux session, `curl http://127.0.0.1:8080/v1/models` should answer. Stop it with Ctrl+C. + +The file must be at `/root/bridge.js` in a distro called `ubuntu`: the app's Start button runs exactly that. + +## 4. Visor + +1. Settings > Meta AI > Provider > **Claude subscription**. It fills in server `http://127.0.0.1:8080/v1` and model `claude-code`; leave the key blank. +2. Make sure the bridge is running (step 3, or the Start button below) and ask a question through the glasses. + +## 5. Start and stop from the app (optional) + +Without this, you start the bridge by hand in Termux each time, and it stops when its command ends (a proot kills what it started, so it must run as the foreground command of its own session). + +1. In Termux (outside Ubuntu): `mkdir -p ~/.termux && echo "allow-external-apps = true" >> ~/.termux/termux.properties`, then `termux-reload-settings`. Close Termux fully and open it again. +2. In Visor, on the Claude subscription tab, tap **Allow** on the Bridge row and grant the Termux permission. +3. Tap **Start**. The row shows "Running in Termux." when the bridge answers. +4. Turn off battery optimisation for Termux, or Android will kill it in the background. + +## 6. Glasses settings that matter + +- Follow-ups without the wake word: Settings > Meta AI > **Respond without "Hey Meta"**. +- A custom wake word ("Hey Jarvis") needs the US English language pack on the glasses. +- Music by voice: start something in Spotify once so it has a media session, then pick it in Visor's media settings. + +## If it does not work + +See the troubleshooting table in [findings.md](findings.md). The first checks: the Bridge row says Running, `claude -p "Reply with OK"` works in Ubuntu, and the provider is **Claude subscription**, not **Claude API**. + +## Not verified + +These steps were written from the working setup, not from a clean install: the Node version, the browser sign-in flow and the `curl` download were not re-tested from scratch. If a step differs on your phone, say so and fix it here. diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index feeeec3b..aca410c1 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -28,6 +28,8 @@ + + @@ -35,6 +37,7 @@ + diff --git a/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiScreen.kt b/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiScreen.kt index a787ff0d..b300504e 100644 --- a/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/metaai/MetaAiScreen.kt @@ -119,7 +119,7 @@ fun MetaAiScreen(onSettings: () -> Unit, onMemories: () -> Unit, onSession: (Str val attention = rememberUpdatedState(glasses.device?.assistantAttention) val speech = rememberSpeech(turns, { attention.value }, light) var phoneListening by remember { mutableStateOf(false) } - val configured = config.apiKey.isNotBlank() || config.provider == Provider.OPENAI_COMPATIBLE + val configured = config.apiKey.isNotBlank() || config.provider != Provider.CLAUDE val now = rememberNow() val newest = turns.lastOrNull() val waiting = turns.takeLast(4).any { it.answer.isEmpty() && it.problem.isEmpty() && System.currentTimeMillis() - it.at < WAITING_MS } diff --git a/app/src/main/kotlin/com/macedonga/visor/phone/PhoneActions.kt b/app/src/main/kotlin/com/macedonga/visor/phone/PhoneActions.kt index e4fef737..cb002370 100644 --- a/app/src/main/kotlin/com/macedonga/visor/phone/PhoneActions.kt +++ b/app/src/main/kotlin/com/macedonga/visor/phone/PhoneActions.kt @@ -140,6 +140,14 @@ class PhoneActions @Inject constructor(@ApplicationContext private val context: return PhoneDone(label(c.packageName)) } + override suspend fun openApp(app: String): PhoneDone { + val pkg = launchable(app) ?: return PhoneDone("", "there is no app called $app on the phone") + val label = label(pkg) + note("Opening $label") + val launch = pm.getLaunchIntentForPackage(pkg) ?: return PhoneDone(label, "$label has no screen to open") + return open(launch, OPEN_ID, context.getString(R.string.phone_open, label), label) + } + override fun nowPlaying(): NowPlaying { val sessions = sessions() ?: return NowPlaying(unknown = "Visor has no notification access, so it cannot see what plays") val c = sessions.firstOrNull { it.playing() } ?: sessions.firstOrNull() ?: return NowPlaying(unknown = "nothing is playing") @@ -263,6 +271,19 @@ class PhoneActions @Inject constructor(@ApplicationContext private val context: .filter { it.packageName != context.packageName && pm.getLaunchIntentForPackage(it.packageName) != null } }.getOrNull() + /** The installed app with a launcher called [name]: the same label, else the closest one that contains it. */ + private fun launchable(name: String): String? { + val want = simple(name) + if (want.isEmpty()) return null + val apps = pm.queryIntentActivities(Intent(Intent.ACTION_MAIN).addCategory(Intent.CATEGORY_LAUNCHER), 0) + .map { it.activityInfo.packageName }.filter { it != context.packageName }.toSet() + .associateWith { simple(label(it)) } + // An app called X must not open for "Zorbax": the name said may contain the app's only from four letters up. + return apps.entries.firstOrNull { it.value == want }?.key + ?: apps.entries.filter { it.value.isNotEmpty() && (it.value.contains(want) || it.value.length >= 4 && want.contains(it.value)) } + .maxByOrNull { it.value.length }?.key + } + /** The installed app called [name], among the ones that play media. */ private fun named(name: String, sessions: List?): String? { val want = simple(name) @@ -350,6 +371,7 @@ class PhoneActions @Inject constructor(@ApplicationContext private val context: const val CHANNEL = "assistant_actions" const val DIRECTIONS_ID = 11 const val PLAY_ID = 12 + const val OPEN_ID = 13 const val LAST = "last_player" const val PLAY_WAIT_MS = 6_000L const val POLL_MS = 250L diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/AssistantSettings.kt b/app/src/main/kotlin/com/macedonga/visor/settings/AssistantSettings.kt index 8cacdaf9..dcaf32cd 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/AssistantSettings.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/AssistantSettings.kt @@ -88,7 +88,11 @@ fun AssistantSettingsScreen(open: (NavKey) -> Unit, model: AssistantViewModel = val askHearing = rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { hearing = it } var check by remember { mutableStateOf(KeyCheck.None) } - LaunchedEffect(config.provider, config.apiKey, config.baseUrl) { + var bridgeTick by remember { mutableStateOf(0) } + var bridgeNote by remember { mutableStateOf(null) } + var termux by remember { mutableStateOf(context.checkSelfPermission(TermuxBridge.PERMISSION) == PackageManager.PERMISSION_GRANTED) } + val askTermux = rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { termux = it } + LaunchedEffect(config.provider, config.apiKey, config.baseUrl, bridgeTick) { if (config.provider == Provider.CLAUDE && config.apiKey.isBlank()) { check = KeyCheck.None return@LaunchedEffect @@ -125,14 +129,45 @@ fun AssistantSettingsScreen(open: (NavKey) -> Unit, model: AssistantViewModel = { PillsRow( stringResource(R.string.ai_provider), - listOf(Provider.CLAUDE to "Claude", Provider.OPENAI_COMPATIBLE to stringResource(R.string.ai_openai_compatible)), + listOf( + Provider.CLAUDE to "Claude API", + Provider.CLAUDE_SUBSCRIPTION to stringResource(R.string.ai_claude_subscription), + Provider.OPENAI_COMPATIBLE to stringResource(R.string.ai_openai_compatible), + ), config.provider, - onPick = { p -> model.store.update { it.copy(provider = p) } }, + onPick = { p -> + model.store.update { + // The bridge's own address and model, unless the address already points at this phone. + if (p == Provider.CLAUDE_SUBSCRIPTION) it.copy( + provider = p, model = "claude-code", + baseUrl = it.baseUrl.takeIf { u -> u.startsWith("http://127.0.0.1") } ?: "http://127.0.0.1:8080/v1", + ) else it.copy(provider = p) + } + }, ) }, if (!claude) row { ValueRow(stringResource(R.string.ai_base_url), config.baseUrl, onClick = { sheet = "url" }) } else null, + if (config.provider == Provider.CLAUDE_SUBSCRIPTION) row { + val up = check is KeyCheck.Works + RowFrame( + stringResource(R.string.ai_bridge), onClick = null, + sub = (bridgeNote ?: stringResource(if (up) R.string.ai_bridge_up else R.string.ai_bridge_down)) + + "\n" + stringResource(R.string.ai_bridge_note), + ) { + ActionPill( + stringResource(if (!termux) R.string.access_allow else if (up) R.string.ai_bridge_stop else R.string.ai_bridge_start), + { + if (!termux) askTermux.launch(TermuxBridge.PERMISSION) else { + bridgeNote = TermuxBridge.run(context, !up) + scope.launch { delay(6_000); bridgeTick++ } + } + }, + modifier = Modifier.padding(start = 12.dp), + ) + } + } else null, { val (sub, problem) = when (val c = check) { KeyCheck.None -> (if (claude) stringResource(R.string.ai_key_needed) else stringResource(R.string.ai_key_optional)) to null diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt index c9909f75..f182e220 100644 --- a/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/macedonga/visor/settings/SettingsScreen.kt @@ -609,7 +609,7 @@ private fun summary(topic: Topic, model: SettingsViewModel, state: SettingsState @Composable internal fun assistantWords(config: AssistantConfig): String { - val configured = config.apiKey.isNotBlank() || config.provider == Provider.OPENAI_COMPATIBLE + val configured = config.apiKey.isNotBlank() || config.provider != Provider.CLAUDE return when { !config.enabled -> stringResource(R.string.sum_ai_off) !configured -> stringResource(R.string.sum_ai_unset) diff --git a/app/src/main/kotlin/com/macedonga/visor/settings/TermuxBridge.kt b/app/src/main/kotlin/com/macedonga/visor/settings/TermuxBridge.kt new file mode 100644 index 00000000..6d782048 --- /dev/null +++ b/app/src/main/kotlin/com/macedonga/visor/settings/TermuxBridge.kt @@ -0,0 +1,31 @@ +package com.macedonga.visor.settings + +import android.content.ComponentName +import android.content.Context +import android.content.Intent + +/** Starts or stops the Claude bridge through Termux's RUN_COMMAND service; Termux needs allow-external-apps on. */ +object TermuxBridge { + const val PERMISSION = "com.termux.permission.RUN_COMMAND" + private const val FILES = "/data/data/com.termux/files" + + /** + * Null when the request went to Termux, else why it did not. Start runs node as the command itself, because + * anything the proot leaves running is killed when its command ends; stop is a pkill from Termux's side. + */ + fun run(context: Context, start: Boolean): String? = runCatching { + val (path, args) = if (start) { + "$FILES/usr/bin/proot-distro" to arrayOf("login", "ubuntu", "--", "bash", "-c", "cd /root && exec node bridge.js >> bridge.log 2>&1") + } else { + "$FILES/usr/bin/pkill" to arrayOf("-f", "node bridge.js") + } + val intent = Intent("com.termux.RUN_COMMAND") + .setComponent(ComponentName("com.termux", "com.termux.app.RunCommandService")) + .putExtra("com.termux.RUN_COMMAND_PATH", path) + .putExtra("com.termux.RUN_COMMAND_ARGUMENTS", args) + .putExtra("com.termux.RUN_COMMAND_WORKDIR", "$FILES/home") + .putExtra("com.termux.RUN_COMMAND_BACKGROUND", true) + checkNotNull(context.startService(intent)) { "Termux is not installed" } + null + }.getOrElse { it.message ?: it.javaClass.simpleName } +} diff --git a/app/src/main/res/values-it/strings.xml b/app/src/main/res/values-it/strings.xml index 8bf34cfd..3cdb4739 100644 --- a/app/src/main/res/values-it/strings.xml +++ b/app/src/main/res/values-it/strings.xml @@ -472,6 +472,13 @@ Spento, l\'assistente degli occhiali non ha un telefono dietro. Modello Fornitore + Abbonamento Claude + Bridge + Avvia + Ferma + Attivo in Termux. + Non attivo. + L\'abbonamento Claude funziona solo mentre il bridge è attivo. Per avviarlo da qui serve Termux con allow-external-apps attivo. Compatibile OpenAI Server Chiave API @@ -928,6 +935,7 @@ Riprova Indicazioni per %1$s Riproduci %1$s + Apri %1$s Wi-Fi creato dagli occhiali Una VPN tiene Visor fuori dalla rete degli occhiali. Escludi Visor dalla VPN. Lo streaming si interrompe spesso diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index e97f41f7..ce0757d3 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -485,6 +485,13 @@ Если выключено, собственный ассистент очков работает сам по себе, без телефона. Модель Провайдер + Подписка Claude + Мост + Запустить + Остановить + Работает в Termux. + Не запущен. + Подписка Claude работает, только пока работает мост. Чтобы запускать его отсюда, в Termux должен быть включён allow-external-apps. Совместимый с OpenAI Адрес API-ключ @@ -929,6 +936,7 @@ Повторить Маршрут до %1$s Воспроизвести %1$s + Открыть %1$s Wi-Fi создают очки VPN не пускает Visor в сеть очков. Исключите Visor из VPN. Трансляция часто обрывается diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index fcdd7b67..8fe11e2d 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -473,6 +473,13 @@ Off, the glasses\' own assistant has no phone behind it. Model Provider + Claude subscription + Bridge + Start + Stop + Running in Termux. + Not running. + The Claude subscription only works while the bridge is running. Starting it from here needs Termux with allow-external-apps on. OpenAI-compatible Server API key @@ -930,6 +937,7 @@ Try again Directions to %1$s Play %1$s + Open %1$s Glasses host the Wi-Fi A VPN keeps Visor off the glasses\' network. Exclude Visor from it. The stream keeps dropping diff --git a/bridge.js b/bridge.js new file mode 100644 index 00000000..292f0e7e --- /dev/null +++ b/bridge.js @@ -0,0 +1,274 @@ +const http = require('http'); +const { spawn } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { randomUUID } = require('crypto'); + +const PORT = Number(process.env.BRIDGE_PORT) || 8080; +const MODEL = process.env.CLAUDE_MODEL || 'haiku'; +// Optional: other apps on the phone can reach a loopback port, so set this and Visor's API key to the same secret. +const TOKEN = process.env.BRIDGE_TOKEN || ''; +const TIMEOUT_MS = 40000; +const IDLE_MS = (Number(process.env.BRIDGE_IDLE_MIN) || 10) * 60000; +const MAX_TURNS = 40; +const MAX_BODY = 20e6; +const MAX_TOOL_ROUNDS = 4; +const WORKDIR = path.join(os.homedir(), '.visor-bridge'); +const SYSTEM = 'You are a voice assistant on smart glasses. Your answer is spoken aloud: reply in one or two short ' + + 'sentences, plain words, no markdown, lists or emoji. If an image comes with the question, it is what the wearer sees. ' + + 'The question comes from speech recognition, which can clip or mishear the first word (for example "and times two" ' + + 'arrives as "n times two"): when it only makes sense as a follow-up to your last answer, read it that way. ' + + 'The glasses keep listening after every answer. Judge by meaning, not by exact words: when the wearer is closing the ' + + 'conversation (thanks, thx, aight, cool that is it, nah I am good, nvm, bye, or anything else that dismisses you) and ' + + 'is not asking anything new, answer in a few words and end with the word END_CONVERSATION. Do not end it when they ' + + 'are still asking or when your answer asked them something.'; +const END_TOOL = 'end_conversation'; +const END = /\s*END_CONVERSATION\W*/gi; +const TOOL_CALL = /TOOL_CALL\s*(\{[\s\S]*\})/; +const FALLBACK = "Sorry, I couldn't process that request."; + +// One long-lived Claude Code process answers every question: starting `claude -p` per request is most of the wait. +let worker = null; +// Tool call ids the bridge handed to Visor, so a result can be named when it comes back. +const calls = new Map(); + +function start() { + fs.mkdirSync(WORKDIR, { recursive: true }); + const child = spawn('claude', [ + '-p', '--input-format', 'stream-json', '--output-format', 'stream-json', '--verbose', + '--model', MODEL, '--system-prompt', SYSTEM, + '--tools', '', '--strict-mcp-config', '--disable-slash-commands', '--setting-sources', '', '--no-session-persistence', + ], { cwd: WORKDIR, stdio: ['pipe', 'pipe', 'pipe'] }); + const w = { child, buf: '', err: '', pending: null, turns: 0, last: Date.now() }; + const lost = error => { + if (worker === w) worker = null; + if (w.pending) w.pending.reject(error); + }; + child.stdout.on('data', chunk => { + w.buf += chunk; + let i; + while ((i = w.buf.indexOf('\n')) >= 0) { + const line = w.buf.slice(0, i); + w.buf = w.buf.slice(i + 1); + let event; + try { event = JSON.parse(line); } catch (e) { continue; } + if (event.type === 'result' && w.pending) w.pending.resolve(event); + } + }); + child.stderr.on('data', chunk => { w.err = (w.err + chunk).slice(-500); }); + child.on('error', lost); + child.on('close', () => lost(new Error(w.err || 'claude exited'))); + child.stdin.on('error', () => {}); + return w; +} + +// Drops the conversation; the next one starts on a process that is already up. +function reset() { + const old = worker; + worker = start(); + if (old) { + old.pending = null; + old.child.kill('SIGTERM'); + } +} + +let queue = Promise.resolve(); +function ask(content, tools) { + const run = queue.then(() => send(content, tools)); + queue = run.catch(() => {}); + return run; +} + +function send(content, tools) { + return new Promise((resolve, reject) => { + if (worker && worker.turns >= MAX_TURNS) reset(); + const w = worker || (worker = start()); + // The warm process keeps the list it was given, so it is sent again only when it changes. + if (tools && w.catalog !== tools) { + content = [{ type: 'text', text: tools }, ...content]; + w.catalog = tools; + } + const settle = (fn, value) => () => { clearTimeout(timer); w.pending = null; fn(value); }; + const timer = setTimeout(() => { reset(); reject(new Error('timeout')); }, TIMEOUT_MS); + w.pending = { + resolve: event => { + w.last = Date.now(); + if (event.is_error || typeof event.result !== 'string' || !event.result.trim()) { + reset(); + settle(reject, new Error(event.result || 'Claude Code returned no text'))(); + } else { + settle(resolve, event.result.trim())(); + } + }, + reject: error => settle(reject, error)(), + }; + w.turns++; + w.child.stdin.write(JSON.stringify({ type: 'user', message: { role: 'user', content } }) + '\n'); + }); +} + +// OpenAI message content to Claude's blocks; images arrive as data URLs. +function blocks(content) { + if (typeof content === 'string') return [{ type: 'text', text: content }]; + const out = []; + for (const part of content || []) { + if (part.type === 'text' && part.text) { + out.push({ type: 'text', text: part.text }); + } else if (part.type === 'image_url') { + const url = (part.image_url && part.image_url.url) || ''; + const cut = url.indexOf(';base64,'); + if (url.startsWith('data:image/') && cut > 0) { + out.push({ type: 'image', source: { type: 'base64', media_type: url.slice(5, cut), data: url.slice(cut + 8) } }); + } + } + } + return out; +} + +// Visor's tools for this question, one line each. The model answers with a TOOL_CALL line to use one. +function catalog(tools) { + const lines = tools.map(t => { + const fn = t.function; + const props = Object.keys((fn.parameters && fn.parameters.properties) || {}); + return `- ${fn.name}(${props.join(', ')}): ${(fn.description || '').slice(0, 160)}`; + }); + return 'Tools you can use:\n' + lines.join('\n') + '\n' + + 'To use one, reply with exactly one line and nothing else: TOOL_CALL {"name": "", "arguments": {...}}. ' + + 'Its result comes back as a TOOL_RESULT line; then answer the wearer in one or two short sentences. ' + + 'Use a tool only when the wearer asks for an action or a fact that needs it.'; +} + +// Tool results since the wearer's last question: a model that keeps calling tools is stopped after MAX_TOOL_ROUNDS. +function toolRounds(messages) { + let n = 0; + for (let i = messages.length - 1; i >= 0 && messages[i].role !== 'user'; i--) if (messages[i].role === 'tool') n++; + return n; +} + +// The tool call in an answer, if it names one of the tools offered; null otherwise. +function parseCall(answer, tools) { + const m = answer.match(TOOL_CALL); + if (!m) return null; + try { + const call = JSON.parse(m[1]); + return tools.some(t => t.function.name === call.name) ? { name: call.name, args: call.arguments || {} } : null; + } catch (e) { + return null; + } +} + +function completion(res, message, finish, model) { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ + id: 'chatcmpl-termux-bridge', + object: 'chat.completion', + created: Math.floor(Date.now() / 1000), + model: model || 'claude-code', + choices: [{ index: 0, message, finish_reason: finish }], + })); +} + +// ends: Visor offered end_conversation and the wearer is done, so answer with that call; Visor then asks once more for the words. +function say(res, text, model, ends) { + const message = { role: 'assistant', content: text }; + if (ends) message.tool_calls = [{ id: 'call_end', type: 'function', function: { name: END_TOOL, arguments: '{}' } }]; + completion(res, message, ends ? 'tool_calls' : 'stop', model); +} + +function callReply(res, id, call, model) { + calls.set(id, call.name); + if (calls.size > 500) calls.clear(); + const tool_calls = [{ id, type: 'function', function: { name: call.name, arguments: JSON.stringify(call.args) } }]; + completion(res, { role: 'assistant', content: null, tool_calls }, 'tool_calls', model); +} + +function fail(res, status, message) { + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: message })); +} + +const server = http.createServer((req, res) => { + if (TOKEN && req.headers.authorization !== `Bearer ${TOKEN}`) return fail(res, 401, 'missing or invalid bearer token'); + + if (req.method === 'GET') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + return res.end(JSON.stringify({ + object: 'list', + data: [{ id: 'claude-code', object: 'model', created: 1700000000, owned_by: 'local' }], + })); + } + + if (req.method !== 'POST' || !req.url.includes('chat/completions')) return fail(res, 404, `Route not found: ${req.url}`); + + let body = ''; + req.on('data', chunk => { + body += chunk; + if (body.length > MAX_BODY) req.destroy(); + }); + req.on('end', async () => { + let request; + let messages; + let content; + let isResult = false; + try { + request = JSON.parse(body); + messages = request.messages || []; + const last = messages[messages.length - 1] || {}; + // The round after end_conversation: nothing to ask Claude, the words already went out with the call. + if (last.role === 'tool' && last.tool_call_id === 'call_end') { + const before = messages.slice().reverse().find(m => m.role === 'assistant'); + return say(res, (before && typeof before.content === 'string' && before.content) || 'Okay.', request.model); + } + if (last.role === 'tool') { + // A tool's result: the warm process already holds the question, so it only needs the result. + isResult = true; + const name = calls.get(last.tool_call_id) || 'tool'; + const result = typeof last.content === 'string' ? last.content : JSON.stringify(last.content); + content = [{ type: 'text', text: `TOOL_RESULT ${name}: ${result}` }]; + } else { + const users = messages.filter(m => m.role === 'user'); + content = blocks(users.length ? users[users.length - 1].content : ''); + if (!content.length) throw new Error('No user message found.'); + } + } catch (error) { + return fail(res, 400, 'Invalid payload'); + } + + const spoken = content.filter(b => b.type === 'text').map(b => b.text).join(' ').trim().toLowerCase(); + if (!isResult && (spoken === '/clear' || spoken === 'clear session')) { + reset(); + return say(res, 'Session cleared. Starting a fresh conversation.', request.model); + } + + const tools = (request.tools || []).filter(t => t.function && t.function.name !== END_TOOL); + const offered = (request.tools || []).some(t => t.function && t.function.name === END_TOOL); + const useTools = tools.length > 0 && toolRounds(messages) < MAX_TOOL_ROUNDS; + const list = useTools ? catalog(tools) : null; + + const started = Date.now(); + try { + const answer = await ask(content, list); + const done = new RegExp(END.source, 'i').test(answer); + const call = useTools ? parseCall(answer, tools) : null; + console.log(`[${new Date().toISOString()}] answered in ${Date.now() - started} ms` + + `${call ? ` (tool ${call.name})` : ''}${done ? ' (conversation ends)' : ''}`); + if (call) return callReply(res, 'call_' + randomUUID().slice(0, 12), call, request.model); + const text = answer.replace(TOOL_CALL, ' ').replace(END, ' ').trim(); + say(res, text || 'Okay.', request.model, done && offered); + } catch (error) { + console.error('Claude error:', error.message); + say(res, FALLBACK, request.model); + } + }); +}); + +server.listen(PORT, '127.0.0.1', () => { + worker = start(); + setInterval(() => { if (worker && worker.turns && Date.now() - worker.last > IDLE_MS) reset(); }, 60000).unref(); + console.log(`Claude voice bridge on http://127.0.0.1:${PORT} (model ${MODEL})`); +}); + +process.on('exit', () => { if (worker) worker.child.kill('SIGTERM'); }); +process.on('SIGINT', () => process.exit(0)); +process.on('SIGTERM', () => process.exit(0)); diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt index 9edb2b64..b62a6fc0 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Brain.kt @@ -24,7 +24,8 @@ import org.json.JSONArray import org.json.JSONObject /** Where an answer comes from. */ -enum class Provider { CLAUDE, OPENAI_COMPATIBLE } +/** CLAUDE_SUBSCRIPTION is the OpenAI-compatible path pointed at a Claude Code bridge on the phone. */ +enum class Provider { CLAUDE, OPENAI_COMPATIBLE, CLAUDE_SUBSCRIPTION } /** One question, its answer and what the tools did for it: the conversation behind the next question. */ data class Exchange(val question: String, val answer: String, val actions: List = emptyList()) @@ -97,7 +98,7 @@ class Brain( canEnd: Boolean = false, ): Reply = withContext(Dispatchers.IO) { val c = config() - check(c.apiKey.isNotBlank() || c.provider == Provider.OPENAI_COMPATIBLE) { "no API key is set" } + check(c.apiKey.isNotBlank() || c.provider != Provider.CLAUDE) { "no API key is set" } // The tools never stand in the way of an answer: without them the model still answers. val turn = Turn(hub?.let { runCatching { it.toolbox() }.getOrNull() } ?: Toolbox(emptyList(), ""), image, canEnd) val system = (if (audience == Audience.PHONE) PHONE_PROMPT else SYSTEM_PROMPT) + @@ -114,7 +115,7 @@ class Brain( val tokens = if (audience == Audience.PHONE) PHONE_TOKENS else GLASSES_TOKENS val text = when (c.provider) { Provider.CLAUDE -> claude(c, c.model.ifBlank { DEFAULT_CLAUDE_MODEL }, system, recent, question, image, tokens, c.webSearch, turn) - Provider.OPENAI_COMPATIBLE -> openAi(c, c.model, system, recent, question, image, tokens, turn) + Provider.OPENAI_COMPATIBLE, Provider.CLAUDE_SUBSCRIPTION -> openAi(c, c.model, system, recent, question, image, tokens, turn) }.replace(VISOR_NOTE, "").trim() check(text.isNotEmpty()) { "the model gave an empty answer" } Reply(text, turn.remembered, turn.actions, turn.ends) @@ -123,7 +124,7 @@ class Brain( /** One sentence for the wearer, in [locale]'s language, written from [what]; outside the conversation. */ suspend fun phrase(what: String, locale: Locale): String = withContext(Dispatchers.IO) { val c = config() - check(c.apiKey.isNotBlank() || c.provider == Provider.OPENAI_COMPATIBLE) { "no API key is set" } + check(c.apiKey.isNotBlank() || c.provider != Provider.CLAUDE) { "no API key is set" } val system = "You write one short sentence that smart glasses speak to their wearer, in " + "${locale.getDisplayLanguage(Locale.ENGLISH)}. Answer with that sentence only." write(c, c.model, system, what, 200) @@ -132,7 +133,7 @@ class Brain( /** A day's lines for the diary, in [locale]'s language, from [exchanges] in order; with the lines' own model if one is set. */ suspend fun dayLines(exchanges: String, locale: Locale): String = withContext(Dispatchers.IO) { val c = config() - check(c.apiKey.isNotBlank() || c.provider == Provider.OPENAI_COMPATIBLE) { "no API key is set" } + check(c.apiKey.isNotBlank() || c.provider != Provider.CLAUDE) { "no API key is set" } val system = "You keep the diary of a person who wears smart glasses with a voice assistant. From the day's " + "exchanges with the assistant, write the day's entry: one to three short sentences in " + "${locale.getDisplayLanguage(Locale.ENGLISH)}, in the second person (\"You asked about...\"), dry and precise, " + @@ -143,7 +144,7 @@ class Brain( private suspend fun write(c: AssistantConfig, model: String, system: String, text: String, tokens: Int): String = when (c.provider) { Provider.CLAUDE -> claude(c, model.ifBlank { DEFAULT_CLAUDE_MODEL }, system, emptyList(), text, null, tokens, false, null) - Provider.OPENAI_COMPATIBLE -> openAi(c, model, system, emptyList(), text, null, tokens, null) + Provider.OPENAI_COMPATIBLE, Provider.CLAUDE_SUBSCRIPTION -> openAi(c, model, system, emptyList(), text, null, tokens, null) }.trim().also { check(it.isNotEmpty()) { "the model gave an empty answer" } } /** What the model knows of the wearer, and how to keep it. Empty when there is nowhere to keep it. */ diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Models.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Models.kt index cfa4b0db..e803a16d 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Models.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Models.kt @@ -34,7 +34,7 @@ suspend fun listModels(config: AssistantConfig): List = withContext(D } while (after != null) all } - Provider.OPENAI_COMPATIBLE -> { + Provider.OPENAI_COMPATIBLE, Provider.CLAUDE_SUBSCRIPTION -> { val headers = if (config.apiKey.isBlank()) emptyMap() else mapOf("Authorization" to "Bearer ${config.apiKey}") val page = get(config.baseUrl.trimEnd('/') + "/models", headers) val data = page.optJSONArray("data") diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Phone.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Phone.kt index 7a1faf93..990ea9ca 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Phone.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Phone.kt @@ -18,6 +18,9 @@ interface Phone { suspend fun control(control: MediaControl): PhoneDone + /** The installed app called [app], opened on the phone. */ + suspend fun openApp(app: String): PhoneDone + fun nowPlaying(): NowPlaying } @@ -55,7 +58,7 @@ internal class PhoneTools(private val phone: Phone) { val prompt = " You can also act on the wearer's phone: directions opens the route to a place in the map app, play_media " + "plays music, podcasts or radio in a phone app, media_control pauses, resumes or skips what plays, now_playing " + - "says what plays. Use them whenever the wearer asks for these, without asking first; say it was done only when the " + + "says what plays, open_app opens an app on the phone by its name. Use them whenever the wearer asks for these, without asking first; say it was done only when the " + "tool says so." /** Name, description and input schema of each tool. */ @@ -81,6 +84,10 @@ internal class PhoneTools(private val phone: Phone) { schema("action" to text("What to do.", *MediaControl.entries.map { it.name.lowercase() }.toTypedArray()), required = "action"), ), Triple("now_playing", "What plays on the phone now.", schema()), + Triple( + "open_app", "Open an app on the phone by its name, such as Maps, Spotify or WhatsApp.", + schema("app" to text("The app's name as the wearer said it."), required = "app"), + ), ) /** Runs [name] when it is one of these tools: what the model is told, and the diary's line for it. Null for another tool. */ @@ -116,6 +123,15 @@ internal class PhoneTools(private val phone: Phone) { if (control == null) ToolResult("unknown action", true) to null else phone.control(control).let { done -> told(done) { "Done: ${control.done}." } to control.action(done) } } + "open_app" -> { + val app = input.optString("app").trim() + if (app.isEmpty()) ToolResult("no app was named", true) to null + else phone.openApp(app).let { done -> + told(done) { + if (it.tapToOpen) "${it.app} waits in a notification on the phone; tapping it opens it." else "${it.app} is open on the phone." + } to ToolAction(done.app.ifEmpty { PHONE }, "Open · $app", done.failure == null && !done.tapToOpen) + } + } "now_playing" -> phone.nowPlaying().let { now -> val text = now.unknown ?: (if (now.playing) "Playing" else "Paused") + " in ${now.app}: " + now.title.ifEmpty { "an untitled item" } + if (now.artist.isNotEmpty()) " by ${now.artist}" else "" diff --git a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Vision.kt b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Vision.kt index 53c37633..a39cf7e6 100644 --- a/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Vision.kt +++ b/core/assistant/src/main/kotlin/com/macedonga/visor/core/assistant/Vision.kt @@ -23,12 +23,14 @@ internal class Visual( val context = understanding[4].asStruct() val onDevice = request[55].asStruct() val inline = context[4].asBytes().takeIf { it.size > 100 } ?: context[16].asBytes().takeIf { it.size > 100 } + val intent = onDevice[1].asText() return Visual( inline = inline, imageUuid = context[3].asText(), captureHandle = context[8].asText(), - glassesWantImage = context[20] == true || understanding[5] == true || onDevice[8] == true, - intent = onDevice[1].asText(), + // IMPLICIT_DESCRIBE_IMAGE ("what can I cook with this") sets none of the flags, but it is a picture question. + glassesWantImage = context[20] == true || understanding[5] == true || onDevice[8] == true || "IMAGE" in intent, + intent = intent, ) } }