From 9ec4c619e0400450c6e1e93399b4fd6e8c37f3ba Mon Sep 17 00:00:00 2001 From: Shadow Date: Tue, 27 Jan 2026 14:46:38 -0600 Subject: [PATCH] Branding: remove legacy android packages --- .../com/clawdbot/android/CameraHudState.kt | 14 - .../java/com/clawdbot/android/DeviceNames.kt | 26 - .../java/com/clawdbot/android/LocationMode.kt | 15 - .../java/com/clawdbot/android/MainActivity.kt | 130 -- .../com/clawdbot/android/MainViewModel.kt | 174 --- .../main/java/com/clawdbot/android/NodeApp.kt | 26 - .../clawdbot/android/NodeForegroundService.kt | 180 --- .../java/com/clawdbot/android/NodeRuntime.kt | 1268 ----------------- .../clawdbot/android/PermissionRequester.kt | 133 -- .../android/ScreenCaptureRequester.kt | 65 - .../java/com/clawdbot/android/SecurePrefs.kt | 308 ---- .../java/com/clawdbot/android/SessionKey.kt | 13 - .../com/clawdbot/android/VoiceWakeMode.kt | 14 - .../java/com/clawdbot/android/WakeWords.kt | 21 - .../clawdbot/android/chat/ChatController.kt | 524 ------- .../com/clawdbot/android/chat/ChatModels.kt | 44 - .../android/gateway/BonjourEscapes.kt | 35 - .../android/gateway/DeviceAuthStore.kt | 26 - .../android/gateway/DeviceIdentityStore.kt | 146 -- .../android/gateway/GatewayDiscovery.kt | 519 ------- .../android/gateway/GatewayEndpoint.kt | 26 - .../android/gateway/GatewayProtocol.kt | 3 - .../android/gateway/GatewaySession.kt | 683 --------- .../clawdbot/android/gateway/GatewayTls.kt | 90 -- .../android/node/CameraCaptureManager.kt | 316 ---- .../clawdbot/android/node/CanvasController.kt | 264 ---- .../clawdbot/android/node/JpegSizeLimiter.kt | 61 - .../android/node/LocationCaptureManager.kt | 117 -- .../android/node/ScreenRecordManager.kt | 199 --- .../com/clawdbot/android/node/SmsManager.kt | 230 --- .../protocol/ClawdbotCanvasA2UIAction.kt | 66 - .../protocol/ClawdbotProtocolConstants.kt | 71 - .../com/clawdbot/android/tools/ToolDisplay.kt | 222 --- .../clawdbot/android/ui/CameraHudOverlay.kt | 44 - .../java/com/clawdbot/android/ui/ChatSheet.kt | 10 - .../com/clawdbot/android/ui/ClawdbotTheme.kt | 32 - .../com/clawdbot/android/ui/RootScreen.kt | 449 ------ .../com/clawdbot/android/ui/SettingsSheet.kt | 686 --------- .../com/clawdbot/android/ui/StatusPill.kt | 114 -- .../com/clawdbot/android/ui/TalkOrbOverlay.kt | 134 -- .../clawdbot/android/ui/chat/ChatComposer.kt | 285 ---- .../clawdbot/android/ui/chat/ChatMarkdown.kt | 215 --- .../android/ui/chat/ChatMessageListCard.kt | 111 -- .../android/ui/chat/ChatMessageViews.kt | 252 ---- .../android/ui/chat/ChatSessionsDialog.kt | 92 -- .../android/ui/chat/ChatSheetContent.kt | 147 -- .../android/ui/chat/SessionFilters.kt | 49 - .../android/voice/StreamingMediaDataSource.kt | 98 -- .../android/voice/TalkDirectiveParser.kt | 191 --- .../clawdbot/android/voice/TalkModeManager.kt | 1257 ---------------- .../voice/VoiceWakeCommandExtractor.kt | 40 - .../android/voice/VoiceWakeManager.kt | 173 --- .../android/NodeForegroundServiceTest.kt | 43 - .../com/clawdbot/android/WakeWordsTest.kt | 50 - .../android/gateway/BonjourEscapesTest.kt | 19 - .../CanvasControllerSnapshotParamsTest.kt | 43 - .../android/node/JpegSizeLimiterTest.kt | 47 - .../clawdbot/android/node/SmsManagerTest.kt | 91 -- .../protocol/ClawdbotCanvasA2UIActionTest.kt | 49 - .../protocol/ClawdbotProtocolConstantsTest.kt | 35 - .../android/ui/chat/SessionFiltersTest.kt | 35 - .../android/voice/TalkDirectiveParserTest.kt | 55 - .../voice/VoiceWakeCommandExtractorTest.kt | 25 - 63 files changed, 10900 deletions(-) delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/CameraHudState.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/DeviceNames.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/LocationMode.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/MainActivity.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/MainViewModel.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/NodeApp.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/NodeForegroundService.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/NodeRuntime.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/PermissionRequester.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ScreenCaptureRequester.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/SecurePrefs.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/SessionKey.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/VoiceWakeMode.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/WakeWords.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/chat/ChatController.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/chat/ChatModels.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/gateway/BonjourEscapes.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/gateway/DeviceAuthStore.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/gateway/DeviceIdentityStore.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayDiscovery.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayEndpoint.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayProtocol.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewaySession.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayTls.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/node/CameraCaptureManager.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/node/CanvasController.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/node/JpegSizeLimiter.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/node/LocationCaptureManager.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/node/ScreenRecordManager.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/node/SmsManager.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/protocol/ClawdbotCanvasA2UIAction.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/protocol/ClawdbotProtocolConstants.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/tools/ToolDisplay.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/CameraHudOverlay.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/ChatSheet.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/ClawdbotTheme.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/RootScreen.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/SettingsSheet.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/StatusPill.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/TalkOrbOverlay.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatComposer.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMarkdown.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMessageListCard.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMessageViews.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatSessionsDialog.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatSheetContent.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/ui/chat/SessionFilters.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/voice/StreamingMediaDataSource.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/voice/TalkDirectiveParser.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/voice/TalkModeManager.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/voice/VoiceWakeCommandExtractor.kt delete mode 100644 apps/android/app/src/main/java/com/clawdbot/android/voice/VoiceWakeManager.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/NodeForegroundServiceTest.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/WakeWordsTest.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/gateway/BonjourEscapesTest.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/node/CanvasControllerSnapshotParamsTest.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/node/JpegSizeLimiterTest.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/node/SmsManagerTest.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/protocol/ClawdbotCanvasA2UIActionTest.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/protocol/ClawdbotProtocolConstantsTest.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/ui/chat/SessionFiltersTest.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/voice/TalkDirectiveParserTest.kt delete mode 100644 apps/android/app/src/test/java/com/clawdbot/android/voice/VoiceWakeCommandExtractorTest.kt diff --git a/apps/android/app/src/main/java/com/clawdbot/android/CameraHudState.kt b/apps/android/app/src/main/java/com/clawdbot/android/CameraHudState.kt deleted file mode 100644 index 1c9b3986f..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/CameraHudState.kt +++ /dev/null @@ -1,14 +0,0 @@ -package com.clawdbot.android - -enum class CameraHudKind { - Photo, - Recording, - Success, - Error, -} - -data class CameraHudState( - val token: Long, - val kind: CameraHudKind, - val message: String, -) diff --git a/apps/android/app/src/main/java/com/clawdbot/android/DeviceNames.kt b/apps/android/app/src/main/java/com/clawdbot/android/DeviceNames.kt deleted file mode 100644 index dfe5c590b..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/DeviceNames.kt +++ /dev/null @@ -1,26 +0,0 @@ -package com.clawdbot.android - -import android.content.Context -import android.os.Build -import android.provider.Settings - -object DeviceNames { - fun bestDefaultNodeName(context: Context): String { - val deviceName = - runCatching { - Settings.Global.getString(context.contentResolver, "device_name") - } - .getOrNull() - ?.trim() - .orEmpty() - - if (deviceName.isNotEmpty()) return deviceName - - val model = - listOfNotNull(Build.MANUFACTURER?.takeIf { it.isNotBlank() }, Build.MODEL?.takeIf { it.isNotBlank() }) - .joinToString(" ") - .trim() - - return model.ifEmpty { "Android Node" } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/LocationMode.kt b/apps/android/app/src/main/java/com/clawdbot/android/LocationMode.kt deleted file mode 100644 index 4df77f632..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/LocationMode.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.clawdbot.android - -enum class LocationMode(val rawValue: String) { - Off("off"), - WhileUsing("whileUsing"), - Always("always"), - ; - - companion object { - fun fromRawValue(raw: String?): LocationMode { - val normalized = raw?.trim()?.lowercase() - return entries.firstOrNull { it.rawValue.lowercase() == normalized } ?: Off - } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/MainActivity.kt b/apps/android/app/src/main/java/com/clawdbot/android/MainActivity.kt deleted file mode 100644 index 92ad6077a..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/MainActivity.kt +++ /dev/null @@ -1,130 +0,0 @@ -package com.clawdbot.android - -import android.Manifest -import android.content.pm.ApplicationInfo -import android.os.Bundle -import android.os.Build -import android.view.WindowManager -import android.webkit.WebView -import androidx.activity.ComponentActivity -import androidx.activity.compose.setContent -import androidx.activity.viewModels -import androidx.compose.material3.Surface -import androidx.compose.ui.Modifier -import androidx.core.content.ContextCompat -import androidx.core.view.WindowCompat -import androidx.core.view.WindowInsetsCompat -import androidx.core.view.WindowInsetsControllerCompat -import androidx.lifecycle.Lifecycle -import androidx.lifecycle.lifecycleScope -import androidx.lifecycle.repeatOnLifecycle -import com.clawdbot.android.ui.RootScreen -import com.clawdbot.android.ui.MoltbotTheme -import kotlinx.coroutines.launch - -class MainActivity : ComponentActivity() { - private val viewModel: MainViewModel by viewModels() - private lateinit var permissionRequester: PermissionRequester - private lateinit var screenCaptureRequester: ScreenCaptureRequester - - override fun onCreate(savedInstanceState: Bundle?) { - super.onCreate(savedInstanceState) - val isDebuggable = (applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE) != 0 - WebView.setWebContentsDebuggingEnabled(isDebuggable) - applyImmersiveMode() - requestDiscoveryPermissionsIfNeeded() - requestNotificationPermissionIfNeeded() - NodeForegroundService.start(this) - permissionRequester = PermissionRequester(this) - screenCaptureRequester = ScreenCaptureRequester(this) - viewModel.camera.attachLifecycleOwner(this) - viewModel.camera.attachPermissionRequester(permissionRequester) - viewModel.sms.attachPermissionRequester(permissionRequester) - viewModel.screenRecorder.attachScreenCaptureRequester(screenCaptureRequester) - viewModel.screenRecorder.attachPermissionRequester(permissionRequester) - - lifecycleScope.launch { - repeatOnLifecycle(Lifecycle.State.STARTED) { - viewModel.preventSleep.collect { enabled -> - if (enabled) { - window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) - } else { - window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) - } - } - } - } - - setContent { - MoltbotTheme { - Surface(modifier = Modifier) { - RootScreen(viewModel = viewModel) - } - } - } - } - - override fun onResume() { - super.onResume() - applyImmersiveMode() - } - - override fun onWindowFocusChanged(hasFocus: Boolean) { - super.onWindowFocusChanged(hasFocus) - if (hasFocus) { - applyImmersiveMode() - } - } - - override fun onStart() { - super.onStart() - viewModel.setForeground(true) - } - - override fun onStop() { - viewModel.setForeground(false) - super.onStop() - } - - private fun applyImmersiveMode() { - WindowCompat.setDecorFitsSystemWindows(window, false) - val controller = WindowInsetsControllerCompat(window, window.decorView) - controller.systemBarsBehavior = - WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE - controller.hide(WindowInsetsCompat.Type.systemBars()) - } - - private fun requestDiscoveryPermissionsIfNeeded() { - if (Build.VERSION.SDK_INT >= 33) { - val ok = - ContextCompat.checkSelfPermission( - this, - Manifest.permission.NEARBY_WIFI_DEVICES, - ) == android.content.pm.PackageManager.PERMISSION_GRANTED - if (!ok) { - requestPermissions(arrayOf(Manifest.permission.NEARBY_WIFI_DEVICES), 100) - } - } else { - val ok = - ContextCompat.checkSelfPermission( - this, - Manifest.permission.ACCESS_FINE_LOCATION, - ) == android.content.pm.PackageManager.PERMISSION_GRANTED - if (!ok) { - requestPermissions(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION), 101) - } - } - } - - private fun requestNotificationPermissionIfNeeded() { - if (Build.VERSION.SDK_INT < 33) return - val ok = - ContextCompat.checkSelfPermission( - this, - Manifest.permission.POST_NOTIFICATIONS, - ) == android.content.pm.PackageManager.PERMISSION_GRANTED - if (!ok) { - requestPermissions(arrayOf(Manifest.permission.POST_NOTIFICATIONS), 102) - } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/MainViewModel.kt b/apps/android/app/src/main/java/com/clawdbot/android/MainViewModel.kt deleted file mode 100644 index 1329f06d4..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/MainViewModel.kt +++ /dev/null @@ -1,174 +0,0 @@ -package com.clawdbot.android - -import android.app.Application -import androidx.lifecycle.AndroidViewModel -import com.clawdbot.android.gateway.GatewayEndpoint -import com.clawdbot.android.chat.OutgoingAttachment -import com.clawdbot.android.node.CameraCaptureManager -import com.clawdbot.android.node.CanvasController -import com.clawdbot.android.node.ScreenRecordManager -import com.clawdbot.android.node.SmsManager -import kotlinx.coroutines.flow.StateFlow - -class MainViewModel(app: Application) : AndroidViewModel(app) { - private val runtime: NodeRuntime = (app as NodeApp).runtime - - val canvas: CanvasController = runtime.canvas - val camera: CameraCaptureManager = runtime.camera - val screenRecorder: ScreenRecordManager = runtime.screenRecorder - val sms: SmsManager = runtime.sms - - val gateways: StateFlow> = runtime.gateways - val discoveryStatusText: StateFlow = runtime.discoveryStatusText - - val isConnected: StateFlow = runtime.isConnected - val statusText: StateFlow = runtime.statusText - val serverName: StateFlow = runtime.serverName - val remoteAddress: StateFlow = runtime.remoteAddress - val isForeground: StateFlow = runtime.isForeground - val seamColorArgb: StateFlow = runtime.seamColorArgb - val mainSessionKey: StateFlow = runtime.mainSessionKey - - val cameraHud: StateFlow = runtime.cameraHud - val cameraFlashToken: StateFlow = runtime.cameraFlashToken - val screenRecordActive: StateFlow = runtime.screenRecordActive - - val instanceId: StateFlow = runtime.instanceId - val displayName: StateFlow = runtime.displayName - val cameraEnabled: StateFlow = runtime.cameraEnabled - val locationMode: StateFlow = runtime.locationMode - val locationPreciseEnabled: StateFlow = runtime.locationPreciseEnabled - val preventSleep: StateFlow = runtime.preventSleep - val wakeWords: StateFlow> = runtime.wakeWords - val voiceWakeMode: StateFlow = runtime.voiceWakeMode - val voiceWakeStatusText: StateFlow = runtime.voiceWakeStatusText - val voiceWakeIsListening: StateFlow = runtime.voiceWakeIsListening - val talkEnabled: StateFlow = runtime.talkEnabled - val talkStatusText: StateFlow = runtime.talkStatusText - val talkIsListening: StateFlow = runtime.talkIsListening - val talkIsSpeaking: StateFlow = runtime.talkIsSpeaking - val manualEnabled: StateFlow = runtime.manualEnabled - val manualHost: StateFlow = runtime.manualHost - val manualPort: StateFlow = runtime.manualPort - val manualTls: StateFlow = runtime.manualTls - val canvasDebugStatusEnabled: StateFlow = runtime.canvasDebugStatusEnabled - - val chatSessionKey: StateFlow = runtime.chatSessionKey - val chatSessionId: StateFlow = runtime.chatSessionId - val chatMessages = runtime.chatMessages - val chatError: StateFlow = runtime.chatError - val chatHealthOk: StateFlow = runtime.chatHealthOk - val chatThinkingLevel: StateFlow = runtime.chatThinkingLevel - val chatStreamingAssistantText: StateFlow = runtime.chatStreamingAssistantText - val chatPendingToolCalls = runtime.chatPendingToolCalls - val chatSessions = runtime.chatSessions - val pendingRunCount: StateFlow = runtime.pendingRunCount - - fun setForeground(value: Boolean) { - runtime.setForeground(value) - } - - fun setDisplayName(value: String) { - runtime.setDisplayName(value) - } - - fun setCameraEnabled(value: Boolean) { - runtime.setCameraEnabled(value) - } - - fun setLocationMode(mode: LocationMode) { - runtime.setLocationMode(mode) - } - - fun setLocationPreciseEnabled(value: Boolean) { - runtime.setLocationPreciseEnabled(value) - } - - fun setPreventSleep(value: Boolean) { - runtime.setPreventSleep(value) - } - - fun setManualEnabled(value: Boolean) { - runtime.setManualEnabled(value) - } - - fun setManualHost(value: String) { - runtime.setManualHost(value) - } - - fun setManualPort(value: Int) { - runtime.setManualPort(value) - } - - fun setManualTls(value: Boolean) { - runtime.setManualTls(value) - } - - fun setCanvasDebugStatusEnabled(value: Boolean) { - runtime.setCanvasDebugStatusEnabled(value) - } - - fun setWakeWords(words: List) { - runtime.setWakeWords(words) - } - - fun resetWakeWordsDefaults() { - runtime.resetWakeWordsDefaults() - } - - fun setVoiceWakeMode(mode: VoiceWakeMode) { - runtime.setVoiceWakeMode(mode) - } - - fun setTalkEnabled(enabled: Boolean) { - runtime.setTalkEnabled(enabled) - } - - fun refreshGatewayConnection() { - runtime.refreshGatewayConnection() - } - - fun connect(endpoint: GatewayEndpoint) { - runtime.connect(endpoint) - } - - fun connectManual() { - runtime.connectManual() - } - - fun disconnect() { - runtime.disconnect() - } - - fun handleCanvasA2UIActionFromWebView(payloadJson: String) { - runtime.handleCanvasA2UIActionFromWebView(payloadJson) - } - - fun loadChat(sessionKey: String) { - runtime.loadChat(sessionKey) - } - - fun refreshChat() { - runtime.refreshChat() - } - - fun refreshChatSessions(limit: Int? = null) { - runtime.refreshChatSessions(limit = limit) - } - - fun setChatThinkingLevel(level: String) { - runtime.setChatThinkingLevel(level) - } - - fun switchChatSession(sessionKey: String) { - runtime.switchChatSession(sessionKey) - } - - fun abortChat() { - runtime.abortChat() - } - - fun sendChat(message: String, thinking: String, attachments: List) { - runtime.sendChat(message = message, thinking = thinking, attachments = attachments) - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/NodeApp.kt b/apps/android/app/src/main/java/com/clawdbot/android/NodeApp.kt deleted file mode 100644 index 228794ff3..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/NodeApp.kt +++ /dev/null @@ -1,26 +0,0 @@ -package com.clawdbot.android - -import android.app.Application -import android.os.StrictMode - -class NodeApp : Application() { - val runtime: NodeRuntime by lazy { NodeRuntime(this) } - - override fun onCreate() { - super.onCreate() - if (BuildConfig.DEBUG) { - StrictMode.setThreadPolicy( - StrictMode.ThreadPolicy.Builder() - .detectAll() - .penaltyLog() - .build(), - ) - StrictMode.setVmPolicy( - StrictMode.VmPolicy.Builder() - .detectAll() - .penaltyLog() - .build(), - ) - } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/NodeForegroundService.kt b/apps/android/app/src/main/java/com/clawdbot/android/NodeForegroundService.kt deleted file mode 100644 index a3074f8b1..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/NodeForegroundService.kt +++ /dev/null @@ -1,180 +0,0 @@ -package com.clawdbot.android - -import android.app.Notification -import android.app.NotificationChannel -import android.app.NotificationManager -import android.app.Service -import android.app.PendingIntent -import android.Manifest -import android.content.Context -import android.content.Intent -import android.content.pm.PackageManager -import android.content.pm.ServiceInfo -import androidx.core.app.NotificationCompat -import androidx.core.content.ContextCompat -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.cancel -import kotlinx.coroutines.flow.combine -import kotlinx.coroutines.launch - -class NodeForegroundService : Service() { - private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main) - private var notificationJob: Job? = null - private var lastRequiresMic = false - private var didStartForeground = false - - override fun onCreate() { - super.onCreate() - ensureChannel() - val initial = buildNotification(title = "Moltbot Node", text = "Starting…") - startForegroundWithTypes(notification = initial, requiresMic = false) - - val runtime = (application as NodeApp).runtime - notificationJob = - scope.launch { - combine( - runtime.statusText, - runtime.serverName, - runtime.isConnected, - runtime.voiceWakeMode, - runtime.voiceWakeIsListening, - ) { status, server, connected, voiceMode, voiceListening -> - Quint(status, server, connected, voiceMode, voiceListening) - }.collect { (status, server, connected, voiceMode, voiceListening) -> - val title = if (connected) "Moltbot Node · Connected" else "Moltbot Node" - val voiceSuffix = - if (voiceMode == VoiceWakeMode.Always) { - if (voiceListening) " · Voice Wake: Listening" else " · Voice Wake: Paused" - } else { - "" - } - val text = (server?.let { "$status · $it" } ?: status) + voiceSuffix - - val requiresMic = - voiceMode == VoiceWakeMode.Always && hasRecordAudioPermission() - startForegroundWithTypes( - notification = buildNotification(title = title, text = text), - requiresMic = requiresMic, - ) - } - } - } - - override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { - when (intent?.action) { - ACTION_STOP -> { - (application as NodeApp).runtime.disconnect() - stopSelf() - return START_NOT_STICKY - } - } - // Keep running; connection is managed by NodeRuntime (auto-reconnect + manual). - return START_STICKY - } - - override fun onDestroy() { - notificationJob?.cancel() - scope.cancel() - super.onDestroy() - } - - override fun onBind(intent: Intent?) = null - - private fun ensureChannel() { - val mgr = getSystemService(NotificationManager::class.java) - val channel = - NotificationChannel( - CHANNEL_ID, - "Connection", - NotificationManager.IMPORTANCE_LOW, - ).apply { - description = "Moltbot node connection status" - setShowBadge(false) - } - mgr.createNotificationChannel(channel) - } - - private fun buildNotification(title: String, text: String): Notification { - val launchIntent = Intent(this, MainActivity::class.java).apply { - flags = Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP - } - val launchPending = - PendingIntent.getActivity( - this, - 1, - launchIntent, - PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE, - ) - - val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP) - val stopPending = - PendingIntent.getService( - this, - 2, - stopIntent, - PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE, - ) - - return NotificationCompat.Builder(this, CHANNEL_ID) - .setSmallIcon(R.mipmap.ic_launcher) - .setContentTitle(title) - .setContentText(text) - .setContentIntent(launchPending) - .setOngoing(true) - .setOnlyAlertOnce(true) - .setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE) - .addAction(0, "Disconnect", stopPending) - .build() - } - - private fun updateNotification(notification: Notification) { - val mgr = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - mgr.notify(NOTIFICATION_ID, notification) - } - - private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) { - if (didStartForeground && requiresMic == lastRequiresMic) { - updateNotification(notification) - return - } - - lastRequiresMic = requiresMic - val types = - if (requiresMic) { - ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE - } else { - ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC - } - startForeground(NOTIFICATION_ID, notification, types) - didStartForeground = true - } - - private fun hasRecordAudioPermission(): Boolean { - return ( - ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == - PackageManager.PERMISSION_GRANTED - ) - } - - companion object { - private const val CHANNEL_ID = "connection" - private const val NOTIFICATION_ID = 1 - - private const val ACTION_STOP = "com.clawdbot.android.action.STOP" - - fun start(context: Context) { - val intent = Intent(context, NodeForegroundService::class.java) - context.startForegroundService(intent) - } - - fun stop(context: Context) { - val intent = Intent(context, NodeForegroundService::class.java).setAction(ACTION_STOP) - context.startService(intent) - } - } -} - -private data class Quint(val first: A, val second: B, val third: C, val fourth: D, val fifth: E) diff --git a/apps/android/app/src/main/java/com/clawdbot/android/NodeRuntime.kt b/apps/android/app/src/main/java/com/clawdbot/android/NodeRuntime.kt deleted file mode 100644 index 46e486100..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/NodeRuntime.kt +++ /dev/null @@ -1,1268 +0,0 @@ -package com.clawdbot.android - -import android.Manifest -import android.content.Context -import android.content.pm.PackageManager -import android.location.LocationManager -import android.os.Build -import android.os.SystemClock -import androidx.core.content.ContextCompat -import com.clawdbot.android.chat.ChatController -import com.clawdbot.android.chat.ChatMessage -import com.clawdbot.android.chat.ChatPendingToolCall -import com.clawdbot.android.chat.ChatSessionEntry -import com.clawdbot.android.chat.OutgoingAttachment -import com.clawdbot.android.gateway.DeviceAuthStore -import com.clawdbot.android.gateway.DeviceIdentityStore -import com.clawdbot.android.gateway.GatewayClientInfo -import com.clawdbot.android.gateway.GatewayConnectOptions -import com.clawdbot.android.gateway.GatewayDiscovery -import com.clawdbot.android.gateway.GatewayEndpoint -import com.clawdbot.android.gateway.GatewaySession -import com.clawdbot.android.gateway.GatewayTlsParams -import com.clawdbot.android.node.CameraCaptureManager -import com.clawdbot.android.node.LocationCaptureManager -import com.clawdbot.android.BuildConfig -import com.clawdbot.android.node.CanvasController -import com.clawdbot.android.node.ScreenRecordManager -import com.clawdbot.android.node.SmsManager -import com.clawdbot.android.protocol.MoltbotCapability -import com.clawdbot.android.protocol.MoltbotCameraCommand -import com.clawdbot.android.protocol.MoltbotCanvasA2UIAction -import com.clawdbot.android.protocol.MoltbotCanvasA2UICommand -import com.clawdbot.android.protocol.MoltbotCanvasCommand -import com.clawdbot.android.protocol.MoltbotScreenCommand -import com.clawdbot.android.protocol.MoltbotLocationCommand -import com.clawdbot.android.protocol.MoltbotSmsCommand -import com.clawdbot.android.voice.TalkModeManager -import com.clawdbot.android.voice.VoiceWakeManager -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.flow.combine -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.distinctUntilChanged -import kotlinx.coroutines.launch -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonNull -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive -import kotlinx.serialization.json.buildJsonObject -import java.util.concurrent.atomic.AtomicLong - -class NodeRuntime(context: Context) { - private val appContext = context.applicationContext - private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) - - val prefs = SecurePrefs(appContext) - private val deviceAuthStore = DeviceAuthStore(prefs) - val canvas = CanvasController() - val camera = CameraCaptureManager(appContext) - val location = LocationCaptureManager(appContext) - val screenRecorder = ScreenRecordManager(appContext) - val sms = SmsManager(appContext) - private val json = Json { ignoreUnknownKeys = true } - - private val externalAudioCaptureActive = MutableStateFlow(false) - - private val voiceWake: VoiceWakeManager by lazy { - VoiceWakeManager( - context = appContext, - scope = scope, - onCommand = { command -> - nodeSession.sendNodeEvent( - event = "agent.request", - payloadJson = - buildJsonObject { - put("message", JsonPrimitive(command)) - put("sessionKey", JsonPrimitive(resolveMainSessionKey())) - put("thinking", JsonPrimitive(chatThinkingLevel.value)) - put("deliver", JsonPrimitive(false)) - }.toString(), - ) - }, - ) - } - - val voiceWakeIsListening: StateFlow - get() = voiceWake.isListening - - val voiceWakeStatusText: StateFlow - get() = voiceWake.statusText - - val talkStatusText: StateFlow - get() = talkMode.statusText - - val talkIsListening: StateFlow - get() = talkMode.isListening - - val talkIsSpeaking: StateFlow - get() = talkMode.isSpeaking - - private val discovery = GatewayDiscovery(appContext, scope = scope) - val gateways: StateFlow> = discovery.gateways - val discoveryStatusText: StateFlow = discovery.statusText - - private val identityStore = DeviceIdentityStore(appContext) - - private val _isConnected = MutableStateFlow(false) - val isConnected: StateFlow = _isConnected.asStateFlow() - - private val _statusText = MutableStateFlow("Offline") - val statusText: StateFlow = _statusText.asStateFlow() - - private val _mainSessionKey = MutableStateFlow("main") - val mainSessionKey: StateFlow = _mainSessionKey.asStateFlow() - - private val cameraHudSeq = AtomicLong(0) - private val _cameraHud = MutableStateFlow(null) - val cameraHud: StateFlow = _cameraHud.asStateFlow() - - private val _cameraFlashToken = MutableStateFlow(0L) - val cameraFlashToken: StateFlow = _cameraFlashToken.asStateFlow() - - private val _screenRecordActive = MutableStateFlow(false) - val screenRecordActive: StateFlow = _screenRecordActive.asStateFlow() - - private val _serverName = MutableStateFlow(null) - val serverName: StateFlow = _serverName.asStateFlow() - - private val _remoteAddress = MutableStateFlow(null) - val remoteAddress: StateFlow = _remoteAddress.asStateFlow() - - private val _seamColorArgb = MutableStateFlow(DEFAULT_SEAM_COLOR_ARGB) - val seamColorArgb: StateFlow = _seamColorArgb.asStateFlow() - - private val _isForeground = MutableStateFlow(true) - val isForeground: StateFlow = _isForeground.asStateFlow() - - private var lastAutoA2uiUrl: String? = null - private var operatorConnected = false - private var nodeConnected = false - private var operatorStatusText: String = "Offline" - private var nodeStatusText: String = "Offline" - private var connectedEndpoint: GatewayEndpoint? = null - - private val operatorSession = - GatewaySession( - scope = scope, - identityStore = identityStore, - deviceAuthStore = deviceAuthStore, - onConnected = { name, remote, mainSessionKey -> - operatorConnected = true - operatorStatusText = "Connected" - _serverName.value = name - _remoteAddress.value = remote - _seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB - applyMainSessionKey(mainSessionKey) - updateStatus() - scope.launch { refreshBrandingFromGateway() } - scope.launch { refreshWakeWordsFromGateway() } - }, - onDisconnected = { message -> - operatorConnected = false - operatorStatusText = message - _serverName.value = null - _remoteAddress.value = null - _seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB - if (!isCanonicalMainSessionKey(_mainSessionKey.value)) { - _mainSessionKey.value = "main" - } - val mainKey = resolveMainSessionKey() - talkMode.setMainSessionKey(mainKey) - chat.applyMainSessionKey(mainKey) - chat.onDisconnected(message) - updateStatus() - }, - onEvent = { event, payloadJson -> - handleGatewayEvent(event, payloadJson) - }, - ) - - private val nodeSession = - GatewaySession( - scope = scope, - identityStore = identityStore, - deviceAuthStore = deviceAuthStore, - onConnected = { _, _, _ -> - nodeConnected = true - nodeStatusText = "Connected" - updateStatus() - maybeNavigateToA2uiOnConnect() - }, - onDisconnected = { message -> - nodeConnected = false - nodeStatusText = message - updateStatus() - showLocalCanvasOnDisconnect() - }, - onEvent = { _, _ -> }, - onInvoke = { req -> - handleInvoke(req.command, req.paramsJson) - }, - onTlsFingerprint = { stableId, fingerprint -> - prefs.saveGatewayTlsFingerprint(stableId, fingerprint) - }, - ) - - private val chat: ChatController = - ChatController( - scope = scope, - session = operatorSession, - json = json, - supportsChatSubscribe = false, - ) - private val talkMode: TalkModeManager by lazy { - TalkModeManager( - context = appContext, - scope = scope, - session = operatorSession, - supportsChatSubscribe = false, - isConnected = { operatorConnected }, - ) - } - - private fun applyMainSessionKey(candidate: String?) { - val trimmed = candidate?.trim().orEmpty() - if (trimmed.isEmpty()) return - if (isCanonicalMainSessionKey(_mainSessionKey.value)) return - if (_mainSessionKey.value == trimmed) return - _mainSessionKey.value = trimmed - talkMode.setMainSessionKey(trimmed) - chat.applyMainSessionKey(trimmed) - } - - private fun updateStatus() { - _isConnected.value = operatorConnected - _statusText.value = - when { - operatorConnected && nodeConnected -> "Connected" - operatorConnected && !nodeConnected -> "Connected (node offline)" - !operatorConnected && nodeConnected -> "Connected (operator offline)" - operatorStatusText.isNotBlank() && operatorStatusText != "Offline" -> operatorStatusText - else -> nodeStatusText - } - } - - private fun resolveMainSessionKey(): String { - val trimmed = _mainSessionKey.value.trim() - return if (trimmed.isEmpty()) "main" else trimmed - } - - private fun maybeNavigateToA2uiOnConnect() { - val a2uiUrl = resolveA2uiHostUrl() ?: return - val current = canvas.currentUrl()?.trim().orEmpty() - if (current.isEmpty() || current == lastAutoA2uiUrl) { - lastAutoA2uiUrl = a2uiUrl - canvas.navigate(a2uiUrl) - } - } - - private fun showLocalCanvasOnDisconnect() { - lastAutoA2uiUrl = null - canvas.navigate("") - } - - val instanceId: StateFlow = prefs.instanceId - val displayName: StateFlow = prefs.displayName - val cameraEnabled: StateFlow = prefs.cameraEnabled - val locationMode: StateFlow = prefs.locationMode - val locationPreciseEnabled: StateFlow = prefs.locationPreciseEnabled - val preventSleep: StateFlow = prefs.preventSleep - val wakeWords: StateFlow> = prefs.wakeWords - val voiceWakeMode: StateFlow = prefs.voiceWakeMode - val talkEnabled: StateFlow = prefs.talkEnabled - val manualEnabled: StateFlow = prefs.manualEnabled - val manualHost: StateFlow = prefs.manualHost - val manualPort: StateFlow = prefs.manualPort - val manualTls: StateFlow = prefs.manualTls - val lastDiscoveredStableId: StateFlow = prefs.lastDiscoveredStableId - val canvasDebugStatusEnabled: StateFlow = prefs.canvasDebugStatusEnabled - - private var didAutoConnect = false - private var suppressWakeWordsSync = false - private var wakeWordsSyncJob: Job? = null - - val chatSessionKey: StateFlow = chat.sessionKey - val chatSessionId: StateFlow = chat.sessionId - val chatMessages: StateFlow> = chat.messages - val chatError: StateFlow = chat.errorText - val chatHealthOk: StateFlow = chat.healthOk - val chatThinkingLevel: StateFlow = chat.thinkingLevel - val chatStreamingAssistantText: StateFlow = chat.streamingAssistantText - val chatPendingToolCalls: StateFlow> = chat.pendingToolCalls - val chatSessions: StateFlow> = chat.sessions - val pendingRunCount: StateFlow = chat.pendingRunCount - - init { - scope.launch { - combine( - voiceWakeMode, - isForeground, - externalAudioCaptureActive, - wakeWords, - ) { mode, foreground, externalAudio, words -> - Quad(mode, foreground, externalAudio, words) - }.distinctUntilChanged() - .collect { (mode, foreground, externalAudio, words) -> - voiceWake.setTriggerWords(words) - - val shouldListen = - when (mode) { - VoiceWakeMode.Off -> false - VoiceWakeMode.Foreground -> foreground - VoiceWakeMode.Always -> true - } && !externalAudio - - if (!shouldListen) { - voiceWake.stop(statusText = if (mode == VoiceWakeMode.Off) "Off" else "Paused") - return@collect - } - - if (!hasRecordAudioPermission()) { - voiceWake.stop(statusText = "Microphone permission required") - return@collect - } - - voiceWake.start() - } - } - - scope.launch { - talkEnabled.collect { enabled -> - talkMode.setEnabled(enabled) - externalAudioCaptureActive.value = enabled - } - } - - scope.launch(Dispatchers.Default) { - gateways.collect { list -> - if (list.isNotEmpty()) { - // Persist the last discovered gateway (best-effort UX parity with iOS). - prefs.setLastDiscoveredStableId(list.last().stableId) - } - - if (didAutoConnect) return@collect - if (_isConnected.value) return@collect - - if (manualEnabled.value) { - val host = manualHost.value.trim() - val port = manualPort.value - if (host.isNotEmpty() && port in 1..65535) { - didAutoConnect = true - connect(GatewayEndpoint.manual(host = host, port = port)) - } - return@collect - } - - val targetStableId = lastDiscoveredStableId.value.trim() - if (targetStableId.isEmpty()) return@collect - val target = list.firstOrNull { it.stableId == targetStableId } ?: return@collect - didAutoConnect = true - connect(target) - } - } - - scope.launch { - combine( - canvasDebugStatusEnabled, - statusText, - serverName, - remoteAddress, - ) { debugEnabled, status, server, remote -> - Quad(debugEnabled, status, server, remote) - }.distinctUntilChanged() - .collect { (debugEnabled, status, server, remote) -> - canvas.setDebugStatusEnabled(debugEnabled) - if (!debugEnabled) return@collect - canvas.setDebugStatus(status, server ?: remote) - } - } - } - - fun setForeground(value: Boolean) { - _isForeground.value = value - } - - fun setDisplayName(value: String) { - prefs.setDisplayName(value) - } - - fun setCameraEnabled(value: Boolean) { - prefs.setCameraEnabled(value) - } - - fun setLocationMode(mode: LocationMode) { - prefs.setLocationMode(mode) - } - - fun setLocationPreciseEnabled(value: Boolean) { - prefs.setLocationPreciseEnabled(value) - } - - fun setPreventSleep(value: Boolean) { - prefs.setPreventSleep(value) - } - - fun setManualEnabled(value: Boolean) { - prefs.setManualEnabled(value) - } - - fun setManualHost(value: String) { - prefs.setManualHost(value) - } - - fun setManualPort(value: Int) { - prefs.setManualPort(value) - } - - fun setManualTls(value: Boolean) { - prefs.setManualTls(value) - } - - fun setCanvasDebugStatusEnabled(value: Boolean) { - prefs.setCanvasDebugStatusEnabled(value) - } - - fun setWakeWords(words: List) { - prefs.setWakeWords(words) - scheduleWakeWordsSyncIfNeeded() - } - - fun resetWakeWordsDefaults() { - setWakeWords(SecurePrefs.defaultWakeWords) - } - - fun setVoiceWakeMode(mode: VoiceWakeMode) { - prefs.setVoiceWakeMode(mode) - } - - fun setTalkEnabled(value: Boolean) { - prefs.setTalkEnabled(value) - } - - private fun buildInvokeCommands(): List = - buildList { - add(MoltbotCanvasCommand.Present.rawValue) - add(MoltbotCanvasCommand.Hide.rawValue) - add(MoltbotCanvasCommand.Navigate.rawValue) - add(MoltbotCanvasCommand.Eval.rawValue) - add(MoltbotCanvasCommand.Snapshot.rawValue) - add(MoltbotCanvasA2UICommand.Push.rawValue) - add(MoltbotCanvasA2UICommand.PushJSONL.rawValue) - add(MoltbotCanvasA2UICommand.Reset.rawValue) - add(MoltbotScreenCommand.Record.rawValue) - if (cameraEnabled.value) { - add(MoltbotCameraCommand.Snap.rawValue) - add(MoltbotCameraCommand.Clip.rawValue) - } - if (locationMode.value != LocationMode.Off) { - add(MoltbotLocationCommand.Get.rawValue) - } - if (sms.canSendSms()) { - add(MoltbotSmsCommand.Send.rawValue) - } - } - - private fun buildCapabilities(): List = - buildList { - add(MoltbotCapability.Canvas.rawValue) - add(MoltbotCapability.Screen.rawValue) - if (cameraEnabled.value) add(MoltbotCapability.Camera.rawValue) - if (sms.canSendSms()) add(MoltbotCapability.Sms.rawValue) - if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) { - add(MoltbotCapability.VoiceWake.rawValue) - } - if (locationMode.value != LocationMode.Off) { - add(MoltbotCapability.Location.rawValue) - } - } - - private fun resolvedVersionName(): String { - val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" } - return if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) { - "$versionName-dev" - } else { - versionName - } - } - - private fun resolveModelIdentifier(): String? { - return listOfNotNull(Build.MANUFACTURER, Build.MODEL) - .joinToString(" ") - .trim() - .ifEmpty { null } - } - - private fun buildUserAgent(): String { - val version = resolvedVersionName() - val release = Build.VERSION.RELEASE?.trim().orEmpty() - val releaseLabel = if (release.isEmpty()) "unknown" else release - return "MoltbotAndroid/$version (Android $releaseLabel; SDK ${Build.VERSION.SDK_INT})" - } - - private fun buildClientInfo(clientId: String, clientMode: String): GatewayClientInfo { - return GatewayClientInfo( - id = clientId, - displayName = displayName.value, - version = resolvedVersionName(), - platform = "android", - mode = clientMode, - instanceId = instanceId.value, - deviceFamily = "Android", - modelIdentifier = resolveModelIdentifier(), - ) - } - - private fun buildNodeConnectOptions(): GatewayConnectOptions { - return GatewayConnectOptions( - role = "node", - scopes = emptyList(), - caps = buildCapabilities(), - commands = buildInvokeCommands(), - permissions = emptyMap(), - client = buildClientInfo(clientId = "moltbot-android", clientMode = "node"), - userAgent = buildUserAgent(), - ) - } - - private fun buildOperatorConnectOptions(): GatewayConnectOptions { - return GatewayConnectOptions( - role = "operator", - scopes = emptyList(), - caps = emptyList(), - commands = emptyList(), - permissions = emptyMap(), - client = buildClientInfo(clientId = "moltbot-control-ui", clientMode = "ui"), - userAgent = buildUserAgent(), - ) - } - - fun refreshGatewayConnection() { - val endpoint = connectedEndpoint ?: return - val token = prefs.loadGatewayToken() - val password = prefs.loadGatewayPassword() - val tls = resolveTlsParams(endpoint) - operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls) - nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls) - operatorSession.reconnect() - nodeSession.reconnect() - } - - fun connect(endpoint: GatewayEndpoint) { - connectedEndpoint = endpoint - operatorStatusText = "Connecting…" - nodeStatusText = "Connecting…" - updateStatus() - val token = prefs.loadGatewayToken() - val password = prefs.loadGatewayPassword() - val tls = resolveTlsParams(endpoint) - operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls) - nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls) - } - - private fun hasRecordAudioPermission(): Boolean { - return ( - ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) == - PackageManager.PERMISSION_GRANTED - ) - } - - private fun hasFineLocationPermission(): Boolean { - return ( - ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_FINE_LOCATION) == - PackageManager.PERMISSION_GRANTED - ) - } - - private fun hasCoarseLocationPermission(): Boolean { - return ( - ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_COARSE_LOCATION) == - PackageManager.PERMISSION_GRANTED - ) - } - - private fun hasBackgroundLocationPermission(): Boolean { - return ( - ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_BACKGROUND_LOCATION) == - PackageManager.PERMISSION_GRANTED - ) - } - - fun connectManual() { - val host = manualHost.value.trim() - val port = manualPort.value - if (host.isEmpty() || port <= 0 || port > 65535) { - _statusText.value = "Failed: invalid manual host/port" - return - } - connect(GatewayEndpoint.manual(host = host, port = port)) - } - - fun disconnect() { - connectedEndpoint = null - operatorSession.disconnect() - nodeSession.disconnect() - } - - private fun resolveTlsParams(endpoint: GatewayEndpoint): GatewayTlsParams? { - val stored = prefs.loadGatewayTlsFingerprint(endpoint.stableId) - val hinted = endpoint.tlsEnabled || !endpoint.tlsFingerprintSha256.isNullOrBlank() - val manual = endpoint.stableId.startsWith("manual|") - - if (manual) { - if (!manualTls.value) return null - return GatewayTlsParams( - required = true, - expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored, - allowTOFU = stored == null, - stableId = endpoint.stableId, - ) - } - - if (hinted) { - return GatewayTlsParams( - required = true, - expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored, - allowTOFU = stored == null, - stableId = endpoint.stableId, - ) - } - - if (!stored.isNullOrBlank()) { - return GatewayTlsParams( - required = true, - expectedFingerprint = stored, - allowTOFU = false, - stableId = endpoint.stableId, - ) - } - - return null - } - - fun handleCanvasA2UIActionFromWebView(payloadJson: String) { - scope.launch { - val trimmed = payloadJson.trim() - if (trimmed.isEmpty()) return@launch - - val root = - try { - json.parseToJsonElement(trimmed).asObjectOrNull() ?: return@launch - } catch (_: Throwable) { - return@launch - } - - val userActionObj = (root["userAction"] as? JsonObject) ?: root - val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { - java.util.UUID.randomUUID().toString() - } - val name = MoltbotCanvasA2UIAction.extractActionName(userActionObj) ?: return@launch - - val surfaceId = - (userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" } - val sourceComponentId = - (userActionObj["sourceComponentId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "-" } - val contextJson = (userActionObj["context"] as? JsonObject)?.toString() - - val sessionKey = resolveMainSessionKey() - val message = - MoltbotCanvasA2UIAction.formatAgentMessage( - actionName = name, - sessionKey = sessionKey, - surfaceId = surfaceId, - sourceComponentId = sourceComponentId, - host = displayName.value, - instanceId = instanceId.value.lowercase(), - contextJson = contextJson, - ) - - val connected = nodeConnected - var error: String? = null - if (connected) { - try { - nodeSession.sendNodeEvent( - event = "agent.request", - payloadJson = - buildJsonObject { - put("message", JsonPrimitive(message)) - put("sessionKey", JsonPrimitive(sessionKey)) - put("thinking", JsonPrimitive("low")) - put("deliver", JsonPrimitive(false)) - put("key", JsonPrimitive(actionId)) - }.toString(), - ) - } catch (e: Throwable) { - error = e.message ?: "send failed" - } - } else { - error = "gateway not connected" - } - - try { - canvas.eval( - MoltbotCanvasA2UIAction.jsDispatchA2UIActionStatus( - actionId = actionId, - ok = connected && error == null, - error = error, - ), - ) - } catch (_: Throwable) { - // ignore - } - } - } - - fun loadChat(sessionKey: String) { - val key = sessionKey.trim().ifEmpty { resolveMainSessionKey() } - chat.load(key) - } - - fun refreshChat() { - chat.refresh() - } - - fun refreshChatSessions(limit: Int? = null) { - chat.refreshSessions(limit = limit) - } - - fun setChatThinkingLevel(level: String) { - chat.setThinkingLevel(level) - } - - fun switchChatSession(sessionKey: String) { - chat.switchSession(sessionKey) - } - - fun abortChat() { - chat.abort() - } - - fun sendChat(message: String, thinking: String, attachments: List) { - chat.sendMessage(message = message, thinkingLevel = thinking, attachments = attachments) - } - - private fun handleGatewayEvent(event: String, payloadJson: String?) { - if (event == "voicewake.changed") { - if (payloadJson.isNullOrBlank()) return - try { - val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return - val array = payload["triggers"] as? JsonArray ?: return - val triggers = array.mapNotNull { it.asStringOrNull() } - applyWakeWordsFromGateway(triggers) - } catch (_: Throwable) { - // ignore - } - return - } - - talkMode.handleGatewayEvent(event, payloadJson) - chat.handleGatewayEvent(event, payloadJson) - } - - private fun applyWakeWordsFromGateway(words: List) { - suppressWakeWordsSync = true - prefs.setWakeWords(words) - suppressWakeWordsSync = false - } - - private fun scheduleWakeWordsSyncIfNeeded() { - if (suppressWakeWordsSync) return - if (!_isConnected.value) return - - val snapshot = prefs.wakeWords.value - wakeWordsSyncJob?.cancel() - wakeWordsSyncJob = - scope.launch { - delay(650) - val jsonList = snapshot.joinToString(separator = ",") { it.toJsonString() } - val params = """{"triggers":[$jsonList]}""" - try { - operatorSession.request("voicewake.set", params) - } catch (_: Throwable) { - // ignore - } - } - } - - private suspend fun refreshWakeWordsFromGateway() { - if (!_isConnected.value) return - try { - val res = operatorSession.request("voicewake.get", "{}") - val payload = json.parseToJsonElement(res).asObjectOrNull() ?: return - val array = payload["triggers"] as? JsonArray ?: return - val triggers = array.mapNotNull { it.asStringOrNull() } - applyWakeWordsFromGateway(triggers) - } catch (_: Throwable) { - // ignore - } - } - - private suspend fun refreshBrandingFromGateway() { - if (!_isConnected.value) return - try { - val res = operatorSession.request("config.get", "{}") - val root = json.parseToJsonElement(res).asObjectOrNull() - val config = root?.get("config").asObjectOrNull() - val ui = config?.get("ui").asObjectOrNull() - val raw = ui?.get("seamColor").asStringOrNull()?.trim() - val sessionCfg = config?.get("session").asObjectOrNull() - val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull()) - applyMainSessionKey(mainKey) - - val parsed = parseHexColorArgb(raw) - _seamColorArgb.value = parsed ?: DEFAULT_SEAM_COLOR_ARGB - } catch (_: Throwable) { - // ignore - } - } - - private suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult { - if ( - command.startsWith(MoltbotCanvasCommand.NamespacePrefix) || - command.startsWith(MoltbotCanvasA2UICommand.NamespacePrefix) || - command.startsWith(MoltbotCameraCommand.NamespacePrefix) || - command.startsWith(MoltbotScreenCommand.NamespacePrefix) - ) { - if (!isForeground.value) { - return GatewaySession.InvokeResult.error( - code = "NODE_BACKGROUND_UNAVAILABLE", - message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground", - ) - } - } - if (command.startsWith(MoltbotCameraCommand.NamespacePrefix) && !cameraEnabled.value) { - return GatewaySession.InvokeResult.error( - code = "CAMERA_DISABLED", - message = "CAMERA_DISABLED: enable Camera in Settings", - ) - } - if (command.startsWith(MoltbotLocationCommand.NamespacePrefix) && - locationMode.value == LocationMode.Off - ) { - return GatewaySession.InvokeResult.error( - code = "LOCATION_DISABLED", - message = "LOCATION_DISABLED: enable Location in Settings", - ) - } - - return when (command) { - MoltbotCanvasCommand.Present.rawValue -> { - val url = CanvasController.parseNavigateUrl(paramsJson) - canvas.navigate(url) - GatewaySession.InvokeResult.ok(null) - } - MoltbotCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null) - MoltbotCanvasCommand.Navigate.rawValue -> { - val url = CanvasController.parseNavigateUrl(paramsJson) - canvas.navigate(url) - GatewaySession.InvokeResult.ok(null) - } - MoltbotCanvasCommand.Eval.rawValue -> { - val js = - CanvasController.parseEvalJs(paramsJson) - ?: return GatewaySession.InvokeResult.error( - code = "INVALID_REQUEST", - message = "INVALID_REQUEST: javaScript required", - ) - val result = - try { - canvas.eval(js) - } catch (err: Throwable) { - return GatewaySession.InvokeResult.error( - code = "NODE_BACKGROUND_UNAVAILABLE", - message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable", - ) - } - GatewaySession.InvokeResult.ok("""{"result":${result.toJsonString()}}""") - } - MoltbotCanvasCommand.Snapshot.rawValue -> { - val snapshotParams = CanvasController.parseSnapshotParams(paramsJson) - val base64 = - try { - canvas.snapshotBase64( - format = snapshotParams.format, - quality = snapshotParams.quality, - maxWidth = snapshotParams.maxWidth, - ) - } catch (err: Throwable) { - return GatewaySession.InvokeResult.error( - code = "NODE_BACKGROUND_UNAVAILABLE", - message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable", - ) - } - GatewaySession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""") - } - MoltbotCanvasA2UICommand.Reset.rawValue -> { - val a2uiUrl = resolveA2uiHostUrl() - ?: return GatewaySession.InvokeResult.error( - code = "A2UI_HOST_NOT_CONFIGURED", - message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host", - ) - val ready = ensureA2uiReady(a2uiUrl) - if (!ready) { - return GatewaySession.InvokeResult.error( - code = "A2UI_HOST_UNAVAILABLE", - message = "A2UI host not reachable", - ) - } - val res = canvas.eval(a2uiResetJS) - GatewaySession.InvokeResult.ok(res) - } - MoltbotCanvasA2UICommand.Push.rawValue, MoltbotCanvasA2UICommand.PushJSONL.rawValue -> { - val messages = - try { - decodeA2uiMessages(command, paramsJson) - } catch (err: Throwable) { - return GatewaySession.InvokeResult.error(code = "INVALID_REQUEST", message = err.message ?: "invalid A2UI payload") - } - val a2uiUrl = resolveA2uiHostUrl() - ?: return GatewaySession.InvokeResult.error( - code = "A2UI_HOST_NOT_CONFIGURED", - message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host", - ) - val ready = ensureA2uiReady(a2uiUrl) - if (!ready) { - return GatewaySession.InvokeResult.error( - code = "A2UI_HOST_UNAVAILABLE", - message = "A2UI host not reachable", - ) - } - val js = a2uiApplyMessagesJS(messages) - val res = canvas.eval(js) - GatewaySession.InvokeResult.ok(res) - } - MoltbotCameraCommand.Snap.rawValue -> { - showCameraHud(message = "Taking photo…", kind = CameraHudKind.Photo) - triggerCameraFlash() - val res = - try { - camera.snap(paramsJson) - } catch (err: Throwable) { - val (code, message) = invokeErrorFromThrowable(err) - showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2200) - return GatewaySession.InvokeResult.error(code = code, message = message) - } - showCameraHud(message = "Photo captured", kind = CameraHudKind.Success, autoHideMs = 1600) - GatewaySession.InvokeResult.ok(res.payloadJson) - } - MoltbotCameraCommand.Clip.rawValue -> { - val includeAudio = paramsJson?.contains("\"includeAudio\":true") != false - if (includeAudio) externalAudioCaptureActive.value = true - try { - showCameraHud(message = "Recording…", kind = CameraHudKind.Recording) - val res = - try { - camera.clip(paramsJson) - } catch (err: Throwable) { - val (code, message) = invokeErrorFromThrowable(err) - showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2400) - return GatewaySession.InvokeResult.error(code = code, message = message) - } - showCameraHud(message = "Clip captured", kind = CameraHudKind.Success, autoHideMs = 1800) - GatewaySession.InvokeResult.ok(res.payloadJson) - } finally { - if (includeAudio) externalAudioCaptureActive.value = false - } - } - MoltbotLocationCommand.Get.rawValue -> { - val mode = locationMode.value - if (!isForeground.value && mode != LocationMode.Always) { - return GatewaySession.InvokeResult.error( - code = "LOCATION_BACKGROUND_UNAVAILABLE", - message = "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always", - ) - } - if (!hasFineLocationPermission() && !hasCoarseLocationPermission()) { - return GatewaySession.InvokeResult.error( - code = "LOCATION_PERMISSION_REQUIRED", - message = "LOCATION_PERMISSION_REQUIRED: grant Location permission", - ) - } - if (!isForeground.value && mode == LocationMode.Always && !hasBackgroundLocationPermission()) { - return GatewaySession.InvokeResult.error( - code = "LOCATION_PERMISSION_REQUIRED", - message = "LOCATION_PERMISSION_REQUIRED: enable Always in system Settings", - ) - } - val (maxAgeMs, timeoutMs, desiredAccuracy) = parseLocationParams(paramsJson) - val preciseEnabled = locationPreciseEnabled.value - val accuracy = - when (desiredAccuracy) { - "precise" -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced" - "coarse" -> "coarse" - else -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced" - } - val providers = - when (accuracy) { - "precise" -> listOf(LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER) - "coarse" -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER) - else -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER) - } - try { - val payload = - location.getLocation( - desiredProviders = providers, - maxAgeMs = maxAgeMs, - timeoutMs = timeoutMs, - isPrecise = accuracy == "precise", - ) - GatewaySession.InvokeResult.ok(payload.payloadJson) - } catch (err: TimeoutCancellationException) { - GatewaySession.InvokeResult.error( - code = "LOCATION_TIMEOUT", - message = "LOCATION_TIMEOUT: no fix in time", - ) - } catch (err: Throwable) { - val message = err.message ?: "LOCATION_UNAVAILABLE: no fix" - GatewaySession.InvokeResult.error(code = "LOCATION_UNAVAILABLE", message = message) - } - } - MoltbotScreenCommand.Record.rawValue -> { - // Status pill mirrors screen recording state so it stays visible without overlay stacking. - _screenRecordActive.value = true - try { - val res = - try { - screenRecorder.record(paramsJson) - } catch (err: Throwable) { - val (code, message) = invokeErrorFromThrowable(err) - return GatewaySession.InvokeResult.error(code = code, message = message) - } - GatewaySession.InvokeResult.ok(res.payloadJson) - } finally { - _screenRecordActive.value = false - } - } - MoltbotSmsCommand.Send.rawValue -> { - val res = sms.send(paramsJson) - if (res.ok) { - GatewaySession.InvokeResult.ok(res.payloadJson) - } else { - val error = res.error ?: "SMS_SEND_FAILED" - val idx = error.indexOf(':') - val code = if (idx > 0) error.substring(0, idx).trim() else "SMS_SEND_FAILED" - GatewaySession.InvokeResult.error(code = code, message = error) - } - } - else -> - GatewaySession.InvokeResult.error( - code = "INVALID_REQUEST", - message = "INVALID_REQUEST: unknown command", - ) - } - } - - private fun triggerCameraFlash() { - // Token is used as a pulse trigger; value doesn't matter as long as it changes. - _cameraFlashToken.value = SystemClock.elapsedRealtimeNanos() - } - - private fun showCameraHud(message: String, kind: CameraHudKind, autoHideMs: Long? = null) { - val token = cameraHudSeq.incrementAndGet() - _cameraHud.value = CameraHudState(token = token, kind = kind, message = message) - - if (autoHideMs != null && autoHideMs > 0) { - scope.launch { - delay(autoHideMs) - if (_cameraHud.value?.token == token) _cameraHud.value = null - } - } - } - - private fun invokeErrorFromThrowable(err: Throwable): Pair { - val raw = (err.message ?: "").trim() - if (raw.isEmpty()) return "UNAVAILABLE" to "UNAVAILABLE: camera error" - - val idx = raw.indexOf(':') - if (idx <= 0) return "UNAVAILABLE" to raw - val code = raw.substring(0, idx).trim().ifEmpty { "UNAVAILABLE" } - val message = raw.substring(idx + 1).trim().ifEmpty { raw } - // Preserve full string for callers/logging, but keep the returned message human-friendly. - return code to "$code: $message" - } - - private fun parseLocationParams(paramsJson: String?): Triple { - if (paramsJson.isNullOrBlank()) { - return Triple(null, 10_000L, null) - } - val root = - try { - json.parseToJsonElement(paramsJson).asObjectOrNull() - } catch (_: Throwable) { - null - } - val maxAgeMs = (root?.get("maxAgeMs") as? JsonPrimitive)?.content?.toLongOrNull() - val timeoutMs = - (root?.get("timeoutMs") as? JsonPrimitive)?.content?.toLongOrNull()?.coerceIn(1_000L, 60_000L) - ?: 10_000L - val desiredAccuracy = - (root?.get("desiredAccuracy") as? JsonPrimitive)?.content?.trim()?.lowercase() - return Triple(maxAgeMs, timeoutMs, desiredAccuracy) - } - - private fun resolveA2uiHostUrl(): String? { - val nodeRaw = nodeSession.currentCanvasHostUrl()?.trim().orEmpty() - val operatorRaw = operatorSession.currentCanvasHostUrl()?.trim().orEmpty() - val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw - if (raw.isBlank()) return null - val base = raw.trimEnd('/') - return "${base}/__moltbot__/a2ui/?platform=android" - } - - private suspend fun ensureA2uiReady(a2uiUrl: String): Boolean { - try { - val already = canvas.eval(a2uiReadyCheckJS) - if (already == "true") return true - } catch (_: Throwable) { - // ignore - } - - canvas.navigate(a2uiUrl) - repeat(50) { - try { - val ready = canvas.eval(a2uiReadyCheckJS) - if (ready == "true") return true - } catch (_: Throwable) { - // ignore - } - delay(120) - } - return false - } - - private fun decodeA2uiMessages(command: String, paramsJson: String?): String { - val raw = paramsJson?.trim().orEmpty() - if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required") - - val obj = - json.parseToJsonElement(raw) as? JsonObject - ?: throw IllegalArgumentException("INVALID_REQUEST: expected object params") - - val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty() - val hasMessagesArray = obj["messages"] is JsonArray - - if (command == MoltbotCanvasA2UICommand.PushJSONL.rawValue || (!hasMessagesArray && jsonlField.isNotBlank())) { - val jsonl = jsonlField - if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required") - val messages = - jsonl - .lineSequence() - .map { it.trim() } - .filter { it.isNotBlank() } - .mapIndexed { idx, line -> - val el = json.parseToJsonElement(line) - val msg = - el as? JsonObject - ?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object") - validateA2uiV0_8(msg, idx + 1) - msg - } - .toList() - return JsonArray(messages).toString() - } - - val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required") - val out = - arr.mapIndexed { idx, el -> - val msg = - el as? JsonObject - ?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object") - validateA2uiV0_8(msg, idx + 1) - msg - } - return JsonArray(out).toString() - } - - private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) { - if (msg.containsKey("createSurface")) { - throw IllegalArgumentException( - "A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.", - ) - } - val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface") - val matched = msg.keys.filter { allowed.contains(it) } - if (matched.size != 1) { - val found = msg.keys.sorted().joinToString(", ") - throw IllegalArgumentException( - "A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found", - ) - } - } -} - -private data class Quad(val first: A, val second: B, val third: C, val fourth: D) - -private const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A - -private const val a2uiReadyCheckJS: String = - """ - (() => { - try { - return !!globalThis.clawdbotA2UI && typeof globalThis.clawdbotA2UI.applyMessages === 'function'; - } catch (_) { - return false; - } - })() - """ - -private const val a2uiResetJS: String = - """ - (() => { - try { - if (!globalThis.clawdbotA2UI) return { ok: false, error: "missing moltbotA2UI" }; - return globalThis.clawdbotA2UI.reset(); - } catch (e) { - return { ok: false, error: String(e?.message ?? e) }; - } - })() - """ - -private fun a2uiApplyMessagesJS(messagesJson: String): String { - return """ - (() => { - try { - if (!globalThis.clawdbotA2UI) return { ok: false, error: "missing moltbotA2UI" }; - const messages = $messagesJson; - return globalThis.clawdbotA2UI.applyMessages(messages); - } catch (e) { - return { ok: false, error: String(e?.message ?? e) }; - } - })() - """.trimIndent() -} - -private fun String.toJsonString(): String { - val escaped = - this.replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\n", "\\n") - .replace("\r", "\\r") - return "\"$escaped\"" -} - -private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject - -private fun JsonElement?.asStringOrNull(): String? = - when (this) { - is JsonNull -> null - is JsonPrimitive -> content - else -> null - } - -private fun parseHexColorArgb(raw: String?): Long? { - val trimmed = raw?.trim().orEmpty() - if (trimmed.isEmpty()) return null - val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed - if (hex.length != 6) return null - val rgb = hex.toLongOrNull(16) ?: return null - return 0xFF000000L or rgb -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/PermissionRequester.kt b/apps/android/app/src/main/java/com/clawdbot/android/PermissionRequester.kt deleted file mode 100644 index 5e95d7b27..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/PermissionRequester.kt +++ /dev/null @@ -1,133 +0,0 @@ -package com.clawdbot.android - -import android.content.pm.PackageManager -import android.content.Intent -import android.Manifest -import android.net.Uri -import android.provider.Settings -import androidx.appcompat.app.AlertDialog -import androidx.activity.ComponentActivity -import androidx.activity.result.ActivityResultLauncher -import androidx.activity.result.contract.ActivityResultContracts -import androidx.core.content.ContextCompat -import androidx.core.app.ActivityCompat -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import kotlinx.coroutines.withContext -import kotlinx.coroutines.suspendCancellableCoroutine -import kotlin.coroutines.resume - -class PermissionRequester(private val activity: ComponentActivity) { - private val mutex = Mutex() - private var pending: CompletableDeferred>? = null - - private val launcher: ActivityResultLauncher> = - activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result -> - val p = pending - pending = null - p?.complete(result) - } - - suspend fun requestIfMissing( - permissions: List, - timeoutMs: Long = 20_000, - ): Map = - mutex.withLock { - val missing = - permissions.filter { perm -> - ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED - } - if (missing.isEmpty()) { - return permissions.associateWith { true } - } - - val needsRationale = - missing.any { ActivityCompat.shouldShowRequestPermissionRationale(activity, it) } - if (needsRationale) { - val proceed = showRationaleDialog(missing) - if (!proceed) { - return permissions.associateWith { perm -> - ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED - } - } - } - - val deferred = CompletableDeferred>() - pending = deferred - withContext(Dispatchers.Main) { - launcher.launch(missing.toTypedArray()) - } - - val result = - withContext(Dispatchers.Default) { - kotlinx.coroutines.withTimeout(timeoutMs) { deferred.await() } - } - - // Merge: if something was already granted, treat it as granted even if launcher omitted it. - val merged = - permissions.associateWith { perm -> - val nowGranted = - ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED - result[perm] == true || nowGranted - } - - val denied = - merged.filterValues { !it }.keys.filter { - !ActivityCompat.shouldShowRequestPermissionRationale(activity, it) - } - if (denied.isNotEmpty()) { - showSettingsDialog(denied) - } - - return merged - } - - private suspend fun showRationaleDialog(permissions: List): Boolean = - withContext(Dispatchers.Main) { - suspendCancellableCoroutine { cont -> - AlertDialog.Builder(activity) - .setTitle("Permission required") - .setMessage(buildRationaleMessage(permissions)) - .setPositiveButton("Continue") { _, _ -> cont.resume(true) } - .setNegativeButton("Not now") { _, _ -> cont.resume(false) } - .setOnCancelListener { cont.resume(false) } - .show() - } - } - - private fun showSettingsDialog(permissions: List) { - AlertDialog.Builder(activity) - .setTitle("Enable permission in Settings") - .setMessage(buildSettingsMessage(permissions)) - .setPositiveButton("Open Settings") { _, _ -> - val intent = - Intent( - Settings.ACTION_APPLICATION_DETAILS_SETTINGS, - Uri.fromParts("package", activity.packageName, null), - ) - activity.startActivity(intent) - } - .setNegativeButton("Cancel", null) - .show() - } - - private fun buildRationaleMessage(permissions: List): String { - val labels = permissions.map { permissionLabel(it) } - return "Moltbot needs ${labels.joinToString(", ")} permissions to continue." - } - - private fun buildSettingsMessage(permissions: List): String { - val labels = permissions.map { permissionLabel(it) } - return "Please enable ${labels.joinToString(", ")} in Android Settings to continue." - } - - private fun permissionLabel(permission: String): String = - when (permission) { - Manifest.permission.CAMERA -> "Camera" - Manifest.permission.RECORD_AUDIO -> "Microphone" - Manifest.permission.SEND_SMS -> "SMS" - else -> permission - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ScreenCaptureRequester.kt b/apps/android/app/src/main/java/com/clawdbot/android/ScreenCaptureRequester.kt deleted file mode 100644 index f7cf6708c..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ScreenCaptureRequester.kt +++ /dev/null @@ -1,65 +0,0 @@ -package com.clawdbot.android - -import android.app.Activity -import android.content.Context -import android.content.Intent -import android.media.projection.MediaProjectionManager -import androidx.activity.ComponentActivity -import androidx.activity.result.ActivityResultLauncher -import androidx.activity.result.contract.ActivityResultContracts -import androidx.appcompat.app.AlertDialog -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import kotlinx.coroutines.withContext -import kotlinx.coroutines.withTimeout -import kotlinx.coroutines.suspendCancellableCoroutine -import kotlin.coroutines.resume - -class ScreenCaptureRequester(private val activity: ComponentActivity) { - data class CaptureResult(val resultCode: Int, val data: Intent) - - private val mutex = Mutex() - private var pending: CompletableDeferred? = null - - private val launcher: ActivityResultLauncher = - activity.registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result -> - val p = pending - pending = null - val data = result.data - if (result.resultCode == Activity.RESULT_OK && data != null) { - p?.complete(CaptureResult(result.resultCode, data)) - } else { - p?.complete(null) - } - } - - suspend fun requestCapture(timeoutMs: Long = 20_000): CaptureResult? = - mutex.withLock { - val proceed = showRationaleDialog() - if (!proceed) return null - - val mgr = activity.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager - val intent = mgr.createScreenCaptureIntent() - - val deferred = CompletableDeferred() - pending = deferred - withContext(Dispatchers.Main) { launcher.launch(intent) } - - withContext(Dispatchers.Default) { withTimeout(timeoutMs) { deferred.await() } } - } - - private suspend fun showRationaleDialog(): Boolean = - withContext(Dispatchers.Main) { - suspendCancellableCoroutine { cont -> - AlertDialog.Builder(activity) - .setTitle("Screen recording required") - .setMessage("Moltbot needs to record the screen for this command.") - .setPositiveButton("Continue") { _, _ -> cont.resume(true) } - .setNegativeButton("Not now") { _, _ -> cont.resume(false) } - .setOnCancelListener { cont.resume(false) } - .show() - } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/SecurePrefs.kt b/apps/android/app/src/main/java/com/clawdbot/android/SecurePrefs.kt deleted file mode 100644 index 1c464f961..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/SecurePrefs.kt +++ /dev/null @@ -1,308 +0,0 @@ -@file:Suppress("DEPRECATION") - -package com.clawdbot.android - -import android.content.Context -import androidx.core.content.edit -import androidx.security.crypto.EncryptedSharedPreferences -import androidx.security.crypto.MasterKey -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonNull -import kotlinx.serialization.json.JsonPrimitive -import java.util.UUID - -class SecurePrefs(context: Context) { - companion object { - val defaultWakeWords: List = listOf("clawd", "claude") - private const val displayNameKey = "node.displayName" - private const val voiceWakeModeKey = "voiceWake.mode" - } - - private val json = Json { ignoreUnknownKeys = true } - - private val masterKey = - MasterKey.Builder(context) - .setKeyScheme(MasterKey.KeyScheme.AES256_GCM) - .build() - - private val prefs = - EncryptedSharedPreferences.create( - context, - "moltbot.node.secure", - masterKey, - EncryptedSharedPreferences.PrefKeyEncryptionScheme.AES256_SIV, - EncryptedSharedPreferences.PrefValueEncryptionScheme.AES256_GCM, - ) - - private val _instanceId = MutableStateFlow(loadOrCreateInstanceId()) - val instanceId: StateFlow = _instanceId - - private val _displayName = - MutableStateFlow(loadOrMigrateDisplayName(context = context)) - val displayName: StateFlow = _displayName - - private val _cameraEnabled = MutableStateFlow(prefs.getBoolean("camera.enabled", true)) - val cameraEnabled: StateFlow = _cameraEnabled - - private val _locationMode = - MutableStateFlow(LocationMode.fromRawValue(prefs.getString("location.enabledMode", "off"))) - val locationMode: StateFlow = _locationMode - - private val _locationPreciseEnabled = - MutableStateFlow(prefs.getBoolean("location.preciseEnabled", true)) - val locationPreciseEnabled: StateFlow = _locationPreciseEnabled - - private val _preventSleep = MutableStateFlow(prefs.getBoolean("screen.preventSleep", true)) - val preventSleep: StateFlow = _preventSleep - - private val _manualEnabled = - MutableStateFlow(readBoolWithMigration("gateway.manual.enabled", "bridge.manual.enabled", false)) - val manualEnabled: StateFlow = _manualEnabled - - private val _manualHost = - MutableStateFlow(readStringWithMigration("gateway.manual.host", "bridge.manual.host", "")) - val manualHost: StateFlow = _manualHost - - private val _manualPort = - MutableStateFlow(readIntWithMigration("gateway.manual.port", "bridge.manual.port", 18789)) - val manualPort: StateFlow = _manualPort - - private val _manualTls = - MutableStateFlow(readBoolWithMigration("gateway.manual.tls", null, true)) - val manualTls: StateFlow = _manualTls - - private val _lastDiscoveredStableId = - MutableStateFlow( - readStringWithMigration( - "gateway.lastDiscoveredStableID", - "bridge.lastDiscoveredStableId", - "", - ), - ) - val lastDiscoveredStableId: StateFlow = _lastDiscoveredStableId - - private val _canvasDebugStatusEnabled = - MutableStateFlow(prefs.getBoolean("canvas.debugStatusEnabled", false)) - val canvasDebugStatusEnabled: StateFlow = _canvasDebugStatusEnabled - - private val _wakeWords = MutableStateFlow(loadWakeWords()) - val wakeWords: StateFlow> = _wakeWords - - private val _voiceWakeMode = MutableStateFlow(loadVoiceWakeMode()) - val voiceWakeMode: StateFlow = _voiceWakeMode - - private val _talkEnabled = MutableStateFlow(prefs.getBoolean("talk.enabled", false)) - val talkEnabled: StateFlow = _talkEnabled - - fun setLastDiscoveredStableId(value: String) { - val trimmed = value.trim() - prefs.edit { putString("gateway.lastDiscoveredStableID", trimmed) } - _lastDiscoveredStableId.value = trimmed - } - - fun setDisplayName(value: String) { - val trimmed = value.trim() - prefs.edit { putString(displayNameKey, trimmed) } - _displayName.value = trimmed - } - - fun setCameraEnabled(value: Boolean) { - prefs.edit { putBoolean("camera.enabled", value) } - _cameraEnabled.value = value - } - - fun setLocationMode(mode: LocationMode) { - prefs.edit { putString("location.enabledMode", mode.rawValue) } - _locationMode.value = mode - } - - fun setLocationPreciseEnabled(value: Boolean) { - prefs.edit { putBoolean("location.preciseEnabled", value) } - _locationPreciseEnabled.value = value - } - - fun setPreventSleep(value: Boolean) { - prefs.edit { putBoolean("screen.preventSleep", value) } - _preventSleep.value = value - } - - fun setManualEnabled(value: Boolean) { - prefs.edit { putBoolean("gateway.manual.enabled", value) } - _manualEnabled.value = value - } - - fun setManualHost(value: String) { - val trimmed = value.trim() - prefs.edit { putString("gateway.manual.host", trimmed) } - _manualHost.value = trimmed - } - - fun setManualPort(value: Int) { - prefs.edit { putInt("gateway.manual.port", value) } - _manualPort.value = value - } - - fun setManualTls(value: Boolean) { - prefs.edit { putBoolean("gateway.manual.tls", value) } - _manualTls.value = value - } - - fun setCanvasDebugStatusEnabled(value: Boolean) { - prefs.edit { putBoolean("canvas.debugStatusEnabled", value) } - _canvasDebugStatusEnabled.value = value - } - - fun loadGatewayToken(): String? { - val key = "gateway.token.${_instanceId.value}" - val stored = prefs.getString(key, null)?.trim() - if (!stored.isNullOrEmpty()) return stored - val legacy = prefs.getString("bridge.token.${_instanceId.value}", null)?.trim() - return legacy?.takeIf { it.isNotEmpty() } - } - - fun saveGatewayToken(token: String) { - val key = "gateway.token.${_instanceId.value}" - prefs.edit { putString(key, token.trim()) } - } - - fun loadGatewayPassword(): String? { - val key = "gateway.password.${_instanceId.value}" - val stored = prefs.getString(key, null)?.trim() - return stored?.takeIf { it.isNotEmpty() } - } - - fun saveGatewayPassword(password: String) { - val key = "gateway.password.${_instanceId.value}" - prefs.edit { putString(key, password.trim()) } - } - - fun loadGatewayTlsFingerprint(stableId: String): String? { - val key = "gateway.tls.$stableId" - return prefs.getString(key, null)?.trim()?.takeIf { it.isNotEmpty() } - } - - fun saveGatewayTlsFingerprint(stableId: String, fingerprint: String) { - val key = "gateway.tls.$stableId" - prefs.edit { putString(key, fingerprint.trim()) } - } - - fun getString(key: String): String? { - return prefs.getString(key, null) - } - - fun putString(key: String, value: String) { - prefs.edit { putString(key, value) } - } - - fun remove(key: String) { - prefs.edit { remove(key) } - } - - private fun loadOrCreateInstanceId(): String { - val existing = prefs.getString("node.instanceId", null)?.trim() - if (!existing.isNullOrBlank()) return existing - val fresh = UUID.randomUUID().toString() - prefs.edit { putString("node.instanceId", fresh) } - return fresh - } - - private fun loadOrMigrateDisplayName(context: Context): String { - val existing = prefs.getString(displayNameKey, null)?.trim().orEmpty() - if (existing.isNotEmpty() && existing != "Android Node") return existing - - val candidate = DeviceNames.bestDefaultNodeName(context).trim() - val resolved = candidate.ifEmpty { "Android Node" } - - prefs.edit { putString(displayNameKey, resolved) } - return resolved - } - - fun setWakeWords(words: List) { - val sanitized = WakeWords.sanitize(words, defaultWakeWords) - val encoded = - JsonArray(sanitized.map { JsonPrimitive(it) }).toString() - prefs.edit { putString("voiceWake.triggerWords", encoded) } - _wakeWords.value = sanitized - } - - fun setVoiceWakeMode(mode: VoiceWakeMode) { - prefs.edit { putString(voiceWakeModeKey, mode.rawValue) } - _voiceWakeMode.value = mode - } - - fun setTalkEnabled(value: Boolean) { - prefs.edit { putBoolean("talk.enabled", value) } - _talkEnabled.value = value - } - - private fun loadVoiceWakeMode(): VoiceWakeMode { - val raw = prefs.getString(voiceWakeModeKey, null) - val resolved = VoiceWakeMode.fromRawValue(raw) - - // Default ON (foreground) when unset. - if (raw.isNullOrBlank()) { - prefs.edit { putString(voiceWakeModeKey, resolved.rawValue) } - } - - return resolved - } - - private fun loadWakeWords(): List { - val raw = prefs.getString("voiceWake.triggerWords", null)?.trim() - if (raw.isNullOrEmpty()) return defaultWakeWords - return try { - val element = json.parseToJsonElement(raw) - val array = element as? JsonArray ?: return defaultWakeWords - val decoded = - array.mapNotNull { item -> - when (item) { - is JsonNull -> null - is JsonPrimitive -> item.content.trim().takeIf { it.isNotEmpty() } - else -> null - } - } - WakeWords.sanitize(decoded, defaultWakeWords) - } catch (_: Throwable) { - defaultWakeWords - } - } - - private fun readBoolWithMigration(newKey: String, oldKey: String?, defaultValue: Boolean): Boolean { - if (prefs.contains(newKey)) { - return prefs.getBoolean(newKey, defaultValue) - } - if (oldKey != null && prefs.contains(oldKey)) { - val value = prefs.getBoolean(oldKey, defaultValue) - prefs.edit { putBoolean(newKey, value) } - return value - } - return defaultValue - } - - private fun readStringWithMigration(newKey: String, oldKey: String?, defaultValue: String): String { - if (prefs.contains(newKey)) { - return prefs.getString(newKey, defaultValue) ?: defaultValue - } - if (oldKey != null && prefs.contains(oldKey)) { - val value = prefs.getString(oldKey, defaultValue) ?: defaultValue - prefs.edit { putString(newKey, value) } - return value - } - return defaultValue - } - - private fun readIntWithMigration(newKey: String, oldKey: String?, defaultValue: Int): Int { - if (prefs.contains(newKey)) { - return prefs.getInt(newKey, defaultValue) - } - if (oldKey != null && prefs.contains(oldKey)) { - val value = prefs.getInt(oldKey, defaultValue) - prefs.edit { putInt(newKey, value) } - return value - } - return defaultValue - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/SessionKey.kt b/apps/android/app/src/main/java/com/clawdbot/android/SessionKey.kt deleted file mode 100644 index e1aae9ec0..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/SessionKey.kt +++ /dev/null @@ -1,13 +0,0 @@ -package com.clawdbot.android - -internal fun normalizeMainKey(raw: String?): String { - val trimmed = raw?.trim() - return if (!trimmed.isNullOrEmpty()) trimmed else "main" -} - -internal fun isCanonicalMainSessionKey(raw: String?): Boolean { - val trimmed = raw?.trim().orEmpty() - if (trimmed.isEmpty()) return false - if (trimmed == "global") return true - return trimmed.startsWith("agent:") -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/VoiceWakeMode.kt b/apps/android/app/src/main/java/com/clawdbot/android/VoiceWakeMode.kt deleted file mode 100644 index 6c3e2c201..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/VoiceWakeMode.kt +++ /dev/null @@ -1,14 +0,0 @@ -package com.clawdbot.android - -enum class VoiceWakeMode(val rawValue: String) { - Off("off"), - Foreground("foreground"), - Always("always"), - ; - - companion object { - fun fromRawValue(raw: String?): VoiceWakeMode { - return entries.firstOrNull { it.rawValue == raw?.trim()?.lowercase() } ?: Foreground - } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/WakeWords.kt b/apps/android/app/src/main/java/com/clawdbot/android/WakeWords.kt deleted file mode 100644 index d54ed1e08..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/WakeWords.kt +++ /dev/null @@ -1,21 +0,0 @@ -package com.clawdbot.android - -object WakeWords { - const val maxWords: Int = 32 - const val maxWordLength: Int = 64 - - fun parseCommaSeparated(input: String): List { - return input.split(",").map { it.trim() }.filter { it.isNotEmpty() } - } - - fun parseIfChanged(input: String, current: List): List? { - val parsed = parseCommaSeparated(input) - return if (parsed == current) null else parsed - } - - fun sanitize(words: List, defaults: List): List { - val cleaned = - words.map { it.trim() }.filter { it.isNotEmpty() }.take(maxWords).map { it.take(maxWordLength) } - return cleaned.ifEmpty { defaults } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/chat/ChatController.kt b/apps/android/app/src/main/java/com/clawdbot/android/chat/ChatController.kt deleted file mode 100644 index a8e64048c..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/chat/ChatController.kt +++ /dev/null @@ -1,524 +0,0 @@ -package com.clawdbot.android.chat - -import com.clawdbot.android.gateway.GatewaySession -import java.util.UUID -import java.util.concurrent.ConcurrentHashMap -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.launch -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonNull -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive -import kotlinx.serialization.json.buildJsonObject - -class ChatController( - private val scope: CoroutineScope, - private val session: GatewaySession, - private val json: Json, - private val supportsChatSubscribe: Boolean, -) { - private val _sessionKey = MutableStateFlow("main") - val sessionKey: StateFlow = _sessionKey.asStateFlow() - - private val _sessionId = MutableStateFlow(null) - val sessionId: StateFlow = _sessionId.asStateFlow() - - private val _messages = MutableStateFlow>(emptyList()) - val messages: StateFlow> = _messages.asStateFlow() - - private val _errorText = MutableStateFlow(null) - val errorText: StateFlow = _errorText.asStateFlow() - - private val _healthOk = MutableStateFlow(false) - val healthOk: StateFlow = _healthOk.asStateFlow() - - private val _thinkingLevel = MutableStateFlow("off") - val thinkingLevel: StateFlow = _thinkingLevel.asStateFlow() - - private val _pendingRunCount = MutableStateFlow(0) - val pendingRunCount: StateFlow = _pendingRunCount.asStateFlow() - - private val _streamingAssistantText = MutableStateFlow(null) - val streamingAssistantText: StateFlow = _streamingAssistantText.asStateFlow() - - private val pendingToolCallsById = ConcurrentHashMap() - private val _pendingToolCalls = MutableStateFlow>(emptyList()) - val pendingToolCalls: StateFlow> = _pendingToolCalls.asStateFlow() - - private val _sessions = MutableStateFlow>(emptyList()) - val sessions: StateFlow> = _sessions.asStateFlow() - - private val pendingRuns = mutableSetOf() - private val pendingRunTimeoutJobs = ConcurrentHashMap() - private val pendingRunTimeoutMs = 120_000L - - private var lastHealthPollAtMs: Long? = null - - fun onDisconnected(message: String) { - _healthOk.value = false - // Not an error; keep connection status in the UI pill. - _errorText.value = null - clearPendingRuns() - pendingToolCallsById.clear() - publishPendingToolCalls() - _streamingAssistantText.value = null - _sessionId.value = null - } - - fun load(sessionKey: String) { - val key = sessionKey.trim().ifEmpty { "main" } - _sessionKey.value = key - scope.launch { bootstrap(forceHealth = true) } - } - - fun applyMainSessionKey(mainSessionKey: String) { - val trimmed = mainSessionKey.trim() - if (trimmed.isEmpty()) return - if (_sessionKey.value == trimmed) return - if (_sessionKey.value != "main") return - _sessionKey.value = trimmed - scope.launch { bootstrap(forceHealth = true) } - } - - fun refresh() { - scope.launch { bootstrap(forceHealth = true) } - } - - fun refreshSessions(limit: Int? = null) { - scope.launch { fetchSessions(limit = limit) } - } - - fun setThinkingLevel(thinkingLevel: String) { - val normalized = normalizeThinking(thinkingLevel) - if (normalized == _thinkingLevel.value) return - _thinkingLevel.value = normalized - } - - fun switchSession(sessionKey: String) { - val key = sessionKey.trim() - if (key.isEmpty()) return - if (key == _sessionKey.value) return - _sessionKey.value = key - scope.launch { bootstrap(forceHealth = true) } - } - - fun sendMessage( - message: String, - thinkingLevel: String, - attachments: List, - ) { - val trimmed = message.trim() - if (trimmed.isEmpty() && attachments.isEmpty()) return - if (!_healthOk.value) { - _errorText.value = "Gateway health not OK; cannot send" - return - } - - val runId = UUID.randomUUID().toString() - val text = if (trimmed.isEmpty() && attachments.isNotEmpty()) "See attached." else trimmed - val sessionKey = _sessionKey.value - val thinking = normalizeThinking(thinkingLevel) - - // Optimistic user message. - val userContent = - buildList { - add(ChatMessageContent(type = "text", text = text)) - for (att in attachments) { - add( - ChatMessageContent( - type = att.type, - mimeType = att.mimeType, - fileName = att.fileName, - base64 = att.base64, - ), - ) - } - } - _messages.value = - _messages.value + - ChatMessage( - id = UUID.randomUUID().toString(), - role = "user", - content = userContent, - timestampMs = System.currentTimeMillis(), - ) - - armPendingRunTimeout(runId) - synchronized(pendingRuns) { - pendingRuns.add(runId) - _pendingRunCount.value = pendingRuns.size - } - - _errorText.value = null - _streamingAssistantText.value = null - pendingToolCallsById.clear() - publishPendingToolCalls() - - scope.launch { - try { - val params = - buildJsonObject { - put("sessionKey", JsonPrimitive(sessionKey)) - put("message", JsonPrimitive(text)) - put("thinking", JsonPrimitive(thinking)) - put("timeoutMs", JsonPrimitive(30_000)) - put("idempotencyKey", JsonPrimitive(runId)) - if (attachments.isNotEmpty()) { - put( - "attachments", - JsonArray( - attachments.map { att -> - buildJsonObject { - put("type", JsonPrimitive(att.type)) - put("mimeType", JsonPrimitive(att.mimeType)) - put("fileName", JsonPrimitive(att.fileName)) - put("content", JsonPrimitive(att.base64)) - } - }, - ), - ) - } - } - val res = session.request("chat.send", params.toString()) - val actualRunId = parseRunId(res) ?: runId - if (actualRunId != runId) { - clearPendingRun(runId) - armPendingRunTimeout(actualRunId) - synchronized(pendingRuns) { - pendingRuns.add(actualRunId) - _pendingRunCount.value = pendingRuns.size - } - } - } catch (err: Throwable) { - clearPendingRun(runId) - _errorText.value = err.message - } - } - } - - fun abort() { - val runIds = - synchronized(pendingRuns) { - pendingRuns.toList() - } - if (runIds.isEmpty()) return - scope.launch { - for (runId in runIds) { - try { - val params = - buildJsonObject { - put("sessionKey", JsonPrimitive(_sessionKey.value)) - put("runId", JsonPrimitive(runId)) - } - session.request("chat.abort", params.toString()) - } catch (_: Throwable) { - // best-effort - } - } - } - } - - fun handleGatewayEvent(event: String, payloadJson: String?) { - when (event) { - "tick" -> { - scope.launch { pollHealthIfNeeded(force = false) } - } - "health" -> { - // If we receive a health snapshot, the gateway is reachable. - _healthOk.value = true - } - "seqGap" -> { - _errorText.value = "Event stream interrupted; try refreshing." - clearPendingRuns() - } - "chat" -> { - if (payloadJson.isNullOrBlank()) return - handleChatEvent(payloadJson) - } - "agent" -> { - if (payloadJson.isNullOrBlank()) return - handleAgentEvent(payloadJson) - } - } - } - - private suspend fun bootstrap(forceHealth: Boolean) { - _errorText.value = null - _healthOk.value = false - clearPendingRuns() - pendingToolCallsById.clear() - publishPendingToolCalls() - _streamingAssistantText.value = null - _sessionId.value = null - - val key = _sessionKey.value - try { - if (supportsChatSubscribe) { - try { - session.sendNodeEvent("chat.subscribe", """{"sessionKey":"$key"}""") - } catch (_: Throwable) { - // best-effort - } - } - - val historyJson = session.request("chat.history", """{"sessionKey":"$key"}""") - val history = parseHistory(historyJson, sessionKey = key) - _messages.value = history.messages - _sessionId.value = history.sessionId - history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it } - - pollHealthIfNeeded(force = forceHealth) - fetchSessions(limit = 50) - } catch (err: Throwable) { - _errorText.value = err.message - } - } - - private suspend fun fetchSessions(limit: Int?) { - try { - val params = - buildJsonObject { - put("includeGlobal", JsonPrimitive(true)) - put("includeUnknown", JsonPrimitive(false)) - if (limit != null && limit > 0) put("limit", JsonPrimitive(limit)) - } - val res = session.request("sessions.list", params.toString()) - _sessions.value = parseSessions(res) - } catch (_: Throwable) { - // best-effort - } - } - - private suspend fun pollHealthIfNeeded(force: Boolean) { - val now = System.currentTimeMillis() - val last = lastHealthPollAtMs - if (!force && last != null && now - last < 10_000) return - lastHealthPollAtMs = now - try { - session.request("health", null) - _healthOk.value = true - } catch (_: Throwable) { - _healthOk.value = false - } - } - - private fun handleChatEvent(payloadJson: String) { - val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return - val sessionKey = payload["sessionKey"].asStringOrNull()?.trim() - if (!sessionKey.isNullOrEmpty() && sessionKey != _sessionKey.value) return - - val runId = payload["runId"].asStringOrNull() - if (runId != null) { - val isPending = - synchronized(pendingRuns) { - pendingRuns.contains(runId) - } - if (!isPending) return - } - - val state = payload["state"].asStringOrNull() - when (state) { - "final", "aborted", "error" -> { - if (state == "error") { - _errorText.value = payload["errorMessage"].asStringOrNull() ?: "Chat failed" - } - if (runId != null) clearPendingRun(runId) else clearPendingRuns() - pendingToolCallsById.clear() - publishPendingToolCalls() - _streamingAssistantText.value = null - scope.launch { - try { - val historyJson = - session.request("chat.history", """{"sessionKey":"${_sessionKey.value}"}""") - val history = parseHistory(historyJson, sessionKey = _sessionKey.value) - _messages.value = history.messages - _sessionId.value = history.sessionId - history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it } - } catch (_: Throwable) { - // best-effort - } - } - } - } - } - - private fun handleAgentEvent(payloadJson: String) { - val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return - val runId = payload["runId"].asStringOrNull() - val sessionId = _sessionId.value - if (sessionId != null && runId != sessionId) return - - val stream = payload["stream"].asStringOrNull() - val data = payload["data"].asObjectOrNull() - - when (stream) { - "assistant" -> { - val text = data?.get("text")?.asStringOrNull() - if (!text.isNullOrEmpty()) { - _streamingAssistantText.value = text - } - } - "tool" -> { - val phase = data?.get("phase")?.asStringOrNull() - val name = data?.get("name")?.asStringOrNull() - val toolCallId = data?.get("toolCallId")?.asStringOrNull() - if (phase.isNullOrEmpty() || name.isNullOrEmpty() || toolCallId.isNullOrEmpty()) return - - val ts = payload["ts"].asLongOrNull() ?: System.currentTimeMillis() - if (phase == "start") { - val args = data?.get("args").asObjectOrNull() - pendingToolCallsById[toolCallId] = - ChatPendingToolCall( - toolCallId = toolCallId, - name = name, - args = args, - startedAtMs = ts, - isError = null, - ) - publishPendingToolCalls() - } else if (phase == "result") { - pendingToolCallsById.remove(toolCallId) - publishPendingToolCalls() - } - } - "error" -> { - _errorText.value = "Event stream interrupted; try refreshing." - clearPendingRuns() - pendingToolCallsById.clear() - publishPendingToolCalls() - _streamingAssistantText.value = null - } - } - } - - private fun publishPendingToolCalls() { - _pendingToolCalls.value = - pendingToolCallsById.values.sortedBy { it.startedAtMs } - } - - private fun armPendingRunTimeout(runId: String) { - pendingRunTimeoutJobs[runId]?.cancel() - pendingRunTimeoutJobs[runId] = - scope.launch { - delay(pendingRunTimeoutMs) - val stillPending = - synchronized(pendingRuns) { - pendingRuns.contains(runId) - } - if (!stillPending) return@launch - clearPendingRun(runId) - _errorText.value = "Timed out waiting for a reply; try again or refresh." - } - } - - private fun clearPendingRun(runId: String) { - pendingRunTimeoutJobs.remove(runId)?.cancel() - synchronized(pendingRuns) { - pendingRuns.remove(runId) - _pendingRunCount.value = pendingRuns.size - } - } - - private fun clearPendingRuns() { - for ((_, job) in pendingRunTimeoutJobs) { - job.cancel() - } - pendingRunTimeoutJobs.clear() - synchronized(pendingRuns) { - pendingRuns.clear() - _pendingRunCount.value = 0 - } - } - - private fun parseHistory(historyJson: String, sessionKey: String): ChatHistory { - val root = json.parseToJsonElement(historyJson).asObjectOrNull() ?: return ChatHistory(sessionKey, null, null, emptyList()) - val sid = root["sessionId"].asStringOrNull() - val thinkingLevel = root["thinkingLevel"].asStringOrNull() - val array = root["messages"].asArrayOrNull() ?: JsonArray(emptyList()) - - val messages = - array.mapNotNull { item -> - val obj = item.asObjectOrNull() ?: return@mapNotNull null - val role = obj["role"].asStringOrNull() ?: return@mapNotNull null - val content = obj["content"].asArrayOrNull()?.mapNotNull(::parseMessageContent) ?: emptyList() - val ts = obj["timestamp"].asLongOrNull() - ChatMessage( - id = UUID.randomUUID().toString(), - role = role, - content = content, - timestampMs = ts, - ) - } - - return ChatHistory(sessionKey = sessionKey, sessionId = sid, thinkingLevel = thinkingLevel, messages = messages) - } - - private fun parseMessageContent(el: JsonElement): ChatMessageContent? { - val obj = el.asObjectOrNull() ?: return null - val type = obj["type"].asStringOrNull() ?: "text" - return if (type == "text") { - ChatMessageContent(type = "text", text = obj["text"].asStringOrNull()) - } else { - ChatMessageContent( - type = type, - mimeType = obj["mimeType"].asStringOrNull(), - fileName = obj["fileName"].asStringOrNull(), - base64 = obj["content"].asStringOrNull(), - ) - } - } - - private fun parseSessions(jsonString: String): List { - val root = json.parseToJsonElement(jsonString).asObjectOrNull() ?: return emptyList() - val sessions = root["sessions"].asArrayOrNull() ?: return emptyList() - return sessions.mapNotNull { item -> - val obj = item.asObjectOrNull() ?: return@mapNotNull null - val key = obj["key"].asStringOrNull()?.trim().orEmpty() - if (key.isEmpty()) return@mapNotNull null - val updatedAt = obj["updatedAt"].asLongOrNull() - val displayName = obj["displayName"].asStringOrNull()?.trim() - ChatSessionEntry(key = key, updatedAtMs = updatedAt, displayName = displayName) - } - } - - private fun parseRunId(resJson: String): String? { - return try { - json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull() - } catch (_: Throwable) { - null - } - } - - private fun normalizeThinking(raw: String): String { - return when (raw.trim().lowercase()) { - "low" -> "low" - "medium" -> "medium" - "high" -> "high" - else -> "off" - } - } -} - -private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject - -private fun JsonElement?.asArrayOrNull(): JsonArray? = this as? JsonArray - -private fun JsonElement?.asStringOrNull(): String? = - when (this) { - is JsonNull -> null - is JsonPrimitive -> content - else -> null - } - -private fun JsonElement?.asLongOrNull(): Long? = - when (this) { - is JsonPrimitive -> content.toLongOrNull() - else -> null - } diff --git a/apps/android/app/src/main/java/com/clawdbot/android/chat/ChatModels.kt b/apps/android/app/src/main/java/com/clawdbot/android/chat/ChatModels.kt deleted file mode 100644 index ad84e8c69..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/chat/ChatModels.kt +++ /dev/null @@ -1,44 +0,0 @@ -package com.clawdbot.android.chat - -data class ChatMessage( - val id: String, - val role: String, - val content: List, - val timestampMs: Long?, -) - -data class ChatMessageContent( - val type: String = "text", - val text: String? = null, - val mimeType: String? = null, - val fileName: String? = null, - val base64: String? = null, -) - -data class ChatPendingToolCall( - val toolCallId: String, - val name: String, - val args: kotlinx.serialization.json.JsonObject? = null, - val startedAtMs: Long, - val isError: Boolean? = null, -) - -data class ChatSessionEntry( - val key: String, - val updatedAtMs: Long?, - val displayName: String? = null, -) - -data class ChatHistory( - val sessionKey: String, - val sessionId: String?, - val thinkingLevel: String?, - val messages: List, -) - -data class OutgoingAttachment( - val type: String, - val mimeType: String, - val fileName: String, - val base64: String, -) diff --git a/apps/android/app/src/main/java/com/clawdbot/android/gateway/BonjourEscapes.kt b/apps/android/app/src/main/java/com/clawdbot/android/gateway/BonjourEscapes.kt deleted file mode 100644 index c05d41b4b..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/gateway/BonjourEscapes.kt +++ /dev/null @@ -1,35 +0,0 @@ -package com.clawdbot.android.gateway - -object BonjourEscapes { - fun decode(input: String): String { - if (input.isEmpty()) return input - - val bytes = mutableListOf() - var i = 0 - while (i < input.length) { - if (input[i] == '\\' && i + 3 < input.length) { - val d0 = input[i + 1] - val d1 = input[i + 2] - val d2 = input[i + 3] - if (d0.isDigit() && d1.isDigit() && d2.isDigit()) { - val value = - ((d0.code - '0'.code) * 100) + ((d1.code - '0'.code) * 10) + (d2.code - '0'.code) - if (value in 0..255) { - bytes.add(value.toByte()) - i += 4 - continue - } - } - } - - val codePoint = Character.codePointAt(input, i) - val charBytes = String(Character.toChars(codePoint)).toByteArray(Charsets.UTF_8) - for (b in charBytes) { - bytes.add(b) - } - i += Character.charCount(codePoint) - } - - return String(bytes.toByteArray(), Charsets.UTF_8) - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/gateway/DeviceAuthStore.kt b/apps/android/app/src/main/java/com/clawdbot/android/gateway/DeviceAuthStore.kt deleted file mode 100644 index 88643d8d7..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/gateway/DeviceAuthStore.kt +++ /dev/null @@ -1,26 +0,0 @@ -package com.clawdbot.android.gateway - -import com.clawdbot.android.SecurePrefs - -class DeviceAuthStore(private val prefs: SecurePrefs) { - fun loadToken(deviceId: String, role: String): String? { - val key = tokenKey(deviceId, role) - return prefs.getString(key)?.trim()?.takeIf { it.isNotEmpty() } - } - - fun saveToken(deviceId: String, role: String, token: String) { - val key = tokenKey(deviceId, role) - prefs.putString(key, token.trim()) - } - - fun clearToken(deviceId: String, role: String) { - val key = tokenKey(deviceId, role) - prefs.remove(key) - } - - private fun tokenKey(deviceId: String, role: String): String { - val normalizedDevice = deviceId.trim().lowercase() - val normalizedRole = role.trim().lowercase() - return "gateway.deviceToken.$normalizedDevice.$normalizedRole" - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/gateway/DeviceIdentityStore.kt b/apps/android/app/src/main/java/com/clawdbot/android/gateway/DeviceIdentityStore.kt deleted file mode 100644 index 4499e0ce7..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/gateway/DeviceIdentityStore.kt +++ /dev/null @@ -1,146 +0,0 @@ -package com.clawdbot.android.gateway - -import android.content.Context -import android.util.Base64 -import java.io.File -import java.security.KeyFactory -import java.security.KeyPairGenerator -import java.security.MessageDigest -import java.security.Signature -import java.security.spec.PKCS8EncodedKeySpec -import kotlinx.serialization.Serializable -import kotlinx.serialization.json.Json - -@Serializable -data class DeviceIdentity( - val deviceId: String, - val publicKeyRawBase64: String, - val privateKeyPkcs8Base64: String, - val createdAtMs: Long, -) - -class DeviceIdentityStore(context: Context) { - private val json = Json { ignoreUnknownKeys = true } - private val identityFile = File(context.filesDir, "moltbot/identity/device.json") - - @Synchronized - fun loadOrCreate(): DeviceIdentity { - val existing = load() - if (existing != null) { - val derived = deriveDeviceId(existing.publicKeyRawBase64) - if (derived != null && derived != existing.deviceId) { - val updated = existing.copy(deviceId = derived) - save(updated) - return updated - } - return existing - } - val fresh = generate() - save(fresh) - return fresh - } - - fun signPayload(payload: String, identity: DeviceIdentity): String? { - return try { - val privateKeyBytes = Base64.decode(identity.privateKeyPkcs8Base64, Base64.DEFAULT) - val keySpec = PKCS8EncodedKeySpec(privateKeyBytes) - val keyFactory = KeyFactory.getInstance("Ed25519") - val privateKey = keyFactory.generatePrivate(keySpec) - val signature = Signature.getInstance("Ed25519") - signature.initSign(privateKey) - signature.update(payload.toByteArray(Charsets.UTF_8)) - base64UrlEncode(signature.sign()) - } catch (_: Throwable) { - null - } - } - - fun publicKeyBase64Url(identity: DeviceIdentity): String? { - return try { - val raw = Base64.decode(identity.publicKeyRawBase64, Base64.DEFAULT) - base64UrlEncode(raw) - } catch (_: Throwable) { - null - } - } - - private fun load(): DeviceIdentity? { - return try { - if (!identityFile.exists()) return null - val raw = identityFile.readText(Charsets.UTF_8) - val decoded = json.decodeFromString(DeviceIdentity.serializer(), raw) - if (decoded.deviceId.isBlank() || - decoded.publicKeyRawBase64.isBlank() || - decoded.privateKeyPkcs8Base64.isBlank() - ) { - null - } else { - decoded - } - } catch (_: Throwable) { - null - } - } - - private fun save(identity: DeviceIdentity) { - try { - identityFile.parentFile?.mkdirs() - val encoded = json.encodeToString(DeviceIdentity.serializer(), identity) - identityFile.writeText(encoded, Charsets.UTF_8) - } catch (_: Throwable) { - // best-effort only - } - } - - private fun generate(): DeviceIdentity { - val keyPair = KeyPairGenerator.getInstance("Ed25519").generateKeyPair() - val spki = keyPair.public.encoded - val rawPublic = stripSpkiPrefix(spki) - val deviceId = sha256Hex(rawPublic) - val privateKey = keyPair.private.encoded - return DeviceIdentity( - deviceId = deviceId, - publicKeyRawBase64 = Base64.encodeToString(rawPublic, Base64.NO_WRAP), - privateKeyPkcs8Base64 = Base64.encodeToString(privateKey, Base64.NO_WRAP), - createdAtMs = System.currentTimeMillis(), - ) - } - - private fun deriveDeviceId(publicKeyRawBase64: String): String? { - return try { - val raw = Base64.decode(publicKeyRawBase64, Base64.DEFAULT) - sha256Hex(raw) - } catch (_: Throwable) { - null - } - } - - private fun stripSpkiPrefix(spki: ByteArray): ByteArray { - if (spki.size == ED25519_SPKI_PREFIX.size + 32 && - spki.copyOfRange(0, ED25519_SPKI_PREFIX.size).contentEquals(ED25519_SPKI_PREFIX) - ) { - return spki.copyOfRange(ED25519_SPKI_PREFIX.size, spki.size) - } - return spki - } - - private fun sha256Hex(data: ByteArray): String { - val digest = MessageDigest.getInstance("SHA-256").digest(data) - val out = StringBuilder(digest.size * 2) - for (byte in digest) { - out.append(String.format("%02x", byte)) - } - return out.toString() - } - - private fun base64UrlEncode(data: ByteArray): String { - return Base64.encodeToString(data, Base64.URL_SAFE or Base64.NO_WRAP or Base64.NO_PADDING) - } - - companion object { - private val ED25519_SPKI_PREFIX = - byteArrayOf( - 0x30, 0x2a, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x03, 0x21, 0x00, - ) - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayDiscovery.kt b/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayDiscovery.kt deleted file mode 100644 index b1d50e28b..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayDiscovery.kt +++ /dev/null @@ -1,519 +0,0 @@ -package com.clawdbot.android.gateway - -import android.content.Context -import android.net.ConnectivityManager -import android.net.DnsResolver -import android.net.NetworkCapabilities -import android.net.nsd.NsdManager -import android.net.nsd.NsdServiceInfo -import android.os.CancellationSignal -import android.util.Log -import java.io.IOException -import java.net.InetSocketAddress -import java.nio.ByteBuffer -import java.nio.charset.CodingErrorAction -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.Executor -import java.util.concurrent.Executors -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.launch -import kotlinx.coroutines.suspendCancellableCoroutine -import org.xbill.DNS.AAAARecord -import org.xbill.DNS.ARecord -import org.xbill.DNS.DClass -import org.xbill.DNS.ExtendedResolver -import org.xbill.DNS.Message -import org.xbill.DNS.Name -import org.xbill.DNS.PTRRecord -import org.xbill.DNS.Record -import org.xbill.DNS.Rcode -import org.xbill.DNS.Resolver -import org.xbill.DNS.SRVRecord -import org.xbill.DNS.Section -import org.xbill.DNS.SimpleResolver -import org.xbill.DNS.TextParseException -import org.xbill.DNS.TXTRecord -import org.xbill.DNS.Type -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException - -@Suppress("DEPRECATION") -class GatewayDiscovery( - context: Context, - private val scope: CoroutineScope, -) { - private val nsd = context.getSystemService(NsdManager::class.java) - private val connectivity = context.getSystemService(ConnectivityManager::class.java) - private val dns = DnsResolver.getInstance() - private val serviceType = "_moltbot-gw._tcp." - private val wideAreaDomain = "moltbot.internal." - private val logTag = "Moltbot/GatewayDiscovery" - - private val localById = ConcurrentHashMap() - private val unicastById = ConcurrentHashMap() - private val _gateways = MutableStateFlow>(emptyList()) - val gateways: StateFlow> = _gateways.asStateFlow() - - private val _statusText = MutableStateFlow("Searching…") - val statusText: StateFlow = _statusText.asStateFlow() - - private var unicastJob: Job? = null - private val dnsExecutor: Executor = Executors.newCachedThreadPool() - - @Volatile private var lastWideAreaRcode: Int? = null - @Volatile private var lastWideAreaCount: Int = 0 - - private val discoveryListener = - object : NsdManager.DiscoveryListener { - override fun onStartDiscoveryFailed(serviceType: String, errorCode: Int) {} - override fun onStopDiscoveryFailed(serviceType: String, errorCode: Int) {} - override fun onDiscoveryStarted(serviceType: String) {} - override fun onDiscoveryStopped(serviceType: String) {} - - override fun onServiceFound(serviceInfo: NsdServiceInfo) { - if (serviceInfo.serviceType != this@GatewayDiscovery.serviceType) return - resolve(serviceInfo) - } - - override fun onServiceLost(serviceInfo: NsdServiceInfo) { - val serviceName = BonjourEscapes.decode(serviceInfo.serviceName) - val id = stableId(serviceName, "local.") - localById.remove(id) - publish() - } - } - - init { - startLocalDiscovery() - startUnicastDiscovery(wideAreaDomain) - } - - private fun startLocalDiscovery() { - try { - nsd.discoverServices(serviceType, NsdManager.PROTOCOL_DNS_SD, discoveryListener) - } catch (_: Throwable) { - // ignore (best-effort) - } - } - - private fun stopLocalDiscovery() { - try { - nsd.stopServiceDiscovery(discoveryListener) - } catch (_: Throwable) { - // ignore (best-effort) - } - } - - private fun startUnicastDiscovery(domain: String) { - unicastJob = - scope.launch(Dispatchers.IO) { - while (true) { - try { - refreshUnicast(domain) - } catch (_: Throwable) { - // ignore (best-effort) - } - delay(5000) - } - } - } - - private fun resolve(serviceInfo: NsdServiceInfo) { - nsd.resolveService( - serviceInfo, - object : NsdManager.ResolveListener { - override fun onResolveFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {} - - override fun onServiceResolved(resolved: NsdServiceInfo) { - val host = resolved.host?.hostAddress ?: return - val port = resolved.port - if (port <= 0) return - - val rawServiceName = resolved.serviceName - val serviceName = BonjourEscapes.decode(rawServiceName) - val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName) - val lanHost = txt(resolved, "lanHost") - val tailnetDns = txt(resolved, "tailnetDns") - val gatewayPort = txtInt(resolved, "gatewayPort") - val canvasPort = txtInt(resolved, "canvasPort") - val tlsEnabled = txtBool(resolved, "gatewayTls") - val tlsFingerprint = txt(resolved, "gatewayTlsSha256") - val id = stableId(serviceName, "local.") - localById[id] = - GatewayEndpoint( - stableId = id, - name = displayName, - host = host, - port = port, - lanHost = lanHost, - tailnetDns = tailnetDns, - gatewayPort = gatewayPort, - canvasPort = canvasPort, - tlsEnabled = tlsEnabled, - tlsFingerprintSha256 = tlsFingerprint, - ) - publish() - } - }, - ) - } - - private fun publish() { - _gateways.value = - (localById.values + unicastById.values).sortedBy { it.name.lowercase() } - _statusText.value = buildStatusText() - } - - private fun buildStatusText(): String { - val localCount = localById.size - val wideRcode = lastWideAreaRcode - val wideCount = lastWideAreaCount - - val wide = - when (wideRcode) { - null -> "Wide: ?" - Rcode.NOERROR -> "Wide: $wideCount" - Rcode.NXDOMAIN -> "Wide: NXDOMAIN" - else -> "Wide: ${Rcode.string(wideRcode)}" - } - - return when { - localCount == 0 && wideRcode == null -> "Searching for gateways…" - localCount == 0 -> "$wide" - else -> "Local: $localCount • $wide" - } - } - - private fun stableId(serviceName: String, domain: String): String { - return "${serviceType}|${domain}|${normalizeName(serviceName)}" - } - - private fun normalizeName(raw: String): String { - return raw.trim().split(Regex("\\s+")).joinToString(" ") - } - - private fun txt(info: NsdServiceInfo, key: String): String? { - val bytes = info.attributes[key] ?: return null - return try { - String(bytes, Charsets.UTF_8).trim().ifEmpty { null } - } catch (_: Throwable) { - null - } - } - - private fun txtInt(info: NsdServiceInfo, key: String): Int? { - return txt(info, key)?.toIntOrNull() - } - - private fun txtBool(info: NsdServiceInfo, key: String): Boolean { - val raw = txt(info, key)?.trim()?.lowercase() ?: return false - return raw == "1" || raw == "true" || raw == "yes" - } - - private suspend fun refreshUnicast(domain: String) { - val ptrName = "${serviceType}${domain}" - val ptrMsg = lookupUnicastMessage(ptrName, Type.PTR) ?: return - val ptrRecords = records(ptrMsg, Section.ANSWER).mapNotNull { it as? PTRRecord } - - val next = LinkedHashMap() - for (ptr in ptrRecords) { - val instanceFqdn = ptr.target.toString() - val srv = - recordByName(ptrMsg, instanceFqdn, Type.SRV) as? SRVRecord - ?: run { - val msg = lookupUnicastMessage(instanceFqdn, Type.SRV) ?: return@run null - recordByName(msg, instanceFqdn, Type.SRV) as? SRVRecord - } - ?: continue - val port = srv.port - if (port <= 0) continue - - val targetFqdn = srv.target.toString() - val host = - resolveHostFromMessage(ptrMsg, targetFqdn) - ?: resolveHostFromMessage(lookupUnicastMessage(instanceFqdn, Type.SRV), targetFqdn) - ?: resolveHostUnicast(targetFqdn) - ?: continue - - val txtFromPtr = - recordsByName(ptrMsg, Section.ADDITIONAL)[keyName(instanceFqdn)] - .orEmpty() - .mapNotNull { it as? TXTRecord } - val txt = - if (txtFromPtr.isNotEmpty()) { - txtFromPtr - } else { - val msg = lookupUnicastMessage(instanceFqdn, Type.TXT) - records(msg, Section.ANSWER).mapNotNull { it as? TXTRecord } - } - val instanceName = BonjourEscapes.decode(decodeInstanceName(instanceFqdn, domain)) - val displayName = BonjourEscapes.decode(txtValue(txt, "displayName") ?: instanceName) - val lanHost = txtValue(txt, "lanHost") - val tailnetDns = txtValue(txt, "tailnetDns") - val gatewayPort = txtIntValue(txt, "gatewayPort") - val canvasPort = txtIntValue(txt, "canvasPort") - val tlsEnabled = txtBoolValue(txt, "gatewayTls") - val tlsFingerprint = txtValue(txt, "gatewayTlsSha256") - val id = stableId(instanceName, domain) - next[id] = - GatewayEndpoint( - stableId = id, - name = displayName, - host = host, - port = port, - lanHost = lanHost, - tailnetDns = tailnetDns, - gatewayPort = gatewayPort, - canvasPort = canvasPort, - tlsEnabled = tlsEnabled, - tlsFingerprintSha256 = tlsFingerprint, - ) - } - - unicastById.clear() - unicastById.putAll(next) - lastWideAreaRcode = ptrMsg.header.rcode - lastWideAreaCount = next.size - publish() - - if (next.isEmpty()) { - Log.d( - logTag, - "wide-area discovery: 0 results for $ptrName (rcode=${Rcode.string(ptrMsg.header.rcode)})", - ) - } - } - - private fun decodeInstanceName(instanceFqdn: String, domain: String): String { - val suffix = "${serviceType}${domain}" - val withoutSuffix = - if (instanceFqdn.endsWith(suffix)) { - instanceFqdn.removeSuffix(suffix) - } else { - instanceFqdn.substringBefore(serviceType) - } - return normalizeName(stripTrailingDot(withoutSuffix)) - } - - private fun stripTrailingDot(raw: String): String { - return raw.removeSuffix(".") - } - - private suspend fun lookupUnicastMessage(name: String, type: Int): Message? { - val query = - try { - Message.newQuery( - org.xbill.DNS.Record.newRecord( - Name.fromString(name), - type, - DClass.IN, - ), - ) - } catch (_: TextParseException) { - return null - } - - val system = queryViaSystemDns(query) - if (records(system, Section.ANSWER).any { it.type == type }) return system - - val direct = createDirectResolver() ?: return system - return try { - val msg = direct.send(query) - if (records(msg, Section.ANSWER).any { it.type == type }) msg else system - } catch (_: Throwable) { - system - } - } - - private suspend fun queryViaSystemDns(query: Message): Message? { - val network = preferredDnsNetwork() - val bytes = - try { - rawQuery(network, query.toWire()) - } catch (_: Throwable) { - return null - } - - return try { - Message(bytes) - } catch (_: IOException) { - null - } - } - - private fun records(msg: Message?, section: Int): List { - return msg?.getSectionArray(section)?.toList() ?: emptyList() - } - - private fun keyName(raw: String): String { - return raw.trim().lowercase() - } - - private fun recordsByName(msg: Message, section: Int): Map> { - val next = LinkedHashMap>() - for (r in records(msg, section)) { - val name = r.name?.toString() ?: continue - next.getOrPut(keyName(name)) { mutableListOf() }.add(r) - } - return next - } - - private fun recordByName(msg: Message, fqdn: String, type: Int): Record? { - val key = keyName(fqdn) - val byNameAnswer = recordsByName(msg, Section.ANSWER) - val fromAnswer = byNameAnswer[key].orEmpty().firstOrNull { it.type == type } - if (fromAnswer != null) return fromAnswer - - val byNameAdditional = recordsByName(msg, Section.ADDITIONAL) - return byNameAdditional[key].orEmpty().firstOrNull { it.type == type } - } - - private fun resolveHostFromMessage(msg: Message?, hostname: String): String? { - val m = msg ?: return null - val key = keyName(hostname) - val additional = recordsByName(m, Section.ADDITIONAL)[key].orEmpty() - val a = additional.mapNotNull { it as? ARecord }.mapNotNull { it.address?.hostAddress } - val aaaa = additional.mapNotNull { it as? AAAARecord }.mapNotNull { it.address?.hostAddress } - return a.firstOrNull() ?: aaaa.firstOrNull() - } - - private fun preferredDnsNetwork(): android.net.Network? { - val cm = connectivity ?: return null - - // Prefer VPN (Tailscale) when present; otherwise use the active network. - cm.allNetworks.firstOrNull { n -> - val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false - caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN) - }?.let { return it } - - return cm.activeNetwork - } - - private fun createDirectResolver(): Resolver? { - val cm = connectivity ?: return null - - val candidateNetworks = - buildList { - cm.allNetworks - .firstOrNull { n -> - val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false - caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN) - }?.let(::add) - cm.activeNetwork?.let(::add) - }.distinct() - - val servers = - candidateNetworks - .asSequence() - .flatMap { n -> - cm.getLinkProperties(n)?.dnsServers?.asSequence() ?: emptySequence() - } - .distinctBy { it.hostAddress ?: it.toString() } - .toList() - if (servers.isEmpty()) return null - - return try { - val resolvers = - servers.mapNotNull { addr -> - try { - SimpleResolver().apply { - setAddress(InetSocketAddress(addr, 53)) - setTimeout(3) - } - } catch (_: Throwable) { - null - } - } - if (resolvers.isEmpty()) return null - ExtendedResolver(resolvers.toTypedArray()).apply { setTimeout(3) } - } catch (_: Throwable) { - null - } - } - - private suspend fun rawQuery(network: android.net.Network?, wireQuery: ByteArray): ByteArray = - suspendCancellableCoroutine { cont -> - val signal = CancellationSignal() - cont.invokeOnCancellation { signal.cancel() } - - dns.rawQuery( - network, - wireQuery, - DnsResolver.FLAG_EMPTY, - dnsExecutor, - signal, - object : DnsResolver.Callback { - override fun onAnswer(answer: ByteArray, rcode: Int) { - cont.resume(answer) - } - - override fun onError(error: DnsResolver.DnsException) { - cont.resumeWithException(error) - } - }, - ) - } - - private fun txtValue(records: List, key: String): String? { - val prefix = "$key=" - for (r in records) { - val strings: List = - try { - r.strings.mapNotNull { it as? String } - } catch (_: Throwable) { - emptyList() - } - for (s in strings) { - val trimmed = decodeDnsTxtString(s).trim() - if (trimmed.startsWith(prefix)) { - return trimmed.removePrefix(prefix).trim().ifEmpty { null } - } - } - } - return null - } - - private fun txtIntValue(records: List, key: String): Int? { - return txtValue(records, key)?.toIntOrNull() - } - - private fun txtBoolValue(records: List, key: String): Boolean { - val raw = txtValue(records, key)?.trim()?.lowercase() ?: return false - return raw == "1" || raw == "true" || raw == "yes" - } - - private fun decodeDnsTxtString(raw: String): String { - // dnsjava treats TXT as opaque bytes and decodes as ISO-8859-1 to preserve bytes. - // Our TXT payload is UTF-8 (written by the gateway), so re-decode when possible. - val bytes = raw.toByteArray(Charsets.ISO_8859_1) - val decoder = - Charsets.UTF_8 - .newDecoder() - .onMalformedInput(CodingErrorAction.REPORT) - .onUnmappableCharacter(CodingErrorAction.REPORT) - return try { - decoder.decode(ByteBuffer.wrap(bytes)).toString() - } catch (_: Throwable) { - raw - } - } - - private suspend fun resolveHostUnicast(hostname: String): String? { - val a = - records(lookupUnicastMessage(hostname, Type.A), Section.ANSWER) - .mapNotNull { it as? ARecord } - .mapNotNull { it.address?.hostAddress } - val aaaa = - records(lookupUnicastMessage(hostname, Type.AAAA), Section.ANSWER) - .mapNotNull { it as? AAAARecord } - .mapNotNull { it.address?.hostAddress } - - return a.firstOrNull() ?: aaaa.firstOrNull() - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayEndpoint.kt b/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayEndpoint.kt deleted file mode 100644 index ab8aeacc9..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayEndpoint.kt +++ /dev/null @@ -1,26 +0,0 @@ -package com.clawdbot.android.gateway - -data class GatewayEndpoint( - val stableId: String, - val name: String, - val host: String, - val port: Int, - val lanHost: String? = null, - val tailnetDns: String? = null, - val gatewayPort: Int? = null, - val canvasPort: Int? = null, - val tlsEnabled: Boolean = false, - val tlsFingerprintSha256: String? = null, -) { - companion object { - fun manual(host: String, port: Int): GatewayEndpoint = - GatewayEndpoint( - stableId = "manual|${host.lowercase()}|$port", - name = "$host:$port", - host = host, - port = port, - tlsEnabled = false, - tlsFingerprintSha256 = null, - ) - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayProtocol.kt b/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayProtocol.kt deleted file mode 100644 index 4873de122..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayProtocol.kt +++ /dev/null @@ -1,3 +0,0 @@ -package com.clawdbot.android.gateway - -const val GATEWAY_PROTOCOL_VERSION = 3 diff --git a/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewaySession.kt b/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewaySession.kt deleted file mode 100644 index a54460e0a..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewaySession.kt +++ /dev/null @@ -1,683 +0,0 @@ -package com.clawdbot.android.gateway - -import android.util.Log -import java.util.Locale -import java.util.UUID -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicBoolean -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.cancelAndJoin -import kotlinx.coroutines.delay -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import kotlinx.coroutines.withContext -import kotlinx.coroutines.withTimeout -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonNull -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive -import kotlinx.serialization.json.buildJsonObject -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.Response -import okhttp3.WebSocket -import okhttp3.WebSocketListener - -data class GatewayClientInfo( - val id: String, - val displayName: String?, - val version: String, - val platform: String, - val mode: String, - val instanceId: String?, - val deviceFamily: String?, - val modelIdentifier: String?, -) - -data class GatewayConnectOptions( - val role: String, - val scopes: List, - val caps: List, - val commands: List, - val permissions: Map, - val client: GatewayClientInfo, - val userAgent: String? = null, -) - -class GatewaySession( - private val scope: CoroutineScope, - private val identityStore: DeviceIdentityStore, - private val deviceAuthStore: DeviceAuthStore, - private val onConnected: (serverName: String?, remoteAddress: String?, mainSessionKey: String?) -> Unit, - private val onDisconnected: (message: String) -> Unit, - private val onEvent: (event: String, payloadJson: String?) -> Unit, - private val onInvoke: (suspend (InvokeRequest) -> InvokeResult)? = null, - private val onTlsFingerprint: ((stableId: String, fingerprint: String) -> Unit)? = null, -) { - data class InvokeRequest( - val id: String, - val nodeId: String, - val command: String, - val paramsJson: String?, - val timeoutMs: Long?, - ) - - data class InvokeResult(val ok: Boolean, val payloadJson: String?, val error: ErrorShape?) { - companion object { - fun ok(payloadJson: String?) = InvokeResult(ok = true, payloadJson = payloadJson, error = null) - fun error(code: String, message: String) = - InvokeResult(ok = false, payloadJson = null, error = ErrorShape(code = code, message = message)) - } - } - - data class ErrorShape(val code: String, val message: String) - - private val json = Json { ignoreUnknownKeys = true } - private val writeLock = Mutex() - private val pending = ConcurrentHashMap>() - - @Volatile private var canvasHostUrl: String? = null - @Volatile private var mainSessionKey: String? = null - - private data class DesiredConnection( - val endpoint: GatewayEndpoint, - val token: String?, - val password: String?, - val options: GatewayConnectOptions, - val tls: GatewayTlsParams?, - ) - - private var desired: DesiredConnection? = null - private var job: Job? = null - @Volatile private var currentConnection: Connection? = null - - fun connect( - endpoint: GatewayEndpoint, - token: String?, - password: String?, - options: GatewayConnectOptions, - tls: GatewayTlsParams? = null, - ) { - desired = DesiredConnection(endpoint, token, password, options, tls) - if (job == null) { - job = scope.launch(Dispatchers.IO) { runLoop() } - } - } - - fun disconnect() { - desired = null - currentConnection?.closeQuietly() - scope.launch(Dispatchers.IO) { - job?.cancelAndJoin() - job = null - canvasHostUrl = null - mainSessionKey = null - onDisconnected("Offline") - } - } - - fun reconnect() { - currentConnection?.closeQuietly() - } - - fun currentCanvasHostUrl(): String? = canvasHostUrl - fun currentMainSessionKey(): String? = mainSessionKey - - suspend fun sendNodeEvent(event: String, payloadJson: String?) { - val conn = currentConnection ?: return - val parsedPayload = payloadJson?.let { parseJsonOrNull(it) } - val params = - buildJsonObject { - put("event", JsonPrimitive(event)) - if (parsedPayload != null) { - put("payload", parsedPayload) - } else if (payloadJson != null) { - put("payloadJSON", JsonPrimitive(payloadJson)) - } else { - put("payloadJSON", JsonNull) - } - } - try { - conn.request("node.event", params, timeoutMs = 8_000) - } catch (err: Throwable) { - Log.w("MoltbotGateway", "node.event failed: ${err.message ?: err::class.java.simpleName}") - } - } - - suspend fun request(method: String, paramsJson: String?, timeoutMs: Long = 15_000): String { - val conn = currentConnection ?: throw IllegalStateException("not connected") - val params = - if (paramsJson.isNullOrBlank()) { - null - } else { - json.parseToJsonElement(paramsJson) - } - val res = conn.request(method, params, timeoutMs) - if (res.ok) return res.payloadJson ?: "" - val err = res.error - throw IllegalStateException("${err?.code ?: "UNAVAILABLE"}: ${err?.message ?: "request failed"}") - } - - private data class RpcResponse(val id: String, val ok: Boolean, val payloadJson: String?, val error: ErrorShape?) - - private inner class Connection( - private val endpoint: GatewayEndpoint, - private val token: String?, - private val password: String?, - private val options: GatewayConnectOptions, - private val tls: GatewayTlsParams?, - ) { - private val connectDeferred = CompletableDeferred() - private val closedDeferred = CompletableDeferred() - private val isClosed = AtomicBoolean(false) - private val connectNonceDeferred = CompletableDeferred() - private val client: OkHttpClient = buildClient() - private var socket: WebSocket? = null - private val loggerTag = "MoltbotGateway" - - val remoteAddress: String = - if (endpoint.host.contains(":")) { - "[${endpoint.host}]:${endpoint.port}" - } else { - "${endpoint.host}:${endpoint.port}" - } - - suspend fun connect() { - val scheme = if (tls != null) "wss" else "ws" - val url = "$scheme://${endpoint.host}:${endpoint.port}" - val request = Request.Builder().url(url).build() - socket = client.newWebSocket(request, Listener()) - try { - connectDeferred.await() - } catch (err: Throwable) { - throw err - } - } - - suspend fun request(method: String, params: JsonElement?, timeoutMs: Long): RpcResponse { - val id = UUID.randomUUID().toString() - val deferred = CompletableDeferred() - pending[id] = deferred - val frame = - buildJsonObject { - put("type", JsonPrimitive("req")) - put("id", JsonPrimitive(id)) - put("method", JsonPrimitive(method)) - if (params != null) put("params", params) - } - sendJson(frame) - return try { - withTimeout(timeoutMs) { deferred.await() } - } catch (err: TimeoutCancellationException) { - pending.remove(id) - throw IllegalStateException("request timeout") - } - } - - suspend fun sendJson(obj: JsonObject) { - val jsonString = obj.toString() - writeLock.withLock { - socket?.send(jsonString) - } - } - - suspend fun awaitClose() = closedDeferred.await() - - fun closeQuietly() { - if (isClosed.compareAndSet(false, true)) { - socket?.close(1000, "bye") - socket = null - closedDeferred.complete(Unit) - } - } - - private fun buildClient(): OkHttpClient { - val builder = OkHttpClient.Builder() - val tlsConfig = buildGatewayTlsConfig(tls) { fingerprint -> - onTlsFingerprint?.invoke(tls?.stableId ?: endpoint.stableId, fingerprint) - } - if (tlsConfig != null) { - builder.sslSocketFactory(tlsConfig.sslSocketFactory, tlsConfig.trustManager) - builder.hostnameVerifier(tlsConfig.hostnameVerifier) - } - return builder.build() - } - - private inner class Listener : WebSocketListener() { - override fun onOpen(webSocket: WebSocket, response: Response) { - scope.launch { - try { - val nonce = awaitConnectNonce() - sendConnect(nonce) - } catch (err: Throwable) { - connectDeferred.completeExceptionally(err) - closeQuietly() - } - } - } - - override fun onMessage(webSocket: WebSocket, text: String) { - scope.launch { handleMessage(text) } - } - - override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { - if (!connectDeferred.isCompleted) { - connectDeferred.completeExceptionally(t) - } - if (isClosed.compareAndSet(false, true)) { - failPending() - closedDeferred.complete(Unit) - onDisconnected("Gateway error: ${t.message ?: t::class.java.simpleName}") - } - } - - override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { - if (!connectDeferred.isCompleted) { - connectDeferred.completeExceptionally(IllegalStateException("Gateway closed: $reason")) - } - if (isClosed.compareAndSet(false, true)) { - failPending() - closedDeferred.complete(Unit) - onDisconnected("Gateway closed: $reason") - } - } - } - - private suspend fun sendConnect(connectNonce: String?) { - val identity = identityStore.loadOrCreate() - val storedToken = deviceAuthStore.loadToken(identity.deviceId, options.role) - val trimmedToken = token?.trim().orEmpty() - val authToken = if (storedToken.isNullOrBlank()) trimmedToken else storedToken - val canFallbackToShared = !storedToken.isNullOrBlank() && trimmedToken.isNotBlank() - val payload = buildConnectParams(identity, connectNonce, authToken, password?.trim()) - val res = request("connect", payload, timeoutMs = 8_000) - if (!res.ok) { - val msg = res.error?.message ?: "connect failed" - if (canFallbackToShared) { - deviceAuthStore.clearToken(identity.deviceId, options.role) - } - throw IllegalStateException(msg) - } - val payloadJson = res.payloadJson ?: throw IllegalStateException("connect failed: missing payload") - val obj = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: throw IllegalStateException("connect failed") - val serverName = obj["server"].asObjectOrNull()?.get("host").asStringOrNull() - val authObj = obj["auth"].asObjectOrNull() - val deviceToken = authObj?.get("deviceToken").asStringOrNull() - val authRole = authObj?.get("role").asStringOrNull() ?: options.role - if (!deviceToken.isNullOrBlank()) { - deviceAuthStore.saveToken(identity.deviceId, authRole, deviceToken) - } - val rawCanvas = obj["canvasHostUrl"].asStringOrNull() - canvasHostUrl = normalizeCanvasHostUrl(rawCanvas, endpoint) - val sessionDefaults = - obj["snapshot"].asObjectOrNull() - ?.get("sessionDefaults").asObjectOrNull() - mainSessionKey = sessionDefaults?.get("mainSessionKey").asStringOrNull() - onConnected(serverName, remoteAddress, mainSessionKey) - connectDeferred.complete(Unit) - } - - private fun buildConnectParams( - identity: DeviceIdentity, - connectNonce: String?, - authToken: String, - authPassword: String?, - ): JsonObject { - val client = options.client - val locale = Locale.getDefault().toLanguageTag() - val clientObj = - buildJsonObject { - put("id", JsonPrimitive(client.id)) - client.displayName?.let { put("displayName", JsonPrimitive(it)) } - put("version", JsonPrimitive(client.version)) - put("platform", JsonPrimitive(client.platform)) - put("mode", JsonPrimitive(client.mode)) - client.instanceId?.let { put("instanceId", JsonPrimitive(it)) } - client.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) } - client.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) } - } - - val password = authPassword?.trim().orEmpty() - val authJson = - when { - authToken.isNotEmpty() -> - buildJsonObject { - put("token", JsonPrimitive(authToken)) - } - password.isNotEmpty() -> - buildJsonObject { - put("password", JsonPrimitive(password)) - } - else -> null - } - - val signedAtMs = System.currentTimeMillis() - val payload = - buildDeviceAuthPayload( - deviceId = identity.deviceId, - clientId = client.id, - clientMode = client.mode, - role = options.role, - scopes = options.scopes, - signedAtMs = signedAtMs, - token = if (authToken.isNotEmpty()) authToken else null, - nonce = connectNonce, - ) - val signature = identityStore.signPayload(payload, identity) - val publicKey = identityStore.publicKeyBase64Url(identity) - val deviceJson = - if (!signature.isNullOrBlank() && !publicKey.isNullOrBlank()) { - buildJsonObject { - put("id", JsonPrimitive(identity.deviceId)) - put("publicKey", JsonPrimitive(publicKey)) - put("signature", JsonPrimitive(signature)) - put("signedAt", JsonPrimitive(signedAtMs)) - if (!connectNonce.isNullOrBlank()) { - put("nonce", JsonPrimitive(connectNonce)) - } - } - } else { - null - } - - return buildJsonObject { - put("minProtocol", JsonPrimitive(GATEWAY_PROTOCOL_VERSION)) - put("maxProtocol", JsonPrimitive(GATEWAY_PROTOCOL_VERSION)) - put("client", clientObj) - if (options.caps.isNotEmpty()) put("caps", JsonArray(options.caps.map(::JsonPrimitive))) - if (options.commands.isNotEmpty()) put("commands", JsonArray(options.commands.map(::JsonPrimitive))) - if (options.permissions.isNotEmpty()) { - put( - "permissions", - buildJsonObject { - options.permissions.forEach { (key, value) -> - put(key, JsonPrimitive(value)) - } - }, - ) - } - put("role", JsonPrimitive(options.role)) - if (options.scopes.isNotEmpty()) put("scopes", JsonArray(options.scopes.map(::JsonPrimitive))) - authJson?.let { put("auth", it) } - deviceJson?.let { put("device", it) } - put("locale", JsonPrimitive(locale)) - options.userAgent?.trim()?.takeIf { it.isNotEmpty() }?.let { - put("userAgent", JsonPrimitive(it)) - } - } - } - - private suspend fun handleMessage(text: String) { - val frame = json.parseToJsonElement(text).asObjectOrNull() ?: return - when (frame["type"].asStringOrNull()) { - "res" -> handleResponse(frame) - "event" -> handleEvent(frame) - } - } - - private fun handleResponse(frame: JsonObject) { - val id = frame["id"].asStringOrNull() ?: return - val ok = frame["ok"].asBooleanOrNull() ?: false - val payloadJson = frame["payload"]?.let { payload -> payload.toString() } - val error = - frame["error"]?.asObjectOrNull()?.let { obj -> - val code = obj["code"].asStringOrNull() ?: "UNAVAILABLE" - val msg = obj["message"].asStringOrNull() ?: "request failed" - ErrorShape(code, msg) - } - pending.remove(id)?.complete(RpcResponse(id, ok, payloadJson, error)) - } - - private fun handleEvent(frame: JsonObject) { - val event = frame["event"].asStringOrNull() ?: return - val payloadJson = - frame["payload"]?.let { it.toString() } ?: frame["payloadJSON"].asStringOrNull() - if (event == "connect.challenge") { - val nonce = extractConnectNonce(payloadJson) - if (!connectNonceDeferred.isCompleted) { - connectNonceDeferred.complete(nonce) - } - return - } - if (event == "node.invoke.request" && payloadJson != null && onInvoke != null) { - handleInvokeEvent(payloadJson) - return - } - onEvent(event, payloadJson) - } - - private suspend fun awaitConnectNonce(): String? { - if (isLoopbackHost(endpoint.host)) return null - return try { - withTimeout(2_000) { connectNonceDeferred.await() } - } catch (_: Throwable) { - null - } - } - - private fun extractConnectNonce(payloadJson: String?): String? { - if (payloadJson.isNullOrBlank()) return null - val obj = parseJsonOrNull(payloadJson)?.asObjectOrNull() ?: return null - return obj["nonce"].asStringOrNull() - } - - private fun handleInvokeEvent(payloadJson: String) { - val payload = - try { - json.parseToJsonElement(payloadJson).asObjectOrNull() - } catch (_: Throwable) { - null - } ?: return - val id = payload["id"].asStringOrNull() ?: return - val nodeId = payload["nodeId"].asStringOrNull() ?: return - val command = payload["command"].asStringOrNull() ?: return - val params = - payload["paramsJSON"].asStringOrNull() - ?: payload["params"]?.let { value -> if (value is JsonNull) null else value.toString() } - val timeoutMs = payload["timeoutMs"].asLongOrNull() - scope.launch { - val result = - try { - onInvoke?.invoke(InvokeRequest(id, nodeId, command, params, timeoutMs)) - ?: InvokeResult.error("UNAVAILABLE", "invoke handler missing") - } catch (err: Throwable) { - invokeErrorFromThrowable(err) - } - sendInvokeResult(id, nodeId, result) - } - } - - private suspend fun sendInvokeResult(id: String, nodeId: String, result: InvokeResult) { - val parsedPayload = result.payloadJson?.let { parseJsonOrNull(it) } - val params = - buildJsonObject { - put("id", JsonPrimitive(id)) - put("nodeId", JsonPrimitive(nodeId)) - put("ok", JsonPrimitive(result.ok)) - if (parsedPayload != null) { - put("payload", parsedPayload) - } else if (result.payloadJson != null) { - put("payloadJSON", JsonPrimitive(result.payloadJson)) - } - result.error?.let { err -> - put( - "error", - buildJsonObject { - put("code", JsonPrimitive(err.code)) - put("message", JsonPrimitive(err.message)) - }, - ) - } - } - try { - request("node.invoke.result", params, timeoutMs = 15_000) - } catch (err: Throwable) { - Log.w(loggerTag, "node.invoke.result failed: ${err.message ?: err::class.java.simpleName}") - } - } - - private fun invokeErrorFromThrowable(err: Throwable): InvokeResult { - val msg = err.message?.trim().takeIf { !it.isNullOrEmpty() } ?: err::class.java.simpleName - val parts = msg.split(":", limit = 2) - if (parts.size == 2) { - val code = parts[0].trim() - val rest = parts[1].trim() - if (code.isNotEmpty() && code.all { it.isUpperCase() || it == '_' }) { - return InvokeResult.error(code = code, message = rest.ifEmpty { msg }) - } - } - return InvokeResult.error(code = "UNAVAILABLE", message = msg) - } - - private fun failPending() { - for ((_, waiter) in pending) { - waiter.cancel() - } - pending.clear() - } - } - - private suspend fun runLoop() { - var attempt = 0 - while (scope.isActive) { - val target = desired - if (target == null) { - currentConnection?.closeQuietly() - currentConnection = null - delay(250) - continue - } - - try { - onDisconnected(if (attempt == 0) "Connecting…" else "Reconnecting…") - connectOnce(target) - attempt = 0 - } catch (err: Throwable) { - attempt += 1 - onDisconnected("Gateway error: ${err.message ?: err::class.java.simpleName}") - val sleepMs = minOf(8_000L, (350.0 * Math.pow(1.7, attempt.toDouble())).toLong()) - delay(sleepMs) - } - } - } - - private suspend fun connectOnce(target: DesiredConnection) = withContext(Dispatchers.IO) { - val conn = Connection(target.endpoint, target.token, target.password, target.options, target.tls) - currentConnection = conn - try { - conn.connect() - conn.awaitClose() - } finally { - currentConnection = null - canvasHostUrl = null - mainSessionKey = null - } - } - - private fun buildDeviceAuthPayload( - deviceId: String, - clientId: String, - clientMode: String, - role: String, - scopes: List, - signedAtMs: Long, - token: String?, - nonce: String?, - ): String { - val scopeString = scopes.joinToString(",") - val authToken = token.orEmpty() - val version = if (nonce.isNullOrBlank()) "v1" else "v2" - val parts = - mutableListOf( - version, - deviceId, - clientId, - clientMode, - role, - scopeString, - signedAtMs.toString(), - authToken, - ) - if (!nonce.isNullOrBlank()) { - parts.add(nonce) - } - return parts.joinToString("|") - } - - private fun normalizeCanvasHostUrl(raw: String?, endpoint: GatewayEndpoint): String? { - val trimmed = raw?.trim().orEmpty() - val parsed = trimmed.takeIf { it.isNotBlank() }?.let { runCatching { java.net.URI(it) }.getOrNull() } - val host = parsed?.host?.trim().orEmpty() - val port = parsed?.port ?: -1 - val scheme = parsed?.scheme?.trim().orEmpty().ifBlank { "http" } - - if (trimmed.isNotBlank() && !isLoopbackHost(host)) { - return trimmed - } - - val fallbackHost = - endpoint.tailnetDns?.trim().takeIf { !it.isNullOrEmpty() } - ?: endpoint.lanHost?.trim().takeIf { !it.isNullOrEmpty() } - ?: endpoint.host.trim() - if (fallbackHost.isEmpty()) return trimmed.ifBlank { null } - - val fallbackPort = endpoint.canvasPort ?: if (port > 0) port else 18793 - val formattedHost = if (fallbackHost.contains(":")) "[${fallbackHost}]" else fallbackHost - return "$scheme://$formattedHost:$fallbackPort" - } - - private fun isLoopbackHost(raw: String?): Boolean { - val host = raw?.trim()?.lowercase().orEmpty() - if (host.isEmpty()) return false - if (host == "localhost") return true - if (host == "::1") return true - if (host == "0.0.0.0" || host == "::") return true - return host.startsWith("127.") - } -} - -private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject - -private fun JsonElement?.asStringOrNull(): String? = - when (this) { - is JsonNull -> null - is JsonPrimitive -> content - else -> null - } - -private fun JsonElement?.asBooleanOrNull(): Boolean? = - when (this) { - is JsonPrimitive -> { - val c = content.trim() - when { - c.equals("true", ignoreCase = true) -> true - c.equals("false", ignoreCase = true) -> false - else -> null - } - } - else -> null - } - -private fun JsonElement?.asLongOrNull(): Long? = - when (this) { - is JsonPrimitive -> content.toLongOrNull() - else -> null - } - -private fun parseJsonOrNull(payload: String): JsonElement? { - val trimmed = payload.trim() - if (trimmed.isEmpty()) return null - return try { - Json.parseToJsonElement(trimmed) - } catch (_: Throwable) { - null - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayTls.kt b/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayTls.kt deleted file mode 100644 index bcca51583..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/gateway/GatewayTls.kt +++ /dev/null @@ -1,90 +0,0 @@ -package com.clawdbot.android.gateway - -import android.annotation.SuppressLint -import java.security.MessageDigest -import java.security.SecureRandom -import java.security.cert.CertificateException -import java.security.cert.X509Certificate -import javax.net.ssl.HostnameVerifier -import javax.net.ssl.SSLContext -import javax.net.ssl.SSLSocketFactory -import javax.net.ssl.TrustManagerFactory -import javax.net.ssl.X509TrustManager - -data class GatewayTlsParams( - val required: Boolean, - val expectedFingerprint: String?, - val allowTOFU: Boolean, - val stableId: String, -) - -data class GatewayTlsConfig( - val sslSocketFactory: SSLSocketFactory, - val trustManager: X509TrustManager, - val hostnameVerifier: HostnameVerifier, -) - -fun buildGatewayTlsConfig( - params: GatewayTlsParams?, - onStore: ((String) -> Unit)? = null, -): GatewayTlsConfig? { - if (params == null) return null - val expected = params.expectedFingerprint?.let(::normalizeFingerprint) - val defaultTrust = defaultTrustManager() - @SuppressLint("CustomX509TrustManager") - val trustManager = - object : X509TrustManager { - override fun checkClientTrusted(chain: Array, authType: String) { - defaultTrust.checkClientTrusted(chain, authType) - } - - override fun checkServerTrusted(chain: Array, authType: String) { - if (chain.isEmpty()) throw CertificateException("empty certificate chain") - val fingerprint = sha256Hex(chain[0].encoded) - if (expected != null) { - if (fingerprint != expected) { - throw CertificateException("gateway TLS fingerprint mismatch") - } - return - } - if (params.allowTOFU) { - onStore?.invoke(fingerprint) - return - } - defaultTrust.checkServerTrusted(chain, authType) - } - - override fun getAcceptedIssuers(): Array = defaultTrust.acceptedIssuers - } - - val context = SSLContext.getInstance("TLS") - context.init(null, arrayOf(trustManager), SecureRandom()) - return GatewayTlsConfig( - sslSocketFactory = context.socketFactory, - trustManager = trustManager, - hostnameVerifier = HostnameVerifier { _, _ -> true }, - ) -} - -private fun defaultTrustManager(): X509TrustManager { - val factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) - factory.init(null as java.security.KeyStore?) - val trust = - factory.trustManagers.firstOrNull { it is X509TrustManager } as? X509TrustManager - return trust ?: throw IllegalStateException("No default X509TrustManager found") -} - -private fun sha256Hex(data: ByteArray): String { - val digest = MessageDigest.getInstance("SHA-256").digest(data) - val out = StringBuilder(digest.size * 2) - for (byte in digest) { - out.append(String.format("%02x", byte)) - } - return out.toString() -} - -private fun normalizeFingerprint(raw: String): String { - val stripped = raw.trim() - .replace(Regex("^sha-?256\\s*:?\\s*", RegexOption.IGNORE_CASE), "") - return stripped.lowercase().filter { it in '0'..'9' || it in 'a'..'f' } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/node/CameraCaptureManager.kt b/apps/android/app/src/main/java/com/clawdbot/android/node/CameraCaptureManager.kt deleted file mode 100644 index f4c4d5794..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/node/CameraCaptureManager.kt +++ /dev/null @@ -1,316 +0,0 @@ -package com.clawdbot.android.node - -import android.Manifest -import android.content.Context -import android.annotation.SuppressLint -import android.graphics.Bitmap -import android.graphics.BitmapFactory -import android.graphics.Matrix -import android.util.Base64 -import android.content.pm.PackageManager -import androidx.exifinterface.media.ExifInterface -import androidx.lifecycle.LifecycleOwner -import androidx.camera.core.CameraSelector -import androidx.camera.core.ImageCapture -import androidx.camera.core.ImageCaptureException -import androidx.camera.lifecycle.ProcessCameraProvider -import androidx.camera.video.FileOutputOptions -import androidx.camera.video.Recorder -import androidx.camera.video.Recording -import androidx.camera.video.VideoCapture -import androidx.camera.video.VideoRecordEvent -import androidx.core.content.ContextCompat -import androidx.core.content.ContextCompat.checkSelfPermission -import androidx.core.graphics.scale -import com.clawdbot.android.PermissionRequester -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.suspendCancellableCoroutine -import kotlinx.coroutines.withTimeout -import kotlinx.coroutines.withContext -import java.io.ByteArrayOutputStream -import java.io.File -import java.util.concurrent.Executor -import kotlin.math.roundToInt -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException - -class CameraCaptureManager(private val context: Context) { - data class Payload(val payloadJson: String) - - @Volatile private var lifecycleOwner: LifecycleOwner? = null - @Volatile private var permissionRequester: PermissionRequester? = null - - fun attachLifecycleOwner(owner: LifecycleOwner) { - lifecycleOwner = owner - } - - fun attachPermissionRequester(requester: PermissionRequester) { - permissionRequester = requester - } - - private suspend fun ensureCameraPermission() { - val granted = checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED - if (granted) return - - val requester = permissionRequester - ?: throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission") - val results = requester.requestIfMissing(listOf(Manifest.permission.CAMERA)) - if (results[Manifest.permission.CAMERA] != true) { - throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission") - } - } - - private suspend fun ensureMicPermission() { - val granted = checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED - if (granted) return - - val requester = permissionRequester - ?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") - val results = requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO)) - if (results[Manifest.permission.RECORD_AUDIO] != true) { - throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") - } - } - - suspend fun snap(paramsJson: String?): Payload = - withContext(Dispatchers.Main) { - ensureCameraPermission() - val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready") - val facing = parseFacing(paramsJson) ?: "front" - val quality = (parseQuality(paramsJson) ?: 0.9).coerceIn(0.1, 1.0) - val maxWidth = parseMaxWidth(paramsJson) - - val provider = context.cameraProvider() - val capture = ImageCapture.Builder().build() - val selector = - if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA - - provider.unbindAll() - provider.bindToLifecycle(owner, selector, capture) - - val (bytes, orientation) = capture.takeJpegWithExif(context.mainExecutor()) - val decoded = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) - ?: throw IllegalStateException("UNAVAILABLE: failed to decode captured image") - val rotated = rotateBitmapByExif(decoded, orientation) - val scaled = - if (maxWidth != null && maxWidth > 0 && rotated.width > maxWidth) { - val h = - (rotated.height.toDouble() * (maxWidth.toDouble() / rotated.width.toDouble())) - .toInt() - .coerceAtLeast(1) - rotated.scale(maxWidth, h) - } else { - rotated - } - - val maxPayloadBytes = 5 * 1024 * 1024 - // Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit). - val maxEncodedBytes = (maxPayloadBytes / 4) * 3 - val result = - JpegSizeLimiter.compressToLimit( - initialWidth = scaled.width, - initialHeight = scaled.height, - startQuality = (quality * 100.0).roundToInt().coerceIn(10, 100), - maxBytes = maxEncodedBytes, - encode = { width, height, q -> - val bitmap = - if (width == scaled.width && height == scaled.height) { - scaled - } else { - scaled.scale(width, height) - } - val out = ByteArrayOutputStream() - if (!bitmap.compress(Bitmap.CompressFormat.JPEG, q, out)) { - if (bitmap !== scaled) bitmap.recycle() - throw IllegalStateException("UNAVAILABLE: failed to encode JPEG") - } - if (bitmap !== scaled) { - bitmap.recycle() - } - out.toByteArray() - }, - ) - val base64 = Base64.encodeToString(result.bytes, Base64.NO_WRAP) - Payload( - """{"format":"jpg","base64":"$base64","width":${result.width},"height":${result.height}}""", - ) - } - - @SuppressLint("MissingPermission") - suspend fun clip(paramsJson: String?): Payload = - withContext(Dispatchers.Main) { - ensureCameraPermission() - val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready") - val facing = parseFacing(paramsJson) ?: "front" - val durationMs = (parseDurationMs(paramsJson) ?: 3_000).coerceIn(200, 60_000) - val includeAudio = parseIncludeAudio(paramsJson) ?: true - if (includeAudio) ensureMicPermission() - - val provider = context.cameraProvider() - val recorder = Recorder.Builder().build() - val videoCapture = VideoCapture.withOutput(recorder) - val selector = - if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA - - provider.unbindAll() - provider.bindToLifecycle(owner, selector, videoCapture) - - val file = File.createTempFile("moltbot-clip-", ".mp4") - val outputOptions = FileOutputOptions.Builder(file).build() - - val finalized = kotlinx.coroutines.CompletableDeferred() - val recording: Recording = - videoCapture.output - .prepareRecording(context, outputOptions) - .apply { - if (includeAudio) withAudioEnabled() - } - .start(context.mainExecutor()) { event -> - if (event is VideoRecordEvent.Finalize) { - finalized.complete(event) - } - } - - try { - kotlinx.coroutines.delay(durationMs.toLong()) - } finally { - recording.stop() - } - - val finalizeEvent = - try { - withTimeout(10_000) { finalized.await() } - } catch (err: Throwable) { - file.delete() - throw IllegalStateException("UNAVAILABLE: camera clip finalize timed out") - } - if (finalizeEvent.hasError()) { - file.delete() - throw IllegalStateException("UNAVAILABLE: camera clip failed") - } - - val bytes = file.readBytes() - file.delete() - val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP) - Payload( - """{"format":"mp4","base64":"$base64","durationMs":$durationMs,"hasAudio":${includeAudio}}""", - ) - } - - private fun rotateBitmapByExif(bitmap: Bitmap, orientation: Int): Bitmap { - val matrix = Matrix() - when (orientation) { - ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f) - ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f) - ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f) - ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f) - ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f) - ExifInterface.ORIENTATION_TRANSPOSE -> { - matrix.postRotate(90f) - matrix.postScale(-1f, 1f) - } - ExifInterface.ORIENTATION_TRANSVERSE -> { - matrix.postRotate(-90f) - matrix.postScale(-1f, 1f) - } - else -> return bitmap - } - val rotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true) - if (rotated !== bitmap) { - bitmap.recycle() - } - return rotated - } - - private fun parseFacing(paramsJson: String?): String? = - when { - paramsJson?.contains("\"front\"") == true -> "front" - paramsJson?.contains("\"back\"") == true -> "back" - else -> null - } - - private fun parseQuality(paramsJson: String?): Double? = - parseNumber(paramsJson, key = "quality")?.toDoubleOrNull() - - private fun parseMaxWidth(paramsJson: String?): Int? = - parseNumber(paramsJson, key = "maxWidth")?.toIntOrNull() - - private fun parseDurationMs(paramsJson: String?): Int? = - parseNumber(paramsJson, key = "durationMs")?.toIntOrNull() - - private fun parseIncludeAudio(paramsJson: String?): Boolean? { - val raw = paramsJson ?: return null - val key = "\"includeAudio\"" - val idx = raw.indexOf(key) - if (idx < 0) return null - val colon = raw.indexOf(':', idx + key.length) - if (colon < 0) return null - val tail = raw.substring(colon + 1).trimStart() - return when { - tail.startsWith("true") -> true - tail.startsWith("false") -> false - else -> null - } - } - - private fun parseNumber(paramsJson: String?, key: String): String? { - val raw = paramsJson ?: return null - val needle = "\"$key\"" - val idx = raw.indexOf(needle) - if (idx < 0) return null - val colon = raw.indexOf(':', idx + needle.length) - if (colon < 0) return null - val tail = raw.substring(colon + 1).trimStart() - return tail.takeWhile { it.isDigit() || it == '.' } - } - - private fun Context.mainExecutor(): Executor = ContextCompat.getMainExecutor(this) -} - -private suspend fun Context.cameraProvider(): ProcessCameraProvider = - suspendCancellableCoroutine { cont -> - val future = ProcessCameraProvider.getInstance(this) - future.addListener( - { - try { - cont.resume(future.get()) - } catch (e: Exception) { - cont.resumeWithException(e) - } - }, - ContextCompat.getMainExecutor(this), - ) - } - -/** Returns (jpegBytes, exifOrientation) so caller can rotate the decoded bitmap. */ -private suspend fun ImageCapture.takeJpegWithExif(executor: Executor): Pair = - suspendCancellableCoroutine { cont -> - val file = File.createTempFile("moltbot-snap-", ".jpg") - val options = ImageCapture.OutputFileOptions.Builder(file).build() - takePicture( - options, - executor, - object : ImageCapture.OnImageSavedCallback { - override fun onError(exception: ImageCaptureException) { - file.delete() - cont.resumeWithException(exception) - } - - override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) { - try { - val exif = ExifInterface(file.absolutePath) - val orientation = exif.getAttributeInt( - ExifInterface.TAG_ORIENTATION, - ExifInterface.ORIENTATION_NORMAL, - ) - val bytes = file.readBytes() - cont.resume(Pair(bytes, orientation)) - } catch (e: Exception) { - cont.resumeWithException(e) - } finally { - file.delete() - } - } - }, - ) - } diff --git a/apps/android/app/src/main/java/com/clawdbot/android/node/CanvasController.kt b/apps/android/app/src/main/java/com/clawdbot/android/node/CanvasController.kt deleted file mode 100644 index 4c955f7ea..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/node/CanvasController.kt +++ /dev/null @@ -1,264 +0,0 @@ -package com.clawdbot.android.node - -import android.graphics.Bitmap -import android.graphics.Canvas -import android.os.Looper -import android.util.Log -import android.webkit.WebView -import androidx.core.graphics.createBitmap -import androidx.core.graphics.scale -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.suspendCancellableCoroutine -import kotlinx.coroutines.withContext -import java.io.ByteArrayOutputStream -import android.util.Base64 -import org.json.JSONObject -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive -import com.clawdbot.android.BuildConfig -import kotlin.coroutines.resume - -class CanvasController { - enum class SnapshotFormat(val rawValue: String) { - Png("png"), - Jpeg("jpeg"), - } - - @Volatile private var webView: WebView? = null - @Volatile private var url: String? = null - @Volatile private var debugStatusEnabled: Boolean = false - @Volatile private var debugStatusTitle: String? = null - @Volatile private var debugStatusSubtitle: String? = null - - private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html" - - private fun clampJpegQuality(quality: Double?): Int { - val q = (quality ?: 0.82).coerceIn(0.1, 1.0) - return (q * 100.0).toInt().coerceIn(1, 100) - } - - fun attach(webView: WebView) { - this.webView = webView - reload() - applyDebugStatus() - } - - fun navigate(url: String) { - val trimmed = url.trim() - this.url = if (trimmed.isBlank() || trimmed == "/") null else trimmed - reload() - } - - fun currentUrl(): String? = url - - fun isDefaultCanvas(): Boolean = url == null - - fun setDebugStatusEnabled(enabled: Boolean) { - debugStatusEnabled = enabled - applyDebugStatus() - } - - fun setDebugStatus(title: String?, subtitle: String?) { - debugStatusTitle = title - debugStatusSubtitle = subtitle - applyDebugStatus() - } - - fun onPageFinished() { - applyDebugStatus() - } - - private inline fun withWebViewOnMain(crossinline block: (WebView) -> Unit) { - val wv = webView ?: return - if (Looper.myLooper() == Looper.getMainLooper()) { - block(wv) - } else { - wv.post { block(wv) } - } - } - - private fun reload() { - val currentUrl = url - withWebViewOnMain { wv -> - if (currentUrl == null) { - if (BuildConfig.DEBUG) { - Log.d("MoltbotCanvas", "load scaffold: $scaffoldAssetUrl") - } - wv.loadUrl(scaffoldAssetUrl) - } else { - if (BuildConfig.DEBUG) { - Log.d("MoltbotCanvas", "load url: $currentUrl") - } - wv.loadUrl(currentUrl) - } - } - } - - private fun applyDebugStatus() { - val enabled = debugStatusEnabled - val title = debugStatusTitle - val subtitle = debugStatusSubtitle - withWebViewOnMain { wv -> - val titleJs = title?.let { JSONObject.quote(it) } ?: "null" - val subtitleJs = subtitle?.let { JSONObject.quote(it) } ?: "null" - val js = """ - (() => { - try { - const api = globalThis.__moltbot; - if (!api) return; - if (typeof api.setDebugStatusEnabled === 'function') { - api.setDebugStatusEnabled(${if (enabled) "true" else "false"}); - } - if (!${if (enabled) "true" else "false"}) return; - if (typeof api.setStatus === 'function') { - api.setStatus($titleJs, $subtitleJs); - } - } catch (_) {} - })(); - """.trimIndent() - wv.evaluateJavascript(js, null) - } - } - - suspend fun eval(javaScript: String): String = - withContext(Dispatchers.Main) { - val wv = webView ?: throw IllegalStateException("no webview") - suspendCancellableCoroutine { cont -> - wv.evaluateJavascript(javaScript) { result -> - cont.resume(result ?: "") - } - } - } - - suspend fun snapshotPngBase64(maxWidth: Int?): String = - withContext(Dispatchers.Main) { - val wv = webView ?: throw IllegalStateException("no webview") - val bmp = wv.captureBitmap() - val scaled = - if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) { - val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1) - bmp.scale(maxWidth, h) - } else { - bmp - } - - val out = ByteArrayOutputStream() - scaled.compress(Bitmap.CompressFormat.PNG, 100, out) - Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) - } - - suspend fun snapshotBase64(format: SnapshotFormat, quality: Double?, maxWidth: Int?): String = - withContext(Dispatchers.Main) { - val wv = webView ?: throw IllegalStateException("no webview") - val bmp = wv.captureBitmap() - val scaled = - if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) { - val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1) - bmp.scale(maxWidth, h) - } else { - bmp - } - - val out = ByteArrayOutputStream() - val (compressFormat, compressQuality) = - when (format) { - SnapshotFormat.Png -> Bitmap.CompressFormat.PNG to 100 - SnapshotFormat.Jpeg -> Bitmap.CompressFormat.JPEG to clampJpegQuality(quality) - } - scaled.compress(compressFormat, compressQuality, out) - Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP) - } - - private suspend fun WebView.captureBitmap(): Bitmap = - suspendCancellableCoroutine { cont -> - val width = width.coerceAtLeast(1) - val height = height.coerceAtLeast(1) - val bitmap = createBitmap(width, height, Bitmap.Config.ARGB_8888) - - // WebView isn't supported by PixelCopy.request(...) directly; draw() is the most reliable - // cross-version snapshot for this lightweight "canvas" use-case. - draw(Canvas(bitmap)) - cont.resume(bitmap) - } - - companion object { - data class SnapshotParams(val format: SnapshotFormat, val quality: Double?, val maxWidth: Int?) - - fun parseNavigateUrl(paramsJson: String?): String { - val obj = parseParamsObject(paramsJson) ?: return "" - return obj.string("url").trim() - } - - fun parseEvalJs(paramsJson: String?): String? { - val obj = parseParamsObject(paramsJson) ?: return null - val js = obj.string("javaScript").trim() - return js.takeIf { it.isNotBlank() } - } - - fun parseSnapshotMaxWidth(paramsJson: String?): Int? { - val obj = parseParamsObject(paramsJson) ?: return null - if (!obj.containsKey("maxWidth")) return null - val width = obj.int("maxWidth") ?: 0 - return width.takeIf { it > 0 } - } - - fun parseSnapshotFormat(paramsJson: String?): SnapshotFormat { - val obj = parseParamsObject(paramsJson) ?: return SnapshotFormat.Jpeg - val raw = obj.string("format").trim().lowercase() - return when (raw) { - "png" -> SnapshotFormat.Png - "jpeg", "jpg" -> SnapshotFormat.Jpeg - "" -> SnapshotFormat.Jpeg - else -> SnapshotFormat.Jpeg - } - } - - fun parseSnapshotQuality(paramsJson: String?): Double? { - val obj = parseParamsObject(paramsJson) ?: return null - if (!obj.containsKey("quality")) return null - val q = obj.double("quality") ?: Double.NaN - if (!q.isFinite()) return null - return q.coerceIn(0.1, 1.0) - } - - fun parseSnapshotParams(paramsJson: String?): SnapshotParams { - return SnapshotParams( - format = parseSnapshotFormat(paramsJson), - quality = parseSnapshotQuality(paramsJson), - maxWidth = parseSnapshotMaxWidth(paramsJson), - ) - } - - private val json = Json { ignoreUnknownKeys = true } - - private fun parseParamsObject(paramsJson: String?): JsonObject? { - val raw = paramsJson?.trim().orEmpty() - if (raw.isEmpty()) return null - return try { - json.parseToJsonElement(raw).asObjectOrNull() - } catch (_: Throwable) { - null - } - } - - private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject - - private fun JsonObject.string(key: String): String { - val prim = this[key] as? JsonPrimitive ?: return "" - val raw = prim.content - return raw.takeIf { it != "null" }.orEmpty() - } - - private fun JsonObject.int(key: String): Int? { - val prim = this[key] as? JsonPrimitive ?: return null - return prim.content.toIntOrNull() - } - - private fun JsonObject.double(key: String): Double? { - val prim = this[key] as? JsonPrimitive ?: return null - return prim.content.toDoubleOrNull() - } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/node/JpegSizeLimiter.kt b/apps/android/app/src/main/java/com/clawdbot/android/node/JpegSizeLimiter.kt deleted file mode 100644 index ec71e9a4b..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/node/JpegSizeLimiter.kt +++ /dev/null @@ -1,61 +0,0 @@ -package com.clawdbot.android.node - -import kotlin.math.max -import kotlin.math.min -import kotlin.math.roundToInt - -internal data class JpegSizeLimiterResult( - val bytes: ByteArray, - val width: Int, - val height: Int, - val quality: Int, -) - -internal object JpegSizeLimiter { - fun compressToLimit( - initialWidth: Int, - initialHeight: Int, - startQuality: Int, - maxBytes: Int, - minQuality: Int = 20, - minSize: Int = 256, - scaleStep: Double = 0.85, - maxScaleAttempts: Int = 6, - maxQualityAttempts: Int = 6, - encode: (width: Int, height: Int, quality: Int) -> ByteArray, - ): JpegSizeLimiterResult { - require(initialWidth > 0 && initialHeight > 0) { "Invalid image size" } - require(maxBytes > 0) { "Invalid maxBytes" } - - var width = initialWidth - var height = initialHeight - val clampedStartQuality = startQuality.coerceIn(minQuality, 100) - var best = JpegSizeLimiterResult(bytes = encode(width, height, clampedStartQuality), width = width, height = height, quality = clampedStartQuality) - if (best.bytes.size <= maxBytes) return best - - repeat(maxScaleAttempts) { - var quality = clampedStartQuality - repeat(maxQualityAttempts) { - val bytes = encode(width, height, quality) - best = JpegSizeLimiterResult(bytes = bytes, width = width, height = height, quality = quality) - if (bytes.size <= maxBytes) return best - if (quality <= minQuality) return@repeat - quality = max(minQuality, (quality * 0.75).roundToInt()) - } - - val minScale = (minSize.toDouble() / min(width, height).toDouble()).coerceAtMost(1.0) - val nextScale = max(scaleStep, minScale) - val nextWidth = max(minSize, (width * nextScale).roundToInt()) - val nextHeight = max(minSize, (height * nextScale).roundToInt()) - if (nextWidth == width && nextHeight == height) return@repeat - width = min(nextWidth, width) - height = min(nextHeight, height) - } - - if (best.bytes.size > maxBytes) { - throw IllegalStateException("CAMERA_TOO_LARGE: ${best.bytes.size} bytes > $maxBytes bytes") - } - - return best - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/node/LocationCaptureManager.kt b/apps/android/app/src/main/java/com/clawdbot/android/node/LocationCaptureManager.kt deleted file mode 100644 index c701be70d..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/node/LocationCaptureManager.kt +++ /dev/null @@ -1,117 +0,0 @@ -package com.clawdbot.android.node - -import android.Manifest -import android.content.Context -import android.content.pm.PackageManager -import android.location.Location -import android.location.LocationManager -import android.os.CancellationSignal -import androidx.core.content.ContextCompat -import java.time.Instant -import java.time.format.DateTimeFormatter -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import kotlinx.coroutines.withTimeout -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException -import kotlinx.coroutines.suspendCancellableCoroutine - -class LocationCaptureManager(private val context: Context) { - data class Payload(val payloadJson: String) - - suspend fun getLocation( - desiredProviders: List, - maxAgeMs: Long?, - timeoutMs: Long, - isPrecise: Boolean, - ): Payload = - withContext(Dispatchers.Main) { - val manager = context.getSystemService(Context.LOCATION_SERVICE) as LocationManager - if (!manager.isProviderEnabled(LocationManager.GPS_PROVIDER) && - !manager.isProviderEnabled(LocationManager.NETWORK_PROVIDER) - ) { - throw IllegalStateException("LOCATION_UNAVAILABLE: no location providers enabled") - } - - val cached = bestLastKnown(manager, desiredProviders, maxAgeMs) - val location = - cached ?: requestCurrent(manager, desiredProviders, timeoutMs) - - val timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(location.time)) - val source = location.provider - val altitudeMeters = if (location.hasAltitude()) location.altitude else null - val speedMps = if (location.hasSpeed()) location.speed.toDouble() else null - val headingDeg = if (location.hasBearing()) location.bearing.toDouble() else null - Payload( - buildString { - append("{\"lat\":") - append(location.latitude) - append(",\"lon\":") - append(location.longitude) - append(",\"accuracyMeters\":") - append(location.accuracy.toDouble()) - if (altitudeMeters != null) append(",\"altitudeMeters\":").append(altitudeMeters) - if (speedMps != null) append(",\"speedMps\":").append(speedMps) - if (headingDeg != null) append(",\"headingDeg\":").append(headingDeg) - append(",\"timestamp\":\"").append(timestamp).append('"') - append(",\"isPrecise\":").append(isPrecise) - append(",\"source\":\"").append(source).append('"') - append('}') - }, - ) - } - - private fun bestLastKnown( - manager: LocationManager, - providers: List, - maxAgeMs: Long?, - ): Location? { - val fineOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == - PackageManager.PERMISSION_GRANTED - val coarseOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == - PackageManager.PERMISSION_GRANTED - if (!fineOk && !coarseOk) { - throw IllegalStateException("LOCATION_PERMISSION_REQUIRED: grant Location permission") - } - val now = System.currentTimeMillis() - val candidates = - providers.mapNotNull { provider -> manager.getLastKnownLocation(provider) } - val freshest = candidates.maxByOrNull { it.time } ?: return null - if (maxAgeMs != null && now - freshest.time > maxAgeMs) return null - return freshest - } - - private suspend fun requestCurrent( - manager: LocationManager, - providers: List, - timeoutMs: Long, - ): Location { - val fineOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == - PackageManager.PERMISSION_GRANTED - val coarseOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == - PackageManager.PERMISSION_GRANTED - if (!fineOk && !coarseOk) { - throw IllegalStateException("LOCATION_PERMISSION_REQUIRED: grant Location permission") - } - val resolved = - providers.firstOrNull { manager.isProviderEnabled(it) } - ?: throw IllegalStateException("LOCATION_UNAVAILABLE: no providers available") - return withTimeout(timeoutMs.coerceAtLeast(1)) { - suspendCancellableCoroutine { cont -> - val signal = CancellationSignal() - cont.invokeOnCancellation { signal.cancel() } - manager.getCurrentLocation(resolved, signal, context.mainExecutor) { location -> - if (location != null) { - cont.resume(location) - } else { - cont.resumeWithException(IllegalStateException("LOCATION_UNAVAILABLE: no fix")) - } - } - } - } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/node/ScreenRecordManager.kt b/apps/android/app/src/main/java/com/clawdbot/android/node/ScreenRecordManager.kt deleted file mode 100644 index 4486fc5f0..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/node/ScreenRecordManager.kt +++ /dev/null @@ -1,199 +0,0 @@ -package com.clawdbot.android.node - -import android.content.Context -import android.hardware.display.DisplayManager -import android.media.MediaRecorder -import android.media.projection.MediaProjectionManager -import android.os.Build -import android.util.Base64 -import com.clawdbot.android.ScreenCaptureRequester -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.delay -import kotlinx.coroutines.withContext -import java.io.File -import kotlin.math.roundToInt - -class ScreenRecordManager(private val context: Context) { - data class Payload(val payloadJson: String) - - @Volatile private var screenCaptureRequester: ScreenCaptureRequester? = null - @Volatile private var permissionRequester: com.clawdbot.android.PermissionRequester? = null - - fun attachScreenCaptureRequester(requester: ScreenCaptureRequester) { - screenCaptureRequester = requester - } - - fun attachPermissionRequester(requester: com.clawdbot.android.PermissionRequester) { - permissionRequester = requester - } - - suspend fun record(paramsJson: String?): Payload = - withContext(Dispatchers.Default) { - val requester = - screenCaptureRequester - ?: throw IllegalStateException( - "SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission", - ) - - val durationMs = (parseDurationMs(paramsJson) ?: 10_000).coerceIn(250, 60_000) - val fps = (parseFps(paramsJson) ?: 10.0).coerceIn(1.0, 60.0) - val fpsInt = fps.roundToInt().coerceIn(1, 60) - val screenIndex = parseScreenIndex(paramsJson) - val includeAudio = parseIncludeAudio(paramsJson) ?: true - val format = parseString(paramsJson, key = "format") - if (format != null && format.lowercase() != "mp4") { - throw IllegalArgumentException("INVALID_REQUEST: screen format must be mp4") - } - if (screenIndex != null && screenIndex != 0) { - throw IllegalArgumentException("INVALID_REQUEST: screenIndex must be 0 on Android") - } - - val capture = requester.requestCapture() - ?: throw IllegalStateException( - "SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission", - ) - - val mgr = - context.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager - val projection = mgr.getMediaProjection(capture.resultCode, capture.data) - ?: throw IllegalStateException("UNAVAILABLE: screen capture unavailable") - - val metrics = context.resources.displayMetrics - val width = metrics.widthPixels - val height = metrics.heightPixels - val densityDpi = metrics.densityDpi - - val file = File.createTempFile("moltbot-screen-", ".mp4") - if (includeAudio) ensureMicPermission() - - val recorder = createMediaRecorder() - var virtualDisplay: android.hardware.display.VirtualDisplay? = null - try { - if (includeAudio) { - recorder.setAudioSource(MediaRecorder.AudioSource.MIC) - } - recorder.setVideoSource(MediaRecorder.VideoSource.SURFACE) - recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4) - recorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264) - if (includeAudio) { - recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC) - recorder.setAudioChannels(1) - recorder.setAudioSamplingRate(44_100) - recorder.setAudioEncodingBitRate(96_000) - } - recorder.setVideoSize(width, height) - recorder.setVideoFrameRate(fpsInt) - recorder.setVideoEncodingBitRate(estimateBitrate(width, height, fpsInt)) - recorder.setOutputFile(file.absolutePath) - recorder.prepare() - - val surface = recorder.surface - virtualDisplay = - projection.createVirtualDisplay( - "moltbot-screen", - width, - height, - densityDpi, - DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, - surface, - null, - null, - ) - - recorder.start() - delay(durationMs.toLong()) - } finally { - try { - recorder.stop() - } catch (_: Throwable) { - // ignore - } - recorder.reset() - recorder.release() - virtualDisplay?.release() - projection.stop() - } - - val bytes = withContext(Dispatchers.IO) { file.readBytes() } - file.delete() - val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP) - Payload( - """{"format":"mp4","base64":"$base64","durationMs":$durationMs,"fps":$fpsInt,"screenIndex":0,"hasAudio":$includeAudio}""", - ) - } - - private fun createMediaRecorder(): MediaRecorder = MediaRecorder(context) - - private suspend fun ensureMicPermission() { - val granted = - androidx.core.content.ContextCompat.checkSelfPermission( - context, - android.Manifest.permission.RECORD_AUDIO, - ) == android.content.pm.PackageManager.PERMISSION_GRANTED - if (granted) return - - val requester = - permissionRequester - ?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") - val results = requester.requestIfMissing(listOf(android.Manifest.permission.RECORD_AUDIO)) - if (results[android.Manifest.permission.RECORD_AUDIO] != true) { - throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission") - } - } - - private fun parseDurationMs(paramsJson: String?): Int? = - parseNumber(paramsJson, key = "durationMs")?.toIntOrNull() - - private fun parseFps(paramsJson: String?): Double? = - parseNumber(paramsJson, key = "fps")?.toDoubleOrNull() - - private fun parseScreenIndex(paramsJson: String?): Int? = - parseNumber(paramsJson, key = "screenIndex")?.toIntOrNull() - - private fun parseIncludeAudio(paramsJson: String?): Boolean? { - val raw = paramsJson ?: return null - val key = "\"includeAudio\"" - val idx = raw.indexOf(key) - if (idx < 0) return null - val colon = raw.indexOf(':', idx + key.length) - if (colon < 0) return null - val tail = raw.substring(colon + 1).trimStart() - return when { - tail.startsWith("true") -> true - tail.startsWith("false") -> false - else -> null - } - } - - private fun parseNumber(paramsJson: String?, key: String): String? { - val raw = paramsJson ?: return null - val needle = "\"$key\"" - val idx = raw.indexOf(needle) - if (idx < 0) return null - val colon = raw.indexOf(':', idx + needle.length) - if (colon < 0) return null - val tail = raw.substring(colon + 1).trimStart() - return tail.takeWhile { it.isDigit() || it == '.' || it == '-' } - } - - private fun parseString(paramsJson: String?, key: String): String? { - val raw = paramsJson ?: return null - val needle = "\"$key\"" - val idx = raw.indexOf(needle) - if (idx < 0) return null - val colon = raw.indexOf(':', idx + needle.length) - if (colon < 0) return null - val tail = raw.substring(colon + 1).trimStart() - if (!tail.startsWith('\"')) return null - val rest = tail.drop(1) - val end = rest.indexOf('\"') - if (end < 0) return null - return rest.substring(0, end) - } - - private fun estimateBitrate(width: Int, height: Int, fps: Int): Int { - val pixels = width.toLong() * height.toLong() - val raw = (pixels * fps.toLong() * 2L).toInt() - return raw.coerceIn(1_000_000, 12_000_000) - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/node/SmsManager.kt b/apps/android/app/src/main/java/com/clawdbot/android/node/SmsManager.kt deleted file mode 100644 index 3e12a56df..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/node/SmsManager.kt +++ /dev/null @@ -1,230 +0,0 @@ -package com.clawdbot.android.node - -import android.Manifest -import android.content.Context -import android.content.pm.PackageManager -import android.telephony.SmsManager as AndroidSmsManager -import androidx.core.content.ContextCompat -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive -import kotlinx.serialization.json.jsonObject -import kotlinx.serialization.encodeToString -import com.clawdbot.android.PermissionRequester - -/** - * Sends SMS messages via the Android SMS API. - * Requires SEND_SMS permission to be granted. - */ -class SmsManager(private val context: Context) { - - private val json = JsonConfig - @Volatile private var permissionRequester: PermissionRequester? = null - - data class SendResult( - val ok: Boolean, - val to: String, - val message: String?, - val error: String? = null, - val payloadJson: String, - ) - - internal data class ParsedParams( - val to: String, - val message: String, - ) - - internal sealed class ParseResult { - data class Ok(val params: ParsedParams) : ParseResult() - data class Error( - val error: String, - val to: String = "", - val message: String? = null, - ) : ParseResult() - } - - internal data class SendPlan( - val parts: List, - val useMultipart: Boolean, - ) - - companion object { - internal val JsonConfig = Json { ignoreUnknownKeys = true } - - internal fun parseParams(paramsJson: String?, json: Json = JsonConfig): ParseResult { - val params = paramsJson?.trim().orEmpty() - if (params.isEmpty()) { - return ParseResult.Error(error = "INVALID_REQUEST: paramsJSON required") - } - - val obj = try { - json.parseToJsonElement(params).jsonObject - } catch (_: Throwable) { - null - } - - if (obj == null) { - return ParseResult.Error(error = "INVALID_REQUEST: expected JSON object") - } - - val to = (obj["to"] as? JsonPrimitive)?.content?.trim().orEmpty() - val message = (obj["message"] as? JsonPrimitive)?.content.orEmpty() - - if (to.isEmpty()) { - return ParseResult.Error( - error = "INVALID_REQUEST: 'to' phone number required", - message = message, - ) - } - - if (message.isEmpty()) { - return ParseResult.Error( - error = "INVALID_REQUEST: 'message' text required", - to = to, - ) - } - - return ParseResult.Ok(ParsedParams(to = to, message = message)) - } - - internal fun buildSendPlan( - message: String, - divider: (String) -> List, - ): SendPlan { - val parts = divider(message).ifEmpty { listOf(message) } - return SendPlan(parts = parts, useMultipart = parts.size > 1) - } - - internal fun buildPayloadJson( - json: Json = JsonConfig, - ok: Boolean, - to: String, - error: String?, - ): String { - val payload = - mutableMapOf( - "ok" to JsonPrimitive(ok), - "to" to JsonPrimitive(to), - ) - if (!ok) { - payload["error"] = JsonPrimitive(error ?: "SMS_SEND_FAILED") - } - return json.encodeToString(JsonObject.serializer(), JsonObject(payload)) - } - } - - fun hasSmsPermission(): Boolean { - return ContextCompat.checkSelfPermission( - context, - Manifest.permission.SEND_SMS - ) == PackageManager.PERMISSION_GRANTED - } - - fun canSendSms(): Boolean { - return hasSmsPermission() && hasTelephonyFeature() - } - - fun hasTelephonyFeature(): Boolean { - return context.packageManager?.hasSystemFeature(PackageManager.FEATURE_TELEPHONY) == true - } - - fun attachPermissionRequester(requester: PermissionRequester) { - permissionRequester = requester - } - - /** - * Send an SMS message. - * - * @param paramsJson JSON with "to" (phone number) and "message" (text) fields - * @return SendResult indicating success or failure - */ - suspend fun send(paramsJson: String?): SendResult { - if (!hasTelephonyFeature()) { - return errorResult( - error = "SMS_UNAVAILABLE: telephony not available", - ) - } - - if (!ensureSmsPermission()) { - return errorResult( - error = "SMS_PERMISSION_REQUIRED: grant SMS permission", - ) - } - - val parseResult = parseParams(paramsJson, json) - if (parseResult is ParseResult.Error) { - return errorResult( - error = parseResult.error, - to = parseResult.to, - message = parseResult.message, - ) - } - val params = (parseResult as ParseResult.Ok).params - - return try { - val smsManager = context.getSystemService(AndroidSmsManager::class.java) - ?: throw IllegalStateException("SMS_UNAVAILABLE: SmsManager not available") - - val plan = buildSendPlan(params.message) { smsManager.divideMessage(it) } - if (plan.useMultipart) { - smsManager.sendMultipartTextMessage( - params.to, // destination - null, // service center (null = default) - ArrayList(plan.parts), // message parts - null, // sent intents - null, // delivery intents - ) - } else { - smsManager.sendTextMessage( - params.to, // destination - null, // service center (null = default) - params.message,// message - null, // sent intent - null, // delivery intent - ) - } - - okResult(to = params.to, message = params.message) - } catch (e: SecurityException) { - errorResult( - error = "SMS_PERMISSION_REQUIRED: ${e.message}", - to = params.to, - message = params.message, - ) - } catch (e: Throwable) { - errorResult( - error = "SMS_SEND_FAILED: ${e.message ?: "unknown error"}", - to = params.to, - message = params.message, - ) - } - } - - private suspend fun ensureSmsPermission(): Boolean { - if (hasSmsPermission()) return true - val requester = permissionRequester ?: return false - val results = requester.requestIfMissing(listOf(Manifest.permission.SEND_SMS)) - return results[Manifest.permission.SEND_SMS] == true - } - - private fun okResult(to: String, message: String): SendResult { - return SendResult( - ok = true, - to = to, - message = message, - error = null, - payloadJson = buildPayloadJson(json = json, ok = true, to = to, error = null), - ) - } - - private fun errorResult(error: String, to: String = "", message: String? = null): SendResult { - return SendResult( - ok = false, - to = to, - message = message, - error = error, - payloadJson = buildPayloadJson(json = json, ok = false, to = to, error = error), - ) - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/protocol/ClawdbotCanvasA2UIAction.kt b/apps/android/app/src/main/java/com/clawdbot/android/protocol/ClawdbotCanvasA2UIAction.kt deleted file mode 100644 index 4ff1a7421..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/protocol/ClawdbotCanvasA2UIAction.kt +++ /dev/null @@ -1,66 +0,0 @@ -package com.clawdbot.android.protocol - -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive - -object MoltbotCanvasA2UIAction { - fun extractActionName(userAction: JsonObject): String? { - val name = - (userAction["name"] as? JsonPrimitive) - ?.content - ?.trim() - .orEmpty() - if (name.isNotEmpty()) return name - val action = - (userAction["action"] as? JsonPrimitive) - ?.content - ?.trim() - .orEmpty() - return action.ifEmpty { null } - } - - fun sanitizeTagValue(value: String): String { - val trimmed = value.trim().ifEmpty { "-" } - val normalized = trimmed.replace(" ", "_") - val out = StringBuilder(normalized.length) - for (c in normalized) { - val ok = - c.isLetterOrDigit() || - c == '_' || - c == '-' || - c == '.' || - c == ':' - out.append(if (ok) c else '_') - } - return out.toString() - } - - fun formatAgentMessage( - actionName: String, - sessionKey: String, - surfaceId: String, - sourceComponentId: String, - host: String, - instanceId: String, - contextJson: String?, - ): String { - val ctxSuffix = contextJson?.takeIf { it.isNotBlank() }?.let { " ctx=$it" }.orEmpty() - return listOf( - "CANVAS_A2UI", - "action=${sanitizeTagValue(actionName)}", - "session=${sanitizeTagValue(sessionKey)}", - "surface=${sanitizeTagValue(surfaceId)}", - "component=${sanitizeTagValue(sourceComponentId)}", - "host=${sanitizeTagValue(host)}", - "instance=${sanitizeTagValue(instanceId)}$ctxSuffix", - "default=update_canvas", - ).joinToString(separator = " ") - } - - fun jsDispatchA2UIActionStatus(actionId: String, ok: Boolean, error: String?): String { - val err = (error ?: "").replace("\\", "\\\\").replace("\"", "\\\"") - val okLiteral = if (ok) "true" else "false" - val idEscaped = actionId.replace("\\", "\\\\").replace("\"", "\\\"") - return "window.dispatchEvent(new CustomEvent('moltbot:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));" - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/protocol/ClawdbotProtocolConstants.kt b/apps/android/app/src/main/java/com/clawdbot/android/protocol/ClawdbotProtocolConstants.kt deleted file mode 100644 index 09a8bb49d..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/protocol/ClawdbotProtocolConstants.kt +++ /dev/null @@ -1,71 +0,0 @@ -package com.clawdbot.android.protocol - -enum class MoltbotCapability(val rawValue: String) { - Canvas("canvas"), - Camera("camera"), - Screen("screen"), - Sms("sms"), - VoiceWake("voiceWake"), - Location("location"), -} - -enum class MoltbotCanvasCommand(val rawValue: String) { - Present("canvas.present"), - Hide("canvas.hide"), - Navigate("canvas.navigate"), - Eval("canvas.eval"), - Snapshot("canvas.snapshot"), - ; - - companion object { - const val NamespacePrefix: String = "canvas." - } -} - -enum class MoltbotCanvasA2UICommand(val rawValue: String) { - Push("canvas.a2ui.push"), - PushJSONL("canvas.a2ui.pushJSONL"), - Reset("canvas.a2ui.reset"), - ; - - companion object { - const val NamespacePrefix: String = "canvas.a2ui." - } -} - -enum class MoltbotCameraCommand(val rawValue: String) { - Snap("camera.snap"), - Clip("camera.clip"), - ; - - companion object { - const val NamespacePrefix: String = "camera." - } -} - -enum class MoltbotScreenCommand(val rawValue: String) { - Record("screen.record"), - ; - - companion object { - const val NamespacePrefix: String = "screen." - } -} - -enum class MoltbotSmsCommand(val rawValue: String) { - Send("sms.send"), - ; - - companion object { - const val NamespacePrefix: String = "sms." - } -} - -enum class MoltbotLocationCommand(val rawValue: String) { - Get("location.get"), - ; - - companion object { - const val NamespacePrefix: String = "location." - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/tools/ToolDisplay.kt b/apps/android/app/src/main/java/com/clawdbot/android/tools/ToolDisplay.kt deleted file mode 100644 index aed5d0b4b..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/tools/ToolDisplay.kt +++ /dev/null @@ -1,222 +0,0 @@ -package com.clawdbot.android.tools - -import android.content.Context -import kotlinx.serialization.Serializable -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive -import kotlinx.serialization.json.contentOrNull - -@Serializable -private data class ToolDisplayActionSpec( - val label: String? = null, - val detailKeys: List? = null, -) - -@Serializable -private data class ToolDisplaySpec( - val emoji: String? = null, - val title: String? = null, - val label: String? = null, - val detailKeys: List? = null, - val actions: Map? = null, -) - -@Serializable -private data class ToolDisplayConfig( - val version: Int? = null, - val fallback: ToolDisplaySpec? = null, - val tools: Map? = null, -) - -data class ToolDisplaySummary( - val name: String, - val emoji: String, - val title: String, - val label: String, - val verb: String?, - val detail: String?, -) { - val detailLine: String? - get() { - val parts = mutableListOf() - if (!verb.isNullOrBlank()) parts.add(verb) - if (!detail.isNullOrBlank()) parts.add(detail) - return if (parts.isEmpty()) null else parts.joinToString(" · ") - } - - val summaryLine: String - get() = if (detailLine != null) "${emoji} ${label}: ${detailLine}" else "${emoji} ${label}" -} - -object ToolDisplayRegistry { - private const val CONFIG_ASSET = "tool-display.json" - - private val json = Json { ignoreUnknownKeys = true } - @Volatile private var cachedConfig: ToolDisplayConfig? = null - - fun resolve( - context: Context, - name: String?, - args: JsonObject?, - meta: String? = null, - ): ToolDisplaySummary { - val trimmedName = name?.trim().orEmpty().ifEmpty { "tool" } - val key = trimmedName.lowercase() - val config = loadConfig(context) - val spec = config.tools?.get(key) - val fallback = config.fallback - - val emoji = spec?.emoji ?: fallback?.emoji ?: "🧩" - val title = spec?.title ?: titleFromName(trimmedName) - val label = spec?.label ?: trimmedName - - val actionRaw = args?.get("action")?.asStringOrNull()?.trim() - val action = actionRaw?.takeIf { it.isNotEmpty() } - val actionSpec = action?.let { spec?.actions?.get(it) } - val verb = normalizeVerb(actionSpec?.label ?: action) - - var detail: String? = null - if (key == "read") { - detail = readDetail(args) - } else if (key == "write" || key == "edit" || key == "attach") { - detail = pathDetail(args) - } - - val detailKeys = actionSpec?.detailKeys ?: spec?.detailKeys ?: fallback?.detailKeys ?: emptyList() - if (detail == null) { - detail = firstValue(args, detailKeys) - } - - if (detail == null) { - detail = meta - } - - if (detail != null) { - detail = shortenHomeInString(detail) - } - - return ToolDisplaySummary( - name = trimmedName, - emoji = emoji, - title = title, - label = label, - verb = verb, - detail = detail, - ) - } - - private fun loadConfig(context: Context): ToolDisplayConfig { - val existing = cachedConfig - if (existing != null) return existing - return try { - val jsonString = context.assets.open(CONFIG_ASSET).bufferedReader().use { it.readText() } - val decoded = json.decodeFromString(ToolDisplayConfig.serializer(), jsonString) - cachedConfig = decoded - decoded - } catch (_: Throwable) { - val fallback = ToolDisplayConfig() - cachedConfig = fallback - fallback - } - } - - private fun titleFromName(name: String): String { - val cleaned = name.replace("_", " ").trim() - if (cleaned.isEmpty()) return "Tool" - return cleaned - .split(Regex("\\s+")) - .joinToString(" ") { part -> - val upper = part.uppercase() - if (part.length <= 2 && part == upper) part - else upper.firstOrNull()?.toString().orEmpty() + part.lowercase().drop(1) - } - } - - private fun normalizeVerb(value: String?): String? { - val trimmed = value?.trim().orEmpty() - if (trimmed.isEmpty()) return null - return trimmed.replace("_", " ") - } - - private fun readDetail(args: JsonObject?): String? { - val path = args?.get("path")?.asStringOrNull() ?: return null - val offset = args["offset"].asNumberOrNull() - val limit = args["limit"].asNumberOrNull() - return if (offset != null && limit != null) { - val end = offset + limit - "${path}:${offset.toInt()}-${end.toInt()}" - } else { - path - } - } - - private fun pathDetail(args: JsonObject?): String? { - return args?.get("path")?.asStringOrNull() - } - - private fun firstValue(args: JsonObject?, keys: List): String? { - for (key in keys) { - val value = valueForPath(args, key) - val rendered = renderValue(value) - if (!rendered.isNullOrBlank()) return rendered - } - return null - } - - private fun valueForPath(args: JsonObject?, path: String): JsonElement? { - var current: JsonElement? = args - for (segment in path.split(".")) { - if (segment.isBlank()) return null - val obj = current as? JsonObject ?: return null - current = obj[segment] - } - return current - } - - private fun renderValue(value: JsonElement?): String? { - if (value == null) return null - if (value is JsonPrimitive) { - if (value.isString) { - val trimmed = value.contentOrNull?.trim().orEmpty() - if (trimmed.isEmpty()) return null - val firstLine = trimmed.lineSequence().firstOrNull()?.trim().orEmpty() - if (firstLine.isEmpty()) return null - return if (firstLine.length > 160) "${firstLine.take(157)}…" else firstLine - } - val raw = value.contentOrNull?.trim().orEmpty() - raw.toBooleanStrictOrNull()?.let { return it.toString() } - raw.toLongOrNull()?.let { return it.toString() } - raw.toDoubleOrNull()?.let { return it.toString() } - } - if (value is JsonArray) { - val items = value.mapNotNull { renderValue(it) } - if (items.isEmpty()) return null - val preview = items.take(3).joinToString(", ") - return if (items.size > 3) "${preview}…" else preview - } - return null - } - - private fun shortenHomeInString(value: String): String { - val home = System.getProperty("user.home")?.takeIf { it.isNotBlank() } - ?: System.getenv("HOME")?.takeIf { it.isNotBlank() } - if (home.isNullOrEmpty()) return value - return value.replace(home, "~") - .replace(Regex("/Users/[^/]+"), "~") - .replace(Regex("/home/[^/]+"), "~") - } - - private fun JsonElement?.asStringOrNull(): String? { - val primitive = this as? JsonPrimitive ?: return null - return if (primitive.isString) primitive.contentOrNull else primitive.toString() - } - - private fun JsonElement?.asNumberOrNull(): Double? { - val primitive = this as? JsonPrimitive ?: return null - val raw = primitive.contentOrNull ?: return null - return raw.toDoubleOrNull() - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/CameraHudOverlay.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/CameraHudOverlay.kt deleted file mode 100644 index 2143ba7f8..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/CameraHudOverlay.kt +++ /dev/null @@ -1,44 +0,0 @@ -package com.clawdbot.android.ui - -import androidx.compose.foundation.background -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.fillMaxSize -import androidx.compose.runtime.Composable -import androidx.compose.runtime.LaunchedEffect -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableFloatStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.setValue -import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.alpha -import androidx.compose.ui.graphics.Color -import kotlinx.coroutines.delay - -@Composable -fun CameraFlashOverlay( - token: Long, - modifier: Modifier = Modifier, -) { - Box(modifier = modifier.fillMaxSize()) { - CameraFlash(token = token) - } -} - -@Composable -private fun CameraFlash(token: Long) { - var alpha by remember { mutableFloatStateOf(0f) } - LaunchedEffect(token) { - if (token == 0L) return@LaunchedEffect - alpha = 0.85f - delay(110) - alpha = 0f - } - - Box( - modifier = - Modifier - .fillMaxSize() - .alpha(alpha) - .background(Color.White), - ) -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/ChatSheet.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/ChatSheet.kt deleted file mode 100644 index 6f15e5922..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/ChatSheet.kt +++ /dev/null @@ -1,10 +0,0 @@ -package com.clawdbot.android.ui - -import androidx.compose.runtime.Composable -import com.clawdbot.android.MainViewModel -import com.clawdbot.android.ui.chat.ChatSheetContent - -@Composable -fun ChatSheet(viewModel: MainViewModel) { - ChatSheetContent(viewModel = viewModel) -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/ClawdbotTheme.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/ClawdbotTheme.kt deleted file mode 100644 index 01d5a6796..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/ClawdbotTheme.kt +++ /dev/null @@ -1,32 +0,0 @@ -package com.clawdbot.android.ui - -import androidx.compose.foundation.isSystemInDarkTheme -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.dynamicDarkColorScheme -import androidx.compose.material3.dynamicLightColorScheme -import androidx.compose.runtime.Composable -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.platform.LocalContext - -@Composable -fun MoltbotTheme(content: @Composable () -> Unit) { - val context = LocalContext.current - val isDark = isSystemInDarkTheme() - val colorScheme = if (isDark) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context) - - MaterialTheme(colorScheme = colorScheme, content = content) -} - -@Composable -fun overlayContainerColor(): Color { - val scheme = MaterialTheme.colorScheme - val isDark = isSystemInDarkTheme() - val base = if (isDark) scheme.surfaceContainerLow else scheme.surfaceContainerHigh - // Light mode: background stays dark (canvas), so clamp overlays away from pure-white glare. - return if (isDark) base else base.copy(alpha = 0.88f) -} - -@Composable -fun overlayIconColor(): Color { - return MaterialTheme.colorScheme.onSurfaceVariant -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/RootScreen.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/RootScreen.kt deleted file mode 100644 index 763052559..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/RootScreen.kt +++ /dev/null @@ -1,449 +0,0 @@ -package com.clawdbot.android.ui - -import android.annotation.SuppressLint -import android.Manifest -import android.content.pm.PackageManager -import android.graphics.Color -import android.util.Log -import android.view.View -import android.webkit.JavascriptInterface -import android.webkit.ConsoleMessage -import android.webkit.WebChromeClient -import android.webkit.WebView -import android.webkit.WebSettings -import android.webkit.WebResourceError -import android.webkit.WebResourceRequest -import android.webkit.WebResourceResponse -import android.webkit.WebViewClient -import androidx.activity.compose.rememberLauncherForActivityResult -import androidx.activity.result.contract.ActivityResultContracts -import androidx.webkit.WebSettingsCompat -import androidx.webkit.WebViewFeature -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.WindowInsets -import androidx.compose.foundation.layout.WindowInsetsSides -import androidx.compose.foundation.layout.only -import androidx.compose.foundation.layout.fillMaxSize -import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.layout.safeDrawing -import androidx.compose.foundation.layout.size -import androidx.compose.foundation.layout.windowInsetsPadding -import androidx.compose.material3.ExperimentalMaterial3Api -import androidx.compose.material3.FilledTonalIconButton -import androidx.compose.material3.Icon -import androidx.compose.material3.IconButtonDefaults -import androidx.compose.material3.LocalContentColor -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.ModalBottomSheet -import androidx.compose.material3.rememberModalBottomSheetState -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.automirrored.filled.ScreenShare -import androidx.compose.material.icons.filled.ChatBubble -import androidx.compose.material.icons.filled.CheckCircle -import androidx.compose.material.icons.filled.Error -import androidx.compose.material.icons.filled.FiberManualRecord -import androidx.compose.material.icons.filled.PhotoCamera -import androidx.compose.material.icons.filled.RecordVoiceOver -import androidx.compose.material.icons.filled.Refresh -import androidx.compose.material.icons.filled.Report -import androidx.compose.material.icons.filled.Settings -import androidx.compose.runtime.Composable -import androidx.compose.runtime.collectAsState -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.setValue -import androidx.compose.ui.Alignment -import androidx.compose.ui.Modifier -import androidx.compose.ui.graphics.Color as ComposeColor -import androidx.compose.ui.graphics.lerp -import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.unit.dp -import androidx.compose.ui.viewinterop.AndroidView -import androidx.compose.ui.window.Popup -import androidx.compose.ui.window.PopupProperties -import androidx.core.content.ContextCompat -import com.clawdbot.android.CameraHudKind -import com.clawdbot.android.MainViewModel - -@OptIn(ExperimentalMaterial3Api::class) -@Composable -fun RootScreen(viewModel: MainViewModel) { - var sheet by remember { mutableStateOf(null) } - val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true) - val safeOverlayInsets = WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal) - val context = LocalContext.current - val serverName by viewModel.serverName.collectAsState() - val statusText by viewModel.statusText.collectAsState() - val cameraHud by viewModel.cameraHud.collectAsState() - val cameraFlashToken by viewModel.cameraFlashToken.collectAsState() - val screenRecordActive by viewModel.screenRecordActive.collectAsState() - val isForeground by viewModel.isForeground.collectAsState() - val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState() - val talkEnabled by viewModel.talkEnabled.collectAsState() - val talkStatusText by viewModel.talkStatusText.collectAsState() - val talkIsListening by viewModel.talkIsListening.collectAsState() - val talkIsSpeaking by viewModel.talkIsSpeaking.collectAsState() - val seamColorArgb by viewModel.seamColorArgb.collectAsState() - val seamColor = remember(seamColorArgb) { ComposeColor(seamColorArgb) } - val audioPermissionLauncher = - rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted -> - if (granted) viewModel.setTalkEnabled(true) - } - val activity = - remember(cameraHud, screenRecordActive, isForeground, statusText, voiceWakeStatusText) { - // Status pill owns transient activity state so it doesn't overlap the connection indicator. - if (!isForeground) { - return@remember StatusActivity( - title = "Foreground required", - icon = Icons.Default.Report, - contentDescription = "Foreground required", - ) - } - - val lowerStatus = statusText.lowercase() - if (lowerStatus.contains("repair")) { - return@remember StatusActivity( - title = "Repairing…", - icon = Icons.Default.Refresh, - contentDescription = "Repairing", - ) - } - if (lowerStatus.contains("pairing") || lowerStatus.contains("approval")) { - return@remember StatusActivity( - title = "Approval pending", - icon = Icons.Default.RecordVoiceOver, - contentDescription = "Approval pending", - ) - } - // Avoid duplicating the primary gateway status ("Connecting…") in the activity slot. - - if (screenRecordActive) { - return@remember StatusActivity( - title = "Recording screen…", - icon = Icons.AutoMirrored.Filled.ScreenShare, - contentDescription = "Recording screen", - tint = androidx.compose.ui.graphics.Color.Red, - ) - } - - cameraHud?.let { hud -> - return@remember when (hud.kind) { - CameraHudKind.Photo -> - StatusActivity( - title = hud.message, - icon = Icons.Default.PhotoCamera, - contentDescription = "Taking photo", - ) - CameraHudKind.Recording -> - StatusActivity( - title = hud.message, - icon = Icons.Default.FiberManualRecord, - contentDescription = "Recording", - tint = androidx.compose.ui.graphics.Color.Red, - ) - CameraHudKind.Success -> - StatusActivity( - title = hud.message, - icon = Icons.Default.CheckCircle, - contentDescription = "Capture finished", - ) - CameraHudKind.Error -> - StatusActivity( - title = hud.message, - icon = Icons.Default.Error, - contentDescription = "Capture failed", - tint = androidx.compose.ui.graphics.Color.Red, - ) - } - } - - if (voiceWakeStatusText.contains("Microphone permission", ignoreCase = true)) { - return@remember StatusActivity( - title = "Mic permission", - icon = Icons.Default.Error, - contentDescription = "Mic permission required", - ) - } - if (voiceWakeStatusText == "Paused") { - val suffix = if (!isForeground) " (background)" else "" - return@remember StatusActivity( - title = "Voice Wake paused$suffix", - icon = Icons.Default.RecordVoiceOver, - contentDescription = "Voice Wake paused", - ) - } - - null - } - - val gatewayState = - remember(serverName, statusText) { - when { - serverName != null -> GatewayState.Connected - statusText.contains("connecting", ignoreCase = true) || - statusText.contains("reconnecting", ignoreCase = true) -> GatewayState.Connecting - statusText.contains("error", ignoreCase = true) -> GatewayState.Error - else -> GatewayState.Disconnected - } - } - - val voiceEnabled = - ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == - PackageManager.PERMISSION_GRANTED - - Box(modifier = Modifier.fillMaxSize()) { - CanvasView(viewModel = viewModel, modifier = Modifier.fillMaxSize()) - } - - // Camera flash must be in a Popup to render above the WebView. - Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) { - CameraFlashOverlay(token = cameraFlashToken, modifier = Modifier.fillMaxSize()) - } - - // Keep the overlay buttons above the WebView canvas (AndroidView), otherwise they may not receive touches. - Popup(alignment = Alignment.TopStart, properties = PopupProperties(focusable = false)) { - StatusPill( - gateway = gatewayState, - voiceEnabled = voiceEnabled, - activity = activity, - onClick = { sheet = Sheet.Settings }, - modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(start = 12.dp, top = 12.dp), - ) - } - - Popup(alignment = Alignment.TopEnd, properties = PopupProperties(focusable = false)) { - Column( - modifier = Modifier.windowInsetsPadding(safeOverlayInsets).padding(end = 12.dp, top = 12.dp), - verticalArrangement = Arrangement.spacedBy(10.dp), - horizontalAlignment = Alignment.End, - ) { - OverlayIconButton( - onClick = { sheet = Sheet.Chat }, - icon = { Icon(Icons.Default.ChatBubble, contentDescription = "Chat") }, - ) - - // Talk mode gets a dedicated side bubble instead of burying it in settings. - val baseOverlay = overlayContainerColor() - val talkContainer = - lerp( - baseOverlay, - seamColor.copy(alpha = baseOverlay.alpha), - if (talkEnabled) 0.35f else 0.22f, - ) - val talkContent = if (talkEnabled) seamColor else overlayIconColor() - OverlayIconButton( - onClick = { - val next = !talkEnabled - if (next) { - val micOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == - PackageManager.PERMISSION_GRANTED - if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) - viewModel.setTalkEnabled(true) - } else { - viewModel.setTalkEnabled(false) - } - }, - containerColor = talkContainer, - contentColor = talkContent, - icon = { - Icon( - Icons.Default.RecordVoiceOver, - contentDescription = "Talk Mode", - ) - }, - ) - - OverlayIconButton( - onClick = { sheet = Sheet.Settings }, - icon = { Icon(Icons.Default.Settings, contentDescription = "Settings") }, - ) - } - } - - if (talkEnabled) { - Popup(alignment = Alignment.Center, properties = PopupProperties(focusable = false)) { - TalkOrbOverlay( - seamColor = seamColor, - statusText = talkStatusText, - isListening = talkIsListening, - isSpeaking = talkIsSpeaking, - ) - } - } - - val currentSheet = sheet - if (currentSheet != null) { - ModalBottomSheet( - onDismissRequest = { sheet = null }, - sheetState = sheetState, - ) { - when (currentSheet) { - Sheet.Chat -> ChatSheet(viewModel = viewModel) - Sheet.Settings -> SettingsSheet(viewModel = viewModel) - } - } - } -} - -private enum class Sheet { - Chat, - Settings, -} - -@Composable -private fun OverlayIconButton( - onClick: () -> Unit, - icon: @Composable () -> Unit, - containerColor: ComposeColor? = null, - contentColor: ComposeColor? = null, -) { - FilledTonalIconButton( - onClick = onClick, - modifier = Modifier.size(44.dp), - colors = - IconButtonDefaults.filledTonalIconButtonColors( - containerColor = containerColor ?: overlayContainerColor(), - contentColor = contentColor ?: overlayIconColor(), - ), - ) { - icon() - } -} - -@SuppressLint("SetJavaScriptEnabled") -@Composable -private fun CanvasView(viewModel: MainViewModel, modifier: Modifier = Modifier) { - val context = LocalContext.current - val isDebuggable = (context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0 - AndroidView( - modifier = modifier, - factory = { - WebView(context).apply { - settings.javaScriptEnabled = true - // Some embedded web UIs (incl. the "background website") use localStorage/sessionStorage. - settings.domStorageEnabled = true - settings.mixedContentMode = WebSettings.MIXED_CONTENT_COMPATIBILITY_MODE - if (WebViewFeature.isFeatureSupported(WebViewFeature.ALGORITHMIC_DARKENING)) { - WebSettingsCompat.setAlgorithmicDarkeningAllowed(settings, false) - } else { - disableForceDarkIfSupported(settings) - } - if (isDebuggable) { - Log.d("MoltbotWebView", "userAgent: ${settings.userAgentString}") - } - isScrollContainer = true - overScrollMode = View.OVER_SCROLL_IF_CONTENT_SCROLLS - isVerticalScrollBarEnabled = true - isHorizontalScrollBarEnabled = true - webViewClient = - object : WebViewClient() { - override fun onReceivedError( - view: WebView, - request: WebResourceRequest, - error: WebResourceError, - ) { - if (!isDebuggable) return - if (!request.isForMainFrame) return - Log.e("MoltbotWebView", "onReceivedError: ${error.errorCode} ${error.description} ${request.url}") - } - - override fun onReceivedHttpError( - view: WebView, - request: WebResourceRequest, - errorResponse: WebResourceResponse, - ) { - if (!isDebuggable) return - if (!request.isForMainFrame) return - Log.e( - "MoltbotWebView", - "onReceivedHttpError: ${errorResponse.statusCode} ${errorResponse.reasonPhrase} ${request.url}", - ) - } - - override fun onPageFinished(view: WebView, url: String?) { - if (isDebuggable) { - Log.d("MoltbotWebView", "onPageFinished: $url") - } - viewModel.canvas.onPageFinished() - } - - override fun onRenderProcessGone( - view: WebView, - detail: android.webkit.RenderProcessGoneDetail, - ): Boolean { - if (isDebuggable) { - Log.e( - "MoltbotWebView", - "onRenderProcessGone didCrash=${detail.didCrash()} priorityAtExit=${detail.rendererPriorityAtExit()}", - ) - } - return true - } - } - webChromeClient = - object : WebChromeClient() { - override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean { - if (!isDebuggable) return false - val msg = consoleMessage ?: return false - Log.d( - "MoltbotWebView", - "console ${msg.messageLevel()} @ ${msg.sourceId()}:${msg.lineNumber()} ${msg.message()}", - ) - return false - } - } - // Use default layer/background; avoid forcing a black fill over WebView content. - - val a2uiBridge = - CanvasA2UIActionBridge { payload -> - viewModel.handleCanvasA2UIActionFromWebView(payload) - } - addJavascriptInterface(a2uiBridge, CanvasA2UIActionBridge.interfaceName) - addJavascriptInterface( - CanvasA2UIActionLegacyBridge(a2uiBridge), - CanvasA2UIActionLegacyBridge.interfaceName, - ) - viewModel.canvas.attach(this) - } - }, - ) -} - -private fun disableForceDarkIfSupported(settings: WebSettings) { - if (!WebViewFeature.isFeatureSupported(WebViewFeature.FORCE_DARK)) return - @Suppress("DEPRECATION") - WebSettingsCompat.setForceDark(settings, WebSettingsCompat.FORCE_DARK_OFF) -} - -private class CanvasA2UIActionBridge(private val onMessage: (String) -> Unit) { - @JavascriptInterface - fun postMessage(payload: String?) { - val msg = payload?.trim().orEmpty() - if (msg.isEmpty()) return - onMessage(msg) - } - - companion object { - const val interfaceName: String = "moltbotCanvasA2UIAction" - } -} - -private class CanvasA2UIActionLegacyBridge(private val bridge: CanvasA2UIActionBridge) { - @JavascriptInterface - fun canvasAction(payload: String?) { - bridge.postMessage(payload) - } - - @JavascriptInterface - fun postMessage(payload: String?) { - bridge.postMessage(payload) - } - - companion object { - const val interfaceName: String = "Android" - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/SettingsSheet.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/SettingsSheet.kt deleted file mode 100644 index 6b3564e14..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/SettingsSheet.kt +++ /dev/null @@ -1,686 +0,0 @@ -package com.clawdbot.android.ui - -import android.Manifest -import android.content.Context -import android.content.Intent -import android.content.pm.PackageManager -import android.net.Uri -import android.os.Build -import android.provider.Settings -import androidx.activity.compose.rememberLauncherForActivityResult -import androidx.activity.result.contract.ActivityResultContracts -import androidx.compose.animation.AnimatedVisibility -import androidx.compose.foundation.clickable -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.PaddingValues -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.Spacer -import androidx.compose.foundation.layout.WindowInsets -import androidx.compose.foundation.layout.WindowInsetsSides -import androidx.compose.foundation.layout.fillMaxHeight -import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.layout.height -import androidx.compose.foundation.layout.imePadding -import androidx.compose.foundation.layout.only -import androidx.compose.foundation.layout.safeDrawing -import androidx.compose.foundation.layout.windowInsetsPadding -import androidx.compose.foundation.lazy.LazyColumn -import androidx.compose.foundation.lazy.items -import androidx.compose.foundation.lazy.rememberLazyListState -import androidx.compose.foundation.text.KeyboardActions -import androidx.compose.foundation.text.KeyboardOptions -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.ExpandLess -import androidx.compose.material.icons.filled.ExpandMore -import androidx.compose.material3.Button -import androidx.compose.material3.HorizontalDivider -import androidx.compose.material3.Icon -import androidx.compose.material3.ListItem -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.OutlinedTextField -import androidx.compose.material3.RadioButton -import androidx.compose.material3.Switch -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.runtime.LaunchedEffect -import androidx.compose.runtime.collectAsState -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.setValue -import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.alpha -import androidx.compose.ui.focus.onFocusChanged -import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.platform.LocalFocusManager -import androidx.compose.ui.text.input.ImeAction -import androidx.compose.ui.text.style.TextAlign -import androidx.compose.ui.unit.dp -import androidx.core.content.ContextCompat -import com.clawdbot.android.BuildConfig -import com.clawdbot.android.LocationMode -import com.clawdbot.android.MainViewModel -import com.clawdbot.android.NodeForegroundService -import com.clawdbot.android.VoiceWakeMode -import com.clawdbot.android.WakeWords - -@Composable -fun SettingsSheet(viewModel: MainViewModel) { - val context = LocalContext.current - val instanceId by viewModel.instanceId.collectAsState() - val displayName by viewModel.displayName.collectAsState() - val cameraEnabled by viewModel.cameraEnabled.collectAsState() - val locationMode by viewModel.locationMode.collectAsState() - val locationPreciseEnabled by viewModel.locationPreciseEnabled.collectAsState() - val preventSleep by viewModel.preventSleep.collectAsState() - val wakeWords by viewModel.wakeWords.collectAsState() - val voiceWakeMode by viewModel.voiceWakeMode.collectAsState() - val voiceWakeStatusText by viewModel.voiceWakeStatusText.collectAsState() - val isConnected by viewModel.isConnected.collectAsState() - val manualEnabled by viewModel.manualEnabled.collectAsState() - val manualHost by viewModel.manualHost.collectAsState() - val manualPort by viewModel.manualPort.collectAsState() - val manualTls by viewModel.manualTls.collectAsState() - val canvasDebugStatusEnabled by viewModel.canvasDebugStatusEnabled.collectAsState() - val statusText by viewModel.statusText.collectAsState() - val serverName by viewModel.serverName.collectAsState() - val remoteAddress by viewModel.remoteAddress.collectAsState() - val gateways by viewModel.gateways.collectAsState() - val discoveryStatusText by viewModel.discoveryStatusText.collectAsState() - - val listState = rememberLazyListState() - val (wakeWordsText, setWakeWordsText) = remember { mutableStateOf("") } - val (advancedExpanded, setAdvancedExpanded) = remember { mutableStateOf(false) } - val focusManager = LocalFocusManager.current - var wakeWordsHadFocus by remember { mutableStateOf(false) } - val deviceModel = - remember { - listOfNotNull(Build.MANUFACTURER, Build.MODEL) - .joinToString(" ") - .trim() - .ifEmpty { "Android" } - } - val appVersion = - remember { - val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" } - if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) { - "$versionName-dev" - } else { - versionName - } - } - - LaunchedEffect(wakeWords) { setWakeWordsText(wakeWords.joinToString(", ")) } - val commitWakeWords = { - val parsed = WakeWords.parseIfChanged(wakeWordsText, wakeWords) - if (parsed != null) { - viewModel.setWakeWords(parsed) - } - } - - val permissionLauncher = - rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms -> - val cameraOk = perms[Manifest.permission.CAMERA] == true - viewModel.setCameraEnabled(cameraOk) - } - - var pendingLocationMode by remember { mutableStateOf(null) } - var pendingPreciseToggle by remember { mutableStateOf(false) } - - val locationPermissionLauncher = - rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms -> - val fineOk = perms[Manifest.permission.ACCESS_FINE_LOCATION] == true - val coarseOk = perms[Manifest.permission.ACCESS_COARSE_LOCATION] == true - val granted = fineOk || coarseOk - val requestedMode = pendingLocationMode - pendingLocationMode = null - - if (pendingPreciseToggle) { - pendingPreciseToggle = false - viewModel.setLocationPreciseEnabled(fineOk) - return@rememberLauncherForActivityResult - } - - if (!granted) { - viewModel.setLocationMode(LocationMode.Off) - return@rememberLauncherForActivityResult - } - - if (requestedMode != null) { - viewModel.setLocationMode(requestedMode) - if (requestedMode == LocationMode.Always) { - val backgroundOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_BACKGROUND_LOCATION) == - PackageManager.PERMISSION_GRANTED - if (!backgroundOk) { - openAppSettings(context) - } - } - } - } - - val audioPermissionLauncher = - rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { _ -> - // Status text is handled by NodeRuntime. - } - - val smsPermissionAvailable = - remember { - context.packageManager?.hasSystemFeature(PackageManager.FEATURE_TELEPHONY) == true - } - var smsPermissionGranted by - remember { - mutableStateOf( - ContextCompat.checkSelfPermission(context, Manifest.permission.SEND_SMS) == - PackageManager.PERMISSION_GRANTED, - ) - } - val smsPermissionLauncher = - rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted -> - smsPermissionGranted = granted - viewModel.refreshGatewayConnection() - } - - fun setCameraEnabledChecked(checked: Boolean) { - if (!checked) { - viewModel.setCameraEnabled(false) - return - } - - val cameraOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == - PackageManager.PERMISSION_GRANTED - if (cameraOk) { - viewModel.setCameraEnabled(true) - } else { - permissionLauncher.launch(arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO)) - } - } - - fun requestLocationPermissions(targetMode: LocationMode) { - val fineOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == - PackageManager.PERMISSION_GRANTED - val coarseOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) == - PackageManager.PERMISSION_GRANTED - if (fineOk || coarseOk) { - viewModel.setLocationMode(targetMode) - if (targetMode == LocationMode.Always) { - val backgroundOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_BACKGROUND_LOCATION) == - PackageManager.PERMISSION_GRANTED - if (!backgroundOk) { - openAppSettings(context) - } - } - } else { - pendingLocationMode = targetMode - locationPermissionLauncher.launch( - arrayOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION), - ) - } - } - - fun setPreciseLocationChecked(checked: Boolean) { - if (!checked) { - viewModel.setLocationPreciseEnabled(false) - return - } - val fineOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) == - PackageManager.PERMISSION_GRANTED - if (fineOk) { - viewModel.setLocationPreciseEnabled(true) - } else { - pendingPreciseToggle = true - locationPermissionLauncher.launch(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION)) - } - } - - val visibleGateways = - if (isConnected && remoteAddress != null) { - gateways.filterNot { "${it.host}:${it.port}" == remoteAddress } - } else { - gateways - } - - val gatewayDiscoveryFooterText = - if (visibleGateways.isEmpty()) { - discoveryStatusText - } else if (isConnected) { - "Discovery active • ${visibleGateways.size} other gateway${if (visibleGateways.size == 1) "" else "s"} found" - } else { - "Discovery active • ${visibleGateways.size} gateway${if (visibleGateways.size == 1) "" else "s"} found" - } - - LazyColumn( - state = listState, - modifier = - Modifier - .fillMaxWidth() - .fillMaxHeight() - .imePadding() - .windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Bottom)), - contentPadding = PaddingValues(16.dp), - verticalArrangement = Arrangement.spacedBy(6.dp), - ) { - // Order parity: Node → Gateway → Voice → Camera → Messaging → Location → Screen. - item { Text("Node", style = MaterialTheme.typography.titleSmall) } - item { - OutlinedTextField( - value = displayName, - onValueChange = viewModel::setDisplayName, - label = { Text("Name") }, - modifier = Modifier.fillMaxWidth(), - ) - } - item { Text("Instance ID: $instanceId", color = MaterialTheme.colorScheme.onSurfaceVariant) } - item { Text("Device: $deviceModel", color = MaterialTheme.colorScheme.onSurfaceVariant) } - item { Text("Version: $appVersion", color = MaterialTheme.colorScheme.onSurfaceVariant) } - - item { HorizontalDivider() } - - // Gateway - item { Text("Gateway", style = MaterialTheme.typography.titleSmall) } - item { ListItem(headlineContent = { Text("Status") }, supportingContent = { Text(statusText) }) } - if (serverName != null) { - item { ListItem(headlineContent = { Text("Server") }, supportingContent = { Text(serverName!!) }) } - } - if (remoteAddress != null) { - item { ListItem(headlineContent = { Text("Address") }, supportingContent = { Text(remoteAddress!!) }) } - } - item { - // UI sanity: "Disconnect" only when we have an active remote. - if (isConnected && remoteAddress != null) { - Button( - onClick = { - viewModel.disconnect() - NodeForegroundService.stop(context) - }, - ) { - Text("Disconnect") - } - } - } - - item { HorizontalDivider() } - - if (!isConnected || visibleGateways.isNotEmpty()) { - item { - Text( - if (isConnected) "Other Gateways" else "Discovered Gateways", - style = MaterialTheme.typography.titleSmall, - ) - } - if (!isConnected && visibleGateways.isEmpty()) { - item { Text("No gateways found yet.", color = MaterialTheme.colorScheme.onSurfaceVariant) } - } else { - items(items = visibleGateways, key = { it.stableId }) { gateway -> - val detailLines = - buildList { - add("IP: ${gateway.host}:${gateway.port}") - gateway.lanHost?.let { add("LAN: $it") } - gateway.tailnetDns?.let { add("Tailnet: $it") } - if (gateway.gatewayPort != null || gateway.canvasPort != null) { - val gw = (gateway.gatewayPort ?: gateway.port).toString() - val canvas = gateway.canvasPort?.toString() ?: "—" - add("Ports: gw $gw · canvas $canvas") - } - } - ListItem( - headlineContent = { Text(gateway.name) }, - supportingContent = { - Column(verticalArrangement = Arrangement.spacedBy(2.dp)) { - detailLines.forEach { line -> - Text(line, color = MaterialTheme.colorScheme.onSurfaceVariant) - } - } - }, - trailingContent = { - Button( - onClick = { - NodeForegroundService.start(context) - viewModel.connect(gateway) - }, - ) { - Text("Connect") - } - }, - ) - } - } - item { - Text( - gatewayDiscoveryFooterText, - modifier = Modifier.fillMaxWidth(), - textAlign = TextAlign.Center, - style = MaterialTheme.typography.labelMedium, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } - } - - item { HorizontalDivider() } - - item { - ListItem( - headlineContent = { Text("Advanced") }, - supportingContent = { Text("Manual gateway connection") }, - trailingContent = { - Icon( - imageVector = if (advancedExpanded) Icons.Filled.ExpandLess else Icons.Filled.ExpandMore, - contentDescription = if (advancedExpanded) "Collapse" else "Expand", - ) - }, - modifier = - Modifier.clickable { - setAdvancedExpanded(!advancedExpanded) - }, - ) - } - item { - AnimatedVisibility(visible = advancedExpanded) { - Column(verticalArrangement = Arrangement.spacedBy(10.dp), modifier = Modifier.fillMaxWidth()) { - ListItem( - headlineContent = { Text("Use Manual Gateway") }, - supportingContent = { Text("Use this when discovery is blocked.") }, - trailingContent = { Switch(checked = manualEnabled, onCheckedChange = viewModel::setManualEnabled) }, - ) - - OutlinedTextField( - value = manualHost, - onValueChange = viewModel::setManualHost, - label = { Text("Host") }, - modifier = Modifier.fillMaxWidth(), - enabled = manualEnabled, - ) - OutlinedTextField( - value = manualPort.toString(), - onValueChange = { v -> viewModel.setManualPort(v.toIntOrNull() ?: 0) }, - label = { Text("Port") }, - modifier = Modifier.fillMaxWidth(), - enabled = manualEnabled, - ) - ListItem( - headlineContent = { Text("Require TLS") }, - supportingContent = { Text("Pin the gateway certificate on first connect.") }, - trailingContent = { Switch(checked = manualTls, onCheckedChange = viewModel::setManualTls, enabled = manualEnabled) }, - modifier = Modifier.alpha(if (manualEnabled) 1f else 0.5f), - ) - - val hostOk = manualHost.trim().isNotEmpty() - val portOk = manualPort in 1..65535 - Button( - onClick = { - NodeForegroundService.start(context) - viewModel.connectManual() - }, - enabled = manualEnabled && hostOk && portOk, - ) { - Text("Connect (Manual)") - } - } - } - } - - item { HorizontalDivider() } - - // Voice - item { Text("Voice", style = MaterialTheme.typography.titleSmall) } - item { - val enabled = voiceWakeMode != VoiceWakeMode.Off - ListItem( - headlineContent = { Text("Voice Wake") }, - supportingContent = { Text(voiceWakeStatusText) }, - trailingContent = { - Switch( - checked = enabled, - onCheckedChange = { on -> - if (on) { - val micOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == - PackageManager.PERMISSION_GRANTED - if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) - viewModel.setVoiceWakeMode(VoiceWakeMode.Foreground) - } else { - viewModel.setVoiceWakeMode(VoiceWakeMode.Off) - } - }, - ) - }, - ) - } - item { - AnimatedVisibility(visible = voiceWakeMode != VoiceWakeMode.Off) { - Column(verticalArrangement = Arrangement.spacedBy(6.dp), modifier = Modifier.fillMaxWidth()) { - ListItem( - headlineContent = { Text("Foreground Only") }, - supportingContent = { Text("Listens only while Moltbot is open.") }, - trailingContent = { - RadioButton( - selected = voiceWakeMode == VoiceWakeMode.Foreground, - onClick = { - val micOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == - PackageManager.PERMISSION_GRANTED - if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) - viewModel.setVoiceWakeMode(VoiceWakeMode.Foreground) - }, - ) - }, - ) - ListItem( - headlineContent = { Text("Always") }, - supportingContent = { Text("Keeps listening in the background (shows a persistent notification).") }, - trailingContent = { - RadioButton( - selected = voiceWakeMode == VoiceWakeMode.Always, - onClick = { - val micOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == - PackageManager.PERMISSION_GRANTED - if (!micOk) audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) - viewModel.setVoiceWakeMode(VoiceWakeMode.Always) - }, - ) - }, - ) - } - } - } - item { - OutlinedTextField( - value = wakeWordsText, - onValueChange = setWakeWordsText, - label = { Text("Wake Words (comma-separated)") }, - modifier = - Modifier.fillMaxWidth().onFocusChanged { focusState -> - if (focusState.isFocused) { - wakeWordsHadFocus = true - } else if (wakeWordsHadFocus) { - wakeWordsHadFocus = false - commitWakeWords() - } - }, - singleLine = true, - keyboardOptions = KeyboardOptions(imeAction = ImeAction.Done), - keyboardActions = - KeyboardActions( - onDone = { - commitWakeWords() - focusManager.clearFocus() - }, - ), - ) - } - item { Button(onClick = viewModel::resetWakeWordsDefaults) { Text("Reset defaults") } } - item { - Text( - if (isConnected) { - "Any node can edit wake words. Changes sync via the gateway." - } else { - "Connect to a gateway to sync wake words globally." - }, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } - - item { HorizontalDivider() } - - // Camera - item { Text("Camera", style = MaterialTheme.typography.titleSmall) } - item { - ListItem( - headlineContent = { Text("Allow Camera") }, - supportingContent = { Text("Allows the gateway to request photos or short video clips (foreground only).") }, - trailingContent = { Switch(checked = cameraEnabled, onCheckedChange = ::setCameraEnabledChecked) }, - ) - } - item { - Text( - "Tip: grant Microphone permission for video clips with audio.", - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } - - item { HorizontalDivider() } - - // Messaging - item { Text("Messaging", style = MaterialTheme.typography.titleSmall) } - item { - val buttonLabel = - when { - !smsPermissionAvailable -> "Unavailable" - smsPermissionGranted -> "Manage" - else -> "Grant" - } - ListItem( - headlineContent = { Text("SMS Permission") }, - supportingContent = { - Text( - if (smsPermissionAvailable) { - "Allow the gateway to send SMS from this device." - } else { - "SMS requires a device with telephony hardware." - }, - ) - }, - trailingContent = { - Button( - onClick = { - if (!smsPermissionAvailable) return@Button - if (smsPermissionGranted) { - openAppSettings(context) - } else { - smsPermissionLauncher.launch(Manifest.permission.SEND_SMS) - } - }, - enabled = smsPermissionAvailable, - ) { - Text(buttonLabel) - } - }, - ) - } - - item { HorizontalDivider() } - - // Location - item { Text("Location", style = MaterialTheme.typography.titleSmall) } - item { - Column(verticalArrangement = Arrangement.spacedBy(6.dp), modifier = Modifier.fillMaxWidth()) { - ListItem( - headlineContent = { Text("Off") }, - supportingContent = { Text("Disable location sharing.") }, - trailingContent = { - RadioButton( - selected = locationMode == LocationMode.Off, - onClick = { viewModel.setLocationMode(LocationMode.Off) }, - ) - }, - ) - ListItem( - headlineContent = { Text("While Using") }, - supportingContent = { Text("Only while Moltbot is open.") }, - trailingContent = { - RadioButton( - selected = locationMode == LocationMode.WhileUsing, - onClick = { requestLocationPermissions(LocationMode.WhileUsing) }, - ) - }, - ) - ListItem( - headlineContent = { Text("Always") }, - supportingContent = { Text("Allow background location (requires system permission).") }, - trailingContent = { - RadioButton( - selected = locationMode == LocationMode.Always, - onClick = { requestLocationPermissions(LocationMode.Always) }, - ) - }, - ) - } - } - item { - ListItem( - headlineContent = { Text("Precise Location") }, - supportingContent = { Text("Use precise GPS when available.") }, - trailingContent = { - Switch( - checked = locationPreciseEnabled, - onCheckedChange = ::setPreciseLocationChecked, - enabled = locationMode != LocationMode.Off, - ) - }, - ) - } - item { - Text( - "Always may require Android Settings to allow background location.", - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } - - item { HorizontalDivider() } - - // Screen - item { Text("Screen", style = MaterialTheme.typography.titleSmall) } - item { - ListItem( - headlineContent = { Text("Prevent Sleep") }, - supportingContent = { Text("Keeps the screen awake while Moltbot is open.") }, - trailingContent = { Switch(checked = preventSleep, onCheckedChange = viewModel::setPreventSleep) }, - ) - } - - item { HorizontalDivider() } - - // Debug - item { Text("Debug", style = MaterialTheme.typography.titleSmall) } - item { - ListItem( - headlineContent = { Text("Debug Canvas Status") }, - supportingContent = { Text("Show status text in the canvas when debug is enabled.") }, - trailingContent = { - Switch( - checked = canvasDebugStatusEnabled, - onCheckedChange = viewModel::setCanvasDebugStatusEnabled, - ) - }, - ) - } - - item { Spacer(modifier = Modifier.height(20.dp)) } - } -} - -private fun openAppSettings(context: Context) { - val intent = - Intent( - Settings.ACTION_APPLICATION_DETAILS_SETTINGS, - Uri.fromParts("package", context.packageName, null), - ) - context.startActivity(intent) -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/StatusPill.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/StatusPill.kt deleted file mode 100644 index 564d96b52..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/StatusPill.kt +++ /dev/null @@ -1,114 +0,0 @@ -package com.clawdbot.android.ui - -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.Spacer -import androidx.compose.foundation.layout.height -import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.layout.size -import androidx.compose.foundation.layout.width -import androidx.compose.foundation.shape.CircleShape -import androidx.compose.foundation.shape.RoundedCornerShape -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.Mic -import androidx.compose.material.icons.filled.MicOff -import androidx.compose.material3.Icon -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.Surface -import androidx.compose.material3.Text -import androidx.compose.material3.VerticalDivider -import androidx.compose.runtime.Composable -import androidx.compose.ui.Alignment -import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.alpha -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.unit.dp - -@Composable -fun StatusPill( - gateway: GatewayState, - voiceEnabled: Boolean, - onClick: () -> Unit, - modifier: Modifier = Modifier, - activity: StatusActivity? = null, -) { - Surface( - onClick = onClick, - modifier = modifier, - shape = RoundedCornerShape(14.dp), - color = overlayContainerColor(), - tonalElevation = 3.dp, - shadowElevation = 0.dp, - ) { - Row( - modifier = Modifier.padding(horizontal = 12.dp, vertical = 8.dp), - horizontalArrangement = Arrangement.spacedBy(10.dp), - verticalAlignment = Alignment.CenterVertically, - ) { - Row(horizontalArrangement = Arrangement.spacedBy(8.dp), verticalAlignment = Alignment.CenterVertically) { - Surface( - modifier = Modifier.size(9.dp), - shape = CircleShape, - color = gateway.color, - ) {} - - Text( - text = gateway.title, - style = MaterialTheme.typography.labelLarge, - ) - } - - VerticalDivider( - modifier = Modifier.height(14.dp).alpha(0.35f), - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - - if (activity != null) { - Row( - horizontalArrangement = Arrangement.spacedBy(6.dp), - verticalAlignment = Alignment.CenterVertically, - ) { - Icon( - imageVector = activity.icon, - contentDescription = activity.contentDescription, - tint = activity.tint ?: overlayIconColor(), - modifier = Modifier.size(18.dp), - ) - Text( - text = activity.title, - style = MaterialTheme.typography.labelLarge, - maxLines = 1, - ) - } - } else { - Icon( - imageVector = if (voiceEnabled) Icons.Default.Mic else Icons.Default.MicOff, - contentDescription = if (voiceEnabled) "Voice enabled" else "Voice disabled", - tint = - if (voiceEnabled) { - overlayIconColor() - } else { - MaterialTheme.colorScheme.onSurfaceVariant - }, - modifier = Modifier.size(18.dp), - ) - } - - Spacer(modifier = Modifier.width(2.dp)) - } - } -} - -data class StatusActivity( - val title: String, - val icon: androidx.compose.ui.graphics.vector.ImageVector, - val contentDescription: String, - val tint: Color? = null, -) - -enum class GatewayState(val title: String, val color: Color) { - Connected("Connected", Color(0xFF2ECC71)), - Connecting("Connecting…", Color(0xFFF1C40F)), - Error("Error", Color(0xFFE74C3C)), - Disconnected("Offline", Color(0xFF9E9E9E)), -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/TalkOrbOverlay.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/TalkOrbOverlay.kt deleted file mode 100644 index 32225b486..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/TalkOrbOverlay.kt +++ /dev/null @@ -1,134 +0,0 @@ -package com.clawdbot.android.ui - -import androidx.compose.animation.core.LinearEasing -import androidx.compose.animation.core.RepeatMode -import androidx.compose.animation.core.animateFloat -import androidx.compose.animation.core.infiniteRepeatable -import androidx.compose.animation.core.rememberInfiniteTransition -import androidx.compose.animation.core.tween -import androidx.compose.foundation.Canvas -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.layout.size -import androidx.compose.foundation.shape.CircleShape -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.Surface -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.runtime.getValue -import androidx.compose.ui.Alignment -import androidx.compose.ui.Modifier -import androidx.compose.ui.graphics.Brush -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.graphics.drawscope.Stroke -import androidx.compose.ui.text.font.FontWeight -import androidx.compose.ui.unit.dp - -@Composable -fun TalkOrbOverlay( - seamColor: Color, - statusText: String, - isListening: Boolean, - isSpeaking: Boolean, - modifier: Modifier = Modifier, -) { - val transition = rememberInfiniteTransition(label = "talk-orb") - val t by - transition.animateFloat( - initialValue = 0f, - targetValue = 1f, - animationSpec = - infiniteRepeatable( - animation = tween(durationMillis = 1500, easing = LinearEasing), - repeatMode = RepeatMode.Restart, - ), - label = "pulse", - ) - - val trimmed = statusText.trim() - val showStatus = trimmed.isNotEmpty() && trimmed != "Off" - val phase = - when { - isSpeaking -> "Speaking" - isListening -> "Listening" - else -> "Thinking" - } - - Column( - modifier = modifier.padding(24.dp), - horizontalAlignment = Alignment.CenterHorizontally, - verticalArrangement = Arrangement.spacedBy(12.dp), - ) { - Box(contentAlignment = Alignment.Center) { - Canvas(modifier = Modifier.size(360.dp)) { - val center = this.center - val baseRadius = size.minDimension * 0.30f - - val ring1 = 1.05f + (t * 0.25f) - val ring2 = 1.20f + (t * 0.55f) - val ringAlpha1 = (1f - t) * 0.34f - val ringAlpha2 = (1f - t) * 0.22f - - drawCircle( - color = seamColor.copy(alpha = ringAlpha1), - radius = baseRadius * ring1, - center = center, - style = Stroke(width = 3.dp.toPx()), - ) - drawCircle( - color = seamColor.copy(alpha = ringAlpha2), - radius = baseRadius * ring2, - center = center, - style = Stroke(width = 3.dp.toPx()), - ) - - drawCircle( - brush = - Brush.radialGradient( - colors = - listOf( - seamColor.copy(alpha = 0.92f), - seamColor.copy(alpha = 0.40f), - Color.Black.copy(alpha = 0.56f), - ), - center = center, - radius = baseRadius * 1.35f, - ), - radius = baseRadius, - center = center, - ) - - drawCircle( - color = seamColor.copy(alpha = 0.34f), - radius = baseRadius, - center = center, - style = Stroke(width = 1.dp.toPx()), - ) - } - } - - if (showStatus) { - Surface( - color = Color.Black.copy(alpha = 0.40f), - shape = CircleShape, - ) { - Text( - text = trimmed, - modifier = Modifier.padding(horizontal = 14.dp, vertical = 8.dp), - color = Color.White.copy(alpha = 0.92f), - style = MaterialTheme.typography.labelLarge, - fontWeight = FontWeight.SemiBold, - ) - } - } else { - Text( - text = phase, - color = Color.White.copy(alpha = 0.80f), - style = MaterialTheme.typography.labelLarge, - fontWeight = FontWeight.SemiBold, - ) - } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatComposer.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatComposer.kt deleted file mode 100644 index 1f30938e0..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatComposer.kt +++ /dev/null @@ -1,285 +0,0 @@ -package com.clawdbot.android.ui.chat - -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.Spacer -import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.layout.size -import androidx.compose.foundation.layout.width -import androidx.compose.foundation.rememberScrollState -import androidx.compose.foundation.shape.RoundedCornerShape -import androidx.compose.foundation.horizontalScroll -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.ArrowUpward -import androidx.compose.material.icons.filled.AttachFile -import androidx.compose.material.icons.filled.Refresh -import androidx.compose.material.icons.filled.Stop -import androidx.compose.material3.ButtonDefaults -import androidx.compose.material3.DropdownMenu -import androidx.compose.material3.DropdownMenuItem -import androidx.compose.material3.FilledTonalButton -import androidx.compose.material3.FilledTonalIconButton -import androidx.compose.material3.Icon -import androidx.compose.material3.IconButtonDefaults -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.OutlinedTextField -import androidx.compose.material3.Surface -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.saveable.rememberSaveable -import androidx.compose.runtime.setValue -import androidx.compose.ui.Alignment -import androidx.compose.ui.Modifier -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.unit.dp -import com.clawdbot.android.chat.ChatSessionEntry - -@Composable -fun ChatComposer( - sessionKey: String, - sessions: List, - mainSessionKey: String, - healthOk: Boolean, - thinkingLevel: String, - pendingRunCount: Int, - errorText: String?, - attachments: List, - onPickImages: () -> Unit, - onRemoveAttachment: (id: String) -> Unit, - onSetThinkingLevel: (level: String) -> Unit, - onSelectSession: (sessionKey: String) -> Unit, - onRefresh: () -> Unit, - onAbort: () -> Unit, - onSend: (text: String) -> Unit, -) { - var input by rememberSaveable { mutableStateOf("") } - var showThinkingMenu by remember { mutableStateOf(false) } - var showSessionMenu by remember { mutableStateOf(false) } - - val sessionOptions = resolveSessionChoices(sessionKey, sessions, mainSessionKey = mainSessionKey) - val currentSessionLabel = - sessionOptions.firstOrNull { it.key == sessionKey }?.displayName ?: sessionKey - - val canSend = pendingRunCount == 0 && (input.trim().isNotEmpty() || attachments.isNotEmpty()) && healthOk - - Surface( - shape = MaterialTheme.shapes.large, - color = MaterialTheme.colorScheme.surfaceContainer, - tonalElevation = 0.dp, - shadowElevation = 0.dp, - ) { - Column(modifier = Modifier.padding(10.dp), verticalArrangement = Arrangement.spacedBy(8.dp)) { - Row( - modifier = Modifier.fillMaxWidth(), - horizontalArrangement = Arrangement.spacedBy(8.dp), - verticalAlignment = Alignment.CenterVertically, - ) { - Box { - FilledTonalButton( - onClick = { showSessionMenu = true }, - contentPadding = ButtonDefaults.ContentPadding, - ) { - Text("Session: $currentSessionLabel") - } - - DropdownMenu(expanded = showSessionMenu, onDismissRequest = { showSessionMenu = false }) { - for (entry in sessionOptions) { - DropdownMenuItem( - text = { Text(entry.displayName ?: entry.key) }, - onClick = { - onSelectSession(entry.key) - showSessionMenu = false - }, - trailingIcon = { - if (entry.key == sessionKey) { - Text("✓") - } else { - Spacer(modifier = Modifier.width(10.dp)) - } - }, - ) - } - } - } - - Box { - FilledTonalButton( - onClick = { showThinkingMenu = true }, - contentPadding = ButtonDefaults.ContentPadding, - ) { - Text("Thinking: ${thinkingLabel(thinkingLevel)}") - } - - DropdownMenu(expanded = showThinkingMenu, onDismissRequest = { showThinkingMenu = false }) { - ThinkingMenuItem("off", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } - ThinkingMenuItem("low", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } - ThinkingMenuItem("medium", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } - ThinkingMenuItem("high", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false } - } - } - - Spacer(modifier = Modifier.weight(1f)) - - FilledTonalIconButton(onClick = onRefresh, modifier = Modifier.size(42.dp)) { - Icon(Icons.Default.Refresh, contentDescription = "Refresh") - } - - FilledTonalIconButton(onClick = onPickImages, modifier = Modifier.size(42.dp)) { - Icon(Icons.Default.AttachFile, contentDescription = "Add image") - } - } - - if (attachments.isNotEmpty()) { - AttachmentsStrip(attachments = attachments, onRemoveAttachment = onRemoveAttachment) - } - - OutlinedTextField( - value = input, - onValueChange = { input = it }, - modifier = Modifier.fillMaxWidth(), - placeholder = { Text("Message Clawd…") }, - minLines = 2, - maxLines = 6, - ) - - Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) { - ConnectionPill(sessionLabel = currentSessionLabel, healthOk = healthOk) - Spacer(modifier = Modifier.weight(1f)) - - if (pendingRunCount > 0) { - FilledTonalIconButton( - onClick = onAbort, - colors = - IconButtonDefaults.filledTonalIconButtonColors( - containerColor = Color(0x33E74C3C), - contentColor = Color(0xFFE74C3C), - ), - ) { - Icon(Icons.Default.Stop, contentDescription = "Abort") - } - } else { - FilledTonalIconButton(onClick = { - val text = input - input = "" - onSend(text) - }, enabled = canSend) { - Icon(Icons.Default.ArrowUpward, contentDescription = "Send") - } - } - } - - if (!errorText.isNullOrBlank()) { - Text( - text = errorText, - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.error, - maxLines = 2, - ) - } - } - } -} - -@Composable -private fun ConnectionPill(sessionLabel: String, healthOk: Boolean) { - Surface( - shape = RoundedCornerShape(999.dp), - color = MaterialTheme.colorScheme.surfaceContainerHighest, - ) { - Row( - modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp), - horizontalArrangement = Arrangement.spacedBy(8.dp), - verticalAlignment = Alignment.CenterVertically, - ) { - Surface( - modifier = Modifier.size(7.dp), - shape = androidx.compose.foundation.shape.CircleShape, - color = if (healthOk) Color(0xFF2ECC71) else Color(0xFFF39C12), - ) {} - Text(sessionLabel, style = MaterialTheme.typography.labelSmall) - Text( - if (healthOk) "Connected" else "Connecting…", - style = MaterialTheme.typography.labelSmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } - } -} - -@Composable -private fun ThinkingMenuItem( - value: String, - current: String, - onSet: (String) -> Unit, - onDismiss: () -> Unit, -) { - DropdownMenuItem( - text = { Text(thinkingLabel(value)) }, - onClick = { - onSet(value) - onDismiss() - }, - trailingIcon = { - if (value == current.trim().lowercase()) { - Text("✓") - } else { - Spacer(modifier = Modifier.width(10.dp)) - } - }, - ) -} - -private fun thinkingLabel(raw: String): String { - return when (raw.trim().lowercase()) { - "low" -> "Low" - "medium" -> "Medium" - "high" -> "High" - else -> "Off" - } -} - -@Composable -private fun AttachmentsStrip( - attachments: List, - onRemoveAttachment: (id: String) -> Unit, -) { - Row( - modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()), - horizontalArrangement = Arrangement.spacedBy(8.dp), - ) { - for (att in attachments) { - AttachmentChip( - fileName = att.fileName, - onRemove = { onRemoveAttachment(att.id) }, - ) - } - } -} - -@Composable -private fun AttachmentChip(fileName: String, onRemove: () -> Unit) { - Surface( - shape = RoundedCornerShape(999.dp), - color = MaterialTheme.colorScheme.primary.copy(alpha = 0.10f), - ) { - Row( - modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(8.dp), - ) { - Text(text = fileName, style = MaterialTheme.typography.bodySmall, maxLines = 1) - FilledTonalIconButton( - onClick = onRemove, - modifier = Modifier.size(30.dp), - ) { - Text("×") - } - } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMarkdown.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMarkdown.kt deleted file mode 100644 index f15673fb3..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMarkdown.kt +++ /dev/null @@ -1,215 +0,0 @@ -package com.clawdbot.android.ui.chat - -import android.graphics.BitmapFactory -import android.util.Base64 -import androidx.compose.foundation.Image -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.text.selection.SelectionContainer -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.runtime.LaunchedEffect -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.setValue -import androidx.compose.ui.Modifier -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.graphics.asImageBitmap -import androidx.compose.ui.layout.ContentScale -import androidx.compose.ui.text.AnnotatedString -import androidx.compose.ui.text.SpanStyle -import androidx.compose.ui.text.buildAnnotatedString -import androidx.compose.ui.text.font.FontFamily -import androidx.compose.ui.text.font.FontStyle -import androidx.compose.ui.text.font.FontWeight -import androidx.compose.ui.text.withStyle -import androidx.compose.ui.unit.dp -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext - -@Composable -fun ChatMarkdown(text: String, textColor: Color) { - val blocks = remember(text) { splitMarkdown(text) } - val inlineCodeBg = MaterialTheme.colorScheme.surfaceContainerLow - - Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { - for (b in blocks) { - when (b) { - is ChatMarkdownBlock.Text -> { - val trimmed = b.text.trimEnd() - if (trimmed.isEmpty()) continue - Text( - text = parseInlineMarkdown(trimmed, inlineCodeBg = inlineCodeBg), - style = MaterialTheme.typography.bodyMedium, - color = textColor, - ) - } - is ChatMarkdownBlock.Code -> { - SelectionContainer(modifier = Modifier.fillMaxWidth()) { - ChatCodeBlock(code = b.code, language = b.language) - } - } - is ChatMarkdownBlock.InlineImage -> { - InlineBase64Image(base64 = b.base64, mimeType = b.mimeType) - } - } - } - } -} - -private sealed interface ChatMarkdownBlock { - data class Text(val text: String) : ChatMarkdownBlock - data class Code(val code: String, val language: String?) : ChatMarkdownBlock - data class InlineImage(val mimeType: String?, val base64: String) : ChatMarkdownBlock -} - -private fun splitMarkdown(raw: String): List { - if (raw.isEmpty()) return emptyList() - - val out = ArrayList() - var idx = 0 - while (idx < raw.length) { - val fenceStart = raw.indexOf("```", startIndex = idx) - if (fenceStart < 0) { - out.addAll(splitInlineImages(raw.substring(idx))) - break - } - - if (fenceStart > idx) { - out.addAll(splitInlineImages(raw.substring(idx, fenceStart))) - } - - val langLineStart = fenceStart + 3 - val langLineEnd = raw.indexOf('\n', startIndex = langLineStart).let { if (it < 0) raw.length else it } - val language = raw.substring(langLineStart, langLineEnd).trim().ifEmpty { null } - - val codeStart = if (langLineEnd < raw.length && raw[langLineEnd] == '\n') langLineEnd + 1 else langLineEnd - val fenceEnd = raw.indexOf("```", startIndex = codeStart) - if (fenceEnd < 0) { - out.addAll(splitInlineImages(raw.substring(fenceStart))) - break - } - val code = raw.substring(codeStart, fenceEnd) - out.add(ChatMarkdownBlock.Code(code = code, language = language)) - - idx = fenceEnd + 3 - } - - return out -} - -private fun splitInlineImages(text: String): List { - if (text.isEmpty()) return emptyList() - val regex = Regex("data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)") - val out = ArrayList() - - var idx = 0 - while (idx < text.length) { - val m = regex.find(text, startIndex = idx) ?: break - val start = m.range.first - val end = m.range.last + 1 - if (start > idx) out.add(ChatMarkdownBlock.Text(text.substring(idx, start))) - - val mime = "image/" + (m.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png") - val b64 = m.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty() - if (b64.isNotEmpty()) { - out.add(ChatMarkdownBlock.InlineImage(mimeType = mime, base64 = b64)) - } - idx = end - } - - if (idx < text.length) out.add(ChatMarkdownBlock.Text(text.substring(idx))) - return out -} - -private fun parseInlineMarkdown(text: String, inlineCodeBg: androidx.compose.ui.graphics.Color): AnnotatedString { - if (text.isEmpty()) return AnnotatedString("") - - val out = buildAnnotatedString { - var i = 0 - while (i < text.length) { - if (text.startsWith("**", startIndex = i)) { - val end = text.indexOf("**", startIndex = i + 2) - if (end > i + 2) { - withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) { - append(text.substring(i + 2, end)) - } - i = end + 2 - continue - } - } - - if (text[i] == '`') { - val end = text.indexOf('`', startIndex = i + 1) - if (end > i + 1) { - withStyle( - SpanStyle( - fontFamily = FontFamily.Monospace, - background = inlineCodeBg, - ), - ) { - append(text.substring(i + 1, end)) - } - i = end + 1 - continue - } - } - - if (text[i] == '*' && (i + 1 < text.length && text[i + 1] != '*')) { - val end = text.indexOf('*', startIndex = i + 1) - if (end > i + 1) { - withStyle(SpanStyle(fontStyle = FontStyle.Italic)) { - append(text.substring(i + 1, end)) - } - i = end + 1 - continue - } - } - - append(text[i]) - i += 1 - } - } - return out -} - -@Composable -private fun InlineBase64Image(base64: String, mimeType: String?) { - var image by remember(base64) { mutableStateOf(null) } - var failed by remember(base64) { mutableStateOf(false) } - - LaunchedEffect(base64) { - failed = false - image = - withContext(Dispatchers.Default) { - try { - val bytes = Base64.decode(base64, Base64.DEFAULT) - val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null - bitmap.asImageBitmap() - } catch (_: Throwable) { - null - } - } - if (image == null) failed = true - } - - if (image != null) { - Image( - bitmap = image!!, - contentDescription = mimeType ?: "image", - contentScale = ContentScale.Fit, - modifier = Modifier.fillMaxWidth(), - ) - } else if (failed) { - Text( - text = "Image unavailable", - modifier = Modifier.padding(vertical = 2.dp), - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMessageListCard.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMessageListCard.kt deleted file mode 100644 index a3229d4a2..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMessageListCard.kt +++ /dev/null @@ -1,111 +0,0 @@ -package com.clawdbot.android.ui.chat - -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.fillMaxSize -import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.lazy.LazyColumn -import androidx.compose.foundation.lazy.rememberLazyListState -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.ArrowCircleDown -import androidx.compose.material3.Card -import androidx.compose.material3.CardDefaults -import androidx.compose.material3.Icon -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.runtime.LaunchedEffect -import androidx.compose.ui.Alignment -import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.alpha -import androidx.compose.ui.unit.dp -import com.clawdbot.android.chat.ChatMessage -import com.clawdbot.android.chat.ChatPendingToolCall - -@Composable -fun ChatMessageListCard( - messages: List, - pendingRunCount: Int, - pendingToolCalls: List, - streamingAssistantText: String?, - modifier: Modifier = Modifier, -) { - val listState = rememberLazyListState() - - LaunchedEffect(messages.size, pendingRunCount, pendingToolCalls.size, streamingAssistantText) { - val total = - messages.size + - (if (pendingRunCount > 0) 1 else 0) + - (if (pendingToolCalls.isNotEmpty()) 1 else 0) + - (if (!streamingAssistantText.isNullOrBlank()) 1 else 0) - if (total <= 0) return@LaunchedEffect - listState.animateScrollToItem(index = total - 1) - } - - Card( - modifier = modifier.fillMaxWidth(), - shape = MaterialTheme.shapes.large, - colors = - CardDefaults.cardColors( - containerColor = MaterialTheme.colorScheme.surfaceContainer, - ), - elevation = CardDefaults.cardElevation(defaultElevation = 0.dp), - ) { - Box(modifier = Modifier.fillMaxSize()) { - LazyColumn( - modifier = Modifier.fillMaxSize(), - state = listState, - verticalArrangement = Arrangement.spacedBy(14.dp), - contentPadding = androidx.compose.foundation.layout.PaddingValues(top = 12.dp, bottom = 12.dp, start = 12.dp, end = 12.dp), - ) { - items(count = messages.size, key = { idx -> messages[idx].id }) { idx -> - ChatMessageBubble(message = messages[idx]) - } - - if (pendingRunCount > 0) { - item(key = "typing") { - ChatTypingIndicatorBubble() - } - } - - if (pendingToolCalls.isNotEmpty()) { - item(key = "tools") { - ChatPendingToolsBubble(toolCalls = pendingToolCalls) - } - } - - val stream = streamingAssistantText?.trim() - if (!stream.isNullOrEmpty()) { - item(key = "stream") { - ChatStreamingAssistantBubble(text = stream) - } - } - } - - if (messages.isEmpty() && pendingRunCount == 0 && pendingToolCalls.isEmpty() && streamingAssistantText.isNullOrBlank()) { - EmptyChatHint(modifier = Modifier.align(Alignment.Center)) - } - } - } -} - -@Composable -private fun EmptyChatHint(modifier: Modifier = Modifier) { - Row( - modifier = modifier.alpha(0.7f), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(8.dp), - ) { - Icon( - imageVector = Icons.Default.ArrowCircleDown, - contentDescription = null, - tint = MaterialTheme.colorScheme.onSurfaceVariant, - ) - Text( - text = "Message Clawd…", - style = MaterialTheme.typography.bodyMedium, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMessageViews.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMessageViews.kt deleted file mode 100644 index 59479744e..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatMessageViews.kt +++ /dev/null @@ -1,252 +0,0 @@ -package com.clawdbot.android.ui.chat - -import android.graphics.BitmapFactory -import android.util.Base64 -import androidx.compose.foundation.background -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.layout.size -import androidx.compose.foundation.shape.CircleShape -import androidx.compose.foundation.shape.RoundedCornerShape -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.Surface -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.runtime.LaunchedEffect -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.setValue -import androidx.compose.ui.Alignment -import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.alpha -import androidx.compose.ui.graphics.Brush -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.graphics.asImageBitmap -import androidx.compose.ui.layout.ContentScale -import androidx.compose.ui.text.font.FontFamily -import androidx.compose.ui.unit.dp -import androidx.compose.foundation.Image -import com.clawdbot.android.chat.ChatMessage -import com.clawdbot.android.chat.ChatMessageContent -import com.clawdbot.android.chat.ChatPendingToolCall -import com.clawdbot.android.tools.ToolDisplayRegistry -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import androidx.compose.ui.platform.LocalContext - -@Composable -fun ChatMessageBubble(message: ChatMessage) { - val isUser = message.role.lowercase() == "user" - - Row( - modifier = Modifier.fillMaxWidth(), - horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start, - ) { - Surface( - shape = RoundedCornerShape(16.dp), - tonalElevation = 0.dp, - shadowElevation = 0.dp, - color = Color.Transparent, - modifier = Modifier.fillMaxWidth(0.92f), - ) { - Box( - modifier = - Modifier - .background(bubbleBackground(isUser)) - .padding(horizontal = 12.dp, vertical = 10.dp), - ) { - val textColor = textColorOverBubble(isUser) - ChatMessageBody(content = message.content, textColor = textColor) - } - } - } -} - -@Composable -private fun ChatMessageBody(content: List, textColor: Color) { - Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { - for (part in content) { - when (part.type) { - "text" -> { - val text = part.text ?: continue - ChatMarkdown(text = text, textColor = textColor) - } - else -> { - val b64 = part.base64 ?: continue - ChatBase64Image(base64 = b64, mimeType = part.mimeType) - } - } - } - } -} - -@Composable -fun ChatTypingIndicatorBubble() { - Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) { - Surface( - shape = RoundedCornerShape(16.dp), - color = MaterialTheme.colorScheme.surfaceContainer, - ) { - Row( - modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(8.dp), - ) { - DotPulse() - Text("Thinking…", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant) - } - } - } -} - -@Composable -fun ChatPendingToolsBubble(toolCalls: List) { - val context = LocalContext.current - val displays = - remember(toolCalls, context) { - toolCalls.map { ToolDisplayRegistry.resolve(context, it.name, it.args) } - } - Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) { - Surface( - shape = RoundedCornerShape(16.dp), - color = MaterialTheme.colorScheme.surfaceContainer, - ) { - Column(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), verticalArrangement = Arrangement.spacedBy(6.dp)) { - Text("Running tools…", style = MaterialTheme.typography.labelLarge, color = MaterialTheme.colorScheme.onSurface) - for (display in displays.take(6)) { - Column(verticalArrangement = Arrangement.spacedBy(2.dp)) { - Text( - "${display.emoji} ${display.label}", - style = MaterialTheme.typography.bodyMedium, - color = MaterialTheme.colorScheme.onSurfaceVariant, - fontFamily = FontFamily.Monospace, - ) - display.detailLine?.let { detail -> - Text( - detail, - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - fontFamily = FontFamily.Monospace, - ) - } - } - } - if (toolCalls.size > 6) { - Text( - "… +${toolCalls.size - 6} more", - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } - } - } - } -} - -@Composable -fun ChatStreamingAssistantBubble(text: String) { - Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) { - Surface( - shape = RoundedCornerShape(16.dp), - color = MaterialTheme.colorScheme.surfaceContainer, - ) { - Box(modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp)) { - ChatMarkdown(text = text, textColor = MaterialTheme.colorScheme.onSurface) - } - } - } -} - -@Composable -private fun bubbleBackground(isUser: Boolean): Brush { - return if (isUser) { - Brush.linearGradient( - colors = listOf(MaterialTheme.colorScheme.primary, MaterialTheme.colorScheme.primary.copy(alpha = 0.78f)), - ) - } else { - Brush.linearGradient( - colors = listOf(MaterialTheme.colorScheme.surfaceContainer, MaterialTheme.colorScheme.surfaceContainerHigh), - ) - } -} - -@Composable -private fun textColorOverBubble(isUser: Boolean): Color { - return if (isUser) { - MaterialTheme.colorScheme.onPrimary - } else { - MaterialTheme.colorScheme.onSurface - } -} - -@Composable -private fun ChatBase64Image(base64: String, mimeType: String?) { - var image by remember(base64) { mutableStateOf(null) } - var failed by remember(base64) { mutableStateOf(false) } - - LaunchedEffect(base64) { - failed = false - image = - withContext(Dispatchers.Default) { - try { - val bytes = Base64.decode(base64, Base64.DEFAULT) - val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null - bitmap.asImageBitmap() - } catch (_: Throwable) { - null - } - } - if (image == null) failed = true - } - - if (image != null) { - Image( - bitmap = image!!, - contentDescription = mimeType ?: "attachment", - contentScale = ContentScale.Fit, - modifier = Modifier.fillMaxWidth(), - ) - } else if (failed) { - Text("Unsupported attachment", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant) - } -} - -@Composable -private fun DotPulse() { - Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) { - PulseDot(alpha = 0.38f) - PulseDot(alpha = 0.62f) - PulseDot(alpha = 0.90f) - } -} - -@Composable -private fun PulseDot(alpha: Float) { - Surface( - modifier = Modifier.size(6.dp).alpha(alpha), - shape = CircleShape, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) {} -} - -@Composable -fun ChatCodeBlock(code: String, language: String?) { - Surface( - shape = RoundedCornerShape(12.dp), - color = MaterialTheme.colorScheme.surfaceContainerLowest, - modifier = Modifier.fillMaxWidth(), - ) { - Text( - text = code.trimEnd(), - modifier = Modifier.padding(10.dp), - fontFamily = FontFamily.Monospace, - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurface, - ) - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatSessionsDialog.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatSessionsDialog.kt deleted file mode 100644 index 9474b2362..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatSessionsDialog.kt +++ /dev/null @@ -1,92 +0,0 @@ -package com.clawdbot.android.ui.chat - -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.Spacer -import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.lazy.LazyColumn -import androidx.compose.foundation.lazy.items -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.Refresh -import androidx.compose.material3.AlertDialog -import androidx.compose.material3.FilledTonalIconButton -import androidx.compose.material3.Icon -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.Surface -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.ui.Alignment -import androidx.compose.ui.Modifier -import androidx.compose.ui.unit.dp -import com.clawdbot.android.chat.ChatSessionEntry - -@Composable -fun ChatSessionsDialog( - currentSessionKey: String, - sessions: List, - onDismiss: () -> Unit, - onRefresh: () -> Unit, - onSelect: (sessionKey: String) -> Unit, -) { - AlertDialog( - onDismissRequest = onDismiss, - confirmButton = {}, - title = { - Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth()) { - Text("Sessions", style = MaterialTheme.typography.titleMedium) - Spacer(modifier = Modifier.weight(1f)) - FilledTonalIconButton(onClick = onRefresh) { - Icon(Icons.Default.Refresh, contentDescription = "Refresh") - } - } - }, - text = { - if (sessions.isEmpty()) { - Text("No sessions", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant) - } else { - LazyColumn(verticalArrangement = Arrangement.spacedBy(8.dp)) { - items(sessions, key = { it.key }) { entry -> - SessionRow( - entry = entry, - isCurrent = entry.key == currentSessionKey, - onClick = { onSelect(entry.key) }, - ) - } - } - } - }, - ) -} - -@Composable -private fun SessionRow( - entry: ChatSessionEntry, - isCurrent: Boolean, - onClick: () -> Unit, -) { - Surface( - onClick = onClick, - shape = MaterialTheme.shapes.medium, - color = - if (isCurrent) { - MaterialTheme.colorScheme.primary.copy(alpha = 0.14f) - } else { - MaterialTheme.colorScheme.surfaceContainer - }, - modifier = Modifier.fillMaxWidth(), - ) { - Row( - modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(10.dp), - ) { - Text(entry.displayName ?: entry.key, style = MaterialTheme.typography.bodyMedium) - Spacer(modifier = Modifier.weight(1f)) - if (isCurrent) { - Text("Current", style = MaterialTheme.typography.labelSmall, color = MaterialTheme.colorScheme.onSurfaceVariant) - } - } - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatSheetContent.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatSheetContent.kt deleted file mode 100644 index 2b58c626b..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/ChatSheetContent.kt +++ /dev/null @@ -1,147 +0,0 @@ -package com.clawdbot.android.ui.chat - -import android.content.ContentResolver -import android.net.Uri -import android.util.Base64 -import androidx.activity.compose.rememberLauncherForActivityResult -import androidx.activity.result.contract.ActivityResultContracts -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.fillMaxSize -import androidx.compose.foundation.layout.padding -import androidx.compose.runtime.Composable -import androidx.compose.runtime.LaunchedEffect -import androidx.compose.runtime.collectAsState -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateListOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.rememberCoroutineScope -import androidx.compose.ui.Modifier -import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.unit.dp -import com.clawdbot.android.MainViewModel -import com.clawdbot.android.chat.OutgoingAttachment -import java.io.ByteArrayOutputStream -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext - -@Composable -fun ChatSheetContent(viewModel: MainViewModel) { - val messages by viewModel.chatMessages.collectAsState() - val errorText by viewModel.chatError.collectAsState() - val pendingRunCount by viewModel.pendingRunCount.collectAsState() - val healthOk by viewModel.chatHealthOk.collectAsState() - val sessionKey by viewModel.chatSessionKey.collectAsState() - val mainSessionKey by viewModel.mainSessionKey.collectAsState() - val thinkingLevel by viewModel.chatThinkingLevel.collectAsState() - val streamingAssistantText by viewModel.chatStreamingAssistantText.collectAsState() - val pendingToolCalls by viewModel.chatPendingToolCalls.collectAsState() - val sessions by viewModel.chatSessions.collectAsState() - - LaunchedEffect(mainSessionKey) { - viewModel.loadChat(mainSessionKey) - viewModel.refreshChatSessions(limit = 200) - } - - val context = LocalContext.current - val resolver = context.contentResolver - val scope = rememberCoroutineScope() - - val attachments = remember { mutableStateListOf() } - - val pickImages = - rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris -> - if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult - scope.launch(Dispatchers.IO) { - val next = - uris.take(8).mapNotNull { uri -> - try { - loadImageAttachment(resolver, uri) - } catch (_: Throwable) { - null - } - } - withContext(Dispatchers.Main) { - attachments.addAll(next) - } - } - } - - Column( - modifier = - Modifier - .fillMaxSize() - .padding(horizontal = 12.dp, vertical = 12.dp), - verticalArrangement = Arrangement.spacedBy(10.dp), - ) { - ChatMessageListCard( - messages = messages, - pendingRunCount = pendingRunCount, - pendingToolCalls = pendingToolCalls, - streamingAssistantText = streamingAssistantText, - modifier = Modifier.weight(1f, fill = true), - ) - - ChatComposer( - sessionKey = sessionKey, - sessions = sessions, - mainSessionKey = mainSessionKey, - healthOk = healthOk, - thinkingLevel = thinkingLevel, - pendingRunCount = pendingRunCount, - errorText = errorText, - attachments = attachments, - onPickImages = { pickImages.launch("image/*") }, - onRemoveAttachment = { id -> attachments.removeAll { it.id == id } }, - onSetThinkingLevel = { level -> viewModel.setChatThinkingLevel(level) }, - onSelectSession = { key -> viewModel.switchChatSession(key) }, - onRefresh = { - viewModel.refreshChat() - viewModel.refreshChatSessions(limit = 200) - }, - onAbort = { viewModel.abortChat() }, - onSend = { text -> - val outgoing = - attachments.map { att -> - OutgoingAttachment( - type = "image", - mimeType = att.mimeType, - fileName = att.fileName, - base64 = att.base64, - ) - } - viewModel.sendChat(message = text, thinking = thinkingLevel, attachments = outgoing) - attachments.clear() - }, - ) - } -} - -data class PendingImageAttachment( - val id: String, - val fileName: String, - val mimeType: String, - val base64: String, -) - -private suspend fun loadImageAttachment(resolver: ContentResolver, uri: Uri): PendingImageAttachment { - val mimeType = resolver.getType(uri) ?: "image/*" - val fileName = (uri.lastPathSegment ?: "image").substringAfterLast('/') - val bytes = - withContext(Dispatchers.IO) { - resolver.openInputStream(uri)?.use { input -> - val out = ByteArrayOutputStream() - input.copyTo(out) - out.toByteArray() - } ?: ByteArray(0) - } - if (bytes.isEmpty()) throw IllegalStateException("empty attachment") - val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP) - return PendingImageAttachment( - id = uri.toString() + "#" + System.currentTimeMillis().toString(), - fileName = fileName, - mimeType = mimeType, - base64 = base64, - ) -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/SessionFilters.kt b/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/SessionFilters.kt deleted file mode 100644 index da08dbd1e..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/ui/chat/SessionFilters.kt +++ /dev/null @@ -1,49 +0,0 @@ -package com.clawdbot.android.ui.chat - -import com.clawdbot.android.chat.ChatSessionEntry - -private const val RECENT_WINDOW_MS = 24 * 60 * 60 * 1000L - -fun resolveSessionChoices( - currentSessionKey: String, - sessions: List, - mainSessionKey: String, - nowMs: Long = System.currentTimeMillis(), -): List { - val mainKey = mainSessionKey.trim().ifEmpty { "main" } - val current = currentSessionKey.trim().let { if (it == "main" && mainKey != "main") mainKey else it } - val aliasKey = if (mainKey == "main") null else "main" - val cutoff = nowMs - RECENT_WINDOW_MS - val sorted = sessions.sortedByDescending { it.updatedAtMs ?: 0L } - val recent = mutableListOf() - val seen = mutableSetOf() - for (entry in sorted) { - if (aliasKey != null && entry.key == aliasKey) continue - if (!seen.add(entry.key)) continue - if ((entry.updatedAtMs ?: 0L) < cutoff) continue - recent.add(entry) - } - - val result = mutableListOf() - val included = mutableSetOf() - val mainEntry = sorted.firstOrNull { it.key == mainKey } - if (mainEntry != null) { - result.add(mainEntry) - included.add(mainKey) - } else if (current == mainKey) { - result.add(ChatSessionEntry(key = mainKey, updatedAtMs = null)) - included.add(mainKey) - } - - for (entry in recent) { - if (included.add(entry.key)) { - result.add(entry) - } - } - - if (current.isNotEmpty() && !included.contains(current)) { - result.add(ChatSessionEntry(key = current, updatedAtMs = null)) - } - - return result -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/voice/StreamingMediaDataSource.kt b/apps/android/app/src/main/java/com/clawdbot/android/voice/StreamingMediaDataSource.kt deleted file mode 100644 index 6b1536ad5..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/voice/StreamingMediaDataSource.kt +++ /dev/null @@ -1,98 +0,0 @@ -package com.clawdbot.android.voice - -import android.media.MediaDataSource -import kotlin.math.min - -internal class StreamingMediaDataSource : MediaDataSource() { - private data class Chunk(val start: Long, val data: ByteArray) - - private val lock = Object() - private val chunks = ArrayList() - private var totalSize: Long = 0 - private var closed = false - private var finished = false - private var lastReadIndex = 0 - - fun append(data: ByteArray) { - if (data.isEmpty()) return - synchronized(lock) { - if (closed || finished) return - val chunk = Chunk(totalSize, data) - chunks.add(chunk) - totalSize += data.size.toLong() - lock.notifyAll() - } - } - - fun finish() { - synchronized(lock) { - if (closed) return - finished = true - lock.notifyAll() - } - } - - fun fail() { - synchronized(lock) { - closed = true - lock.notifyAll() - } - } - - override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int { - if (position < 0) return -1 - synchronized(lock) { - while (!closed && !finished && position >= totalSize) { - lock.wait() - } - if (closed) return -1 - if (position >= totalSize && finished) return -1 - - val available = (totalSize - position).toInt() - val toRead = min(size, available) - var remaining = toRead - var destOffset = offset - var pos = position - - var index = findChunkIndex(pos) - while (remaining > 0 && index < chunks.size) { - val chunk = chunks[index] - val inChunkOffset = (pos - chunk.start).toInt() - if (inChunkOffset >= chunk.data.size) { - index++ - continue - } - val copyLen = min(remaining, chunk.data.size - inChunkOffset) - System.arraycopy(chunk.data, inChunkOffset, buffer, destOffset, copyLen) - remaining -= copyLen - destOffset += copyLen - pos += copyLen - if (inChunkOffset + copyLen >= chunk.data.size) { - index++ - } - } - - return toRead - remaining - } - } - - override fun getSize(): Long = -1 - - override fun close() { - synchronized(lock) { - closed = true - lock.notifyAll() - } - } - - private fun findChunkIndex(position: Long): Int { - var index = lastReadIndex - while (index < chunks.size) { - val chunk = chunks[index] - if (position < chunk.start + chunk.data.size) break - index++ - } - lastReadIndex = index - return index - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/voice/TalkDirectiveParser.kt b/apps/android/app/src/main/java/com/clawdbot/android/voice/TalkDirectiveParser.kt deleted file mode 100644 index 02d2c3967..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/voice/TalkDirectiveParser.kt +++ /dev/null @@ -1,191 +0,0 @@ -package com.clawdbot.android.voice - -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive - -private val directiveJson = Json { ignoreUnknownKeys = true } - -data class TalkDirective( - val voiceId: String? = null, - val modelId: String? = null, - val speed: Double? = null, - val rateWpm: Int? = null, - val stability: Double? = null, - val similarity: Double? = null, - val style: Double? = null, - val speakerBoost: Boolean? = null, - val seed: Long? = null, - val normalize: String? = null, - val language: String? = null, - val outputFormat: String? = null, - val latencyTier: Int? = null, - val once: Boolean? = null, -) - -data class TalkDirectiveParseResult( - val directive: TalkDirective?, - val stripped: String, - val unknownKeys: List, -) - -object TalkDirectiveParser { - fun parse(text: String): TalkDirectiveParseResult { - val normalized = text.replace("\r\n", "\n") - val lines = normalized.split("\n").toMutableList() - if (lines.isEmpty()) return TalkDirectiveParseResult(null, text, emptyList()) - - val firstNonEmpty = lines.indexOfFirst { it.trim().isNotEmpty() } - if (firstNonEmpty == -1) return TalkDirectiveParseResult(null, text, emptyList()) - - val head = lines[firstNonEmpty].trim() - if (!head.startsWith("{") || !head.endsWith("}")) { - return TalkDirectiveParseResult(null, text, emptyList()) - } - - val obj = parseJsonObject(head) ?: return TalkDirectiveParseResult(null, text, emptyList()) - - val speakerBoost = - boolValue(obj, listOf("speaker_boost", "speakerBoost")) - ?: boolValue(obj, listOf("no_speaker_boost", "noSpeakerBoost"))?.not() - - val directive = TalkDirective( - voiceId = stringValue(obj, listOf("voice", "voice_id", "voiceId")), - modelId = stringValue(obj, listOf("model", "model_id", "modelId")), - speed = doubleValue(obj, listOf("speed")), - rateWpm = intValue(obj, listOf("rate", "wpm")), - stability = doubleValue(obj, listOf("stability")), - similarity = doubleValue(obj, listOf("similarity", "similarity_boost", "similarityBoost")), - style = doubleValue(obj, listOf("style")), - speakerBoost = speakerBoost, - seed = longValue(obj, listOf("seed")), - normalize = stringValue(obj, listOf("normalize", "apply_text_normalization")), - language = stringValue(obj, listOf("lang", "language_code", "language")), - outputFormat = stringValue(obj, listOf("output_format", "format")), - latencyTier = intValue(obj, listOf("latency", "latency_tier", "latencyTier")), - once = boolValue(obj, listOf("once")), - ) - - val hasDirective = listOf( - directive.voiceId, - directive.modelId, - directive.speed, - directive.rateWpm, - directive.stability, - directive.similarity, - directive.style, - directive.speakerBoost, - directive.seed, - directive.normalize, - directive.language, - directive.outputFormat, - directive.latencyTier, - directive.once, - ).any { it != null } - - if (!hasDirective) return TalkDirectiveParseResult(null, text, emptyList()) - - val knownKeys = setOf( - "voice", "voice_id", "voiceid", - "model", "model_id", "modelid", - "speed", "rate", "wpm", - "stability", "similarity", "similarity_boost", "similarityboost", - "style", - "speaker_boost", "speakerboost", - "no_speaker_boost", "nospeakerboost", - "seed", - "normalize", "apply_text_normalization", - "lang", "language_code", "language", - "output_format", "format", - "latency", "latency_tier", "latencytier", - "once", - ) - val unknownKeys = obj.keys.filter { !knownKeys.contains(it.lowercase()) }.sorted() - - lines.removeAt(firstNonEmpty) - if (firstNonEmpty < lines.size) { - if (lines[firstNonEmpty].trim().isEmpty()) { - lines.removeAt(firstNonEmpty) - } - } - - return TalkDirectiveParseResult(directive, lines.joinToString("\n"), unknownKeys) - } - - private fun parseJsonObject(line: String): JsonObject? { - return try { - directiveJson.parseToJsonElement(line) as? JsonObject - } catch (_: Throwable) { - null - } - } - - private fun stringValue(obj: JsonObject, keys: List): String? { - for (key in keys) { - val value = obj[key].asStringOrNull()?.trim() - if (!value.isNullOrEmpty()) return value - } - return null - } - - private fun doubleValue(obj: JsonObject, keys: List): Double? { - for (key in keys) { - val value = obj[key].asDoubleOrNull() - if (value != null) return value - } - return null - } - - private fun intValue(obj: JsonObject, keys: List): Int? { - for (key in keys) { - val value = obj[key].asIntOrNull() - if (value != null) return value - } - return null - } - - private fun longValue(obj: JsonObject, keys: List): Long? { - for (key in keys) { - val value = obj[key].asLongOrNull() - if (value != null) return value - } - return null - } - - private fun boolValue(obj: JsonObject, keys: List): Boolean? { - for (key in keys) { - val value = obj[key].asBooleanOrNull() - if (value != null) return value - } - return null - } -} - -private fun JsonElement?.asStringOrNull(): String? = - (this as? JsonPrimitive)?.takeIf { it.isString }?.content - -private fun JsonElement?.asDoubleOrNull(): Double? { - val primitive = this as? JsonPrimitive ?: return null - return primitive.content.toDoubleOrNull() -} - -private fun JsonElement?.asIntOrNull(): Int? { - val primitive = this as? JsonPrimitive ?: return null - return primitive.content.toIntOrNull() -} - -private fun JsonElement?.asLongOrNull(): Long? { - val primitive = this as? JsonPrimitive ?: return null - return primitive.content.toLongOrNull() -} - -private fun JsonElement?.asBooleanOrNull(): Boolean? { - val primitive = this as? JsonPrimitive ?: return null - val content = primitive.content.trim().lowercase() - return when (content) { - "true", "yes", "1" -> true - "false", "no", "0" -> false - else -> null - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/voice/TalkModeManager.kt b/apps/android/app/src/main/java/com/clawdbot/android/voice/TalkModeManager.kt deleted file mode 100644 index 41f98140d..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/voice/TalkModeManager.kt +++ /dev/null @@ -1,1257 +0,0 @@ -package com.clawdbot.android.voice - -import android.Manifest -import android.content.Context -import android.content.Intent -import android.content.pm.PackageManager -import android.media.AudioAttributes -import android.media.AudioFormat -import android.media.AudioManager -import android.media.AudioTrack -import android.media.MediaPlayer -import android.os.Bundle -import android.os.Handler -import android.os.Looper -import android.os.SystemClock -import android.speech.RecognitionListener -import android.speech.RecognizerIntent -import android.speech.SpeechRecognizer -import android.speech.tts.TextToSpeech -import android.speech.tts.UtteranceProgressListener -import android.util.Log -import androidx.core.content.ContextCompat -import com.clawdbot.android.gateway.GatewaySession -import com.clawdbot.android.isCanonicalMainSessionKey -import com.clawdbot.android.normalizeMainKey -import java.net.HttpURLConnection -import java.net.URL -import java.util.UUID -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive -import kotlinx.serialization.json.buildJsonObject -import kotlin.math.max - -class TalkModeManager( - private val context: Context, - private val scope: CoroutineScope, - private val session: GatewaySession, - private val supportsChatSubscribe: Boolean, - private val isConnected: () -> Boolean, -) { - companion object { - private const val tag = "TalkMode" - private const val defaultModelIdFallback = "eleven_v3" - private const val defaultOutputFormatFallback = "pcm_24000" - } - - private val mainHandler = Handler(Looper.getMainLooper()) - private val json = Json { ignoreUnknownKeys = true } - - private val _isEnabled = MutableStateFlow(false) - val isEnabled: StateFlow = _isEnabled - - private val _isListening = MutableStateFlow(false) - val isListening: StateFlow = _isListening - - private val _isSpeaking = MutableStateFlow(false) - val isSpeaking: StateFlow = _isSpeaking - - private val _statusText = MutableStateFlow("Off") - val statusText: StateFlow = _statusText - - private val _lastAssistantText = MutableStateFlow(null) - val lastAssistantText: StateFlow = _lastAssistantText - - private val _usingFallbackTts = MutableStateFlow(false) - val usingFallbackTts: StateFlow = _usingFallbackTts - - private var recognizer: SpeechRecognizer? = null - private var restartJob: Job? = null - private var stopRequested = false - private var listeningMode = false - - private var silenceJob: Job? = null - private val silenceWindowMs = 700L - private var lastTranscript: String = "" - private var lastHeardAtMs: Long? = null - private var lastSpokenText: String? = null - private var lastInterruptedAtSeconds: Double? = null - - private var defaultVoiceId: String? = null - private var currentVoiceId: String? = null - private var fallbackVoiceId: String? = null - private var defaultModelId: String? = null - private var currentModelId: String? = null - private var defaultOutputFormat: String? = null - private var apiKey: String? = null - private var voiceAliases: Map = emptyMap() - private var interruptOnSpeech: Boolean = true - private var voiceOverrideActive = false - private var modelOverrideActive = false - private var mainSessionKey: String = "main" - - private var pendingRunId: String? = null - private var pendingFinal: CompletableDeferred? = null - private var chatSubscribedSessionKey: String? = null - - private var player: MediaPlayer? = null - private var streamingSource: StreamingMediaDataSource? = null - private var pcmTrack: AudioTrack? = null - @Volatile private var pcmStopRequested = false - private var systemTts: TextToSpeech? = null - private var systemTtsPending: CompletableDeferred? = null - private var systemTtsPendingId: String? = null - - fun setMainSessionKey(sessionKey: String?) { - val trimmed = sessionKey?.trim().orEmpty() - if (trimmed.isEmpty()) return - if (isCanonicalMainSessionKey(mainSessionKey)) return - mainSessionKey = trimmed - } - - fun setEnabled(enabled: Boolean) { - if (_isEnabled.value == enabled) return - _isEnabled.value = enabled - if (enabled) { - Log.d(tag, "enabled") - start() - } else { - Log.d(tag, "disabled") - stop() - } - } - - fun handleGatewayEvent(event: String, payloadJson: String?) { - if (event != "chat") return - if (payloadJson.isNullOrBlank()) return - val pending = pendingRunId ?: return - val obj = - try { - json.parseToJsonElement(payloadJson).asObjectOrNull() - } catch (_: Throwable) { - null - } ?: return - val runId = obj["runId"].asStringOrNull() ?: return - if (runId != pending) return - val state = obj["state"].asStringOrNull() ?: return - if (state == "final") { - pendingFinal?.complete(true) - pendingFinal = null - pendingRunId = null - } - } - - private fun start() { - mainHandler.post { - if (_isListening.value) return@post - stopRequested = false - listeningMode = true - Log.d(tag, "start") - - if (!SpeechRecognizer.isRecognitionAvailable(context)) { - _statusText.value = "Speech recognizer unavailable" - Log.w(tag, "speech recognizer unavailable") - return@post - } - - val micOk = - ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == - PackageManager.PERMISSION_GRANTED - if (!micOk) { - _statusText.value = "Microphone permission required" - Log.w(tag, "microphone permission required") - return@post - } - - try { - recognizer?.destroy() - recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } - startListeningInternal(markListening = true) - startSilenceMonitor() - Log.d(tag, "listening") - } catch (err: Throwable) { - _statusText.value = "Start failed: ${err.message ?: err::class.simpleName}" - Log.w(tag, "start failed: ${err.message ?: err::class.simpleName}") - } - } - } - - private fun stop() { - stopRequested = true - listeningMode = false - restartJob?.cancel() - restartJob = null - silenceJob?.cancel() - silenceJob = null - lastTranscript = "" - lastHeardAtMs = null - _isListening.value = false - _statusText.value = "Off" - stopSpeaking() - _usingFallbackTts.value = false - chatSubscribedSessionKey = null - - mainHandler.post { - recognizer?.cancel() - recognizer?.destroy() - recognizer = null - } - systemTts?.stop() - systemTtsPending?.cancel() - systemTtsPending = null - systemTtsPendingId = null - } - - private fun startListeningInternal(markListening: Boolean) { - val r = recognizer ?: return - val intent = - Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { - putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) - putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) - putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) - putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName) - } - - if (markListening) { - _statusText.value = "Listening" - _isListening.value = true - } - r.startListening(intent) - } - - private fun scheduleRestart(delayMs: Long = 350) { - if (stopRequested) return - restartJob?.cancel() - restartJob = - scope.launch { - delay(delayMs) - mainHandler.post { - if (stopRequested) return@post - try { - recognizer?.cancel() - val shouldListen = listeningMode - val shouldInterrupt = _isSpeaking.value && interruptOnSpeech - if (!shouldListen && !shouldInterrupt) return@post - startListeningInternal(markListening = shouldListen) - } catch (_: Throwable) { - // handled by onError - } - } - } - } - - private fun handleTranscript(text: String, isFinal: Boolean) { - val trimmed = text.trim() - if (_isSpeaking.value && interruptOnSpeech) { - if (shouldInterrupt(trimmed)) { - stopSpeaking() - } - return - } - - if (!_isListening.value) return - - if (trimmed.isNotEmpty()) { - lastTranscript = trimmed - lastHeardAtMs = SystemClock.elapsedRealtime() - } - - if (isFinal) { - lastTranscript = trimmed - } - } - - private fun startSilenceMonitor() { - silenceJob?.cancel() - silenceJob = - scope.launch { - while (_isEnabled.value) { - delay(200) - checkSilence() - } - } - } - - private fun checkSilence() { - if (!_isListening.value) return - val transcript = lastTranscript.trim() - if (transcript.isEmpty()) return - val lastHeard = lastHeardAtMs ?: return - val elapsed = SystemClock.elapsedRealtime() - lastHeard - if (elapsed < silenceWindowMs) return - scope.launch { finalizeTranscript(transcript) } - } - - private suspend fun finalizeTranscript(transcript: String) { - listeningMode = false - _isListening.value = false - _statusText.value = "Thinking…" - lastTranscript = "" - lastHeardAtMs = null - - reloadConfig() - val prompt = buildPrompt(transcript) - if (!isConnected()) { - _statusText.value = "Gateway not connected" - Log.w(tag, "finalize: gateway not connected") - start() - return - } - - try { - val startedAt = System.currentTimeMillis().toDouble() / 1000.0 - subscribeChatIfNeeded(session = session, sessionKey = mainSessionKey) - Log.d(tag, "chat.send start sessionKey=${mainSessionKey.ifBlank { "main" }} chars=${prompt.length}") - val runId = sendChat(prompt, session) - Log.d(tag, "chat.send ok runId=$runId") - val ok = waitForChatFinal(runId) - if (!ok) { - Log.w(tag, "chat final timeout runId=$runId; attempting history fallback") - } - val assistant = waitForAssistantText(session, startedAt, if (ok) 12_000 else 25_000) - if (assistant.isNullOrBlank()) { - _statusText.value = "No reply" - Log.w(tag, "assistant text timeout runId=$runId") - start() - return - } - Log.d(tag, "assistant text ok chars=${assistant.length}") - playAssistant(assistant) - } catch (err: Throwable) { - _statusText.value = "Talk failed: ${err.message ?: err::class.simpleName}" - Log.w(tag, "finalize failed: ${err.message ?: err::class.simpleName}") - } - - if (_isEnabled.value) { - start() - } - } - - private suspend fun subscribeChatIfNeeded(session: GatewaySession, sessionKey: String) { - if (!supportsChatSubscribe) return - val key = sessionKey.trim() - if (key.isEmpty()) return - if (chatSubscribedSessionKey == key) return - try { - session.sendNodeEvent("chat.subscribe", """{"sessionKey":"$key"}""") - chatSubscribedSessionKey = key - Log.d(tag, "chat.subscribe ok sessionKey=$key") - } catch (err: Throwable) { - Log.w(tag, "chat.subscribe failed sessionKey=$key err=${err.message ?: err::class.java.simpleName}") - } - } - - private fun buildPrompt(transcript: String): String { - val lines = mutableListOf( - "Talk Mode active. Reply in a concise, spoken tone.", - "You may optionally prefix the response with JSON (first line) to set ElevenLabs voice (id or alias), e.g. {\"voice\":\"\",\"once\":true}.", - ) - lastInterruptedAtSeconds?.let { - lines.add("Assistant speech interrupted at ${"%.1f".format(it)}s.") - lastInterruptedAtSeconds = null - } - lines.add("") - lines.add(transcript) - return lines.joinToString("\n") - } - - private suspend fun sendChat(message: String, session: GatewaySession): String { - val runId = UUID.randomUUID().toString() - val params = - buildJsonObject { - put("sessionKey", JsonPrimitive(mainSessionKey.ifBlank { "main" })) - put("message", JsonPrimitive(message)) - put("thinking", JsonPrimitive("low")) - put("timeoutMs", JsonPrimitive(30_000)) - put("idempotencyKey", JsonPrimitive(runId)) - } - val res = session.request("chat.send", params.toString()) - val parsed = parseRunId(res) ?: runId - if (parsed != runId) { - pendingRunId = parsed - } - return parsed - } - - private suspend fun waitForChatFinal(runId: String): Boolean { - pendingFinal?.cancel() - val deferred = CompletableDeferred() - pendingRunId = runId - pendingFinal = deferred - - val result = - withContext(Dispatchers.IO) { - try { - kotlinx.coroutines.withTimeout(120_000) { deferred.await() } - } catch (_: Throwable) { - false - } - } - - if (!result) { - pendingFinal = null - pendingRunId = null - } - return result - } - - private suspend fun waitForAssistantText( - session: GatewaySession, - sinceSeconds: Double, - timeoutMs: Long, - ): String? { - val deadline = SystemClock.elapsedRealtime() + timeoutMs - while (SystemClock.elapsedRealtime() < deadline) { - val text = fetchLatestAssistantText(session, sinceSeconds) - if (!text.isNullOrBlank()) return text - delay(300) - } - return null - } - - private suspend fun fetchLatestAssistantText( - session: GatewaySession, - sinceSeconds: Double? = null, - ): String? { - val key = mainSessionKey.ifBlank { "main" } - val res = session.request("chat.history", "{\"sessionKey\":\"$key\"}") - val root = json.parseToJsonElement(res).asObjectOrNull() ?: return null - val messages = root["messages"] as? JsonArray ?: return null - for (item in messages.reversed()) { - val obj = item.asObjectOrNull() ?: continue - if (obj["role"].asStringOrNull() != "assistant") continue - if (sinceSeconds != null) { - val timestamp = obj["timestamp"].asDoubleOrNull() - if (timestamp != null && !TalkModeRuntime.isMessageTimestampAfter(timestamp, sinceSeconds)) continue - } - val content = obj["content"] as? JsonArray ?: continue - val text = - content.mapNotNull { entry -> - entry.asObjectOrNull()?.get("text")?.asStringOrNull()?.trim() - }.filter { it.isNotEmpty() } - if (text.isNotEmpty()) return text.joinToString("\n") - } - return null - } - - private suspend fun playAssistant(text: String) { - val parsed = TalkDirectiveParser.parse(text) - if (parsed.unknownKeys.isNotEmpty()) { - Log.w(tag, "Unknown talk directive keys: ${parsed.unknownKeys}") - } - val directive = parsed.directive - val cleaned = parsed.stripped.trim() - if (cleaned.isEmpty()) return - _lastAssistantText.value = cleaned - - val requestedVoice = directive?.voiceId?.trim()?.takeIf { it.isNotEmpty() } - val resolvedVoice = resolveVoiceAlias(requestedVoice) - if (requestedVoice != null && resolvedVoice == null) { - Log.w(tag, "unknown voice alias: $requestedVoice") - } - - if (directive?.voiceId != null) { - if (directive.once != true) { - currentVoiceId = resolvedVoice - voiceOverrideActive = true - } - } - if (directive?.modelId != null) { - if (directive.once != true) { - currentModelId = directive.modelId - modelOverrideActive = true - } - } - - val apiKey = - apiKey?.trim()?.takeIf { it.isNotEmpty() } - ?: System.getenv("ELEVENLABS_API_KEY")?.trim() - val preferredVoice = resolvedVoice ?: currentVoiceId ?: defaultVoiceId - val voiceId = - if (!apiKey.isNullOrEmpty()) { - resolveVoiceId(preferredVoice, apiKey) - } else { - null - } - - _statusText.value = "Speaking…" - _isSpeaking.value = true - lastSpokenText = cleaned - ensureInterruptListener() - - try { - val canUseElevenLabs = !voiceId.isNullOrBlank() && !apiKey.isNullOrEmpty() - if (!canUseElevenLabs) { - if (voiceId.isNullOrBlank()) { - Log.w(tag, "missing voiceId; falling back to system voice") - } - if (apiKey.isNullOrEmpty()) { - Log.w(tag, "missing ELEVENLABS_API_KEY; falling back to system voice") - } - _usingFallbackTts.value = true - _statusText.value = "Speaking (System)…" - speakWithSystemTts(cleaned) - } else { - _usingFallbackTts.value = false - val ttsStarted = SystemClock.elapsedRealtime() - val modelId = directive?.modelId ?: currentModelId ?: defaultModelId - val request = - ElevenLabsRequest( - text = cleaned, - modelId = modelId, - outputFormat = - TalkModeRuntime.validatedOutputFormat(directive?.outputFormat ?: defaultOutputFormat), - speed = TalkModeRuntime.resolveSpeed(directive?.speed, directive?.rateWpm), - stability = TalkModeRuntime.validatedStability(directive?.stability, modelId), - similarity = TalkModeRuntime.validatedUnit(directive?.similarity), - style = TalkModeRuntime.validatedUnit(directive?.style), - speakerBoost = directive?.speakerBoost, - seed = TalkModeRuntime.validatedSeed(directive?.seed), - normalize = TalkModeRuntime.validatedNormalize(directive?.normalize), - language = TalkModeRuntime.validatedLanguage(directive?.language), - latencyTier = TalkModeRuntime.validatedLatencyTier(directive?.latencyTier), - ) - streamAndPlay(voiceId = voiceId!!, apiKey = apiKey!!, request = request) - Log.d(tag, "elevenlabs stream ok durMs=${SystemClock.elapsedRealtime() - ttsStarted}") - } - } catch (err: Throwable) { - Log.w(tag, "speak failed: ${err.message ?: err::class.simpleName}; falling back to system voice") - try { - _usingFallbackTts.value = true - _statusText.value = "Speaking (System)…" - speakWithSystemTts(cleaned) - } catch (fallbackErr: Throwable) { - _statusText.value = "Speak failed: ${fallbackErr.message ?: fallbackErr::class.simpleName}" - Log.w(tag, "system voice failed: ${fallbackErr.message ?: fallbackErr::class.simpleName}") - } - } - - _isSpeaking.value = false - } - - private suspend fun streamAndPlay(voiceId: String, apiKey: String, request: ElevenLabsRequest) { - stopSpeaking(resetInterrupt = false) - - pcmStopRequested = false - val pcmSampleRate = TalkModeRuntime.parsePcmSampleRate(request.outputFormat) - if (pcmSampleRate != null) { - try { - streamAndPlayPcm(voiceId = voiceId, apiKey = apiKey, request = request, sampleRate = pcmSampleRate) - return - } catch (err: Throwable) { - if (pcmStopRequested) return - Log.w(tag, "pcm playback failed; falling back to mp3: ${err.message ?: err::class.simpleName}") - } - } - - streamAndPlayMp3(voiceId = voiceId, apiKey = apiKey, request = request) - } - - private suspend fun streamAndPlayMp3(voiceId: String, apiKey: String, request: ElevenLabsRequest) { - val dataSource = StreamingMediaDataSource() - streamingSource = dataSource - - val player = MediaPlayer() - this.player = player - - val prepared = CompletableDeferred() - val finished = CompletableDeferred() - - player.setAudioAttributes( - AudioAttributes.Builder() - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .setUsage(AudioAttributes.USAGE_ASSISTANT) - .build(), - ) - player.setOnPreparedListener { - it.start() - prepared.complete(Unit) - } - player.setOnCompletionListener { - finished.complete(Unit) - } - player.setOnErrorListener { _, _, _ -> - finished.completeExceptionally(IllegalStateException("MediaPlayer error")) - true - } - - player.setDataSource(dataSource) - withContext(Dispatchers.Main) { - player.prepareAsync() - } - - val fetchError = CompletableDeferred() - val fetchJob = - scope.launch(Dispatchers.IO) { - try { - streamTts(voiceId = voiceId, apiKey = apiKey, request = request, sink = dataSource) - fetchError.complete(null) - } catch (err: Throwable) { - dataSource.fail() - fetchError.complete(err) - } - } - - Log.d(tag, "play start") - try { - prepared.await() - finished.await() - fetchError.await()?.let { throw it } - } finally { - fetchJob.cancel() - cleanupPlayer() - } - Log.d(tag, "play done") - } - - private suspend fun streamAndPlayPcm( - voiceId: String, - apiKey: String, - request: ElevenLabsRequest, - sampleRate: Int, - ) { - val minBuffer = - AudioTrack.getMinBufferSize( - sampleRate, - AudioFormat.CHANNEL_OUT_MONO, - AudioFormat.ENCODING_PCM_16BIT, - ) - if (minBuffer <= 0) { - throw IllegalStateException("AudioTrack buffer size invalid: $minBuffer") - } - - val bufferSize = max(minBuffer * 2, 8 * 1024) - val track = - AudioTrack( - AudioAttributes.Builder() - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .setUsage(AudioAttributes.USAGE_ASSISTANT) - .build(), - AudioFormat.Builder() - .setSampleRate(sampleRate) - .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) - .setEncoding(AudioFormat.ENCODING_PCM_16BIT) - .build(), - bufferSize, - AudioTrack.MODE_STREAM, - AudioManager.AUDIO_SESSION_ID_GENERATE, - ) - if (track.state != AudioTrack.STATE_INITIALIZED) { - track.release() - throw IllegalStateException("AudioTrack init failed") - } - pcmTrack = track - track.play() - - Log.d(tag, "pcm play start sampleRate=$sampleRate bufferSize=$bufferSize") - try { - streamPcm(voiceId = voiceId, apiKey = apiKey, request = request, track = track) - } finally { - cleanupPcmTrack() - } - Log.d(tag, "pcm play done") - } - - private suspend fun speakWithSystemTts(text: String) { - val trimmed = text.trim() - if (trimmed.isEmpty()) return - val ok = ensureSystemTts() - if (!ok) { - throw IllegalStateException("system TTS unavailable") - } - - val tts = systemTts ?: throw IllegalStateException("system TTS unavailable") - val utteranceId = "talk-${UUID.randomUUID()}" - val deferred = CompletableDeferred() - systemTtsPending?.cancel() - systemTtsPending = deferred - systemTtsPendingId = utteranceId - - withContext(Dispatchers.Main) { - val params = Bundle() - tts.speak(trimmed, TextToSpeech.QUEUE_FLUSH, params, utteranceId) - } - - withContext(Dispatchers.IO) { - try { - kotlinx.coroutines.withTimeout(180_000) { deferred.await() } - } catch (err: Throwable) { - throw err - } - } - } - - private suspend fun ensureSystemTts(): Boolean { - if (systemTts != null) return true - return withContext(Dispatchers.Main) { - val deferred = CompletableDeferred() - val tts = - try { - TextToSpeech(context) { status -> - deferred.complete(status == TextToSpeech.SUCCESS) - } - } catch (_: Throwable) { - deferred.complete(false) - null - } - if (tts == null) return@withContext false - - tts.setOnUtteranceProgressListener( - object : UtteranceProgressListener() { - override fun onStart(utteranceId: String?) {} - - override fun onDone(utteranceId: String?) { - if (utteranceId == null) return - if (utteranceId != systemTtsPendingId) return - systemTtsPending?.complete(Unit) - systemTtsPending = null - systemTtsPendingId = null - } - - @Suppress("OVERRIDE_DEPRECATION") - @Deprecated("Deprecated in Java") - override fun onError(utteranceId: String?) { - if (utteranceId == null) return - if (utteranceId != systemTtsPendingId) return - systemTtsPending?.completeExceptionally(IllegalStateException("system TTS error")) - systemTtsPending = null - systemTtsPendingId = null - } - - override fun onError(utteranceId: String?, errorCode: Int) { - if (utteranceId == null) return - if (utteranceId != systemTtsPendingId) return - systemTtsPending?.completeExceptionally(IllegalStateException("system TTS error $errorCode")) - systemTtsPending = null - systemTtsPendingId = null - } - }, - ) - - val ok = - try { - deferred.await() - } catch (_: Throwable) { - false - } - if (ok) { - systemTts = tts - } else { - tts.shutdown() - } - ok - } - } - - private fun stopSpeaking(resetInterrupt: Boolean = true) { - pcmStopRequested = true - if (!_isSpeaking.value) { - cleanupPlayer() - cleanupPcmTrack() - systemTts?.stop() - systemTtsPending?.cancel() - systemTtsPending = null - systemTtsPendingId = null - return - } - if (resetInterrupt) { - val currentMs = player?.currentPosition?.toDouble() ?: 0.0 - lastInterruptedAtSeconds = currentMs / 1000.0 - } - cleanupPlayer() - cleanupPcmTrack() - systemTts?.stop() - systemTtsPending?.cancel() - systemTtsPending = null - systemTtsPendingId = null - _isSpeaking.value = false - } - - private fun cleanupPlayer() { - player?.stop() - player?.release() - player = null - streamingSource?.close() - streamingSource = null - } - - private fun cleanupPcmTrack() { - val track = pcmTrack ?: return - try { - track.pause() - track.flush() - track.stop() - } catch (_: Throwable) { - // ignore cleanup errors - } finally { - track.release() - } - pcmTrack = null - } - - private fun shouldInterrupt(transcript: String): Boolean { - val trimmed = transcript.trim() - if (trimmed.length < 3) return false - val spoken = lastSpokenText?.lowercase() - if (spoken != null && spoken.contains(trimmed.lowercase())) return false - return true - } - - private suspend fun reloadConfig() { - val envVoice = System.getenv("ELEVENLABS_VOICE_ID")?.trim() - val sagVoice = System.getenv("SAG_VOICE_ID")?.trim() - val envKey = System.getenv("ELEVENLABS_API_KEY")?.trim() - try { - val res = session.request("config.get", "{}") - val root = json.parseToJsonElement(res).asObjectOrNull() - val config = root?.get("config").asObjectOrNull() - val talk = config?.get("talk").asObjectOrNull() - val sessionCfg = config?.get("session").asObjectOrNull() - val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull()) - val voice = talk?.get("voiceId")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } - val aliases = - talk?.get("voiceAliases").asObjectOrNull()?.entries?.mapNotNull { (key, value) -> - val id = value.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } ?: return@mapNotNull null - normalizeAliasKey(key).takeIf { it.isNotEmpty() }?.let { it to id } - }?.toMap().orEmpty() - val model = talk?.get("modelId")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } - val outputFormat = talk?.get("outputFormat")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } - val key = talk?.get("apiKey")?.asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() } - val interrupt = talk?.get("interruptOnSpeech")?.asBooleanOrNull() - - if (!isCanonicalMainSessionKey(mainSessionKey)) { - mainSessionKey = mainKey - } - defaultVoiceId = voice ?: envVoice?.takeIf { it.isNotEmpty() } ?: sagVoice?.takeIf { it.isNotEmpty() } - voiceAliases = aliases - if (!voiceOverrideActive) currentVoiceId = defaultVoiceId - defaultModelId = model ?: defaultModelIdFallback - if (!modelOverrideActive) currentModelId = defaultModelId - defaultOutputFormat = outputFormat ?: defaultOutputFormatFallback - apiKey = key ?: envKey?.takeIf { it.isNotEmpty() } - if (interrupt != null) interruptOnSpeech = interrupt - } catch (_: Throwable) { - defaultVoiceId = envVoice?.takeIf { it.isNotEmpty() } ?: sagVoice?.takeIf { it.isNotEmpty() } - defaultModelId = defaultModelIdFallback - if (!modelOverrideActive) currentModelId = defaultModelId - apiKey = envKey?.takeIf { it.isNotEmpty() } - voiceAliases = emptyMap() - defaultOutputFormat = defaultOutputFormatFallback - } - } - - private fun parseRunId(jsonString: String): String? { - val obj = json.parseToJsonElement(jsonString).asObjectOrNull() ?: return null - return obj["runId"].asStringOrNull() - } - - private suspend fun streamTts( - voiceId: String, - apiKey: String, - request: ElevenLabsRequest, - sink: StreamingMediaDataSource, - ) { - withContext(Dispatchers.IO) { - val conn = openTtsConnection(voiceId = voiceId, apiKey = apiKey, request = request) - try { - val payload = buildRequestPayload(request) - conn.outputStream.use { it.write(payload.toByteArray()) } - - val code = conn.responseCode - if (code >= 400) { - val message = conn.errorStream?.readBytes()?.toString(Charsets.UTF_8) ?: "" - sink.fail() - throw IllegalStateException("ElevenLabs failed: $code $message") - } - - val buffer = ByteArray(8 * 1024) - conn.inputStream.use { input -> - while (true) { - val read = input.read(buffer) - if (read <= 0) break - sink.append(buffer.copyOf(read)) - } - } - sink.finish() - } finally { - conn.disconnect() - } - } - } - - private suspend fun streamPcm( - voiceId: String, - apiKey: String, - request: ElevenLabsRequest, - track: AudioTrack, - ) { - withContext(Dispatchers.IO) { - val conn = openTtsConnection(voiceId = voiceId, apiKey = apiKey, request = request) - try { - val payload = buildRequestPayload(request) - conn.outputStream.use { it.write(payload.toByteArray()) } - - val code = conn.responseCode - if (code >= 400) { - val message = conn.errorStream?.readBytes()?.toString(Charsets.UTF_8) ?: "" - throw IllegalStateException("ElevenLabs failed: $code $message") - } - - val buffer = ByteArray(8 * 1024) - conn.inputStream.use { input -> - while (true) { - if (pcmStopRequested) return@withContext - val read = input.read(buffer) - if (read <= 0) break - var offset = 0 - while (offset < read) { - if (pcmStopRequested) return@withContext - val wrote = - try { - track.write(buffer, offset, read - offset) - } catch (err: Throwable) { - if (pcmStopRequested) return@withContext - throw err - } - if (wrote <= 0) { - if (pcmStopRequested) return@withContext - throw IllegalStateException("AudioTrack write failed: $wrote") - } - offset += wrote - } - } - } - } finally { - conn.disconnect() - } - } - } - - private fun openTtsConnection( - voiceId: String, - apiKey: String, - request: ElevenLabsRequest, - ): HttpURLConnection { - val baseUrl = "https://api.elevenlabs.io/v1/text-to-speech/$voiceId/stream" - val latencyTier = request.latencyTier - val url = - if (latencyTier != null) { - URL("$baseUrl?optimize_streaming_latency=$latencyTier") - } else { - URL(baseUrl) - } - val conn = url.openConnection() as HttpURLConnection - conn.requestMethod = "POST" - conn.connectTimeout = 30_000 - conn.readTimeout = 30_000 - conn.setRequestProperty("Content-Type", "application/json") - conn.setRequestProperty("Accept", resolveAcceptHeader(request.outputFormat)) - conn.setRequestProperty("xi-api-key", apiKey) - conn.doOutput = true - return conn - } - - private fun resolveAcceptHeader(outputFormat: String?): String { - val normalized = outputFormat?.trim()?.lowercase().orEmpty() - return if (normalized.startsWith("pcm_")) "audio/pcm" else "audio/mpeg" - } - - private fun buildRequestPayload(request: ElevenLabsRequest): String { - val voiceSettingsEntries = - buildJsonObject { - request.speed?.let { put("speed", JsonPrimitive(it)) } - request.stability?.let { put("stability", JsonPrimitive(it)) } - request.similarity?.let { put("similarity_boost", JsonPrimitive(it)) } - request.style?.let { put("style", JsonPrimitive(it)) } - request.speakerBoost?.let { put("use_speaker_boost", JsonPrimitive(it)) } - } - - val payload = - buildJsonObject { - put("text", JsonPrimitive(request.text)) - request.modelId?.takeIf { it.isNotEmpty() }?.let { put("model_id", JsonPrimitive(it)) } - request.outputFormat?.takeIf { it.isNotEmpty() }?.let { put("output_format", JsonPrimitive(it)) } - request.seed?.let { put("seed", JsonPrimitive(it)) } - request.normalize?.let { put("apply_text_normalization", JsonPrimitive(it)) } - request.language?.let { put("language_code", JsonPrimitive(it)) } - if (voiceSettingsEntries.isNotEmpty()) { - put("voice_settings", voiceSettingsEntries) - } - } - - return payload.toString() - } - - private data class ElevenLabsRequest( - val text: String, - val modelId: String?, - val outputFormat: String?, - val speed: Double?, - val stability: Double?, - val similarity: Double?, - val style: Double?, - val speakerBoost: Boolean?, - val seed: Long?, - val normalize: String?, - val language: String?, - val latencyTier: Int?, - ) - - private object TalkModeRuntime { - fun resolveSpeed(speed: Double?, rateWpm: Int?): Double? { - if (rateWpm != null && rateWpm > 0) { - val resolved = rateWpm.toDouble() / 175.0 - if (resolved <= 0.5 || resolved >= 2.0) return null - return resolved - } - if (speed != null) { - if (speed <= 0.5 || speed >= 2.0) return null - return speed - } - return null - } - - fun validatedUnit(value: Double?): Double? { - if (value == null) return null - if (value < 0 || value > 1) return null - return value - } - - fun validatedStability(value: Double?, modelId: String?): Double? { - if (value == null) return null - val normalized = modelId?.trim()?.lowercase() - if (normalized == "eleven_v3") { - return if (value == 0.0 || value == 0.5 || value == 1.0) value else null - } - return validatedUnit(value) - } - - fun validatedSeed(value: Long?): Long? { - if (value == null) return null - if (value < 0 || value > 4294967295L) return null - return value - } - - fun validatedNormalize(value: String?): String? { - val normalized = value?.trim()?.lowercase() ?: return null - return if (normalized in listOf("auto", "on", "off")) normalized else null - } - - fun validatedLanguage(value: String?): String? { - val normalized = value?.trim()?.lowercase() ?: return null - if (normalized.length != 2) return null - if (!normalized.all { it in 'a'..'z' }) return null - return normalized - } - - fun validatedOutputFormat(value: String?): String? { - val trimmed = value?.trim()?.lowercase() ?: return null - if (trimmed.isEmpty()) return null - if (trimmed.startsWith("mp3_")) return trimmed - return if (parsePcmSampleRate(trimmed) != null) trimmed else null - } - - fun validatedLatencyTier(value: Int?): Int? { - if (value == null) return null - if (value < 0 || value > 4) return null - return value - } - - fun parsePcmSampleRate(value: String?): Int? { - val trimmed = value?.trim()?.lowercase() ?: return null - if (!trimmed.startsWith("pcm_")) return null - val suffix = trimmed.removePrefix("pcm_") - val digits = suffix.takeWhile { it.isDigit() } - val rate = digits.toIntOrNull() ?: return null - return if (rate in setOf(16000, 22050, 24000, 44100)) rate else null - } - - fun isMessageTimestampAfter(timestamp: Double, sinceSeconds: Double): Boolean { - val sinceMs = sinceSeconds * 1000 - return if (timestamp > 10_000_000_000) { - timestamp >= sinceMs - 500 - } else { - timestamp >= sinceSeconds - 0.5 - } - } - } - - private fun ensureInterruptListener() { - if (!interruptOnSpeech || !_isEnabled.value) return - mainHandler.post { - if (stopRequested) return@post - if (!SpeechRecognizer.isRecognitionAvailable(context)) return@post - try { - if (recognizer == null) { - recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } - } - recognizer?.cancel() - startListeningInternal(markListening = false) - } catch (_: Throwable) { - // ignore - } - } - } - - private fun resolveVoiceAlias(value: String?): String? { - val trimmed = value?.trim().orEmpty() - if (trimmed.isEmpty()) return null - val normalized = normalizeAliasKey(trimmed) - voiceAliases[normalized]?.let { return it } - if (voiceAliases.values.any { it.equals(trimmed, ignoreCase = true) }) return trimmed - return if (isLikelyVoiceId(trimmed)) trimmed else null - } - - private suspend fun resolveVoiceId(preferred: String?, apiKey: String): String? { - val trimmed = preferred?.trim().orEmpty() - if (trimmed.isNotEmpty()) { - val resolved = resolveVoiceAlias(trimmed) - if (resolved != null) return resolved - Log.w(tag, "unknown voice alias $trimmed") - } - fallbackVoiceId?.let { return it } - - return try { - val voices = listVoices(apiKey) - val first = voices.firstOrNull() ?: return null - fallbackVoiceId = first.voiceId - if (defaultVoiceId.isNullOrBlank()) { - defaultVoiceId = first.voiceId - } - if (!voiceOverrideActive) { - currentVoiceId = first.voiceId - } - val name = first.name ?: "unknown" - Log.d(tag, "default voice selected $name (${first.voiceId})") - first.voiceId - } catch (err: Throwable) { - Log.w(tag, "list voices failed: ${err.message ?: err::class.simpleName}") - null - } - } - - private suspend fun listVoices(apiKey: String): List { - return withContext(Dispatchers.IO) { - val url = URL("https://api.elevenlabs.io/v1/voices") - val conn = url.openConnection() as HttpURLConnection - conn.requestMethod = "GET" - conn.connectTimeout = 15_000 - conn.readTimeout = 15_000 - conn.setRequestProperty("xi-api-key", apiKey) - - val code = conn.responseCode - val stream = if (code >= 400) conn.errorStream else conn.inputStream - val data = stream.readBytes() - if (code >= 400) { - val message = data.toString(Charsets.UTF_8) - throw IllegalStateException("ElevenLabs voices failed: $code $message") - } - - val root = json.parseToJsonElement(data.toString(Charsets.UTF_8)).asObjectOrNull() - val voices = (root?.get("voices") as? JsonArray) ?: JsonArray(emptyList()) - voices.mapNotNull { entry -> - val obj = entry.asObjectOrNull() ?: return@mapNotNull null - val voiceId = obj["voice_id"].asStringOrNull() ?: return@mapNotNull null - val name = obj["name"].asStringOrNull() - ElevenLabsVoice(voiceId, name) - } - } - } - - private fun isLikelyVoiceId(value: String): Boolean { - if (value.length < 10) return false - return value.all { it.isLetterOrDigit() || it == '-' || it == '_' } - } - - private fun normalizeAliasKey(value: String): String = - value.trim().lowercase() - - private data class ElevenLabsVoice(val voiceId: String, val name: String?) - - private val listener = - object : RecognitionListener { - override fun onReadyForSpeech(params: Bundle?) { - if (_isEnabled.value) { - _statusText.value = if (_isListening.value) "Listening" else _statusText.value - } - } - - override fun onBeginningOfSpeech() {} - - override fun onRmsChanged(rmsdB: Float) {} - - override fun onBufferReceived(buffer: ByteArray?) {} - - override fun onEndOfSpeech() { - scheduleRestart() - } - - override fun onError(error: Int) { - if (stopRequested) return - _isListening.value = false - if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) { - _statusText.value = "Microphone permission required" - return - } - - _statusText.value = - when (error) { - SpeechRecognizer.ERROR_AUDIO -> "Audio error" - SpeechRecognizer.ERROR_CLIENT -> "Client error" - SpeechRecognizer.ERROR_NETWORK -> "Network error" - SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout" - SpeechRecognizer.ERROR_NO_MATCH -> "Listening" - SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy" - SpeechRecognizer.ERROR_SERVER -> "Server error" - SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening" - else -> "Speech error ($error)" - } - scheduleRestart(delayMs = 600) - } - - override fun onResults(results: Bundle?) { - val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() - list.firstOrNull()?.let { handleTranscript(it, isFinal = true) } - scheduleRestart() - } - - override fun onPartialResults(partialResults: Bundle?) { - val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() - list.firstOrNull()?.let { handleTranscript(it, isFinal = false) } - } - - override fun onEvent(eventType: Int, params: Bundle?) {} - } -} - -private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject - -private fun JsonElement?.asStringOrNull(): String? = - (this as? JsonPrimitive)?.takeIf { it.isString }?.content - -private fun JsonElement?.asDoubleOrNull(): Double? { - val primitive = this as? JsonPrimitive ?: return null - return primitive.content.toDoubleOrNull() -} - -private fun JsonElement?.asBooleanOrNull(): Boolean? { - val primitive = this as? JsonPrimitive ?: return null - val content = primitive.content.trim().lowercase() - return when (content) { - "true", "yes", "1" -> true - "false", "no", "0" -> false - else -> null - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/voice/VoiceWakeCommandExtractor.kt b/apps/android/app/src/main/java/com/clawdbot/android/voice/VoiceWakeCommandExtractor.kt deleted file mode 100644 index 1f527b8c8..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/voice/VoiceWakeCommandExtractor.kt +++ /dev/null @@ -1,40 +0,0 @@ -package com.clawdbot.android.voice - -object VoiceWakeCommandExtractor { - fun extractCommand(text: String, triggerWords: List): String? { - val raw = text.trim() - if (raw.isEmpty()) return null - - val triggers = - triggerWords - .map { it.trim().lowercase() } - .filter { it.isNotEmpty() } - .distinct() - if (triggers.isEmpty()) return null - - val alternation = triggers.joinToString("|") { Regex.escape(it) } - // Match: " " - val regex = Regex("(?i)(?:^|\\s)($alternation)\\b[\\s\\p{Punct}]*([\\s\\S]+)$") - val match = regex.find(raw) ?: return null - val extracted = match.groupValues.getOrNull(2)?.trim().orEmpty() - if (extracted.isEmpty()) return null - - val cleaned = extracted.trimStart { it.isWhitespace() || it.isPunctuation() }.trim() - if (cleaned.isEmpty()) return null - return cleaned - } -} - -private fun Char.isPunctuation(): Boolean { - return when (Character.getType(this)) { - Character.CONNECTOR_PUNCTUATION.toInt(), - Character.DASH_PUNCTUATION.toInt(), - Character.START_PUNCTUATION.toInt(), - Character.END_PUNCTUATION.toInt(), - Character.INITIAL_QUOTE_PUNCTUATION.toInt(), - Character.FINAL_QUOTE_PUNCTUATION.toInt(), - Character.OTHER_PUNCTUATION.toInt(), - -> true - else -> false - } -} diff --git a/apps/android/app/src/main/java/com/clawdbot/android/voice/VoiceWakeManager.kt b/apps/android/app/src/main/java/com/clawdbot/android/voice/VoiceWakeManager.kt deleted file mode 100644 index 69863b4cc..000000000 --- a/apps/android/app/src/main/java/com/clawdbot/android/voice/VoiceWakeManager.kt +++ /dev/null @@ -1,173 +0,0 @@ -package com.clawdbot.android.voice - -import android.content.Context -import android.content.Intent -import android.os.Bundle -import android.os.Handler -import android.os.Looper -import android.speech.RecognitionListener -import android.speech.RecognizerIntent -import android.speech.SpeechRecognizer -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.launch - -class VoiceWakeManager( - private val context: Context, - private val scope: CoroutineScope, - private val onCommand: suspend (String) -> Unit, -) { - private val mainHandler = Handler(Looper.getMainLooper()) - - private val _isListening = MutableStateFlow(false) - val isListening: StateFlow = _isListening - - private val _statusText = MutableStateFlow("Off") - val statusText: StateFlow = _statusText - - var triggerWords: List = emptyList() - private set - - private var recognizer: SpeechRecognizer? = null - private var restartJob: Job? = null - private var lastDispatched: String? = null - private var stopRequested = false - - fun setTriggerWords(words: List) { - triggerWords = words - } - - fun start() { - mainHandler.post { - if (_isListening.value) return@post - stopRequested = false - - if (!SpeechRecognizer.isRecognitionAvailable(context)) { - _isListening.value = false - _statusText.value = "Speech recognizer unavailable" - return@post - } - - try { - recognizer?.destroy() - recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } - startListeningInternal() - } catch (err: Throwable) { - _isListening.value = false - _statusText.value = "Start failed: ${err.message ?: err::class.simpleName}" - } - } - } - - fun stop(statusText: String = "Off") { - stopRequested = true - restartJob?.cancel() - restartJob = null - mainHandler.post { - _isListening.value = false - _statusText.value = statusText - recognizer?.cancel() - recognizer?.destroy() - recognizer = null - } - } - - private fun startListeningInternal() { - val r = recognizer ?: return - val intent = - Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { - putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) - putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) - putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) - putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName) - } - - _statusText.value = "Listening" - _isListening.value = true - r.startListening(intent) - } - - private fun scheduleRestart(delayMs: Long = 350) { - if (stopRequested) return - restartJob?.cancel() - restartJob = - scope.launch { - delay(delayMs) - mainHandler.post { - if (stopRequested) return@post - try { - recognizer?.cancel() - startListeningInternal() - } catch (_: Throwable) { - // Will be picked up by onError and retry again. - } - } - } - } - - private fun handleTranscription(text: String) { - val command = VoiceWakeCommandExtractor.extractCommand(text, triggerWords) ?: return - if (command == lastDispatched) return - lastDispatched = command - - scope.launch { onCommand(command) } - _statusText.value = "Triggered" - scheduleRestart(delayMs = 650) - } - - private val listener = - object : RecognitionListener { - override fun onReadyForSpeech(params: Bundle?) { - _statusText.value = "Listening" - } - - override fun onBeginningOfSpeech() {} - - override fun onRmsChanged(rmsdB: Float) {} - - override fun onBufferReceived(buffer: ByteArray?) {} - - override fun onEndOfSpeech() { - scheduleRestart() - } - - override fun onError(error: Int) { - if (stopRequested) return - _isListening.value = false - if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) { - _statusText.value = "Microphone permission required" - return - } - - _statusText.value = - when (error) { - SpeechRecognizer.ERROR_AUDIO -> "Audio error" - SpeechRecognizer.ERROR_CLIENT -> "Client error" - SpeechRecognizer.ERROR_NETWORK -> "Network error" - SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout" - SpeechRecognizer.ERROR_NO_MATCH -> "Listening" - SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy" - SpeechRecognizer.ERROR_SERVER -> "Server error" - SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening" - else -> "Speech error ($error)" - } - scheduleRestart(delayMs = 600) - } - - override fun onResults(results: Bundle?) { - val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() - list.firstOrNull()?.let(::handleTranscription) - scheduleRestart() - } - - override fun onPartialResults(partialResults: Bundle?) { - val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() - list.firstOrNull()?.let(::handleTranscription) - } - - override fun onEvent(eventType: Int, params: Bundle?) {} - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/NodeForegroundServiceTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/NodeForegroundServiceTest.kt deleted file mode 100644 index cb1c8b898..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/NodeForegroundServiceTest.kt +++ /dev/null @@ -1,43 +0,0 @@ -package com.clawdbot.android - -import android.app.Notification -import android.content.Intent -import org.junit.Assert.assertEquals -import org.junit.Assert.assertNotNull -import org.junit.Test -import org.junit.runner.RunWith -import org.robolectric.Robolectric -import org.robolectric.RobolectricTestRunner -import org.robolectric.Shadows -import org.robolectric.annotation.Config - -@RunWith(RobolectricTestRunner::class) -@Config(sdk = [34]) -class NodeForegroundServiceTest { - @Test - fun buildNotificationSetsLaunchIntent() { - val service = Robolectric.buildService(NodeForegroundService::class.java).get() - val notification = buildNotification(service) - - val pendingIntent = notification.contentIntent - assertNotNull(pendingIntent) - - val savedIntent = Shadows.shadowOf(pendingIntent).savedIntent - assertNotNull(savedIntent) - assertEquals(MainActivity::class.java.name, savedIntent.component?.className) - - val expectedFlags = Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP - assertEquals(expectedFlags, savedIntent.flags and expectedFlags) - } - - private fun buildNotification(service: NodeForegroundService): Notification { - val method = - NodeForegroundService::class.java.getDeclaredMethod( - "buildNotification", - String::class.java, - String::class.java, - ) - method.isAccessible = true - return method.invoke(service, "Title", "Text") as Notification - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/WakeWordsTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/WakeWordsTest.kt deleted file mode 100644 index 9363e810c..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/WakeWordsTest.kt +++ /dev/null @@ -1,50 +0,0 @@ -package com.clawdbot.android - -import org.junit.Assert.assertEquals -import org.junit.Assert.assertNull -import org.junit.Test - -class WakeWordsTest { - @Test - fun parseCommaSeparatedTrimsAndDropsEmpty() { - assertEquals(listOf("clawd", "claude"), WakeWords.parseCommaSeparated(" clawd , claude, , ")) - } - - @Test - fun sanitizeTrimsCapsAndFallsBack() { - val defaults = listOf("clawd", "claude") - val long = "x".repeat(WakeWords.maxWordLength + 10) - val words = listOf(" ", " hello ", long) - - val sanitized = WakeWords.sanitize(words, defaults) - assertEquals(2, sanitized.size) - assertEquals("hello", sanitized[0]) - assertEquals("x".repeat(WakeWords.maxWordLength), sanitized[1]) - - assertEquals(defaults, WakeWords.sanitize(listOf(" ", ""), defaults)) - } - - @Test - fun sanitizeLimitsWordCount() { - val defaults = listOf("clawd") - val words = (1..(WakeWords.maxWords + 5)).map { "w$it" } - val sanitized = WakeWords.sanitize(words, defaults) - assertEquals(WakeWords.maxWords, sanitized.size) - assertEquals("w1", sanitized.first()) - assertEquals("w${WakeWords.maxWords}", sanitized.last()) - } - - @Test - fun parseIfChangedSkipsWhenUnchanged() { - val current = listOf("clawd", "claude") - val parsed = WakeWords.parseIfChanged(" clawd , claude ", current) - assertNull(parsed) - } - - @Test - fun parseIfChangedReturnsUpdatedList() { - val current = listOf("clawd") - val parsed = WakeWords.parseIfChanged(" clawd , jarvis ", current) - assertEquals(listOf("clawd", "jarvis"), parsed) - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/gateway/BonjourEscapesTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/gateway/BonjourEscapesTest.kt deleted file mode 100644 index e6acf833e..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/gateway/BonjourEscapesTest.kt +++ /dev/null @@ -1,19 +0,0 @@ -package com.clawdbot.android.gateway - -import org.junit.Assert.assertEquals -import org.junit.Test - -class BonjourEscapesTest { - @Test - fun decodeNoop() { - assertEquals("", BonjourEscapes.decode("")) - assertEquals("hello", BonjourEscapes.decode("hello")) - } - - @Test - fun decodeDecodesDecimalEscapes() { - assertEquals("Moltbot Gateway", BonjourEscapes.decode("Moltbot\\032Gateway")) - assertEquals("A B", BonjourEscapes.decode("A\\032B")) - assertEquals("Peter\u2019s Mac", BonjourEscapes.decode("Peter\\226\\128\\153s Mac")) - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/node/CanvasControllerSnapshotParamsTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/node/CanvasControllerSnapshotParamsTest.kt deleted file mode 100644 index 0b0a42ed5..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/node/CanvasControllerSnapshotParamsTest.kt +++ /dev/null @@ -1,43 +0,0 @@ -package com.clawdbot.android.node - -import org.junit.Assert.assertEquals -import org.junit.Assert.assertNull -import org.junit.Test - -class CanvasControllerSnapshotParamsTest { - @Test - fun parseSnapshotParamsDefaultsToJpeg() { - val params = CanvasController.parseSnapshotParams(null) - assertEquals(CanvasController.SnapshotFormat.Jpeg, params.format) - assertNull(params.quality) - assertNull(params.maxWidth) - } - - @Test - fun parseSnapshotParamsParsesPng() { - val params = CanvasController.parseSnapshotParams("""{"format":"png","maxWidth":900}""") - assertEquals(CanvasController.SnapshotFormat.Png, params.format) - assertEquals(900, params.maxWidth) - } - - @Test - fun parseSnapshotParamsParsesJpegAliases() { - assertEquals( - CanvasController.SnapshotFormat.Jpeg, - CanvasController.parseSnapshotParams("""{"format":"jpeg"}""").format, - ) - assertEquals( - CanvasController.SnapshotFormat.Jpeg, - CanvasController.parseSnapshotParams("""{"format":"jpg"}""").format, - ) - } - - @Test - fun parseSnapshotParamsClampsQuality() { - val low = CanvasController.parseSnapshotParams("""{"quality":0.01}""") - assertEquals(0.1, low.quality) - - val high = CanvasController.parseSnapshotParams("""{"quality":5}""") - assertEquals(1.0, high.quality) - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/node/JpegSizeLimiterTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/node/JpegSizeLimiterTest.kt deleted file mode 100644 index 2c22c2d6a..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/node/JpegSizeLimiterTest.kt +++ /dev/null @@ -1,47 +0,0 @@ -package com.clawdbot.android.node - -import org.junit.Assert.assertEquals -import org.junit.Assert.assertTrue -import org.junit.Test -import kotlin.math.min - -class JpegSizeLimiterTest { - @Test - fun compressesLargePayloadsUnderLimit() { - val maxBytes = 5 * 1024 * 1024 - val result = - JpegSizeLimiter.compressToLimit( - initialWidth = 4000, - initialHeight = 3000, - startQuality = 95, - maxBytes = maxBytes, - encode = { width, height, quality -> - val estimated = (width.toLong() * height.toLong() * quality.toLong()) / 100 - val size = min(maxBytes.toLong() * 2, estimated).toInt() - ByteArray(size) - }, - ) - - assertTrue(result.bytes.size <= maxBytes) - assertTrue(result.width <= 4000) - assertTrue(result.height <= 3000) - assertTrue(result.quality <= 95) - } - - @Test - fun keepsSmallPayloadsAsIs() { - val maxBytes = 5 * 1024 * 1024 - val result = - JpegSizeLimiter.compressToLimit( - initialWidth = 800, - initialHeight = 600, - startQuality = 90, - maxBytes = maxBytes, - encode = { _, _, _ -> ByteArray(120_000) }, - ) - - assertEquals(800, result.width) - assertEquals(600, result.height) - assertEquals(90, result.quality) - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/node/SmsManagerTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/node/SmsManagerTest.kt deleted file mode 100644 index 4748a5683..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/node/SmsManagerTest.kt +++ /dev/null @@ -1,91 +0,0 @@ -package com.clawdbot.android.node - -import kotlinx.serialization.json.jsonObject -import kotlinx.serialization.json.jsonPrimitive -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertTrue -import org.junit.Test - -class SmsManagerTest { - private val json = SmsManager.JsonConfig - - @Test - fun parseParamsRejectsEmptyPayload() { - val result = SmsManager.parseParams("", json) - assertTrue(result is SmsManager.ParseResult.Error) - val error = result as SmsManager.ParseResult.Error - assertEquals("INVALID_REQUEST: paramsJSON required", error.error) - } - - @Test - fun parseParamsRejectsInvalidJson() { - val result = SmsManager.parseParams("not-json", json) - assertTrue(result is SmsManager.ParseResult.Error) - val error = result as SmsManager.ParseResult.Error - assertEquals("INVALID_REQUEST: expected JSON object", error.error) - } - - @Test - fun parseParamsRejectsNonObjectJson() { - val result = SmsManager.parseParams("[]", json) - assertTrue(result is SmsManager.ParseResult.Error) - val error = result as SmsManager.ParseResult.Error - assertEquals("INVALID_REQUEST: expected JSON object", error.error) - } - - @Test - fun parseParamsRejectsMissingTo() { - val result = SmsManager.parseParams("{\"message\":\"Hi\"}", json) - assertTrue(result is SmsManager.ParseResult.Error) - val error = result as SmsManager.ParseResult.Error - assertEquals("INVALID_REQUEST: 'to' phone number required", error.error) - assertEquals("Hi", error.message) - } - - @Test - fun parseParamsRejectsMissingMessage() { - val result = SmsManager.parseParams("{\"to\":\"+1234\"}", json) - assertTrue(result is SmsManager.ParseResult.Error) - val error = result as SmsManager.ParseResult.Error - assertEquals("INVALID_REQUEST: 'message' text required", error.error) - assertEquals("+1234", error.to) - } - - @Test - fun parseParamsTrimsToField() { - val result = SmsManager.parseParams("{\"to\":\" +1555 \",\"message\":\"Hello\"}", json) - assertTrue(result is SmsManager.ParseResult.Ok) - val ok = result as SmsManager.ParseResult.Ok - assertEquals("+1555", ok.params.to) - assertEquals("Hello", ok.params.message) - } - - @Test - fun buildPayloadJsonEscapesFields() { - val payload = SmsManager.buildPayloadJson( - json = json, - ok = false, - to = "+1\"23", - error = "SMS_SEND_FAILED: \"nope\"", - ) - val parsed = json.parseToJsonElement(payload).jsonObject - assertEquals("false", parsed["ok"]?.jsonPrimitive?.content) - assertEquals("+1\"23", parsed["to"]?.jsonPrimitive?.content) - assertEquals("SMS_SEND_FAILED: \"nope\"", parsed["error"]?.jsonPrimitive?.content) - } - - @Test - fun buildSendPlanUsesMultipartWhenMultipleParts() { - val plan = SmsManager.buildSendPlan("hello") { listOf("a", "b") } - assertTrue(plan.useMultipart) - assertEquals(listOf("a", "b"), plan.parts) - } - - @Test - fun buildSendPlanFallsBackToSinglePartWhenDividerEmpty() { - val plan = SmsManager.buildSendPlan("hello") { emptyList() } - assertFalse(plan.useMultipart) - assertEquals(listOf("hello"), plan.parts) - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/protocol/ClawdbotCanvasA2UIActionTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/protocol/ClawdbotCanvasA2UIActionTest.kt deleted file mode 100644 index adb522767..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/protocol/ClawdbotCanvasA2UIActionTest.kt +++ /dev/null @@ -1,49 +0,0 @@ -package com.clawdbot.android.protocol - -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.jsonObject -import org.junit.Assert.assertEquals -import org.junit.Test - -class MoltbotCanvasA2UIActionTest { - @Test - fun extractActionNameAcceptsNameOrAction() { - val nameObj = Json.parseToJsonElement("{\"name\":\"Hello\"}").jsonObject - assertEquals("Hello", MoltbotCanvasA2UIAction.extractActionName(nameObj)) - - val actionObj = Json.parseToJsonElement("{\"action\":\"Wave\"}").jsonObject - assertEquals("Wave", MoltbotCanvasA2UIAction.extractActionName(actionObj)) - - val fallbackObj = - Json.parseToJsonElement("{\"name\":\" \",\"action\":\"Fallback\"}").jsonObject - assertEquals("Fallback", MoltbotCanvasA2UIAction.extractActionName(fallbackObj)) - } - - @Test - fun formatAgentMessageMatchesSharedSpec() { - val msg = - MoltbotCanvasA2UIAction.formatAgentMessage( - actionName = "Get Weather", - sessionKey = "main", - surfaceId = "main", - sourceComponentId = "btnWeather", - host = "Peter’s iPad", - instanceId = "ipad16,6", - contextJson = "{\"city\":\"Vienna\"}", - ) - - assertEquals( - "CANVAS_A2UI action=Get_Weather session=main surface=main component=btnWeather host=Peter_s_iPad instance=ipad16_6 ctx={\"city\":\"Vienna\"} default=update_canvas", - msg, - ) - } - - @Test - fun jsDispatchA2uiStatusIsStable() { - val js = MoltbotCanvasA2UIAction.jsDispatchA2UIActionStatus(actionId = "a1", ok = true, error = null) - assertEquals( - "window.dispatchEvent(new CustomEvent('moltbot:a2ui-action-status', { detail: { id: \"a1\", ok: true, error: \"\" } }));", - js, - ) - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/protocol/ClawdbotProtocolConstantsTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/protocol/ClawdbotProtocolConstantsTest.kt deleted file mode 100644 index 1b96ee9a9..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/protocol/ClawdbotProtocolConstantsTest.kt +++ /dev/null @@ -1,35 +0,0 @@ -package com.clawdbot.android.protocol - -import org.junit.Assert.assertEquals -import org.junit.Test - -class MoltbotProtocolConstantsTest { - @Test - fun canvasCommandsUseStableStrings() { - assertEquals("canvas.present", MoltbotCanvasCommand.Present.rawValue) - assertEquals("canvas.hide", MoltbotCanvasCommand.Hide.rawValue) - assertEquals("canvas.navigate", MoltbotCanvasCommand.Navigate.rawValue) - assertEquals("canvas.eval", MoltbotCanvasCommand.Eval.rawValue) - assertEquals("canvas.snapshot", MoltbotCanvasCommand.Snapshot.rawValue) - } - - @Test - fun a2uiCommandsUseStableStrings() { - assertEquals("canvas.a2ui.push", MoltbotCanvasA2UICommand.Push.rawValue) - assertEquals("canvas.a2ui.pushJSONL", MoltbotCanvasA2UICommand.PushJSONL.rawValue) - assertEquals("canvas.a2ui.reset", MoltbotCanvasA2UICommand.Reset.rawValue) - } - - @Test - fun capabilitiesUseStableStrings() { - assertEquals("canvas", MoltbotCapability.Canvas.rawValue) - assertEquals("camera", MoltbotCapability.Camera.rawValue) - assertEquals("screen", MoltbotCapability.Screen.rawValue) - assertEquals("voiceWake", MoltbotCapability.VoiceWake.rawValue) - } - - @Test - fun screenCommandsUseStableStrings() { - assertEquals("screen.record", MoltbotScreenCommand.Record.rawValue) - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/ui/chat/SessionFiltersTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/ui/chat/SessionFiltersTest.kt deleted file mode 100644 index b945ad66f..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/ui/chat/SessionFiltersTest.kt +++ /dev/null @@ -1,35 +0,0 @@ -package com.clawdbot.android.ui.chat - -import com.clawdbot.android.chat.ChatSessionEntry -import org.junit.Assert.assertEquals -import org.junit.Test - -class SessionFiltersTest { - @Test - fun sessionChoicesPreferMainAndRecent() { - val now = 1_700_000_000_000L - val recent1 = now - 2 * 60 * 60 * 1000L - val recent2 = now - 5 * 60 * 60 * 1000L - val stale = now - 26 * 60 * 60 * 1000L - val sessions = - listOf( - ChatSessionEntry(key = "recent-1", updatedAtMs = recent1), - ChatSessionEntry(key = "main", updatedAtMs = stale), - ChatSessionEntry(key = "old-1", updatedAtMs = stale), - ChatSessionEntry(key = "recent-2", updatedAtMs = recent2), - ) - - val result = resolveSessionChoices("main", sessions, mainSessionKey = "main", nowMs = now).map { it.key } - assertEquals(listOf("main", "recent-1", "recent-2"), result) - } - - @Test - fun sessionChoicesIncludeCurrentWhenMissing() { - val now = 1_700_000_000_000L - val recent = now - 10 * 60 * 1000L - val sessions = listOf(ChatSessionEntry(key = "main", updatedAtMs = recent)) - - val result = resolveSessionChoices("custom", sessions, mainSessionKey = "main", nowMs = now).map { it.key } - assertEquals(listOf("main", "custom"), result) - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/voice/TalkDirectiveParserTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/voice/TalkDirectiveParserTest.kt deleted file mode 100644 index a42b88e3a..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/voice/TalkDirectiveParserTest.kt +++ /dev/null @@ -1,55 +0,0 @@ -package com.clawdbot.android.voice - -import org.junit.Assert.assertEquals -import org.junit.Assert.assertNull -import org.junit.Assert.assertTrue -import org.junit.Test - -class TalkDirectiveParserTest { - @Test - fun parsesDirectiveAndStripsHeader() { - val input = """ - {"voice":"voice-123","once":true} - Hello from talk mode. - """.trimIndent() - val result = TalkDirectiveParser.parse(input) - assertEquals("voice-123", result.directive?.voiceId) - assertEquals(true, result.directive?.once) - assertEquals("Hello from talk mode.", result.stripped.trim()) - } - - @Test - fun ignoresUnknownKeysButReportsThem() { - val input = """ - {"voice":"abc","foo":1,"bar":"baz"} - Hi there. - """.trimIndent() - val result = TalkDirectiveParser.parse(input) - assertEquals("abc", result.directive?.voiceId) - assertTrue(result.unknownKeys.containsAll(listOf("bar", "foo"))) - } - - @Test - fun parsesAlternateKeys() { - val input = """ - {"model_id":"eleven_v3","similarity_boost":0.4,"no_speaker_boost":true,"rate":200} - Speak. - """.trimIndent() - val result = TalkDirectiveParser.parse(input) - assertEquals("eleven_v3", result.directive?.modelId) - assertEquals(0.4, result.directive?.similarity) - assertEquals(false, result.directive?.speakerBoost) - assertEquals(200, result.directive?.rateWpm) - } - - @Test - fun returnsNullWhenNoDirectivePresent() { - val input = """ - {} - Hello. - """.trimIndent() - val result = TalkDirectiveParser.parse(input) - assertNull(result.directive) - assertEquals(input, result.stripped) - } -} diff --git a/apps/android/app/src/test/java/com/clawdbot/android/voice/VoiceWakeCommandExtractorTest.kt b/apps/android/app/src/test/java/com/clawdbot/android/voice/VoiceWakeCommandExtractorTest.kt deleted file mode 100644 index f6e512fa3..000000000 --- a/apps/android/app/src/test/java/com/clawdbot/android/voice/VoiceWakeCommandExtractorTest.kt +++ /dev/null @@ -1,25 +0,0 @@ -package com.clawdbot.android.voice - -import org.junit.Assert.assertEquals -import org.junit.Assert.assertNull -import org.junit.Test - -class VoiceWakeCommandExtractorTest { - @Test - fun extractsCommandAfterTriggerWord() { - val res = VoiceWakeCommandExtractor.extractCommand("Claude take a photo", listOf("clawd", "claude")) - assertEquals("take a photo", res) - } - - @Test - fun extractsCommandWithPunctuation() { - val res = VoiceWakeCommandExtractor.extractCommand("hey clawd, what's the weather?", listOf("clawd")) - assertEquals("what's the weather?", res) - } - - @Test - fun returnsNullWhenNoCommandProvided() { - assertNull(VoiceWakeCommandExtractor.extractCommand("claude", listOf("claude"))) - assertNull(VoiceWakeCommandExtractor.extractCommand("hey claude!", listOf("claude"))) - } -}