Include full contents of all nested repositories

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-27 16:25:02 +01:00
parent 14ff8fd54c
commit 2401ed446f
7271 changed files with 1310112 additions and 6 deletions

View File

@@ -0,0 +1,156 @@
import com.android.build.api.variant.impl.VariantOutputImpl
plugins {
id("com.android.application")
id("org.jetbrains.kotlin.plugin.compose")
id("org.jetbrains.kotlin.plugin.serialization")
}
android {
namespace = "ai.openclaw.android"
compileSdk = 36
sourceSets {
getByName("main") {
assets.directories.add("../../shared/OpenClawKit/Sources/OpenClawKit/Resources")
}
}
defaultConfig {
applicationId = "ai.openclaw.android"
minSdk = 31
targetSdk = 36
versionCode = 202602260
versionName = "2026.2.26"
ndk {
// Support all major ABIs — native libs are tiny (~47 KB per ABI)
abiFilters += listOf("armeabi-v7a", "arm64-v8a", "x86", "x86_64")
}
}
buildTypes {
release {
isMinifyEnabled = true
isShrinkResources = true
proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
}
debug {
isMinifyEnabled = false
}
}
buildFeatures {
compose = true
buildConfig = true
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
}
packaging {
resources {
excludes += setOf(
"/META-INF/{AL2.0,LGPL2.1}",
"/META-INF/*.version",
"/META-INF/LICENSE*.txt",
"DebugProbesKt.bin",
"kotlin-tooling-metadata.json",
)
}
}
lint {
disable += setOf(
"GradleDependency",
"IconLauncherShape",
"NewerVersionAvailable",
)
warningsAsErrors = true
}
testOptions {
unitTests.isIncludeAndroidResources = true
}
}
androidComponents {
onVariants { variant ->
variant.outputs
.filterIsInstance<VariantOutputImpl>()
.forEach { output ->
val versionName = output.versionName.orNull ?: "0"
val buildType = variant.buildType
val outputFileName = "openclaw-${versionName}-${buildType}.apk"
output.outputFileName = outputFileName
}
}
}
kotlin {
compilerOptions {
jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17)
allWarningsAsErrors.set(true)
}
}
dependencies {
val composeBom = platform("androidx.compose:compose-bom:2026.02.00")
implementation(composeBom)
androidTestImplementation(composeBom)
implementation("androidx.core:core-ktx:1.17.0")
implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.10.0")
implementation("androidx.activity:activity-compose:1.12.2")
implementation("androidx.webkit:webkit:1.15.0")
implementation("androidx.compose.ui:ui")
implementation("androidx.compose.ui:ui-tooling-preview")
implementation("androidx.compose.material3:material3")
// material-icons-extended pulled in full icon set (~20 MB DEX). Only ~18 icons used.
// R8 will tree-shake unused icons when minify is enabled on release builds.
implementation("androidx.compose.material:material-icons-extended")
implementation("androidx.navigation:navigation-compose:2.9.7")
debugImplementation("androidx.compose.ui:ui-tooling")
// Material Components (XML theme + resources)
implementation("com.google.android.material:material:1.13.0")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.10.2")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.10.0")
implementation("androidx.security:security-crypto:1.1.0")
implementation("androidx.exifinterface:exifinterface:1.4.2")
implementation("com.squareup.okhttp3:okhttp:5.3.2")
implementation("org.bouncycastle:bcprov-jdk18on:1.83")
implementation("org.commonmark:commonmark:0.27.1")
implementation("org.commonmark:commonmark-ext-autolink:0.27.1")
implementation("org.commonmark:commonmark-ext-gfm-strikethrough:0.27.1")
implementation("org.commonmark:commonmark-ext-gfm-tables:0.27.1")
implementation("org.commonmark:commonmark-ext-task-list-items:0.27.1")
// CameraX (for node.invoke camera.* parity)
implementation("androidx.camera:camera-core:1.5.2")
implementation("androidx.camera:camera-camera2:1.5.2")
implementation("androidx.camera:camera-lifecycle:1.5.2")
implementation("androidx.camera:camera-video:1.5.2")
implementation("androidx.camera:camera-view:1.5.2")
implementation("com.journeyapps:zxing-android-embedded:4.3.0")
// Unicast DNS-SD (Wide-Area Bonjour) for tailnet discovery domains.
implementation("dnsjava:dnsjava:3.6.4")
testImplementation("junit:junit:4.13.2")
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
testImplementation("io.kotest:kotest-runner-junit5-jvm:6.1.3")
testImplementation("io.kotest:kotest-assertions-core-jvm:6.1.3")
testImplementation("com.squareup.okhttp3:mockwebserver:5.3.2")
testImplementation("org.robolectric:robolectric:4.16.1")
testRuntimeOnly("org.junit.vintage:junit-vintage-engine:6.0.2")
}
tasks.withType<Test>().configureEach {
useJUnitPlatform()
}

View File

@@ -0,0 +1,28 @@
# ── App classes ───────────────────────────────────────────────────
-keep class ai.openclaw.android.** { *; }
# ── Bouncy Castle ─────────────────────────────────────────────────
-keep class org.bouncycastle.** { *; }
-dontwarn org.bouncycastle.**
# ── CameraX ───────────────────────────────────────────────────────
-keep class androidx.camera.** { *; }
# ── kotlinx.serialization ────────────────────────────────────────
-keep class kotlinx.serialization.** { *; }
-keepclassmembers class * {
@kotlinx.serialization.Serializable *;
}
-keepattributes *Annotation*, InnerClasses
# ── OkHttp ────────────────────────────────────────────────────────
-dontwarn okhttp3.**
-dontwarn okio.**
-keep class okhttp3.internal.platform.** { *; }
# ── Misc suppressions ────────────────────────────────────────────
-dontwarn com.sun.jna.**
-dontwarn javax.naming.**
-dontwarn lombok.Generated
-dontwarn org.slf4j.impl.StaticLoggerBinder
-dontwarn sun.net.spi.nameservice.NameServiceDescriptor

View File

@@ -0,0 +1,74 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<uses-permission
android:name="android.permission.NEARBY_WIFI_DEVICES"
android:usesPermissionFlags="neverForLocation" />
<uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />
<uses-permission android:name="android.permission.ACCESS_COARSE_LOCATION" />
<uses-permission android:name="android.permission.ACCESS_BACKGROUND_LOCATION" />
<uses-permission android:name="android.permission.CAMERA" />
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<uses-permission android:name="android.permission.SEND_SMS" />
<uses-permission android:name="android.permission.REQUEST_INSTALL_PACKAGES" />
<uses-feature
android:name="android.hardware.camera"
android:required="false" />
<uses-feature
android:name="android.hardware.telephony"
android:required="false" />
<application
android:name=".NodeApp"
android:allowBackup="true"
android:dataExtractionRules="@xml/data_extraction_rules"
android:fullBackupContent="@xml/backup_rules"
android:icon="@mipmap/ic_launcher"
android:roundIcon="@mipmap/ic_launcher_round"
android:label="@string/app_name"
android:supportsRtl="true"
android:networkSecurityConfig="@xml/network_security_config"
android:theme="@style/Theme.OpenClawNode">
<service
android:name=".NodeForegroundService"
android:exported="false"
android:foregroundServiceType="dataSync|microphone|mediaProjection" />
<service
android:name=".node.DeviceNotificationListenerService"
android:label="@string/app_name"
android:permission="android.permission.BIND_NOTIFICATION_LISTENER_SERVICE"
android:exported="false">
<intent-filter>
<action android:name="android.service.notification.NotificationListenerService" />
</intent-filter>
</service>
<provider
android:name="androidx.core.content.FileProvider"
android:authorities="${applicationId}.fileprovider"
android:exported="false"
android:grantUriPermissions="true">
<meta-data
android:name="android.support.FILE_PROVIDER_PATHS"
android:resource="@xml/file_paths" />
</provider>
<activity
android:name=".MainActivity"
android:exported="true"
android:windowSoftInputMode="adjustResize"
android:configChanges="orientation|screenSize|screenLayout|smallestScreenSize|uiMode|density|keyboard|keyboardHidden|navigation">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
<receiver
android:name=".InstallResultReceiver"
android:exported="false" />
</application>
</manifest>

View File

@@ -0,0 +1,14 @@
package ai.openclaw.android
enum class CameraHudKind {
Photo,
Recording,
Success,
Error,
}
data class CameraHudState(
val token: Long,
val kind: CameraHudKind,
val message: String,
)

View File

@@ -0,0 +1,26 @@
package ai.openclaw.android
import android.content.Context
import android.os.Build
import android.provider.Settings
object DeviceNames {
fun bestDefaultNodeName(context: Context): String {
val deviceName =
runCatching {
Settings.Global.getString(context.contentResolver, "device_name")
}
.getOrNull()
?.trim()
.orEmpty()
if (deviceName.isNotEmpty()) return deviceName
val model =
listOfNotNull(Build.MANUFACTURER?.takeIf { it.isNotBlank() }, Build.MODEL?.takeIf { it.isNotBlank() })
.joinToString(" ")
.trim()
return model.ifEmpty { "Android Node" }
}
}

View File

@@ -0,0 +1,33 @@
package ai.openclaw.android
import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.pm.PackageInstaller
import android.util.Log
class InstallResultReceiver : BroadcastReceiver() {
override fun onReceive(context: Context, intent: Intent) {
val status = intent.getIntExtra(PackageInstaller.EXTRA_STATUS, PackageInstaller.STATUS_FAILURE)
val message = intent.getStringExtra(PackageInstaller.EXTRA_STATUS_MESSAGE)
when (status) {
PackageInstaller.STATUS_PENDING_USER_ACTION -> {
// System needs user confirmation — launch the confirmation activity
@Suppress("DEPRECATION")
val confirmIntent = intent.getParcelableExtra<Intent>(Intent.EXTRA_INTENT)
if (confirmIntent != null) {
confirmIntent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
context.startActivity(confirmIntent)
Log.w("openclaw", "app.update: user confirmation requested, launching install dialog")
}
}
PackageInstaller.STATUS_SUCCESS -> {
Log.w("openclaw", "app.update: install SUCCESS")
}
else -> {
Log.e("openclaw", "app.update: install FAILED status=$status message=$message")
}
}
}
}

View File

@@ -0,0 +1,15 @@
package ai.openclaw.android
enum class LocationMode(val rawValue: String) {
Off("off"),
WhileUsing("whileUsing"),
Always("always"),
;
companion object {
fun fromRawValue(raw: String?): LocationMode {
val normalized = raw?.trim()?.lowercase()
return entries.firstOrNull { it.rawValue.lowercase() == normalized } ?: Off
}
}
}

View File

@@ -0,0 +1,67 @@
package ai.openclaw.android
import android.os.Bundle
import android.view.WindowManager
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.activity.viewModels
import androidx.core.view.WindowCompat
import androidx.compose.material3.Surface
import androidx.compose.ui.Modifier
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.lifecycleScope
import androidx.lifecycle.repeatOnLifecycle
import ai.openclaw.android.ui.RootScreen
import ai.openclaw.android.ui.OpenClawTheme
import kotlinx.coroutines.launch
class MainActivity : ComponentActivity() {
private val viewModel: MainViewModel by viewModels()
private lateinit var permissionRequester: PermissionRequester
private lateinit var screenCaptureRequester: ScreenCaptureRequester
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
WindowCompat.setDecorFitsSystemWindows(window, false)
permissionRequester = PermissionRequester(this)
screenCaptureRequester = ScreenCaptureRequester(this)
viewModel.camera.attachLifecycleOwner(this)
viewModel.camera.attachPermissionRequester(permissionRequester)
viewModel.sms.attachPermissionRequester(permissionRequester)
viewModel.screenRecorder.attachScreenCaptureRequester(screenCaptureRequester)
viewModel.screenRecorder.attachPermissionRequester(permissionRequester)
lifecycleScope.launch {
repeatOnLifecycle(Lifecycle.State.STARTED) {
viewModel.preventSleep.collect { enabled ->
if (enabled) {
window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
} else {
window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
}
}
}
}
setContent {
OpenClawTheme {
Surface(modifier = Modifier) {
RootScreen(viewModel = viewModel)
}
}
}
// Keep startup path lean: start foreground service after first frame.
window.decorView.post { NodeForegroundService.start(this) }
}
override fun onStart() {
super.onStart()
viewModel.setForeground(true)
}
override fun onStop() {
viewModel.setForeground(false)
super.onStop()
}
}

View File

@@ -0,0 +1,195 @@
package ai.openclaw.android
import android.app.Application
import androidx.lifecycle.AndroidViewModel
import ai.openclaw.android.gateway.GatewayEndpoint
import ai.openclaw.android.chat.OutgoingAttachment
import ai.openclaw.android.node.CameraCaptureManager
import ai.openclaw.android.node.CanvasController
import ai.openclaw.android.node.ScreenRecordManager
import ai.openclaw.android.node.SmsManager
import ai.openclaw.android.voice.VoiceConversationEntry
import kotlinx.coroutines.flow.StateFlow
class MainViewModel(app: Application) : AndroidViewModel(app) {
private val runtime: NodeRuntime = (app as NodeApp).runtime
val canvas: CanvasController = runtime.canvas
val canvasCurrentUrl: StateFlow<String?> = runtime.canvas.currentUrl
val canvasA2uiHydrated: StateFlow<Boolean> = runtime.canvasA2uiHydrated
val canvasRehydratePending: StateFlow<Boolean> = runtime.canvasRehydratePending
val canvasRehydrateErrorText: StateFlow<String?> = runtime.canvasRehydrateErrorText
val camera: CameraCaptureManager = runtime.camera
val screenRecorder: ScreenRecordManager = runtime.screenRecorder
val sms: SmsManager = runtime.sms
val gateways: StateFlow<List<GatewayEndpoint>> = runtime.gateways
val discoveryStatusText: StateFlow<String> = runtime.discoveryStatusText
val isConnected: StateFlow<Boolean> = runtime.isConnected
val isNodeConnected: StateFlow<Boolean> = runtime.nodeConnected
val statusText: StateFlow<String> = runtime.statusText
val serverName: StateFlow<String?> = runtime.serverName
val remoteAddress: StateFlow<String?> = runtime.remoteAddress
val pendingGatewayTrust: StateFlow<NodeRuntime.GatewayTrustPrompt?> = runtime.pendingGatewayTrust
val isForeground: StateFlow<Boolean> = runtime.isForeground
val seamColorArgb: StateFlow<Long> = runtime.seamColorArgb
val mainSessionKey: StateFlow<String> = runtime.mainSessionKey
val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
val screenRecordActive: StateFlow<Boolean> = runtime.screenRecordActive
val instanceId: StateFlow<String> = runtime.instanceId
val displayName: StateFlow<String> = runtime.displayName
val cameraEnabled: StateFlow<Boolean> = runtime.cameraEnabled
val locationMode: StateFlow<LocationMode> = runtime.locationMode
val locationPreciseEnabled: StateFlow<Boolean> = runtime.locationPreciseEnabled
val preventSleep: StateFlow<Boolean> = runtime.preventSleep
val micEnabled: StateFlow<Boolean> = runtime.micEnabled
val micStatusText: StateFlow<String> = runtime.micStatusText
val micLiveTranscript: StateFlow<String?> = runtime.micLiveTranscript
val micIsListening: StateFlow<Boolean> = runtime.micIsListening
val micQueuedMessages: StateFlow<List<String>> = runtime.micQueuedMessages
val micConversation: StateFlow<List<VoiceConversationEntry>> = runtime.micConversation
val micInputLevel: StateFlow<Float> = runtime.micInputLevel
val micIsSending: StateFlow<Boolean> = runtime.micIsSending
val manualEnabled: StateFlow<Boolean> = runtime.manualEnabled
val manualHost: StateFlow<String> = runtime.manualHost
val manualPort: StateFlow<Int> = runtime.manualPort
val manualTls: StateFlow<Boolean> = runtime.manualTls
val gatewayToken: StateFlow<String> = runtime.gatewayToken
val onboardingCompleted: StateFlow<Boolean> = runtime.onboardingCompleted
val canvasDebugStatusEnabled: StateFlow<Boolean> = runtime.canvasDebugStatusEnabled
val chatSessionKey: StateFlow<String> = runtime.chatSessionKey
val chatSessionId: StateFlow<String?> = runtime.chatSessionId
val chatMessages = runtime.chatMessages
val chatError: StateFlow<String?> = runtime.chatError
val chatHealthOk: StateFlow<Boolean> = runtime.chatHealthOk
val chatThinkingLevel: StateFlow<String> = runtime.chatThinkingLevel
val chatStreamingAssistantText: StateFlow<String?> = runtime.chatStreamingAssistantText
val chatPendingToolCalls = runtime.chatPendingToolCalls
val chatSessions = runtime.chatSessions
val pendingRunCount: StateFlow<Int> = runtime.pendingRunCount
fun setForeground(value: Boolean) {
runtime.setForeground(value)
}
fun setDisplayName(value: String) {
runtime.setDisplayName(value)
}
fun setCameraEnabled(value: Boolean) {
runtime.setCameraEnabled(value)
}
fun setLocationMode(mode: LocationMode) {
runtime.setLocationMode(mode)
}
fun setLocationPreciseEnabled(value: Boolean) {
runtime.setLocationPreciseEnabled(value)
}
fun setPreventSleep(value: Boolean) {
runtime.setPreventSleep(value)
}
fun setManualEnabled(value: Boolean) {
runtime.setManualEnabled(value)
}
fun setManualHost(value: String) {
runtime.setManualHost(value)
}
fun setManualPort(value: Int) {
runtime.setManualPort(value)
}
fun setManualTls(value: Boolean) {
runtime.setManualTls(value)
}
fun setGatewayToken(value: String) {
runtime.setGatewayToken(value)
}
fun setGatewayPassword(value: String) {
runtime.setGatewayPassword(value)
}
fun setOnboardingCompleted(value: Boolean) {
runtime.setOnboardingCompleted(value)
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
runtime.setCanvasDebugStatusEnabled(value)
}
fun setMicEnabled(enabled: Boolean) {
runtime.setMicEnabled(enabled)
}
fun refreshGatewayConnection() {
runtime.refreshGatewayConnection()
}
fun connect(endpoint: GatewayEndpoint) {
runtime.connect(endpoint)
}
fun connectManual() {
runtime.connectManual()
}
fun disconnect() {
runtime.disconnect()
}
fun acceptGatewayTrustPrompt() {
runtime.acceptGatewayTrustPrompt()
}
fun declineGatewayTrustPrompt() {
runtime.declineGatewayTrustPrompt()
}
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
runtime.handleCanvasA2UIActionFromWebView(payloadJson)
}
fun requestCanvasRehydrate(source: String = "screen_tab") {
runtime.requestCanvasRehydrate(source = source, force = true)
}
fun loadChat(sessionKey: String) {
runtime.loadChat(sessionKey)
}
fun refreshChat() {
runtime.refreshChat()
}
fun refreshChatSessions(limit: Int? = null) {
runtime.refreshChatSessions(limit = limit)
}
fun setChatThinkingLevel(level: String) {
runtime.setChatThinkingLevel(level)
}
fun switchChatSession(sessionKey: String) {
runtime.switchChatSession(sessionKey)
}
fun abortChat() {
runtime.abortChat()
}
fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
runtime.sendChat(message = message, thinking = thinking, attachments = attachments)
}
}

View File

@@ -0,0 +1,26 @@
package ai.openclaw.android
import android.app.Application
import android.os.StrictMode
class NodeApp : Application() {
val runtime: NodeRuntime by lazy { NodeRuntime(this) }
override fun onCreate() {
super.onCreate()
if (BuildConfig.DEBUG) {
StrictMode.setThreadPolicy(
StrictMode.ThreadPolicy.Builder()
.detectAll()
.penaltyLog()
.build(),
)
StrictMode.setVmPolicy(
StrictMode.VmPolicy.Builder()
.detectAll()
.penaltyLog()
.build(),
)
}
}
}

View File

@@ -0,0 +1,180 @@
package ai.openclaw.android
import android.app.Notification
import android.app.NotificationChannel
import android.app.NotificationManager
import android.app.Service
import android.app.PendingIntent
import android.Manifest
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.content.pm.ServiceInfo
import androidx.core.app.NotificationCompat
import androidx.core.content.ContextCompat
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.launch
class NodeForegroundService : Service() {
private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
private var notificationJob: Job? = null
private var lastRequiresMic = false
private var didStartForeground = false
override fun onCreate() {
super.onCreate()
ensureChannel()
val initial = buildNotification(title = "OpenClaw Node", text = "Starting…")
startForegroundWithTypes(notification = initial, requiresMic = false)
val runtime = (application as NodeApp).runtime
notificationJob =
scope.launch {
combine(
runtime.statusText,
runtime.serverName,
runtime.isConnected,
runtime.micEnabled,
runtime.micIsListening,
) { status, server, connected, micEnabled, micListening ->
Quint(status, server, connected, micEnabled, micListening)
}.collect { (status, server, connected, micEnabled, micListening) ->
val title = if (connected) "OpenClaw Node · Connected" else "OpenClaw Node"
val micSuffix =
if (micEnabled) {
if (micListening) " · Mic: Listening" else " · Mic: Pending"
} else {
""
}
val text = (server?.let { "$status · $it" } ?: status) + micSuffix
val requiresMic =
micEnabled && hasRecordAudioPermission()
startForegroundWithTypes(
notification = buildNotification(title = title, text = text),
requiresMic = requiresMic,
)
}
}
}
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
when (intent?.action) {
ACTION_STOP -> {
(application as NodeApp).runtime.disconnect()
stopSelf()
return START_NOT_STICKY
}
}
// Keep running; connection is managed by NodeRuntime (auto-reconnect + manual).
return START_STICKY
}
override fun onDestroy() {
notificationJob?.cancel()
scope.cancel()
super.onDestroy()
}
override fun onBind(intent: Intent?) = null
private fun ensureChannel() {
val mgr = getSystemService(NotificationManager::class.java)
val channel =
NotificationChannel(
CHANNEL_ID,
"Connection",
NotificationManager.IMPORTANCE_LOW,
).apply {
description = "OpenClaw node connection status"
setShowBadge(false)
}
mgr.createNotificationChannel(channel)
}
private fun buildNotification(title: String, text: String): Notification {
val launchIntent = Intent(this, MainActivity::class.java).apply {
flags = Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP
}
val launchPending =
PendingIntent.getActivity(
this,
1,
launchIntent,
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
)
val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP)
val stopPending =
PendingIntent.getService(
this,
2,
stopIntent,
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
)
return NotificationCompat.Builder(this, CHANNEL_ID)
.setSmallIcon(R.mipmap.ic_launcher)
.setContentTitle(title)
.setContentText(text)
.setContentIntent(launchPending)
.setOngoing(true)
.setOnlyAlertOnce(true)
.setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE)
.addAction(0, "Disconnect", stopPending)
.build()
}
private fun updateNotification(notification: Notification) {
val mgr = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
mgr.notify(NOTIFICATION_ID, notification)
}
private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) {
if (didStartForeground && requiresMic == lastRequiresMic) {
updateNotification(notification)
return
}
lastRequiresMic = requiresMic
val types =
if (requiresMic) {
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
} else {
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC
}
startForeground(NOTIFICATION_ID, notification, types)
didStartForeground = true
}
private fun hasRecordAudioPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
)
}
companion object {
private const val CHANNEL_ID = "connection"
private const val NOTIFICATION_ID = 1
private const val ACTION_STOP = "ai.openclaw.android.action.STOP"
fun start(context: Context) {
val intent = Intent(context, NodeForegroundService::class.java)
context.startForegroundService(intent)
}
fun stop(context: Context) {
val intent = Intent(context, NodeForegroundService::class.java).setAction(ACTION_STOP)
context.startService(intent)
}
}
}
private data class Quint<A, B, C, D, E>(val first: A, val second: B, val third: C, val fourth: D, val fifth: E)

View File

@@ -0,0 +1,813 @@
package ai.openclaw.android
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.os.SystemClock
import android.util.Log
import androidx.core.content.ContextCompat
import ai.openclaw.android.chat.ChatController
import ai.openclaw.android.chat.ChatMessage
import ai.openclaw.android.chat.ChatPendingToolCall
import ai.openclaw.android.chat.ChatSessionEntry
import ai.openclaw.android.chat.OutgoingAttachment
import ai.openclaw.android.gateway.DeviceAuthStore
import ai.openclaw.android.gateway.DeviceIdentityStore
import ai.openclaw.android.gateway.GatewayDiscovery
import ai.openclaw.android.gateway.GatewayEndpoint
import ai.openclaw.android.gateway.GatewaySession
import ai.openclaw.android.gateway.probeGatewayTlsFingerprint
import ai.openclaw.android.node.*
import ai.openclaw.android.protocol.OpenClawCanvasA2UIAction
import ai.openclaw.android.voice.MicCaptureManager
import ai.openclaw.android.voice.VoiceConversationEntry
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.distinctUntilChanged
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import java.util.UUID
import java.util.concurrent.atomic.AtomicLong
class NodeRuntime(context: Context) {
private val appContext = context.applicationContext
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
val prefs = SecurePrefs(appContext)
private val deviceAuthStore = DeviceAuthStore(prefs)
val canvas = CanvasController()
val camera = CameraCaptureManager(appContext)
val location = LocationCaptureManager(appContext)
val screenRecorder = ScreenRecordManager(appContext)
val sms = SmsManager(appContext)
private val json = Json { ignoreUnknownKeys = true }
private val externalAudioCaptureActive = MutableStateFlow(false)
private val discovery = GatewayDiscovery(appContext, scope = scope)
val gateways: StateFlow<List<GatewayEndpoint>> = discovery.gateways
val discoveryStatusText: StateFlow<String> = discovery.statusText
private val identityStore = DeviceIdentityStore(appContext)
private var connectedEndpoint: GatewayEndpoint? = null
private val cameraHandler: CameraHandler = CameraHandler(
appContext = appContext,
camera = camera,
externalAudioCaptureActive = externalAudioCaptureActive,
showCameraHud = ::showCameraHud,
triggerCameraFlash = ::triggerCameraFlash,
invokeErrorFromThrowable = { invokeErrorFromThrowable(it) },
)
private val debugHandler: DebugHandler = DebugHandler(
appContext = appContext,
identityStore = identityStore,
)
private val appUpdateHandler: AppUpdateHandler = AppUpdateHandler(
appContext = appContext,
connectedEndpoint = { connectedEndpoint },
)
private val locationHandler: LocationHandler = LocationHandler(
appContext = appContext,
location = location,
json = json,
isForeground = { _isForeground.value },
locationMode = { locationMode.value },
locationPreciseEnabled = { locationPreciseEnabled.value },
)
private val deviceHandler: DeviceHandler = DeviceHandler(
appContext = appContext,
)
private val notificationsHandler: NotificationsHandler = NotificationsHandler(
appContext = appContext,
)
private val screenHandler: ScreenHandler = ScreenHandler(
screenRecorder = screenRecorder,
setScreenRecordActive = { _screenRecordActive.value = it },
invokeErrorFromThrowable = { invokeErrorFromThrowable(it) },
)
private val smsHandlerImpl: SmsHandler = SmsHandler(
sms = sms,
)
private val a2uiHandler: A2UIHandler = A2UIHandler(
canvas = canvas,
json = json,
getNodeCanvasHostUrl = { nodeSession.currentCanvasHostUrl() },
getOperatorCanvasHostUrl = { operatorSession.currentCanvasHostUrl() },
)
private val connectionManager: ConnectionManager = ConnectionManager(
prefs = prefs,
cameraEnabled = { cameraEnabled.value },
locationMode = { locationMode.value },
voiceWakeMode = { VoiceWakeMode.Off },
smsAvailable = { sms.canSendSms() },
hasRecordAudioPermission = { hasRecordAudioPermission() },
manualTls = { manualTls.value },
)
private val invokeDispatcher: InvokeDispatcher = InvokeDispatcher(
canvas = canvas,
cameraHandler = cameraHandler,
locationHandler = locationHandler,
deviceHandler = deviceHandler,
notificationsHandler = notificationsHandler,
screenHandler = screenHandler,
smsHandler = smsHandlerImpl,
a2uiHandler = a2uiHandler,
debugHandler = debugHandler,
appUpdateHandler = appUpdateHandler,
isForeground = { _isForeground.value },
cameraEnabled = { cameraEnabled.value },
locationEnabled = { locationMode.value != LocationMode.Off },
smsAvailable = { sms.canSendSms() },
debugBuild = { BuildConfig.DEBUG },
refreshNodeCanvasCapability = { nodeSession.refreshNodeCanvasCapability() },
onCanvasA2uiPush = {
_canvasA2uiHydrated.value = true
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = null
},
onCanvasA2uiReset = { _canvasA2uiHydrated.value = false },
)
data class GatewayTrustPrompt(
val endpoint: GatewayEndpoint,
val fingerprintSha256: String,
)
private val _isConnected = MutableStateFlow(false)
val isConnected: StateFlow<Boolean> = _isConnected.asStateFlow()
private val _nodeConnected = MutableStateFlow(false)
val nodeConnected: StateFlow<Boolean> = _nodeConnected.asStateFlow()
private val _statusText = MutableStateFlow("Offline")
val statusText: StateFlow<String> = _statusText.asStateFlow()
private val _pendingGatewayTrust = MutableStateFlow<GatewayTrustPrompt?>(null)
val pendingGatewayTrust: StateFlow<GatewayTrustPrompt?> = _pendingGatewayTrust.asStateFlow()
private val _mainSessionKey = MutableStateFlow("main")
val mainSessionKey: StateFlow<String> = _mainSessionKey.asStateFlow()
private val cameraHudSeq = AtomicLong(0)
private val _cameraHud = MutableStateFlow<CameraHudState?>(null)
val cameraHud: StateFlow<CameraHudState?> = _cameraHud.asStateFlow()
private val _cameraFlashToken = MutableStateFlow(0L)
val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
private val _screenRecordActive = MutableStateFlow(false)
val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
private val _canvasA2uiHydrated = MutableStateFlow(false)
val canvasA2uiHydrated: StateFlow<Boolean> = _canvasA2uiHydrated.asStateFlow()
private val _canvasRehydratePending = MutableStateFlow(false)
val canvasRehydratePending: StateFlow<Boolean> = _canvasRehydratePending.asStateFlow()
private val _canvasRehydrateErrorText = MutableStateFlow<String?>(null)
val canvasRehydrateErrorText: StateFlow<String?> = _canvasRehydrateErrorText.asStateFlow()
private val _serverName = MutableStateFlow<String?>(null)
val serverName: StateFlow<String?> = _serverName.asStateFlow()
private val _remoteAddress = MutableStateFlow<String?>(null)
val remoteAddress: StateFlow<String?> = _remoteAddress.asStateFlow()
private val _seamColorArgb = MutableStateFlow(DEFAULT_SEAM_COLOR_ARGB)
val seamColorArgb: StateFlow<Long> = _seamColorArgb.asStateFlow()
private val _isForeground = MutableStateFlow(true)
val isForeground: StateFlow<Boolean> = _isForeground.asStateFlow()
private var lastAutoA2uiUrl: String? = null
private var didAutoRequestCanvasRehydrate = false
private val canvasRehydrateSeq = AtomicLong(0)
private var operatorConnected = false
private var operatorStatusText: String = "Offline"
private var nodeStatusText: String = "Offline"
private val operatorSession =
GatewaySession(
scope = scope,
identityStore = identityStore,
deviceAuthStore = deviceAuthStore,
onConnected = { name, remote, mainSessionKey ->
operatorConnected = true
operatorStatusText = "Connected"
_serverName.value = name
_remoteAddress.value = remote
_seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
applyMainSessionKey(mainSessionKey)
updateStatus()
micCapture.onGatewayConnectionChanged(true)
scope.launch { refreshBrandingFromGateway() }
},
onDisconnected = { message ->
operatorConnected = false
operatorStatusText = message
_serverName.value = null
_remoteAddress.value = null
_seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
if (!isCanonicalMainSessionKey(_mainSessionKey.value)) {
_mainSessionKey.value = "main"
}
chat.applyMainSessionKey(resolveMainSessionKey())
chat.onDisconnected(message)
updateStatus()
micCapture.onGatewayConnectionChanged(false)
},
onEvent = { event, payloadJson ->
handleGatewayEvent(event, payloadJson)
},
)
private val nodeSession =
GatewaySession(
scope = scope,
identityStore = identityStore,
deviceAuthStore = deviceAuthStore,
onConnected = { _, _, _ ->
_nodeConnected.value = true
nodeStatusText = "Connected"
didAutoRequestCanvasRehydrate = false
_canvasA2uiHydrated.value = false
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = null
updateStatus()
maybeNavigateToA2uiOnConnect()
},
onDisconnected = { message ->
_nodeConnected.value = false
nodeStatusText = message
didAutoRequestCanvasRehydrate = false
_canvasA2uiHydrated.value = false
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = null
updateStatus()
showLocalCanvasOnDisconnect()
},
onEvent = { _, _ -> },
onInvoke = { req ->
invokeDispatcher.handleInvoke(req.command, req.paramsJson)
},
onTlsFingerprint = { stableId, fingerprint ->
prefs.saveGatewayTlsFingerprint(stableId, fingerprint)
},
)
private val chat: ChatController =
ChatController(
scope = scope,
session = operatorSession,
json = json,
supportsChatSubscribe = false,
)
private val micCapture: MicCaptureManager by lazy {
MicCaptureManager(
context = appContext,
scope = scope,
sendToGateway = { message ->
val idempotencyKey = UUID.randomUUID().toString()
val params =
buildJsonObject {
put("sessionKey", JsonPrimitive(resolveMainSessionKey()))
put("message", JsonPrimitive(message))
put("thinking", JsonPrimitive(chatThinkingLevel.value))
put("timeoutMs", JsonPrimitive(30_000))
put("idempotencyKey", JsonPrimitive(idempotencyKey))
}
val response = operatorSession.request("chat.send", params.toString())
parseChatSendRunId(response) ?: idempotencyKey
},
)
}
val micStatusText: StateFlow<String>
get() = micCapture.statusText
val micLiveTranscript: StateFlow<String?>
get() = micCapture.liveTranscript
val micIsListening: StateFlow<Boolean>
get() = micCapture.isListening
val micEnabled: StateFlow<Boolean>
get() = micCapture.micEnabled
val micQueuedMessages: StateFlow<List<String>>
get() = micCapture.queuedMessages
val micConversation: StateFlow<List<VoiceConversationEntry>>
get() = micCapture.conversation
val micInputLevel: StateFlow<Float>
get() = micCapture.inputLevel
val micIsSending: StateFlow<Boolean>
get() = micCapture.isSending
private fun applyMainSessionKey(candidate: String?) {
val trimmed = normalizeMainKey(candidate) ?: return
if (isCanonicalMainSessionKey(_mainSessionKey.value)) return
if (_mainSessionKey.value == trimmed) return
_mainSessionKey.value = trimmed
chat.applyMainSessionKey(trimmed)
}
private fun updateStatus() {
_isConnected.value = operatorConnected
val operator = operatorStatusText.trim()
val node = nodeStatusText.trim()
_statusText.value =
when {
operatorConnected && _nodeConnected.value -> "Connected"
operatorConnected && !_nodeConnected.value -> "Connected (node offline)"
!operatorConnected && _nodeConnected.value ->
if (operator.isNotEmpty() && operator != "Offline") {
"Connected (operator: $operator)"
} else {
"Connected (operator offline)"
}
operator.isNotBlank() && operator != "Offline" -> operator
else -> node
}
}
private fun resolveMainSessionKey(): String {
val trimmed = _mainSessionKey.value.trim()
return if (trimmed.isEmpty()) "main" else trimmed
}
private fun maybeNavigateToA2uiOnConnect() {
val a2uiUrl = a2uiHandler.resolveA2uiHostUrl() ?: return
val current = canvas.currentUrl()?.trim().orEmpty()
if (current.isEmpty() || current == lastAutoA2uiUrl) {
lastAutoA2uiUrl = a2uiUrl
canvas.navigate(a2uiUrl)
}
}
private fun showLocalCanvasOnDisconnect() {
lastAutoA2uiUrl = null
_canvasA2uiHydrated.value = false
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = null
canvas.navigate("")
}
fun requestCanvasRehydrate(source: String = "manual", force: Boolean = true) {
scope.launch {
if (!_nodeConnected.value) {
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = "Node offline. Reconnect and retry."
return@launch
}
if (!force && didAutoRequestCanvasRehydrate) return@launch
didAutoRequestCanvasRehydrate = true
val requestId = canvasRehydrateSeq.incrementAndGet()
_canvasRehydratePending.value = true
_canvasRehydrateErrorText.value = null
val sessionKey = resolveMainSessionKey()
val prompt =
"Restore canvas now for session=$sessionKey source=$source. " +
"If existing A2UI state exists, replay it immediately. " +
"If not, create and render a compact mobile-friendly dashboard in Canvas."
val sent =
nodeSession.sendNodeEvent(
event = "agent.request",
payloadJson =
buildJsonObject {
put("message", JsonPrimitive(prompt))
put("sessionKey", JsonPrimitive(sessionKey))
put("thinking", JsonPrimitive("low"))
put("deliver", JsonPrimitive(false))
}.toString(),
)
if (!sent) {
if (!force) {
didAutoRequestCanvasRehydrate = false
}
if (canvasRehydrateSeq.get() == requestId) {
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = "Failed to request restore. Tap to retry."
}
Log.w("OpenClawCanvas", "canvas rehydrate request failed ($source): transport unavailable")
return@launch
}
scope.launch {
delay(20_000)
if (canvasRehydrateSeq.get() != requestId) return@launch
if (!_canvasRehydratePending.value) return@launch
if (_canvasA2uiHydrated.value) return@launch
_canvasRehydratePending.value = false
_canvasRehydrateErrorText.value = "No canvas update yet. Tap to retry."
}
}
}
val instanceId: StateFlow<String> = prefs.instanceId
val displayName: StateFlow<String> = prefs.displayName
val cameraEnabled: StateFlow<Boolean> = prefs.cameraEnabled
val locationMode: StateFlow<LocationMode> = prefs.locationMode
val locationPreciseEnabled: StateFlow<Boolean> = prefs.locationPreciseEnabled
val preventSleep: StateFlow<Boolean> = prefs.preventSleep
val manualEnabled: StateFlow<Boolean> = prefs.manualEnabled
val manualHost: StateFlow<String> = prefs.manualHost
val manualPort: StateFlow<Int> = prefs.manualPort
val manualTls: StateFlow<Boolean> = prefs.manualTls
val gatewayToken: StateFlow<String> = prefs.gatewayToken
val onboardingCompleted: StateFlow<Boolean> = prefs.onboardingCompleted
fun setGatewayToken(value: String) = prefs.setGatewayToken(value)
fun setGatewayPassword(value: String) = prefs.setGatewayPassword(value)
fun setOnboardingCompleted(value: Boolean) = prefs.setOnboardingCompleted(value)
val lastDiscoveredStableId: StateFlow<String> = prefs.lastDiscoveredStableId
val canvasDebugStatusEnabled: StateFlow<Boolean> = prefs.canvasDebugStatusEnabled
private var didAutoConnect = false
val chatSessionKey: StateFlow<String> = chat.sessionKey
val chatSessionId: StateFlow<String?> = chat.sessionId
val chatMessages: StateFlow<List<ChatMessage>> = chat.messages
val chatError: StateFlow<String?> = chat.errorText
val chatHealthOk: StateFlow<Boolean> = chat.healthOk
val chatThinkingLevel: StateFlow<String> = chat.thinkingLevel
val chatStreamingAssistantText: StateFlow<String?> = chat.streamingAssistantText
val chatPendingToolCalls: StateFlow<List<ChatPendingToolCall>> = chat.pendingToolCalls
val chatSessions: StateFlow<List<ChatSessionEntry>> = chat.sessions
val pendingRunCount: StateFlow<Int> = chat.pendingRunCount
init {
if (prefs.voiceWakeMode.value != VoiceWakeMode.Off) {
prefs.setVoiceWakeMode(VoiceWakeMode.Off)
}
scope.launch {
prefs.loadGatewayToken()
}
scope.launch {
prefs.talkEnabled.collect { enabled ->
micCapture.setMicEnabled(enabled)
externalAudioCaptureActive.value = enabled
}
}
scope.launch(Dispatchers.Default) {
gateways.collect { list ->
if (list.isNotEmpty()) {
// Security: don't let an unauthenticated discovery feed continuously steer autoconnect.
// UX parity with iOS: only set once when unset.
if (lastDiscoveredStableId.value.trim().isEmpty()) {
prefs.setLastDiscoveredStableId(list.first().stableId)
}
}
if (didAutoConnect) return@collect
if (_isConnected.value) return@collect
if (manualEnabled.value) {
val host = manualHost.value.trim()
val port = manualPort.value
if (host.isNotEmpty() && port in 1..65535) {
// Security: autoconnect only to previously trusted gateways (stored TLS pin).
if (!manualTls.value) return@collect
val stableId = GatewayEndpoint.manual(host = host, port = port).stableId
val storedFingerprint = prefs.loadGatewayTlsFingerprint(stableId)?.trim().orEmpty()
if (storedFingerprint.isEmpty()) return@collect
didAutoConnect = true
connect(GatewayEndpoint.manual(host = host, port = port))
}
return@collect
}
val targetStableId = lastDiscoveredStableId.value.trim()
if (targetStableId.isEmpty()) return@collect
val target = list.firstOrNull { it.stableId == targetStableId } ?: return@collect
// Security: autoconnect only to previously trusted gateways (stored TLS pin).
val storedFingerprint = prefs.loadGatewayTlsFingerprint(target.stableId)?.trim().orEmpty()
if (storedFingerprint.isEmpty()) return@collect
didAutoConnect = true
connect(target)
}
}
scope.launch {
combine(
canvasDebugStatusEnabled,
statusText,
serverName,
remoteAddress,
) { debugEnabled, status, server, remote ->
Quad(debugEnabled, status, server, remote)
}.distinctUntilChanged()
.collect { (debugEnabled, status, server, remote) ->
canvas.setDebugStatusEnabled(debugEnabled)
if (!debugEnabled) return@collect
canvas.setDebugStatus(status, server ?: remote)
}
}
}
fun setForeground(value: Boolean) {
_isForeground.value = value
}
fun setDisplayName(value: String) {
prefs.setDisplayName(value)
}
fun setCameraEnabled(value: Boolean) {
prefs.setCameraEnabled(value)
}
fun setLocationMode(mode: LocationMode) {
prefs.setLocationMode(mode)
}
fun setLocationPreciseEnabled(value: Boolean) {
prefs.setLocationPreciseEnabled(value)
}
fun setPreventSleep(value: Boolean) {
prefs.setPreventSleep(value)
}
fun setManualEnabled(value: Boolean) {
prefs.setManualEnabled(value)
}
fun setManualHost(value: String) {
prefs.setManualHost(value)
}
fun setManualPort(value: Int) {
prefs.setManualPort(value)
}
fun setManualTls(value: Boolean) {
prefs.setManualTls(value)
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
prefs.setCanvasDebugStatusEnabled(value)
}
fun setMicEnabled(value: Boolean) {
prefs.setTalkEnabled(value)
micCapture.setMicEnabled(value)
externalAudioCaptureActive.value = value
}
fun refreshGatewayConnection() {
val endpoint =
connectedEndpoint ?: run {
_statusText.value = "Failed: no cached gateway endpoint"
return
}
operatorStatusText = "Connecting…"
updateStatus()
val token = prefs.loadGatewayToken()
val password = prefs.loadGatewayPassword()
val tls = connectionManager.resolveTlsParams(endpoint)
operatorSession.connect(endpoint, token, password, connectionManager.buildOperatorConnectOptions(), tls)
nodeSession.connect(endpoint, token, password, connectionManager.buildNodeConnectOptions(), tls)
operatorSession.reconnect()
nodeSession.reconnect()
}
fun connect(endpoint: GatewayEndpoint) {
val tls = connectionManager.resolveTlsParams(endpoint)
if (tls?.required == true && tls.expectedFingerprint.isNullOrBlank()) {
// First-time TLS: capture fingerprint, ask user to verify out-of-band, then store and connect.
_statusText.value = "Verify gateway TLS fingerprint…"
scope.launch {
val fp = probeGatewayTlsFingerprint(endpoint.host, endpoint.port) ?: run {
_statusText.value = "Failed: can't read TLS fingerprint"
return@launch
}
_pendingGatewayTrust.value = GatewayTrustPrompt(endpoint = endpoint, fingerprintSha256 = fp)
}
return
}
connectedEndpoint = endpoint
operatorStatusText = "Connecting…"
nodeStatusText = "Connecting…"
updateStatus()
val token = prefs.loadGatewayToken()
val password = prefs.loadGatewayPassword()
operatorSession.connect(endpoint, token, password, connectionManager.buildOperatorConnectOptions(), tls)
nodeSession.connect(endpoint, token, password, connectionManager.buildNodeConnectOptions(), tls)
}
fun acceptGatewayTrustPrompt() {
val prompt = _pendingGatewayTrust.value ?: return
_pendingGatewayTrust.value = null
prefs.saveGatewayTlsFingerprint(prompt.endpoint.stableId, prompt.fingerprintSha256)
connect(prompt.endpoint)
}
fun declineGatewayTrustPrompt() {
_pendingGatewayTrust.value = null
_statusText.value = "Offline"
}
private fun hasRecordAudioPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
)
}
fun connectManual() {
val host = manualHost.value.trim()
val port = manualPort.value
if (host.isEmpty() || port <= 0 || port > 65535) {
_statusText.value = "Failed: invalid manual host/port"
return
}
connect(GatewayEndpoint.manual(host = host, port = port))
}
fun disconnect() {
connectedEndpoint = null
_pendingGatewayTrust.value = null
operatorSession.disconnect()
nodeSession.disconnect()
}
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
scope.launch {
val trimmed = payloadJson.trim()
if (trimmed.isEmpty()) return@launch
val root =
try {
json.parseToJsonElement(trimmed).asObjectOrNull() ?: return@launch
} catch (_: Throwable) {
return@launch
}
val userActionObj = (root["userAction"] as? JsonObject) ?: root
val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty {
java.util.UUID.randomUUID().toString()
}
val name = OpenClawCanvasA2UIAction.extractActionName(userActionObj) ?: return@launch
val surfaceId =
(userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" }
val sourceComponentId =
(userActionObj["sourceComponentId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "-" }
val contextJson = (userActionObj["context"] as? JsonObject)?.toString()
val sessionKey = resolveMainSessionKey()
val message =
OpenClawCanvasA2UIAction.formatAgentMessage(
actionName = name,
sessionKey = sessionKey,
surfaceId = surfaceId,
sourceComponentId = sourceComponentId,
host = displayName.value,
instanceId = instanceId.value.lowercase(),
contextJson = contextJson,
)
val connected = _nodeConnected.value
var error: String? = null
if (connected) {
val sent =
nodeSession.sendNodeEvent(
event = "agent.request",
payloadJson =
buildJsonObject {
put("message", JsonPrimitive(message))
put("sessionKey", JsonPrimitive(sessionKey))
put("thinking", JsonPrimitive("low"))
put("deliver", JsonPrimitive(false))
put("key", JsonPrimitive(actionId))
}.toString(),
)
if (!sent) {
error = "send failed"
}
} else {
error = "gateway not connected"
}
try {
canvas.eval(
OpenClawCanvasA2UIAction.jsDispatchA2UIActionStatus(
actionId = actionId,
ok = connected && error == null,
error = error,
),
)
} catch (_: Throwable) {
// ignore
}
}
}
fun loadChat(sessionKey: String) {
val key = sessionKey.trim().ifEmpty { resolveMainSessionKey() }
chat.load(key)
}
fun refreshChat() {
chat.refresh()
}
fun refreshChatSessions(limit: Int? = null) {
chat.refreshSessions(limit = limit)
}
fun setChatThinkingLevel(level: String) {
chat.setThinkingLevel(level)
}
fun switchChatSession(sessionKey: String) {
chat.switchSession(sessionKey)
}
fun abortChat() {
chat.abort()
}
fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
chat.sendMessage(message = message, thinkingLevel = thinking, attachments = attachments)
}
private fun handleGatewayEvent(event: String, payloadJson: String?) {
micCapture.handleGatewayEvent(event, payloadJson)
chat.handleGatewayEvent(event, payloadJson)
}
private fun parseChatSendRunId(response: String): String? {
return try {
val root = json.parseToJsonElement(response).asObjectOrNull() ?: return null
root["runId"].asStringOrNull()
} catch (_: Throwable) {
null
}
}
private suspend fun refreshBrandingFromGateway() {
if (!_isConnected.value) return
try {
val res = operatorSession.request("config.get", "{}")
val root = json.parseToJsonElement(res).asObjectOrNull()
val config = root?.get("config").asObjectOrNull()
val ui = config?.get("ui").asObjectOrNull()
val raw = ui?.get("seamColor").asStringOrNull()?.trim()
val sessionCfg = config?.get("session").asObjectOrNull()
val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull())
applyMainSessionKey(mainKey)
val parsed = parseHexColorArgb(raw)
_seamColorArgb.value = parsed ?: DEFAULT_SEAM_COLOR_ARGB
} catch (_: Throwable) {
// ignore
}
}
private fun triggerCameraFlash() {
// Token is used as a pulse trigger; value doesn't matter as long as it changes.
_cameraFlashToken.value = SystemClock.elapsedRealtimeNanos()
}
private fun showCameraHud(message: String, kind: CameraHudKind, autoHideMs: Long? = null) {
val token = cameraHudSeq.incrementAndGet()
_cameraHud.value = CameraHudState(token = token, kind = kind, message = message)
if (autoHideMs != null && autoHideMs > 0) {
scope.launch {
delay(autoHideMs)
if (_cameraHud.value?.token == token) _cameraHud.value = null
}
}
}
}

View File

@@ -0,0 +1,133 @@
package ai.openclaw.android
import android.content.pm.PackageManager
import android.content.Intent
import android.Manifest
import android.net.Uri
import android.provider.Settings
import androidx.appcompat.app.AlertDialog
import androidx.activity.ComponentActivity
import androidx.activity.result.ActivityResultLauncher
import androidx.activity.result.contract.ActivityResultContracts
import androidx.core.content.ContextCompat
import androidx.core.app.ActivityCompat
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlin.coroutines.resume
class PermissionRequester(private val activity: ComponentActivity) {
private val mutex = Mutex()
private var pending: CompletableDeferred<Map<String, Boolean>>? = null
private val launcher: ActivityResultLauncher<Array<String>> =
activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result ->
val p = pending
pending = null
p?.complete(result)
}
suspend fun requestIfMissing(
permissions: List<String>,
timeoutMs: Long = 20_000,
): Map<String, Boolean> =
mutex.withLock {
val missing =
permissions.filter { perm ->
ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED
}
if (missing.isEmpty()) {
return permissions.associateWith { true }
}
val needsRationale =
missing.any { ActivityCompat.shouldShowRequestPermissionRationale(activity, it) }
if (needsRationale) {
val proceed = showRationaleDialog(missing)
if (!proceed) {
return permissions.associateWith { perm ->
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
}
}
}
val deferred = CompletableDeferred<Map<String, Boolean>>()
pending = deferred
withContext(Dispatchers.Main) {
launcher.launch(missing.toTypedArray())
}
val result =
withContext(Dispatchers.Default) {
kotlinx.coroutines.withTimeout(timeoutMs) { deferred.await() }
}
// Merge: if something was already granted, treat it as granted even if launcher omitted it.
val merged =
permissions.associateWith { perm ->
val nowGranted =
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
result[perm] == true || nowGranted
}
val denied =
merged.filterValues { !it }.keys.filter {
!ActivityCompat.shouldShowRequestPermissionRationale(activity, it)
}
if (denied.isNotEmpty()) {
showSettingsDialog(denied)
}
return merged
}
private suspend fun showRationaleDialog(permissions: List<String>): Boolean =
withContext(Dispatchers.Main) {
suspendCancellableCoroutine { cont ->
AlertDialog.Builder(activity)
.setTitle("Permission required")
.setMessage(buildRationaleMessage(permissions))
.setPositiveButton("Continue") { _, _ -> cont.resume(true) }
.setNegativeButton("Not now") { _, _ -> cont.resume(false) }
.setOnCancelListener { cont.resume(false) }
.show()
}
}
private fun showSettingsDialog(permissions: List<String>) {
AlertDialog.Builder(activity)
.setTitle("Enable permission in Settings")
.setMessage(buildSettingsMessage(permissions))
.setPositiveButton("Open Settings") { _, _ ->
val intent =
Intent(
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
Uri.fromParts("package", activity.packageName, null),
)
activity.startActivity(intent)
}
.setNegativeButton("Cancel", null)
.show()
}
private fun buildRationaleMessage(permissions: List<String>): String {
val labels = permissions.map { permissionLabel(it) }
return "OpenClaw needs ${labels.joinToString(", ")} permissions to continue."
}
private fun buildSettingsMessage(permissions: List<String>): String {
val labels = permissions.map { permissionLabel(it) }
return "Please enable ${labels.joinToString(", ")} in Android Settings to continue."
}
private fun permissionLabel(permission: String): String =
when (permission) {
Manifest.permission.CAMERA -> "Camera"
Manifest.permission.RECORD_AUDIO -> "Microphone"
Manifest.permission.SEND_SMS -> "SMS"
else -> permission
}
}

View File

@@ -0,0 +1,65 @@
package ai.openclaw.android
import android.app.Activity
import android.content.Context
import android.content.Intent
import android.media.projection.MediaProjectionManager
import androidx.activity.ComponentActivity
import androidx.activity.result.ActivityResultLauncher
import androidx.activity.result.contract.ActivityResultContracts
import androidx.appcompat.app.AlertDialog
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeout
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlin.coroutines.resume
class ScreenCaptureRequester(private val activity: ComponentActivity) {
data class CaptureResult(val resultCode: Int, val data: Intent)
private val mutex = Mutex()
private var pending: CompletableDeferred<CaptureResult?>? = null
private val launcher: ActivityResultLauncher<Intent> =
activity.registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
val p = pending
pending = null
val data = result.data
if (result.resultCode == Activity.RESULT_OK && data != null) {
p?.complete(CaptureResult(result.resultCode, data))
} else {
p?.complete(null)
}
}
suspend fun requestCapture(timeoutMs: Long = 20_000): CaptureResult? =
mutex.withLock {
val proceed = showRationaleDialog()
if (!proceed) return null
val mgr = activity.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
val intent = mgr.createScreenCaptureIntent()
val deferred = CompletableDeferred<CaptureResult?>()
pending = deferred
withContext(Dispatchers.Main) { launcher.launch(intent) }
withContext(Dispatchers.Default) { withTimeout(timeoutMs) { deferred.await() } }
}
private suspend fun showRationaleDialog(): Boolean =
withContext(Dispatchers.Main) {
suspendCancellableCoroutine { cont ->
AlertDialog.Builder(activity)
.setTitle("Screen recording required")
.setMessage("OpenClaw needs to record the screen for this command.")
.setPositiveButton("Continue") { _, _ -> cont.resume(true) }
.setNegativeButton("Not now") { _, _ -> cont.resume(false) }
.setOnCancelListener { cont.resume(false) }
.show()
}
}
}

View File

@@ -0,0 +1,304 @@
@file:Suppress("DEPRECATION")
package ai.openclaw.android
import android.content.Context
import android.content.SharedPreferences
import androidx.core.content.edit
import androidx.security.crypto.EncryptedSharedPreferences
import androidx.security.crypto.MasterKey
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonPrimitive
import java.util.UUID
class SecurePrefs(context: Context) {
companion object {
val defaultWakeWords: List<String> = listOf("openclaw", "claude")
private const val displayNameKey = "node.displayName"
private const val voiceWakeModeKey = "voiceWake.mode"
private const val plainPrefsName = "openclaw.node"
private const val securePrefsName = "openclaw.node.secure"
}
private val appContext = context.applicationContext
private val json = Json { ignoreUnknownKeys = true }
private val plainPrefs: SharedPreferences =
appContext.getSharedPreferences(plainPrefsName, Context.MODE_PRIVATE)
private val masterKey by lazy {
MasterKey.Builder(appContext)
.setKeyScheme(MasterKey.KeyScheme.AES256_GCM)
.build()
}
private val securePrefs: SharedPreferences by lazy { createSecurePrefs(appContext, securePrefsName) }
private val _instanceId = MutableStateFlow(loadOrCreateInstanceId())
val instanceId: StateFlow<String> = _instanceId
private val _displayName =
MutableStateFlow(loadOrMigrateDisplayName(context = context))
val displayName: StateFlow<String> = _displayName
private val _cameraEnabled = MutableStateFlow(plainPrefs.getBoolean("camera.enabled", true))
val cameraEnabled: StateFlow<Boolean> = _cameraEnabled
private val _locationMode =
MutableStateFlow(LocationMode.fromRawValue(plainPrefs.getString("location.enabledMode", "off")))
val locationMode: StateFlow<LocationMode> = _locationMode
private val _locationPreciseEnabled =
MutableStateFlow(plainPrefs.getBoolean("location.preciseEnabled", true))
val locationPreciseEnabled: StateFlow<Boolean> = _locationPreciseEnabled
private val _preventSleep = MutableStateFlow(plainPrefs.getBoolean("screen.preventSleep", true))
val preventSleep: StateFlow<Boolean> = _preventSleep
private val _manualEnabled =
MutableStateFlow(plainPrefs.getBoolean("gateway.manual.enabled", false))
val manualEnabled: StateFlow<Boolean> = _manualEnabled
private val _manualHost =
MutableStateFlow(plainPrefs.getString("gateway.manual.host", "") ?: "")
val manualHost: StateFlow<String> = _manualHost
private val _manualPort =
MutableStateFlow(plainPrefs.getInt("gateway.manual.port", 18789))
val manualPort: StateFlow<Int> = _manualPort
private val _manualTls =
MutableStateFlow(plainPrefs.getBoolean("gateway.manual.tls", true))
val manualTls: StateFlow<Boolean> = _manualTls
private val _gatewayToken = MutableStateFlow("")
val gatewayToken: StateFlow<String> = _gatewayToken
private val _onboardingCompleted =
MutableStateFlow(plainPrefs.getBoolean("onboarding.completed", false))
val onboardingCompleted: StateFlow<Boolean> = _onboardingCompleted
private val _lastDiscoveredStableId =
MutableStateFlow(
plainPrefs.getString("gateway.lastDiscoveredStableID", "") ?: "",
)
val lastDiscoveredStableId: StateFlow<String> = _lastDiscoveredStableId
private val _canvasDebugStatusEnabled =
MutableStateFlow(plainPrefs.getBoolean("canvas.debugStatusEnabled", false))
val canvasDebugStatusEnabled: StateFlow<Boolean> = _canvasDebugStatusEnabled
private val _wakeWords = MutableStateFlow(loadWakeWords())
val wakeWords: StateFlow<List<String>> = _wakeWords
private val _voiceWakeMode = MutableStateFlow(loadVoiceWakeMode())
val voiceWakeMode: StateFlow<VoiceWakeMode> = _voiceWakeMode
private val _talkEnabled = MutableStateFlow(plainPrefs.getBoolean("talk.enabled", false))
val talkEnabled: StateFlow<Boolean> = _talkEnabled
fun setLastDiscoveredStableId(value: String) {
val trimmed = value.trim()
plainPrefs.edit { putString("gateway.lastDiscoveredStableID", trimmed) }
_lastDiscoveredStableId.value = trimmed
}
fun setDisplayName(value: String) {
val trimmed = value.trim()
plainPrefs.edit { putString(displayNameKey, trimmed) }
_displayName.value = trimmed
}
fun setCameraEnabled(value: Boolean) {
plainPrefs.edit { putBoolean("camera.enabled", value) }
_cameraEnabled.value = value
}
fun setLocationMode(mode: LocationMode) {
plainPrefs.edit { putString("location.enabledMode", mode.rawValue) }
_locationMode.value = mode
}
fun setLocationPreciseEnabled(value: Boolean) {
plainPrefs.edit { putBoolean("location.preciseEnabled", value) }
_locationPreciseEnabled.value = value
}
fun setPreventSleep(value: Boolean) {
plainPrefs.edit { putBoolean("screen.preventSleep", value) }
_preventSleep.value = value
}
fun setManualEnabled(value: Boolean) {
plainPrefs.edit { putBoolean("gateway.manual.enabled", value) }
_manualEnabled.value = value
}
fun setManualHost(value: String) {
val trimmed = value.trim()
plainPrefs.edit { putString("gateway.manual.host", trimmed) }
_manualHost.value = trimmed
}
fun setManualPort(value: Int) {
plainPrefs.edit { putInt("gateway.manual.port", value) }
_manualPort.value = value
}
fun setManualTls(value: Boolean) {
plainPrefs.edit { putBoolean("gateway.manual.tls", value) }
_manualTls.value = value
}
fun setGatewayToken(value: String) {
val trimmed = value.trim()
securePrefs.edit { putString("gateway.manual.token", trimmed) }
_gatewayToken.value = trimmed
}
fun setGatewayPassword(value: String) {
saveGatewayPassword(value)
}
fun setOnboardingCompleted(value: Boolean) {
plainPrefs.edit { putBoolean("onboarding.completed", value) }
_onboardingCompleted.value = value
}
fun setCanvasDebugStatusEnabled(value: Boolean) {
plainPrefs.edit { putBoolean("canvas.debugStatusEnabled", value) }
_canvasDebugStatusEnabled.value = value
}
fun loadGatewayToken(): String? {
val manual =
_gatewayToken.value.trim().ifEmpty {
val stored = securePrefs.getString("gateway.manual.token", null)?.trim().orEmpty()
if (stored.isNotEmpty()) _gatewayToken.value = stored
stored
}
if (manual.isNotEmpty()) return manual
val key = "gateway.token.${_instanceId.value}"
val stored = securePrefs.getString(key, null)?.trim()
return stored?.takeIf { it.isNotEmpty() }
}
fun saveGatewayToken(token: String) {
val key = "gateway.token.${_instanceId.value}"
securePrefs.edit { putString(key, token.trim()) }
}
fun loadGatewayPassword(): String? {
val key = "gateway.password.${_instanceId.value}"
val stored = securePrefs.getString(key, null)?.trim()
return stored?.takeIf { it.isNotEmpty() }
}
fun saveGatewayPassword(password: String) {
val key = "gateway.password.${_instanceId.value}"
securePrefs.edit { putString(key, password.trim()) }
}
fun loadGatewayTlsFingerprint(stableId: String): String? {
val key = "gateway.tls.$stableId"
return plainPrefs.getString(key, null)?.trim()?.takeIf { it.isNotEmpty() }
}
fun saveGatewayTlsFingerprint(stableId: String, fingerprint: String) {
val key = "gateway.tls.$stableId"
plainPrefs.edit { putString(key, fingerprint.trim()) }
}
fun getString(key: String): String? {
return securePrefs.getString(key, null)
}
fun putString(key: String, value: String) {
securePrefs.edit { putString(key, value) }
}
fun remove(key: String) {
securePrefs.edit { remove(key) }
}
private fun createSecurePrefs(context: Context, name: String): SharedPreferences {
return EncryptedSharedPreferences.create(
context,
name,
masterKey,
EncryptedSharedPreferences.PrefKeyEncryptionScheme.AES256_SIV,
EncryptedSharedPreferences.PrefValueEncryptionScheme.AES256_GCM,
)
}
private fun loadOrCreateInstanceId(): String {
val existing = plainPrefs.getString("node.instanceId", null)?.trim()
if (!existing.isNullOrBlank()) return existing
val fresh = UUID.randomUUID().toString()
plainPrefs.edit { putString("node.instanceId", fresh) }
return fresh
}
private fun loadOrMigrateDisplayName(context: Context): String {
val existing = plainPrefs.getString(displayNameKey, null)?.trim().orEmpty()
if (existing.isNotEmpty() && existing != "Android Node") return existing
val candidate = DeviceNames.bestDefaultNodeName(context).trim()
val resolved = candidate.ifEmpty { "Android Node" }
plainPrefs.edit { putString(displayNameKey, resolved) }
return resolved
}
fun setWakeWords(words: List<String>) {
val sanitized = WakeWords.sanitize(words, defaultWakeWords)
val encoded =
JsonArray(sanitized.map { JsonPrimitive(it) }).toString()
plainPrefs.edit { putString("voiceWake.triggerWords", encoded) }
_wakeWords.value = sanitized
}
fun setVoiceWakeMode(mode: VoiceWakeMode) {
plainPrefs.edit { putString(voiceWakeModeKey, mode.rawValue) }
_voiceWakeMode.value = mode
}
fun setTalkEnabled(value: Boolean) {
plainPrefs.edit { putBoolean("talk.enabled", value) }
_talkEnabled.value = value
}
private fun loadVoiceWakeMode(): VoiceWakeMode {
val raw = plainPrefs.getString(voiceWakeModeKey, null)
val resolved = VoiceWakeMode.fromRawValue(raw)
// Default ON (foreground) when unset.
if (raw.isNullOrBlank()) {
plainPrefs.edit { putString(voiceWakeModeKey, resolved.rawValue) }
}
return resolved
}
private fun loadWakeWords(): List<String> {
val raw = plainPrefs.getString("voiceWake.triggerWords", null)?.trim()
if (raw.isNullOrEmpty()) return defaultWakeWords
return try {
val element = json.parseToJsonElement(raw)
val array = element as? JsonArray ?: return defaultWakeWords
val decoded =
array.mapNotNull { item ->
when (item) {
is JsonNull -> null
is JsonPrimitive -> item.content.trim().takeIf { it.isNotEmpty() }
else -> null
}
}
WakeWords.sanitize(decoded, defaultWakeWords)
} catch (_: Throwable) {
defaultWakeWords
}
}
}

View File

@@ -0,0 +1,13 @@
package ai.openclaw.android
internal fun normalizeMainKey(raw: String?): String {
val trimmed = raw?.trim()
return if (!trimmed.isNullOrEmpty()) trimmed else "main"
}
internal fun isCanonicalMainSessionKey(raw: String?): Boolean {
val trimmed = raw?.trim().orEmpty()
if (trimmed.isEmpty()) return false
if (trimmed == "global") return true
return trimmed.startsWith("agent:")
}

View File

@@ -0,0 +1,14 @@
package ai.openclaw.android
enum class VoiceWakeMode(val rawValue: String) {
Off("off"),
Foreground("foreground"),
Always("always"),
;
companion object {
fun fromRawValue(raw: String?): VoiceWakeMode {
return entries.firstOrNull { it.rawValue == raw?.trim()?.lowercase() } ?: Foreground
}
}
}

View File

@@ -0,0 +1,21 @@
package ai.openclaw.android
object WakeWords {
const val maxWords: Int = 32
const val maxWordLength: Int = 64
fun parseCommaSeparated(input: String): List<String> {
return input.split(",").map { it.trim() }.filter { it.isNotEmpty() }
}
fun parseIfChanged(input: String, current: List<String>): List<String>? {
val parsed = parseCommaSeparated(input)
return if (parsed == current) null else parsed
}
fun sanitize(words: List<String>, defaults: List<String>): List<String> {
val cleaned =
words.map { it.trim() }.filter { it.isNotEmpty() }.take(maxWords).map { it.take(maxWordLength) }
return cleaned.ifEmpty { defaults }
}
}

View File

@@ -0,0 +1,540 @@
package ai.openclaw.android.chat
import ai.openclaw.android.gateway.GatewaySession
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
class ChatController(
private val scope: CoroutineScope,
private val session: GatewaySession,
private val json: Json,
private val supportsChatSubscribe: Boolean,
) {
private val _sessionKey = MutableStateFlow("main")
val sessionKey: StateFlow<String> = _sessionKey.asStateFlow()
private val _sessionId = MutableStateFlow<String?>(null)
val sessionId: StateFlow<String?> = _sessionId.asStateFlow()
private val _messages = MutableStateFlow<List<ChatMessage>>(emptyList())
val messages: StateFlow<List<ChatMessage>> = _messages.asStateFlow()
private val _errorText = MutableStateFlow<String?>(null)
val errorText: StateFlow<String?> = _errorText.asStateFlow()
private val _healthOk = MutableStateFlow(false)
val healthOk: StateFlow<Boolean> = _healthOk.asStateFlow()
private val _thinkingLevel = MutableStateFlow("off")
val thinkingLevel: StateFlow<String> = _thinkingLevel.asStateFlow()
private val _pendingRunCount = MutableStateFlow(0)
val pendingRunCount: StateFlow<Int> = _pendingRunCount.asStateFlow()
private val _streamingAssistantText = MutableStateFlow<String?>(null)
val streamingAssistantText: StateFlow<String?> = _streamingAssistantText.asStateFlow()
private val pendingToolCallsById = ConcurrentHashMap<String, ChatPendingToolCall>()
private val _pendingToolCalls = MutableStateFlow<List<ChatPendingToolCall>>(emptyList())
val pendingToolCalls: StateFlow<List<ChatPendingToolCall>> = _pendingToolCalls.asStateFlow()
private val _sessions = MutableStateFlow<List<ChatSessionEntry>>(emptyList())
val sessions: StateFlow<List<ChatSessionEntry>> = _sessions.asStateFlow()
private val pendingRuns = mutableSetOf<String>()
private val pendingRunTimeoutJobs = ConcurrentHashMap<String, Job>()
private val pendingRunTimeoutMs = 120_000L
private var lastHealthPollAtMs: Long? = null
fun onDisconnected(message: String) {
_healthOk.value = false
// Not an error; keep connection status in the UI pill.
_errorText.value = null
clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
_sessionId.value = null
}
fun load(sessionKey: String) {
val key = sessionKey.trim().ifEmpty { "main" }
_sessionKey.value = key
scope.launch { bootstrap(forceHealth = true) }
}
fun applyMainSessionKey(mainSessionKey: String) {
val trimmed = mainSessionKey.trim()
if (trimmed.isEmpty()) return
if (_sessionKey.value == trimmed) return
if (_sessionKey.value != "main") return
_sessionKey.value = trimmed
scope.launch { bootstrap(forceHealth = true) }
}
fun refresh() {
scope.launch { bootstrap(forceHealth = true) }
}
fun refreshSessions(limit: Int? = null) {
scope.launch { fetchSessions(limit = limit) }
}
fun setThinkingLevel(thinkingLevel: String) {
val normalized = normalizeThinking(thinkingLevel)
if (normalized == _thinkingLevel.value) return
_thinkingLevel.value = normalized
}
fun switchSession(sessionKey: String) {
val key = sessionKey.trim()
if (key.isEmpty()) return
if (key == _sessionKey.value) return
_sessionKey.value = key
scope.launch { bootstrap(forceHealth = true) }
}
fun sendMessage(
message: String,
thinkingLevel: String,
attachments: List<OutgoingAttachment>,
) {
val trimmed = message.trim()
if (trimmed.isEmpty() && attachments.isEmpty()) return
if (!_healthOk.value) {
_errorText.value = "Gateway health not OK; cannot send"
return
}
val runId = UUID.randomUUID().toString()
val text = if (trimmed.isEmpty() && attachments.isNotEmpty()) "See attached." else trimmed
val sessionKey = _sessionKey.value
val thinking = normalizeThinking(thinkingLevel)
// Optimistic user message.
val userContent =
buildList {
add(ChatMessageContent(type = "text", text = text))
for (att in attachments) {
add(
ChatMessageContent(
type = att.type,
mimeType = att.mimeType,
fileName = att.fileName,
base64 = att.base64,
),
)
}
}
_messages.value =
_messages.value +
ChatMessage(
id = UUID.randomUUID().toString(),
role = "user",
content = userContent,
timestampMs = System.currentTimeMillis(),
)
armPendingRunTimeout(runId)
synchronized(pendingRuns) {
pendingRuns.add(runId)
_pendingRunCount.value = pendingRuns.size
}
_errorText.value = null
_streamingAssistantText.value = null
pendingToolCallsById.clear()
publishPendingToolCalls()
scope.launch {
try {
val params =
buildJsonObject {
put("sessionKey", JsonPrimitive(sessionKey))
put("message", JsonPrimitive(text))
put("thinking", JsonPrimitive(thinking))
put("timeoutMs", JsonPrimitive(30_000))
put("idempotencyKey", JsonPrimitive(runId))
if (attachments.isNotEmpty()) {
put(
"attachments",
JsonArray(
attachments.map { att ->
buildJsonObject {
put("type", JsonPrimitive(att.type))
put("mimeType", JsonPrimitive(att.mimeType))
put("fileName", JsonPrimitive(att.fileName))
put("content", JsonPrimitive(att.base64))
}
},
),
)
}
}
val res = session.request("chat.send", params.toString())
val actualRunId = parseRunId(res) ?: runId
if (actualRunId != runId) {
clearPendingRun(runId)
armPendingRunTimeout(actualRunId)
synchronized(pendingRuns) {
pendingRuns.add(actualRunId)
_pendingRunCount.value = pendingRuns.size
}
}
} catch (err: Throwable) {
clearPendingRun(runId)
_errorText.value = err.message
}
}
}
fun abort() {
val runIds =
synchronized(pendingRuns) {
pendingRuns.toList()
}
if (runIds.isEmpty()) return
scope.launch {
for (runId in runIds) {
try {
val params =
buildJsonObject {
put("sessionKey", JsonPrimitive(_sessionKey.value))
put("runId", JsonPrimitive(runId))
}
session.request("chat.abort", params.toString())
} catch (_: Throwable) {
// best-effort
}
}
}
}
fun handleGatewayEvent(event: String, payloadJson: String?) {
when (event) {
"tick" -> {
scope.launch { pollHealthIfNeeded(force = false) }
}
"health" -> {
// If we receive a health snapshot, the gateway is reachable.
_healthOk.value = true
}
"seqGap" -> {
_errorText.value = "Event stream interrupted; try refreshing."
clearPendingRuns()
}
"chat" -> {
if (payloadJson.isNullOrBlank()) return
handleChatEvent(payloadJson)
}
"agent" -> {
if (payloadJson.isNullOrBlank()) return
handleAgentEvent(payloadJson)
}
}
}
private suspend fun bootstrap(forceHealth: Boolean) {
_errorText.value = null
_healthOk.value = false
clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
_sessionId.value = null
val key = _sessionKey.value
try {
if (supportsChatSubscribe) {
session.sendNodeEvent("chat.subscribe", """{"sessionKey":"$key"}""")
}
val historyJson = session.request("chat.history", """{"sessionKey":"$key"}""")
val history = parseHistory(historyJson, sessionKey = key)
_messages.value = history.messages
_sessionId.value = history.sessionId
history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it }
pollHealthIfNeeded(force = forceHealth)
fetchSessions(limit = 50)
} catch (err: Throwable) {
_errorText.value = err.message
}
}
private suspend fun fetchSessions(limit: Int?) {
try {
val params =
buildJsonObject {
put("includeGlobal", JsonPrimitive(true))
put("includeUnknown", JsonPrimitive(false))
if (limit != null && limit > 0) put("limit", JsonPrimitive(limit))
}
val res = session.request("sessions.list", params.toString())
_sessions.value = parseSessions(res)
} catch (_: Throwable) {
// best-effort
}
}
private suspend fun pollHealthIfNeeded(force: Boolean) {
val now = System.currentTimeMillis()
val last = lastHealthPollAtMs
if (!force && last != null && now - last < 10_000) return
lastHealthPollAtMs = now
try {
session.request("health", null)
_healthOk.value = true
} catch (_: Throwable) {
_healthOk.value = false
}
}
private fun handleChatEvent(payloadJson: String) {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val sessionKey = payload["sessionKey"].asStringOrNull()?.trim()
if (!sessionKey.isNullOrEmpty() && sessionKey != _sessionKey.value) return
val runId = payload["runId"].asStringOrNull()
if (runId != null) {
val isPending =
synchronized(pendingRuns) {
pendingRuns.contains(runId)
}
if (!isPending) return
}
val state = payload["state"].asStringOrNull()
when (state) {
"delta" -> {
val text = parseAssistantDeltaText(payload)
if (!text.isNullOrEmpty()) {
_streamingAssistantText.value = text
}
}
"final", "aborted", "error" -> {
if (state == "error") {
_errorText.value = payload["errorMessage"].asStringOrNull() ?: "Chat failed"
}
if (runId != null) clearPendingRun(runId) else clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
scope.launch {
try {
val historyJson =
session.request("chat.history", """{"sessionKey":"${_sessionKey.value}"}""")
val history = parseHistory(historyJson, sessionKey = _sessionKey.value)
_messages.value = history.messages
_sessionId.value = history.sessionId
history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it }
} catch (_: Throwable) {
// best-effort
}
}
}
}
}
private fun handleAgentEvent(payloadJson: String) {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val sessionKey = payload["sessionKey"].asStringOrNull()?.trim()
if (!sessionKey.isNullOrEmpty() && sessionKey != _sessionKey.value) return
val stream = payload["stream"].asStringOrNull()
val data = payload["data"].asObjectOrNull()
when (stream) {
"assistant" -> {
val text = data?.get("text")?.asStringOrNull()
if (!text.isNullOrEmpty()) {
_streamingAssistantText.value = text
}
}
"tool" -> {
val phase = data?.get("phase")?.asStringOrNull()
val name = data?.get("name")?.asStringOrNull()
val toolCallId = data?.get("toolCallId")?.asStringOrNull()
if (phase.isNullOrEmpty() || name.isNullOrEmpty() || toolCallId.isNullOrEmpty()) return
val ts = payload["ts"].asLongOrNull() ?: System.currentTimeMillis()
if (phase == "start") {
val args = data?.get("args").asObjectOrNull()
pendingToolCallsById[toolCallId] =
ChatPendingToolCall(
toolCallId = toolCallId,
name = name,
args = args,
startedAtMs = ts,
isError = null,
)
publishPendingToolCalls()
} else if (phase == "result") {
pendingToolCallsById.remove(toolCallId)
publishPendingToolCalls()
}
}
"error" -> {
_errorText.value = "Event stream interrupted; try refreshing."
clearPendingRuns()
pendingToolCallsById.clear()
publishPendingToolCalls()
_streamingAssistantText.value = null
}
}
}
private fun parseAssistantDeltaText(payload: JsonObject): String? {
val message = payload["message"].asObjectOrNull() ?: return null
if (message["role"].asStringOrNull() != "assistant") return null
val content = message["content"].asArrayOrNull() ?: return null
for (item in content) {
val obj = item.asObjectOrNull() ?: continue
if (obj["type"].asStringOrNull() != "text") continue
val text = obj["text"].asStringOrNull()
if (!text.isNullOrEmpty()) {
return text
}
}
return null
}
private fun publishPendingToolCalls() {
_pendingToolCalls.value =
pendingToolCallsById.values.sortedBy { it.startedAtMs }
}
private fun armPendingRunTimeout(runId: String) {
pendingRunTimeoutJobs[runId]?.cancel()
pendingRunTimeoutJobs[runId] =
scope.launch {
delay(pendingRunTimeoutMs)
val stillPending =
synchronized(pendingRuns) {
pendingRuns.contains(runId)
}
if (!stillPending) return@launch
clearPendingRun(runId)
_errorText.value = "Timed out waiting for a reply; try again or refresh."
}
}
private fun clearPendingRun(runId: String) {
pendingRunTimeoutJobs.remove(runId)?.cancel()
synchronized(pendingRuns) {
pendingRuns.remove(runId)
_pendingRunCount.value = pendingRuns.size
}
}
private fun clearPendingRuns() {
for ((_, job) in pendingRunTimeoutJobs) {
job.cancel()
}
pendingRunTimeoutJobs.clear()
synchronized(pendingRuns) {
pendingRuns.clear()
_pendingRunCount.value = 0
}
}
private fun parseHistory(historyJson: String, sessionKey: String): ChatHistory {
val root = json.parseToJsonElement(historyJson).asObjectOrNull() ?: return ChatHistory(sessionKey, null, null, emptyList())
val sid = root["sessionId"].asStringOrNull()
val thinkingLevel = root["thinkingLevel"].asStringOrNull()
val array = root["messages"].asArrayOrNull() ?: JsonArray(emptyList())
val messages =
array.mapNotNull { item ->
val obj = item.asObjectOrNull() ?: return@mapNotNull null
val role = obj["role"].asStringOrNull() ?: return@mapNotNull null
val content = obj["content"].asArrayOrNull()?.mapNotNull(::parseMessageContent) ?: emptyList()
val ts = obj["timestamp"].asLongOrNull()
ChatMessage(
id = UUID.randomUUID().toString(),
role = role,
content = content,
timestampMs = ts,
)
}
return ChatHistory(sessionKey = sessionKey, sessionId = sid, thinkingLevel = thinkingLevel, messages = messages)
}
private fun parseMessageContent(el: JsonElement): ChatMessageContent? {
val obj = el.asObjectOrNull() ?: return null
val type = obj["type"].asStringOrNull() ?: "text"
return if (type == "text") {
ChatMessageContent(type = "text", text = obj["text"].asStringOrNull())
} else {
ChatMessageContent(
type = type,
mimeType = obj["mimeType"].asStringOrNull(),
fileName = obj["fileName"].asStringOrNull(),
base64 = obj["content"].asStringOrNull(),
)
}
}
private fun parseSessions(jsonString: String): List<ChatSessionEntry> {
val root = json.parseToJsonElement(jsonString).asObjectOrNull() ?: return emptyList()
val sessions = root["sessions"].asArrayOrNull() ?: return emptyList()
return sessions.mapNotNull { item ->
val obj = item.asObjectOrNull() ?: return@mapNotNull null
val key = obj["key"].asStringOrNull()?.trim().orEmpty()
if (key.isEmpty()) return@mapNotNull null
val updatedAt = obj["updatedAt"].asLongOrNull()
val displayName = obj["displayName"].asStringOrNull()?.trim()
ChatSessionEntry(key = key, updatedAtMs = updatedAt, displayName = displayName)
}
}
private fun parseRunId(resJson: String): String? {
return try {
json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull()
} catch (_: Throwable) {
null
}
}
private fun normalizeThinking(raw: String): String {
return when (raw.trim().lowercase()) {
"low" -> "low"
"medium" -> "medium"
"high" -> "high"
else -> "off"
}
}
}
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
private fun JsonElement?.asArrayOrNull(): JsonArray? = this as? JsonArray
private fun JsonElement?.asStringOrNull(): String? =
when (this) {
is JsonNull -> null
is JsonPrimitive -> content
else -> null
}
private fun JsonElement?.asLongOrNull(): Long? =
when (this) {
is JsonPrimitive -> content.toLongOrNull()
else -> null
}

View File

@@ -0,0 +1,44 @@
package ai.openclaw.android.chat
data class ChatMessage(
val id: String,
val role: String,
val content: List<ChatMessageContent>,
val timestampMs: Long?,
)
data class ChatMessageContent(
val type: String = "text",
val text: String? = null,
val mimeType: String? = null,
val fileName: String? = null,
val base64: String? = null,
)
data class ChatPendingToolCall(
val toolCallId: String,
val name: String,
val args: kotlinx.serialization.json.JsonObject? = null,
val startedAtMs: Long,
val isError: Boolean? = null,
)
data class ChatSessionEntry(
val key: String,
val updatedAtMs: Long?,
val displayName: String? = null,
)
data class ChatHistory(
val sessionKey: String,
val sessionId: String?,
val thinkingLevel: String?,
val messages: List<ChatMessage>,
)
data class OutgoingAttachment(
val type: String,
val mimeType: String,
val fileName: String,
val base64: String,
)

View File

@@ -0,0 +1,35 @@
package ai.openclaw.android.gateway
object BonjourEscapes {
fun decode(input: String): String {
if (input.isEmpty()) return input
val bytes = mutableListOf<Byte>()
var i = 0
while (i < input.length) {
if (input[i] == '\\' && i + 3 < input.length) {
val d0 = input[i + 1]
val d1 = input[i + 2]
val d2 = input[i + 3]
if (d0.isDigit() && d1.isDigit() && d2.isDigit()) {
val value =
((d0.code - '0'.code) * 100) + ((d1.code - '0'.code) * 10) + (d2.code - '0'.code)
if (value in 0..255) {
bytes.add(value.toByte())
i += 4
continue
}
}
}
val codePoint = Character.codePointAt(input, i)
val charBytes = String(Character.toChars(codePoint)).toByteArray(Charsets.UTF_8)
for (b in charBytes) {
bytes.add(b)
}
i += Character.charCount(codePoint)
}
return String(bytes.toByteArray(), Charsets.UTF_8)
}
}

View File

@@ -0,0 +1,52 @@
package ai.openclaw.android.gateway
internal object DeviceAuthPayload {
fun buildV3(
deviceId: String,
clientId: String,
clientMode: String,
role: String,
scopes: List<String>,
signedAtMs: Long,
token: String?,
nonce: String,
platform: String?,
deviceFamily: String?,
): String {
val scopeString = scopes.joinToString(",")
val authToken = token.orEmpty()
val platformNorm = normalizeMetadataField(platform)
val deviceFamilyNorm = normalizeMetadataField(deviceFamily)
return listOf(
"v3",
deviceId,
clientId,
clientMode,
role,
scopeString,
signedAtMs.toString(),
authToken,
nonce,
platformNorm,
deviceFamilyNorm,
).joinToString("|")
}
internal fun normalizeMetadataField(value: String?): String {
val trimmed = value?.trim().orEmpty()
if (trimmed.isEmpty()) {
return ""
}
// Keep cross-runtime normalization deterministic (TS/Swift/Kotlin):
// lowercase ASCII A-Z only for auth payload metadata fields.
val out = StringBuilder(trimmed.length)
for (ch in trimmed) {
if (ch in 'A'..'Z') {
out.append((ch.code + 32).toChar())
} else {
out.append(ch)
}
}
return out.toString()
}
}

View File

@@ -0,0 +1,31 @@
package ai.openclaw.android.gateway
import ai.openclaw.android.SecurePrefs
interface DeviceAuthTokenStore {
fun loadToken(deviceId: String, role: String): String?
fun saveToken(deviceId: String, role: String, token: String)
}
class DeviceAuthStore(private val prefs: SecurePrefs) : DeviceAuthTokenStore {
override fun loadToken(deviceId: String, role: String): String? {
val key = tokenKey(deviceId, role)
return prefs.getString(key)?.trim()?.takeIf { it.isNotEmpty() }
}
override fun saveToken(deviceId: String, role: String, token: String) {
val key = tokenKey(deviceId, role)
prefs.putString(key, token.trim())
}
fun clearToken(deviceId: String, role: String) {
val key = tokenKey(deviceId, role)
prefs.remove(key)
}
private fun tokenKey(deviceId: String, role: String): String {
val normalizedDevice = deviceId.trim().lowercase()
val normalizedRole = role.trim().lowercase()
return "gateway.deviceToken.$normalizedDevice.$normalizedRole"
}
}

View File

@@ -0,0 +1,174 @@
package ai.openclaw.android.gateway
import android.content.Context
import android.util.Base64
import java.io.File
import java.security.MessageDigest
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
@Serializable
data class DeviceIdentity(
val deviceId: String,
val publicKeyRawBase64: String,
val privateKeyPkcs8Base64: String,
val createdAtMs: Long,
)
class DeviceIdentityStore(context: Context) {
private val json = Json { ignoreUnknownKeys = true }
private val identityFile = File(context.filesDir, "openclaw/identity/device.json")
@Volatile private var cachedIdentity: DeviceIdentity? = null
@Synchronized
fun loadOrCreate(): DeviceIdentity {
cachedIdentity?.let { return it }
val existing = load()
if (existing != null) {
val derived = deriveDeviceId(existing.publicKeyRawBase64)
if (derived != null && derived != existing.deviceId) {
val updated = existing.copy(deviceId = derived)
save(updated)
cachedIdentity = updated
return updated
}
cachedIdentity = existing
return existing
}
val fresh = generate()
save(fresh)
cachedIdentity = fresh
return fresh
}
fun signPayload(payload: String, identity: DeviceIdentity): String? {
return try {
// Use BC lightweight API directly — JCA provider registration is broken by R8
val privateKeyBytes = Base64.decode(identity.privateKeyPkcs8Base64, Base64.DEFAULT)
val pkInfo = org.bouncycastle.asn1.pkcs.PrivateKeyInfo.getInstance(privateKeyBytes)
val parsed = pkInfo.parsePrivateKey()
val rawPrivate = org.bouncycastle.asn1.DEROctetString.getInstance(parsed).octets
val privateKey = org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters(rawPrivate, 0)
val signer = org.bouncycastle.crypto.signers.Ed25519Signer()
signer.init(true, privateKey)
val payloadBytes = payload.toByteArray(Charsets.UTF_8)
signer.update(payloadBytes, 0, payloadBytes.size)
base64UrlEncode(signer.generateSignature())
} catch (e: Throwable) {
android.util.Log.e("DeviceAuth", "signPayload FAILED: ${e.javaClass.simpleName}: ${e.message}", e)
null
}
}
fun verifySelfSignature(payload: String, signatureBase64Url: String, identity: DeviceIdentity): Boolean {
return try {
val rawPublicKey = Base64.decode(identity.publicKeyRawBase64, Base64.DEFAULT)
val pubKey = org.bouncycastle.crypto.params.Ed25519PublicKeyParameters(rawPublicKey, 0)
val sigBytes = base64UrlDecode(signatureBase64Url)
val verifier = org.bouncycastle.crypto.signers.Ed25519Signer()
verifier.init(false, pubKey)
val payloadBytes = payload.toByteArray(Charsets.UTF_8)
verifier.update(payloadBytes, 0, payloadBytes.size)
verifier.verifySignature(sigBytes)
} catch (e: Throwable) {
android.util.Log.e("DeviceAuth", "self-verify exception: ${e.message}", e)
false
}
}
private fun base64UrlDecode(input: String): ByteArray {
val normalized = input.replace('-', '+').replace('_', '/')
val padded = normalized + "=".repeat((4 - normalized.length % 4) % 4)
return Base64.decode(padded, Base64.DEFAULT)
}
fun publicKeyBase64Url(identity: DeviceIdentity): String? {
return try {
val raw = Base64.decode(identity.publicKeyRawBase64, Base64.DEFAULT)
base64UrlEncode(raw)
} catch (_: Throwable) {
null
}
}
private fun load(): DeviceIdentity? {
return readIdentity(identityFile)
}
private fun readIdentity(file: File): DeviceIdentity? {
return try {
if (!file.exists()) return null
val raw = file.readText(Charsets.UTF_8)
val decoded = json.decodeFromString(DeviceIdentity.serializer(), raw)
if (decoded.deviceId.isBlank() ||
decoded.publicKeyRawBase64.isBlank() ||
decoded.privateKeyPkcs8Base64.isBlank()
) {
null
} else {
decoded
}
} catch (_: Throwable) {
null
}
}
private fun save(identity: DeviceIdentity) {
try {
identityFile.parentFile?.mkdirs()
val encoded = json.encodeToString(DeviceIdentity.serializer(), identity)
identityFile.writeText(encoded, Charsets.UTF_8)
} catch (_: Throwable) {
// best-effort only
}
}
private fun generate(): DeviceIdentity {
// Use BC lightweight API directly to avoid JCA provider issues with R8
val kpGen = org.bouncycastle.crypto.generators.Ed25519KeyPairGenerator()
kpGen.init(org.bouncycastle.crypto.params.Ed25519KeyGenerationParameters(java.security.SecureRandom()))
val kp = kpGen.generateKeyPair()
val pubKey = kp.public as org.bouncycastle.crypto.params.Ed25519PublicKeyParameters
val privKey = kp.private as org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters
val rawPublic = pubKey.encoded // 32 bytes
val deviceId = sha256Hex(rawPublic)
// Encode private key as PKCS8 for storage
val privKeyInfo = org.bouncycastle.crypto.util.PrivateKeyInfoFactory.createPrivateKeyInfo(privKey)
val pkcs8Bytes = privKeyInfo.encoded
return DeviceIdentity(
deviceId = deviceId,
publicKeyRawBase64 = Base64.encodeToString(rawPublic, Base64.NO_WRAP),
privateKeyPkcs8Base64 = Base64.encodeToString(pkcs8Bytes, Base64.NO_WRAP),
createdAtMs = System.currentTimeMillis(),
)
}
private fun deriveDeviceId(publicKeyRawBase64: String): String? {
return try {
val raw = Base64.decode(publicKeyRawBase64, Base64.DEFAULT)
sha256Hex(raw)
} catch (_: Throwable) {
null
}
}
private fun sha256Hex(data: ByteArray): String {
val digest = MessageDigest.getInstance("SHA-256").digest(data)
val out = CharArray(digest.size * 2)
var i = 0
for (byte in digest) {
val v = byte.toInt() and 0xff
out[i++] = HEX[v ushr 4]
out[i++] = HEX[v and 0x0f]
}
return String(out)
}
private fun base64UrlEncode(data: ByteArray): String {
return Base64.encodeToString(data, Base64.URL_SAFE or Base64.NO_WRAP or Base64.NO_PADDING)
}
companion object {
private val HEX = "0123456789abcdef".toCharArray()
}
}

View File

@@ -0,0 +1,521 @@
package ai.openclaw.android.gateway
import android.content.Context
import android.net.ConnectivityManager
import android.net.DnsResolver
import android.net.NetworkCapabilities
import android.net.nsd.NsdManager
import android.net.nsd.NsdServiceInfo
import android.os.CancellationSignal
import android.util.Log
import java.io.IOException
import java.net.InetSocketAddress
import java.nio.ByteBuffer
import java.nio.charset.CodingErrorAction
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.Executor
import java.util.concurrent.Executors
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import org.xbill.DNS.AAAARecord
import org.xbill.DNS.ARecord
import org.xbill.DNS.DClass
import org.xbill.DNS.ExtendedResolver
import org.xbill.DNS.Message
import org.xbill.DNS.Name
import org.xbill.DNS.PTRRecord
import org.xbill.DNS.Record
import org.xbill.DNS.Rcode
import org.xbill.DNS.Resolver
import org.xbill.DNS.SRVRecord
import org.xbill.DNS.Section
import org.xbill.DNS.SimpleResolver
import org.xbill.DNS.TextParseException
import org.xbill.DNS.TXTRecord
import org.xbill.DNS.Type
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
@Suppress("DEPRECATION")
class GatewayDiscovery(
context: Context,
private val scope: CoroutineScope,
) {
private val nsd = context.getSystemService(NsdManager::class.java)
private val connectivity = context.getSystemService(ConnectivityManager::class.java)
private val dns = DnsResolver.getInstance()
private val serviceType = "_openclaw-gw._tcp."
private val wideAreaDomain = System.getenv("OPENCLAW_WIDE_AREA_DOMAIN")
private val logTag = "OpenClaw/GatewayDiscovery"
private val localById = ConcurrentHashMap<String, GatewayEndpoint>()
private val unicastById = ConcurrentHashMap<String, GatewayEndpoint>()
private val _gateways = MutableStateFlow<List<GatewayEndpoint>>(emptyList())
val gateways: StateFlow<List<GatewayEndpoint>> = _gateways.asStateFlow()
private val _statusText = MutableStateFlow("Searching…")
val statusText: StateFlow<String> = _statusText.asStateFlow()
private var unicastJob: Job? = null
private val dnsExecutor: Executor = Executors.newCachedThreadPool()
@Volatile private var lastWideAreaRcode: Int? = null
@Volatile private var lastWideAreaCount: Int = 0
private val discoveryListener =
object : NsdManager.DiscoveryListener {
override fun onStartDiscoveryFailed(serviceType: String, errorCode: Int) {}
override fun onStopDiscoveryFailed(serviceType: String, errorCode: Int) {}
override fun onDiscoveryStarted(serviceType: String) {}
override fun onDiscoveryStopped(serviceType: String) {}
override fun onServiceFound(serviceInfo: NsdServiceInfo) {
if (serviceInfo.serviceType != this@GatewayDiscovery.serviceType) return
resolve(serviceInfo)
}
override fun onServiceLost(serviceInfo: NsdServiceInfo) {
val serviceName = BonjourEscapes.decode(serviceInfo.serviceName)
val id = stableId(serviceName, "local.")
localById.remove(id)
publish()
}
}
init {
startLocalDiscovery()
if (!wideAreaDomain.isNullOrBlank()) {
startUnicastDiscovery(wideAreaDomain)
}
}
private fun startLocalDiscovery() {
try {
nsd.discoverServices(serviceType, NsdManager.PROTOCOL_DNS_SD, discoveryListener)
} catch (_: Throwable) {
// ignore (best-effort)
}
}
private fun stopLocalDiscovery() {
try {
nsd.stopServiceDiscovery(discoveryListener)
} catch (_: Throwable) {
// ignore (best-effort)
}
}
private fun startUnicastDiscovery(domain: String) {
unicastJob =
scope.launch(Dispatchers.IO) {
while (true) {
try {
refreshUnicast(domain)
} catch (_: Throwable) {
// ignore (best-effort)
}
delay(5000)
}
}
}
private fun resolve(serviceInfo: NsdServiceInfo) {
nsd.resolveService(
serviceInfo,
object : NsdManager.ResolveListener {
override fun onResolveFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {}
override fun onServiceResolved(resolved: NsdServiceInfo) {
val host = resolved.host?.hostAddress ?: return
val port = resolved.port
if (port <= 0) return
val rawServiceName = resolved.serviceName
val serviceName = BonjourEscapes.decode(rawServiceName)
val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName)
val lanHost = txt(resolved, "lanHost")
val tailnetDns = txt(resolved, "tailnetDns")
val gatewayPort = txtInt(resolved, "gatewayPort")
val canvasPort = txtInt(resolved, "canvasPort")
val tlsEnabled = txtBool(resolved, "gatewayTls")
val tlsFingerprint = txt(resolved, "gatewayTlsSha256")
val id = stableId(serviceName, "local.")
localById[id] =
GatewayEndpoint(
stableId = id,
name = displayName,
host = host,
port = port,
lanHost = lanHost,
tailnetDns = tailnetDns,
gatewayPort = gatewayPort,
canvasPort = canvasPort,
tlsEnabled = tlsEnabled,
tlsFingerprintSha256 = tlsFingerprint,
)
publish()
}
},
)
}
private fun publish() {
_gateways.value =
(localById.values + unicastById.values).sortedBy { it.name.lowercase() }
_statusText.value = buildStatusText()
}
private fun buildStatusText(): String {
val localCount = localById.size
val wideRcode = lastWideAreaRcode
val wideCount = lastWideAreaCount
val wide =
when (wideRcode) {
null -> "Wide: ?"
Rcode.NOERROR -> "Wide: $wideCount"
Rcode.NXDOMAIN -> "Wide: NXDOMAIN"
else -> "Wide: ${Rcode.string(wideRcode)}"
}
return when {
localCount == 0 && wideRcode == null -> "Searching for gateways…"
localCount == 0 -> "$wide"
else -> "Local: $localCount$wide"
}
}
private fun stableId(serviceName: String, domain: String): String {
return "${serviceType}|${domain}|${normalizeName(serviceName)}"
}
private fun normalizeName(raw: String): String {
return raw.trim().split(Regex("\\s+")).joinToString(" ")
}
private fun txt(info: NsdServiceInfo, key: String): String? {
val bytes = info.attributes[key] ?: return null
return try {
String(bytes, Charsets.UTF_8).trim().ifEmpty { null }
} catch (_: Throwable) {
null
}
}
private fun txtInt(info: NsdServiceInfo, key: String): Int? {
return txt(info, key)?.toIntOrNull()
}
private fun txtBool(info: NsdServiceInfo, key: String): Boolean {
val raw = txt(info, key)?.trim()?.lowercase() ?: return false
return raw == "1" || raw == "true" || raw == "yes"
}
private suspend fun refreshUnicast(domain: String) {
val ptrName = "${serviceType}${domain}"
val ptrMsg = lookupUnicastMessage(ptrName, Type.PTR) ?: return
val ptrRecords = records(ptrMsg, Section.ANSWER).mapNotNull { it as? PTRRecord }
val next = LinkedHashMap<String, GatewayEndpoint>()
for (ptr in ptrRecords) {
val instanceFqdn = ptr.target.toString()
val srv =
recordByName(ptrMsg, instanceFqdn, Type.SRV) as? SRVRecord
?: run {
val msg = lookupUnicastMessage(instanceFqdn, Type.SRV) ?: return@run null
recordByName(msg, instanceFqdn, Type.SRV) as? SRVRecord
}
?: continue
val port = srv.port
if (port <= 0) continue
val targetFqdn = srv.target.toString()
val host =
resolveHostFromMessage(ptrMsg, targetFqdn)
?: resolveHostFromMessage(lookupUnicastMessage(instanceFqdn, Type.SRV), targetFqdn)
?: resolveHostUnicast(targetFqdn)
?: continue
val txtFromPtr =
recordsByName(ptrMsg, Section.ADDITIONAL)[keyName(instanceFqdn)]
.orEmpty()
.mapNotNull { it as? TXTRecord }
val txt =
if (txtFromPtr.isNotEmpty()) {
txtFromPtr
} else {
val msg = lookupUnicastMessage(instanceFqdn, Type.TXT)
records(msg, Section.ANSWER).mapNotNull { it as? TXTRecord }
}
val instanceName = BonjourEscapes.decode(decodeInstanceName(instanceFqdn, domain))
val displayName = BonjourEscapes.decode(txtValue(txt, "displayName") ?: instanceName)
val lanHost = txtValue(txt, "lanHost")
val tailnetDns = txtValue(txt, "tailnetDns")
val gatewayPort = txtIntValue(txt, "gatewayPort")
val canvasPort = txtIntValue(txt, "canvasPort")
val tlsEnabled = txtBoolValue(txt, "gatewayTls")
val tlsFingerprint = txtValue(txt, "gatewayTlsSha256")
val id = stableId(instanceName, domain)
next[id] =
GatewayEndpoint(
stableId = id,
name = displayName,
host = host,
port = port,
lanHost = lanHost,
tailnetDns = tailnetDns,
gatewayPort = gatewayPort,
canvasPort = canvasPort,
tlsEnabled = tlsEnabled,
tlsFingerprintSha256 = tlsFingerprint,
)
}
unicastById.clear()
unicastById.putAll(next)
lastWideAreaRcode = ptrMsg.header.rcode
lastWideAreaCount = next.size
publish()
if (next.isEmpty()) {
Log.d(
logTag,
"wide-area discovery: 0 results for $ptrName (rcode=${Rcode.string(ptrMsg.header.rcode)})",
)
}
}
private fun decodeInstanceName(instanceFqdn: String, domain: String): String {
val suffix = "${serviceType}${domain}"
val withoutSuffix =
if (instanceFqdn.endsWith(suffix)) {
instanceFqdn.removeSuffix(suffix)
} else {
instanceFqdn.substringBefore(serviceType)
}
return normalizeName(stripTrailingDot(withoutSuffix))
}
private fun stripTrailingDot(raw: String): String {
return raw.removeSuffix(".")
}
private suspend fun lookupUnicastMessage(name: String, type: Int): Message? {
val query =
try {
Message.newQuery(
org.xbill.DNS.Record.newRecord(
Name.fromString(name),
type,
DClass.IN,
),
)
} catch (_: TextParseException) {
return null
}
val system = queryViaSystemDns(query)
if (records(system, Section.ANSWER).any { it.type == type }) return system
val direct = createDirectResolver() ?: return system
return try {
val msg = direct.send(query)
if (records(msg, Section.ANSWER).any { it.type == type }) msg else system
} catch (_: Throwable) {
system
}
}
private suspend fun queryViaSystemDns(query: Message): Message? {
val network = preferredDnsNetwork()
val bytes =
try {
rawQuery(network, query.toWire())
} catch (_: Throwable) {
return null
}
return try {
Message(bytes)
} catch (_: IOException) {
null
}
}
private fun records(msg: Message?, section: Int): List<Record> {
return msg?.getSectionArray(section)?.toList() ?: emptyList()
}
private fun keyName(raw: String): String {
return raw.trim().lowercase()
}
private fun recordsByName(msg: Message, section: Int): Map<String, List<Record>> {
val next = LinkedHashMap<String, MutableList<Record>>()
for (r in records(msg, section)) {
val name = r.name?.toString() ?: continue
next.getOrPut(keyName(name)) { mutableListOf() }.add(r)
}
return next
}
private fun recordByName(msg: Message, fqdn: String, type: Int): Record? {
val key = keyName(fqdn)
val byNameAnswer = recordsByName(msg, Section.ANSWER)
val fromAnswer = byNameAnswer[key].orEmpty().firstOrNull { it.type == type }
if (fromAnswer != null) return fromAnswer
val byNameAdditional = recordsByName(msg, Section.ADDITIONAL)
return byNameAdditional[key].orEmpty().firstOrNull { it.type == type }
}
private fun resolveHostFromMessage(msg: Message?, hostname: String): String? {
val m = msg ?: return null
val key = keyName(hostname)
val additional = recordsByName(m, Section.ADDITIONAL)[key].orEmpty()
val a = additional.mapNotNull { it as? ARecord }.mapNotNull { it.address?.hostAddress }
val aaaa = additional.mapNotNull { it as? AAAARecord }.mapNotNull { it.address?.hostAddress }
return a.firstOrNull() ?: aaaa.firstOrNull()
}
private fun preferredDnsNetwork(): android.net.Network? {
val cm = connectivity ?: return null
// Prefer VPN (Tailscale) when present; otherwise use the active network.
cm.allNetworks.firstOrNull { n ->
val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false
caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN)
}?.let { return it }
return cm.activeNetwork
}
private fun createDirectResolver(): Resolver? {
val cm = connectivity ?: return null
val candidateNetworks =
buildList {
cm.allNetworks
.firstOrNull { n ->
val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false
caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN)
}?.let(::add)
cm.activeNetwork?.let(::add)
}.distinct()
val servers =
candidateNetworks
.asSequence()
.flatMap { n ->
cm.getLinkProperties(n)?.dnsServers?.asSequence() ?: emptySequence()
}
.distinctBy { it.hostAddress ?: it.toString() }
.toList()
if (servers.isEmpty()) return null
return try {
val resolvers =
servers.mapNotNull { addr ->
try {
SimpleResolver().apply {
setAddress(InetSocketAddress(addr, 53))
setTimeout(3)
}
} catch (_: Throwable) {
null
}
}
if (resolvers.isEmpty()) return null
ExtendedResolver(resolvers.toTypedArray()).apply { setTimeout(3) }
} catch (_: Throwable) {
null
}
}
private suspend fun rawQuery(network: android.net.Network?, wireQuery: ByteArray): ByteArray =
suspendCancellableCoroutine { cont ->
val signal = CancellationSignal()
cont.invokeOnCancellation { signal.cancel() }
dns.rawQuery(
network,
wireQuery,
DnsResolver.FLAG_EMPTY,
dnsExecutor,
signal,
object : DnsResolver.Callback<ByteArray> {
override fun onAnswer(answer: ByteArray, rcode: Int) {
cont.resume(answer)
}
override fun onError(error: DnsResolver.DnsException) {
cont.resumeWithException(error)
}
},
)
}
private fun txtValue(records: List<TXTRecord>, key: String): String? {
val prefix = "$key="
for (r in records) {
val strings: List<String> =
try {
r.strings.mapNotNull { it as? String }
} catch (_: Throwable) {
emptyList()
}
for (s in strings) {
val trimmed = decodeDnsTxtString(s).trim()
if (trimmed.startsWith(prefix)) {
return trimmed.removePrefix(prefix).trim().ifEmpty { null }
}
}
}
return null
}
private fun txtIntValue(records: List<TXTRecord>, key: String): Int? {
return txtValue(records, key)?.toIntOrNull()
}
private fun txtBoolValue(records: List<TXTRecord>, key: String): Boolean {
val raw = txtValue(records, key)?.trim()?.lowercase() ?: return false
return raw == "1" || raw == "true" || raw == "yes"
}
private fun decodeDnsTxtString(raw: String): String {
// dnsjava treats TXT as opaque bytes and decodes as ISO-8859-1 to preserve bytes.
// Our TXT payload is UTF-8 (written by the gateway), so re-decode when possible.
val bytes = raw.toByteArray(Charsets.ISO_8859_1)
val decoder =
Charsets.UTF_8
.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT)
return try {
decoder.decode(ByteBuffer.wrap(bytes)).toString()
} catch (_: Throwable) {
raw
}
}
private suspend fun resolveHostUnicast(hostname: String): String? {
val a =
records(lookupUnicastMessage(hostname, Type.A), Section.ANSWER)
.mapNotNull { it as? ARecord }
.mapNotNull { it.address?.hostAddress }
val aaaa =
records(lookupUnicastMessage(hostname, Type.AAAA), Section.ANSWER)
.mapNotNull { it as? AAAARecord }
.mapNotNull { it.address?.hostAddress }
return a.firstOrNull() ?: aaaa.firstOrNull()
}
}

View File

@@ -0,0 +1,26 @@
package ai.openclaw.android.gateway
data class GatewayEndpoint(
val stableId: String,
val name: String,
val host: String,
val port: Int,
val lanHost: String? = null,
val tailnetDns: String? = null,
val gatewayPort: Int? = null,
val canvasPort: Int? = null,
val tlsEnabled: Boolean = false,
val tlsFingerprintSha256: String? = null,
) {
companion object {
fun manual(host: String, port: Int): GatewayEndpoint =
GatewayEndpoint(
stableId = "manual|${host.lowercase()}|$port",
name = "$host:$port",
host = host,
port = port,
tlsEnabled = false,
tlsFingerprintSha256 = null,
)
}
}

View File

@@ -0,0 +1,3 @@
package ai.openclaw.android.gateway
const val GATEWAY_PROTOCOL_VERSION = 3

View File

@@ -0,0 +1,760 @@
package ai.openclaw.android.gateway
import android.util.Log
import java.util.Locale
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicBoolean
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.TimeoutCancellationException
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeout
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import okhttp3.WebSocket
import okhttp3.WebSocketListener
data class GatewayClientInfo(
val id: String,
val displayName: String?,
val version: String,
val platform: String,
val mode: String,
val instanceId: String?,
val deviceFamily: String?,
val modelIdentifier: String?,
)
data class GatewayConnectOptions(
val role: String,
val scopes: List<String>,
val caps: List<String>,
val commands: List<String>,
val permissions: Map<String, Boolean>,
val client: GatewayClientInfo,
val userAgent: String? = null,
)
class GatewaySession(
private val scope: CoroutineScope,
private val identityStore: DeviceIdentityStore,
private val deviceAuthStore: DeviceAuthTokenStore,
private val onConnected: (serverName: String?, remoteAddress: String?, mainSessionKey: String?) -> Unit,
private val onDisconnected: (message: String) -> Unit,
private val onEvent: (event: String, payloadJson: String?) -> Unit,
private val onInvoke: (suspend (InvokeRequest) -> InvokeResult)? = null,
private val onTlsFingerprint: ((stableId: String, fingerprint: String) -> Unit)? = null,
) {
private companion object {
// Keep connect timeout above observed gateway unauthorized close on lower-end devices.
private const val CONNECT_RPC_TIMEOUT_MS = 12_000L
}
data class InvokeRequest(
val id: String,
val nodeId: String,
val command: String,
val paramsJson: String?,
val timeoutMs: Long?,
)
data class InvokeResult(val ok: Boolean, val payloadJson: String?, val error: ErrorShape?) {
companion object {
fun ok(payloadJson: String?) = InvokeResult(ok = true, payloadJson = payloadJson, error = null)
fun error(code: String, message: String) =
InvokeResult(ok = false, payloadJson = null, error = ErrorShape(code = code, message = message))
}
}
data class ErrorShape(val code: String, val message: String)
private val json = Json { ignoreUnknownKeys = true }
private val writeLock = Mutex()
private val pending = ConcurrentHashMap<String, CompletableDeferred<RpcResponse>>()
@Volatile private var canvasHostUrl: String? = null
@Volatile private var mainSessionKey: String? = null
private data class DesiredConnection(
val endpoint: GatewayEndpoint,
val token: String?,
val password: String?,
val options: GatewayConnectOptions,
val tls: GatewayTlsParams?,
)
private var desired: DesiredConnection? = null
private var job: Job? = null
@Volatile private var currentConnection: Connection? = null
fun connect(
endpoint: GatewayEndpoint,
token: String?,
password: String?,
options: GatewayConnectOptions,
tls: GatewayTlsParams? = null,
) {
desired = DesiredConnection(endpoint, token, password, options, tls)
if (job == null) {
job = scope.launch(Dispatchers.IO) { runLoop() }
}
}
fun disconnect() {
desired = null
currentConnection?.closeQuietly()
scope.launch(Dispatchers.IO) {
job?.cancelAndJoin()
job = null
canvasHostUrl = null
mainSessionKey = null
onDisconnected("Offline")
}
}
fun reconnect() {
currentConnection?.closeQuietly()
}
fun currentCanvasHostUrl(): String? = canvasHostUrl
fun currentMainSessionKey(): String? = mainSessionKey
suspend fun sendNodeEvent(event: String, payloadJson: String?): Boolean {
val conn = currentConnection ?: return false
val parsedPayload = payloadJson?.let { parseJsonOrNull(it) }
val params =
buildJsonObject {
put("event", JsonPrimitive(event))
if (parsedPayload != null) {
put("payload", parsedPayload)
} else if (payloadJson != null) {
put("payloadJSON", JsonPrimitive(payloadJson))
} else {
put("payloadJSON", JsonNull)
}
}
try {
conn.request("node.event", params, timeoutMs = 8_000)
return true
} catch (err: Throwable) {
Log.w("OpenClawGateway", "node.event failed: ${err.message ?: err::class.java.simpleName}")
return false
}
}
suspend fun request(method: String, paramsJson: String?, timeoutMs: Long = 15_000): String {
val conn = currentConnection ?: throw IllegalStateException("not connected")
val params =
if (paramsJson.isNullOrBlank()) {
null
} else {
json.parseToJsonElement(paramsJson)
}
val res = conn.request(method, params, timeoutMs)
if (res.ok) return res.payloadJson ?: ""
val err = res.error
throw IllegalStateException("${err?.code ?: "UNAVAILABLE"}: ${err?.message ?: "request failed"}")
}
suspend fun refreshNodeCanvasCapability(timeoutMs: Long = 8_000): Boolean {
val conn = currentConnection ?: return false
val response =
try {
conn.request(
"node.canvas.capability.refresh",
params = buildJsonObject {},
timeoutMs = timeoutMs,
)
} catch (err: Throwable) {
Log.w("OpenClawGateway", "node.canvas.capability.refresh failed: ${err.message ?: err::class.java.simpleName}")
return false
}
if (!response.ok) {
val err = response.error
Log.w(
"OpenClawGateway",
"node.canvas.capability.refresh rejected: ${err?.code ?: "UNAVAILABLE"}: ${err?.message ?: "request failed"}",
)
return false
}
val payloadObj = response.payloadJson?.let(::parseJsonOrNull)?.asObjectOrNull()
val refreshedCapability = payloadObj?.get("canvasCapability").asStringOrNull()?.trim().orEmpty()
if (refreshedCapability.isEmpty()) {
Log.w("OpenClawGateway", "node.canvas.capability.refresh missing canvasCapability")
return false
}
val scopedCanvasHostUrl = canvasHostUrl?.trim().orEmpty()
if (scopedCanvasHostUrl.isEmpty()) {
Log.w("OpenClawGateway", "node.canvas.capability.refresh missing local canvasHostUrl")
return false
}
val refreshedUrl = replaceCanvasCapabilityInScopedHostUrl(scopedCanvasHostUrl, refreshedCapability)
if (refreshedUrl == null) {
Log.w("OpenClawGateway", "node.canvas.capability.refresh unable to rewrite scoped canvas URL")
return false
}
canvasHostUrl = refreshedUrl
return true
}
private data class RpcResponse(val id: String, val ok: Boolean, val payloadJson: String?, val error: ErrorShape?)
private inner class Connection(
private val endpoint: GatewayEndpoint,
private val token: String?,
private val password: String?,
private val options: GatewayConnectOptions,
private val tls: GatewayTlsParams?,
) {
private val connectDeferred = CompletableDeferred<Unit>()
private val closedDeferred = CompletableDeferred<Unit>()
private val isClosed = AtomicBoolean(false)
private val connectNonceDeferred = CompletableDeferred<String>()
private val client: OkHttpClient = buildClient()
private var socket: WebSocket? = null
private val loggerTag = "OpenClawGateway"
val remoteAddress: String =
if (endpoint.host.contains(":")) {
"[${endpoint.host}]:${endpoint.port}"
} else {
"${endpoint.host}:${endpoint.port}"
}
suspend fun connect() {
val scheme = if (tls != null) "wss" else "ws"
val url = "$scheme://${endpoint.host}:${endpoint.port}"
val request = Request.Builder().url(url).build()
socket = client.newWebSocket(request, Listener())
try {
connectDeferred.await()
} catch (err: Throwable) {
throw err
}
}
suspend fun request(method: String, params: JsonElement?, timeoutMs: Long): RpcResponse {
val id = UUID.randomUUID().toString()
val deferred = CompletableDeferred<RpcResponse>()
pending[id] = deferred
val frame =
buildJsonObject {
put("type", JsonPrimitive("req"))
put("id", JsonPrimitive(id))
put("method", JsonPrimitive(method))
if (params != null) put("params", params)
}
sendJson(frame)
return try {
withTimeout(timeoutMs) { deferred.await() }
} catch (err: TimeoutCancellationException) {
pending.remove(id)
throw IllegalStateException("request timeout")
}
}
suspend fun sendJson(obj: JsonObject) {
val jsonString = obj.toString()
writeLock.withLock {
socket?.send(jsonString)
}
}
suspend fun awaitClose() = closedDeferred.await()
fun closeQuietly() {
if (isClosed.compareAndSet(false, true)) {
socket?.close(1000, "bye")
socket = null
closedDeferred.complete(Unit)
}
}
private fun buildClient(): OkHttpClient {
val builder = OkHttpClient.Builder()
.writeTimeout(60, java.util.concurrent.TimeUnit.SECONDS)
.readTimeout(0, java.util.concurrent.TimeUnit.SECONDS)
.pingInterval(30, java.util.concurrent.TimeUnit.SECONDS)
val tlsConfig = buildGatewayTlsConfig(tls) { fingerprint ->
onTlsFingerprint?.invoke(tls?.stableId ?: endpoint.stableId, fingerprint)
}
if (tlsConfig != null) {
builder.sslSocketFactory(tlsConfig.sslSocketFactory, tlsConfig.trustManager)
builder.hostnameVerifier(tlsConfig.hostnameVerifier)
}
return builder.build()
}
private inner class Listener : WebSocketListener() {
override fun onOpen(webSocket: WebSocket, response: Response) {
scope.launch {
try {
val nonce = awaitConnectNonce()
sendConnect(nonce)
} catch (err: Throwable) {
connectDeferred.completeExceptionally(err)
closeQuietly()
}
}
}
override fun onMessage(webSocket: WebSocket, text: String) {
scope.launch { handleMessage(text) }
}
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
if (!connectDeferred.isCompleted) {
connectDeferred.completeExceptionally(t)
}
if (isClosed.compareAndSet(false, true)) {
failPending()
closedDeferred.complete(Unit)
onDisconnected("Gateway error: ${t.message ?: t::class.java.simpleName}")
}
}
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
if (!connectDeferred.isCompleted) {
connectDeferred.completeExceptionally(IllegalStateException("Gateway closed: $reason"))
}
if (isClosed.compareAndSet(false, true)) {
failPending()
closedDeferred.complete(Unit)
onDisconnected("Gateway closed: $reason")
}
}
}
private suspend fun sendConnect(connectNonce: String) {
val identity = identityStore.loadOrCreate()
val storedToken = deviceAuthStore.loadToken(identity.deviceId, options.role)
val trimmedToken = token?.trim().orEmpty()
// QR/setup/manual shared token must take precedence; stale role tokens can survive re-onboarding.
val authToken = if (trimmedToken.isNotBlank()) trimmedToken else storedToken.orEmpty()
val payload = buildConnectParams(identity, connectNonce, authToken, password?.trim())
val res = request("connect", payload, timeoutMs = CONNECT_RPC_TIMEOUT_MS)
if (!res.ok) {
val msg = res.error?.message ?: "connect failed"
throw IllegalStateException(msg)
}
handleConnectSuccess(res, identity.deviceId)
connectDeferred.complete(Unit)
}
private fun handleConnectSuccess(res: RpcResponse, deviceId: String) {
val payloadJson = res.payloadJson ?: throw IllegalStateException("connect failed: missing payload")
val obj = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: throw IllegalStateException("connect failed")
val serverName = obj["server"].asObjectOrNull()?.get("host").asStringOrNull()
val authObj = obj["auth"].asObjectOrNull()
val deviceToken = authObj?.get("deviceToken").asStringOrNull()
val authRole = authObj?.get("role").asStringOrNull() ?: options.role
if (!deviceToken.isNullOrBlank()) {
deviceAuthStore.saveToken(deviceId, authRole, deviceToken)
}
val rawCanvas = obj["canvasHostUrl"].asStringOrNull()
canvasHostUrl = normalizeCanvasHostUrl(rawCanvas, endpoint, isTlsConnection = tls != null)
val sessionDefaults =
obj["snapshot"].asObjectOrNull()
?.get("sessionDefaults").asObjectOrNull()
mainSessionKey = sessionDefaults?.get("mainSessionKey").asStringOrNull()
onConnected(serverName, remoteAddress, mainSessionKey)
}
private fun buildConnectParams(
identity: DeviceIdentity,
connectNonce: String,
authToken: String,
authPassword: String?,
): JsonObject {
val client = options.client
val locale = Locale.getDefault().toLanguageTag()
val clientObj =
buildJsonObject {
put("id", JsonPrimitive(client.id))
client.displayName?.let { put("displayName", JsonPrimitive(it)) }
put("version", JsonPrimitive(client.version))
put("platform", JsonPrimitive(client.platform))
put("mode", JsonPrimitive(client.mode))
client.instanceId?.let { put("instanceId", JsonPrimitive(it)) }
client.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
client.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
}
val password = authPassword?.trim().orEmpty()
val authJson =
when {
authToken.isNotEmpty() ->
buildJsonObject {
put("token", JsonPrimitive(authToken))
}
password.isNotEmpty() ->
buildJsonObject {
put("password", JsonPrimitive(password))
}
else -> null
}
val signedAtMs = System.currentTimeMillis()
val payload =
DeviceAuthPayload.buildV3(
deviceId = identity.deviceId,
clientId = client.id,
clientMode = client.mode,
role = options.role,
scopes = options.scopes,
signedAtMs = signedAtMs,
token = if (authToken.isNotEmpty()) authToken else null,
nonce = connectNonce,
platform = client.platform,
deviceFamily = client.deviceFamily,
)
val signature = identityStore.signPayload(payload, identity)
val publicKey = identityStore.publicKeyBase64Url(identity)
val deviceJson =
if (!signature.isNullOrBlank() && !publicKey.isNullOrBlank()) {
buildJsonObject {
put("id", JsonPrimitive(identity.deviceId))
put("publicKey", JsonPrimitive(publicKey))
put("signature", JsonPrimitive(signature))
put("signedAt", JsonPrimitive(signedAtMs))
put("nonce", JsonPrimitive(connectNonce))
}
} else {
null
}
return buildJsonObject {
put("minProtocol", JsonPrimitive(GATEWAY_PROTOCOL_VERSION))
put("maxProtocol", JsonPrimitive(GATEWAY_PROTOCOL_VERSION))
put("client", clientObj)
if (options.caps.isNotEmpty()) put("caps", JsonArray(options.caps.map(::JsonPrimitive)))
if (options.commands.isNotEmpty()) put("commands", JsonArray(options.commands.map(::JsonPrimitive)))
if (options.permissions.isNotEmpty()) {
put(
"permissions",
buildJsonObject {
options.permissions.forEach { (key, value) ->
put(key, JsonPrimitive(value))
}
},
)
}
put("role", JsonPrimitive(options.role))
if (options.scopes.isNotEmpty()) put("scopes", JsonArray(options.scopes.map(::JsonPrimitive)))
authJson?.let { put("auth", it) }
deviceJson?.let { put("device", it) }
put("locale", JsonPrimitive(locale))
options.userAgent?.trim()?.takeIf { it.isNotEmpty() }?.let {
put("userAgent", JsonPrimitive(it))
}
}
}
private suspend fun handleMessage(text: String) {
val frame = json.parseToJsonElement(text).asObjectOrNull() ?: return
when (frame["type"].asStringOrNull()) {
"res" -> handleResponse(frame)
"event" -> handleEvent(frame)
}
}
private fun handleResponse(frame: JsonObject) {
val id = frame["id"].asStringOrNull() ?: return
val ok = frame["ok"].asBooleanOrNull() ?: false
val payloadJson = frame["payload"]?.let { payload -> payload.toString() }
val error =
frame["error"]?.asObjectOrNull()?.let { obj ->
val code = obj["code"].asStringOrNull() ?: "UNAVAILABLE"
val msg = obj["message"].asStringOrNull() ?: "request failed"
ErrorShape(code, msg)
}
pending.remove(id)?.complete(RpcResponse(id, ok, payloadJson, error))
}
private fun handleEvent(frame: JsonObject) {
val event = frame["event"].asStringOrNull() ?: return
val payloadJson =
frame["payload"]?.let { it.toString() } ?: frame["payloadJSON"].asStringOrNull()
if (event == "connect.challenge") {
val nonce = extractConnectNonce(payloadJson)
if (!connectNonceDeferred.isCompleted && !nonce.isNullOrBlank()) {
connectNonceDeferred.complete(nonce.trim())
}
return
}
if (event == "node.invoke.request" && payloadJson != null && onInvoke != null) {
handleInvokeEvent(payloadJson)
return
}
onEvent(event, payloadJson)
}
private suspend fun awaitConnectNonce(): String {
return try {
withTimeout(2_000) { connectNonceDeferred.await() }
} catch (err: Throwable) {
throw IllegalStateException("connect challenge timeout", err)
}
}
private fun extractConnectNonce(payloadJson: String?): String? {
if (payloadJson.isNullOrBlank()) return null
val obj = parseJsonOrNull(payloadJson)?.asObjectOrNull() ?: return null
return obj["nonce"].asStringOrNull()
}
private fun handleInvokeEvent(payloadJson: String) {
val payload =
try {
json.parseToJsonElement(payloadJson).asObjectOrNull()
} catch (_: Throwable) {
null
} ?: return
val id = payload["id"].asStringOrNull() ?: return
val nodeId = payload["nodeId"].asStringOrNull() ?: return
val command = payload["command"].asStringOrNull() ?: return
val params =
payload["paramsJSON"].asStringOrNull()
?: payload["params"]?.let { value -> if (value is JsonNull) null else value.toString() }
val timeoutMs = payload["timeoutMs"].asLongOrNull()
scope.launch {
val result =
try {
onInvoke?.invoke(InvokeRequest(id, nodeId, command, params, timeoutMs))
?: InvokeResult.error("UNAVAILABLE", "invoke handler missing")
} catch (err: Throwable) {
invokeErrorFromThrowable(err)
}
sendInvokeResult(id, nodeId, result, timeoutMs)
}
}
private suspend fun sendInvokeResult(
id: String,
nodeId: String,
result: InvokeResult,
invokeTimeoutMs: Long?,
) {
val parsedPayload = result.payloadJson?.let { parseJsonOrNull(it) }
val params =
buildJsonObject {
put("id", JsonPrimitive(id))
put("nodeId", JsonPrimitive(nodeId))
put("ok", JsonPrimitive(result.ok))
if (parsedPayload != null) {
put("payload", parsedPayload)
} else if (result.payloadJson != null) {
put("payloadJSON", JsonPrimitive(result.payloadJson))
}
result.error?.let { err ->
put(
"error",
buildJsonObject {
put("code", JsonPrimitive(err.code))
put("message", JsonPrimitive(err.message))
},
)
}
}
val ackTimeoutMs = resolveInvokeResultAckTimeoutMs(invokeTimeoutMs)
try {
request("node.invoke.result", params, timeoutMs = ackTimeoutMs)
} catch (err: Throwable) {
Log.w(
loggerTag,
"node.invoke.result failed (ackTimeoutMs=$ackTimeoutMs): ${err.message ?: err::class.java.simpleName}",
)
}
}
private fun invokeErrorFromThrowable(err: Throwable): InvokeResult {
val parsed = parseInvokeErrorFromThrowable(err, fallbackMessage = err::class.java.simpleName)
return InvokeResult.error(code = parsed.code, message = parsed.message)
}
private fun failPending() {
for ((_, waiter) in pending) {
waiter.cancel()
}
pending.clear()
}
}
private suspend fun runLoop() {
var attempt = 0
while (scope.isActive) {
val target = desired
if (target == null) {
currentConnection?.closeQuietly()
currentConnection = null
delay(250)
continue
}
try {
onDisconnected(if (attempt == 0) "Connecting…" else "Reconnecting…")
connectOnce(target)
attempt = 0
} catch (err: Throwable) {
attempt += 1
onDisconnected("Gateway error: ${err.message ?: err::class.java.simpleName}")
val sleepMs = minOf(8_000L, (350.0 * Math.pow(1.7, attempt.toDouble())).toLong())
delay(sleepMs)
}
}
}
private suspend fun connectOnce(target: DesiredConnection) = withContext(Dispatchers.IO) {
val conn = Connection(target.endpoint, target.token, target.password, target.options, target.tls)
currentConnection = conn
try {
conn.connect()
conn.awaitClose()
} finally {
currentConnection = null
canvasHostUrl = null
mainSessionKey = null
}
}
private fun normalizeCanvasHostUrl(
raw: String?,
endpoint: GatewayEndpoint,
isTlsConnection: Boolean,
): String? {
val trimmed = raw?.trim().orEmpty()
val parsed = trimmed.takeIf { it.isNotBlank() }?.let { runCatching { java.net.URI(it) }.getOrNull() }
val host = parsed?.host?.trim().orEmpty()
val port = parsed?.port ?: -1
val scheme = parsed?.scheme?.trim().orEmpty().ifBlank { "http" }
val suffix = buildUrlSuffix(parsed)
// If raw URL is a non-loopback address and this connection uses TLS,
// normalize scheme/port to the endpoint we actually connected to.
if (trimmed.isNotBlank() && host.isNotBlank() && !isLoopbackHost(host)) {
val needsTlsRewrite =
isTlsConnection &&
(
!scheme.equals("https", ignoreCase = true) ||
(port > 0 && port != endpoint.port) ||
(port <= 0 && endpoint.port != 443)
)
if (needsTlsRewrite) {
return buildCanvasUrl(host = host, scheme = "https", port = endpoint.port, suffix = suffix)
}
return trimmed
}
val fallbackHost =
endpoint.tailnetDns?.trim().takeIf { !it.isNullOrEmpty() }
?: endpoint.lanHost?.trim().takeIf { !it.isNullOrEmpty() }
?: endpoint.host.trim()
if (fallbackHost.isEmpty()) return trimmed.ifBlank { null }
// For TLS connections, use the connected endpoint's scheme/port instead of raw canvas metadata.
val fallbackScheme = if (isTlsConnection) "https" else scheme
// For TLS, always use the connected endpoint port.
val fallbackPort = if (isTlsConnection) endpoint.port else (endpoint.canvasPort ?: endpoint.port)
return buildCanvasUrl(host = fallbackHost, scheme = fallbackScheme, port = fallbackPort, suffix = suffix)
}
private fun buildCanvasUrl(host: String, scheme: String, port: Int, suffix: String): String {
val loweredScheme = scheme.lowercase()
val formattedHost = if (host.contains(":")) "[${host}]" else host
val portSuffix = if ((loweredScheme == "https" && port == 443) || (loweredScheme == "http" && port == 80)) "" else ":$port"
return "$loweredScheme://$formattedHost$portSuffix$suffix"
}
private fun buildUrlSuffix(uri: java.net.URI?): String {
if (uri == null) return ""
val path = uri.rawPath?.takeIf { it.isNotBlank() } ?: ""
val query = uri.rawQuery?.takeIf { it.isNotBlank() }?.let { "?$it" } ?: ""
val fragment = uri.rawFragment?.takeIf { it.isNotBlank() }?.let { "#$it" } ?: ""
return "$path$query$fragment"
}
private fun isLoopbackHost(raw: String?): Boolean {
val host = raw?.trim()?.lowercase().orEmpty()
if (host.isEmpty()) return false
if (host == "localhost") return true
if (host == "::1") return true
if (host == "0.0.0.0" || host == "::") return true
return host.startsWith("127.")
}
}
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
private fun JsonElement?.asStringOrNull(): String? =
when (this) {
is JsonNull -> null
is JsonPrimitive -> content
else -> null
}
private fun JsonElement?.asBooleanOrNull(): Boolean? =
when (this) {
is JsonPrimitive -> {
val c = content.trim()
when {
c.equals("true", ignoreCase = true) -> true
c.equals("false", ignoreCase = true) -> false
else -> null
}
}
else -> null
}
private fun JsonElement?.asLongOrNull(): Long? =
when (this) {
is JsonPrimitive -> content.toLongOrNull()
else -> null
}
private fun parseJsonOrNull(payload: String): JsonElement? {
val trimmed = payload.trim()
if (trimmed.isEmpty()) return null
return try {
Json.parseToJsonElement(trimmed)
} catch (_: Throwable) {
null
}
}
internal fun replaceCanvasCapabilityInScopedHostUrl(
scopedUrl: String,
capability: String,
): String? {
val marker = "/__openclaw__/cap/"
val markerStart = scopedUrl.indexOf(marker)
if (markerStart < 0) return null
val capabilityStart = markerStart + marker.length
val slashEnd = scopedUrl.indexOf("/", capabilityStart).takeIf { it >= 0 }
val queryEnd = scopedUrl.indexOf("?", capabilityStart).takeIf { it >= 0 }
val fragmentEnd = scopedUrl.indexOf("#", capabilityStart).takeIf { it >= 0 }
val capabilityEnd = listOfNotNull(slashEnd, queryEnd, fragmentEnd).minOrNull() ?: scopedUrl.length
if (capabilityEnd <= capabilityStart) return null
return scopedUrl.substring(0, capabilityStart) + capability + scopedUrl.substring(capabilityEnd)
}
internal fun resolveInvokeResultAckTimeoutMs(invokeTimeoutMs: Long?): Long {
val normalized = invokeTimeoutMs?.takeIf { it > 0L } ?: 15_000L
return normalized.coerceIn(15_000L, 120_000L)
}

View File

@@ -0,0 +1,159 @@
package ai.openclaw.android.gateway
import android.annotation.SuppressLint
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import java.net.InetSocketAddress
import java.security.MessageDigest
import java.security.SecureRandom
import java.security.cert.CertificateException
import java.security.cert.X509Certificate
import java.util.Locale
import javax.net.ssl.HttpsURLConnection
import javax.net.ssl.HostnameVerifier
import javax.net.ssl.SSLContext
import javax.net.ssl.SSLParameters
import javax.net.ssl.SSLSocketFactory
import javax.net.ssl.SNIHostName
import javax.net.ssl.SSLSocket
import javax.net.ssl.TrustManagerFactory
import javax.net.ssl.X509TrustManager
data class GatewayTlsParams(
val required: Boolean,
val expectedFingerprint: String?,
val allowTOFU: Boolean,
val stableId: String,
)
data class GatewayTlsConfig(
val sslSocketFactory: SSLSocketFactory,
val trustManager: X509TrustManager,
val hostnameVerifier: HostnameVerifier,
)
fun buildGatewayTlsConfig(
params: GatewayTlsParams?,
onStore: ((String) -> Unit)? = null,
): GatewayTlsConfig? {
if (params == null) return null
val expected = params.expectedFingerprint?.let(::normalizeFingerprint)
val defaultTrust = defaultTrustManager()
@SuppressLint("CustomX509TrustManager")
val trustManager =
object : X509TrustManager {
override fun checkClientTrusted(chain: Array<X509Certificate>, authType: String) {
defaultTrust.checkClientTrusted(chain, authType)
}
override fun checkServerTrusted(chain: Array<X509Certificate>, authType: String) {
if (chain.isEmpty()) throw CertificateException("empty certificate chain")
val fingerprint = sha256Hex(chain[0].encoded)
if (expected != null) {
if (fingerprint != expected) {
throw CertificateException("gateway TLS fingerprint mismatch")
}
return
}
if (params.allowTOFU) {
onStore?.invoke(fingerprint)
return
}
defaultTrust.checkServerTrusted(chain, authType)
}
override fun getAcceptedIssuers(): Array<X509Certificate> = defaultTrust.acceptedIssuers
}
val context = SSLContext.getInstance("TLS")
context.init(null, arrayOf(trustManager), SecureRandom())
val verifier =
if (expected != null || params.allowTOFU) {
// When pinning, we intentionally ignore hostname mismatch (service discovery often yields IPs).
HostnameVerifier { _, _ -> true }
} else {
HttpsURLConnection.getDefaultHostnameVerifier()
}
return GatewayTlsConfig(
sslSocketFactory = context.socketFactory,
trustManager = trustManager,
hostnameVerifier = verifier,
)
}
suspend fun probeGatewayTlsFingerprint(
host: String,
port: Int,
timeoutMs: Int = 3_000,
): String? {
val trimmedHost = host.trim()
if (trimmedHost.isEmpty()) return null
if (port !in 1..65535) return null
return withContext(Dispatchers.IO) {
val trustAll =
@SuppressLint("CustomX509TrustManager", "TrustAllX509TrustManager")
object : X509TrustManager {
@SuppressLint("TrustAllX509TrustManager")
override fun checkClientTrusted(chain: Array<X509Certificate>, authType: String) {}
@SuppressLint("TrustAllX509TrustManager")
override fun checkServerTrusted(chain: Array<X509Certificate>, authType: String) {}
override fun getAcceptedIssuers(): Array<X509Certificate> = emptyArray()
}
val context = SSLContext.getInstance("TLS")
context.init(null, arrayOf(trustAll), SecureRandom())
val socket = (context.socketFactory.createSocket() as SSLSocket)
try {
socket.soTimeout = timeoutMs
socket.connect(InetSocketAddress(trimmedHost, port), timeoutMs)
// Best-effort SNI for hostnames (avoid crashing on IP literals).
try {
if (trimmedHost.any { it.isLetter() }) {
val params = SSLParameters()
params.serverNames = listOf(SNIHostName(trimmedHost))
socket.sslParameters = params
}
} catch (_: Throwable) {
// ignore
}
socket.startHandshake()
val cert = socket.session.peerCertificates.firstOrNull() as? X509Certificate ?: return@withContext null
sha256Hex(cert.encoded)
} catch (_: Throwable) {
null
} finally {
try {
socket.close()
} catch (_: Throwable) {
// ignore
}
}
}
}
private fun defaultTrustManager(): X509TrustManager {
val factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
factory.init(null as java.security.KeyStore?)
val trust =
factory.trustManagers.firstOrNull { it is X509TrustManager } as? X509TrustManager
return trust ?: throw IllegalStateException("No default X509TrustManager found")
}
private fun sha256Hex(data: ByteArray): String {
val digest = MessageDigest.getInstance("SHA-256").digest(data)
val out = StringBuilder(digest.size * 2)
for (byte in digest) {
out.append(String.format(Locale.US, "%02x", byte))
}
return out.toString()
}
private fun normalizeFingerprint(raw: String): String {
val stripped = raw.trim()
.replace(Regex("^sha-?256\\s*:?\\s*", RegexOption.IGNORE_CASE), "")
return stripped.lowercase(Locale.US).filter { it in '0'..'9' || it in 'a'..'f' }
}

View File

@@ -0,0 +1,39 @@
package ai.openclaw.android.gateway
data class ParsedInvokeError(
val code: String,
val message: String,
val hadExplicitCode: Boolean,
) {
val prefixedMessage: String
get() = "$code: $message"
}
fun parseInvokeErrorMessage(raw: String): ParsedInvokeError {
val trimmed = raw.trim()
if (trimmed.isEmpty()) {
return ParsedInvokeError(code = "UNAVAILABLE", message = "error", hadExplicitCode = false)
}
val parts = trimmed.split(":", limit = 2)
if (parts.size == 2) {
val code = parts[0].trim()
val rest = parts[1].trim()
if (code.isNotEmpty() && code.all { it.isUpperCase() || it == '_' }) {
return ParsedInvokeError(
code = code,
message = rest.ifEmpty { trimmed },
hadExplicitCode = true,
)
}
}
return ParsedInvokeError(code = "UNAVAILABLE", message = trimmed, hadExplicitCode = false)
}
fun parseInvokeErrorFromThrowable(
err: Throwable,
fallbackMessage: String = "error",
): ParsedInvokeError {
val raw = err.message?.trim().takeIf { !it.isNullOrEmpty() } ?: fallbackMessage
return parseInvokeErrorMessage(raw)
}

View File

@@ -0,0 +1,146 @@
package ai.openclaw.android.node
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.coroutines.delay
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
class A2UIHandler(
private val canvas: CanvasController,
private val json: Json,
private val getNodeCanvasHostUrl: () -> String?,
private val getOperatorCanvasHostUrl: () -> String?,
) {
fun resolveA2uiHostUrl(): String? {
val nodeRaw = getNodeCanvasHostUrl()?.trim().orEmpty()
val operatorRaw = getOperatorCanvasHostUrl()?.trim().orEmpty()
val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw
if (raw.isBlank()) return null
val base = raw.trimEnd('/')
return "${base}/__openclaw__/a2ui/?platform=android"
}
suspend fun ensureA2uiReady(a2uiUrl: String): Boolean {
try {
val already = canvas.eval(a2uiReadyCheckJS)
if (already == "true") return true
} catch (_: Throwable) {
// ignore
}
canvas.navigate(a2uiUrl)
repeat(50) {
try {
val ready = canvas.eval(a2uiReadyCheckJS)
if (ready == "true") return true
} catch (_: Throwable) {
// ignore
}
delay(120)
}
return false
}
fun decodeA2uiMessages(command: String, paramsJson: String?): String {
val raw = paramsJson?.trim().orEmpty()
if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required")
val obj =
json.parseToJsonElement(raw) as? JsonObject
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
val hasMessagesArray = obj["messages"] is JsonArray
if (command == "canvas.a2ui.pushJSONL" || (!hasMessagesArray && jsonlField.isNotBlank())) {
val jsonl = jsonlField
if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required")
val messages =
jsonl
.lineSequence()
.map { it.trim() }
.filter { it.isNotBlank() }
.mapIndexed { idx, line ->
val el = json.parseToJsonElement(line)
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
.toList()
return JsonArray(messages).toString()
}
val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required")
val out =
arr.mapIndexed { idx, el ->
val msg =
el as? JsonObject
?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object")
validateA2uiV0_8(msg, idx + 1)
msg
}
return JsonArray(out).toString()
}
private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) {
if (msg.containsKey("createSurface")) {
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.",
)
}
val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface")
val matched = msg.keys.filter { allowed.contains(it) }
if (matched.size != 1) {
val found = msg.keys.sorted().joinToString(", ")
throw IllegalArgumentException(
"A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found",
)
}
}
companion object {
const val a2uiReadyCheckJS: String =
"""
(() => {
try {
const host = globalThis.openclawA2UI;
return !!host && typeof host.applyMessages === 'function';
} catch (_) {
return false;
}
})()
"""
const val a2uiResetJS: String =
"""
(() => {
try {
const host = globalThis.openclawA2UI;
if (!host) return { ok: false, error: "missing openclawA2UI" };
return host.reset();
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
"""
fun a2uiApplyMessagesJS(messagesJson: String): String {
return """
(() => {
try {
const host = globalThis.openclawA2UI;
if (!host) return { ok: false, error: "missing openclawA2UI" };
const messages = $messagesJson;
return host.applyMessages(messages);
} catch (e) {
return { ok: false, error: String(e?.message ?? e) };
}
})()
""".trimIndent()
}
}
}

View File

@@ -0,0 +1,295 @@
package ai.openclaw.android.node
import android.app.PendingIntent
import android.content.Context
import android.content.Intent
import ai.openclaw.android.InstallResultReceiver
import ai.openclaw.android.MainActivity
import ai.openclaw.android.gateway.GatewayEndpoint
import ai.openclaw.android.gateway.GatewaySession
import java.io.File
import java.net.URI
import java.security.MessageDigest
import java.util.Locale
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
private val SHA256_HEX = Regex("^[a-fA-F0-9]{64}$")
internal data class AppUpdateRequest(
val url: String,
val expectedSha256: String,
)
internal fun parseAppUpdateRequest(paramsJson: String?, connectedHost: String?): AppUpdateRequest {
val params =
try {
paramsJson?.let { Json.parseToJsonElement(it).jsonObject }
} catch (_: Throwable) {
throw IllegalArgumentException("params must be valid JSON")
} ?: throw IllegalArgumentException("missing 'url' parameter")
val urlRaw =
params["url"]?.jsonPrimitive?.content?.trim().orEmpty()
.ifEmpty { throw IllegalArgumentException("missing 'url' parameter") }
val sha256Raw =
params["sha256"]?.jsonPrimitive?.content?.trim().orEmpty()
.ifEmpty { throw IllegalArgumentException("missing 'sha256' parameter") }
if (!SHA256_HEX.matches(sha256Raw)) {
throw IllegalArgumentException("invalid 'sha256' parameter (expected 64 hex chars)")
}
val uri =
try {
URI(urlRaw)
} catch (_: Throwable) {
throw IllegalArgumentException("invalid 'url' parameter")
}
val scheme = uri.scheme?.lowercase(Locale.US).orEmpty()
if (scheme != "https") {
throw IllegalArgumentException("url must use https")
}
if (!uri.userInfo.isNullOrBlank()) {
throw IllegalArgumentException("url must not include credentials")
}
val host = uri.host?.lowercase(Locale.US) ?: throw IllegalArgumentException("url host required")
val connectedHostNormalized = connectedHost?.trim()?.lowercase(Locale.US).orEmpty()
if (connectedHostNormalized.isNotEmpty() && host != connectedHostNormalized) {
throw IllegalArgumentException("url host must match connected gateway host")
}
return AppUpdateRequest(
url = uri.toASCIIString(),
expectedSha256 = sha256Raw.lowercase(Locale.US),
)
}
internal fun sha256Hex(file: File): String {
val digest = MessageDigest.getInstance("SHA-256")
file.inputStream().use { input ->
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
while (true) {
val read = input.read(buffer)
if (read < 0) break
if (read == 0) continue
digest.update(buffer, 0, read)
}
}
val out = StringBuilder(64)
for (byte in digest.digest()) {
out.append(String.format(Locale.US, "%02x", byte))
}
return out.toString()
}
class AppUpdateHandler(
private val appContext: Context,
private val connectedEndpoint: () -> GatewayEndpoint?,
) {
fun handleUpdate(paramsJson: String?): GatewaySession.InvokeResult {
try {
val updateRequest =
try {
parseAppUpdateRequest(paramsJson, connectedEndpoint()?.host)
} catch (err: IllegalArgumentException) {
return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: ${err.message ?: "invalid app.update params"}",
)
}
val url = updateRequest.url
val expectedSha256 = updateRequest.expectedSha256
android.util.Log.w("openclaw", "app.update: downloading from $url")
val notifId = 9001
val channelId = "app_update"
val notifManager = appContext.getSystemService(android.content.Context.NOTIFICATION_SERVICE) as android.app.NotificationManager
// Create notification channel (required for Android 8+)
val channel = android.app.NotificationChannel(channelId, "App Updates", android.app.NotificationManager.IMPORTANCE_LOW)
notifManager.createNotificationChannel(channel)
// PendingIntent to open the app when notification is tapped
val launchIntent = Intent(appContext, MainActivity::class.java).apply {
flags = Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP
}
val launchPi = PendingIntent.getActivity(appContext, 0, launchIntent, PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE)
// Launch download async so the invoke returns immediately
CoroutineScope(Dispatchers.IO).launch {
try {
val cacheDir = java.io.File(appContext.cacheDir, "updates")
cacheDir.mkdirs()
val file = java.io.File(cacheDir, "update.apk")
if (file.exists()) file.delete()
// Show initial progress notification
fun buildProgressNotif(progress: Int, max: Int, text: String): android.app.Notification {
return android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_sys_download)
.setContentTitle("OpenClaw Update")
.setContentText(text)
.setProgress(max, progress, max == 0)
.setContentIntent(launchPi)
.setOngoing(true)
.build()
}
notifManager.notify(notifId, buildProgressNotif(0, 0, "Connecting..."))
val client = okhttp3.OkHttpClient.Builder()
.connectTimeout(30, java.util.concurrent.TimeUnit.SECONDS)
.readTimeout(300, java.util.concurrent.TimeUnit.SECONDS)
.build()
val request = okhttp3.Request.Builder().url(url).build()
val response = client.newCall(request).execute()
if (!response.isSuccessful) {
notifManager.cancel(notifId)
notifManager.notify(notifId, android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_notify_error)
.setContentTitle("Update Failed")
.setContentIntent(launchPi)
.setContentText("HTTP ${response.code}")
.build())
return@launch
}
val contentLength = response.body?.contentLength() ?: -1L
val body = response.body ?: run {
notifManager.cancel(notifId)
return@launch
}
// Download with progress tracking
var totalBytes = 0L
var lastNotifUpdate = 0L
body.byteStream().use { input ->
file.outputStream().use { output ->
val buffer = ByteArray(8192)
while (true) {
val bytesRead = input.read(buffer)
if (bytesRead == -1) break
output.write(buffer, 0, bytesRead)
totalBytes += bytesRead
// Update notification at most every 500ms
val now = System.currentTimeMillis()
if (now - lastNotifUpdate > 500) {
lastNotifUpdate = now
if (contentLength > 0) {
val pct = ((totalBytes * 100) / contentLength).toInt()
val mb = String.format(Locale.US, "%.1f", totalBytes / 1048576.0)
val totalMb = String.format(Locale.US, "%.1f", contentLength / 1048576.0)
notifManager.notify(notifId, buildProgressNotif(pct, 100, "$mb / $totalMb MB ($pct%)"))
} else {
val mb = String.format(Locale.US, "%.1f", totalBytes / 1048576.0)
notifManager.notify(notifId, buildProgressNotif(0, 0, "${mb} MB downloaded"))
}
}
}
}
}
android.util.Log.w("openclaw", "app.update: downloaded ${file.length()} bytes")
val actualSha256 = sha256Hex(file)
if (actualSha256 != expectedSha256) {
android.util.Log.e(
"openclaw",
"app.update: sha256 mismatch expected=$expectedSha256 actual=$actualSha256",
)
file.delete()
notifManager.cancel(notifId)
notifManager.notify(
notifId,
android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_notify_error)
.setContentTitle("Update Failed")
.setContentIntent(launchPi)
.setContentText("SHA-256 mismatch")
.build(),
)
return@launch
}
// Verify file is a valid APK (basic check: ZIP magic bytes)
val magic = file.inputStream().use { it.read().toByte() to it.read().toByte() }
if (magic.first != 0x50.toByte() || magic.second != 0x4B.toByte()) {
android.util.Log.e("openclaw", "app.update: invalid APK (bad magic: ${magic.first}, ${magic.second})")
file.delete()
notifManager.cancel(notifId)
notifManager.notify(notifId, android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_notify_error)
.setContentTitle("Update Failed")
.setContentIntent(launchPi)
.setContentText("Downloaded file is not a valid APK")
.build())
return@launch
}
// Use PackageInstaller session API — works from background on API 34+
// The system handles showing the install confirmation dialog
notifManager.cancel(notifId)
notifManager.notify(
notifId,
android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_sys_download_done)
.setContentTitle("Installing Update...")
.setContentIntent(launchPi)
.setContentText("${String.format(Locale.US, "%.1f", totalBytes / 1048576.0)} MB downloaded")
.build(),
)
val installer = appContext.packageManager.packageInstaller
val params = android.content.pm.PackageInstaller.SessionParams(
android.content.pm.PackageInstaller.SessionParams.MODE_FULL_INSTALL
)
params.setSize(file.length())
val sessionId = installer.createSession(params)
val session = installer.openSession(sessionId)
session.openWrite("openclaw-update.apk", 0, file.length()).use { out ->
file.inputStream().use { inp -> inp.copyTo(out) }
session.fsync(out)
}
// Commit with FLAG_MUTABLE PendingIntent — system requires mutable for PackageInstaller status
val callbackIntent = android.content.Intent(appContext, InstallResultReceiver::class.java)
val pi = android.app.PendingIntent.getBroadcast(
appContext, sessionId, callbackIntent,
android.app.PendingIntent.FLAG_UPDATE_CURRENT or android.app.PendingIntent.FLAG_MUTABLE
)
session.commit(pi.intentSender)
android.util.Log.w("openclaw", "app.update: PackageInstaller session committed, waiting for user confirmation")
} catch (err: Throwable) {
android.util.Log.e("openclaw", "app.update: async error", err)
notifManager.cancel(notifId)
notifManager.notify(notifId, android.app.Notification.Builder(appContext, channelId)
.setSmallIcon(android.R.drawable.stat_notify_error)
.setContentTitle("Update Failed")
.setContentIntent(launchPi)
.setContentText(err.message ?: "Unknown error")
.build())
}
}
// Return immediately — download happens in background
return GatewaySession.InvokeResult.ok(buildJsonObject {
put("status", "downloading")
put("url", url)
put("sha256", expectedSha256)
}.toString())
} catch (err: Throwable) {
android.util.Log.e("openclaw", "app.update: error", err)
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = err.message ?: "update failed")
}
}
}

View File

@@ -0,0 +1,442 @@
package ai.openclaw.android.node
import android.Manifest
import android.annotation.SuppressLint
import android.content.Context
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.graphics.Matrix
import android.content.pm.PackageManager
import android.hardware.camera2.CameraCharacteristics
import android.util.Base64
import androidx.camera.camera2.interop.Camera2CameraInfo
import androidx.camera.core.CameraInfo
import androidx.exifinterface.media.ExifInterface
import androidx.lifecycle.LifecycleOwner
import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageCapture
import androidx.camera.core.ImageCaptureException
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.video.FileOutputOptions
import androidx.camera.video.FallbackStrategy
import androidx.camera.video.Quality
import androidx.camera.video.QualitySelector
import androidx.camera.video.Recorder
import androidx.camera.video.Recording
import androidx.camera.video.VideoCapture
import androidx.camera.video.VideoRecordEvent
import androidx.core.content.ContextCompat
import androidx.core.content.ContextCompat.checkSelfPermission
import androidx.core.graphics.scale
import ai.openclaw.android.PermissionRequester
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withTimeout
import kotlinx.coroutines.withContext
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.contentOrNull
import java.io.ByteArrayOutputStream
import java.io.File
import java.util.concurrent.Executor
import kotlin.math.roundToInt
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
class CameraCaptureManager(private val context: Context) {
data class Payload(val payloadJson: String)
data class FilePayload(val file: File, val durationMs: Long, val hasAudio: Boolean)
data class CameraDeviceInfo(
val id: String,
val name: String,
val position: String,
val deviceType: String,
)
@Volatile private var lifecycleOwner: LifecycleOwner? = null
@Volatile private var permissionRequester: PermissionRequester? = null
fun attachLifecycleOwner(owner: LifecycleOwner) {
lifecycleOwner = owner
}
fun attachPermissionRequester(requester: PermissionRequester) {
permissionRequester = requester
}
suspend fun listDevices(): List<CameraDeviceInfo> =
withContext(Dispatchers.Main) {
val provider = context.cameraProvider()
provider.availableCameraInfos
.mapNotNull { info -> cameraDeviceInfoOrNull(info) }
.sortedBy { it.id }
}
private suspend fun ensureCameraPermission() {
val granted = checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED
if (granted) return
val requester = permissionRequester
?: throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission")
val results = requester.requestIfMissing(listOf(Manifest.permission.CAMERA))
if (results[Manifest.permission.CAMERA] != true) {
throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission")
}
}
private suspend fun ensureMicPermission() {
val granted = checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
if (granted) return
val requester = permissionRequester
?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
val results = requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO))
if (results[Manifest.permission.RECORD_AUDIO] != true) {
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
}
}
suspend fun snap(paramsJson: String?): Payload =
withContext(Dispatchers.Main) {
ensureCameraPermission()
val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready")
val params = parseParamsObject(paramsJson)
val facing = parseFacing(params) ?: "front"
val quality = (parseQuality(params) ?: 0.95).coerceIn(0.1, 1.0)
val maxWidth = parseMaxWidth(params) ?: 1600
val deviceId = parseDeviceId(params)
val provider = context.cameraProvider()
val capture = ImageCapture.Builder().build()
val selector = resolveCameraSelector(provider, facing, deviceId)
provider.unbindAll()
provider.bindToLifecycle(owner, selector, capture)
val (bytes, orientation) = capture.takeJpegWithExif(context.mainExecutor())
val decoded = BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
?: throw IllegalStateException("UNAVAILABLE: failed to decode captured image")
val rotated = rotateBitmapByExif(decoded, orientation)
val scaled =
if (maxWidth > 0 && rotated.width > maxWidth) {
val h =
(rotated.height.toDouble() * (maxWidth.toDouble() / rotated.width.toDouble()))
.toInt()
.coerceAtLeast(1)
rotated.scale(maxWidth, h)
} else {
rotated
}
val maxPayloadBytes = 5 * 1024 * 1024
// Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
val maxEncodedBytes = (maxPayloadBytes / 4) * 3
val result =
JpegSizeLimiter.compressToLimit(
initialWidth = scaled.width,
initialHeight = scaled.height,
startQuality = (quality * 100.0).roundToInt().coerceIn(10, 100),
maxBytes = maxEncodedBytes,
encode = { width, height, q ->
val bitmap =
if (width == scaled.width && height == scaled.height) {
scaled
} else {
scaled.scale(width, height)
}
val out = ByteArrayOutputStream()
if (!bitmap.compress(Bitmap.CompressFormat.JPEG, q, out)) {
if (bitmap !== scaled) bitmap.recycle()
throw IllegalStateException("UNAVAILABLE: failed to encode JPEG")
}
if (bitmap !== scaled) {
bitmap.recycle()
}
out.toByteArray()
},
)
val base64 = Base64.encodeToString(result.bytes, Base64.NO_WRAP)
Payload(
"""{"format":"jpg","base64":"$base64","width":${result.width},"height":${result.height}}""",
)
}
@SuppressLint("MissingPermission")
suspend fun clip(paramsJson: String?): FilePayload =
withContext(Dispatchers.Main) {
ensureCameraPermission()
val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready")
val params = parseParamsObject(paramsJson)
val facing = parseFacing(params) ?: "front"
val durationMs = (parseDurationMs(params) ?: 3_000).coerceIn(200, 60_000)
val includeAudio = parseIncludeAudio(params) ?: true
val deviceId = parseDeviceId(params)
if (includeAudio) ensureMicPermission()
android.util.Log.w("CameraCaptureManager", "clip: start facing=$facing duration=$durationMs audio=$includeAudio deviceId=${deviceId ?: "-"}")
val provider = context.cameraProvider()
android.util.Log.w("CameraCaptureManager", "clip: got camera provider")
// Use LOWEST quality for smallest files over WebSocket
val recorder = Recorder.Builder()
.setQualitySelector(
QualitySelector.from(Quality.LOWEST, FallbackStrategy.lowerQualityOrHigherThan(Quality.LOWEST))
)
.build()
val videoCapture = VideoCapture.withOutput(recorder)
val selector = resolveCameraSelector(provider, facing, deviceId)
// CameraX requires a Preview use case for the camera to start producing frames;
// without it, the encoder may get no data (ERROR_NO_VALID_DATA).
val preview = androidx.camera.core.Preview.Builder().build()
// Provide a dummy SurfaceTexture so the preview pipeline activates
val surfaceTexture = android.graphics.SurfaceTexture(0)
surfaceTexture.setDefaultBufferSize(640, 480)
preview.setSurfaceProvider { request ->
val surface = android.view.Surface(surfaceTexture)
request.provideSurface(surface, context.mainExecutor()) { result ->
surface.release()
surfaceTexture.release()
}
}
provider.unbindAll()
android.util.Log.w("CameraCaptureManager", "clip: binding preview + videoCapture to lifecycle")
val camera = provider.bindToLifecycle(owner, selector, preview, videoCapture)
android.util.Log.w("CameraCaptureManager", "clip: bound, cameraInfo=${camera.cameraInfo}")
// Give camera pipeline time to initialize before recording
android.util.Log.w("CameraCaptureManager", "clip: warming up camera 1.5s...")
kotlinx.coroutines.delay(1_500)
val file = File.createTempFile("openclaw-clip-", ".mp4")
val outputOptions = FileOutputOptions.Builder(file).build()
val finalized = kotlinx.coroutines.CompletableDeferred<VideoRecordEvent.Finalize>()
android.util.Log.w("CameraCaptureManager", "clip: starting recording to ${file.absolutePath}")
val recording: Recording =
videoCapture.output
.prepareRecording(context, outputOptions)
.apply {
if (includeAudio) withAudioEnabled()
}
.start(context.mainExecutor()) { event ->
android.util.Log.w("CameraCaptureManager", "clip: event ${event.javaClass.simpleName}")
if (event is VideoRecordEvent.Status) {
android.util.Log.w("CameraCaptureManager", "clip: recording status update")
}
if (event is VideoRecordEvent.Finalize) {
android.util.Log.w("CameraCaptureManager", "clip: finalize hasError=${event.hasError()} error=${event.error} cause=${event.cause}")
finalized.complete(event)
}
}
android.util.Log.w("CameraCaptureManager", "clip: recording started, delaying ${durationMs}ms")
try {
kotlinx.coroutines.delay(durationMs.toLong())
} finally {
android.util.Log.w("CameraCaptureManager", "clip: stopping recording")
recording.stop()
}
val finalizeEvent =
try {
withTimeout(15_000) { finalized.await() }
} catch (err: Throwable) {
android.util.Log.e("CameraCaptureManager", "clip: finalize timed out", err)
withContext(Dispatchers.IO) { file.delete() }
provider.unbindAll()
throw IllegalStateException("UNAVAILABLE: camera clip finalize timed out")
}
if (finalizeEvent.hasError()) {
android.util.Log.e("CameraCaptureManager", "clip: FAILED error=${finalizeEvent.error}, cause=${finalizeEvent.cause}", finalizeEvent.cause)
// Check file size for debugging
val fileSize = withContext(Dispatchers.IO) { if (file.exists()) file.length() else -1 }
android.util.Log.e("CameraCaptureManager", "clip: file exists=${file.exists()} size=$fileSize")
withContext(Dispatchers.IO) { file.delete() }
provider.unbindAll()
throw IllegalStateException("UNAVAILABLE: camera clip failed (error=${finalizeEvent.error})")
}
val fileSize = withContext(Dispatchers.IO) { file.length() }
android.util.Log.w("CameraCaptureManager", "clip: SUCCESS file size=$fileSize")
provider.unbindAll()
FilePayload(file = file, durationMs = durationMs.toLong(), hasAudio = includeAudio)
}
private fun rotateBitmapByExif(bitmap: Bitmap, orientation: Int): Bitmap {
val matrix = Matrix()
when (orientation) {
ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f)
ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f)
ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f)
ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f)
ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f)
ExifInterface.ORIENTATION_TRANSPOSE -> {
matrix.postRotate(90f)
matrix.postScale(-1f, 1f)
}
ExifInterface.ORIENTATION_TRANSVERSE -> {
matrix.postRotate(-90f)
matrix.postScale(-1f, 1f)
}
else -> return bitmap
}
val rotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true)
if (rotated !== bitmap) {
bitmap.recycle()
}
return rotated
}
private fun parseParamsObject(paramsJson: String?): JsonObject? {
if (paramsJson.isNullOrBlank()) return null
return try {
Json.parseToJsonElement(paramsJson).asObjectOrNull()
} catch (_: Throwable) {
null
}
}
private fun readPrimitive(params: JsonObject?, key: String): JsonPrimitive? =
params?.get(key) as? JsonPrimitive
private fun parseFacing(params: JsonObject?): String? {
val value = readPrimitive(params, "facing")?.contentOrNull?.trim()?.lowercase() ?: return null
return when (value) {
"front", "back" -> value
else -> null
}
}
private fun parseQuality(params: JsonObject?): Double? =
readPrimitive(params, "quality")?.contentOrNull?.toDoubleOrNull()
private fun parseMaxWidth(params: JsonObject?): Int? =
readPrimitive(params, "maxWidth")
?.contentOrNull
?.toIntOrNull()
?.takeIf { it > 0 }
private fun parseDurationMs(params: JsonObject?): Int? =
readPrimitive(params, "durationMs")?.contentOrNull?.toIntOrNull()
private fun parseDeviceId(params: JsonObject?): String? =
readPrimitive(params, "deviceId")
?.contentOrNull
?.trim()
?.takeIf { it.isNotEmpty() }
private fun parseIncludeAudio(params: JsonObject?): Boolean? {
val value = readPrimitive(params, "includeAudio")?.contentOrNull?.trim()?.lowercase()
return when (value) {
"true" -> true
"false" -> false
else -> null
}
}
private fun Context.mainExecutor(): Executor = ContextCompat.getMainExecutor(this)
private fun resolveCameraSelector(
provider: ProcessCameraProvider,
facing: String,
deviceId: String?,
): CameraSelector {
if (deviceId.isNullOrEmpty()) {
return if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA
}
val availableIds = provider.availableCameraInfos.mapNotNull { cameraIdOrNull(it) }.toSet()
if (!availableIds.contains(deviceId)) {
throw IllegalStateException("INVALID_REQUEST: unknown camera deviceId '$deviceId'")
}
return CameraSelector.Builder()
.addCameraFilter { infos -> infos.filter { cameraIdOrNull(it) == deviceId } }
.build()
}
private fun cameraDeviceInfoOrNull(info: CameraInfo): CameraDeviceInfo? {
val cameraId = cameraIdOrNull(info) ?: return null
val lensFacing =
runCatching {
Camera2CameraInfo.from(info).getCameraCharacteristic(CameraCharacteristics.LENS_FACING)
}.getOrNull()
val position =
when (lensFacing) {
CameraCharacteristics.LENS_FACING_FRONT -> "front"
CameraCharacteristics.LENS_FACING_BACK -> "back"
CameraCharacteristics.LENS_FACING_EXTERNAL -> "external"
else -> "unspecified"
}
val deviceType =
if (lensFacing == CameraCharacteristics.LENS_FACING_EXTERNAL) "external" else "builtIn"
val name =
when (position) {
"front" -> "Front Camera"
"back" -> "Back Camera"
"external" -> "External Camera"
else -> "Camera $cameraId"
}
return CameraDeviceInfo(
id = cameraId,
name = name,
position = position,
deviceType = deviceType,
)
}
private fun cameraIdOrNull(info: CameraInfo): String? =
runCatching { Camera2CameraInfo.from(info).cameraId }.getOrNull()
}
private suspend fun Context.cameraProvider(): ProcessCameraProvider =
suspendCancellableCoroutine { cont ->
val future = ProcessCameraProvider.getInstance(this)
future.addListener(
{
try {
cont.resume(future.get())
} catch (e: Exception) {
cont.resumeWithException(e)
}
},
ContextCompat.getMainExecutor(this),
)
}
/** Returns (jpegBytes, exifOrientation) so caller can rotate the decoded bitmap. */
private suspend fun ImageCapture.takeJpegWithExif(executor: Executor): Pair<ByteArray, Int> =
suspendCancellableCoroutine { cont ->
val file = File.createTempFile("openclaw-snap-", ".jpg")
val options = ImageCapture.OutputFileOptions.Builder(file).build()
takePicture(
options,
executor,
object : ImageCapture.OnImageSavedCallback {
override fun onError(exception: ImageCaptureException) {
file.delete()
cont.resumeWithException(exception)
}
override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) {
try {
val exif = ExifInterface(file.absolutePath)
val orientation = exif.getAttributeInt(
ExifInterface.TAG_ORIENTATION,
ExifInterface.ORIENTATION_NORMAL,
)
val bytes = file.readBytes()
cont.resume(Pair(bytes, orientation))
} catch (e: Exception) {
cont.resumeWithException(e)
} finally {
file.delete()
}
}
},
)
}

View File

@@ -0,0 +1,175 @@
package ai.openclaw.android.node
import android.content.Context
import ai.openclaw.android.CameraHudKind
import ai.openclaw.android.BuildConfig
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.withContext
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonArray
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.put
internal const val CAMERA_CLIP_MAX_RAW_BYTES: Long = 18L * 1024L * 1024L
internal fun isCameraClipWithinPayloadLimit(rawBytes: Long): Boolean =
rawBytes in 0L..CAMERA_CLIP_MAX_RAW_BYTES
class CameraHandler(
private val appContext: Context,
private val camera: CameraCaptureManager,
private val externalAudioCaptureActive: MutableStateFlow<Boolean>,
private val showCameraHud: (message: String, kind: CameraHudKind, autoHideMs: Long?) -> Unit,
private val triggerCameraFlash: () -> Unit,
private val invokeErrorFromThrowable: (err: Throwable) -> Pair<String, String>,
) {
suspend fun handleList(_paramsJson: String?): GatewaySession.InvokeResult {
return try {
val devices = camera.listDevices()
val payload =
buildJsonObject {
put(
"devices",
buildJsonArray {
devices.forEach { device ->
add(
buildJsonObject {
put("id", JsonPrimitive(device.id))
put("name", JsonPrimitive(device.name))
put("position", JsonPrimitive(device.position))
put("deviceType", JsonPrimitive(device.deviceType))
},
)
}
},
)
}.toString()
GatewaySession.InvokeResult.ok(payload)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
GatewaySession.InvokeResult.error(code = code, message = message)
}
}
suspend fun handleSnap(paramsJson: String?): GatewaySession.InvokeResult {
val logFile = if (BuildConfig.DEBUG) java.io.File(appContext.cacheDir, "camera_debug.log") else null
fun camLog(msg: String) {
if (!BuildConfig.DEBUG) return
val ts = java.text.SimpleDateFormat("HH:mm:ss.SSS", java.util.Locale.US).format(java.util.Date())
logFile?.appendText("[$ts] $msg\n")
android.util.Log.w("openclaw", "camera.snap: $msg")
}
try {
logFile?.writeText("") // clear
camLog("starting, params=$paramsJson")
camLog("calling showCameraHud")
showCameraHud("Taking photo…", CameraHudKind.Photo, null)
camLog("calling triggerCameraFlash")
triggerCameraFlash()
val res =
try {
camLog("calling camera.snap()")
val r = camera.snap(paramsJson)
camLog("success, payload size=${r.payloadJson.length}")
r
} catch (err: Throwable) {
camLog("inner error: ${err::class.java.simpleName}: ${err.message}")
camLog("stack: ${err.stackTraceToString().take(2000)}")
val (code, message) = invokeErrorFromThrowable(err)
showCameraHud(message, CameraHudKind.Error, 2200)
return GatewaySession.InvokeResult.error(code = code, message = message)
}
camLog("returning result")
showCameraHud("Photo captured", CameraHudKind.Success, 1600)
return GatewaySession.InvokeResult.ok(res.payloadJson)
} catch (err: Throwable) {
camLog("outer error: ${err::class.java.simpleName}: ${err.message}")
camLog("stack: ${err.stackTraceToString().take(2000)}")
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = err.message ?: "camera snap failed")
}
}
suspend fun handleClip(paramsJson: String?): GatewaySession.InvokeResult {
val clipLogFile = if (BuildConfig.DEBUG) java.io.File(appContext.cacheDir, "camera_debug.log") else null
fun clipLog(msg: String) {
if (!BuildConfig.DEBUG) return
val ts = java.text.SimpleDateFormat("HH:mm:ss.SSS", java.util.Locale.US).format(java.util.Date())
clipLogFile?.appendText("[CLIP $ts] $msg\n")
android.util.Log.w("openclaw", "camera.clip: $msg")
}
val includeAudio = parseIncludeAudio(paramsJson) ?: true
if (includeAudio) externalAudioCaptureActive.value = true
try {
clipLogFile?.writeText("") // clear
clipLog("starting, params=$paramsJson includeAudio=$includeAudio")
clipLog("calling showCameraHud")
showCameraHud("Recording…", CameraHudKind.Recording, null)
val filePayload =
try {
clipLog("calling camera.clip()")
val r = camera.clip(paramsJson)
clipLog("success, file size=${r.file.length()}")
r
} catch (err: Throwable) {
clipLog("inner error: ${err::class.java.simpleName}: ${err.message}")
clipLog("stack: ${err.stackTraceToString().take(2000)}")
val (code, message) = invokeErrorFromThrowable(err)
showCameraHud(message, CameraHudKind.Error, 2400)
return GatewaySession.InvokeResult.error(code = code, message = message)
}
val rawBytes = filePayload.file.length()
if (!isCameraClipWithinPayloadLimit(rawBytes)) {
clipLog("payload too large: bytes=$rawBytes max=$CAMERA_CLIP_MAX_RAW_BYTES")
withContext(Dispatchers.IO) { filePayload.file.delete() }
showCameraHud("Clip too large", CameraHudKind.Error, 2400)
return GatewaySession.InvokeResult.error(
code = "PAYLOAD_TOO_LARGE",
message =
"PAYLOAD_TOO_LARGE: camera clip is $rawBytes bytes; max is $CAMERA_CLIP_MAX_RAW_BYTES bytes. Reduce durationMs and retry.",
)
}
val bytes = withContext(Dispatchers.IO) {
val b = filePayload.file.readBytes()
filePayload.file.delete()
b
}
val base64 = android.util.Base64.encodeToString(bytes, android.util.Base64.NO_WRAP)
clipLog("returning base64 payload")
showCameraHud("Clip captured", CameraHudKind.Success, 1800)
return GatewaySession.InvokeResult.ok(
"""{"format":"mp4","base64":"$base64","durationMs":${filePayload.durationMs},"hasAudio":${filePayload.hasAudio}}"""
)
} catch (err: Throwable) {
clipLog("outer error: ${err::class.java.simpleName}: ${err.message}")
clipLog("stack: ${err.stackTraceToString().take(2000)}")
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = err.message ?: "camera clip failed")
} finally {
if (includeAudio) externalAudioCaptureActive.value = false
}
}
private fun parseIncludeAudio(paramsJson: String?): Boolean? {
if (paramsJson.isNullOrBlank()) return null
val root =
try {
Json.parseToJsonElement(paramsJson).asObjectOrNull()
} catch (_: Throwable) {
null
} ?: return null
val value =
(root["includeAudio"] as? JsonPrimitive)
?.contentOrNull
?.trim()
?.lowercase()
return when (value) {
"true" -> true
"false" -> false
else -> null
}
}
}

View File

@@ -0,0 +1,276 @@
package ai.openclaw.android.node
import android.graphics.Bitmap
import android.graphics.Canvas
import android.os.Looper
import android.util.Log
import android.webkit.WebView
import androidx.core.graphics.createBitmap
import androidx.core.graphics.scale
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import java.io.ByteArrayOutputStream
import android.util.Base64
import org.json.JSONObject
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import ai.openclaw.android.BuildConfig
import kotlin.coroutines.resume
class CanvasController {
enum class SnapshotFormat(val rawValue: String) {
Png("png"),
Jpeg("jpeg"),
}
@Volatile private var webView: WebView? = null
@Volatile private var url: String? = null
@Volatile private var debugStatusEnabled: Boolean = false
@Volatile private var debugStatusTitle: String? = null
@Volatile private var debugStatusSubtitle: String? = null
private val _currentUrl = MutableStateFlow<String?>(null)
val currentUrl: StateFlow<String?> = _currentUrl.asStateFlow()
private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html"
private fun clampJpegQuality(quality: Double?): Int {
val q = (quality ?: 0.82).coerceIn(0.1, 1.0)
return (q * 100.0).toInt().coerceIn(1, 100)
}
fun attach(webView: WebView) {
this.webView = webView
reload()
applyDebugStatus()
}
fun detach(webView: WebView) {
if (this.webView === webView) {
this.webView = null
}
}
fun navigate(url: String) {
val trimmed = url.trim()
this.url = if (trimmed.isBlank() || trimmed == "/") null else trimmed
_currentUrl.value = this.url
reload()
}
fun currentUrl(): String? = url
fun isDefaultCanvas(): Boolean = url == null
fun setDebugStatusEnabled(enabled: Boolean) {
debugStatusEnabled = enabled
applyDebugStatus()
}
fun setDebugStatus(title: String?, subtitle: String?) {
debugStatusTitle = title
debugStatusSubtitle = subtitle
applyDebugStatus()
}
fun onPageFinished() {
applyDebugStatus()
}
private inline fun withWebViewOnMain(crossinline block: (WebView) -> Unit) {
val wv = webView ?: return
if (Looper.myLooper() == Looper.getMainLooper()) {
block(wv)
} else {
wv.post { block(wv) }
}
}
private fun reload() {
val currentUrl = url
withWebViewOnMain { wv ->
if (currentUrl == null) {
if (BuildConfig.DEBUG) {
Log.d("OpenClawCanvas", "load scaffold: $scaffoldAssetUrl")
}
wv.loadUrl(scaffoldAssetUrl)
} else {
if (BuildConfig.DEBUG) {
Log.d("OpenClawCanvas", "load url: $currentUrl")
}
wv.loadUrl(currentUrl)
}
}
}
private fun applyDebugStatus() {
val enabled = debugStatusEnabled
val title = debugStatusTitle
val subtitle = debugStatusSubtitle
withWebViewOnMain { wv ->
val titleJs = title?.let { JSONObject.quote(it) } ?: "null"
val subtitleJs = subtitle?.let { JSONObject.quote(it) } ?: "null"
val js = """
(() => {
try {
const api = globalThis.__openclaw;
if (!api) return;
if (typeof api.setDebugStatusEnabled === 'function') {
api.setDebugStatusEnabled(${if (enabled) "true" else "false"});
}
if (!${if (enabled) "true" else "false"}) return;
if (typeof api.setStatus === 'function') {
api.setStatus($titleJs, $subtitleJs);
}
} catch (_) {}
})();
""".trimIndent()
wv.evaluateJavascript(js, null)
}
}
suspend fun eval(javaScript: String): String =
withContext(Dispatchers.Main) {
val wv = webView ?: throw IllegalStateException("no webview")
suspendCancellableCoroutine { cont ->
wv.evaluateJavascript(javaScript) { result ->
cont.resume(result ?: "")
}
}
}
suspend fun snapshotPngBase64(maxWidth: Int?): String =
withContext(Dispatchers.Main) {
val wv = webView ?: throw IllegalStateException("no webview")
val bmp = wv.captureBitmap()
val scaled =
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
bmp.scale(maxWidth, h)
} else {
bmp
}
val out = ByteArrayOutputStream()
scaled.compress(Bitmap.CompressFormat.PNG, 100, out)
Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP)
}
suspend fun snapshotBase64(format: SnapshotFormat, quality: Double?, maxWidth: Int?): String =
withContext(Dispatchers.Main) {
val wv = webView ?: throw IllegalStateException("no webview")
val bmp = wv.captureBitmap()
val scaled =
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
bmp.scale(maxWidth, h)
} else {
bmp
}
val out = ByteArrayOutputStream()
val (compressFormat, compressQuality) =
when (format) {
SnapshotFormat.Png -> Bitmap.CompressFormat.PNG to 100
SnapshotFormat.Jpeg -> Bitmap.CompressFormat.JPEG to clampJpegQuality(quality)
}
scaled.compress(compressFormat, compressQuality, out)
Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP)
}
private suspend fun WebView.captureBitmap(): Bitmap =
suspendCancellableCoroutine { cont ->
val width = width.coerceAtLeast(1)
val height = height.coerceAtLeast(1)
val bitmap = createBitmap(width, height, Bitmap.Config.ARGB_8888)
// WebView isn't supported by PixelCopy.request(...) directly; draw() is the most reliable
// cross-version snapshot for this lightweight "canvas" use-case.
draw(Canvas(bitmap))
cont.resume(bitmap)
}
companion object {
data class SnapshotParams(val format: SnapshotFormat, val quality: Double?, val maxWidth: Int?)
fun parseNavigateUrl(paramsJson: String?): String {
val obj = parseParamsObject(paramsJson) ?: return ""
return obj.string("url").trim()
}
fun parseEvalJs(paramsJson: String?): String? {
val obj = parseParamsObject(paramsJson) ?: return null
val js = obj.string("javaScript").trim()
return js.takeIf { it.isNotBlank() }
}
fun parseSnapshotMaxWidth(paramsJson: String?): Int? {
val obj = parseParamsObject(paramsJson) ?: return null
if (!obj.containsKey("maxWidth")) return null
val width = obj.int("maxWidth") ?: 0
return width.takeIf { it > 0 }
}
fun parseSnapshotFormat(paramsJson: String?): SnapshotFormat {
val obj = parseParamsObject(paramsJson) ?: return SnapshotFormat.Jpeg
val raw = obj.string("format").trim().lowercase()
return when (raw) {
"png" -> SnapshotFormat.Png
"jpeg", "jpg" -> SnapshotFormat.Jpeg
"" -> SnapshotFormat.Jpeg
else -> SnapshotFormat.Jpeg
}
}
fun parseSnapshotQuality(paramsJson: String?): Double? {
val obj = parseParamsObject(paramsJson) ?: return null
if (!obj.containsKey("quality")) return null
val q = obj.double("quality") ?: Double.NaN
if (!q.isFinite()) return null
return q.coerceIn(0.1, 1.0)
}
fun parseSnapshotParams(paramsJson: String?): SnapshotParams {
return SnapshotParams(
format = parseSnapshotFormat(paramsJson),
quality = parseSnapshotQuality(paramsJson),
maxWidth = parseSnapshotMaxWidth(paramsJson),
)
}
private val json = Json { ignoreUnknownKeys = true }
private fun parseParamsObject(paramsJson: String?): JsonObject? {
val raw = paramsJson?.trim().orEmpty()
if (raw.isEmpty()) return null
return try {
json.parseToJsonElement(raw).asObjectOrNull()
} catch (_: Throwable) {
null
}
}
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
private fun JsonObject.string(key: String): String {
val prim = this[key] as? JsonPrimitive ?: return ""
val raw = prim.content
return raw.takeIf { it != "null" }.orEmpty()
}
private fun JsonObject.int(key: String): Int? {
val prim = this[key] as? JsonPrimitive ?: return null
return prim.content.toIntOrNull()
}
private fun JsonObject.double(key: String): Double? {
val prim = this[key] as? JsonPrimitive ?: return null
return prim.content.toDoubleOrNull()
}
}
}

View File

@@ -0,0 +1,163 @@
package ai.openclaw.android.node
import android.os.Build
import ai.openclaw.android.BuildConfig
import ai.openclaw.android.SecurePrefs
import ai.openclaw.android.gateway.GatewayClientInfo
import ai.openclaw.android.gateway.GatewayConnectOptions
import ai.openclaw.android.gateway.GatewayEndpoint
import ai.openclaw.android.gateway.GatewayTlsParams
import ai.openclaw.android.protocol.OpenClawCapability
import ai.openclaw.android.LocationMode
import ai.openclaw.android.VoiceWakeMode
class ConnectionManager(
private val prefs: SecurePrefs,
private val cameraEnabled: () -> Boolean,
private val locationMode: () -> LocationMode,
private val voiceWakeMode: () -> VoiceWakeMode,
private val smsAvailable: () -> Boolean,
private val hasRecordAudioPermission: () -> Boolean,
private val manualTls: () -> Boolean,
) {
companion object {
internal fun resolveTlsParamsForEndpoint(
endpoint: GatewayEndpoint,
storedFingerprint: String?,
manualTlsEnabled: Boolean,
): GatewayTlsParams? {
val stableId = endpoint.stableId
val stored = storedFingerprint?.trim().takeIf { !it.isNullOrEmpty() }
val isManual = stableId.startsWith("manual|")
if (isManual) {
if (!manualTlsEnabled) return null
if (!stored.isNullOrBlank()) {
return GatewayTlsParams(
required = true,
expectedFingerprint = stored,
allowTOFU = false,
stableId = stableId,
)
}
return GatewayTlsParams(
required = true,
expectedFingerprint = null,
allowTOFU = false,
stableId = stableId,
)
}
// Prefer stored pins. Never let discovery-provided TXT override a stored fingerprint.
if (!stored.isNullOrBlank()) {
return GatewayTlsParams(
required = true,
expectedFingerprint = stored,
allowTOFU = false,
stableId = stableId,
)
}
val hinted = endpoint.tlsEnabled || !endpoint.tlsFingerprintSha256.isNullOrBlank()
if (hinted) {
// TXT is unauthenticated. Do not treat the advertised fingerprint as authoritative.
return GatewayTlsParams(
required = true,
expectedFingerprint = null,
allowTOFU = false,
stableId = stableId,
)
}
return null
}
}
fun buildInvokeCommands(): List<String> =
InvokeCommandRegistry.advertisedCommands(
cameraEnabled = cameraEnabled(),
locationEnabled = locationMode() != LocationMode.Off,
smsAvailable = smsAvailable(),
debugBuild = BuildConfig.DEBUG,
)
fun buildCapabilities(): List<String> =
buildList {
add(OpenClawCapability.Canvas.rawValue)
add(OpenClawCapability.Screen.rawValue)
add(OpenClawCapability.Device.rawValue)
if (cameraEnabled()) add(OpenClawCapability.Camera.rawValue)
if (smsAvailable()) add(OpenClawCapability.Sms.rawValue)
if (voiceWakeMode() != VoiceWakeMode.Off && hasRecordAudioPermission()) {
add(OpenClawCapability.VoiceWake.rawValue)
}
if (locationMode() != LocationMode.Off) {
add(OpenClawCapability.Location.rawValue)
}
}
fun resolvedVersionName(): String {
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
return if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
}
fun resolveModelIdentifier(): String? {
return listOfNotNull(Build.MANUFACTURER, Build.MODEL)
.joinToString(" ")
.trim()
.ifEmpty { null }
}
fun buildUserAgent(): String {
val version = resolvedVersionName()
val release = Build.VERSION.RELEASE?.trim().orEmpty()
val releaseLabel = if (release.isEmpty()) "unknown" else release
return "OpenClawAndroid/$version (Android $releaseLabel; SDK ${Build.VERSION.SDK_INT})"
}
fun buildClientInfo(clientId: String, clientMode: String): GatewayClientInfo {
return GatewayClientInfo(
id = clientId,
displayName = prefs.displayName.value,
version = resolvedVersionName(),
platform = "android",
mode = clientMode,
instanceId = prefs.instanceId.value,
deviceFamily = "Android",
modelIdentifier = resolveModelIdentifier(),
)
}
fun buildNodeConnectOptions(): GatewayConnectOptions {
return GatewayConnectOptions(
role = "node",
scopes = emptyList(),
caps = buildCapabilities(),
commands = buildInvokeCommands(),
permissions = emptyMap(),
client = buildClientInfo(clientId = "openclaw-android", clientMode = "node"),
userAgent = buildUserAgent(),
)
}
fun buildOperatorConnectOptions(): GatewayConnectOptions {
return GatewayConnectOptions(
role = "operator",
scopes = listOf("operator.read", "operator.write", "operator.talk.secrets"),
caps = emptyList(),
commands = emptyList(),
permissions = emptyMap(),
client = buildClientInfo(clientId = "openclaw-android", clientMode = "ui"),
userAgent = buildUserAgent(),
)
}
fun resolveTlsParams(endpoint: GatewayEndpoint): GatewayTlsParams? {
val stored = prefs.loadGatewayTlsFingerprint(endpoint.stableId)
return resolveTlsParamsForEndpoint(endpoint, storedFingerprint = stored, manualTlsEnabled = manualTls())
}
}

View File

@@ -0,0 +1,118 @@
package ai.openclaw.android.node
import android.content.Context
import ai.openclaw.android.BuildConfig
import ai.openclaw.android.gateway.DeviceIdentityStore
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.serialization.json.JsonPrimitive
class DebugHandler(
private val appContext: Context,
private val identityStore: DeviceIdentityStore,
) {
fun handleEd25519(): GatewaySession.InvokeResult {
if (!BuildConfig.DEBUG) {
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = "debug commands are disabled in release builds")
}
// Self-test Ed25519 signing and return diagnostic info
try {
val identity = identityStore.loadOrCreate()
val testPayload = "test|${identity.deviceId}|${System.currentTimeMillis()}"
val results = mutableListOf<String>()
results.add("deviceId: ${identity.deviceId}")
results.add("publicKeyRawBase64: ${identity.publicKeyRawBase64.take(20)}...")
results.add("privateKeyPkcs8Base64: ${identity.privateKeyPkcs8Base64.take(20)}...")
// Test publicKeyBase64Url
val pubKeyUrl = identityStore.publicKeyBase64Url(identity)
results.add("publicKeyBase64Url: ${pubKeyUrl ?: "NULL (FAILED)"}")
// Test signing
val signature = identityStore.signPayload(testPayload, identity)
results.add("signPayload: ${if (signature != null) "${signature.take(20)}... (OK)" else "NULL (FAILED)"}")
// Test self-verify
if (signature != null) {
val verifyOk = identityStore.verifySelfSignature(testPayload, signature, identity)
results.add("verifySelfSignature: $verifyOk")
}
// Check available providers
val providers = java.security.Security.getProviders()
val ed25519Providers = providers.filter { p ->
p.services.any { s -> s.algorithm.contains("Ed25519", ignoreCase = true) }
}
results.add("Ed25519 providers: ${ed25519Providers.map { "${it.name} v${it.version}" }}")
results.add("Provider order: ${providers.take(5).map { it.name }}")
// Test KeyFactory directly
try {
val kf = java.security.KeyFactory.getInstance("Ed25519")
results.add("KeyFactory.Ed25519: ${kf.provider.name} (OK)")
} catch (e: Throwable) {
results.add("KeyFactory.Ed25519: FAILED - ${e.javaClass.simpleName}: ${e.message}")
}
// Test Signature directly
try {
val sig = java.security.Signature.getInstance("Ed25519")
results.add("Signature.Ed25519: ${sig.provider.name} (OK)")
} catch (e: Throwable) {
results.add("Signature.Ed25519: FAILED - ${e.javaClass.simpleName}: ${e.message}")
}
val diagnostics = results.joinToString("\n")
return GatewaySession.InvokeResult.ok("""{"diagnostics":${JsonPrimitive(diagnostics)}}""")
} catch (e: Throwable) {
return GatewaySession.InvokeResult.error(code = "ED25519_TEST_FAILED", message = "${e.javaClass.simpleName}: ${e.message}\n${e.stackTraceToString().take(500)}")
}
}
fun handleLogs(): GatewaySession.InvokeResult {
if (!BuildConfig.DEBUG) {
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = "debug commands are disabled in release builds")
}
val pid = android.os.Process.myPid()
val rt = Runtime.getRuntime()
val info = "v6 pid=$pid thread=${Thread.currentThread().name} free=${rt.freeMemory()/1024}K total=${rt.totalMemory()/1024}K max=${rt.maxMemory()/1024}K uptime=${android.os.SystemClock.elapsedRealtime()/1000}s sdk=${android.os.Build.VERSION.SDK_INT} device=${android.os.Build.MODEL}\n"
// Run logcat on current dispatcher thread (no withContext) with file redirect
val logResult = try {
val tmpFile = java.io.File(appContext.cacheDir, "debug_logs.txt")
if (tmpFile.exists()) tmpFile.delete()
val pb = ProcessBuilder("logcat", "-d", "-t", "200", "--pid=$pid")
pb.redirectOutput(tmpFile)
pb.redirectErrorStream(true)
val proc = pb.start()
val finished = proc.waitFor(4, java.util.concurrent.TimeUnit.SECONDS)
if (!finished) proc.destroyForcibly()
val raw = if (tmpFile.exists() && tmpFile.length() > 0) {
tmpFile.readText().take(128000)
} else {
"(no output, finished=$finished, exists=${tmpFile.exists()})"
}
tmpFile.delete()
val spamPatterns = listOf("setRequestedFrameRate", "I View :", "BLASTBufferQueue", "VRI[Pop-Up",
"InsetsController:", "VRI[MainActivity", "InsetsSource:", "handleResized", "ProfileInstaller",
"I VRI[", "onStateChanged: host=", "D StrictMode:", "E StrictMode:", "ImeFocusController",
"InputTransport", "IncorrectContextUseViolation")
val sb = StringBuilder()
for (line in raw.lineSequence()) {
if (line.isBlank()) continue
if (spamPatterns.any { line.contains(it) }) continue
if (sb.length + line.length > 16000) { sb.append("\n(truncated)"); break }
if (sb.isNotEmpty()) sb.append('\n')
sb.append(line)
}
sb.toString().ifEmpty { "(all ${raw.lines().size} lines filtered as spam)" }
} catch (e: Throwable) {
"(logcat error: ${e::class.java.simpleName}: ${e.message})"
}
// Also include camera debug log if it exists
val camLogFile = java.io.File(appContext.cacheDir, "camera_debug.log")
val camLog = if (camLogFile.exists() && camLogFile.length() > 0) {
"\n--- camera_debug.log ---\n" + camLogFile.readText().take(4000)
} else ""
return GatewaySession.InvokeResult.ok("""{"logs":${JsonPrimitive(info + logResult + camLog)}}""")
}
}

View File

@@ -0,0 +1,365 @@
package ai.openclaw.android.node
import android.Manifest
import android.app.ActivityManager
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.content.pm.PackageManager
import android.net.ConnectivityManager
import android.net.NetworkCapabilities
import android.os.BatteryManager
import android.os.Build
import android.os.Environment
import android.os.PowerManager
import android.os.StatFs
import android.os.SystemClock
import androidx.core.content.ContextCompat
import ai.openclaw.android.BuildConfig
import ai.openclaw.android.gateway.GatewaySession
import java.util.Locale
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonArray
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.put
class DeviceHandler(
private val appContext: Context,
) {
private data class BatterySnapshot(
val status: Int,
val plugged: Int,
val levelFraction: Double?,
val temperatureC: Double?,
)
fun handleDeviceStatus(_paramsJson: String?): GatewaySession.InvokeResult {
return GatewaySession.InvokeResult.ok(statusPayloadJson())
}
fun handleDeviceInfo(_paramsJson: String?): GatewaySession.InvokeResult {
return GatewaySession.InvokeResult.ok(infoPayloadJson())
}
fun handleDevicePermissions(_paramsJson: String?): GatewaySession.InvokeResult {
return GatewaySession.InvokeResult.ok(permissionsPayloadJson())
}
fun handleDeviceHealth(_paramsJson: String?): GatewaySession.InvokeResult {
return GatewaySession.InvokeResult.ok(healthPayloadJson())
}
private fun statusPayloadJson(): String {
val battery = readBatterySnapshot()
val powerManager = appContext.getSystemService(PowerManager::class.java)
val storage = StatFs(Environment.getDataDirectory().absolutePath)
val totalBytes = storage.totalBytes
val freeBytes = storage.availableBytes
val usedBytes = (totalBytes - freeBytes).coerceAtLeast(0L)
val connectivity = appContext.getSystemService(ConnectivityManager::class.java)
val activeNetwork = connectivity?.activeNetwork
val caps = activeNetwork?.let { connectivity.getNetworkCapabilities(it) }
val uptimeSeconds = SystemClock.elapsedRealtime() / 1_000.0
return buildJsonObject {
put(
"battery",
buildJsonObject {
battery.levelFraction?.let { put("level", JsonPrimitive(it)) }
put("state", JsonPrimitive(mapBatteryState(battery.status)))
put("lowPowerModeEnabled", JsonPrimitive(powerManager?.isPowerSaveMode == true))
},
)
put(
"thermal",
buildJsonObject {
put("state", JsonPrimitive(mapThermalState(powerManager)))
},
)
put(
"storage",
buildJsonObject {
put("totalBytes", JsonPrimitive(totalBytes))
put("freeBytes", JsonPrimitive(freeBytes))
put("usedBytes", JsonPrimitive(usedBytes))
},
)
put(
"network",
buildJsonObject {
put("status", JsonPrimitive(mapNetworkStatus(caps)))
put(
"isExpensive",
JsonPrimitive(
caps?.hasCapability(NetworkCapabilities.NET_CAPABILITY_NOT_METERED)?.not() ?: false,
),
)
put(
"isConstrained",
JsonPrimitive(
caps?.hasCapability(NetworkCapabilities.NET_CAPABILITY_NOT_RESTRICTED)?.not() ?: false,
),
)
put("interfaces", networkInterfacesJson(caps))
},
)
put("uptimeSeconds", JsonPrimitive(uptimeSeconds))
}.toString()
}
private fun infoPayloadJson(): String {
val model = Build.MODEL?.trim().orEmpty()
val manufacturer = Build.MANUFACTURER?.trim().orEmpty()
val modelIdentifier = Build.DEVICE?.trim().orEmpty()
val systemVersion = Build.VERSION.RELEASE?.trim().orEmpty()
val locale = Locale.getDefault().toLanguageTag().trim()
val appVersion = BuildConfig.VERSION_NAME.trim()
val appBuild = BuildConfig.VERSION_CODE.toString()
return buildJsonObject {
put("deviceName", JsonPrimitive(model.ifEmpty { "Android" }))
put("modelIdentifier", JsonPrimitive(modelIdentifier.ifEmpty { listOf(manufacturer, model).filter { it.isNotEmpty() }.joinToString(" ") }))
put("systemName", JsonPrimitive("Android"))
put("systemVersion", JsonPrimitive(systemVersion.ifEmpty { Build.VERSION.SDK_INT.toString() }))
put("appVersion", JsonPrimitive(appVersion.ifEmpty { "dev" }))
put("appBuild", JsonPrimitive(appBuild.ifEmpty { "0" }))
put("locale", JsonPrimitive(locale.ifEmpty { Locale.getDefault().toString() }))
}.toString()
}
private fun permissionsPayloadJson(): String {
val canSendSms = appContext.packageManager.hasSystemFeature(PackageManager.FEATURE_TELEPHONY)
val notificationAccess = DeviceNotificationListenerService.isAccessEnabled(appContext)
return buildJsonObject {
put(
"permissions",
buildJsonObject {
put(
"camera",
permissionStateJson(
granted = hasPermission(Manifest.permission.CAMERA),
promptableWhenDenied = true,
),
)
put(
"microphone",
permissionStateJson(
granted = hasPermission(Manifest.permission.RECORD_AUDIO),
promptableWhenDenied = true,
),
)
put(
"location",
permissionStateJson(
granted =
hasPermission(Manifest.permission.ACCESS_FINE_LOCATION) ||
hasPermission(Manifest.permission.ACCESS_COARSE_LOCATION),
promptableWhenDenied = true,
),
)
put(
"backgroundLocation",
permissionStateJson(
granted = hasPermission(Manifest.permission.ACCESS_BACKGROUND_LOCATION),
promptableWhenDenied = true,
),
)
put(
"sms",
permissionStateJson(
granted = hasPermission(Manifest.permission.SEND_SMS) && canSendSms,
promptableWhenDenied = canSendSms,
),
)
put(
"notificationListener",
permissionStateJson(
granted = notificationAccess,
promptableWhenDenied = true,
),
)
// Screen capture on Android is interactive per-capture consent, not a sticky app permission.
put(
"screenCapture",
permissionStateJson(
granted = false,
promptableWhenDenied = true,
),
)
},
)
}.toString()
}
private fun healthPayloadJson(): String {
val battery = readBatterySnapshot()
val batteryManager = appContext.getSystemService(BatteryManager::class.java)
val currentNowUa = batteryManager?.getLongProperty(BatteryManager.BATTERY_PROPERTY_CURRENT_NOW)
val currentNowMa =
if (currentNowUa == null || currentNowUa == Long.MIN_VALUE) {
null
} else {
currentNowUa.toDouble() / 1_000.0
}
val powerManager = appContext.getSystemService(PowerManager::class.java)
val activityManager = appContext.getSystemService(ActivityManager::class.java)
val memoryInfo = ActivityManager.MemoryInfo()
activityManager?.getMemoryInfo(memoryInfo)
val totalRamBytes = memoryInfo.totalMem.coerceAtLeast(0L)
val availableRamBytes = memoryInfo.availMem.coerceAtLeast(0L)
val usedRamBytes = (totalRamBytes - availableRamBytes).coerceAtLeast(0L)
val lowMemory = memoryInfo.lowMemory
val memoryPressure = mapMemoryPressure(totalRamBytes, availableRamBytes, lowMemory)
return buildJsonObject {
put(
"memory",
buildJsonObject {
put("pressure", JsonPrimitive(memoryPressure))
put("totalRamBytes", JsonPrimitive(totalRamBytes))
put("availableRamBytes", JsonPrimitive(availableRamBytes))
put("usedRamBytes", JsonPrimitive(usedRamBytes))
put("thresholdBytes", JsonPrimitive(memoryInfo.threshold.coerceAtLeast(0L)))
put("lowMemory", JsonPrimitive(lowMemory))
},
)
put(
"battery",
buildJsonObject {
put("state", JsonPrimitive(mapBatteryState(battery.status)))
put("chargingType", JsonPrimitive(mapChargingType(battery.plugged)))
battery.temperatureC?.let { put("temperatureC", JsonPrimitive(it)) }
currentNowMa?.let { put("currentMa", JsonPrimitive(it)) }
},
)
put(
"power",
buildJsonObject {
put("dozeModeEnabled", JsonPrimitive(powerManager?.isDeviceIdleMode == true))
put("lowPowerModeEnabled", JsonPrimitive(powerManager?.isPowerSaveMode == true))
},
)
put(
"system",
buildJsonObject {
Build.VERSION.SECURITY_PATCH
?.trim()
?.takeIf { it.isNotEmpty() }
?.let { put("securityPatchLevel", JsonPrimitive(it)) }
},
)
}.toString()
}
private fun readBatterySnapshot(): BatterySnapshot {
val intent = appContext.registerReceiver(null, IntentFilter(Intent.ACTION_BATTERY_CHANGED))
val status =
intent?.getIntExtra(BatteryManager.EXTRA_STATUS, BatteryManager.BATTERY_STATUS_UNKNOWN)
?: BatteryManager.BATTERY_STATUS_UNKNOWN
val plugged = intent?.getIntExtra(BatteryManager.EXTRA_PLUGGED, 0) ?: 0
val temperatureC =
intent
?.getIntExtra(BatteryManager.EXTRA_TEMPERATURE, Int.MIN_VALUE)
?.takeIf { it != Int.MIN_VALUE }
?.toDouble()
?.div(10.0)
return BatterySnapshot(
status = status,
plugged = plugged,
levelFraction = batteryLevelFraction(intent),
temperatureC = temperatureC,
)
}
private fun batteryLevelFraction(intent: Intent?): Double? {
val rawLevel = intent?.getIntExtra(BatteryManager.EXTRA_LEVEL, -1) ?: -1
val rawScale = intent?.getIntExtra(BatteryManager.EXTRA_SCALE, -1) ?: -1
if (rawLevel < 0 || rawScale <= 0) return null
return rawLevel.toDouble() / rawScale.toDouble()
}
private fun mapBatteryState(status: Int): String {
return when (status) {
BatteryManager.BATTERY_STATUS_CHARGING -> "charging"
BatteryManager.BATTERY_STATUS_FULL -> "full"
BatteryManager.BATTERY_STATUS_DISCHARGING, BatteryManager.BATTERY_STATUS_NOT_CHARGING -> "unplugged"
else -> "unknown"
}
}
private fun mapChargingType(plugged: Int): String {
return when (plugged) {
BatteryManager.BATTERY_PLUGGED_AC -> "ac"
BatteryManager.BATTERY_PLUGGED_USB -> "usb"
BatteryManager.BATTERY_PLUGGED_WIRELESS -> "wireless"
BatteryManager.BATTERY_PLUGGED_DOCK -> "dock"
else -> "none"
}
}
private fun mapThermalState(powerManager: PowerManager?): String {
val thermal = powerManager?.currentThermalStatus ?: return "nominal"
return when (thermal) {
PowerManager.THERMAL_STATUS_NONE, PowerManager.THERMAL_STATUS_LIGHT -> "nominal"
PowerManager.THERMAL_STATUS_MODERATE -> "fair"
PowerManager.THERMAL_STATUS_SEVERE -> "serious"
PowerManager.THERMAL_STATUS_CRITICAL,
PowerManager.THERMAL_STATUS_EMERGENCY,
PowerManager.THERMAL_STATUS_SHUTDOWN -> "critical"
else -> "nominal"
}
}
private fun mapNetworkStatus(caps: NetworkCapabilities?): String {
if (caps == null) return "unsatisfied"
return when {
caps.hasCapability(NetworkCapabilities.NET_CAPABILITY_VALIDATED) -> "satisfied"
caps.hasCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET) -> "requiresConnection"
else -> "unsatisfied"
}
}
private fun permissionStateJson(granted: Boolean, promptableWhenDenied: Boolean) =
buildJsonObject {
put("status", JsonPrimitive(if (granted) "granted" else "denied"))
put("promptable", JsonPrimitive(!granted && promptableWhenDenied))
}
private fun hasPermission(permission: String): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, permission) == PackageManager.PERMISSION_GRANTED
)
}
private fun mapMemoryPressure(totalBytes: Long, availableBytes: Long, lowMemory: Boolean): String {
if (totalBytes <= 0L) return if (lowMemory) "critical" else "unknown"
if (lowMemory) return "critical"
val freeRatio = availableBytes.toDouble() / totalBytes.toDouble()
return when {
freeRatio <= 0.05 -> "critical"
freeRatio <= 0.15 -> "high"
freeRatio <= 0.30 -> "moderate"
else -> "normal"
}
}
private fun networkInterfacesJson(caps: NetworkCapabilities?) =
buildJsonArray {
if (caps == null) return@buildJsonArray
var hasKnownTransport = false
if (caps.hasTransport(NetworkCapabilities.TRANSPORT_WIFI)) {
hasKnownTransport = true
add(JsonPrimitive("wifi"))
}
if (caps.hasTransport(NetworkCapabilities.TRANSPORT_CELLULAR)) {
hasKnownTransport = true
add(JsonPrimitive("cellular"))
}
if (caps.hasTransport(NetworkCapabilities.TRANSPORT_ETHERNET)) {
hasKnownTransport = true
add(JsonPrimitive("wired"))
}
if (!hasKnownTransport) add(JsonPrimitive("other"))
}
}

View File

@@ -0,0 +1,315 @@
package ai.openclaw.android.node
import android.app.Notification
import android.app.NotificationManager
import android.app.RemoteInput
import android.content.ComponentName
import android.content.Context
import android.content.Intent
import android.os.Build
import android.service.notification.NotificationListenerService
import android.service.notification.StatusBarNotification
private const val MAX_NOTIFICATION_TEXT_CHARS = 512
internal fun sanitizeNotificationText(value: CharSequence?): String? {
val normalized = value?.toString()?.trim().orEmpty()
return normalized.take(MAX_NOTIFICATION_TEXT_CHARS).ifEmpty { null }
}
data class DeviceNotificationEntry(
val key: String,
val packageName: String,
val title: String?,
val text: String?,
val subText: String?,
val category: String?,
val channelId: String?,
val postTimeMs: Long,
val isOngoing: Boolean,
val isClearable: Boolean,
)
data class DeviceNotificationSnapshot(
val enabled: Boolean,
val connected: Boolean,
val notifications: List<DeviceNotificationEntry>,
)
enum class NotificationActionKind {
Open,
Dismiss,
Reply,
}
data class NotificationActionRequest(
val key: String,
val kind: NotificationActionKind,
val replyText: String? = null,
)
data class NotificationActionResult(
val ok: Boolean,
val code: String? = null,
val message: String? = null,
)
internal fun actionRequiresClearableNotification(kind: NotificationActionKind): Boolean {
return kind == NotificationActionKind.Dismiss
}
private object DeviceNotificationStore {
private val lock = Any()
private var connected = false
private val byKey = LinkedHashMap<String, DeviceNotificationEntry>()
fun replace(entries: List<DeviceNotificationEntry>) {
synchronized(lock) {
byKey.clear()
for (entry in entries) {
byKey[entry.key] = entry
}
}
}
fun upsert(entry: DeviceNotificationEntry) {
synchronized(lock) {
byKey[entry.key] = entry
}
}
fun remove(key: String) {
synchronized(lock) {
byKey.remove(key)
}
}
fun setConnected(value: Boolean) {
synchronized(lock) {
connected = value
if (!value) {
byKey.clear()
}
}
}
fun snapshot(enabled: Boolean): DeviceNotificationSnapshot {
val (isConnected, entries) =
synchronized(lock) {
connected to byKey.values.sortedByDescending { it.postTimeMs }
}
return DeviceNotificationSnapshot(
enabled = enabled,
connected = isConnected,
notifications = entries,
)
}
}
class DeviceNotificationListenerService : NotificationListenerService() {
override fun onListenerConnected() {
super.onListenerConnected()
activeService = this
DeviceNotificationStore.setConnected(true)
refreshActiveNotifications()
}
override fun onListenerDisconnected() {
if (activeService === this) {
activeService = null
}
DeviceNotificationStore.setConnected(false)
super.onListenerDisconnected()
}
override fun onDestroy() {
if (activeService === this) {
activeService = null
}
super.onDestroy()
}
override fun onNotificationPosted(sbn: StatusBarNotification?) {
super.onNotificationPosted(sbn)
val entry = sbn?.toEntry() ?: return
DeviceNotificationStore.upsert(entry)
}
override fun onNotificationRemoved(sbn: StatusBarNotification?) {
super.onNotificationRemoved(sbn)
val key = sbn?.key ?: return
DeviceNotificationStore.remove(key)
}
private fun refreshActiveNotifications() {
val entries =
runCatching {
activeNotifications
?.mapNotNull { it.toEntry() }
?: emptyList()
}.getOrElse { emptyList() }
DeviceNotificationStore.replace(entries)
}
private fun StatusBarNotification.toEntry(): DeviceNotificationEntry {
val extras = notification.extras
val keyValue = key.takeIf { it.isNotBlank() } ?: "$packageName:$id:$postTime"
val title = sanitizeNotificationText(extras?.getCharSequence(Notification.EXTRA_TITLE))
val body =
sanitizeNotificationText(extras?.getCharSequence(Notification.EXTRA_BIG_TEXT))
?: sanitizeNotificationText(extras?.getCharSequence(Notification.EXTRA_TEXT))
val subText = sanitizeNotificationText(extras?.getCharSequence(Notification.EXTRA_SUB_TEXT))
return DeviceNotificationEntry(
key = keyValue,
packageName = packageName,
title = title,
text = body,
subText = subText,
category = notification.category?.trim()?.ifEmpty { null },
channelId = notification.channelId?.trim()?.ifEmpty { null },
postTimeMs = postTime,
isOngoing = isOngoing,
isClearable = isClearable,
)
}
companion object {
@Volatile private var activeService: DeviceNotificationListenerService? = null
private fun serviceComponent(context: Context): ComponentName {
return ComponentName(context, DeviceNotificationListenerService::class.java)
}
fun isAccessEnabled(context: Context): Boolean {
val manager = context.getSystemService(NotificationManager::class.java) ?: return false
return manager.isNotificationListenerAccessGranted(serviceComponent(context))
}
fun snapshot(context: Context, enabled: Boolean = isAccessEnabled(context)): DeviceNotificationSnapshot {
return DeviceNotificationStore.snapshot(enabled = enabled)
}
fun requestServiceRebind(context: Context) {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) {
return
}
runCatching {
NotificationListenerService.requestRebind(serviceComponent(context))
}
}
fun executeAction(context: Context, request: NotificationActionRequest): NotificationActionResult {
if (!isAccessEnabled(context)) {
return NotificationActionResult(
ok = false,
code = "NOTIFICATIONS_DISABLED",
message = "NOTIFICATIONS_DISABLED: enable notification access in system Settings",
)
}
val service = activeService
?: return NotificationActionResult(
ok = false,
code = "NOTIFICATIONS_UNAVAILABLE",
message = "NOTIFICATIONS_UNAVAILABLE: notification listener not connected",
)
return service.executeActionInternal(request)
}
}
private fun executeActionInternal(request: NotificationActionRequest): NotificationActionResult {
val sbn =
activeNotifications
?.firstOrNull { it.key == request.key }
?: return NotificationActionResult(
ok = false,
code = "NOTIFICATION_NOT_FOUND",
message = "NOTIFICATION_NOT_FOUND: notification key not found",
)
if (actionRequiresClearableNotification(request.kind) && !sbn.isClearable) {
return NotificationActionResult(
ok = false,
code = "NOTIFICATION_NOT_CLEARABLE",
message = "NOTIFICATION_NOT_CLEARABLE: notification is ongoing or protected",
)
}
return when (request.kind) {
NotificationActionKind.Open -> {
val pendingIntent = sbn.notification.contentIntent
?: return NotificationActionResult(
ok = false,
code = "ACTION_UNAVAILABLE",
message = "ACTION_UNAVAILABLE: notification has no open action",
)
runCatching {
pendingIntent.send()
}.fold(
onSuccess = { NotificationActionResult(ok = true) },
onFailure = { err ->
NotificationActionResult(
ok = false,
code = "ACTION_FAILED",
message = "ACTION_FAILED: ${err.message ?: "open failed"}",
)
},
)
}
NotificationActionKind.Dismiss -> {
runCatching {
cancelNotification(sbn.key)
DeviceNotificationStore.remove(sbn.key)
}.fold(
onSuccess = { NotificationActionResult(ok = true) },
onFailure = { err ->
NotificationActionResult(
ok = false,
code = "ACTION_FAILED",
message = "ACTION_FAILED: ${err.message ?: "dismiss failed"}",
)
},
)
}
NotificationActionKind.Reply -> {
val replyText = request.replyText?.trim().orEmpty()
if (replyText.isEmpty()) {
return NotificationActionResult(
ok = false,
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: replyText required for reply action",
)
}
val action =
sbn.notification.actions
?.firstOrNull { candidate ->
candidate.actionIntent != null && !candidate.remoteInputs.isNullOrEmpty()
}
?: return NotificationActionResult(
ok = false,
code = "ACTION_UNAVAILABLE",
message = "ACTION_UNAVAILABLE: notification has no reply action",
)
val remoteInputs = action.remoteInputs ?: emptyArray()
val fillInIntent = Intent()
val replyBundle = android.os.Bundle()
for (remoteInput in remoteInputs) {
replyBundle.putCharSequence(remoteInput.resultKey, replyText)
}
RemoteInput.addResultsToIntent(remoteInputs, fillInIntent, replyBundle)
runCatching {
action.actionIntent.send(this, 0, fillInIntent)
}.fold(
onSuccess = { NotificationActionResult(ok = true) },
onFailure = { err ->
NotificationActionResult(
ok = false,
code = "ACTION_FAILED",
message = "ACTION_FAILED: ${err.message ?: "reply failed"}",
)
},
)
}
}
}
}

View File

@@ -0,0 +1,71 @@
package ai.openclaw.android.node
import ai.openclaw.android.SecurePrefs
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
class GatewayEventHandler(
private val scope: CoroutineScope,
private val prefs: SecurePrefs,
private val json: Json,
private val operatorSession: GatewaySession,
private val isConnected: () -> Boolean,
) {
private var suppressWakeWordsSync = false
private var wakeWordsSyncJob: Job? = null
fun applyWakeWordsFromGateway(words: List<String>) {
suppressWakeWordsSync = true
prefs.setWakeWords(words)
suppressWakeWordsSync = false
}
fun scheduleWakeWordsSyncIfNeeded() {
if (suppressWakeWordsSync) return
if (!isConnected()) return
val snapshot = prefs.wakeWords.value
wakeWordsSyncJob?.cancel()
wakeWordsSyncJob =
scope.launch {
delay(650)
val jsonList = snapshot.joinToString(separator = ",") { it.toJsonString() }
val params = """{"triggers":[$jsonList]}"""
try {
operatorSession.request("voicewake.set", params)
} catch (_: Throwable) {
// ignore
}
}
}
suspend fun refreshWakeWordsFromGateway() {
if (!isConnected()) return
try {
val res = operatorSession.request("voicewake.get", "{}")
val payload = json.parseToJsonElement(res).asObjectOrNull() ?: return
val array = payload["triggers"] as? JsonArray ?: return
val triggers = array.mapNotNull { it.asStringOrNull() }
applyWakeWordsFromGateway(triggers)
} catch (_: Throwable) {
// ignore
}
}
fun handleVoiceWakeChangedEvent(payloadJson: String?) {
if (payloadJson.isNullOrBlank()) return
try {
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
val array = payload["triggers"] as? JsonArray ?: return
val triggers = array.mapNotNull { it.asStringOrNull() }
applyWakeWordsFromGateway(triggers)
} catch (_: Throwable) {
// ignore
}
}
}

View File

@@ -0,0 +1,139 @@
package ai.openclaw.android.node
import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
import ai.openclaw.android.protocol.OpenClawCanvasCommand
import ai.openclaw.android.protocol.OpenClawCameraCommand
import ai.openclaw.android.protocol.OpenClawDeviceCommand
import ai.openclaw.android.protocol.OpenClawLocationCommand
import ai.openclaw.android.protocol.OpenClawNotificationsCommand
import ai.openclaw.android.protocol.OpenClawScreenCommand
import ai.openclaw.android.protocol.OpenClawSmsCommand
enum class InvokeCommandAvailability {
Always,
CameraEnabled,
LocationEnabled,
SmsAvailable,
DebugBuild,
}
data class InvokeCommandSpec(
val name: String,
val requiresForeground: Boolean = false,
val availability: InvokeCommandAvailability = InvokeCommandAvailability.Always,
)
object InvokeCommandRegistry {
val all: List<InvokeCommandSpec> =
listOf(
InvokeCommandSpec(
name = OpenClawCanvasCommand.Present.rawValue,
requiresForeground = true,
),
InvokeCommandSpec(
name = OpenClawCanvasCommand.Hide.rawValue,
requiresForeground = true,
),
InvokeCommandSpec(
name = OpenClawCanvasCommand.Navigate.rawValue,
requiresForeground = true,
),
InvokeCommandSpec(
name = OpenClawCanvasCommand.Eval.rawValue,
requiresForeground = true,
),
InvokeCommandSpec(
name = OpenClawCanvasCommand.Snapshot.rawValue,
requiresForeground = true,
),
InvokeCommandSpec(
name = OpenClawCanvasA2UICommand.Push.rawValue,
requiresForeground = true,
),
InvokeCommandSpec(
name = OpenClawCanvasA2UICommand.PushJSONL.rawValue,
requiresForeground = true,
),
InvokeCommandSpec(
name = OpenClawCanvasA2UICommand.Reset.rawValue,
requiresForeground = true,
),
InvokeCommandSpec(
name = OpenClawScreenCommand.Record.rawValue,
requiresForeground = true,
),
InvokeCommandSpec(
name = OpenClawCameraCommand.List.rawValue,
requiresForeground = true,
availability = InvokeCommandAvailability.CameraEnabled,
),
InvokeCommandSpec(
name = OpenClawCameraCommand.Snap.rawValue,
requiresForeground = true,
availability = InvokeCommandAvailability.CameraEnabled,
),
InvokeCommandSpec(
name = OpenClawCameraCommand.Clip.rawValue,
requiresForeground = true,
availability = InvokeCommandAvailability.CameraEnabled,
),
InvokeCommandSpec(
name = OpenClawLocationCommand.Get.rawValue,
availability = InvokeCommandAvailability.LocationEnabled,
),
InvokeCommandSpec(
name = OpenClawDeviceCommand.Status.rawValue,
),
InvokeCommandSpec(
name = OpenClawDeviceCommand.Info.rawValue,
),
InvokeCommandSpec(
name = OpenClawDeviceCommand.Permissions.rawValue,
),
InvokeCommandSpec(
name = OpenClawDeviceCommand.Health.rawValue,
),
InvokeCommandSpec(
name = OpenClawNotificationsCommand.List.rawValue,
),
InvokeCommandSpec(
name = OpenClawNotificationsCommand.Actions.rawValue,
),
InvokeCommandSpec(
name = OpenClawSmsCommand.Send.rawValue,
availability = InvokeCommandAvailability.SmsAvailable,
),
InvokeCommandSpec(
name = "debug.logs",
availability = InvokeCommandAvailability.DebugBuild,
),
InvokeCommandSpec(
name = "debug.ed25519",
availability = InvokeCommandAvailability.DebugBuild,
),
InvokeCommandSpec(name = "app.update"),
)
private val byNameInternal: Map<String, InvokeCommandSpec> = all.associateBy { it.name }
fun find(command: String): InvokeCommandSpec? = byNameInternal[command]
fun advertisedCommands(
cameraEnabled: Boolean,
locationEnabled: Boolean,
smsAvailable: Boolean,
debugBuild: Boolean,
): List<String> {
return all
.filter { spec ->
when (spec.availability) {
InvokeCommandAvailability.Always -> true
InvokeCommandAvailability.CameraEnabled -> cameraEnabled
InvokeCommandAvailability.LocationEnabled -> locationEnabled
InvokeCommandAvailability.SmsAvailable -> smsAvailable
InvokeCommandAvailability.DebugBuild -> debugBuild
}
}
.map { it.name }
}
}

View File

@@ -0,0 +1,235 @@
package ai.openclaw.android.node
import ai.openclaw.android.gateway.GatewaySession
import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
import ai.openclaw.android.protocol.OpenClawCanvasCommand
import ai.openclaw.android.protocol.OpenClawCameraCommand
import ai.openclaw.android.protocol.OpenClawDeviceCommand
import ai.openclaw.android.protocol.OpenClawLocationCommand
import ai.openclaw.android.protocol.OpenClawNotificationsCommand
import ai.openclaw.android.protocol.OpenClawScreenCommand
import ai.openclaw.android.protocol.OpenClawSmsCommand
class InvokeDispatcher(
private val canvas: CanvasController,
private val cameraHandler: CameraHandler,
private val locationHandler: LocationHandler,
private val deviceHandler: DeviceHandler,
private val notificationsHandler: NotificationsHandler,
private val screenHandler: ScreenHandler,
private val smsHandler: SmsHandler,
private val a2uiHandler: A2UIHandler,
private val debugHandler: DebugHandler,
private val appUpdateHandler: AppUpdateHandler,
private val isForeground: () -> Boolean,
private val cameraEnabled: () -> Boolean,
private val locationEnabled: () -> Boolean,
private val smsAvailable: () -> Boolean,
private val debugBuild: () -> Boolean,
private val refreshNodeCanvasCapability: suspend () -> Boolean,
private val onCanvasA2uiPush: () -> Unit,
private val onCanvasA2uiReset: () -> Unit,
) {
suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult {
val spec =
InvokeCommandRegistry.find(command)
?: return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: unknown command",
)
if (spec.requiresForeground && !isForeground()) {
return GatewaySession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground",
)
}
availabilityError(spec.availability)?.let { return it }
return when (command) {
// Canvas commands
OpenClawCanvasCommand.Present.rawValue -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
GatewaySession.InvokeResult.ok(null)
}
OpenClawCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null)
OpenClawCanvasCommand.Navigate.rawValue -> {
val url = CanvasController.parseNavigateUrl(paramsJson)
canvas.navigate(url)
GatewaySession.InvokeResult.ok(null)
}
OpenClawCanvasCommand.Eval.rawValue -> {
val js =
CanvasController.parseEvalJs(paramsJson)
?: return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: javaScript required",
)
withCanvasAvailable {
val result = canvas.eval(js)
GatewaySession.InvokeResult.ok("""{"result":${result.toJsonString()}}""")
}
}
OpenClawCanvasCommand.Snapshot.rawValue -> {
val snapshotParams = CanvasController.parseSnapshotParams(paramsJson)
withCanvasAvailable {
val base64 =
canvas.snapshotBase64(
format = snapshotParams.format,
quality = snapshotParams.quality,
maxWidth = snapshotParams.maxWidth,
)
GatewaySession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""")
}
}
// A2UI commands
OpenClawCanvasA2UICommand.Reset.rawValue ->
withReadyA2ui {
withCanvasAvailable {
val res = canvas.eval(A2UIHandler.a2uiResetJS)
onCanvasA2uiReset()
GatewaySession.InvokeResult.ok(res)
}
}
OpenClawCanvasA2UICommand.Push.rawValue, OpenClawCanvasA2UICommand.PushJSONL.rawValue -> {
val messages =
try {
a2uiHandler.decodeA2uiMessages(command, paramsJson)
} catch (err: Throwable) {
return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = err.message ?: "invalid A2UI payload"
)
}
withReadyA2ui {
withCanvasAvailable {
val js = A2UIHandler.a2uiApplyMessagesJS(messages)
val res = canvas.eval(js)
onCanvasA2uiPush()
GatewaySession.InvokeResult.ok(res)
}
}
}
// Camera commands
OpenClawCameraCommand.List.rawValue -> cameraHandler.handleList(paramsJson)
OpenClawCameraCommand.Snap.rawValue -> cameraHandler.handleSnap(paramsJson)
OpenClawCameraCommand.Clip.rawValue -> cameraHandler.handleClip(paramsJson)
// Location command
OpenClawLocationCommand.Get.rawValue -> locationHandler.handleLocationGet(paramsJson)
// Device commands
OpenClawDeviceCommand.Status.rawValue -> deviceHandler.handleDeviceStatus(paramsJson)
OpenClawDeviceCommand.Info.rawValue -> deviceHandler.handleDeviceInfo(paramsJson)
OpenClawDeviceCommand.Permissions.rawValue -> deviceHandler.handleDevicePermissions(paramsJson)
OpenClawDeviceCommand.Health.rawValue -> deviceHandler.handleDeviceHealth(paramsJson)
// Notifications command
OpenClawNotificationsCommand.List.rawValue -> notificationsHandler.handleNotificationsList(paramsJson)
OpenClawNotificationsCommand.Actions.rawValue -> notificationsHandler.handleNotificationsActions(paramsJson)
// Screen command
OpenClawScreenCommand.Record.rawValue -> screenHandler.handleScreenRecord(paramsJson)
// SMS command
OpenClawSmsCommand.Send.rawValue -> smsHandler.handleSmsSend(paramsJson)
// Debug commands
"debug.ed25519" -> debugHandler.handleEd25519()
"debug.logs" -> debugHandler.handleLogs()
// App update
"app.update" -> appUpdateHandler.handleUpdate(paramsJson)
else -> GatewaySession.InvokeResult.error(code = "INVALID_REQUEST", message = "INVALID_REQUEST: unknown command")
}
}
private suspend fun withReadyA2ui(
block: suspend () -> GatewaySession.InvokeResult,
): GatewaySession.InvokeResult {
var a2uiUrl = a2uiHandler.resolveA2uiHostUrl()
?: return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_NOT_CONFIGURED",
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
)
val readyOnFirstCheck = a2uiHandler.ensureA2uiReady(a2uiUrl)
if (!readyOnFirstCheck) {
if (!refreshNodeCanvasCapability()) {
return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_UNAVAILABLE",
message = "A2UI_HOST_UNAVAILABLE: A2UI host not reachable",
)
}
a2uiUrl = a2uiHandler.resolveA2uiHostUrl()
?: return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_NOT_CONFIGURED",
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
)
if (!a2uiHandler.ensureA2uiReady(a2uiUrl)) {
return GatewaySession.InvokeResult.error(
code = "A2UI_HOST_UNAVAILABLE",
message = "A2UI_HOST_UNAVAILABLE: A2UI host not reachable",
)
}
}
return block()
}
private suspend fun withCanvasAvailable(
block: suspend () -> GatewaySession.InvokeResult,
): GatewaySession.InvokeResult {
return try {
block()
} catch (_: Throwable) {
GatewaySession.InvokeResult.error(
code = "NODE_BACKGROUND_UNAVAILABLE",
message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
)
}
}
private fun availabilityError(availability: InvokeCommandAvailability): GatewaySession.InvokeResult? {
return when (availability) {
InvokeCommandAvailability.Always -> null
InvokeCommandAvailability.CameraEnabled ->
if (cameraEnabled()) {
null
} else {
GatewaySession.InvokeResult.error(
code = "CAMERA_DISABLED",
message = "CAMERA_DISABLED: enable Camera in Settings",
)
}
InvokeCommandAvailability.LocationEnabled ->
if (locationEnabled()) {
null
} else {
GatewaySession.InvokeResult.error(
code = "LOCATION_DISABLED",
message = "LOCATION_DISABLED: enable Location in Settings",
)
}
InvokeCommandAvailability.SmsAvailable ->
if (smsAvailable()) {
null
} else {
GatewaySession.InvokeResult.error(
code = "SMS_UNAVAILABLE",
message = "SMS_UNAVAILABLE: SMS not available on this device",
)
}
InvokeCommandAvailability.DebugBuild ->
if (debugBuild()) {
null
} else {
GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: unknown command",
)
}
}
}
}

View File

@@ -0,0 +1,61 @@
package ai.openclaw.android.node
import kotlin.math.max
import kotlin.math.min
import kotlin.math.roundToInt
internal data class JpegSizeLimiterResult(
val bytes: ByteArray,
val width: Int,
val height: Int,
val quality: Int,
)
internal object JpegSizeLimiter {
fun compressToLimit(
initialWidth: Int,
initialHeight: Int,
startQuality: Int,
maxBytes: Int,
minQuality: Int = 20,
minSize: Int = 256,
scaleStep: Double = 0.85,
maxScaleAttempts: Int = 6,
maxQualityAttempts: Int = 6,
encode: (width: Int, height: Int, quality: Int) -> ByteArray,
): JpegSizeLimiterResult {
require(initialWidth > 0 && initialHeight > 0) { "Invalid image size" }
require(maxBytes > 0) { "Invalid maxBytes" }
var width = initialWidth
var height = initialHeight
val clampedStartQuality = startQuality.coerceIn(minQuality, 100)
var best = JpegSizeLimiterResult(bytes = encode(width, height, clampedStartQuality), width = width, height = height, quality = clampedStartQuality)
if (best.bytes.size <= maxBytes) return best
repeat(maxScaleAttempts) {
var quality = clampedStartQuality
repeat(maxQualityAttempts) {
val bytes = encode(width, height, quality)
best = JpegSizeLimiterResult(bytes = bytes, width = width, height = height, quality = quality)
if (bytes.size <= maxBytes) return best
if (quality <= minQuality) return@repeat
quality = max(minQuality, (quality * 0.75).roundToInt())
}
val minScale = (minSize.toDouble() / min(width, height).toDouble()).coerceAtMost(1.0)
val nextScale = max(scaleStep, minScale)
val nextWidth = max(minSize, (width * nextScale).roundToInt())
val nextHeight = max(minSize, (height * nextScale).roundToInt())
if (nextWidth == width && nextHeight == height) return@repeat
width = min(nextWidth, width)
height = min(nextHeight, height)
}
if (best.bytes.size > maxBytes) {
throw IllegalStateException("CAMERA_TOO_LARGE: ${best.bytes.size} bytes > $maxBytes bytes")
}
return best
}
}

View File

@@ -0,0 +1,117 @@
package ai.openclaw.android.node
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.location.Location
import android.location.LocationManager
import android.os.CancellationSignal
import androidx.core.content.ContextCompat
import java.time.Instant
import java.time.format.DateTimeFormatter
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeout
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
import kotlinx.coroutines.suspendCancellableCoroutine
class LocationCaptureManager(private val context: Context) {
data class Payload(val payloadJson: String)
suspend fun getLocation(
desiredProviders: List<String>,
maxAgeMs: Long?,
timeoutMs: Long,
isPrecise: Boolean,
): Payload =
withContext(Dispatchers.Main) {
val manager = context.getSystemService(Context.LOCATION_SERVICE) as LocationManager
if (!manager.isProviderEnabled(LocationManager.GPS_PROVIDER) &&
!manager.isProviderEnabled(LocationManager.NETWORK_PROVIDER)
) {
throw IllegalStateException("LOCATION_UNAVAILABLE: no location providers enabled")
}
val cached = bestLastKnown(manager, desiredProviders, maxAgeMs)
val location =
cached ?: requestCurrent(manager, desiredProviders, timeoutMs)
val timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(location.time))
val source = location.provider
val altitudeMeters = if (location.hasAltitude()) location.altitude else null
val speedMps = if (location.hasSpeed()) location.speed.toDouble() else null
val headingDeg = if (location.hasBearing()) location.bearing.toDouble() else null
Payload(
buildString {
append("{\"lat\":")
append(location.latitude)
append(",\"lon\":")
append(location.longitude)
append(",\"accuracyMeters\":")
append(location.accuracy.toDouble())
if (altitudeMeters != null) append(",\"altitudeMeters\":").append(altitudeMeters)
if (speedMps != null) append(",\"speedMps\":").append(speedMps)
if (headingDeg != null) append(",\"headingDeg\":").append(headingDeg)
append(",\"timestamp\":\"").append(timestamp).append('"')
append(",\"isPrecise\":").append(isPrecise)
append(",\"source\":\"").append(source).append('"')
append('}')
},
)
}
private fun bestLastKnown(
manager: LocationManager,
providers: List<String>,
maxAgeMs: Long?,
): Location? {
val fineOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
val coarseOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
if (!fineOk && !coarseOk) {
throw IllegalStateException("LOCATION_PERMISSION_REQUIRED: grant Location permission")
}
val now = System.currentTimeMillis()
val candidates =
providers.mapNotNull { provider -> manager.getLastKnownLocation(provider) }
val freshest = candidates.maxByOrNull { it.time } ?: return null
if (maxAgeMs != null && now - freshest.time > maxAgeMs) return null
return freshest
}
private suspend fun requestCurrent(
manager: LocationManager,
providers: List<String>,
timeoutMs: Long,
): Location {
val fineOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
val coarseOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
if (!fineOk && !coarseOk) {
throw IllegalStateException("LOCATION_PERMISSION_REQUIRED: grant Location permission")
}
val resolved =
providers.firstOrNull { manager.isProviderEnabled(it) }
?: throw IllegalStateException("LOCATION_UNAVAILABLE: no providers available")
return withTimeout(timeoutMs.coerceAtLeast(1)) {
suspendCancellableCoroutine { cont ->
val signal = CancellationSignal()
cont.invokeOnCancellation { signal.cancel() }
manager.getCurrentLocation(resolved, signal, context.mainExecutor) { location ->
if (location != null) {
cont.resume(location)
} else {
cont.resumeWithException(IllegalStateException("LOCATION_UNAVAILABLE: no fix"))
}
}
}
}
}
}

View File

@@ -0,0 +1,116 @@
package ai.openclaw.android.node
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.location.LocationManager
import androidx.core.content.ContextCompat
import ai.openclaw.android.LocationMode
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.coroutines.TimeoutCancellationException
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
class LocationHandler(
private val appContext: Context,
private val location: LocationCaptureManager,
private val json: Json,
private val isForeground: () -> Boolean,
private val locationMode: () -> LocationMode,
private val locationPreciseEnabled: () -> Boolean,
) {
fun hasFineLocationPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_FINE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
)
}
fun hasCoarseLocationPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_COARSE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
)
}
fun hasBackgroundLocationPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ==
PackageManager.PERMISSION_GRANTED
)
}
suspend fun handleLocationGet(paramsJson: String?): GatewaySession.InvokeResult {
val mode = locationMode()
if (!isForeground() && mode != LocationMode.Always) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_BACKGROUND_UNAVAILABLE",
message = "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always",
)
}
if (!hasFineLocationPermission() && !hasCoarseLocationPermission()) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_PERMISSION_REQUIRED",
message = "LOCATION_PERMISSION_REQUIRED: grant Location permission",
)
}
if (!isForeground() && mode == LocationMode.Always && !hasBackgroundLocationPermission()) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_PERMISSION_REQUIRED",
message = "LOCATION_PERMISSION_REQUIRED: enable Always in system Settings",
)
}
val (maxAgeMs, timeoutMs, desiredAccuracy) = parseLocationParams(paramsJson)
val preciseEnabled = locationPreciseEnabled()
val accuracy =
when (desiredAccuracy) {
"precise" -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
"coarse" -> "coarse"
else -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
}
val providers =
when (accuracy) {
"precise" -> listOf(LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER)
"coarse" -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
else -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
}
try {
val payload =
location.getLocation(
desiredProviders = providers,
maxAgeMs = maxAgeMs,
timeoutMs = timeoutMs,
isPrecise = accuracy == "precise",
)
return GatewaySession.InvokeResult.ok(payload.payloadJson)
} catch (err: TimeoutCancellationException) {
return GatewaySession.InvokeResult.error(
code = "LOCATION_TIMEOUT",
message = "LOCATION_TIMEOUT: no fix in time",
)
} catch (err: Throwable) {
val message = err.message ?: "LOCATION_UNAVAILABLE: no fix"
return GatewaySession.InvokeResult.error(code = "LOCATION_UNAVAILABLE", message = message)
}
}
private fun parseLocationParams(paramsJson: String?): Triple<Long?, Long, String?> {
if (paramsJson.isNullOrBlank()) {
return Triple(null, 10_000L, null)
}
val root =
try {
json.parseToJsonElement(paramsJson).asObjectOrNull()
} catch (_: Throwable) {
null
}
val maxAgeMs = (root?.get("maxAgeMs") as? JsonPrimitive)?.content?.toLongOrNull()
val timeoutMs =
(root?.get("timeoutMs") as? JsonPrimitive)?.content?.toLongOrNull()?.coerceIn(1_000L, 60_000L)
?: 10_000L
val desiredAccuracy =
(root?.get("desiredAccuracy") as? JsonPrimitive)?.content?.trim()?.lowercase()
return Triple(maxAgeMs, timeoutMs, desiredAccuracy)
}
}

View File

@@ -0,0 +1,53 @@
package ai.openclaw.android.node
import ai.openclaw.android.gateway.parseInvokeErrorFromThrowable
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonNull
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A
data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
fun String.toJsonString(): String {
val escaped =
this.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r")
return "\"$escaped\""
}
fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
fun JsonElement?.asStringOrNull(): String? =
when (this) {
is JsonNull -> null
is JsonPrimitive -> content
else -> null
}
fun parseHexColorArgb(raw: String?): Long? {
val trimmed = raw?.trim().orEmpty()
if (trimmed.isEmpty()) return null
val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed
if (hex.length != 6) return null
val rgb = hex.toLongOrNull(16) ?: return null
return 0xFF000000L or rgb
}
fun invokeErrorFromThrowable(err: Throwable): Pair<String, String> {
val parsed = parseInvokeErrorFromThrowable(err, fallbackMessage = "UNAVAILABLE: error")
val message = if (parsed.hadExplicitCode) parsed.prefixedMessage else parsed.message
return parsed.code to message
}
fun normalizeMainKey(raw: String?): String? {
val trimmed = raw?.trim().orEmpty()
return if (trimmed.isEmpty()) null else trimmed
}
fun isCanonicalMainSessionKey(key: String): Boolean {
return key == "main"
}

View File

@@ -0,0 +1,174 @@
package ai.openclaw.android.node
import android.content.Context
import ai.openclaw.android.gateway.GatewaySession
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.put
internal interface NotificationsStateProvider {
fun readSnapshot(context: Context): DeviceNotificationSnapshot
fun requestServiceRebind(context: Context)
fun executeAction(context: Context, request: NotificationActionRequest): NotificationActionResult
}
private object SystemNotificationsStateProvider : NotificationsStateProvider {
override fun readSnapshot(context: Context): DeviceNotificationSnapshot {
val enabled = DeviceNotificationListenerService.isAccessEnabled(context)
if (!enabled) {
return DeviceNotificationSnapshot(
enabled = false,
connected = false,
notifications = emptyList(),
)
}
return DeviceNotificationListenerService.snapshot(context, enabled = true)
}
override fun requestServiceRebind(context: Context) {
DeviceNotificationListenerService.requestServiceRebind(context)
}
override fun executeAction(context: Context, request: NotificationActionRequest): NotificationActionResult {
return DeviceNotificationListenerService.executeAction(context, request)
}
}
class NotificationsHandler private constructor(
private val appContext: Context,
private val stateProvider: NotificationsStateProvider,
) {
constructor(appContext: Context) : this(appContext = appContext, stateProvider = SystemNotificationsStateProvider)
suspend fun handleNotificationsList(_paramsJson: String?): GatewaySession.InvokeResult {
val snapshot = readSnapshotWithRebind()
return GatewaySession.InvokeResult.ok(snapshotPayloadJson(snapshot))
}
suspend fun handleNotificationsActions(paramsJson: String?): GatewaySession.InvokeResult {
readSnapshotWithRebind()
val params = parseParamsObject(paramsJson)
?: return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: expected JSON object",
)
val key =
readString(params, "key")
?: return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: key required",
)
val actionRaw =
readString(params, "action")?.lowercase()
?: return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: action required (open|dismiss|reply)",
)
val action =
when (actionRaw) {
"open" -> NotificationActionKind.Open
"dismiss" -> NotificationActionKind.Dismiss
"reply" -> NotificationActionKind.Reply
else ->
return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: action must be open|dismiss|reply",
)
}
val replyText = readString(params, "replyText")
if (action == NotificationActionKind.Reply && replyText.isNullOrBlank()) {
return GatewaySession.InvokeResult.error(
code = "INVALID_REQUEST",
message = "INVALID_REQUEST: replyText required for reply action",
)
}
val result =
stateProvider.executeAction(
appContext,
NotificationActionRequest(
key = key,
kind = action,
replyText = replyText,
),
)
if (!result.ok) {
return GatewaySession.InvokeResult.error(
code = result.code ?: "UNAVAILABLE",
message = result.message ?: "notification action failed",
)
}
val payload =
buildJsonObject {
put("ok", JsonPrimitive(true))
put("key", JsonPrimitive(key))
put("action", JsonPrimitive(actionRaw))
}.toString()
return GatewaySession.InvokeResult.ok(payload)
}
private fun readSnapshotWithRebind(): DeviceNotificationSnapshot {
val snapshot = stateProvider.readSnapshot(appContext)
if (snapshot.enabled && !snapshot.connected) {
stateProvider.requestServiceRebind(appContext)
}
return snapshot
}
private fun snapshotPayloadJson(snapshot: DeviceNotificationSnapshot): String {
return buildJsonObject {
put("enabled", JsonPrimitive(snapshot.enabled))
put("connected", JsonPrimitive(snapshot.connected))
put("count", JsonPrimitive(snapshot.notifications.size))
put(
"notifications",
JsonArray(
snapshot.notifications.map { entry ->
buildJsonObject {
put("key", JsonPrimitive(entry.key))
put("packageName", JsonPrimitive(entry.packageName))
put("postTimeMs", JsonPrimitive(entry.postTimeMs))
put("isOngoing", JsonPrimitive(entry.isOngoing))
put("isClearable", JsonPrimitive(entry.isClearable))
entry.title?.let { put("title", JsonPrimitive(it)) }
entry.text?.let { put("text", JsonPrimitive(it)) }
entry.subText?.let { put("subText", JsonPrimitive(it)) }
entry.category?.let { put("category", JsonPrimitive(it)) }
entry.channelId?.let { put("channelId", JsonPrimitive(it)) }
}
},
),
)
}.toString()
}
private fun parseParamsObject(paramsJson: String?): JsonObject? {
if (paramsJson.isNullOrBlank()) return null
return try {
Json.parseToJsonElement(paramsJson).asObjectOrNull()
} catch (_: Throwable) {
null
}
}
private fun readString(params: JsonObject, key: String): String? =
(params[key] as? JsonPrimitive)
?.contentOrNull
?.trim()
?.takeIf { it.isNotEmpty() }
companion object {
internal fun forTesting(
appContext: Context,
stateProvider: NotificationsStateProvider,
): NotificationsHandler = NotificationsHandler(appContext = appContext, stateProvider = stateProvider)
}
}

View File

@@ -0,0 +1,25 @@
package ai.openclaw.android.node
import ai.openclaw.android.gateway.GatewaySession
class ScreenHandler(
private val screenRecorder: ScreenRecordManager,
private val setScreenRecordActive: (Boolean) -> Unit,
private val invokeErrorFromThrowable: (Throwable) -> Pair<String, String>,
) {
suspend fun handleScreenRecord(paramsJson: String?): GatewaySession.InvokeResult {
setScreenRecordActive(true)
try {
val res =
try {
screenRecorder.record(paramsJson)
} catch (err: Throwable) {
val (code, message) = invokeErrorFromThrowable(err)
return GatewaySession.InvokeResult.error(code = code, message = message)
}
return GatewaySession.InvokeResult.ok(res.payloadJson)
} finally {
setScreenRecordActive(false)
}
}
}

View File

@@ -0,0 +1,187 @@
package ai.openclaw.android.node
import android.content.Context
import android.hardware.display.DisplayManager
import android.media.MediaRecorder
import android.media.projection.MediaProjectionManager
import android.os.Build
import android.util.Base64
import ai.openclaw.android.ScreenCaptureRequester
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.contentOrNull
import java.io.File
import kotlin.math.roundToInt
class ScreenRecordManager(private val context: Context) {
data class Payload(val payloadJson: String)
@Volatile private var screenCaptureRequester: ScreenCaptureRequester? = null
@Volatile private var permissionRequester: ai.openclaw.android.PermissionRequester? = null
fun attachScreenCaptureRequester(requester: ScreenCaptureRequester) {
screenCaptureRequester = requester
}
fun attachPermissionRequester(requester: ai.openclaw.android.PermissionRequester) {
permissionRequester = requester
}
suspend fun record(paramsJson: String?): Payload =
withContext(Dispatchers.Default) {
val requester =
screenCaptureRequester
?: throw IllegalStateException(
"SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission",
)
val params = parseParamsObject(paramsJson)
val durationMs = (parseDurationMs(params) ?: 10_000).coerceIn(250, 60_000)
val fps = (parseFps(params) ?: 10.0).coerceIn(1.0, 60.0)
val fpsInt = fps.roundToInt().coerceIn(1, 60)
val screenIndex = parseScreenIndex(params)
val includeAudio = parseIncludeAudio(params) ?: true
val format = parseString(params, key = "format")
if (format != null && format.lowercase() != "mp4") {
throw IllegalArgumentException("INVALID_REQUEST: screen format must be mp4")
}
if (screenIndex != null && screenIndex != 0) {
throw IllegalArgumentException("INVALID_REQUEST: screenIndex must be 0 on Android")
}
val capture = requester.requestCapture()
?: throw IllegalStateException(
"SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission",
)
val mgr =
context.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
val projection = mgr.getMediaProjection(capture.resultCode, capture.data)
?: throw IllegalStateException("UNAVAILABLE: screen capture unavailable")
val metrics = context.resources.displayMetrics
val width = metrics.widthPixels
val height = metrics.heightPixels
val densityDpi = metrics.densityDpi
val file = File.createTempFile("openclaw-screen-", ".mp4")
if (includeAudio) ensureMicPermission()
val recorder = createMediaRecorder()
var virtualDisplay: android.hardware.display.VirtualDisplay? = null
try {
if (includeAudio) {
recorder.setAudioSource(MediaRecorder.AudioSource.MIC)
}
recorder.setVideoSource(MediaRecorder.VideoSource.SURFACE)
recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
recorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264)
if (includeAudio) {
recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
recorder.setAudioChannels(1)
recorder.setAudioSamplingRate(44_100)
recorder.setAudioEncodingBitRate(96_000)
}
recorder.setVideoSize(width, height)
recorder.setVideoFrameRate(fpsInt)
recorder.setVideoEncodingBitRate(estimateBitrate(width, height, fpsInt))
recorder.setOutputFile(file.absolutePath)
recorder.prepare()
val surface = recorder.surface
virtualDisplay =
projection.createVirtualDisplay(
"openclaw-screen",
width,
height,
densityDpi,
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
surface,
null,
null,
)
recorder.start()
delay(durationMs.toLong())
} finally {
try {
recorder.stop()
} catch (_: Throwable) {
// ignore
}
recorder.reset()
recorder.release()
virtualDisplay?.release()
projection.stop()
}
val bytes = withContext(Dispatchers.IO) { file.readBytes() }
file.delete()
val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP)
Payload(
"""{"format":"mp4","base64":"$base64","durationMs":$durationMs,"fps":$fpsInt,"screenIndex":0,"hasAudio":$includeAudio}""",
)
}
private fun createMediaRecorder(): MediaRecorder = MediaRecorder(context)
private suspend fun ensureMicPermission() {
val granted =
androidx.core.content.ContextCompat.checkSelfPermission(
context,
android.Manifest.permission.RECORD_AUDIO,
) == android.content.pm.PackageManager.PERMISSION_GRANTED
if (granted) return
val requester =
permissionRequester
?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
val results = requester.requestIfMissing(listOf(android.Manifest.permission.RECORD_AUDIO))
if (results[android.Manifest.permission.RECORD_AUDIO] != true) {
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
}
}
private fun parseParamsObject(paramsJson: String?): JsonObject? {
if (paramsJson.isNullOrBlank()) return null
return try {
Json.parseToJsonElement(paramsJson).asObjectOrNull()
} catch (_: Throwable) {
null
}
}
private fun readPrimitive(params: JsonObject?, key: String): JsonPrimitive? =
params?.get(key) as? JsonPrimitive
private fun parseDurationMs(params: JsonObject?): Int? =
readPrimitive(params, "durationMs")?.contentOrNull?.toIntOrNull()
private fun parseFps(params: JsonObject?): Double? =
readPrimitive(params, "fps")?.contentOrNull?.toDoubleOrNull()
private fun parseScreenIndex(params: JsonObject?): Int? =
readPrimitive(params, "screenIndex")?.contentOrNull?.toIntOrNull()
private fun parseIncludeAudio(params: JsonObject?): Boolean? {
val value = readPrimitive(params, "includeAudio")?.contentOrNull?.trim()?.lowercase()
return when (value) {
"true" -> true
"false" -> false
else -> null
}
}
private fun parseString(params: JsonObject?, key: String): String? =
readPrimitive(params, key)?.contentOrNull
private fun estimateBitrate(width: Int, height: Int, fps: Int): Int {
val pixels = width.toLong() * height.toLong()
val raw = (pixels * fps.toLong() * 2L).toInt()
return raw.coerceIn(1_000_000, 12_000_000)
}
}

View File

@@ -0,0 +1,19 @@
package ai.openclaw.android.node
import ai.openclaw.android.gateway.GatewaySession
class SmsHandler(
private val sms: SmsManager,
) {
suspend fun handleSmsSend(paramsJson: String?): GatewaySession.InvokeResult {
val res = sms.send(paramsJson)
if (res.ok) {
return GatewaySession.InvokeResult.ok(res.payloadJson)
} else {
val error = res.error ?: "SMS_SEND_FAILED"
val idx = error.indexOf(':')
val code = if (idx > 0) error.substring(0, idx).trim() else "SMS_SEND_FAILED"
return GatewaySession.InvokeResult.error(code = code, message = error)
}
}
}

View File

@@ -0,0 +1,230 @@
package ai.openclaw.android.node
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.telephony.SmsManager as AndroidSmsManager
import androidx.core.content.ContextCompat
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.encodeToString
import ai.openclaw.android.PermissionRequester
/**
* Sends SMS messages via the Android SMS API.
* Requires SEND_SMS permission to be granted.
*/
class SmsManager(private val context: Context) {
private val json = JsonConfig
@Volatile private var permissionRequester: PermissionRequester? = null
data class SendResult(
val ok: Boolean,
val to: String,
val message: String?,
val error: String? = null,
val payloadJson: String,
)
internal data class ParsedParams(
val to: String,
val message: String,
)
internal sealed class ParseResult {
data class Ok(val params: ParsedParams) : ParseResult()
data class Error(
val error: String,
val to: String = "",
val message: String? = null,
) : ParseResult()
}
internal data class SendPlan(
val parts: List<String>,
val useMultipart: Boolean,
)
companion object {
internal val JsonConfig = Json { ignoreUnknownKeys = true }
internal fun parseParams(paramsJson: String?, json: Json = JsonConfig): ParseResult {
val params = paramsJson?.trim().orEmpty()
if (params.isEmpty()) {
return ParseResult.Error(error = "INVALID_REQUEST: paramsJSON required")
}
val obj = try {
json.parseToJsonElement(params).jsonObject
} catch (_: Throwable) {
null
}
if (obj == null) {
return ParseResult.Error(error = "INVALID_REQUEST: expected JSON object")
}
val to = (obj["to"] as? JsonPrimitive)?.content?.trim().orEmpty()
val message = (obj["message"] as? JsonPrimitive)?.content.orEmpty()
if (to.isEmpty()) {
return ParseResult.Error(
error = "INVALID_REQUEST: 'to' phone number required",
message = message,
)
}
if (message.isEmpty()) {
return ParseResult.Error(
error = "INVALID_REQUEST: 'message' text required",
to = to,
)
}
return ParseResult.Ok(ParsedParams(to = to, message = message))
}
internal fun buildSendPlan(
message: String,
divider: (String) -> List<String>,
): SendPlan {
val parts = divider(message).ifEmpty { listOf(message) }
return SendPlan(parts = parts, useMultipart = parts.size > 1)
}
internal fun buildPayloadJson(
json: Json = JsonConfig,
ok: Boolean,
to: String,
error: String?,
): String {
val payload =
mutableMapOf<String, JsonElement>(
"ok" to JsonPrimitive(ok),
"to" to JsonPrimitive(to),
)
if (!ok) {
payload["error"] = JsonPrimitive(error ?: "SMS_SEND_FAILED")
}
return json.encodeToString(JsonObject.serializer(), JsonObject(payload))
}
}
fun hasSmsPermission(): Boolean {
return ContextCompat.checkSelfPermission(
context,
Manifest.permission.SEND_SMS
) == PackageManager.PERMISSION_GRANTED
}
fun canSendSms(): Boolean {
return hasSmsPermission() && hasTelephonyFeature()
}
fun hasTelephonyFeature(): Boolean {
return context.packageManager?.hasSystemFeature(PackageManager.FEATURE_TELEPHONY) == true
}
fun attachPermissionRequester(requester: PermissionRequester) {
permissionRequester = requester
}
/**
* Send an SMS message.
*
* @param paramsJson JSON with "to" (phone number) and "message" (text) fields
* @return SendResult indicating success or failure
*/
suspend fun send(paramsJson: String?): SendResult {
if (!hasTelephonyFeature()) {
return errorResult(
error = "SMS_UNAVAILABLE: telephony not available",
)
}
if (!ensureSmsPermission()) {
return errorResult(
error = "SMS_PERMISSION_REQUIRED: grant SMS permission",
)
}
val parseResult = parseParams(paramsJson, json)
if (parseResult is ParseResult.Error) {
return errorResult(
error = parseResult.error,
to = parseResult.to,
message = parseResult.message,
)
}
val params = (parseResult as ParseResult.Ok).params
return try {
val smsManager = context.getSystemService(AndroidSmsManager::class.java)
?: throw IllegalStateException("SMS_UNAVAILABLE: SmsManager not available")
val plan = buildSendPlan(params.message) { smsManager.divideMessage(it) }
if (plan.useMultipart) {
smsManager.sendMultipartTextMessage(
params.to, // destination
null, // service center (null = default)
ArrayList(plan.parts), // message parts
null, // sent intents
null, // delivery intents
)
} else {
smsManager.sendTextMessage(
params.to, // destination
null, // service center (null = default)
params.message,// message
null, // sent intent
null, // delivery intent
)
}
okResult(to = params.to, message = params.message)
} catch (e: SecurityException) {
errorResult(
error = "SMS_PERMISSION_REQUIRED: ${e.message}",
to = params.to,
message = params.message,
)
} catch (e: Throwable) {
errorResult(
error = "SMS_SEND_FAILED: ${e.message ?: "unknown error"}",
to = params.to,
message = params.message,
)
}
}
private suspend fun ensureSmsPermission(): Boolean {
if (hasSmsPermission()) return true
val requester = permissionRequester ?: return false
val results = requester.requestIfMissing(listOf(Manifest.permission.SEND_SMS))
return results[Manifest.permission.SEND_SMS] == true
}
private fun okResult(to: String, message: String): SendResult {
return SendResult(
ok = true,
to = to,
message = message,
error = null,
payloadJson = buildPayloadJson(json = json, ok = true, to = to, error = null),
)
}
private fun errorResult(error: String, to: String = "", message: String? = null): SendResult {
return SendResult(
ok = false,
to = to,
message = message,
error = error,
payloadJson = buildPayloadJson(json = json, ok = false, to = to, error = error),
)
}
}

View File

@@ -0,0 +1,66 @@
package ai.openclaw.android.protocol
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
object OpenClawCanvasA2UIAction {
fun extractActionName(userAction: JsonObject): String? {
val name =
(userAction["name"] as? JsonPrimitive)
?.content
?.trim()
.orEmpty()
if (name.isNotEmpty()) return name
val action =
(userAction["action"] as? JsonPrimitive)
?.content
?.trim()
.orEmpty()
return action.ifEmpty { null }
}
fun sanitizeTagValue(value: String): String {
val trimmed = value.trim().ifEmpty { "-" }
val normalized = trimmed.replace(" ", "_")
val out = StringBuilder(normalized.length)
for (c in normalized) {
val ok =
c.isLetterOrDigit() ||
c == '_' ||
c == '-' ||
c == '.' ||
c == ':'
out.append(if (ok) c else '_')
}
return out.toString()
}
fun formatAgentMessage(
actionName: String,
sessionKey: String,
surfaceId: String,
sourceComponentId: String,
host: String,
instanceId: String,
contextJson: String?,
): String {
val ctxSuffix = contextJson?.takeIf { it.isNotBlank() }?.let { " ctx=$it" }.orEmpty()
return listOf(
"CANVAS_A2UI",
"action=${sanitizeTagValue(actionName)}",
"session=${sanitizeTagValue(sessionKey)}",
"surface=${sanitizeTagValue(surfaceId)}",
"component=${sanitizeTagValue(sourceComponentId)}",
"host=${sanitizeTagValue(host)}",
"instance=${sanitizeTagValue(instanceId)}$ctxSuffix",
"default=update_canvas",
).joinToString(separator = " ")
}
fun jsDispatchA2UIActionStatus(actionId: String, ok: Boolean, error: String?): String {
val err = (error ?: "").replace("\\", "\\\\").replace("\"", "\\\"")
val okLiteral = if (ok) "true" else "false"
val idEscaped = actionId.replace("\\", "\\\\").replace("\"", "\\\"")
return "window.dispatchEvent(new CustomEvent('openclaw:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));"
}
}

View File

@@ -0,0 +1,95 @@
package ai.openclaw.android.protocol
enum class OpenClawCapability(val rawValue: String) {
Canvas("canvas"),
Camera("camera"),
Screen("screen"),
Sms("sms"),
VoiceWake("voiceWake"),
Location("location"),
Device("device"),
}
enum class OpenClawCanvasCommand(val rawValue: String) {
Present("canvas.present"),
Hide("canvas.hide"),
Navigate("canvas.navigate"),
Eval("canvas.eval"),
Snapshot("canvas.snapshot"),
;
companion object {
const val NamespacePrefix: String = "canvas."
}
}
enum class OpenClawCanvasA2UICommand(val rawValue: String) {
Push("canvas.a2ui.push"),
PushJSONL("canvas.a2ui.pushJSONL"),
Reset("canvas.a2ui.reset"),
;
companion object {
const val NamespacePrefix: String = "canvas.a2ui."
}
}
enum class OpenClawCameraCommand(val rawValue: String) {
List("camera.list"),
Snap("camera.snap"),
Clip("camera.clip"),
;
companion object {
const val NamespacePrefix: String = "camera."
}
}
enum class OpenClawScreenCommand(val rawValue: String) {
Record("screen.record"),
;
companion object {
const val NamespacePrefix: String = "screen."
}
}
enum class OpenClawSmsCommand(val rawValue: String) {
Send("sms.send"),
;
companion object {
const val NamespacePrefix: String = "sms."
}
}
enum class OpenClawLocationCommand(val rawValue: String) {
Get("location.get"),
;
companion object {
const val NamespacePrefix: String = "location."
}
}
enum class OpenClawDeviceCommand(val rawValue: String) {
Status("device.status"),
Info("device.info"),
Permissions("device.permissions"),
Health("device.health"),
;
companion object {
const val NamespacePrefix: String = "device."
}
}
enum class OpenClawNotificationsCommand(val rawValue: String) {
List("notifications.list"),
Actions("notifications.actions"),
;
companion object {
const val NamespacePrefix: String = "notifications."
}
}

View File

@@ -0,0 +1,222 @@
package ai.openclaw.android.tools
import android.content.Context
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.contentOrNull
@Serializable
private data class ToolDisplayActionSpec(
val label: String? = null,
val detailKeys: List<String>? = null,
)
@Serializable
private data class ToolDisplaySpec(
val emoji: String? = null,
val title: String? = null,
val label: String? = null,
val detailKeys: List<String>? = null,
val actions: Map<String, ToolDisplayActionSpec>? = null,
)
@Serializable
private data class ToolDisplayConfig(
val version: Int? = null,
val fallback: ToolDisplaySpec? = null,
val tools: Map<String, ToolDisplaySpec>? = null,
)
data class ToolDisplaySummary(
val name: String,
val emoji: String,
val title: String,
val label: String,
val verb: String?,
val detail: String?,
) {
val detailLine: String?
get() {
val parts = mutableListOf<String>()
if (!verb.isNullOrBlank()) parts.add(verb)
if (!detail.isNullOrBlank()) parts.add(detail)
return if (parts.isEmpty()) null else parts.joinToString(" · ")
}
val summaryLine: String
get() = if (detailLine != null) "${emoji} ${label}: ${detailLine}" else "${emoji} ${label}"
}
object ToolDisplayRegistry {
private const val CONFIG_ASSET = "tool-display.json"
private val json = Json { ignoreUnknownKeys = true }
@Volatile private var cachedConfig: ToolDisplayConfig? = null
fun resolve(
context: Context,
name: String?,
args: JsonObject?,
meta: String? = null,
): ToolDisplaySummary {
val trimmedName = name?.trim().orEmpty().ifEmpty { "tool" }
val key = trimmedName.lowercase()
val config = loadConfig(context)
val spec = config.tools?.get(key)
val fallback = config.fallback
val emoji = spec?.emoji ?: fallback?.emoji ?: "🧩"
val title = spec?.title ?: titleFromName(trimmedName)
val label = spec?.label ?: trimmedName
val actionRaw = args?.get("action")?.asStringOrNull()?.trim()
val action = actionRaw?.takeIf { it.isNotEmpty() }
val actionSpec = action?.let { spec?.actions?.get(it) }
val verb = normalizeVerb(actionSpec?.label ?: action)
var detail: String? = null
if (key == "read") {
detail = readDetail(args)
} else if (key == "write" || key == "edit" || key == "attach") {
detail = pathDetail(args)
}
val detailKeys = actionSpec?.detailKeys ?: spec?.detailKeys ?: fallback?.detailKeys ?: emptyList()
if (detail == null) {
detail = firstValue(args, detailKeys)
}
if (detail == null) {
detail = meta
}
if (detail != null) {
detail = shortenHomeInString(detail)
}
return ToolDisplaySummary(
name = trimmedName,
emoji = emoji,
title = title,
label = label,
verb = verb,
detail = detail,
)
}
private fun loadConfig(context: Context): ToolDisplayConfig {
val existing = cachedConfig
if (existing != null) return existing
return try {
val jsonString = context.assets.open(CONFIG_ASSET).bufferedReader().use { it.readText() }
val decoded = json.decodeFromString(ToolDisplayConfig.serializer(), jsonString)
cachedConfig = decoded
decoded
} catch (_: Throwable) {
val fallback = ToolDisplayConfig()
cachedConfig = fallback
fallback
}
}
private fun titleFromName(name: String): String {
val cleaned = name.replace("_", " ").trim()
if (cleaned.isEmpty()) return "Tool"
return cleaned
.split(Regex("\\s+"))
.joinToString(" ") { part ->
val upper = part.uppercase()
if (part.length <= 2 && part == upper) part
else upper.firstOrNull()?.toString().orEmpty() + part.lowercase().drop(1)
}
}
private fun normalizeVerb(value: String?): String? {
val trimmed = value?.trim().orEmpty()
if (trimmed.isEmpty()) return null
return trimmed.replace("_", " ")
}
private fun readDetail(args: JsonObject?): String? {
val path = args?.get("path")?.asStringOrNull() ?: return null
val offset = args["offset"].asNumberOrNull()
val limit = args["limit"].asNumberOrNull()
return if (offset != null && limit != null) {
val end = offset + limit
"${path}:${offset.toInt()}-${end.toInt()}"
} else {
path
}
}
private fun pathDetail(args: JsonObject?): String? {
return args?.get("path")?.asStringOrNull()
}
private fun firstValue(args: JsonObject?, keys: List<String>): String? {
for (key in keys) {
val value = valueForPath(args, key)
val rendered = renderValue(value)
if (!rendered.isNullOrBlank()) return rendered
}
return null
}
private fun valueForPath(args: JsonObject?, path: String): JsonElement? {
var current: JsonElement? = args
for (segment in path.split(".")) {
if (segment.isBlank()) return null
val obj = current as? JsonObject ?: return null
current = obj[segment]
}
return current
}
private fun renderValue(value: JsonElement?): String? {
if (value == null) return null
if (value is JsonPrimitive) {
if (value.isString) {
val trimmed = value.contentOrNull?.trim().orEmpty()
if (trimmed.isEmpty()) return null
val firstLine = trimmed.lineSequence().firstOrNull()?.trim().orEmpty()
if (firstLine.isEmpty()) return null
return if (firstLine.length > 160) "${firstLine.take(157)}" else firstLine
}
val raw = value.contentOrNull?.trim().orEmpty()
raw.toBooleanStrictOrNull()?.let { return it.toString() }
raw.toLongOrNull()?.let { return it.toString() }
raw.toDoubleOrNull()?.let { return it.toString() }
}
if (value is JsonArray) {
val items = value.mapNotNull { renderValue(it) }
if (items.isEmpty()) return null
val preview = items.take(3).joinToString(", ")
return if (items.size > 3) "${preview}" else preview
}
return null
}
private fun shortenHomeInString(value: String): String {
val home = System.getProperty("user.home")?.takeIf { it.isNotBlank() }
?: System.getenv("HOME")?.takeIf { it.isNotBlank() }
if (home.isNullOrEmpty()) return value
return value.replace(home, "~")
.replace(Regex("/Users/[^/]+"), "~")
.replace(Regex("/home/[^/]+"), "~")
}
private fun JsonElement?.asStringOrNull(): String? {
val primitive = this as? JsonPrimitive ?: return null
return if (primitive.isString) primitive.contentOrNull else primitive.toString()
}
private fun JsonElement?.asNumberOrNull(): Double? {
val primitive = this as? JsonPrimitive ?: return null
val raw = primitive.contentOrNull ?: return null
return raw.toDoubleOrNull()
}
}

View File

@@ -0,0 +1,44 @@
package ai.openclaw.android.ui
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableFloatStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Color
import kotlinx.coroutines.delay
@Composable
fun CameraFlashOverlay(
token: Long,
modifier: Modifier = Modifier,
) {
Box(modifier = modifier.fillMaxSize()) {
CameraFlash(token = token)
}
}
@Composable
private fun CameraFlash(token: Long) {
var alpha by remember { mutableFloatStateOf(0f) }
LaunchedEffect(token) {
if (token == 0L) return@LaunchedEffect
alpha = 0.85f
delay(110)
alpha = 0f
}
Box(
modifier =
Modifier
.fillMaxSize()
.alpha(alpha)
.background(Color.White),
)
}

View File

@@ -0,0 +1,150 @@
package ai.openclaw.android.ui
import android.annotation.SuppressLint
import android.util.Log
import android.view.View
import android.webkit.ConsoleMessage
import android.webkit.JavascriptInterface
import android.webkit.WebChromeClient
import android.webkit.WebResourceError
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebSettings
import android.webkit.WebView
import android.webkit.WebViewClient
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.viewinterop.AndroidView
import androidx.webkit.WebSettingsCompat
import androidx.webkit.WebViewFeature
import ai.openclaw.android.MainViewModel
@SuppressLint("SetJavaScriptEnabled")
@Composable
fun CanvasScreen(viewModel: MainViewModel, modifier: Modifier = Modifier) {
val context = LocalContext.current
val isDebuggable = (context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0
val webViewRef = remember { mutableStateOf<WebView?>(null) }
DisposableEffect(viewModel) {
onDispose {
val webView = webViewRef.value ?: return@onDispose
viewModel.canvas.detach(webView)
webView.removeJavascriptInterface(CanvasA2UIActionBridge.interfaceName)
webView.stopLoading()
webView.destroy()
webViewRef.value = null
}
}
AndroidView(
modifier = modifier,
factory = {
WebView(context).apply {
settings.javaScriptEnabled = true
settings.domStorageEnabled = true
settings.mixedContentMode = WebSettings.MIXED_CONTENT_COMPATIBILITY_MODE
settings.useWideViewPort = false
settings.loadWithOverviewMode = false
settings.builtInZoomControls = false
settings.displayZoomControls = false
settings.setSupportZoom(false)
if (WebViewFeature.isFeatureSupported(WebViewFeature.ALGORITHMIC_DARKENING)) {
WebSettingsCompat.setAlgorithmicDarkeningAllowed(settings, false)
} else {
disableForceDarkIfSupported(settings)
}
if (isDebuggable) {
Log.d("OpenClawWebView", "userAgent: ${settings.userAgentString}")
}
isScrollContainer = true
overScrollMode = View.OVER_SCROLL_IF_CONTENT_SCROLLS
isVerticalScrollBarEnabled = true
isHorizontalScrollBarEnabled = true
webViewClient =
object : WebViewClient() {
override fun onReceivedError(
view: WebView,
request: WebResourceRequest,
error: WebResourceError,
) {
if (!isDebuggable || !request.isForMainFrame) return
Log.e("OpenClawWebView", "onReceivedError: ${error.errorCode} ${error.description} ${request.url}")
}
override fun onReceivedHttpError(
view: WebView,
request: WebResourceRequest,
errorResponse: WebResourceResponse,
) {
if (!isDebuggable || !request.isForMainFrame) return
Log.e(
"OpenClawWebView",
"onReceivedHttpError: ${errorResponse.statusCode} ${errorResponse.reasonPhrase} ${request.url}",
)
}
override fun onPageFinished(view: WebView, url: String?) {
if (isDebuggable) {
Log.d("OpenClawWebView", "onPageFinished: $url")
}
viewModel.canvas.onPageFinished()
}
override fun onRenderProcessGone(
view: WebView,
detail: android.webkit.RenderProcessGoneDetail,
): Boolean {
if (isDebuggable) {
Log.e(
"OpenClawWebView",
"onRenderProcessGone didCrash=${detail.didCrash()} priorityAtExit=${detail.rendererPriorityAtExit()}",
)
}
return true
}
}
webChromeClient =
object : WebChromeClient() {
override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean {
if (!isDebuggable) return false
val msg = consoleMessage ?: return false
Log.d(
"OpenClawWebView",
"console ${msg.messageLevel()} @ ${msg.sourceId()}:${msg.lineNumber()} ${msg.message()}",
)
return false
}
}
val bridge = CanvasA2UIActionBridge { payload -> viewModel.handleCanvasA2UIActionFromWebView(payload) }
addJavascriptInterface(bridge, CanvasA2UIActionBridge.interfaceName)
viewModel.canvas.attach(this)
webViewRef.value = this
}
},
)
}
private fun disableForceDarkIfSupported(settings: WebSettings) {
if (!WebViewFeature.isFeatureSupported(WebViewFeature.FORCE_DARK)) return
@Suppress("DEPRECATION")
WebSettingsCompat.setForceDark(settings, WebSettingsCompat.FORCE_DARK_OFF)
}
private class CanvasA2UIActionBridge(private val onMessage: (String) -> Unit) {
@JavascriptInterface
fun postMessage(payload: String?) {
val msg = payload?.trim().orEmpty()
if (msg.isEmpty()) return
onMessage(msg)
}
companion object {
const val interfaceName: String = "openclawCanvasA2UIAction"
}
}

View File

@@ -0,0 +1,10 @@
package ai.openclaw.android.ui
import androidx.compose.runtime.Composable
import ai.openclaw.android.MainViewModel
import ai.openclaw.android.ui.chat.ChatSheetContent
@Composable
fun ChatSheet(viewModel: MainViewModel) {
ChatSheetContent(viewModel = viewModel)
}

View File

@@ -0,0 +1,493 @@
package ai.openclaw.android.ui
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.foundation.text.KeyboardOptions
import androidx.compose.foundation.verticalScroll
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.ExpandLess
import androidx.compose.material.icons.filled.ExpandMore
import androidx.compose.material3.AlertDialog
import androidx.compose.material3.Button
import androidx.compose.material3.ButtonDefaults
import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.Icon
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.OutlinedTextFieldDefaults
import androidx.compose.material3.Surface
import androidx.compose.material3.Switch
import androidx.compose.material3.SwitchDefaults
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.input.KeyboardType
import androidx.compose.ui.unit.dp
import ai.openclaw.android.MainViewModel
private enum class ConnectInputMode {
SetupCode,
Manual,
}
@Composable
fun ConnectTabScreen(viewModel: MainViewModel) {
val statusText by viewModel.statusText.collectAsState()
val isConnected by viewModel.isConnected.collectAsState()
val remoteAddress by viewModel.remoteAddress.collectAsState()
val manualHost by viewModel.manualHost.collectAsState()
val manualPort by viewModel.manualPort.collectAsState()
val manualTls by viewModel.manualTls.collectAsState()
val manualEnabled by viewModel.manualEnabled.collectAsState()
val gatewayToken by viewModel.gatewayToken.collectAsState()
val pendingTrust by viewModel.pendingGatewayTrust.collectAsState()
var advancedOpen by rememberSaveable { mutableStateOf(false) }
var inputMode by
remember(manualEnabled, manualHost, gatewayToken) {
mutableStateOf(
if (manualEnabled || manualHost.isNotBlank() || gatewayToken.trim().isNotEmpty()) {
ConnectInputMode.Manual
} else {
ConnectInputMode.SetupCode
},
)
}
var setupCode by rememberSaveable { mutableStateOf("") }
var manualHostInput by rememberSaveable { mutableStateOf(manualHost.ifBlank { "10.0.2.2" }) }
var manualPortInput by rememberSaveable { mutableStateOf(manualPort.toString()) }
var manualTlsInput by rememberSaveable { mutableStateOf(manualTls) }
var passwordInput by rememberSaveable { mutableStateOf("") }
var validationText by rememberSaveable { mutableStateOf<String?>(null) }
if (pendingTrust != null) {
val prompt = pendingTrust!!
AlertDialog(
onDismissRequest = { viewModel.declineGatewayTrustPrompt() },
title = { Text("Trust this gateway?") },
text = {
Text(
"First-time TLS connection.\n\nVerify this SHA-256 fingerprint before trusting:\n${prompt.fingerprintSha256}",
style = mobileCallout,
)
},
confirmButton = {
TextButton(onClick = { viewModel.acceptGatewayTrustPrompt() }) {
Text("Trust and continue")
}
},
dismissButton = {
TextButton(onClick = { viewModel.declineGatewayTrustPrompt() }) {
Text("Cancel")
}
},
)
}
val setupResolvedEndpoint = remember(setupCode) { decodeGatewaySetupCode(setupCode)?.url?.let { parseGatewayEndpoint(it)?.displayUrl } }
val manualResolvedEndpoint = remember(manualHostInput, manualPortInput, manualTlsInput) {
composeGatewayManualUrl(manualHostInput, manualPortInput, manualTlsInput)?.let { parseGatewayEndpoint(it)?.displayUrl }
}
val activeEndpoint =
remember(isConnected, remoteAddress, setupResolvedEndpoint, manualResolvedEndpoint, inputMode) {
when {
isConnected && !remoteAddress.isNullOrBlank() -> remoteAddress!!
inputMode == ConnectInputMode.SetupCode -> setupResolvedEndpoint ?: "Not set"
else -> manualResolvedEndpoint ?: "Not set"
}
}
val primaryLabel = if (isConnected) "Disconnect Gateway" else "Connect Gateway"
Column(
modifier = Modifier.verticalScroll(rememberScrollState()).padding(horizontal = 20.dp, vertical = 16.dp),
verticalArrangement = Arrangement.spacedBy(14.dp),
) {
Column(verticalArrangement = Arrangement.spacedBy(6.dp)) {
Text("Connection Control", style = mobileCaption1.copy(fontWeight = FontWeight.Bold), color = mobileAccent)
Text("Gateway Connection", style = mobileTitle1, color = mobileText)
Text(
"One primary action. Open advanced controls only when needed.",
style = mobileCallout,
color = mobileTextSecondary,
)
}
Surface(
modifier = Modifier.fillMaxWidth(),
shape = RoundedCornerShape(14.dp),
color = mobileSurface,
border = BorderStroke(1.dp, mobileBorder),
) {
Column(modifier = Modifier.padding(horizontal = 14.dp, vertical = 12.dp), verticalArrangement = Arrangement.spacedBy(4.dp)) {
Text("Active endpoint", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
Text(activeEndpoint, style = mobileBody.copy(fontFamily = FontFamily.Monospace), color = mobileText)
}
}
Surface(
modifier = Modifier.fillMaxWidth(),
shape = RoundedCornerShape(14.dp),
color = mobileSurface,
border = BorderStroke(1.dp, mobileBorder),
) {
Column(modifier = Modifier.padding(horizontal = 14.dp, vertical = 12.dp), verticalArrangement = Arrangement.spacedBy(4.dp)) {
Text("Gateway state", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
Text(statusText, style = mobileBody, color = mobileText)
}
}
Button(
onClick = {
if (isConnected) {
viewModel.disconnect()
validationText = null
return@Button
}
if (statusText.contains("operator offline", ignoreCase = true)) {
validationText = null
viewModel.refreshGatewayConnection()
return@Button
}
val config =
resolveGatewayConnectConfig(
useSetupCode = inputMode == ConnectInputMode.SetupCode,
setupCode = setupCode,
manualHost = manualHostInput,
manualPort = manualPortInput,
manualTls = manualTlsInput,
fallbackToken = gatewayToken,
fallbackPassword = passwordInput,
)
if (config == null) {
validationText =
if (inputMode == ConnectInputMode.SetupCode) {
"Paste a valid setup code to connect."
} else {
"Enter a valid manual host and port to connect."
}
return@Button
}
validationText = null
viewModel.setManualEnabled(true)
viewModel.setManualHost(config.host)
viewModel.setManualPort(config.port)
viewModel.setManualTls(config.tls)
if (config.token.isNotBlank()) {
viewModel.setGatewayToken(config.token)
}
viewModel.setGatewayPassword(config.password)
viewModel.connectManual()
},
modifier = Modifier.fillMaxWidth().height(52.dp),
shape = RoundedCornerShape(14.dp),
colors =
ButtonDefaults.buttonColors(
containerColor = if (isConnected) mobileDanger else mobileAccent,
contentColor = Color.White,
),
) {
Text(primaryLabel, style = mobileHeadline.copy(fontWeight = FontWeight.Bold))
}
Surface(
modifier = Modifier.fillMaxWidth(),
shape = RoundedCornerShape(14.dp),
color = mobileSurface,
border = BorderStroke(1.dp, mobileBorder),
onClick = { advancedOpen = !advancedOpen },
) {
Row(
modifier = Modifier.fillMaxWidth().padding(horizontal = 14.dp, vertical = 12.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.SpaceBetween,
) {
Column(verticalArrangement = Arrangement.spacedBy(2.dp)) {
Text("Advanced controls", style = mobileHeadline, color = mobileText)
Text("Setup code, endpoint, TLS, token, password, onboarding.", style = mobileCaption1, color = mobileTextSecondary)
}
Icon(
imageVector = if (advancedOpen) Icons.Default.ExpandLess else Icons.Default.ExpandMore,
contentDescription = if (advancedOpen) "Collapse advanced controls" else "Expand advanced controls",
tint = mobileTextSecondary,
)
}
}
AnimatedVisibility(visible = advancedOpen) {
Surface(
modifier = Modifier.fillMaxWidth(),
shape = RoundedCornerShape(14.dp),
color = Color.White,
border = BorderStroke(1.dp, mobileBorder),
) {
Column(
modifier = Modifier.fillMaxWidth().padding(horizontal = 14.dp, vertical = 14.dp),
verticalArrangement = Arrangement.spacedBy(12.dp),
) {
Text("Connection method", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
MethodChip(
label = "Setup Code",
active = inputMode == ConnectInputMode.SetupCode,
onClick = { inputMode = ConnectInputMode.SetupCode },
)
MethodChip(
label = "Manual",
active = inputMode == ConnectInputMode.Manual,
onClick = { inputMode = ConnectInputMode.Manual },
)
}
Text("Run these on the gateway host:", style = mobileCallout, color = mobileTextSecondary)
CommandBlock("openclaw qr --setup-code-only")
CommandBlock("openclaw qr --json")
if (inputMode == ConnectInputMode.SetupCode) {
Text("Setup Code", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
OutlinedTextField(
value = setupCode,
onValueChange = {
setupCode = it
validationText = null
},
placeholder = { Text("Paste setup code", style = mobileBody, color = mobileTextTertiary) },
modifier = Modifier.fillMaxWidth(),
minLines = 3,
maxLines = 5,
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Ascii),
textStyle = mobileBody.copy(fontFamily = FontFamily.Monospace, color = mobileText),
shape = RoundedCornerShape(14.dp),
colors = outlinedColors(),
)
if (!setupResolvedEndpoint.isNullOrBlank()) {
EndpointPreview(endpoint = setupResolvedEndpoint)
}
} else {
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
QuickFillChip(
label = "Android Emulator",
onClick = {
manualHostInput = "10.0.2.2"
manualPortInput = "18789"
manualTlsInput = false
validationText = null
},
)
QuickFillChip(
label = "Localhost",
onClick = {
manualHostInput = "127.0.0.1"
manualPortInput = "18789"
manualTlsInput = false
validationText = null
},
)
}
Text("Host", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
OutlinedTextField(
value = manualHostInput,
onValueChange = {
manualHostInput = it
validationText = null
},
placeholder = { Text("10.0.2.2", style = mobileBody, color = mobileTextTertiary) },
modifier = Modifier.fillMaxWidth(),
singleLine = true,
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Uri),
textStyle = mobileBody.copy(color = mobileText),
shape = RoundedCornerShape(14.dp),
colors = outlinedColors(),
)
Text("Port", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
OutlinedTextField(
value = manualPortInput,
onValueChange = {
manualPortInput = it
validationText = null
},
placeholder = { Text("18789", style = mobileBody, color = mobileTextTertiary) },
modifier = Modifier.fillMaxWidth(),
singleLine = true,
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number),
textStyle = mobileBody.copy(fontFamily = FontFamily.Monospace, color = mobileText),
shape = RoundedCornerShape(14.dp),
colors = outlinedColors(),
)
Row(
modifier = Modifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.SpaceBetween,
) {
Column(verticalArrangement = Arrangement.spacedBy(2.dp)) {
Text("Use TLS", style = mobileHeadline, color = mobileText)
Text("Switch to secure websocket (`wss`).", style = mobileCallout, color = mobileTextSecondary)
}
Switch(
checked = manualTlsInput,
onCheckedChange = {
manualTlsInput = it
validationText = null
},
colors =
SwitchDefaults.colors(
checkedTrackColor = mobileAccent,
uncheckedTrackColor = mobileBorderStrong,
checkedThumbColor = Color.White,
uncheckedThumbColor = Color.White,
),
)
}
Text("Token (optional)", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
OutlinedTextField(
value = gatewayToken,
onValueChange = { viewModel.setGatewayToken(it) },
placeholder = { Text("token", style = mobileBody, color = mobileTextTertiary) },
modifier = Modifier.fillMaxWidth(),
singleLine = true,
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Ascii),
textStyle = mobileBody.copy(color = mobileText),
shape = RoundedCornerShape(14.dp),
colors = outlinedColors(),
)
Text("Password (optional)", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
OutlinedTextField(
value = passwordInput,
onValueChange = { passwordInput = it },
placeholder = { Text("password", style = mobileBody, color = mobileTextTertiary) },
modifier = Modifier.fillMaxWidth(),
singleLine = true,
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Ascii),
textStyle = mobileBody.copy(color = mobileText),
shape = RoundedCornerShape(14.dp),
colors = outlinedColors(),
)
if (!manualResolvedEndpoint.isNullOrBlank()) {
EndpointPreview(endpoint = manualResolvedEndpoint)
}
}
HorizontalDivider(color = mobileBorder)
TextButton(onClick = { viewModel.setOnboardingCompleted(false) }) {
Text("Run onboarding again", style = mobileCallout.copy(fontWeight = FontWeight.SemiBold), color = mobileAccent)
}
}
}
}
if (!validationText.isNullOrBlank()) {
Text(validationText!!, style = mobileCaption1, color = mobileWarning)
}
}
}
@Composable
private fun MethodChip(label: String, active: Boolean, onClick: () -> Unit) {
Button(
onClick = onClick,
modifier = Modifier.height(40.dp),
shape = RoundedCornerShape(12.dp),
contentPadding = PaddingValues(horizontal = 12.dp, vertical = 8.dp),
colors =
ButtonDefaults.buttonColors(
containerColor = if (active) mobileAccent else mobileSurface,
contentColor = if (active) Color.White else mobileText,
),
border = BorderStroke(1.dp, if (active) Color(0xFF184DAF) else mobileBorderStrong),
) {
Text(label, style = mobileCaption1.copy(fontWeight = FontWeight.Bold))
}
}
@Composable
private fun QuickFillChip(label: String, onClick: () -> Unit) {
Button(
onClick = onClick,
shape = RoundedCornerShape(999.dp),
contentPadding = PaddingValues(horizontal = 12.dp, vertical = 6.dp),
colors =
ButtonDefaults.buttonColors(
containerColor = mobileAccentSoft,
contentColor = mobileAccent,
),
elevation = null,
) {
Text(label, style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold))
}
}
@Composable
private fun CommandBlock(command: String) {
Surface(
modifier = Modifier.fillMaxWidth(),
shape = RoundedCornerShape(12.dp),
color = mobileCodeBg,
border = BorderStroke(1.dp, Color(0xFF2B2E35)),
) {
Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) {
Box(modifier = Modifier.width(3.dp).height(42.dp).background(Color(0xFF3FC97A)))
Text(
text = command,
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
style = mobileCallout.copy(fontFamily = FontFamily.Monospace),
color = mobileCodeText,
)
}
}
}
@Composable
private fun EndpointPreview(endpoint: String) {
Column(verticalArrangement = Arrangement.spacedBy(4.dp)) {
HorizontalDivider(color = mobileBorder)
Text("Resolved endpoint", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
Text(endpoint, style = mobileCallout.copy(fontFamily = FontFamily.Monospace), color = mobileText)
HorizontalDivider(color = mobileBorder)
}
}
@Composable
private fun outlinedColors() =
OutlinedTextFieldDefaults.colors(
focusedContainerColor = mobileSurface,
unfocusedContainerColor = mobileSurface,
focusedBorderColor = mobileAccent,
unfocusedBorderColor = mobileBorder,
focusedTextColor = mobileText,
unfocusedTextColor = mobileText,
cursorColor = mobileAccent,
)

View File

@@ -0,0 +1,142 @@
package ai.openclaw.android.ui
import androidx.core.net.toUri
import java.util.Base64
import java.util.Locale
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonObject
internal data class GatewayEndpointConfig(
val host: String,
val port: Int,
val tls: Boolean,
val displayUrl: String,
)
internal data class GatewaySetupCode(
val url: String,
val token: String?,
val password: String?,
)
internal data class GatewayConnectConfig(
val host: String,
val port: Int,
val tls: Boolean,
val token: String,
val password: String,
)
private val gatewaySetupJson = Json { ignoreUnknownKeys = true }
internal fun resolveGatewayConnectConfig(
useSetupCode: Boolean,
setupCode: String,
manualHost: String,
manualPort: String,
manualTls: Boolean,
fallbackToken: String,
fallbackPassword: String,
): GatewayConnectConfig? {
if (useSetupCode) {
val setup = decodeGatewaySetupCode(setupCode) ?: return null
val parsed = parseGatewayEndpoint(setup.url) ?: return null
return GatewayConnectConfig(
host = parsed.host,
port = parsed.port,
tls = parsed.tls,
token = setup.token ?: fallbackToken.trim(),
password = setup.password ?: fallbackPassword.trim(),
)
}
val manualUrl = composeGatewayManualUrl(manualHost, manualPort, manualTls) ?: return null
val parsed = parseGatewayEndpoint(manualUrl) ?: return null
return GatewayConnectConfig(
host = parsed.host,
port = parsed.port,
tls = parsed.tls,
token = fallbackToken.trim(),
password = fallbackPassword.trim(),
)
}
internal fun parseGatewayEndpoint(rawInput: String): GatewayEndpointConfig? {
val raw = rawInput.trim()
if (raw.isEmpty()) return null
val normalized = if (raw.contains("://")) raw else "https://$raw"
val uri = normalized.toUri()
val host = uri.host?.trim().orEmpty()
if (host.isEmpty()) return null
val scheme = uri.scheme?.trim()?.lowercase(Locale.US).orEmpty()
val tls =
when (scheme) {
"ws", "http" -> false
"wss", "https" -> true
else -> true
}
val port = uri.port.takeIf { it in 1..65535 } ?: 18789
val displayUrl = "${if (tls) "https" else "http"}://$host:$port"
return GatewayEndpointConfig(host = host, port = port, tls = tls, displayUrl = displayUrl)
}
internal fun decodeGatewaySetupCode(rawInput: String): GatewaySetupCode? {
val trimmed = rawInput.trim()
if (trimmed.isEmpty()) return null
val padded =
trimmed
.replace('-', '+')
.replace('_', '/')
.let { normalized ->
val remainder = normalized.length % 4
if (remainder == 0) normalized else normalized + "=".repeat(4 - remainder)
}
return try {
val decoded = String(Base64.getDecoder().decode(padded), Charsets.UTF_8)
val obj = parseJsonObject(decoded) ?: return null
val url = jsonField(obj, "url").orEmpty()
if (url.isEmpty()) return null
val token = jsonField(obj, "token")
val password = jsonField(obj, "password")
GatewaySetupCode(url = url, token = token, password = password)
} catch (_: IllegalArgumentException) {
null
}
}
internal fun resolveScannedSetupCode(rawInput: String): String? {
val setupCode = resolveSetupCodeCandidate(rawInput) ?: return null
return setupCode.takeIf { decodeGatewaySetupCode(it) != null }
}
internal fun composeGatewayManualUrl(hostInput: String, portInput: String, tls: Boolean): String? {
val host = hostInput.trim()
val port = portInput.trim().toIntOrNull() ?: return null
if (host.isEmpty() || port !in 1..65535) return null
val scheme = if (tls) "https" else "http"
return "$scheme://$host:$port"
}
private fun parseJsonObject(input: String): JsonObject? {
return runCatching { gatewaySetupJson.parseToJsonElement(input).jsonObject }.getOrNull()
}
private fun resolveSetupCodeCandidate(rawInput: String): String? {
val trimmed = rawInput.trim()
if (trimmed.isEmpty()) return null
val qrSetupCode = parseJsonObject(trimmed)?.let { jsonField(it, "setupCode") }
return qrSetupCode ?: trimmed
}
private fun jsonField(obj: JsonObject, key: String): String? {
val value = (obj[key] as? JsonPrimitive)?.contentOrNull?.trim().orEmpty()
return value.ifEmpty { null }
}

View File

@@ -0,0 +1,106 @@
package ai.openclaw.android.ui
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.TextStyle
import androidx.compose.ui.text.font.Font
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.sp
import ai.openclaw.android.R
internal val mobileBackgroundGradient =
Brush.verticalGradient(
listOf(
Color(0xFFFFFFFF),
Color(0xFFF7F8FA),
Color(0xFFEFF1F5),
),
)
internal val mobileSurface = Color(0xFFF6F7FA)
internal val mobileSurfaceStrong = Color(0xFFECEEF3)
internal val mobileBorder = Color(0xFFE5E7EC)
internal val mobileBorderStrong = Color(0xFFD6DAE2)
internal val mobileText = Color(0xFF17181C)
internal val mobileTextSecondary = Color(0xFF5D6472)
internal val mobileTextTertiary = Color(0xFF99A0AE)
internal val mobileAccent = Color(0xFF1D5DD8)
internal val mobileAccentSoft = Color(0xFFECF3FF)
internal val mobileSuccess = Color(0xFF2F8C5A)
internal val mobileSuccessSoft = Color(0xFFEEF9F3)
internal val mobileWarning = Color(0xFFC8841A)
internal val mobileWarningSoft = Color(0xFFFFF8EC)
internal val mobileDanger = Color(0xFFD04B4B)
internal val mobileDangerSoft = Color(0xFFFFF2F2)
internal val mobileCodeBg = Color(0xFF15171B)
internal val mobileCodeText = Color(0xFFE8EAEE)
internal val mobileFontFamily =
FontFamily(
Font(resId = R.font.manrope_400_regular, weight = FontWeight.Normal),
Font(resId = R.font.manrope_500_medium, weight = FontWeight.Medium),
Font(resId = R.font.manrope_600_semibold, weight = FontWeight.SemiBold),
Font(resId = R.font.manrope_700_bold, weight = FontWeight.Bold),
)
internal val mobileTitle1 =
TextStyle(
fontFamily = mobileFontFamily,
fontWeight = FontWeight.SemiBold,
fontSize = 24.sp,
lineHeight = 30.sp,
letterSpacing = (-0.5).sp,
)
internal val mobileTitle2 =
TextStyle(
fontFamily = mobileFontFamily,
fontWeight = FontWeight.SemiBold,
fontSize = 20.sp,
lineHeight = 26.sp,
letterSpacing = (-0.3).sp,
)
internal val mobileHeadline =
TextStyle(
fontFamily = mobileFontFamily,
fontWeight = FontWeight.SemiBold,
fontSize = 16.sp,
lineHeight = 22.sp,
letterSpacing = (-0.1).sp,
)
internal val mobileBody =
TextStyle(
fontFamily = mobileFontFamily,
fontWeight = FontWeight.Medium,
fontSize = 15.sp,
lineHeight = 22.sp,
)
internal val mobileCallout =
TextStyle(
fontFamily = mobileFontFamily,
fontWeight = FontWeight.Medium,
fontSize = 14.sp,
lineHeight = 20.sp,
)
internal val mobileCaption1 =
TextStyle(
fontFamily = mobileFontFamily,
fontWeight = FontWeight.Medium,
fontSize = 12.sp,
lineHeight = 16.sp,
letterSpacing = 0.2.sp,
)
internal val mobileCaption2 =
TextStyle(
fontFamily = mobileFontFamily,
fontWeight = FontWeight.Medium,
fontSize = 11.sp,
lineHeight = 14.sp,
letterSpacing = 0.4.sp,
)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,32 @@
package ai.openclaw.android.ui
import androidx.compose.foundation.isSystemInDarkTheme
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.dynamicDarkColorScheme
import androidx.compose.material3.dynamicLightColorScheme
import androidx.compose.runtime.Composable
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.platform.LocalContext
@Composable
fun OpenClawTheme(content: @Composable () -> Unit) {
val context = LocalContext.current
val isDark = isSystemInDarkTheme()
val colorScheme = if (isDark) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)
MaterialTheme(colorScheme = colorScheme, content = content)
}
@Composable
fun overlayContainerColor(): Color {
val scheme = MaterialTheme.colorScheme
val isDark = isSystemInDarkTheme()
val base = if (isDark) scheme.surfaceContainerLow else scheme.surfaceContainerHigh
// Light mode: background stays dark (canvas), so clamp overlays away from pure-white glare.
return if (isDark) base else base.copy(alpha = 0.88f)
}
@Composable
fun overlayIconColor(): Color {
return MaterialTheme.colorScheme.onSurfaceVariant
}

View File

@@ -0,0 +1,320 @@
package ai.openclaw.android.ui
import androidx.compose.foundation.background
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.WindowInsets
import androidx.compose.foundation.layout.WindowInsetsSides
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.layout.ime
import androidx.compose.foundation.layout.navigationBars
import androidx.compose.foundation.layout.only
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.safeDrawing
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.foundation.layout.consumeWindowInsets
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.automirrored.filled.ScreenShare
import androidx.compose.material.icons.filled.ChatBubble
import androidx.compose.material.icons.filled.CheckCircle
import androidx.compose.material.icons.filled.RecordVoiceOver
import androidx.compose.material.icons.filled.Settings
import androidx.compose.material3.Icon
import androidx.compose.material3.Scaffold
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.vector.ImageVector
import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
import ai.openclaw.android.MainViewModel
private enum class HomeTab(
val label: String,
val icon: ImageVector,
) {
Connect(label = "Connect", icon = Icons.Default.CheckCircle),
Chat(label = "Chat", icon = Icons.Default.ChatBubble),
Voice(label = "Voice", icon = Icons.Default.RecordVoiceOver),
Screen(label = "Screen", icon = Icons.AutoMirrored.Filled.ScreenShare),
Settings(label = "Settings", icon = Icons.Default.Settings),
}
private enum class StatusVisual {
Connected,
Connecting,
Warning,
Error,
Offline,
}
@Composable
fun PostOnboardingTabs(viewModel: MainViewModel, modifier: Modifier = Modifier) {
var activeTab by rememberSaveable { mutableStateOf(HomeTab.Connect) }
val statusText by viewModel.statusText.collectAsState()
val isConnected by viewModel.isConnected.collectAsState()
val statusVisual =
remember(statusText, isConnected) {
val lower = statusText.lowercase()
when {
isConnected -> StatusVisual.Connected
lower.contains("connecting") || lower.contains("reconnecting") -> StatusVisual.Connecting
lower.contains("pairing") || lower.contains("approval") || lower.contains("auth") -> StatusVisual.Warning
lower.contains("error") || lower.contains("failed") -> StatusVisual.Error
else -> StatusVisual.Offline
}
}
val density = LocalDensity.current
val imeVisible = WindowInsets.ime.getBottom(density) > 0
val hideBottomTabBar = activeTab == HomeTab.Chat && imeVisible
Scaffold(
modifier = modifier,
containerColor = Color.Transparent,
contentWindowInsets = WindowInsets(0, 0, 0, 0),
topBar = {
TopStatusBar(
statusText = statusText,
statusVisual = statusVisual,
)
},
bottomBar = {
if (!hideBottomTabBar) {
BottomTabBar(
activeTab = activeTab,
onSelect = { activeTab = it },
)
}
},
) { innerPadding ->
Box(
modifier =
Modifier
.fillMaxSize()
.padding(innerPadding)
.consumeWindowInsets(innerPadding)
.background(mobileBackgroundGradient),
) {
when (activeTab) {
HomeTab.Connect -> ConnectTabScreen(viewModel = viewModel)
HomeTab.Chat -> ChatSheet(viewModel = viewModel)
HomeTab.Voice -> VoiceTabScreen(viewModel = viewModel)
HomeTab.Screen -> ScreenTabScreen(viewModel = viewModel)
HomeTab.Settings -> SettingsSheet(viewModel = viewModel)
}
}
}
}
@Composable
private fun ScreenTabScreen(viewModel: MainViewModel) {
val isConnected by viewModel.isConnected.collectAsState()
val isNodeConnected by viewModel.isNodeConnected.collectAsState()
val canvasUrl by viewModel.canvasCurrentUrl.collectAsState()
val canvasA2uiHydrated by viewModel.canvasA2uiHydrated.collectAsState()
val canvasRehydratePending by viewModel.canvasRehydratePending.collectAsState()
val canvasRehydrateErrorText by viewModel.canvasRehydrateErrorText.collectAsState()
val isA2uiUrl = canvasUrl?.contains("/__openclaw__/a2ui/") == true
val showRestoreCta = isConnected && isNodeConnected && (canvasUrl.isNullOrBlank() || (isA2uiUrl && !canvasA2uiHydrated))
val restoreCtaText =
when {
canvasRehydratePending -> "Restore requested. Waiting for agent…"
!canvasRehydrateErrorText.isNullOrBlank() -> canvasRehydrateErrorText!!
else -> "Canvas reset. Tap to restore dashboard."
}
Box(modifier = Modifier.fillMaxSize()) {
CanvasScreen(viewModel = viewModel, modifier = Modifier.fillMaxSize())
if (showRestoreCta) {
Surface(
onClick = {
if (canvasRehydratePending) return@Surface
viewModel.requestCanvasRehydrate(source = "screen_tab_cta")
},
modifier = Modifier.align(Alignment.TopCenter).padding(horizontal = 16.dp, vertical = 16.dp),
shape = RoundedCornerShape(12.dp),
color = mobileSurface.copy(alpha = 0.9f),
border = BorderStroke(1.dp, mobileBorder),
shadowElevation = 4.dp,
) {
Text(
text = restoreCtaText,
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
style = mobileCallout.copy(fontWeight = FontWeight.Medium),
color = mobileText,
)
}
}
}
}
@Composable
private fun TopStatusBar(
statusText: String,
statusVisual: StatusVisual,
) {
val safeInsets = WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal)
val (chipBg, chipDot, chipText, chipBorder) =
when (statusVisual) {
StatusVisual.Connected ->
listOf(
mobileSuccessSoft,
mobileSuccess,
mobileSuccess,
Color(0xFFCFEBD8),
)
StatusVisual.Connecting ->
listOf(
mobileAccentSoft,
mobileAccent,
mobileAccent,
Color(0xFFD5E2FA),
)
StatusVisual.Warning ->
listOf(
mobileWarningSoft,
mobileWarning,
mobileWarning,
Color(0xFFEED8B8),
)
StatusVisual.Error ->
listOf(
mobileDangerSoft,
mobileDanger,
mobileDanger,
Color(0xFFF3C8C8),
)
StatusVisual.Offline ->
listOf(
mobileSurface,
mobileTextTertiary,
mobileTextSecondary,
mobileBorder,
)
}
Surface(
modifier = Modifier.fillMaxWidth().windowInsetsPadding(safeInsets),
color = Color.Transparent,
shadowElevation = 0.dp,
) {
Row(
modifier = Modifier.fillMaxWidth().padding(horizontal = 18.dp, vertical = 12.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.SpaceBetween,
) {
Text(
text = "OpenClaw",
style = mobileTitle2,
color = mobileText,
)
Surface(
shape = RoundedCornerShape(999.dp),
color = chipBg,
border = androidx.compose.foundation.BorderStroke(1.dp, chipBorder),
) {
Row(
modifier = Modifier.padding(horizontal = 10.dp, vertical = 5.dp),
horizontalArrangement = Arrangement.spacedBy(6.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Surface(
modifier = Modifier.padding(top = 1.dp),
color = chipDot,
shape = RoundedCornerShape(999.dp),
) {
Box(modifier = Modifier.padding(4.dp))
}
Text(
text = statusText.trim().ifEmpty { "Offline" },
style = mobileCaption1,
color = chipText,
maxLines = 1,
)
}
}
}
}
}
@Composable
private fun BottomTabBar(
activeTab: HomeTab,
onSelect: (HomeTab) -> Unit,
) {
val safeInsets = WindowInsets.navigationBars.only(WindowInsetsSides.Bottom + WindowInsetsSides.Horizontal)
Box(
modifier =
Modifier
.fillMaxWidth(),
) {
Surface(
modifier = Modifier.fillMaxWidth(),
color = Color.White.copy(alpha = 0.97f),
shape = RoundedCornerShape(topStart = 24.dp, topEnd = 24.dp),
border = BorderStroke(1.dp, mobileBorder),
shadowElevation = 6.dp,
) {
Row(
modifier =
Modifier
.fillMaxWidth()
.windowInsetsPadding(safeInsets)
.padding(horizontal = 10.dp, vertical = 10.dp),
horizontalArrangement = Arrangement.spacedBy(6.dp),
verticalAlignment = Alignment.CenterVertically,
) {
HomeTab.entries.forEach { tab ->
val active = tab == activeTab
Surface(
onClick = { onSelect(tab) },
modifier = Modifier.weight(1f).heightIn(min = 58.dp),
shape = RoundedCornerShape(16.dp),
color = if (active) mobileAccentSoft else Color.Transparent,
border = if (active) BorderStroke(1.dp, Color(0xFFD5E2FA)) else null,
shadowElevation = 0.dp,
) {
Column(
modifier = Modifier.fillMaxWidth().padding(horizontal = 6.dp, vertical = 7.dp),
horizontalAlignment = Alignment.CenterHorizontally,
verticalArrangement = Arrangement.spacedBy(2.dp),
) {
Icon(
imageVector = tab.icon,
contentDescription = tab.label,
tint = if (active) mobileAccent else mobileTextTertiary,
)
Text(
text = tab.label,
color = if (active) mobileAccent else mobileTextSecondary,
style = mobileCaption2.copy(fontWeight = if (active) FontWeight.Bold else FontWeight.Medium),
)
}
}
}
}
}
}
}

View File

@@ -0,0 +1,20 @@
package ai.openclaw.android.ui
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.ui.Modifier
import ai.openclaw.android.MainViewModel
@Composable
fun RootScreen(viewModel: MainViewModel) {
val onboardingCompleted by viewModel.onboardingCompleted.collectAsState()
if (!onboardingCompleted) {
OnboardingFlow(viewModel = viewModel, modifier = Modifier.fillMaxSize())
return
}
PostOnboardingTabs(viewModel = viewModel, modifier = Modifier.fillMaxSize())
}

View File

@@ -0,0 +1,605 @@
package ai.openclaw.android.ui
import android.Manifest
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.net.Uri
import android.os.Build
import android.provider.Settings
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.foundation.background
import androidx.compose.foundation.border
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.WindowInsets
import androidx.compose.foundation.layout.WindowInsetsSides
import androidx.compose.foundation.layout.fillMaxHeight
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.imePadding
import androidx.compose.foundation.layout.only
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.safeDrawing
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.foundation.lazy.rememberLazyListState
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material3.Button
import androidx.compose.material3.ButtonDefaults
import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.ListItem
import androidx.compose.material3.ListItemDefaults
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.OutlinedTextFieldDefaults
import androidx.compose.material3.RadioButton
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.sp
import androidx.compose.ui.unit.dp
import androidx.core.content.ContextCompat
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleEventObserver
import androidx.lifecycle.compose.LocalLifecycleOwner
import ai.openclaw.android.BuildConfig
import ai.openclaw.android.LocationMode
import ai.openclaw.android.MainViewModel
@Composable
fun SettingsSheet(viewModel: MainViewModel) {
val context = LocalContext.current
val lifecycleOwner = LocalLifecycleOwner.current
val instanceId by viewModel.instanceId.collectAsState()
val displayName by viewModel.displayName.collectAsState()
val cameraEnabled by viewModel.cameraEnabled.collectAsState()
val locationMode by viewModel.locationMode.collectAsState()
val locationPreciseEnabled by viewModel.locationPreciseEnabled.collectAsState()
val preventSleep by viewModel.preventSleep.collectAsState()
val canvasDebugStatusEnabled by viewModel.canvasDebugStatusEnabled.collectAsState()
val listState = rememberLazyListState()
val deviceModel =
remember {
listOfNotNull(Build.MANUFACTURER, Build.MODEL)
.joinToString(" ")
.trim()
.ifEmpty { "Android" }
}
val appVersion =
remember {
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
"$versionName-dev"
} else {
versionName
}
}
val listItemColors =
ListItemDefaults.colors(
containerColor = Color.Transparent,
headlineColor = mobileText,
supportingColor = mobileTextSecondary,
trailingIconColor = mobileTextSecondary,
leadingIconColor = mobileTextSecondary,
)
val permissionLauncher =
rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms ->
val cameraOk = perms[Manifest.permission.CAMERA] == true
viewModel.setCameraEnabled(cameraOk)
}
var pendingLocationMode by remember { mutableStateOf<LocationMode?>(null) }
var pendingPreciseToggle by remember { mutableStateOf(false) }
val locationPermissionLauncher =
rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms ->
val fineOk = perms[Manifest.permission.ACCESS_FINE_LOCATION] == true
val coarseOk = perms[Manifest.permission.ACCESS_COARSE_LOCATION] == true
val granted = fineOk || coarseOk
val requestedMode = pendingLocationMode
pendingLocationMode = null
if (pendingPreciseToggle) {
pendingPreciseToggle = false
viewModel.setLocationPreciseEnabled(fineOk)
return@rememberLauncherForActivityResult
}
if (!granted) {
viewModel.setLocationMode(LocationMode.Off)
return@rememberLauncherForActivityResult
}
if (requestedMode != null) {
viewModel.setLocationMode(requestedMode)
if (requestedMode == LocationMode.Always) {
val backgroundOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ==
PackageManager.PERMISSION_GRANTED
if (!backgroundOk) {
openAppSettings(context)
}
}
}
}
var micPermissionGranted by
remember {
mutableStateOf(
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED,
)
}
val audioPermissionLauncher =
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
micPermissionGranted = granted
}
val smsPermissionAvailable =
remember {
context.packageManager?.hasSystemFeature(PackageManager.FEATURE_TELEPHONY) == true
}
var smsPermissionGranted by
remember {
mutableStateOf(
ContextCompat.checkSelfPermission(context, Manifest.permission.SEND_SMS) ==
PackageManager.PERMISSION_GRANTED,
)
}
val smsPermissionLauncher =
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
smsPermissionGranted = granted
viewModel.refreshGatewayConnection()
}
DisposableEffect(lifecycleOwner, context) {
val observer =
LifecycleEventObserver { _, event ->
if (event == Lifecycle.Event.ON_RESUME) {
micPermissionGranted =
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
smsPermissionGranted =
ContextCompat.checkSelfPermission(context, Manifest.permission.SEND_SMS) ==
PackageManager.PERMISSION_GRANTED
}
}
lifecycleOwner.lifecycle.addObserver(observer)
onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }
}
fun setCameraEnabledChecked(checked: Boolean) {
if (!checked) {
viewModel.setCameraEnabled(false)
return
}
val cameraOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) ==
PackageManager.PERMISSION_GRANTED
if (cameraOk) {
viewModel.setCameraEnabled(true)
} else {
permissionLauncher.launch(arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO))
}
}
fun requestLocationPermissions(targetMode: LocationMode) {
val fineOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
val coarseOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
if (fineOk || coarseOk) {
viewModel.setLocationMode(targetMode)
if (targetMode == LocationMode.Always) {
val backgroundOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ==
PackageManager.PERMISSION_GRANTED
if (!backgroundOk) {
openAppSettings(context)
}
}
} else {
pendingLocationMode = targetMode
locationPermissionLauncher.launch(
arrayOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION),
)
}
}
fun setPreciseLocationChecked(checked: Boolean) {
if (!checked) {
viewModel.setLocationPreciseEnabled(false)
return
}
val fineOk =
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) ==
PackageManager.PERMISSION_GRANTED
if (fineOk) {
viewModel.setLocationPreciseEnabled(true)
} else {
pendingPreciseToggle = true
locationPermissionLauncher.launch(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION))
}
}
Box(
modifier =
Modifier
.fillMaxSize()
.background(mobileBackgroundGradient),
) {
LazyColumn(
state = listState,
modifier =
Modifier
.fillMaxWidth()
.fillMaxHeight()
.imePadding()
.windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Bottom)),
contentPadding = PaddingValues(horizontal = 20.dp, vertical = 16.dp),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
item {
Column(verticalArrangement = Arrangement.spacedBy(6.dp)) {
Text(
"SETTINGS",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
color = mobileAccent,
)
Text("Device Configuration", style = mobileTitle2, color = mobileText)
Text(
"Manage capabilities, permissions, and diagnostics.",
style = mobileCallout,
color = mobileTextSecondary,
)
}
}
item { HorizontalDivider(color = mobileBorder) }
// Order parity: Node → Voice → Camera → Messaging → Location → Screen.
item {
Text(
"NODE",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
color = mobileAccent,
)
}
item {
OutlinedTextField(
value = displayName,
onValueChange = viewModel::setDisplayName,
label = { Text("Name", style = mobileCaption1, color = mobileTextSecondary) },
modifier = Modifier.fillMaxWidth(),
textStyle = mobileBody.copy(color = mobileText),
colors = settingsTextFieldColors(),
)
}
item { Text("Instance ID: $instanceId", style = mobileCallout.copy(fontFamily = FontFamily.Monospace), color = mobileTextSecondary) }
item { Text("Device: $deviceModel", style = mobileCallout, color = mobileTextSecondary) }
item { Text("Version: $appVersion", style = mobileCallout, color = mobileTextSecondary) }
item { HorizontalDivider(color = mobileBorder) }
// Voice
item {
Text(
"VOICE",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
color = mobileAccent,
)
}
item {
ListItem(
modifier = settingsRowModifier(),
colors = listItemColors,
headlineContent = { Text("Microphone permission", style = mobileHeadline) },
supportingContent = {
Text(
if (micPermissionGranted) {
"Granted. Use the Voice tab mic button to capture transcript."
} else {
"Required for Voice tab transcription."
},
style = mobileCallout,
)
},
trailingContent = {
Button(
onClick = {
if (micPermissionGranted) {
openAppSettings(context)
} else {
audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
}
},
colors = settingsPrimaryButtonColors(),
shape = RoundedCornerShape(14.dp),
) {
Text(
if (micPermissionGranted) "Manage" else "Grant",
style = mobileCallout.copy(fontWeight = FontWeight.Bold),
)
}
},
)
}
item {
Text(
"Voice wake and talk modes were removed. Voice now uses one mic on/off flow in the Voice tab.",
style = mobileCallout,
color = mobileTextSecondary,
)
}
item { HorizontalDivider(color = mobileBorder) }
// Camera
item {
Text(
"CAMERA",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
color = mobileAccent,
)
}
item {
ListItem(
modifier = settingsRowModifier(),
colors = listItemColors,
headlineContent = { Text("Allow Camera", style = mobileHeadline) },
supportingContent = { Text("Allows the gateway to request photos or short video clips (foreground only).", style = mobileCallout) },
trailingContent = { Switch(checked = cameraEnabled, onCheckedChange = ::setCameraEnabledChecked) },
)
}
item {
Text(
"Tip: grant Microphone permission for video clips with audio.",
style = mobileCallout,
color = mobileTextSecondary,
)
}
item { HorizontalDivider(color = mobileBorder) }
// Messaging
item {
Text(
"MESSAGING",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
color = mobileAccent,
)
}
item {
val buttonLabel =
when {
!smsPermissionAvailable -> "Unavailable"
smsPermissionGranted -> "Manage"
else -> "Grant"
}
ListItem(
modifier = settingsRowModifier(),
colors = listItemColors,
headlineContent = { Text("SMS Permission", style = mobileHeadline) },
supportingContent = {
Text(
if (smsPermissionAvailable) {
"Allow the gateway to send SMS from this device."
} else {
"SMS requires a device with telephony hardware."
},
style = mobileCallout,
)
},
trailingContent = {
Button(
onClick = {
if (!smsPermissionAvailable) return@Button
if (smsPermissionGranted) {
openAppSettings(context)
} else {
smsPermissionLauncher.launch(Manifest.permission.SEND_SMS)
}
},
enabled = smsPermissionAvailable,
colors = settingsPrimaryButtonColors(),
shape = RoundedCornerShape(14.dp),
) {
Text(buttonLabel, style = mobileCallout.copy(fontWeight = FontWeight.Bold))
}
},
)
}
item { HorizontalDivider(color = mobileBorder) }
// Location
item {
Text(
"LOCATION",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
color = mobileAccent,
)
}
item {
Column(modifier = settingsRowModifier(), verticalArrangement = Arrangement.spacedBy(0.dp)) {
ListItem(
modifier = Modifier.fillMaxWidth(),
colors = listItemColors,
headlineContent = { Text("Off", style = mobileHeadline) },
supportingContent = { Text("Disable location sharing.", style = mobileCallout) },
trailingContent = {
RadioButton(
selected = locationMode == LocationMode.Off,
onClick = { viewModel.setLocationMode(LocationMode.Off) },
)
},
)
HorizontalDivider(color = mobileBorder)
ListItem(
modifier = Modifier.fillMaxWidth(),
colors = listItemColors,
headlineContent = { Text("While Using", style = mobileHeadline) },
supportingContent = { Text("Only while OpenClaw is open.", style = mobileCallout) },
trailingContent = {
RadioButton(
selected = locationMode == LocationMode.WhileUsing,
onClick = { requestLocationPermissions(LocationMode.WhileUsing) },
)
},
)
HorizontalDivider(color = mobileBorder)
ListItem(
modifier = Modifier.fillMaxWidth(),
colors = listItemColors,
headlineContent = { Text("Always", style = mobileHeadline) },
supportingContent = { Text("Allow background location (requires system permission).", style = mobileCallout) },
trailingContent = {
RadioButton(
selected = locationMode == LocationMode.Always,
onClick = { requestLocationPermissions(LocationMode.Always) },
)
},
)
HorizontalDivider(color = mobileBorder)
ListItem(
modifier = Modifier.fillMaxWidth(),
colors = listItemColors,
headlineContent = { Text("Precise Location", style = mobileHeadline) },
supportingContent = { Text("Use precise GPS when available.", style = mobileCallout) },
trailingContent = {
Switch(
checked = locationPreciseEnabled,
onCheckedChange = ::setPreciseLocationChecked,
enabled = locationMode != LocationMode.Off,
)
},
)
}
}
item {
Text(
"Always may require Android Settings to allow background location.",
style = mobileCallout,
color = mobileTextSecondary,
)
}
item { HorizontalDivider(color = mobileBorder) }
// Screen
item {
Text(
"SCREEN",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
color = mobileAccent,
)
}
item {
ListItem(
modifier = settingsRowModifier(),
colors = listItemColors,
headlineContent = { Text("Prevent Sleep", style = mobileHeadline) },
supportingContent = { Text("Keeps the screen awake while OpenClaw is open.", style = mobileCallout) },
trailingContent = { Switch(checked = preventSleep, onCheckedChange = viewModel::setPreventSleep) },
)
}
item { HorizontalDivider(color = mobileBorder) }
// Debug
item {
Text(
"DEBUG",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
color = mobileAccent,
)
}
item {
ListItem(
modifier = settingsRowModifier(),
colors = listItemColors,
headlineContent = { Text("Debug Canvas Status", style = mobileHeadline) },
supportingContent = { Text("Show status text in the canvas when debug is enabled.", style = mobileCallout) },
trailingContent = {
Switch(
checked = canvasDebugStatusEnabled,
onCheckedChange = viewModel::setCanvasDebugStatusEnabled,
)
},
)
}
item { Spacer(modifier = Modifier.height(24.dp)) }
}
}
}
@Composable
private fun settingsTextFieldColors() =
OutlinedTextFieldDefaults.colors(
focusedContainerColor = mobileSurface,
unfocusedContainerColor = mobileSurface,
focusedBorderColor = mobileAccent,
unfocusedBorderColor = mobileBorder,
focusedTextColor = mobileText,
unfocusedTextColor = mobileText,
cursorColor = mobileAccent,
)
private fun settingsRowModifier() =
Modifier
.fillMaxWidth()
.border(width = 1.dp, color = mobileBorder, shape = RoundedCornerShape(14.dp))
.background(Color.White, RoundedCornerShape(14.dp))
@Composable
private fun settingsPrimaryButtonColors() =
ButtonDefaults.buttonColors(
containerColor = mobileAccent,
contentColor = Color.White,
disabledContainerColor = mobileAccent.copy(alpha = 0.45f),
disabledContentColor = Color.White.copy(alpha = 0.9f),
)
@Composable
private fun settingsDangerButtonColors() =
ButtonDefaults.buttonColors(
containerColor = mobileDanger,
contentColor = Color.White,
disabledContainerColor = mobileDanger.copy(alpha = 0.45f),
disabledContentColor = Color.White.copy(alpha = 0.9f),
)
private fun openAppSettings(context: Context) {
val intent =
Intent(
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
Uri.fromParts("package", context.packageName, null),
)
context.startActivity(intent)
}

View File

@@ -0,0 +1,134 @@
package ai.openclaw.android.ui
import androidx.compose.animation.core.LinearEasing
import androidx.compose.animation.core.RepeatMode
import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
@Composable
fun TalkOrbOverlay(
seamColor: Color,
statusText: String,
isListening: Boolean,
isSpeaking: Boolean,
modifier: Modifier = Modifier,
) {
val transition = rememberInfiniteTransition(label = "talk-orb")
val t by
transition.animateFloat(
initialValue = 0f,
targetValue = 1f,
animationSpec =
infiniteRepeatable(
animation = tween(durationMillis = 1500, easing = LinearEasing),
repeatMode = RepeatMode.Restart,
),
label = "pulse",
)
val trimmed = statusText.trim()
val showStatus = trimmed.isNotEmpty() && trimmed != "Off"
val phase =
when {
isSpeaking -> "Speaking"
isListening -> "Listening"
else -> "Thinking"
}
Column(
modifier = modifier.padding(24.dp),
horizontalAlignment = Alignment.CenterHorizontally,
verticalArrangement = Arrangement.spacedBy(12.dp),
) {
Box(contentAlignment = Alignment.Center) {
Canvas(modifier = Modifier.size(360.dp)) {
val center = this.center
val baseRadius = size.minDimension * 0.30f
val ring1 = 1.05f + (t * 0.25f)
val ring2 = 1.20f + (t * 0.55f)
val ringAlpha1 = (1f - t) * 0.34f
val ringAlpha2 = (1f - t) * 0.22f
drawCircle(
color = seamColor.copy(alpha = ringAlpha1),
radius = baseRadius * ring1,
center = center,
style = Stroke(width = 3.dp.toPx()),
)
drawCircle(
color = seamColor.copy(alpha = ringAlpha2),
radius = baseRadius * ring2,
center = center,
style = Stroke(width = 3.dp.toPx()),
)
drawCircle(
brush =
Brush.radialGradient(
colors =
listOf(
seamColor.copy(alpha = 0.92f),
seamColor.copy(alpha = 0.40f),
Color.Black.copy(alpha = 0.56f),
),
center = center,
radius = baseRadius * 1.35f,
),
radius = baseRadius,
center = center,
)
drawCircle(
color = seamColor.copy(alpha = 0.34f),
radius = baseRadius,
center = center,
style = Stroke(width = 1.dp.toPx()),
)
}
}
if (showStatus) {
Surface(
color = Color.Black.copy(alpha = 0.40f),
shape = CircleShape,
) {
Text(
text = trimmed,
modifier = Modifier.padding(horizontal = 14.dp, vertical = 8.dp),
color = Color.White.copy(alpha = 0.92f),
style = MaterialTheme.typography.labelLarge,
fontWeight = FontWeight.SemiBold,
)
}
} else {
Text(
text = phase,
color = Color.White.copy(alpha = 0.80f),
style = MaterialTheme.typography.labelLarge,
fontWeight = FontWeight.SemiBold,
)
}
}
}

View File

@@ -0,0 +1,451 @@
package ai.openclaw.android.ui
import android.Manifest
import android.app.Activity
import android.content.Context
import android.content.ContextWrapper
import android.content.Intent
import android.content.pm.PackageManager
import android.net.Uri
import android.provider.Settings
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.animation.core.LinearEasing
import androidx.compose.animation.core.RepeatMode
import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.WindowInsets
import androidx.compose.foundation.layout.WindowInsetsSides
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.layout.imePadding
import androidx.compose.foundation.layout.only
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.safeDrawing
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.items
import androidx.compose.foundation.lazy.rememberLazyListState
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Mic
import androidx.compose.material.icons.filled.MicOff
import androidx.compose.material3.Button
import androidx.compose.material3.ButtonDefaults
import androidx.compose.material3.Icon
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.core.app.ActivityCompat
import androidx.core.content.ContextCompat
import androidx.lifecycle.Lifecycle
import androidx.lifecycle.LifecycleEventObserver
import androidx.lifecycle.compose.LocalLifecycleOwner
import ai.openclaw.android.MainViewModel
import ai.openclaw.android.voice.VoiceConversationEntry
import ai.openclaw.android.voice.VoiceConversationRole
import kotlin.math.PI
import kotlin.math.max
import kotlin.math.sin
@Composable
fun VoiceTabScreen(viewModel: MainViewModel) {
val context = LocalContext.current
val lifecycleOwner = LocalLifecycleOwner.current
val activity = remember(context) { context.findActivity() }
val listState = rememberLazyListState()
val isConnected by viewModel.isConnected.collectAsState()
val gatewayStatus by viewModel.statusText.collectAsState()
val micEnabled by viewModel.micEnabled.collectAsState()
val micStatusText by viewModel.micStatusText.collectAsState()
val micLiveTranscript by viewModel.micLiveTranscript.collectAsState()
val micQueuedMessages by viewModel.micQueuedMessages.collectAsState()
val micConversation by viewModel.micConversation.collectAsState()
val micInputLevel by viewModel.micInputLevel.collectAsState()
val micIsSending by viewModel.micIsSending.collectAsState()
val hasStreamingAssistant = micConversation.any { it.role == VoiceConversationRole.Assistant && it.isStreaming }
val showThinkingBubble = micIsSending && !hasStreamingAssistant
var hasMicPermission by remember { mutableStateOf(context.hasRecordAudioPermission()) }
var pendingMicEnable by remember { mutableStateOf(false) }
DisposableEffect(lifecycleOwner, context) {
val observer =
LifecycleEventObserver { _, event ->
if (event == Lifecycle.Event.ON_RESUME) {
hasMicPermission = context.hasRecordAudioPermission()
}
}
lifecycleOwner.lifecycle.addObserver(observer)
onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }
}
val requestMicPermission =
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
hasMicPermission = granted
if (granted && pendingMicEnable) {
viewModel.setMicEnabled(true)
}
pendingMicEnable = false
}
LaunchedEffect(micConversation.size, showThinkingBubble) {
val total = micConversation.size + if (showThinkingBubble) 1 else 0
if (total > 0) {
listState.animateScrollToItem(total - 1)
}
}
Column(
modifier =
Modifier
.fillMaxSize()
.background(mobileBackgroundGradient)
.imePadding()
.windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Bottom))
.padding(horizontal = 20.dp, vertical = 14.dp),
verticalArrangement = Arrangement.spacedBy(10.dp),
) {
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.SpaceBetween,
verticalAlignment = Alignment.CenterVertically,
) {
Column(verticalArrangement = Arrangement.spacedBy(4.dp)) {
Text(
"VOICE",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
color = mobileAccent,
)
Text("Voice mode", style = mobileTitle2, color = mobileText)
}
Surface(
shape = RoundedCornerShape(999.dp),
color = if (isConnected) mobileAccentSoft else mobileSurfaceStrong,
border = BorderStroke(1.dp, if (isConnected) mobileAccent.copy(alpha = 0.25f) else mobileBorderStrong),
) {
Text(
if (isConnected) "Connected" else "Offline",
modifier = Modifier.padding(horizontal = 12.dp, vertical = 6.dp),
style = mobileCaption1,
color = if (isConnected) mobileAccent else mobileTextSecondary,
)
}
}
LazyColumn(
state = listState,
modifier = Modifier.fillMaxWidth().weight(1f),
contentPadding = PaddingValues(vertical = 4.dp),
verticalArrangement = Arrangement.spacedBy(10.dp),
) {
if (micConversation.isEmpty() && !showThinkingBubble) {
item {
Column(
modifier = Modifier.fillMaxWidth().padding(top = 12.dp),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
Text(
"Tap the mic and speak. Each pause sends a turn automatically.",
style = mobileCallout,
color = mobileTextSecondary,
)
}
}
}
items(items = micConversation, key = { it.id }) { entry ->
VoiceTurnBubble(entry = entry)
}
if (showThinkingBubble) {
item {
VoiceThinkingBubble()
}
}
}
Surface(
modifier = Modifier.fillMaxWidth(),
shape = RoundedCornerShape(20.dp),
color = Color.White,
border = BorderStroke(1.dp, mobileBorder),
) {
Column(
modifier = Modifier.fillMaxWidth().padding(horizontal = 14.dp, vertical = 12.dp),
horizontalAlignment = Alignment.CenterHorizontally,
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
Surface(
shape = RoundedCornerShape(999.dp),
color = mobileSurface,
border = BorderStroke(1.dp, mobileBorder),
) {
val queueCount = micQueuedMessages.size
val stateText =
when {
queueCount > 0 -> "$queueCount queued"
micIsSending -> "Sending"
micEnabled -> "Listening"
else -> "Mic off"
}
Text(
"$gatewayStatus · $stateText",
modifier = Modifier.padding(horizontal = 12.dp, vertical = 7.dp),
style = mobileCaption1,
color = mobileTextSecondary,
)
}
if (!micLiveTranscript.isNullOrBlank()) {
Surface(
modifier = Modifier.fillMaxWidth(),
shape = RoundedCornerShape(14.dp),
color = mobileAccentSoft,
border = BorderStroke(1.dp, mobileAccent.copy(alpha = 0.2f)),
) {
Text(
micLiveTranscript!!.trim(),
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
style = mobileCallout,
color = mobileText,
)
}
}
MicWaveform(level = micInputLevel, active = micEnabled)
Button(
onClick = {
if (micEnabled) {
viewModel.setMicEnabled(false)
return@Button
}
if (hasMicPermission) {
viewModel.setMicEnabled(true)
} else {
pendingMicEnable = true
requestMicPermission.launch(Manifest.permission.RECORD_AUDIO)
}
},
shape = CircleShape,
contentPadding = PaddingValues(0.dp),
modifier = Modifier.size(86.dp),
colors =
ButtonDefaults.buttonColors(
containerColor = if (micEnabled) mobileDanger else mobileAccent,
contentColor = Color.White,
),
) {
Icon(
imageVector = if (micEnabled) Icons.Default.MicOff else Icons.Default.Mic,
contentDescription = if (micEnabled) "Turn microphone off" else "Turn microphone on",
modifier = Modifier.size(30.dp),
)
}
Text(
if (micEnabled) "Tap to stop" else "Tap to speak",
style = mobileCallout,
color = mobileTextSecondary,
)
if (!hasMicPermission) {
val showRationale =
if (activity == null) {
false
} else {
ActivityCompat.shouldShowRequestPermissionRationale(activity, Manifest.permission.RECORD_AUDIO)
}
Text(
if (showRationale) {
"Microphone permission is required for voice mode."
} else {
"Microphone blocked. Open app settings to enable it."
},
style = mobileCaption1,
color = mobileWarning,
textAlign = TextAlign.Center,
)
Button(
onClick = { openAppSettings(context) },
shape = RoundedCornerShape(12.dp),
colors = ButtonDefaults.buttonColors(containerColor = mobileSurfaceStrong, contentColor = mobileText),
) {
Text("Open settings", style = mobileCallout.copy(fontWeight = FontWeight.SemiBold))
}
}
Text(
micStatusText,
style = mobileCaption1,
color = mobileTextTertiary,
)
}
}
}
}
@Composable
private fun VoiceTurnBubble(entry: VoiceConversationEntry) {
val isUser = entry.role == VoiceConversationRole.User
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start,
) {
Surface(
modifier = Modifier.fillMaxWidth(0.90f),
shape = RoundedCornerShape(14.dp),
color = if (isUser) mobileAccentSoft else mobileSurface,
border = BorderStroke(1.dp, if (isUser) mobileAccent.copy(alpha = 0.2f) else mobileBorder),
) {
Column(
modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 10.dp),
verticalArrangement = Arrangement.spacedBy(6.dp),
) {
Text(
if (isUser) "You" else "OpenClaw",
style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold),
color = mobileTextSecondary,
)
Text(
if (entry.isStreaming && entry.text.isBlank()) "Listening response…" else entry.text,
style = mobileCallout,
color = mobileText,
)
}
}
}
}
@Composable
private fun VoiceThinkingBubble() {
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) {
Surface(
modifier = Modifier.fillMaxWidth(0.68f),
shape = RoundedCornerShape(14.dp),
color = mobileSurface,
border = BorderStroke(1.dp, mobileBorder),
) {
Row(
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
ThinkingDots(color = mobileTextSecondary)
Text("OpenClaw is thinking…", style = mobileCallout, color = mobileTextSecondary)
}
}
}
}
@Composable
private fun ThinkingDots(color: Color) {
Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) {
ThinkingDot(alpha = 0.38f, color = color)
ThinkingDot(alpha = 0.62f, color = color)
ThinkingDot(alpha = 0.90f, color = color)
}
}
@Composable
private fun ThinkingDot(alpha: Float, color: Color) {
Surface(
modifier = Modifier.size(6.dp).alpha(alpha),
shape = CircleShape,
color = color,
) {}
}
@Composable
private fun MicWaveform(level: Float, active: Boolean) {
val transition = rememberInfiniteTransition(label = "voiceWave")
val phase by
transition.animateFloat(
initialValue = 0f,
targetValue = 1f,
animationSpec = infiniteRepeatable(animation = tween(1_000, easing = LinearEasing), repeatMode = RepeatMode.Restart),
label = "voiceWavePhase",
)
val effective = if (active) level.coerceIn(0f, 1f) else 0f
val base = max(effective, if (active) 0.05f else 0f)
Row(
modifier = Modifier.fillMaxWidth().heightIn(min = 40.dp),
horizontalArrangement = Arrangement.spacedBy(4.dp, Alignment.CenterHorizontally),
verticalAlignment = Alignment.CenterVertically,
) {
repeat(16) { index ->
val pulse =
if (!active) {
0f
} else {
((sin(((phase * 2f * PI) + (index * 0.55f)).toDouble()) + 1.0) * 0.5).toFloat()
}
val barHeight = 6.dp + (24.dp * (base * pulse))
Box(
modifier =
Modifier
.width(5.dp)
.height(barHeight)
.background(if (active) mobileAccent else mobileBorderStrong, RoundedCornerShape(999.dp)),
)
}
}
}
private fun Context.hasRecordAudioPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
)
}
private fun Context.findActivity(): Activity? =
when (this) {
is Activity -> this
is ContextWrapper -> baseContext.findActivity()
else -> null
}
private fun openAppSettings(context: Context) {
val intent =
Intent(
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
Uri.fromParts("package", context.packageName, null),
)
context.startActivity(intent)
}

View File

@@ -0,0 +1,353 @@
package ai.openclaw.android.ui.chat
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.horizontalScroll
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.automirrored.filled.Send
import androidx.compose.material.icons.filled.ArrowDropDown
import androidx.compose.material.icons.filled.AttachFile
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Stop
import androidx.compose.material3.Button
import androidx.compose.material3.ButtonDefaults
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.DropdownMenu
import androidx.compose.material3.DropdownMenuItem
import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.OutlinedTextFieldDefaults
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.saveable.rememberSaveable
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import ai.openclaw.android.ui.mobileAccent
import ai.openclaw.android.ui.mobileAccentSoft
import ai.openclaw.android.ui.mobileBorder
import ai.openclaw.android.ui.mobileBorderStrong
import ai.openclaw.android.ui.mobileCallout
import ai.openclaw.android.ui.mobileCaption1
import ai.openclaw.android.ui.mobileHeadline
import ai.openclaw.android.ui.mobileSurface
import ai.openclaw.android.ui.mobileText
import ai.openclaw.android.ui.mobileTextSecondary
import ai.openclaw.android.ui.mobileTextTertiary
@Composable
fun ChatComposer(
healthOk: Boolean,
thinkingLevel: String,
pendingRunCount: Int,
attachments: List<PendingImageAttachment>,
onPickImages: () -> Unit,
onRemoveAttachment: (id: String) -> Unit,
onSetThinkingLevel: (level: String) -> Unit,
onRefresh: () -> Unit,
onAbort: () -> Unit,
onSend: (text: String) -> Unit,
) {
var input by rememberSaveable { mutableStateOf("") }
var showThinkingMenu by remember { mutableStateOf(false) }
val canSend = pendingRunCount == 0 && (input.trim().isNotEmpty() || attachments.isNotEmpty()) && healthOk
val sendBusy = pendingRunCount > 0
Column(modifier = Modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(8.dp)) {
Row(
modifier = Modifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Box(modifier = Modifier.weight(1f)) {
Surface(
onClick = { showThinkingMenu = true },
shape = RoundedCornerShape(14.dp),
color = mobileAccentSoft,
border = BorderStroke(1.dp, mobileBorderStrong),
) {
Row(
modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 8.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.SpaceBetween,
) {
Text(
text = "Thinking: ${thinkingLabel(thinkingLevel)}",
style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold),
color = mobileText,
)
Icon(Icons.Default.ArrowDropDown, contentDescription = "Select thinking level", tint = mobileTextSecondary)
}
}
DropdownMenu(expanded = showThinkingMenu, onDismissRequest = { showThinkingMenu = false }) {
ThinkingMenuItem("off", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
ThinkingMenuItem("low", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
ThinkingMenuItem("medium", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
ThinkingMenuItem("high", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
}
}
SecondaryActionButton(
label = "Attach",
icon = Icons.Default.AttachFile,
enabled = true,
onClick = onPickImages,
)
}
if (attachments.isNotEmpty()) {
AttachmentsStrip(attachments = attachments, onRemoveAttachment = onRemoveAttachment)
}
HorizontalDivider(color = mobileBorder)
Text(
text = "MESSAGE",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 0.9.sp),
color = mobileTextSecondary,
)
OutlinedTextField(
value = input,
onValueChange = { input = it },
modifier = Modifier.fillMaxWidth().height(92.dp),
placeholder = { Text("Type a message", style = mobileBodyStyle(), color = mobileTextTertiary) },
minLines = 2,
maxLines = 5,
textStyle = mobileBodyStyle().copy(color = mobileText),
shape = RoundedCornerShape(14.dp),
colors = chatTextFieldColors(),
)
if (!healthOk) {
Text(
text = "Gateway is offline. Connect first in the Connect tab.",
style = mobileCallout,
color = ai.openclaw.android.ui.mobileWarning,
)
}
Row(
modifier = Modifier.fillMaxWidth(),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(10.dp),
) {
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
SecondaryActionButton(
label = "Refresh",
icon = Icons.Default.Refresh,
enabled = true,
compact = true,
onClick = onRefresh,
)
SecondaryActionButton(
label = "Abort",
icon = Icons.Default.Stop,
enabled = pendingRunCount > 0,
compact = true,
onClick = onAbort,
)
}
Button(
onClick = {
val text = input
input = ""
onSend(text)
},
enabled = canSend,
modifier = Modifier.weight(1f).height(48.dp),
shape = RoundedCornerShape(14.dp),
colors =
ButtonDefaults.buttonColors(
containerColor = mobileAccent,
contentColor = Color.White,
disabledContainerColor = mobileBorderStrong,
disabledContentColor = mobileTextTertiary,
),
border = BorderStroke(1.dp, if (canSend) Color(0xFF154CAD) else mobileBorderStrong),
) {
if (sendBusy) {
CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp, color = Color.White)
} else {
Icon(Icons.AutoMirrored.Filled.Send, contentDescription = null, modifier = Modifier.size(16.dp))
}
Spacer(modifier = Modifier.width(8.dp))
Text(
text = "Send",
style = mobileHeadline.copy(fontWeight = FontWeight.Bold),
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
}
}
}
@Composable
private fun SecondaryActionButton(
label: String,
icon: androidx.compose.ui.graphics.vector.ImageVector,
enabled: Boolean,
compact: Boolean = false,
onClick: () -> Unit,
) {
Button(
onClick = onClick,
enabled = enabled,
modifier = if (compact) Modifier.size(44.dp) else Modifier.height(44.dp),
shape = RoundedCornerShape(14.dp),
colors =
ButtonDefaults.buttonColors(
containerColor = Color.White,
contentColor = mobileTextSecondary,
disabledContainerColor = Color.White,
disabledContentColor = mobileTextTertiary,
),
border = BorderStroke(1.dp, mobileBorderStrong),
contentPadding = if (compact) PaddingValues(0.dp) else ButtonDefaults.ContentPadding,
) {
Icon(icon, contentDescription = label, modifier = Modifier.size(14.dp))
if (!compact) {
Spacer(modifier = Modifier.width(5.dp))
Text(
text = label,
style = mobileCallout.copy(fontWeight = FontWeight.SemiBold),
color = if (enabled) mobileTextSecondary else mobileTextTertiary,
)
}
}
}
@Composable
private fun ThinkingMenuItem(
value: String,
current: String,
onSet: (String) -> Unit,
onDismiss: () -> Unit,
) {
DropdownMenuItem(
text = { Text(thinkingLabel(value), style = mobileCallout, color = mobileText) },
onClick = {
onSet(value)
onDismiss()
},
trailingIcon = {
if (value == current.trim().lowercase()) {
Text("", style = mobileCallout, color = mobileAccent)
} else {
Spacer(modifier = Modifier.width(10.dp))
}
},
)
}
private fun thinkingLabel(raw: String): String {
return when (raw.trim().lowercase()) {
"low" -> "Low"
"medium" -> "Medium"
"high" -> "High"
else -> "Off"
}
}
@Composable
private fun AttachmentsStrip(
attachments: List<PendingImageAttachment>,
onRemoveAttachment: (id: String) -> Unit,
) {
Row(
modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()),
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
for (att in attachments) {
AttachmentChip(
fileName = att.fileName,
onRemove = { onRemoveAttachment(att.id) },
)
}
}
}
@Composable
private fun AttachmentChip(fileName: String, onRemove: () -> Unit) {
Surface(
shape = RoundedCornerShape(999.dp),
color = mobileAccentSoft,
border = BorderStroke(1.dp, mobileBorderStrong),
) {
Row(
modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
Text(
text = fileName,
style = mobileCaption1,
color = mobileText,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
Surface(
onClick = onRemove,
shape = RoundedCornerShape(999.dp),
color = Color.White,
border = BorderStroke(1.dp, mobileBorderStrong),
) {
Text(
text = "×",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold),
color = mobileTextSecondary,
modifier = Modifier.padding(horizontal = 8.dp, vertical = 2.dp),
)
}
}
}
}
@Composable
private fun chatTextFieldColors() =
OutlinedTextFieldDefaults.colors(
focusedContainerColor = mobileSurface,
unfocusedContainerColor = mobileSurface,
focusedBorderColor = mobileAccent,
unfocusedBorderColor = mobileBorder,
focusedTextColor = mobileText,
unfocusedTextColor = mobileText,
cursorColor = mobileAccent,
)
@Composable
private fun mobileBodyStyle() =
MaterialTheme.typography.bodyMedium.copy(
fontFamily = ai.openclaw.android.ui.mobileFontFamily,
fontWeight = FontWeight.Medium,
fontSize = 15.sp,
lineHeight = 22.sp,
)

View File

@@ -0,0 +1,591 @@
package ai.openclaw.android.ui.chat
import android.graphics.BitmapFactory
import android.util.Base64
import androidx.compose.foundation.Image
import androidx.compose.foundation.background
import androidx.compose.foundation.border
import androidx.compose.foundation.horizontalScroll
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.IntrinsicSize
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxHeight
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.text.selection.SelectionContainer
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.text.AnnotatedString
import androidx.compose.ui.text.SpanStyle
import androidx.compose.ui.text.TextStyle
import androidx.compose.ui.text.buildAnnotatedString
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontStyle
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextDecoration
import androidx.compose.ui.text.withStyle
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import ai.openclaw.android.ui.mobileAccent
import ai.openclaw.android.ui.mobileCallout
import ai.openclaw.android.ui.mobileCaption1
import ai.openclaw.android.ui.mobileCodeBg
import ai.openclaw.android.ui.mobileCodeText
import ai.openclaw.android.ui.mobileTextSecondary
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.commonmark.Extension
import org.commonmark.ext.autolink.AutolinkExtension
import org.commonmark.ext.gfm.strikethrough.Strikethrough
import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension
import org.commonmark.ext.gfm.tables.TableBlock
import org.commonmark.ext.gfm.tables.TableBody
import org.commonmark.ext.gfm.tables.TableCell
import org.commonmark.ext.gfm.tables.TableHead
import org.commonmark.ext.gfm.tables.TableRow
import org.commonmark.ext.gfm.tables.TablesExtension
import org.commonmark.ext.task.list.items.TaskListItemMarker
import org.commonmark.ext.task.list.items.TaskListItemsExtension
import org.commonmark.node.BlockQuote
import org.commonmark.node.BulletList
import org.commonmark.node.Code
import org.commonmark.node.Document
import org.commonmark.node.Emphasis
import org.commonmark.node.FencedCodeBlock
import org.commonmark.node.Heading
import org.commonmark.node.HardLineBreak
import org.commonmark.node.HtmlBlock
import org.commonmark.node.HtmlInline
import org.commonmark.node.Image as MarkdownImage
import org.commonmark.node.IndentedCodeBlock
import org.commonmark.node.Link
import org.commonmark.node.ListItem
import org.commonmark.node.Node
import org.commonmark.node.OrderedList
import org.commonmark.node.Paragraph
import org.commonmark.node.SoftLineBreak
import org.commonmark.node.StrongEmphasis
import org.commonmark.node.Text as MarkdownTextNode
import org.commonmark.node.ThematicBreak
import org.commonmark.parser.Parser
private const val LIST_INDENT_DP = 14
private val dataImageRegex = Regex("^data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)$")
private val markdownParser: Parser by lazy {
val extensions: List<Extension> =
listOf(
AutolinkExtension.create(),
StrikethroughExtension.create(),
TablesExtension.create(),
TaskListItemsExtension.create(),
)
Parser.builder()
.extensions(extensions)
.build()
}
@Composable
fun ChatMarkdown(text: String, textColor: Color) {
val document = remember(text) { markdownParser.parse(text) as Document }
val inlineStyles = InlineStyles(inlineCodeBg = mobileCodeBg, inlineCodeColor = mobileCodeText)
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
RenderMarkdownBlocks(
start = document.firstChild,
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = 0,
)
}
}
@Composable
private fun RenderMarkdownBlocks(
start: Node?,
textColor: Color,
inlineStyles: InlineStyles,
listDepth: Int,
) {
var node = start
while (node != null) {
val current = node
when (current) {
is Paragraph -> {
RenderParagraph(current, textColor = textColor, inlineStyles = inlineStyles)
}
is Heading -> {
val headingText = remember(current) { buildInlineMarkdown(current.firstChild, inlineStyles) }
Text(
text = headingText,
style = headingStyle(current.level),
color = textColor,
)
}
is FencedCodeBlock -> {
SelectionContainer(modifier = Modifier.fillMaxWidth()) {
ChatCodeBlock(code = current.literal.orEmpty(), language = current.info?.trim()?.ifEmpty { null })
}
}
is IndentedCodeBlock -> {
SelectionContainer(modifier = Modifier.fillMaxWidth()) {
ChatCodeBlock(code = current.literal.orEmpty(), language = null)
}
}
is BlockQuote -> {
Row(
modifier = Modifier
.fillMaxWidth()
.height(IntrinsicSize.Min)
.padding(vertical = 2.dp),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.Top,
) {
Box(
modifier = Modifier
.width(2.dp)
.fillMaxHeight()
.background(mobileTextSecondary.copy(alpha = 0.35f)),
)
Column(
modifier = Modifier.weight(1f),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
RenderMarkdownBlocks(
start = current.firstChild,
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth,
)
}
}
}
is BulletList -> {
RenderBulletList(
list = current,
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth,
)
}
is OrderedList -> {
RenderOrderedList(
list = current,
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth,
)
}
is TableBlock -> {
RenderTableBlock(
table = current,
textColor = textColor,
inlineStyles = inlineStyles,
)
}
is ThematicBreak -> {
Box(
modifier = Modifier
.fillMaxWidth()
.height(1.dp)
.background(mobileTextSecondary.copy(alpha = 0.25f)),
)
}
is HtmlBlock -> {
val literal = current.literal.orEmpty().trim()
if (literal.isNotEmpty()) {
Text(
text = literal,
style = mobileCallout.copy(fontFamily = FontFamily.Monospace),
color = textColor,
)
}
}
}
node = current.next
}
}
@Composable
private fun RenderParagraph(
paragraph: Paragraph,
textColor: Color,
inlineStyles: InlineStyles,
) {
val standaloneImage = remember(paragraph) { standaloneDataImage(paragraph) }
if (standaloneImage != null) {
InlineBase64Image(base64 = standaloneImage.base64, mimeType = standaloneImage.mimeType)
return
}
val annotated = remember(paragraph) { buildInlineMarkdown(paragraph.firstChild, inlineStyles) }
if (annotated.text.trimEnd().isEmpty()) {
return
}
Text(
text = annotated,
style = mobileCallout,
color = textColor,
)
}
@Composable
private fun RenderBulletList(
list: BulletList,
textColor: Color,
inlineStyles: InlineStyles,
listDepth: Int,
) {
Column(
modifier = Modifier.padding(start = (LIST_INDENT_DP * listDepth).dp),
verticalArrangement = Arrangement.spacedBy(6.dp),
) {
var item = list.firstChild
while (item != null) {
if (item is ListItem) {
RenderListItem(
item = item,
markerText = "",
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth,
)
}
item = item.next
}
}
}
@Composable
private fun RenderOrderedList(
list: OrderedList,
textColor: Color,
inlineStyles: InlineStyles,
listDepth: Int,
) {
Column(
modifier = Modifier.padding(start = (LIST_INDENT_DP * listDepth).dp),
verticalArrangement = Arrangement.spacedBy(6.dp),
) {
var index = list.markerStartNumber ?: 1
var item = list.firstChild
while (item != null) {
if (item is ListItem) {
RenderListItem(
item = item,
markerText = "$index.",
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth,
)
index += 1
}
item = item.next
}
}
}
@Composable
private fun RenderListItem(
item: ListItem,
markerText: String,
textColor: Color,
inlineStyles: InlineStyles,
listDepth: Int,
) {
var contentStart = item.firstChild
var marker = markerText
val task = contentStart as? TaskListItemMarker
if (task != null) {
marker = if (task.isChecked) "" else ""
contentStart = task.next
}
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(8.dp),
verticalAlignment = Alignment.Top,
) {
Text(
text = marker,
style = mobileCallout.copy(fontWeight = FontWeight.SemiBold),
color = textColor,
modifier = Modifier.width(24.dp),
)
Column(
modifier = Modifier.weight(1f),
verticalArrangement = Arrangement.spacedBy(4.dp),
) {
RenderMarkdownBlocks(
start = contentStart,
textColor = textColor,
inlineStyles = inlineStyles,
listDepth = listDepth + 1,
)
}
}
}
@Composable
private fun RenderTableBlock(
table: TableBlock,
textColor: Color,
inlineStyles: InlineStyles,
) {
val rows = remember(table) { buildTableRows(table, inlineStyles) }
if (rows.isEmpty()) return
val maxCols = rows.maxOf { row -> row.cells.size }.coerceAtLeast(1)
val scrollState = rememberScrollState()
Column(
modifier = Modifier
.fillMaxWidth()
.horizontalScroll(scrollState)
.border(1.dp, mobileTextSecondary.copy(alpha = 0.25f)),
) {
for (row in rows) {
Row(
modifier = Modifier.fillMaxWidth(),
) {
for (index in 0 until maxCols) {
val cell = row.cells.getOrNull(index) ?: AnnotatedString("")
Text(
text = cell,
style = if (row.isHeader) mobileCaption1.copy(fontWeight = FontWeight.SemiBold) else mobileCallout,
color = textColor,
modifier = Modifier
.border(1.dp, mobileTextSecondary.copy(alpha = 0.22f))
.padding(horizontal = 8.dp, vertical = 6.dp)
.width(160.dp),
)
}
}
}
}
}
private fun buildTableRows(table: TableBlock, inlineStyles: InlineStyles): List<TableRenderRow> {
val rows = mutableListOf<TableRenderRow>()
var child = table.firstChild
while (child != null) {
when (child) {
is TableHead -> rows.addAll(readTableSection(child, isHeader = true, inlineStyles = inlineStyles))
is TableBody -> rows.addAll(readTableSection(child, isHeader = false, inlineStyles = inlineStyles))
is TableRow -> rows.add(readTableRow(child, isHeader = false, inlineStyles = inlineStyles))
}
child = child.next
}
return rows
}
private fun readTableSection(section: Node, isHeader: Boolean, inlineStyles: InlineStyles): List<TableRenderRow> {
val rows = mutableListOf<TableRenderRow>()
var row = section.firstChild
while (row != null) {
if (row is TableRow) {
rows.add(readTableRow(row, isHeader = isHeader, inlineStyles = inlineStyles))
}
row = row.next
}
return rows
}
private fun readTableRow(row: TableRow, isHeader: Boolean, inlineStyles: InlineStyles): TableRenderRow {
val cells = mutableListOf<AnnotatedString>()
var cellNode = row.firstChild
while (cellNode != null) {
if (cellNode is TableCell) {
cells.add(buildInlineMarkdown(cellNode.firstChild, inlineStyles))
}
cellNode = cellNode.next
}
return TableRenderRow(isHeader = isHeader, cells = cells)
}
private fun buildInlineMarkdown(start: Node?, inlineStyles: InlineStyles): AnnotatedString {
return buildAnnotatedString {
appendInlineNode(
node = start,
inlineCodeBg = inlineStyles.inlineCodeBg,
inlineCodeColor = inlineStyles.inlineCodeColor,
)
}
}
private fun AnnotatedString.Builder.appendInlineNode(
node: Node?,
inlineCodeBg: Color,
inlineCodeColor: Color,
) {
var current = node
while (current != null) {
when (current) {
is MarkdownTextNode -> append(current.literal)
is SoftLineBreak -> append('\n')
is HardLineBreak -> append('\n')
is Code -> {
withStyle(
SpanStyle(
fontFamily = FontFamily.Monospace,
background = inlineCodeBg,
color = inlineCodeColor,
),
) {
append(current.literal)
}
}
is Emphasis -> {
withStyle(SpanStyle(fontStyle = FontStyle.Italic)) {
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
}
}
is StrongEmphasis -> {
withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) {
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
}
}
is Strikethrough -> {
withStyle(SpanStyle(textDecoration = TextDecoration.LineThrough)) {
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
}
}
is Link -> {
withStyle(
SpanStyle(
color = mobileAccent,
textDecoration = TextDecoration.Underline,
),
) {
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
}
}
is MarkdownImage -> {
val alt = buildPlainText(current.firstChild)
if (alt.isNotBlank()) {
append(alt)
} else {
append("image")
}
}
is HtmlInline -> {
if (!current.literal.isNullOrBlank()) {
append(current.literal)
}
}
else -> {
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
}
}
current = current.next
}
}
private fun buildPlainText(start: Node?): String {
val sb = StringBuilder()
var node = start
while (node != null) {
when (node) {
is MarkdownTextNode -> sb.append(node.literal)
is SoftLineBreak, is HardLineBreak -> sb.append('\n')
else -> sb.append(buildPlainText(node.firstChild))
}
node = node.next
}
return sb.toString()
}
private fun standaloneDataImage(paragraph: Paragraph): ParsedDataImage? {
val only = paragraph.firstChild as? MarkdownImage ?: return null
if (only.next != null) return null
return parseDataImageDestination(only.destination)
}
private fun parseDataImageDestination(destination: String?): ParsedDataImage? {
val raw = destination?.trim().orEmpty()
if (raw.isEmpty()) return null
val match = dataImageRegex.matchEntire(raw) ?: return null
val subtype = match.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png"
val base64 = match.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty()
if (base64.isEmpty()) return null
return ParsedDataImage(mimeType = "image/$subtype", base64 = base64)
}
private fun headingStyle(level: Int): TextStyle {
return when (level.coerceIn(1, 6)) {
1 -> mobileCallout.copy(fontSize = 22.sp, lineHeight = 28.sp, fontWeight = FontWeight.Bold)
2 -> mobileCallout.copy(fontSize = 20.sp, lineHeight = 26.sp, fontWeight = FontWeight.Bold)
3 -> mobileCallout.copy(fontSize = 18.sp, lineHeight = 24.sp, fontWeight = FontWeight.SemiBold)
4 -> mobileCallout.copy(fontSize = 16.sp, lineHeight = 22.sp, fontWeight = FontWeight.SemiBold)
else -> mobileCallout.copy(fontWeight = FontWeight.SemiBold)
}
}
private data class InlineStyles(
val inlineCodeBg: Color,
val inlineCodeColor: Color,
)
private data class TableRenderRow(
val isHeader: Boolean,
val cells: List<AnnotatedString>,
)
private data class ParsedDataImage(
val mimeType: String,
val base64: String,
)
@Composable
private fun InlineBase64Image(base64: String, mimeType: String?) {
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }
var failed by remember(base64) { mutableStateOf(false) }
LaunchedEffect(base64) {
failed = false
image =
withContext(Dispatchers.Default) {
try {
val bytes = Base64.decode(base64, Base64.DEFAULT)
val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null
bitmap.asImageBitmap()
} catch (_: Throwable) {
null
}
}
if (image == null) failed = true
}
if (image != null) {
Image(
bitmap = image!!,
contentDescription = mimeType ?: "image",
contentScale = ContentScale.Fit,
modifier = Modifier.fillMaxWidth(),
)
} else if (failed) {
Text(
text = "Image unavailable",
modifier = Modifier.padding(vertical = 2.dp),
style = mobileCaption1,
color = mobileTextSecondary,
)
}
}

View File

@@ -0,0 +1,108 @@
package ai.openclaw.android.ui.chat
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.lazy.LazyColumn
import androidx.compose.foundation.lazy.rememberLazyListState
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.unit.dp
import ai.openclaw.android.chat.ChatMessage
import ai.openclaw.android.chat.ChatPendingToolCall
import ai.openclaw.android.ui.mobileBorder
import ai.openclaw.android.ui.mobileCallout
import ai.openclaw.android.ui.mobileHeadline
import ai.openclaw.android.ui.mobileText
import ai.openclaw.android.ui.mobileTextSecondary
@Composable
fun ChatMessageListCard(
messages: List<ChatMessage>,
pendingRunCount: Int,
pendingToolCalls: List<ChatPendingToolCall>,
streamingAssistantText: String?,
healthOk: Boolean,
modifier: Modifier = Modifier,
) {
val listState = rememberLazyListState()
// With reverseLayout the newest item is at index 0 (bottom of screen).
LaunchedEffect(messages.size, pendingRunCount, pendingToolCalls.size, streamingAssistantText) {
listState.animateScrollToItem(index = 0)
}
Box(modifier = modifier.fillMaxWidth()) {
LazyColumn(
modifier = Modifier.fillMaxSize(),
state = listState,
reverseLayout = true,
verticalArrangement = Arrangement.spacedBy(10.dp),
contentPadding = androidx.compose.foundation.layout.PaddingValues(bottom = 8.dp),
) {
// With reverseLayout = true, index 0 renders at the BOTTOM.
// So we emit newest items first: streaming → tools → typing → messages (newest→oldest).
val stream = streamingAssistantText?.trim()
if (!stream.isNullOrEmpty()) {
item(key = "stream") {
ChatStreamingAssistantBubble(text = stream)
}
}
if (pendingToolCalls.isNotEmpty()) {
item(key = "tools") {
ChatPendingToolsBubble(toolCalls = pendingToolCalls)
}
}
if (pendingRunCount > 0) {
item(key = "typing") {
ChatTypingIndicatorBubble()
}
}
items(count = messages.size, key = { idx -> messages[messages.size - 1 - idx].id }) { idx ->
ChatMessageBubble(message = messages[messages.size - 1 - idx])
}
}
if (messages.isEmpty() && pendingRunCount == 0 && pendingToolCalls.isEmpty() && streamingAssistantText.isNullOrBlank()) {
EmptyChatHint(modifier = Modifier.align(Alignment.Center), healthOk = healthOk)
}
}
}
@Composable
private fun EmptyChatHint(modifier: Modifier = Modifier, healthOk: Boolean) {
Surface(
modifier = modifier.fillMaxWidth(),
shape = RoundedCornerShape(14.dp),
color = androidx.compose.ui.graphics.Color.White.copy(alpha = 0.9f),
border = androidx.compose.foundation.BorderStroke(1.dp, mobileBorder),
) {
androidx.compose.foundation.layout.Column(
modifier = Modifier.padding(horizontal = 12.dp, vertical = 12.dp),
verticalArrangement = Arrangement.spacedBy(4.dp),
) {
Text("No messages yet", style = mobileHeadline, color = mobileText)
Text(
text =
if (healthOk) {
"Send the first prompt to start this session."
} else {
"Connect gateway first, then return to chat."
},
style = mobileCallout,
color = mobileTextSecondary,
)
}
}
}

View File

@@ -0,0 +1,323 @@
package ai.openclaw.android.ui.chat
import android.graphics.BitmapFactory
import android.util.Base64
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.Image
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import ai.openclaw.android.chat.ChatMessage
import ai.openclaw.android.chat.ChatMessageContent
import ai.openclaw.android.chat.ChatPendingToolCall
import ai.openclaw.android.tools.ToolDisplayRegistry
import ai.openclaw.android.ui.mobileAccent
import ai.openclaw.android.ui.mobileAccentSoft
import ai.openclaw.android.ui.mobileBorder
import ai.openclaw.android.ui.mobileBorderStrong
import ai.openclaw.android.ui.mobileCallout
import ai.openclaw.android.ui.mobileCaption1
import ai.openclaw.android.ui.mobileCaption2
import ai.openclaw.android.ui.mobileCodeBg
import ai.openclaw.android.ui.mobileCodeText
import ai.openclaw.android.ui.mobileHeadline
import ai.openclaw.android.ui.mobileText
import ai.openclaw.android.ui.mobileTextSecondary
import ai.openclaw.android.ui.mobileWarning
import ai.openclaw.android.ui.mobileWarningSoft
import java.util.Locale
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
private data class ChatBubbleStyle(
val alignEnd: Boolean,
val containerColor: Color,
val borderColor: Color,
val roleColor: Color,
)
@Composable
fun ChatMessageBubble(message: ChatMessage) {
val role = message.role.trim().lowercase(Locale.US)
val style = bubbleStyle(role)
// Filter to only displayable content parts (text with content, or base64 images).
val displayableContent =
message.content.filter { part ->
when (part.type) {
"text" -> !part.text.isNullOrBlank()
else -> part.base64 != null
}
}
if (displayableContent.isEmpty()) return
ChatBubbleContainer(style = style, roleLabel = roleLabel(role)) {
ChatMessageBody(content = displayableContent, textColor = mobileText)
}
}
@Composable
private fun ChatBubbleContainer(
style: ChatBubbleStyle,
roleLabel: String,
modifier: Modifier = Modifier,
content: @Composable () -> Unit,
) {
Row(
modifier = modifier.fillMaxWidth(),
horizontalArrangement = if (style.alignEnd) Arrangement.End else Arrangement.Start,
) {
Surface(
shape = RoundedCornerShape(12.dp),
border = BorderStroke(1.dp, style.borderColor),
color = style.containerColor,
tonalElevation = 0.dp,
shadowElevation = 0.dp,
modifier = Modifier.fillMaxWidth(0.90f),
) {
Column(
modifier = Modifier.padding(horizontal = 11.dp, vertical = 8.dp),
verticalArrangement = Arrangement.spacedBy(3.dp),
) {
Text(
text = roleLabel,
style = mobileCaption2.copy(fontWeight = FontWeight.SemiBold, letterSpacing = 0.6.sp),
color = style.roleColor,
)
content()
}
}
}
}
@Composable
private fun ChatMessageBody(content: List<ChatMessageContent>, textColor: Color) {
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
for (part in content) {
when (part.type) {
"text" -> {
val text = part.text ?: continue
ChatMarkdown(text = text, textColor = textColor)
}
else -> {
val b64 = part.base64 ?: continue
ChatBase64Image(base64 = b64, mimeType = part.mimeType)
}
}
}
}
}
@Composable
fun ChatTypingIndicatorBubble() {
ChatBubbleContainer(
style = bubbleStyle("assistant"),
roleLabel = roleLabel("assistant"),
) {
Row(
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
DotPulse(color = mobileTextSecondary)
Text("Thinking...", style = mobileCallout, color = mobileTextSecondary)
}
}
}
@Composable
fun ChatPendingToolsBubble(toolCalls: List<ChatPendingToolCall>) {
val context = LocalContext.current
val displays =
remember(toolCalls, context) {
toolCalls.map { ToolDisplayRegistry.resolve(context, it.name, it.args) }
}
ChatBubbleContainer(
style = bubbleStyle("assistant"),
roleLabel = "TOOLS",
) {
Column(verticalArrangement = Arrangement.spacedBy(4.dp)) {
Text("Running tools...", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
for (display in displays.take(6)) {
Column(verticalArrangement = Arrangement.spacedBy(2.dp)) {
Text(
"${display.emoji} ${display.label}",
style = mobileCallout,
color = mobileTextSecondary,
fontFamily = FontFamily.Monospace,
)
display.detailLine?.let { detail ->
Text(
detail,
style = mobileCaption1,
color = mobileTextSecondary,
fontFamily = FontFamily.Monospace,
)
}
}
}
if (toolCalls.size > 6) {
Text(
text = "... +${toolCalls.size - 6} more",
style = mobileCaption1,
color = mobileTextSecondary,
)
}
}
}
}
@Composable
fun ChatStreamingAssistantBubble(text: String) {
ChatBubbleContainer(
style = bubbleStyle("assistant").copy(borderColor = mobileAccent),
roleLabel = "ASSISTANT · LIVE",
) {
ChatMarkdown(text = text, textColor = mobileText)
}
}
private fun bubbleStyle(role: String): ChatBubbleStyle {
return when (role) {
"user" ->
ChatBubbleStyle(
alignEnd = true,
containerColor = mobileAccentSoft,
borderColor = mobileAccent,
roleColor = mobileAccent,
)
"system" ->
ChatBubbleStyle(
alignEnd = false,
containerColor = mobileWarningSoft,
borderColor = mobileWarning.copy(alpha = 0.45f),
roleColor = mobileWarning,
)
else ->
ChatBubbleStyle(
alignEnd = false,
containerColor = Color.White,
borderColor = mobileBorderStrong,
roleColor = mobileTextSecondary,
)
}
}
private fun roleLabel(role: String): String {
return when (role) {
"user" -> "USER"
"system" -> "SYSTEM"
else -> "ASSISTANT"
}
}
@Composable
private fun ChatBase64Image(base64: String, mimeType: String?) {
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }
var failed by remember(base64) { mutableStateOf(false) }
LaunchedEffect(base64) {
failed = false
image =
withContext(Dispatchers.Default) {
try {
val bytes = Base64.decode(base64, Base64.DEFAULT)
val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null
bitmap.asImageBitmap()
} catch (_: Throwable) {
null
}
}
if (image == null) failed = true
}
if (image != null) {
Surface(
shape = RoundedCornerShape(10.dp),
border = BorderStroke(1.dp, mobileBorder),
color = Color.White,
modifier = Modifier.fillMaxWidth(),
) {
Image(
bitmap = image!!,
contentDescription = mimeType ?: "attachment",
contentScale = ContentScale.Fit,
modifier = Modifier.fillMaxWidth(),
)
}
} else if (failed) {
Text("Unsupported attachment", style = mobileCaption1, color = mobileTextSecondary)
}
}
@Composable
private fun DotPulse(color: Color) {
Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) {
PulseDot(alpha = 0.38f, color = color)
PulseDot(alpha = 0.62f, color = color)
PulseDot(alpha = 0.90f, color = color)
}
}
@Composable
private fun PulseDot(alpha: Float, color: Color) {
Surface(
modifier = Modifier.size(6.dp).alpha(alpha),
shape = CircleShape,
color = color,
) {}
}
@Composable
fun ChatCodeBlock(code: String, language: String?) {
Surface(
shape = RoundedCornerShape(8.dp),
color = mobileCodeBg,
border = BorderStroke(1.dp, Color(0xFF2B2E35)),
modifier = Modifier.fillMaxWidth(),
) {
Column(modifier = Modifier.padding(horizontal = 10.dp, vertical = 8.dp), verticalArrangement = Arrangement.spacedBy(4.dp)) {
if (!language.isNullOrBlank()) {
Text(
text = language.uppercase(Locale.US),
style = mobileCaption2.copy(letterSpacing = 0.4.sp),
color = mobileTextSecondary,
)
}
Text(
text = code.trimEnd(),
fontFamily = FontFamily.Monospace,
style = mobileCallout,
color = mobileCodeText,
)
}
}
}

View File

@@ -0,0 +1,282 @@
package ai.openclaw.android.ui.chat
import android.content.ContentResolver
import android.net.Uri
import android.util.Base64
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.horizontalScroll
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.imePadding
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateListOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import ai.openclaw.android.MainViewModel
import ai.openclaw.android.chat.ChatSessionEntry
import ai.openclaw.android.chat.OutgoingAttachment
import ai.openclaw.android.ui.mobileAccent
import ai.openclaw.android.ui.mobileBorder
import ai.openclaw.android.ui.mobileBorderStrong
import ai.openclaw.android.ui.mobileCallout
import ai.openclaw.android.ui.mobileCaption1
import ai.openclaw.android.ui.mobileCaption2
import ai.openclaw.android.ui.mobileDanger
import ai.openclaw.android.ui.mobileSuccess
import ai.openclaw.android.ui.mobileSuccessSoft
import ai.openclaw.android.ui.mobileText
import ai.openclaw.android.ui.mobileTextSecondary
import ai.openclaw.android.ui.mobileWarning
import ai.openclaw.android.ui.mobileWarningSoft
import java.io.ByteArrayOutputStream
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
@Composable
fun ChatSheetContent(viewModel: MainViewModel) {
val messages by viewModel.chatMessages.collectAsState()
val errorText by viewModel.chatError.collectAsState()
val pendingRunCount by viewModel.pendingRunCount.collectAsState()
val healthOk by viewModel.chatHealthOk.collectAsState()
val sessionKey by viewModel.chatSessionKey.collectAsState()
val mainSessionKey by viewModel.mainSessionKey.collectAsState()
val thinkingLevel by viewModel.chatThinkingLevel.collectAsState()
val streamingAssistantText by viewModel.chatStreamingAssistantText.collectAsState()
val pendingToolCalls by viewModel.chatPendingToolCalls.collectAsState()
val sessions by viewModel.chatSessions.collectAsState()
LaunchedEffect(mainSessionKey) {
viewModel.loadChat(mainSessionKey)
viewModel.refreshChatSessions(limit = 200)
}
val context = LocalContext.current
val resolver = context.contentResolver
val scope = rememberCoroutineScope()
val attachments = remember { mutableStateListOf<PendingImageAttachment>() }
val pickImages =
rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris ->
if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult
scope.launch(Dispatchers.IO) {
val next =
uris.take(8).mapNotNull { uri ->
try {
loadImageAttachment(resolver, uri)
} catch (_: Throwable) {
null
}
}
withContext(Dispatchers.Main) {
attachments.addAll(next)
}
}
}
Column(
modifier =
Modifier
.fillMaxSize()
.padding(horizontal = 20.dp, vertical = 12.dp),
verticalArrangement = Arrangement.spacedBy(8.dp),
) {
ChatThreadSelector(
sessionKey = sessionKey,
sessions = sessions,
mainSessionKey = mainSessionKey,
healthOk = healthOk,
onSelectSession = { key -> viewModel.switchChatSession(key) },
)
if (!errorText.isNullOrBlank()) {
ChatErrorRail(errorText = errorText!!)
}
ChatMessageListCard(
messages = messages,
pendingRunCount = pendingRunCount,
pendingToolCalls = pendingToolCalls,
streamingAssistantText = streamingAssistantText,
healthOk = healthOk,
modifier = Modifier.weight(1f, fill = true),
)
Row(modifier = Modifier.fillMaxWidth().imePadding()) {
ChatComposer(
healthOk = healthOk,
thinkingLevel = thinkingLevel,
pendingRunCount = pendingRunCount,
attachments = attachments,
onPickImages = { pickImages.launch("image/*") },
onRemoveAttachment = { id -> attachments.removeAll { it.id == id } },
onSetThinkingLevel = { level -> viewModel.setChatThinkingLevel(level) },
onRefresh = {
viewModel.refreshChat()
viewModel.refreshChatSessions(limit = 200)
},
onAbort = { viewModel.abortChat() },
onSend = { text ->
val outgoing =
attachments.map { att ->
OutgoingAttachment(
type = "image",
mimeType = att.mimeType,
fileName = att.fileName,
base64 = att.base64,
)
}
viewModel.sendChat(message = text, thinking = thinkingLevel, attachments = outgoing)
attachments.clear()
},
)
}
}
}
@Composable
private fun ChatThreadSelector(
sessionKey: String,
sessions: List<ChatSessionEntry>,
mainSessionKey: String,
healthOk: Boolean,
onSelectSession: (String) -> Unit,
) {
val sessionOptions = resolveSessionChoices(sessionKey, sessions, mainSessionKey = mainSessionKey)
val currentSessionLabel =
friendlySessionName(sessionOptions.firstOrNull { it.key == sessionKey }?.displayName ?: sessionKey)
Column(modifier = Modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(8.dp)) {
Row(
modifier = Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.SpaceBetween,
verticalAlignment = androidx.compose.ui.Alignment.CenterVertically,
) {
Text(
text = "SESSION",
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 0.8.sp),
color = mobileTextSecondary,
)
Row(horizontalArrangement = Arrangement.spacedBy(6.dp), verticalAlignment = androidx.compose.ui.Alignment.CenterVertically) {
Text(
text = currentSessionLabel,
style = mobileCallout.copy(fontWeight = FontWeight.SemiBold),
color = mobileText,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
ChatConnectionPill(healthOk = healthOk)
}
}
Row(
modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()),
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
for (entry in sessionOptions) {
val active = entry.key == sessionKey
Surface(
onClick = { onSelectSession(entry.key) },
shape = RoundedCornerShape(14.dp),
color = if (active) mobileAccent else Color.White,
border = BorderStroke(1.dp, if (active) Color(0xFF154CAD) else mobileBorderStrong),
tonalElevation = 0.dp,
shadowElevation = 0.dp,
) {
Text(
text = friendlySessionName(entry.displayName ?: entry.key),
style = mobileCaption1.copy(fontWeight = if (active) FontWeight.Bold else FontWeight.SemiBold),
color = if (active) Color.White else mobileText,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
modifier = Modifier.padding(horizontal = 12.dp, vertical = 8.dp),
)
}
}
}
}
}
@Composable
private fun ChatConnectionPill(healthOk: Boolean) {
Surface(
shape = RoundedCornerShape(999.dp),
color = if (healthOk) mobileSuccessSoft else mobileWarningSoft,
border = BorderStroke(1.dp, if (healthOk) mobileSuccess.copy(alpha = 0.35f) else mobileWarning.copy(alpha = 0.35f)),
) {
Text(
text = if (healthOk) "Connected" else "Offline",
style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold),
color = if (healthOk) mobileSuccess else mobileWarning,
modifier = Modifier.padding(horizontal = 8.dp, vertical = 3.dp),
)
}
}
@Composable
private fun ChatErrorRail(errorText: String) {
Surface(
modifier = Modifier.fillMaxWidth(),
color = androidx.compose.ui.graphics.Color.White,
shape = RoundedCornerShape(12.dp),
border = androidx.compose.foundation.BorderStroke(1.dp, mobileDanger),
) {
Column(modifier = Modifier.padding(horizontal = 10.dp, vertical = 8.dp), verticalArrangement = Arrangement.spacedBy(2.dp)) {
Text(
text = "CHAT ERROR",
style = mobileCaption2.copy(letterSpacing = 0.6.sp),
color = mobileDanger,
)
Text(text = errorText, style = mobileCallout, color = mobileText)
}
}
}
data class PendingImageAttachment(
val id: String,
val fileName: String,
val mimeType: String,
val base64: String,
)
private suspend fun loadImageAttachment(resolver: ContentResolver, uri: Uri): PendingImageAttachment {
val mimeType = resolver.getType(uri) ?: "image/*"
val fileName = (uri.lastPathSegment ?: "image").substringAfterLast('/')
val bytes =
withContext(Dispatchers.IO) {
resolver.openInputStream(uri)?.use { input ->
val out = ByteArrayOutputStream()
input.copyTo(out)
out.toByteArray()
} ?: ByteArray(0)
}
if (bytes.isEmpty()) throw IllegalStateException("empty attachment")
val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP)
return PendingImageAttachment(
id = uri.toString() + "#" + System.currentTimeMillis().toString(),
fileName = fileName,
mimeType = mimeType,
base64 = base64,
)
}

View File

@@ -0,0 +1,73 @@
package ai.openclaw.android.ui.chat
import ai.openclaw.android.chat.ChatSessionEntry
private const val RECENT_WINDOW_MS = 24 * 60 * 60 * 1000L
/**
* Derive a human-friendly label from a raw session key.
* Examples:
* "telegram:g-agent-main-main" -> "Main"
* "agent:main:main" -> "Main"
* "discord:g-server-channel" -> "Server Channel"
* "my-custom-session" -> "My Custom Session"
*/
fun friendlySessionName(key: String): String {
// Strip common prefixes like "telegram:", "agent:", "discord:" etc.
val stripped = key.substringAfterLast(":")
// Remove leading "g-" prefix (gateway artifact)
val cleaned = if (stripped.startsWith("g-")) stripped.removePrefix("g-") else stripped
// Split on hyphens/underscores, title-case each word, collapse "main main" -> "Main"
val words = cleaned.split('-', '_').filter { it.isNotBlank() }.map { word ->
word.replaceFirstChar { it.uppercaseChar() }
}.distinct()
val result = words.joinToString(" ")
return result.ifBlank { key }
}
fun resolveSessionChoices(
currentSessionKey: String,
sessions: List<ChatSessionEntry>,
mainSessionKey: String,
nowMs: Long = System.currentTimeMillis(),
): List<ChatSessionEntry> {
val mainKey = mainSessionKey.trim().ifEmpty { "main" }
val current = currentSessionKey.trim().let { if (it == "main" && mainKey != "main") mainKey else it }
val aliasKey = if (mainKey == "main") null else "main"
val cutoff = nowMs - RECENT_WINDOW_MS
val sorted = sessions.sortedByDescending { it.updatedAtMs ?: 0L }
val recent = mutableListOf<ChatSessionEntry>()
val seen = mutableSetOf<String>()
for (entry in sorted) {
if (aliasKey != null && entry.key == aliasKey) continue
if (!seen.add(entry.key)) continue
if ((entry.updatedAtMs ?: 0L) < cutoff) continue
recent.add(entry)
}
val result = mutableListOf<ChatSessionEntry>()
val included = mutableSetOf<String>()
val mainEntry = sorted.firstOrNull { it.key == mainKey }
if (mainEntry != null) {
result.add(mainEntry)
included.add(mainKey)
} else if (current == mainKey) {
result.add(ChatSessionEntry(key = mainKey, updatedAtMs = null))
included.add(mainKey)
}
for (entry in recent) {
if (included.add(entry.key)) {
result.add(entry)
}
}
if (current.isNotEmpty() && !included.contains(current)) {
result.add(ChatSessionEntry(key = current, updatedAtMs = null))
}
return result
}

View File

@@ -0,0 +1,523 @@
package ai.openclaw.android.voice
import android.Manifest
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.speech.RecognitionListener
import android.speech.RecognizerIntent
import android.speech.SpeechRecognizer
import androidx.core.content.ContextCompat
import java.util.UUID
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
enum class VoiceConversationRole {
User,
Assistant,
}
data class VoiceConversationEntry(
val id: String,
val role: VoiceConversationRole,
val text: String,
val isStreaming: Boolean = false,
)
class MicCaptureManager(
private val context: Context,
private val scope: CoroutineScope,
private val sendToGateway: suspend (String) -> String?,
) {
companion object {
private const val speechMinSessionMs = 30_000L
private const val speechCompleteSilenceMs = 1_500L
private const val speechPossibleSilenceMs = 900L
private const val maxConversationEntries = 40
private const val pendingRunTimeoutMs = 45_000L
}
private data class QueuedUtterance(
val text: String,
)
private val mainHandler = Handler(Looper.getMainLooper())
private val json = Json { ignoreUnknownKeys = true }
private val _micEnabled = MutableStateFlow(false)
val micEnabled: StateFlow<Boolean> = _micEnabled
private val _isListening = MutableStateFlow(false)
val isListening: StateFlow<Boolean> = _isListening
private val _statusText = MutableStateFlow("Mic off")
val statusText: StateFlow<String> = _statusText
private val _liveTranscript = MutableStateFlow<String?>(null)
val liveTranscript: StateFlow<String?> = _liveTranscript
private val _queuedMessages = MutableStateFlow<List<String>>(emptyList())
val queuedMessages: StateFlow<List<String>> = _queuedMessages
private val _conversation = MutableStateFlow<List<VoiceConversationEntry>>(emptyList())
val conversation: StateFlow<List<VoiceConversationEntry>> = _conversation
private val _inputLevel = MutableStateFlow(0f)
val inputLevel: StateFlow<Float> = _inputLevel
private val _isSending = MutableStateFlow(false)
val isSending: StateFlow<Boolean> = _isSending
private val messageQueue = ArrayDeque<QueuedUtterance>()
private val sessionSegments = mutableListOf<String>()
private var lastFinalSegment: String? = null
private var pendingRunId: String? = null
private var pendingAssistantEntryId: String? = null
private var gatewayConnected = false
private var recognizer: SpeechRecognizer? = null
private var restartJob: Job? = null
private var pendingRunTimeoutJob: Job? = null
private var stopRequested = false
fun setMicEnabled(enabled: Boolean) {
if (_micEnabled.value == enabled) return
_micEnabled.value = enabled
if (enabled) {
start()
sendQueuedIfIdle()
} else {
stop()
flushSessionToQueue()
sendQueuedIfIdle()
}
}
fun onGatewayConnectionChanged(connected: Boolean) {
gatewayConnected = connected
if (connected) {
sendQueuedIfIdle()
return
}
if (messageQueue.isNotEmpty()) {
_statusText.value = queuedWaitingStatus()
}
}
fun handleGatewayEvent(event: String, payloadJson: String?) {
if (event != "chat") return
if (payloadJson.isNullOrBlank()) return
val payload =
try {
json.parseToJsonElement(payloadJson).asObjectOrNull()
} catch (_: Throwable) {
null
} ?: return
val runId = pendingRunId ?: return
val eventRunId = payload["runId"].asStringOrNull() ?: return
if (eventRunId != runId) return
when (payload["state"].asStringOrNull()) {
"delta" -> {
val deltaText = parseAssistantText(payload)
if (!deltaText.isNullOrBlank()) {
upsertPendingAssistant(text = deltaText.trim(), isStreaming = true)
}
}
"final" -> {
val finalText = parseAssistantText(payload)?.trim().orEmpty()
if (finalText.isNotEmpty()) {
upsertPendingAssistant(text = finalText, isStreaming = false)
} else if (pendingAssistantEntryId != null) {
updateConversationEntry(pendingAssistantEntryId!!, text = null, isStreaming = false)
}
completePendingTurn()
}
"error" -> {
val errorMessage = payload["errorMessage"].asStringOrNull()?.trim().orEmpty().ifEmpty { "Voice request failed" }
upsertPendingAssistant(text = errorMessage, isStreaming = false)
completePendingTurn()
}
"aborted" -> {
upsertPendingAssistant(text = "Response aborted", isStreaming = false)
completePendingTurn()
}
}
}
private fun start() {
stopRequested = false
if (!SpeechRecognizer.isRecognitionAvailable(context)) {
_statusText.value = "Speech recognizer unavailable"
_micEnabled.value = false
return
}
if (!hasMicPermission()) {
_statusText.value = "Microphone permission required"
_micEnabled.value = false
return
}
mainHandler.post {
try {
if (recognizer == null) {
recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) }
}
startListeningSession()
} catch (err: Throwable) {
_statusText.value = "Start failed: ${err.message ?: err::class.simpleName}"
_micEnabled.value = false
}
}
}
private fun stop() {
stopRequested = true
restartJob?.cancel()
restartJob = null
_isListening.value = false
_statusText.value = if (_isSending.value) "Mic off · sending…" else "Mic off"
_inputLevel.value = 0f
mainHandler.post {
recognizer?.cancel()
recognizer?.destroy()
recognizer = null
}
}
private fun startListeningSession() {
val recognizerInstance = recognizer ?: return
val intent =
Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3)
putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName)
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_MINIMUM_LENGTH_MILLIS, speechMinSessionMs)
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, speechCompleteSilenceMs)
putExtra(
RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS,
speechPossibleSilenceMs,
)
}
_statusText.value =
when {
_isSending.value -> "Listening · sending queued voice"
messageQueue.isNotEmpty() -> "Listening · ${messageQueue.size} queued"
else -> "Listening"
}
_isListening.value = true
recognizerInstance.startListening(intent)
}
private fun scheduleRestart(delayMs: Long = 300L) {
if (stopRequested) return
if (!_micEnabled.value) return
restartJob?.cancel()
restartJob =
scope.launch {
delay(delayMs)
mainHandler.post {
if (stopRequested || !_micEnabled.value) return@post
try {
startListeningSession()
} catch (_: Throwable) {
// retry through onError
}
}
}
}
private fun flushSessionToQueue() {
val message = sessionSegments.joinToString(" ").trim()
sessionSegments.clear()
_liveTranscript.value = null
lastFinalSegment = null
if (message.isEmpty()) return
appendConversation(
role = VoiceConversationRole.User,
text = message,
)
messageQueue.addLast(QueuedUtterance(text = message))
publishQueue()
}
private fun publishQueue() {
_queuedMessages.value = messageQueue.map { it.text }
}
private fun sendQueuedIfIdle() {
if (_isSending.value) return
if (messageQueue.isEmpty()) {
if (_micEnabled.value) {
_statusText.value = "Listening"
} else {
_statusText.value = "Mic off"
}
return
}
if (!gatewayConnected) {
_statusText.value = queuedWaitingStatus()
return
}
val next = messageQueue.first()
_isSending.value = true
pendingRunTimeoutJob?.cancel()
pendingRunTimeoutJob = null
_statusText.value = if (_micEnabled.value) "Listening · sending queued voice" else "Sending queued voice"
scope.launch {
try {
val runId = sendToGateway(next.text)
pendingRunId = runId
if (runId == null) {
pendingRunTimeoutJob?.cancel()
pendingRunTimeoutJob = null
messageQueue.removeFirst()
publishQueue()
_isSending.value = false
pendingAssistantEntryId = null
sendQueuedIfIdle()
} else {
armPendingRunTimeout(runId)
}
} catch (err: Throwable) {
pendingRunTimeoutJob?.cancel()
pendingRunTimeoutJob = null
_isSending.value = false
pendingRunId = null
pendingAssistantEntryId = null
_statusText.value =
if (!gatewayConnected) {
queuedWaitingStatus()
} else {
"Send failed: ${err.message ?: err::class.simpleName}"
}
}
}
}
private fun armPendingRunTimeout(runId: String) {
pendingRunTimeoutJob?.cancel()
pendingRunTimeoutJob =
scope.launch {
delay(pendingRunTimeoutMs)
if (pendingRunId != runId) return@launch
pendingRunId = null
pendingAssistantEntryId = null
_isSending.value = false
_statusText.value =
if (gatewayConnected) {
"Voice reply timed out; retrying queued turn"
} else {
queuedWaitingStatus()
}
sendQueuedIfIdle()
}
}
private fun completePendingTurn() {
pendingRunTimeoutJob?.cancel()
pendingRunTimeoutJob = null
if (messageQueue.isNotEmpty()) {
messageQueue.removeFirst()
publishQueue()
}
pendingRunId = null
pendingAssistantEntryId = null
_isSending.value = false
sendQueuedIfIdle()
}
private fun queuedWaitingStatus(): String {
return "${messageQueue.size} queued · waiting for gateway"
}
private fun appendConversation(
role: VoiceConversationRole,
text: String,
isStreaming: Boolean = false,
): String {
val id = UUID.randomUUID().toString()
_conversation.value =
(_conversation.value + VoiceConversationEntry(id = id, role = role, text = text, isStreaming = isStreaming))
.takeLast(maxConversationEntries)
return id
}
private fun updateConversationEntry(id: String, text: String?, isStreaming: Boolean) {
val current = _conversation.value
_conversation.value =
current.map { entry ->
if (entry.id == id) {
val updatedText = text ?: entry.text
entry.copy(text = updatedText, isStreaming = isStreaming)
} else {
entry
}
}
}
private fun upsertPendingAssistant(text: String, isStreaming: Boolean) {
val currentId = pendingAssistantEntryId
if (currentId == null) {
pendingAssistantEntryId =
appendConversation(
role = VoiceConversationRole.Assistant,
text = text,
isStreaming = isStreaming,
)
return
}
updateConversationEntry(id = currentId, text = text, isStreaming = isStreaming)
}
private fun onFinalTranscript(text: String) {
val trimmed = text.trim()
if (trimmed.isEmpty()) return
_liveTranscript.value = trimmed
if (lastFinalSegment == trimmed) return
lastFinalSegment = trimmed
sessionSegments.add(trimmed)
}
private fun disableMic(status: String) {
stopRequested = true
restartJob?.cancel()
restartJob = null
_micEnabled.value = false
_isListening.value = false
_inputLevel.value = 0f
_statusText.value = status
mainHandler.post {
recognizer?.cancel()
recognizer?.destroy()
recognizer = null
}
}
private fun hasMicPermission(): Boolean {
return (
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
PackageManager.PERMISSION_GRANTED
)
}
private fun parseAssistantText(payload: JsonObject): String? {
val message = payload["message"].asObjectOrNull() ?: return null
if (message["role"].asStringOrNull() != "assistant") return null
val content = message["content"] as? JsonArray ?: return null
val parts =
content.mapNotNull { item ->
val obj = item.asObjectOrNull() ?: return@mapNotNull null
if (obj["type"].asStringOrNull() != "text") return@mapNotNull null
obj["text"].asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() }
}
if (parts.isEmpty()) return null
return parts.joinToString("\n")
}
private val listener =
object : RecognitionListener {
override fun onReadyForSpeech(params: Bundle?) {
_isListening.value = true
}
override fun onBeginningOfSpeech() {}
override fun onRmsChanged(rmsdB: Float) {
val level = ((rmsdB + 2f) / 12f).coerceIn(0f, 1f)
_inputLevel.value = level
}
override fun onBufferReceived(buffer: ByteArray?) {}
override fun onEndOfSpeech() {
_inputLevel.value = 0f
scheduleRestart()
}
override fun onError(error: Int) {
if (stopRequested) return
_isListening.value = false
_inputLevel.value = 0f
val status =
when (error) {
SpeechRecognizer.ERROR_AUDIO -> "Audio error"
SpeechRecognizer.ERROR_CLIENT -> "Client error"
SpeechRecognizer.ERROR_NETWORK -> "Network error"
SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout"
SpeechRecognizer.ERROR_NO_MATCH -> "Listening"
SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy"
SpeechRecognizer.ERROR_SERVER -> "Server error"
SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening"
SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS -> "Microphone permission required"
SpeechRecognizer.ERROR_LANGUAGE_NOT_SUPPORTED -> "Language not supported on this device"
SpeechRecognizer.ERROR_LANGUAGE_UNAVAILABLE -> "Language unavailable on this device"
SpeechRecognizer.ERROR_SERVER_DISCONNECTED -> "Speech service disconnected"
SpeechRecognizer.ERROR_TOO_MANY_REQUESTS -> "Speech requests limited; retrying"
else -> "Speech error ($error)"
}
_statusText.value = status
if (
error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS ||
error == SpeechRecognizer.ERROR_LANGUAGE_NOT_SUPPORTED ||
error == SpeechRecognizer.ERROR_LANGUAGE_UNAVAILABLE
) {
disableMic(status)
return
}
val restartDelayMs =
when (error) {
SpeechRecognizer.ERROR_NO_MATCH,
SpeechRecognizer.ERROR_SPEECH_TIMEOUT,
-> 1_200L
SpeechRecognizer.ERROR_TOO_MANY_REQUESTS -> 2_500L
else -> 600L
}
scheduleRestart(delayMs = restartDelayMs)
}
override fun onResults(results: Bundle?) {
val text = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty().firstOrNull()
if (!text.isNullOrBlank()) {
onFinalTranscript(text)
flushSessionToQueue()
sendQueuedIfIdle()
}
scheduleRestart()
}
override fun onPartialResults(partialResults: Bundle?) {
val text = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty().firstOrNull()
if (!text.isNullOrBlank()) {
_liveTranscript.value = text.trim()
}
}
override fun onEvent(eventType: Int, params: Bundle?) {}
}
}
private fun kotlinx.serialization.json.JsonElement?.asObjectOrNull(): JsonObject? =
this as? JsonObject
private fun kotlinx.serialization.json.JsonElement?.asStringOrNull(): String? =
(this as? JsonPrimitive)?.takeIf { it.isString }?.content

View File

@@ -0,0 +1,98 @@
package ai.openclaw.android.voice
import android.media.MediaDataSource
import kotlin.math.min
internal class StreamingMediaDataSource : MediaDataSource() {
private data class Chunk(val start: Long, val data: ByteArray)
private val lock = Object()
private val chunks = ArrayList<Chunk>()
private var totalSize: Long = 0
private var closed = false
private var finished = false
private var lastReadIndex = 0
fun append(data: ByteArray) {
if (data.isEmpty()) return
synchronized(lock) {
if (closed || finished) return
val chunk = Chunk(totalSize, data)
chunks.add(chunk)
totalSize += data.size.toLong()
lock.notifyAll()
}
}
fun finish() {
synchronized(lock) {
if (closed) return
finished = true
lock.notifyAll()
}
}
fun fail() {
synchronized(lock) {
closed = true
lock.notifyAll()
}
}
override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int {
if (position < 0) return -1
synchronized(lock) {
while (!closed && !finished && position >= totalSize) {
lock.wait()
}
if (closed) return -1
if (position >= totalSize && finished) return -1
val available = (totalSize - position).toInt()
val toRead = min(size, available)
var remaining = toRead
var destOffset = offset
var pos = position
var index = findChunkIndex(pos)
while (remaining > 0 && index < chunks.size) {
val chunk = chunks[index]
val inChunkOffset = (pos - chunk.start).toInt()
if (inChunkOffset >= chunk.data.size) {
index++
continue
}
val copyLen = min(remaining, chunk.data.size - inChunkOffset)
System.arraycopy(chunk.data, inChunkOffset, buffer, destOffset, copyLen)
remaining -= copyLen
destOffset += copyLen
pos += copyLen
if (inChunkOffset + copyLen >= chunk.data.size) {
index++
}
}
return toRead - remaining
}
}
override fun getSize(): Long = -1
override fun close() {
synchronized(lock) {
closed = true
lock.notifyAll()
}
}
private fun findChunkIndex(position: Long): Int {
var index = lastReadIndex
while (index < chunks.size) {
val chunk = chunks[index]
if (position < chunk.start + chunk.data.size) break
index++
}
lastReadIndex = index
return index
}
}

View File

@@ -0,0 +1,191 @@
package ai.openclaw.android.voice
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
private val directiveJson = Json { ignoreUnknownKeys = true }
data class TalkDirective(
val voiceId: String? = null,
val modelId: String? = null,
val speed: Double? = null,
val rateWpm: Int? = null,
val stability: Double? = null,
val similarity: Double? = null,
val style: Double? = null,
val speakerBoost: Boolean? = null,
val seed: Long? = null,
val normalize: String? = null,
val language: String? = null,
val outputFormat: String? = null,
val latencyTier: Int? = null,
val once: Boolean? = null,
)
data class TalkDirectiveParseResult(
val directive: TalkDirective?,
val stripped: String,
val unknownKeys: List<String>,
)
object TalkDirectiveParser {
fun parse(text: String): TalkDirectiveParseResult {
val normalized = text.replace("\r\n", "\n")
val lines = normalized.split("\n").toMutableList()
if (lines.isEmpty()) return TalkDirectiveParseResult(null, text, emptyList())
val firstNonEmpty = lines.indexOfFirst { it.trim().isNotEmpty() }
if (firstNonEmpty == -1) return TalkDirectiveParseResult(null, text, emptyList())
val head = lines[firstNonEmpty].trim()
if (!head.startsWith("{") || !head.endsWith("}")) {
return TalkDirectiveParseResult(null, text, emptyList())
}
val obj = parseJsonObject(head) ?: return TalkDirectiveParseResult(null, text, emptyList())
val speakerBoost =
boolValue(obj, listOf("speaker_boost", "speakerBoost"))
?: boolValue(obj, listOf("no_speaker_boost", "noSpeakerBoost"))?.not()
val directive = TalkDirective(
voiceId = stringValue(obj, listOf("voice", "voice_id", "voiceId")),
modelId = stringValue(obj, listOf("model", "model_id", "modelId")),
speed = doubleValue(obj, listOf("speed")),
rateWpm = intValue(obj, listOf("rate", "wpm")),
stability = doubleValue(obj, listOf("stability")),
similarity = doubleValue(obj, listOf("similarity", "similarity_boost", "similarityBoost")),
style = doubleValue(obj, listOf("style")),
speakerBoost = speakerBoost,
seed = longValue(obj, listOf("seed")),
normalize = stringValue(obj, listOf("normalize", "apply_text_normalization")),
language = stringValue(obj, listOf("lang", "language_code", "language")),
outputFormat = stringValue(obj, listOf("output_format", "format")),
latencyTier = intValue(obj, listOf("latency", "latency_tier", "latencyTier")),
once = boolValue(obj, listOf("once")),
)
val hasDirective = listOf(
directive.voiceId,
directive.modelId,
directive.speed,
directive.rateWpm,
directive.stability,
directive.similarity,
directive.style,
directive.speakerBoost,
directive.seed,
directive.normalize,
directive.language,
directive.outputFormat,
directive.latencyTier,
directive.once,
).any { it != null }
if (!hasDirective) return TalkDirectiveParseResult(null, text, emptyList())
val knownKeys = setOf(
"voice", "voice_id", "voiceid",
"model", "model_id", "modelid",
"speed", "rate", "wpm",
"stability", "similarity", "similarity_boost", "similarityboost",
"style",
"speaker_boost", "speakerboost",
"no_speaker_boost", "nospeakerboost",
"seed",
"normalize", "apply_text_normalization",
"lang", "language_code", "language",
"output_format", "format",
"latency", "latency_tier", "latencytier",
"once",
)
val unknownKeys = obj.keys.filter { !knownKeys.contains(it.lowercase()) }.sorted()
lines.removeAt(firstNonEmpty)
if (firstNonEmpty < lines.size) {
if (lines[firstNonEmpty].trim().isEmpty()) {
lines.removeAt(firstNonEmpty)
}
}
return TalkDirectiveParseResult(directive, lines.joinToString("\n"), unknownKeys)
}
private fun parseJsonObject(line: String): JsonObject? {
return try {
directiveJson.parseToJsonElement(line) as? JsonObject
} catch (_: Throwable) {
null
}
}
private fun stringValue(obj: JsonObject, keys: List<String>): String? {
for (key in keys) {
val value = obj[key].asStringOrNull()?.trim()
if (!value.isNullOrEmpty()) return value
}
return null
}
private fun doubleValue(obj: JsonObject, keys: List<String>): Double? {
for (key in keys) {
val value = obj[key].asDoubleOrNull()
if (value != null) return value
}
return null
}
private fun intValue(obj: JsonObject, keys: List<String>): Int? {
for (key in keys) {
val value = obj[key].asIntOrNull()
if (value != null) return value
}
return null
}
private fun longValue(obj: JsonObject, keys: List<String>): Long? {
for (key in keys) {
val value = obj[key].asLongOrNull()
if (value != null) return value
}
return null
}
private fun boolValue(obj: JsonObject, keys: List<String>): Boolean? {
for (key in keys) {
val value = obj[key].asBooleanOrNull()
if (value != null) return value
}
return null
}
}
private fun JsonElement?.asStringOrNull(): String? =
(this as? JsonPrimitive)?.takeIf { it.isString }?.content
private fun JsonElement?.asDoubleOrNull(): Double? {
val primitive = this as? JsonPrimitive ?: return null
return primitive.content.toDoubleOrNull()
}
private fun JsonElement?.asIntOrNull(): Int? {
val primitive = this as? JsonPrimitive ?: return null
return primitive.content.toIntOrNull()
}
private fun JsonElement?.asLongOrNull(): Long? {
val primitive = this as? JsonPrimitive ?: return null
return primitive.content.toLongOrNull()
}
private fun JsonElement?.asBooleanOrNull(): Boolean? {
val primitive = this as? JsonPrimitive ?: return null
val content = primitive.content.trim().lowercase()
return when (content) {
"true", "yes", "1" -> true
"false", "no", "0" -> false
else -> null
}
}

View File

@@ -0,0 +1,40 @@
package ai.openclaw.android.voice
object VoiceWakeCommandExtractor {
fun extractCommand(text: String, triggerWords: List<String>): String? {
val raw = text.trim()
if (raw.isEmpty()) return null
val triggers =
triggerWords
.map { it.trim().lowercase() }
.filter { it.isNotEmpty() }
.distinct()
if (triggers.isEmpty()) return null
val alternation = triggers.joinToString("|") { Regex.escape(it) }
// Match: "<anything> <trigger><punct/space> <command>"
val regex = Regex("(?i)(?:^|\\s)($alternation)\\b[\\s\\p{Punct}]*([\\s\\S]+)$")
val match = regex.find(raw) ?: return null
val extracted = match.groupValues.getOrNull(2)?.trim().orEmpty()
if (extracted.isEmpty()) return null
val cleaned = extracted.trimStart { it.isWhitespace() || it.isPunctuation() }.trim()
if (cleaned.isEmpty()) return null
return cleaned
}
}
private fun Char.isPunctuation(): Boolean {
return when (Character.getType(this)) {
Character.CONNECTOR_PUNCTUATION.toInt(),
Character.DASH_PUNCTUATION.toInt(),
Character.START_PUNCTUATION.toInt(),
Character.END_PUNCTUATION.toInt(),
Character.INITIAL_QUOTE_PUNCTUATION.toInt(),
Character.FINAL_QUOTE_PUNCTUATION.toInt(),
Character.OTHER_PUNCTUATION.toInt(),
-> true
else -> false
}
}

View File

@@ -0,0 +1,173 @@
package ai.openclaw.android.voice
import android.content.Context
import android.content.Intent
import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.speech.RecognitionListener
import android.speech.RecognizerIntent
import android.speech.SpeechRecognizer
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch
class VoiceWakeManager(
private val context: Context,
private val scope: CoroutineScope,
private val onCommand: suspend (String) -> Unit,
) {
private val mainHandler = Handler(Looper.getMainLooper())
private val _isListening = MutableStateFlow(false)
val isListening: StateFlow<Boolean> = _isListening
private val _statusText = MutableStateFlow("Off")
val statusText: StateFlow<String> = _statusText
var triggerWords: List<String> = emptyList()
private set
private var recognizer: SpeechRecognizer? = null
private var restartJob: Job? = null
private var lastDispatched: String? = null
private var stopRequested = false
fun setTriggerWords(words: List<String>) {
triggerWords = words
}
fun start() {
mainHandler.post {
if (_isListening.value) return@post
stopRequested = false
if (!SpeechRecognizer.isRecognitionAvailable(context)) {
_isListening.value = false
_statusText.value = "Speech recognizer unavailable"
return@post
}
try {
recognizer?.destroy()
recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) }
startListeningInternal()
} catch (err: Throwable) {
_isListening.value = false
_statusText.value = "Start failed: ${err.message ?: err::class.simpleName}"
}
}
}
fun stop(statusText: String = "Off") {
stopRequested = true
restartJob?.cancel()
restartJob = null
mainHandler.post {
_isListening.value = false
_statusText.value = statusText
recognizer?.cancel()
recognizer?.destroy()
recognizer = null
}
}
private fun startListeningInternal() {
val r = recognizer ?: return
val intent =
Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3)
putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName)
}
_statusText.value = "Listening"
_isListening.value = true
r.startListening(intent)
}
private fun scheduleRestart(delayMs: Long = 350) {
if (stopRequested) return
restartJob?.cancel()
restartJob =
scope.launch {
delay(delayMs)
mainHandler.post {
if (stopRequested) return@post
try {
recognizer?.cancel()
startListeningInternal()
} catch (_: Throwable) {
// Will be picked up by onError and retry again.
}
}
}
}
private fun handleTranscription(text: String) {
val command = VoiceWakeCommandExtractor.extractCommand(text, triggerWords) ?: return
if (command == lastDispatched) return
lastDispatched = command
scope.launch { onCommand(command) }
_statusText.value = "Triggered"
scheduleRestart(delayMs = 650)
}
private val listener =
object : RecognitionListener {
override fun onReadyForSpeech(params: Bundle?) {
_statusText.value = "Listening"
}
override fun onBeginningOfSpeech() {}
override fun onRmsChanged(rmsdB: Float) {}
override fun onBufferReceived(buffer: ByteArray?) {}
override fun onEndOfSpeech() {
scheduleRestart()
}
override fun onError(error: Int) {
if (stopRequested) return
_isListening.value = false
if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) {
_statusText.value = "Microphone permission required"
return
}
_statusText.value =
when (error) {
SpeechRecognizer.ERROR_AUDIO -> "Audio error"
SpeechRecognizer.ERROR_CLIENT -> "Client error"
SpeechRecognizer.ERROR_NETWORK -> "Network error"
SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout"
SpeechRecognizer.ERROR_NO_MATCH -> "Listening"
SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy"
SpeechRecognizer.ERROR_SERVER -> "Server error"
SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening"
else -> "Speech error ($error)"
}
scheduleRestart(delayMs = 600)
}
override fun onResults(results: Bundle?) {
val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
list.firstOrNull()?.let(::handleTranscription)
scheduleRestart()
}
override fun onPartialResults(partialResults: Bundle?) {
val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
list.firstOrNull()?.let(::handleTranscription)
}
override fun onEvent(eventType: Int, params: Bundle?) {}
}
}

View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@color/ic_launcher_background" />
<foreground android:drawable="@mipmap/ic_launcher_foreground" />
<monochrome android:drawable="@mipmap/ic_launcher_foreground" />
</adaptive-icon>

View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
<background android:drawable="@color/ic_launcher_background" />
<foreground android:drawable="@mipmap/ic_launcher_foreground" />
<monochrome android:drawable="@mipmap/ic_launcher_foreground" />
</adaptive-icon>

Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 21 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 73 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 155 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 60 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 270 KiB

View File

@@ -0,0 +1,3 @@
<resources>
<color name="ic_launcher_background">#0A0A0A</color>
</resources>

View File

@@ -0,0 +1,3 @@
<resources>
<string name="app_name">OpenClaw Node</string>
</resources>

View File

@@ -0,0 +1,7 @@
<resources>
<style name="Theme.OpenClawNode" parent="Theme.Material3.DayNight.NoActionBar">
<item name="android:statusBarColor">@android:color/transparent</item>
<item name="android:navigationBarColor">@android:color/transparent</item>
<item name="android:windowLightStatusBar">false</item>
</style>
</resources>

View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<full-backup-content>
<include domain="file" path="." />
</full-backup-content>

View File

@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<data-extraction-rules>
<cloud-backup>
<include domain="file" path="." />
</cloud-backup>
<device-transfer>
<include domain="file" path="." />
</device-transfer>
</data-extraction-rules>

Some files were not shown because too many files have changed in this diff Show More