Include full contents of all nested repositories
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
156
openclaw/apps/android/app/build.gradle.kts
Normal file
@@ -0,0 +1,156 @@
|
||||
import com.android.build.api.variant.impl.VariantOutputImpl
|
||||
|
||||
plugins {
|
||||
id("com.android.application")
|
||||
id("org.jetbrains.kotlin.plugin.compose")
|
||||
id("org.jetbrains.kotlin.plugin.serialization")
|
||||
}
|
||||
|
||||
android {
|
||||
namespace = "ai.openclaw.android"
|
||||
compileSdk = 36
|
||||
|
||||
sourceSets {
|
||||
getByName("main") {
|
||||
assets.directories.add("../../shared/OpenClawKit/Sources/OpenClawKit/Resources")
|
||||
}
|
||||
}
|
||||
|
||||
defaultConfig {
|
||||
applicationId = "ai.openclaw.android"
|
||||
minSdk = 31
|
||||
targetSdk = 36
|
||||
versionCode = 202602260
|
||||
versionName = "2026.2.26"
|
||||
ndk {
|
||||
// Support all major ABIs — native libs are tiny (~47 KB per ABI)
|
||||
abiFilters += listOf("armeabi-v7a", "arm64-v8a", "x86", "x86_64")
|
||||
}
|
||||
}
|
||||
|
||||
buildTypes {
|
||||
release {
|
||||
isMinifyEnabled = true
|
||||
isShrinkResources = true
|
||||
proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
|
||||
}
|
||||
debug {
|
||||
isMinifyEnabled = false
|
||||
}
|
||||
}
|
||||
|
||||
buildFeatures {
|
||||
compose = true
|
||||
buildConfig = true
|
||||
}
|
||||
|
||||
compileOptions {
|
||||
sourceCompatibility = JavaVersion.VERSION_17
|
||||
targetCompatibility = JavaVersion.VERSION_17
|
||||
}
|
||||
|
||||
packaging {
|
||||
resources {
|
||||
excludes += setOf(
|
||||
"/META-INF/{AL2.0,LGPL2.1}",
|
||||
"/META-INF/*.version",
|
||||
"/META-INF/LICENSE*.txt",
|
||||
"DebugProbesKt.bin",
|
||||
"kotlin-tooling-metadata.json",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
lint {
|
||||
disable += setOf(
|
||||
"GradleDependency",
|
||||
"IconLauncherShape",
|
||||
"NewerVersionAvailable",
|
||||
)
|
||||
warningsAsErrors = true
|
||||
}
|
||||
|
||||
testOptions {
|
||||
unitTests.isIncludeAndroidResources = true
|
||||
}
|
||||
}
|
||||
|
||||
androidComponents {
|
||||
onVariants { variant ->
|
||||
variant.outputs
|
||||
.filterIsInstance<VariantOutputImpl>()
|
||||
.forEach { output ->
|
||||
val versionName = output.versionName.orNull ?: "0"
|
||||
val buildType = variant.buildType
|
||||
|
||||
val outputFileName = "openclaw-${versionName}-${buildType}.apk"
|
||||
output.outputFileName = outputFileName
|
||||
}
|
||||
}
|
||||
}
|
||||
kotlin {
|
||||
compilerOptions {
|
||||
jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17)
|
||||
allWarningsAsErrors.set(true)
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
val composeBom = platform("androidx.compose:compose-bom:2026.02.00")
|
||||
implementation(composeBom)
|
||||
androidTestImplementation(composeBom)
|
||||
|
||||
implementation("androidx.core:core-ktx:1.17.0")
|
||||
implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.10.0")
|
||||
implementation("androidx.activity:activity-compose:1.12.2")
|
||||
implementation("androidx.webkit:webkit:1.15.0")
|
||||
|
||||
implementation("androidx.compose.ui:ui")
|
||||
implementation("androidx.compose.ui:ui-tooling-preview")
|
||||
implementation("androidx.compose.material3:material3")
|
||||
// material-icons-extended pulled in full icon set (~20 MB DEX). Only ~18 icons used.
|
||||
// R8 will tree-shake unused icons when minify is enabled on release builds.
|
||||
implementation("androidx.compose.material:material-icons-extended")
|
||||
implementation("androidx.navigation:navigation-compose:2.9.7")
|
||||
|
||||
debugImplementation("androidx.compose.ui:ui-tooling")
|
||||
|
||||
// Material Components (XML theme + resources)
|
||||
implementation("com.google.android.material:material:1.13.0")
|
||||
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.10.2")
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.10.0")
|
||||
|
||||
implementation("androidx.security:security-crypto:1.1.0")
|
||||
implementation("androidx.exifinterface:exifinterface:1.4.2")
|
||||
implementation("com.squareup.okhttp3:okhttp:5.3.2")
|
||||
implementation("org.bouncycastle:bcprov-jdk18on:1.83")
|
||||
implementation("org.commonmark:commonmark:0.27.1")
|
||||
implementation("org.commonmark:commonmark-ext-autolink:0.27.1")
|
||||
implementation("org.commonmark:commonmark-ext-gfm-strikethrough:0.27.1")
|
||||
implementation("org.commonmark:commonmark-ext-gfm-tables:0.27.1")
|
||||
implementation("org.commonmark:commonmark-ext-task-list-items:0.27.1")
|
||||
|
||||
// CameraX (for node.invoke camera.* parity)
|
||||
implementation("androidx.camera:camera-core:1.5.2")
|
||||
implementation("androidx.camera:camera-camera2:1.5.2")
|
||||
implementation("androidx.camera:camera-lifecycle:1.5.2")
|
||||
implementation("androidx.camera:camera-video:1.5.2")
|
||||
implementation("androidx.camera:camera-view:1.5.2")
|
||||
implementation("com.journeyapps:zxing-android-embedded:4.3.0")
|
||||
|
||||
// Unicast DNS-SD (Wide-Area Bonjour) for tailnet discovery domains.
|
||||
implementation("dnsjava:dnsjava:3.6.4")
|
||||
|
||||
testImplementation("junit:junit:4.13.2")
|
||||
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
|
||||
testImplementation("io.kotest:kotest-runner-junit5-jvm:6.1.3")
|
||||
testImplementation("io.kotest:kotest-assertions-core-jvm:6.1.3")
|
||||
testImplementation("com.squareup.okhttp3:mockwebserver:5.3.2")
|
||||
testImplementation("org.robolectric:robolectric:4.16.1")
|
||||
testRuntimeOnly("org.junit.vintage:junit-vintage-engine:6.0.2")
|
||||
}
|
||||
|
||||
tasks.withType<Test>().configureEach {
|
||||
useJUnitPlatform()
|
||||
}
|
||||
28
openclaw/apps/android/app/proguard-rules.pro
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
# ── App classes ───────────────────────────────────────────────────
|
||||
-keep class ai.openclaw.android.** { *; }
|
||||
|
||||
# ── Bouncy Castle ─────────────────────────────────────────────────
|
||||
-keep class org.bouncycastle.** { *; }
|
||||
-dontwarn org.bouncycastle.**
|
||||
|
||||
# ── CameraX ───────────────────────────────────────────────────────
|
||||
-keep class androidx.camera.** { *; }
|
||||
|
||||
# ── kotlinx.serialization ────────────────────────────────────────
|
||||
-keep class kotlinx.serialization.** { *; }
|
||||
-keepclassmembers class * {
|
||||
@kotlinx.serialization.Serializable *;
|
||||
}
|
||||
-keepattributes *Annotation*, InnerClasses
|
||||
|
||||
# ── OkHttp ────────────────────────────────────────────────────────
|
||||
-dontwarn okhttp3.**
|
||||
-dontwarn okio.**
|
||||
-keep class okhttp3.internal.platform.** { *; }
|
||||
|
||||
# ── Misc suppressions ────────────────────────────────────────────
|
||||
-dontwarn com.sun.jna.**
|
||||
-dontwarn javax.naming.**
|
||||
-dontwarn lombok.Generated
|
||||
-dontwarn org.slf4j.impl.StaticLoggerBinder
|
||||
-dontwarn sun.net.spi.nameservice.NameServiceDescriptor
|
||||
74
openclaw/apps/android/app/src/main/AndroidManifest.xml
Normal file
@@ -0,0 +1,74 @@
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
|
||||
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
|
||||
<uses-permission
|
||||
android:name="android.permission.NEARBY_WIFI_DEVICES"
|
||||
android:usesPermissionFlags="neverForLocation" />
|
||||
<uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />
|
||||
<uses-permission android:name="android.permission.ACCESS_COARSE_LOCATION" />
|
||||
<uses-permission android:name="android.permission.ACCESS_BACKGROUND_LOCATION" />
|
||||
<uses-permission android:name="android.permission.CAMERA" />
|
||||
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||
<uses-permission android:name="android.permission.SEND_SMS" />
|
||||
<uses-permission android:name="android.permission.REQUEST_INSTALL_PACKAGES" />
|
||||
<uses-feature
|
||||
android:name="android.hardware.camera"
|
||||
android:required="false" />
|
||||
<uses-feature
|
||||
android:name="android.hardware.telephony"
|
||||
android:required="false" />
|
||||
|
||||
<application
|
||||
android:name=".NodeApp"
|
||||
android:allowBackup="true"
|
||||
android:dataExtractionRules="@xml/data_extraction_rules"
|
||||
android:fullBackupContent="@xml/backup_rules"
|
||||
android:icon="@mipmap/ic_launcher"
|
||||
android:roundIcon="@mipmap/ic_launcher_round"
|
||||
android:label="@string/app_name"
|
||||
android:supportsRtl="true"
|
||||
android:networkSecurityConfig="@xml/network_security_config"
|
||||
android:theme="@style/Theme.OpenClawNode">
|
||||
<service
|
||||
android:name=".NodeForegroundService"
|
||||
android:exported="false"
|
||||
android:foregroundServiceType="dataSync|microphone|mediaProjection" />
|
||||
<service
|
||||
android:name=".node.DeviceNotificationListenerService"
|
||||
android:label="@string/app_name"
|
||||
android:permission="android.permission.BIND_NOTIFICATION_LISTENER_SERVICE"
|
||||
android:exported="false">
|
||||
<intent-filter>
|
||||
<action android:name="android.service.notification.NotificationListenerService" />
|
||||
</intent-filter>
|
||||
</service>
|
||||
<provider
|
||||
android:name="androidx.core.content.FileProvider"
|
||||
android:authorities="${applicationId}.fileprovider"
|
||||
android:exported="false"
|
||||
android:grantUriPermissions="true">
|
||||
<meta-data
|
||||
android:name="android.support.FILE_PROVIDER_PATHS"
|
||||
android:resource="@xml/file_paths" />
|
||||
</provider>
|
||||
<activity
|
||||
android:name=".MainActivity"
|
||||
android:exported="true"
|
||||
android:windowSoftInputMode="adjustResize"
|
||||
android:configChanges="orientation|screenSize|screenLayout|smallestScreenSize|uiMode|density|keyboard|keyboardHidden|navigation">
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
|
||||
<receiver
|
||||
android:name=".InstallResultReceiver"
|
||||
android:exported="false" />
|
||||
</application>
|
||||
</manifest>
|
||||
@@ -0,0 +1,14 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
enum class CameraHudKind {
|
||||
Photo,
|
||||
Recording,
|
||||
Success,
|
||||
Error,
|
||||
}
|
||||
|
||||
data class CameraHudState(
|
||||
val token: Long,
|
||||
val kind: CameraHudKind,
|
||||
val message: String,
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
import android.content.Context
|
||||
import android.os.Build
|
||||
import android.provider.Settings
|
||||
|
||||
object DeviceNames {
|
||||
fun bestDefaultNodeName(context: Context): String {
|
||||
val deviceName =
|
||||
runCatching {
|
||||
Settings.Global.getString(context.contentResolver, "device_name")
|
||||
}
|
||||
.getOrNull()
|
||||
?.trim()
|
||||
.orEmpty()
|
||||
|
||||
if (deviceName.isNotEmpty()) return deviceName
|
||||
|
||||
val model =
|
||||
listOfNotNull(Build.MANUFACTURER?.takeIf { it.isNotBlank() }, Build.MODEL?.takeIf { it.isNotBlank() })
|
||||
.joinToString(" ")
|
||||
.trim()
|
||||
|
||||
return model.ifEmpty { "Android Node" }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
import android.content.BroadcastReceiver
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageInstaller
|
||||
import android.util.Log
|
||||
|
||||
class InstallResultReceiver : BroadcastReceiver() {
|
||||
override fun onReceive(context: Context, intent: Intent) {
|
||||
val status = intent.getIntExtra(PackageInstaller.EXTRA_STATUS, PackageInstaller.STATUS_FAILURE)
|
||||
val message = intent.getStringExtra(PackageInstaller.EXTRA_STATUS_MESSAGE)
|
||||
|
||||
when (status) {
|
||||
PackageInstaller.STATUS_PENDING_USER_ACTION -> {
|
||||
// System needs user confirmation — launch the confirmation activity
|
||||
@Suppress("DEPRECATION")
|
||||
val confirmIntent = intent.getParcelableExtra<Intent>(Intent.EXTRA_INTENT)
|
||||
if (confirmIntent != null) {
|
||||
confirmIntent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
|
||||
context.startActivity(confirmIntent)
|
||||
Log.w("openclaw", "app.update: user confirmation requested, launching install dialog")
|
||||
}
|
||||
}
|
||||
PackageInstaller.STATUS_SUCCESS -> {
|
||||
Log.w("openclaw", "app.update: install SUCCESS")
|
||||
}
|
||||
else -> {
|
||||
Log.e("openclaw", "app.update: install FAILED status=$status message=$message")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
enum class LocationMode(val rawValue: String) {
|
||||
Off("off"),
|
||||
WhileUsing("whileUsing"),
|
||||
Always("always"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
fun fromRawValue(raw: String?): LocationMode {
|
||||
val normalized = raw?.trim()?.lowercase()
|
||||
return entries.firstOrNull { it.rawValue.lowercase() == normalized } ?: Off
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
import android.os.Bundle
|
||||
import android.view.WindowManager
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.compose.setContent
|
||||
import androidx.activity.viewModels
|
||||
import androidx.core.view.WindowCompat
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.lifecycleScope
|
||||
import androidx.lifecycle.repeatOnLifecycle
|
||||
import ai.openclaw.android.ui.RootScreen
|
||||
import ai.openclaw.android.ui.OpenClawTheme
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
class MainActivity : ComponentActivity() {
|
||||
private val viewModel: MainViewModel by viewModels()
|
||||
private lateinit var permissionRequester: PermissionRequester
|
||||
private lateinit var screenCaptureRequester: ScreenCaptureRequester
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
WindowCompat.setDecorFitsSystemWindows(window, false)
|
||||
permissionRequester = PermissionRequester(this)
|
||||
screenCaptureRequester = ScreenCaptureRequester(this)
|
||||
viewModel.camera.attachLifecycleOwner(this)
|
||||
viewModel.camera.attachPermissionRequester(permissionRequester)
|
||||
viewModel.sms.attachPermissionRequester(permissionRequester)
|
||||
viewModel.screenRecorder.attachScreenCaptureRequester(screenCaptureRequester)
|
||||
viewModel.screenRecorder.attachPermissionRequester(permissionRequester)
|
||||
|
||||
lifecycleScope.launch {
|
||||
repeatOnLifecycle(Lifecycle.State.STARTED) {
|
||||
viewModel.preventSleep.collect { enabled ->
|
||||
if (enabled) {
|
||||
window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
|
||||
} else {
|
||||
window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setContent {
|
||||
OpenClawTheme {
|
||||
Surface(modifier = Modifier) {
|
||||
RootScreen(viewModel = viewModel)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keep startup path lean: start foreground service after first frame.
|
||||
window.decorView.post { NodeForegroundService.start(this) }
|
||||
}
|
||||
|
||||
override fun onStart() {
|
||||
super.onStart()
|
||||
viewModel.setForeground(true)
|
||||
}
|
||||
|
||||
override fun onStop() {
|
||||
viewModel.setForeground(false)
|
||||
super.onStop()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
import android.app.Application
|
||||
import androidx.lifecycle.AndroidViewModel
|
||||
import ai.openclaw.android.gateway.GatewayEndpoint
|
||||
import ai.openclaw.android.chat.OutgoingAttachment
|
||||
import ai.openclaw.android.node.CameraCaptureManager
|
||||
import ai.openclaw.android.node.CanvasController
|
||||
import ai.openclaw.android.node.ScreenRecordManager
|
||||
import ai.openclaw.android.node.SmsManager
|
||||
import ai.openclaw.android.voice.VoiceConversationEntry
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
|
||||
class MainViewModel(app: Application) : AndroidViewModel(app) {
|
||||
private val runtime: NodeRuntime = (app as NodeApp).runtime
|
||||
|
||||
val canvas: CanvasController = runtime.canvas
|
||||
val canvasCurrentUrl: StateFlow<String?> = runtime.canvas.currentUrl
|
||||
val canvasA2uiHydrated: StateFlow<Boolean> = runtime.canvasA2uiHydrated
|
||||
val canvasRehydratePending: StateFlow<Boolean> = runtime.canvasRehydratePending
|
||||
val canvasRehydrateErrorText: StateFlow<String?> = runtime.canvasRehydrateErrorText
|
||||
val camera: CameraCaptureManager = runtime.camera
|
||||
val screenRecorder: ScreenRecordManager = runtime.screenRecorder
|
||||
val sms: SmsManager = runtime.sms
|
||||
|
||||
val gateways: StateFlow<List<GatewayEndpoint>> = runtime.gateways
|
||||
val discoveryStatusText: StateFlow<String> = runtime.discoveryStatusText
|
||||
|
||||
val isConnected: StateFlow<Boolean> = runtime.isConnected
|
||||
val isNodeConnected: StateFlow<Boolean> = runtime.nodeConnected
|
||||
val statusText: StateFlow<String> = runtime.statusText
|
||||
val serverName: StateFlow<String?> = runtime.serverName
|
||||
val remoteAddress: StateFlow<String?> = runtime.remoteAddress
|
||||
val pendingGatewayTrust: StateFlow<NodeRuntime.GatewayTrustPrompt?> = runtime.pendingGatewayTrust
|
||||
val isForeground: StateFlow<Boolean> = runtime.isForeground
|
||||
val seamColorArgb: StateFlow<Long> = runtime.seamColorArgb
|
||||
val mainSessionKey: StateFlow<String> = runtime.mainSessionKey
|
||||
|
||||
val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
|
||||
val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
|
||||
val screenRecordActive: StateFlow<Boolean> = runtime.screenRecordActive
|
||||
|
||||
val instanceId: StateFlow<String> = runtime.instanceId
|
||||
val displayName: StateFlow<String> = runtime.displayName
|
||||
val cameraEnabled: StateFlow<Boolean> = runtime.cameraEnabled
|
||||
val locationMode: StateFlow<LocationMode> = runtime.locationMode
|
||||
val locationPreciseEnabled: StateFlow<Boolean> = runtime.locationPreciseEnabled
|
||||
val preventSleep: StateFlow<Boolean> = runtime.preventSleep
|
||||
val micEnabled: StateFlow<Boolean> = runtime.micEnabled
|
||||
val micStatusText: StateFlow<String> = runtime.micStatusText
|
||||
val micLiveTranscript: StateFlow<String?> = runtime.micLiveTranscript
|
||||
val micIsListening: StateFlow<Boolean> = runtime.micIsListening
|
||||
val micQueuedMessages: StateFlow<List<String>> = runtime.micQueuedMessages
|
||||
val micConversation: StateFlow<List<VoiceConversationEntry>> = runtime.micConversation
|
||||
val micInputLevel: StateFlow<Float> = runtime.micInputLevel
|
||||
val micIsSending: StateFlow<Boolean> = runtime.micIsSending
|
||||
val manualEnabled: StateFlow<Boolean> = runtime.manualEnabled
|
||||
val manualHost: StateFlow<String> = runtime.manualHost
|
||||
val manualPort: StateFlow<Int> = runtime.manualPort
|
||||
val manualTls: StateFlow<Boolean> = runtime.manualTls
|
||||
val gatewayToken: StateFlow<String> = runtime.gatewayToken
|
||||
val onboardingCompleted: StateFlow<Boolean> = runtime.onboardingCompleted
|
||||
val canvasDebugStatusEnabled: StateFlow<Boolean> = runtime.canvasDebugStatusEnabled
|
||||
|
||||
val chatSessionKey: StateFlow<String> = runtime.chatSessionKey
|
||||
val chatSessionId: StateFlow<String?> = runtime.chatSessionId
|
||||
val chatMessages = runtime.chatMessages
|
||||
val chatError: StateFlow<String?> = runtime.chatError
|
||||
val chatHealthOk: StateFlow<Boolean> = runtime.chatHealthOk
|
||||
val chatThinkingLevel: StateFlow<String> = runtime.chatThinkingLevel
|
||||
val chatStreamingAssistantText: StateFlow<String?> = runtime.chatStreamingAssistantText
|
||||
val chatPendingToolCalls = runtime.chatPendingToolCalls
|
||||
val chatSessions = runtime.chatSessions
|
||||
val pendingRunCount: StateFlow<Int> = runtime.pendingRunCount
|
||||
|
||||
fun setForeground(value: Boolean) {
|
||||
runtime.setForeground(value)
|
||||
}
|
||||
|
||||
fun setDisplayName(value: String) {
|
||||
runtime.setDisplayName(value)
|
||||
}
|
||||
|
||||
fun setCameraEnabled(value: Boolean) {
|
||||
runtime.setCameraEnabled(value)
|
||||
}
|
||||
|
||||
fun setLocationMode(mode: LocationMode) {
|
||||
runtime.setLocationMode(mode)
|
||||
}
|
||||
|
||||
fun setLocationPreciseEnabled(value: Boolean) {
|
||||
runtime.setLocationPreciseEnabled(value)
|
||||
}
|
||||
|
||||
fun setPreventSleep(value: Boolean) {
|
||||
runtime.setPreventSleep(value)
|
||||
}
|
||||
|
||||
fun setManualEnabled(value: Boolean) {
|
||||
runtime.setManualEnabled(value)
|
||||
}
|
||||
|
||||
fun setManualHost(value: String) {
|
||||
runtime.setManualHost(value)
|
||||
}
|
||||
|
||||
fun setManualPort(value: Int) {
|
||||
runtime.setManualPort(value)
|
||||
}
|
||||
|
||||
fun setManualTls(value: Boolean) {
|
||||
runtime.setManualTls(value)
|
||||
}
|
||||
|
||||
fun setGatewayToken(value: String) {
|
||||
runtime.setGatewayToken(value)
|
||||
}
|
||||
|
||||
fun setGatewayPassword(value: String) {
|
||||
runtime.setGatewayPassword(value)
|
||||
}
|
||||
|
||||
fun setOnboardingCompleted(value: Boolean) {
|
||||
runtime.setOnboardingCompleted(value)
|
||||
}
|
||||
|
||||
fun setCanvasDebugStatusEnabled(value: Boolean) {
|
||||
runtime.setCanvasDebugStatusEnabled(value)
|
||||
}
|
||||
|
||||
fun setMicEnabled(enabled: Boolean) {
|
||||
runtime.setMicEnabled(enabled)
|
||||
}
|
||||
|
||||
fun refreshGatewayConnection() {
|
||||
runtime.refreshGatewayConnection()
|
||||
}
|
||||
|
||||
fun connect(endpoint: GatewayEndpoint) {
|
||||
runtime.connect(endpoint)
|
||||
}
|
||||
|
||||
fun connectManual() {
|
||||
runtime.connectManual()
|
||||
}
|
||||
|
||||
fun disconnect() {
|
||||
runtime.disconnect()
|
||||
}
|
||||
|
||||
fun acceptGatewayTrustPrompt() {
|
||||
runtime.acceptGatewayTrustPrompt()
|
||||
}
|
||||
|
||||
fun declineGatewayTrustPrompt() {
|
||||
runtime.declineGatewayTrustPrompt()
|
||||
}
|
||||
|
||||
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
|
||||
runtime.handleCanvasA2UIActionFromWebView(payloadJson)
|
||||
}
|
||||
|
||||
fun requestCanvasRehydrate(source: String = "screen_tab") {
|
||||
runtime.requestCanvasRehydrate(source = source, force = true)
|
||||
}
|
||||
|
||||
fun loadChat(sessionKey: String) {
|
||||
runtime.loadChat(sessionKey)
|
||||
}
|
||||
|
||||
fun refreshChat() {
|
||||
runtime.refreshChat()
|
||||
}
|
||||
|
||||
fun refreshChatSessions(limit: Int? = null) {
|
||||
runtime.refreshChatSessions(limit = limit)
|
||||
}
|
||||
|
||||
fun setChatThinkingLevel(level: String) {
|
||||
runtime.setChatThinkingLevel(level)
|
||||
}
|
||||
|
||||
fun switchChatSession(sessionKey: String) {
|
||||
runtime.switchChatSession(sessionKey)
|
||||
}
|
||||
|
||||
fun abortChat() {
|
||||
runtime.abortChat()
|
||||
}
|
||||
|
||||
fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
|
||||
runtime.sendChat(message = message, thinking = thinking, attachments = attachments)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
import android.app.Application
|
||||
import android.os.StrictMode
|
||||
|
||||
class NodeApp : Application() {
|
||||
val runtime: NodeRuntime by lazy { NodeRuntime(this) }
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
if (BuildConfig.DEBUG) {
|
||||
StrictMode.setThreadPolicy(
|
||||
StrictMode.ThreadPolicy.Builder()
|
||||
.detectAll()
|
||||
.penaltyLog()
|
||||
.build(),
|
||||
)
|
||||
StrictMode.setVmPolicy(
|
||||
StrictMode.VmPolicy.Builder()
|
||||
.detectAll()
|
||||
.penaltyLog()
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
import android.app.Notification
|
||||
import android.app.NotificationChannel
|
||||
import android.app.NotificationManager
|
||||
import android.app.Service
|
||||
import android.app.PendingIntent
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.content.pm.ServiceInfo
|
||||
import androidx.core.app.NotificationCompat
|
||||
import androidx.core.content.ContextCompat
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.flow.combine
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
class NodeForegroundService : Service() {
|
||||
private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
|
||||
private var notificationJob: Job? = null
|
||||
private var lastRequiresMic = false
|
||||
private var didStartForeground = false
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
ensureChannel()
|
||||
val initial = buildNotification(title = "OpenClaw Node", text = "Starting…")
|
||||
startForegroundWithTypes(notification = initial, requiresMic = false)
|
||||
|
||||
val runtime = (application as NodeApp).runtime
|
||||
notificationJob =
|
||||
scope.launch {
|
||||
combine(
|
||||
runtime.statusText,
|
||||
runtime.serverName,
|
||||
runtime.isConnected,
|
||||
runtime.micEnabled,
|
||||
runtime.micIsListening,
|
||||
) { status, server, connected, micEnabled, micListening ->
|
||||
Quint(status, server, connected, micEnabled, micListening)
|
||||
}.collect { (status, server, connected, micEnabled, micListening) ->
|
||||
val title = if (connected) "OpenClaw Node · Connected" else "OpenClaw Node"
|
||||
val micSuffix =
|
||||
if (micEnabled) {
|
||||
if (micListening) " · Mic: Listening" else " · Mic: Pending"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
val text = (server?.let { "$status · $it" } ?: status) + micSuffix
|
||||
|
||||
val requiresMic =
|
||||
micEnabled && hasRecordAudioPermission()
|
||||
startForegroundWithTypes(
|
||||
notification = buildNotification(title = title, text = text),
|
||||
requiresMic = requiresMic,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
||||
when (intent?.action) {
|
||||
ACTION_STOP -> {
|
||||
(application as NodeApp).runtime.disconnect()
|
||||
stopSelf()
|
||||
return START_NOT_STICKY
|
||||
}
|
||||
}
|
||||
// Keep running; connection is managed by NodeRuntime (auto-reconnect + manual).
|
||||
return START_STICKY
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
notificationJob?.cancel()
|
||||
scope.cancel()
|
||||
super.onDestroy()
|
||||
}
|
||||
|
||||
override fun onBind(intent: Intent?) = null
|
||||
|
||||
private fun ensureChannel() {
|
||||
val mgr = getSystemService(NotificationManager::class.java)
|
||||
val channel =
|
||||
NotificationChannel(
|
||||
CHANNEL_ID,
|
||||
"Connection",
|
||||
NotificationManager.IMPORTANCE_LOW,
|
||||
).apply {
|
||||
description = "OpenClaw node connection status"
|
||||
setShowBadge(false)
|
||||
}
|
||||
mgr.createNotificationChannel(channel)
|
||||
}
|
||||
|
||||
private fun buildNotification(title: String, text: String): Notification {
|
||||
val launchIntent = Intent(this, MainActivity::class.java).apply {
|
||||
flags = Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP
|
||||
}
|
||||
val launchPending =
|
||||
PendingIntent.getActivity(
|
||||
this,
|
||||
1,
|
||||
launchIntent,
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
|
||||
)
|
||||
|
||||
val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP)
|
||||
val stopPending =
|
||||
PendingIntent.getService(
|
||||
this,
|
||||
2,
|
||||
stopIntent,
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
|
||||
)
|
||||
|
||||
return NotificationCompat.Builder(this, CHANNEL_ID)
|
||||
.setSmallIcon(R.mipmap.ic_launcher)
|
||||
.setContentTitle(title)
|
||||
.setContentText(text)
|
||||
.setContentIntent(launchPending)
|
||||
.setOngoing(true)
|
||||
.setOnlyAlertOnce(true)
|
||||
.setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE)
|
||||
.addAction(0, "Disconnect", stopPending)
|
||||
.build()
|
||||
}
|
||||
|
||||
private fun updateNotification(notification: Notification) {
|
||||
val mgr = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
|
||||
mgr.notify(NOTIFICATION_ID, notification)
|
||||
}
|
||||
|
||||
private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) {
|
||||
if (didStartForeground && requiresMic == lastRequiresMic) {
|
||||
updateNotification(notification)
|
||||
return
|
||||
}
|
||||
|
||||
lastRequiresMic = requiresMic
|
||||
val types =
|
||||
if (requiresMic) {
|
||||
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
|
||||
} else {
|
||||
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC
|
||||
}
|
||||
startForeground(NOTIFICATION_ID, notification, types)
|
||||
didStartForeground = true
|
||||
}
|
||||
|
||||
private fun hasRecordAudioPermission(): Boolean {
|
||||
return (
|
||||
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val CHANNEL_ID = "connection"
|
||||
private const val NOTIFICATION_ID = 1
|
||||
|
||||
private const val ACTION_STOP = "ai.openclaw.android.action.STOP"
|
||||
|
||||
fun start(context: Context) {
|
||||
val intent = Intent(context, NodeForegroundService::class.java)
|
||||
context.startForegroundService(intent)
|
||||
}
|
||||
|
||||
fun stop(context: Context) {
|
||||
val intent = Intent(context, NodeForegroundService::class.java).setAction(ACTION_STOP)
|
||||
context.startService(intent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private data class Quint<A, B, C, D, E>(val first: A, val second: B, val third: C, val fourth: D, val fifth: E)
|
||||
@@ -0,0 +1,813 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.os.SystemClock
|
||||
import android.util.Log
|
||||
import androidx.core.content.ContextCompat
|
||||
import ai.openclaw.android.chat.ChatController
|
||||
import ai.openclaw.android.chat.ChatMessage
|
||||
import ai.openclaw.android.chat.ChatPendingToolCall
|
||||
import ai.openclaw.android.chat.ChatSessionEntry
|
||||
import ai.openclaw.android.chat.OutgoingAttachment
|
||||
import ai.openclaw.android.gateway.DeviceAuthStore
|
||||
import ai.openclaw.android.gateway.DeviceIdentityStore
|
||||
import ai.openclaw.android.gateway.GatewayDiscovery
|
||||
import ai.openclaw.android.gateway.GatewayEndpoint
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import ai.openclaw.android.gateway.probeGatewayTlsFingerprint
|
||||
import ai.openclaw.android.node.*
|
||||
import ai.openclaw.android.protocol.OpenClawCanvasA2UIAction
|
||||
import ai.openclaw.android.voice.MicCaptureManager
|
||||
import ai.openclaw.android.voice.VoiceConversationEntry
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.flow.combine
|
||||
import kotlinx.coroutines.flow.distinctUntilChanged
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import java.util.UUID
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class NodeRuntime(context: Context) {
|
||||
private val appContext = context.applicationContext
|
||||
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
|
||||
val prefs = SecurePrefs(appContext)
|
||||
private val deviceAuthStore = DeviceAuthStore(prefs)
|
||||
val canvas = CanvasController()
|
||||
val camera = CameraCaptureManager(appContext)
|
||||
val location = LocationCaptureManager(appContext)
|
||||
val screenRecorder = ScreenRecordManager(appContext)
|
||||
val sms = SmsManager(appContext)
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
|
||||
private val externalAudioCaptureActive = MutableStateFlow(false)
|
||||
|
||||
private val discovery = GatewayDiscovery(appContext, scope = scope)
|
||||
val gateways: StateFlow<List<GatewayEndpoint>> = discovery.gateways
|
||||
val discoveryStatusText: StateFlow<String> = discovery.statusText
|
||||
|
||||
private val identityStore = DeviceIdentityStore(appContext)
|
||||
private var connectedEndpoint: GatewayEndpoint? = null
|
||||
|
||||
private val cameraHandler: CameraHandler = CameraHandler(
|
||||
appContext = appContext,
|
||||
camera = camera,
|
||||
externalAudioCaptureActive = externalAudioCaptureActive,
|
||||
showCameraHud = ::showCameraHud,
|
||||
triggerCameraFlash = ::triggerCameraFlash,
|
||||
invokeErrorFromThrowable = { invokeErrorFromThrowable(it) },
|
||||
)
|
||||
|
||||
private val debugHandler: DebugHandler = DebugHandler(
|
||||
appContext = appContext,
|
||||
identityStore = identityStore,
|
||||
)
|
||||
|
||||
private val appUpdateHandler: AppUpdateHandler = AppUpdateHandler(
|
||||
appContext = appContext,
|
||||
connectedEndpoint = { connectedEndpoint },
|
||||
)
|
||||
|
||||
private val locationHandler: LocationHandler = LocationHandler(
|
||||
appContext = appContext,
|
||||
location = location,
|
||||
json = json,
|
||||
isForeground = { _isForeground.value },
|
||||
locationMode = { locationMode.value },
|
||||
locationPreciseEnabled = { locationPreciseEnabled.value },
|
||||
)
|
||||
|
||||
private val deviceHandler: DeviceHandler = DeviceHandler(
|
||||
appContext = appContext,
|
||||
)
|
||||
|
||||
private val notificationsHandler: NotificationsHandler = NotificationsHandler(
|
||||
appContext = appContext,
|
||||
)
|
||||
|
||||
private val screenHandler: ScreenHandler = ScreenHandler(
|
||||
screenRecorder = screenRecorder,
|
||||
setScreenRecordActive = { _screenRecordActive.value = it },
|
||||
invokeErrorFromThrowable = { invokeErrorFromThrowable(it) },
|
||||
)
|
||||
|
||||
private val smsHandlerImpl: SmsHandler = SmsHandler(
|
||||
sms = sms,
|
||||
)
|
||||
|
||||
private val a2uiHandler: A2UIHandler = A2UIHandler(
|
||||
canvas = canvas,
|
||||
json = json,
|
||||
getNodeCanvasHostUrl = { nodeSession.currentCanvasHostUrl() },
|
||||
getOperatorCanvasHostUrl = { operatorSession.currentCanvasHostUrl() },
|
||||
)
|
||||
|
||||
private val connectionManager: ConnectionManager = ConnectionManager(
|
||||
prefs = prefs,
|
||||
cameraEnabled = { cameraEnabled.value },
|
||||
locationMode = { locationMode.value },
|
||||
voiceWakeMode = { VoiceWakeMode.Off },
|
||||
smsAvailable = { sms.canSendSms() },
|
||||
hasRecordAudioPermission = { hasRecordAudioPermission() },
|
||||
manualTls = { manualTls.value },
|
||||
)
|
||||
|
||||
private val invokeDispatcher: InvokeDispatcher = InvokeDispatcher(
|
||||
canvas = canvas,
|
||||
cameraHandler = cameraHandler,
|
||||
locationHandler = locationHandler,
|
||||
deviceHandler = deviceHandler,
|
||||
notificationsHandler = notificationsHandler,
|
||||
screenHandler = screenHandler,
|
||||
smsHandler = smsHandlerImpl,
|
||||
a2uiHandler = a2uiHandler,
|
||||
debugHandler = debugHandler,
|
||||
appUpdateHandler = appUpdateHandler,
|
||||
isForeground = { _isForeground.value },
|
||||
cameraEnabled = { cameraEnabled.value },
|
||||
locationEnabled = { locationMode.value != LocationMode.Off },
|
||||
smsAvailable = { sms.canSendSms() },
|
||||
debugBuild = { BuildConfig.DEBUG },
|
||||
refreshNodeCanvasCapability = { nodeSession.refreshNodeCanvasCapability() },
|
||||
onCanvasA2uiPush = {
|
||||
_canvasA2uiHydrated.value = true
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = null
|
||||
},
|
||||
onCanvasA2uiReset = { _canvasA2uiHydrated.value = false },
|
||||
)
|
||||
|
||||
data class GatewayTrustPrompt(
|
||||
val endpoint: GatewayEndpoint,
|
||||
val fingerprintSha256: String,
|
||||
)
|
||||
|
||||
private val _isConnected = MutableStateFlow(false)
|
||||
val isConnected: StateFlow<Boolean> = _isConnected.asStateFlow()
|
||||
private val _nodeConnected = MutableStateFlow(false)
|
||||
val nodeConnected: StateFlow<Boolean> = _nodeConnected.asStateFlow()
|
||||
|
||||
private val _statusText = MutableStateFlow("Offline")
|
||||
val statusText: StateFlow<String> = _statusText.asStateFlow()
|
||||
|
||||
private val _pendingGatewayTrust = MutableStateFlow<GatewayTrustPrompt?>(null)
|
||||
val pendingGatewayTrust: StateFlow<GatewayTrustPrompt?> = _pendingGatewayTrust.asStateFlow()
|
||||
|
||||
private val _mainSessionKey = MutableStateFlow("main")
|
||||
val mainSessionKey: StateFlow<String> = _mainSessionKey.asStateFlow()
|
||||
|
||||
private val cameraHudSeq = AtomicLong(0)
|
||||
private val _cameraHud = MutableStateFlow<CameraHudState?>(null)
|
||||
val cameraHud: StateFlow<CameraHudState?> = _cameraHud.asStateFlow()
|
||||
|
||||
private val _cameraFlashToken = MutableStateFlow(0L)
|
||||
val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
|
||||
|
||||
private val _screenRecordActive = MutableStateFlow(false)
|
||||
val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
|
||||
|
||||
private val _canvasA2uiHydrated = MutableStateFlow(false)
|
||||
val canvasA2uiHydrated: StateFlow<Boolean> = _canvasA2uiHydrated.asStateFlow()
|
||||
private val _canvasRehydratePending = MutableStateFlow(false)
|
||||
val canvasRehydratePending: StateFlow<Boolean> = _canvasRehydratePending.asStateFlow()
|
||||
private val _canvasRehydrateErrorText = MutableStateFlow<String?>(null)
|
||||
val canvasRehydrateErrorText: StateFlow<String?> = _canvasRehydrateErrorText.asStateFlow()
|
||||
|
||||
private val _serverName = MutableStateFlow<String?>(null)
|
||||
val serverName: StateFlow<String?> = _serverName.asStateFlow()
|
||||
|
||||
private val _remoteAddress = MutableStateFlow<String?>(null)
|
||||
val remoteAddress: StateFlow<String?> = _remoteAddress.asStateFlow()
|
||||
|
||||
private val _seamColorArgb = MutableStateFlow(DEFAULT_SEAM_COLOR_ARGB)
|
||||
val seamColorArgb: StateFlow<Long> = _seamColorArgb.asStateFlow()
|
||||
|
||||
private val _isForeground = MutableStateFlow(true)
|
||||
val isForeground: StateFlow<Boolean> = _isForeground.asStateFlow()
|
||||
|
||||
private var lastAutoA2uiUrl: String? = null
|
||||
private var didAutoRequestCanvasRehydrate = false
|
||||
private val canvasRehydrateSeq = AtomicLong(0)
|
||||
private var operatorConnected = false
|
||||
private var operatorStatusText: String = "Offline"
|
||||
private var nodeStatusText: String = "Offline"
|
||||
|
||||
private val operatorSession =
|
||||
GatewaySession(
|
||||
scope = scope,
|
||||
identityStore = identityStore,
|
||||
deviceAuthStore = deviceAuthStore,
|
||||
onConnected = { name, remote, mainSessionKey ->
|
||||
operatorConnected = true
|
||||
operatorStatusText = "Connected"
|
||||
_serverName.value = name
|
||||
_remoteAddress.value = remote
|
||||
_seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
|
||||
applyMainSessionKey(mainSessionKey)
|
||||
updateStatus()
|
||||
micCapture.onGatewayConnectionChanged(true)
|
||||
scope.launch { refreshBrandingFromGateway() }
|
||||
},
|
||||
onDisconnected = { message ->
|
||||
operatorConnected = false
|
||||
operatorStatusText = message
|
||||
_serverName.value = null
|
||||
_remoteAddress.value = null
|
||||
_seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
|
||||
if (!isCanonicalMainSessionKey(_mainSessionKey.value)) {
|
||||
_mainSessionKey.value = "main"
|
||||
}
|
||||
chat.applyMainSessionKey(resolveMainSessionKey())
|
||||
chat.onDisconnected(message)
|
||||
updateStatus()
|
||||
micCapture.onGatewayConnectionChanged(false)
|
||||
},
|
||||
onEvent = { event, payloadJson ->
|
||||
handleGatewayEvent(event, payloadJson)
|
||||
},
|
||||
)
|
||||
|
||||
private val nodeSession =
|
||||
GatewaySession(
|
||||
scope = scope,
|
||||
identityStore = identityStore,
|
||||
deviceAuthStore = deviceAuthStore,
|
||||
onConnected = { _, _, _ ->
|
||||
_nodeConnected.value = true
|
||||
nodeStatusText = "Connected"
|
||||
didAutoRequestCanvasRehydrate = false
|
||||
_canvasA2uiHydrated.value = false
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = null
|
||||
updateStatus()
|
||||
maybeNavigateToA2uiOnConnect()
|
||||
},
|
||||
onDisconnected = { message ->
|
||||
_nodeConnected.value = false
|
||||
nodeStatusText = message
|
||||
didAutoRequestCanvasRehydrate = false
|
||||
_canvasA2uiHydrated.value = false
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = null
|
||||
updateStatus()
|
||||
showLocalCanvasOnDisconnect()
|
||||
},
|
||||
onEvent = { _, _ -> },
|
||||
onInvoke = { req ->
|
||||
invokeDispatcher.handleInvoke(req.command, req.paramsJson)
|
||||
},
|
||||
onTlsFingerprint = { stableId, fingerprint ->
|
||||
prefs.saveGatewayTlsFingerprint(stableId, fingerprint)
|
||||
},
|
||||
)
|
||||
|
||||
private val chat: ChatController =
|
||||
ChatController(
|
||||
scope = scope,
|
||||
session = operatorSession,
|
||||
json = json,
|
||||
supportsChatSubscribe = false,
|
||||
)
|
||||
private val micCapture: MicCaptureManager by lazy {
|
||||
MicCaptureManager(
|
||||
context = appContext,
|
||||
scope = scope,
|
||||
sendToGateway = { message ->
|
||||
val idempotencyKey = UUID.randomUUID().toString()
|
||||
val params =
|
||||
buildJsonObject {
|
||||
put("sessionKey", JsonPrimitive(resolveMainSessionKey()))
|
||||
put("message", JsonPrimitive(message))
|
||||
put("thinking", JsonPrimitive(chatThinkingLevel.value))
|
||||
put("timeoutMs", JsonPrimitive(30_000))
|
||||
put("idempotencyKey", JsonPrimitive(idempotencyKey))
|
||||
}
|
||||
val response = operatorSession.request("chat.send", params.toString())
|
||||
parseChatSendRunId(response) ?: idempotencyKey
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
val micStatusText: StateFlow<String>
|
||||
get() = micCapture.statusText
|
||||
|
||||
val micLiveTranscript: StateFlow<String?>
|
||||
get() = micCapture.liveTranscript
|
||||
|
||||
val micIsListening: StateFlow<Boolean>
|
||||
get() = micCapture.isListening
|
||||
|
||||
val micEnabled: StateFlow<Boolean>
|
||||
get() = micCapture.micEnabled
|
||||
|
||||
val micQueuedMessages: StateFlow<List<String>>
|
||||
get() = micCapture.queuedMessages
|
||||
|
||||
val micConversation: StateFlow<List<VoiceConversationEntry>>
|
||||
get() = micCapture.conversation
|
||||
|
||||
val micInputLevel: StateFlow<Float>
|
||||
get() = micCapture.inputLevel
|
||||
|
||||
val micIsSending: StateFlow<Boolean>
|
||||
get() = micCapture.isSending
|
||||
|
||||
private fun applyMainSessionKey(candidate: String?) {
|
||||
val trimmed = normalizeMainKey(candidate) ?: return
|
||||
if (isCanonicalMainSessionKey(_mainSessionKey.value)) return
|
||||
if (_mainSessionKey.value == trimmed) return
|
||||
_mainSessionKey.value = trimmed
|
||||
chat.applyMainSessionKey(trimmed)
|
||||
}
|
||||
|
||||
private fun updateStatus() {
|
||||
_isConnected.value = operatorConnected
|
||||
val operator = operatorStatusText.trim()
|
||||
val node = nodeStatusText.trim()
|
||||
_statusText.value =
|
||||
when {
|
||||
operatorConnected && _nodeConnected.value -> "Connected"
|
||||
operatorConnected && !_nodeConnected.value -> "Connected (node offline)"
|
||||
!operatorConnected && _nodeConnected.value ->
|
||||
if (operator.isNotEmpty() && operator != "Offline") {
|
||||
"Connected (operator: $operator)"
|
||||
} else {
|
||||
"Connected (operator offline)"
|
||||
}
|
||||
operator.isNotBlank() && operator != "Offline" -> operator
|
||||
else -> node
|
||||
}
|
||||
}
|
||||
|
||||
private fun resolveMainSessionKey(): String {
|
||||
val trimmed = _mainSessionKey.value.trim()
|
||||
return if (trimmed.isEmpty()) "main" else trimmed
|
||||
}
|
||||
|
||||
private fun maybeNavigateToA2uiOnConnect() {
|
||||
val a2uiUrl = a2uiHandler.resolveA2uiHostUrl() ?: return
|
||||
val current = canvas.currentUrl()?.trim().orEmpty()
|
||||
if (current.isEmpty() || current == lastAutoA2uiUrl) {
|
||||
lastAutoA2uiUrl = a2uiUrl
|
||||
canvas.navigate(a2uiUrl)
|
||||
}
|
||||
}
|
||||
|
||||
private fun showLocalCanvasOnDisconnect() {
|
||||
lastAutoA2uiUrl = null
|
||||
_canvasA2uiHydrated.value = false
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = null
|
||||
canvas.navigate("")
|
||||
}
|
||||
|
||||
fun requestCanvasRehydrate(source: String = "manual", force: Boolean = true) {
|
||||
scope.launch {
|
||||
if (!_nodeConnected.value) {
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = "Node offline. Reconnect and retry."
|
||||
return@launch
|
||||
}
|
||||
if (!force && didAutoRequestCanvasRehydrate) return@launch
|
||||
didAutoRequestCanvasRehydrate = true
|
||||
val requestId = canvasRehydrateSeq.incrementAndGet()
|
||||
_canvasRehydratePending.value = true
|
||||
_canvasRehydrateErrorText.value = null
|
||||
|
||||
val sessionKey = resolveMainSessionKey()
|
||||
val prompt =
|
||||
"Restore canvas now for session=$sessionKey source=$source. " +
|
||||
"If existing A2UI state exists, replay it immediately. " +
|
||||
"If not, create and render a compact mobile-friendly dashboard in Canvas."
|
||||
val sent =
|
||||
nodeSession.sendNodeEvent(
|
||||
event = "agent.request",
|
||||
payloadJson =
|
||||
buildJsonObject {
|
||||
put("message", JsonPrimitive(prompt))
|
||||
put("sessionKey", JsonPrimitive(sessionKey))
|
||||
put("thinking", JsonPrimitive("low"))
|
||||
put("deliver", JsonPrimitive(false))
|
||||
}.toString(),
|
||||
)
|
||||
if (!sent) {
|
||||
if (!force) {
|
||||
didAutoRequestCanvasRehydrate = false
|
||||
}
|
||||
if (canvasRehydrateSeq.get() == requestId) {
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = "Failed to request restore. Tap to retry."
|
||||
}
|
||||
Log.w("OpenClawCanvas", "canvas rehydrate request failed ($source): transport unavailable")
|
||||
return@launch
|
||||
}
|
||||
scope.launch {
|
||||
delay(20_000)
|
||||
if (canvasRehydrateSeq.get() != requestId) return@launch
|
||||
if (!_canvasRehydratePending.value) return@launch
|
||||
if (_canvasA2uiHydrated.value) return@launch
|
||||
_canvasRehydratePending.value = false
|
||||
_canvasRehydrateErrorText.value = "No canvas update yet. Tap to retry."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val instanceId: StateFlow<String> = prefs.instanceId
|
||||
val displayName: StateFlow<String> = prefs.displayName
|
||||
val cameraEnabled: StateFlow<Boolean> = prefs.cameraEnabled
|
||||
val locationMode: StateFlow<LocationMode> = prefs.locationMode
|
||||
val locationPreciseEnabled: StateFlow<Boolean> = prefs.locationPreciseEnabled
|
||||
val preventSleep: StateFlow<Boolean> = prefs.preventSleep
|
||||
val manualEnabled: StateFlow<Boolean> = prefs.manualEnabled
|
||||
val manualHost: StateFlow<String> = prefs.manualHost
|
||||
val manualPort: StateFlow<Int> = prefs.manualPort
|
||||
val manualTls: StateFlow<Boolean> = prefs.manualTls
|
||||
val gatewayToken: StateFlow<String> = prefs.gatewayToken
|
||||
val onboardingCompleted: StateFlow<Boolean> = prefs.onboardingCompleted
|
||||
fun setGatewayToken(value: String) = prefs.setGatewayToken(value)
|
||||
fun setGatewayPassword(value: String) = prefs.setGatewayPassword(value)
|
||||
fun setOnboardingCompleted(value: Boolean) = prefs.setOnboardingCompleted(value)
|
||||
val lastDiscoveredStableId: StateFlow<String> = prefs.lastDiscoveredStableId
|
||||
val canvasDebugStatusEnabled: StateFlow<Boolean> = prefs.canvasDebugStatusEnabled
|
||||
|
||||
private var didAutoConnect = false
|
||||
|
||||
val chatSessionKey: StateFlow<String> = chat.sessionKey
|
||||
val chatSessionId: StateFlow<String?> = chat.sessionId
|
||||
val chatMessages: StateFlow<List<ChatMessage>> = chat.messages
|
||||
val chatError: StateFlow<String?> = chat.errorText
|
||||
val chatHealthOk: StateFlow<Boolean> = chat.healthOk
|
||||
val chatThinkingLevel: StateFlow<String> = chat.thinkingLevel
|
||||
val chatStreamingAssistantText: StateFlow<String?> = chat.streamingAssistantText
|
||||
val chatPendingToolCalls: StateFlow<List<ChatPendingToolCall>> = chat.pendingToolCalls
|
||||
val chatSessions: StateFlow<List<ChatSessionEntry>> = chat.sessions
|
||||
val pendingRunCount: StateFlow<Int> = chat.pendingRunCount
|
||||
|
||||
init {
|
||||
if (prefs.voiceWakeMode.value != VoiceWakeMode.Off) {
|
||||
prefs.setVoiceWakeMode(VoiceWakeMode.Off)
|
||||
}
|
||||
|
||||
scope.launch {
|
||||
prefs.loadGatewayToken()
|
||||
}
|
||||
|
||||
scope.launch {
|
||||
prefs.talkEnabled.collect { enabled ->
|
||||
micCapture.setMicEnabled(enabled)
|
||||
externalAudioCaptureActive.value = enabled
|
||||
}
|
||||
}
|
||||
|
||||
scope.launch(Dispatchers.Default) {
|
||||
gateways.collect { list ->
|
||||
if (list.isNotEmpty()) {
|
||||
// Security: don't let an unauthenticated discovery feed continuously steer autoconnect.
|
||||
// UX parity with iOS: only set once when unset.
|
||||
if (lastDiscoveredStableId.value.trim().isEmpty()) {
|
||||
prefs.setLastDiscoveredStableId(list.first().stableId)
|
||||
}
|
||||
}
|
||||
|
||||
if (didAutoConnect) return@collect
|
||||
if (_isConnected.value) return@collect
|
||||
|
||||
if (manualEnabled.value) {
|
||||
val host = manualHost.value.trim()
|
||||
val port = manualPort.value
|
||||
if (host.isNotEmpty() && port in 1..65535) {
|
||||
// Security: autoconnect only to previously trusted gateways (stored TLS pin).
|
||||
if (!manualTls.value) return@collect
|
||||
val stableId = GatewayEndpoint.manual(host = host, port = port).stableId
|
||||
val storedFingerprint = prefs.loadGatewayTlsFingerprint(stableId)?.trim().orEmpty()
|
||||
if (storedFingerprint.isEmpty()) return@collect
|
||||
|
||||
didAutoConnect = true
|
||||
connect(GatewayEndpoint.manual(host = host, port = port))
|
||||
}
|
||||
return@collect
|
||||
}
|
||||
|
||||
val targetStableId = lastDiscoveredStableId.value.trim()
|
||||
if (targetStableId.isEmpty()) return@collect
|
||||
val target = list.firstOrNull { it.stableId == targetStableId } ?: return@collect
|
||||
|
||||
// Security: autoconnect only to previously trusted gateways (stored TLS pin).
|
||||
val storedFingerprint = prefs.loadGatewayTlsFingerprint(target.stableId)?.trim().orEmpty()
|
||||
if (storedFingerprint.isEmpty()) return@collect
|
||||
|
||||
didAutoConnect = true
|
||||
connect(target)
|
||||
}
|
||||
}
|
||||
|
||||
scope.launch {
|
||||
combine(
|
||||
canvasDebugStatusEnabled,
|
||||
statusText,
|
||||
serverName,
|
||||
remoteAddress,
|
||||
) { debugEnabled, status, server, remote ->
|
||||
Quad(debugEnabled, status, server, remote)
|
||||
}.distinctUntilChanged()
|
||||
.collect { (debugEnabled, status, server, remote) ->
|
||||
canvas.setDebugStatusEnabled(debugEnabled)
|
||||
if (!debugEnabled) return@collect
|
||||
canvas.setDebugStatus(status, server ?: remote)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun setForeground(value: Boolean) {
|
||||
_isForeground.value = value
|
||||
}
|
||||
|
||||
fun setDisplayName(value: String) {
|
||||
prefs.setDisplayName(value)
|
||||
}
|
||||
|
||||
fun setCameraEnabled(value: Boolean) {
|
||||
prefs.setCameraEnabled(value)
|
||||
}
|
||||
|
||||
fun setLocationMode(mode: LocationMode) {
|
||||
prefs.setLocationMode(mode)
|
||||
}
|
||||
|
||||
fun setLocationPreciseEnabled(value: Boolean) {
|
||||
prefs.setLocationPreciseEnabled(value)
|
||||
}
|
||||
|
||||
fun setPreventSleep(value: Boolean) {
|
||||
prefs.setPreventSleep(value)
|
||||
}
|
||||
|
||||
fun setManualEnabled(value: Boolean) {
|
||||
prefs.setManualEnabled(value)
|
||||
}
|
||||
|
||||
fun setManualHost(value: String) {
|
||||
prefs.setManualHost(value)
|
||||
}
|
||||
|
||||
fun setManualPort(value: Int) {
|
||||
prefs.setManualPort(value)
|
||||
}
|
||||
|
||||
fun setManualTls(value: Boolean) {
|
||||
prefs.setManualTls(value)
|
||||
}
|
||||
|
||||
fun setCanvasDebugStatusEnabled(value: Boolean) {
|
||||
prefs.setCanvasDebugStatusEnabled(value)
|
||||
}
|
||||
|
||||
fun setMicEnabled(value: Boolean) {
|
||||
prefs.setTalkEnabled(value)
|
||||
micCapture.setMicEnabled(value)
|
||||
externalAudioCaptureActive.value = value
|
||||
}
|
||||
|
||||
fun refreshGatewayConnection() {
|
||||
val endpoint =
|
||||
connectedEndpoint ?: run {
|
||||
_statusText.value = "Failed: no cached gateway endpoint"
|
||||
return
|
||||
}
|
||||
operatorStatusText = "Connecting…"
|
||||
updateStatus()
|
||||
val token = prefs.loadGatewayToken()
|
||||
val password = prefs.loadGatewayPassword()
|
||||
val tls = connectionManager.resolveTlsParams(endpoint)
|
||||
operatorSession.connect(endpoint, token, password, connectionManager.buildOperatorConnectOptions(), tls)
|
||||
nodeSession.connect(endpoint, token, password, connectionManager.buildNodeConnectOptions(), tls)
|
||||
operatorSession.reconnect()
|
||||
nodeSession.reconnect()
|
||||
}
|
||||
|
||||
fun connect(endpoint: GatewayEndpoint) {
|
||||
val tls = connectionManager.resolveTlsParams(endpoint)
|
||||
if (tls?.required == true && tls.expectedFingerprint.isNullOrBlank()) {
|
||||
// First-time TLS: capture fingerprint, ask user to verify out-of-band, then store and connect.
|
||||
_statusText.value = "Verify gateway TLS fingerprint…"
|
||||
scope.launch {
|
||||
val fp = probeGatewayTlsFingerprint(endpoint.host, endpoint.port) ?: run {
|
||||
_statusText.value = "Failed: can't read TLS fingerprint"
|
||||
return@launch
|
||||
}
|
||||
_pendingGatewayTrust.value = GatewayTrustPrompt(endpoint = endpoint, fingerprintSha256 = fp)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
connectedEndpoint = endpoint
|
||||
operatorStatusText = "Connecting…"
|
||||
nodeStatusText = "Connecting…"
|
||||
updateStatus()
|
||||
val token = prefs.loadGatewayToken()
|
||||
val password = prefs.loadGatewayPassword()
|
||||
operatorSession.connect(endpoint, token, password, connectionManager.buildOperatorConnectOptions(), tls)
|
||||
nodeSession.connect(endpoint, token, password, connectionManager.buildNodeConnectOptions(), tls)
|
||||
}
|
||||
|
||||
fun acceptGatewayTrustPrompt() {
|
||||
val prompt = _pendingGatewayTrust.value ?: return
|
||||
_pendingGatewayTrust.value = null
|
||||
prefs.saveGatewayTlsFingerprint(prompt.endpoint.stableId, prompt.fingerprintSha256)
|
||||
connect(prompt.endpoint)
|
||||
}
|
||||
|
||||
fun declineGatewayTrustPrompt() {
|
||||
_pendingGatewayTrust.value = null
|
||||
_statusText.value = "Offline"
|
||||
}
|
||||
|
||||
private fun hasRecordAudioPermission(): Boolean {
|
||||
return (
|
||||
ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
)
|
||||
}
|
||||
|
||||
fun connectManual() {
|
||||
val host = manualHost.value.trim()
|
||||
val port = manualPort.value
|
||||
if (host.isEmpty() || port <= 0 || port > 65535) {
|
||||
_statusText.value = "Failed: invalid manual host/port"
|
||||
return
|
||||
}
|
||||
connect(GatewayEndpoint.manual(host = host, port = port))
|
||||
}
|
||||
|
||||
fun disconnect() {
|
||||
connectedEndpoint = null
|
||||
_pendingGatewayTrust.value = null
|
||||
operatorSession.disconnect()
|
||||
nodeSession.disconnect()
|
||||
}
|
||||
|
||||
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
|
||||
scope.launch {
|
||||
val trimmed = payloadJson.trim()
|
||||
if (trimmed.isEmpty()) return@launch
|
||||
|
||||
val root =
|
||||
try {
|
||||
json.parseToJsonElement(trimmed).asObjectOrNull() ?: return@launch
|
||||
} catch (_: Throwable) {
|
||||
return@launch
|
||||
}
|
||||
|
||||
val userActionObj = (root["userAction"] as? JsonObject) ?: root
|
||||
val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty {
|
||||
java.util.UUID.randomUUID().toString()
|
||||
}
|
||||
val name = OpenClawCanvasA2UIAction.extractActionName(userActionObj) ?: return@launch
|
||||
|
||||
val surfaceId =
|
||||
(userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" }
|
||||
val sourceComponentId =
|
||||
(userActionObj["sourceComponentId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "-" }
|
||||
val contextJson = (userActionObj["context"] as? JsonObject)?.toString()
|
||||
|
||||
val sessionKey = resolveMainSessionKey()
|
||||
val message =
|
||||
OpenClawCanvasA2UIAction.formatAgentMessage(
|
||||
actionName = name,
|
||||
sessionKey = sessionKey,
|
||||
surfaceId = surfaceId,
|
||||
sourceComponentId = sourceComponentId,
|
||||
host = displayName.value,
|
||||
instanceId = instanceId.value.lowercase(),
|
||||
contextJson = contextJson,
|
||||
)
|
||||
|
||||
val connected = _nodeConnected.value
|
||||
var error: String? = null
|
||||
if (connected) {
|
||||
val sent =
|
||||
nodeSession.sendNodeEvent(
|
||||
event = "agent.request",
|
||||
payloadJson =
|
||||
buildJsonObject {
|
||||
put("message", JsonPrimitive(message))
|
||||
put("sessionKey", JsonPrimitive(sessionKey))
|
||||
put("thinking", JsonPrimitive("low"))
|
||||
put("deliver", JsonPrimitive(false))
|
||||
put("key", JsonPrimitive(actionId))
|
||||
}.toString(),
|
||||
)
|
||||
if (!sent) {
|
||||
error = "send failed"
|
||||
}
|
||||
} else {
|
||||
error = "gateway not connected"
|
||||
}
|
||||
|
||||
try {
|
||||
canvas.eval(
|
||||
OpenClawCanvasA2UIAction.jsDispatchA2UIActionStatus(
|
||||
actionId = actionId,
|
||||
ok = connected && error == null,
|
||||
error = error,
|
||||
),
|
||||
)
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun loadChat(sessionKey: String) {
|
||||
val key = sessionKey.trim().ifEmpty { resolveMainSessionKey() }
|
||||
chat.load(key)
|
||||
}
|
||||
|
||||
fun refreshChat() {
|
||||
chat.refresh()
|
||||
}
|
||||
|
||||
fun refreshChatSessions(limit: Int? = null) {
|
||||
chat.refreshSessions(limit = limit)
|
||||
}
|
||||
|
||||
fun setChatThinkingLevel(level: String) {
|
||||
chat.setThinkingLevel(level)
|
||||
}
|
||||
|
||||
fun switchChatSession(sessionKey: String) {
|
||||
chat.switchSession(sessionKey)
|
||||
}
|
||||
|
||||
fun abortChat() {
|
||||
chat.abort()
|
||||
}
|
||||
|
||||
fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
|
||||
chat.sendMessage(message = message, thinkingLevel = thinking, attachments = attachments)
|
||||
}
|
||||
|
||||
private fun handleGatewayEvent(event: String, payloadJson: String?) {
|
||||
micCapture.handleGatewayEvent(event, payloadJson)
|
||||
chat.handleGatewayEvent(event, payloadJson)
|
||||
}
|
||||
|
||||
private fun parseChatSendRunId(response: String): String? {
|
||||
return try {
|
||||
val root = json.parseToJsonElement(response).asObjectOrNull() ?: return null
|
||||
root["runId"].asStringOrNull()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun refreshBrandingFromGateway() {
|
||||
if (!_isConnected.value) return
|
||||
try {
|
||||
val res = operatorSession.request("config.get", "{}")
|
||||
val root = json.parseToJsonElement(res).asObjectOrNull()
|
||||
val config = root?.get("config").asObjectOrNull()
|
||||
val ui = config?.get("ui").asObjectOrNull()
|
||||
val raw = ui?.get("seamColor").asStringOrNull()?.trim()
|
||||
val sessionCfg = config?.get("session").asObjectOrNull()
|
||||
val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull())
|
||||
applyMainSessionKey(mainKey)
|
||||
|
||||
val parsed = parseHexColorArgb(raw)
|
||||
_seamColorArgb.value = parsed ?: DEFAULT_SEAM_COLOR_ARGB
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
private fun triggerCameraFlash() {
|
||||
// Token is used as a pulse trigger; value doesn't matter as long as it changes.
|
||||
_cameraFlashToken.value = SystemClock.elapsedRealtimeNanos()
|
||||
}
|
||||
|
||||
private fun showCameraHud(message: String, kind: CameraHudKind, autoHideMs: Long? = null) {
|
||||
val token = cameraHudSeq.incrementAndGet()
|
||||
_cameraHud.value = CameraHudState(token = token, kind = kind, message = message)
|
||||
|
||||
if (autoHideMs != null && autoHideMs > 0) {
|
||||
scope.launch {
|
||||
delay(autoHideMs)
|
||||
if (_cameraHud.value?.token == token) _cameraHud.value = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
import android.content.pm.PackageManager
|
||||
import android.content.Intent
|
||||
import android.Manifest
|
||||
import android.net.Uri
|
||||
import android.provider.Settings
|
||||
import androidx.appcompat.app.AlertDialog
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.result.ActivityResultLauncher
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.core.app.ActivityCompat
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlin.coroutines.resume
|
||||
|
||||
class PermissionRequester(private val activity: ComponentActivity) {
|
||||
private val mutex = Mutex()
|
||||
private var pending: CompletableDeferred<Map<String, Boolean>>? = null
|
||||
|
||||
private val launcher: ActivityResultLauncher<Array<String>> =
|
||||
activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result ->
|
||||
val p = pending
|
||||
pending = null
|
||||
p?.complete(result)
|
||||
}
|
||||
|
||||
suspend fun requestIfMissing(
|
||||
permissions: List<String>,
|
||||
timeoutMs: Long = 20_000,
|
||||
): Map<String, Boolean> =
|
||||
mutex.withLock {
|
||||
val missing =
|
||||
permissions.filter { perm ->
|
||||
ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
if (missing.isEmpty()) {
|
||||
return permissions.associateWith { true }
|
||||
}
|
||||
|
||||
val needsRationale =
|
||||
missing.any { ActivityCompat.shouldShowRequestPermissionRationale(activity, it) }
|
||||
if (needsRationale) {
|
||||
val proceed = showRationaleDialog(missing)
|
||||
if (!proceed) {
|
||||
return permissions.associateWith { perm ->
|
||||
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val deferred = CompletableDeferred<Map<String, Boolean>>()
|
||||
pending = deferred
|
||||
withContext(Dispatchers.Main) {
|
||||
launcher.launch(missing.toTypedArray())
|
||||
}
|
||||
|
||||
val result =
|
||||
withContext(Dispatchers.Default) {
|
||||
kotlinx.coroutines.withTimeout(timeoutMs) { deferred.await() }
|
||||
}
|
||||
|
||||
// Merge: if something was already granted, treat it as granted even if launcher omitted it.
|
||||
val merged =
|
||||
permissions.associateWith { perm ->
|
||||
val nowGranted =
|
||||
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
|
||||
result[perm] == true || nowGranted
|
||||
}
|
||||
|
||||
val denied =
|
||||
merged.filterValues { !it }.keys.filter {
|
||||
!ActivityCompat.shouldShowRequestPermissionRationale(activity, it)
|
||||
}
|
||||
if (denied.isNotEmpty()) {
|
||||
showSettingsDialog(denied)
|
||||
}
|
||||
|
||||
return merged
|
||||
}
|
||||
|
||||
private suspend fun showRationaleDialog(permissions: List<String>): Boolean =
|
||||
withContext(Dispatchers.Main) {
|
||||
suspendCancellableCoroutine { cont ->
|
||||
AlertDialog.Builder(activity)
|
||||
.setTitle("Permission required")
|
||||
.setMessage(buildRationaleMessage(permissions))
|
||||
.setPositiveButton("Continue") { _, _ -> cont.resume(true) }
|
||||
.setNegativeButton("Not now") { _, _ -> cont.resume(false) }
|
||||
.setOnCancelListener { cont.resume(false) }
|
||||
.show()
|
||||
}
|
||||
}
|
||||
|
||||
private fun showSettingsDialog(permissions: List<String>) {
|
||||
AlertDialog.Builder(activity)
|
||||
.setTitle("Enable permission in Settings")
|
||||
.setMessage(buildSettingsMessage(permissions))
|
||||
.setPositiveButton("Open Settings") { _, _ ->
|
||||
val intent =
|
||||
Intent(
|
||||
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
|
||||
Uri.fromParts("package", activity.packageName, null),
|
||||
)
|
||||
activity.startActivity(intent)
|
||||
}
|
||||
.setNegativeButton("Cancel", null)
|
||||
.show()
|
||||
}
|
||||
|
||||
private fun buildRationaleMessage(permissions: List<String>): String {
|
||||
val labels = permissions.map { permissionLabel(it) }
|
||||
return "OpenClaw needs ${labels.joinToString(", ")} permissions to continue."
|
||||
}
|
||||
|
||||
private fun buildSettingsMessage(permissions: List<String>): String {
|
||||
val labels = permissions.map { permissionLabel(it) }
|
||||
return "Please enable ${labels.joinToString(", ")} in Android Settings to continue."
|
||||
}
|
||||
|
||||
private fun permissionLabel(permission: String): String =
|
||||
when (permission) {
|
||||
Manifest.permission.CAMERA -> "Camera"
|
||||
Manifest.permission.RECORD_AUDIO -> "Microphone"
|
||||
Manifest.permission.SEND_SMS -> "SMS"
|
||||
else -> permission
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
import android.app.Activity
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.media.projection.MediaProjectionManager
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.result.ActivityResultLauncher
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.appcompat.app.AlertDialog
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.withTimeout
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlin.coroutines.resume
|
||||
|
||||
class ScreenCaptureRequester(private val activity: ComponentActivity) {
|
||||
data class CaptureResult(val resultCode: Int, val data: Intent)
|
||||
|
||||
private val mutex = Mutex()
|
||||
private var pending: CompletableDeferred<CaptureResult?>? = null
|
||||
|
||||
private val launcher: ActivityResultLauncher<Intent> =
|
||||
activity.registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
|
||||
val p = pending
|
||||
pending = null
|
||||
val data = result.data
|
||||
if (result.resultCode == Activity.RESULT_OK && data != null) {
|
||||
p?.complete(CaptureResult(result.resultCode, data))
|
||||
} else {
|
||||
p?.complete(null)
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun requestCapture(timeoutMs: Long = 20_000): CaptureResult? =
|
||||
mutex.withLock {
|
||||
val proceed = showRationaleDialog()
|
||||
if (!proceed) return null
|
||||
|
||||
val mgr = activity.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||
val intent = mgr.createScreenCaptureIntent()
|
||||
|
||||
val deferred = CompletableDeferred<CaptureResult?>()
|
||||
pending = deferred
|
||||
withContext(Dispatchers.Main) { launcher.launch(intent) }
|
||||
|
||||
withContext(Dispatchers.Default) { withTimeout(timeoutMs) { deferred.await() } }
|
||||
}
|
||||
|
||||
private suspend fun showRationaleDialog(): Boolean =
|
||||
withContext(Dispatchers.Main) {
|
||||
suspendCancellableCoroutine { cont ->
|
||||
AlertDialog.Builder(activity)
|
||||
.setTitle("Screen recording required")
|
||||
.setMessage("OpenClaw needs to record the screen for this command.")
|
||||
.setPositiveButton("Continue") { _, _ -> cont.resume(true) }
|
||||
.setNegativeButton("Not now") { _, _ -> cont.resume(false) }
|
||||
.setOnCancelListener { cont.resume(false) }
|
||||
.show()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
@file:Suppress("DEPRECATION")
|
||||
|
||||
package ai.openclaw.android
|
||||
|
||||
import android.content.Context
|
||||
import android.content.SharedPreferences
|
||||
import androidx.core.content.edit
|
||||
import androidx.security.crypto.EncryptedSharedPreferences
|
||||
import androidx.security.crypto.MasterKey
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonNull
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import java.util.UUID
|
||||
|
||||
class SecurePrefs(context: Context) {
|
||||
companion object {
|
||||
val defaultWakeWords: List<String> = listOf("openclaw", "claude")
|
||||
private const val displayNameKey = "node.displayName"
|
||||
private const val voiceWakeModeKey = "voiceWake.mode"
|
||||
private const val plainPrefsName = "openclaw.node"
|
||||
private const val securePrefsName = "openclaw.node.secure"
|
||||
}
|
||||
|
||||
private val appContext = context.applicationContext
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
private val plainPrefs: SharedPreferences =
|
||||
appContext.getSharedPreferences(plainPrefsName, Context.MODE_PRIVATE)
|
||||
|
||||
private val masterKey by lazy {
|
||||
MasterKey.Builder(appContext)
|
||||
.setKeyScheme(MasterKey.KeyScheme.AES256_GCM)
|
||||
.build()
|
||||
}
|
||||
private val securePrefs: SharedPreferences by lazy { createSecurePrefs(appContext, securePrefsName) }
|
||||
|
||||
private val _instanceId = MutableStateFlow(loadOrCreateInstanceId())
|
||||
val instanceId: StateFlow<String> = _instanceId
|
||||
|
||||
private val _displayName =
|
||||
MutableStateFlow(loadOrMigrateDisplayName(context = context))
|
||||
val displayName: StateFlow<String> = _displayName
|
||||
|
||||
private val _cameraEnabled = MutableStateFlow(plainPrefs.getBoolean("camera.enabled", true))
|
||||
val cameraEnabled: StateFlow<Boolean> = _cameraEnabled
|
||||
|
||||
private val _locationMode =
|
||||
MutableStateFlow(LocationMode.fromRawValue(plainPrefs.getString("location.enabledMode", "off")))
|
||||
val locationMode: StateFlow<LocationMode> = _locationMode
|
||||
|
||||
private val _locationPreciseEnabled =
|
||||
MutableStateFlow(plainPrefs.getBoolean("location.preciseEnabled", true))
|
||||
val locationPreciseEnabled: StateFlow<Boolean> = _locationPreciseEnabled
|
||||
|
||||
private val _preventSleep = MutableStateFlow(plainPrefs.getBoolean("screen.preventSleep", true))
|
||||
val preventSleep: StateFlow<Boolean> = _preventSleep
|
||||
|
||||
private val _manualEnabled =
|
||||
MutableStateFlow(plainPrefs.getBoolean("gateway.manual.enabled", false))
|
||||
val manualEnabled: StateFlow<Boolean> = _manualEnabled
|
||||
|
||||
private val _manualHost =
|
||||
MutableStateFlow(plainPrefs.getString("gateway.manual.host", "") ?: "")
|
||||
val manualHost: StateFlow<String> = _manualHost
|
||||
|
||||
private val _manualPort =
|
||||
MutableStateFlow(plainPrefs.getInt("gateway.manual.port", 18789))
|
||||
val manualPort: StateFlow<Int> = _manualPort
|
||||
|
||||
private val _manualTls =
|
||||
MutableStateFlow(plainPrefs.getBoolean("gateway.manual.tls", true))
|
||||
val manualTls: StateFlow<Boolean> = _manualTls
|
||||
|
||||
private val _gatewayToken = MutableStateFlow("")
|
||||
val gatewayToken: StateFlow<String> = _gatewayToken
|
||||
|
||||
private val _onboardingCompleted =
|
||||
MutableStateFlow(plainPrefs.getBoolean("onboarding.completed", false))
|
||||
val onboardingCompleted: StateFlow<Boolean> = _onboardingCompleted
|
||||
|
||||
private val _lastDiscoveredStableId =
|
||||
MutableStateFlow(
|
||||
plainPrefs.getString("gateway.lastDiscoveredStableID", "") ?: "",
|
||||
)
|
||||
val lastDiscoveredStableId: StateFlow<String> = _lastDiscoveredStableId
|
||||
|
||||
private val _canvasDebugStatusEnabled =
|
||||
MutableStateFlow(plainPrefs.getBoolean("canvas.debugStatusEnabled", false))
|
||||
val canvasDebugStatusEnabled: StateFlow<Boolean> = _canvasDebugStatusEnabled
|
||||
|
||||
private val _wakeWords = MutableStateFlow(loadWakeWords())
|
||||
val wakeWords: StateFlow<List<String>> = _wakeWords
|
||||
|
||||
private val _voiceWakeMode = MutableStateFlow(loadVoiceWakeMode())
|
||||
val voiceWakeMode: StateFlow<VoiceWakeMode> = _voiceWakeMode
|
||||
|
||||
private val _talkEnabled = MutableStateFlow(plainPrefs.getBoolean("talk.enabled", false))
|
||||
val talkEnabled: StateFlow<Boolean> = _talkEnabled
|
||||
|
||||
fun setLastDiscoveredStableId(value: String) {
|
||||
val trimmed = value.trim()
|
||||
plainPrefs.edit { putString("gateway.lastDiscoveredStableID", trimmed) }
|
||||
_lastDiscoveredStableId.value = trimmed
|
||||
}
|
||||
|
||||
fun setDisplayName(value: String) {
|
||||
val trimmed = value.trim()
|
||||
plainPrefs.edit { putString(displayNameKey, trimmed) }
|
||||
_displayName.value = trimmed
|
||||
}
|
||||
|
||||
fun setCameraEnabled(value: Boolean) {
|
||||
plainPrefs.edit { putBoolean("camera.enabled", value) }
|
||||
_cameraEnabled.value = value
|
||||
}
|
||||
|
||||
fun setLocationMode(mode: LocationMode) {
|
||||
plainPrefs.edit { putString("location.enabledMode", mode.rawValue) }
|
||||
_locationMode.value = mode
|
||||
}
|
||||
|
||||
fun setLocationPreciseEnabled(value: Boolean) {
|
||||
plainPrefs.edit { putBoolean("location.preciseEnabled", value) }
|
||||
_locationPreciseEnabled.value = value
|
||||
}
|
||||
|
||||
fun setPreventSleep(value: Boolean) {
|
||||
plainPrefs.edit { putBoolean("screen.preventSleep", value) }
|
||||
_preventSleep.value = value
|
||||
}
|
||||
|
||||
fun setManualEnabled(value: Boolean) {
|
||||
plainPrefs.edit { putBoolean("gateway.manual.enabled", value) }
|
||||
_manualEnabled.value = value
|
||||
}
|
||||
|
||||
fun setManualHost(value: String) {
|
||||
val trimmed = value.trim()
|
||||
plainPrefs.edit { putString("gateway.manual.host", trimmed) }
|
||||
_manualHost.value = trimmed
|
||||
}
|
||||
|
||||
fun setManualPort(value: Int) {
|
||||
plainPrefs.edit { putInt("gateway.manual.port", value) }
|
||||
_manualPort.value = value
|
||||
}
|
||||
|
||||
fun setManualTls(value: Boolean) {
|
||||
plainPrefs.edit { putBoolean("gateway.manual.tls", value) }
|
||||
_manualTls.value = value
|
||||
}
|
||||
|
||||
fun setGatewayToken(value: String) {
|
||||
val trimmed = value.trim()
|
||||
securePrefs.edit { putString("gateway.manual.token", trimmed) }
|
||||
_gatewayToken.value = trimmed
|
||||
}
|
||||
|
||||
fun setGatewayPassword(value: String) {
|
||||
saveGatewayPassword(value)
|
||||
}
|
||||
|
||||
fun setOnboardingCompleted(value: Boolean) {
|
||||
plainPrefs.edit { putBoolean("onboarding.completed", value) }
|
||||
_onboardingCompleted.value = value
|
||||
}
|
||||
|
||||
fun setCanvasDebugStatusEnabled(value: Boolean) {
|
||||
plainPrefs.edit { putBoolean("canvas.debugStatusEnabled", value) }
|
||||
_canvasDebugStatusEnabled.value = value
|
||||
}
|
||||
|
||||
fun loadGatewayToken(): String? {
|
||||
val manual =
|
||||
_gatewayToken.value.trim().ifEmpty {
|
||||
val stored = securePrefs.getString("gateway.manual.token", null)?.trim().orEmpty()
|
||||
if (stored.isNotEmpty()) _gatewayToken.value = stored
|
||||
stored
|
||||
}
|
||||
if (manual.isNotEmpty()) return manual
|
||||
val key = "gateway.token.${_instanceId.value}"
|
||||
val stored = securePrefs.getString(key, null)?.trim()
|
||||
return stored?.takeIf { it.isNotEmpty() }
|
||||
}
|
||||
|
||||
fun saveGatewayToken(token: String) {
|
||||
val key = "gateway.token.${_instanceId.value}"
|
||||
securePrefs.edit { putString(key, token.trim()) }
|
||||
}
|
||||
|
||||
fun loadGatewayPassword(): String? {
|
||||
val key = "gateway.password.${_instanceId.value}"
|
||||
val stored = securePrefs.getString(key, null)?.trim()
|
||||
return stored?.takeIf { it.isNotEmpty() }
|
||||
}
|
||||
|
||||
fun saveGatewayPassword(password: String) {
|
||||
val key = "gateway.password.${_instanceId.value}"
|
||||
securePrefs.edit { putString(key, password.trim()) }
|
||||
}
|
||||
|
||||
fun loadGatewayTlsFingerprint(stableId: String): String? {
|
||||
val key = "gateway.tls.$stableId"
|
||||
return plainPrefs.getString(key, null)?.trim()?.takeIf { it.isNotEmpty() }
|
||||
}
|
||||
|
||||
fun saveGatewayTlsFingerprint(stableId: String, fingerprint: String) {
|
||||
val key = "gateway.tls.$stableId"
|
||||
plainPrefs.edit { putString(key, fingerprint.trim()) }
|
||||
}
|
||||
|
||||
fun getString(key: String): String? {
|
||||
return securePrefs.getString(key, null)
|
||||
}
|
||||
|
||||
fun putString(key: String, value: String) {
|
||||
securePrefs.edit { putString(key, value) }
|
||||
}
|
||||
|
||||
fun remove(key: String) {
|
||||
securePrefs.edit { remove(key) }
|
||||
}
|
||||
|
||||
private fun createSecurePrefs(context: Context, name: String): SharedPreferences {
|
||||
return EncryptedSharedPreferences.create(
|
||||
context,
|
||||
name,
|
||||
masterKey,
|
||||
EncryptedSharedPreferences.PrefKeyEncryptionScheme.AES256_SIV,
|
||||
EncryptedSharedPreferences.PrefValueEncryptionScheme.AES256_GCM,
|
||||
)
|
||||
}
|
||||
|
||||
private fun loadOrCreateInstanceId(): String {
|
||||
val existing = plainPrefs.getString("node.instanceId", null)?.trim()
|
||||
if (!existing.isNullOrBlank()) return existing
|
||||
val fresh = UUID.randomUUID().toString()
|
||||
plainPrefs.edit { putString("node.instanceId", fresh) }
|
||||
return fresh
|
||||
}
|
||||
|
||||
private fun loadOrMigrateDisplayName(context: Context): String {
|
||||
val existing = plainPrefs.getString(displayNameKey, null)?.trim().orEmpty()
|
||||
if (existing.isNotEmpty() && existing != "Android Node") return existing
|
||||
|
||||
val candidate = DeviceNames.bestDefaultNodeName(context).trim()
|
||||
val resolved = candidate.ifEmpty { "Android Node" }
|
||||
|
||||
plainPrefs.edit { putString(displayNameKey, resolved) }
|
||||
return resolved
|
||||
}
|
||||
|
||||
fun setWakeWords(words: List<String>) {
|
||||
val sanitized = WakeWords.sanitize(words, defaultWakeWords)
|
||||
val encoded =
|
||||
JsonArray(sanitized.map { JsonPrimitive(it) }).toString()
|
||||
plainPrefs.edit { putString("voiceWake.triggerWords", encoded) }
|
||||
_wakeWords.value = sanitized
|
||||
}
|
||||
|
||||
fun setVoiceWakeMode(mode: VoiceWakeMode) {
|
||||
plainPrefs.edit { putString(voiceWakeModeKey, mode.rawValue) }
|
||||
_voiceWakeMode.value = mode
|
||||
}
|
||||
|
||||
fun setTalkEnabled(value: Boolean) {
|
||||
plainPrefs.edit { putBoolean("talk.enabled", value) }
|
||||
_talkEnabled.value = value
|
||||
}
|
||||
|
||||
private fun loadVoiceWakeMode(): VoiceWakeMode {
|
||||
val raw = plainPrefs.getString(voiceWakeModeKey, null)
|
||||
val resolved = VoiceWakeMode.fromRawValue(raw)
|
||||
|
||||
// Default ON (foreground) when unset.
|
||||
if (raw.isNullOrBlank()) {
|
||||
plainPrefs.edit { putString(voiceWakeModeKey, resolved.rawValue) }
|
||||
}
|
||||
|
||||
return resolved
|
||||
}
|
||||
|
||||
private fun loadWakeWords(): List<String> {
|
||||
val raw = plainPrefs.getString("voiceWake.triggerWords", null)?.trim()
|
||||
if (raw.isNullOrEmpty()) return defaultWakeWords
|
||||
return try {
|
||||
val element = json.parseToJsonElement(raw)
|
||||
val array = element as? JsonArray ?: return defaultWakeWords
|
||||
val decoded =
|
||||
array.mapNotNull { item ->
|
||||
when (item) {
|
||||
is JsonNull -> null
|
||||
is JsonPrimitive -> item.content.trim().takeIf { it.isNotEmpty() }
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
WakeWords.sanitize(decoded, defaultWakeWords)
|
||||
} catch (_: Throwable) {
|
||||
defaultWakeWords
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
internal fun normalizeMainKey(raw: String?): String {
|
||||
val trimmed = raw?.trim()
|
||||
return if (!trimmed.isNullOrEmpty()) trimmed else "main"
|
||||
}
|
||||
|
||||
internal fun isCanonicalMainSessionKey(raw: String?): Boolean {
|
||||
val trimmed = raw?.trim().orEmpty()
|
||||
if (trimmed.isEmpty()) return false
|
||||
if (trimmed == "global") return true
|
||||
return trimmed.startsWith("agent:")
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
enum class VoiceWakeMode(val rawValue: String) {
|
||||
Off("off"),
|
||||
Foreground("foreground"),
|
||||
Always("always"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
fun fromRawValue(raw: String?): VoiceWakeMode {
|
||||
return entries.firstOrNull { it.rawValue == raw?.trim()?.lowercase() } ?: Foreground
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package ai.openclaw.android
|
||||
|
||||
object WakeWords {
|
||||
const val maxWords: Int = 32
|
||||
const val maxWordLength: Int = 64
|
||||
|
||||
fun parseCommaSeparated(input: String): List<String> {
|
||||
return input.split(",").map { it.trim() }.filter { it.isNotEmpty() }
|
||||
}
|
||||
|
||||
fun parseIfChanged(input: String, current: List<String>): List<String>? {
|
||||
val parsed = parseCommaSeparated(input)
|
||||
return if (parsed == current) null else parsed
|
||||
}
|
||||
|
||||
fun sanitize(words: List<String>, defaults: List<String>): List<String> {
|
||||
val cleaned =
|
||||
words.map { it.trim() }.filter { it.isNotEmpty() }.take(maxWords).map { it.take(maxWordLength) }
|
||||
return cleaned.ifEmpty { defaults }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,540 @@
|
||||
package ai.openclaw.android.chat
|
||||
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import java.util.UUID
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import kotlinx.serialization.json.JsonNull
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
|
||||
class ChatController(
|
||||
private val scope: CoroutineScope,
|
||||
private val session: GatewaySession,
|
||||
private val json: Json,
|
||||
private val supportsChatSubscribe: Boolean,
|
||||
) {
|
||||
private val _sessionKey = MutableStateFlow("main")
|
||||
val sessionKey: StateFlow<String> = _sessionKey.asStateFlow()
|
||||
|
||||
private val _sessionId = MutableStateFlow<String?>(null)
|
||||
val sessionId: StateFlow<String?> = _sessionId.asStateFlow()
|
||||
|
||||
private val _messages = MutableStateFlow<List<ChatMessage>>(emptyList())
|
||||
val messages: StateFlow<List<ChatMessage>> = _messages.asStateFlow()
|
||||
|
||||
private val _errorText = MutableStateFlow<String?>(null)
|
||||
val errorText: StateFlow<String?> = _errorText.asStateFlow()
|
||||
|
||||
private val _healthOk = MutableStateFlow(false)
|
||||
val healthOk: StateFlow<Boolean> = _healthOk.asStateFlow()
|
||||
|
||||
private val _thinkingLevel = MutableStateFlow("off")
|
||||
val thinkingLevel: StateFlow<String> = _thinkingLevel.asStateFlow()
|
||||
|
||||
private val _pendingRunCount = MutableStateFlow(0)
|
||||
val pendingRunCount: StateFlow<Int> = _pendingRunCount.asStateFlow()
|
||||
|
||||
private val _streamingAssistantText = MutableStateFlow<String?>(null)
|
||||
val streamingAssistantText: StateFlow<String?> = _streamingAssistantText.asStateFlow()
|
||||
|
||||
private val pendingToolCallsById = ConcurrentHashMap<String, ChatPendingToolCall>()
|
||||
private val _pendingToolCalls = MutableStateFlow<List<ChatPendingToolCall>>(emptyList())
|
||||
val pendingToolCalls: StateFlow<List<ChatPendingToolCall>> = _pendingToolCalls.asStateFlow()
|
||||
|
||||
private val _sessions = MutableStateFlow<List<ChatSessionEntry>>(emptyList())
|
||||
val sessions: StateFlow<List<ChatSessionEntry>> = _sessions.asStateFlow()
|
||||
|
||||
private val pendingRuns = mutableSetOf<String>()
|
||||
private val pendingRunTimeoutJobs = ConcurrentHashMap<String, Job>()
|
||||
private val pendingRunTimeoutMs = 120_000L
|
||||
|
||||
private var lastHealthPollAtMs: Long? = null
|
||||
|
||||
fun onDisconnected(message: String) {
|
||||
_healthOk.value = false
|
||||
// Not an error; keep connection status in the UI pill.
|
||||
_errorText.value = null
|
||||
clearPendingRuns()
|
||||
pendingToolCallsById.clear()
|
||||
publishPendingToolCalls()
|
||||
_streamingAssistantText.value = null
|
||||
_sessionId.value = null
|
||||
}
|
||||
|
||||
fun load(sessionKey: String) {
|
||||
val key = sessionKey.trim().ifEmpty { "main" }
|
||||
_sessionKey.value = key
|
||||
scope.launch { bootstrap(forceHealth = true) }
|
||||
}
|
||||
|
||||
fun applyMainSessionKey(mainSessionKey: String) {
|
||||
val trimmed = mainSessionKey.trim()
|
||||
if (trimmed.isEmpty()) return
|
||||
if (_sessionKey.value == trimmed) return
|
||||
if (_sessionKey.value != "main") return
|
||||
_sessionKey.value = trimmed
|
||||
scope.launch { bootstrap(forceHealth = true) }
|
||||
}
|
||||
|
||||
fun refresh() {
|
||||
scope.launch { bootstrap(forceHealth = true) }
|
||||
}
|
||||
|
||||
fun refreshSessions(limit: Int? = null) {
|
||||
scope.launch { fetchSessions(limit = limit) }
|
||||
}
|
||||
|
||||
fun setThinkingLevel(thinkingLevel: String) {
|
||||
val normalized = normalizeThinking(thinkingLevel)
|
||||
if (normalized == _thinkingLevel.value) return
|
||||
_thinkingLevel.value = normalized
|
||||
}
|
||||
|
||||
fun switchSession(sessionKey: String) {
|
||||
val key = sessionKey.trim()
|
||||
if (key.isEmpty()) return
|
||||
if (key == _sessionKey.value) return
|
||||
_sessionKey.value = key
|
||||
scope.launch { bootstrap(forceHealth = true) }
|
||||
}
|
||||
|
||||
fun sendMessage(
|
||||
message: String,
|
||||
thinkingLevel: String,
|
||||
attachments: List<OutgoingAttachment>,
|
||||
) {
|
||||
val trimmed = message.trim()
|
||||
if (trimmed.isEmpty() && attachments.isEmpty()) return
|
||||
if (!_healthOk.value) {
|
||||
_errorText.value = "Gateway health not OK; cannot send"
|
||||
return
|
||||
}
|
||||
|
||||
val runId = UUID.randomUUID().toString()
|
||||
val text = if (trimmed.isEmpty() && attachments.isNotEmpty()) "See attached." else trimmed
|
||||
val sessionKey = _sessionKey.value
|
||||
val thinking = normalizeThinking(thinkingLevel)
|
||||
|
||||
// Optimistic user message.
|
||||
val userContent =
|
||||
buildList {
|
||||
add(ChatMessageContent(type = "text", text = text))
|
||||
for (att in attachments) {
|
||||
add(
|
||||
ChatMessageContent(
|
||||
type = att.type,
|
||||
mimeType = att.mimeType,
|
||||
fileName = att.fileName,
|
||||
base64 = att.base64,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
_messages.value =
|
||||
_messages.value +
|
||||
ChatMessage(
|
||||
id = UUID.randomUUID().toString(),
|
||||
role = "user",
|
||||
content = userContent,
|
||||
timestampMs = System.currentTimeMillis(),
|
||||
)
|
||||
|
||||
armPendingRunTimeout(runId)
|
||||
synchronized(pendingRuns) {
|
||||
pendingRuns.add(runId)
|
||||
_pendingRunCount.value = pendingRuns.size
|
||||
}
|
||||
|
||||
_errorText.value = null
|
||||
_streamingAssistantText.value = null
|
||||
pendingToolCallsById.clear()
|
||||
publishPendingToolCalls()
|
||||
|
||||
scope.launch {
|
||||
try {
|
||||
val params =
|
||||
buildJsonObject {
|
||||
put("sessionKey", JsonPrimitive(sessionKey))
|
||||
put("message", JsonPrimitive(text))
|
||||
put("thinking", JsonPrimitive(thinking))
|
||||
put("timeoutMs", JsonPrimitive(30_000))
|
||||
put("idempotencyKey", JsonPrimitive(runId))
|
||||
if (attachments.isNotEmpty()) {
|
||||
put(
|
||||
"attachments",
|
||||
JsonArray(
|
||||
attachments.map { att ->
|
||||
buildJsonObject {
|
||||
put("type", JsonPrimitive(att.type))
|
||||
put("mimeType", JsonPrimitive(att.mimeType))
|
||||
put("fileName", JsonPrimitive(att.fileName))
|
||||
put("content", JsonPrimitive(att.base64))
|
||||
}
|
||||
},
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
val res = session.request("chat.send", params.toString())
|
||||
val actualRunId = parseRunId(res) ?: runId
|
||||
if (actualRunId != runId) {
|
||||
clearPendingRun(runId)
|
||||
armPendingRunTimeout(actualRunId)
|
||||
synchronized(pendingRuns) {
|
||||
pendingRuns.add(actualRunId)
|
||||
_pendingRunCount.value = pendingRuns.size
|
||||
}
|
||||
}
|
||||
} catch (err: Throwable) {
|
||||
clearPendingRun(runId)
|
||||
_errorText.value = err.message
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun abort() {
|
||||
val runIds =
|
||||
synchronized(pendingRuns) {
|
||||
pendingRuns.toList()
|
||||
}
|
||||
if (runIds.isEmpty()) return
|
||||
scope.launch {
|
||||
for (runId in runIds) {
|
||||
try {
|
||||
val params =
|
||||
buildJsonObject {
|
||||
put("sessionKey", JsonPrimitive(_sessionKey.value))
|
||||
put("runId", JsonPrimitive(runId))
|
||||
}
|
||||
session.request("chat.abort", params.toString())
|
||||
} catch (_: Throwable) {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun handleGatewayEvent(event: String, payloadJson: String?) {
|
||||
when (event) {
|
||||
"tick" -> {
|
||||
scope.launch { pollHealthIfNeeded(force = false) }
|
||||
}
|
||||
"health" -> {
|
||||
// If we receive a health snapshot, the gateway is reachable.
|
||||
_healthOk.value = true
|
||||
}
|
||||
"seqGap" -> {
|
||||
_errorText.value = "Event stream interrupted; try refreshing."
|
||||
clearPendingRuns()
|
||||
}
|
||||
"chat" -> {
|
||||
if (payloadJson.isNullOrBlank()) return
|
||||
handleChatEvent(payloadJson)
|
||||
}
|
||||
"agent" -> {
|
||||
if (payloadJson.isNullOrBlank()) return
|
||||
handleAgentEvent(payloadJson)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun bootstrap(forceHealth: Boolean) {
|
||||
_errorText.value = null
|
||||
_healthOk.value = false
|
||||
clearPendingRuns()
|
||||
pendingToolCallsById.clear()
|
||||
publishPendingToolCalls()
|
||||
_streamingAssistantText.value = null
|
||||
_sessionId.value = null
|
||||
|
||||
val key = _sessionKey.value
|
||||
try {
|
||||
if (supportsChatSubscribe) {
|
||||
session.sendNodeEvent("chat.subscribe", """{"sessionKey":"$key"}""")
|
||||
}
|
||||
|
||||
val historyJson = session.request("chat.history", """{"sessionKey":"$key"}""")
|
||||
val history = parseHistory(historyJson, sessionKey = key)
|
||||
_messages.value = history.messages
|
||||
_sessionId.value = history.sessionId
|
||||
history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it }
|
||||
|
||||
pollHealthIfNeeded(force = forceHealth)
|
||||
fetchSessions(limit = 50)
|
||||
} catch (err: Throwable) {
|
||||
_errorText.value = err.message
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun fetchSessions(limit: Int?) {
|
||||
try {
|
||||
val params =
|
||||
buildJsonObject {
|
||||
put("includeGlobal", JsonPrimitive(true))
|
||||
put("includeUnknown", JsonPrimitive(false))
|
||||
if (limit != null && limit > 0) put("limit", JsonPrimitive(limit))
|
||||
}
|
||||
val res = session.request("sessions.list", params.toString())
|
||||
_sessions.value = parseSessions(res)
|
||||
} catch (_: Throwable) {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun pollHealthIfNeeded(force: Boolean) {
|
||||
val now = System.currentTimeMillis()
|
||||
val last = lastHealthPollAtMs
|
||||
if (!force && last != null && now - last < 10_000) return
|
||||
lastHealthPollAtMs = now
|
||||
try {
|
||||
session.request("health", null)
|
||||
_healthOk.value = true
|
||||
} catch (_: Throwable) {
|
||||
_healthOk.value = false
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleChatEvent(payloadJson: String) {
|
||||
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
|
||||
val sessionKey = payload["sessionKey"].asStringOrNull()?.trim()
|
||||
if (!sessionKey.isNullOrEmpty() && sessionKey != _sessionKey.value) return
|
||||
|
||||
val runId = payload["runId"].asStringOrNull()
|
||||
if (runId != null) {
|
||||
val isPending =
|
||||
synchronized(pendingRuns) {
|
||||
pendingRuns.contains(runId)
|
||||
}
|
||||
if (!isPending) return
|
||||
}
|
||||
|
||||
val state = payload["state"].asStringOrNull()
|
||||
when (state) {
|
||||
"delta" -> {
|
||||
val text = parseAssistantDeltaText(payload)
|
||||
if (!text.isNullOrEmpty()) {
|
||||
_streamingAssistantText.value = text
|
||||
}
|
||||
}
|
||||
"final", "aborted", "error" -> {
|
||||
if (state == "error") {
|
||||
_errorText.value = payload["errorMessage"].asStringOrNull() ?: "Chat failed"
|
||||
}
|
||||
if (runId != null) clearPendingRun(runId) else clearPendingRuns()
|
||||
pendingToolCallsById.clear()
|
||||
publishPendingToolCalls()
|
||||
_streamingAssistantText.value = null
|
||||
scope.launch {
|
||||
try {
|
||||
val historyJson =
|
||||
session.request("chat.history", """{"sessionKey":"${_sessionKey.value}"}""")
|
||||
val history = parseHistory(historyJson, sessionKey = _sessionKey.value)
|
||||
_messages.value = history.messages
|
||||
_sessionId.value = history.sessionId
|
||||
history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it }
|
||||
} catch (_: Throwable) {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleAgentEvent(payloadJson: String) {
|
||||
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
|
||||
val sessionKey = payload["sessionKey"].asStringOrNull()?.trim()
|
||||
if (!sessionKey.isNullOrEmpty() && sessionKey != _sessionKey.value) return
|
||||
|
||||
val stream = payload["stream"].asStringOrNull()
|
||||
val data = payload["data"].asObjectOrNull()
|
||||
|
||||
when (stream) {
|
||||
"assistant" -> {
|
||||
val text = data?.get("text")?.asStringOrNull()
|
||||
if (!text.isNullOrEmpty()) {
|
||||
_streamingAssistantText.value = text
|
||||
}
|
||||
}
|
||||
"tool" -> {
|
||||
val phase = data?.get("phase")?.asStringOrNull()
|
||||
val name = data?.get("name")?.asStringOrNull()
|
||||
val toolCallId = data?.get("toolCallId")?.asStringOrNull()
|
||||
if (phase.isNullOrEmpty() || name.isNullOrEmpty() || toolCallId.isNullOrEmpty()) return
|
||||
|
||||
val ts = payload["ts"].asLongOrNull() ?: System.currentTimeMillis()
|
||||
if (phase == "start") {
|
||||
val args = data?.get("args").asObjectOrNull()
|
||||
pendingToolCallsById[toolCallId] =
|
||||
ChatPendingToolCall(
|
||||
toolCallId = toolCallId,
|
||||
name = name,
|
||||
args = args,
|
||||
startedAtMs = ts,
|
||||
isError = null,
|
||||
)
|
||||
publishPendingToolCalls()
|
||||
} else if (phase == "result") {
|
||||
pendingToolCallsById.remove(toolCallId)
|
||||
publishPendingToolCalls()
|
||||
}
|
||||
}
|
||||
"error" -> {
|
||||
_errorText.value = "Event stream interrupted; try refreshing."
|
||||
clearPendingRuns()
|
||||
pendingToolCallsById.clear()
|
||||
publishPendingToolCalls()
|
||||
_streamingAssistantText.value = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseAssistantDeltaText(payload: JsonObject): String? {
|
||||
val message = payload["message"].asObjectOrNull() ?: return null
|
||||
if (message["role"].asStringOrNull() != "assistant") return null
|
||||
val content = message["content"].asArrayOrNull() ?: return null
|
||||
for (item in content) {
|
||||
val obj = item.asObjectOrNull() ?: continue
|
||||
if (obj["type"].asStringOrNull() != "text") continue
|
||||
val text = obj["text"].asStringOrNull()
|
||||
if (!text.isNullOrEmpty()) {
|
||||
return text
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun publishPendingToolCalls() {
|
||||
_pendingToolCalls.value =
|
||||
pendingToolCallsById.values.sortedBy { it.startedAtMs }
|
||||
}
|
||||
|
||||
private fun armPendingRunTimeout(runId: String) {
|
||||
pendingRunTimeoutJobs[runId]?.cancel()
|
||||
pendingRunTimeoutJobs[runId] =
|
||||
scope.launch {
|
||||
delay(pendingRunTimeoutMs)
|
||||
val stillPending =
|
||||
synchronized(pendingRuns) {
|
||||
pendingRuns.contains(runId)
|
||||
}
|
||||
if (!stillPending) return@launch
|
||||
clearPendingRun(runId)
|
||||
_errorText.value = "Timed out waiting for a reply; try again or refresh."
|
||||
}
|
||||
}
|
||||
|
||||
private fun clearPendingRun(runId: String) {
|
||||
pendingRunTimeoutJobs.remove(runId)?.cancel()
|
||||
synchronized(pendingRuns) {
|
||||
pendingRuns.remove(runId)
|
||||
_pendingRunCount.value = pendingRuns.size
|
||||
}
|
||||
}
|
||||
|
||||
private fun clearPendingRuns() {
|
||||
for ((_, job) in pendingRunTimeoutJobs) {
|
||||
job.cancel()
|
||||
}
|
||||
pendingRunTimeoutJobs.clear()
|
||||
synchronized(pendingRuns) {
|
||||
pendingRuns.clear()
|
||||
_pendingRunCount.value = 0
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseHistory(historyJson: String, sessionKey: String): ChatHistory {
|
||||
val root = json.parseToJsonElement(historyJson).asObjectOrNull() ?: return ChatHistory(sessionKey, null, null, emptyList())
|
||||
val sid = root["sessionId"].asStringOrNull()
|
||||
val thinkingLevel = root["thinkingLevel"].asStringOrNull()
|
||||
val array = root["messages"].asArrayOrNull() ?: JsonArray(emptyList())
|
||||
|
||||
val messages =
|
||||
array.mapNotNull { item ->
|
||||
val obj = item.asObjectOrNull() ?: return@mapNotNull null
|
||||
val role = obj["role"].asStringOrNull() ?: return@mapNotNull null
|
||||
val content = obj["content"].asArrayOrNull()?.mapNotNull(::parseMessageContent) ?: emptyList()
|
||||
val ts = obj["timestamp"].asLongOrNull()
|
||||
ChatMessage(
|
||||
id = UUID.randomUUID().toString(),
|
||||
role = role,
|
||||
content = content,
|
||||
timestampMs = ts,
|
||||
)
|
||||
}
|
||||
|
||||
return ChatHistory(sessionKey = sessionKey, sessionId = sid, thinkingLevel = thinkingLevel, messages = messages)
|
||||
}
|
||||
|
||||
private fun parseMessageContent(el: JsonElement): ChatMessageContent? {
|
||||
val obj = el.asObjectOrNull() ?: return null
|
||||
val type = obj["type"].asStringOrNull() ?: "text"
|
||||
return if (type == "text") {
|
||||
ChatMessageContent(type = "text", text = obj["text"].asStringOrNull())
|
||||
} else {
|
||||
ChatMessageContent(
|
||||
type = type,
|
||||
mimeType = obj["mimeType"].asStringOrNull(),
|
||||
fileName = obj["fileName"].asStringOrNull(),
|
||||
base64 = obj["content"].asStringOrNull(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseSessions(jsonString: String): List<ChatSessionEntry> {
|
||||
val root = json.parseToJsonElement(jsonString).asObjectOrNull() ?: return emptyList()
|
||||
val sessions = root["sessions"].asArrayOrNull() ?: return emptyList()
|
||||
return sessions.mapNotNull { item ->
|
||||
val obj = item.asObjectOrNull() ?: return@mapNotNull null
|
||||
val key = obj["key"].asStringOrNull()?.trim().orEmpty()
|
||||
if (key.isEmpty()) return@mapNotNull null
|
||||
val updatedAt = obj["updatedAt"].asLongOrNull()
|
||||
val displayName = obj["displayName"].asStringOrNull()?.trim()
|
||||
ChatSessionEntry(key = key, updatedAtMs = updatedAt, displayName = displayName)
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseRunId(resJson: String): String? {
|
||||
return try {
|
||||
json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun normalizeThinking(raw: String): String {
|
||||
return when (raw.trim().lowercase()) {
|
||||
"low" -> "low"
|
||||
"medium" -> "medium"
|
||||
"high" -> "high"
|
||||
else -> "off"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
|
||||
|
||||
private fun JsonElement?.asArrayOrNull(): JsonArray? = this as? JsonArray
|
||||
|
||||
private fun JsonElement?.asStringOrNull(): String? =
|
||||
when (this) {
|
||||
is JsonNull -> null
|
||||
is JsonPrimitive -> content
|
||||
else -> null
|
||||
}
|
||||
|
||||
private fun JsonElement?.asLongOrNull(): Long? =
|
||||
when (this) {
|
||||
is JsonPrimitive -> content.toLongOrNull()
|
||||
else -> null
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package ai.openclaw.android.chat
|
||||
|
||||
data class ChatMessage(
|
||||
val id: String,
|
||||
val role: String,
|
||||
val content: List<ChatMessageContent>,
|
||||
val timestampMs: Long?,
|
||||
)
|
||||
|
||||
data class ChatMessageContent(
|
||||
val type: String = "text",
|
||||
val text: String? = null,
|
||||
val mimeType: String? = null,
|
||||
val fileName: String? = null,
|
||||
val base64: String? = null,
|
||||
)
|
||||
|
||||
data class ChatPendingToolCall(
|
||||
val toolCallId: String,
|
||||
val name: String,
|
||||
val args: kotlinx.serialization.json.JsonObject? = null,
|
||||
val startedAtMs: Long,
|
||||
val isError: Boolean? = null,
|
||||
)
|
||||
|
||||
data class ChatSessionEntry(
|
||||
val key: String,
|
||||
val updatedAtMs: Long?,
|
||||
val displayName: String? = null,
|
||||
)
|
||||
|
||||
data class ChatHistory(
|
||||
val sessionKey: String,
|
||||
val sessionId: String?,
|
||||
val thinkingLevel: String?,
|
||||
val messages: List<ChatMessage>,
|
||||
)
|
||||
|
||||
data class OutgoingAttachment(
|
||||
val type: String,
|
||||
val mimeType: String,
|
||||
val fileName: String,
|
||||
val base64: String,
|
||||
)
|
||||
@@ -0,0 +1,35 @@
|
||||
package ai.openclaw.android.gateway
|
||||
|
||||
object BonjourEscapes {
|
||||
fun decode(input: String): String {
|
||||
if (input.isEmpty()) return input
|
||||
|
||||
val bytes = mutableListOf<Byte>()
|
||||
var i = 0
|
||||
while (i < input.length) {
|
||||
if (input[i] == '\\' && i + 3 < input.length) {
|
||||
val d0 = input[i + 1]
|
||||
val d1 = input[i + 2]
|
||||
val d2 = input[i + 3]
|
||||
if (d0.isDigit() && d1.isDigit() && d2.isDigit()) {
|
||||
val value =
|
||||
((d0.code - '0'.code) * 100) + ((d1.code - '0'.code) * 10) + (d2.code - '0'.code)
|
||||
if (value in 0..255) {
|
||||
bytes.add(value.toByte())
|
||||
i += 4
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val codePoint = Character.codePointAt(input, i)
|
||||
val charBytes = String(Character.toChars(codePoint)).toByteArray(Charsets.UTF_8)
|
||||
for (b in charBytes) {
|
||||
bytes.add(b)
|
||||
}
|
||||
i += Character.charCount(codePoint)
|
||||
}
|
||||
|
||||
return String(bytes.toByteArray(), Charsets.UTF_8)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package ai.openclaw.android.gateway
|
||||
|
||||
internal object DeviceAuthPayload {
|
||||
fun buildV3(
|
||||
deviceId: String,
|
||||
clientId: String,
|
||||
clientMode: String,
|
||||
role: String,
|
||||
scopes: List<String>,
|
||||
signedAtMs: Long,
|
||||
token: String?,
|
||||
nonce: String,
|
||||
platform: String?,
|
||||
deviceFamily: String?,
|
||||
): String {
|
||||
val scopeString = scopes.joinToString(",")
|
||||
val authToken = token.orEmpty()
|
||||
val platformNorm = normalizeMetadataField(platform)
|
||||
val deviceFamilyNorm = normalizeMetadataField(deviceFamily)
|
||||
return listOf(
|
||||
"v3",
|
||||
deviceId,
|
||||
clientId,
|
||||
clientMode,
|
||||
role,
|
||||
scopeString,
|
||||
signedAtMs.toString(),
|
||||
authToken,
|
||||
nonce,
|
||||
platformNorm,
|
||||
deviceFamilyNorm,
|
||||
).joinToString("|")
|
||||
}
|
||||
|
||||
internal fun normalizeMetadataField(value: String?): String {
|
||||
val trimmed = value?.trim().orEmpty()
|
||||
if (trimmed.isEmpty()) {
|
||||
return ""
|
||||
}
|
||||
// Keep cross-runtime normalization deterministic (TS/Swift/Kotlin):
|
||||
// lowercase ASCII A-Z only for auth payload metadata fields.
|
||||
val out = StringBuilder(trimmed.length)
|
||||
for (ch in trimmed) {
|
||||
if (ch in 'A'..'Z') {
|
||||
out.append((ch.code + 32).toChar())
|
||||
} else {
|
||||
out.append(ch)
|
||||
}
|
||||
}
|
||||
return out.toString()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package ai.openclaw.android.gateway
|
||||
|
||||
import ai.openclaw.android.SecurePrefs
|
||||
|
||||
interface DeviceAuthTokenStore {
|
||||
fun loadToken(deviceId: String, role: String): String?
|
||||
fun saveToken(deviceId: String, role: String, token: String)
|
||||
}
|
||||
|
||||
class DeviceAuthStore(private val prefs: SecurePrefs) : DeviceAuthTokenStore {
|
||||
override fun loadToken(deviceId: String, role: String): String? {
|
||||
val key = tokenKey(deviceId, role)
|
||||
return prefs.getString(key)?.trim()?.takeIf { it.isNotEmpty() }
|
||||
}
|
||||
|
||||
override fun saveToken(deviceId: String, role: String, token: String) {
|
||||
val key = tokenKey(deviceId, role)
|
||||
prefs.putString(key, token.trim())
|
||||
}
|
||||
|
||||
fun clearToken(deviceId: String, role: String) {
|
||||
val key = tokenKey(deviceId, role)
|
||||
prefs.remove(key)
|
||||
}
|
||||
|
||||
private fun tokenKey(deviceId: String, role: String): String {
|
||||
val normalizedDevice = deviceId.trim().lowercase()
|
||||
val normalizedRole = role.trim().lowercase()
|
||||
return "gateway.deviceToken.$normalizedDevice.$normalizedRole"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package ai.openclaw.android.gateway
|
||||
|
||||
import android.content.Context
|
||||
import android.util.Base64
|
||||
import java.io.File
|
||||
import java.security.MessageDigest
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.Json
|
||||
|
||||
@Serializable
|
||||
data class DeviceIdentity(
|
||||
val deviceId: String,
|
||||
val publicKeyRawBase64: String,
|
||||
val privateKeyPkcs8Base64: String,
|
||||
val createdAtMs: Long,
|
||||
)
|
||||
|
||||
class DeviceIdentityStore(context: Context) {
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
private val identityFile = File(context.filesDir, "openclaw/identity/device.json")
|
||||
@Volatile private var cachedIdentity: DeviceIdentity? = null
|
||||
|
||||
@Synchronized
|
||||
fun loadOrCreate(): DeviceIdentity {
|
||||
cachedIdentity?.let { return it }
|
||||
val existing = load()
|
||||
if (existing != null) {
|
||||
val derived = deriveDeviceId(existing.publicKeyRawBase64)
|
||||
if (derived != null && derived != existing.deviceId) {
|
||||
val updated = existing.copy(deviceId = derived)
|
||||
save(updated)
|
||||
cachedIdentity = updated
|
||||
return updated
|
||||
}
|
||||
cachedIdentity = existing
|
||||
return existing
|
||||
}
|
||||
val fresh = generate()
|
||||
save(fresh)
|
||||
cachedIdentity = fresh
|
||||
return fresh
|
||||
}
|
||||
|
||||
fun signPayload(payload: String, identity: DeviceIdentity): String? {
|
||||
return try {
|
||||
// Use BC lightweight API directly — JCA provider registration is broken by R8
|
||||
val privateKeyBytes = Base64.decode(identity.privateKeyPkcs8Base64, Base64.DEFAULT)
|
||||
val pkInfo = org.bouncycastle.asn1.pkcs.PrivateKeyInfo.getInstance(privateKeyBytes)
|
||||
val parsed = pkInfo.parsePrivateKey()
|
||||
val rawPrivate = org.bouncycastle.asn1.DEROctetString.getInstance(parsed).octets
|
||||
val privateKey = org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters(rawPrivate, 0)
|
||||
val signer = org.bouncycastle.crypto.signers.Ed25519Signer()
|
||||
signer.init(true, privateKey)
|
||||
val payloadBytes = payload.toByteArray(Charsets.UTF_8)
|
||||
signer.update(payloadBytes, 0, payloadBytes.size)
|
||||
base64UrlEncode(signer.generateSignature())
|
||||
} catch (e: Throwable) {
|
||||
android.util.Log.e("DeviceAuth", "signPayload FAILED: ${e.javaClass.simpleName}: ${e.message}", e)
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
fun verifySelfSignature(payload: String, signatureBase64Url: String, identity: DeviceIdentity): Boolean {
|
||||
return try {
|
||||
val rawPublicKey = Base64.decode(identity.publicKeyRawBase64, Base64.DEFAULT)
|
||||
val pubKey = org.bouncycastle.crypto.params.Ed25519PublicKeyParameters(rawPublicKey, 0)
|
||||
val sigBytes = base64UrlDecode(signatureBase64Url)
|
||||
val verifier = org.bouncycastle.crypto.signers.Ed25519Signer()
|
||||
verifier.init(false, pubKey)
|
||||
val payloadBytes = payload.toByteArray(Charsets.UTF_8)
|
||||
verifier.update(payloadBytes, 0, payloadBytes.size)
|
||||
verifier.verifySignature(sigBytes)
|
||||
} catch (e: Throwable) {
|
||||
android.util.Log.e("DeviceAuth", "self-verify exception: ${e.message}", e)
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
private fun base64UrlDecode(input: String): ByteArray {
|
||||
val normalized = input.replace('-', '+').replace('_', '/')
|
||||
val padded = normalized + "=".repeat((4 - normalized.length % 4) % 4)
|
||||
return Base64.decode(padded, Base64.DEFAULT)
|
||||
}
|
||||
|
||||
fun publicKeyBase64Url(identity: DeviceIdentity): String? {
|
||||
return try {
|
||||
val raw = Base64.decode(identity.publicKeyRawBase64, Base64.DEFAULT)
|
||||
base64UrlEncode(raw)
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun load(): DeviceIdentity? {
|
||||
return readIdentity(identityFile)
|
||||
}
|
||||
|
||||
private fun readIdentity(file: File): DeviceIdentity? {
|
||||
return try {
|
||||
if (!file.exists()) return null
|
||||
val raw = file.readText(Charsets.UTF_8)
|
||||
val decoded = json.decodeFromString(DeviceIdentity.serializer(), raw)
|
||||
if (decoded.deviceId.isBlank() ||
|
||||
decoded.publicKeyRawBase64.isBlank() ||
|
||||
decoded.privateKeyPkcs8Base64.isBlank()
|
||||
) {
|
||||
null
|
||||
} else {
|
||||
decoded
|
||||
}
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun save(identity: DeviceIdentity) {
|
||||
try {
|
||||
identityFile.parentFile?.mkdirs()
|
||||
val encoded = json.encodeToString(DeviceIdentity.serializer(), identity)
|
||||
identityFile.writeText(encoded, Charsets.UTF_8)
|
||||
} catch (_: Throwable) {
|
||||
// best-effort only
|
||||
}
|
||||
}
|
||||
|
||||
private fun generate(): DeviceIdentity {
|
||||
// Use BC lightweight API directly to avoid JCA provider issues with R8
|
||||
val kpGen = org.bouncycastle.crypto.generators.Ed25519KeyPairGenerator()
|
||||
kpGen.init(org.bouncycastle.crypto.params.Ed25519KeyGenerationParameters(java.security.SecureRandom()))
|
||||
val kp = kpGen.generateKeyPair()
|
||||
val pubKey = kp.public as org.bouncycastle.crypto.params.Ed25519PublicKeyParameters
|
||||
val privKey = kp.private as org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters
|
||||
val rawPublic = pubKey.encoded // 32 bytes
|
||||
val deviceId = sha256Hex(rawPublic)
|
||||
// Encode private key as PKCS8 for storage
|
||||
val privKeyInfo = org.bouncycastle.crypto.util.PrivateKeyInfoFactory.createPrivateKeyInfo(privKey)
|
||||
val pkcs8Bytes = privKeyInfo.encoded
|
||||
return DeviceIdentity(
|
||||
deviceId = deviceId,
|
||||
publicKeyRawBase64 = Base64.encodeToString(rawPublic, Base64.NO_WRAP),
|
||||
privateKeyPkcs8Base64 = Base64.encodeToString(pkcs8Bytes, Base64.NO_WRAP),
|
||||
createdAtMs = System.currentTimeMillis(),
|
||||
)
|
||||
}
|
||||
|
||||
private fun deriveDeviceId(publicKeyRawBase64: String): String? {
|
||||
return try {
|
||||
val raw = Base64.decode(publicKeyRawBase64, Base64.DEFAULT)
|
||||
sha256Hex(raw)
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun sha256Hex(data: ByteArray): String {
|
||||
val digest = MessageDigest.getInstance("SHA-256").digest(data)
|
||||
val out = CharArray(digest.size * 2)
|
||||
var i = 0
|
||||
for (byte in digest) {
|
||||
val v = byte.toInt() and 0xff
|
||||
out[i++] = HEX[v ushr 4]
|
||||
out[i++] = HEX[v and 0x0f]
|
||||
}
|
||||
return String(out)
|
||||
}
|
||||
|
||||
private fun base64UrlEncode(data: ByteArray): String {
|
||||
return Base64.encodeToString(data, Base64.URL_SAFE or Base64.NO_WRAP or Base64.NO_PADDING)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val HEX = "0123456789abcdef".toCharArray()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,521 @@
|
||||
package ai.openclaw.android.gateway
|
||||
|
||||
import android.content.Context
|
||||
import android.net.ConnectivityManager
|
||||
import android.net.DnsResolver
|
||||
import android.net.NetworkCapabilities
|
||||
import android.net.nsd.NsdManager
|
||||
import android.net.nsd.NsdServiceInfo
|
||||
import android.os.CancellationSignal
|
||||
import android.util.Log
|
||||
import java.io.IOException
|
||||
import java.net.InetSocketAddress
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.charset.CodingErrorAction
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.Executor
|
||||
import java.util.concurrent.Executors
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import org.xbill.DNS.AAAARecord
|
||||
import org.xbill.DNS.ARecord
|
||||
import org.xbill.DNS.DClass
|
||||
import org.xbill.DNS.ExtendedResolver
|
||||
import org.xbill.DNS.Message
|
||||
import org.xbill.DNS.Name
|
||||
import org.xbill.DNS.PTRRecord
|
||||
import org.xbill.DNS.Record
|
||||
import org.xbill.DNS.Rcode
|
||||
import org.xbill.DNS.Resolver
|
||||
import org.xbill.DNS.SRVRecord
|
||||
import org.xbill.DNS.Section
|
||||
import org.xbill.DNS.SimpleResolver
|
||||
import org.xbill.DNS.TextParseException
|
||||
import org.xbill.DNS.TXTRecord
|
||||
import org.xbill.DNS.Type
|
||||
import kotlin.coroutines.resume
|
||||
import kotlin.coroutines.resumeWithException
|
||||
|
||||
@Suppress("DEPRECATION")
|
||||
class GatewayDiscovery(
|
||||
context: Context,
|
||||
private val scope: CoroutineScope,
|
||||
) {
|
||||
private val nsd = context.getSystemService(NsdManager::class.java)
|
||||
private val connectivity = context.getSystemService(ConnectivityManager::class.java)
|
||||
private val dns = DnsResolver.getInstance()
|
||||
private val serviceType = "_openclaw-gw._tcp."
|
||||
private val wideAreaDomain = System.getenv("OPENCLAW_WIDE_AREA_DOMAIN")
|
||||
private val logTag = "OpenClaw/GatewayDiscovery"
|
||||
|
||||
private val localById = ConcurrentHashMap<String, GatewayEndpoint>()
|
||||
private val unicastById = ConcurrentHashMap<String, GatewayEndpoint>()
|
||||
private val _gateways = MutableStateFlow<List<GatewayEndpoint>>(emptyList())
|
||||
val gateways: StateFlow<List<GatewayEndpoint>> = _gateways.asStateFlow()
|
||||
|
||||
private val _statusText = MutableStateFlow("Searching…")
|
||||
val statusText: StateFlow<String> = _statusText.asStateFlow()
|
||||
|
||||
private var unicastJob: Job? = null
|
||||
private val dnsExecutor: Executor = Executors.newCachedThreadPool()
|
||||
|
||||
@Volatile private var lastWideAreaRcode: Int? = null
|
||||
@Volatile private var lastWideAreaCount: Int = 0
|
||||
|
||||
private val discoveryListener =
|
||||
object : NsdManager.DiscoveryListener {
|
||||
override fun onStartDiscoveryFailed(serviceType: String, errorCode: Int) {}
|
||||
override fun onStopDiscoveryFailed(serviceType: String, errorCode: Int) {}
|
||||
override fun onDiscoveryStarted(serviceType: String) {}
|
||||
override fun onDiscoveryStopped(serviceType: String) {}
|
||||
|
||||
override fun onServiceFound(serviceInfo: NsdServiceInfo) {
|
||||
if (serviceInfo.serviceType != this@GatewayDiscovery.serviceType) return
|
||||
resolve(serviceInfo)
|
||||
}
|
||||
|
||||
override fun onServiceLost(serviceInfo: NsdServiceInfo) {
|
||||
val serviceName = BonjourEscapes.decode(serviceInfo.serviceName)
|
||||
val id = stableId(serviceName, "local.")
|
||||
localById.remove(id)
|
||||
publish()
|
||||
}
|
||||
}
|
||||
|
||||
init {
|
||||
startLocalDiscovery()
|
||||
if (!wideAreaDomain.isNullOrBlank()) {
|
||||
startUnicastDiscovery(wideAreaDomain)
|
||||
}
|
||||
}
|
||||
|
||||
private fun startLocalDiscovery() {
|
||||
try {
|
||||
nsd.discoverServices(serviceType, NsdManager.PROTOCOL_DNS_SD, discoveryListener)
|
||||
} catch (_: Throwable) {
|
||||
// ignore (best-effort)
|
||||
}
|
||||
}
|
||||
|
||||
private fun stopLocalDiscovery() {
|
||||
try {
|
||||
nsd.stopServiceDiscovery(discoveryListener)
|
||||
} catch (_: Throwable) {
|
||||
// ignore (best-effort)
|
||||
}
|
||||
}
|
||||
|
||||
private fun startUnicastDiscovery(domain: String) {
|
||||
unicastJob =
|
||||
scope.launch(Dispatchers.IO) {
|
||||
while (true) {
|
||||
try {
|
||||
refreshUnicast(domain)
|
||||
} catch (_: Throwable) {
|
||||
// ignore (best-effort)
|
||||
}
|
||||
delay(5000)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun resolve(serviceInfo: NsdServiceInfo) {
|
||||
nsd.resolveService(
|
||||
serviceInfo,
|
||||
object : NsdManager.ResolveListener {
|
||||
override fun onResolveFailed(serviceInfo: NsdServiceInfo, errorCode: Int) {}
|
||||
|
||||
override fun onServiceResolved(resolved: NsdServiceInfo) {
|
||||
val host = resolved.host?.hostAddress ?: return
|
||||
val port = resolved.port
|
||||
if (port <= 0) return
|
||||
|
||||
val rawServiceName = resolved.serviceName
|
||||
val serviceName = BonjourEscapes.decode(rawServiceName)
|
||||
val displayName = BonjourEscapes.decode(txt(resolved, "displayName") ?: serviceName)
|
||||
val lanHost = txt(resolved, "lanHost")
|
||||
val tailnetDns = txt(resolved, "tailnetDns")
|
||||
val gatewayPort = txtInt(resolved, "gatewayPort")
|
||||
val canvasPort = txtInt(resolved, "canvasPort")
|
||||
val tlsEnabled = txtBool(resolved, "gatewayTls")
|
||||
val tlsFingerprint = txt(resolved, "gatewayTlsSha256")
|
||||
val id = stableId(serviceName, "local.")
|
||||
localById[id] =
|
||||
GatewayEndpoint(
|
||||
stableId = id,
|
||||
name = displayName,
|
||||
host = host,
|
||||
port = port,
|
||||
lanHost = lanHost,
|
||||
tailnetDns = tailnetDns,
|
||||
gatewayPort = gatewayPort,
|
||||
canvasPort = canvasPort,
|
||||
tlsEnabled = tlsEnabled,
|
||||
tlsFingerprintSha256 = tlsFingerprint,
|
||||
)
|
||||
publish()
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
private fun publish() {
|
||||
_gateways.value =
|
||||
(localById.values + unicastById.values).sortedBy { it.name.lowercase() }
|
||||
_statusText.value = buildStatusText()
|
||||
}
|
||||
|
||||
private fun buildStatusText(): String {
|
||||
val localCount = localById.size
|
||||
val wideRcode = lastWideAreaRcode
|
||||
val wideCount = lastWideAreaCount
|
||||
|
||||
val wide =
|
||||
when (wideRcode) {
|
||||
null -> "Wide: ?"
|
||||
Rcode.NOERROR -> "Wide: $wideCount"
|
||||
Rcode.NXDOMAIN -> "Wide: NXDOMAIN"
|
||||
else -> "Wide: ${Rcode.string(wideRcode)}"
|
||||
}
|
||||
|
||||
return when {
|
||||
localCount == 0 && wideRcode == null -> "Searching for gateways…"
|
||||
localCount == 0 -> "$wide"
|
||||
else -> "Local: $localCount • $wide"
|
||||
}
|
||||
}
|
||||
|
||||
private fun stableId(serviceName: String, domain: String): String {
|
||||
return "${serviceType}|${domain}|${normalizeName(serviceName)}"
|
||||
}
|
||||
|
||||
private fun normalizeName(raw: String): String {
|
||||
return raw.trim().split(Regex("\\s+")).joinToString(" ")
|
||||
}
|
||||
|
||||
private fun txt(info: NsdServiceInfo, key: String): String? {
|
||||
val bytes = info.attributes[key] ?: return null
|
||||
return try {
|
||||
String(bytes, Charsets.UTF_8).trim().ifEmpty { null }
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun txtInt(info: NsdServiceInfo, key: String): Int? {
|
||||
return txt(info, key)?.toIntOrNull()
|
||||
}
|
||||
|
||||
private fun txtBool(info: NsdServiceInfo, key: String): Boolean {
|
||||
val raw = txt(info, key)?.trim()?.lowercase() ?: return false
|
||||
return raw == "1" || raw == "true" || raw == "yes"
|
||||
}
|
||||
|
||||
private suspend fun refreshUnicast(domain: String) {
|
||||
val ptrName = "${serviceType}${domain}"
|
||||
val ptrMsg = lookupUnicastMessage(ptrName, Type.PTR) ?: return
|
||||
val ptrRecords = records(ptrMsg, Section.ANSWER).mapNotNull { it as? PTRRecord }
|
||||
|
||||
val next = LinkedHashMap<String, GatewayEndpoint>()
|
||||
for (ptr in ptrRecords) {
|
||||
val instanceFqdn = ptr.target.toString()
|
||||
val srv =
|
||||
recordByName(ptrMsg, instanceFqdn, Type.SRV) as? SRVRecord
|
||||
?: run {
|
||||
val msg = lookupUnicastMessage(instanceFqdn, Type.SRV) ?: return@run null
|
||||
recordByName(msg, instanceFqdn, Type.SRV) as? SRVRecord
|
||||
}
|
||||
?: continue
|
||||
val port = srv.port
|
||||
if (port <= 0) continue
|
||||
|
||||
val targetFqdn = srv.target.toString()
|
||||
val host =
|
||||
resolveHostFromMessage(ptrMsg, targetFqdn)
|
||||
?: resolveHostFromMessage(lookupUnicastMessage(instanceFqdn, Type.SRV), targetFqdn)
|
||||
?: resolveHostUnicast(targetFqdn)
|
||||
?: continue
|
||||
|
||||
val txtFromPtr =
|
||||
recordsByName(ptrMsg, Section.ADDITIONAL)[keyName(instanceFqdn)]
|
||||
.orEmpty()
|
||||
.mapNotNull { it as? TXTRecord }
|
||||
val txt =
|
||||
if (txtFromPtr.isNotEmpty()) {
|
||||
txtFromPtr
|
||||
} else {
|
||||
val msg = lookupUnicastMessage(instanceFqdn, Type.TXT)
|
||||
records(msg, Section.ANSWER).mapNotNull { it as? TXTRecord }
|
||||
}
|
||||
val instanceName = BonjourEscapes.decode(decodeInstanceName(instanceFqdn, domain))
|
||||
val displayName = BonjourEscapes.decode(txtValue(txt, "displayName") ?: instanceName)
|
||||
val lanHost = txtValue(txt, "lanHost")
|
||||
val tailnetDns = txtValue(txt, "tailnetDns")
|
||||
val gatewayPort = txtIntValue(txt, "gatewayPort")
|
||||
val canvasPort = txtIntValue(txt, "canvasPort")
|
||||
val tlsEnabled = txtBoolValue(txt, "gatewayTls")
|
||||
val tlsFingerprint = txtValue(txt, "gatewayTlsSha256")
|
||||
val id = stableId(instanceName, domain)
|
||||
next[id] =
|
||||
GatewayEndpoint(
|
||||
stableId = id,
|
||||
name = displayName,
|
||||
host = host,
|
||||
port = port,
|
||||
lanHost = lanHost,
|
||||
tailnetDns = tailnetDns,
|
||||
gatewayPort = gatewayPort,
|
||||
canvasPort = canvasPort,
|
||||
tlsEnabled = tlsEnabled,
|
||||
tlsFingerprintSha256 = tlsFingerprint,
|
||||
)
|
||||
}
|
||||
|
||||
unicastById.clear()
|
||||
unicastById.putAll(next)
|
||||
lastWideAreaRcode = ptrMsg.header.rcode
|
||||
lastWideAreaCount = next.size
|
||||
publish()
|
||||
|
||||
if (next.isEmpty()) {
|
||||
Log.d(
|
||||
logTag,
|
||||
"wide-area discovery: 0 results for $ptrName (rcode=${Rcode.string(ptrMsg.header.rcode)})",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun decodeInstanceName(instanceFqdn: String, domain: String): String {
|
||||
val suffix = "${serviceType}${domain}"
|
||||
val withoutSuffix =
|
||||
if (instanceFqdn.endsWith(suffix)) {
|
||||
instanceFqdn.removeSuffix(suffix)
|
||||
} else {
|
||||
instanceFqdn.substringBefore(serviceType)
|
||||
}
|
||||
return normalizeName(stripTrailingDot(withoutSuffix))
|
||||
}
|
||||
|
||||
private fun stripTrailingDot(raw: String): String {
|
||||
return raw.removeSuffix(".")
|
||||
}
|
||||
|
||||
private suspend fun lookupUnicastMessage(name: String, type: Int): Message? {
|
||||
val query =
|
||||
try {
|
||||
Message.newQuery(
|
||||
org.xbill.DNS.Record.newRecord(
|
||||
Name.fromString(name),
|
||||
type,
|
||||
DClass.IN,
|
||||
),
|
||||
)
|
||||
} catch (_: TextParseException) {
|
||||
return null
|
||||
}
|
||||
|
||||
val system = queryViaSystemDns(query)
|
||||
if (records(system, Section.ANSWER).any { it.type == type }) return system
|
||||
|
||||
val direct = createDirectResolver() ?: return system
|
||||
return try {
|
||||
val msg = direct.send(query)
|
||||
if (records(msg, Section.ANSWER).any { it.type == type }) msg else system
|
||||
} catch (_: Throwable) {
|
||||
system
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun queryViaSystemDns(query: Message): Message? {
|
||||
val network = preferredDnsNetwork()
|
||||
val bytes =
|
||||
try {
|
||||
rawQuery(network, query.toWire())
|
||||
} catch (_: Throwable) {
|
||||
return null
|
||||
}
|
||||
|
||||
return try {
|
||||
Message(bytes)
|
||||
} catch (_: IOException) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun records(msg: Message?, section: Int): List<Record> {
|
||||
return msg?.getSectionArray(section)?.toList() ?: emptyList()
|
||||
}
|
||||
|
||||
private fun keyName(raw: String): String {
|
||||
return raw.trim().lowercase()
|
||||
}
|
||||
|
||||
private fun recordsByName(msg: Message, section: Int): Map<String, List<Record>> {
|
||||
val next = LinkedHashMap<String, MutableList<Record>>()
|
||||
for (r in records(msg, section)) {
|
||||
val name = r.name?.toString() ?: continue
|
||||
next.getOrPut(keyName(name)) { mutableListOf() }.add(r)
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
private fun recordByName(msg: Message, fqdn: String, type: Int): Record? {
|
||||
val key = keyName(fqdn)
|
||||
val byNameAnswer = recordsByName(msg, Section.ANSWER)
|
||||
val fromAnswer = byNameAnswer[key].orEmpty().firstOrNull { it.type == type }
|
||||
if (fromAnswer != null) return fromAnswer
|
||||
|
||||
val byNameAdditional = recordsByName(msg, Section.ADDITIONAL)
|
||||
return byNameAdditional[key].orEmpty().firstOrNull { it.type == type }
|
||||
}
|
||||
|
||||
private fun resolveHostFromMessage(msg: Message?, hostname: String): String? {
|
||||
val m = msg ?: return null
|
||||
val key = keyName(hostname)
|
||||
val additional = recordsByName(m, Section.ADDITIONAL)[key].orEmpty()
|
||||
val a = additional.mapNotNull { it as? ARecord }.mapNotNull { it.address?.hostAddress }
|
||||
val aaaa = additional.mapNotNull { it as? AAAARecord }.mapNotNull { it.address?.hostAddress }
|
||||
return a.firstOrNull() ?: aaaa.firstOrNull()
|
||||
}
|
||||
|
||||
private fun preferredDnsNetwork(): android.net.Network? {
|
||||
val cm = connectivity ?: return null
|
||||
|
||||
// Prefer VPN (Tailscale) when present; otherwise use the active network.
|
||||
cm.allNetworks.firstOrNull { n ->
|
||||
val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false
|
||||
caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN)
|
||||
}?.let { return it }
|
||||
|
||||
return cm.activeNetwork
|
||||
}
|
||||
|
||||
private fun createDirectResolver(): Resolver? {
|
||||
val cm = connectivity ?: return null
|
||||
|
||||
val candidateNetworks =
|
||||
buildList {
|
||||
cm.allNetworks
|
||||
.firstOrNull { n ->
|
||||
val caps = cm.getNetworkCapabilities(n) ?: return@firstOrNull false
|
||||
caps.hasTransport(NetworkCapabilities.TRANSPORT_VPN)
|
||||
}?.let(::add)
|
||||
cm.activeNetwork?.let(::add)
|
||||
}.distinct()
|
||||
|
||||
val servers =
|
||||
candidateNetworks
|
||||
.asSequence()
|
||||
.flatMap { n ->
|
||||
cm.getLinkProperties(n)?.dnsServers?.asSequence() ?: emptySequence()
|
||||
}
|
||||
.distinctBy { it.hostAddress ?: it.toString() }
|
||||
.toList()
|
||||
if (servers.isEmpty()) return null
|
||||
|
||||
return try {
|
||||
val resolvers =
|
||||
servers.mapNotNull { addr ->
|
||||
try {
|
||||
SimpleResolver().apply {
|
||||
setAddress(InetSocketAddress(addr, 53))
|
||||
setTimeout(3)
|
||||
}
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
if (resolvers.isEmpty()) return null
|
||||
ExtendedResolver(resolvers.toTypedArray()).apply { setTimeout(3) }
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun rawQuery(network: android.net.Network?, wireQuery: ByteArray): ByteArray =
|
||||
suspendCancellableCoroutine { cont ->
|
||||
val signal = CancellationSignal()
|
||||
cont.invokeOnCancellation { signal.cancel() }
|
||||
|
||||
dns.rawQuery(
|
||||
network,
|
||||
wireQuery,
|
||||
DnsResolver.FLAG_EMPTY,
|
||||
dnsExecutor,
|
||||
signal,
|
||||
object : DnsResolver.Callback<ByteArray> {
|
||||
override fun onAnswer(answer: ByteArray, rcode: Int) {
|
||||
cont.resume(answer)
|
||||
}
|
||||
|
||||
override fun onError(error: DnsResolver.DnsException) {
|
||||
cont.resumeWithException(error)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
private fun txtValue(records: List<TXTRecord>, key: String): String? {
|
||||
val prefix = "$key="
|
||||
for (r in records) {
|
||||
val strings: List<String> =
|
||||
try {
|
||||
r.strings.mapNotNull { it as? String }
|
||||
} catch (_: Throwable) {
|
||||
emptyList()
|
||||
}
|
||||
for (s in strings) {
|
||||
val trimmed = decodeDnsTxtString(s).trim()
|
||||
if (trimmed.startsWith(prefix)) {
|
||||
return trimmed.removePrefix(prefix).trim().ifEmpty { null }
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun txtIntValue(records: List<TXTRecord>, key: String): Int? {
|
||||
return txtValue(records, key)?.toIntOrNull()
|
||||
}
|
||||
|
||||
private fun txtBoolValue(records: List<TXTRecord>, key: String): Boolean {
|
||||
val raw = txtValue(records, key)?.trim()?.lowercase() ?: return false
|
||||
return raw == "1" || raw == "true" || raw == "yes"
|
||||
}
|
||||
|
||||
private fun decodeDnsTxtString(raw: String): String {
|
||||
// dnsjava treats TXT as opaque bytes and decodes as ISO-8859-1 to preserve bytes.
|
||||
// Our TXT payload is UTF-8 (written by the gateway), so re-decode when possible.
|
||||
val bytes = raw.toByteArray(Charsets.ISO_8859_1)
|
||||
val decoder =
|
||||
Charsets.UTF_8
|
||||
.newDecoder()
|
||||
.onMalformedInput(CodingErrorAction.REPORT)
|
||||
.onUnmappableCharacter(CodingErrorAction.REPORT)
|
||||
return try {
|
||||
decoder.decode(ByteBuffer.wrap(bytes)).toString()
|
||||
} catch (_: Throwable) {
|
||||
raw
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun resolveHostUnicast(hostname: String): String? {
|
||||
val a =
|
||||
records(lookupUnicastMessage(hostname, Type.A), Section.ANSWER)
|
||||
.mapNotNull { it as? ARecord }
|
||||
.mapNotNull { it.address?.hostAddress }
|
||||
val aaaa =
|
||||
records(lookupUnicastMessage(hostname, Type.AAAA), Section.ANSWER)
|
||||
.mapNotNull { it as? AAAARecord }
|
||||
.mapNotNull { it.address?.hostAddress }
|
||||
|
||||
return a.firstOrNull() ?: aaaa.firstOrNull()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package ai.openclaw.android.gateway
|
||||
|
||||
data class GatewayEndpoint(
|
||||
val stableId: String,
|
||||
val name: String,
|
||||
val host: String,
|
||||
val port: Int,
|
||||
val lanHost: String? = null,
|
||||
val tailnetDns: String? = null,
|
||||
val gatewayPort: Int? = null,
|
||||
val canvasPort: Int? = null,
|
||||
val tlsEnabled: Boolean = false,
|
||||
val tlsFingerprintSha256: String? = null,
|
||||
) {
|
||||
companion object {
|
||||
fun manual(host: String, port: Int): GatewayEndpoint =
|
||||
GatewayEndpoint(
|
||||
stableId = "manual|${host.lowercase()}|$port",
|
||||
name = "$host:$port",
|
||||
host = host,
|
||||
port = port,
|
||||
tlsEnabled = false,
|
||||
tlsFingerprintSha256 = null,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
package ai.openclaw.android.gateway
|
||||
|
||||
const val GATEWAY_PROTOCOL_VERSION = 3
|
||||
@@ -0,0 +1,760 @@
|
||||
package ai.openclaw.android.gateway
|
||||
|
||||
import android.util.Log
|
||||
import java.util.Locale
|
||||
import java.util.UUID
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.TimeoutCancellationException
|
||||
import kotlinx.coroutines.cancelAndJoin
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.withTimeout
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import kotlinx.serialization.json.JsonNull
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import okhttp3.WebSocket
|
||||
import okhttp3.WebSocketListener
|
||||
|
||||
data class GatewayClientInfo(
|
||||
val id: String,
|
||||
val displayName: String?,
|
||||
val version: String,
|
||||
val platform: String,
|
||||
val mode: String,
|
||||
val instanceId: String?,
|
||||
val deviceFamily: String?,
|
||||
val modelIdentifier: String?,
|
||||
)
|
||||
|
||||
data class GatewayConnectOptions(
|
||||
val role: String,
|
||||
val scopes: List<String>,
|
||||
val caps: List<String>,
|
||||
val commands: List<String>,
|
||||
val permissions: Map<String, Boolean>,
|
||||
val client: GatewayClientInfo,
|
||||
val userAgent: String? = null,
|
||||
)
|
||||
|
||||
class GatewaySession(
|
||||
private val scope: CoroutineScope,
|
||||
private val identityStore: DeviceIdentityStore,
|
||||
private val deviceAuthStore: DeviceAuthTokenStore,
|
||||
private val onConnected: (serverName: String?, remoteAddress: String?, mainSessionKey: String?) -> Unit,
|
||||
private val onDisconnected: (message: String) -> Unit,
|
||||
private val onEvent: (event: String, payloadJson: String?) -> Unit,
|
||||
private val onInvoke: (suspend (InvokeRequest) -> InvokeResult)? = null,
|
||||
private val onTlsFingerprint: ((stableId: String, fingerprint: String) -> Unit)? = null,
|
||||
) {
|
||||
private companion object {
|
||||
// Keep connect timeout above observed gateway unauthorized close on lower-end devices.
|
||||
private const val CONNECT_RPC_TIMEOUT_MS = 12_000L
|
||||
}
|
||||
|
||||
data class InvokeRequest(
|
||||
val id: String,
|
||||
val nodeId: String,
|
||||
val command: String,
|
||||
val paramsJson: String?,
|
||||
val timeoutMs: Long?,
|
||||
)
|
||||
|
||||
data class InvokeResult(val ok: Boolean, val payloadJson: String?, val error: ErrorShape?) {
|
||||
companion object {
|
||||
fun ok(payloadJson: String?) = InvokeResult(ok = true, payloadJson = payloadJson, error = null)
|
||||
fun error(code: String, message: String) =
|
||||
InvokeResult(ok = false, payloadJson = null, error = ErrorShape(code = code, message = message))
|
||||
}
|
||||
}
|
||||
|
||||
data class ErrorShape(val code: String, val message: String)
|
||||
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
private val writeLock = Mutex()
|
||||
private val pending = ConcurrentHashMap<String, CompletableDeferred<RpcResponse>>()
|
||||
|
||||
@Volatile private var canvasHostUrl: String? = null
|
||||
@Volatile private var mainSessionKey: String? = null
|
||||
|
||||
private data class DesiredConnection(
|
||||
val endpoint: GatewayEndpoint,
|
||||
val token: String?,
|
||||
val password: String?,
|
||||
val options: GatewayConnectOptions,
|
||||
val tls: GatewayTlsParams?,
|
||||
)
|
||||
|
||||
private var desired: DesiredConnection? = null
|
||||
private var job: Job? = null
|
||||
@Volatile private var currentConnection: Connection? = null
|
||||
|
||||
fun connect(
|
||||
endpoint: GatewayEndpoint,
|
||||
token: String?,
|
||||
password: String?,
|
||||
options: GatewayConnectOptions,
|
||||
tls: GatewayTlsParams? = null,
|
||||
) {
|
||||
desired = DesiredConnection(endpoint, token, password, options, tls)
|
||||
if (job == null) {
|
||||
job = scope.launch(Dispatchers.IO) { runLoop() }
|
||||
}
|
||||
}
|
||||
|
||||
fun disconnect() {
|
||||
desired = null
|
||||
currentConnection?.closeQuietly()
|
||||
scope.launch(Dispatchers.IO) {
|
||||
job?.cancelAndJoin()
|
||||
job = null
|
||||
canvasHostUrl = null
|
||||
mainSessionKey = null
|
||||
onDisconnected("Offline")
|
||||
}
|
||||
}
|
||||
|
||||
fun reconnect() {
|
||||
currentConnection?.closeQuietly()
|
||||
}
|
||||
|
||||
fun currentCanvasHostUrl(): String? = canvasHostUrl
|
||||
fun currentMainSessionKey(): String? = mainSessionKey
|
||||
|
||||
suspend fun sendNodeEvent(event: String, payloadJson: String?): Boolean {
|
||||
val conn = currentConnection ?: return false
|
||||
val parsedPayload = payloadJson?.let { parseJsonOrNull(it) }
|
||||
val params =
|
||||
buildJsonObject {
|
||||
put("event", JsonPrimitive(event))
|
||||
if (parsedPayload != null) {
|
||||
put("payload", parsedPayload)
|
||||
} else if (payloadJson != null) {
|
||||
put("payloadJSON", JsonPrimitive(payloadJson))
|
||||
} else {
|
||||
put("payloadJSON", JsonNull)
|
||||
}
|
||||
}
|
||||
try {
|
||||
conn.request("node.event", params, timeoutMs = 8_000)
|
||||
return true
|
||||
} catch (err: Throwable) {
|
||||
Log.w("OpenClawGateway", "node.event failed: ${err.message ?: err::class.java.simpleName}")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun request(method: String, paramsJson: String?, timeoutMs: Long = 15_000): String {
|
||||
val conn = currentConnection ?: throw IllegalStateException("not connected")
|
||||
val params =
|
||||
if (paramsJson.isNullOrBlank()) {
|
||||
null
|
||||
} else {
|
||||
json.parseToJsonElement(paramsJson)
|
||||
}
|
||||
val res = conn.request(method, params, timeoutMs)
|
||||
if (res.ok) return res.payloadJson ?: ""
|
||||
val err = res.error
|
||||
throw IllegalStateException("${err?.code ?: "UNAVAILABLE"}: ${err?.message ?: "request failed"}")
|
||||
}
|
||||
|
||||
suspend fun refreshNodeCanvasCapability(timeoutMs: Long = 8_000): Boolean {
|
||||
val conn = currentConnection ?: return false
|
||||
val response =
|
||||
try {
|
||||
conn.request(
|
||||
"node.canvas.capability.refresh",
|
||||
params = buildJsonObject {},
|
||||
timeoutMs = timeoutMs,
|
||||
)
|
||||
} catch (err: Throwable) {
|
||||
Log.w("OpenClawGateway", "node.canvas.capability.refresh failed: ${err.message ?: err::class.java.simpleName}")
|
||||
return false
|
||||
}
|
||||
if (!response.ok) {
|
||||
val err = response.error
|
||||
Log.w(
|
||||
"OpenClawGateway",
|
||||
"node.canvas.capability.refresh rejected: ${err?.code ?: "UNAVAILABLE"}: ${err?.message ?: "request failed"}",
|
||||
)
|
||||
return false
|
||||
}
|
||||
val payloadObj = response.payloadJson?.let(::parseJsonOrNull)?.asObjectOrNull()
|
||||
val refreshedCapability = payloadObj?.get("canvasCapability").asStringOrNull()?.trim().orEmpty()
|
||||
if (refreshedCapability.isEmpty()) {
|
||||
Log.w("OpenClawGateway", "node.canvas.capability.refresh missing canvasCapability")
|
||||
return false
|
||||
}
|
||||
val scopedCanvasHostUrl = canvasHostUrl?.trim().orEmpty()
|
||||
if (scopedCanvasHostUrl.isEmpty()) {
|
||||
Log.w("OpenClawGateway", "node.canvas.capability.refresh missing local canvasHostUrl")
|
||||
return false
|
||||
}
|
||||
val refreshedUrl = replaceCanvasCapabilityInScopedHostUrl(scopedCanvasHostUrl, refreshedCapability)
|
||||
if (refreshedUrl == null) {
|
||||
Log.w("OpenClawGateway", "node.canvas.capability.refresh unable to rewrite scoped canvas URL")
|
||||
return false
|
||||
}
|
||||
canvasHostUrl = refreshedUrl
|
||||
return true
|
||||
}
|
||||
|
||||
private data class RpcResponse(val id: String, val ok: Boolean, val payloadJson: String?, val error: ErrorShape?)
|
||||
|
||||
private inner class Connection(
|
||||
private val endpoint: GatewayEndpoint,
|
||||
private val token: String?,
|
||||
private val password: String?,
|
||||
private val options: GatewayConnectOptions,
|
||||
private val tls: GatewayTlsParams?,
|
||||
) {
|
||||
private val connectDeferred = CompletableDeferred<Unit>()
|
||||
private val closedDeferred = CompletableDeferred<Unit>()
|
||||
private val isClosed = AtomicBoolean(false)
|
||||
private val connectNonceDeferred = CompletableDeferred<String>()
|
||||
private val client: OkHttpClient = buildClient()
|
||||
private var socket: WebSocket? = null
|
||||
private val loggerTag = "OpenClawGateway"
|
||||
|
||||
val remoteAddress: String =
|
||||
if (endpoint.host.contains(":")) {
|
||||
"[${endpoint.host}]:${endpoint.port}"
|
||||
} else {
|
||||
"${endpoint.host}:${endpoint.port}"
|
||||
}
|
||||
|
||||
suspend fun connect() {
|
||||
val scheme = if (tls != null) "wss" else "ws"
|
||||
val url = "$scheme://${endpoint.host}:${endpoint.port}"
|
||||
val request = Request.Builder().url(url).build()
|
||||
socket = client.newWebSocket(request, Listener())
|
||||
try {
|
||||
connectDeferred.await()
|
||||
} catch (err: Throwable) {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun request(method: String, params: JsonElement?, timeoutMs: Long): RpcResponse {
|
||||
val id = UUID.randomUUID().toString()
|
||||
val deferred = CompletableDeferred<RpcResponse>()
|
||||
pending[id] = deferred
|
||||
val frame =
|
||||
buildJsonObject {
|
||||
put("type", JsonPrimitive("req"))
|
||||
put("id", JsonPrimitive(id))
|
||||
put("method", JsonPrimitive(method))
|
||||
if (params != null) put("params", params)
|
||||
}
|
||||
sendJson(frame)
|
||||
return try {
|
||||
withTimeout(timeoutMs) { deferred.await() }
|
||||
} catch (err: TimeoutCancellationException) {
|
||||
pending.remove(id)
|
||||
throw IllegalStateException("request timeout")
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun sendJson(obj: JsonObject) {
|
||||
val jsonString = obj.toString()
|
||||
writeLock.withLock {
|
||||
socket?.send(jsonString)
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun awaitClose() = closedDeferred.await()
|
||||
|
||||
fun closeQuietly() {
|
||||
if (isClosed.compareAndSet(false, true)) {
|
||||
socket?.close(1000, "bye")
|
||||
socket = null
|
||||
closedDeferred.complete(Unit)
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildClient(): OkHttpClient {
|
||||
val builder = OkHttpClient.Builder()
|
||||
.writeTimeout(60, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.readTimeout(0, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.pingInterval(30, java.util.concurrent.TimeUnit.SECONDS)
|
||||
val tlsConfig = buildGatewayTlsConfig(tls) { fingerprint ->
|
||||
onTlsFingerprint?.invoke(tls?.stableId ?: endpoint.stableId, fingerprint)
|
||||
}
|
||||
if (tlsConfig != null) {
|
||||
builder.sslSocketFactory(tlsConfig.sslSocketFactory, tlsConfig.trustManager)
|
||||
builder.hostnameVerifier(tlsConfig.hostnameVerifier)
|
||||
}
|
||||
return builder.build()
|
||||
}
|
||||
|
||||
private inner class Listener : WebSocketListener() {
|
||||
override fun onOpen(webSocket: WebSocket, response: Response) {
|
||||
scope.launch {
|
||||
try {
|
||||
val nonce = awaitConnectNonce()
|
||||
sendConnect(nonce)
|
||||
} catch (err: Throwable) {
|
||||
connectDeferred.completeExceptionally(err)
|
||||
closeQuietly()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onMessage(webSocket: WebSocket, text: String) {
|
||||
scope.launch { handleMessage(text) }
|
||||
}
|
||||
|
||||
override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) {
|
||||
if (!connectDeferred.isCompleted) {
|
||||
connectDeferred.completeExceptionally(t)
|
||||
}
|
||||
if (isClosed.compareAndSet(false, true)) {
|
||||
failPending()
|
||||
closedDeferred.complete(Unit)
|
||||
onDisconnected("Gateway error: ${t.message ?: t::class.java.simpleName}")
|
||||
}
|
||||
}
|
||||
|
||||
override fun onClosed(webSocket: WebSocket, code: Int, reason: String) {
|
||||
if (!connectDeferred.isCompleted) {
|
||||
connectDeferred.completeExceptionally(IllegalStateException("Gateway closed: $reason"))
|
||||
}
|
||||
if (isClosed.compareAndSet(false, true)) {
|
||||
failPending()
|
||||
closedDeferred.complete(Unit)
|
||||
onDisconnected("Gateway closed: $reason")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun sendConnect(connectNonce: String) {
|
||||
val identity = identityStore.loadOrCreate()
|
||||
val storedToken = deviceAuthStore.loadToken(identity.deviceId, options.role)
|
||||
val trimmedToken = token?.trim().orEmpty()
|
||||
// QR/setup/manual shared token must take precedence; stale role tokens can survive re-onboarding.
|
||||
val authToken = if (trimmedToken.isNotBlank()) trimmedToken else storedToken.orEmpty()
|
||||
val payload = buildConnectParams(identity, connectNonce, authToken, password?.trim())
|
||||
val res = request("connect", payload, timeoutMs = CONNECT_RPC_TIMEOUT_MS)
|
||||
if (!res.ok) {
|
||||
val msg = res.error?.message ?: "connect failed"
|
||||
throw IllegalStateException(msg)
|
||||
}
|
||||
handleConnectSuccess(res, identity.deviceId)
|
||||
connectDeferred.complete(Unit)
|
||||
}
|
||||
|
||||
private fun handleConnectSuccess(res: RpcResponse, deviceId: String) {
|
||||
val payloadJson = res.payloadJson ?: throw IllegalStateException("connect failed: missing payload")
|
||||
val obj = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: throw IllegalStateException("connect failed")
|
||||
val serverName = obj["server"].asObjectOrNull()?.get("host").asStringOrNull()
|
||||
val authObj = obj["auth"].asObjectOrNull()
|
||||
val deviceToken = authObj?.get("deviceToken").asStringOrNull()
|
||||
val authRole = authObj?.get("role").asStringOrNull() ?: options.role
|
||||
if (!deviceToken.isNullOrBlank()) {
|
||||
deviceAuthStore.saveToken(deviceId, authRole, deviceToken)
|
||||
}
|
||||
val rawCanvas = obj["canvasHostUrl"].asStringOrNull()
|
||||
canvasHostUrl = normalizeCanvasHostUrl(rawCanvas, endpoint, isTlsConnection = tls != null)
|
||||
val sessionDefaults =
|
||||
obj["snapshot"].asObjectOrNull()
|
||||
?.get("sessionDefaults").asObjectOrNull()
|
||||
mainSessionKey = sessionDefaults?.get("mainSessionKey").asStringOrNull()
|
||||
onConnected(serverName, remoteAddress, mainSessionKey)
|
||||
}
|
||||
|
||||
private fun buildConnectParams(
|
||||
identity: DeviceIdentity,
|
||||
connectNonce: String,
|
||||
authToken: String,
|
||||
authPassword: String?,
|
||||
): JsonObject {
|
||||
val client = options.client
|
||||
val locale = Locale.getDefault().toLanguageTag()
|
||||
val clientObj =
|
||||
buildJsonObject {
|
||||
put("id", JsonPrimitive(client.id))
|
||||
client.displayName?.let { put("displayName", JsonPrimitive(it)) }
|
||||
put("version", JsonPrimitive(client.version))
|
||||
put("platform", JsonPrimitive(client.platform))
|
||||
put("mode", JsonPrimitive(client.mode))
|
||||
client.instanceId?.let { put("instanceId", JsonPrimitive(it)) }
|
||||
client.deviceFamily?.let { put("deviceFamily", JsonPrimitive(it)) }
|
||||
client.modelIdentifier?.let { put("modelIdentifier", JsonPrimitive(it)) }
|
||||
}
|
||||
|
||||
val password = authPassword?.trim().orEmpty()
|
||||
val authJson =
|
||||
when {
|
||||
authToken.isNotEmpty() ->
|
||||
buildJsonObject {
|
||||
put("token", JsonPrimitive(authToken))
|
||||
}
|
||||
password.isNotEmpty() ->
|
||||
buildJsonObject {
|
||||
put("password", JsonPrimitive(password))
|
||||
}
|
||||
else -> null
|
||||
}
|
||||
|
||||
val signedAtMs = System.currentTimeMillis()
|
||||
val payload =
|
||||
DeviceAuthPayload.buildV3(
|
||||
deviceId = identity.deviceId,
|
||||
clientId = client.id,
|
||||
clientMode = client.mode,
|
||||
role = options.role,
|
||||
scopes = options.scopes,
|
||||
signedAtMs = signedAtMs,
|
||||
token = if (authToken.isNotEmpty()) authToken else null,
|
||||
nonce = connectNonce,
|
||||
platform = client.platform,
|
||||
deviceFamily = client.deviceFamily,
|
||||
)
|
||||
val signature = identityStore.signPayload(payload, identity)
|
||||
val publicKey = identityStore.publicKeyBase64Url(identity)
|
||||
val deviceJson =
|
||||
if (!signature.isNullOrBlank() && !publicKey.isNullOrBlank()) {
|
||||
buildJsonObject {
|
||||
put("id", JsonPrimitive(identity.deviceId))
|
||||
put("publicKey", JsonPrimitive(publicKey))
|
||||
put("signature", JsonPrimitive(signature))
|
||||
put("signedAt", JsonPrimitive(signedAtMs))
|
||||
put("nonce", JsonPrimitive(connectNonce))
|
||||
}
|
||||
} else {
|
||||
null
|
||||
}
|
||||
|
||||
return buildJsonObject {
|
||||
put("minProtocol", JsonPrimitive(GATEWAY_PROTOCOL_VERSION))
|
||||
put("maxProtocol", JsonPrimitive(GATEWAY_PROTOCOL_VERSION))
|
||||
put("client", clientObj)
|
||||
if (options.caps.isNotEmpty()) put("caps", JsonArray(options.caps.map(::JsonPrimitive)))
|
||||
if (options.commands.isNotEmpty()) put("commands", JsonArray(options.commands.map(::JsonPrimitive)))
|
||||
if (options.permissions.isNotEmpty()) {
|
||||
put(
|
||||
"permissions",
|
||||
buildJsonObject {
|
||||
options.permissions.forEach { (key, value) ->
|
||||
put(key, JsonPrimitive(value))
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
put("role", JsonPrimitive(options.role))
|
||||
if (options.scopes.isNotEmpty()) put("scopes", JsonArray(options.scopes.map(::JsonPrimitive)))
|
||||
authJson?.let { put("auth", it) }
|
||||
deviceJson?.let { put("device", it) }
|
||||
put("locale", JsonPrimitive(locale))
|
||||
options.userAgent?.trim()?.takeIf { it.isNotEmpty() }?.let {
|
||||
put("userAgent", JsonPrimitive(it))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun handleMessage(text: String) {
|
||||
val frame = json.parseToJsonElement(text).asObjectOrNull() ?: return
|
||||
when (frame["type"].asStringOrNull()) {
|
||||
"res" -> handleResponse(frame)
|
||||
"event" -> handleEvent(frame)
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleResponse(frame: JsonObject) {
|
||||
val id = frame["id"].asStringOrNull() ?: return
|
||||
val ok = frame["ok"].asBooleanOrNull() ?: false
|
||||
val payloadJson = frame["payload"]?.let { payload -> payload.toString() }
|
||||
val error =
|
||||
frame["error"]?.asObjectOrNull()?.let { obj ->
|
||||
val code = obj["code"].asStringOrNull() ?: "UNAVAILABLE"
|
||||
val msg = obj["message"].asStringOrNull() ?: "request failed"
|
||||
ErrorShape(code, msg)
|
||||
}
|
||||
pending.remove(id)?.complete(RpcResponse(id, ok, payloadJson, error))
|
||||
}
|
||||
|
||||
private fun handleEvent(frame: JsonObject) {
|
||||
val event = frame["event"].asStringOrNull() ?: return
|
||||
val payloadJson =
|
||||
frame["payload"]?.let { it.toString() } ?: frame["payloadJSON"].asStringOrNull()
|
||||
if (event == "connect.challenge") {
|
||||
val nonce = extractConnectNonce(payloadJson)
|
||||
if (!connectNonceDeferred.isCompleted && !nonce.isNullOrBlank()) {
|
||||
connectNonceDeferred.complete(nonce.trim())
|
||||
}
|
||||
return
|
||||
}
|
||||
if (event == "node.invoke.request" && payloadJson != null && onInvoke != null) {
|
||||
handleInvokeEvent(payloadJson)
|
||||
return
|
||||
}
|
||||
onEvent(event, payloadJson)
|
||||
}
|
||||
|
||||
private suspend fun awaitConnectNonce(): String {
|
||||
return try {
|
||||
withTimeout(2_000) { connectNonceDeferred.await() }
|
||||
} catch (err: Throwable) {
|
||||
throw IllegalStateException("connect challenge timeout", err)
|
||||
}
|
||||
}
|
||||
|
||||
private fun extractConnectNonce(payloadJson: String?): String? {
|
||||
if (payloadJson.isNullOrBlank()) return null
|
||||
val obj = parseJsonOrNull(payloadJson)?.asObjectOrNull() ?: return null
|
||||
return obj["nonce"].asStringOrNull()
|
||||
}
|
||||
|
||||
private fun handleInvokeEvent(payloadJson: String) {
|
||||
val payload =
|
||||
try {
|
||||
json.parseToJsonElement(payloadJson).asObjectOrNull()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
} ?: return
|
||||
val id = payload["id"].asStringOrNull() ?: return
|
||||
val nodeId = payload["nodeId"].asStringOrNull() ?: return
|
||||
val command = payload["command"].asStringOrNull() ?: return
|
||||
val params =
|
||||
payload["paramsJSON"].asStringOrNull()
|
||||
?: payload["params"]?.let { value -> if (value is JsonNull) null else value.toString() }
|
||||
val timeoutMs = payload["timeoutMs"].asLongOrNull()
|
||||
scope.launch {
|
||||
val result =
|
||||
try {
|
||||
onInvoke?.invoke(InvokeRequest(id, nodeId, command, params, timeoutMs))
|
||||
?: InvokeResult.error("UNAVAILABLE", "invoke handler missing")
|
||||
} catch (err: Throwable) {
|
||||
invokeErrorFromThrowable(err)
|
||||
}
|
||||
sendInvokeResult(id, nodeId, result, timeoutMs)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun sendInvokeResult(
|
||||
id: String,
|
||||
nodeId: String,
|
||||
result: InvokeResult,
|
||||
invokeTimeoutMs: Long?,
|
||||
) {
|
||||
val parsedPayload = result.payloadJson?.let { parseJsonOrNull(it) }
|
||||
val params =
|
||||
buildJsonObject {
|
||||
put("id", JsonPrimitive(id))
|
||||
put("nodeId", JsonPrimitive(nodeId))
|
||||
put("ok", JsonPrimitive(result.ok))
|
||||
if (parsedPayload != null) {
|
||||
put("payload", parsedPayload)
|
||||
} else if (result.payloadJson != null) {
|
||||
put("payloadJSON", JsonPrimitive(result.payloadJson))
|
||||
}
|
||||
result.error?.let { err ->
|
||||
put(
|
||||
"error",
|
||||
buildJsonObject {
|
||||
put("code", JsonPrimitive(err.code))
|
||||
put("message", JsonPrimitive(err.message))
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
val ackTimeoutMs = resolveInvokeResultAckTimeoutMs(invokeTimeoutMs)
|
||||
try {
|
||||
request("node.invoke.result", params, timeoutMs = ackTimeoutMs)
|
||||
} catch (err: Throwable) {
|
||||
Log.w(
|
||||
loggerTag,
|
||||
"node.invoke.result failed (ackTimeoutMs=$ackTimeoutMs): ${err.message ?: err::class.java.simpleName}",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun invokeErrorFromThrowable(err: Throwable): InvokeResult {
|
||||
val parsed = parseInvokeErrorFromThrowable(err, fallbackMessage = err::class.java.simpleName)
|
||||
return InvokeResult.error(code = parsed.code, message = parsed.message)
|
||||
}
|
||||
|
||||
private fun failPending() {
|
||||
for ((_, waiter) in pending) {
|
||||
waiter.cancel()
|
||||
}
|
||||
pending.clear()
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun runLoop() {
|
||||
var attempt = 0
|
||||
while (scope.isActive) {
|
||||
val target = desired
|
||||
if (target == null) {
|
||||
currentConnection?.closeQuietly()
|
||||
currentConnection = null
|
||||
delay(250)
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
onDisconnected(if (attempt == 0) "Connecting…" else "Reconnecting…")
|
||||
connectOnce(target)
|
||||
attempt = 0
|
||||
} catch (err: Throwable) {
|
||||
attempt += 1
|
||||
onDisconnected("Gateway error: ${err.message ?: err::class.java.simpleName}")
|
||||
val sleepMs = minOf(8_000L, (350.0 * Math.pow(1.7, attempt.toDouble())).toLong())
|
||||
delay(sleepMs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun connectOnce(target: DesiredConnection) = withContext(Dispatchers.IO) {
|
||||
val conn = Connection(target.endpoint, target.token, target.password, target.options, target.tls)
|
||||
currentConnection = conn
|
||||
try {
|
||||
conn.connect()
|
||||
conn.awaitClose()
|
||||
} finally {
|
||||
currentConnection = null
|
||||
canvasHostUrl = null
|
||||
mainSessionKey = null
|
||||
}
|
||||
}
|
||||
|
||||
private fun normalizeCanvasHostUrl(
|
||||
raw: String?,
|
||||
endpoint: GatewayEndpoint,
|
||||
isTlsConnection: Boolean,
|
||||
): String? {
|
||||
val trimmed = raw?.trim().orEmpty()
|
||||
val parsed = trimmed.takeIf { it.isNotBlank() }?.let { runCatching { java.net.URI(it) }.getOrNull() }
|
||||
val host = parsed?.host?.trim().orEmpty()
|
||||
val port = parsed?.port ?: -1
|
||||
val scheme = parsed?.scheme?.trim().orEmpty().ifBlank { "http" }
|
||||
val suffix = buildUrlSuffix(parsed)
|
||||
|
||||
// If raw URL is a non-loopback address and this connection uses TLS,
|
||||
// normalize scheme/port to the endpoint we actually connected to.
|
||||
if (trimmed.isNotBlank() && host.isNotBlank() && !isLoopbackHost(host)) {
|
||||
val needsTlsRewrite =
|
||||
isTlsConnection &&
|
||||
(
|
||||
!scheme.equals("https", ignoreCase = true) ||
|
||||
(port > 0 && port != endpoint.port) ||
|
||||
(port <= 0 && endpoint.port != 443)
|
||||
)
|
||||
if (needsTlsRewrite) {
|
||||
return buildCanvasUrl(host = host, scheme = "https", port = endpoint.port, suffix = suffix)
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
val fallbackHost =
|
||||
endpoint.tailnetDns?.trim().takeIf { !it.isNullOrEmpty() }
|
||||
?: endpoint.lanHost?.trim().takeIf { !it.isNullOrEmpty() }
|
||||
?: endpoint.host.trim()
|
||||
if (fallbackHost.isEmpty()) return trimmed.ifBlank { null }
|
||||
|
||||
// For TLS connections, use the connected endpoint's scheme/port instead of raw canvas metadata.
|
||||
val fallbackScheme = if (isTlsConnection) "https" else scheme
|
||||
// For TLS, always use the connected endpoint port.
|
||||
val fallbackPort = if (isTlsConnection) endpoint.port else (endpoint.canvasPort ?: endpoint.port)
|
||||
return buildCanvasUrl(host = fallbackHost, scheme = fallbackScheme, port = fallbackPort, suffix = suffix)
|
||||
}
|
||||
|
||||
private fun buildCanvasUrl(host: String, scheme: String, port: Int, suffix: String): String {
|
||||
val loweredScheme = scheme.lowercase()
|
||||
val formattedHost = if (host.contains(":")) "[${host}]" else host
|
||||
val portSuffix = if ((loweredScheme == "https" && port == 443) || (loweredScheme == "http" && port == 80)) "" else ":$port"
|
||||
return "$loweredScheme://$formattedHost$portSuffix$suffix"
|
||||
}
|
||||
|
||||
private fun buildUrlSuffix(uri: java.net.URI?): String {
|
||||
if (uri == null) return ""
|
||||
val path = uri.rawPath?.takeIf { it.isNotBlank() } ?: ""
|
||||
val query = uri.rawQuery?.takeIf { it.isNotBlank() }?.let { "?$it" } ?: ""
|
||||
val fragment = uri.rawFragment?.takeIf { it.isNotBlank() }?.let { "#$it" } ?: ""
|
||||
return "$path$query$fragment"
|
||||
}
|
||||
|
||||
private fun isLoopbackHost(raw: String?): Boolean {
|
||||
val host = raw?.trim()?.lowercase().orEmpty()
|
||||
if (host.isEmpty()) return false
|
||||
if (host == "localhost") return true
|
||||
if (host == "::1") return true
|
||||
if (host == "0.0.0.0" || host == "::") return true
|
||||
return host.startsWith("127.")
|
||||
}
|
||||
}
|
||||
|
||||
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
|
||||
|
||||
private fun JsonElement?.asStringOrNull(): String? =
|
||||
when (this) {
|
||||
is JsonNull -> null
|
||||
is JsonPrimitive -> content
|
||||
else -> null
|
||||
}
|
||||
|
||||
private fun JsonElement?.asBooleanOrNull(): Boolean? =
|
||||
when (this) {
|
||||
is JsonPrimitive -> {
|
||||
val c = content.trim()
|
||||
when {
|
||||
c.equals("true", ignoreCase = true) -> true
|
||||
c.equals("false", ignoreCase = true) -> false
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
else -> null
|
||||
}
|
||||
|
||||
private fun JsonElement?.asLongOrNull(): Long? =
|
||||
when (this) {
|
||||
is JsonPrimitive -> content.toLongOrNull()
|
||||
else -> null
|
||||
}
|
||||
|
||||
private fun parseJsonOrNull(payload: String): JsonElement? {
|
||||
val trimmed = payload.trim()
|
||||
if (trimmed.isEmpty()) return null
|
||||
return try {
|
||||
Json.parseToJsonElement(trimmed)
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
internal fun replaceCanvasCapabilityInScopedHostUrl(
|
||||
scopedUrl: String,
|
||||
capability: String,
|
||||
): String? {
|
||||
val marker = "/__openclaw__/cap/"
|
||||
val markerStart = scopedUrl.indexOf(marker)
|
||||
if (markerStart < 0) return null
|
||||
val capabilityStart = markerStart + marker.length
|
||||
val slashEnd = scopedUrl.indexOf("/", capabilityStart).takeIf { it >= 0 }
|
||||
val queryEnd = scopedUrl.indexOf("?", capabilityStart).takeIf { it >= 0 }
|
||||
val fragmentEnd = scopedUrl.indexOf("#", capabilityStart).takeIf { it >= 0 }
|
||||
val capabilityEnd = listOfNotNull(slashEnd, queryEnd, fragmentEnd).minOrNull() ?: scopedUrl.length
|
||||
if (capabilityEnd <= capabilityStart) return null
|
||||
return scopedUrl.substring(0, capabilityStart) + capability + scopedUrl.substring(capabilityEnd)
|
||||
}
|
||||
|
||||
internal fun resolveInvokeResultAckTimeoutMs(invokeTimeoutMs: Long?): Long {
|
||||
val normalized = invokeTimeoutMs?.takeIf { it > 0L } ?: 15_000L
|
||||
return normalized.coerceIn(15_000L, 120_000L)
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package ai.openclaw.android.gateway
|
||||
|
||||
import android.annotation.SuppressLint
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.net.InetSocketAddress
|
||||
import java.security.MessageDigest
|
||||
import java.security.SecureRandom
|
||||
import java.security.cert.CertificateException
|
||||
import java.security.cert.X509Certificate
|
||||
import java.util.Locale
|
||||
import javax.net.ssl.HttpsURLConnection
|
||||
import javax.net.ssl.HostnameVerifier
|
||||
import javax.net.ssl.SSLContext
|
||||
import javax.net.ssl.SSLParameters
|
||||
import javax.net.ssl.SSLSocketFactory
|
||||
import javax.net.ssl.SNIHostName
|
||||
import javax.net.ssl.SSLSocket
|
||||
import javax.net.ssl.TrustManagerFactory
|
||||
import javax.net.ssl.X509TrustManager
|
||||
|
||||
data class GatewayTlsParams(
|
||||
val required: Boolean,
|
||||
val expectedFingerprint: String?,
|
||||
val allowTOFU: Boolean,
|
||||
val stableId: String,
|
||||
)
|
||||
|
||||
data class GatewayTlsConfig(
|
||||
val sslSocketFactory: SSLSocketFactory,
|
||||
val trustManager: X509TrustManager,
|
||||
val hostnameVerifier: HostnameVerifier,
|
||||
)
|
||||
|
||||
fun buildGatewayTlsConfig(
|
||||
params: GatewayTlsParams?,
|
||||
onStore: ((String) -> Unit)? = null,
|
||||
): GatewayTlsConfig? {
|
||||
if (params == null) return null
|
||||
val expected = params.expectedFingerprint?.let(::normalizeFingerprint)
|
||||
val defaultTrust = defaultTrustManager()
|
||||
@SuppressLint("CustomX509TrustManager")
|
||||
val trustManager =
|
||||
object : X509TrustManager {
|
||||
override fun checkClientTrusted(chain: Array<X509Certificate>, authType: String) {
|
||||
defaultTrust.checkClientTrusted(chain, authType)
|
||||
}
|
||||
|
||||
override fun checkServerTrusted(chain: Array<X509Certificate>, authType: String) {
|
||||
if (chain.isEmpty()) throw CertificateException("empty certificate chain")
|
||||
val fingerprint = sha256Hex(chain[0].encoded)
|
||||
if (expected != null) {
|
||||
if (fingerprint != expected) {
|
||||
throw CertificateException("gateway TLS fingerprint mismatch")
|
||||
}
|
||||
return
|
||||
}
|
||||
if (params.allowTOFU) {
|
||||
onStore?.invoke(fingerprint)
|
||||
return
|
||||
}
|
||||
defaultTrust.checkServerTrusted(chain, authType)
|
||||
}
|
||||
|
||||
override fun getAcceptedIssuers(): Array<X509Certificate> = defaultTrust.acceptedIssuers
|
||||
}
|
||||
|
||||
val context = SSLContext.getInstance("TLS")
|
||||
context.init(null, arrayOf(trustManager), SecureRandom())
|
||||
val verifier =
|
||||
if (expected != null || params.allowTOFU) {
|
||||
// When pinning, we intentionally ignore hostname mismatch (service discovery often yields IPs).
|
||||
HostnameVerifier { _, _ -> true }
|
||||
} else {
|
||||
HttpsURLConnection.getDefaultHostnameVerifier()
|
||||
}
|
||||
return GatewayTlsConfig(
|
||||
sslSocketFactory = context.socketFactory,
|
||||
trustManager = trustManager,
|
||||
hostnameVerifier = verifier,
|
||||
)
|
||||
}
|
||||
|
||||
suspend fun probeGatewayTlsFingerprint(
|
||||
host: String,
|
||||
port: Int,
|
||||
timeoutMs: Int = 3_000,
|
||||
): String? {
|
||||
val trimmedHost = host.trim()
|
||||
if (trimmedHost.isEmpty()) return null
|
||||
if (port !in 1..65535) return null
|
||||
|
||||
return withContext(Dispatchers.IO) {
|
||||
val trustAll =
|
||||
@SuppressLint("CustomX509TrustManager", "TrustAllX509TrustManager")
|
||||
object : X509TrustManager {
|
||||
@SuppressLint("TrustAllX509TrustManager")
|
||||
override fun checkClientTrusted(chain: Array<X509Certificate>, authType: String) {}
|
||||
@SuppressLint("TrustAllX509TrustManager")
|
||||
override fun checkServerTrusted(chain: Array<X509Certificate>, authType: String) {}
|
||||
override fun getAcceptedIssuers(): Array<X509Certificate> = emptyArray()
|
||||
}
|
||||
|
||||
val context = SSLContext.getInstance("TLS")
|
||||
context.init(null, arrayOf(trustAll), SecureRandom())
|
||||
|
||||
val socket = (context.socketFactory.createSocket() as SSLSocket)
|
||||
try {
|
||||
socket.soTimeout = timeoutMs
|
||||
socket.connect(InetSocketAddress(trimmedHost, port), timeoutMs)
|
||||
|
||||
// Best-effort SNI for hostnames (avoid crashing on IP literals).
|
||||
try {
|
||||
if (trimmedHost.any { it.isLetter() }) {
|
||||
val params = SSLParameters()
|
||||
params.serverNames = listOf(SNIHostName(trimmedHost))
|
||||
socket.sslParameters = params
|
||||
}
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
socket.startHandshake()
|
||||
val cert = socket.session.peerCertificates.firstOrNull() as? X509Certificate ?: return@withContext null
|
||||
sha256Hex(cert.encoded)
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
} finally {
|
||||
try {
|
||||
socket.close()
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun defaultTrustManager(): X509TrustManager {
|
||||
val factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm())
|
||||
factory.init(null as java.security.KeyStore?)
|
||||
val trust =
|
||||
factory.trustManagers.firstOrNull { it is X509TrustManager } as? X509TrustManager
|
||||
return trust ?: throw IllegalStateException("No default X509TrustManager found")
|
||||
}
|
||||
|
||||
private fun sha256Hex(data: ByteArray): String {
|
||||
val digest = MessageDigest.getInstance("SHA-256").digest(data)
|
||||
val out = StringBuilder(digest.size * 2)
|
||||
for (byte in digest) {
|
||||
out.append(String.format(Locale.US, "%02x", byte))
|
||||
}
|
||||
return out.toString()
|
||||
}
|
||||
|
||||
private fun normalizeFingerprint(raw: String): String {
|
||||
val stripped = raw.trim()
|
||||
.replace(Regex("^sha-?256\\s*:?\\s*", RegexOption.IGNORE_CASE), "")
|
||||
return stripped.lowercase(Locale.US).filter { it in '0'..'9' || it in 'a'..'f' }
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package ai.openclaw.android.gateway
|
||||
|
||||
data class ParsedInvokeError(
|
||||
val code: String,
|
||||
val message: String,
|
||||
val hadExplicitCode: Boolean,
|
||||
) {
|
||||
val prefixedMessage: String
|
||||
get() = "$code: $message"
|
||||
}
|
||||
|
||||
fun parseInvokeErrorMessage(raw: String): ParsedInvokeError {
|
||||
val trimmed = raw.trim()
|
||||
if (trimmed.isEmpty()) {
|
||||
return ParsedInvokeError(code = "UNAVAILABLE", message = "error", hadExplicitCode = false)
|
||||
}
|
||||
|
||||
val parts = trimmed.split(":", limit = 2)
|
||||
if (parts.size == 2) {
|
||||
val code = parts[0].trim()
|
||||
val rest = parts[1].trim()
|
||||
if (code.isNotEmpty() && code.all { it.isUpperCase() || it == '_' }) {
|
||||
return ParsedInvokeError(
|
||||
code = code,
|
||||
message = rest.ifEmpty { trimmed },
|
||||
hadExplicitCode = true,
|
||||
)
|
||||
}
|
||||
}
|
||||
return ParsedInvokeError(code = "UNAVAILABLE", message = trimmed, hadExplicitCode = false)
|
||||
}
|
||||
|
||||
fun parseInvokeErrorFromThrowable(
|
||||
err: Throwable,
|
||||
fallbackMessage: String = "error",
|
||||
): ParsedInvokeError {
|
||||
val raw = err.message?.trim().takeIf { !it.isNullOrEmpty() } ?: fallbackMessage
|
||||
return parseInvokeErrorMessage(raw)
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
|
||||
class A2UIHandler(
|
||||
private val canvas: CanvasController,
|
||||
private val json: Json,
|
||||
private val getNodeCanvasHostUrl: () -> String?,
|
||||
private val getOperatorCanvasHostUrl: () -> String?,
|
||||
) {
|
||||
fun resolveA2uiHostUrl(): String? {
|
||||
val nodeRaw = getNodeCanvasHostUrl()?.trim().orEmpty()
|
||||
val operatorRaw = getOperatorCanvasHostUrl()?.trim().orEmpty()
|
||||
val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw
|
||||
if (raw.isBlank()) return null
|
||||
val base = raw.trimEnd('/')
|
||||
return "${base}/__openclaw__/a2ui/?platform=android"
|
||||
}
|
||||
|
||||
suspend fun ensureA2uiReady(a2uiUrl: String): Boolean {
|
||||
try {
|
||||
val already = canvas.eval(a2uiReadyCheckJS)
|
||||
if (already == "true") return true
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
canvas.navigate(a2uiUrl)
|
||||
repeat(50) {
|
||||
try {
|
||||
val ready = canvas.eval(a2uiReadyCheckJS)
|
||||
if (ready == "true") return true
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
delay(120)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
fun decodeA2uiMessages(command: String, paramsJson: String?): String {
|
||||
val raw = paramsJson?.trim().orEmpty()
|
||||
if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required")
|
||||
|
||||
val obj =
|
||||
json.parseToJsonElement(raw) as? JsonObject
|
||||
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
|
||||
|
||||
val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
|
||||
val hasMessagesArray = obj["messages"] is JsonArray
|
||||
|
||||
if (command == "canvas.a2ui.pushJSONL" || (!hasMessagesArray && jsonlField.isNotBlank())) {
|
||||
val jsonl = jsonlField
|
||||
if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required")
|
||||
val messages =
|
||||
jsonl
|
||||
.lineSequence()
|
||||
.map { it.trim() }
|
||||
.filter { it.isNotBlank() }
|
||||
.mapIndexed { idx, line ->
|
||||
val el = json.parseToJsonElement(line)
|
||||
val msg =
|
||||
el as? JsonObject
|
||||
?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object")
|
||||
validateA2uiV0_8(msg, idx + 1)
|
||||
msg
|
||||
}
|
||||
.toList()
|
||||
return JsonArray(messages).toString()
|
||||
}
|
||||
|
||||
val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required")
|
||||
val out =
|
||||
arr.mapIndexed { idx, el ->
|
||||
val msg =
|
||||
el as? JsonObject
|
||||
?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object")
|
||||
validateA2uiV0_8(msg, idx + 1)
|
||||
msg
|
||||
}
|
||||
return JsonArray(out).toString()
|
||||
}
|
||||
|
||||
private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) {
|
||||
if (msg.containsKey("createSurface")) {
|
||||
throw IllegalArgumentException(
|
||||
"A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.",
|
||||
)
|
||||
}
|
||||
val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface")
|
||||
val matched = msg.keys.filter { allowed.contains(it) }
|
||||
if (matched.size != 1) {
|
||||
val found = msg.keys.sorted().joinToString(", ")
|
||||
throw IllegalArgumentException(
|
||||
"A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val a2uiReadyCheckJS: String =
|
||||
"""
|
||||
(() => {
|
||||
try {
|
||||
const host = globalThis.openclawA2UI;
|
||||
return !!host && typeof host.applyMessages === 'function';
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
})()
|
||||
"""
|
||||
|
||||
const val a2uiResetJS: String =
|
||||
"""
|
||||
(() => {
|
||||
try {
|
||||
const host = globalThis.openclawA2UI;
|
||||
if (!host) return { ok: false, error: "missing openclawA2UI" };
|
||||
return host.reset();
|
||||
} catch (e) {
|
||||
return { ok: false, error: String(e?.message ?? e) };
|
||||
}
|
||||
})()
|
||||
"""
|
||||
|
||||
fun a2uiApplyMessagesJS(messagesJson: String): String {
|
||||
return """
|
||||
(() => {
|
||||
try {
|
||||
const host = globalThis.openclawA2UI;
|
||||
if (!host) return { ok: false, error: "missing openclawA2UI" };
|
||||
const messages = $messagesJson;
|
||||
return host.applyMessages(messages);
|
||||
} catch (e) {
|
||||
return { ok: false, error: String(e?.message ?? e) };
|
||||
}
|
||||
})()
|
||||
""".trimIndent()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.app.PendingIntent
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import ai.openclaw.android.InstallResultReceiver
|
||||
import ai.openclaw.android.MainActivity
|
||||
import ai.openclaw.android.gateway.GatewayEndpoint
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import java.io.File
|
||||
import java.net.URI
|
||||
import java.security.MessageDigest
|
||||
import java.util.Locale
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import kotlinx.serialization.json.jsonPrimitive
|
||||
import kotlinx.serialization.json.put
|
||||
|
||||
private val SHA256_HEX = Regex("^[a-fA-F0-9]{64}$")
|
||||
|
||||
internal data class AppUpdateRequest(
|
||||
val url: String,
|
||||
val expectedSha256: String,
|
||||
)
|
||||
|
||||
internal fun parseAppUpdateRequest(paramsJson: String?, connectedHost: String?): AppUpdateRequest {
|
||||
val params =
|
||||
try {
|
||||
paramsJson?.let { Json.parseToJsonElement(it).jsonObject }
|
||||
} catch (_: Throwable) {
|
||||
throw IllegalArgumentException("params must be valid JSON")
|
||||
} ?: throw IllegalArgumentException("missing 'url' parameter")
|
||||
|
||||
val urlRaw =
|
||||
params["url"]?.jsonPrimitive?.content?.trim().orEmpty()
|
||||
.ifEmpty { throw IllegalArgumentException("missing 'url' parameter") }
|
||||
val sha256Raw =
|
||||
params["sha256"]?.jsonPrimitive?.content?.trim().orEmpty()
|
||||
.ifEmpty { throw IllegalArgumentException("missing 'sha256' parameter") }
|
||||
if (!SHA256_HEX.matches(sha256Raw)) {
|
||||
throw IllegalArgumentException("invalid 'sha256' parameter (expected 64 hex chars)")
|
||||
}
|
||||
|
||||
val uri =
|
||||
try {
|
||||
URI(urlRaw)
|
||||
} catch (_: Throwable) {
|
||||
throw IllegalArgumentException("invalid 'url' parameter")
|
||||
}
|
||||
val scheme = uri.scheme?.lowercase(Locale.US).orEmpty()
|
||||
if (scheme != "https") {
|
||||
throw IllegalArgumentException("url must use https")
|
||||
}
|
||||
if (!uri.userInfo.isNullOrBlank()) {
|
||||
throw IllegalArgumentException("url must not include credentials")
|
||||
}
|
||||
val host = uri.host?.lowercase(Locale.US) ?: throw IllegalArgumentException("url host required")
|
||||
val connectedHostNormalized = connectedHost?.trim()?.lowercase(Locale.US).orEmpty()
|
||||
if (connectedHostNormalized.isNotEmpty() && host != connectedHostNormalized) {
|
||||
throw IllegalArgumentException("url host must match connected gateway host")
|
||||
}
|
||||
|
||||
return AppUpdateRequest(
|
||||
url = uri.toASCIIString(),
|
||||
expectedSha256 = sha256Raw.lowercase(Locale.US),
|
||||
)
|
||||
}
|
||||
|
||||
internal fun sha256Hex(file: File): String {
|
||||
val digest = MessageDigest.getInstance("SHA-256")
|
||||
file.inputStream().use { input ->
|
||||
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
|
||||
while (true) {
|
||||
val read = input.read(buffer)
|
||||
if (read < 0) break
|
||||
if (read == 0) continue
|
||||
digest.update(buffer, 0, read)
|
||||
}
|
||||
}
|
||||
val out = StringBuilder(64)
|
||||
for (byte in digest.digest()) {
|
||||
out.append(String.format(Locale.US, "%02x", byte))
|
||||
}
|
||||
return out.toString()
|
||||
}
|
||||
|
||||
class AppUpdateHandler(
|
||||
private val appContext: Context,
|
||||
private val connectedEndpoint: () -> GatewayEndpoint?,
|
||||
) {
|
||||
|
||||
fun handleUpdate(paramsJson: String?): GatewaySession.InvokeResult {
|
||||
try {
|
||||
val updateRequest =
|
||||
try {
|
||||
parseAppUpdateRequest(paramsJson, connectedEndpoint()?.host)
|
||||
} catch (err: IllegalArgumentException) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "INVALID_REQUEST",
|
||||
message = "INVALID_REQUEST: ${err.message ?: "invalid app.update params"}",
|
||||
)
|
||||
}
|
||||
val url = updateRequest.url
|
||||
val expectedSha256 = updateRequest.expectedSha256
|
||||
|
||||
android.util.Log.w("openclaw", "app.update: downloading from $url")
|
||||
|
||||
val notifId = 9001
|
||||
val channelId = "app_update"
|
||||
val notifManager = appContext.getSystemService(android.content.Context.NOTIFICATION_SERVICE) as android.app.NotificationManager
|
||||
|
||||
// Create notification channel (required for Android 8+)
|
||||
val channel = android.app.NotificationChannel(channelId, "App Updates", android.app.NotificationManager.IMPORTANCE_LOW)
|
||||
notifManager.createNotificationChannel(channel)
|
||||
|
||||
// PendingIntent to open the app when notification is tapped
|
||||
val launchIntent = Intent(appContext, MainActivity::class.java).apply {
|
||||
flags = Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP
|
||||
}
|
||||
val launchPi = PendingIntent.getActivity(appContext, 0, launchIntent, PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE)
|
||||
|
||||
// Launch download async so the invoke returns immediately
|
||||
CoroutineScope(Dispatchers.IO).launch {
|
||||
try {
|
||||
val cacheDir = java.io.File(appContext.cacheDir, "updates")
|
||||
cacheDir.mkdirs()
|
||||
val file = java.io.File(cacheDir, "update.apk")
|
||||
if (file.exists()) file.delete()
|
||||
|
||||
// Show initial progress notification
|
||||
fun buildProgressNotif(progress: Int, max: Int, text: String): android.app.Notification {
|
||||
return android.app.Notification.Builder(appContext, channelId)
|
||||
.setSmallIcon(android.R.drawable.stat_sys_download)
|
||||
.setContentTitle("OpenClaw Update")
|
||||
.setContentText(text)
|
||||
.setProgress(max, progress, max == 0)
|
||||
|
||||
.setContentIntent(launchPi)
|
||||
.setOngoing(true)
|
||||
.build()
|
||||
}
|
||||
notifManager.notify(notifId, buildProgressNotif(0, 0, "Connecting..."))
|
||||
|
||||
val client = okhttp3.OkHttpClient.Builder()
|
||||
.connectTimeout(30, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.readTimeout(300, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.build()
|
||||
val request = okhttp3.Request.Builder().url(url).build()
|
||||
val response = client.newCall(request).execute()
|
||||
if (!response.isSuccessful) {
|
||||
notifManager.cancel(notifId)
|
||||
notifManager.notify(notifId, android.app.Notification.Builder(appContext, channelId)
|
||||
.setSmallIcon(android.R.drawable.stat_notify_error)
|
||||
.setContentTitle("Update Failed")
|
||||
|
||||
.setContentIntent(launchPi)
|
||||
.setContentText("HTTP ${response.code}")
|
||||
.build())
|
||||
return@launch
|
||||
}
|
||||
|
||||
val contentLength = response.body?.contentLength() ?: -1L
|
||||
val body = response.body ?: run {
|
||||
notifManager.cancel(notifId)
|
||||
return@launch
|
||||
}
|
||||
|
||||
// Download with progress tracking
|
||||
var totalBytes = 0L
|
||||
var lastNotifUpdate = 0L
|
||||
body.byteStream().use { input ->
|
||||
file.outputStream().use { output ->
|
||||
val buffer = ByteArray(8192)
|
||||
while (true) {
|
||||
val bytesRead = input.read(buffer)
|
||||
if (bytesRead == -1) break
|
||||
output.write(buffer, 0, bytesRead)
|
||||
totalBytes += bytesRead
|
||||
|
||||
// Update notification at most every 500ms
|
||||
val now = System.currentTimeMillis()
|
||||
if (now - lastNotifUpdate > 500) {
|
||||
lastNotifUpdate = now
|
||||
if (contentLength > 0) {
|
||||
val pct = ((totalBytes * 100) / contentLength).toInt()
|
||||
val mb = String.format(Locale.US, "%.1f", totalBytes / 1048576.0)
|
||||
val totalMb = String.format(Locale.US, "%.1f", contentLength / 1048576.0)
|
||||
notifManager.notify(notifId, buildProgressNotif(pct, 100, "$mb / $totalMb MB ($pct%)"))
|
||||
} else {
|
||||
val mb = String.format(Locale.US, "%.1f", totalBytes / 1048576.0)
|
||||
notifManager.notify(notifId, buildProgressNotif(0, 0, "${mb} MB downloaded"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
android.util.Log.w("openclaw", "app.update: downloaded ${file.length()} bytes")
|
||||
val actualSha256 = sha256Hex(file)
|
||||
if (actualSha256 != expectedSha256) {
|
||||
android.util.Log.e(
|
||||
"openclaw",
|
||||
"app.update: sha256 mismatch expected=$expectedSha256 actual=$actualSha256",
|
||||
)
|
||||
file.delete()
|
||||
notifManager.cancel(notifId)
|
||||
notifManager.notify(
|
||||
notifId,
|
||||
android.app.Notification.Builder(appContext, channelId)
|
||||
.setSmallIcon(android.R.drawable.stat_notify_error)
|
||||
.setContentTitle("Update Failed")
|
||||
.setContentIntent(launchPi)
|
||||
.setContentText("SHA-256 mismatch")
|
||||
.build(),
|
||||
)
|
||||
return@launch
|
||||
}
|
||||
|
||||
// Verify file is a valid APK (basic check: ZIP magic bytes)
|
||||
val magic = file.inputStream().use { it.read().toByte() to it.read().toByte() }
|
||||
if (magic.first != 0x50.toByte() || magic.second != 0x4B.toByte()) {
|
||||
android.util.Log.e("openclaw", "app.update: invalid APK (bad magic: ${magic.first}, ${magic.second})")
|
||||
file.delete()
|
||||
notifManager.cancel(notifId)
|
||||
notifManager.notify(notifId, android.app.Notification.Builder(appContext, channelId)
|
||||
.setSmallIcon(android.R.drawable.stat_notify_error)
|
||||
.setContentTitle("Update Failed")
|
||||
|
||||
.setContentIntent(launchPi)
|
||||
.setContentText("Downloaded file is not a valid APK")
|
||||
.build())
|
||||
return@launch
|
||||
}
|
||||
|
||||
// Use PackageInstaller session API — works from background on API 34+
|
||||
// The system handles showing the install confirmation dialog
|
||||
notifManager.cancel(notifId)
|
||||
notifManager.notify(
|
||||
notifId,
|
||||
android.app.Notification.Builder(appContext, channelId)
|
||||
.setSmallIcon(android.R.drawable.stat_sys_download_done)
|
||||
.setContentTitle("Installing Update...")
|
||||
.setContentIntent(launchPi)
|
||||
.setContentText("${String.format(Locale.US, "%.1f", totalBytes / 1048576.0)} MB downloaded")
|
||||
.build(),
|
||||
)
|
||||
|
||||
val installer = appContext.packageManager.packageInstaller
|
||||
val params = android.content.pm.PackageInstaller.SessionParams(
|
||||
android.content.pm.PackageInstaller.SessionParams.MODE_FULL_INSTALL
|
||||
)
|
||||
params.setSize(file.length())
|
||||
val sessionId = installer.createSession(params)
|
||||
val session = installer.openSession(sessionId)
|
||||
session.openWrite("openclaw-update.apk", 0, file.length()).use { out ->
|
||||
file.inputStream().use { inp -> inp.copyTo(out) }
|
||||
session.fsync(out)
|
||||
}
|
||||
// Commit with FLAG_MUTABLE PendingIntent — system requires mutable for PackageInstaller status
|
||||
val callbackIntent = android.content.Intent(appContext, InstallResultReceiver::class.java)
|
||||
val pi = android.app.PendingIntent.getBroadcast(
|
||||
appContext, sessionId, callbackIntent,
|
||||
android.app.PendingIntent.FLAG_UPDATE_CURRENT or android.app.PendingIntent.FLAG_MUTABLE
|
||||
)
|
||||
session.commit(pi.intentSender)
|
||||
android.util.Log.w("openclaw", "app.update: PackageInstaller session committed, waiting for user confirmation")
|
||||
} catch (err: Throwable) {
|
||||
android.util.Log.e("openclaw", "app.update: async error", err)
|
||||
notifManager.cancel(notifId)
|
||||
notifManager.notify(notifId, android.app.Notification.Builder(appContext, channelId)
|
||||
.setSmallIcon(android.R.drawable.stat_notify_error)
|
||||
.setContentTitle("Update Failed")
|
||||
|
||||
.setContentIntent(launchPi)
|
||||
.setContentText(err.message ?: "Unknown error")
|
||||
.build())
|
||||
}
|
||||
}
|
||||
|
||||
// Return immediately — download happens in background
|
||||
return GatewaySession.InvokeResult.ok(buildJsonObject {
|
||||
put("status", "downloading")
|
||||
put("url", url)
|
||||
put("sha256", expectedSha256)
|
||||
}.toString())
|
||||
} catch (err: Throwable) {
|
||||
android.util.Log.e("openclaw", "app.update: error", err)
|
||||
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = err.message ?: "update failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.Manifest
|
||||
import android.annotation.SuppressLint
|
||||
import android.content.Context
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.BitmapFactory
|
||||
import android.graphics.Matrix
|
||||
import android.content.pm.PackageManager
|
||||
import android.hardware.camera2.CameraCharacteristics
|
||||
import android.util.Base64
|
||||
import androidx.camera.camera2.interop.Camera2CameraInfo
|
||||
import androidx.camera.core.CameraInfo
|
||||
import androidx.exifinterface.media.ExifInterface
|
||||
import androidx.lifecycle.LifecycleOwner
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.camera.core.ImageCapture
|
||||
import androidx.camera.core.ImageCaptureException
|
||||
import androidx.camera.lifecycle.ProcessCameraProvider
|
||||
import androidx.camera.video.FileOutputOptions
|
||||
import androidx.camera.video.FallbackStrategy
|
||||
import androidx.camera.video.Quality
|
||||
import androidx.camera.video.QualitySelector
|
||||
import androidx.camera.video.Recorder
|
||||
import androidx.camera.video.Recording
|
||||
import androidx.camera.video.VideoCapture
|
||||
import androidx.camera.video.VideoRecordEvent
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.core.content.ContextCompat.checkSelfPermission
|
||||
import androidx.core.graphics.scale
|
||||
import ai.openclaw.android.PermissionRequester
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlinx.coroutines.withTimeout
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.contentOrNull
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.io.File
|
||||
import java.util.concurrent.Executor
|
||||
import kotlin.math.roundToInt
|
||||
import kotlin.coroutines.resume
|
||||
import kotlin.coroutines.resumeWithException
|
||||
|
||||
class CameraCaptureManager(private val context: Context) {
|
||||
data class Payload(val payloadJson: String)
|
||||
data class FilePayload(val file: File, val durationMs: Long, val hasAudio: Boolean)
|
||||
data class CameraDeviceInfo(
|
||||
val id: String,
|
||||
val name: String,
|
||||
val position: String,
|
||||
val deviceType: String,
|
||||
)
|
||||
|
||||
@Volatile private var lifecycleOwner: LifecycleOwner? = null
|
||||
@Volatile private var permissionRequester: PermissionRequester? = null
|
||||
|
||||
fun attachLifecycleOwner(owner: LifecycleOwner) {
|
||||
lifecycleOwner = owner
|
||||
}
|
||||
|
||||
fun attachPermissionRequester(requester: PermissionRequester) {
|
||||
permissionRequester = requester
|
||||
}
|
||||
|
||||
suspend fun listDevices(): List<CameraDeviceInfo> =
|
||||
withContext(Dispatchers.Main) {
|
||||
val provider = context.cameraProvider()
|
||||
provider.availableCameraInfos
|
||||
.mapNotNull { info -> cameraDeviceInfoOrNull(info) }
|
||||
.sortedBy { it.id }
|
||||
}
|
||||
|
||||
private suspend fun ensureCameraPermission() {
|
||||
val granted = checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED
|
||||
if (granted) return
|
||||
|
||||
val requester = permissionRequester
|
||||
?: throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission")
|
||||
val results = requester.requestIfMissing(listOf(Manifest.permission.CAMERA))
|
||||
if (results[Manifest.permission.CAMERA] != true) {
|
||||
throw IllegalStateException("CAMERA_PERMISSION_REQUIRED: grant Camera permission")
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun ensureMicPermission() {
|
||||
val granted = checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
|
||||
if (granted) return
|
||||
|
||||
val requester = permissionRequester
|
||||
?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
|
||||
val results = requester.requestIfMissing(listOf(Manifest.permission.RECORD_AUDIO))
|
||||
if (results[Manifest.permission.RECORD_AUDIO] != true) {
|
||||
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun snap(paramsJson: String?): Payload =
|
||||
withContext(Dispatchers.Main) {
|
||||
ensureCameraPermission()
|
||||
val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready")
|
||||
val params = parseParamsObject(paramsJson)
|
||||
val facing = parseFacing(params) ?: "front"
|
||||
val quality = (parseQuality(params) ?: 0.95).coerceIn(0.1, 1.0)
|
||||
val maxWidth = parseMaxWidth(params) ?: 1600
|
||||
val deviceId = parseDeviceId(params)
|
||||
|
||||
val provider = context.cameraProvider()
|
||||
val capture = ImageCapture.Builder().build()
|
||||
val selector = resolveCameraSelector(provider, facing, deviceId)
|
||||
|
||||
provider.unbindAll()
|
||||
provider.bindToLifecycle(owner, selector, capture)
|
||||
|
||||
val (bytes, orientation) = capture.takeJpegWithExif(context.mainExecutor())
|
||||
val decoded = BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
|
||||
?: throw IllegalStateException("UNAVAILABLE: failed to decode captured image")
|
||||
val rotated = rotateBitmapByExif(decoded, orientation)
|
||||
val scaled =
|
||||
if (maxWidth > 0 && rotated.width > maxWidth) {
|
||||
val h =
|
||||
(rotated.height.toDouble() * (maxWidth.toDouble() / rotated.width.toDouble()))
|
||||
.toInt()
|
||||
.coerceAtLeast(1)
|
||||
rotated.scale(maxWidth, h)
|
||||
} else {
|
||||
rotated
|
||||
}
|
||||
|
||||
val maxPayloadBytes = 5 * 1024 * 1024
|
||||
// Base64 inflates payloads by ~4/3; cap encoded bytes so the payload stays under 5MB (API limit).
|
||||
val maxEncodedBytes = (maxPayloadBytes / 4) * 3
|
||||
val result =
|
||||
JpegSizeLimiter.compressToLimit(
|
||||
initialWidth = scaled.width,
|
||||
initialHeight = scaled.height,
|
||||
startQuality = (quality * 100.0).roundToInt().coerceIn(10, 100),
|
||||
maxBytes = maxEncodedBytes,
|
||||
encode = { width, height, q ->
|
||||
val bitmap =
|
||||
if (width == scaled.width && height == scaled.height) {
|
||||
scaled
|
||||
} else {
|
||||
scaled.scale(width, height)
|
||||
}
|
||||
val out = ByteArrayOutputStream()
|
||||
if (!bitmap.compress(Bitmap.CompressFormat.JPEG, q, out)) {
|
||||
if (bitmap !== scaled) bitmap.recycle()
|
||||
throw IllegalStateException("UNAVAILABLE: failed to encode JPEG")
|
||||
}
|
||||
if (bitmap !== scaled) {
|
||||
bitmap.recycle()
|
||||
}
|
||||
out.toByteArray()
|
||||
},
|
||||
)
|
||||
val base64 = Base64.encodeToString(result.bytes, Base64.NO_WRAP)
|
||||
Payload(
|
||||
"""{"format":"jpg","base64":"$base64","width":${result.width},"height":${result.height}}""",
|
||||
)
|
||||
}
|
||||
|
||||
@SuppressLint("MissingPermission")
|
||||
suspend fun clip(paramsJson: String?): FilePayload =
|
||||
withContext(Dispatchers.Main) {
|
||||
ensureCameraPermission()
|
||||
val owner = lifecycleOwner ?: throw IllegalStateException("UNAVAILABLE: camera not ready")
|
||||
val params = parseParamsObject(paramsJson)
|
||||
val facing = parseFacing(params) ?: "front"
|
||||
val durationMs = (parseDurationMs(params) ?: 3_000).coerceIn(200, 60_000)
|
||||
val includeAudio = parseIncludeAudio(params) ?: true
|
||||
val deviceId = parseDeviceId(params)
|
||||
if (includeAudio) ensureMicPermission()
|
||||
|
||||
android.util.Log.w("CameraCaptureManager", "clip: start facing=$facing duration=$durationMs audio=$includeAudio deviceId=${deviceId ?: "-"}")
|
||||
|
||||
val provider = context.cameraProvider()
|
||||
android.util.Log.w("CameraCaptureManager", "clip: got camera provider")
|
||||
|
||||
// Use LOWEST quality for smallest files over WebSocket
|
||||
val recorder = Recorder.Builder()
|
||||
.setQualitySelector(
|
||||
QualitySelector.from(Quality.LOWEST, FallbackStrategy.lowerQualityOrHigherThan(Quality.LOWEST))
|
||||
)
|
||||
.build()
|
||||
val videoCapture = VideoCapture.withOutput(recorder)
|
||||
val selector = resolveCameraSelector(provider, facing, deviceId)
|
||||
|
||||
// CameraX requires a Preview use case for the camera to start producing frames;
|
||||
// without it, the encoder may get no data (ERROR_NO_VALID_DATA).
|
||||
val preview = androidx.camera.core.Preview.Builder().build()
|
||||
// Provide a dummy SurfaceTexture so the preview pipeline activates
|
||||
val surfaceTexture = android.graphics.SurfaceTexture(0)
|
||||
surfaceTexture.setDefaultBufferSize(640, 480)
|
||||
preview.setSurfaceProvider { request ->
|
||||
val surface = android.view.Surface(surfaceTexture)
|
||||
request.provideSurface(surface, context.mainExecutor()) { result ->
|
||||
surface.release()
|
||||
surfaceTexture.release()
|
||||
}
|
||||
}
|
||||
|
||||
provider.unbindAll()
|
||||
android.util.Log.w("CameraCaptureManager", "clip: binding preview + videoCapture to lifecycle")
|
||||
val camera = provider.bindToLifecycle(owner, selector, preview, videoCapture)
|
||||
android.util.Log.w("CameraCaptureManager", "clip: bound, cameraInfo=${camera.cameraInfo}")
|
||||
|
||||
// Give camera pipeline time to initialize before recording
|
||||
android.util.Log.w("CameraCaptureManager", "clip: warming up camera 1.5s...")
|
||||
kotlinx.coroutines.delay(1_500)
|
||||
|
||||
val file = File.createTempFile("openclaw-clip-", ".mp4")
|
||||
val outputOptions = FileOutputOptions.Builder(file).build()
|
||||
|
||||
val finalized = kotlinx.coroutines.CompletableDeferred<VideoRecordEvent.Finalize>()
|
||||
android.util.Log.w("CameraCaptureManager", "clip: starting recording to ${file.absolutePath}")
|
||||
val recording: Recording =
|
||||
videoCapture.output
|
||||
.prepareRecording(context, outputOptions)
|
||||
.apply {
|
||||
if (includeAudio) withAudioEnabled()
|
||||
}
|
||||
.start(context.mainExecutor()) { event ->
|
||||
android.util.Log.w("CameraCaptureManager", "clip: event ${event.javaClass.simpleName}")
|
||||
if (event is VideoRecordEvent.Status) {
|
||||
android.util.Log.w("CameraCaptureManager", "clip: recording status update")
|
||||
}
|
||||
if (event is VideoRecordEvent.Finalize) {
|
||||
android.util.Log.w("CameraCaptureManager", "clip: finalize hasError=${event.hasError()} error=${event.error} cause=${event.cause}")
|
||||
finalized.complete(event)
|
||||
}
|
||||
}
|
||||
|
||||
android.util.Log.w("CameraCaptureManager", "clip: recording started, delaying ${durationMs}ms")
|
||||
try {
|
||||
kotlinx.coroutines.delay(durationMs.toLong())
|
||||
} finally {
|
||||
android.util.Log.w("CameraCaptureManager", "clip: stopping recording")
|
||||
recording.stop()
|
||||
}
|
||||
|
||||
val finalizeEvent =
|
||||
try {
|
||||
withTimeout(15_000) { finalized.await() }
|
||||
} catch (err: Throwable) {
|
||||
android.util.Log.e("CameraCaptureManager", "clip: finalize timed out", err)
|
||||
withContext(Dispatchers.IO) { file.delete() }
|
||||
provider.unbindAll()
|
||||
throw IllegalStateException("UNAVAILABLE: camera clip finalize timed out")
|
||||
}
|
||||
if (finalizeEvent.hasError()) {
|
||||
android.util.Log.e("CameraCaptureManager", "clip: FAILED error=${finalizeEvent.error}, cause=${finalizeEvent.cause}", finalizeEvent.cause)
|
||||
// Check file size for debugging
|
||||
val fileSize = withContext(Dispatchers.IO) { if (file.exists()) file.length() else -1 }
|
||||
android.util.Log.e("CameraCaptureManager", "clip: file exists=${file.exists()} size=$fileSize")
|
||||
withContext(Dispatchers.IO) { file.delete() }
|
||||
provider.unbindAll()
|
||||
throw IllegalStateException("UNAVAILABLE: camera clip failed (error=${finalizeEvent.error})")
|
||||
}
|
||||
|
||||
val fileSize = withContext(Dispatchers.IO) { file.length() }
|
||||
android.util.Log.w("CameraCaptureManager", "clip: SUCCESS file size=$fileSize")
|
||||
|
||||
provider.unbindAll()
|
||||
|
||||
FilePayload(file = file, durationMs = durationMs.toLong(), hasAudio = includeAudio)
|
||||
}
|
||||
|
||||
private fun rotateBitmapByExif(bitmap: Bitmap, orientation: Int): Bitmap {
|
||||
val matrix = Matrix()
|
||||
when (orientation) {
|
||||
ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f)
|
||||
ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f)
|
||||
ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f)
|
||||
ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f)
|
||||
ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f)
|
||||
ExifInterface.ORIENTATION_TRANSPOSE -> {
|
||||
matrix.postRotate(90f)
|
||||
matrix.postScale(-1f, 1f)
|
||||
}
|
||||
ExifInterface.ORIENTATION_TRANSVERSE -> {
|
||||
matrix.postRotate(-90f)
|
||||
matrix.postScale(-1f, 1f)
|
||||
}
|
||||
else -> return bitmap
|
||||
}
|
||||
val rotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true)
|
||||
if (rotated !== bitmap) {
|
||||
bitmap.recycle()
|
||||
}
|
||||
return rotated
|
||||
}
|
||||
|
||||
private fun parseParamsObject(paramsJson: String?): JsonObject? {
|
||||
if (paramsJson.isNullOrBlank()) return null
|
||||
return try {
|
||||
Json.parseToJsonElement(paramsJson).asObjectOrNull()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun readPrimitive(params: JsonObject?, key: String): JsonPrimitive? =
|
||||
params?.get(key) as? JsonPrimitive
|
||||
|
||||
private fun parseFacing(params: JsonObject?): String? {
|
||||
val value = readPrimitive(params, "facing")?.contentOrNull?.trim()?.lowercase() ?: return null
|
||||
return when (value) {
|
||||
"front", "back" -> value
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseQuality(params: JsonObject?): Double? =
|
||||
readPrimitive(params, "quality")?.contentOrNull?.toDoubleOrNull()
|
||||
|
||||
private fun parseMaxWidth(params: JsonObject?): Int? =
|
||||
readPrimitive(params, "maxWidth")
|
||||
?.contentOrNull
|
||||
?.toIntOrNull()
|
||||
?.takeIf { it > 0 }
|
||||
|
||||
private fun parseDurationMs(params: JsonObject?): Int? =
|
||||
readPrimitive(params, "durationMs")?.contentOrNull?.toIntOrNull()
|
||||
|
||||
private fun parseDeviceId(params: JsonObject?): String? =
|
||||
readPrimitive(params, "deviceId")
|
||||
?.contentOrNull
|
||||
?.trim()
|
||||
?.takeIf { it.isNotEmpty() }
|
||||
|
||||
private fun parseIncludeAudio(params: JsonObject?): Boolean? {
|
||||
val value = readPrimitive(params, "includeAudio")?.contentOrNull?.trim()?.lowercase()
|
||||
return when (value) {
|
||||
"true" -> true
|
||||
"false" -> false
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
private fun Context.mainExecutor(): Executor = ContextCompat.getMainExecutor(this)
|
||||
|
||||
private fun resolveCameraSelector(
|
||||
provider: ProcessCameraProvider,
|
||||
facing: String,
|
||||
deviceId: String?,
|
||||
): CameraSelector {
|
||||
if (deviceId.isNullOrEmpty()) {
|
||||
return if (facing == "front") CameraSelector.DEFAULT_FRONT_CAMERA else CameraSelector.DEFAULT_BACK_CAMERA
|
||||
}
|
||||
val availableIds = provider.availableCameraInfos.mapNotNull { cameraIdOrNull(it) }.toSet()
|
||||
if (!availableIds.contains(deviceId)) {
|
||||
throw IllegalStateException("INVALID_REQUEST: unknown camera deviceId '$deviceId'")
|
||||
}
|
||||
return CameraSelector.Builder()
|
||||
.addCameraFilter { infos -> infos.filter { cameraIdOrNull(it) == deviceId } }
|
||||
.build()
|
||||
}
|
||||
|
||||
private fun cameraDeviceInfoOrNull(info: CameraInfo): CameraDeviceInfo? {
|
||||
val cameraId = cameraIdOrNull(info) ?: return null
|
||||
val lensFacing =
|
||||
runCatching {
|
||||
Camera2CameraInfo.from(info).getCameraCharacteristic(CameraCharacteristics.LENS_FACING)
|
||||
}.getOrNull()
|
||||
val position =
|
||||
when (lensFacing) {
|
||||
CameraCharacteristics.LENS_FACING_FRONT -> "front"
|
||||
CameraCharacteristics.LENS_FACING_BACK -> "back"
|
||||
CameraCharacteristics.LENS_FACING_EXTERNAL -> "external"
|
||||
else -> "unspecified"
|
||||
}
|
||||
val deviceType =
|
||||
if (lensFacing == CameraCharacteristics.LENS_FACING_EXTERNAL) "external" else "builtIn"
|
||||
val name =
|
||||
when (position) {
|
||||
"front" -> "Front Camera"
|
||||
"back" -> "Back Camera"
|
||||
"external" -> "External Camera"
|
||||
else -> "Camera $cameraId"
|
||||
}
|
||||
return CameraDeviceInfo(
|
||||
id = cameraId,
|
||||
name = name,
|
||||
position = position,
|
||||
deviceType = deviceType,
|
||||
)
|
||||
}
|
||||
|
||||
private fun cameraIdOrNull(info: CameraInfo): String? =
|
||||
runCatching { Camera2CameraInfo.from(info).cameraId }.getOrNull()
|
||||
}
|
||||
|
||||
private suspend fun Context.cameraProvider(): ProcessCameraProvider =
|
||||
suspendCancellableCoroutine { cont ->
|
||||
val future = ProcessCameraProvider.getInstance(this)
|
||||
future.addListener(
|
||||
{
|
||||
try {
|
||||
cont.resume(future.get())
|
||||
} catch (e: Exception) {
|
||||
cont.resumeWithException(e)
|
||||
}
|
||||
},
|
||||
ContextCompat.getMainExecutor(this),
|
||||
)
|
||||
}
|
||||
|
||||
/** Returns (jpegBytes, exifOrientation) so caller can rotate the decoded bitmap. */
|
||||
private suspend fun ImageCapture.takeJpegWithExif(executor: Executor): Pair<ByteArray, Int> =
|
||||
suspendCancellableCoroutine { cont ->
|
||||
val file = File.createTempFile("openclaw-snap-", ".jpg")
|
||||
val options = ImageCapture.OutputFileOptions.Builder(file).build()
|
||||
takePicture(
|
||||
options,
|
||||
executor,
|
||||
object : ImageCapture.OnImageSavedCallback {
|
||||
override fun onError(exception: ImageCaptureException) {
|
||||
file.delete()
|
||||
cont.resumeWithException(exception)
|
||||
}
|
||||
|
||||
override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) {
|
||||
try {
|
||||
val exif = ExifInterface(file.absolutePath)
|
||||
val orientation = exif.getAttributeInt(
|
||||
ExifInterface.TAG_ORIENTATION,
|
||||
ExifInterface.ORIENTATION_NORMAL,
|
||||
)
|
||||
val bytes = file.readBytes()
|
||||
cont.resume(Pair(bytes, orientation))
|
||||
} catch (e: Exception) {
|
||||
cont.resumeWithException(e)
|
||||
} finally {
|
||||
file.delete()
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.content.Context
|
||||
import ai.openclaw.android.CameraHudKind
|
||||
import ai.openclaw.android.BuildConfig
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonArray
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import kotlinx.serialization.json.contentOrNull
|
||||
import kotlinx.serialization.json.put
|
||||
|
||||
internal const val CAMERA_CLIP_MAX_RAW_BYTES: Long = 18L * 1024L * 1024L
|
||||
|
||||
internal fun isCameraClipWithinPayloadLimit(rawBytes: Long): Boolean =
|
||||
rawBytes in 0L..CAMERA_CLIP_MAX_RAW_BYTES
|
||||
|
||||
class CameraHandler(
|
||||
private val appContext: Context,
|
||||
private val camera: CameraCaptureManager,
|
||||
private val externalAudioCaptureActive: MutableStateFlow<Boolean>,
|
||||
private val showCameraHud: (message: String, kind: CameraHudKind, autoHideMs: Long?) -> Unit,
|
||||
private val triggerCameraFlash: () -> Unit,
|
||||
private val invokeErrorFromThrowable: (err: Throwable) -> Pair<String, String>,
|
||||
) {
|
||||
suspend fun handleList(_paramsJson: String?): GatewaySession.InvokeResult {
|
||||
return try {
|
||||
val devices = camera.listDevices()
|
||||
val payload =
|
||||
buildJsonObject {
|
||||
put(
|
||||
"devices",
|
||||
buildJsonArray {
|
||||
devices.forEach { device ->
|
||||
add(
|
||||
buildJsonObject {
|
||||
put("id", JsonPrimitive(device.id))
|
||||
put("name", JsonPrimitive(device.name))
|
||||
put("position", JsonPrimitive(device.position))
|
||||
put("deviceType", JsonPrimitive(device.deviceType))
|
||||
},
|
||||
)
|
||||
}
|
||||
},
|
||||
)
|
||||
}.toString()
|
||||
GatewaySession.InvokeResult.ok(payload)
|
||||
} catch (err: Throwable) {
|
||||
val (code, message) = invokeErrorFromThrowable(err)
|
||||
GatewaySession.InvokeResult.error(code = code, message = message)
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun handleSnap(paramsJson: String?): GatewaySession.InvokeResult {
|
||||
val logFile = if (BuildConfig.DEBUG) java.io.File(appContext.cacheDir, "camera_debug.log") else null
|
||||
fun camLog(msg: String) {
|
||||
if (!BuildConfig.DEBUG) return
|
||||
val ts = java.text.SimpleDateFormat("HH:mm:ss.SSS", java.util.Locale.US).format(java.util.Date())
|
||||
logFile?.appendText("[$ts] $msg\n")
|
||||
android.util.Log.w("openclaw", "camera.snap: $msg")
|
||||
}
|
||||
try {
|
||||
logFile?.writeText("") // clear
|
||||
camLog("starting, params=$paramsJson")
|
||||
camLog("calling showCameraHud")
|
||||
showCameraHud("Taking photo…", CameraHudKind.Photo, null)
|
||||
camLog("calling triggerCameraFlash")
|
||||
triggerCameraFlash()
|
||||
val res =
|
||||
try {
|
||||
camLog("calling camera.snap()")
|
||||
val r = camera.snap(paramsJson)
|
||||
camLog("success, payload size=${r.payloadJson.length}")
|
||||
r
|
||||
} catch (err: Throwable) {
|
||||
camLog("inner error: ${err::class.java.simpleName}: ${err.message}")
|
||||
camLog("stack: ${err.stackTraceToString().take(2000)}")
|
||||
val (code, message) = invokeErrorFromThrowable(err)
|
||||
showCameraHud(message, CameraHudKind.Error, 2200)
|
||||
return GatewaySession.InvokeResult.error(code = code, message = message)
|
||||
}
|
||||
camLog("returning result")
|
||||
showCameraHud("Photo captured", CameraHudKind.Success, 1600)
|
||||
return GatewaySession.InvokeResult.ok(res.payloadJson)
|
||||
} catch (err: Throwable) {
|
||||
camLog("outer error: ${err::class.java.simpleName}: ${err.message}")
|
||||
camLog("stack: ${err.stackTraceToString().take(2000)}")
|
||||
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = err.message ?: "camera snap failed")
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun handleClip(paramsJson: String?): GatewaySession.InvokeResult {
|
||||
val clipLogFile = if (BuildConfig.DEBUG) java.io.File(appContext.cacheDir, "camera_debug.log") else null
|
||||
fun clipLog(msg: String) {
|
||||
if (!BuildConfig.DEBUG) return
|
||||
val ts = java.text.SimpleDateFormat("HH:mm:ss.SSS", java.util.Locale.US).format(java.util.Date())
|
||||
clipLogFile?.appendText("[CLIP $ts] $msg\n")
|
||||
android.util.Log.w("openclaw", "camera.clip: $msg")
|
||||
}
|
||||
val includeAudio = parseIncludeAudio(paramsJson) ?: true
|
||||
if (includeAudio) externalAudioCaptureActive.value = true
|
||||
try {
|
||||
clipLogFile?.writeText("") // clear
|
||||
clipLog("starting, params=$paramsJson includeAudio=$includeAudio")
|
||||
clipLog("calling showCameraHud")
|
||||
showCameraHud("Recording…", CameraHudKind.Recording, null)
|
||||
val filePayload =
|
||||
try {
|
||||
clipLog("calling camera.clip()")
|
||||
val r = camera.clip(paramsJson)
|
||||
clipLog("success, file size=${r.file.length()}")
|
||||
r
|
||||
} catch (err: Throwable) {
|
||||
clipLog("inner error: ${err::class.java.simpleName}: ${err.message}")
|
||||
clipLog("stack: ${err.stackTraceToString().take(2000)}")
|
||||
val (code, message) = invokeErrorFromThrowable(err)
|
||||
showCameraHud(message, CameraHudKind.Error, 2400)
|
||||
return GatewaySession.InvokeResult.error(code = code, message = message)
|
||||
}
|
||||
val rawBytes = filePayload.file.length()
|
||||
if (!isCameraClipWithinPayloadLimit(rawBytes)) {
|
||||
clipLog("payload too large: bytes=$rawBytes max=$CAMERA_CLIP_MAX_RAW_BYTES")
|
||||
withContext(Dispatchers.IO) { filePayload.file.delete() }
|
||||
showCameraHud("Clip too large", CameraHudKind.Error, 2400)
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "PAYLOAD_TOO_LARGE",
|
||||
message =
|
||||
"PAYLOAD_TOO_LARGE: camera clip is $rawBytes bytes; max is $CAMERA_CLIP_MAX_RAW_BYTES bytes. Reduce durationMs and retry.",
|
||||
)
|
||||
}
|
||||
|
||||
val bytes = withContext(Dispatchers.IO) {
|
||||
val b = filePayload.file.readBytes()
|
||||
filePayload.file.delete()
|
||||
b
|
||||
}
|
||||
val base64 = android.util.Base64.encodeToString(bytes, android.util.Base64.NO_WRAP)
|
||||
clipLog("returning base64 payload")
|
||||
showCameraHud("Clip captured", CameraHudKind.Success, 1800)
|
||||
return GatewaySession.InvokeResult.ok(
|
||||
"""{"format":"mp4","base64":"$base64","durationMs":${filePayload.durationMs},"hasAudio":${filePayload.hasAudio}}"""
|
||||
)
|
||||
} catch (err: Throwable) {
|
||||
clipLog("outer error: ${err::class.java.simpleName}: ${err.message}")
|
||||
clipLog("stack: ${err.stackTraceToString().take(2000)}")
|
||||
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = err.message ?: "camera clip failed")
|
||||
} finally {
|
||||
if (includeAudio) externalAudioCaptureActive.value = false
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseIncludeAudio(paramsJson: String?): Boolean? {
|
||||
if (paramsJson.isNullOrBlank()) return null
|
||||
val root =
|
||||
try {
|
||||
Json.parseToJsonElement(paramsJson).asObjectOrNull()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
} ?: return null
|
||||
val value =
|
||||
(root["includeAudio"] as? JsonPrimitive)
|
||||
?.contentOrNull
|
||||
?.trim()
|
||||
?.lowercase()
|
||||
return when (value) {
|
||||
"true" -> true
|
||||
"false" -> false
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.os.Looper
|
||||
import android.util.Log
|
||||
import android.webkit.WebView
|
||||
import androidx.core.graphics.createBitmap
|
||||
import androidx.core.graphics.scale
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import java.io.ByteArrayOutputStream
|
||||
import android.util.Base64
|
||||
import org.json.JSONObject
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import ai.openclaw.android.BuildConfig
|
||||
import kotlin.coroutines.resume
|
||||
|
||||
class CanvasController {
|
||||
enum class SnapshotFormat(val rawValue: String) {
|
||||
Png("png"),
|
||||
Jpeg("jpeg"),
|
||||
}
|
||||
|
||||
@Volatile private var webView: WebView? = null
|
||||
@Volatile private var url: String? = null
|
||||
@Volatile private var debugStatusEnabled: Boolean = false
|
||||
@Volatile private var debugStatusTitle: String? = null
|
||||
@Volatile private var debugStatusSubtitle: String? = null
|
||||
private val _currentUrl = MutableStateFlow<String?>(null)
|
||||
val currentUrl: StateFlow<String?> = _currentUrl.asStateFlow()
|
||||
|
||||
private val scaffoldAssetUrl = "file:///android_asset/CanvasScaffold/scaffold.html"
|
||||
|
||||
private fun clampJpegQuality(quality: Double?): Int {
|
||||
val q = (quality ?: 0.82).coerceIn(0.1, 1.0)
|
||||
return (q * 100.0).toInt().coerceIn(1, 100)
|
||||
}
|
||||
|
||||
fun attach(webView: WebView) {
|
||||
this.webView = webView
|
||||
reload()
|
||||
applyDebugStatus()
|
||||
}
|
||||
|
||||
fun detach(webView: WebView) {
|
||||
if (this.webView === webView) {
|
||||
this.webView = null
|
||||
}
|
||||
}
|
||||
|
||||
fun navigate(url: String) {
|
||||
val trimmed = url.trim()
|
||||
this.url = if (trimmed.isBlank() || trimmed == "/") null else trimmed
|
||||
_currentUrl.value = this.url
|
||||
reload()
|
||||
}
|
||||
|
||||
fun currentUrl(): String? = url
|
||||
|
||||
fun isDefaultCanvas(): Boolean = url == null
|
||||
|
||||
fun setDebugStatusEnabled(enabled: Boolean) {
|
||||
debugStatusEnabled = enabled
|
||||
applyDebugStatus()
|
||||
}
|
||||
|
||||
fun setDebugStatus(title: String?, subtitle: String?) {
|
||||
debugStatusTitle = title
|
||||
debugStatusSubtitle = subtitle
|
||||
applyDebugStatus()
|
||||
}
|
||||
|
||||
fun onPageFinished() {
|
||||
applyDebugStatus()
|
||||
}
|
||||
|
||||
private inline fun withWebViewOnMain(crossinline block: (WebView) -> Unit) {
|
||||
val wv = webView ?: return
|
||||
if (Looper.myLooper() == Looper.getMainLooper()) {
|
||||
block(wv)
|
||||
} else {
|
||||
wv.post { block(wv) }
|
||||
}
|
||||
}
|
||||
|
||||
private fun reload() {
|
||||
val currentUrl = url
|
||||
withWebViewOnMain { wv ->
|
||||
if (currentUrl == null) {
|
||||
if (BuildConfig.DEBUG) {
|
||||
Log.d("OpenClawCanvas", "load scaffold: $scaffoldAssetUrl")
|
||||
}
|
||||
wv.loadUrl(scaffoldAssetUrl)
|
||||
} else {
|
||||
if (BuildConfig.DEBUG) {
|
||||
Log.d("OpenClawCanvas", "load url: $currentUrl")
|
||||
}
|
||||
wv.loadUrl(currentUrl)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun applyDebugStatus() {
|
||||
val enabled = debugStatusEnabled
|
||||
val title = debugStatusTitle
|
||||
val subtitle = debugStatusSubtitle
|
||||
withWebViewOnMain { wv ->
|
||||
val titleJs = title?.let { JSONObject.quote(it) } ?: "null"
|
||||
val subtitleJs = subtitle?.let { JSONObject.quote(it) } ?: "null"
|
||||
val js = """
|
||||
(() => {
|
||||
try {
|
||||
const api = globalThis.__openclaw;
|
||||
if (!api) return;
|
||||
if (typeof api.setDebugStatusEnabled === 'function') {
|
||||
api.setDebugStatusEnabled(${if (enabled) "true" else "false"});
|
||||
}
|
||||
if (!${if (enabled) "true" else "false"}) return;
|
||||
if (typeof api.setStatus === 'function') {
|
||||
api.setStatus($titleJs, $subtitleJs);
|
||||
}
|
||||
} catch (_) {}
|
||||
})();
|
||||
""".trimIndent()
|
||||
wv.evaluateJavascript(js, null)
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun eval(javaScript: String): String =
|
||||
withContext(Dispatchers.Main) {
|
||||
val wv = webView ?: throw IllegalStateException("no webview")
|
||||
suspendCancellableCoroutine { cont ->
|
||||
wv.evaluateJavascript(javaScript) { result ->
|
||||
cont.resume(result ?: "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun snapshotPngBase64(maxWidth: Int?): String =
|
||||
withContext(Dispatchers.Main) {
|
||||
val wv = webView ?: throw IllegalStateException("no webview")
|
||||
val bmp = wv.captureBitmap()
|
||||
val scaled =
|
||||
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
|
||||
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
|
||||
bmp.scale(maxWidth, h)
|
||||
} else {
|
||||
bmp
|
||||
}
|
||||
|
||||
val out = ByteArrayOutputStream()
|
||||
scaled.compress(Bitmap.CompressFormat.PNG, 100, out)
|
||||
Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP)
|
||||
}
|
||||
|
||||
suspend fun snapshotBase64(format: SnapshotFormat, quality: Double?, maxWidth: Int?): String =
|
||||
withContext(Dispatchers.Main) {
|
||||
val wv = webView ?: throw IllegalStateException("no webview")
|
||||
val bmp = wv.captureBitmap()
|
||||
val scaled =
|
||||
if (maxWidth != null && maxWidth > 0 && bmp.width > maxWidth) {
|
||||
val h = (bmp.height.toDouble() * (maxWidth.toDouble() / bmp.width.toDouble())).toInt().coerceAtLeast(1)
|
||||
bmp.scale(maxWidth, h)
|
||||
} else {
|
||||
bmp
|
||||
}
|
||||
|
||||
val out = ByteArrayOutputStream()
|
||||
val (compressFormat, compressQuality) =
|
||||
when (format) {
|
||||
SnapshotFormat.Png -> Bitmap.CompressFormat.PNG to 100
|
||||
SnapshotFormat.Jpeg -> Bitmap.CompressFormat.JPEG to clampJpegQuality(quality)
|
||||
}
|
||||
scaled.compress(compressFormat, compressQuality, out)
|
||||
Base64.encodeToString(out.toByteArray(), Base64.NO_WRAP)
|
||||
}
|
||||
|
||||
private suspend fun WebView.captureBitmap(): Bitmap =
|
||||
suspendCancellableCoroutine { cont ->
|
||||
val width = width.coerceAtLeast(1)
|
||||
val height = height.coerceAtLeast(1)
|
||||
val bitmap = createBitmap(width, height, Bitmap.Config.ARGB_8888)
|
||||
|
||||
// WebView isn't supported by PixelCopy.request(...) directly; draw() is the most reliable
|
||||
// cross-version snapshot for this lightweight "canvas" use-case.
|
||||
draw(Canvas(bitmap))
|
||||
cont.resume(bitmap)
|
||||
}
|
||||
|
||||
companion object {
|
||||
data class SnapshotParams(val format: SnapshotFormat, val quality: Double?, val maxWidth: Int?)
|
||||
|
||||
fun parseNavigateUrl(paramsJson: String?): String {
|
||||
val obj = parseParamsObject(paramsJson) ?: return ""
|
||||
return obj.string("url").trim()
|
||||
}
|
||||
|
||||
fun parseEvalJs(paramsJson: String?): String? {
|
||||
val obj = parseParamsObject(paramsJson) ?: return null
|
||||
val js = obj.string("javaScript").trim()
|
||||
return js.takeIf { it.isNotBlank() }
|
||||
}
|
||||
|
||||
fun parseSnapshotMaxWidth(paramsJson: String?): Int? {
|
||||
val obj = parseParamsObject(paramsJson) ?: return null
|
||||
if (!obj.containsKey("maxWidth")) return null
|
||||
val width = obj.int("maxWidth") ?: 0
|
||||
return width.takeIf { it > 0 }
|
||||
}
|
||||
|
||||
fun parseSnapshotFormat(paramsJson: String?): SnapshotFormat {
|
||||
val obj = parseParamsObject(paramsJson) ?: return SnapshotFormat.Jpeg
|
||||
val raw = obj.string("format").trim().lowercase()
|
||||
return when (raw) {
|
||||
"png" -> SnapshotFormat.Png
|
||||
"jpeg", "jpg" -> SnapshotFormat.Jpeg
|
||||
"" -> SnapshotFormat.Jpeg
|
||||
else -> SnapshotFormat.Jpeg
|
||||
}
|
||||
}
|
||||
|
||||
fun parseSnapshotQuality(paramsJson: String?): Double? {
|
||||
val obj = parseParamsObject(paramsJson) ?: return null
|
||||
if (!obj.containsKey("quality")) return null
|
||||
val q = obj.double("quality") ?: Double.NaN
|
||||
if (!q.isFinite()) return null
|
||||
return q.coerceIn(0.1, 1.0)
|
||||
}
|
||||
|
||||
fun parseSnapshotParams(paramsJson: String?): SnapshotParams {
|
||||
return SnapshotParams(
|
||||
format = parseSnapshotFormat(paramsJson),
|
||||
quality = parseSnapshotQuality(paramsJson),
|
||||
maxWidth = parseSnapshotMaxWidth(paramsJson),
|
||||
)
|
||||
}
|
||||
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
|
||||
private fun parseParamsObject(paramsJson: String?): JsonObject? {
|
||||
val raw = paramsJson?.trim().orEmpty()
|
||||
if (raw.isEmpty()) return null
|
||||
return try {
|
||||
json.parseToJsonElement(raw).asObjectOrNull()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
|
||||
|
||||
private fun JsonObject.string(key: String): String {
|
||||
val prim = this[key] as? JsonPrimitive ?: return ""
|
||||
val raw = prim.content
|
||||
return raw.takeIf { it != "null" }.orEmpty()
|
||||
}
|
||||
|
||||
private fun JsonObject.int(key: String): Int? {
|
||||
val prim = this[key] as? JsonPrimitive ?: return null
|
||||
return prim.content.toIntOrNull()
|
||||
}
|
||||
|
||||
private fun JsonObject.double(key: String): Double? {
|
||||
val prim = this[key] as? JsonPrimitive ?: return null
|
||||
return prim.content.toDoubleOrNull()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.os.Build
|
||||
import ai.openclaw.android.BuildConfig
|
||||
import ai.openclaw.android.SecurePrefs
|
||||
import ai.openclaw.android.gateway.GatewayClientInfo
|
||||
import ai.openclaw.android.gateway.GatewayConnectOptions
|
||||
import ai.openclaw.android.gateway.GatewayEndpoint
|
||||
import ai.openclaw.android.gateway.GatewayTlsParams
|
||||
import ai.openclaw.android.protocol.OpenClawCapability
|
||||
import ai.openclaw.android.LocationMode
|
||||
import ai.openclaw.android.VoiceWakeMode
|
||||
|
||||
class ConnectionManager(
|
||||
private val prefs: SecurePrefs,
|
||||
private val cameraEnabled: () -> Boolean,
|
||||
private val locationMode: () -> LocationMode,
|
||||
private val voiceWakeMode: () -> VoiceWakeMode,
|
||||
private val smsAvailable: () -> Boolean,
|
||||
private val hasRecordAudioPermission: () -> Boolean,
|
||||
private val manualTls: () -> Boolean,
|
||||
) {
|
||||
companion object {
|
||||
internal fun resolveTlsParamsForEndpoint(
|
||||
endpoint: GatewayEndpoint,
|
||||
storedFingerprint: String?,
|
||||
manualTlsEnabled: Boolean,
|
||||
): GatewayTlsParams? {
|
||||
val stableId = endpoint.stableId
|
||||
val stored = storedFingerprint?.trim().takeIf { !it.isNullOrEmpty() }
|
||||
val isManual = stableId.startsWith("manual|")
|
||||
|
||||
if (isManual) {
|
||||
if (!manualTlsEnabled) return null
|
||||
if (!stored.isNullOrBlank()) {
|
||||
return GatewayTlsParams(
|
||||
required = true,
|
||||
expectedFingerprint = stored,
|
||||
allowTOFU = false,
|
||||
stableId = stableId,
|
||||
)
|
||||
}
|
||||
return GatewayTlsParams(
|
||||
required = true,
|
||||
expectedFingerprint = null,
|
||||
allowTOFU = false,
|
||||
stableId = stableId,
|
||||
)
|
||||
}
|
||||
|
||||
// Prefer stored pins. Never let discovery-provided TXT override a stored fingerprint.
|
||||
if (!stored.isNullOrBlank()) {
|
||||
return GatewayTlsParams(
|
||||
required = true,
|
||||
expectedFingerprint = stored,
|
||||
allowTOFU = false,
|
||||
stableId = stableId,
|
||||
)
|
||||
}
|
||||
|
||||
val hinted = endpoint.tlsEnabled || !endpoint.tlsFingerprintSha256.isNullOrBlank()
|
||||
if (hinted) {
|
||||
// TXT is unauthenticated. Do not treat the advertised fingerprint as authoritative.
|
||||
return GatewayTlsParams(
|
||||
required = true,
|
||||
expectedFingerprint = null,
|
||||
allowTOFU = false,
|
||||
stableId = stableId,
|
||||
)
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
fun buildInvokeCommands(): List<String> =
|
||||
InvokeCommandRegistry.advertisedCommands(
|
||||
cameraEnabled = cameraEnabled(),
|
||||
locationEnabled = locationMode() != LocationMode.Off,
|
||||
smsAvailable = smsAvailable(),
|
||||
debugBuild = BuildConfig.DEBUG,
|
||||
)
|
||||
|
||||
fun buildCapabilities(): List<String> =
|
||||
buildList {
|
||||
add(OpenClawCapability.Canvas.rawValue)
|
||||
add(OpenClawCapability.Screen.rawValue)
|
||||
add(OpenClawCapability.Device.rawValue)
|
||||
if (cameraEnabled()) add(OpenClawCapability.Camera.rawValue)
|
||||
if (smsAvailable()) add(OpenClawCapability.Sms.rawValue)
|
||||
if (voiceWakeMode() != VoiceWakeMode.Off && hasRecordAudioPermission()) {
|
||||
add(OpenClawCapability.VoiceWake.rawValue)
|
||||
}
|
||||
if (locationMode() != LocationMode.Off) {
|
||||
add(OpenClawCapability.Location.rawValue)
|
||||
}
|
||||
}
|
||||
|
||||
fun resolvedVersionName(): String {
|
||||
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
|
||||
return if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
|
||||
"$versionName-dev"
|
||||
} else {
|
||||
versionName
|
||||
}
|
||||
}
|
||||
|
||||
fun resolveModelIdentifier(): String? {
|
||||
return listOfNotNull(Build.MANUFACTURER, Build.MODEL)
|
||||
.joinToString(" ")
|
||||
.trim()
|
||||
.ifEmpty { null }
|
||||
}
|
||||
|
||||
fun buildUserAgent(): String {
|
||||
val version = resolvedVersionName()
|
||||
val release = Build.VERSION.RELEASE?.trim().orEmpty()
|
||||
val releaseLabel = if (release.isEmpty()) "unknown" else release
|
||||
return "OpenClawAndroid/$version (Android $releaseLabel; SDK ${Build.VERSION.SDK_INT})"
|
||||
}
|
||||
|
||||
fun buildClientInfo(clientId: String, clientMode: String): GatewayClientInfo {
|
||||
return GatewayClientInfo(
|
||||
id = clientId,
|
||||
displayName = prefs.displayName.value,
|
||||
version = resolvedVersionName(),
|
||||
platform = "android",
|
||||
mode = clientMode,
|
||||
instanceId = prefs.instanceId.value,
|
||||
deviceFamily = "Android",
|
||||
modelIdentifier = resolveModelIdentifier(),
|
||||
)
|
||||
}
|
||||
|
||||
fun buildNodeConnectOptions(): GatewayConnectOptions {
|
||||
return GatewayConnectOptions(
|
||||
role = "node",
|
||||
scopes = emptyList(),
|
||||
caps = buildCapabilities(),
|
||||
commands = buildInvokeCommands(),
|
||||
permissions = emptyMap(),
|
||||
client = buildClientInfo(clientId = "openclaw-android", clientMode = "node"),
|
||||
userAgent = buildUserAgent(),
|
||||
)
|
||||
}
|
||||
|
||||
fun buildOperatorConnectOptions(): GatewayConnectOptions {
|
||||
return GatewayConnectOptions(
|
||||
role = "operator",
|
||||
scopes = listOf("operator.read", "operator.write", "operator.talk.secrets"),
|
||||
caps = emptyList(),
|
||||
commands = emptyList(),
|
||||
permissions = emptyMap(),
|
||||
client = buildClientInfo(clientId = "openclaw-android", clientMode = "ui"),
|
||||
userAgent = buildUserAgent(),
|
||||
)
|
||||
}
|
||||
|
||||
fun resolveTlsParams(endpoint: GatewayEndpoint): GatewayTlsParams? {
|
||||
val stored = prefs.loadGatewayTlsFingerprint(endpoint.stableId)
|
||||
return resolveTlsParamsForEndpoint(endpoint, storedFingerprint = stored, manualTlsEnabled = manualTls())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.content.Context
|
||||
import ai.openclaw.android.BuildConfig
|
||||
import ai.openclaw.android.gateway.DeviceIdentityStore
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
|
||||
class DebugHandler(
|
||||
private val appContext: Context,
|
||||
private val identityStore: DeviceIdentityStore,
|
||||
) {
|
||||
|
||||
fun handleEd25519(): GatewaySession.InvokeResult {
|
||||
if (!BuildConfig.DEBUG) {
|
||||
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = "debug commands are disabled in release builds")
|
||||
}
|
||||
// Self-test Ed25519 signing and return diagnostic info
|
||||
try {
|
||||
val identity = identityStore.loadOrCreate()
|
||||
val testPayload = "test|${identity.deviceId}|${System.currentTimeMillis()}"
|
||||
val results = mutableListOf<String>()
|
||||
results.add("deviceId: ${identity.deviceId}")
|
||||
results.add("publicKeyRawBase64: ${identity.publicKeyRawBase64.take(20)}...")
|
||||
results.add("privateKeyPkcs8Base64: ${identity.privateKeyPkcs8Base64.take(20)}...")
|
||||
|
||||
// Test publicKeyBase64Url
|
||||
val pubKeyUrl = identityStore.publicKeyBase64Url(identity)
|
||||
results.add("publicKeyBase64Url: ${pubKeyUrl ?: "NULL (FAILED)"}")
|
||||
|
||||
// Test signing
|
||||
val signature = identityStore.signPayload(testPayload, identity)
|
||||
results.add("signPayload: ${if (signature != null) "${signature.take(20)}... (OK)" else "NULL (FAILED)"}")
|
||||
|
||||
// Test self-verify
|
||||
if (signature != null) {
|
||||
val verifyOk = identityStore.verifySelfSignature(testPayload, signature, identity)
|
||||
results.add("verifySelfSignature: $verifyOk")
|
||||
}
|
||||
|
||||
// Check available providers
|
||||
val providers = java.security.Security.getProviders()
|
||||
val ed25519Providers = providers.filter { p ->
|
||||
p.services.any { s -> s.algorithm.contains("Ed25519", ignoreCase = true) }
|
||||
}
|
||||
results.add("Ed25519 providers: ${ed25519Providers.map { "${it.name} v${it.version}" }}")
|
||||
results.add("Provider order: ${providers.take(5).map { it.name }}")
|
||||
|
||||
// Test KeyFactory directly
|
||||
try {
|
||||
val kf = java.security.KeyFactory.getInstance("Ed25519")
|
||||
results.add("KeyFactory.Ed25519: ${kf.provider.name} (OK)")
|
||||
} catch (e: Throwable) {
|
||||
results.add("KeyFactory.Ed25519: FAILED - ${e.javaClass.simpleName}: ${e.message}")
|
||||
}
|
||||
|
||||
// Test Signature directly
|
||||
try {
|
||||
val sig = java.security.Signature.getInstance("Ed25519")
|
||||
results.add("Signature.Ed25519: ${sig.provider.name} (OK)")
|
||||
} catch (e: Throwable) {
|
||||
results.add("Signature.Ed25519: FAILED - ${e.javaClass.simpleName}: ${e.message}")
|
||||
}
|
||||
|
||||
val diagnostics = results.joinToString("\n")
|
||||
return GatewaySession.InvokeResult.ok("""{"diagnostics":${JsonPrimitive(diagnostics)}}""")
|
||||
} catch (e: Throwable) {
|
||||
return GatewaySession.InvokeResult.error(code = "ED25519_TEST_FAILED", message = "${e.javaClass.simpleName}: ${e.message}\n${e.stackTraceToString().take(500)}")
|
||||
}
|
||||
}
|
||||
|
||||
fun handleLogs(): GatewaySession.InvokeResult {
|
||||
if (!BuildConfig.DEBUG) {
|
||||
return GatewaySession.InvokeResult.error(code = "UNAVAILABLE", message = "debug commands are disabled in release builds")
|
||||
}
|
||||
val pid = android.os.Process.myPid()
|
||||
val rt = Runtime.getRuntime()
|
||||
val info = "v6 pid=$pid thread=${Thread.currentThread().name} free=${rt.freeMemory()/1024}K total=${rt.totalMemory()/1024}K max=${rt.maxMemory()/1024}K uptime=${android.os.SystemClock.elapsedRealtime()/1000}s sdk=${android.os.Build.VERSION.SDK_INT} device=${android.os.Build.MODEL}\n"
|
||||
// Run logcat on current dispatcher thread (no withContext) with file redirect
|
||||
val logResult = try {
|
||||
val tmpFile = java.io.File(appContext.cacheDir, "debug_logs.txt")
|
||||
if (tmpFile.exists()) tmpFile.delete()
|
||||
val pb = ProcessBuilder("logcat", "-d", "-t", "200", "--pid=$pid")
|
||||
pb.redirectOutput(tmpFile)
|
||||
pb.redirectErrorStream(true)
|
||||
val proc = pb.start()
|
||||
val finished = proc.waitFor(4, java.util.concurrent.TimeUnit.SECONDS)
|
||||
if (!finished) proc.destroyForcibly()
|
||||
val raw = if (tmpFile.exists() && tmpFile.length() > 0) {
|
||||
tmpFile.readText().take(128000)
|
||||
} else {
|
||||
"(no output, finished=$finished, exists=${tmpFile.exists()})"
|
||||
}
|
||||
tmpFile.delete()
|
||||
val spamPatterns = listOf("setRequestedFrameRate", "I View :", "BLASTBufferQueue", "VRI[Pop-Up",
|
||||
"InsetsController:", "VRI[MainActivity", "InsetsSource:", "handleResized", "ProfileInstaller",
|
||||
"I VRI[", "onStateChanged: host=", "D StrictMode:", "E StrictMode:", "ImeFocusController",
|
||||
"InputTransport", "IncorrectContextUseViolation")
|
||||
val sb = StringBuilder()
|
||||
for (line in raw.lineSequence()) {
|
||||
if (line.isBlank()) continue
|
||||
if (spamPatterns.any { line.contains(it) }) continue
|
||||
if (sb.length + line.length > 16000) { sb.append("\n(truncated)"); break }
|
||||
if (sb.isNotEmpty()) sb.append('\n')
|
||||
sb.append(line)
|
||||
}
|
||||
sb.toString().ifEmpty { "(all ${raw.lines().size} lines filtered as spam)" }
|
||||
} catch (e: Throwable) {
|
||||
"(logcat error: ${e::class.java.simpleName}: ${e.message})"
|
||||
}
|
||||
// Also include camera debug log if it exists
|
||||
val camLogFile = java.io.File(appContext.cacheDir, "camera_debug.log")
|
||||
val camLog = if (camLogFile.exists() && camLogFile.length() > 0) {
|
||||
"\n--- camera_debug.log ---\n" + camLogFile.readText().take(4000)
|
||||
} else ""
|
||||
return GatewaySession.InvokeResult.ok("""{"logs":${JsonPrimitive(info + logResult + camLog)}}""")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.Manifest
|
||||
import android.app.ActivityManager
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.IntentFilter
|
||||
import android.content.pm.PackageManager
|
||||
import android.net.ConnectivityManager
|
||||
import android.net.NetworkCapabilities
|
||||
import android.os.BatteryManager
|
||||
import android.os.Build
|
||||
import android.os.Environment
|
||||
import android.os.PowerManager
|
||||
import android.os.StatFs
|
||||
import android.os.SystemClock
|
||||
import androidx.core.content.ContextCompat
|
||||
import ai.openclaw.android.BuildConfig
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import java.util.Locale
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonArray
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import kotlinx.serialization.json.put
|
||||
|
||||
class DeviceHandler(
|
||||
private val appContext: Context,
|
||||
) {
|
||||
private data class BatterySnapshot(
|
||||
val status: Int,
|
||||
val plugged: Int,
|
||||
val levelFraction: Double?,
|
||||
val temperatureC: Double?,
|
||||
)
|
||||
|
||||
fun handleDeviceStatus(_paramsJson: String?): GatewaySession.InvokeResult {
|
||||
return GatewaySession.InvokeResult.ok(statusPayloadJson())
|
||||
}
|
||||
|
||||
fun handleDeviceInfo(_paramsJson: String?): GatewaySession.InvokeResult {
|
||||
return GatewaySession.InvokeResult.ok(infoPayloadJson())
|
||||
}
|
||||
|
||||
fun handleDevicePermissions(_paramsJson: String?): GatewaySession.InvokeResult {
|
||||
return GatewaySession.InvokeResult.ok(permissionsPayloadJson())
|
||||
}
|
||||
|
||||
fun handleDeviceHealth(_paramsJson: String?): GatewaySession.InvokeResult {
|
||||
return GatewaySession.InvokeResult.ok(healthPayloadJson())
|
||||
}
|
||||
|
||||
private fun statusPayloadJson(): String {
|
||||
val battery = readBatterySnapshot()
|
||||
val powerManager = appContext.getSystemService(PowerManager::class.java)
|
||||
val storage = StatFs(Environment.getDataDirectory().absolutePath)
|
||||
val totalBytes = storage.totalBytes
|
||||
val freeBytes = storage.availableBytes
|
||||
val usedBytes = (totalBytes - freeBytes).coerceAtLeast(0L)
|
||||
val connectivity = appContext.getSystemService(ConnectivityManager::class.java)
|
||||
val activeNetwork = connectivity?.activeNetwork
|
||||
val caps = activeNetwork?.let { connectivity.getNetworkCapabilities(it) }
|
||||
val uptimeSeconds = SystemClock.elapsedRealtime() / 1_000.0
|
||||
|
||||
return buildJsonObject {
|
||||
put(
|
||||
"battery",
|
||||
buildJsonObject {
|
||||
battery.levelFraction?.let { put("level", JsonPrimitive(it)) }
|
||||
put("state", JsonPrimitive(mapBatteryState(battery.status)))
|
||||
put("lowPowerModeEnabled", JsonPrimitive(powerManager?.isPowerSaveMode == true))
|
||||
},
|
||||
)
|
||||
put(
|
||||
"thermal",
|
||||
buildJsonObject {
|
||||
put("state", JsonPrimitive(mapThermalState(powerManager)))
|
||||
},
|
||||
)
|
||||
put(
|
||||
"storage",
|
||||
buildJsonObject {
|
||||
put("totalBytes", JsonPrimitive(totalBytes))
|
||||
put("freeBytes", JsonPrimitive(freeBytes))
|
||||
put("usedBytes", JsonPrimitive(usedBytes))
|
||||
},
|
||||
)
|
||||
put(
|
||||
"network",
|
||||
buildJsonObject {
|
||||
put("status", JsonPrimitive(mapNetworkStatus(caps)))
|
||||
put(
|
||||
"isExpensive",
|
||||
JsonPrimitive(
|
||||
caps?.hasCapability(NetworkCapabilities.NET_CAPABILITY_NOT_METERED)?.not() ?: false,
|
||||
),
|
||||
)
|
||||
put(
|
||||
"isConstrained",
|
||||
JsonPrimitive(
|
||||
caps?.hasCapability(NetworkCapabilities.NET_CAPABILITY_NOT_RESTRICTED)?.not() ?: false,
|
||||
),
|
||||
)
|
||||
put("interfaces", networkInterfacesJson(caps))
|
||||
},
|
||||
)
|
||||
put("uptimeSeconds", JsonPrimitive(uptimeSeconds))
|
||||
}.toString()
|
||||
}
|
||||
|
||||
private fun infoPayloadJson(): String {
|
||||
val model = Build.MODEL?.trim().orEmpty()
|
||||
val manufacturer = Build.MANUFACTURER?.trim().orEmpty()
|
||||
val modelIdentifier = Build.DEVICE?.trim().orEmpty()
|
||||
val systemVersion = Build.VERSION.RELEASE?.trim().orEmpty()
|
||||
val locale = Locale.getDefault().toLanguageTag().trim()
|
||||
val appVersion = BuildConfig.VERSION_NAME.trim()
|
||||
val appBuild = BuildConfig.VERSION_CODE.toString()
|
||||
|
||||
return buildJsonObject {
|
||||
put("deviceName", JsonPrimitive(model.ifEmpty { "Android" }))
|
||||
put("modelIdentifier", JsonPrimitive(modelIdentifier.ifEmpty { listOf(manufacturer, model).filter { it.isNotEmpty() }.joinToString(" ") }))
|
||||
put("systemName", JsonPrimitive("Android"))
|
||||
put("systemVersion", JsonPrimitive(systemVersion.ifEmpty { Build.VERSION.SDK_INT.toString() }))
|
||||
put("appVersion", JsonPrimitive(appVersion.ifEmpty { "dev" }))
|
||||
put("appBuild", JsonPrimitive(appBuild.ifEmpty { "0" }))
|
||||
put("locale", JsonPrimitive(locale.ifEmpty { Locale.getDefault().toString() }))
|
||||
}.toString()
|
||||
}
|
||||
|
||||
private fun permissionsPayloadJson(): String {
|
||||
val canSendSms = appContext.packageManager.hasSystemFeature(PackageManager.FEATURE_TELEPHONY)
|
||||
val notificationAccess = DeviceNotificationListenerService.isAccessEnabled(appContext)
|
||||
return buildJsonObject {
|
||||
put(
|
||||
"permissions",
|
||||
buildJsonObject {
|
||||
put(
|
||||
"camera",
|
||||
permissionStateJson(
|
||||
granted = hasPermission(Manifest.permission.CAMERA),
|
||||
promptableWhenDenied = true,
|
||||
),
|
||||
)
|
||||
put(
|
||||
"microphone",
|
||||
permissionStateJson(
|
||||
granted = hasPermission(Manifest.permission.RECORD_AUDIO),
|
||||
promptableWhenDenied = true,
|
||||
),
|
||||
)
|
||||
put(
|
||||
"location",
|
||||
permissionStateJson(
|
||||
granted =
|
||||
hasPermission(Manifest.permission.ACCESS_FINE_LOCATION) ||
|
||||
hasPermission(Manifest.permission.ACCESS_COARSE_LOCATION),
|
||||
promptableWhenDenied = true,
|
||||
),
|
||||
)
|
||||
put(
|
||||
"backgroundLocation",
|
||||
permissionStateJson(
|
||||
granted = hasPermission(Manifest.permission.ACCESS_BACKGROUND_LOCATION),
|
||||
promptableWhenDenied = true,
|
||||
),
|
||||
)
|
||||
put(
|
||||
"sms",
|
||||
permissionStateJson(
|
||||
granted = hasPermission(Manifest.permission.SEND_SMS) && canSendSms,
|
||||
promptableWhenDenied = canSendSms,
|
||||
),
|
||||
)
|
||||
put(
|
||||
"notificationListener",
|
||||
permissionStateJson(
|
||||
granted = notificationAccess,
|
||||
promptableWhenDenied = true,
|
||||
),
|
||||
)
|
||||
// Screen capture on Android is interactive per-capture consent, not a sticky app permission.
|
||||
put(
|
||||
"screenCapture",
|
||||
permissionStateJson(
|
||||
granted = false,
|
||||
promptableWhenDenied = true,
|
||||
),
|
||||
)
|
||||
},
|
||||
)
|
||||
}.toString()
|
||||
}
|
||||
|
||||
private fun healthPayloadJson(): String {
|
||||
val battery = readBatterySnapshot()
|
||||
val batteryManager = appContext.getSystemService(BatteryManager::class.java)
|
||||
val currentNowUa = batteryManager?.getLongProperty(BatteryManager.BATTERY_PROPERTY_CURRENT_NOW)
|
||||
val currentNowMa =
|
||||
if (currentNowUa == null || currentNowUa == Long.MIN_VALUE) {
|
||||
null
|
||||
} else {
|
||||
currentNowUa.toDouble() / 1_000.0
|
||||
}
|
||||
|
||||
val powerManager = appContext.getSystemService(PowerManager::class.java)
|
||||
val activityManager = appContext.getSystemService(ActivityManager::class.java)
|
||||
val memoryInfo = ActivityManager.MemoryInfo()
|
||||
activityManager?.getMemoryInfo(memoryInfo)
|
||||
val totalRamBytes = memoryInfo.totalMem.coerceAtLeast(0L)
|
||||
val availableRamBytes = memoryInfo.availMem.coerceAtLeast(0L)
|
||||
val usedRamBytes = (totalRamBytes - availableRamBytes).coerceAtLeast(0L)
|
||||
val lowMemory = memoryInfo.lowMemory
|
||||
val memoryPressure = mapMemoryPressure(totalRamBytes, availableRamBytes, lowMemory)
|
||||
|
||||
return buildJsonObject {
|
||||
put(
|
||||
"memory",
|
||||
buildJsonObject {
|
||||
put("pressure", JsonPrimitive(memoryPressure))
|
||||
put("totalRamBytes", JsonPrimitive(totalRamBytes))
|
||||
put("availableRamBytes", JsonPrimitive(availableRamBytes))
|
||||
put("usedRamBytes", JsonPrimitive(usedRamBytes))
|
||||
put("thresholdBytes", JsonPrimitive(memoryInfo.threshold.coerceAtLeast(0L)))
|
||||
put("lowMemory", JsonPrimitive(lowMemory))
|
||||
},
|
||||
)
|
||||
put(
|
||||
"battery",
|
||||
buildJsonObject {
|
||||
put("state", JsonPrimitive(mapBatteryState(battery.status)))
|
||||
put("chargingType", JsonPrimitive(mapChargingType(battery.plugged)))
|
||||
battery.temperatureC?.let { put("temperatureC", JsonPrimitive(it)) }
|
||||
currentNowMa?.let { put("currentMa", JsonPrimitive(it)) }
|
||||
},
|
||||
)
|
||||
put(
|
||||
"power",
|
||||
buildJsonObject {
|
||||
put("dozeModeEnabled", JsonPrimitive(powerManager?.isDeviceIdleMode == true))
|
||||
put("lowPowerModeEnabled", JsonPrimitive(powerManager?.isPowerSaveMode == true))
|
||||
},
|
||||
)
|
||||
put(
|
||||
"system",
|
||||
buildJsonObject {
|
||||
Build.VERSION.SECURITY_PATCH
|
||||
?.trim()
|
||||
?.takeIf { it.isNotEmpty() }
|
||||
?.let { put("securityPatchLevel", JsonPrimitive(it)) }
|
||||
},
|
||||
)
|
||||
}.toString()
|
||||
}
|
||||
|
||||
private fun readBatterySnapshot(): BatterySnapshot {
|
||||
val intent = appContext.registerReceiver(null, IntentFilter(Intent.ACTION_BATTERY_CHANGED))
|
||||
val status =
|
||||
intent?.getIntExtra(BatteryManager.EXTRA_STATUS, BatteryManager.BATTERY_STATUS_UNKNOWN)
|
||||
?: BatteryManager.BATTERY_STATUS_UNKNOWN
|
||||
val plugged = intent?.getIntExtra(BatteryManager.EXTRA_PLUGGED, 0) ?: 0
|
||||
val temperatureC =
|
||||
intent
|
||||
?.getIntExtra(BatteryManager.EXTRA_TEMPERATURE, Int.MIN_VALUE)
|
||||
?.takeIf { it != Int.MIN_VALUE }
|
||||
?.toDouble()
|
||||
?.div(10.0)
|
||||
return BatterySnapshot(
|
||||
status = status,
|
||||
plugged = plugged,
|
||||
levelFraction = batteryLevelFraction(intent),
|
||||
temperatureC = temperatureC,
|
||||
)
|
||||
}
|
||||
|
||||
private fun batteryLevelFraction(intent: Intent?): Double? {
|
||||
val rawLevel = intent?.getIntExtra(BatteryManager.EXTRA_LEVEL, -1) ?: -1
|
||||
val rawScale = intent?.getIntExtra(BatteryManager.EXTRA_SCALE, -1) ?: -1
|
||||
if (rawLevel < 0 || rawScale <= 0) return null
|
||||
return rawLevel.toDouble() / rawScale.toDouble()
|
||||
}
|
||||
|
||||
private fun mapBatteryState(status: Int): String {
|
||||
return when (status) {
|
||||
BatteryManager.BATTERY_STATUS_CHARGING -> "charging"
|
||||
BatteryManager.BATTERY_STATUS_FULL -> "full"
|
||||
BatteryManager.BATTERY_STATUS_DISCHARGING, BatteryManager.BATTERY_STATUS_NOT_CHARGING -> "unplugged"
|
||||
else -> "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
private fun mapChargingType(plugged: Int): String {
|
||||
return when (plugged) {
|
||||
BatteryManager.BATTERY_PLUGGED_AC -> "ac"
|
||||
BatteryManager.BATTERY_PLUGGED_USB -> "usb"
|
||||
BatteryManager.BATTERY_PLUGGED_WIRELESS -> "wireless"
|
||||
BatteryManager.BATTERY_PLUGGED_DOCK -> "dock"
|
||||
else -> "none"
|
||||
}
|
||||
}
|
||||
|
||||
private fun mapThermalState(powerManager: PowerManager?): String {
|
||||
val thermal = powerManager?.currentThermalStatus ?: return "nominal"
|
||||
return when (thermal) {
|
||||
PowerManager.THERMAL_STATUS_NONE, PowerManager.THERMAL_STATUS_LIGHT -> "nominal"
|
||||
PowerManager.THERMAL_STATUS_MODERATE -> "fair"
|
||||
PowerManager.THERMAL_STATUS_SEVERE -> "serious"
|
||||
PowerManager.THERMAL_STATUS_CRITICAL,
|
||||
PowerManager.THERMAL_STATUS_EMERGENCY,
|
||||
PowerManager.THERMAL_STATUS_SHUTDOWN -> "critical"
|
||||
else -> "nominal"
|
||||
}
|
||||
}
|
||||
|
||||
private fun mapNetworkStatus(caps: NetworkCapabilities?): String {
|
||||
if (caps == null) return "unsatisfied"
|
||||
return when {
|
||||
caps.hasCapability(NetworkCapabilities.NET_CAPABILITY_VALIDATED) -> "satisfied"
|
||||
caps.hasCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET) -> "requiresConnection"
|
||||
else -> "unsatisfied"
|
||||
}
|
||||
}
|
||||
|
||||
private fun permissionStateJson(granted: Boolean, promptableWhenDenied: Boolean) =
|
||||
buildJsonObject {
|
||||
put("status", JsonPrimitive(if (granted) "granted" else "denied"))
|
||||
put("promptable", JsonPrimitive(!granted && promptableWhenDenied))
|
||||
}
|
||||
|
||||
private fun hasPermission(permission: String): Boolean {
|
||||
return (
|
||||
ContextCompat.checkSelfPermission(appContext, permission) == PackageManager.PERMISSION_GRANTED
|
||||
)
|
||||
}
|
||||
|
||||
private fun mapMemoryPressure(totalBytes: Long, availableBytes: Long, lowMemory: Boolean): String {
|
||||
if (totalBytes <= 0L) return if (lowMemory) "critical" else "unknown"
|
||||
if (lowMemory) return "critical"
|
||||
val freeRatio = availableBytes.toDouble() / totalBytes.toDouble()
|
||||
return when {
|
||||
freeRatio <= 0.05 -> "critical"
|
||||
freeRatio <= 0.15 -> "high"
|
||||
freeRatio <= 0.30 -> "moderate"
|
||||
else -> "normal"
|
||||
}
|
||||
}
|
||||
|
||||
private fun networkInterfacesJson(caps: NetworkCapabilities?) =
|
||||
buildJsonArray {
|
||||
if (caps == null) return@buildJsonArray
|
||||
var hasKnownTransport = false
|
||||
if (caps.hasTransport(NetworkCapabilities.TRANSPORT_WIFI)) {
|
||||
hasKnownTransport = true
|
||||
add(JsonPrimitive("wifi"))
|
||||
}
|
||||
if (caps.hasTransport(NetworkCapabilities.TRANSPORT_CELLULAR)) {
|
||||
hasKnownTransport = true
|
||||
add(JsonPrimitive("cellular"))
|
||||
}
|
||||
if (caps.hasTransport(NetworkCapabilities.TRANSPORT_ETHERNET)) {
|
||||
hasKnownTransport = true
|
||||
add(JsonPrimitive("wired"))
|
||||
}
|
||||
if (!hasKnownTransport) add(JsonPrimitive("other"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.app.Notification
|
||||
import android.app.NotificationManager
|
||||
import android.app.RemoteInput
|
||||
import android.content.ComponentName
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.os.Build
|
||||
import android.service.notification.NotificationListenerService
|
||||
import android.service.notification.StatusBarNotification
|
||||
|
||||
private const val MAX_NOTIFICATION_TEXT_CHARS = 512
|
||||
|
||||
internal fun sanitizeNotificationText(value: CharSequence?): String? {
|
||||
val normalized = value?.toString()?.trim().orEmpty()
|
||||
return normalized.take(MAX_NOTIFICATION_TEXT_CHARS).ifEmpty { null }
|
||||
}
|
||||
|
||||
data class DeviceNotificationEntry(
|
||||
val key: String,
|
||||
val packageName: String,
|
||||
val title: String?,
|
||||
val text: String?,
|
||||
val subText: String?,
|
||||
val category: String?,
|
||||
val channelId: String?,
|
||||
val postTimeMs: Long,
|
||||
val isOngoing: Boolean,
|
||||
val isClearable: Boolean,
|
||||
)
|
||||
|
||||
data class DeviceNotificationSnapshot(
|
||||
val enabled: Boolean,
|
||||
val connected: Boolean,
|
||||
val notifications: List<DeviceNotificationEntry>,
|
||||
)
|
||||
|
||||
enum class NotificationActionKind {
|
||||
Open,
|
||||
Dismiss,
|
||||
Reply,
|
||||
}
|
||||
|
||||
data class NotificationActionRequest(
|
||||
val key: String,
|
||||
val kind: NotificationActionKind,
|
||||
val replyText: String? = null,
|
||||
)
|
||||
|
||||
data class NotificationActionResult(
|
||||
val ok: Boolean,
|
||||
val code: String? = null,
|
||||
val message: String? = null,
|
||||
)
|
||||
|
||||
internal fun actionRequiresClearableNotification(kind: NotificationActionKind): Boolean {
|
||||
return kind == NotificationActionKind.Dismiss
|
||||
}
|
||||
|
||||
private object DeviceNotificationStore {
|
||||
private val lock = Any()
|
||||
private var connected = false
|
||||
private val byKey = LinkedHashMap<String, DeviceNotificationEntry>()
|
||||
|
||||
fun replace(entries: List<DeviceNotificationEntry>) {
|
||||
synchronized(lock) {
|
||||
byKey.clear()
|
||||
for (entry in entries) {
|
||||
byKey[entry.key] = entry
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun upsert(entry: DeviceNotificationEntry) {
|
||||
synchronized(lock) {
|
||||
byKey[entry.key] = entry
|
||||
}
|
||||
}
|
||||
|
||||
fun remove(key: String) {
|
||||
synchronized(lock) {
|
||||
byKey.remove(key)
|
||||
}
|
||||
}
|
||||
|
||||
fun setConnected(value: Boolean) {
|
||||
synchronized(lock) {
|
||||
connected = value
|
||||
if (!value) {
|
||||
byKey.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun snapshot(enabled: Boolean): DeviceNotificationSnapshot {
|
||||
val (isConnected, entries) =
|
||||
synchronized(lock) {
|
||||
connected to byKey.values.sortedByDescending { it.postTimeMs }
|
||||
}
|
||||
return DeviceNotificationSnapshot(
|
||||
enabled = enabled,
|
||||
connected = isConnected,
|
||||
notifications = entries,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class DeviceNotificationListenerService : NotificationListenerService() {
|
||||
override fun onListenerConnected() {
|
||||
super.onListenerConnected()
|
||||
activeService = this
|
||||
DeviceNotificationStore.setConnected(true)
|
||||
refreshActiveNotifications()
|
||||
}
|
||||
|
||||
override fun onListenerDisconnected() {
|
||||
if (activeService === this) {
|
||||
activeService = null
|
||||
}
|
||||
DeviceNotificationStore.setConnected(false)
|
||||
super.onListenerDisconnected()
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
if (activeService === this) {
|
||||
activeService = null
|
||||
}
|
||||
super.onDestroy()
|
||||
}
|
||||
|
||||
override fun onNotificationPosted(sbn: StatusBarNotification?) {
|
||||
super.onNotificationPosted(sbn)
|
||||
val entry = sbn?.toEntry() ?: return
|
||||
DeviceNotificationStore.upsert(entry)
|
||||
}
|
||||
|
||||
override fun onNotificationRemoved(sbn: StatusBarNotification?) {
|
||||
super.onNotificationRemoved(sbn)
|
||||
val key = sbn?.key ?: return
|
||||
DeviceNotificationStore.remove(key)
|
||||
}
|
||||
|
||||
private fun refreshActiveNotifications() {
|
||||
val entries =
|
||||
runCatching {
|
||||
activeNotifications
|
||||
?.mapNotNull { it.toEntry() }
|
||||
?: emptyList()
|
||||
}.getOrElse { emptyList() }
|
||||
DeviceNotificationStore.replace(entries)
|
||||
}
|
||||
|
||||
private fun StatusBarNotification.toEntry(): DeviceNotificationEntry {
|
||||
val extras = notification.extras
|
||||
val keyValue = key.takeIf { it.isNotBlank() } ?: "$packageName:$id:$postTime"
|
||||
val title = sanitizeNotificationText(extras?.getCharSequence(Notification.EXTRA_TITLE))
|
||||
val body =
|
||||
sanitizeNotificationText(extras?.getCharSequence(Notification.EXTRA_BIG_TEXT))
|
||||
?: sanitizeNotificationText(extras?.getCharSequence(Notification.EXTRA_TEXT))
|
||||
val subText = sanitizeNotificationText(extras?.getCharSequence(Notification.EXTRA_SUB_TEXT))
|
||||
return DeviceNotificationEntry(
|
||||
key = keyValue,
|
||||
packageName = packageName,
|
||||
title = title,
|
||||
text = body,
|
||||
subText = subText,
|
||||
category = notification.category?.trim()?.ifEmpty { null },
|
||||
channelId = notification.channelId?.trim()?.ifEmpty { null },
|
||||
postTimeMs = postTime,
|
||||
isOngoing = isOngoing,
|
||||
isClearable = isClearable,
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
@Volatile private var activeService: DeviceNotificationListenerService? = null
|
||||
|
||||
private fun serviceComponent(context: Context): ComponentName {
|
||||
return ComponentName(context, DeviceNotificationListenerService::class.java)
|
||||
}
|
||||
|
||||
fun isAccessEnabled(context: Context): Boolean {
|
||||
val manager = context.getSystemService(NotificationManager::class.java) ?: return false
|
||||
return manager.isNotificationListenerAccessGranted(serviceComponent(context))
|
||||
}
|
||||
|
||||
fun snapshot(context: Context, enabled: Boolean = isAccessEnabled(context)): DeviceNotificationSnapshot {
|
||||
return DeviceNotificationStore.snapshot(enabled = enabled)
|
||||
}
|
||||
|
||||
fun requestServiceRebind(context: Context) {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) {
|
||||
return
|
||||
}
|
||||
runCatching {
|
||||
NotificationListenerService.requestRebind(serviceComponent(context))
|
||||
}
|
||||
}
|
||||
|
||||
fun executeAction(context: Context, request: NotificationActionRequest): NotificationActionResult {
|
||||
if (!isAccessEnabled(context)) {
|
||||
return NotificationActionResult(
|
||||
ok = false,
|
||||
code = "NOTIFICATIONS_DISABLED",
|
||||
message = "NOTIFICATIONS_DISABLED: enable notification access in system Settings",
|
||||
)
|
||||
}
|
||||
val service = activeService
|
||||
?: return NotificationActionResult(
|
||||
ok = false,
|
||||
code = "NOTIFICATIONS_UNAVAILABLE",
|
||||
message = "NOTIFICATIONS_UNAVAILABLE: notification listener not connected",
|
||||
)
|
||||
return service.executeActionInternal(request)
|
||||
}
|
||||
}
|
||||
|
||||
private fun executeActionInternal(request: NotificationActionRequest): NotificationActionResult {
|
||||
val sbn =
|
||||
activeNotifications
|
||||
?.firstOrNull { it.key == request.key }
|
||||
?: return NotificationActionResult(
|
||||
ok = false,
|
||||
code = "NOTIFICATION_NOT_FOUND",
|
||||
message = "NOTIFICATION_NOT_FOUND: notification key not found",
|
||||
)
|
||||
if (actionRequiresClearableNotification(request.kind) && !sbn.isClearable) {
|
||||
return NotificationActionResult(
|
||||
ok = false,
|
||||
code = "NOTIFICATION_NOT_CLEARABLE",
|
||||
message = "NOTIFICATION_NOT_CLEARABLE: notification is ongoing or protected",
|
||||
)
|
||||
}
|
||||
|
||||
return when (request.kind) {
|
||||
NotificationActionKind.Open -> {
|
||||
val pendingIntent = sbn.notification.contentIntent
|
||||
?: return NotificationActionResult(
|
||||
ok = false,
|
||||
code = "ACTION_UNAVAILABLE",
|
||||
message = "ACTION_UNAVAILABLE: notification has no open action",
|
||||
)
|
||||
runCatching {
|
||||
pendingIntent.send()
|
||||
}.fold(
|
||||
onSuccess = { NotificationActionResult(ok = true) },
|
||||
onFailure = { err ->
|
||||
NotificationActionResult(
|
||||
ok = false,
|
||||
code = "ACTION_FAILED",
|
||||
message = "ACTION_FAILED: ${err.message ?: "open failed"}",
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
NotificationActionKind.Dismiss -> {
|
||||
runCatching {
|
||||
cancelNotification(sbn.key)
|
||||
DeviceNotificationStore.remove(sbn.key)
|
||||
}.fold(
|
||||
onSuccess = { NotificationActionResult(ok = true) },
|
||||
onFailure = { err ->
|
||||
NotificationActionResult(
|
||||
ok = false,
|
||||
code = "ACTION_FAILED",
|
||||
message = "ACTION_FAILED: ${err.message ?: "dismiss failed"}",
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
NotificationActionKind.Reply -> {
|
||||
val replyText = request.replyText?.trim().orEmpty()
|
||||
if (replyText.isEmpty()) {
|
||||
return NotificationActionResult(
|
||||
ok = false,
|
||||
code = "INVALID_REQUEST",
|
||||
message = "INVALID_REQUEST: replyText required for reply action",
|
||||
)
|
||||
}
|
||||
val action =
|
||||
sbn.notification.actions
|
||||
?.firstOrNull { candidate ->
|
||||
candidate.actionIntent != null && !candidate.remoteInputs.isNullOrEmpty()
|
||||
}
|
||||
?: return NotificationActionResult(
|
||||
ok = false,
|
||||
code = "ACTION_UNAVAILABLE",
|
||||
message = "ACTION_UNAVAILABLE: notification has no reply action",
|
||||
)
|
||||
val remoteInputs = action.remoteInputs ?: emptyArray()
|
||||
val fillInIntent = Intent()
|
||||
val replyBundle = android.os.Bundle()
|
||||
for (remoteInput in remoteInputs) {
|
||||
replyBundle.putCharSequence(remoteInput.resultKey, replyText)
|
||||
}
|
||||
RemoteInput.addResultsToIntent(remoteInputs, fillInIntent, replyBundle)
|
||||
runCatching {
|
||||
action.actionIntent.send(this, 0, fillInIntent)
|
||||
}.fold(
|
||||
onSuccess = { NotificationActionResult(ok = true) },
|
||||
onFailure = { err ->
|
||||
NotificationActionResult(
|
||||
ok = false,
|
||||
code = "ACTION_FAILED",
|
||||
message = "ACTION_FAILED: ${err.message ?: "reply failed"}",
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import ai.openclaw.android.SecurePrefs
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
|
||||
class GatewayEventHandler(
|
||||
private val scope: CoroutineScope,
|
||||
private val prefs: SecurePrefs,
|
||||
private val json: Json,
|
||||
private val operatorSession: GatewaySession,
|
||||
private val isConnected: () -> Boolean,
|
||||
) {
|
||||
private var suppressWakeWordsSync = false
|
||||
private var wakeWordsSyncJob: Job? = null
|
||||
|
||||
fun applyWakeWordsFromGateway(words: List<String>) {
|
||||
suppressWakeWordsSync = true
|
||||
prefs.setWakeWords(words)
|
||||
suppressWakeWordsSync = false
|
||||
}
|
||||
|
||||
fun scheduleWakeWordsSyncIfNeeded() {
|
||||
if (suppressWakeWordsSync) return
|
||||
if (!isConnected()) return
|
||||
|
||||
val snapshot = prefs.wakeWords.value
|
||||
wakeWordsSyncJob?.cancel()
|
||||
wakeWordsSyncJob =
|
||||
scope.launch {
|
||||
delay(650)
|
||||
val jsonList = snapshot.joinToString(separator = ",") { it.toJsonString() }
|
||||
val params = """{"triggers":[$jsonList]}"""
|
||||
try {
|
||||
operatorSession.request("voicewake.set", params)
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun refreshWakeWordsFromGateway() {
|
||||
if (!isConnected()) return
|
||||
try {
|
||||
val res = operatorSession.request("voicewake.get", "{}")
|
||||
val payload = json.parseToJsonElement(res).asObjectOrNull() ?: return
|
||||
val array = payload["triggers"] as? JsonArray ?: return
|
||||
val triggers = array.mapNotNull { it.asStringOrNull() }
|
||||
applyWakeWordsFromGateway(triggers)
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
fun handleVoiceWakeChangedEvent(payloadJson: String?) {
|
||||
if (payloadJson.isNullOrBlank()) return
|
||||
try {
|
||||
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
|
||||
val array = payload["triggers"] as? JsonArray ?: return
|
||||
val triggers = array.mapNotNull { it.asStringOrNull() }
|
||||
applyWakeWordsFromGateway(triggers)
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
|
||||
import ai.openclaw.android.protocol.OpenClawCanvasCommand
|
||||
import ai.openclaw.android.protocol.OpenClawCameraCommand
|
||||
import ai.openclaw.android.protocol.OpenClawDeviceCommand
|
||||
import ai.openclaw.android.protocol.OpenClawLocationCommand
|
||||
import ai.openclaw.android.protocol.OpenClawNotificationsCommand
|
||||
import ai.openclaw.android.protocol.OpenClawScreenCommand
|
||||
import ai.openclaw.android.protocol.OpenClawSmsCommand
|
||||
|
||||
enum class InvokeCommandAvailability {
|
||||
Always,
|
||||
CameraEnabled,
|
||||
LocationEnabled,
|
||||
SmsAvailable,
|
||||
DebugBuild,
|
||||
}
|
||||
|
||||
data class InvokeCommandSpec(
|
||||
val name: String,
|
||||
val requiresForeground: Boolean = false,
|
||||
val availability: InvokeCommandAvailability = InvokeCommandAvailability.Always,
|
||||
)
|
||||
|
||||
object InvokeCommandRegistry {
|
||||
val all: List<InvokeCommandSpec> =
|
||||
listOf(
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCanvasCommand.Present.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCanvasCommand.Hide.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCanvasCommand.Navigate.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCanvasCommand.Eval.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCanvasCommand.Snapshot.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCanvasA2UICommand.Push.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCanvasA2UICommand.PushJSONL.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCanvasA2UICommand.Reset.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawScreenCommand.Record.rawValue,
|
||||
requiresForeground = true,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCameraCommand.List.rawValue,
|
||||
requiresForeground = true,
|
||||
availability = InvokeCommandAvailability.CameraEnabled,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCameraCommand.Snap.rawValue,
|
||||
requiresForeground = true,
|
||||
availability = InvokeCommandAvailability.CameraEnabled,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawCameraCommand.Clip.rawValue,
|
||||
requiresForeground = true,
|
||||
availability = InvokeCommandAvailability.CameraEnabled,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawLocationCommand.Get.rawValue,
|
||||
availability = InvokeCommandAvailability.LocationEnabled,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawDeviceCommand.Status.rawValue,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawDeviceCommand.Info.rawValue,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawDeviceCommand.Permissions.rawValue,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawDeviceCommand.Health.rawValue,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawNotificationsCommand.List.rawValue,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawNotificationsCommand.Actions.rawValue,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = OpenClawSmsCommand.Send.rawValue,
|
||||
availability = InvokeCommandAvailability.SmsAvailable,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = "debug.logs",
|
||||
availability = InvokeCommandAvailability.DebugBuild,
|
||||
),
|
||||
InvokeCommandSpec(
|
||||
name = "debug.ed25519",
|
||||
availability = InvokeCommandAvailability.DebugBuild,
|
||||
),
|
||||
InvokeCommandSpec(name = "app.update"),
|
||||
)
|
||||
|
||||
private val byNameInternal: Map<String, InvokeCommandSpec> = all.associateBy { it.name }
|
||||
|
||||
fun find(command: String): InvokeCommandSpec? = byNameInternal[command]
|
||||
|
||||
fun advertisedCommands(
|
||||
cameraEnabled: Boolean,
|
||||
locationEnabled: Boolean,
|
||||
smsAvailable: Boolean,
|
||||
debugBuild: Boolean,
|
||||
): List<String> {
|
||||
return all
|
||||
.filter { spec ->
|
||||
when (spec.availability) {
|
||||
InvokeCommandAvailability.Always -> true
|
||||
InvokeCommandAvailability.CameraEnabled -> cameraEnabled
|
||||
InvokeCommandAvailability.LocationEnabled -> locationEnabled
|
||||
InvokeCommandAvailability.SmsAvailable -> smsAvailable
|
||||
InvokeCommandAvailability.DebugBuild -> debugBuild
|
||||
}
|
||||
}
|
||||
.map { it.name }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
|
||||
import ai.openclaw.android.protocol.OpenClawCanvasCommand
|
||||
import ai.openclaw.android.protocol.OpenClawCameraCommand
|
||||
import ai.openclaw.android.protocol.OpenClawDeviceCommand
|
||||
import ai.openclaw.android.protocol.OpenClawLocationCommand
|
||||
import ai.openclaw.android.protocol.OpenClawNotificationsCommand
|
||||
import ai.openclaw.android.protocol.OpenClawScreenCommand
|
||||
import ai.openclaw.android.protocol.OpenClawSmsCommand
|
||||
|
||||
class InvokeDispatcher(
|
||||
private val canvas: CanvasController,
|
||||
private val cameraHandler: CameraHandler,
|
||||
private val locationHandler: LocationHandler,
|
||||
private val deviceHandler: DeviceHandler,
|
||||
private val notificationsHandler: NotificationsHandler,
|
||||
private val screenHandler: ScreenHandler,
|
||||
private val smsHandler: SmsHandler,
|
||||
private val a2uiHandler: A2UIHandler,
|
||||
private val debugHandler: DebugHandler,
|
||||
private val appUpdateHandler: AppUpdateHandler,
|
||||
private val isForeground: () -> Boolean,
|
||||
private val cameraEnabled: () -> Boolean,
|
||||
private val locationEnabled: () -> Boolean,
|
||||
private val smsAvailable: () -> Boolean,
|
||||
private val debugBuild: () -> Boolean,
|
||||
private val refreshNodeCanvasCapability: suspend () -> Boolean,
|
||||
private val onCanvasA2uiPush: () -> Unit,
|
||||
private val onCanvasA2uiReset: () -> Unit,
|
||||
) {
|
||||
suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult {
|
||||
val spec =
|
||||
InvokeCommandRegistry.find(command)
|
||||
?: return GatewaySession.InvokeResult.error(
|
||||
code = "INVALID_REQUEST",
|
||||
message = "INVALID_REQUEST: unknown command",
|
||||
)
|
||||
if (spec.requiresForeground && !isForeground()) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "NODE_BACKGROUND_UNAVAILABLE",
|
||||
message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground",
|
||||
)
|
||||
}
|
||||
availabilityError(spec.availability)?.let { return it }
|
||||
|
||||
return when (command) {
|
||||
// Canvas commands
|
||||
OpenClawCanvasCommand.Present.rawValue -> {
|
||||
val url = CanvasController.parseNavigateUrl(paramsJson)
|
||||
canvas.navigate(url)
|
||||
GatewaySession.InvokeResult.ok(null)
|
||||
}
|
||||
OpenClawCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null)
|
||||
OpenClawCanvasCommand.Navigate.rawValue -> {
|
||||
val url = CanvasController.parseNavigateUrl(paramsJson)
|
||||
canvas.navigate(url)
|
||||
GatewaySession.InvokeResult.ok(null)
|
||||
}
|
||||
OpenClawCanvasCommand.Eval.rawValue -> {
|
||||
val js =
|
||||
CanvasController.parseEvalJs(paramsJson)
|
||||
?: return GatewaySession.InvokeResult.error(
|
||||
code = "INVALID_REQUEST",
|
||||
message = "INVALID_REQUEST: javaScript required",
|
||||
)
|
||||
withCanvasAvailable {
|
||||
val result = canvas.eval(js)
|
||||
GatewaySession.InvokeResult.ok("""{"result":${result.toJsonString()}}""")
|
||||
}
|
||||
}
|
||||
OpenClawCanvasCommand.Snapshot.rawValue -> {
|
||||
val snapshotParams = CanvasController.parseSnapshotParams(paramsJson)
|
||||
withCanvasAvailable {
|
||||
val base64 =
|
||||
canvas.snapshotBase64(
|
||||
format = snapshotParams.format,
|
||||
quality = snapshotParams.quality,
|
||||
maxWidth = snapshotParams.maxWidth,
|
||||
)
|
||||
GatewaySession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""")
|
||||
}
|
||||
}
|
||||
|
||||
// A2UI commands
|
||||
OpenClawCanvasA2UICommand.Reset.rawValue ->
|
||||
withReadyA2ui {
|
||||
withCanvasAvailable {
|
||||
val res = canvas.eval(A2UIHandler.a2uiResetJS)
|
||||
onCanvasA2uiReset()
|
||||
GatewaySession.InvokeResult.ok(res)
|
||||
}
|
||||
}
|
||||
OpenClawCanvasA2UICommand.Push.rawValue, OpenClawCanvasA2UICommand.PushJSONL.rawValue -> {
|
||||
val messages =
|
||||
try {
|
||||
a2uiHandler.decodeA2uiMessages(command, paramsJson)
|
||||
} catch (err: Throwable) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "INVALID_REQUEST",
|
||||
message = err.message ?: "invalid A2UI payload"
|
||||
)
|
||||
}
|
||||
withReadyA2ui {
|
||||
withCanvasAvailable {
|
||||
val js = A2UIHandler.a2uiApplyMessagesJS(messages)
|
||||
val res = canvas.eval(js)
|
||||
onCanvasA2uiPush()
|
||||
GatewaySession.InvokeResult.ok(res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Camera commands
|
||||
OpenClawCameraCommand.List.rawValue -> cameraHandler.handleList(paramsJson)
|
||||
OpenClawCameraCommand.Snap.rawValue -> cameraHandler.handleSnap(paramsJson)
|
||||
OpenClawCameraCommand.Clip.rawValue -> cameraHandler.handleClip(paramsJson)
|
||||
|
||||
// Location command
|
||||
OpenClawLocationCommand.Get.rawValue -> locationHandler.handleLocationGet(paramsJson)
|
||||
|
||||
// Device commands
|
||||
OpenClawDeviceCommand.Status.rawValue -> deviceHandler.handleDeviceStatus(paramsJson)
|
||||
OpenClawDeviceCommand.Info.rawValue -> deviceHandler.handleDeviceInfo(paramsJson)
|
||||
OpenClawDeviceCommand.Permissions.rawValue -> deviceHandler.handleDevicePermissions(paramsJson)
|
||||
OpenClawDeviceCommand.Health.rawValue -> deviceHandler.handleDeviceHealth(paramsJson)
|
||||
|
||||
// Notifications command
|
||||
OpenClawNotificationsCommand.List.rawValue -> notificationsHandler.handleNotificationsList(paramsJson)
|
||||
OpenClawNotificationsCommand.Actions.rawValue -> notificationsHandler.handleNotificationsActions(paramsJson)
|
||||
|
||||
// Screen command
|
||||
OpenClawScreenCommand.Record.rawValue -> screenHandler.handleScreenRecord(paramsJson)
|
||||
|
||||
// SMS command
|
||||
OpenClawSmsCommand.Send.rawValue -> smsHandler.handleSmsSend(paramsJson)
|
||||
|
||||
// Debug commands
|
||||
"debug.ed25519" -> debugHandler.handleEd25519()
|
||||
"debug.logs" -> debugHandler.handleLogs()
|
||||
|
||||
// App update
|
||||
"app.update" -> appUpdateHandler.handleUpdate(paramsJson)
|
||||
|
||||
else -> GatewaySession.InvokeResult.error(code = "INVALID_REQUEST", message = "INVALID_REQUEST: unknown command")
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun withReadyA2ui(
|
||||
block: suspend () -> GatewaySession.InvokeResult,
|
||||
): GatewaySession.InvokeResult {
|
||||
var a2uiUrl = a2uiHandler.resolveA2uiHostUrl()
|
||||
?: return GatewaySession.InvokeResult.error(
|
||||
code = "A2UI_HOST_NOT_CONFIGURED",
|
||||
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
|
||||
)
|
||||
val readyOnFirstCheck = a2uiHandler.ensureA2uiReady(a2uiUrl)
|
||||
if (!readyOnFirstCheck) {
|
||||
if (!refreshNodeCanvasCapability()) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "A2UI_HOST_UNAVAILABLE",
|
||||
message = "A2UI_HOST_UNAVAILABLE: A2UI host not reachable",
|
||||
)
|
||||
}
|
||||
a2uiUrl = a2uiHandler.resolveA2uiHostUrl()
|
||||
?: return GatewaySession.InvokeResult.error(
|
||||
code = "A2UI_HOST_NOT_CONFIGURED",
|
||||
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
|
||||
)
|
||||
if (!a2uiHandler.ensureA2uiReady(a2uiUrl)) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "A2UI_HOST_UNAVAILABLE",
|
||||
message = "A2UI_HOST_UNAVAILABLE: A2UI host not reachable",
|
||||
)
|
||||
}
|
||||
}
|
||||
return block()
|
||||
}
|
||||
|
||||
private suspend fun withCanvasAvailable(
|
||||
block: suspend () -> GatewaySession.InvokeResult,
|
||||
): GatewaySession.InvokeResult {
|
||||
return try {
|
||||
block()
|
||||
} catch (_: Throwable) {
|
||||
GatewaySession.InvokeResult.error(
|
||||
code = "NODE_BACKGROUND_UNAVAILABLE",
|
||||
message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun availabilityError(availability: InvokeCommandAvailability): GatewaySession.InvokeResult? {
|
||||
return when (availability) {
|
||||
InvokeCommandAvailability.Always -> null
|
||||
InvokeCommandAvailability.CameraEnabled ->
|
||||
if (cameraEnabled()) {
|
||||
null
|
||||
} else {
|
||||
GatewaySession.InvokeResult.error(
|
||||
code = "CAMERA_DISABLED",
|
||||
message = "CAMERA_DISABLED: enable Camera in Settings",
|
||||
)
|
||||
}
|
||||
InvokeCommandAvailability.LocationEnabled ->
|
||||
if (locationEnabled()) {
|
||||
null
|
||||
} else {
|
||||
GatewaySession.InvokeResult.error(
|
||||
code = "LOCATION_DISABLED",
|
||||
message = "LOCATION_DISABLED: enable Location in Settings",
|
||||
)
|
||||
}
|
||||
InvokeCommandAvailability.SmsAvailable ->
|
||||
if (smsAvailable()) {
|
||||
null
|
||||
} else {
|
||||
GatewaySession.InvokeResult.error(
|
||||
code = "SMS_UNAVAILABLE",
|
||||
message = "SMS_UNAVAILABLE: SMS not available on this device",
|
||||
)
|
||||
}
|
||||
InvokeCommandAvailability.DebugBuild ->
|
||||
if (debugBuild()) {
|
||||
null
|
||||
} else {
|
||||
GatewaySession.InvokeResult.error(
|
||||
code = "INVALID_REQUEST",
|
||||
message = "INVALID_REQUEST: unknown command",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import kotlin.math.max
|
||||
import kotlin.math.min
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
internal data class JpegSizeLimiterResult(
|
||||
val bytes: ByteArray,
|
||||
val width: Int,
|
||||
val height: Int,
|
||||
val quality: Int,
|
||||
)
|
||||
|
||||
internal object JpegSizeLimiter {
|
||||
fun compressToLimit(
|
||||
initialWidth: Int,
|
||||
initialHeight: Int,
|
||||
startQuality: Int,
|
||||
maxBytes: Int,
|
||||
minQuality: Int = 20,
|
||||
minSize: Int = 256,
|
||||
scaleStep: Double = 0.85,
|
||||
maxScaleAttempts: Int = 6,
|
||||
maxQualityAttempts: Int = 6,
|
||||
encode: (width: Int, height: Int, quality: Int) -> ByteArray,
|
||||
): JpegSizeLimiterResult {
|
||||
require(initialWidth > 0 && initialHeight > 0) { "Invalid image size" }
|
||||
require(maxBytes > 0) { "Invalid maxBytes" }
|
||||
|
||||
var width = initialWidth
|
||||
var height = initialHeight
|
||||
val clampedStartQuality = startQuality.coerceIn(minQuality, 100)
|
||||
var best = JpegSizeLimiterResult(bytes = encode(width, height, clampedStartQuality), width = width, height = height, quality = clampedStartQuality)
|
||||
if (best.bytes.size <= maxBytes) return best
|
||||
|
||||
repeat(maxScaleAttempts) {
|
||||
var quality = clampedStartQuality
|
||||
repeat(maxQualityAttempts) {
|
||||
val bytes = encode(width, height, quality)
|
||||
best = JpegSizeLimiterResult(bytes = bytes, width = width, height = height, quality = quality)
|
||||
if (bytes.size <= maxBytes) return best
|
||||
if (quality <= minQuality) return@repeat
|
||||
quality = max(minQuality, (quality * 0.75).roundToInt())
|
||||
}
|
||||
|
||||
val minScale = (minSize.toDouble() / min(width, height).toDouble()).coerceAtMost(1.0)
|
||||
val nextScale = max(scaleStep, minScale)
|
||||
val nextWidth = max(minSize, (width * nextScale).roundToInt())
|
||||
val nextHeight = max(minSize, (height * nextScale).roundToInt())
|
||||
if (nextWidth == width && nextHeight == height) return@repeat
|
||||
width = min(nextWidth, width)
|
||||
height = min(nextHeight, height)
|
||||
}
|
||||
|
||||
if (best.bytes.size > maxBytes) {
|
||||
throw IllegalStateException("CAMERA_TOO_LARGE: ${best.bytes.size} bytes > $maxBytes bytes")
|
||||
}
|
||||
|
||||
return best
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.location.Location
|
||||
import android.location.LocationManager
|
||||
import android.os.CancellationSignal
|
||||
import androidx.core.content.ContextCompat
|
||||
import java.time.Instant
|
||||
import java.time.format.DateTimeFormatter
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.withTimeout
|
||||
import kotlin.coroutines.resume
|
||||
import kotlin.coroutines.resumeWithException
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
|
||||
class LocationCaptureManager(private val context: Context) {
|
||||
data class Payload(val payloadJson: String)
|
||||
|
||||
suspend fun getLocation(
|
||||
desiredProviders: List<String>,
|
||||
maxAgeMs: Long?,
|
||||
timeoutMs: Long,
|
||||
isPrecise: Boolean,
|
||||
): Payload =
|
||||
withContext(Dispatchers.Main) {
|
||||
val manager = context.getSystemService(Context.LOCATION_SERVICE) as LocationManager
|
||||
if (!manager.isProviderEnabled(LocationManager.GPS_PROVIDER) &&
|
||||
!manager.isProviderEnabled(LocationManager.NETWORK_PROVIDER)
|
||||
) {
|
||||
throw IllegalStateException("LOCATION_UNAVAILABLE: no location providers enabled")
|
||||
}
|
||||
|
||||
val cached = bestLastKnown(manager, desiredProviders, maxAgeMs)
|
||||
val location =
|
||||
cached ?: requestCurrent(manager, desiredProviders, timeoutMs)
|
||||
|
||||
val timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(location.time))
|
||||
val source = location.provider
|
||||
val altitudeMeters = if (location.hasAltitude()) location.altitude else null
|
||||
val speedMps = if (location.hasSpeed()) location.speed.toDouble() else null
|
||||
val headingDeg = if (location.hasBearing()) location.bearing.toDouble() else null
|
||||
Payload(
|
||||
buildString {
|
||||
append("{\"lat\":")
|
||||
append(location.latitude)
|
||||
append(",\"lon\":")
|
||||
append(location.longitude)
|
||||
append(",\"accuracyMeters\":")
|
||||
append(location.accuracy.toDouble())
|
||||
if (altitudeMeters != null) append(",\"altitudeMeters\":").append(altitudeMeters)
|
||||
if (speedMps != null) append(",\"speedMps\":").append(speedMps)
|
||||
if (headingDeg != null) append(",\"headingDeg\":").append(headingDeg)
|
||||
append(",\"timestamp\":\"").append(timestamp).append('"')
|
||||
append(",\"isPrecise\":").append(isPrecise)
|
||||
append(",\"source\":\"").append(source).append('"')
|
||||
append('}')
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
private fun bestLastKnown(
|
||||
manager: LocationManager,
|
||||
providers: List<String>,
|
||||
maxAgeMs: Long?,
|
||||
): Location? {
|
||||
val fineOk =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
val coarseOk =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
if (!fineOk && !coarseOk) {
|
||||
throw IllegalStateException("LOCATION_PERMISSION_REQUIRED: grant Location permission")
|
||||
}
|
||||
val now = System.currentTimeMillis()
|
||||
val candidates =
|
||||
providers.mapNotNull { provider -> manager.getLastKnownLocation(provider) }
|
||||
val freshest = candidates.maxByOrNull { it.time } ?: return null
|
||||
if (maxAgeMs != null && now - freshest.time > maxAgeMs) return null
|
||||
return freshest
|
||||
}
|
||||
|
||||
private suspend fun requestCurrent(
|
||||
manager: LocationManager,
|
||||
providers: List<String>,
|
||||
timeoutMs: Long,
|
||||
): Location {
|
||||
val fineOk =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
val coarseOk =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
if (!fineOk && !coarseOk) {
|
||||
throw IllegalStateException("LOCATION_PERMISSION_REQUIRED: grant Location permission")
|
||||
}
|
||||
val resolved =
|
||||
providers.firstOrNull { manager.isProviderEnabled(it) }
|
||||
?: throw IllegalStateException("LOCATION_UNAVAILABLE: no providers available")
|
||||
return withTimeout(timeoutMs.coerceAtLeast(1)) {
|
||||
suspendCancellableCoroutine { cont ->
|
||||
val signal = CancellationSignal()
|
||||
cont.invokeOnCancellation { signal.cancel() }
|
||||
manager.getCurrentLocation(resolved, signal, context.mainExecutor) { location ->
|
||||
if (location != null) {
|
||||
cont.resume(location)
|
||||
} else {
|
||||
cont.resumeWithException(IllegalStateException("LOCATION_UNAVAILABLE: no fix"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.location.LocationManager
|
||||
import androidx.core.content.ContextCompat
|
||||
import ai.openclaw.android.LocationMode
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import kotlinx.coroutines.TimeoutCancellationException
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
|
||||
class LocationHandler(
|
||||
private val appContext: Context,
|
||||
private val location: LocationCaptureManager,
|
||||
private val json: Json,
|
||||
private val isForeground: () -> Boolean,
|
||||
private val locationMode: () -> LocationMode,
|
||||
private val locationPreciseEnabled: () -> Boolean,
|
||||
) {
|
||||
fun hasFineLocationPermission(): Boolean {
|
||||
return (
|
||||
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_FINE_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
)
|
||||
}
|
||||
|
||||
fun hasCoarseLocationPermission(): Boolean {
|
||||
return (
|
||||
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_COARSE_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
)
|
||||
}
|
||||
|
||||
fun hasBackgroundLocationPermission(): Boolean {
|
||||
return (
|
||||
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
)
|
||||
}
|
||||
|
||||
suspend fun handleLocationGet(paramsJson: String?): GatewaySession.InvokeResult {
|
||||
val mode = locationMode()
|
||||
if (!isForeground() && mode != LocationMode.Always) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "LOCATION_BACKGROUND_UNAVAILABLE",
|
||||
message = "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always",
|
||||
)
|
||||
}
|
||||
if (!hasFineLocationPermission() && !hasCoarseLocationPermission()) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "LOCATION_PERMISSION_REQUIRED",
|
||||
message = "LOCATION_PERMISSION_REQUIRED: grant Location permission",
|
||||
)
|
||||
}
|
||||
if (!isForeground() && mode == LocationMode.Always && !hasBackgroundLocationPermission()) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "LOCATION_PERMISSION_REQUIRED",
|
||||
message = "LOCATION_PERMISSION_REQUIRED: enable Always in system Settings",
|
||||
)
|
||||
}
|
||||
val (maxAgeMs, timeoutMs, desiredAccuracy) = parseLocationParams(paramsJson)
|
||||
val preciseEnabled = locationPreciseEnabled()
|
||||
val accuracy =
|
||||
when (desiredAccuracy) {
|
||||
"precise" -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
|
||||
"coarse" -> "coarse"
|
||||
else -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
|
||||
}
|
||||
val providers =
|
||||
when (accuracy) {
|
||||
"precise" -> listOf(LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER)
|
||||
"coarse" -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
|
||||
else -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
|
||||
}
|
||||
try {
|
||||
val payload =
|
||||
location.getLocation(
|
||||
desiredProviders = providers,
|
||||
maxAgeMs = maxAgeMs,
|
||||
timeoutMs = timeoutMs,
|
||||
isPrecise = accuracy == "precise",
|
||||
)
|
||||
return GatewaySession.InvokeResult.ok(payload.payloadJson)
|
||||
} catch (err: TimeoutCancellationException) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "LOCATION_TIMEOUT",
|
||||
message = "LOCATION_TIMEOUT: no fix in time",
|
||||
)
|
||||
} catch (err: Throwable) {
|
||||
val message = err.message ?: "LOCATION_UNAVAILABLE: no fix"
|
||||
return GatewaySession.InvokeResult.error(code = "LOCATION_UNAVAILABLE", message = message)
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseLocationParams(paramsJson: String?): Triple<Long?, Long, String?> {
|
||||
if (paramsJson.isNullOrBlank()) {
|
||||
return Triple(null, 10_000L, null)
|
||||
}
|
||||
val root =
|
||||
try {
|
||||
json.parseToJsonElement(paramsJson).asObjectOrNull()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
val maxAgeMs = (root?.get("maxAgeMs") as? JsonPrimitive)?.content?.toLongOrNull()
|
||||
val timeoutMs =
|
||||
(root?.get("timeoutMs") as? JsonPrimitive)?.content?.toLongOrNull()?.coerceIn(1_000L, 60_000L)
|
||||
?: 10_000L
|
||||
val desiredAccuracy =
|
||||
(root?.get("desiredAccuracy") as? JsonPrimitive)?.content?.trim()?.lowercase()
|
||||
return Triple(maxAgeMs, timeoutMs, desiredAccuracy)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import ai.openclaw.android.gateway.parseInvokeErrorFromThrowable
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import kotlinx.serialization.json.JsonNull
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
|
||||
const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A
|
||||
|
||||
data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
|
||||
|
||||
fun String.toJsonString(): String {
|
||||
val escaped =
|
||||
this.replace("\\", "\\\\")
|
||||
.replace("\"", "\\\"")
|
||||
.replace("\n", "\\n")
|
||||
.replace("\r", "\\r")
|
||||
return "\"$escaped\""
|
||||
}
|
||||
|
||||
fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
|
||||
|
||||
fun JsonElement?.asStringOrNull(): String? =
|
||||
when (this) {
|
||||
is JsonNull -> null
|
||||
is JsonPrimitive -> content
|
||||
else -> null
|
||||
}
|
||||
|
||||
fun parseHexColorArgb(raw: String?): Long? {
|
||||
val trimmed = raw?.trim().orEmpty()
|
||||
if (trimmed.isEmpty()) return null
|
||||
val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed
|
||||
if (hex.length != 6) return null
|
||||
val rgb = hex.toLongOrNull(16) ?: return null
|
||||
return 0xFF000000L or rgb
|
||||
}
|
||||
|
||||
fun invokeErrorFromThrowable(err: Throwable): Pair<String, String> {
|
||||
val parsed = parseInvokeErrorFromThrowable(err, fallbackMessage = "UNAVAILABLE: error")
|
||||
val message = if (parsed.hadExplicitCode) parsed.prefixedMessage else parsed.message
|
||||
return parsed.code to message
|
||||
}
|
||||
|
||||
fun normalizeMainKey(raw: String?): String? {
|
||||
val trimmed = raw?.trim().orEmpty()
|
||||
return if (trimmed.isEmpty()) null else trimmed
|
||||
}
|
||||
|
||||
fun isCanonicalMainSessionKey(key: String): Boolean {
|
||||
return key == "main"
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.content.Context
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import kotlinx.serialization.json.contentOrNull
|
||||
import kotlinx.serialization.json.put
|
||||
|
||||
internal interface NotificationsStateProvider {
|
||||
fun readSnapshot(context: Context): DeviceNotificationSnapshot
|
||||
|
||||
fun requestServiceRebind(context: Context)
|
||||
|
||||
fun executeAction(context: Context, request: NotificationActionRequest): NotificationActionResult
|
||||
}
|
||||
|
||||
private object SystemNotificationsStateProvider : NotificationsStateProvider {
|
||||
override fun readSnapshot(context: Context): DeviceNotificationSnapshot {
|
||||
val enabled = DeviceNotificationListenerService.isAccessEnabled(context)
|
||||
if (!enabled) {
|
||||
return DeviceNotificationSnapshot(
|
||||
enabled = false,
|
||||
connected = false,
|
||||
notifications = emptyList(),
|
||||
)
|
||||
}
|
||||
return DeviceNotificationListenerService.snapshot(context, enabled = true)
|
||||
}
|
||||
|
||||
override fun requestServiceRebind(context: Context) {
|
||||
DeviceNotificationListenerService.requestServiceRebind(context)
|
||||
}
|
||||
|
||||
override fun executeAction(context: Context, request: NotificationActionRequest): NotificationActionResult {
|
||||
return DeviceNotificationListenerService.executeAction(context, request)
|
||||
}
|
||||
}
|
||||
|
||||
class NotificationsHandler private constructor(
|
||||
private val appContext: Context,
|
||||
private val stateProvider: NotificationsStateProvider,
|
||||
) {
|
||||
constructor(appContext: Context) : this(appContext = appContext, stateProvider = SystemNotificationsStateProvider)
|
||||
|
||||
suspend fun handleNotificationsList(_paramsJson: String?): GatewaySession.InvokeResult {
|
||||
val snapshot = readSnapshotWithRebind()
|
||||
return GatewaySession.InvokeResult.ok(snapshotPayloadJson(snapshot))
|
||||
}
|
||||
|
||||
suspend fun handleNotificationsActions(paramsJson: String?): GatewaySession.InvokeResult {
|
||||
readSnapshotWithRebind()
|
||||
|
||||
val params = parseParamsObject(paramsJson)
|
||||
?: return GatewaySession.InvokeResult.error(
|
||||
code = "INVALID_REQUEST",
|
||||
message = "INVALID_REQUEST: expected JSON object",
|
||||
)
|
||||
val key =
|
||||
readString(params, "key")
|
||||
?: return GatewaySession.InvokeResult.error(
|
||||
code = "INVALID_REQUEST",
|
||||
message = "INVALID_REQUEST: key required",
|
||||
)
|
||||
val actionRaw =
|
||||
readString(params, "action")?.lowercase()
|
||||
?: return GatewaySession.InvokeResult.error(
|
||||
code = "INVALID_REQUEST",
|
||||
message = "INVALID_REQUEST: action required (open|dismiss|reply)",
|
||||
)
|
||||
val action =
|
||||
when (actionRaw) {
|
||||
"open" -> NotificationActionKind.Open
|
||||
"dismiss" -> NotificationActionKind.Dismiss
|
||||
"reply" -> NotificationActionKind.Reply
|
||||
else ->
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "INVALID_REQUEST",
|
||||
message = "INVALID_REQUEST: action must be open|dismiss|reply",
|
||||
)
|
||||
}
|
||||
val replyText = readString(params, "replyText")
|
||||
if (action == NotificationActionKind.Reply && replyText.isNullOrBlank()) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = "INVALID_REQUEST",
|
||||
message = "INVALID_REQUEST: replyText required for reply action",
|
||||
)
|
||||
}
|
||||
|
||||
val result =
|
||||
stateProvider.executeAction(
|
||||
appContext,
|
||||
NotificationActionRequest(
|
||||
key = key,
|
||||
kind = action,
|
||||
replyText = replyText,
|
||||
),
|
||||
)
|
||||
if (!result.ok) {
|
||||
return GatewaySession.InvokeResult.error(
|
||||
code = result.code ?: "UNAVAILABLE",
|
||||
message = result.message ?: "notification action failed",
|
||||
)
|
||||
}
|
||||
|
||||
val payload =
|
||||
buildJsonObject {
|
||||
put("ok", JsonPrimitive(true))
|
||||
put("key", JsonPrimitive(key))
|
||||
put("action", JsonPrimitive(actionRaw))
|
||||
}.toString()
|
||||
return GatewaySession.InvokeResult.ok(payload)
|
||||
}
|
||||
|
||||
private fun readSnapshotWithRebind(): DeviceNotificationSnapshot {
|
||||
val snapshot = stateProvider.readSnapshot(appContext)
|
||||
if (snapshot.enabled && !snapshot.connected) {
|
||||
stateProvider.requestServiceRebind(appContext)
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
private fun snapshotPayloadJson(snapshot: DeviceNotificationSnapshot): String {
|
||||
return buildJsonObject {
|
||||
put("enabled", JsonPrimitive(snapshot.enabled))
|
||||
put("connected", JsonPrimitive(snapshot.connected))
|
||||
put("count", JsonPrimitive(snapshot.notifications.size))
|
||||
put(
|
||||
"notifications",
|
||||
JsonArray(
|
||||
snapshot.notifications.map { entry ->
|
||||
buildJsonObject {
|
||||
put("key", JsonPrimitive(entry.key))
|
||||
put("packageName", JsonPrimitive(entry.packageName))
|
||||
put("postTimeMs", JsonPrimitive(entry.postTimeMs))
|
||||
put("isOngoing", JsonPrimitive(entry.isOngoing))
|
||||
put("isClearable", JsonPrimitive(entry.isClearable))
|
||||
entry.title?.let { put("title", JsonPrimitive(it)) }
|
||||
entry.text?.let { put("text", JsonPrimitive(it)) }
|
||||
entry.subText?.let { put("subText", JsonPrimitive(it)) }
|
||||
entry.category?.let { put("category", JsonPrimitive(it)) }
|
||||
entry.channelId?.let { put("channelId", JsonPrimitive(it)) }
|
||||
}
|
||||
},
|
||||
),
|
||||
)
|
||||
}.toString()
|
||||
}
|
||||
|
||||
private fun parseParamsObject(paramsJson: String?): JsonObject? {
|
||||
if (paramsJson.isNullOrBlank()) return null
|
||||
return try {
|
||||
Json.parseToJsonElement(paramsJson).asObjectOrNull()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun readString(params: JsonObject, key: String): String? =
|
||||
(params[key] as? JsonPrimitive)
|
||||
?.contentOrNull
|
||||
?.trim()
|
||||
?.takeIf { it.isNotEmpty() }
|
||||
|
||||
companion object {
|
||||
internal fun forTesting(
|
||||
appContext: Context,
|
||||
stateProvider: NotificationsStateProvider,
|
||||
): NotificationsHandler = NotificationsHandler(appContext = appContext, stateProvider = stateProvider)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
|
||||
class ScreenHandler(
|
||||
private val screenRecorder: ScreenRecordManager,
|
||||
private val setScreenRecordActive: (Boolean) -> Unit,
|
||||
private val invokeErrorFromThrowable: (Throwable) -> Pair<String, String>,
|
||||
) {
|
||||
suspend fun handleScreenRecord(paramsJson: String?): GatewaySession.InvokeResult {
|
||||
setScreenRecordActive(true)
|
||||
try {
|
||||
val res =
|
||||
try {
|
||||
screenRecorder.record(paramsJson)
|
||||
} catch (err: Throwable) {
|
||||
val (code, message) = invokeErrorFromThrowable(err)
|
||||
return GatewaySession.InvokeResult.error(code = code, message = message)
|
||||
}
|
||||
return GatewaySession.InvokeResult.ok(res.payloadJson)
|
||||
} finally {
|
||||
setScreenRecordActive(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.content.Context
|
||||
import android.hardware.display.DisplayManager
|
||||
import android.media.MediaRecorder
|
||||
import android.media.projection.MediaProjectionManager
|
||||
import android.os.Build
|
||||
import android.util.Base64
|
||||
import ai.openclaw.android.ScreenCaptureRequester
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.contentOrNull
|
||||
import java.io.File
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
class ScreenRecordManager(private val context: Context) {
|
||||
data class Payload(val payloadJson: String)
|
||||
|
||||
@Volatile private var screenCaptureRequester: ScreenCaptureRequester? = null
|
||||
@Volatile private var permissionRequester: ai.openclaw.android.PermissionRequester? = null
|
||||
|
||||
fun attachScreenCaptureRequester(requester: ScreenCaptureRequester) {
|
||||
screenCaptureRequester = requester
|
||||
}
|
||||
|
||||
fun attachPermissionRequester(requester: ai.openclaw.android.PermissionRequester) {
|
||||
permissionRequester = requester
|
||||
}
|
||||
|
||||
suspend fun record(paramsJson: String?): Payload =
|
||||
withContext(Dispatchers.Default) {
|
||||
val requester =
|
||||
screenCaptureRequester
|
||||
?: throw IllegalStateException(
|
||||
"SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission",
|
||||
)
|
||||
|
||||
val params = parseParamsObject(paramsJson)
|
||||
val durationMs = (parseDurationMs(params) ?: 10_000).coerceIn(250, 60_000)
|
||||
val fps = (parseFps(params) ?: 10.0).coerceIn(1.0, 60.0)
|
||||
val fpsInt = fps.roundToInt().coerceIn(1, 60)
|
||||
val screenIndex = parseScreenIndex(params)
|
||||
val includeAudio = parseIncludeAudio(params) ?: true
|
||||
val format = parseString(params, key = "format")
|
||||
if (format != null && format.lowercase() != "mp4") {
|
||||
throw IllegalArgumentException("INVALID_REQUEST: screen format must be mp4")
|
||||
}
|
||||
if (screenIndex != null && screenIndex != 0) {
|
||||
throw IllegalArgumentException("INVALID_REQUEST: screenIndex must be 0 on Android")
|
||||
}
|
||||
|
||||
val capture = requester.requestCapture()
|
||||
?: throw IllegalStateException(
|
||||
"SCREEN_PERMISSION_REQUIRED: grant Screen Recording permission",
|
||||
)
|
||||
|
||||
val mgr =
|
||||
context.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||
val projection = mgr.getMediaProjection(capture.resultCode, capture.data)
|
||||
?: throw IllegalStateException("UNAVAILABLE: screen capture unavailable")
|
||||
|
||||
val metrics = context.resources.displayMetrics
|
||||
val width = metrics.widthPixels
|
||||
val height = metrics.heightPixels
|
||||
val densityDpi = metrics.densityDpi
|
||||
|
||||
val file = File.createTempFile("openclaw-screen-", ".mp4")
|
||||
if (includeAudio) ensureMicPermission()
|
||||
|
||||
val recorder = createMediaRecorder()
|
||||
var virtualDisplay: android.hardware.display.VirtualDisplay? = null
|
||||
try {
|
||||
if (includeAudio) {
|
||||
recorder.setAudioSource(MediaRecorder.AudioSource.MIC)
|
||||
}
|
||||
recorder.setVideoSource(MediaRecorder.VideoSource.SURFACE)
|
||||
recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
|
||||
recorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264)
|
||||
if (includeAudio) {
|
||||
recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
|
||||
recorder.setAudioChannels(1)
|
||||
recorder.setAudioSamplingRate(44_100)
|
||||
recorder.setAudioEncodingBitRate(96_000)
|
||||
}
|
||||
recorder.setVideoSize(width, height)
|
||||
recorder.setVideoFrameRate(fpsInt)
|
||||
recorder.setVideoEncodingBitRate(estimateBitrate(width, height, fpsInt))
|
||||
recorder.setOutputFile(file.absolutePath)
|
||||
recorder.prepare()
|
||||
|
||||
val surface = recorder.surface
|
||||
virtualDisplay =
|
||||
projection.createVirtualDisplay(
|
||||
"openclaw-screen",
|
||||
width,
|
||||
height,
|
||||
densityDpi,
|
||||
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
|
||||
surface,
|
||||
null,
|
||||
null,
|
||||
)
|
||||
|
||||
recorder.start()
|
||||
delay(durationMs.toLong())
|
||||
} finally {
|
||||
try {
|
||||
recorder.stop()
|
||||
} catch (_: Throwable) {
|
||||
// ignore
|
||||
}
|
||||
recorder.reset()
|
||||
recorder.release()
|
||||
virtualDisplay?.release()
|
||||
projection.stop()
|
||||
}
|
||||
|
||||
val bytes = withContext(Dispatchers.IO) { file.readBytes() }
|
||||
file.delete()
|
||||
val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP)
|
||||
Payload(
|
||||
"""{"format":"mp4","base64":"$base64","durationMs":$durationMs,"fps":$fpsInt,"screenIndex":0,"hasAudio":$includeAudio}""",
|
||||
)
|
||||
}
|
||||
|
||||
private fun createMediaRecorder(): MediaRecorder = MediaRecorder(context)
|
||||
|
||||
private suspend fun ensureMicPermission() {
|
||||
val granted =
|
||||
androidx.core.content.ContextCompat.checkSelfPermission(
|
||||
context,
|
||||
android.Manifest.permission.RECORD_AUDIO,
|
||||
) == android.content.pm.PackageManager.PERMISSION_GRANTED
|
||||
if (granted) return
|
||||
|
||||
val requester =
|
||||
permissionRequester
|
||||
?: throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
|
||||
val results = requester.requestIfMissing(listOf(android.Manifest.permission.RECORD_AUDIO))
|
||||
if (results[android.Manifest.permission.RECORD_AUDIO] != true) {
|
||||
throw IllegalStateException("MIC_PERMISSION_REQUIRED: grant Microphone permission")
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseParamsObject(paramsJson: String?): JsonObject? {
|
||||
if (paramsJson.isNullOrBlank()) return null
|
||||
return try {
|
||||
Json.parseToJsonElement(paramsJson).asObjectOrNull()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun readPrimitive(params: JsonObject?, key: String): JsonPrimitive? =
|
||||
params?.get(key) as? JsonPrimitive
|
||||
|
||||
private fun parseDurationMs(params: JsonObject?): Int? =
|
||||
readPrimitive(params, "durationMs")?.contentOrNull?.toIntOrNull()
|
||||
|
||||
private fun parseFps(params: JsonObject?): Double? =
|
||||
readPrimitive(params, "fps")?.contentOrNull?.toDoubleOrNull()
|
||||
|
||||
private fun parseScreenIndex(params: JsonObject?): Int? =
|
||||
readPrimitive(params, "screenIndex")?.contentOrNull?.toIntOrNull()
|
||||
|
||||
private fun parseIncludeAudio(params: JsonObject?): Boolean? {
|
||||
val value = readPrimitive(params, "includeAudio")?.contentOrNull?.trim()?.lowercase()
|
||||
return when (value) {
|
||||
"true" -> true
|
||||
"false" -> false
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseString(params: JsonObject?, key: String): String? =
|
||||
readPrimitive(params, key)?.contentOrNull
|
||||
|
||||
private fun estimateBitrate(width: Int, height: Int, fps: Int): Int {
|
||||
val pixels = width.toLong() * height.toLong()
|
||||
val raw = (pixels * fps.toLong() * 2L).toInt()
|
||||
return raw.coerceIn(1_000_000, 12_000_000)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import ai.openclaw.android.gateway.GatewaySession
|
||||
|
||||
class SmsHandler(
|
||||
private val sms: SmsManager,
|
||||
) {
|
||||
suspend fun handleSmsSend(paramsJson: String?): GatewaySession.InvokeResult {
|
||||
val res = sms.send(paramsJson)
|
||||
if (res.ok) {
|
||||
return GatewaySession.InvokeResult.ok(res.payloadJson)
|
||||
} else {
|
||||
val error = res.error ?: "SMS_SEND_FAILED"
|
||||
val idx = error.indexOf(':')
|
||||
val code = if (idx > 0) error.substring(0, idx).trim() else "SMS_SEND_FAILED"
|
||||
return GatewaySession.InvokeResult.error(code = code, message = error)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package ai.openclaw.android.node
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.telephony.SmsManager as AndroidSmsManager
|
||||
import androidx.core.content.ContextCompat
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import kotlinx.serialization.encodeToString
|
||||
import ai.openclaw.android.PermissionRequester
|
||||
|
||||
/**
|
||||
* Sends SMS messages via the Android SMS API.
|
||||
* Requires SEND_SMS permission to be granted.
|
||||
*/
|
||||
class SmsManager(private val context: Context) {
|
||||
|
||||
private val json = JsonConfig
|
||||
@Volatile private var permissionRequester: PermissionRequester? = null
|
||||
|
||||
data class SendResult(
|
||||
val ok: Boolean,
|
||||
val to: String,
|
||||
val message: String?,
|
||||
val error: String? = null,
|
||||
val payloadJson: String,
|
||||
)
|
||||
|
||||
internal data class ParsedParams(
|
||||
val to: String,
|
||||
val message: String,
|
||||
)
|
||||
|
||||
internal sealed class ParseResult {
|
||||
data class Ok(val params: ParsedParams) : ParseResult()
|
||||
data class Error(
|
||||
val error: String,
|
||||
val to: String = "",
|
||||
val message: String? = null,
|
||||
) : ParseResult()
|
||||
}
|
||||
|
||||
internal data class SendPlan(
|
||||
val parts: List<String>,
|
||||
val useMultipart: Boolean,
|
||||
)
|
||||
|
||||
companion object {
|
||||
internal val JsonConfig = Json { ignoreUnknownKeys = true }
|
||||
|
||||
internal fun parseParams(paramsJson: String?, json: Json = JsonConfig): ParseResult {
|
||||
val params = paramsJson?.trim().orEmpty()
|
||||
if (params.isEmpty()) {
|
||||
return ParseResult.Error(error = "INVALID_REQUEST: paramsJSON required")
|
||||
}
|
||||
|
||||
val obj = try {
|
||||
json.parseToJsonElement(params).jsonObject
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
|
||||
if (obj == null) {
|
||||
return ParseResult.Error(error = "INVALID_REQUEST: expected JSON object")
|
||||
}
|
||||
|
||||
val to = (obj["to"] as? JsonPrimitive)?.content?.trim().orEmpty()
|
||||
val message = (obj["message"] as? JsonPrimitive)?.content.orEmpty()
|
||||
|
||||
if (to.isEmpty()) {
|
||||
return ParseResult.Error(
|
||||
error = "INVALID_REQUEST: 'to' phone number required",
|
||||
message = message,
|
||||
)
|
||||
}
|
||||
|
||||
if (message.isEmpty()) {
|
||||
return ParseResult.Error(
|
||||
error = "INVALID_REQUEST: 'message' text required",
|
||||
to = to,
|
||||
)
|
||||
}
|
||||
|
||||
return ParseResult.Ok(ParsedParams(to = to, message = message))
|
||||
}
|
||||
|
||||
internal fun buildSendPlan(
|
||||
message: String,
|
||||
divider: (String) -> List<String>,
|
||||
): SendPlan {
|
||||
val parts = divider(message).ifEmpty { listOf(message) }
|
||||
return SendPlan(parts = parts, useMultipart = parts.size > 1)
|
||||
}
|
||||
|
||||
internal fun buildPayloadJson(
|
||||
json: Json = JsonConfig,
|
||||
ok: Boolean,
|
||||
to: String,
|
||||
error: String?,
|
||||
): String {
|
||||
val payload =
|
||||
mutableMapOf<String, JsonElement>(
|
||||
"ok" to JsonPrimitive(ok),
|
||||
"to" to JsonPrimitive(to),
|
||||
)
|
||||
if (!ok) {
|
||||
payload["error"] = JsonPrimitive(error ?: "SMS_SEND_FAILED")
|
||||
}
|
||||
return json.encodeToString(JsonObject.serializer(), JsonObject(payload))
|
||||
}
|
||||
}
|
||||
|
||||
fun hasSmsPermission(): Boolean {
|
||||
return ContextCompat.checkSelfPermission(
|
||||
context,
|
||||
Manifest.permission.SEND_SMS
|
||||
) == PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
|
||||
fun canSendSms(): Boolean {
|
||||
return hasSmsPermission() && hasTelephonyFeature()
|
||||
}
|
||||
|
||||
fun hasTelephonyFeature(): Boolean {
|
||||
return context.packageManager?.hasSystemFeature(PackageManager.FEATURE_TELEPHONY) == true
|
||||
}
|
||||
|
||||
fun attachPermissionRequester(requester: PermissionRequester) {
|
||||
permissionRequester = requester
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an SMS message.
|
||||
*
|
||||
* @param paramsJson JSON with "to" (phone number) and "message" (text) fields
|
||||
* @return SendResult indicating success or failure
|
||||
*/
|
||||
suspend fun send(paramsJson: String?): SendResult {
|
||||
if (!hasTelephonyFeature()) {
|
||||
return errorResult(
|
||||
error = "SMS_UNAVAILABLE: telephony not available",
|
||||
)
|
||||
}
|
||||
|
||||
if (!ensureSmsPermission()) {
|
||||
return errorResult(
|
||||
error = "SMS_PERMISSION_REQUIRED: grant SMS permission",
|
||||
)
|
||||
}
|
||||
|
||||
val parseResult = parseParams(paramsJson, json)
|
||||
if (parseResult is ParseResult.Error) {
|
||||
return errorResult(
|
||||
error = parseResult.error,
|
||||
to = parseResult.to,
|
||||
message = parseResult.message,
|
||||
)
|
||||
}
|
||||
val params = (parseResult as ParseResult.Ok).params
|
||||
|
||||
return try {
|
||||
val smsManager = context.getSystemService(AndroidSmsManager::class.java)
|
||||
?: throw IllegalStateException("SMS_UNAVAILABLE: SmsManager not available")
|
||||
|
||||
val plan = buildSendPlan(params.message) { smsManager.divideMessage(it) }
|
||||
if (plan.useMultipart) {
|
||||
smsManager.sendMultipartTextMessage(
|
||||
params.to, // destination
|
||||
null, // service center (null = default)
|
||||
ArrayList(plan.parts), // message parts
|
||||
null, // sent intents
|
||||
null, // delivery intents
|
||||
)
|
||||
} else {
|
||||
smsManager.sendTextMessage(
|
||||
params.to, // destination
|
||||
null, // service center (null = default)
|
||||
params.message,// message
|
||||
null, // sent intent
|
||||
null, // delivery intent
|
||||
)
|
||||
}
|
||||
|
||||
okResult(to = params.to, message = params.message)
|
||||
} catch (e: SecurityException) {
|
||||
errorResult(
|
||||
error = "SMS_PERMISSION_REQUIRED: ${e.message}",
|
||||
to = params.to,
|
||||
message = params.message,
|
||||
)
|
||||
} catch (e: Throwable) {
|
||||
errorResult(
|
||||
error = "SMS_SEND_FAILED: ${e.message ?: "unknown error"}",
|
||||
to = params.to,
|
||||
message = params.message,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun ensureSmsPermission(): Boolean {
|
||||
if (hasSmsPermission()) return true
|
||||
val requester = permissionRequester ?: return false
|
||||
val results = requester.requestIfMissing(listOf(Manifest.permission.SEND_SMS))
|
||||
return results[Manifest.permission.SEND_SMS] == true
|
||||
}
|
||||
|
||||
private fun okResult(to: String, message: String): SendResult {
|
||||
return SendResult(
|
||||
ok = true,
|
||||
to = to,
|
||||
message = message,
|
||||
error = null,
|
||||
payloadJson = buildPayloadJson(json = json, ok = true, to = to, error = null),
|
||||
)
|
||||
}
|
||||
|
||||
private fun errorResult(error: String, to: String = "", message: String? = null): SendResult {
|
||||
return SendResult(
|
||||
ok = false,
|
||||
to = to,
|
||||
message = message,
|
||||
error = error,
|
||||
payloadJson = buildPayloadJson(json = json, ok = false, to = to, error = error),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package ai.openclaw.android.protocol
|
||||
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
|
||||
object OpenClawCanvasA2UIAction {
|
||||
fun extractActionName(userAction: JsonObject): String? {
|
||||
val name =
|
||||
(userAction["name"] as? JsonPrimitive)
|
||||
?.content
|
||||
?.trim()
|
||||
.orEmpty()
|
||||
if (name.isNotEmpty()) return name
|
||||
val action =
|
||||
(userAction["action"] as? JsonPrimitive)
|
||||
?.content
|
||||
?.trim()
|
||||
.orEmpty()
|
||||
return action.ifEmpty { null }
|
||||
}
|
||||
|
||||
fun sanitizeTagValue(value: String): String {
|
||||
val trimmed = value.trim().ifEmpty { "-" }
|
||||
val normalized = trimmed.replace(" ", "_")
|
||||
val out = StringBuilder(normalized.length)
|
||||
for (c in normalized) {
|
||||
val ok =
|
||||
c.isLetterOrDigit() ||
|
||||
c == '_' ||
|
||||
c == '-' ||
|
||||
c == '.' ||
|
||||
c == ':'
|
||||
out.append(if (ok) c else '_')
|
||||
}
|
||||
return out.toString()
|
||||
}
|
||||
|
||||
fun formatAgentMessage(
|
||||
actionName: String,
|
||||
sessionKey: String,
|
||||
surfaceId: String,
|
||||
sourceComponentId: String,
|
||||
host: String,
|
||||
instanceId: String,
|
||||
contextJson: String?,
|
||||
): String {
|
||||
val ctxSuffix = contextJson?.takeIf { it.isNotBlank() }?.let { " ctx=$it" }.orEmpty()
|
||||
return listOf(
|
||||
"CANVAS_A2UI",
|
||||
"action=${sanitizeTagValue(actionName)}",
|
||||
"session=${sanitizeTagValue(sessionKey)}",
|
||||
"surface=${sanitizeTagValue(surfaceId)}",
|
||||
"component=${sanitizeTagValue(sourceComponentId)}",
|
||||
"host=${sanitizeTagValue(host)}",
|
||||
"instance=${sanitizeTagValue(instanceId)}$ctxSuffix",
|
||||
"default=update_canvas",
|
||||
).joinToString(separator = " ")
|
||||
}
|
||||
|
||||
fun jsDispatchA2UIActionStatus(actionId: String, ok: Boolean, error: String?): String {
|
||||
val err = (error ?: "").replace("\\", "\\\\").replace("\"", "\\\"")
|
||||
val okLiteral = if (ok) "true" else "false"
|
||||
val idEscaped = actionId.replace("\\", "\\\\").replace("\"", "\\\"")
|
||||
return "window.dispatchEvent(new CustomEvent('openclaw:a2ui-action-status', { detail: { id: \"${idEscaped}\", ok: ${okLiteral}, error: \"${err}\" } }));"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package ai.openclaw.android.protocol
|
||||
|
||||
enum class OpenClawCapability(val rawValue: String) {
|
||||
Canvas("canvas"),
|
||||
Camera("camera"),
|
||||
Screen("screen"),
|
||||
Sms("sms"),
|
||||
VoiceWake("voiceWake"),
|
||||
Location("location"),
|
||||
Device("device"),
|
||||
}
|
||||
|
||||
enum class OpenClawCanvasCommand(val rawValue: String) {
|
||||
Present("canvas.present"),
|
||||
Hide("canvas.hide"),
|
||||
Navigate("canvas.navigate"),
|
||||
Eval("canvas.eval"),
|
||||
Snapshot("canvas.snapshot"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
const val NamespacePrefix: String = "canvas."
|
||||
}
|
||||
}
|
||||
|
||||
enum class OpenClawCanvasA2UICommand(val rawValue: String) {
|
||||
Push("canvas.a2ui.push"),
|
||||
PushJSONL("canvas.a2ui.pushJSONL"),
|
||||
Reset("canvas.a2ui.reset"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
const val NamespacePrefix: String = "canvas.a2ui."
|
||||
}
|
||||
}
|
||||
|
||||
enum class OpenClawCameraCommand(val rawValue: String) {
|
||||
List("camera.list"),
|
||||
Snap("camera.snap"),
|
||||
Clip("camera.clip"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
const val NamespacePrefix: String = "camera."
|
||||
}
|
||||
}
|
||||
|
||||
enum class OpenClawScreenCommand(val rawValue: String) {
|
||||
Record("screen.record"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
const val NamespacePrefix: String = "screen."
|
||||
}
|
||||
}
|
||||
|
||||
enum class OpenClawSmsCommand(val rawValue: String) {
|
||||
Send("sms.send"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
const val NamespacePrefix: String = "sms."
|
||||
}
|
||||
}
|
||||
|
||||
enum class OpenClawLocationCommand(val rawValue: String) {
|
||||
Get("location.get"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
const val NamespacePrefix: String = "location."
|
||||
}
|
||||
}
|
||||
|
||||
enum class OpenClawDeviceCommand(val rawValue: String) {
|
||||
Status("device.status"),
|
||||
Info("device.info"),
|
||||
Permissions("device.permissions"),
|
||||
Health("device.health"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
const val NamespacePrefix: String = "device."
|
||||
}
|
||||
}
|
||||
|
||||
enum class OpenClawNotificationsCommand(val rawValue: String) {
|
||||
List("notifications.list"),
|
||||
Actions("notifications.actions"),
|
||||
;
|
||||
|
||||
companion object {
|
||||
const val NamespacePrefix: String = "notifications."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package ai.openclaw.android.tools
|
||||
|
||||
import android.content.Context
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.contentOrNull
|
||||
|
||||
@Serializable
|
||||
private data class ToolDisplayActionSpec(
|
||||
val label: String? = null,
|
||||
val detailKeys: List<String>? = null,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
private data class ToolDisplaySpec(
|
||||
val emoji: String? = null,
|
||||
val title: String? = null,
|
||||
val label: String? = null,
|
||||
val detailKeys: List<String>? = null,
|
||||
val actions: Map<String, ToolDisplayActionSpec>? = null,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
private data class ToolDisplayConfig(
|
||||
val version: Int? = null,
|
||||
val fallback: ToolDisplaySpec? = null,
|
||||
val tools: Map<String, ToolDisplaySpec>? = null,
|
||||
)
|
||||
|
||||
data class ToolDisplaySummary(
|
||||
val name: String,
|
||||
val emoji: String,
|
||||
val title: String,
|
||||
val label: String,
|
||||
val verb: String?,
|
||||
val detail: String?,
|
||||
) {
|
||||
val detailLine: String?
|
||||
get() {
|
||||
val parts = mutableListOf<String>()
|
||||
if (!verb.isNullOrBlank()) parts.add(verb)
|
||||
if (!detail.isNullOrBlank()) parts.add(detail)
|
||||
return if (parts.isEmpty()) null else parts.joinToString(" · ")
|
||||
}
|
||||
|
||||
val summaryLine: String
|
||||
get() = if (detailLine != null) "${emoji} ${label}: ${detailLine}" else "${emoji} ${label}"
|
||||
}
|
||||
|
||||
object ToolDisplayRegistry {
|
||||
private const val CONFIG_ASSET = "tool-display.json"
|
||||
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
@Volatile private var cachedConfig: ToolDisplayConfig? = null
|
||||
|
||||
fun resolve(
|
||||
context: Context,
|
||||
name: String?,
|
||||
args: JsonObject?,
|
||||
meta: String? = null,
|
||||
): ToolDisplaySummary {
|
||||
val trimmedName = name?.trim().orEmpty().ifEmpty { "tool" }
|
||||
val key = trimmedName.lowercase()
|
||||
val config = loadConfig(context)
|
||||
val spec = config.tools?.get(key)
|
||||
val fallback = config.fallback
|
||||
|
||||
val emoji = spec?.emoji ?: fallback?.emoji ?: "🧩"
|
||||
val title = spec?.title ?: titleFromName(trimmedName)
|
||||
val label = spec?.label ?: trimmedName
|
||||
|
||||
val actionRaw = args?.get("action")?.asStringOrNull()?.trim()
|
||||
val action = actionRaw?.takeIf { it.isNotEmpty() }
|
||||
val actionSpec = action?.let { spec?.actions?.get(it) }
|
||||
val verb = normalizeVerb(actionSpec?.label ?: action)
|
||||
|
||||
var detail: String? = null
|
||||
if (key == "read") {
|
||||
detail = readDetail(args)
|
||||
} else if (key == "write" || key == "edit" || key == "attach") {
|
||||
detail = pathDetail(args)
|
||||
}
|
||||
|
||||
val detailKeys = actionSpec?.detailKeys ?: spec?.detailKeys ?: fallback?.detailKeys ?: emptyList()
|
||||
if (detail == null) {
|
||||
detail = firstValue(args, detailKeys)
|
||||
}
|
||||
|
||||
if (detail == null) {
|
||||
detail = meta
|
||||
}
|
||||
|
||||
if (detail != null) {
|
||||
detail = shortenHomeInString(detail)
|
||||
}
|
||||
|
||||
return ToolDisplaySummary(
|
||||
name = trimmedName,
|
||||
emoji = emoji,
|
||||
title = title,
|
||||
label = label,
|
||||
verb = verb,
|
||||
detail = detail,
|
||||
)
|
||||
}
|
||||
|
||||
private fun loadConfig(context: Context): ToolDisplayConfig {
|
||||
val existing = cachedConfig
|
||||
if (existing != null) return existing
|
||||
return try {
|
||||
val jsonString = context.assets.open(CONFIG_ASSET).bufferedReader().use { it.readText() }
|
||||
val decoded = json.decodeFromString(ToolDisplayConfig.serializer(), jsonString)
|
||||
cachedConfig = decoded
|
||||
decoded
|
||||
} catch (_: Throwable) {
|
||||
val fallback = ToolDisplayConfig()
|
||||
cachedConfig = fallback
|
||||
fallback
|
||||
}
|
||||
}
|
||||
|
||||
private fun titleFromName(name: String): String {
|
||||
val cleaned = name.replace("_", " ").trim()
|
||||
if (cleaned.isEmpty()) return "Tool"
|
||||
return cleaned
|
||||
.split(Regex("\\s+"))
|
||||
.joinToString(" ") { part ->
|
||||
val upper = part.uppercase()
|
||||
if (part.length <= 2 && part == upper) part
|
||||
else upper.firstOrNull()?.toString().orEmpty() + part.lowercase().drop(1)
|
||||
}
|
||||
}
|
||||
|
||||
private fun normalizeVerb(value: String?): String? {
|
||||
val trimmed = value?.trim().orEmpty()
|
||||
if (trimmed.isEmpty()) return null
|
||||
return trimmed.replace("_", " ")
|
||||
}
|
||||
|
||||
private fun readDetail(args: JsonObject?): String? {
|
||||
val path = args?.get("path")?.asStringOrNull() ?: return null
|
||||
val offset = args["offset"].asNumberOrNull()
|
||||
val limit = args["limit"].asNumberOrNull()
|
||||
return if (offset != null && limit != null) {
|
||||
val end = offset + limit
|
||||
"${path}:${offset.toInt()}-${end.toInt()}"
|
||||
} else {
|
||||
path
|
||||
}
|
||||
}
|
||||
|
||||
private fun pathDetail(args: JsonObject?): String? {
|
||||
return args?.get("path")?.asStringOrNull()
|
||||
}
|
||||
|
||||
private fun firstValue(args: JsonObject?, keys: List<String>): String? {
|
||||
for (key in keys) {
|
||||
val value = valueForPath(args, key)
|
||||
val rendered = renderValue(value)
|
||||
if (!rendered.isNullOrBlank()) return rendered
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun valueForPath(args: JsonObject?, path: String): JsonElement? {
|
||||
var current: JsonElement? = args
|
||||
for (segment in path.split(".")) {
|
||||
if (segment.isBlank()) return null
|
||||
val obj = current as? JsonObject ?: return null
|
||||
current = obj[segment]
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
private fun renderValue(value: JsonElement?): String? {
|
||||
if (value == null) return null
|
||||
if (value is JsonPrimitive) {
|
||||
if (value.isString) {
|
||||
val trimmed = value.contentOrNull?.trim().orEmpty()
|
||||
if (trimmed.isEmpty()) return null
|
||||
val firstLine = trimmed.lineSequence().firstOrNull()?.trim().orEmpty()
|
||||
if (firstLine.isEmpty()) return null
|
||||
return if (firstLine.length > 160) "${firstLine.take(157)}…" else firstLine
|
||||
}
|
||||
val raw = value.contentOrNull?.trim().orEmpty()
|
||||
raw.toBooleanStrictOrNull()?.let { return it.toString() }
|
||||
raw.toLongOrNull()?.let { return it.toString() }
|
||||
raw.toDoubleOrNull()?.let { return it.toString() }
|
||||
}
|
||||
if (value is JsonArray) {
|
||||
val items = value.mapNotNull { renderValue(it) }
|
||||
if (items.isEmpty()) return null
|
||||
val preview = items.take(3).joinToString(", ")
|
||||
return if (items.size > 3) "${preview}…" else preview
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun shortenHomeInString(value: String): String {
|
||||
val home = System.getProperty("user.home")?.takeIf { it.isNotBlank() }
|
||||
?: System.getenv("HOME")?.takeIf { it.isNotBlank() }
|
||||
if (home.isNullOrEmpty()) return value
|
||||
return value.replace(home, "~")
|
||||
.replace(Regex("/Users/[^/]+"), "~")
|
||||
.replace(Regex("/home/[^/]+"), "~")
|
||||
}
|
||||
|
||||
private fun JsonElement?.asStringOrNull(): String? {
|
||||
val primitive = this as? JsonPrimitive ?: return null
|
||||
return if (primitive.isString) primitive.contentOrNull else primitive.toString()
|
||||
}
|
||||
|
||||
private fun JsonElement?.asNumberOrNull(): Double? {
|
||||
val primitive = this as? JsonPrimitive ?: return null
|
||||
val raw = primitive.contentOrNull ?: return null
|
||||
return raw.toDoubleOrNull()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableFloatStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.draw.alpha
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import kotlinx.coroutines.delay
|
||||
|
||||
@Composable
|
||||
fun CameraFlashOverlay(
|
||||
token: Long,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
Box(modifier = modifier.fillMaxSize()) {
|
||||
CameraFlash(token = token)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun CameraFlash(token: Long) {
|
||||
var alpha by remember { mutableFloatStateOf(0f) }
|
||||
LaunchedEffect(token) {
|
||||
if (token == 0L) return@LaunchedEffect
|
||||
alpha = 0.85f
|
||||
delay(110)
|
||||
alpha = 0f
|
||||
}
|
||||
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxSize()
|
||||
.alpha(alpha)
|
||||
.background(Color.White),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import android.annotation.SuppressLint
|
||||
import android.util.Log
|
||||
import android.view.View
|
||||
import android.webkit.ConsoleMessage
|
||||
import android.webkit.JavascriptInterface
|
||||
import android.webkit.WebChromeClient
|
||||
import android.webkit.WebResourceError
|
||||
import android.webkit.WebResourceRequest
|
||||
import android.webkit.WebResourceResponse
|
||||
import android.webkit.WebSettings
|
||||
import android.webkit.WebView
|
||||
import android.webkit.WebViewClient
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.viewinterop.AndroidView
|
||||
import androidx.webkit.WebSettingsCompat
|
||||
import androidx.webkit.WebViewFeature
|
||||
import ai.openclaw.android.MainViewModel
|
||||
|
||||
@SuppressLint("SetJavaScriptEnabled")
|
||||
@Composable
|
||||
fun CanvasScreen(viewModel: MainViewModel, modifier: Modifier = Modifier) {
|
||||
val context = LocalContext.current
|
||||
val isDebuggable = (context.applicationInfo.flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0
|
||||
val webViewRef = remember { mutableStateOf<WebView?>(null) }
|
||||
|
||||
DisposableEffect(viewModel) {
|
||||
onDispose {
|
||||
val webView = webViewRef.value ?: return@onDispose
|
||||
viewModel.canvas.detach(webView)
|
||||
webView.removeJavascriptInterface(CanvasA2UIActionBridge.interfaceName)
|
||||
webView.stopLoading()
|
||||
webView.destroy()
|
||||
webViewRef.value = null
|
||||
}
|
||||
}
|
||||
|
||||
AndroidView(
|
||||
modifier = modifier,
|
||||
factory = {
|
||||
WebView(context).apply {
|
||||
settings.javaScriptEnabled = true
|
||||
settings.domStorageEnabled = true
|
||||
settings.mixedContentMode = WebSettings.MIXED_CONTENT_COMPATIBILITY_MODE
|
||||
settings.useWideViewPort = false
|
||||
settings.loadWithOverviewMode = false
|
||||
settings.builtInZoomControls = false
|
||||
settings.displayZoomControls = false
|
||||
settings.setSupportZoom(false)
|
||||
if (WebViewFeature.isFeatureSupported(WebViewFeature.ALGORITHMIC_DARKENING)) {
|
||||
WebSettingsCompat.setAlgorithmicDarkeningAllowed(settings, false)
|
||||
} else {
|
||||
disableForceDarkIfSupported(settings)
|
||||
}
|
||||
if (isDebuggable) {
|
||||
Log.d("OpenClawWebView", "userAgent: ${settings.userAgentString}")
|
||||
}
|
||||
isScrollContainer = true
|
||||
overScrollMode = View.OVER_SCROLL_IF_CONTENT_SCROLLS
|
||||
isVerticalScrollBarEnabled = true
|
||||
isHorizontalScrollBarEnabled = true
|
||||
webViewClient =
|
||||
object : WebViewClient() {
|
||||
override fun onReceivedError(
|
||||
view: WebView,
|
||||
request: WebResourceRequest,
|
||||
error: WebResourceError,
|
||||
) {
|
||||
if (!isDebuggable || !request.isForMainFrame) return
|
||||
Log.e("OpenClawWebView", "onReceivedError: ${error.errorCode} ${error.description} ${request.url}")
|
||||
}
|
||||
|
||||
override fun onReceivedHttpError(
|
||||
view: WebView,
|
||||
request: WebResourceRequest,
|
||||
errorResponse: WebResourceResponse,
|
||||
) {
|
||||
if (!isDebuggable || !request.isForMainFrame) return
|
||||
Log.e(
|
||||
"OpenClawWebView",
|
||||
"onReceivedHttpError: ${errorResponse.statusCode} ${errorResponse.reasonPhrase} ${request.url}",
|
||||
)
|
||||
}
|
||||
|
||||
override fun onPageFinished(view: WebView, url: String?) {
|
||||
if (isDebuggable) {
|
||||
Log.d("OpenClawWebView", "onPageFinished: $url")
|
||||
}
|
||||
viewModel.canvas.onPageFinished()
|
||||
}
|
||||
|
||||
override fun onRenderProcessGone(
|
||||
view: WebView,
|
||||
detail: android.webkit.RenderProcessGoneDetail,
|
||||
): Boolean {
|
||||
if (isDebuggable) {
|
||||
Log.e(
|
||||
"OpenClawWebView",
|
||||
"onRenderProcessGone didCrash=${detail.didCrash()} priorityAtExit=${detail.rendererPriorityAtExit()}",
|
||||
)
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
webChromeClient =
|
||||
object : WebChromeClient() {
|
||||
override fun onConsoleMessage(consoleMessage: ConsoleMessage?): Boolean {
|
||||
if (!isDebuggable) return false
|
||||
val msg = consoleMessage ?: return false
|
||||
Log.d(
|
||||
"OpenClawWebView",
|
||||
"console ${msg.messageLevel()} @ ${msg.sourceId()}:${msg.lineNumber()} ${msg.message()}",
|
||||
)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
val bridge = CanvasA2UIActionBridge { payload -> viewModel.handleCanvasA2UIActionFromWebView(payload) }
|
||||
addJavascriptInterface(bridge, CanvasA2UIActionBridge.interfaceName)
|
||||
viewModel.canvas.attach(this)
|
||||
webViewRef.value = this
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
private fun disableForceDarkIfSupported(settings: WebSettings) {
|
||||
if (!WebViewFeature.isFeatureSupported(WebViewFeature.FORCE_DARK)) return
|
||||
@Suppress("DEPRECATION")
|
||||
WebSettingsCompat.setForceDark(settings, WebSettingsCompat.FORCE_DARK_OFF)
|
||||
}
|
||||
|
||||
private class CanvasA2UIActionBridge(private val onMessage: (String) -> Unit) {
|
||||
@JavascriptInterface
|
||||
fun postMessage(payload: String?) {
|
||||
val msg = payload?.trim().orEmpty()
|
||||
if (msg.isEmpty()) return
|
||||
onMessage(msg)
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val interfaceName: String = "openclawCanvasA2UIAction"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import androidx.compose.runtime.Composable
|
||||
import ai.openclaw.android.MainViewModel
|
||||
import ai.openclaw.android.ui.chat.ChatSheetContent
|
||||
|
||||
@Composable
|
||||
fun ChatSheet(viewModel: MainViewModel) {
|
||||
ChatSheetContent(viewModel = viewModel)
|
||||
}
|
||||
@@ -0,0 +1,493 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import androidx.compose.animation.AnimatedVisibility
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.PaddingValues
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.foundation.text.KeyboardOptions
|
||||
import androidx.compose.foundation.verticalScroll
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.filled.ExpandLess
|
||||
import androidx.compose.material.icons.filled.ExpandMore
|
||||
import androidx.compose.material3.AlertDialog
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.ButtonDefaults
|
||||
import androidx.compose.material3.HorizontalDivider
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.OutlinedTextField
|
||||
import androidx.compose.material3.OutlinedTextFieldDefaults
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.SwitchDefaults
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.material3.TextButton
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.collectAsState
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.saveable.rememberSaveable
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.text.font.FontFamily
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.text.input.KeyboardType
|
||||
import androidx.compose.ui.unit.dp
|
||||
import ai.openclaw.android.MainViewModel
|
||||
|
||||
private enum class ConnectInputMode {
|
||||
SetupCode,
|
||||
Manual,
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun ConnectTabScreen(viewModel: MainViewModel) {
|
||||
val statusText by viewModel.statusText.collectAsState()
|
||||
val isConnected by viewModel.isConnected.collectAsState()
|
||||
val remoteAddress by viewModel.remoteAddress.collectAsState()
|
||||
val manualHost by viewModel.manualHost.collectAsState()
|
||||
val manualPort by viewModel.manualPort.collectAsState()
|
||||
val manualTls by viewModel.manualTls.collectAsState()
|
||||
val manualEnabled by viewModel.manualEnabled.collectAsState()
|
||||
val gatewayToken by viewModel.gatewayToken.collectAsState()
|
||||
val pendingTrust by viewModel.pendingGatewayTrust.collectAsState()
|
||||
|
||||
var advancedOpen by rememberSaveable { mutableStateOf(false) }
|
||||
var inputMode by
|
||||
remember(manualEnabled, manualHost, gatewayToken) {
|
||||
mutableStateOf(
|
||||
if (manualEnabled || manualHost.isNotBlank() || gatewayToken.trim().isNotEmpty()) {
|
||||
ConnectInputMode.Manual
|
||||
} else {
|
||||
ConnectInputMode.SetupCode
|
||||
},
|
||||
)
|
||||
}
|
||||
var setupCode by rememberSaveable { mutableStateOf("") }
|
||||
var manualHostInput by rememberSaveable { mutableStateOf(manualHost.ifBlank { "10.0.2.2" }) }
|
||||
var manualPortInput by rememberSaveable { mutableStateOf(manualPort.toString()) }
|
||||
var manualTlsInput by rememberSaveable { mutableStateOf(manualTls) }
|
||||
var passwordInput by rememberSaveable { mutableStateOf("") }
|
||||
var validationText by rememberSaveable { mutableStateOf<String?>(null) }
|
||||
|
||||
if (pendingTrust != null) {
|
||||
val prompt = pendingTrust!!
|
||||
AlertDialog(
|
||||
onDismissRequest = { viewModel.declineGatewayTrustPrompt() },
|
||||
title = { Text("Trust this gateway?") },
|
||||
text = {
|
||||
Text(
|
||||
"First-time TLS connection.\n\nVerify this SHA-256 fingerprint before trusting:\n${prompt.fingerprintSha256}",
|
||||
style = mobileCallout,
|
||||
)
|
||||
},
|
||||
confirmButton = {
|
||||
TextButton(onClick = { viewModel.acceptGatewayTrustPrompt() }) {
|
||||
Text("Trust and continue")
|
||||
}
|
||||
},
|
||||
dismissButton = {
|
||||
TextButton(onClick = { viewModel.declineGatewayTrustPrompt() }) {
|
||||
Text("Cancel")
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
val setupResolvedEndpoint = remember(setupCode) { decodeGatewaySetupCode(setupCode)?.url?.let { parseGatewayEndpoint(it)?.displayUrl } }
|
||||
val manualResolvedEndpoint = remember(manualHostInput, manualPortInput, manualTlsInput) {
|
||||
composeGatewayManualUrl(manualHostInput, manualPortInput, manualTlsInput)?.let { parseGatewayEndpoint(it)?.displayUrl }
|
||||
}
|
||||
|
||||
val activeEndpoint =
|
||||
remember(isConnected, remoteAddress, setupResolvedEndpoint, manualResolvedEndpoint, inputMode) {
|
||||
when {
|
||||
isConnected && !remoteAddress.isNullOrBlank() -> remoteAddress!!
|
||||
inputMode == ConnectInputMode.SetupCode -> setupResolvedEndpoint ?: "Not set"
|
||||
else -> manualResolvedEndpoint ?: "Not set"
|
||||
}
|
||||
}
|
||||
|
||||
val primaryLabel = if (isConnected) "Disconnect Gateway" else "Connect Gateway"
|
||||
|
||||
Column(
|
||||
modifier = Modifier.verticalScroll(rememberScrollState()).padding(horizontal = 20.dp, vertical = 16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(14.dp),
|
||||
) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(6.dp)) {
|
||||
Text("Connection Control", style = mobileCaption1.copy(fontWeight = FontWeight.Bold), color = mobileAccent)
|
||||
Text("Gateway Connection", style = mobileTitle1, color = mobileText)
|
||||
Text(
|
||||
"One primary action. Open advanced controls only when needed.",
|
||||
style = mobileCallout,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
color = mobileSurface,
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
) {
|
||||
Column(modifier = Modifier.padding(horizontal = 14.dp, vertical = 12.dp), verticalArrangement = Arrangement.spacedBy(4.dp)) {
|
||||
Text("Active endpoint", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
|
||||
Text(activeEndpoint, style = mobileBody.copy(fontFamily = FontFamily.Monospace), color = mobileText)
|
||||
}
|
||||
}
|
||||
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
color = mobileSurface,
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
) {
|
||||
Column(modifier = Modifier.padding(horizontal = 14.dp, vertical = 12.dp), verticalArrangement = Arrangement.spacedBy(4.dp)) {
|
||||
Text("Gateway state", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
|
||||
Text(statusText, style = mobileBody, color = mobileText)
|
||||
}
|
||||
}
|
||||
|
||||
Button(
|
||||
onClick = {
|
||||
if (isConnected) {
|
||||
viewModel.disconnect()
|
||||
validationText = null
|
||||
return@Button
|
||||
}
|
||||
if (statusText.contains("operator offline", ignoreCase = true)) {
|
||||
validationText = null
|
||||
viewModel.refreshGatewayConnection()
|
||||
return@Button
|
||||
}
|
||||
|
||||
val config =
|
||||
resolveGatewayConnectConfig(
|
||||
useSetupCode = inputMode == ConnectInputMode.SetupCode,
|
||||
setupCode = setupCode,
|
||||
manualHost = manualHostInput,
|
||||
manualPort = manualPortInput,
|
||||
manualTls = manualTlsInput,
|
||||
fallbackToken = gatewayToken,
|
||||
fallbackPassword = passwordInput,
|
||||
)
|
||||
|
||||
if (config == null) {
|
||||
validationText =
|
||||
if (inputMode == ConnectInputMode.SetupCode) {
|
||||
"Paste a valid setup code to connect."
|
||||
} else {
|
||||
"Enter a valid manual host and port to connect."
|
||||
}
|
||||
return@Button
|
||||
}
|
||||
|
||||
validationText = null
|
||||
viewModel.setManualEnabled(true)
|
||||
viewModel.setManualHost(config.host)
|
||||
viewModel.setManualPort(config.port)
|
||||
viewModel.setManualTls(config.tls)
|
||||
if (config.token.isNotBlank()) {
|
||||
viewModel.setGatewayToken(config.token)
|
||||
}
|
||||
viewModel.setGatewayPassword(config.password)
|
||||
viewModel.connectManual()
|
||||
},
|
||||
modifier = Modifier.fillMaxWidth().height(52.dp),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors =
|
||||
ButtonDefaults.buttonColors(
|
||||
containerColor = if (isConnected) mobileDanger else mobileAccent,
|
||||
contentColor = Color.White,
|
||||
),
|
||||
) {
|
||||
Text(primaryLabel, style = mobileHeadline.copy(fontWeight = FontWeight.Bold))
|
||||
}
|
||||
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
color = mobileSurface,
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
onClick = { advancedOpen = !advancedOpen },
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 14.dp, vertical = 12.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(2.dp)) {
|
||||
Text("Advanced controls", style = mobileHeadline, color = mobileText)
|
||||
Text("Setup code, endpoint, TLS, token, password, onboarding.", style = mobileCaption1, color = mobileTextSecondary)
|
||||
}
|
||||
Icon(
|
||||
imageVector = if (advancedOpen) Icons.Default.ExpandLess else Icons.Default.ExpandMore,
|
||||
contentDescription = if (advancedOpen) "Collapse advanced controls" else "Expand advanced controls",
|
||||
tint = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
AnimatedVisibility(visible = advancedOpen) {
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
color = Color.White,
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 14.dp, vertical = 14.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(12.dp),
|
||||
) {
|
||||
Text("Connection method", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
MethodChip(
|
||||
label = "Setup Code",
|
||||
active = inputMode == ConnectInputMode.SetupCode,
|
||||
onClick = { inputMode = ConnectInputMode.SetupCode },
|
||||
)
|
||||
MethodChip(
|
||||
label = "Manual",
|
||||
active = inputMode == ConnectInputMode.Manual,
|
||||
onClick = { inputMode = ConnectInputMode.Manual },
|
||||
)
|
||||
}
|
||||
|
||||
Text("Run these on the gateway host:", style = mobileCallout, color = mobileTextSecondary)
|
||||
CommandBlock("openclaw qr --setup-code-only")
|
||||
CommandBlock("openclaw qr --json")
|
||||
|
||||
if (inputMode == ConnectInputMode.SetupCode) {
|
||||
Text("Setup Code", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
|
||||
OutlinedTextField(
|
||||
value = setupCode,
|
||||
onValueChange = {
|
||||
setupCode = it
|
||||
validationText = null
|
||||
},
|
||||
placeholder = { Text("Paste setup code", style = mobileBody, color = mobileTextTertiary) },
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
minLines = 3,
|
||||
maxLines = 5,
|
||||
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Ascii),
|
||||
textStyle = mobileBody.copy(fontFamily = FontFamily.Monospace, color = mobileText),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors = outlinedColors(),
|
||||
)
|
||||
if (!setupResolvedEndpoint.isNullOrBlank()) {
|
||||
EndpointPreview(endpoint = setupResolvedEndpoint)
|
||||
}
|
||||
} else {
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
QuickFillChip(
|
||||
label = "Android Emulator",
|
||||
onClick = {
|
||||
manualHostInput = "10.0.2.2"
|
||||
manualPortInput = "18789"
|
||||
manualTlsInput = false
|
||||
validationText = null
|
||||
},
|
||||
)
|
||||
QuickFillChip(
|
||||
label = "Localhost",
|
||||
onClick = {
|
||||
manualHostInput = "127.0.0.1"
|
||||
manualPortInput = "18789"
|
||||
manualTlsInput = false
|
||||
validationText = null
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Text("Host", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
|
||||
OutlinedTextField(
|
||||
value = manualHostInput,
|
||||
onValueChange = {
|
||||
manualHostInput = it
|
||||
validationText = null
|
||||
},
|
||||
placeholder = { Text("10.0.2.2", style = mobileBody, color = mobileTextTertiary) },
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
singleLine = true,
|
||||
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Uri),
|
||||
textStyle = mobileBody.copy(color = mobileText),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors = outlinedColors(),
|
||||
)
|
||||
|
||||
Text("Port", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
|
||||
OutlinedTextField(
|
||||
value = manualPortInput,
|
||||
onValueChange = {
|
||||
manualPortInput = it
|
||||
validationText = null
|
||||
},
|
||||
placeholder = { Text("18789", style = mobileBody, color = mobileTextTertiary) },
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
singleLine = true,
|
||||
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number),
|
||||
textStyle = mobileBody.copy(fontFamily = FontFamily.Monospace, color = mobileText),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors = outlinedColors(),
|
||||
)
|
||||
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(2.dp)) {
|
||||
Text("Use TLS", style = mobileHeadline, color = mobileText)
|
||||
Text("Switch to secure websocket (`wss`).", style = mobileCallout, color = mobileTextSecondary)
|
||||
}
|
||||
Switch(
|
||||
checked = manualTlsInput,
|
||||
onCheckedChange = {
|
||||
manualTlsInput = it
|
||||
validationText = null
|
||||
},
|
||||
colors =
|
||||
SwitchDefaults.colors(
|
||||
checkedTrackColor = mobileAccent,
|
||||
uncheckedTrackColor = mobileBorderStrong,
|
||||
checkedThumbColor = Color.White,
|
||||
uncheckedThumbColor = Color.White,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
Text("Token (optional)", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
|
||||
OutlinedTextField(
|
||||
value = gatewayToken,
|
||||
onValueChange = { viewModel.setGatewayToken(it) },
|
||||
placeholder = { Text("token", style = mobileBody, color = mobileTextTertiary) },
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
singleLine = true,
|
||||
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Ascii),
|
||||
textStyle = mobileBody.copy(color = mobileText),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors = outlinedColors(),
|
||||
)
|
||||
|
||||
Text("Password (optional)", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
|
||||
OutlinedTextField(
|
||||
value = passwordInput,
|
||||
onValueChange = { passwordInput = it },
|
||||
placeholder = { Text("password", style = mobileBody, color = mobileTextTertiary) },
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
singleLine = true,
|
||||
keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Ascii),
|
||||
textStyle = mobileBody.copy(color = mobileText),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors = outlinedColors(),
|
||||
)
|
||||
|
||||
if (!manualResolvedEndpoint.isNullOrBlank()) {
|
||||
EndpointPreview(endpoint = manualResolvedEndpoint)
|
||||
}
|
||||
}
|
||||
|
||||
HorizontalDivider(color = mobileBorder)
|
||||
|
||||
TextButton(onClick = { viewModel.setOnboardingCompleted(false) }) {
|
||||
Text("Run onboarding again", style = mobileCallout.copy(fontWeight = FontWeight.SemiBold), color = mobileAccent)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!validationText.isNullOrBlank()) {
|
||||
Text(validationText!!, style = mobileCaption1, color = mobileWarning)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun MethodChip(label: String, active: Boolean, onClick: () -> Unit) {
|
||||
Button(
|
||||
onClick = onClick,
|
||||
modifier = Modifier.height(40.dp),
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
contentPadding = PaddingValues(horizontal = 12.dp, vertical = 8.dp),
|
||||
colors =
|
||||
ButtonDefaults.buttonColors(
|
||||
containerColor = if (active) mobileAccent else mobileSurface,
|
||||
contentColor = if (active) Color.White else mobileText,
|
||||
),
|
||||
border = BorderStroke(1.dp, if (active) Color(0xFF184DAF) else mobileBorderStrong),
|
||||
) {
|
||||
Text(label, style = mobileCaption1.copy(fontWeight = FontWeight.Bold))
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun QuickFillChip(label: String, onClick: () -> Unit) {
|
||||
Button(
|
||||
onClick = onClick,
|
||||
shape = RoundedCornerShape(999.dp),
|
||||
contentPadding = PaddingValues(horizontal = 12.dp, vertical = 6.dp),
|
||||
colors =
|
||||
ButtonDefaults.buttonColors(
|
||||
containerColor = mobileAccentSoft,
|
||||
contentColor = mobileAccent,
|
||||
),
|
||||
elevation = null,
|
||||
) {
|
||||
Text(label, style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold))
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun CommandBlock(command: String) {
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
color = mobileCodeBg,
|
||||
border = BorderStroke(1.dp, Color(0xFF2B2E35)),
|
||||
) {
|
||||
Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) {
|
||||
Box(modifier = Modifier.width(3.dp).height(42.dp).background(Color(0xFF3FC97A)))
|
||||
Text(
|
||||
text = command,
|
||||
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
|
||||
style = mobileCallout.copy(fontFamily = FontFamily.Monospace),
|
||||
color = mobileCodeText,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun EndpointPreview(endpoint: String) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(4.dp)) {
|
||||
HorizontalDivider(color = mobileBorder)
|
||||
Text("Resolved endpoint", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
|
||||
Text(endpoint, style = mobileCallout.copy(fontFamily = FontFamily.Monospace), color = mobileText)
|
||||
HorizontalDivider(color = mobileBorder)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun outlinedColors() =
|
||||
OutlinedTextFieldDefaults.colors(
|
||||
focusedContainerColor = mobileSurface,
|
||||
unfocusedContainerColor = mobileSurface,
|
||||
focusedBorderColor = mobileAccent,
|
||||
unfocusedBorderColor = mobileBorder,
|
||||
focusedTextColor = mobileText,
|
||||
unfocusedTextColor = mobileText,
|
||||
cursorColor = mobileAccent,
|
||||
)
|
||||
@@ -0,0 +1,142 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import androidx.core.net.toUri
|
||||
import java.util.Base64
|
||||
import java.util.Locale
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.contentOrNull
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
|
||||
internal data class GatewayEndpointConfig(
|
||||
val host: String,
|
||||
val port: Int,
|
||||
val tls: Boolean,
|
||||
val displayUrl: String,
|
||||
)
|
||||
|
||||
internal data class GatewaySetupCode(
|
||||
val url: String,
|
||||
val token: String?,
|
||||
val password: String?,
|
||||
)
|
||||
|
||||
internal data class GatewayConnectConfig(
|
||||
val host: String,
|
||||
val port: Int,
|
||||
val tls: Boolean,
|
||||
val token: String,
|
||||
val password: String,
|
||||
)
|
||||
|
||||
private val gatewaySetupJson = Json { ignoreUnknownKeys = true }
|
||||
|
||||
internal fun resolveGatewayConnectConfig(
|
||||
useSetupCode: Boolean,
|
||||
setupCode: String,
|
||||
manualHost: String,
|
||||
manualPort: String,
|
||||
manualTls: Boolean,
|
||||
fallbackToken: String,
|
||||
fallbackPassword: String,
|
||||
): GatewayConnectConfig? {
|
||||
if (useSetupCode) {
|
||||
val setup = decodeGatewaySetupCode(setupCode) ?: return null
|
||||
val parsed = parseGatewayEndpoint(setup.url) ?: return null
|
||||
return GatewayConnectConfig(
|
||||
host = parsed.host,
|
||||
port = parsed.port,
|
||||
tls = parsed.tls,
|
||||
token = setup.token ?: fallbackToken.trim(),
|
||||
password = setup.password ?: fallbackPassword.trim(),
|
||||
)
|
||||
}
|
||||
|
||||
val manualUrl = composeGatewayManualUrl(manualHost, manualPort, manualTls) ?: return null
|
||||
val parsed = parseGatewayEndpoint(manualUrl) ?: return null
|
||||
return GatewayConnectConfig(
|
||||
host = parsed.host,
|
||||
port = parsed.port,
|
||||
tls = parsed.tls,
|
||||
token = fallbackToken.trim(),
|
||||
password = fallbackPassword.trim(),
|
||||
)
|
||||
}
|
||||
|
||||
internal fun parseGatewayEndpoint(rawInput: String): GatewayEndpointConfig? {
|
||||
val raw = rawInput.trim()
|
||||
if (raw.isEmpty()) return null
|
||||
|
||||
val normalized = if (raw.contains("://")) raw else "https://$raw"
|
||||
val uri = normalized.toUri()
|
||||
val host = uri.host?.trim().orEmpty()
|
||||
if (host.isEmpty()) return null
|
||||
|
||||
val scheme = uri.scheme?.trim()?.lowercase(Locale.US).orEmpty()
|
||||
val tls =
|
||||
when (scheme) {
|
||||
"ws", "http" -> false
|
||||
"wss", "https" -> true
|
||||
else -> true
|
||||
}
|
||||
val port = uri.port.takeIf { it in 1..65535 } ?: 18789
|
||||
val displayUrl = "${if (tls) "https" else "http"}://$host:$port"
|
||||
|
||||
return GatewayEndpointConfig(host = host, port = port, tls = tls, displayUrl = displayUrl)
|
||||
}
|
||||
|
||||
internal fun decodeGatewaySetupCode(rawInput: String): GatewaySetupCode? {
|
||||
val trimmed = rawInput.trim()
|
||||
if (trimmed.isEmpty()) return null
|
||||
|
||||
val padded =
|
||||
trimmed
|
||||
.replace('-', '+')
|
||||
.replace('_', '/')
|
||||
.let { normalized ->
|
||||
val remainder = normalized.length % 4
|
||||
if (remainder == 0) normalized else normalized + "=".repeat(4 - remainder)
|
||||
}
|
||||
|
||||
return try {
|
||||
val decoded = String(Base64.getDecoder().decode(padded), Charsets.UTF_8)
|
||||
val obj = parseJsonObject(decoded) ?: return null
|
||||
val url = jsonField(obj, "url").orEmpty()
|
||||
if (url.isEmpty()) return null
|
||||
val token = jsonField(obj, "token")
|
||||
val password = jsonField(obj, "password")
|
||||
GatewaySetupCode(url = url, token = token, password = password)
|
||||
} catch (_: IllegalArgumentException) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
internal fun resolveScannedSetupCode(rawInput: String): String? {
|
||||
val setupCode = resolveSetupCodeCandidate(rawInput) ?: return null
|
||||
return setupCode.takeIf { decodeGatewaySetupCode(it) != null }
|
||||
}
|
||||
|
||||
internal fun composeGatewayManualUrl(hostInput: String, portInput: String, tls: Boolean): String? {
|
||||
val host = hostInput.trim()
|
||||
val port = portInput.trim().toIntOrNull() ?: return null
|
||||
if (host.isEmpty() || port !in 1..65535) return null
|
||||
val scheme = if (tls) "https" else "http"
|
||||
return "$scheme://$host:$port"
|
||||
}
|
||||
|
||||
private fun parseJsonObject(input: String): JsonObject? {
|
||||
return runCatching { gatewaySetupJson.parseToJsonElement(input).jsonObject }.getOrNull()
|
||||
}
|
||||
|
||||
private fun resolveSetupCodeCandidate(rawInput: String): String? {
|
||||
val trimmed = rawInput.trim()
|
||||
if (trimmed.isEmpty()) return null
|
||||
val qrSetupCode = parseJsonObject(trimmed)?.let { jsonField(it, "setupCode") }
|
||||
return qrSetupCode ?: trimmed
|
||||
}
|
||||
|
||||
private fun jsonField(obj: JsonObject, key: String): String? {
|
||||
val value = (obj[key] as? JsonPrimitive)?.contentOrNull?.trim().orEmpty()
|
||||
return value.ifEmpty { null }
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import androidx.compose.ui.graphics.Brush
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.text.TextStyle
|
||||
import androidx.compose.ui.text.font.Font
|
||||
import androidx.compose.ui.text.font.FontFamily
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.unit.sp
|
||||
import ai.openclaw.android.R
|
||||
|
||||
internal val mobileBackgroundGradient =
|
||||
Brush.verticalGradient(
|
||||
listOf(
|
||||
Color(0xFFFFFFFF),
|
||||
Color(0xFFF7F8FA),
|
||||
Color(0xFFEFF1F5),
|
||||
),
|
||||
)
|
||||
|
||||
internal val mobileSurface = Color(0xFFF6F7FA)
|
||||
internal val mobileSurfaceStrong = Color(0xFFECEEF3)
|
||||
internal val mobileBorder = Color(0xFFE5E7EC)
|
||||
internal val mobileBorderStrong = Color(0xFFD6DAE2)
|
||||
internal val mobileText = Color(0xFF17181C)
|
||||
internal val mobileTextSecondary = Color(0xFF5D6472)
|
||||
internal val mobileTextTertiary = Color(0xFF99A0AE)
|
||||
internal val mobileAccent = Color(0xFF1D5DD8)
|
||||
internal val mobileAccentSoft = Color(0xFFECF3FF)
|
||||
internal val mobileSuccess = Color(0xFF2F8C5A)
|
||||
internal val mobileSuccessSoft = Color(0xFFEEF9F3)
|
||||
internal val mobileWarning = Color(0xFFC8841A)
|
||||
internal val mobileWarningSoft = Color(0xFFFFF8EC)
|
||||
internal val mobileDanger = Color(0xFFD04B4B)
|
||||
internal val mobileDangerSoft = Color(0xFFFFF2F2)
|
||||
internal val mobileCodeBg = Color(0xFF15171B)
|
||||
internal val mobileCodeText = Color(0xFFE8EAEE)
|
||||
|
||||
internal val mobileFontFamily =
|
||||
FontFamily(
|
||||
Font(resId = R.font.manrope_400_regular, weight = FontWeight.Normal),
|
||||
Font(resId = R.font.manrope_500_medium, weight = FontWeight.Medium),
|
||||
Font(resId = R.font.manrope_600_semibold, weight = FontWeight.SemiBold),
|
||||
Font(resId = R.font.manrope_700_bold, weight = FontWeight.Bold),
|
||||
)
|
||||
|
||||
internal val mobileTitle1 =
|
||||
TextStyle(
|
||||
fontFamily = mobileFontFamily,
|
||||
fontWeight = FontWeight.SemiBold,
|
||||
fontSize = 24.sp,
|
||||
lineHeight = 30.sp,
|
||||
letterSpacing = (-0.5).sp,
|
||||
)
|
||||
|
||||
internal val mobileTitle2 =
|
||||
TextStyle(
|
||||
fontFamily = mobileFontFamily,
|
||||
fontWeight = FontWeight.SemiBold,
|
||||
fontSize = 20.sp,
|
||||
lineHeight = 26.sp,
|
||||
letterSpacing = (-0.3).sp,
|
||||
)
|
||||
|
||||
internal val mobileHeadline =
|
||||
TextStyle(
|
||||
fontFamily = mobileFontFamily,
|
||||
fontWeight = FontWeight.SemiBold,
|
||||
fontSize = 16.sp,
|
||||
lineHeight = 22.sp,
|
||||
letterSpacing = (-0.1).sp,
|
||||
)
|
||||
|
||||
internal val mobileBody =
|
||||
TextStyle(
|
||||
fontFamily = mobileFontFamily,
|
||||
fontWeight = FontWeight.Medium,
|
||||
fontSize = 15.sp,
|
||||
lineHeight = 22.sp,
|
||||
)
|
||||
|
||||
internal val mobileCallout =
|
||||
TextStyle(
|
||||
fontFamily = mobileFontFamily,
|
||||
fontWeight = FontWeight.Medium,
|
||||
fontSize = 14.sp,
|
||||
lineHeight = 20.sp,
|
||||
)
|
||||
|
||||
internal val mobileCaption1 =
|
||||
TextStyle(
|
||||
fontFamily = mobileFontFamily,
|
||||
fontWeight = FontWeight.Medium,
|
||||
fontSize = 12.sp,
|
||||
lineHeight = 16.sp,
|
||||
letterSpacing = 0.2.sp,
|
||||
)
|
||||
|
||||
internal val mobileCaption2 =
|
||||
TextStyle(
|
||||
fontFamily = mobileFontFamily,
|
||||
fontWeight = FontWeight.Medium,
|
||||
fontSize = 11.sp,
|
||||
lineHeight = 14.sp,
|
||||
letterSpacing = 0.4.sp,
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import androidx.compose.foundation.isSystemInDarkTheme
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.dynamicDarkColorScheme
|
||||
import androidx.compose.material3.dynamicLightColorScheme
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
|
||||
@Composable
|
||||
fun OpenClawTheme(content: @Composable () -> Unit) {
|
||||
val context = LocalContext.current
|
||||
val isDark = isSystemInDarkTheme()
|
||||
val colorScheme = if (isDark) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)
|
||||
|
||||
MaterialTheme(colorScheme = colorScheme, content = content)
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun overlayContainerColor(): Color {
|
||||
val scheme = MaterialTheme.colorScheme
|
||||
val isDark = isSystemInDarkTheme()
|
||||
val base = if (isDark) scheme.surfaceContainerLow else scheme.surfaceContainerHigh
|
||||
// Light mode: background stays dark (canvas), so clamp overlays away from pure-white glare.
|
||||
return if (isDark) base else base.copy(alpha = 0.88f)
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun overlayIconColor(): Color {
|
||||
return MaterialTheme.colorScheme.onSurfaceVariant
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.WindowInsets
|
||||
import androidx.compose.foundation.layout.WindowInsetsSides
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.heightIn
|
||||
import androidx.compose.foundation.layout.ime
|
||||
import androidx.compose.foundation.layout.navigationBars
|
||||
import androidx.compose.foundation.layout.only
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.safeDrawing
|
||||
import androidx.compose.foundation.layout.windowInsetsPadding
|
||||
import androidx.compose.foundation.layout.consumeWindowInsets
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.automirrored.filled.ScreenShare
|
||||
import androidx.compose.material.icons.filled.ChatBubble
|
||||
import androidx.compose.material.icons.filled.CheckCircle
|
||||
import androidx.compose.material.icons.filled.RecordVoiceOver
|
||||
import androidx.compose.material.icons.filled.Settings
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.Scaffold
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.collectAsState
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.saveable.rememberSaveable
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.vector.ImageVector
|
||||
import androidx.compose.ui.platform.LocalDensity
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.unit.dp
|
||||
import ai.openclaw.android.MainViewModel
|
||||
|
||||
private enum class HomeTab(
|
||||
val label: String,
|
||||
val icon: ImageVector,
|
||||
) {
|
||||
Connect(label = "Connect", icon = Icons.Default.CheckCircle),
|
||||
Chat(label = "Chat", icon = Icons.Default.ChatBubble),
|
||||
Voice(label = "Voice", icon = Icons.Default.RecordVoiceOver),
|
||||
Screen(label = "Screen", icon = Icons.AutoMirrored.Filled.ScreenShare),
|
||||
Settings(label = "Settings", icon = Icons.Default.Settings),
|
||||
}
|
||||
|
||||
private enum class StatusVisual {
|
||||
Connected,
|
||||
Connecting,
|
||||
Warning,
|
||||
Error,
|
||||
Offline,
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun PostOnboardingTabs(viewModel: MainViewModel, modifier: Modifier = Modifier) {
|
||||
var activeTab by rememberSaveable { mutableStateOf(HomeTab.Connect) }
|
||||
|
||||
val statusText by viewModel.statusText.collectAsState()
|
||||
val isConnected by viewModel.isConnected.collectAsState()
|
||||
|
||||
val statusVisual =
|
||||
remember(statusText, isConnected) {
|
||||
val lower = statusText.lowercase()
|
||||
when {
|
||||
isConnected -> StatusVisual.Connected
|
||||
lower.contains("connecting") || lower.contains("reconnecting") -> StatusVisual.Connecting
|
||||
lower.contains("pairing") || lower.contains("approval") || lower.contains("auth") -> StatusVisual.Warning
|
||||
lower.contains("error") || lower.contains("failed") -> StatusVisual.Error
|
||||
else -> StatusVisual.Offline
|
||||
}
|
||||
}
|
||||
|
||||
val density = LocalDensity.current
|
||||
val imeVisible = WindowInsets.ime.getBottom(density) > 0
|
||||
val hideBottomTabBar = activeTab == HomeTab.Chat && imeVisible
|
||||
|
||||
Scaffold(
|
||||
modifier = modifier,
|
||||
containerColor = Color.Transparent,
|
||||
contentWindowInsets = WindowInsets(0, 0, 0, 0),
|
||||
topBar = {
|
||||
TopStatusBar(
|
||||
statusText = statusText,
|
||||
statusVisual = statusVisual,
|
||||
)
|
||||
},
|
||||
bottomBar = {
|
||||
if (!hideBottomTabBar) {
|
||||
BottomTabBar(
|
||||
activeTab = activeTab,
|
||||
onSelect = { activeTab = it },
|
||||
)
|
||||
}
|
||||
},
|
||||
) { innerPadding ->
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxSize()
|
||||
.padding(innerPadding)
|
||||
.consumeWindowInsets(innerPadding)
|
||||
.background(mobileBackgroundGradient),
|
||||
) {
|
||||
when (activeTab) {
|
||||
HomeTab.Connect -> ConnectTabScreen(viewModel = viewModel)
|
||||
HomeTab.Chat -> ChatSheet(viewModel = viewModel)
|
||||
HomeTab.Voice -> VoiceTabScreen(viewModel = viewModel)
|
||||
HomeTab.Screen -> ScreenTabScreen(viewModel = viewModel)
|
||||
HomeTab.Settings -> SettingsSheet(viewModel = viewModel)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ScreenTabScreen(viewModel: MainViewModel) {
|
||||
val isConnected by viewModel.isConnected.collectAsState()
|
||||
val isNodeConnected by viewModel.isNodeConnected.collectAsState()
|
||||
val canvasUrl by viewModel.canvasCurrentUrl.collectAsState()
|
||||
val canvasA2uiHydrated by viewModel.canvasA2uiHydrated.collectAsState()
|
||||
val canvasRehydratePending by viewModel.canvasRehydratePending.collectAsState()
|
||||
val canvasRehydrateErrorText by viewModel.canvasRehydrateErrorText.collectAsState()
|
||||
val isA2uiUrl = canvasUrl?.contains("/__openclaw__/a2ui/") == true
|
||||
val showRestoreCta = isConnected && isNodeConnected && (canvasUrl.isNullOrBlank() || (isA2uiUrl && !canvasA2uiHydrated))
|
||||
val restoreCtaText =
|
||||
when {
|
||||
canvasRehydratePending -> "Restore requested. Waiting for agent…"
|
||||
!canvasRehydrateErrorText.isNullOrBlank() -> canvasRehydrateErrorText!!
|
||||
else -> "Canvas reset. Tap to restore dashboard."
|
||||
}
|
||||
|
||||
Box(modifier = Modifier.fillMaxSize()) {
|
||||
CanvasScreen(viewModel = viewModel, modifier = Modifier.fillMaxSize())
|
||||
|
||||
if (showRestoreCta) {
|
||||
Surface(
|
||||
onClick = {
|
||||
if (canvasRehydratePending) return@Surface
|
||||
viewModel.requestCanvasRehydrate(source = "screen_tab_cta")
|
||||
},
|
||||
modifier = Modifier.align(Alignment.TopCenter).padding(horizontal = 16.dp, vertical = 16.dp),
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
color = mobileSurface.copy(alpha = 0.9f),
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
shadowElevation = 4.dp,
|
||||
) {
|
||||
Text(
|
||||
text = restoreCtaText,
|
||||
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
|
||||
style = mobileCallout.copy(fontWeight = FontWeight.Medium),
|
||||
color = mobileText,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun TopStatusBar(
|
||||
statusText: String,
|
||||
statusVisual: StatusVisual,
|
||||
) {
|
||||
val safeInsets = WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal)
|
||||
|
||||
val (chipBg, chipDot, chipText, chipBorder) =
|
||||
when (statusVisual) {
|
||||
StatusVisual.Connected ->
|
||||
listOf(
|
||||
mobileSuccessSoft,
|
||||
mobileSuccess,
|
||||
mobileSuccess,
|
||||
Color(0xFFCFEBD8),
|
||||
)
|
||||
StatusVisual.Connecting ->
|
||||
listOf(
|
||||
mobileAccentSoft,
|
||||
mobileAccent,
|
||||
mobileAccent,
|
||||
Color(0xFFD5E2FA),
|
||||
)
|
||||
StatusVisual.Warning ->
|
||||
listOf(
|
||||
mobileWarningSoft,
|
||||
mobileWarning,
|
||||
mobileWarning,
|
||||
Color(0xFFEED8B8),
|
||||
)
|
||||
StatusVisual.Error ->
|
||||
listOf(
|
||||
mobileDangerSoft,
|
||||
mobileDanger,
|
||||
mobileDanger,
|
||||
Color(0xFFF3C8C8),
|
||||
)
|
||||
StatusVisual.Offline ->
|
||||
listOf(
|
||||
mobileSurface,
|
||||
mobileTextTertiary,
|
||||
mobileTextSecondary,
|
||||
mobileBorder,
|
||||
)
|
||||
}
|
||||
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth().windowInsetsPadding(safeInsets),
|
||||
color = Color.Transparent,
|
||||
shadowElevation = 0.dp,
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 18.dp, vertical = 12.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
) {
|
||||
Text(
|
||||
text = "OpenClaw",
|
||||
style = mobileTitle2,
|
||||
color = mobileText,
|
||||
)
|
||||
Surface(
|
||||
shape = RoundedCornerShape(999.dp),
|
||||
color = chipBg,
|
||||
border = androidx.compose.foundation.BorderStroke(1.dp, chipBorder),
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.padding(horizontal = 10.dp, vertical = 5.dp),
|
||||
horizontalArrangement = Arrangement.spacedBy(6.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
Surface(
|
||||
modifier = Modifier.padding(top = 1.dp),
|
||||
color = chipDot,
|
||||
shape = RoundedCornerShape(999.dp),
|
||||
) {
|
||||
Box(modifier = Modifier.padding(4.dp))
|
||||
}
|
||||
Text(
|
||||
text = statusText.trim().ifEmpty { "Offline" },
|
||||
style = mobileCaption1,
|
||||
color = chipText,
|
||||
maxLines = 1,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun BottomTabBar(
|
||||
activeTab: HomeTab,
|
||||
onSelect: (HomeTab) -> Unit,
|
||||
) {
|
||||
val safeInsets = WindowInsets.navigationBars.only(WindowInsetsSides.Bottom + WindowInsetsSides.Horizontal)
|
||||
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxWidth(),
|
||||
) {
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
color = Color.White.copy(alpha = 0.97f),
|
||||
shape = RoundedCornerShape(topStart = 24.dp, topEnd = 24.dp),
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
shadowElevation = 6.dp,
|
||||
) {
|
||||
Row(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxWidth()
|
||||
.windowInsetsPadding(safeInsets)
|
||||
.padding(horizontal = 10.dp, vertical = 10.dp),
|
||||
horizontalArrangement = Arrangement.spacedBy(6.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
HomeTab.entries.forEach { tab ->
|
||||
val active = tab == activeTab
|
||||
Surface(
|
||||
onClick = { onSelect(tab) },
|
||||
modifier = Modifier.weight(1f).heightIn(min = 58.dp),
|
||||
shape = RoundedCornerShape(16.dp),
|
||||
color = if (active) mobileAccentSoft else Color.Transparent,
|
||||
border = if (active) BorderStroke(1.dp, Color(0xFFD5E2FA)) else null,
|
||||
shadowElevation = 0.dp,
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 6.dp, vertical = 7.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally,
|
||||
verticalArrangement = Arrangement.spacedBy(2.dp),
|
||||
) {
|
||||
Icon(
|
||||
imageVector = tab.icon,
|
||||
contentDescription = tab.label,
|
||||
tint = if (active) mobileAccent else mobileTextTertiary,
|
||||
)
|
||||
Text(
|
||||
text = tab.label,
|
||||
color = if (active) mobileAccent else mobileTextSecondary,
|
||||
style = mobileCaption2.copy(fontWeight = if (active) FontWeight.Bold else FontWeight.Medium),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.collectAsState
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.ui.Modifier
|
||||
import ai.openclaw.android.MainViewModel
|
||||
|
||||
@Composable
|
||||
fun RootScreen(viewModel: MainViewModel) {
|
||||
val onboardingCompleted by viewModel.onboardingCompleted.collectAsState()
|
||||
|
||||
if (!onboardingCompleted) {
|
||||
OnboardingFlow(viewModel = viewModel, modifier = Modifier.fillMaxSize())
|
||||
return
|
||||
}
|
||||
|
||||
PostOnboardingTabs(viewModel = viewModel, modifier = Modifier.fillMaxSize())
|
||||
}
|
||||
@@ -0,0 +1,605 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.net.Uri
|
||||
import android.os.Build
|
||||
import android.provider.Settings
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.border
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.PaddingValues
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.WindowInsets
|
||||
import androidx.compose.foundation.layout.WindowInsetsSides
|
||||
import androidx.compose.foundation.layout.fillMaxHeight
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.imePadding
|
||||
import androidx.compose.foundation.layout.only
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.safeDrawing
|
||||
import androidx.compose.foundation.layout.windowInsetsPadding
|
||||
import androidx.compose.foundation.lazy.LazyColumn
|
||||
import androidx.compose.foundation.lazy.items
|
||||
import androidx.compose.foundation.lazy.rememberLazyListState
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.ButtonDefaults
|
||||
import androidx.compose.material3.HorizontalDivider
|
||||
import androidx.compose.material3.ListItem
|
||||
import androidx.compose.material3.ListItemDefaults
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.OutlinedTextField
|
||||
import androidx.compose.material3.OutlinedTextFieldDefaults
|
||||
import androidx.compose.material3.RadioButton
|
||||
import androidx.compose.material3.Switch
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.collectAsState
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.draw.alpha
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.text.font.FontFamily
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.unit.sp
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.LifecycleEventObserver
|
||||
import androidx.lifecycle.compose.LocalLifecycleOwner
|
||||
import ai.openclaw.android.BuildConfig
|
||||
import ai.openclaw.android.LocationMode
|
||||
import ai.openclaw.android.MainViewModel
|
||||
|
||||
@Composable
|
||||
fun SettingsSheet(viewModel: MainViewModel) {
|
||||
val context = LocalContext.current
|
||||
val lifecycleOwner = LocalLifecycleOwner.current
|
||||
val instanceId by viewModel.instanceId.collectAsState()
|
||||
val displayName by viewModel.displayName.collectAsState()
|
||||
val cameraEnabled by viewModel.cameraEnabled.collectAsState()
|
||||
val locationMode by viewModel.locationMode.collectAsState()
|
||||
val locationPreciseEnabled by viewModel.locationPreciseEnabled.collectAsState()
|
||||
val preventSleep by viewModel.preventSleep.collectAsState()
|
||||
val canvasDebugStatusEnabled by viewModel.canvasDebugStatusEnabled.collectAsState()
|
||||
|
||||
val listState = rememberLazyListState()
|
||||
val deviceModel =
|
||||
remember {
|
||||
listOfNotNull(Build.MANUFACTURER, Build.MODEL)
|
||||
.joinToString(" ")
|
||||
.trim()
|
||||
.ifEmpty { "Android" }
|
||||
}
|
||||
val appVersion =
|
||||
remember {
|
||||
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
|
||||
if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
|
||||
"$versionName-dev"
|
||||
} else {
|
||||
versionName
|
||||
}
|
||||
}
|
||||
val listItemColors =
|
||||
ListItemDefaults.colors(
|
||||
containerColor = Color.Transparent,
|
||||
headlineColor = mobileText,
|
||||
supportingColor = mobileTextSecondary,
|
||||
trailingIconColor = mobileTextSecondary,
|
||||
leadingIconColor = mobileTextSecondary,
|
||||
)
|
||||
|
||||
val permissionLauncher =
|
||||
rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms ->
|
||||
val cameraOk = perms[Manifest.permission.CAMERA] == true
|
||||
viewModel.setCameraEnabled(cameraOk)
|
||||
}
|
||||
|
||||
var pendingLocationMode by remember { mutableStateOf<LocationMode?>(null) }
|
||||
var pendingPreciseToggle by remember { mutableStateOf(false) }
|
||||
|
||||
val locationPermissionLauncher =
|
||||
rememberLauncherForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { perms ->
|
||||
val fineOk = perms[Manifest.permission.ACCESS_FINE_LOCATION] == true
|
||||
val coarseOk = perms[Manifest.permission.ACCESS_COARSE_LOCATION] == true
|
||||
val granted = fineOk || coarseOk
|
||||
val requestedMode = pendingLocationMode
|
||||
pendingLocationMode = null
|
||||
|
||||
if (pendingPreciseToggle) {
|
||||
pendingPreciseToggle = false
|
||||
viewModel.setLocationPreciseEnabled(fineOk)
|
||||
return@rememberLauncherForActivityResult
|
||||
}
|
||||
|
||||
if (!granted) {
|
||||
viewModel.setLocationMode(LocationMode.Off)
|
||||
return@rememberLauncherForActivityResult
|
||||
}
|
||||
|
||||
if (requestedMode != null) {
|
||||
viewModel.setLocationMode(requestedMode)
|
||||
if (requestedMode == LocationMode.Always) {
|
||||
val backgroundOk =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
if (!backgroundOk) {
|
||||
openAppSettings(context)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var micPermissionGranted by
|
||||
remember {
|
||||
mutableStateOf(
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
|
||||
PackageManager.PERMISSION_GRANTED,
|
||||
)
|
||||
}
|
||||
val audioPermissionLauncher =
|
||||
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
|
||||
micPermissionGranted = granted
|
||||
}
|
||||
|
||||
val smsPermissionAvailable =
|
||||
remember {
|
||||
context.packageManager?.hasSystemFeature(PackageManager.FEATURE_TELEPHONY) == true
|
||||
}
|
||||
var smsPermissionGranted by
|
||||
remember {
|
||||
mutableStateOf(
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.SEND_SMS) ==
|
||||
PackageManager.PERMISSION_GRANTED,
|
||||
)
|
||||
}
|
||||
val smsPermissionLauncher =
|
||||
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
|
||||
smsPermissionGranted = granted
|
||||
viewModel.refreshGatewayConnection()
|
||||
}
|
||||
|
||||
DisposableEffect(lifecycleOwner, context) {
|
||||
val observer =
|
||||
LifecycleEventObserver { _, event ->
|
||||
if (event == Lifecycle.Event.ON_RESUME) {
|
||||
micPermissionGranted =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
smsPermissionGranted =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.SEND_SMS) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
}
|
||||
lifecycleOwner.lifecycle.addObserver(observer)
|
||||
onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }
|
||||
}
|
||||
|
||||
fun setCameraEnabledChecked(checked: Boolean) {
|
||||
if (!checked) {
|
||||
viewModel.setCameraEnabled(false)
|
||||
return
|
||||
}
|
||||
|
||||
val cameraOk =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
if (cameraOk) {
|
||||
viewModel.setCameraEnabled(true)
|
||||
} else {
|
||||
permissionLauncher.launch(arrayOf(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO))
|
||||
}
|
||||
}
|
||||
|
||||
fun requestLocationPermissions(targetMode: LocationMode) {
|
||||
val fineOk =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
val coarseOk =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_COARSE_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
if (fineOk || coarseOk) {
|
||||
viewModel.setLocationMode(targetMode)
|
||||
if (targetMode == LocationMode.Always) {
|
||||
val backgroundOk =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
if (!backgroundOk) {
|
||||
openAppSettings(context)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
pendingLocationMode = targetMode
|
||||
locationPermissionLauncher.launch(
|
||||
arrayOf(Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.ACCESS_COARSE_LOCATION),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun setPreciseLocationChecked(checked: Boolean) {
|
||||
if (!checked) {
|
||||
viewModel.setLocationPreciseEnabled(false)
|
||||
return
|
||||
}
|
||||
val fineOk =
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.ACCESS_FINE_LOCATION) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
if (fineOk) {
|
||||
viewModel.setLocationPreciseEnabled(true)
|
||||
} else {
|
||||
pendingPreciseToggle = true
|
||||
locationPermissionLauncher.launch(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION))
|
||||
}
|
||||
}
|
||||
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxSize()
|
||||
.background(mobileBackgroundGradient),
|
||||
) {
|
||||
LazyColumn(
|
||||
state = listState,
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxWidth()
|
||||
.fillMaxHeight()
|
||||
.imePadding()
|
||||
.windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Bottom)),
|
||||
contentPadding = PaddingValues(horizontal = 20.dp, vertical = 16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
item {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(6.dp)) {
|
||||
Text(
|
||||
"SETTINGS",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
|
||||
color = mobileAccent,
|
||||
)
|
||||
Text("Device Configuration", style = mobileTitle2, color = mobileText)
|
||||
Text(
|
||||
"Manage capabilities, permissions, and diagnostics.",
|
||||
style = mobileCallout,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
}
|
||||
item { HorizontalDivider(color = mobileBorder) }
|
||||
|
||||
// Order parity: Node → Voice → Camera → Messaging → Location → Screen.
|
||||
item {
|
||||
Text(
|
||||
"NODE",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
|
||||
color = mobileAccent,
|
||||
)
|
||||
}
|
||||
item {
|
||||
OutlinedTextField(
|
||||
value = displayName,
|
||||
onValueChange = viewModel::setDisplayName,
|
||||
label = { Text("Name", style = mobileCaption1, color = mobileTextSecondary) },
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
textStyle = mobileBody.copy(color = mobileText),
|
||||
colors = settingsTextFieldColors(),
|
||||
)
|
||||
}
|
||||
item { Text("Instance ID: $instanceId", style = mobileCallout.copy(fontFamily = FontFamily.Monospace), color = mobileTextSecondary) }
|
||||
item { Text("Device: $deviceModel", style = mobileCallout, color = mobileTextSecondary) }
|
||||
item { Text("Version: $appVersion", style = mobileCallout, color = mobileTextSecondary) }
|
||||
|
||||
item { HorizontalDivider(color = mobileBorder) }
|
||||
|
||||
// Voice
|
||||
item {
|
||||
Text(
|
||||
"VOICE",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
|
||||
color = mobileAccent,
|
||||
)
|
||||
}
|
||||
item {
|
||||
ListItem(
|
||||
modifier = settingsRowModifier(),
|
||||
colors = listItemColors,
|
||||
headlineContent = { Text("Microphone permission", style = mobileHeadline) },
|
||||
supportingContent = {
|
||||
Text(
|
||||
if (micPermissionGranted) {
|
||||
"Granted. Use the Voice tab mic button to capture transcript."
|
||||
} else {
|
||||
"Required for Voice tab transcription."
|
||||
},
|
||||
style = mobileCallout,
|
||||
)
|
||||
},
|
||||
trailingContent = {
|
||||
Button(
|
||||
onClick = {
|
||||
if (micPermissionGranted) {
|
||||
openAppSettings(context)
|
||||
} else {
|
||||
audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
|
||||
}
|
||||
},
|
||||
colors = settingsPrimaryButtonColors(),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
) {
|
||||
Text(
|
||||
if (micPermissionGranted) "Manage" else "Grant",
|
||||
style = mobileCallout.copy(fontWeight = FontWeight.Bold),
|
||||
)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
item {
|
||||
Text(
|
||||
"Voice wake and talk modes were removed. Voice now uses one mic on/off flow in the Voice tab.",
|
||||
style = mobileCallout,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
|
||||
item { HorizontalDivider(color = mobileBorder) }
|
||||
|
||||
// Camera
|
||||
item {
|
||||
Text(
|
||||
"CAMERA",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
|
||||
color = mobileAccent,
|
||||
)
|
||||
}
|
||||
item {
|
||||
ListItem(
|
||||
modifier = settingsRowModifier(),
|
||||
colors = listItemColors,
|
||||
headlineContent = { Text("Allow Camera", style = mobileHeadline) },
|
||||
supportingContent = { Text("Allows the gateway to request photos or short video clips (foreground only).", style = mobileCallout) },
|
||||
trailingContent = { Switch(checked = cameraEnabled, onCheckedChange = ::setCameraEnabledChecked) },
|
||||
)
|
||||
}
|
||||
item {
|
||||
Text(
|
||||
"Tip: grant Microphone permission for video clips with audio.",
|
||||
style = mobileCallout,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
|
||||
item { HorizontalDivider(color = mobileBorder) }
|
||||
|
||||
// Messaging
|
||||
item {
|
||||
Text(
|
||||
"MESSAGING",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
|
||||
color = mobileAccent,
|
||||
)
|
||||
}
|
||||
item {
|
||||
val buttonLabel =
|
||||
when {
|
||||
!smsPermissionAvailable -> "Unavailable"
|
||||
smsPermissionGranted -> "Manage"
|
||||
else -> "Grant"
|
||||
}
|
||||
ListItem(
|
||||
modifier = settingsRowModifier(),
|
||||
colors = listItemColors,
|
||||
headlineContent = { Text("SMS Permission", style = mobileHeadline) },
|
||||
supportingContent = {
|
||||
Text(
|
||||
if (smsPermissionAvailable) {
|
||||
"Allow the gateway to send SMS from this device."
|
||||
} else {
|
||||
"SMS requires a device with telephony hardware."
|
||||
},
|
||||
style = mobileCallout,
|
||||
)
|
||||
},
|
||||
trailingContent = {
|
||||
Button(
|
||||
onClick = {
|
||||
if (!smsPermissionAvailable) return@Button
|
||||
if (smsPermissionGranted) {
|
||||
openAppSettings(context)
|
||||
} else {
|
||||
smsPermissionLauncher.launch(Manifest.permission.SEND_SMS)
|
||||
}
|
||||
},
|
||||
enabled = smsPermissionAvailable,
|
||||
colors = settingsPrimaryButtonColors(),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
) {
|
||||
Text(buttonLabel, style = mobileCallout.copy(fontWeight = FontWeight.Bold))
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
item { HorizontalDivider(color = mobileBorder) }
|
||||
|
||||
// Location
|
||||
item {
|
||||
Text(
|
||||
"LOCATION",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
|
||||
color = mobileAccent,
|
||||
)
|
||||
}
|
||||
item {
|
||||
Column(modifier = settingsRowModifier(), verticalArrangement = Arrangement.spacedBy(0.dp)) {
|
||||
ListItem(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
colors = listItemColors,
|
||||
headlineContent = { Text("Off", style = mobileHeadline) },
|
||||
supportingContent = { Text("Disable location sharing.", style = mobileCallout) },
|
||||
trailingContent = {
|
||||
RadioButton(
|
||||
selected = locationMode == LocationMode.Off,
|
||||
onClick = { viewModel.setLocationMode(LocationMode.Off) },
|
||||
)
|
||||
},
|
||||
)
|
||||
HorizontalDivider(color = mobileBorder)
|
||||
ListItem(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
colors = listItemColors,
|
||||
headlineContent = { Text("While Using", style = mobileHeadline) },
|
||||
supportingContent = { Text("Only while OpenClaw is open.", style = mobileCallout) },
|
||||
trailingContent = {
|
||||
RadioButton(
|
||||
selected = locationMode == LocationMode.WhileUsing,
|
||||
onClick = { requestLocationPermissions(LocationMode.WhileUsing) },
|
||||
)
|
||||
},
|
||||
)
|
||||
HorizontalDivider(color = mobileBorder)
|
||||
ListItem(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
colors = listItemColors,
|
||||
headlineContent = { Text("Always", style = mobileHeadline) },
|
||||
supportingContent = { Text("Allow background location (requires system permission).", style = mobileCallout) },
|
||||
trailingContent = {
|
||||
RadioButton(
|
||||
selected = locationMode == LocationMode.Always,
|
||||
onClick = { requestLocationPermissions(LocationMode.Always) },
|
||||
)
|
||||
},
|
||||
)
|
||||
HorizontalDivider(color = mobileBorder)
|
||||
ListItem(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
colors = listItemColors,
|
||||
headlineContent = { Text("Precise Location", style = mobileHeadline) },
|
||||
supportingContent = { Text("Use precise GPS when available.", style = mobileCallout) },
|
||||
trailingContent = {
|
||||
Switch(
|
||||
checked = locationPreciseEnabled,
|
||||
onCheckedChange = ::setPreciseLocationChecked,
|
||||
enabled = locationMode != LocationMode.Off,
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
item {
|
||||
Text(
|
||||
"Always may require Android Settings to allow background location.",
|
||||
style = mobileCallout,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
|
||||
item { HorizontalDivider(color = mobileBorder) }
|
||||
|
||||
// Screen
|
||||
item {
|
||||
Text(
|
||||
"SCREEN",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
|
||||
color = mobileAccent,
|
||||
)
|
||||
}
|
||||
item {
|
||||
ListItem(
|
||||
modifier = settingsRowModifier(),
|
||||
colors = listItemColors,
|
||||
headlineContent = { Text("Prevent Sleep", style = mobileHeadline) },
|
||||
supportingContent = { Text("Keeps the screen awake while OpenClaw is open.", style = mobileCallout) },
|
||||
trailingContent = { Switch(checked = preventSleep, onCheckedChange = viewModel::setPreventSleep) },
|
||||
)
|
||||
}
|
||||
|
||||
item { HorizontalDivider(color = mobileBorder) }
|
||||
|
||||
// Debug
|
||||
item {
|
||||
Text(
|
||||
"DEBUG",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
|
||||
color = mobileAccent,
|
||||
)
|
||||
}
|
||||
item {
|
||||
ListItem(
|
||||
modifier = settingsRowModifier(),
|
||||
colors = listItemColors,
|
||||
headlineContent = { Text("Debug Canvas Status", style = mobileHeadline) },
|
||||
supportingContent = { Text("Show status text in the canvas when debug is enabled.", style = mobileCallout) },
|
||||
trailingContent = {
|
||||
Switch(
|
||||
checked = canvasDebugStatusEnabled,
|
||||
onCheckedChange = viewModel::setCanvasDebugStatusEnabled,
|
||||
)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
item { Spacer(modifier = Modifier.height(24.dp)) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun settingsTextFieldColors() =
|
||||
OutlinedTextFieldDefaults.colors(
|
||||
focusedContainerColor = mobileSurface,
|
||||
unfocusedContainerColor = mobileSurface,
|
||||
focusedBorderColor = mobileAccent,
|
||||
unfocusedBorderColor = mobileBorder,
|
||||
focusedTextColor = mobileText,
|
||||
unfocusedTextColor = mobileText,
|
||||
cursorColor = mobileAccent,
|
||||
)
|
||||
|
||||
private fun settingsRowModifier() =
|
||||
Modifier
|
||||
.fillMaxWidth()
|
||||
.border(width = 1.dp, color = mobileBorder, shape = RoundedCornerShape(14.dp))
|
||||
.background(Color.White, RoundedCornerShape(14.dp))
|
||||
|
||||
@Composable
|
||||
private fun settingsPrimaryButtonColors() =
|
||||
ButtonDefaults.buttonColors(
|
||||
containerColor = mobileAccent,
|
||||
contentColor = Color.White,
|
||||
disabledContainerColor = mobileAccent.copy(alpha = 0.45f),
|
||||
disabledContentColor = Color.White.copy(alpha = 0.9f),
|
||||
)
|
||||
|
||||
@Composable
|
||||
private fun settingsDangerButtonColors() =
|
||||
ButtonDefaults.buttonColors(
|
||||
containerColor = mobileDanger,
|
||||
contentColor = Color.White,
|
||||
disabledContainerColor = mobileDanger.copy(alpha = 0.45f),
|
||||
disabledContentColor = Color.White.copy(alpha = 0.9f),
|
||||
)
|
||||
|
||||
private fun openAppSettings(context: Context) {
|
||||
val intent =
|
||||
Intent(
|
||||
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
|
||||
Uri.fromParts("package", context.packageName, null),
|
||||
)
|
||||
context.startActivity(intent)
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import androidx.compose.animation.core.LinearEasing
|
||||
import androidx.compose.animation.core.RepeatMode
|
||||
import androidx.compose.animation.core.animateFloat
|
||||
import androidx.compose.animation.core.infiniteRepeatable
|
||||
import androidx.compose.animation.core.rememberInfiniteTransition
|
||||
import androidx.compose.animation.core.tween
|
||||
import androidx.compose.foundation.Canvas
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Brush
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.drawscope.Stroke
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.unit.dp
|
||||
|
||||
@Composable
|
||||
fun TalkOrbOverlay(
|
||||
seamColor: Color,
|
||||
statusText: String,
|
||||
isListening: Boolean,
|
||||
isSpeaking: Boolean,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
val transition = rememberInfiniteTransition(label = "talk-orb")
|
||||
val t by
|
||||
transition.animateFloat(
|
||||
initialValue = 0f,
|
||||
targetValue = 1f,
|
||||
animationSpec =
|
||||
infiniteRepeatable(
|
||||
animation = tween(durationMillis = 1500, easing = LinearEasing),
|
||||
repeatMode = RepeatMode.Restart,
|
||||
),
|
||||
label = "pulse",
|
||||
)
|
||||
|
||||
val trimmed = statusText.trim()
|
||||
val showStatus = trimmed.isNotEmpty() && trimmed != "Off"
|
||||
val phase =
|
||||
when {
|
||||
isSpeaking -> "Speaking"
|
||||
isListening -> "Listening"
|
||||
else -> "Thinking"
|
||||
}
|
||||
|
||||
Column(
|
||||
modifier = modifier.padding(24.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally,
|
||||
verticalArrangement = Arrangement.spacedBy(12.dp),
|
||||
) {
|
||||
Box(contentAlignment = Alignment.Center) {
|
||||
Canvas(modifier = Modifier.size(360.dp)) {
|
||||
val center = this.center
|
||||
val baseRadius = size.minDimension * 0.30f
|
||||
|
||||
val ring1 = 1.05f + (t * 0.25f)
|
||||
val ring2 = 1.20f + (t * 0.55f)
|
||||
val ringAlpha1 = (1f - t) * 0.34f
|
||||
val ringAlpha2 = (1f - t) * 0.22f
|
||||
|
||||
drawCircle(
|
||||
color = seamColor.copy(alpha = ringAlpha1),
|
||||
radius = baseRadius * ring1,
|
||||
center = center,
|
||||
style = Stroke(width = 3.dp.toPx()),
|
||||
)
|
||||
drawCircle(
|
||||
color = seamColor.copy(alpha = ringAlpha2),
|
||||
radius = baseRadius * ring2,
|
||||
center = center,
|
||||
style = Stroke(width = 3.dp.toPx()),
|
||||
)
|
||||
|
||||
drawCircle(
|
||||
brush =
|
||||
Brush.radialGradient(
|
||||
colors =
|
||||
listOf(
|
||||
seamColor.copy(alpha = 0.92f),
|
||||
seamColor.copy(alpha = 0.40f),
|
||||
Color.Black.copy(alpha = 0.56f),
|
||||
),
|
||||
center = center,
|
||||
radius = baseRadius * 1.35f,
|
||||
),
|
||||
radius = baseRadius,
|
||||
center = center,
|
||||
)
|
||||
|
||||
drawCircle(
|
||||
color = seamColor.copy(alpha = 0.34f),
|
||||
radius = baseRadius,
|
||||
center = center,
|
||||
style = Stroke(width = 1.dp.toPx()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (showStatus) {
|
||||
Surface(
|
||||
color = Color.Black.copy(alpha = 0.40f),
|
||||
shape = CircleShape,
|
||||
) {
|
||||
Text(
|
||||
text = trimmed,
|
||||
modifier = Modifier.padding(horizontal = 14.dp, vertical = 8.dp),
|
||||
color = Color.White.copy(alpha = 0.92f),
|
||||
style = MaterialTheme.typography.labelLarge,
|
||||
fontWeight = FontWeight.SemiBold,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
Text(
|
||||
text = phase,
|
||||
color = Color.White.copy(alpha = 0.80f),
|
||||
style = MaterialTheme.typography.labelLarge,
|
||||
fontWeight = FontWeight.SemiBold,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
package ai.openclaw.android.ui
|
||||
|
||||
import android.Manifest
|
||||
import android.app.Activity
|
||||
import android.content.Context
|
||||
import android.content.ContextWrapper
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.net.Uri
|
||||
import android.provider.Settings
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.compose.animation.core.LinearEasing
|
||||
import androidx.compose.animation.core.RepeatMode
|
||||
import androidx.compose.animation.core.animateFloat
|
||||
import androidx.compose.animation.core.infiniteRepeatable
|
||||
import androidx.compose.animation.core.rememberInfiniteTransition
|
||||
import androidx.compose.animation.core.tween
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.PaddingValues
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.WindowInsets
|
||||
import androidx.compose.foundation.layout.WindowInsetsSides
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.heightIn
|
||||
import androidx.compose.foundation.layout.imePadding
|
||||
import androidx.compose.foundation.layout.only
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.safeDrawing
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.layout.windowInsetsPadding
|
||||
import androidx.compose.foundation.lazy.LazyColumn
|
||||
import androidx.compose.foundation.lazy.items
|
||||
import androidx.compose.foundation.lazy.rememberLazyListState
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.filled.Mic
|
||||
import androidx.compose.material.icons.filled.MicOff
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.ButtonDefaults
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.collectAsState
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.draw.alpha
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.text.style.TextAlign
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.unit.sp
|
||||
import androidx.core.app.ActivityCompat
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.LifecycleEventObserver
|
||||
import androidx.lifecycle.compose.LocalLifecycleOwner
|
||||
import ai.openclaw.android.MainViewModel
|
||||
import ai.openclaw.android.voice.VoiceConversationEntry
|
||||
import ai.openclaw.android.voice.VoiceConversationRole
|
||||
import kotlin.math.PI
|
||||
import kotlin.math.max
|
||||
import kotlin.math.sin
|
||||
|
||||
@Composable
|
||||
fun VoiceTabScreen(viewModel: MainViewModel) {
|
||||
val context = LocalContext.current
|
||||
val lifecycleOwner = LocalLifecycleOwner.current
|
||||
val activity = remember(context) { context.findActivity() }
|
||||
val listState = rememberLazyListState()
|
||||
|
||||
val isConnected by viewModel.isConnected.collectAsState()
|
||||
val gatewayStatus by viewModel.statusText.collectAsState()
|
||||
val micEnabled by viewModel.micEnabled.collectAsState()
|
||||
val micStatusText by viewModel.micStatusText.collectAsState()
|
||||
val micLiveTranscript by viewModel.micLiveTranscript.collectAsState()
|
||||
val micQueuedMessages by viewModel.micQueuedMessages.collectAsState()
|
||||
val micConversation by viewModel.micConversation.collectAsState()
|
||||
val micInputLevel by viewModel.micInputLevel.collectAsState()
|
||||
val micIsSending by viewModel.micIsSending.collectAsState()
|
||||
|
||||
val hasStreamingAssistant = micConversation.any { it.role == VoiceConversationRole.Assistant && it.isStreaming }
|
||||
val showThinkingBubble = micIsSending && !hasStreamingAssistant
|
||||
|
||||
var hasMicPermission by remember { mutableStateOf(context.hasRecordAudioPermission()) }
|
||||
var pendingMicEnable by remember { mutableStateOf(false) }
|
||||
|
||||
DisposableEffect(lifecycleOwner, context) {
|
||||
val observer =
|
||||
LifecycleEventObserver { _, event ->
|
||||
if (event == Lifecycle.Event.ON_RESUME) {
|
||||
hasMicPermission = context.hasRecordAudioPermission()
|
||||
}
|
||||
}
|
||||
lifecycleOwner.lifecycle.addObserver(observer)
|
||||
onDispose { lifecycleOwner.lifecycle.removeObserver(observer) }
|
||||
}
|
||||
|
||||
val requestMicPermission =
|
||||
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
|
||||
hasMicPermission = granted
|
||||
if (granted && pendingMicEnable) {
|
||||
viewModel.setMicEnabled(true)
|
||||
}
|
||||
pendingMicEnable = false
|
||||
}
|
||||
|
||||
LaunchedEffect(micConversation.size, showThinkingBubble) {
|
||||
val total = micConversation.size + if (showThinkingBubble) 1 else 0
|
||||
if (total > 0) {
|
||||
listState.animateScrollToItem(total - 1)
|
||||
}
|
||||
}
|
||||
|
||||
Column(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxSize()
|
||||
.background(mobileBackgroundGradient)
|
||||
.imePadding()
|
||||
.windowInsetsPadding(WindowInsets.safeDrawing.only(WindowInsetsSides.Bottom))
|
||||
.padding(horizontal = 20.dp, vertical = 14.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(10.dp),
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(4.dp)) {
|
||||
Text(
|
||||
"VOICE",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 1.sp),
|
||||
color = mobileAccent,
|
||||
)
|
||||
Text("Voice mode", style = mobileTitle2, color = mobileText)
|
||||
}
|
||||
Surface(
|
||||
shape = RoundedCornerShape(999.dp),
|
||||
color = if (isConnected) mobileAccentSoft else mobileSurfaceStrong,
|
||||
border = BorderStroke(1.dp, if (isConnected) mobileAccent.copy(alpha = 0.25f) else mobileBorderStrong),
|
||||
) {
|
||||
Text(
|
||||
if (isConnected) "Connected" else "Offline",
|
||||
modifier = Modifier.padding(horizontal = 12.dp, vertical = 6.dp),
|
||||
style = mobileCaption1,
|
||||
color = if (isConnected) mobileAccent else mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
LazyColumn(
|
||||
state = listState,
|
||||
modifier = Modifier.fillMaxWidth().weight(1f),
|
||||
contentPadding = PaddingValues(vertical = 4.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(10.dp),
|
||||
) {
|
||||
if (micConversation.isEmpty() && !showThinkingBubble) {
|
||||
item {
|
||||
Column(
|
||||
modifier = Modifier.fillMaxWidth().padding(top = 12.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text(
|
||||
"Tap the mic and speak. Each pause sends a turn automatically.",
|
||||
style = mobileCallout,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
items(items = micConversation, key = { it.id }) { entry ->
|
||||
VoiceTurnBubble(entry = entry)
|
||||
}
|
||||
|
||||
if (showThinkingBubble) {
|
||||
item {
|
||||
VoiceThinkingBubble()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
shape = RoundedCornerShape(20.dp),
|
||||
color = Color.White,
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 14.dp, vertical = 12.dp),
|
||||
horizontalAlignment = Alignment.CenterHorizontally,
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Surface(
|
||||
shape = RoundedCornerShape(999.dp),
|
||||
color = mobileSurface,
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
) {
|
||||
val queueCount = micQueuedMessages.size
|
||||
val stateText =
|
||||
when {
|
||||
queueCount > 0 -> "$queueCount queued"
|
||||
micIsSending -> "Sending"
|
||||
micEnabled -> "Listening"
|
||||
else -> "Mic off"
|
||||
}
|
||||
Text(
|
||||
"$gatewayStatus · $stateText",
|
||||
modifier = Modifier.padding(horizontal = 12.dp, vertical = 7.dp),
|
||||
style = mobileCaption1,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
|
||||
if (!micLiveTranscript.isNullOrBlank()) {
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
color = mobileAccentSoft,
|
||||
border = BorderStroke(1.dp, mobileAccent.copy(alpha = 0.2f)),
|
||||
) {
|
||||
Text(
|
||||
micLiveTranscript!!.trim(),
|
||||
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
|
||||
style = mobileCallout,
|
||||
color = mobileText,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
MicWaveform(level = micInputLevel, active = micEnabled)
|
||||
|
||||
Button(
|
||||
onClick = {
|
||||
if (micEnabled) {
|
||||
viewModel.setMicEnabled(false)
|
||||
return@Button
|
||||
}
|
||||
if (hasMicPermission) {
|
||||
viewModel.setMicEnabled(true)
|
||||
} else {
|
||||
pendingMicEnable = true
|
||||
requestMicPermission.launch(Manifest.permission.RECORD_AUDIO)
|
||||
}
|
||||
},
|
||||
shape = CircleShape,
|
||||
contentPadding = PaddingValues(0.dp),
|
||||
modifier = Modifier.size(86.dp),
|
||||
colors =
|
||||
ButtonDefaults.buttonColors(
|
||||
containerColor = if (micEnabled) mobileDanger else mobileAccent,
|
||||
contentColor = Color.White,
|
||||
),
|
||||
) {
|
||||
Icon(
|
||||
imageVector = if (micEnabled) Icons.Default.MicOff else Icons.Default.Mic,
|
||||
contentDescription = if (micEnabled) "Turn microphone off" else "Turn microphone on",
|
||||
modifier = Modifier.size(30.dp),
|
||||
)
|
||||
}
|
||||
|
||||
Text(
|
||||
if (micEnabled) "Tap to stop" else "Tap to speak",
|
||||
style = mobileCallout,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
|
||||
if (!hasMicPermission) {
|
||||
val showRationale =
|
||||
if (activity == null) {
|
||||
false
|
||||
} else {
|
||||
ActivityCompat.shouldShowRequestPermissionRationale(activity, Manifest.permission.RECORD_AUDIO)
|
||||
}
|
||||
Text(
|
||||
if (showRationale) {
|
||||
"Microphone permission is required for voice mode."
|
||||
} else {
|
||||
"Microphone blocked. Open app settings to enable it."
|
||||
},
|
||||
style = mobileCaption1,
|
||||
color = mobileWarning,
|
||||
textAlign = TextAlign.Center,
|
||||
)
|
||||
Button(
|
||||
onClick = { openAppSettings(context) },
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
colors = ButtonDefaults.buttonColors(containerColor = mobileSurfaceStrong, contentColor = mobileText),
|
||||
) {
|
||||
Text("Open settings", style = mobileCallout.copy(fontWeight = FontWeight.SemiBold))
|
||||
}
|
||||
}
|
||||
|
||||
Text(
|
||||
micStatusText,
|
||||
style = mobileCaption1,
|
||||
color = mobileTextTertiary,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun VoiceTurnBubble(entry: VoiceConversationEntry) {
|
||||
val isUser = entry.role == VoiceConversationRole.User
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start,
|
||||
) {
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(0.90f),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
color = if (isUser) mobileAccentSoft else mobileSurface,
|
||||
border = BorderStroke(1.dp, if (isUser) mobileAccent.copy(alpha = 0.2f) else mobileBorder),
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 10.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(6.dp),
|
||||
) {
|
||||
Text(
|
||||
if (isUser) "You" else "OpenClaw",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold),
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
Text(
|
||||
if (entry.isStreaming && entry.text.isBlank()) "Listening response…" else entry.text,
|
||||
style = mobileCallout,
|
||||
color = mobileText,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun VoiceThinkingBubble() {
|
||||
Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Start) {
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(0.68f),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
color = mobileSurface,
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.padding(horizontal = 12.dp, vertical = 10.dp),
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
ThinkingDots(color = mobileTextSecondary)
|
||||
Text("OpenClaw is thinking…", style = mobileCallout, color = mobileTextSecondary)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ThinkingDots(color: Color) {
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) {
|
||||
ThinkingDot(alpha = 0.38f, color = color)
|
||||
ThinkingDot(alpha = 0.62f, color = color)
|
||||
ThinkingDot(alpha = 0.90f, color = color)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ThinkingDot(alpha: Float, color: Color) {
|
||||
Surface(
|
||||
modifier = Modifier.size(6.dp).alpha(alpha),
|
||||
shape = CircleShape,
|
||||
color = color,
|
||||
) {}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun MicWaveform(level: Float, active: Boolean) {
|
||||
val transition = rememberInfiniteTransition(label = "voiceWave")
|
||||
val phase by
|
||||
transition.animateFloat(
|
||||
initialValue = 0f,
|
||||
targetValue = 1f,
|
||||
animationSpec = infiniteRepeatable(animation = tween(1_000, easing = LinearEasing), repeatMode = RepeatMode.Restart),
|
||||
label = "voiceWavePhase",
|
||||
)
|
||||
|
||||
val effective = if (active) level.coerceIn(0f, 1f) else 0f
|
||||
val base = max(effective, if (active) 0.05f else 0f)
|
||||
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth().heightIn(min = 40.dp),
|
||||
horizontalArrangement = Arrangement.spacedBy(4.dp, Alignment.CenterHorizontally),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
) {
|
||||
repeat(16) { index ->
|
||||
val pulse =
|
||||
if (!active) {
|
||||
0f
|
||||
} else {
|
||||
((sin(((phase * 2f * PI) + (index * 0.55f)).toDouble()) + 1.0) * 0.5).toFloat()
|
||||
}
|
||||
val barHeight = 6.dp + (24.dp * (base * pulse))
|
||||
Box(
|
||||
modifier =
|
||||
Modifier
|
||||
.width(5.dp)
|
||||
.height(barHeight)
|
||||
.background(if (active) mobileAccent else mobileBorderStrong, RoundedCornerShape(999.dp)),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun Context.hasRecordAudioPermission(): Boolean {
|
||||
return (
|
||||
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
)
|
||||
}
|
||||
|
||||
private fun Context.findActivity(): Activity? =
|
||||
when (this) {
|
||||
is Activity -> this
|
||||
is ContextWrapper -> baseContext.findActivity()
|
||||
else -> null
|
||||
}
|
||||
|
||||
private fun openAppSettings(context: Context) {
|
||||
val intent =
|
||||
Intent(
|
||||
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
|
||||
Uri.fromParts("package", context.packageName, null),
|
||||
)
|
||||
context.startActivity(intent)
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package ai.openclaw.android.ui.chat
|
||||
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.horizontalScroll
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.PaddingValues
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.automirrored.filled.Send
|
||||
import androidx.compose.material.icons.filled.ArrowDropDown
|
||||
import androidx.compose.material.icons.filled.AttachFile
|
||||
import androidx.compose.material.icons.filled.Refresh
|
||||
import androidx.compose.material.icons.filled.Stop
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.ButtonDefaults
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.DropdownMenu
|
||||
import androidx.compose.material3.DropdownMenuItem
|
||||
import androidx.compose.material3.HorizontalDivider
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.OutlinedTextField
|
||||
import androidx.compose.material3.OutlinedTextFieldDefaults
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.saveable.rememberSaveable
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.text.style.TextOverflow
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.unit.sp
|
||||
import ai.openclaw.android.ui.mobileAccent
|
||||
import ai.openclaw.android.ui.mobileAccentSoft
|
||||
import ai.openclaw.android.ui.mobileBorder
|
||||
import ai.openclaw.android.ui.mobileBorderStrong
|
||||
import ai.openclaw.android.ui.mobileCallout
|
||||
import ai.openclaw.android.ui.mobileCaption1
|
||||
import ai.openclaw.android.ui.mobileHeadline
|
||||
import ai.openclaw.android.ui.mobileSurface
|
||||
import ai.openclaw.android.ui.mobileText
|
||||
import ai.openclaw.android.ui.mobileTextSecondary
|
||||
import ai.openclaw.android.ui.mobileTextTertiary
|
||||
|
||||
@Composable
|
||||
fun ChatComposer(
|
||||
healthOk: Boolean,
|
||||
thinkingLevel: String,
|
||||
pendingRunCount: Int,
|
||||
attachments: List<PendingImageAttachment>,
|
||||
onPickImages: () -> Unit,
|
||||
onRemoveAttachment: (id: String) -> Unit,
|
||||
onSetThinkingLevel: (level: String) -> Unit,
|
||||
onRefresh: () -> Unit,
|
||||
onAbort: () -> Unit,
|
||||
onSend: (text: String) -> Unit,
|
||||
) {
|
||||
var input by rememberSaveable { mutableStateOf("") }
|
||||
var showThinkingMenu by remember { mutableStateOf(false) }
|
||||
|
||||
val canSend = pendingRunCount == 0 && (input.trim().isNotEmpty() || attachments.isNotEmpty()) && healthOk
|
||||
val sendBusy = pendingRunCount > 0
|
||||
|
||||
Column(modifier = Modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Box(modifier = Modifier.weight(1f)) {
|
||||
Surface(
|
||||
onClick = { showThinkingMenu = true },
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
color = mobileAccentSoft,
|
||||
border = BorderStroke(1.dp, mobileBorderStrong),
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 8.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
) {
|
||||
Text(
|
||||
text = "Thinking: ${thinkingLabel(thinkingLevel)}",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold),
|
||||
color = mobileText,
|
||||
)
|
||||
Icon(Icons.Default.ArrowDropDown, contentDescription = "Select thinking level", tint = mobileTextSecondary)
|
||||
}
|
||||
}
|
||||
|
||||
DropdownMenu(expanded = showThinkingMenu, onDismissRequest = { showThinkingMenu = false }) {
|
||||
ThinkingMenuItem("off", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
|
||||
ThinkingMenuItem("low", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
|
||||
ThinkingMenuItem("medium", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
|
||||
ThinkingMenuItem("high", thinkingLevel, onSetThinkingLevel) { showThinkingMenu = false }
|
||||
}
|
||||
}
|
||||
|
||||
SecondaryActionButton(
|
||||
label = "Attach",
|
||||
icon = Icons.Default.AttachFile,
|
||||
enabled = true,
|
||||
onClick = onPickImages,
|
||||
)
|
||||
}
|
||||
|
||||
if (attachments.isNotEmpty()) {
|
||||
AttachmentsStrip(attachments = attachments, onRemoveAttachment = onRemoveAttachment)
|
||||
}
|
||||
|
||||
HorizontalDivider(color = mobileBorder)
|
||||
|
||||
Text(
|
||||
text = "MESSAGE",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 0.9.sp),
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
|
||||
OutlinedTextField(
|
||||
value = input,
|
||||
onValueChange = { input = it },
|
||||
modifier = Modifier.fillMaxWidth().height(92.dp),
|
||||
placeholder = { Text("Type a message", style = mobileBodyStyle(), color = mobileTextTertiary) },
|
||||
minLines = 2,
|
||||
maxLines = 5,
|
||||
textStyle = mobileBodyStyle().copy(color = mobileText),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors = chatTextFieldColors(),
|
||||
)
|
||||
|
||||
if (!healthOk) {
|
||||
Text(
|
||||
text = "Gateway is offline. Connect first in the Connect tab.",
|
||||
style = mobileCallout,
|
||||
color = ai.openclaw.android.ui.mobileWarning,
|
||||
)
|
||||
}
|
||||
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(10.dp),
|
||||
) {
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
SecondaryActionButton(
|
||||
label = "Refresh",
|
||||
icon = Icons.Default.Refresh,
|
||||
enabled = true,
|
||||
compact = true,
|
||||
onClick = onRefresh,
|
||||
)
|
||||
|
||||
SecondaryActionButton(
|
||||
label = "Abort",
|
||||
icon = Icons.Default.Stop,
|
||||
enabled = pendingRunCount > 0,
|
||||
compact = true,
|
||||
onClick = onAbort,
|
||||
)
|
||||
}
|
||||
|
||||
Button(
|
||||
onClick = {
|
||||
val text = input
|
||||
input = ""
|
||||
onSend(text)
|
||||
},
|
||||
enabled = canSend,
|
||||
modifier = Modifier.weight(1f).height(48.dp),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors =
|
||||
ButtonDefaults.buttonColors(
|
||||
containerColor = mobileAccent,
|
||||
contentColor = Color.White,
|
||||
disabledContainerColor = mobileBorderStrong,
|
||||
disabledContentColor = mobileTextTertiary,
|
||||
),
|
||||
border = BorderStroke(1.dp, if (canSend) Color(0xFF154CAD) else mobileBorderStrong),
|
||||
) {
|
||||
if (sendBusy) {
|
||||
CircularProgressIndicator(modifier = Modifier.size(16.dp), strokeWidth = 2.dp, color = Color.White)
|
||||
} else {
|
||||
Icon(Icons.AutoMirrored.Filled.Send, contentDescription = null, modifier = Modifier.size(16.dp))
|
||||
}
|
||||
Spacer(modifier = Modifier.width(8.dp))
|
||||
Text(
|
||||
text = "Send",
|
||||
style = mobileHeadline.copy(fontWeight = FontWeight.Bold),
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun SecondaryActionButton(
|
||||
label: String,
|
||||
icon: androidx.compose.ui.graphics.vector.ImageVector,
|
||||
enabled: Boolean,
|
||||
compact: Boolean = false,
|
||||
onClick: () -> Unit,
|
||||
) {
|
||||
Button(
|
||||
onClick = onClick,
|
||||
enabled = enabled,
|
||||
modifier = if (compact) Modifier.size(44.dp) else Modifier.height(44.dp),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
colors =
|
||||
ButtonDefaults.buttonColors(
|
||||
containerColor = Color.White,
|
||||
contentColor = mobileTextSecondary,
|
||||
disabledContainerColor = Color.White,
|
||||
disabledContentColor = mobileTextTertiary,
|
||||
),
|
||||
border = BorderStroke(1.dp, mobileBorderStrong),
|
||||
contentPadding = if (compact) PaddingValues(0.dp) else ButtonDefaults.ContentPadding,
|
||||
) {
|
||||
Icon(icon, contentDescription = label, modifier = Modifier.size(14.dp))
|
||||
if (!compact) {
|
||||
Spacer(modifier = Modifier.width(5.dp))
|
||||
Text(
|
||||
text = label,
|
||||
style = mobileCallout.copy(fontWeight = FontWeight.SemiBold),
|
||||
color = if (enabled) mobileTextSecondary else mobileTextTertiary,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ThinkingMenuItem(
|
||||
value: String,
|
||||
current: String,
|
||||
onSet: (String) -> Unit,
|
||||
onDismiss: () -> Unit,
|
||||
) {
|
||||
DropdownMenuItem(
|
||||
text = { Text(thinkingLabel(value), style = mobileCallout, color = mobileText) },
|
||||
onClick = {
|
||||
onSet(value)
|
||||
onDismiss()
|
||||
},
|
||||
trailingIcon = {
|
||||
if (value == current.trim().lowercase()) {
|
||||
Text("✓", style = mobileCallout, color = mobileAccent)
|
||||
} else {
|
||||
Spacer(modifier = Modifier.width(10.dp))
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
private fun thinkingLabel(raw: String): String {
|
||||
return when (raw.trim().lowercase()) {
|
||||
"low" -> "Low"
|
||||
"medium" -> "Medium"
|
||||
"high" -> "High"
|
||||
else -> "Off"
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun AttachmentsStrip(
|
||||
attachments: List<PendingImageAttachment>,
|
||||
onRemoveAttachment: (id: String) -> Unit,
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()),
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
for (att in attachments) {
|
||||
AttachmentChip(
|
||||
fileName = att.fileName,
|
||||
onRemove = { onRemoveAttachment(att.id) },
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun AttachmentChip(fileName: String, onRemove: () -> Unit) {
|
||||
Surface(
|
||||
shape = RoundedCornerShape(999.dp),
|
||||
color = mobileAccentSoft,
|
||||
border = BorderStroke(1.dp, mobileBorderStrong),
|
||||
) {
|
||||
Row(
|
||||
modifier = Modifier.padding(horizontal = 10.dp, vertical = 6.dp),
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
Text(
|
||||
text = fileName,
|
||||
style = mobileCaption1,
|
||||
color = mobileText,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
)
|
||||
Surface(
|
||||
onClick = onRemove,
|
||||
shape = RoundedCornerShape(999.dp),
|
||||
color = Color.White,
|
||||
border = BorderStroke(1.dp, mobileBorderStrong),
|
||||
) {
|
||||
Text(
|
||||
text = "×",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold),
|
||||
color = mobileTextSecondary,
|
||||
modifier = Modifier.padding(horizontal = 8.dp, vertical = 2.dp),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun chatTextFieldColors() =
|
||||
OutlinedTextFieldDefaults.colors(
|
||||
focusedContainerColor = mobileSurface,
|
||||
unfocusedContainerColor = mobileSurface,
|
||||
focusedBorderColor = mobileAccent,
|
||||
unfocusedBorderColor = mobileBorder,
|
||||
focusedTextColor = mobileText,
|
||||
unfocusedTextColor = mobileText,
|
||||
cursorColor = mobileAccent,
|
||||
)
|
||||
|
||||
@Composable
|
||||
private fun mobileBodyStyle() =
|
||||
MaterialTheme.typography.bodyMedium.copy(
|
||||
fontFamily = ai.openclaw.android.ui.mobileFontFamily,
|
||||
fontWeight = FontWeight.Medium,
|
||||
fontSize = 15.sp,
|
||||
lineHeight = 22.sp,
|
||||
)
|
||||
@@ -0,0 +1,591 @@
|
||||
package ai.openclaw.android.ui.chat
|
||||
|
||||
import android.graphics.BitmapFactory
|
||||
import android.util.Base64
|
||||
import androidx.compose.foundation.Image
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.border
|
||||
import androidx.compose.foundation.horizontalScroll
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.IntrinsicSize
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxHeight
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.height
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.text.selection.SelectionContainer
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.asImageBitmap
|
||||
import androidx.compose.ui.layout.ContentScale
|
||||
import androidx.compose.ui.text.AnnotatedString
|
||||
import androidx.compose.ui.text.SpanStyle
|
||||
import androidx.compose.ui.text.TextStyle
|
||||
import androidx.compose.ui.text.buildAnnotatedString
|
||||
import androidx.compose.ui.text.font.FontFamily
|
||||
import androidx.compose.ui.text.font.FontStyle
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.text.style.TextDecoration
|
||||
import androidx.compose.ui.text.withStyle
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.unit.sp
|
||||
import ai.openclaw.android.ui.mobileAccent
|
||||
import ai.openclaw.android.ui.mobileCallout
|
||||
import ai.openclaw.android.ui.mobileCaption1
|
||||
import ai.openclaw.android.ui.mobileCodeBg
|
||||
import ai.openclaw.android.ui.mobileCodeText
|
||||
import ai.openclaw.android.ui.mobileTextSecondary
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import org.commonmark.Extension
|
||||
import org.commonmark.ext.autolink.AutolinkExtension
|
||||
import org.commonmark.ext.gfm.strikethrough.Strikethrough
|
||||
import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension
|
||||
import org.commonmark.ext.gfm.tables.TableBlock
|
||||
import org.commonmark.ext.gfm.tables.TableBody
|
||||
import org.commonmark.ext.gfm.tables.TableCell
|
||||
import org.commonmark.ext.gfm.tables.TableHead
|
||||
import org.commonmark.ext.gfm.tables.TableRow
|
||||
import org.commonmark.ext.gfm.tables.TablesExtension
|
||||
import org.commonmark.ext.task.list.items.TaskListItemMarker
|
||||
import org.commonmark.ext.task.list.items.TaskListItemsExtension
|
||||
import org.commonmark.node.BlockQuote
|
||||
import org.commonmark.node.BulletList
|
||||
import org.commonmark.node.Code
|
||||
import org.commonmark.node.Document
|
||||
import org.commonmark.node.Emphasis
|
||||
import org.commonmark.node.FencedCodeBlock
|
||||
import org.commonmark.node.Heading
|
||||
import org.commonmark.node.HardLineBreak
|
||||
import org.commonmark.node.HtmlBlock
|
||||
import org.commonmark.node.HtmlInline
|
||||
import org.commonmark.node.Image as MarkdownImage
|
||||
import org.commonmark.node.IndentedCodeBlock
|
||||
import org.commonmark.node.Link
|
||||
import org.commonmark.node.ListItem
|
||||
import org.commonmark.node.Node
|
||||
import org.commonmark.node.OrderedList
|
||||
import org.commonmark.node.Paragraph
|
||||
import org.commonmark.node.SoftLineBreak
|
||||
import org.commonmark.node.StrongEmphasis
|
||||
import org.commonmark.node.Text as MarkdownTextNode
|
||||
import org.commonmark.node.ThematicBreak
|
||||
import org.commonmark.parser.Parser
|
||||
|
||||
private const val LIST_INDENT_DP = 14
|
||||
private val dataImageRegex = Regex("^data:image/([a-zA-Z0-9+.-]+);base64,([A-Za-z0-9+/=\\n\\r]+)$")
|
||||
|
||||
private val markdownParser: Parser by lazy {
|
||||
val extensions: List<Extension> =
|
||||
listOf(
|
||||
AutolinkExtension.create(),
|
||||
StrikethroughExtension.create(),
|
||||
TablesExtension.create(),
|
||||
TaskListItemsExtension.create(),
|
||||
)
|
||||
Parser.builder()
|
||||
.extensions(extensions)
|
||||
.build()
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun ChatMarkdown(text: String, textColor: Color) {
|
||||
val document = remember(text) { markdownParser.parse(text) as Document }
|
||||
val inlineStyles = InlineStyles(inlineCodeBg = mobileCodeBg, inlineCodeColor = mobileCodeText)
|
||||
|
||||
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
|
||||
RenderMarkdownBlocks(
|
||||
start = document.firstChild,
|
||||
textColor = textColor,
|
||||
inlineStyles = inlineStyles,
|
||||
listDepth = 0,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun RenderMarkdownBlocks(
|
||||
start: Node?,
|
||||
textColor: Color,
|
||||
inlineStyles: InlineStyles,
|
||||
listDepth: Int,
|
||||
) {
|
||||
var node = start
|
||||
while (node != null) {
|
||||
val current = node
|
||||
when (current) {
|
||||
is Paragraph -> {
|
||||
RenderParagraph(current, textColor = textColor, inlineStyles = inlineStyles)
|
||||
}
|
||||
is Heading -> {
|
||||
val headingText = remember(current) { buildInlineMarkdown(current.firstChild, inlineStyles) }
|
||||
Text(
|
||||
text = headingText,
|
||||
style = headingStyle(current.level),
|
||||
color = textColor,
|
||||
)
|
||||
}
|
||||
is FencedCodeBlock -> {
|
||||
SelectionContainer(modifier = Modifier.fillMaxWidth()) {
|
||||
ChatCodeBlock(code = current.literal.orEmpty(), language = current.info?.trim()?.ifEmpty { null })
|
||||
}
|
||||
}
|
||||
is IndentedCodeBlock -> {
|
||||
SelectionContainer(modifier = Modifier.fillMaxWidth()) {
|
||||
ChatCodeBlock(code = current.literal.orEmpty(), language = null)
|
||||
}
|
||||
}
|
||||
is BlockQuote -> {
|
||||
Row(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.height(IntrinsicSize.Min)
|
||||
.padding(vertical = 2.dp),
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
verticalAlignment = Alignment.Top,
|
||||
) {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.width(2.dp)
|
||||
.fillMaxHeight()
|
||||
.background(mobileTextSecondary.copy(alpha = 0.35f)),
|
||||
)
|
||||
Column(
|
||||
modifier = Modifier.weight(1f),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
RenderMarkdownBlocks(
|
||||
start = current.firstChild,
|
||||
textColor = textColor,
|
||||
inlineStyles = inlineStyles,
|
||||
listDepth = listDepth,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
is BulletList -> {
|
||||
RenderBulletList(
|
||||
list = current,
|
||||
textColor = textColor,
|
||||
inlineStyles = inlineStyles,
|
||||
listDepth = listDepth,
|
||||
)
|
||||
}
|
||||
is OrderedList -> {
|
||||
RenderOrderedList(
|
||||
list = current,
|
||||
textColor = textColor,
|
||||
inlineStyles = inlineStyles,
|
||||
listDepth = listDepth,
|
||||
)
|
||||
}
|
||||
is TableBlock -> {
|
||||
RenderTableBlock(
|
||||
table = current,
|
||||
textColor = textColor,
|
||||
inlineStyles = inlineStyles,
|
||||
)
|
||||
}
|
||||
is ThematicBreak -> {
|
||||
Box(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.height(1.dp)
|
||||
.background(mobileTextSecondary.copy(alpha = 0.25f)),
|
||||
)
|
||||
}
|
||||
is HtmlBlock -> {
|
||||
val literal = current.literal.orEmpty().trim()
|
||||
if (literal.isNotEmpty()) {
|
||||
Text(
|
||||
text = literal,
|
||||
style = mobileCallout.copy(fontFamily = FontFamily.Monospace),
|
||||
color = textColor,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
node = current.next
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun RenderParagraph(
|
||||
paragraph: Paragraph,
|
||||
textColor: Color,
|
||||
inlineStyles: InlineStyles,
|
||||
) {
|
||||
val standaloneImage = remember(paragraph) { standaloneDataImage(paragraph) }
|
||||
if (standaloneImage != null) {
|
||||
InlineBase64Image(base64 = standaloneImage.base64, mimeType = standaloneImage.mimeType)
|
||||
return
|
||||
}
|
||||
|
||||
val annotated = remember(paragraph) { buildInlineMarkdown(paragraph.firstChild, inlineStyles) }
|
||||
if (annotated.text.trimEnd().isEmpty()) {
|
||||
return
|
||||
}
|
||||
|
||||
Text(
|
||||
text = annotated,
|
||||
style = mobileCallout,
|
||||
color = textColor,
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun RenderBulletList(
|
||||
list: BulletList,
|
||||
textColor: Color,
|
||||
inlineStyles: InlineStyles,
|
||||
listDepth: Int,
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier.padding(start = (LIST_INDENT_DP * listDepth).dp),
|
||||
verticalArrangement = Arrangement.spacedBy(6.dp),
|
||||
) {
|
||||
var item = list.firstChild
|
||||
while (item != null) {
|
||||
if (item is ListItem) {
|
||||
RenderListItem(
|
||||
item = item,
|
||||
markerText = "•",
|
||||
textColor = textColor,
|
||||
inlineStyles = inlineStyles,
|
||||
listDepth = listDepth,
|
||||
)
|
||||
}
|
||||
item = item.next
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun RenderOrderedList(
|
||||
list: OrderedList,
|
||||
textColor: Color,
|
||||
inlineStyles: InlineStyles,
|
||||
listDepth: Int,
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier.padding(start = (LIST_INDENT_DP * listDepth).dp),
|
||||
verticalArrangement = Arrangement.spacedBy(6.dp),
|
||||
) {
|
||||
var index = list.markerStartNumber ?: 1
|
||||
var item = list.firstChild
|
||||
while (item != null) {
|
||||
if (item is ListItem) {
|
||||
RenderListItem(
|
||||
item = item,
|
||||
markerText = "$index.",
|
||||
textColor = textColor,
|
||||
inlineStyles = inlineStyles,
|
||||
listDepth = listDepth,
|
||||
)
|
||||
index += 1
|
||||
}
|
||||
item = item.next
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun RenderListItem(
|
||||
item: ListItem,
|
||||
markerText: String,
|
||||
textColor: Color,
|
||||
inlineStyles: InlineStyles,
|
||||
listDepth: Int,
|
||||
) {
|
||||
var contentStart = item.firstChild
|
||||
var marker = markerText
|
||||
val task = contentStart as? TaskListItemMarker
|
||||
if (task != null) {
|
||||
marker = if (task.isChecked) "☑" else "☐"
|
||||
contentStart = task.next
|
||||
}
|
||||
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
verticalAlignment = Alignment.Top,
|
||||
) {
|
||||
Text(
|
||||
text = marker,
|
||||
style = mobileCallout.copy(fontWeight = FontWeight.SemiBold),
|
||||
color = textColor,
|
||||
modifier = Modifier.width(24.dp),
|
||||
)
|
||||
|
||||
Column(
|
||||
modifier = Modifier.weight(1f),
|
||||
verticalArrangement = Arrangement.spacedBy(4.dp),
|
||||
) {
|
||||
RenderMarkdownBlocks(
|
||||
start = contentStart,
|
||||
textColor = textColor,
|
||||
inlineStyles = inlineStyles,
|
||||
listDepth = listDepth + 1,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun RenderTableBlock(
|
||||
table: TableBlock,
|
||||
textColor: Color,
|
||||
inlineStyles: InlineStyles,
|
||||
) {
|
||||
val rows = remember(table) { buildTableRows(table, inlineStyles) }
|
||||
if (rows.isEmpty()) return
|
||||
|
||||
val maxCols = rows.maxOf { row -> row.cells.size }.coerceAtLeast(1)
|
||||
val scrollState = rememberScrollState()
|
||||
|
||||
Column(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.horizontalScroll(scrollState)
|
||||
.border(1.dp, mobileTextSecondary.copy(alpha = 0.25f)),
|
||||
) {
|
||||
for (row in rows) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
) {
|
||||
for (index in 0 until maxCols) {
|
||||
val cell = row.cells.getOrNull(index) ?: AnnotatedString("")
|
||||
Text(
|
||||
text = cell,
|
||||
style = if (row.isHeader) mobileCaption1.copy(fontWeight = FontWeight.SemiBold) else mobileCallout,
|
||||
color = textColor,
|
||||
modifier = Modifier
|
||||
.border(1.dp, mobileTextSecondary.copy(alpha = 0.22f))
|
||||
.padding(horizontal = 8.dp, vertical = 6.dp)
|
||||
.width(160.dp),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildTableRows(table: TableBlock, inlineStyles: InlineStyles): List<TableRenderRow> {
|
||||
val rows = mutableListOf<TableRenderRow>()
|
||||
var child = table.firstChild
|
||||
while (child != null) {
|
||||
when (child) {
|
||||
is TableHead -> rows.addAll(readTableSection(child, isHeader = true, inlineStyles = inlineStyles))
|
||||
is TableBody -> rows.addAll(readTableSection(child, isHeader = false, inlineStyles = inlineStyles))
|
||||
is TableRow -> rows.add(readTableRow(child, isHeader = false, inlineStyles = inlineStyles))
|
||||
}
|
||||
child = child.next
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
private fun readTableSection(section: Node, isHeader: Boolean, inlineStyles: InlineStyles): List<TableRenderRow> {
|
||||
val rows = mutableListOf<TableRenderRow>()
|
||||
var row = section.firstChild
|
||||
while (row != null) {
|
||||
if (row is TableRow) {
|
||||
rows.add(readTableRow(row, isHeader = isHeader, inlineStyles = inlineStyles))
|
||||
}
|
||||
row = row.next
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
private fun readTableRow(row: TableRow, isHeader: Boolean, inlineStyles: InlineStyles): TableRenderRow {
|
||||
val cells = mutableListOf<AnnotatedString>()
|
||||
var cellNode = row.firstChild
|
||||
while (cellNode != null) {
|
||||
if (cellNode is TableCell) {
|
||||
cells.add(buildInlineMarkdown(cellNode.firstChild, inlineStyles))
|
||||
}
|
||||
cellNode = cellNode.next
|
||||
}
|
||||
return TableRenderRow(isHeader = isHeader, cells = cells)
|
||||
}
|
||||
|
||||
private fun buildInlineMarkdown(start: Node?, inlineStyles: InlineStyles): AnnotatedString {
|
||||
return buildAnnotatedString {
|
||||
appendInlineNode(
|
||||
node = start,
|
||||
inlineCodeBg = inlineStyles.inlineCodeBg,
|
||||
inlineCodeColor = inlineStyles.inlineCodeColor,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun AnnotatedString.Builder.appendInlineNode(
|
||||
node: Node?,
|
||||
inlineCodeBg: Color,
|
||||
inlineCodeColor: Color,
|
||||
) {
|
||||
var current = node
|
||||
while (current != null) {
|
||||
when (current) {
|
||||
is MarkdownTextNode -> append(current.literal)
|
||||
is SoftLineBreak -> append('\n')
|
||||
is HardLineBreak -> append('\n')
|
||||
is Code -> {
|
||||
withStyle(
|
||||
SpanStyle(
|
||||
fontFamily = FontFamily.Monospace,
|
||||
background = inlineCodeBg,
|
||||
color = inlineCodeColor,
|
||||
),
|
||||
) {
|
||||
append(current.literal)
|
||||
}
|
||||
}
|
||||
is Emphasis -> {
|
||||
withStyle(SpanStyle(fontStyle = FontStyle.Italic)) {
|
||||
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
|
||||
}
|
||||
}
|
||||
is StrongEmphasis -> {
|
||||
withStyle(SpanStyle(fontWeight = FontWeight.SemiBold)) {
|
||||
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
|
||||
}
|
||||
}
|
||||
is Strikethrough -> {
|
||||
withStyle(SpanStyle(textDecoration = TextDecoration.LineThrough)) {
|
||||
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
|
||||
}
|
||||
}
|
||||
is Link -> {
|
||||
withStyle(
|
||||
SpanStyle(
|
||||
color = mobileAccent,
|
||||
textDecoration = TextDecoration.Underline,
|
||||
),
|
||||
) {
|
||||
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
|
||||
}
|
||||
}
|
||||
is MarkdownImage -> {
|
||||
val alt = buildPlainText(current.firstChild)
|
||||
if (alt.isNotBlank()) {
|
||||
append(alt)
|
||||
} else {
|
||||
append("image")
|
||||
}
|
||||
}
|
||||
is HtmlInline -> {
|
||||
if (!current.literal.isNullOrBlank()) {
|
||||
append(current.literal)
|
||||
}
|
||||
}
|
||||
else -> {
|
||||
appendInlineNode(current.firstChild, inlineCodeBg = inlineCodeBg, inlineCodeColor = inlineCodeColor)
|
||||
}
|
||||
}
|
||||
current = current.next
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildPlainText(start: Node?): String {
|
||||
val sb = StringBuilder()
|
||||
var node = start
|
||||
while (node != null) {
|
||||
when (node) {
|
||||
is MarkdownTextNode -> sb.append(node.literal)
|
||||
is SoftLineBreak, is HardLineBreak -> sb.append('\n')
|
||||
else -> sb.append(buildPlainText(node.firstChild))
|
||||
}
|
||||
node = node.next
|
||||
}
|
||||
return sb.toString()
|
||||
}
|
||||
|
||||
private fun standaloneDataImage(paragraph: Paragraph): ParsedDataImage? {
|
||||
val only = paragraph.firstChild as? MarkdownImage ?: return null
|
||||
if (only.next != null) return null
|
||||
return parseDataImageDestination(only.destination)
|
||||
}
|
||||
|
||||
private fun parseDataImageDestination(destination: String?): ParsedDataImage? {
|
||||
val raw = destination?.trim().orEmpty()
|
||||
if (raw.isEmpty()) return null
|
||||
val match = dataImageRegex.matchEntire(raw) ?: return null
|
||||
val subtype = match.groupValues.getOrNull(1)?.trim()?.ifEmpty { "png" } ?: "png"
|
||||
val base64 = match.groupValues.getOrNull(2)?.replace("\n", "")?.replace("\r", "")?.trim().orEmpty()
|
||||
if (base64.isEmpty()) return null
|
||||
return ParsedDataImage(mimeType = "image/$subtype", base64 = base64)
|
||||
}
|
||||
|
||||
private fun headingStyle(level: Int): TextStyle {
|
||||
return when (level.coerceIn(1, 6)) {
|
||||
1 -> mobileCallout.copy(fontSize = 22.sp, lineHeight = 28.sp, fontWeight = FontWeight.Bold)
|
||||
2 -> mobileCallout.copy(fontSize = 20.sp, lineHeight = 26.sp, fontWeight = FontWeight.Bold)
|
||||
3 -> mobileCallout.copy(fontSize = 18.sp, lineHeight = 24.sp, fontWeight = FontWeight.SemiBold)
|
||||
4 -> mobileCallout.copy(fontSize = 16.sp, lineHeight = 22.sp, fontWeight = FontWeight.SemiBold)
|
||||
else -> mobileCallout.copy(fontWeight = FontWeight.SemiBold)
|
||||
}
|
||||
}
|
||||
|
||||
private data class InlineStyles(
|
||||
val inlineCodeBg: Color,
|
||||
val inlineCodeColor: Color,
|
||||
)
|
||||
|
||||
private data class TableRenderRow(
|
||||
val isHeader: Boolean,
|
||||
val cells: List<AnnotatedString>,
|
||||
)
|
||||
|
||||
private data class ParsedDataImage(
|
||||
val mimeType: String,
|
||||
val base64: String,
|
||||
)
|
||||
|
||||
@Composable
|
||||
private fun InlineBase64Image(base64: String, mimeType: String?) {
|
||||
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }
|
||||
var failed by remember(base64) { mutableStateOf(false) }
|
||||
|
||||
LaunchedEffect(base64) {
|
||||
failed = false
|
||||
image =
|
||||
withContext(Dispatchers.Default) {
|
||||
try {
|
||||
val bytes = Base64.decode(base64, Base64.DEFAULT)
|
||||
val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null
|
||||
bitmap.asImageBitmap()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
if (image == null) failed = true
|
||||
}
|
||||
|
||||
if (image != null) {
|
||||
Image(
|
||||
bitmap = image!!,
|
||||
contentDescription = mimeType ?: "image",
|
||||
contentScale = ContentScale.Fit,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
)
|
||||
} else if (failed) {
|
||||
Text(
|
||||
text = "Image unavailable",
|
||||
modifier = Modifier.padding(vertical = 2.dp),
|
||||
style = mobileCaption1,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package ai.openclaw.android.ui.chat
|
||||
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.lazy.LazyColumn
|
||||
import androidx.compose.foundation.lazy.rememberLazyListState
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.unit.dp
|
||||
import ai.openclaw.android.chat.ChatMessage
|
||||
import ai.openclaw.android.chat.ChatPendingToolCall
|
||||
import ai.openclaw.android.ui.mobileBorder
|
||||
import ai.openclaw.android.ui.mobileCallout
|
||||
import ai.openclaw.android.ui.mobileHeadline
|
||||
import ai.openclaw.android.ui.mobileText
|
||||
import ai.openclaw.android.ui.mobileTextSecondary
|
||||
|
||||
@Composable
|
||||
fun ChatMessageListCard(
|
||||
messages: List<ChatMessage>,
|
||||
pendingRunCount: Int,
|
||||
pendingToolCalls: List<ChatPendingToolCall>,
|
||||
streamingAssistantText: String?,
|
||||
healthOk: Boolean,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
val listState = rememberLazyListState()
|
||||
|
||||
// With reverseLayout the newest item is at index 0 (bottom of screen).
|
||||
LaunchedEffect(messages.size, pendingRunCount, pendingToolCalls.size, streamingAssistantText) {
|
||||
listState.animateScrollToItem(index = 0)
|
||||
}
|
||||
|
||||
Box(modifier = modifier.fillMaxWidth()) {
|
||||
LazyColumn(
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
state = listState,
|
||||
reverseLayout = true,
|
||||
verticalArrangement = Arrangement.spacedBy(10.dp),
|
||||
contentPadding = androidx.compose.foundation.layout.PaddingValues(bottom = 8.dp),
|
||||
) {
|
||||
// With reverseLayout = true, index 0 renders at the BOTTOM.
|
||||
// So we emit newest items first: streaming → tools → typing → messages (newest→oldest).
|
||||
|
||||
val stream = streamingAssistantText?.trim()
|
||||
if (!stream.isNullOrEmpty()) {
|
||||
item(key = "stream") {
|
||||
ChatStreamingAssistantBubble(text = stream)
|
||||
}
|
||||
}
|
||||
|
||||
if (pendingToolCalls.isNotEmpty()) {
|
||||
item(key = "tools") {
|
||||
ChatPendingToolsBubble(toolCalls = pendingToolCalls)
|
||||
}
|
||||
}
|
||||
|
||||
if (pendingRunCount > 0) {
|
||||
item(key = "typing") {
|
||||
ChatTypingIndicatorBubble()
|
||||
}
|
||||
}
|
||||
|
||||
items(count = messages.size, key = { idx -> messages[messages.size - 1 - idx].id }) { idx ->
|
||||
ChatMessageBubble(message = messages[messages.size - 1 - idx])
|
||||
}
|
||||
}
|
||||
|
||||
if (messages.isEmpty() && pendingRunCount == 0 && pendingToolCalls.isEmpty() && streamingAssistantText.isNullOrBlank()) {
|
||||
EmptyChatHint(modifier = Modifier.align(Alignment.Center), healthOk = healthOk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun EmptyChatHint(modifier: Modifier = Modifier, healthOk: Boolean) {
|
||||
Surface(
|
||||
modifier = modifier.fillMaxWidth(),
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
color = androidx.compose.ui.graphics.Color.White.copy(alpha = 0.9f),
|
||||
border = androidx.compose.foundation.BorderStroke(1.dp, mobileBorder),
|
||||
) {
|
||||
androidx.compose.foundation.layout.Column(
|
||||
modifier = Modifier.padding(horizontal = 12.dp, vertical = 12.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(4.dp),
|
||||
) {
|
||||
Text("No messages yet", style = mobileHeadline, color = mobileText)
|
||||
Text(
|
||||
text =
|
||||
if (healthOk) {
|
||||
"Send the first prompt to start this session."
|
||||
} else {
|
||||
"Connect gateway first, then return to chat."
|
||||
},
|
||||
style = mobileCallout,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
package ai.openclaw.android.ui.chat
|
||||
|
||||
import android.graphics.BitmapFactory
|
||||
import android.util.Base64
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.Image
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.draw.alpha
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.asImageBitmap
|
||||
import androidx.compose.ui.layout.ContentScale
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.text.font.FontFamily
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.unit.sp
|
||||
import ai.openclaw.android.chat.ChatMessage
|
||||
import ai.openclaw.android.chat.ChatMessageContent
|
||||
import ai.openclaw.android.chat.ChatPendingToolCall
|
||||
import ai.openclaw.android.tools.ToolDisplayRegistry
|
||||
import ai.openclaw.android.ui.mobileAccent
|
||||
import ai.openclaw.android.ui.mobileAccentSoft
|
||||
import ai.openclaw.android.ui.mobileBorder
|
||||
import ai.openclaw.android.ui.mobileBorderStrong
|
||||
import ai.openclaw.android.ui.mobileCallout
|
||||
import ai.openclaw.android.ui.mobileCaption1
|
||||
import ai.openclaw.android.ui.mobileCaption2
|
||||
import ai.openclaw.android.ui.mobileCodeBg
|
||||
import ai.openclaw.android.ui.mobileCodeText
|
||||
import ai.openclaw.android.ui.mobileHeadline
|
||||
import ai.openclaw.android.ui.mobileText
|
||||
import ai.openclaw.android.ui.mobileTextSecondary
|
||||
import ai.openclaw.android.ui.mobileWarning
|
||||
import ai.openclaw.android.ui.mobileWarningSoft
|
||||
import java.util.Locale
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
private data class ChatBubbleStyle(
|
||||
val alignEnd: Boolean,
|
||||
val containerColor: Color,
|
||||
val borderColor: Color,
|
||||
val roleColor: Color,
|
||||
)
|
||||
|
||||
@Composable
|
||||
fun ChatMessageBubble(message: ChatMessage) {
|
||||
val role = message.role.trim().lowercase(Locale.US)
|
||||
val style = bubbleStyle(role)
|
||||
|
||||
// Filter to only displayable content parts (text with content, or base64 images).
|
||||
val displayableContent =
|
||||
message.content.filter { part ->
|
||||
when (part.type) {
|
||||
"text" -> !part.text.isNullOrBlank()
|
||||
else -> part.base64 != null
|
||||
}
|
||||
}
|
||||
|
||||
if (displayableContent.isEmpty()) return
|
||||
|
||||
ChatBubbleContainer(style = style, roleLabel = roleLabel(role)) {
|
||||
ChatMessageBody(content = displayableContent, textColor = mobileText)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ChatBubbleContainer(
|
||||
style: ChatBubbleStyle,
|
||||
roleLabel: String,
|
||||
modifier: Modifier = Modifier,
|
||||
content: @Composable () -> Unit,
|
||||
) {
|
||||
Row(
|
||||
modifier = modifier.fillMaxWidth(),
|
||||
horizontalArrangement = if (style.alignEnd) Arrangement.End else Arrangement.Start,
|
||||
) {
|
||||
Surface(
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
border = BorderStroke(1.dp, style.borderColor),
|
||||
color = style.containerColor,
|
||||
tonalElevation = 0.dp,
|
||||
shadowElevation = 0.dp,
|
||||
modifier = Modifier.fillMaxWidth(0.90f),
|
||||
) {
|
||||
Column(
|
||||
modifier = Modifier.padding(horizontal = 11.dp, vertical = 8.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(3.dp),
|
||||
) {
|
||||
Text(
|
||||
text = roleLabel,
|
||||
style = mobileCaption2.copy(fontWeight = FontWeight.SemiBold, letterSpacing = 0.6.sp),
|
||||
color = style.roleColor,
|
||||
)
|
||||
content()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ChatMessageBody(content: List<ChatMessageContent>, textColor: Color) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
for (part in content) {
|
||||
when (part.type) {
|
||||
"text" -> {
|
||||
val text = part.text ?: continue
|
||||
ChatMarkdown(text = text, textColor = textColor)
|
||||
}
|
||||
else -> {
|
||||
val b64 = part.base64 ?: continue
|
||||
ChatBase64Image(base64 = b64, mimeType = part.mimeType)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun ChatTypingIndicatorBubble() {
|
||||
ChatBubbleContainer(
|
||||
style = bubbleStyle("assistant"),
|
||||
roleLabel = roleLabel("assistant"),
|
||||
) {
|
||||
Row(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
DotPulse(color = mobileTextSecondary)
|
||||
Text("Thinking...", style = mobileCallout, color = mobileTextSecondary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun ChatPendingToolsBubble(toolCalls: List<ChatPendingToolCall>) {
|
||||
val context = LocalContext.current
|
||||
val displays =
|
||||
remember(toolCalls, context) {
|
||||
toolCalls.map { ToolDisplayRegistry.resolve(context, it.name, it.args) }
|
||||
}
|
||||
|
||||
ChatBubbleContainer(
|
||||
style = bubbleStyle("assistant"),
|
||||
roleLabel = "TOOLS",
|
||||
) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(4.dp)) {
|
||||
Text("Running tools...", style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold), color = mobileTextSecondary)
|
||||
for (display in displays.take(6)) {
|
||||
Column(verticalArrangement = Arrangement.spacedBy(2.dp)) {
|
||||
Text(
|
||||
"${display.emoji} ${display.label}",
|
||||
style = mobileCallout,
|
||||
color = mobileTextSecondary,
|
||||
fontFamily = FontFamily.Monospace,
|
||||
)
|
||||
display.detailLine?.let { detail ->
|
||||
Text(
|
||||
detail,
|
||||
style = mobileCaption1,
|
||||
color = mobileTextSecondary,
|
||||
fontFamily = FontFamily.Monospace,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (toolCalls.size > 6) {
|
||||
Text(
|
||||
text = "... +${toolCalls.size - 6} more",
|
||||
style = mobileCaption1,
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun ChatStreamingAssistantBubble(text: String) {
|
||||
ChatBubbleContainer(
|
||||
style = bubbleStyle("assistant").copy(borderColor = mobileAccent),
|
||||
roleLabel = "ASSISTANT · LIVE",
|
||||
) {
|
||||
ChatMarkdown(text = text, textColor = mobileText)
|
||||
}
|
||||
}
|
||||
|
||||
private fun bubbleStyle(role: String): ChatBubbleStyle {
|
||||
return when (role) {
|
||||
"user" ->
|
||||
ChatBubbleStyle(
|
||||
alignEnd = true,
|
||||
containerColor = mobileAccentSoft,
|
||||
borderColor = mobileAccent,
|
||||
roleColor = mobileAccent,
|
||||
)
|
||||
|
||||
"system" ->
|
||||
ChatBubbleStyle(
|
||||
alignEnd = false,
|
||||
containerColor = mobileWarningSoft,
|
||||
borderColor = mobileWarning.copy(alpha = 0.45f),
|
||||
roleColor = mobileWarning,
|
||||
)
|
||||
|
||||
else ->
|
||||
ChatBubbleStyle(
|
||||
alignEnd = false,
|
||||
containerColor = Color.White,
|
||||
borderColor = mobileBorderStrong,
|
||||
roleColor = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun roleLabel(role: String): String {
|
||||
return when (role) {
|
||||
"user" -> "USER"
|
||||
"system" -> "SYSTEM"
|
||||
else -> "ASSISTANT"
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ChatBase64Image(base64: String, mimeType: String?) {
|
||||
var image by remember(base64) { mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null) }
|
||||
var failed by remember(base64) { mutableStateOf(false) }
|
||||
|
||||
LaunchedEffect(base64) {
|
||||
failed = false
|
||||
image =
|
||||
withContext(Dispatchers.Default) {
|
||||
try {
|
||||
val bytes = Base64.decode(base64, Base64.DEFAULT)
|
||||
val bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size) ?: return@withContext null
|
||||
bitmap.asImageBitmap()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
if (image == null) failed = true
|
||||
}
|
||||
|
||||
if (image != null) {
|
||||
Surface(
|
||||
shape = RoundedCornerShape(10.dp),
|
||||
border = BorderStroke(1.dp, mobileBorder),
|
||||
color = Color.White,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
) {
|
||||
Image(
|
||||
bitmap = image!!,
|
||||
contentDescription = mimeType ?: "attachment",
|
||||
contentScale = ContentScale.Fit,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
)
|
||||
}
|
||||
} else if (failed) {
|
||||
Text("Unsupported attachment", style = mobileCaption1, color = mobileTextSecondary)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun DotPulse(color: Color) {
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(5.dp), verticalAlignment = Alignment.CenterVertically) {
|
||||
PulseDot(alpha = 0.38f, color = color)
|
||||
PulseDot(alpha = 0.62f, color = color)
|
||||
PulseDot(alpha = 0.90f, color = color)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun PulseDot(alpha: Float, color: Color) {
|
||||
Surface(
|
||||
modifier = Modifier.size(6.dp).alpha(alpha),
|
||||
shape = CircleShape,
|
||||
color = color,
|
||||
) {}
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun ChatCodeBlock(code: String, language: String?) {
|
||||
Surface(
|
||||
shape = RoundedCornerShape(8.dp),
|
||||
color = mobileCodeBg,
|
||||
border = BorderStroke(1.dp, Color(0xFF2B2E35)),
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
) {
|
||||
Column(modifier = Modifier.padding(horizontal = 10.dp, vertical = 8.dp), verticalArrangement = Arrangement.spacedBy(4.dp)) {
|
||||
if (!language.isNullOrBlank()) {
|
||||
Text(
|
||||
text = language.uppercase(Locale.US),
|
||||
style = mobileCaption2.copy(letterSpacing = 0.4.sp),
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
}
|
||||
Text(
|
||||
text = code.trimEnd(),
|
||||
fontFamily = FontFamily.Monospace,
|
||||
style = mobileCallout,
|
||||
color = mobileCodeText,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
package ai.openclaw.android.ui.chat
|
||||
|
||||
import android.content.ContentResolver
|
||||
import android.net.Uri
|
||||
import android.util.Base64
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.horizontalScroll
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.imePadding
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||
import androidx.compose.material3.Surface
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.collectAsState
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateListOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.rememberCoroutineScope
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.text.style.TextOverflow
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.unit.sp
|
||||
import ai.openclaw.android.MainViewModel
|
||||
import ai.openclaw.android.chat.ChatSessionEntry
|
||||
import ai.openclaw.android.chat.OutgoingAttachment
|
||||
import ai.openclaw.android.ui.mobileAccent
|
||||
import ai.openclaw.android.ui.mobileBorder
|
||||
import ai.openclaw.android.ui.mobileBorderStrong
|
||||
import ai.openclaw.android.ui.mobileCallout
|
||||
import ai.openclaw.android.ui.mobileCaption1
|
||||
import ai.openclaw.android.ui.mobileCaption2
|
||||
import ai.openclaw.android.ui.mobileDanger
|
||||
import ai.openclaw.android.ui.mobileSuccess
|
||||
import ai.openclaw.android.ui.mobileSuccessSoft
|
||||
import ai.openclaw.android.ui.mobileText
|
||||
import ai.openclaw.android.ui.mobileTextSecondary
|
||||
import ai.openclaw.android.ui.mobileWarning
|
||||
import ai.openclaw.android.ui.mobileWarningSoft
|
||||
import java.io.ByteArrayOutputStream
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
@Composable
|
||||
fun ChatSheetContent(viewModel: MainViewModel) {
|
||||
val messages by viewModel.chatMessages.collectAsState()
|
||||
val errorText by viewModel.chatError.collectAsState()
|
||||
val pendingRunCount by viewModel.pendingRunCount.collectAsState()
|
||||
val healthOk by viewModel.chatHealthOk.collectAsState()
|
||||
val sessionKey by viewModel.chatSessionKey.collectAsState()
|
||||
val mainSessionKey by viewModel.mainSessionKey.collectAsState()
|
||||
val thinkingLevel by viewModel.chatThinkingLevel.collectAsState()
|
||||
val streamingAssistantText by viewModel.chatStreamingAssistantText.collectAsState()
|
||||
val pendingToolCalls by viewModel.chatPendingToolCalls.collectAsState()
|
||||
val sessions by viewModel.chatSessions.collectAsState()
|
||||
|
||||
LaunchedEffect(mainSessionKey) {
|
||||
viewModel.loadChat(mainSessionKey)
|
||||
viewModel.refreshChatSessions(limit = 200)
|
||||
}
|
||||
|
||||
val context = LocalContext.current
|
||||
val resolver = context.contentResolver
|
||||
val scope = rememberCoroutineScope()
|
||||
|
||||
val attachments = remember { mutableStateListOf<PendingImageAttachment>() }
|
||||
|
||||
val pickImages =
|
||||
rememberLauncherForActivityResult(ActivityResultContracts.GetMultipleContents()) { uris ->
|
||||
if (uris.isNullOrEmpty()) return@rememberLauncherForActivityResult
|
||||
scope.launch(Dispatchers.IO) {
|
||||
val next =
|
||||
uris.take(8).mapNotNull { uri ->
|
||||
try {
|
||||
loadImageAttachment(resolver, uri)
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
withContext(Dispatchers.Main) {
|
||||
attachments.addAll(next)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Column(
|
||||
modifier =
|
||||
Modifier
|
||||
.fillMaxSize()
|
||||
.padding(horizontal = 20.dp, vertical = 12.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
ChatThreadSelector(
|
||||
sessionKey = sessionKey,
|
||||
sessions = sessions,
|
||||
mainSessionKey = mainSessionKey,
|
||||
healthOk = healthOk,
|
||||
onSelectSession = { key -> viewModel.switchChatSession(key) },
|
||||
)
|
||||
|
||||
if (!errorText.isNullOrBlank()) {
|
||||
ChatErrorRail(errorText = errorText!!)
|
||||
}
|
||||
|
||||
ChatMessageListCard(
|
||||
messages = messages,
|
||||
pendingRunCount = pendingRunCount,
|
||||
pendingToolCalls = pendingToolCalls,
|
||||
streamingAssistantText = streamingAssistantText,
|
||||
healthOk = healthOk,
|
||||
modifier = Modifier.weight(1f, fill = true),
|
||||
)
|
||||
|
||||
Row(modifier = Modifier.fillMaxWidth().imePadding()) {
|
||||
ChatComposer(
|
||||
healthOk = healthOk,
|
||||
thinkingLevel = thinkingLevel,
|
||||
pendingRunCount = pendingRunCount,
|
||||
attachments = attachments,
|
||||
onPickImages = { pickImages.launch("image/*") },
|
||||
onRemoveAttachment = { id -> attachments.removeAll { it.id == id } },
|
||||
onSetThinkingLevel = { level -> viewModel.setChatThinkingLevel(level) },
|
||||
onRefresh = {
|
||||
viewModel.refreshChat()
|
||||
viewModel.refreshChatSessions(limit = 200)
|
||||
},
|
||||
onAbort = { viewModel.abortChat() },
|
||||
onSend = { text ->
|
||||
val outgoing =
|
||||
attachments.map { att ->
|
||||
OutgoingAttachment(
|
||||
type = "image",
|
||||
mimeType = att.mimeType,
|
||||
fileName = att.fileName,
|
||||
base64 = att.base64,
|
||||
)
|
||||
}
|
||||
viewModel.sendChat(message = text, thinking = thinkingLevel, attachments = outgoing)
|
||||
attachments.clear()
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ChatThreadSelector(
|
||||
sessionKey: String,
|
||||
sessions: List<ChatSessionEntry>,
|
||||
mainSessionKey: String,
|
||||
healthOk: Boolean,
|
||||
onSelectSession: (String) -> Unit,
|
||||
) {
|
||||
val sessionOptions = resolveSessionChoices(sessionKey, sessions, mainSessionKey = mainSessionKey)
|
||||
val currentSessionLabel =
|
||||
friendlySessionName(sessionOptions.firstOrNull { it.key == sessionKey }?.displayName ?: sessionKey)
|
||||
|
||||
Column(modifier = Modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
horizontalArrangement = Arrangement.SpaceBetween,
|
||||
verticalAlignment = androidx.compose.ui.Alignment.CenterVertically,
|
||||
) {
|
||||
Text(
|
||||
text = "SESSION",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.Bold, letterSpacing = 0.8.sp),
|
||||
color = mobileTextSecondary,
|
||||
)
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(6.dp), verticalAlignment = androidx.compose.ui.Alignment.CenterVertically) {
|
||||
Text(
|
||||
text = currentSessionLabel,
|
||||
style = mobileCallout.copy(fontWeight = FontWeight.SemiBold),
|
||||
color = mobileText,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
)
|
||||
ChatConnectionPill(healthOk = healthOk)
|
||||
}
|
||||
}
|
||||
|
||||
Row(
|
||||
modifier = Modifier.fillMaxWidth().horizontalScroll(rememberScrollState()),
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
for (entry in sessionOptions) {
|
||||
val active = entry.key == sessionKey
|
||||
Surface(
|
||||
onClick = { onSelectSession(entry.key) },
|
||||
shape = RoundedCornerShape(14.dp),
|
||||
color = if (active) mobileAccent else Color.White,
|
||||
border = BorderStroke(1.dp, if (active) Color(0xFF154CAD) else mobileBorderStrong),
|
||||
tonalElevation = 0.dp,
|
||||
shadowElevation = 0.dp,
|
||||
) {
|
||||
Text(
|
||||
text = friendlySessionName(entry.displayName ?: entry.key),
|
||||
style = mobileCaption1.copy(fontWeight = if (active) FontWeight.Bold else FontWeight.SemiBold),
|
||||
color = if (active) Color.White else mobileText,
|
||||
maxLines = 1,
|
||||
overflow = TextOverflow.Ellipsis,
|
||||
modifier = Modifier.padding(horizontal = 12.dp, vertical = 8.dp),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ChatConnectionPill(healthOk: Boolean) {
|
||||
Surface(
|
||||
shape = RoundedCornerShape(999.dp),
|
||||
color = if (healthOk) mobileSuccessSoft else mobileWarningSoft,
|
||||
border = BorderStroke(1.dp, if (healthOk) mobileSuccess.copy(alpha = 0.35f) else mobileWarning.copy(alpha = 0.35f)),
|
||||
) {
|
||||
Text(
|
||||
text = if (healthOk) "Connected" else "Offline",
|
||||
style = mobileCaption1.copy(fontWeight = FontWeight.SemiBold),
|
||||
color = if (healthOk) mobileSuccess else mobileWarning,
|
||||
modifier = Modifier.padding(horizontal = 8.dp, vertical = 3.dp),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ChatErrorRail(errorText: String) {
|
||||
Surface(
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
color = androidx.compose.ui.graphics.Color.White,
|
||||
shape = RoundedCornerShape(12.dp),
|
||||
border = androidx.compose.foundation.BorderStroke(1.dp, mobileDanger),
|
||||
) {
|
||||
Column(modifier = Modifier.padding(horizontal = 10.dp, vertical = 8.dp), verticalArrangement = Arrangement.spacedBy(2.dp)) {
|
||||
Text(
|
||||
text = "CHAT ERROR",
|
||||
style = mobileCaption2.copy(letterSpacing = 0.6.sp),
|
||||
color = mobileDanger,
|
||||
)
|
||||
Text(text = errorText, style = mobileCallout, color = mobileText)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data class PendingImageAttachment(
|
||||
val id: String,
|
||||
val fileName: String,
|
||||
val mimeType: String,
|
||||
val base64: String,
|
||||
)
|
||||
|
||||
private suspend fun loadImageAttachment(resolver: ContentResolver, uri: Uri): PendingImageAttachment {
|
||||
val mimeType = resolver.getType(uri) ?: "image/*"
|
||||
val fileName = (uri.lastPathSegment ?: "image").substringAfterLast('/')
|
||||
val bytes =
|
||||
withContext(Dispatchers.IO) {
|
||||
resolver.openInputStream(uri)?.use { input ->
|
||||
val out = ByteArrayOutputStream()
|
||||
input.copyTo(out)
|
||||
out.toByteArray()
|
||||
} ?: ByteArray(0)
|
||||
}
|
||||
if (bytes.isEmpty()) throw IllegalStateException("empty attachment")
|
||||
val base64 = Base64.encodeToString(bytes, Base64.NO_WRAP)
|
||||
return PendingImageAttachment(
|
||||
id = uri.toString() + "#" + System.currentTimeMillis().toString(),
|
||||
fileName = fileName,
|
||||
mimeType = mimeType,
|
||||
base64 = base64,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package ai.openclaw.android.ui.chat
|
||||
|
||||
import ai.openclaw.android.chat.ChatSessionEntry
|
||||
|
||||
private const val RECENT_WINDOW_MS = 24 * 60 * 60 * 1000L
|
||||
|
||||
/**
|
||||
* Derive a human-friendly label from a raw session key.
|
||||
* Examples:
|
||||
* "telegram:g-agent-main-main" -> "Main"
|
||||
* "agent:main:main" -> "Main"
|
||||
* "discord:g-server-channel" -> "Server Channel"
|
||||
* "my-custom-session" -> "My Custom Session"
|
||||
*/
|
||||
fun friendlySessionName(key: String): String {
|
||||
// Strip common prefixes like "telegram:", "agent:", "discord:" etc.
|
||||
val stripped = key.substringAfterLast(":")
|
||||
|
||||
// Remove leading "g-" prefix (gateway artifact)
|
||||
val cleaned = if (stripped.startsWith("g-")) stripped.removePrefix("g-") else stripped
|
||||
|
||||
// Split on hyphens/underscores, title-case each word, collapse "main main" -> "Main"
|
||||
val words = cleaned.split('-', '_').filter { it.isNotBlank() }.map { word ->
|
||||
word.replaceFirstChar { it.uppercaseChar() }
|
||||
}.distinct()
|
||||
|
||||
val result = words.joinToString(" ")
|
||||
return result.ifBlank { key }
|
||||
}
|
||||
|
||||
fun resolveSessionChoices(
|
||||
currentSessionKey: String,
|
||||
sessions: List<ChatSessionEntry>,
|
||||
mainSessionKey: String,
|
||||
nowMs: Long = System.currentTimeMillis(),
|
||||
): List<ChatSessionEntry> {
|
||||
val mainKey = mainSessionKey.trim().ifEmpty { "main" }
|
||||
val current = currentSessionKey.trim().let { if (it == "main" && mainKey != "main") mainKey else it }
|
||||
val aliasKey = if (mainKey == "main") null else "main"
|
||||
val cutoff = nowMs - RECENT_WINDOW_MS
|
||||
val sorted = sessions.sortedByDescending { it.updatedAtMs ?: 0L }
|
||||
val recent = mutableListOf<ChatSessionEntry>()
|
||||
val seen = mutableSetOf<String>()
|
||||
for (entry in sorted) {
|
||||
if (aliasKey != null && entry.key == aliasKey) continue
|
||||
if (!seen.add(entry.key)) continue
|
||||
if ((entry.updatedAtMs ?: 0L) < cutoff) continue
|
||||
recent.add(entry)
|
||||
}
|
||||
|
||||
val result = mutableListOf<ChatSessionEntry>()
|
||||
val included = mutableSetOf<String>()
|
||||
val mainEntry = sorted.firstOrNull { it.key == mainKey }
|
||||
if (mainEntry != null) {
|
||||
result.add(mainEntry)
|
||||
included.add(mainKey)
|
||||
} else if (current == mainKey) {
|
||||
result.add(ChatSessionEntry(key = mainKey, updatedAtMs = null))
|
||||
included.add(mainKey)
|
||||
}
|
||||
|
||||
for (entry in recent) {
|
||||
if (included.add(entry.key)) {
|
||||
result.add(entry)
|
||||
}
|
||||
}
|
||||
|
||||
if (current.isNotEmpty() && !included.contains(current)) {
|
||||
result.add(ChatSessionEntry(key = current, updatedAtMs = null))
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,523 @@
|
||||
package ai.openclaw.android.voice
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.os.Bundle
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.speech.RecognitionListener
|
||||
import android.speech.RecognizerIntent
|
||||
import android.speech.SpeechRecognizer
|
||||
import androidx.core.content.ContextCompat
|
||||
import java.util.UUID
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
|
||||
enum class VoiceConversationRole {
|
||||
User,
|
||||
Assistant,
|
||||
}
|
||||
|
||||
data class VoiceConversationEntry(
|
||||
val id: String,
|
||||
val role: VoiceConversationRole,
|
||||
val text: String,
|
||||
val isStreaming: Boolean = false,
|
||||
)
|
||||
|
||||
class MicCaptureManager(
|
||||
private val context: Context,
|
||||
private val scope: CoroutineScope,
|
||||
private val sendToGateway: suspend (String) -> String?,
|
||||
) {
|
||||
companion object {
|
||||
private const val speechMinSessionMs = 30_000L
|
||||
private const val speechCompleteSilenceMs = 1_500L
|
||||
private const val speechPossibleSilenceMs = 900L
|
||||
private const val maxConversationEntries = 40
|
||||
private const val pendingRunTimeoutMs = 45_000L
|
||||
}
|
||||
|
||||
private data class QueuedUtterance(
|
||||
val text: String,
|
||||
)
|
||||
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val json = Json { ignoreUnknownKeys = true }
|
||||
|
||||
private val _micEnabled = MutableStateFlow(false)
|
||||
val micEnabled: StateFlow<Boolean> = _micEnabled
|
||||
|
||||
private val _isListening = MutableStateFlow(false)
|
||||
val isListening: StateFlow<Boolean> = _isListening
|
||||
|
||||
private val _statusText = MutableStateFlow("Mic off")
|
||||
val statusText: StateFlow<String> = _statusText
|
||||
|
||||
private val _liveTranscript = MutableStateFlow<String?>(null)
|
||||
val liveTranscript: StateFlow<String?> = _liveTranscript
|
||||
|
||||
private val _queuedMessages = MutableStateFlow<List<String>>(emptyList())
|
||||
val queuedMessages: StateFlow<List<String>> = _queuedMessages
|
||||
|
||||
private val _conversation = MutableStateFlow<List<VoiceConversationEntry>>(emptyList())
|
||||
val conversation: StateFlow<List<VoiceConversationEntry>> = _conversation
|
||||
|
||||
private val _inputLevel = MutableStateFlow(0f)
|
||||
val inputLevel: StateFlow<Float> = _inputLevel
|
||||
|
||||
private val _isSending = MutableStateFlow(false)
|
||||
val isSending: StateFlow<Boolean> = _isSending
|
||||
|
||||
private val messageQueue = ArrayDeque<QueuedUtterance>()
|
||||
private val sessionSegments = mutableListOf<String>()
|
||||
private var lastFinalSegment: String? = null
|
||||
private var pendingRunId: String? = null
|
||||
private var pendingAssistantEntryId: String? = null
|
||||
private var gatewayConnected = false
|
||||
|
||||
private var recognizer: SpeechRecognizer? = null
|
||||
private var restartJob: Job? = null
|
||||
private var pendingRunTimeoutJob: Job? = null
|
||||
private var stopRequested = false
|
||||
|
||||
fun setMicEnabled(enabled: Boolean) {
|
||||
if (_micEnabled.value == enabled) return
|
||||
_micEnabled.value = enabled
|
||||
if (enabled) {
|
||||
start()
|
||||
sendQueuedIfIdle()
|
||||
} else {
|
||||
stop()
|
||||
flushSessionToQueue()
|
||||
sendQueuedIfIdle()
|
||||
}
|
||||
}
|
||||
|
||||
fun onGatewayConnectionChanged(connected: Boolean) {
|
||||
gatewayConnected = connected
|
||||
if (connected) {
|
||||
sendQueuedIfIdle()
|
||||
return
|
||||
}
|
||||
if (messageQueue.isNotEmpty()) {
|
||||
_statusText.value = queuedWaitingStatus()
|
||||
}
|
||||
}
|
||||
|
||||
fun handleGatewayEvent(event: String, payloadJson: String?) {
|
||||
if (event != "chat") return
|
||||
if (payloadJson.isNullOrBlank()) return
|
||||
val payload =
|
||||
try {
|
||||
json.parseToJsonElement(payloadJson).asObjectOrNull()
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
} ?: return
|
||||
|
||||
val runId = pendingRunId ?: return
|
||||
val eventRunId = payload["runId"].asStringOrNull() ?: return
|
||||
if (eventRunId != runId) return
|
||||
|
||||
when (payload["state"].asStringOrNull()) {
|
||||
"delta" -> {
|
||||
val deltaText = parseAssistantText(payload)
|
||||
if (!deltaText.isNullOrBlank()) {
|
||||
upsertPendingAssistant(text = deltaText.trim(), isStreaming = true)
|
||||
}
|
||||
}
|
||||
"final" -> {
|
||||
val finalText = parseAssistantText(payload)?.trim().orEmpty()
|
||||
if (finalText.isNotEmpty()) {
|
||||
upsertPendingAssistant(text = finalText, isStreaming = false)
|
||||
} else if (pendingAssistantEntryId != null) {
|
||||
updateConversationEntry(pendingAssistantEntryId!!, text = null, isStreaming = false)
|
||||
}
|
||||
completePendingTurn()
|
||||
}
|
||||
"error" -> {
|
||||
val errorMessage = payload["errorMessage"].asStringOrNull()?.trim().orEmpty().ifEmpty { "Voice request failed" }
|
||||
upsertPendingAssistant(text = errorMessage, isStreaming = false)
|
||||
completePendingTurn()
|
||||
}
|
||||
"aborted" -> {
|
||||
upsertPendingAssistant(text = "Response aborted", isStreaming = false)
|
||||
completePendingTurn()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun start() {
|
||||
stopRequested = false
|
||||
if (!SpeechRecognizer.isRecognitionAvailable(context)) {
|
||||
_statusText.value = "Speech recognizer unavailable"
|
||||
_micEnabled.value = false
|
||||
return
|
||||
}
|
||||
if (!hasMicPermission()) {
|
||||
_statusText.value = "Microphone permission required"
|
||||
_micEnabled.value = false
|
||||
return
|
||||
}
|
||||
|
||||
mainHandler.post {
|
||||
try {
|
||||
if (recognizer == null) {
|
||||
recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) }
|
||||
}
|
||||
startListeningSession()
|
||||
} catch (err: Throwable) {
|
||||
_statusText.value = "Start failed: ${err.message ?: err::class.simpleName}"
|
||||
_micEnabled.value = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun stop() {
|
||||
stopRequested = true
|
||||
restartJob?.cancel()
|
||||
restartJob = null
|
||||
_isListening.value = false
|
||||
_statusText.value = if (_isSending.value) "Mic off · sending…" else "Mic off"
|
||||
_inputLevel.value = 0f
|
||||
mainHandler.post {
|
||||
recognizer?.cancel()
|
||||
recognizer?.destroy()
|
||||
recognizer = null
|
||||
}
|
||||
}
|
||||
|
||||
private fun startListeningSession() {
|
||||
val recognizerInstance = recognizer ?: return
|
||||
val intent =
|
||||
Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
|
||||
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
||||
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3)
|
||||
putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName)
|
||||
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_MINIMUM_LENGTH_MILLIS, speechMinSessionMs)
|
||||
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, speechCompleteSilenceMs)
|
||||
putExtra(
|
||||
RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS,
|
||||
speechPossibleSilenceMs,
|
||||
)
|
||||
}
|
||||
_statusText.value =
|
||||
when {
|
||||
_isSending.value -> "Listening · sending queued voice"
|
||||
messageQueue.isNotEmpty() -> "Listening · ${messageQueue.size} queued"
|
||||
else -> "Listening"
|
||||
}
|
||||
_isListening.value = true
|
||||
recognizerInstance.startListening(intent)
|
||||
}
|
||||
|
||||
private fun scheduleRestart(delayMs: Long = 300L) {
|
||||
if (stopRequested) return
|
||||
if (!_micEnabled.value) return
|
||||
restartJob?.cancel()
|
||||
restartJob =
|
||||
scope.launch {
|
||||
delay(delayMs)
|
||||
mainHandler.post {
|
||||
if (stopRequested || !_micEnabled.value) return@post
|
||||
try {
|
||||
startListeningSession()
|
||||
} catch (_: Throwable) {
|
||||
// retry through onError
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun flushSessionToQueue() {
|
||||
val message = sessionSegments.joinToString(" ").trim()
|
||||
sessionSegments.clear()
|
||||
_liveTranscript.value = null
|
||||
lastFinalSegment = null
|
||||
if (message.isEmpty()) return
|
||||
|
||||
appendConversation(
|
||||
role = VoiceConversationRole.User,
|
||||
text = message,
|
||||
)
|
||||
messageQueue.addLast(QueuedUtterance(text = message))
|
||||
publishQueue()
|
||||
}
|
||||
|
||||
private fun publishQueue() {
|
||||
_queuedMessages.value = messageQueue.map { it.text }
|
||||
}
|
||||
|
||||
private fun sendQueuedIfIdle() {
|
||||
if (_isSending.value) return
|
||||
if (messageQueue.isEmpty()) {
|
||||
if (_micEnabled.value) {
|
||||
_statusText.value = "Listening"
|
||||
} else {
|
||||
_statusText.value = "Mic off"
|
||||
}
|
||||
return
|
||||
}
|
||||
if (!gatewayConnected) {
|
||||
_statusText.value = queuedWaitingStatus()
|
||||
return
|
||||
}
|
||||
|
||||
val next = messageQueue.first()
|
||||
_isSending.value = true
|
||||
pendingRunTimeoutJob?.cancel()
|
||||
pendingRunTimeoutJob = null
|
||||
_statusText.value = if (_micEnabled.value) "Listening · sending queued voice" else "Sending queued voice"
|
||||
|
||||
scope.launch {
|
||||
try {
|
||||
val runId = sendToGateway(next.text)
|
||||
pendingRunId = runId
|
||||
if (runId == null) {
|
||||
pendingRunTimeoutJob?.cancel()
|
||||
pendingRunTimeoutJob = null
|
||||
messageQueue.removeFirst()
|
||||
publishQueue()
|
||||
_isSending.value = false
|
||||
pendingAssistantEntryId = null
|
||||
sendQueuedIfIdle()
|
||||
} else {
|
||||
armPendingRunTimeout(runId)
|
||||
}
|
||||
} catch (err: Throwable) {
|
||||
pendingRunTimeoutJob?.cancel()
|
||||
pendingRunTimeoutJob = null
|
||||
_isSending.value = false
|
||||
pendingRunId = null
|
||||
pendingAssistantEntryId = null
|
||||
_statusText.value =
|
||||
if (!gatewayConnected) {
|
||||
queuedWaitingStatus()
|
||||
} else {
|
||||
"Send failed: ${err.message ?: err::class.simpleName}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun armPendingRunTimeout(runId: String) {
|
||||
pendingRunTimeoutJob?.cancel()
|
||||
pendingRunTimeoutJob =
|
||||
scope.launch {
|
||||
delay(pendingRunTimeoutMs)
|
||||
if (pendingRunId != runId) return@launch
|
||||
pendingRunId = null
|
||||
pendingAssistantEntryId = null
|
||||
_isSending.value = false
|
||||
_statusText.value =
|
||||
if (gatewayConnected) {
|
||||
"Voice reply timed out; retrying queued turn"
|
||||
} else {
|
||||
queuedWaitingStatus()
|
||||
}
|
||||
sendQueuedIfIdle()
|
||||
}
|
||||
}
|
||||
|
||||
private fun completePendingTurn() {
|
||||
pendingRunTimeoutJob?.cancel()
|
||||
pendingRunTimeoutJob = null
|
||||
if (messageQueue.isNotEmpty()) {
|
||||
messageQueue.removeFirst()
|
||||
publishQueue()
|
||||
}
|
||||
pendingRunId = null
|
||||
pendingAssistantEntryId = null
|
||||
_isSending.value = false
|
||||
sendQueuedIfIdle()
|
||||
}
|
||||
|
||||
private fun queuedWaitingStatus(): String {
|
||||
return "${messageQueue.size} queued · waiting for gateway"
|
||||
}
|
||||
|
||||
private fun appendConversation(
|
||||
role: VoiceConversationRole,
|
||||
text: String,
|
||||
isStreaming: Boolean = false,
|
||||
): String {
|
||||
val id = UUID.randomUUID().toString()
|
||||
_conversation.value =
|
||||
(_conversation.value + VoiceConversationEntry(id = id, role = role, text = text, isStreaming = isStreaming))
|
||||
.takeLast(maxConversationEntries)
|
||||
return id
|
||||
}
|
||||
|
||||
private fun updateConversationEntry(id: String, text: String?, isStreaming: Boolean) {
|
||||
val current = _conversation.value
|
||||
_conversation.value =
|
||||
current.map { entry ->
|
||||
if (entry.id == id) {
|
||||
val updatedText = text ?: entry.text
|
||||
entry.copy(text = updatedText, isStreaming = isStreaming)
|
||||
} else {
|
||||
entry
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun upsertPendingAssistant(text: String, isStreaming: Boolean) {
|
||||
val currentId = pendingAssistantEntryId
|
||||
if (currentId == null) {
|
||||
pendingAssistantEntryId =
|
||||
appendConversation(
|
||||
role = VoiceConversationRole.Assistant,
|
||||
text = text,
|
||||
isStreaming = isStreaming,
|
||||
)
|
||||
return
|
||||
}
|
||||
updateConversationEntry(id = currentId, text = text, isStreaming = isStreaming)
|
||||
}
|
||||
|
||||
private fun onFinalTranscript(text: String) {
|
||||
val trimmed = text.trim()
|
||||
if (trimmed.isEmpty()) return
|
||||
_liveTranscript.value = trimmed
|
||||
if (lastFinalSegment == trimmed) return
|
||||
lastFinalSegment = trimmed
|
||||
sessionSegments.add(trimmed)
|
||||
}
|
||||
|
||||
private fun disableMic(status: String) {
|
||||
stopRequested = true
|
||||
restartJob?.cancel()
|
||||
restartJob = null
|
||||
_micEnabled.value = false
|
||||
_isListening.value = false
|
||||
_inputLevel.value = 0f
|
||||
_statusText.value = status
|
||||
mainHandler.post {
|
||||
recognizer?.cancel()
|
||||
recognizer?.destroy()
|
||||
recognizer = null
|
||||
}
|
||||
}
|
||||
|
||||
private fun hasMicPermission(): Boolean {
|
||||
return (
|
||||
ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) ==
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
)
|
||||
}
|
||||
|
||||
private fun parseAssistantText(payload: JsonObject): String? {
|
||||
val message = payload["message"].asObjectOrNull() ?: return null
|
||||
if (message["role"].asStringOrNull() != "assistant") return null
|
||||
val content = message["content"] as? JsonArray ?: return null
|
||||
|
||||
val parts =
|
||||
content.mapNotNull { item ->
|
||||
val obj = item.asObjectOrNull() ?: return@mapNotNull null
|
||||
if (obj["type"].asStringOrNull() != "text") return@mapNotNull null
|
||||
obj["text"].asStringOrNull()?.trim()?.takeIf { it.isNotEmpty() }
|
||||
}
|
||||
if (parts.isEmpty()) return null
|
||||
return parts.joinToString("\n")
|
||||
}
|
||||
|
||||
private val listener =
|
||||
object : RecognitionListener {
|
||||
override fun onReadyForSpeech(params: Bundle?) {
|
||||
_isListening.value = true
|
||||
}
|
||||
|
||||
override fun onBeginningOfSpeech() {}
|
||||
|
||||
override fun onRmsChanged(rmsdB: Float) {
|
||||
val level = ((rmsdB + 2f) / 12f).coerceIn(0f, 1f)
|
||||
_inputLevel.value = level
|
||||
}
|
||||
|
||||
override fun onBufferReceived(buffer: ByteArray?) {}
|
||||
|
||||
override fun onEndOfSpeech() {
|
||||
_inputLevel.value = 0f
|
||||
scheduleRestart()
|
||||
}
|
||||
|
||||
override fun onError(error: Int) {
|
||||
if (stopRequested) return
|
||||
_isListening.value = false
|
||||
_inputLevel.value = 0f
|
||||
val status =
|
||||
when (error) {
|
||||
SpeechRecognizer.ERROR_AUDIO -> "Audio error"
|
||||
SpeechRecognizer.ERROR_CLIENT -> "Client error"
|
||||
SpeechRecognizer.ERROR_NETWORK -> "Network error"
|
||||
SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout"
|
||||
SpeechRecognizer.ERROR_NO_MATCH -> "Listening"
|
||||
SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy"
|
||||
SpeechRecognizer.ERROR_SERVER -> "Server error"
|
||||
SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening"
|
||||
SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS -> "Microphone permission required"
|
||||
SpeechRecognizer.ERROR_LANGUAGE_NOT_SUPPORTED -> "Language not supported on this device"
|
||||
SpeechRecognizer.ERROR_LANGUAGE_UNAVAILABLE -> "Language unavailable on this device"
|
||||
SpeechRecognizer.ERROR_SERVER_DISCONNECTED -> "Speech service disconnected"
|
||||
SpeechRecognizer.ERROR_TOO_MANY_REQUESTS -> "Speech requests limited; retrying"
|
||||
else -> "Speech error ($error)"
|
||||
}
|
||||
_statusText.value = status
|
||||
|
||||
if (
|
||||
error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS ||
|
||||
error == SpeechRecognizer.ERROR_LANGUAGE_NOT_SUPPORTED ||
|
||||
error == SpeechRecognizer.ERROR_LANGUAGE_UNAVAILABLE
|
||||
) {
|
||||
disableMic(status)
|
||||
return
|
||||
}
|
||||
|
||||
val restartDelayMs =
|
||||
when (error) {
|
||||
SpeechRecognizer.ERROR_NO_MATCH,
|
||||
SpeechRecognizer.ERROR_SPEECH_TIMEOUT,
|
||||
-> 1_200L
|
||||
SpeechRecognizer.ERROR_TOO_MANY_REQUESTS -> 2_500L
|
||||
else -> 600L
|
||||
}
|
||||
scheduleRestart(delayMs = restartDelayMs)
|
||||
}
|
||||
|
||||
override fun onResults(results: Bundle?) {
|
||||
val text = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty().firstOrNull()
|
||||
if (!text.isNullOrBlank()) {
|
||||
onFinalTranscript(text)
|
||||
flushSessionToQueue()
|
||||
sendQueuedIfIdle()
|
||||
}
|
||||
scheduleRestart()
|
||||
}
|
||||
|
||||
override fun onPartialResults(partialResults: Bundle?) {
|
||||
val text = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty().firstOrNull()
|
||||
if (!text.isNullOrBlank()) {
|
||||
_liveTranscript.value = text.trim()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onEvent(eventType: Int, params: Bundle?) {}
|
||||
}
|
||||
}
|
||||
|
||||
private fun kotlinx.serialization.json.JsonElement?.asObjectOrNull(): JsonObject? =
|
||||
this as? JsonObject
|
||||
|
||||
private fun kotlinx.serialization.json.JsonElement?.asStringOrNull(): String? =
|
||||
(this as? JsonPrimitive)?.takeIf { it.isString }?.content
|
||||
@@ -0,0 +1,98 @@
|
||||
package ai.openclaw.android.voice
|
||||
|
||||
import android.media.MediaDataSource
|
||||
import kotlin.math.min
|
||||
|
||||
internal class StreamingMediaDataSource : MediaDataSource() {
|
||||
private data class Chunk(val start: Long, val data: ByteArray)
|
||||
|
||||
private val lock = Object()
|
||||
private val chunks = ArrayList<Chunk>()
|
||||
private var totalSize: Long = 0
|
||||
private var closed = false
|
||||
private var finished = false
|
||||
private var lastReadIndex = 0
|
||||
|
||||
fun append(data: ByteArray) {
|
||||
if (data.isEmpty()) return
|
||||
synchronized(lock) {
|
||||
if (closed || finished) return
|
||||
val chunk = Chunk(totalSize, data)
|
||||
chunks.add(chunk)
|
||||
totalSize += data.size.toLong()
|
||||
lock.notifyAll()
|
||||
}
|
||||
}
|
||||
|
||||
fun finish() {
|
||||
synchronized(lock) {
|
||||
if (closed) return
|
||||
finished = true
|
||||
lock.notifyAll()
|
||||
}
|
||||
}
|
||||
|
||||
fun fail() {
|
||||
synchronized(lock) {
|
||||
closed = true
|
||||
lock.notifyAll()
|
||||
}
|
||||
}
|
||||
|
||||
override fun readAt(position: Long, buffer: ByteArray, offset: Int, size: Int): Int {
|
||||
if (position < 0) return -1
|
||||
synchronized(lock) {
|
||||
while (!closed && !finished && position >= totalSize) {
|
||||
lock.wait()
|
||||
}
|
||||
if (closed) return -1
|
||||
if (position >= totalSize && finished) return -1
|
||||
|
||||
val available = (totalSize - position).toInt()
|
||||
val toRead = min(size, available)
|
||||
var remaining = toRead
|
||||
var destOffset = offset
|
||||
var pos = position
|
||||
|
||||
var index = findChunkIndex(pos)
|
||||
while (remaining > 0 && index < chunks.size) {
|
||||
val chunk = chunks[index]
|
||||
val inChunkOffset = (pos - chunk.start).toInt()
|
||||
if (inChunkOffset >= chunk.data.size) {
|
||||
index++
|
||||
continue
|
||||
}
|
||||
val copyLen = min(remaining, chunk.data.size - inChunkOffset)
|
||||
System.arraycopy(chunk.data, inChunkOffset, buffer, destOffset, copyLen)
|
||||
remaining -= copyLen
|
||||
destOffset += copyLen
|
||||
pos += copyLen
|
||||
if (inChunkOffset + copyLen >= chunk.data.size) {
|
||||
index++
|
||||
}
|
||||
}
|
||||
|
||||
return toRead - remaining
|
||||
}
|
||||
}
|
||||
|
||||
override fun getSize(): Long = -1
|
||||
|
||||
override fun close() {
|
||||
synchronized(lock) {
|
||||
closed = true
|
||||
lock.notifyAll()
|
||||
}
|
||||
}
|
||||
|
||||
private fun findChunkIndex(position: Long): Int {
|
||||
var index = lastReadIndex
|
||||
while (index < chunks.size) {
|
||||
val chunk = chunks[index]
|
||||
if (position < chunk.start + chunk.data.size) break
|
||||
index++
|
||||
}
|
||||
lastReadIndex = index
|
||||
return index
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package ai.openclaw.android.voice
|
||||
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
|
||||
private val directiveJson = Json { ignoreUnknownKeys = true }
|
||||
|
||||
data class TalkDirective(
|
||||
val voiceId: String? = null,
|
||||
val modelId: String? = null,
|
||||
val speed: Double? = null,
|
||||
val rateWpm: Int? = null,
|
||||
val stability: Double? = null,
|
||||
val similarity: Double? = null,
|
||||
val style: Double? = null,
|
||||
val speakerBoost: Boolean? = null,
|
||||
val seed: Long? = null,
|
||||
val normalize: String? = null,
|
||||
val language: String? = null,
|
||||
val outputFormat: String? = null,
|
||||
val latencyTier: Int? = null,
|
||||
val once: Boolean? = null,
|
||||
)
|
||||
|
||||
data class TalkDirectiveParseResult(
|
||||
val directive: TalkDirective?,
|
||||
val stripped: String,
|
||||
val unknownKeys: List<String>,
|
||||
)
|
||||
|
||||
object TalkDirectiveParser {
|
||||
fun parse(text: String): TalkDirectiveParseResult {
|
||||
val normalized = text.replace("\r\n", "\n")
|
||||
val lines = normalized.split("\n").toMutableList()
|
||||
if (lines.isEmpty()) return TalkDirectiveParseResult(null, text, emptyList())
|
||||
|
||||
val firstNonEmpty = lines.indexOfFirst { it.trim().isNotEmpty() }
|
||||
if (firstNonEmpty == -1) return TalkDirectiveParseResult(null, text, emptyList())
|
||||
|
||||
val head = lines[firstNonEmpty].trim()
|
||||
if (!head.startsWith("{") || !head.endsWith("}")) {
|
||||
return TalkDirectiveParseResult(null, text, emptyList())
|
||||
}
|
||||
|
||||
val obj = parseJsonObject(head) ?: return TalkDirectiveParseResult(null, text, emptyList())
|
||||
|
||||
val speakerBoost =
|
||||
boolValue(obj, listOf("speaker_boost", "speakerBoost"))
|
||||
?: boolValue(obj, listOf("no_speaker_boost", "noSpeakerBoost"))?.not()
|
||||
|
||||
val directive = TalkDirective(
|
||||
voiceId = stringValue(obj, listOf("voice", "voice_id", "voiceId")),
|
||||
modelId = stringValue(obj, listOf("model", "model_id", "modelId")),
|
||||
speed = doubleValue(obj, listOf("speed")),
|
||||
rateWpm = intValue(obj, listOf("rate", "wpm")),
|
||||
stability = doubleValue(obj, listOf("stability")),
|
||||
similarity = doubleValue(obj, listOf("similarity", "similarity_boost", "similarityBoost")),
|
||||
style = doubleValue(obj, listOf("style")),
|
||||
speakerBoost = speakerBoost,
|
||||
seed = longValue(obj, listOf("seed")),
|
||||
normalize = stringValue(obj, listOf("normalize", "apply_text_normalization")),
|
||||
language = stringValue(obj, listOf("lang", "language_code", "language")),
|
||||
outputFormat = stringValue(obj, listOf("output_format", "format")),
|
||||
latencyTier = intValue(obj, listOf("latency", "latency_tier", "latencyTier")),
|
||||
once = boolValue(obj, listOf("once")),
|
||||
)
|
||||
|
||||
val hasDirective = listOf(
|
||||
directive.voiceId,
|
||||
directive.modelId,
|
||||
directive.speed,
|
||||
directive.rateWpm,
|
||||
directive.stability,
|
||||
directive.similarity,
|
||||
directive.style,
|
||||
directive.speakerBoost,
|
||||
directive.seed,
|
||||
directive.normalize,
|
||||
directive.language,
|
||||
directive.outputFormat,
|
||||
directive.latencyTier,
|
||||
directive.once,
|
||||
).any { it != null }
|
||||
|
||||
if (!hasDirective) return TalkDirectiveParseResult(null, text, emptyList())
|
||||
|
||||
val knownKeys = setOf(
|
||||
"voice", "voice_id", "voiceid",
|
||||
"model", "model_id", "modelid",
|
||||
"speed", "rate", "wpm",
|
||||
"stability", "similarity", "similarity_boost", "similarityboost",
|
||||
"style",
|
||||
"speaker_boost", "speakerboost",
|
||||
"no_speaker_boost", "nospeakerboost",
|
||||
"seed",
|
||||
"normalize", "apply_text_normalization",
|
||||
"lang", "language_code", "language",
|
||||
"output_format", "format",
|
||||
"latency", "latency_tier", "latencytier",
|
||||
"once",
|
||||
)
|
||||
val unknownKeys = obj.keys.filter { !knownKeys.contains(it.lowercase()) }.sorted()
|
||||
|
||||
lines.removeAt(firstNonEmpty)
|
||||
if (firstNonEmpty < lines.size) {
|
||||
if (lines[firstNonEmpty].trim().isEmpty()) {
|
||||
lines.removeAt(firstNonEmpty)
|
||||
}
|
||||
}
|
||||
|
||||
return TalkDirectiveParseResult(directive, lines.joinToString("\n"), unknownKeys)
|
||||
}
|
||||
|
||||
private fun parseJsonObject(line: String): JsonObject? {
|
||||
return try {
|
||||
directiveJson.parseToJsonElement(line) as? JsonObject
|
||||
} catch (_: Throwable) {
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun stringValue(obj: JsonObject, keys: List<String>): String? {
|
||||
for (key in keys) {
|
||||
val value = obj[key].asStringOrNull()?.trim()
|
||||
if (!value.isNullOrEmpty()) return value
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun doubleValue(obj: JsonObject, keys: List<String>): Double? {
|
||||
for (key in keys) {
|
||||
val value = obj[key].asDoubleOrNull()
|
||||
if (value != null) return value
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun intValue(obj: JsonObject, keys: List<String>): Int? {
|
||||
for (key in keys) {
|
||||
val value = obj[key].asIntOrNull()
|
||||
if (value != null) return value
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun longValue(obj: JsonObject, keys: List<String>): Long? {
|
||||
for (key in keys) {
|
||||
val value = obj[key].asLongOrNull()
|
||||
if (value != null) return value
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun boolValue(obj: JsonObject, keys: List<String>): Boolean? {
|
||||
for (key in keys) {
|
||||
val value = obj[key].asBooleanOrNull()
|
||||
if (value != null) return value
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private fun JsonElement?.asStringOrNull(): String? =
|
||||
(this as? JsonPrimitive)?.takeIf { it.isString }?.content
|
||||
|
||||
private fun JsonElement?.asDoubleOrNull(): Double? {
|
||||
val primitive = this as? JsonPrimitive ?: return null
|
||||
return primitive.content.toDoubleOrNull()
|
||||
}
|
||||
|
||||
private fun JsonElement?.asIntOrNull(): Int? {
|
||||
val primitive = this as? JsonPrimitive ?: return null
|
||||
return primitive.content.toIntOrNull()
|
||||
}
|
||||
|
||||
private fun JsonElement?.asLongOrNull(): Long? {
|
||||
val primitive = this as? JsonPrimitive ?: return null
|
||||
return primitive.content.toLongOrNull()
|
||||
}
|
||||
|
||||
private fun JsonElement?.asBooleanOrNull(): Boolean? {
|
||||
val primitive = this as? JsonPrimitive ?: return null
|
||||
val content = primitive.content.trim().lowercase()
|
||||
return when (content) {
|
||||
"true", "yes", "1" -> true
|
||||
"false", "no", "0" -> false
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package ai.openclaw.android.voice
|
||||
|
||||
object VoiceWakeCommandExtractor {
|
||||
fun extractCommand(text: String, triggerWords: List<String>): String? {
|
||||
val raw = text.trim()
|
||||
if (raw.isEmpty()) return null
|
||||
|
||||
val triggers =
|
||||
triggerWords
|
||||
.map { it.trim().lowercase() }
|
||||
.filter { it.isNotEmpty() }
|
||||
.distinct()
|
||||
if (triggers.isEmpty()) return null
|
||||
|
||||
val alternation = triggers.joinToString("|") { Regex.escape(it) }
|
||||
// Match: "<anything> <trigger><punct/space> <command>"
|
||||
val regex = Regex("(?i)(?:^|\\s)($alternation)\\b[\\s\\p{Punct}]*([\\s\\S]+)$")
|
||||
val match = regex.find(raw) ?: return null
|
||||
val extracted = match.groupValues.getOrNull(2)?.trim().orEmpty()
|
||||
if (extracted.isEmpty()) return null
|
||||
|
||||
val cleaned = extracted.trimStart { it.isWhitespace() || it.isPunctuation() }.trim()
|
||||
if (cleaned.isEmpty()) return null
|
||||
return cleaned
|
||||
}
|
||||
}
|
||||
|
||||
private fun Char.isPunctuation(): Boolean {
|
||||
return when (Character.getType(this)) {
|
||||
Character.CONNECTOR_PUNCTUATION.toInt(),
|
||||
Character.DASH_PUNCTUATION.toInt(),
|
||||
Character.START_PUNCTUATION.toInt(),
|
||||
Character.END_PUNCTUATION.toInt(),
|
||||
Character.INITIAL_QUOTE_PUNCTUATION.toInt(),
|
||||
Character.FINAL_QUOTE_PUNCTUATION.toInt(),
|
||||
Character.OTHER_PUNCTUATION.toInt(),
|
||||
-> true
|
||||
else -> false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package ai.openclaw.android.voice
|
||||
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.os.Bundle
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.speech.RecognitionListener
|
||||
import android.speech.RecognizerIntent
|
||||
import android.speech.SpeechRecognizer
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
class VoiceWakeManager(
|
||||
private val context: Context,
|
||||
private val scope: CoroutineScope,
|
||||
private val onCommand: suspend (String) -> Unit,
|
||||
) {
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
|
||||
private val _isListening = MutableStateFlow(false)
|
||||
val isListening: StateFlow<Boolean> = _isListening
|
||||
|
||||
private val _statusText = MutableStateFlow("Off")
|
||||
val statusText: StateFlow<String> = _statusText
|
||||
|
||||
var triggerWords: List<String> = emptyList()
|
||||
private set
|
||||
|
||||
private var recognizer: SpeechRecognizer? = null
|
||||
private var restartJob: Job? = null
|
||||
private var lastDispatched: String? = null
|
||||
private var stopRequested = false
|
||||
|
||||
fun setTriggerWords(words: List<String>) {
|
||||
triggerWords = words
|
||||
}
|
||||
|
||||
fun start() {
|
||||
mainHandler.post {
|
||||
if (_isListening.value) return@post
|
||||
stopRequested = false
|
||||
|
||||
if (!SpeechRecognizer.isRecognitionAvailable(context)) {
|
||||
_isListening.value = false
|
||||
_statusText.value = "Speech recognizer unavailable"
|
||||
return@post
|
||||
}
|
||||
|
||||
try {
|
||||
recognizer?.destroy()
|
||||
recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) }
|
||||
startListeningInternal()
|
||||
} catch (err: Throwable) {
|
||||
_isListening.value = false
|
||||
_statusText.value = "Start failed: ${err.message ?: err::class.simpleName}"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun stop(statusText: String = "Off") {
|
||||
stopRequested = true
|
||||
restartJob?.cancel()
|
||||
restartJob = null
|
||||
mainHandler.post {
|
||||
_isListening.value = false
|
||||
_statusText.value = statusText
|
||||
recognizer?.cancel()
|
||||
recognizer?.destroy()
|
||||
recognizer = null
|
||||
}
|
||||
}
|
||||
|
||||
private fun startListeningInternal() {
|
||||
val r = recognizer ?: return
|
||||
val intent =
|
||||
Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
|
||||
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
||||
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3)
|
||||
putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName)
|
||||
}
|
||||
|
||||
_statusText.value = "Listening"
|
||||
_isListening.value = true
|
||||
r.startListening(intent)
|
||||
}
|
||||
|
||||
private fun scheduleRestart(delayMs: Long = 350) {
|
||||
if (stopRequested) return
|
||||
restartJob?.cancel()
|
||||
restartJob =
|
||||
scope.launch {
|
||||
delay(delayMs)
|
||||
mainHandler.post {
|
||||
if (stopRequested) return@post
|
||||
try {
|
||||
recognizer?.cancel()
|
||||
startListeningInternal()
|
||||
} catch (_: Throwable) {
|
||||
// Will be picked up by onError and retry again.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleTranscription(text: String) {
|
||||
val command = VoiceWakeCommandExtractor.extractCommand(text, triggerWords) ?: return
|
||||
if (command == lastDispatched) return
|
||||
lastDispatched = command
|
||||
|
||||
scope.launch { onCommand(command) }
|
||||
_statusText.value = "Triggered"
|
||||
scheduleRestart(delayMs = 650)
|
||||
}
|
||||
|
||||
private val listener =
|
||||
object : RecognitionListener {
|
||||
override fun onReadyForSpeech(params: Bundle?) {
|
||||
_statusText.value = "Listening"
|
||||
}
|
||||
|
||||
override fun onBeginningOfSpeech() {}
|
||||
|
||||
override fun onRmsChanged(rmsdB: Float) {}
|
||||
|
||||
override fun onBufferReceived(buffer: ByteArray?) {}
|
||||
|
||||
override fun onEndOfSpeech() {
|
||||
scheduleRestart()
|
||||
}
|
||||
|
||||
override fun onError(error: Int) {
|
||||
if (stopRequested) return
|
||||
_isListening.value = false
|
||||
if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) {
|
||||
_statusText.value = "Microphone permission required"
|
||||
return
|
||||
}
|
||||
|
||||
_statusText.value =
|
||||
when (error) {
|
||||
SpeechRecognizer.ERROR_AUDIO -> "Audio error"
|
||||
SpeechRecognizer.ERROR_CLIENT -> "Client error"
|
||||
SpeechRecognizer.ERROR_NETWORK -> "Network error"
|
||||
SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout"
|
||||
SpeechRecognizer.ERROR_NO_MATCH -> "Listening"
|
||||
SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy"
|
||||
SpeechRecognizer.ERROR_SERVER -> "Server error"
|
||||
SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "Listening"
|
||||
else -> "Speech error ($error)"
|
||||
}
|
||||
scheduleRestart(delayMs = 600)
|
||||
}
|
||||
|
||||
override fun onResults(results: Bundle?) {
|
||||
val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
|
||||
list.firstOrNull()?.let(::handleTranscription)
|
||||
scheduleRestart()
|
||||
}
|
||||
|
||||
override fun onPartialResults(partialResults: Bundle?) {
|
||||
val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
|
||||
list.firstOrNull()?.let(::handleTranscription)
|
||||
}
|
||||
|
||||
override fun onEvent(eventType: Int, params: Bundle?) {}
|
||||
}
|
||||
}
|
||||
BIN
openclaw/apps/android/app/src/main/res/font/manrope_700_bold.ttf
Normal file
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
<background android:drawable="@color/ic_launcher_background" />
|
||||
<foreground android:drawable="@mipmap/ic_launcher_foreground" />
|
||||
<monochrome android:drawable="@mipmap/ic_launcher_foreground" />
|
||||
</adaptive-icon>
|
||||
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
<background android:drawable="@color/ic_launcher_background" />
|
||||
<foreground android:drawable="@mipmap/ic_launcher_foreground" />
|
||||
<monochrome android:drawable="@mipmap/ic_launcher_foreground" />
|
||||
</adaptive-icon>
|
||||
|
After Width: | Height: | Size: 11 KiB |
|
After Width: | Height: | Size: 44 KiB |
|
After Width: | Height: | Size: 5.5 KiB |
|
After Width: | Height: | Size: 21 KiB |
|
After Width: | Height: | Size: 17 KiB |
|
After Width: | Height: | Size: 73 KiB |
|
After Width: | Height: | Size: 36 KiB |
|
After Width: | Height: | Size: 155 KiB |
|
After Width: | Height: | Size: 60 KiB |
|
After Width: | Height: | Size: 270 KiB |
3
openclaw/apps/android/app/src/main/res/values/colors.xml
Normal file
@@ -0,0 +1,3 @@
|
||||
<resources>
|
||||
<color name="ic_launcher_background">#0A0A0A</color>
|
||||
</resources>
|
||||
@@ -0,0 +1,3 @@
|
||||
<resources>
|
||||
<string name="app_name">OpenClaw Node</string>
|
||||
</resources>
|
||||
7
openclaw/apps/android/app/src/main/res/values/themes.xml
Normal file
@@ -0,0 +1,7 @@
|
||||
<resources>
|
||||
<style name="Theme.OpenClawNode" parent="Theme.Material3.DayNight.NoActionBar">
|
||||
<item name="android:statusBarColor">@android:color/transparent</item>
|
||||
<item name="android:navigationBarColor">@android:color/transparent</item>
|
||||
<item name="android:windowLightStatusBar">false</item>
|
||||
</style>
|
||||
</resources>
|
||||
@@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<full-backup-content>
|
||||
<include domain="file" path="." />
|
||||
</full-backup-content>
|
||||
@@ -0,0 +1,9 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<data-extraction-rules>
|
||||
<cloud-backup>
|
||||
<include domain="file" path="." />
|
||||
</cloud-backup>
|
||||
<device-transfer>
|
||||
<include domain="file" path="." />
|
||||
</device-transfer>
|
||||
</data-extraction-rules>
|
||||