feat(android): Add screen context capture for voice assistant
This commit is contained in:
@@ -9,6 +9,24 @@ import android.app.assist.AssistContent
|
||||
|
||||
class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession(context) {
|
||||
|
||||
private var capturedContext: String? = null
|
||||
private var summarizeButton: android.widget.Button? = null
|
||||
|
||||
override fun onCreateContentView(): android.view.View {
|
||||
val view = layoutInflater.inflate(app.cogwheel.conduit.R.layout.assistant_overlay, null)
|
||||
summarizeButton = view.findViewById(app.cogwheel.conduit.R.id.btn_summarize)
|
||||
summarizeButton?.setOnClickListener {
|
||||
launchAppWithContext()
|
||||
}
|
||||
|
||||
val closeButton = view.findViewById<android.view.View>(app.cogwheel.conduit.R.id.btn_close)
|
||||
closeButton?.setOnClickListener {
|
||||
finish()
|
||||
}
|
||||
|
||||
return view
|
||||
}
|
||||
|
||||
override fun onHandleAssist(
|
||||
data: Bundle?,
|
||||
structure: AssistStructure?,
|
||||
@@ -18,9 +36,56 @@ class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession
|
||||
|
||||
android.util.Log.d("ConduitVoiceSession", "onHandleAssist called")
|
||||
|
||||
// Launch the main activity when the assistant is triggered
|
||||
val intent = Intent(context, MainActivity::class.java)
|
||||
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
|
||||
context.startActivity(intent)
|
||||
val screenContext = StringBuilder()
|
||||
structure?.let {
|
||||
val nodes = it.windowNodeCount
|
||||
for (i in 0 until nodes) {
|
||||
val windowNode = it.getWindowNodeAt(i)
|
||||
traverseNode(windowNode.rootViewNode, screenContext)
|
||||
}
|
||||
}
|
||||
|
||||
capturedContext = screenContext.toString()
|
||||
// Ideally, we could update the UI here to say "Context Ready"
|
||||
}
|
||||
|
||||
private fun launchAppWithContext() {
|
||||
try {
|
||||
android.util.Log.d("ConduitVoiceSession", "Attempting to launch app with context")
|
||||
val intent = Intent(context, MainActivity::class.java)
|
||||
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
|
||||
intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP)
|
||||
intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP)
|
||||
|
||||
if (capturedContext != null) {
|
||||
intent.putExtra("screen_context", capturedContext)
|
||||
android.util.Log.d("ConduitVoiceSession", "Context attached: ${capturedContext?.take(50)}...")
|
||||
} else {
|
||||
android.util.Log.d("ConduitVoiceSession", "No context captured")
|
||||
}
|
||||
|
||||
context.startActivity(intent)
|
||||
android.util.Log.d("ConduitVoiceSession", "App launch requested")
|
||||
finish() // Close the overlay
|
||||
} catch (e: Exception) {
|
||||
android.util.Log.e("ConduitVoiceSession", "Failed to launch app", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun traverseNode(node: AssistStructure.ViewNode?, builder: StringBuilder) {
|
||||
if (node == null) return
|
||||
|
||||
if (node.text != null) {
|
||||
builder.append(node.text).append("\n")
|
||||
}
|
||||
|
||||
// Also check content description for accessibility text
|
||||
if (node.contentDescription != null) {
|
||||
builder.append(node.contentDescription).append("\n")
|
||||
}
|
||||
|
||||
for (i in 0 until node.childCount) {
|
||||
traverseNode(node.getChildAt(i), builder)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,12 +23,32 @@ class MainActivity : FlutterActivity() {
|
||||
windowInsetsController.isAppearanceLightNavigationBars = false
|
||||
}
|
||||
|
||||
private val CHANNEL = "app.cogwheel.conduit/assistant"
|
||||
private var methodChannel: io.flutter.plugin.common.MethodChannel? = null
|
||||
|
||||
override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
|
||||
super.configureFlutterEngine(flutterEngine)
|
||||
|
||||
// Initialize background streaming handler
|
||||
backgroundStreamingHandler = BackgroundStreamingHandler(this)
|
||||
backgroundStreamingHandler.setup(flutterEngine)
|
||||
|
||||
methodChannel = io.flutter.plugin.common.MethodChannel(flutterEngine.dartExecutor.binaryMessenger, CHANNEL)
|
||||
|
||||
// Check if started with context
|
||||
handleIntent(intent)
|
||||
}
|
||||
|
||||
override fun onNewIntent(intent: android.content.Intent) {
|
||||
super.onNewIntent(intent)
|
||||
handleIntent(intent)
|
||||
}
|
||||
|
||||
private fun handleIntent(intent: android.content.Intent) {
|
||||
val screenContext = intent.getStringExtra("screen_context")
|
||||
if (screenContext != null) {
|
||||
methodChannel?.invokeMethod("analyzeScreen", screenContext)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
|
||||
6
android/app/src/main/res/drawable/rounded_button_bg.xml
Normal file
6
android/app/src/main/res/drawable/rounded_button_bg.xml
Normal file
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<shape xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:shape="rectangle">
|
||||
<solid android:color="#2196F3" />
|
||||
<corners android:radius="20dp" />
|
||||
</shape>
|
||||
6
android/app/src/main/res/drawable/rounded_overlay_bg.xml
Normal file
6
android/app/src/main/res/drawable/rounded_overlay_bg.xml
Normal file
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<shape xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:shape="rectangle">
|
||||
<solid android:color="@android:color/white" />
|
||||
<corners android:radius="28dp" />
|
||||
</shape>
|
||||
67
android/app/src/main/res/layout/assistant_overlay.xml
Normal file
67
android/app/src/main/res/layout/assistant_overlay.xml
Normal file
@@ -0,0 +1,67 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:padding="16dp">
|
||||
|
||||
<LinearLayout
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="vertical"
|
||||
android:background="@drawable/rounded_overlay_bg"
|
||||
android:padding="24dp"
|
||||
android:elevation="8dp"
|
||||
android:layout_marginBottom="16dp">
|
||||
|
||||
<LinearLayout
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:orientation="horizontal"
|
||||
android:gravity="center_vertical"
|
||||
android:layout_marginBottom="24dp">
|
||||
|
||||
<!-- App Icon Placeholder (optional) -->
|
||||
<!-- <ImageView ... /> -->
|
||||
|
||||
<TextView
|
||||
android:layout_width="0dp"
|
||||
android:layout_height="wrap_content"
|
||||
android:layout_weight="1"
|
||||
android:text="Conduit"
|
||||
android:textSize="20sp"
|
||||
android:textStyle="bold"
|
||||
android:textColor="@android:color/black"
|
||||
android:fontFamily="sans-serif-medium"/>
|
||||
|
||||
<!-- Close Button (X) -->
|
||||
<ImageView
|
||||
android:id="@+id/btn_close"
|
||||
android:layout_width="24dp"
|
||||
android:layout_height="24dp"
|
||||
android:src="@android:drawable/ic_menu_close_clear_cancel"
|
||||
android:alpha="0.5"/>
|
||||
</LinearLayout>
|
||||
|
||||
<TextView
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="wrap_content"
|
||||
android:text="I can read your screen to help you."
|
||||
android:textSize="16sp"
|
||||
android:textColor="#5F6368"
|
||||
android:layout_marginBottom="24dp"/>
|
||||
|
||||
<Button
|
||||
android:id="@+id/btn_summarize"
|
||||
android:layout_width="match_parent"
|
||||
android:layout_height="56dp"
|
||||
android:text="Summarize this screen"
|
||||
android:textAllCaps="false"
|
||||
android:textSize="16sp"
|
||||
android:textStyle="bold"
|
||||
android:textColor="@android:color/white"
|
||||
android:background="@drawable/rounded_button_bg"
|
||||
android:stateListAnimator="@null"
|
||||
android:elevation="0dp"/>
|
||||
|
||||
</LinearLayout>
|
||||
</FrameLayout>
|
||||
Reference in New Issue
Block a user