From 20e57e9f883eb502b18be19ee458b441e46bc005 Mon Sep 17 00:00:00 2001
From: cogwheel0 <172976095+cogwheel0@users.noreply.github.com>
Date: Fri, 21 Nov 2025 19:16:51 +0530
Subject: [PATCH 1/7] feat(android): Add voice interaction service for app
launch
---
android/app/src/main/AndroidManifest.xml | 12 +++++++++
.../conduit/ConduitVoiceInteractionSession.kt | 26 +++++++++++++++++++
.../ConduitVoiceInteractionSessionService.kt | 11 ++++++++
.../res/xml/voice_interaction_service.xml | 7 +++++
pubspec.lock | 16 ++++++------
5 files changed, 64 insertions(+), 8 deletions(-)
create mode 100644 android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
create mode 100644 android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSessionService.kt
create mode 100644 android/app/src/main/res/xml/voice_interaction_service.xml
diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml
index 7008f09..e56683c 100644
--- a/android/app/src/main/AndroidManifest.xml
+++ b/android/app/src/main/AndroidManifest.xml
@@ -32,6 +32,18 @@
android:allowBackup="false"
android:fullBackupContent="false"
android:usesCleartextTraffic="true">
+
+
+
+
+
+
+
+
diff --git a/pubspec.lock b/pubspec.lock
index 8cfd515..43c4376 100644
--- a/pubspec.lock
+++ b/pubspec.lock
@@ -961,10 +961,10 @@ packages:
dependency: transitive
description:
name: meta
- sha256: e3641ec5d63ebf0d9b41bd43201a66e3fc79a65db5f61fc181f04cd27aab950c
+ sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394"
url: "https://pub.dev"
source: hosted
- version: "1.16.0"
+ version: "1.17.0"
mime:
dependency: transitive
description:
@@ -1606,26 +1606,26 @@ packages:
dependency: transitive
description:
name: test
- sha256: "65e29d831719be0591f7b3b1a32a3cda258ec98c58c7b25f7b84241bc31215bb"
+ sha256: "75906bf273541b676716d1ca7627a17e4c4070a3a16272b7a3dc7da3b9f3f6b7"
url: "https://pub.dev"
source: hosted
- version: "1.26.2"
+ version: "1.26.3"
test_api:
dependency: transitive
description:
name: test_api
- sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00"
+ sha256: ab2726c1a94d3176a45960b6234466ec367179b87dd74f1611adb1f3b5fb9d55
url: "https://pub.dev"
source: hosted
- version: "0.7.6"
+ version: "0.7.7"
test_core:
dependency: transitive
description:
name: test_core
- sha256: "80bf5a02b60af04b09e14f6fe68b921aad119493e26e490deaca5993fef1b05a"
+ sha256: "0cc24b5ff94b38d2ae73e1eb43cc302b77964fbf67abad1e296025b78deb53d0"
url: "https://pub.dev"
source: hosted
- version: "0.6.11"
+ version: "0.6.12"
timezone:
dependency: transitive
description:
From f18edc7fe00a7d6ad65d3c7fb310399334580848 Mon Sep 17 00:00:00 2001
From: cogwheel0 <172976095+cogwheel0@users.noreply.github.com>
Date: Fri, 21 Nov 2025 19:50:39 +0530
Subject: [PATCH 2/7] feat(android): Add screen context capture for voice
assistant
---
.../conduit/ConduitVoiceInteractionSession.kt | 73 ++++++++++++++++++-
.../app/cogwheel/conduit/MainActivity.kt | 20 +++++
.../main/res/drawable/rounded_button_bg.xml | 6 ++
.../main/res/drawable/rounded_overlay_bg.xml | 6 ++
.../src/main/res/layout/assistant_overlay.xml | 67 +++++++++++++++++
lib/core/utils/android_assistant_handler.dart | 35 +++++++++
lib/features/chat/views/chat_page.dart | 25 +++++++
7 files changed, 228 insertions(+), 4 deletions(-)
create mode 100644 android/app/src/main/res/drawable/rounded_button_bg.xml
create mode 100644 android/app/src/main/res/drawable/rounded_overlay_bg.xml
create mode 100644 android/app/src/main/res/layout/assistant_overlay.xml
create mode 100644 lib/core/utils/android_assistant_handler.dart
diff --git a/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt b/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
index 9e52f5b..987206b 100644
--- a/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
+++ b/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
@@ -9,6 +9,24 @@ import android.app.assist.AssistContent
class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession(context) {
+ private var capturedContext: String? = null
+ private var summarizeButton: android.widget.Button? = null
+
+ override fun onCreateContentView(): android.view.View {
+ val view = layoutInflater.inflate(app.cogwheel.conduit.R.layout.assistant_overlay, null)
+ summarizeButton = view.findViewById(app.cogwheel.conduit.R.id.btn_summarize)
+ summarizeButton?.setOnClickListener {
+ launchAppWithContext()
+ }
+
+ val closeButton = view.findViewById(app.cogwheel.conduit.R.id.btn_close)
+ closeButton?.setOnClickListener {
+ finish()
+ }
+
+ return view
+ }
+
override fun onHandleAssist(
data: Bundle?,
structure: AssistStructure?,
@@ -18,9 +36,56 @@ class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession
android.util.Log.d("ConduitVoiceSession", "onHandleAssist called")
- // Launch the main activity when the assistant is triggered
- val intent = Intent(context, MainActivity::class.java)
- intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
- context.startActivity(intent)
+ val screenContext = StringBuilder()
+ structure?.let {
+ val nodes = it.windowNodeCount
+ for (i in 0 until nodes) {
+ val windowNode = it.getWindowNodeAt(i)
+ traverseNode(windowNode.rootViewNode, screenContext)
+ }
+ }
+
+ capturedContext = screenContext.toString()
+ // Ideally, we could update the UI here to say "Context Ready"
+ }
+
+ private fun launchAppWithContext() {
+ try {
+ android.util.Log.d("ConduitVoiceSession", "Attempting to launch app with context")
+ val intent = Intent(context, MainActivity::class.java)
+ intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
+ intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP)
+ intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP)
+
+ if (capturedContext != null) {
+ intent.putExtra("screen_context", capturedContext)
+ android.util.Log.d("ConduitVoiceSession", "Context attached: ${capturedContext?.take(50)}...")
+ } else {
+ android.util.Log.d("ConduitVoiceSession", "No context captured")
+ }
+
+ context.startActivity(intent)
+ android.util.Log.d("ConduitVoiceSession", "App launch requested")
+ finish() // Close the overlay
+ } catch (e: Exception) {
+ android.util.Log.e("ConduitVoiceSession", "Failed to launch app", e)
+ }
+ }
+
+ private fun traverseNode(node: AssistStructure.ViewNode?, builder: StringBuilder) {
+ if (node == null) return
+
+ if (node.text != null) {
+ builder.append(node.text).append("\n")
+ }
+
+ // Also check content description for accessibility text
+ if (node.contentDescription != null) {
+ builder.append(node.contentDescription).append("\n")
+ }
+
+ for (i in 0 until node.childCount) {
+ traverseNode(node.getChildAt(i), builder)
+ }
}
}
diff --git a/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt b/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt
index edecfcf..4343470 100644
--- a/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt
+++ b/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt
@@ -23,12 +23,32 @@ class MainActivity : FlutterActivity() {
windowInsetsController.isAppearanceLightNavigationBars = false
}
+ private val CHANNEL = "app.cogwheel.conduit/assistant"
+ private var methodChannel: io.flutter.plugin.common.MethodChannel? = null
+
override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
super.configureFlutterEngine(flutterEngine)
// Initialize background streaming handler
backgroundStreamingHandler = BackgroundStreamingHandler(this)
backgroundStreamingHandler.setup(flutterEngine)
+
+ methodChannel = io.flutter.plugin.common.MethodChannel(flutterEngine.dartExecutor.binaryMessenger, CHANNEL)
+
+ // Check if started with context
+ handleIntent(intent)
+ }
+
+ override fun onNewIntent(intent: android.content.Intent) {
+ super.onNewIntent(intent)
+ handleIntent(intent)
+ }
+
+ private fun handleIntent(intent: android.content.Intent) {
+ val screenContext = intent.getStringExtra("screen_context")
+ if (screenContext != null) {
+ methodChannel?.invokeMethod("analyzeScreen", screenContext)
+ }
}
override fun onDestroy() {
diff --git a/android/app/src/main/res/drawable/rounded_button_bg.xml b/android/app/src/main/res/drawable/rounded_button_bg.xml
new file mode 100644
index 0000000..ab7b3f1
--- /dev/null
+++ b/android/app/src/main/res/drawable/rounded_button_bg.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
diff --git a/android/app/src/main/res/drawable/rounded_overlay_bg.xml b/android/app/src/main/res/drawable/rounded_overlay_bg.xml
new file mode 100644
index 0000000..53dea9a
--- /dev/null
+++ b/android/app/src/main/res/drawable/rounded_overlay_bg.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
diff --git a/android/app/src/main/res/layout/assistant_overlay.xml b/android/app/src/main/res/layout/assistant_overlay.xml
new file mode 100644
index 0000000..32ea604
--- /dev/null
+++ b/android/app/src/main/res/layout/assistant_overlay.xml
@@ -0,0 +1,67 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/lib/core/utils/android_assistant_handler.dart b/lib/core/utils/android_assistant_handler.dart
new file mode 100644
index 0000000..19b9f44
--- /dev/null
+++ b/lib/core/utils/android_assistant_handler.dart
@@ -0,0 +1,35 @@
+import 'package:flutter/services.dart';
+import 'package:flutter_riverpod/flutter_riverpod.dart';
+
+final androidAssistantProvider = Provider(
+ (ref) => AndroidAssistantHandler(ref),
+);
+
+final screenContextProvider = NotifierProvider(
+ ScreenContextNotifier.new,
+);
+
+class ScreenContextNotifier extends Notifier {
+ @override
+ String? build() => null;
+
+ void setContext(String? context) {
+ state = context;
+ }
+}
+
+class AndroidAssistantHandler {
+ static const platform = MethodChannel('app.cogwheel.conduit/assistant');
+ final Ref _ref;
+
+ AndroidAssistantHandler(this._ref) {
+ platform.setMethodCallHandler(_handleMethodCall);
+ }
+
+ Future _handleMethodCall(MethodCall call) async {
+ if (call.method == 'analyzeScreen') {
+ final String context = call.arguments as String;
+ _ref.read(screenContextProvider.notifier).setContext(context);
+ }
+ }
+}
diff --git a/lib/features/chat/views/chat_page.dart b/lib/features/chat/views/chat_page.dart
index 3453ccb..98ee420 100644
--- a/lib/features/chat/views/chat_page.dart
+++ b/lib/features/chat/views/chat_page.dart
@@ -17,6 +17,7 @@ import '../../../core/utils/debug_logger.dart';
import '../../../core/utils/user_display_name.dart';
import '../../../core/utils/model_icon_utils.dart';
import '../../auth/providers/unified_auth_providers.dart';
+import '../../../core/utils/android_assistant_handler.dart';
import '../widgets/modern_chat_input.dart';
import '../widgets/user_message_bubble.dart';
@@ -288,6 +289,10 @@ class _ChatPageState extends ConsumerState {
// Initialize chat page components
WidgetsBinding.instance.addPostFrameCallback((_) async {
if (!mounted) return;
+
+ // Initialize Android Assistant Handler
+ ref.read(androidAssistantProvider);
+
// First, ensure a model is selected
await _checkAndAutoSelectModel();
if (!mounted) return;
@@ -301,6 +306,26 @@ class _ChatPageState extends ConsumerState {
});
}
+ @override
+ void didChangeDependencies() {
+ super.didChangeDependencies();
+ // Listen for screen context from Android Assistant
+ final screenContext = ref.watch(screenContextProvider);
+ if (screenContext != null && screenContext.isNotEmpty) {
+ // Clear the context so we don't process it again
+ WidgetsBinding.instance.addPostFrameCallback((_) {
+ ref.read(screenContextProvider.notifier).setContext(null);
+ // Pre-fill the input or send a message
+ // For now, let's just pre-fill the input with a prompt
+ // TODO: Ideally we should add this as a system message or attachment
+ _handleMessageSend(
+ "Here is the content of my screen:\n\n$screenContext\n\nCan you summarize this?",
+ null,
+ );
+ });
+ }
+ }
+
@override
void dispose() {
_scrollController.dispose();
From 6a2a452be4a1cc04bb2f32368effab0cd9696179 Mon Sep 17 00:00:00 2001
From: cogwheel0 <172976095+cogwheel0@users.noreply.github.com>
Date: Fri, 21 Nov 2025 20:40:13 +0530
Subject: [PATCH 3/7] feat(ui): Add new interaction buttons and icons for voice
assistant overlay
---
.../conduit/ConduitVoiceInteractionSession.kt | 42 +++-
.../res/drawable/dark_rounded_button_bg.xml | 6 +
android/app/src/main/res/drawable/ic_add.xml | 10 +
android/app/src/main/res/drawable/ic_ask.xml | 10 +
.../app/src/main/res/drawable/ic_share.xml | 10 +
.../app/src/main/res/drawable/ic_sparkle.xml | 13 +
.../src/main/res/drawable/ic_summarize.xml | 10 +
.../app/src/main/res/drawable/ic_voice.xml | 10 +
.../src/main/res/drawable/input_bar_bg.xml | 6 +
.../src/main/res/drawable/voice_button_bg.xml | 5 +
.../src/main/res/layout/assistant_overlay.xml | 232 ++++++++++++++----
11 files changed, 300 insertions(+), 54 deletions(-)
create mode 100644 android/app/src/main/res/drawable/dark_rounded_button_bg.xml
create mode 100644 android/app/src/main/res/drawable/ic_add.xml
create mode 100644 android/app/src/main/res/drawable/ic_ask.xml
create mode 100644 android/app/src/main/res/drawable/ic_share.xml
create mode 100644 android/app/src/main/res/drawable/ic_sparkle.xml
create mode 100644 android/app/src/main/res/drawable/ic_summarize.xml
create mode 100644 android/app/src/main/res/drawable/ic_voice.xml
create mode 100644 android/app/src/main/res/drawable/input_bar_bg.xml
create mode 100644 android/app/src/main/res/drawable/voice_button_bg.xml
diff --git a/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt b/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
index 987206b..1e24238 100644
--- a/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
+++ b/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
@@ -10,20 +10,48 @@ import android.app.assist.AssistContent
class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession(context) {
private var capturedContext: String? = null
- private var summarizeButton: android.widget.Button? = null
override fun onCreateContentView(): android.view.View {
val view = layoutInflater.inflate(app.cogwheel.conduit.R.layout.assistant_overlay, null)
- summarizeButton = view.findViewById(app.cogwheel.conduit.R.id.btn_summarize)
+
+ // Share screen button
+ val shareScreenButton = view.findViewById(app.cogwheel.conduit.R.id.btn_share_screen)
+ shareScreenButton?.setOnClickListener {
+ // TODO: Implement share screen functionality
+ launchAppWithContext()
+ }
+
+ // Summarize page button
+ val summarizeButton = view.findViewById(app.cogwheel.conduit.R.id.btn_summarize)
summarizeButton?.setOnClickListener {
launchAppWithContext()
}
-
- val closeButton = view.findViewById(app.cogwheel.conduit.R.id.btn_close)
- closeButton?.setOnClickListener {
- finish()
+
+ // Ask about page button
+ val askAboutButton = view.findViewById(app.cogwheel.conduit.R.id.btn_ask_about)
+ askAboutButton?.setOnClickListener {
+ launchAppWithContext()
}
-
+
+ // Input area (opens text input)
+ val inputArea = view.findViewById(app.cogwheel.conduit.R.id.input_area)
+ inputArea?.setOnClickListener {
+ launchAppWithContext()
+ }
+
+ // Voice button
+ val voiceButton = view.findViewById(app.cogwheel.conduit.R.id.btn_voice)
+ voiceButton?.setOnClickListener {
+ // TODO: Implement voice input functionality
+ launchAppWithContext()
+ }
+
+ // Sparkle button (AI actions)
+ val sparkleButton = view.findViewById(app.cogwheel.conduit.R.id.btn_sparkle)
+ sparkleButton?.setOnClickListener {
+ launchAppWithContext()
+ }
+
return view
}
diff --git a/android/app/src/main/res/drawable/dark_rounded_button_bg.xml b/android/app/src/main/res/drawable/dark_rounded_button_bg.xml
new file mode 100644
index 0000000..50b72f6
--- /dev/null
+++ b/android/app/src/main/res/drawable/dark_rounded_button_bg.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
diff --git a/android/app/src/main/res/drawable/ic_add.xml b/android/app/src/main/res/drawable/ic_add.xml
new file mode 100644
index 0000000..ea55c4e
--- /dev/null
+++ b/android/app/src/main/res/drawable/ic_add.xml
@@ -0,0 +1,10 @@
+
+
+
+
diff --git a/android/app/src/main/res/drawable/ic_ask.xml b/android/app/src/main/res/drawable/ic_ask.xml
new file mode 100644
index 0000000..15ca1fc
--- /dev/null
+++ b/android/app/src/main/res/drawable/ic_ask.xml
@@ -0,0 +1,10 @@
+
+
+
+
diff --git a/android/app/src/main/res/drawable/ic_share.xml b/android/app/src/main/res/drawable/ic_share.xml
new file mode 100644
index 0000000..690915f
--- /dev/null
+++ b/android/app/src/main/res/drawable/ic_share.xml
@@ -0,0 +1,10 @@
+
+
+
+
diff --git a/android/app/src/main/res/drawable/ic_sparkle.xml b/android/app/src/main/res/drawable/ic_sparkle.xml
new file mode 100644
index 0000000..7700658
--- /dev/null
+++ b/android/app/src/main/res/drawable/ic_sparkle.xml
@@ -0,0 +1,13 @@
+
+
+
+
+
diff --git a/android/app/src/main/res/drawable/ic_summarize.xml b/android/app/src/main/res/drawable/ic_summarize.xml
new file mode 100644
index 0000000..5aa52b7
--- /dev/null
+++ b/android/app/src/main/res/drawable/ic_summarize.xml
@@ -0,0 +1,10 @@
+
+
+
+
diff --git a/android/app/src/main/res/drawable/ic_voice.xml b/android/app/src/main/res/drawable/ic_voice.xml
new file mode 100644
index 0000000..39ba5e2
--- /dev/null
+++ b/android/app/src/main/res/drawable/ic_voice.xml
@@ -0,0 +1,10 @@
+
+
+
+
diff --git a/android/app/src/main/res/drawable/input_bar_bg.xml b/android/app/src/main/res/drawable/input_bar_bg.xml
new file mode 100644
index 0000000..1a791a3
--- /dev/null
+++ b/android/app/src/main/res/drawable/input_bar_bg.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
diff --git a/android/app/src/main/res/drawable/voice_button_bg.xml b/android/app/src/main/res/drawable/voice_button_bg.xml
new file mode 100644
index 0000000..b201dcd
--- /dev/null
+++ b/android/app/src/main/res/drawable/voice_button_bg.xml
@@ -0,0 +1,5 @@
+
+
+
+
diff --git a/android/app/src/main/res/layout/assistant_overlay.xml b/android/app/src/main/res/layout/assistant_overlay.xml
index 32ea604..2988499 100644
--- a/android/app/src/main/res/layout/assistant_overlay.xml
+++ b/android/app/src/main/res/layout/assistant_overlay.xml
@@ -1,67 +1,205 @@
+ android:layout_height="match_parent"
+ android:background="#99000000">
+ android:padding="20dp"
+ android:layout_margin="16dp">
+
-
-
-
+ android:orientation="vertical"
+ android:layout_gravity="start"
+ android:layout_marginBottom="16dp">
-
+
-
-
-
+ android:orientation="horizontal"
+ android:background="@drawable/dark_rounded_button_bg"
+ android:paddingStart="16dp"
+ android:paddingEnd="20dp"
+ android:paddingTop="14dp"
+ android:paddingBottom="14dp"
+ android:gravity="center_vertical"
+ android:layout_marginBottom="12dp"
+ android:clickable="true"
+ android:focusable="true">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
From 577d76c2a2c48fe0d228917db4aeb6eeaa70b17a Mon Sep 17 00:00:00 2001
From: cogwheel0 <172976095+cogwheel0@users.noreply.github.com>
Date: Fri, 21 Nov 2025 20:50:27 +0530
Subject: [PATCH 4/7] refactor(ui): Simplify assistant overlay layout and
remove unused buttons
---
.../conduit/ConduitVoiceInteractionSession.kt | 108 ++++++++++++++----
.../src/main/res/layout/assistant_overlay.xml | 95 ++-------------
2 files changed, 92 insertions(+), 111 deletions(-)
diff --git a/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt b/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
index 1e24238..ffd7c9c 100644
--- a/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
+++ b/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
@@ -6,50 +6,39 @@ import android.service.voice.VoiceInteractionSession
import android.os.Bundle
import android.app.assist.AssistStructure
import android.app.assist.AssistContent
+import android.graphics.Bitmap
class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession(context) {
private var capturedContext: String? = null
+ private var capturedScreenshot: Bitmap? = null
override fun onCreateContentView(): android.view.View {
val view = layoutInflater.inflate(app.cogwheel.conduit.R.layout.assistant_overlay, null)
- // Share screen button
- val shareScreenButton = view.findViewById(app.cogwheel.conduit.R.id.btn_share_screen)
- shareScreenButton?.setOnClickListener {
- // TODO: Implement share screen functionality
- launchAppWithContext()
- }
-
- // Summarize page button
+ // Summarize page button - sends screen context
val summarizeButton = view.findViewById(app.cogwheel.conduit.R.id.btn_summarize)
summarizeButton?.setOnClickListener {
- launchAppWithContext()
+ launchAppWithContext(includeScreenshot = false)
}
- // Ask about page button
+ // Ask about page button - sends screenshot
val askAboutButton = view.findViewById(app.cogwheel.conduit.R.id.btn_ask_about)
askAboutButton?.setOnClickListener {
- launchAppWithContext()
+ launchAppWithScreenshot()
}
// Input area (opens text input)
val inputArea = view.findViewById(app.cogwheel.conduit.R.id.input_area)
inputArea?.setOnClickListener {
- launchAppWithContext()
+ launchApp()
}
// Voice button
val voiceButton = view.findViewById(app.cogwheel.conduit.R.id.btn_voice)
voiceButton?.setOnClickListener {
// TODO: Implement voice input functionality
- launchAppWithContext()
- }
-
- // Sparkle button (AI actions)
- val sparkleButton = view.findViewById(app.cogwheel.conduit.R.id.btn_sparkle)
- sparkleButton?.setOnClickListener {
- launchAppWithContext()
+ launchAppWithContext(includeScreenshot = false)
}
return view
@@ -61,9 +50,10 @@ class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession
content: AssistContent?
) {
super.onHandleAssist(data, structure, content)
-
+
android.util.Log.d("ConduitVoiceSession", "onHandleAssist called")
+ // Capture screen context
val screenContext = StringBuilder()
structure?.let {
val nodes = it.windowNodeCount
@@ -72,26 +62,60 @@ class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession
traverseNode(windowNode.rootViewNode, screenContext)
}
}
-
capturedContext = screenContext.toString()
- // Ideally, we could update the UI here to say "Context Ready"
+
+ // Capture screenshot from assist data
+ data?.let {
+ try {
+ capturedScreenshot = it.getParcelable("screenshot")
+ if (capturedScreenshot == null) {
+ // Try alternative key
+ capturedScreenshot = it.getParcelable("android.intent.extra.ASSIST_SCREENSHOT")
+ }
+ android.util.Log.d("ConduitVoiceSession", "Screenshot captured: ${capturedScreenshot != null}")
+ } catch (e: Exception) {
+ android.util.Log.e("ConduitVoiceSession", "Failed to get screenshot from bundle", e)
+ }
+ }
}
- private fun launchAppWithContext() {
+ override fun onHandleScreenshot(screenshot: Bitmap?) {
+ super.onHandleScreenshot(screenshot)
+ capturedScreenshot = screenshot
+ android.util.Log.d("ConduitVoiceSession", "Screenshot received via onHandleScreenshot: ${screenshot != null}")
+ }
+
+ private fun launchApp() {
+ try {
+ android.util.Log.d("ConduitVoiceSession", "Attempting to launch app")
+ val intent = Intent(context, MainActivity::class.java)
+ intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
+ intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP)
+ intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP)
+
+ context.startActivity(intent)
+ android.util.Log.d("ConduitVoiceSession", "App launch requested")
+ finish() // Close the overlay
+ } catch (e: Exception) {
+ android.util.Log.e("ConduitVoiceSession", "Failed to launch app", e)
+ }
+ }
+
+ private fun launchAppWithContext(includeScreenshot: Boolean) {
try {
android.util.Log.d("ConduitVoiceSession", "Attempting to launch app with context")
val intent = Intent(context, MainActivity::class.java)
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP)
intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP)
-
+
if (capturedContext != null) {
intent.putExtra("screen_context", capturedContext)
android.util.Log.d("ConduitVoiceSession", "Context attached: ${capturedContext?.take(50)}...")
} else {
android.util.Log.d("ConduitVoiceSession", "No context captured")
}
-
+
context.startActivity(intent)
android.util.Log.d("ConduitVoiceSession", "App launch requested")
finish() // Close the overlay
@@ -100,6 +124,40 @@ class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession
}
}
+ private fun launchAppWithScreenshot() {
+ try {
+ android.util.Log.d("ConduitVoiceSession", "Attempting to launch app with screenshot")
+ val intent = Intent(context, MainActivity::class.java)
+ intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
+ intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP)
+ intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP)
+
+ // Save screenshot to cache and pass URI
+ capturedScreenshot?.let { bitmap ->
+ try {
+ val file = java.io.File(context.cacheDir, "assistant_screenshot_${System.currentTimeMillis()}.png")
+ val outputStream = java.io.FileOutputStream(file)
+ bitmap.compress(Bitmap.CompressFormat.PNG, 100, outputStream)
+ outputStream.flush()
+ outputStream.close()
+
+ intent.putExtra("screenshot_path", file.absolutePath)
+ android.util.Log.d("ConduitVoiceSession", "Screenshot saved to: ${file.absolutePath}")
+ } catch (e: Exception) {
+ android.util.Log.e("ConduitVoiceSession", "Failed to save screenshot", e)
+ }
+ } ?: run {
+ android.util.Log.d("ConduitVoiceSession", "No screenshot captured")
+ }
+
+ context.startActivity(intent)
+ android.util.Log.d("ConduitVoiceSession", "App launch requested with screenshot")
+ finish() // Close the overlay
+ } catch (e: Exception) {
+ android.util.Log.e("ConduitVoiceSession", "Failed to launch app with screenshot", e)
+ }
+ }
+
private fun traverseNode(node: AssistStructure.ViewNode?, builder: StringBuilder) {
if (node == null) return
diff --git a/android/app/src/main/res/layout/assistant_overlay.xml b/android/app/src/main/res/layout/assistant_overlay.xml
index 2988499..26b252a 100644
--- a/android/app/src/main/res/layout/assistant_overlay.xml
+++ b/android/app/src/main/res/layout/assistant_overlay.xml
@@ -20,36 +20,6 @@
android:layout_gravity="start"
android:layout_marginBottom="16dp">
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
-
-
-
+ android:focusable="true"/>
-
-
-
-
-
-
+ android:src="@drawable/ic_voice"/>
From daecceb3421c75a276a4fddb3a9fe121e1a5fa2a Mon Sep 17 00:00:00 2001
From: cogwheel0 <172976095+cogwheel0@users.noreply.github.com>
Date: Fri, 21 Nov 2025 20:59:49 +0530
Subject: [PATCH 5/7] feat(android): Add screenshot processing for Android
assistant
---
.../app/cogwheel/conduit/MainActivity.kt | 15 ++++
lib/core/utils/android_assistant_handler.dart | 70 +++++++++++++++++++
2 files changed, 85 insertions(+)
diff --git a/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt b/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt
index 4343470..04fa508 100644
--- a/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt
+++ b/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt
@@ -45,9 +45,24 @@ class MainActivity : FlutterActivity() {
}
private fun handleIntent(intent: android.content.Intent) {
+ android.util.Log.d("MainActivity", "handleIntent called")
+ android.util.Log.d("MainActivity", "Intent extras: ${intent.extras?.keySet()}")
+
val screenContext = intent.getStringExtra("screen_context")
+ val screenshotPath = intent.getStringExtra("screenshot_path")
+
+ android.util.Log.d("MainActivity", "screenContext: $screenContext")
+ android.util.Log.d("MainActivity", "screenshotPath: $screenshotPath")
+ android.util.Log.d("MainActivity", "methodChannel: $methodChannel")
+
if (screenContext != null) {
+ android.util.Log.d("MainActivity", "Invoking analyzeScreen")
methodChannel?.invokeMethod("analyzeScreen", screenContext)
+ } else if (screenshotPath != null) {
+ android.util.Log.d("MainActivity", "Invoking analyzeScreenshot with path: $screenshotPath")
+ methodChannel?.invokeMethod("analyzeScreenshot", screenshotPath)
+ } else {
+ android.util.Log.d("MainActivity", "No screen context or screenshot path found")
}
}
diff --git a/lib/core/utils/android_assistant_handler.dart b/lib/core/utils/android_assistant_handler.dart
index 19b9f44..1e079cb 100644
--- a/lib/core/utils/android_assistant_handler.dart
+++ b/lib/core/utils/android_assistant_handler.dart
@@ -1,5 +1,14 @@
+import 'dart:io';
import 'package:flutter/services.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
+import 'package:path/path.dart' as path;
+import '../../features/chat/providers/chat_providers.dart';
+import '../../features/chat/services/file_attachment_service.dart';
+import '../services/navigation_service.dart';
+import '../../shared/services/tasks/task_queue.dart';
+import '../providers/app_providers.dart';
+import '../../features/auth/providers/unified_auth_providers.dart';
+import 'debug_logger.dart';
final androidAssistantProvider = Provider(
(ref) => AndroidAssistantHandler(ref),
@@ -30,6 +39,67 @@ class AndroidAssistantHandler {
if (call.method == 'analyzeScreen') {
final String context = call.arguments as String;
_ref.read(screenContextProvider.notifier).setContext(context);
+ } else if (call.method == 'analyzeScreenshot') {
+ final String screenshotPath = call.arguments as String;
+ await _processScreenshot(screenshotPath);
+ }
+ }
+
+ Future _processScreenshot(String screenshotPath) async {
+ try {
+ DebugLogger.log('Processing screenshot: $screenshotPath', scope: 'assistant');
+
+ // Wait for app to be ready (authenticated and model available)
+ final navState = _ref.read(authNavigationStateProvider);
+ final model = _ref.read(selectedModelProvider);
+
+ if (navState != AuthNavigationState.authenticated || model == null) {
+ DebugLogger.log('App not ready for screenshot processing', scope: 'assistant');
+ return;
+ }
+
+ // Navigate to chat if not already there
+ final isOnChatRoute = NavigationService.currentRoute == Routes.chat;
+ if (!isOnChatRoute) {
+ // Navigation will happen via auth state
+ return;
+ }
+
+ // Start a fresh chat context
+ startNewChat(_ref);
+
+ // Add screenshot as attachment
+ final file = File(screenshotPath);
+ if (!await file.exists()) {
+ DebugLogger.log('Screenshot file not found: $screenshotPath', scope: 'assistant');
+ return;
+ }
+
+ final svc = _ref.read(fileAttachmentServiceProvider);
+ if (svc != null) {
+ final attachment = LocalAttachment(
+ file: file,
+ displayName: path.basename(screenshotPath),
+ );
+
+ _ref.read(attachedFilesProvider.notifier).addFiles([attachment]);
+
+ // Enqueue upload via task queue
+ final activeConv = _ref.read(activeConversationProvider);
+ try {
+ await _ref.read(taskQueueProvider.notifier).enqueueUploadMedia(
+ conversationId: activeConv?.id,
+ filePath: attachment.file.path,
+ fileName: attachment.displayName,
+ fileSize: await attachment.file.length(),
+ );
+ DebugLogger.log('Screenshot uploaded successfully', scope: 'assistant');
+ } catch (e) {
+ DebugLogger.log('Failed to upload screenshot: $e', scope: 'assistant');
+ }
+ }
+ } catch (e) {
+ DebugLogger.log('Failed to process screenshot: $e', scope: 'assistant');
}
}
}
From 9d47c8a96418f0eb63dd921cf2c35c15b80fff8a Mon Sep 17 00:00:00 2001
From: cogwheel0 <172976095+cogwheel0@users.noreply.github.com>
Date: Fri, 21 Nov 2025 21:05:59 +0530
Subject: [PATCH 6/7] feat(voice-call): Implement direct voice call launch from
assistant
---
.../conduit/ConduitVoiceInteractionSession.kt | 26 +++++++++--
.../app/cogwheel/conduit/MainActivity.kt | 7 ++-
lib/core/utils/android_assistant_handler.dart | 45 +++++++++++++++++++
3 files changed, 73 insertions(+), 5 deletions(-)
diff --git a/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt b/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
index ffd7c9c..178c66c 100644
--- a/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
+++ b/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSession.kt
@@ -34,11 +34,10 @@ class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession
launchApp()
}
- // Voice button
+ // Voice button - opens voice call directly
val voiceButton = view.findViewById(app.cogwheel.conduit.R.id.btn_voice)
voiceButton?.setOnClickListener {
- // TODO: Implement voice input functionality
- launchAppWithContext(includeScreenshot = false)
+ launchAppForVoiceCall()
}
return view
@@ -158,13 +157,32 @@ class ConduitVoiceInteractionSession(context: Context) : VoiceInteractionSession
}
}
+ private fun launchAppForVoiceCall() {
+ try {
+ android.util.Log.d("ConduitVoiceSession", "Attempting to launch app for voice call")
+ val intent = Intent(context, MainActivity::class.java)
+ intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
+ intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP)
+ intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP)
+
+ intent.putExtra("start_voice_call", true)
+ android.util.Log.d("ConduitVoiceSession", "Voice call flag attached")
+
+ context.startActivity(intent)
+ android.util.Log.d("ConduitVoiceSession", "App launch requested for voice call")
+ finish() // Close the overlay
+ } catch (e: Exception) {
+ android.util.Log.e("ConduitVoiceSession", "Failed to launch app for voice call", e)
+ }
+ }
+
private fun traverseNode(node: AssistStructure.ViewNode?, builder: StringBuilder) {
if (node == null) return
if (node.text != null) {
builder.append(node.text).append("\n")
}
-
+
// Also check content description for accessibility text
if (node.contentDescription != null) {
builder.append(node.contentDescription).append("\n")
diff --git a/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt b/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt
index 04fa508..e2cb63f 100644
--- a/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt
+++ b/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt
@@ -50,12 +50,17 @@ class MainActivity : FlutterActivity() {
val screenContext = intent.getStringExtra("screen_context")
val screenshotPath = intent.getStringExtra("screenshot_path")
+ val startVoiceCall = intent.getBooleanExtra("start_voice_call", false)
android.util.Log.d("MainActivity", "screenContext: $screenContext")
android.util.Log.d("MainActivity", "screenshotPath: $screenshotPath")
+ android.util.Log.d("MainActivity", "startVoiceCall: $startVoiceCall")
android.util.Log.d("MainActivity", "methodChannel: $methodChannel")
- if (screenContext != null) {
+ if (startVoiceCall) {
+ android.util.Log.d("MainActivity", "Invoking startVoiceCall")
+ methodChannel?.invokeMethod("startVoiceCall", null)
+ } else if (screenContext != null) {
android.util.Log.d("MainActivity", "Invoking analyzeScreen")
methodChannel?.invokeMethod("analyzeScreen", screenContext)
} else if (screenshotPath != null) {
diff --git a/lib/core/utils/android_assistant_handler.dart b/lib/core/utils/android_assistant_handler.dart
index 1e079cb..4b03461 100644
--- a/lib/core/utils/android_assistant_handler.dart
+++ b/lib/core/utils/android_assistant_handler.dart
@@ -1,9 +1,11 @@
import 'dart:io';
+import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:path/path.dart' as path;
import '../../features/chat/providers/chat_providers.dart';
import '../../features/chat/services/file_attachment_service.dart';
+import '../../features/chat/views/voice_call_page.dart';
import '../services/navigation_service.dart';
import '../../shared/services/tasks/task_queue.dart';
import '../providers/app_providers.dart';
@@ -42,6 +44,8 @@ class AndroidAssistantHandler {
} else if (call.method == 'analyzeScreenshot') {
final String screenshotPath = call.arguments as String;
await _processScreenshot(screenshotPath);
+ } else if (call.method == 'startVoiceCall') {
+ await _startVoiceCall();
}
}
@@ -102,4 +106,45 @@ class AndroidAssistantHandler {
DebugLogger.log('Failed to process screenshot: $e', scope: 'assistant');
}
}
+
+ Future _startVoiceCall() async {
+ try {
+ DebugLogger.log('Starting voice call from assistant', scope: 'assistant');
+
+ // Wait for app to be ready (authenticated and model available)
+ final navState = _ref.read(authNavigationStateProvider);
+ final model = _ref.read(selectedModelProvider);
+
+ if (navState != AuthNavigationState.authenticated || model == null) {
+ DebugLogger.log('App not ready for voice call', scope: 'assistant');
+ return;
+ }
+
+ // Navigate to chat if not already there
+ final isOnChatRoute = NavigationService.currentRoute == Routes.chat;
+ if (!isOnChatRoute) {
+ // Navigation will happen via auth state
+ return;
+ }
+
+ // Get the current BuildContext from the navigation service
+ final context = NavigationService.navigatorKey.currentContext;
+ if (context == null) {
+ DebugLogger.log('No context available for voice call navigation', scope: 'assistant');
+ return;
+ }
+
+ // Navigate to voice call page
+ await Navigator.of(context).push(
+ MaterialPageRoute(
+ builder: (context) => const VoiceCallPage(),
+ fullscreenDialog: true,
+ ),
+ );
+
+ DebugLogger.log('Voice call page launched', scope: 'assistant');
+ } catch (e) {
+ DebugLogger.log('Failed to start voice call: $e', scope: 'assistant');
+ }
+ }
}
From 1a6ec3f9ad7ead5ec4533ad61b7627de415d442c Mon Sep 17 00:00:00 2001
From: cogwheel0 <172976095+cogwheel0@users.noreply.github.com>
Date: Fri, 21 Nov 2025 21:12:39 +0530
Subject: [PATCH 7/7] feat(assistant): Improve screen context processing with
model selection
---
lib/core/utils/android_assistant_handler.dart | 30 ++++++++++++++++++-
lib/features/chat/views/chat_page.dart | 6 ++--
2 files changed, 31 insertions(+), 5 deletions(-)
diff --git a/lib/core/utils/android_assistant_handler.dart b/lib/core/utils/android_assistant_handler.dart
index 4b03461..42a8368 100644
--- a/lib/core/utils/android_assistant_handler.dart
+++ b/lib/core/utils/android_assistant_handler.dart
@@ -40,7 +40,7 @@ class AndroidAssistantHandler {
Future _handleMethodCall(MethodCall call) async {
if (call.method == 'analyzeScreen') {
final String context = call.arguments as String;
- _ref.read(screenContextProvider.notifier).setContext(context);
+ await _processScreenContext(context);
} else if (call.method == 'analyzeScreenshot') {
final String screenshotPath = call.arguments as String;
await _processScreenshot(screenshotPath);
@@ -49,6 +49,34 @@ class AndroidAssistantHandler {
}
}
+ Future _processScreenContext(String context) async {
+ try {
+ DebugLogger.log('Processing screen context', scope: 'assistant');
+
+ // Wait for app to be ready (authenticated and model available)
+ final navState = _ref.read(authNavigationStateProvider);
+ final model = _ref.read(selectedModelProvider);
+
+ if (navState != AuthNavigationState.authenticated || model == null) {
+ DebugLogger.log('App not ready for screen context processing', scope: 'assistant');
+ return;
+ }
+
+ // Navigate to chat if not already there
+ final isOnChatRoute = NavigationService.currentRoute == Routes.chat;
+ if (!isOnChatRoute) {
+ // Navigation will happen via auth state
+ return;
+ }
+
+ // Set the screen context
+ _ref.read(screenContextProvider.notifier).setContext(context);
+ DebugLogger.log('Screen context set successfully', scope: 'assistant');
+ } catch (e) {
+ DebugLogger.log('Failed to process screen context: $e', scope: 'assistant');
+ }
+ }
+
Future _processScreenshot(String screenshotPath) async {
try {
DebugLogger.log('Processing screenshot: $screenshotPath', scope: 'assistant');
diff --git a/lib/features/chat/views/chat_page.dart b/lib/features/chat/views/chat_page.dart
index 98ee420..06fad40 100644
--- a/lib/features/chat/views/chat_page.dart
+++ b/lib/features/chat/views/chat_page.dart
@@ -315,12 +315,10 @@ class _ChatPageState extends ConsumerState {
// Clear the context so we don't process it again
WidgetsBinding.instance.addPostFrameCallback((_) {
ref.read(screenContextProvider.notifier).setContext(null);
- // Pre-fill the input or send a message
- // For now, let's just pre-fill the input with a prompt
- // TODO: Ideally we should add this as a system message or attachment
+ final currentModel = ref.read(selectedModelProvider);
_handleMessageSend(
"Here is the content of my screen:\n\n$screenContext\n\nCan you summarize this?",
- null,
+ currentModel,
);
});
}