diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 7008f09..e56683c 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -32,6 +32,18 @@ android:allowBackup="false" android:fullBackupContent="false" android:usesCleartextTraffic="true"> + + + + + + + (app.cogwheel.conduit.R.id.btn_summarize) + summarizeButton?.setOnClickListener { + launchAppWithContext(includeScreenshot = false) + } + + // Ask about page button - sends screenshot + val askAboutButton = view.findViewById(app.cogwheel.conduit.R.id.btn_ask_about) + askAboutButton?.setOnClickListener { + launchAppWithScreenshot() + } + + // Input area (opens text input) + val inputArea = view.findViewById(app.cogwheel.conduit.R.id.input_area) + inputArea?.setOnClickListener { + launchApp() + } + + // Voice button - opens voice call directly + val voiceButton = view.findViewById(app.cogwheel.conduit.R.id.btn_voice) + voiceButton?.setOnClickListener { + launchAppForVoiceCall() + } + + return view + } + + override fun onHandleAssist( + data: Bundle?, + structure: AssistStructure?, + content: AssistContent? + ) { + super.onHandleAssist(data, structure, content) + + android.util.Log.d("ConduitVoiceSession", "onHandleAssist called") + + // Capture screen context + val screenContext = StringBuilder() + structure?.let { + val nodes = it.windowNodeCount + for (i in 0 until nodes) { + val windowNode = it.getWindowNodeAt(i) + traverseNode(windowNode.rootViewNode, screenContext) + } + } + capturedContext = screenContext.toString() + + // Capture screenshot from assist data + data?.let { + try { + capturedScreenshot = it.getParcelable("screenshot") + if (capturedScreenshot == null) { + // Try alternative key + capturedScreenshot = it.getParcelable("android.intent.extra.ASSIST_SCREENSHOT") + } + android.util.Log.d("ConduitVoiceSession", "Screenshot captured: ${capturedScreenshot != null}") + } catch (e: Exception) { + android.util.Log.e("ConduitVoiceSession", "Failed to get screenshot from bundle", e) + } + } + } + + override fun onHandleScreenshot(screenshot: Bitmap?) { + super.onHandleScreenshot(screenshot) + capturedScreenshot = screenshot + android.util.Log.d("ConduitVoiceSession", "Screenshot received via onHandleScreenshot: ${screenshot != null}") + } + + private fun launchApp() { + try { + android.util.Log.d("ConduitVoiceSession", "Attempting to launch app") + val intent = Intent(context, MainActivity::class.java) + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP) + intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP) + + context.startActivity(intent) + android.util.Log.d("ConduitVoiceSession", "App launch requested") + finish() // Close the overlay + } catch (e: Exception) { + android.util.Log.e("ConduitVoiceSession", "Failed to launch app", e) + } + } + + private fun launchAppWithContext(includeScreenshot: Boolean) { + try { + android.util.Log.d("ConduitVoiceSession", "Attempting to launch app with context") + val intent = Intent(context, MainActivity::class.java) + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP) + intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP) + + if (capturedContext != null) { + intent.putExtra("screen_context", capturedContext) + android.util.Log.d("ConduitVoiceSession", "Context attached: ${capturedContext?.take(50)}...") + } else { + android.util.Log.d("ConduitVoiceSession", "No context captured") + } + + context.startActivity(intent) + android.util.Log.d("ConduitVoiceSession", "App launch requested") + finish() // Close the overlay + } catch (e: Exception) { + android.util.Log.e("ConduitVoiceSession", "Failed to launch app", e) + } + } + + private fun launchAppWithScreenshot() { + try { + android.util.Log.d("ConduitVoiceSession", "Attempting to launch app with screenshot") + val intent = Intent(context, MainActivity::class.java) + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP) + intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP) + + // Save screenshot to cache and pass URI + capturedScreenshot?.let { bitmap -> + try { + val file = java.io.File(context.cacheDir, "assistant_screenshot_${System.currentTimeMillis()}.png") + val outputStream = java.io.FileOutputStream(file) + bitmap.compress(Bitmap.CompressFormat.PNG, 100, outputStream) + outputStream.flush() + outputStream.close() + + intent.putExtra("screenshot_path", file.absolutePath) + android.util.Log.d("ConduitVoiceSession", "Screenshot saved to: ${file.absolutePath}") + } catch (e: Exception) { + android.util.Log.e("ConduitVoiceSession", "Failed to save screenshot", e) + } + } ?: run { + android.util.Log.d("ConduitVoiceSession", "No screenshot captured") + } + + context.startActivity(intent) + android.util.Log.d("ConduitVoiceSession", "App launch requested with screenshot") + finish() // Close the overlay + } catch (e: Exception) { + android.util.Log.e("ConduitVoiceSession", "Failed to launch app with screenshot", e) + } + } + + private fun launchAppForVoiceCall() { + try { + android.util.Log.d("ConduitVoiceSession", "Attempting to launch app for voice call") + val intent = Intent(context, MainActivity::class.java) + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP) + intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP) + + intent.putExtra("start_voice_call", true) + android.util.Log.d("ConduitVoiceSession", "Voice call flag attached") + + context.startActivity(intent) + android.util.Log.d("ConduitVoiceSession", "App launch requested for voice call") + finish() // Close the overlay + } catch (e: Exception) { + android.util.Log.e("ConduitVoiceSession", "Failed to launch app for voice call", e) + } + } + + private fun traverseNode(node: AssistStructure.ViewNode?, builder: StringBuilder) { + if (node == null) return + + if (node.text != null) { + builder.append(node.text).append("\n") + } + + // Also check content description for accessibility text + if (node.contentDescription != null) { + builder.append(node.contentDescription).append("\n") + } + + for (i in 0 until node.childCount) { + traverseNode(node.getChildAt(i), builder) + } + } +} diff --git a/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSessionService.kt b/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSessionService.kt new file mode 100644 index 0000000..dce73ab --- /dev/null +++ b/android/app/src/main/kotlin/app/cogwheel/conduit/ConduitVoiceInteractionSessionService.kt @@ -0,0 +1,11 @@ +package app.cogwheel.conduit + +import android.service.voice.VoiceInteractionSession +import android.service.voice.VoiceInteractionSessionService +import android.os.Bundle + +class ConduitVoiceInteractionSessionService : VoiceInteractionSessionService() { + override fun onNewSession(args: Bundle?): VoiceInteractionSession { + return ConduitVoiceInteractionSession(this) + } +} diff --git a/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt b/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt index edecfcf..e2cb63f 100644 --- a/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt +++ b/android/app/src/main/kotlin/app/cogwheel/conduit/MainActivity.kt @@ -23,12 +23,52 @@ class MainActivity : FlutterActivity() { windowInsetsController.isAppearanceLightNavigationBars = false } + private val CHANNEL = "app.cogwheel.conduit/assistant" + private var methodChannel: io.flutter.plugin.common.MethodChannel? = null + override fun configureFlutterEngine(flutterEngine: FlutterEngine) { super.configureFlutterEngine(flutterEngine) // Initialize background streaming handler backgroundStreamingHandler = BackgroundStreamingHandler(this) backgroundStreamingHandler.setup(flutterEngine) + + methodChannel = io.flutter.plugin.common.MethodChannel(flutterEngine.dartExecutor.binaryMessenger, CHANNEL) + + // Check if started with context + handleIntent(intent) + } + + override fun onNewIntent(intent: android.content.Intent) { + super.onNewIntent(intent) + handleIntent(intent) + } + + private fun handleIntent(intent: android.content.Intent) { + android.util.Log.d("MainActivity", "handleIntent called") + android.util.Log.d("MainActivity", "Intent extras: ${intent.extras?.keySet()}") + + val screenContext = intent.getStringExtra("screen_context") + val screenshotPath = intent.getStringExtra("screenshot_path") + val startVoiceCall = intent.getBooleanExtra("start_voice_call", false) + + android.util.Log.d("MainActivity", "screenContext: $screenContext") + android.util.Log.d("MainActivity", "screenshotPath: $screenshotPath") + android.util.Log.d("MainActivity", "startVoiceCall: $startVoiceCall") + android.util.Log.d("MainActivity", "methodChannel: $methodChannel") + + if (startVoiceCall) { + android.util.Log.d("MainActivity", "Invoking startVoiceCall") + methodChannel?.invokeMethod("startVoiceCall", null) + } else if (screenContext != null) { + android.util.Log.d("MainActivity", "Invoking analyzeScreen") + methodChannel?.invokeMethod("analyzeScreen", screenContext) + } else if (screenshotPath != null) { + android.util.Log.d("MainActivity", "Invoking analyzeScreenshot with path: $screenshotPath") + methodChannel?.invokeMethod("analyzeScreenshot", screenshotPath) + } else { + android.util.Log.d("MainActivity", "No screen context or screenshot path found") + } } override fun onDestroy() { diff --git a/android/app/src/main/res/drawable/dark_rounded_button_bg.xml b/android/app/src/main/res/drawable/dark_rounded_button_bg.xml new file mode 100644 index 0000000..50b72f6 --- /dev/null +++ b/android/app/src/main/res/drawable/dark_rounded_button_bg.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/android/app/src/main/res/drawable/ic_add.xml b/android/app/src/main/res/drawable/ic_add.xml new file mode 100644 index 0000000..ea55c4e --- /dev/null +++ b/android/app/src/main/res/drawable/ic_add.xml @@ -0,0 +1,10 @@ + + + + diff --git a/android/app/src/main/res/drawable/ic_ask.xml b/android/app/src/main/res/drawable/ic_ask.xml new file mode 100644 index 0000000..15ca1fc --- /dev/null +++ b/android/app/src/main/res/drawable/ic_ask.xml @@ -0,0 +1,10 @@ + + + + diff --git a/android/app/src/main/res/drawable/ic_share.xml b/android/app/src/main/res/drawable/ic_share.xml new file mode 100644 index 0000000..690915f --- /dev/null +++ b/android/app/src/main/res/drawable/ic_share.xml @@ -0,0 +1,10 @@ + + + + diff --git a/android/app/src/main/res/drawable/ic_sparkle.xml b/android/app/src/main/res/drawable/ic_sparkle.xml new file mode 100644 index 0000000..7700658 --- /dev/null +++ b/android/app/src/main/res/drawable/ic_sparkle.xml @@ -0,0 +1,13 @@ + + + + + diff --git a/android/app/src/main/res/drawable/ic_summarize.xml b/android/app/src/main/res/drawable/ic_summarize.xml new file mode 100644 index 0000000..5aa52b7 --- /dev/null +++ b/android/app/src/main/res/drawable/ic_summarize.xml @@ -0,0 +1,10 @@ + + + + diff --git a/android/app/src/main/res/drawable/ic_voice.xml b/android/app/src/main/res/drawable/ic_voice.xml new file mode 100644 index 0000000..39ba5e2 --- /dev/null +++ b/android/app/src/main/res/drawable/ic_voice.xml @@ -0,0 +1,10 @@ + + + + diff --git a/android/app/src/main/res/drawable/input_bar_bg.xml b/android/app/src/main/res/drawable/input_bar_bg.xml new file mode 100644 index 0000000..1a791a3 --- /dev/null +++ b/android/app/src/main/res/drawable/input_bar_bg.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/android/app/src/main/res/drawable/rounded_button_bg.xml b/android/app/src/main/res/drawable/rounded_button_bg.xml new file mode 100644 index 0000000..ab7b3f1 --- /dev/null +++ b/android/app/src/main/res/drawable/rounded_button_bg.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/android/app/src/main/res/drawable/rounded_overlay_bg.xml b/android/app/src/main/res/drawable/rounded_overlay_bg.xml new file mode 100644 index 0000000..53dea9a --- /dev/null +++ b/android/app/src/main/res/drawable/rounded_overlay_bg.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/android/app/src/main/res/drawable/voice_button_bg.xml b/android/app/src/main/res/drawable/voice_button_bg.xml new file mode 100644 index 0000000..b201dcd --- /dev/null +++ b/android/app/src/main/res/drawable/voice_button_bg.xml @@ -0,0 +1,5 @@ + + + + diff --git a/android/app/src/main/res/layout/assistant_overlay.xml b/android/app/src/main/res/layout/assistant_overlay.xml new file mode 100644 index 0000000..26b252a --- /dev/null +++ b/android/app/src/main/res/layout/assistant_overlay.xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/app/src/main/res/xml/voice_interaction_service.xml b/android/app/src/main/res/xml/voice_interaction_service.xml new file mode 100644 index 0000000..dd01cc5 --- /dev/null +++ b/android/app/src/main/res/xml/voice_interaction_service.xml @@ -0,0 +1,7 @@ + + diff --git a/lib/core/utils/android_assistant_handler.dart b/lib/core/utils/android_assistant_handler.dart new file mode 100644 index 0000000..42a8368 --- /dev/null +++ b/lib/core/utils/android_assistant_handler.dart @@ -0,0 +1,178 @@ +import 'dart:io'; +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:path/path.dart' as path; +import '../../features/chat/providers/chat_providers.dart'; +import '../../features/chat/services/file_attachment_service.dart'; +import '../../features/chat/views/voice_call_page.dart'; +import '../services/navigation_service.dart'; +import '../../shared/services/tasks/task_queue.dart'; +import '../providers/app_providers.dart'; +import '../../features/auth/providers/unified_auth_providers.dart'; +import 'debug_logger.dart'; + +final androidAssistantProvider = Provider( + (ref) => AndroidAssistantHandler(ref), +); + +final screenContextProvider = NotifierProvider( + ScreenContextNotifier.new, +); + +class ScreenContextNotifier extends Notifier { + @override + String? build() => null; + + void setContext(String? context) { + state = context; + } +} + +class AndroidAssistantHandler { + static const platform = MethodChannel('app.cogwheel.conduit/assistant'); + final Ref _ref; + + AndroidAssistantHandler(this._ref) { + platform.setMethodCallHandler(_handleMethodCall); + } + + Future _handleMethodCall(MethodCall call) async { + if (call.method == 'analyzeScreen') { + final String context = call.arguments as String; + await _processScreenContext(context); + } else if (call.method == 'analyzeScreenshot') { + final String screenshotPath = call.arguments as String; + await _processScreenshot(screenshotPath); + } else if (call.method == 'startVoiceCall') { + await _startVoiceCall(); + } + } + + Future _processScreenContext(String context) async { + try { + DebugLogger.log('Processing screen context', scope: 'assistant'); + + // Wait for app to be ready (authenticated and model available) + final navState = _ref.read(authNavigationStateProvider); + final model = _ref.read(selectedModelProvider); + + if (navState != AuthNavigationState.authenticated || model == null) { + DebugLogger.log('App not ready for screen context processing', scope: 'assistant'); + return; + } + + // Navigate to chat if not already there + final isOnChatRoute = NavigationService.currentRoute == Routes.chat; + if (!isOnChatRoute) { + // Navigation will happen via auth state + return; + } + + // Set the screen context + _ref.read(screenContextProvider.notifier).setContext(context); + DebugLogger.log('Screen context set successfully', scope: 'assistant'); + } catch (e) { + DebugLogger.log('Failed to process screen context: $e', scope: 'assistant'); + } + } + + Future _processScreenshot(String screenshotPath) async { + try { + DebugLogger.log('Processing screenshot: $screenshotPath', scope: 'assistant'); + + // Wait for app to be ready (authenticated and model available) + final navState = _ref.read(authNavigationStateProvider); + final model = _ref.read(selectedModelProvider); + + if (navState != AuthNavigationState.authenticated || model == null) { + DebugLogger.log('App not ready for screenshot processing', scope: 'assistant'); + return; + } + + // Navigate to chat if not already there + final isOnChatRoute = NavigationService.currentRoute == Routes.chat; + if (!isOnChatRoute) { + // Navigation will happen via auth state + return; + } + + // Start a fresh chat context + startNewChat(_ref); + + // Add screenshot as attachment + final file = File(screenshotPath); + if (!await file.exists()) { + DebugLogger.log('Screenshot file not found: $screenshotPath', scope: 'assistant'); + return; + } + + final svc = _ref.read(fileAttachmentServiceProvider); + if (svc != null) { + final attachment = LocalAttachment( + file: file, + displayName: path.basename(screenshotPath), + ); + + _ref.read(attachedFilesProvider.notifier).addFiles([attachment]); + + // Enqueue upload via task queue + final activeConv = _ref.read(activeConversationProvider); + try { + await _ref.read(taskQueueProvider.notifier).enqueueUploadMedia( + conversationId: activeConv?.id, + filePath: attachment.file.path, + fileName: attachment.displayName, + fileSize: await attachment.file.length(), + ); + DebugLogger.log('Screenshot uploaded successfully', scope: 'assistant'); + } catch (e) { + DebugLogger.log('Failed to upload screenshot: $e', scope: 'assistant'); + } + } + } catch (e) { + DebugLogger.log('Failed to process screenshot: $e', scope: 'assistant'); + } + } + + Future _startVoiceCall() async { + try { + DebugLogger.log('Starting voice call from assistant', scope: 'assistant'); + + // Wait for app to be ready (authenticated and model available) + final navState = _ref.read(authNavigationStateProvider); + final model = _ref.read(selectedModelProvider); + + if (navState != AuthNavigationState.authenticated || model == null) { + DebugLogger.log('App not ready for voice call', scope: 'assistant'); + return; + } + + // Navigate to chat if not already there + final isOnChatRoute = NavigationService.currentRoute == Routes.chat; + if (!isOnChatRoute) { + // Navigation will happen via auth state + return; + } + + // Get the current BuildContext from the navigation service + final context = NavigationService.navigatorKey.currentContext; + if (context == null) { + DebugLogger.log('No context available for voice call navigation', scope: 'assistant'); + return; + } + + // Navigate to voice call page + await Navigator.of(context).push( + MaterialPageRoute( + builder: (context) => const VoiceCallPage(), + fullscreenDialog: true, + ), + ); + + DebugLogger.log('Voice call page launched', scope: 'assistant'); + } catch (e) { + DebugLogger.log('Failed to start voice call: $e', scope: 'assistant'); + } + } +} diff --git a/lib/features/chat/views/chat_page.dart b/lib/features/chat/views/chat_page.dart index 3453ccb..06fad40 100644 --- a/lib/features/chat/views/chat_page.dart +++ b/lib/features/chat/views/chat_page.dart @@ -17,6 +17,7 @@ import '../../../core/utils/debug_logger.dart'; import '../../../core/utils/user_display_name.dart'; import '../../../core/utils/model_icon_utils.dart'; import '../../auth/providers/unified_auth_providers.dart'; +import '../../../core/utils/android_assistant_handler.dart'; import '../widgets/modern_chat_input.dart'; import '../widgets/user_message_bubble.dart'; @@ -288,6 +289,10 @@ class _ChatPageState extends ConsumerState { // Initialize chat page components WidgetsBinding.instance.addPostFrameCallback((_) async { if (!mounted) return; + + // Initialize Android Assistant Handler + ref.read(androidAssistantProvider); + // First, ensure a model is selected await _checkAndAutoSelectModel(); if (!mounted) return; @@ -301,6 +306,24 @@ class _ChatPageState extends ConsumerState { }); } + @override + void didChangeDependencies() { + super.didChangeDependencies(); + // Listen for screen context from Android Assistant + final screenContext = ref.watch(screenContextProvider); + if (screenContext != null && screenContext.isNotEmpty) { + // Clear the context so we don't process it again + WidgetsBinding.instance.addPostFrameCallback((_) { + ref.read(screenContextProvider.notifier).setContext(null); + final currentModel = ref.read(selectedModelProvider); + _handleMessageSend( + "Here is the content of my screen:\n\n$screenContext\n\nCan you summarize this?", + currentModel, + ); + }); + } + } + @override void dispose() { _scrollController.dispose(); diff --git a/pubspec.lock b/pubspec.lock index 8cfd515..43c4376 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -961,10 +961,10 @@ packages: dependency: transitive description: name: meta - sha256: e3641ec5d63ebf0d9b41bd43201a66e3fc79a65db5f61fc181f04cd27aab950c + sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394" url: "https://pub.dev" source: hosted - version: "1.16.0" + version: "1.17.0" mime: dependency: transitive description: @@ -1606,26 +1606,26 @@ packages: dependency: transitive description: name: test - sha256: "65e29d831719be0591f7b3b1a32a3cda258ec98c58c7b25f7b84241bc31215bb" + sha256: "75906bf273541b676716d1ca7627a17e4c4070a3a16272b7a3dc7da3b9f3f6b7" url: "https://pub.dev" source: hosted - version: "1.26.2" + version: "1.26.3" test_api: dependency: transitive description: name: test_api - sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00" + sha256: ab2726c1a94d3176a45960b6234466ec367179b87dd74f1611adb1f3b5fb9d55 url: "https://pub.dev" source: hosted - version: "0.7.6" + version: "0.7.7" test_core: dependency: transitive description: name: test_core - sha256: "80bf5a02b60af04b09e14f6fe68b921aad119493e26e490deaca5993fef1b05a" + sha256: "0cc24b5ff94b38d2ae73e1eb43cc302b77964fbf67abad1e296025b78deb53d0" url: "https://pub.dev" source: hosted - version: "0.6.11" + version: "0.6.12" timezone: dependency: transitive description: