feat(callkit): Add CallKit service for native call UI and permissions

This commit is contained in:
cogwheel0
2025-11-24 12:29:44 +05:30
parent a52194f336
commit d38e986d7c
11 changed files with 551 additions and 65 deletions

View File

@@ -18,4 +18,7 @@
# Keep WebSocket functionality # Keep WebSocket functionality
-keep class org.java_websocket.** { *; } -keep class org.java_websocket.** { *; }
-dontwarn org.java_websocket.** -dontwarn org.java_websocket.**
# Keep Flutter CallKit Incoming classes
-keep class com.hiennv.flutter_callkit_incoming.** { *; }

View File

@@ -47,7 +47,7 @@
<activity <activity
android:name=".MainActivity" android:name=".MainActivity"
android:exported="true" android:exported="true"
android:launchMode="singleTop" android:launchMode="singleInstance"
android:theme="@style/LaunchTheme" android:theme="@style/LaunchTheme"
android:configChanges="orientation|keyboardHidden|keyboard|screenSize|smallestScreenSize|locale|layoutDirection|fontScale|screenLayout|density|uiMode" android:configChanges="orientation|keyboardHidden|keyboard|screenSize|smallestScreenSize|locale|layoutDirection|fontScale|screenLayout|density|uiMode"
android:hardwareAccelerated="true" android:hardwareAccelerated="true"

View File

@@ -4,6 +4,7 @@ PODS:
- FlutterMacOS - FlutterMacOS
- connectivity_plus (0.0.1): - connectivity_plus (0.0.1):
- Flutter - Flutter
- CryptoSwift (1.8.4)
- DKImagePickerController/Core (4.3.9): - DKImagePickerController/Core (4.3.9):
- DKImagePickerController/ImageDataManager - DKImagePickerController/ImageDataManager
- DKImagePickerController/Resource - DKImagePickerController/Resource
@@ -39,6 +40,9 @@ PODS:
- DKImagePickerController/PhotoGallery - DKImagePickerController/PhotoGallery
- Flutter - Flutter
- Flutter (1.0.0) - Flutter (1.0.0)
- flutter_callkit_incoming (0.0.1):
- CryptoSwift
- Flutter
- flutter_local_notifications (0.0.1): - flutter_local_notifications (0.0.1):
- Flutter - Flutter
- flutter_native_splash (2.4.3): - flutter_native_splash (2.4.3):
@@ -98,6 +102,7 @@ DEPENDENCIES:
- connectivity_plus (from `.symlinks/plugins/connectivity_plus/ios`) - connectivity_plus (from `.symlinks/plugins/connectivity_plus/ios`)
- file_picker (from `.symlinks/plugins/file_picker/ios`) - file_picker (from `.symlinks/plugins/file_picker/ios`)
- Flutter (from `Flutter`) - Flutter (from `Flutter`)
- flutter_callkit_incoming (from `.symlinks/plugins/flutter_callkit_incoming/ios`)
- flutter_local_notifications (from `.symlinks/plugins/flutter_local_notifications/ios`) - flutter_local_notifications (from `.symlinks/plugins/flutter_local_notifications/ios`)
- flutter_native_splash (from `.symlinks/plugins/flutter_native_splash/ios`) - flutter_native_splash (from `.symlinks/plugins/flutter_native_splash/ios`)
- flutter_secure_storage (from `.symlinks/plugins/flutter_secure_storage/ios`) - flutter_secure_storage (from `.symlinks/plugins/flutter_secure_storage/ios`)
@@ -119,6 +124,7 @@ DEPENDENCIES:
SPEC REPOS: SPEC REPOS:
trunk: trunk:
- CryptoSwift
- DKImagePickerController - DKImagePickerController
- DKPhotoGallery - DKPhotoGallery
- onnxruntime-c - onnxruntime-c
@@ -135,6 +141,8 @@ EXTERNAL SOURCES:
:path: ".symlinks/plugins/file_picker/ios" :path: ".symlinks/plugins/file_picker/ios"
Flutter: Flutter:
:path: Flutter :path: Flutter
flutter_callkit_incoming:
:path: ".symlinks/plugins/flutter_callkit_incoming/ios"
flutter_local_notifications: flutter_local_notifications:
:path: ".symlinks/plugins/flutter_local_notifications/ios" :path: ".symlinks/plugins/flutter_local_notifications/ios"
flutter_native_splash: flutter_native_splash:
@@ -175,10 +183,12 @@ EXTERNAL SOURCES:
SPEC CHECKSUMS: SPEC CHECKSUMS:
audioplayers_darwin: 4f9ca89d92d3d21cec7ec580e78ca888e5fb68bd audioplayers_darwin: 4f9ca89d92d3d21cec7ec580e78ca888e5fb68bd
connectivity_plus: cb623214f4e1f6ef8fe7403d580fdad517d2f7dd connectivity_plus: cb623214f4e1f6ef8fe7403d580fdad517d2f7dd
CryptoSwift: e64e11850ede528a02a0f3e768cec8e9d92ecb90
DKImagePickerController: 946cec48c7873164274ecc4624d19e3da4c1ef3c DKImagePickerController: 946cec48c7873164274ecc4624d19e3da4c1ef3c
DKPhotoGallery: b3834fecb755ee09a593d7c9e389d8b5d6deed60 DKPhotoGallery: b3834fecb755ee09a593d7c9e389d8b5d6deed60
file_picker: a0560bc09d61de87f12d246fc47d2119e6ef37be file_picker: a0560bc09d61de87f12d246fc47d2119e6ef37be
Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467 Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467
flutter_callkit_incoming: cb8138af67cda6dd981f7101a5d709003af21502
flutter_local_notifications: a5a732f069baa862e728d839dd2ebb904737effb flutter_local_notifications: a5a732f069baa862e728d839dd2ebb904737effb
flutter_native_splash: c32d145d68aeda5502d5f543ee38c192065986cf flutter_native_splash: c32d145d68aeda5502d5f543ee38c192065986cf
flutter_secure_storage: 1ed9476fba7e7a782b22888f956cce43e2c62f13 flutter_secure_storage: 1ed9476fba7e7a782b22888f956cce43e2c62f13

View File

@@ -87,11 +87,13 @@
</dict> </dict>
</array> </array>
<!-- Background Modes --> <!-- Background Modes -->
<key>UIBackgroundModes</key> <key>UIBackgroundModes</key>
<array> <array>
<string>audio</string> <string>audio</string>
<string>processing</string> <string>remote-notification</string>
</array> <string>processing</string>
<string>voip</string>
</array>
<!-- Background Task Identifiers --> <!-- Background Task Identifiers -->
<key>BGTaskSchedulerPermittedIdentifiers</key> <key>BGTaskSchedulerPermittedIdentifiers</key>
<array> <array>

View File

@@ -33,3 +33,4 @@ final optimizedStorageServiceProvider = Provider<OptimizedStorageService>((
workerManager: ref.watch(workerManagerProvider), workerManager: ref.watch(workerManagerProvider),
); );
}); });

View File

@@ -0,0 +1,184 @@
import 'dart:developer' as developer;
import 'package:flutter_callkit_incoming/entities/entities.dart';
import 'package:flutter_callkit_incoming/flutter_callkit_incoming.dart';
import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:uuid/uuid.dart';
part 'callkit_service.g.dart';
/// Thin wrapper around `flutter_callkit_incoming` for voice calls.
class CallKitService {
CallKitService({Uuid? uuid}) : _uuid = uuid ?? const Uuid();
final Uuid _uuid;
static const int _defaultCallDurationMs = 2 * 60 * 60 * 1000; // 2 hours
/// Requests the notification/full-screen intent permissions needed on Android.
Future<void> requestPermissions() async {
await _safe(
() => FlutterCallkitIncoming.requestNotificationPermission(
<String, dynamic>{
'title': 'Notification permission',
'rationaleMessagePermission':
'Call alerts need notification permission.',
'postNotificationMessageRequired':
'Allow notifications to show incoming calls.',
},
),
);
await _safe(() => FlutterCallkitIncoming.requestFullIntentPermission());
}
/// Starts an outgoing call with the native UI and returns the call id.
Future<String?> startOutgoingVoiceCall({
required String calleeName,
required String handle,
String? avatar,
int? durationMs,
}) async {
final id = _uuid.v4();
final params = _buildParams(
id: id,
callerName: calleeName,
handle: handle,
avatar: avatar,
durationMs: durationMs ?? _defaultCallDurationMs,
);
try {
await FlutterCallkitIncoming.startCall(params);
return id;
} catch (error, stackTrace) {
developer.log(
'CallKit startCall failed: $error',
name: 'callkit',
error: error,
stackTrace: stackTrace,
);
return null;
}
}
/// Marks the current call as connected so iOS shows an incrementing timer.
Future<void> markCallConnected(String id) async {
try {
await FlutterCallkitIncoming.setCallConnected(id);
} catch (error, stackTrace) {
developer.log(
'CallKit setCallConnected failed: $error',
name: 'callkit',
error: error,
stackTrace: stackTrace,
);
}
}
/// Ends a specific call id.
Future<void> endCall(String id) async {
await _safe(() => FlutterCallkitIncoming.endCall(id));
}
/// Clears all ongoing/missed calls.
Future<void> endAllCalls() async {
await _safe(FlutterCallkitIncoming.endAllCalls);
}
/// Returns the platform VOIP token (iOS PushKit) when available.
Future<String?> getVoipToken() async {
final token = await _safe<dynamic>(
() => FlutterCallkitIncoming.getDevicePushTokenVoIP(),
);
if (token == null) return null;
if (token is String) return token;
return token.toString();
}
/// Returns the raw active call list from the plugin.
Future<List<Map<String, dynamic>>> activeCalls() async {
final calls = await _safe<dynamic>(FlutterCallkitIncoming.activeCalls);
if (calls is List) {
return calls
.whereType<Map<dynamic, dynamic>>()
.map(Map<String, dynamic>.from)
.toList();
}
return <Map<String, dynamic>>[];
}
/// Stream of CallKit events from the native layer.
Stream<CallEvent> get events =>
FlutterCallkitIncoming.onEvent.where((event) => event != null).cast();
CallKitParams _buildParams({
required String id,
required String callerName,
required String handle,
String? avatar,
int durationMs = _defaultCallDurationMs,
}) {
return CallKitParams(
id: id,
nameCaller: callerName,
appName: 'Conduit',
avatar: avatar,
handle: handle,
type: 0, // 0 = audio call
duration: durationMs,
textAccept: 'Accept',
textDecline: 'Decline',
missedCallNotification: const NotificationParams(
showNotification: true,
isShowCallback: true,
subtitle: 'Missed call',
callbackText: 'Call back',
),
callingNotification: const NotificationParams(
showNotification: true,
isShowCallback: true,
subtitle: 'Calling...',
callbackText: 'Hang up',
),
extra: const <String, dynamic>{'transport': 'voice'},
android: const AndroidParams(
isCustomNotification: true,
isShowLogo: true,
ringtonePath: 'system_ringtone_default',
backgroundColor: '#0D1726',
actionColor: '#4CAF50',
incomingCallNotificationChannelName: 'Incoming Call',
missedCallNotificationChannelName: 'Missed Call',
),
ios: const IOSParams(
handleType: 'generic',
supportsVideo: false,
audioSessionMode: 'default',
audioSessionActive: true,
audioSessionPreferredSampleRate: 44100.0,
audioSessionPreferredIOBufferDuration: 0.005,
supportsDTMF: true,
supportsHolding: true,
supportsGrouping: false,
supportsUngrouping: false,
ringtonePath: 'system_ringtone_default',
),
);
}
Future<T?> _safe<T>(Future<T> Function() action) async {
try {
return await action();
} catch (error, stackTrace) {
developer.log(
'CallKit error: $error',
name: 'callkit',
error: error,
stackTrace: stackTrace,
);
return null;
}
}
}
@Riverpod(keepAlive: true)
CallKitService callKitService(Ref ref) => CallKitService();

View File

@@ -69,6 +69,22 @@ class TextToSpeechService {
break; break;
} }
}); });
if (!kIsWeb && Platform.isIOS) {
final context = AudioContext(
iOS: AudioContextIOS(
category: AVAudioSessionCategory.playAndRecord,
options: const {
AVAudioSessionOptions.defaultToSpeaker,
AVAudioSessionOptions.mixWithOthers,
AVAudioSessionOptions.allowBluetooth,
AVAudioSessionOptions.allowBluetoothA2DP,
},
),
android: const AudioContextAndroid(),
);
_player.setAudioContext(context);
}
} }
Future<void> _configureDeviceEngine({ Future<void> _configureDeviceEngine({
@@ -87,12 +103,13 @@ class TextToSpeechService {
if (!kIsWeb && Platform.isIOS) { if (!kIsWeb && Platform.isIOS) {
await _tts.setSharedInstance(true); await _tts.setSharedInstance(true);
await _tts.setIosAudioCategory(IosTextToSpeechAudioCategory.playback, [ await _tts
IosTextToSpeechAudioCategoryOptions.mixWithOthers, .setIosAudioCategory(IosTextToSpeechAudioCategory.playAndRecord, [
IosTextToSpeechAudioCategoryOptions.defaultToSpeaker, IosTextToSpeechAudioCategoryOptions.mixWithOthers,
IosTextToSpeechAudioCategoryOptions.allowBluetooth, IosTextToSpeechAudioCategoryOptions.defaultToSpeaker,
IosTextToSpeechAudioCategoryOptions.allowBluetoothA2DP, IosTextToSpeechAudioCategoryOptions.allowBluetooth,
]); IosTextToSpeechAudioCategoryOptions.allowBluetoothA2DP,
]);
} }
if (_engine != TtsEngine.server) { if (_engine != TtsEngine.server) {

View File

@@ -1,12 +1,15 @@
import 'dart:async'; import 'dart:async';
import 'dart:collection'; import 'dart:collection';
import 'dart:developer' as developer;
import 'package:audioplayers/audioplayers.dart'; import 'package:audioplayers/audioplayers.dart';
import 'package:flutter_callkit_incoming/entities/call_event.dart';
import 'package:riverpod_annotation/riverpod_annotation.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:wakelock_plus/wakelock_plus.dart'; import 'package:wakelock_plus/wakelock_plus.dart';
import '../../../core/providers/app_providers.dart'; import '../../../core/providers/app_providers.dart';
import '../../../core/services/background_streaming_handler.dart'; import '../../../core/services/background_streaming_handler.dart';
import '../../../core/services/callkit_service.dart';
import '../../../core/services/socket_service.dart'; import '../../../core/services/socket_service.dart';
import '../../../core/utils/markdown_to_text.dart'; import '../../../core/utils/markdown_to_text.dart';
import '../providers/chat_providers.dart'; import '../providers/chat_providers.dart';
@@ -38,6 +41,7 @@ class VoiceCallService {
final TextToSpeechService _tts; final TextToSpeechService _tts;
final SocketService _socketService; final SocketService _socketService;
final Ref _ref; final Ref _ref;
final CallKitService _callKitService;
final VoiceCallNotificationService _notificationService = final VoiceCallNotificationService _notificationService =
VoiceCallNotificationService(); VoiceCallNotificationService();
@@ -64,6 +68,9 @@ class VoiceCallService {
bool _serverPipelineActive = false; bool _serverPipelineActive = false;
int _nextServerChunkId = 0; int _nextServerChunkId = 0;
int _nextServerPlaybackId = 0; int _nextServerPlaybackId = 0;
bool _callKitPermissionsRequested = false;
String? _callKitCallId;
bool _callKitConnectedReported = false;
final StreamController<VoiceCallState> _stateController = final StreamController<VoiceCallState> _stateController =
StreamController<VoiceCallState>.broadcast(); StreamController<VoiceCallState>.broadcast();
@@ -73,15 +80,18 @@ class VoiceCallService {
StreamController<String>.broadcast(); StreamController<String>.broadcast();
final StreamController<int> _intensityController = final StreamController<int> _intensityController =
StreamController<int>.broadcast(); StreamController<int>.broadcast();
StreamSubscription<CallEvent>? _callKitEventSubscription;
VoiceCallService({ VoiceCallService({
required VoiceInputService voiceInput, required VoiceInputService voiceInput,
required TextToSpeechService tts, required TextToSpeechService tts,
required SocketService socketService, required SocketService socketService,
required CallKitService callKitService,
required Ref ref, required Ref ref,
}) : _voiceInput = voiceInput, }) : _voiceInput = voiceInput,
_tts = tts, _tts = tts,
_socketService = socketService, _socketService = socketService,
_callKitService = callKitService,
_ref = ref { _ref = ref {
_tts.bindHandlers( _tts.bindHandlers(
onStart: _handleTtsStart, onStart: _handleTtsStart,
@@ -161,10 +171,124 @@ class VoiceCallService {
); );
} }
Future<void> _ensureCallKitPermissions() async {
if (_callKitPermissionsRequested) return;
_callKitPermissionsRequested = true;
await _callKitService.requestPermissions();
}
Future<void> _startCallKitSession({required String modelName}) async {
try {
await _ensureCallKitPermissions();
final callId = await _callKitService.startOutgoingVoiceCall(
calleeName: modelName,
handle: 'Conduit AI',
);
_callKitCallId = callId;
_callKitConnectedReported = false;
} catch (error, stackTrace) {
developer.log(
'CallKit outgoing setup failed: $error',
name: 'voice_call',
error: error,
stackTrace: stackTrace,
);
}
}
Future<void> _endCallKitSession() async {
if (_callKitCallId == null) {
return;
}
final callId = _callKitCallId;
_callKitCallId = null;
_callKitConnectedReported = false;
try {
await _callKitService.endCall(callId!);
} catch (error, stackTrace) {
developer.log(
'CallKit endCall failed: $error',
name: 'voice_call',
error: error,
stackTrace: stackTrace,
);
await _callKitService.endAllCalls();
}
}
void _listenForCallKitEvents() {
_callKitEventSubscription?.cancel();
if (_callKitCallId == null) return;
_callKitEventSubscription = _callKitService.events.listen((callEvent) {
final eventId = _extractCallId(callEvent.body);
if (_callKitCallId != null &&
eventId != null &&
eventId != _callKitCallId) {
return;
}
switch (callEvent.event) {
case Event.actionCallEnded:
case Event.actionCallDecline:
case Event.actionCallTimeout:
if (_state != VoiceCallState.disconnected) {
unawaited(stopCall());
}
break;
case Event.actionCallToggleMute:
_handleCallKitMute(callEvent.body);
break;
case Event.actionCallToggleHold:
_handleCallKitHold(callEvent.body);
break;
case Event.actionCallConnected:
unawaited(_markCallKitConnected());
break;
default:
break;
}
});
}
void _handleCallKitMute(dynamic body) {
final isMuted = body is Map ? body['isMuted'] == true : body == true;
if (_isMuted != isMuted) {
_toggleMute();
}
}
void _handleCallKitHold(dynamic body) {
final onHold = body is Map ? body['isOnHold'] == true : body == true;
if (onHold) {
unawaited(pauseListening(reason: VoiceCallPauseReason.system));
} else {
unawaited(resumeListening(reason: VoiceCallPauseReason.system));
}
}
Future<void> _markCallKitConnected() async {
if (_callKitCallId == null || _callKitConnectedReported) return;
await _callKitService.markCallConnected(_callKitCallId!);
_callKitConnectedReported = true;
}
String? _extractCallId(dynamic body) {
if (body is Map) {
final id = body['id'];
return id?.toString();
}
return null;
}
Future<void> startCall(String? conversationId) async { Future<void> startCall(String? conversationId) async {
if (_isDisposed) return; if (_isDisposed) return;
try { try {
final modelName = _ref.read(selectedModelProvider)?.name ?? 'Assistant';
await _startCallKitSession(modelName: modelName);
_listenForCallKitEvents();
// Update state (this will trigger notification) // Update state (this will trigger notification)
_updateState(VoiceCallState.connecting); _updateState(VoiceCallState.connecting);
@@ -200,15 +324,19 @@ class VoiceCallService {
// Start listening for user voice input // Start listening for user voice input
await _startListening(); await _startListening();
await _markCallKitConnected();
} catch (e) { } catch (e) {
_updateState(VoiceCallState.error); _updateState(VoiceCallState.error);
_keepAliveTimer?.cancel(); _keepAliveTimer?.cancel();
_keepAliveTimer = null; _keepAliveTimer = null;
await _callKitEventSubscription?.cancel();
_callKitEventSubscription = null;
await WakelockPlus.disable(); await WakelockPlus.disable();
await _notificationService.cancelNotification(); await _notificationService.cancelNotification();
await BackgroundStreamingHandler.instance.stopBackgroundExecution(const [ await BackgroundStreamingHandler.instance.stopBackgroundExecution(const [
_voiceCallStreamId, _voiceCallStreamId,
]); ]);
await _endCallKitSession();
rethrow; rethrow;
} }
} }
@@ -551,10 +679,12 @@ class VoiceCallService {
return; return;
} }
_isSpeaking = false; _isSpeaking = false;
_listeningSuspendedForSpeech = false;
if (_serverAudioBuffer.containsKey(_nextServerPlaybackId)) { if (_serverAudioBuffer.containsKey(_nextServerPlaybackId)) {
_maybeStartServerAudio(); _maybeStartServerAudio();
return; return;
} }
_responseCompleted = true;
_maybeResumeListeningAfterSpeech(); _maybeResumeListeningAfterSpeech();
} }
@@ -603,6 +733,7 @@ class VoiceCallService {
return; return;
} }
_listeningSuspendedForSpeech = false;
unawaited(_startListening()); unawaited(_startListening());
} }
@@ -618,6 +749,8 @@ class VoiceCallService {
unawaited(_startNextSpeechChunk()); unawaited(_startNextSpeechChunk());
return; return;
} }
_responseCompleted = true;
_listeningSuspendedForSpeech = false;
_maybeResumeListeningAfterSpeech(); _maybeResumeListeningAfterSpeech();
} }
@@ -641,6 +774,8 @@ class VoiceCallService {
await _transcriptSubscription?.cancel(); await _transcriptSubscription?.cancel();
await _intensitySubscription?.cancel(); await _intensitySubscription?.cancel();
await _callKitEventSubscription?.cancel();
_callKitEventSubscription = null;
_socketSubscription?.dispose(); _socketSubscription?.dispose();
await _voiceInput.stopListening(); await _voiceInput.stopListening();
@@ -653,6 +788,7 @@ class VoiceCallService {
// Cancel notification // Cancel notification
await _notificationService.cancelNotification(); await _notificationService.cancelNotification();
await _endCallKitSession();
// Disable wake lock when call ends // Disable wake lock when call ends
await WakelockPlus.disable(); await WakelockPlus.disable();
@@ -734,6 +870,10 @@ class VoiceCallService {
} }
Future<void> _updateNotification() async { Future<void> _updateNotification() async {
// When CallKit is active, rely on native UI instead of the ongoing
// notification to avoid duplicate surfaces.
if (_callKitCallId != null) return;
// Skip notification for idle, error, and disconnected states // Skip notification for idle, error, and disconnected states
if (_state == VoiceCallState.idle || if (_state == VoiceCallState.idle ||
_state == VoiceCallState.error || _state == VoiceCallState.error ||
@@ -801,6 +941,8 @@ class VoiceCallService {
await _transcriptSubscription?.cancel(); await _transcriptSubscription?.cancel();
await _intensitySubscription?.cancel(); await _intensitySubscription?.cancel();
await _callKitEventSubscription?.cancel();
_callKitEventSubscription = null;
_socketSubscription?.dispose(); _socketSubscription?.dispose();
_voiceInput.dispose(); _voiceInput.dispose();
@@ -809,6 +951,7 @@ class VoiceCallService {
// Cancel notification // Cancel notification
await _notificationService.cancelNotification(); await _notificationService.cancelNotification();
await _endCallKitSession();
// Ensure wake lock is disabled on dispose // Ensure wake lock is disabled on dispose
await WakelockPlus.disable(); await WakelockPlus.disable();
@@ -830,6 +973,7 @@ VoiceCallService voiceCallService(Ref ref) {
final api = ref.watch(apiServiceProvider); final api = ref.watch(apiServiceProvider);
final tts = TextToSpeechService(api: api); final tts = TextToSpeechService(api: api);
final socketService = ref.watch(socketServiceProvider); final socketService = ref.watch(socketServiceProvider);
final callKit = ref.watch(callKitServiceProvider);
if (socketService == null) { if (socketService == null) {
throw Exception('Socket service not available'); throw Exception('Socket service not available');
@@ -839,6 +983,7 @@ VoiceCallService voiceCallService(Ref ref) {
voiceInput: voiceInput, voiceInput: voiceInput,
tts: tts, tts: tts,
socketService: socketService, socketService: socketService,
callKitService: callKit,
ref: ref, ref: ref,
); );

View File

@@ -5,13 +5,15 @@ import 'dart:typed_data';
import 'package:flutter/widgets.dart'; import 'package:flutter/widgets.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:record/record.dart'; import 'package:record/record.dart'
hide IosAudioCategory, IosAudioCategoryOptions;
import 'package:riverpod_annotation/riverpod_annotation.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:stts/stts.dart'; import 'package:stts/stts.dart';
import 'package:vad/vad.dart'; import 'package:vad/vad.dart';
import '../../../core/providers/app_providers.dart'; import '../../../core/providers/app_providers.dart';
import '../../../core/services/api_service.dart'; import '../../../core/services/api_service.dart';
import '../../../core/services/background_streaming_handler.dart';
import '../../../core/services/settings_service.dart'; import '../../../core/services/settings_service.dart';
part 'voice_input_service.g.dart'; part 'voice_input_service.g.dart';
@@ -27,6 +29,7 @@ class VoiceInputService {
static const int _vadSampleRate = 16000; static const int _vadSampleRate = 16000;
static const int _vadFrameSamples = 1536; static const int _vadFrameSamples = 1536;
static const Duration _localeFetchTimeout = Duration(seconds: 2); static const Duration _localeFetchTimeout = Duration(seconds: 2);
static const String _backgroundSttStreamId = 'voice-input-stt';
final VadHandler _vadHandler = VadHandler.create(); final VadHandler _vadHandler = VadHandler.create();
final Stt _speech = Stt(); final Stt _speech = Stt();
@@ -36,11 +39,13 @@ class VoiceInputService {
bool _isInitialized = false; bool _isInitialized = false;
bool _isListening = false; bool _isListening = false;
bool _localSttAvailable = false; bool _localSttAvailable = false;
bool _localSttActive = false;
SttPreference _preference = SttPreference.deviceOnly; SttPreference _preference = SttPreference.deviceOnly;
bool _usingServerStt = false; bool _usingServerStt = false;
String? _selectedLocaleId; String? _selectedLocaleId;
List<LocaleName> _locales = const []; List<LocaleName> _locales = const [];
bool _usingFallbackLocales = false; bool _usingFallbackLocales = false;
Future<void>? _startingLocalStt;
StreamController<String>? _textStreamController; StreamController<String>? _textStreamController;
String _currentText = ''; String _currentText = '';
StreamController<int>? _intensityController; StreamController<int>? _intensityController;
@@ -49,6 +54,7 @@ class VoiceInputService {
int _lastIntensity = 0; int _lastIntensity = 0;
Timer? _intensityDecayTimer; Timer? _intensityDecayTimer;
List<double>? _vadPendingSamples; List<double>? _vadPendingSamples;
bool _backgroundMicPinned = false;
Stream<String> get textStream => Stream<String> get textStream =>
_textStreamController?.stream ?? const Stream<String>.empty(); _textStreamController?.stream ?? const Stream<String>.empty();
@@ -266,22 +272,38 @@ class VoiceInputService {
Future<void> _startLocalRecognition({ Future<void> _startLocalRecognition({
required bool allowOnlineFallback, required bool allowOnlineFallback,
}) async { }) async {
if (_startingLocalStt != null) {
await _startingLocalStt;
}
final completer = Completer<void>();
_startingLocalStt = completer.future;
_localSttActive = false;
await _ensureLocalSttReset();
await _configureIosAudioSession();
if (_selectedLocaleId != null) { if (_selectedLocaleId != null) {
await _speech.setLanguage(_selectedLocaleId!); await _speech.setLanguage(_selectedLocaleId!);
} }
Future<void> attempt(bool offline) => _speech.start( Future<void> attempt(bool offline) async {
SttRecognitionOptions(punctuation: true, offline: offline), await _speech.start(
); SttRecognitionOptions(punctuation: true, offline: offline),
);
_localSttActive = true;
}
try { try {
await attempt(true); await attempt(true);
} catch (error) { } catch (error) {
_localSttActive = false;
await _ensureLocalSttReset();
if (Platform.isIOS && allowOnlineFallback) { if (Platform.isIOS && allowOnlineFallback) {
try { try {
await attempt(false); await attempt(false);
return; return;
} catch (secondary) { } catch (secondary) {
await _ensureLocalSttReset();
throw Exception( throw Exception(
'On-device speech failed ($error); ' 'On-device speech failed ($error); '
'online fallback failed ($secondary).', 'online fallback failed ($secondary).',
@@ -289,16 +311,25 @@ class VoiceInputService {
} }
} }
rethrow; rethrow;
} finally {
completer.complete();
_startingLocalStt = null;
} }
} }
Stream<String> startListening() { Future<Stream<String>> startListening() async {
if (!_isInitialized) { if (!_isInitialized) {
throw Exception('Voice input not initialized'); throw Exception('Voice input not initialized');
} }
if (_startingLocalStt != null) {
try {
await _startingLocalStt;
} catch (_) {}
}
if (_isListening) { if (_isListening) {
unawaited(stopListening()); await stopListening();
} }
_textStreamController = StreamController<String>.broadcast(); _textStreamController = StreamController<String>.broadcast();
@@ -316,62 +347,81 @@ class VoiceInputService {
canUseLocal && _preference != SttPreference.serverOnly; canUseLocal && _preference != SttPreference.serverOnly;
final bool shouldUseServer = final bool shouldUseServer =
serverAvailable && serverAvailable &&
(_preference == SttPreference.serverOnly || !shouldUseLocal); (_preference == SttPreference.serverOnly ||
(!shouldUseLocal && _preference != SttPreference.deviceOnly));
if (shouldUseLocal) { if (shouldUseLocal) {
await _pinBackgroundMicrophone();
_autoStopTimer?.cancel(); _autoStopTimer?.cancel();
_autoStopTimer = Timer(const Duration(seconds: 60), () { _autoStopTimer = Timer(const Duration(seconds: 60), () {
if (_isListening) { if (_isListening) {
unawaited(_stopListening()); unawaited(_stopListening());
} }
}); });
try {
final isStillAvailable = await _speech.isSupported();
if (!isStillAvailable && _isListening) {
_localSttAvailable = false;
_textStreamController?.addError(
Exception('On-device speech recognition unavailable'),
);
await _stopListening();
return _textStreamController!.stream;
}
} catch (_) {
// ignore availability check errors
}
Future.microtask(() async { _sttResultSub = _speech.onResultChanged.listen(
try { (SttRecognition result) {
final isStillAvailable = await _speech.isSupported(); if (!_isListening) return;
if (!isStillAvailable && _isListening) { final prevLen = _currentText.length;
_localSttAvailable = false; _currentText = result.text;
_textStreamController?.addError( _textStreamController?.add(_currentText);
Exception('On-device speech recognition unavailable'), final delta = (_currentText.length - prevLen).clamp(0, 50);
); final mapped = (delta / 5.0).ceil();
_lastIntensity = mapped.clamp(0, 10);
try {
_intensityController?.add(_lastIntensity);
} catch (_) {}
if (result.isFinal) {
unawaited(_stopListening()); unawaited(_stopListening());
} }
} catch (_) { },
// ignore availability check errors onError: (error) {
} debugPrint('Local STT Error: $error');
}); _handleLocalRecognizerError(error);
},
_sttResultSub = _speech.onResultChanged.listen((SttRecognition result) {
if (!_isListening) return;
final prevLen = _currentText.length;
_currentText = result.text;
_textStreamController?.add(_currentText);
final delta = (_currentText.length - prevLen).clamp(0, 50);
final mapped = (delta / 5.0).ceil();
_lastIntensity = mapped.clamp(0, 10);
try {
_intensityController?.add(_lastIntensity);
} catch (_) {}
if (result.isFinal) {
unawaited(_stopListening());
}
}, onError: _handleLocalRecognizerError);
_sttStateSub = _speech.onStateChanged.listen(
(_) {},
onError: _handleLocalRecognizerError,
); );
Future(() async { _sttStateSub = _speech.onStateChanged.listen(
try { (state) {
await _startLocalRecognition(allowOnlineFallback: !prefersDeviceOnly); debugPrint('Local STT State: $state');
} catch (error) { if (state == SttState.start) {
_localSttAvailable = false; _localSttActive = true;
if (!_isListening) return; } else if (state == SttState.stop) {
_textStreamController?.addError(error); _localSttActive = false;
await _stopListening(); }
},
onError: (error) {
debugPrint('Local STT State Error: $error');
_handleLocalRecognizerError(error);
},
);
try {
debugPrint('Starting local recognition...');
await _startLocalRecognition(allowOnlineFallback: !prefersDeviceOnly);
debugPrint('Local recognition started');
} catch (error) {
debugPrint('Failed to start local recognition: $error');
_localSttAvailable = false;
if (!_isListening) {
return _textStreamController!.stream;
} }
}); _textStreamController?.addError(error);
await _stopListening();
}
} else if (shouldUseServer) { } else if (shouldUseServer) {
_usingServerStt = true; _usingServerStt = true;
_autoStopTimer?.cancel(); _autoStopTimer?.cancel();
@@ -406,7 +456,7 @@ class VoiceInputService {
}); });
} }
return _textStreamController!.stream; return _textStreamController?.stream ?? const Stream<String>.empty();
} }
/// Centralized entry point to begin voice recognition. /// Centralized entry point to begin voice recognition.
@@ -417,7 +467,7 @@ class VoiceInputService {
if (!hasMic) { if (!hasMic) {
throw Exception('Microphone permission not granted'); throw Exception('Microphone permission not granted');
} }
return startListening(); return await startListening();
} }
Future<void> stopListening() async { Future<void> stopListening() async {
@@ -453,9 +503,16 @@ class VoiceInputService {
await _closeControllers(); await _closeControllers();
_usingServerStt = false; _usingServerStt = false;
await _releaseBackgroundMicrophone();
} }
Future<void> _stopLocalStt() async { Future<void> _stopLocalStt() async {
final pendingStart = _startingLocalStt;
if (pendingStart != null) {
try {
await pendingStart;
} catch (_) {}
}
if (_sttResultSub != null) { if (_sttResultSub != null) {
try { try {
await _sttResultSub?.cancel(); await _sttResultSub?.cancel();
@@ -469,11 +526,67 @@ class VoiceInputService {
_sttStateSub = null; _sttStateSub = null;
} }
if (_localSttAvailable) { final shouldStopStt = _localSttActive && _localSttAvailable;
_localSttActive = false;
if (shouldStopStt) {
try { try {
await _speech.stop(); await _speech.stop();
} catch (_) {} } catch (_) {}
} }
if (Platform.isIOS) {
try {
await _speech.ios?.setAudioSessionActive(false);
} catch (_) {}
}
}
Future<void> _pinBackgroundMicrophone() async {
if (!Platform.isIOS || _backgroundMicPinned) return;
try {
await BackgroundStreamingHandler.instance.startBackgroundExecution(const [
_backgroundSttStreamId,
], requiresMicrophone: true);
_backgroundMicPinned = true;
} catch (_) {}
}
Future<void> _releaseBackgroundMicrophone() async {
if (!Platform.isIOS || !_backgroundMicPinned) return;
_backgroundMicPinned = false;
try {
await BackgroundStreamingHandler.instance.stopBackgroundExecution(const [
_backgroundSttStreamId,
]);
} catch (_) {}
}
Future<void> _ensureLocalSttReset() async {
try {
await _speech.stop();
} catch (_) {}
if (Platform.isIOS) {
try {
await _speech.ios?.setAudioSessionActive(false);
} catch (_) {}
}
}
Future<void> _configureIosAudioSession() async {
if (!Platform.isIOS) return;
final ios = _speech.ios;
if (ios == null) return;
try {
await ios.setAudioSessionCategory(
category: IosAudioCategory.playAndRecord,
options: [
IosAudioCategoryOptions.allowBluetooth,
IosAudioCategoryOptions.allowBluetoothA2DP,
IosAudioCategoryOptions.defaultToSpeaker,
IosAudioCategoryOptions.duckOthers,
],
);
await ios.setAudioSessionActive(true);
} catch (_) {}
} }
Future<void> _startServerRecording() async { Future<void> _startServerRecording() async {
@@ -548,7 +661,9 @@ class VoiceInputService {
Future<void> _stopVadRecording() async { Future<void> _stopVadRecording() async {
try { try {
await _vadHandler.stopListening(); if (_isListening) {
await _vadHandler.stopListening();
}
} catch (_) {} } catch (_) {}
await _vadSpeechEndSub?.cancel(); await _vadSpeechEndSub?.cancel();
_vadSpeechEndSub = null; _vadSpeechEndSub = null;

View File

@@ -486,6 +486,14 @@ packages:
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "3.4.1" version: "3.4.1"
flutter_callkit_incoming:
dependency: "direct main"
description:
name: flutter_callkit_incoming
sha256: "3589deb8b71e43f2d520a9c8a5240243f611062a8b246cdca4b1fda01fbbf9b8"
url: "https://pub.dev"
source: hosted
version: "3.0.0"
flutter_lints: flutter_lints:
dependency: "direct dev" dependency: "direct dev"
description: description:

View File

@@ -73,6 +73,7 @@ dependencies:
connectivity_plus: ^7.0.0 connectivity_plus: ^7.0.0
flutter_cache_manager: ^3.4.1 flutter_cache_manager: ^3.4.1
http: ^1.5.0 http: ^1.5.0
flutter_callkit_incoming: ^3.0.0
# Clipboard functionality is available through flutter/services (part of Flutter SDK) # Clipboard functionality is available through flutter/services (part of Flutter SDK)