ars/iiEsaywebUIapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Replace mic_stream_recorder with vad and update iOS deployment target

Browse Source
This commit is contained in:
cogwheel0
2025-11-10 01:57:28 +05:30
parent 5f597a3bb5
commit 62c9243e34
7 changed files with 262 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ios/Flutter/AppFrameworkInfo.plist
View File

@@ -21,6 +21,6 @@
<key>CFBundleVersion</key>
<string>1.0</string>
<key>MinimumOSVersion</key>
<string>13.0</string>
<string>15.1</string>
</dict>
</plist>

2
ios/Podfile
View File

@@ -1,5 +1,5 @@
# Uncomment this line to define a global platform for your project
platform :ios, '13.0'
platform :ios, '15.1'
# CocoaPods analytics sends network stats synchronously affecting flutter build latency.
ENV['COCOAPODS_DISABLE_STATS'] = 'true'

30
ios/Podfile.lock
View File

@@ -49,13 +49,18 @@ PODS:
- Flutter
- image_picker_ios (0.0.1):
- Flutter
- mic_stream_recorder (0.0.1):
- Flutter
- onnxruntime-c (1.22.0)
- onnxruntime-objc (1.22.0):
- onnxruntime-objc/Core (= 1.22.0)
- onnxruntime-objc/Core (1.22.0):
- onnxruntime-c (= 1.22.0)
- package_info_plus (0.4.5):
- Flutter
- path_provider_foundation (0.0.1):
- Flutter
- FlutterMacOS
- record_ios (1.1.0):
- Flutter
- SDWebImage (5.21.1):
- SDWebImage/Core (= 5.21.1)
- SDWebImage/Core (5.21.1)
@@ -80,6 +85,9 @@ PODS:
- SwiftyGif (5.4.5)
- url_launcher_ios (0.0.1):
- Flutter
- vad (0.0.6):
- Flutter
- onnxruntime-objc (= 1.22.0)
- wakelock_plus (0.0.1):
- Flutter
- webview_flutter_wkwebview (0.0.1):
@@ -96,9 +104,9 @@ DEPENDENCIES:
- flutter_secure_storage (from `.symlinks/plugins/flutter_secure_storage/ios`)
- flutter_tts (from `.symlinks/plugins/flutter_tts/ios`)
- image_picker_ios (from `.symlinks/plugins/image_picker_ios/ios`)
- mic_stream_recorder (from `.symlinks/plugins/mic_stream_recorder/ios`)
- package_info_plus (from `.symlinks/plugins/package_info_plus/ios`)
- path_provider_foundation (from `.symlinks/plugins/path_provider_foundation/darwin`)
- record_ios (from `.symlinks/plugins/record_ios/ios`)
- share_handler_ios (from `.symlinks/plugins/share_handler_ios/ios`)
- share_handler_ios_models (from `.symlinks/plugins/share_handler_ios/ios/Models`)
- share_plus (from `.symlinks/plugins/share_plus/ios`)
@@ -106,6 +114,7 @@ DEPENDENCIES:
- sqflite_darwin (from `.symlinks/plugins/sqflite_darwin/darwin`)
- stts (from `.symlinks/plugins/stts/ios`)
- url_launcher_ios (from `.symlinks/plugins/url_launcher_ios/ios`)
- vad (from `.symlinks/plugins/vad/ios`)
- wakelock_plus (from `.symlinks/plugins/wakelock_plus/ios`)
- webview_flutter_wkwebview (from `.symlinks/plugins/webview_flutter_wkwebview/darwin`)
@@ -113,6 +122,8 @@ SPEC REPOS:
trunk:
- DKImagePickerController
- DKPhotoGallery
- onnxruntime-c
- onnxruntime-objc
- SDWebImage
- SwiftyGif
@@ -135,12 +146,12 @@ EXTERNAL SOURCES:
:path: ".symlinks/plugins/flutter_tts/ios"
image_picker_ios:
:path: ".symlinks/plugins/image_picker_ios/ios"
mic_stream_recorder:
:path: ".symlinks/plugins/mic_stream_recorder/ios"
package_info_plus:
:path: ".symlinks/plugins/package_info_plus/ios"
path_provider_foundation:
:path: ".symlinks/plugins/path_provider_foundation/darwin"
record_ios:
:path: ".symlinks/plugins/record_ios/ios"
share_handler_ios:
:path: ".symlinks/plugins/share_handler_ios/ios"
share_handler_ios_models:
@@ -155,6 +166,8 @@ EXTERNAL SOURCES:
:path: ".symlinks/plugins/stts/ios"
url_launcher_ios:
:path: ".symlinks/plugins/url_launcher_ios/ios"
vad:
:path: ".symlinks/plugins/vad/ios"
wakelock_plus:
:path: ".symlinks/plugins/wakelock_plus/ios"
webview_flutter_wkwebview:
@@ -172,9 +185,11 @@ SPEC CHECKSUMS:
flutter_secure_storage: 1ed9476fba7e7a782b22888f956cce43e2c62f13
flutter_tts: b88dbc8655d3dc961bc4a796e4e16a4cc1795833
image_picker_ios: 7fe1ff8e34c1790d6fff70a32484959f563a928a
mic_stream_recorder: 27d2d1225563a3a28bf4019fc5cc198cffd7dad1
onnxruntime-c: 7f778680e96145956c0a31945f260321eed2611a
onnxruntime-objc: 83d28b87525bd971259a66e153ea32b5d023de19
package_info_plus: af8e2ca6888548050f16fa2f1938db7b5a5df499
path_provider_foundation: 080d55be775b7414fd5a5ef3ac137b97b097e564
record_ios: f75fa1d57f840012775c0e93a38a7f3ceea1a374
SDWebImage: f29024626962457f3470184232766516dee8dfea
share_handler_ios: e2244e990f826b2c8eaa291ac3831569438ba0fb
share_handler_ios_models: fc638c9b4330dc7f082586c92aee9dfa0b87b871
@@ -184,9 +199,10 @@ SPEC CHECKSUMS:
stts: 1a48df645bb516e86e4121d5253b582749a1d3a6
SwiftyGif: 706c60cf65fa2bc5ee0313beece843c8eb8194d4
url_launcher_ios: 694010445543906933d732453a59da0a173ae33d
vad: 7934867589afe53567f492df66fb1615f2185822
wakelock_plus: e29112ab3ef0b318e58cfa5c32326458be66b556
webview_flutter_wkwebview: 8ebf4fded22593026f7dbff1fbff31ea98573c8d
PODFILE CHECKSUM: df88575cf61e98a1a3edf2f8c887dad2c18c2079
PODFILE CHECKSUM: a6ecbec6401c6461e69650e9ef66360aee70610f
COCOAPODS: 1.16.2

12
ios/Runner.xcodeproj/project.pbxproj
View File

@@ -585,7 +585,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 13.0;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = iphoneos;
SUPPORTED_PLATFORMS = iphoneos;
@@ -722,7 +722,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 13.0;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
MTL_ENABLE_DEBUG_INFO = YES;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = iphoneos;
@@ -773,7 +773,7 @@
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 13.0;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
MTL_ENABLE_DEBUG_INFO = NO;
SDKROOT = iphoneos;
SUPPORTED_PLATFORMS = iphoneos;
@@ -865,7 +865,7 @@
INFOPLIST_FILE = ShareExtension/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = ShareExtension;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
IPHONEOS_DEPLOYMENT_TARGET = 14.0;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
@@ -908,7 +908,7 @@
INFOPLIST_FILE = ShareExtension/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = ShareExtension;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
IPHONEOS_DEPLOYMENT_TARGET = 14.0;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
@@ -948,7 +948,7 @@
INFOPLIST_FILE = ShareExtension/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = ShareExtension;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
IPHONEOS_DEPLOYMENT_TARGET = 14.0;
IPHONEOS_DEPLOYMENT_TARGET = 15.1;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",

255
lib/features/chat/services/voice_input_service.dart
View File

@@ -1,13 +1,13 @@
import 'dart:async';
import 'dart:io' show File, Platform;
import 'dart:convert';
import 'dart:io' show Platform;
import 'dart:typed_data';
import 'package:flutter/widgets.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:mic_stream_recorder/mic_stream_recorder.dart';
import 'package:stts/stts.dart';
import 'package:path/path.dart' as p;
import 'package:path_provider/path_provider.dart';
import 'package:vad/vad.dart';
import '../../../core/providers/app_providers.dart';
import '../../../core/services/api_service.dart';
@@ -23,7 +23,10 @@ class LocaleName {
}
class VoiceInputService {
final MicStreamRecorder _recorder = MicStreamRecorder();
static const int _vadSampleRate = 16000;
static const int _vadFrameSamples = 1536;
final VadHandler _vadHandler = VadHandler.create();
final Stt _speech = Stt();
final ApiService? _api;
final Ref? _ref;
@@ -41,17 +44,17 @@ class VoiceInputService {
_intensityController?.stream ?? const Stream<int>.empty();
int _lastIntensity = 0;
Timer? _intensityDecayTimer;
Timer? _silenceTimer;
bool _hasDetectedSpeech = false;
int _amplitudeCallbackCount = 0;
Timer? _amplitudeFallbackTimer;
List<double>? _vadPendingSamples;
Stream<String> get textStream =>
_textStreamController?.stream ?? const Stream<String>.empty();
Timer? _autoStopTimer;
StreamSubscription<double>? _ampSub;
StreamSubscription<SttRecognition>? _sttResultSub;
StreamSubscription<SttState>? _sttStateSub;
StreamSubscription<List<double>>? _vadSpeechEndSub;
StreamSubscription<({double isSpeech, double notSpeech, List<double> frame})>?
_vadFrameSub;
StreamSubscription<String>? _vadErrorSub;
bool get isSupportedPlatform => Platform.isAndroid || Platform.isIOS;
bool get hasServerStt => _api != null;
@@ -60,9 +63,7 @@ class VoiceInputService {
bool get prefersServerOnly => _preference == SttPreference.serverOnly;
bool get prefersDeviceOnly => _preference == SttPreference.deviceOnly;
VoiceInputService({ApiService? api, Ref? ref})
: _api = api,
_ref = ref;
VoiceInputService({ApiService? api, Ref? ref}) : _api = api, _ref = ref;
void updatePreference(SttPreference preference) {
_preference = preference;
@@ -327,33 +328,27 @@ class VoiceInputService {
_autoStopTimer?.cancel();
_autoStopTimer = null;
_silenceTimer?.cancel();
_silenceTimer = null;
_amplitudeFallbackTimer?.cancel();
_amplitudeFallbackTimer = null;
if (_usingServerStt) {
await _finalizeServerRecording();
await _stopVadRecording();
final samples = _vadPendingSamples;
_vadPendingSamples = null;
if (samples != null && samples.isNotEmpty) {
await _processVadSamples(samples);
}
} else {
await _stopLocalStt();
if (_currentText.isNotEmpty) {
_textStreamController?.add(_currentText);
}
}
await _ampSub?.cancel();
_ampSub = null;
_intensityDecayTimer?.cancel();
_intensityDecayTimer = null;
_lastIntensity = 0;
if (!_usingServerStt && _currentText.isNotEmpty) {
_textStreamController?.add(_currentText);
}
await _closeControllers();
_usingServerStt = false;
_hasDetectedSpeech = false;
}
Future<void> _stopLocalStt() async {
@@ -411,82 +406,100 @@ class VoiceInputService {
}
Future<void> _startServerRecording() async {
final path = await _createRecordingPath();
_hasDetectedSpeech = false;
await _setupVadStreams();
final settings = _ref?.read(appSettingsProvider);
final silenceMs = settings?.voiceSilenceDuration ?? 2000;
final redemptionFrames = _silenceDurationToFrames(silenceMs);
final endPadFrames = redemptionFrames > 4
? (redemptionFrames / 4).round().clamp(1, redemptionFrames)
: 1;
await _recorder.startRecording(path);
try {
await _vadHandler.startListening(
frameSamples: _vadFrameSamples,
redemptionFrames: redemptionFrames,
endSpeechPadFrames: endPadFrames,
preSpeechPadFrames: 2,
minSpeechFrames: 3,
submitUserSpeechOnPause: true,
recordConfig: const RecordConfig(
encoder: AudioEncoder.pcm16bits,
sampleRate: _vadSampleRate,
numChannels: 1,
bitRate: 16,
echoCancel: true,
autoGain: true,
noiseSuppress: true,
androidConfig: AndroidRecordConfig(
audioSource: AndroidAudioSource.voiceCommunication,
audioManagerMode: AudioManagerMode.modeInCommunication,
speakerphone: true,
manageBluetooth: true,
useLegacy: false,
),
),
);
} catch (error) {
_textStreamController?.addError(error);
rethrow;
}
}
await _ampSub?.cancel();
_amplitudeFallbackTimer?.cancel();
_amplitudeCallbackCount = 0;
Future<void> _setupVadStreams() async {
await _vadSpeechEndSub?.cancel();
_vadSpeechEndSub = _vadHandler.onSpeechEnd.listen((samples) {
if (!_isListening || !_usingServerStt) return;
if (samples.isEmpty) return;
_vadPendingSamples = samples;
if (_isListening) {
unawaited(_stopListening());
}
});
_ampSub = _recorder.amplitudeStream.listen((amplitude) {
_amplitudeCallbackCount++;
await _vadFrameSub?.cancel();
_vadFrameSub = _vadHandler.onFrameProcessed.listen((frameData) {
if (!_isListening) return;
_lastIntensity = _normalizedToIntensity(amplitude);
final intensity = _intensityFromVadFrame(frameData.frame);
_lastIntensity = intensity;
try {
_intensityController?.add(_lastIntensity);
} catch (_) {}
_handleServerAmplitude(amplitude);
});
_amplitudeFallbackTimer = Timer(const Duration(seconds: 1), () {
if (_amplitudeCallbackCount == 0) {
_silenceTimer = Timer(const Duration(seconds: 15), () {
if (_isListening && _usingServerStt) {
unawaited(_stopListening());
}
});
await _vadErrorSub?.cancel();
_vadErrorSub = _vadHandler.onError.listen((message) {
_textStreamController?.addError(Exception(message));
if (_isListening) {
unawaited(_stopListening());
}
});
}
void _handleServerAmplitude(double amplitude) {
if (!_usingServerStt || !_isListening) return;
const double speechThreshold = 0.55;
if (amplitude.isNaN || amplitude.isInfinite) return;
if (amplitude > speechThreshold) {
_hasDetectedSpeech = true;
_silenceTimer?.cancel();
_silenceTimer = null;
} else if (_hasDetectedSpeech && _silenceTimer == null) {
final silenceDuration = _ref?.read(appSettingsProvider).voiceSilenceDuration ?? 2000;
_silenceTimer = Timer(Duration(milliseconds: silenceDuration), () {
if (_isListening && _usingServerStt) {
unawaited(_stopListening());
}
});
}
Future<void> _stopVadRecording() async {
try {
await _vadHandler.stopListening();
} catch (_) {}
await _vadSpeechEndSub?.cancel();
_vadSpeechEndSub = null;
await _vadFrameSub?.cancel();
_vadFrameSub = null;
await _vadErrorSub?.cancel();
_vadErrorSub = null;
}
Future<String> _createRecordingPath() async {
final directory = await getTemporaryDirectory();
final timestamp = DateTime.now().millisecondsSinceEpoch;
final fileName = 'conduit_voice_$timestamp.m4a';
return p.join(directory.path, fileName);
}
Future<void> _finalizeServerRecording() async {
Future<void> _processVadSamples(List<double> samples) async {
final api = _api;
if (api == null) return;
final path = await _recorder.stopRecording();
if (path == null || path.isEmpty) return;
final file = File(path);
try {
if (!await file.exists()) return;
final bytes = await file.readAsBytes();
if (bytes.isEmpty) return;
final wavBytes = _samplesToWav(samples);
final fileName =
'conduit_voice_${DateTime.now().millisecondsSinceEpoch}.wav';
final response = await api.transcribeSpeech(
audioBytes: bytes,
fileName: p.basename(path),
mimeType: 'audio/mp4',
audioBytes: wavBytes,
fileName: fileName,
mimeType: 'audio/wav',
language: _languageForServer(),
);
@@ -499,19 +512,72 @@ class VoiceInputService {
}
} catch (error) {
_textStreamController?.addError(error);
} finally {
unawaited(_cleanupRecordingFile(file));
}
}
Future<void> _cleanupRecordingFile(File file) async {
try {
if (await file.exists()) {
await file.delete();
}
} catch (_) {}
int _silenceDurationToFrames(int milliseconds) {
final frameDurationMs = (_vadFrameSamples / _vadSampleRate) * 1000;
final frames = (milliseconds / frameDurationMs).round();
return frames.clamp(4, 50);
}
int _intensityFromVadFrame(List<double> frame) {
if (frame.isEmpty) return 0;
double peak = 0;
for (final sample in frame) {
final value = sample.abs();
if (value > peak) {
peak = value;
}
}
final scaled = (peak * 12).round();
return scaled.clamp(0, 10);
}
Uint8List _samplesToWav(List<double> samples) {
if (samples.isEmpty) {
return Uint8List(0);
}
final Int16List pcm = Int16List(samples.length);
for (var i = 0; i < samples.length; i++) {
final clamped = samples[i].clamp(-1.0, 1.0);
final scaled = (clamped * 32767).round().clamp(-32768, 32767);
pcm[i] = scaled;
}
final dataLength = pcm.lengthInBytes;
final bytesPerSample = 2;
final numChannels = 1;
final byteRate = _vadSampleRate * numChannels * bytesPerSample;
final blockAlign = numChannels * bytesPerSample;
final builder = BytesBuilder();
builder.add(ascii.encode('RIFF'));
builder.add(_int32Le(36 + dataLength));
builder.add(ascii.encode('WAVE'));
builder.add(ascii.encode('fmt '));
builder.add(_int32Le(16));
builder.add(_int16Le(1));
builder.add(_int16Le(numChannels));
builder.add(_int32Le(_vadSampleRate));
builder.add(_int32Le(byteRate));
builder.add(_int16Le(blockAlign));
builder.add(_int16Le(16));
builder.add(ascii.encode('data'));
builder.add(_int32Le(dataLength));
builder.add(Uint8List.view(pcm.buffer));
return builder.toBytes();
}
List<int> _int16Le(int value) => [value & 0xff, (value >> 8) & 0xff];
List<int> _int32Le(int value) => [
value & 0xff,
(value >> 8) & 0xff,
(value >> 16) & 0xff,
(value >> 24) & 0xff,
];
String? _languageForServer() {
final locale = _selectedLocaleId;
if (locale != null && locale.isNotEmpty) {
@@ -611,11 +677,6 @@ class VoiceInputService {
return null;
}
int _normalizedToIntensity(double value) {
if (value.isNaN || value.isInfinite) return 0;
return (value * 10).round().clamp(0, 10);
}
Future<void> _closeControllers() async {
if (_textStreamController != null) {
try {
@@ -647,7 +708,7 @@ class VoiceInputService {
void dispose() {
stopListening();
_silenceTimer?.cancel();
unawaited(_vadHandler.dispose());
try {
_speech.dispose().catchError((_) {});
} catch (_) {}

80
pubspec.lock
View File

@@ -965,14 +965,6 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.16.0"
mic_stream_recorder:
dependency: "direct main"
description:
name: mic_stream_recorder
sha256: "73965991ef5cc93d2b0c1e6d590cbd567a853b9ee7b2d52de43a73f185bb0d9c"
url: "https://pub.dev"
source: hosted
version: "1.1.2"
mime:
dependency: transitive
description:
@@ -1173,6 +1165,70 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.5.0"
record:
dependency: transitive
description:
name: record
sha256: "6bad72fb3ea6708d724cf8b6c97c4e236cf9f43a52259b654efeb6fd9b737f1f"
url: "https://pub.dev"
source: hosted
version: "6.1.2"
record_android:
dependency: transitive
description:
name: record_android
sha256: fb54ee4e28f6829b8c580252a9ef49d9c549cfd263b0660ad7eeac0908658e9f
url: "https://pub.dev"
source: hosted
version: "1.4.4"
record_ios:
dependency: transitive
description:
name: record_ios
sha256: "765b42ac1be019b1674ddd809b811fc721fe5a93f7bb1da7803f0d16772fd6d7"
url: "https://pub.dev"
source: hosted
version: "1.1.4"
record_linux:
dependency: transitive
description:
name: record_linux
sha256: "235b1f1fb84e810f8149cc0c2c731d7d697f8d1c333b32cb820c449bf7bb72d8"
url: "https://pub.dev"
source: hosted
version: "1.2.1"
record_macos:
dependency: transitive
description:
name: record_macos
sha256: "842ea4b7e95f4dd237aacffc686d1b0ff4277e3e5357865f8d28cd28bc18ed95"
url: "https://pub.dev"
source: hosted
version: "1.1.2"
record_platform_interface:
dependency: transitive
description:
name: record_platform_interface
sha256: b0065fdf1ec28f5a634d676724d388a77e43ce7646fb049949f58c69f3fcb4ed
url: "https://pub.dev"
source: hosted
version: "1.4.0"
record_web:
dependency: transitive
description:
name: record_web
sha256: "20ac10d56514cb9f8cecc8f3579383084fdfb43b0d04e05a95244d0d76091d90"
url: "https://pub.dev"
source: hosted
version: "1.2.1"
record_windows:
dependency: transitive
description:
name: record_windows
sha256: "223258060a1d25c62bae18282c16783f28581ec19401d17e56b5205b9f039d78"
url: "https://pub.dev"
source: hosted
version: "1.0.7"
riverpod:
dependency: transitive
description:
@@ -1682,6 +1738,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "4.5.1"
vad:
dependency: "direct main"
description:
name: vad
sha256: ef6c8b12c5af7a6a519ff5684f074b8a2ac00c434705f544af379ea77bccd258
url: "https://pub.dev"
source: hosted
version: "0.0.7+1"
vector_graphics:
dependency: transitive
description:

2
pubspec.yaml
View File

@@ -44,7 +44,7 @@ dependencies:
flutter_animate: ^4.5.0
# Platform Features
mic_stream_recorder: ^1.1.2
vad: ^0.0.7+1
stts: ^1.2.5
flutter_tts: ^4.2.3
audioplayers: ^6.5.1