From d57ddf67c50d27c9df64a47615a856d72efac52f Mon Sep 17 00:00:00 2001 From: cogwheel0 <172976095+cogwheel0@users.noreply.github.com> Date: Sat, 16 Aug 2025 21:17:01 +0530 Subject: [PATCH] fix: image attachment flashing and not persisting --- lib/core/services/api_service.dart | 19 + .../widgets/documentation_message_widget.dart | 155 +++---- .../chat/widgets/modern_message_bubble.dart | 381 ++++++++++-------- test_streaming.md | 93 ----- 4 files changed, 310 insertions(+), 338 deletions(-) delete mode 100644 test_streaming.md diff --git a/lib/core/services/api_service.dart b/lib/core/services/api_service.dart index 478ca3f..cf27cb8 100644 --- a/lib/core/services/api_service.dart +++ b/lib/core/services/api_service.dart @@ -636,12 +636,27 @@ class ApiService { role = 'user'; } + // Parse attachments from 'files' field + List? attachmentIds; + if (msgData['files'] != null) { + final filesList = msgData['files'] as List; + attachmentIds = filesList + .where((file) => file is Map && file['file_id'] != null) + .map((file) => file['file_id'] as String) + .toList(); + + if (attachmentIds.isEmpty) { + attachmentIds = null; + } + } + return ChatMessage( id: msgData['id']?.toString() ?? uuid.v4(), role: role, content: contentString, timestamp: _parseTimestamp(msgData['timestamp']), model: msgData['model'] as String?, + attachmentIds: attachmentIds, ); } @@ -760,6 +775,8 @@ class ApiService { if (msg.role == 'assistant') 'modelIdx': 0, if (msg.role == 'assistant') 'done': true, if (msg.role == 'user' && model != null) 'models': [model], + if (msg.attachmentIds != null && msg.attachmentIds!.isNotEmpty) + 'files': msg.attachmentIds!.map((id) => {'file_id': id}).toList(), }; // Update parent's childrenIds @@ -780,6 +797,8 @@ class ApiService { if (msg.role == 'assistant') 'modelIdx': 0, if (msg.role == 'assistant') 'done': true, if (msg.role == 'user' && model != null) 'models': [model], + if (msg.attachmentIds != null && msg.attachmentIds!.isNotEmpty) + 'files': msg.attachmentIds!.map((id) => {'file_id': id}).toList(), }); previousId = messageId; diff --git a/lib/features/chat/widgets/documentation_message_widget.dart b/lib/features/chat/widgets/documentation_message_widget.dart index 1b67d35..8891e12 100644 --- a/lib/features/chat/widgets/documentation_message_widget.dart +++ b/lib/features/chat/widgets/documentation_message_widget.dart @@ -49,6 +49,7 @@ class _DocumentationMessageWidgetState String _renderedContent = ''; Timer? _throttleTimer; String? _pendingContent; + Widget? _cachedAvatar; @override void initState() { @@ -67,6 +68,13 @@ class _DocumentationMessageWidgetState _updateReasoningContent(); } + @override + void didChangeDependencies() { + super.didChangeDependencies(); + // Build cached avatar when theme context is available + _buildCachedAvatar(); + } + @override void didUpdateWidget(DocumentationMessageWidget oldWidget) { super.didUpdateWidget(oldWidget); @@ -77,6 +85,11 @@ class _DocumentationMessageWidgetState _scheduleRenderUpdate(widget.message.content ?? ''); _updateReasoningContent(); } + + // Rebuild cached avatar if model name changes + if (oldWidget.modelName != widget.modelName) { + _buildCachedAvatar(); + } } void _updateReasoningContent() { @@ -124,6 +137,41 @@ class _DocumentationMessageWidgetState return content; } + void _buildCachedAvatar() { + _cachedAvatar = Padding( + padding: const EdgeInsets.only(bottom: 12), + child: Row( + children: [ + Container( + width: 20, + height: 20, + decoration: BoxDecoration( + color: context.conduitTheme.buttonPrimary, + borderRadius: BorderRadius.circular( + AppBorderRadius.small, + ), + ), + child: Icon( + Icons.auto_awesome, + color: context.conduitTheme.buttonPrimaryText, + size: 12, + ), + ), + const SizedBox(width: Spacing.xs), + Text( + widget.modelName ?? 'Assistant', + style: TextStyle( + color: context.conduitTheme.textSecondary, + fontSize: AppTypography.bodySmall, + fontWeight: FontWeight.w500, + letterSpacing: 0.1, + ), + ), + ], + ), + ); + } + @override void dispose() { _fadeController.dispose(); @@ -158,29 +206,29 @@ class _DocumentationMessageWidgetState return Container( width: double.infinity, margin: const EdgeInsets.only(bottom: 16, left: 50, right: 12), - child: Row( - mainAxisAlignment: MainAxisAlignment.end, + child: Column( + crossAxisAlignment: CrossAxisAlignment.end, children: [ - Flexible( - child: GestureDetector( - onLongPress: () => _toggleActions(), - child: Container( - padding: const EdgeInsets.symmetric( - horizontal: 16, - vertical: 12, - ), - decoration: BoxDecoration( - color: context.conduitTheme.chatBubbleUser, - borderRadius: BorderRadius.circular(AppBorderRadius.lg), - border: Border.all( - color: context.conduitTheme.chatBubbleUserBorder, - width: BorderWidth.regular, - ), - ), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Text( + Row( + mainAxisAlignment: MainAxisAlignment.end, + children: [ + Flexible( + child: GestureDetector( + onLongPress: () => _toggleActions(), + child: Container( + padding: const EdgeInsets.symmetric( + horizontal: 16, + vertical: 12, + ), + decoration: BoxDecoration( + color: context.conduitTheme.chatBubbleUser, + borderRadius: BorderRadius.circular(AppBorderRadius.lg), + border: Border.all( + color: context.conduitTheme.chatBubbleUserBorder, + width: BorderWidth.regular, + ), + ), + child: Text( widget.message.content, style: TextStyle( color: context.conduitTheme.chatBubbleUserText, @@ -189,17 +237,17 @@ class _DocumentationMessageWidgetState letterSpacing: 0.1, ), ), - - // Action buttons for user messages - if (_showActions) ...[ - const SizedBox(height: 12), - _buildUserActionButtons(), - ], - ], + ), ), ), - ), + ], ), + + // Action buttons below the message bubble + if (_showActions) ...[ + const SizedBox(height: Spacing.sm), + _buildUserActionButtons(), + ], ], ), ) @@ -220,39 +268,8 @@ class _DocumentationMessageWidgetState child: Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - // Simplified AI Name and Avatar - Padding( - padding: const EdgeInsets.only(bottom: 12), - child: Row( - children: [ - Container( - width: 20, - height: 20, - decoration: BoxDecoration( - color: context.conduitTheme.buttonPrimary, - borderRadius: BorderRadius.circular( - AppBorderRadius.small, - ), - ), - child: Icon( - Icons.auto_awesome, - color: context.conduitTheme.buttonPrimaryText, - size: 12, - ), - ), - const SizedBox(width: Spacing.xs), - Text( - widget.modelName ?? 'Assistant', - style: TextStyle( - color: context.conduitTheme.textSecondary, - fontSize: AppTypography.bodySmall, - fontWeight: FontWeight.w500, - letterSpacing: 0.1, - ), - ), - ], - ), - ), + // Cached AI Name and Avatar to prevent flashing + _cachedAvatar ?? const SizedBox.shrink(), // Reasoning Section (if present) if (_reasoningContent != null) ...[ @@ -364,16 +381,16 @@ class _DocumentationMessageWidgetState _reasoningContent?.mainContent ?? widget.message.content, ), - - // Action buttons - inline and minimal - if (_showActions) ...[ - const SizedBox(height: Spacing.md), - _buildActionButtons(), - ], ], ), ), ), + + // Action buttons below the message content + if (_showActions) ...[ + const SizedBox(height: Spacing.sm), + _buildActionButtons(), + ], ], ), ) diff --git a/lib/features/chat/widgets/modern_message_bubble.dart b/lib/features/chat/widgets/modern_message_bubble.dart index c6b9d12..b648562 100644 --- a/lib/features/chat/widgets/modern_message_bubble.dart +++ b/lib/features/chat/widgets/modern_message_bubble.dart @@ -46,6 +46,10 @@ class _ModernMessageBubbleState extends ConsumerState // Cache for image base64 data to prevent repeated API calls final Map _imageCache = {}; + + // Cache for rendered image widgets to prevent rebuilding during streaming + final Map _imageWidgetCache = {}; + String? _lastAttachmentIds; @override void initState() { @@ -60,6 +64,207 @@ class _ModernMessageBubbleState extends ConsumerState ); } + + + Widget _buildAttachmentImages() { + if (widget.message.attachmentIds == null || + widget.message.attachmentIds!.isEmpty) { + return const SizedBox.shrink(); + } + + final currentAttachmentIds = widget.message.attachmentIds!.join('_'); + + // Clear cache if attachment IDs changed + if (_lastAttachmentIds != currentAttachmentIds) { + _imageWidgetCache.clear(); + _lastAttachmentIds = currentAttachmentIds; + } + + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: widget.message.attachmentIds!.map((attachmentId) { + // Return cached widget if available + if (_imageWidgetCache.containsKey(attachmentId)) { + return _imageWidgetCache[attachmentId]!; + } + + // Build widget and cache it + final imageWidget = _buildSingleImageWidget(attachmentId); + _imageWidgetCache[attachmentId] = imageWidget; + return imageWidget; + }).toList(), + ); + } + + Widget _buildSingleImageWidget(String attachmentId) { + return Consumer( + builder: (context, ref, child) { + final api = ref.read(apiServiceProvider); + if (api == null) return const SizedBox.shrink(); + + return FutureBuilder( + key: ValueKey('img_$attachmentId'), + future: _getCachedImageBase64(api, attachmentId), + builder: (context, snapshot) { + if (snapshot.connectionState == ConnectionState.waiting) { + return Container( + height: 150, + width: 200, + margin: const EdgeInsets.only(bottom: Spacing.xs), + decoration: BoxDecoration( + color: context.conduitTheme.surfaceBackground.withValues( + alpha: 0.5, + ), + borderRadius: BorderRadius.circular(AppBorderRadius.sm), + ), + child: Center( + child: CircularProgressIndicator( + color: context.conduitTheme.buttonPrimary, + strokeWidth: 2, + ), + ), + ); + } + + if (snapshot.hasError || + !snapshot.hasData || + snapshot.data == null) { + return Container( + height: 100, + width: 150, + margin: const EdgeInsets.only(bottom: Spacing.xs), + decoration: BoxDecoration( + color: context.conduitTheme.surfaceBackground.withValues( + alpha: 0.3, + ), + borderRadius: BorderRadius.circular(AppBorderRadius.sm), + border: Border.all( + color: context.conduitTheme.textSecondary.withValues( + alpha: 0.3, + ), + ), + ), + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Icon( + Icons.broken_image_outlined, + color: context.conduitTheme.textSecondary, + size: 32, + ), + const SizedBox(height: Spacing.xs), + Text( + 'Image unavailable', + style: TextStyle( + color: context.conduitTheme.textSecondary, + fontSize: AppTypography.bodySmall, + ), + ), + ], + ), + ); + } + + final base64Data = snapshot.data!; + try { + // Handle data URLs (data:image/...;base64,...) + String actualBase64; + if (base64Data.startsWith('data:')) { + // Extract base64 part from data URL + final commaIndex = base64Data.indexOf(','); + if (commaIndex != -1) { + actualBase64 = base64Data.substring(commaIndex + 1); + } else { + throw Exception('Invalid data URL format'); + } + } else { + // Direct base64 string + actualBase64 = base64Data; + } + + final imageBytes = base64.decode(actualBase64); + return Container( + margin: const EdgeInsets.only(bottom: Spacing.xs), + constraints: const BoxConstraints( + maxWidth: 300, + maxHeight: 300, + ), + child: ClipRRect( + borderRadius: BorderRadius.circular(AppBorderRadius.sm), + child: Image.memory( + imageBytes, + fit: BoxFit.cover, + errorBuilder: (context, error, stackTrace) { + return Container( + height: 100, + width: 150, + decoration: BoxDecoration( + color: context.conduitTheme.surfaceBackground + .withValues(alpha: 0.3), + borderRadius: BorderRadius.circular( + AppBorderRadius.sm, + ), + ), + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Icon( + Icons.error_outline, + color: context.conduitTheme.error, + size: 32, + ), + const SizedBox(height: Spacing.xs), + Text( + 'Failed to load image', + style: TextStyle( + color: context.conduitTheme.error, + fontSize: AppTypography.bodySmall, + ), + ), + ], + ), + ); + }, + ), + ), + ); + } catch (e) { + return Container( + height: 100, + width: 150, + margin: const EdgeInsets.only(bottom: Spacing.xs), + decoration: BoxDecoration( + color: context.conduitTheme.surfaceBackground.withValues( + alpha: 0.3, + ), + borderRadius: BorderRadius.circular(AppBorderRadius.sm), + ), + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Icon( + Icons.error_outline, + color: context.conduitTheme.error, + size: 32, + ), + const SizedBox(height: Spacing.xs), + Text( + 'Invalid image format', + style: TextStyle( + color: context.conduitTheme.error, + fontSize: AppTypography.bodySmall, + ), + ), + ], + ), + ); + } + }, + ); + }, + ); + } + @override void dispose() { _fadeController.dispose(); @@ -279,183 +484,7 @@ class _ModernMessageBubbleState extends ConsumerState ); } - Widget _buildAttachmentImages() { - if (widget.message.attachmentIds == null || - widget.message.attachmentIds!.isEmpty) { - return const SizedBox.shrink(); - } - return Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: widget.message.attachmentIds!.map((attachmentId) { - return Consumer( - builder: (context, ref, child) { - final api = ref.watch(apiServiceProvider); - if (api == null) return const SizedBox.shrink(); - - return FutureBuilder( - future: _getCachedImageBase64(api, attachmentId), - builder: (context, snapshot) { - if (snapshot.connectionState == ConnectionState.waiting) { - return Container( - height: 150, - width: 200, - margin: const EdgeInsets.only(bottom: Spacing.xs), - decoration: BoxDecoration( - color: context.conduitTheme.surfaceBackground.withValues( - alpha: 0.5, - ), - borderRadius: BorderRadius.circular(AppBorderRadius.sm), - ), - child: Center( - child: CircularProgressIndicator( - color: context.conduitTheme.buttonPrimary, - strokeWidth: 2, - ), - ), - ); - } - - if (snapshot.hasError || - !snapshot.hasData || - snapshot.data == null) { - return Container( - height: 100, - width: 150, - margin: const EdgeInsets.only(bottom: Spacing.xs), - decoration: BoxDecoration( - color: context.conduitTheme.surfaceBackground.withValues( - alpha: 0.3, - ), - borderRadius: BorderRadius.circular(AppBorderRadius.sm), - border: Border.all( - color: context.conduitTheme.textSecondary.withValues( - alpha: 0.3, - ), - ), - ), - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - Icon( - Icons.broken_image_outlined, - color: context.conduitTheme.textSecondary, - size: 32, - ), - const SizedBox(height: Spacing.xs), - Text( - 'Image unavailable', - style: TextStyle( - color: context.conduitTheme.textSecondary, - fontSize: AppTypography.bodySmall, - ), - ), - ], - ), - ); - } - - final base64Data = snapshot.data!; - try { - // Handle data URLs (data:image/...;base64,...) - String actualBase64; - if (base64Data.startsWith('data:')) { - // Extract base64 part from data URL - final commaIndex = base64Data.indexOf(','); - if (commaIndex != -1) { - actualBase64 = base64Data.substring(commaIndex + 1); - } else { - throw Exception('Invalid data URL format'); - } - } else { - // Direct base64 string - actualBase64 = base64Data; - } - - final imageBytes = base64.decode(actualBase64); - return Container( - margin: const EdgeInsets.only(bottom: Spacing.xs), - constraints: const BoxConstraints( - maxWidth: 300, - maxHeight: 300, - ), - child: ClipRRect( - borderRadius: BorderRadius.circular(AppBorderRadius.sm), - child: Image.memory( - imageBytes, - fit: BoxFit.cover, - errorBuilder: (context, error, stackTrace) { - return Container( - height: 100, - width: 150, - decoration: BoxDecoration( - color: context.conduitTheme.surfaceBackground - .withValues(alpha: 0.3), - borderRadius: BorderRadius.circular( - AppBorderRadius.sm, - ), - ), - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - Icon( - Icons.error_outline, - color: context.conduitTheme.error, - size: 32, - ), - const SizedBox(height: Spacing.xs), - Text( - 'Failed to load image', - style: TextStyle( - color: context.conduitTheme.error, - fontSize: AppTypography.bodySmall, - ), - ), - ], - ), - ); - }, - ), - ), - ); - } catch (e) { - return Container( - height: 100, - width: 150, - margin: const EdgeInsets.only(bottom: Spacing.xs), - decoration: BoxDecoration( - color: context.conduitTheme.surfaceBackground.withValues( - alpha: 0.3, - ), - borderRadius: BorderRadius.circular(AppBorderRadius.sm), - ), - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - Icon( - Icons.error_outline, - color: context.conduitTheme.error, - size: 32, - ), - const SizedBox(height: Spacing.xs), - Text( - 'Invalid image format', - style: TextStyle( - color: context.conduitTheme.error, - fontSize: AppTypography.bodySmall, - ), - ), - ], - ), - ); - } - }, - ); - }, - ); - }).toList(), - ); - } Future _getCachedImageBase64(dynamic api, String fileId) async { // Check cache first to prevent repeated API calls diff --git a/test_streaming.md b/test_streaming.md deleted file mode 100644 index 3b7dbcb..0000000 --- a/test_streaming.md +++ /dev/null @@ -1,93 +0,0 @@ -# Testing Background Streaming Resilience - -## Quick Test Steps - -1. **Start a Chat Stream** - - Open the app and start a new conversation - - Send a message that will generate a long response - - Verify streaming starts normally - -2. **Test Background Resilience** - - While response is streaming, switch to another app (press home button) - - Wait 10-15 seconds - - Return to the app - - Verify: Stream continues or resumes without duplicate content - -3. **Test Network Interruption** - - Start streaming a response - - Turn on airplane mode for 5 seconds - - Turn off airplane mode - - Verify: Stream recovers and continues - -4. **Test App Lifecycle** - - Start streaming - - Background the app multiple times rapidly - - Verify: No memory leaks, single active stream - -## Implementation Summary - -### Core Changes Made: - -1. **BackgroundStreamingHandler** (`lib/core/services/background_streaming_handler.dart`) - - Manages stream state across app lifecycle changes - - Handles iOS background tasks and Android foreground services - - Tracks stream metadata for recovery - -2. **Enhanced PersistentStreamingService** (`lib/core/services/persistent_streaming_service.dart`) - - Integrates with BackgroundStreamingHandler - - Monitors connectivity and app lifecycle - - Implements exponential backoff retry logic - - Tracks stream progress for resume capability - -3. **Robust SSE Parser** (`lib/core/services/sse_parser.dart`) - - Heartbeat monitoring with configurable timeout - - Tolerates partial Unicode and network hiccups - - Emits reconnection requests on timeout - - Handles incomplete data gracefully - -4. **Enhanced API Service** (`lib/core/services/api_service.dart`) - - Updated `_streamSSE` method to use persistent service - - Better error handling and recovery - - Longer timeouts for streaming connections - - Progress tracking for resume capability - -5. **iOS Integration** (`ios/Runner/BackgroundStreamingHandler.swift`) - - Proper Flutter plugin registration - - Background task management (~30 seconds) - - Stream state persistence in UserDefaults - -6. **Android Integration** (`android/.../BackgroundStreamingHandler.kt`) - - Foreground service for extended background processing - - Wake lock management for reliable networking - - SharedPreferences for stream state persistence - - Notification handling for user awareness - -### Key Features: - -- **Automatic Recovery**: Streams auto-resume when app returns to foreground -- **Connectivity Awareness**: Pauses on network loss, resumes on reconnection -- **Background Execution**: - - iOS: ~30 seconds of background streaming via background tasks - - Android: Foreground service with wake lock for extended background processing -- **Heartbeat Monitoring**: Detects dead connections and triggers recovery -- **Progress Tracking**: Tracks chunk sequence and content for resumption -- **Exponential Backoff**: Smart retry logic with jitter to avoid thundering herd -- **Cross-Platform**: Works on both iOS and Android with platform-specific optimizations - -### Testing Scenarios Covered: - -✅ App backgrounding during stream -✅ Network connectivity loss/restore -✅ Rapid background/foreground cycles -✅ Long-running streams (>5 min) -✅ Server-side disconnections -✅ Auth token expiration during stream -✅ Multiple concurrent streams - -## Next Steps - -1. Test with real OpenWebUI server -2. Verify memory usage during long streams -3. Test with poor network conditions -4. Add telemetry for recovery success rates -5. Consider adding user notification for background recovery \ No newline at end of file