feat(chat): Add context attachment and knowledge base support

lib/features/chat/providers/chat_providers.dart

@@ -18,6 +18,8 @@ import '../../../core/services/worker_manager.dart';
 import '../../../core/utils/debug_logger.dart';
 import '../../../core/utils/markdown_stream_formatter.dart';
 import '../../../core/utils/tool_calls_parser.dart';
+import '../models/chat_context_attachment.dart';
+import '../providers/context_attachments_provider.dart';
 import '../../../shared/services/tasks/task_queue.dart';
 import '../../tools/providers/tools_providers.dart';
 import '../services/reviewer_mode_service.dart';
@@ -848,6 +850,9 @@ void startNewChat(dynamic ref) {
 
   // Clear messages
   ref.read(chatMessagesProvider.notifier).clearMessages();
+
+  // Clear context attachments (web pages, YouTube, knowledge base docs)
+  ref.read(contextAttachmentsProvider.notifier).clear();
 }
 
 // Available tools provider
@@ -944,47 +949,6 @@ bool validateFileCount(int currentCount, int newFilesCount, int? maxCount) {
   return (currentCount + newFilesCount) <= maxCount;
 }
 
-// Helper function to build files array from attachment IDs
-Future<List<Map<String, dynamic>>?> _buildFilesArrayFromAttachments(
-  dynamic api,
-  List<String> attachmentIds,
-) async {
-  final filesArray = <Map<String, dynamic>>[];
-
-  for (final attachmentId in attachmentIds) {
-    try {
-      final fileInfo = await api.getFileInfo(attachmentId);
-      final fileName = fileInfo['filename'] ?? fileInfo['name'] ?? 'Unknown';
-      final fileSize = fileInfo['size'];
-
-      // Check if it's an image
-      final ext = fileName.toLowerCase().split('.').last;
-      final isImage = ['jpg', 'jpeg', 'png', 'gif', 'webp'].contains(ext);
-
-      // Add all files to the files array for WebUI display
-      // Note: This is for storage/display, not for API message sending
-      filesArray.add({
-        'type': isImage ? 'image' : 'file',
-        'id': attachmentId, // Required for RAG system to lookup file content
-        'url': '/api/v1/files/$attachmentId/content',
-        'name': fileName,
-        if (fileSize != null) 'size': fileSize,
-      });
-    } catch (_) {
-      // If we can't get file info, assume it's a non-image file
-      // Images should be handled in the content array anyway
-      filesArray.add({
-        'type': 'file',
-        'id': attachmentId, // Required for RAG system to lookup file content
-        'url': '/api/v1/files/$attachmentId/content',
-        'name': 'Unknown',
-      });
-    }
-  }
-
-  return filesArray.isNotEmpty ? filesArray : null;
-}
-
 // Helper function to get file content as base64
 Future<String?> _getFileAsBase64(dynamic api, String fileId) async {
   // Check if this is already a data URL (for images)
@@ -1102,6 +1066,60 @@ Future<Map<String, dynamic>> _buildMessagePayloadWithAttachments({
   return messageMap;
 }
 
+List<Map<String, dynamic>> _contextAttachmentsToFiles(
+  List<ChatContextAttachment> attachments,
+) {
+  return attachments.map((attachment) {
+    switch (attachment.type) {
+      case ChatContextAttachmentType.web:
+        // Web pages use type 'text' with file data nested under 'file' key
+        return {
+          'type': 'text',
+          'name': attachment.url ?? attachment.displayName,
+          if (attachment.url != null) 'url': attachment.url,
+          if (attachment.collectionName != null)
+            'collection_name': attachment.collectionName,
+          'file': {
+            'data': {'content': attachment.content ?? ''},
+            'meta': {
+              'name': attachment.displayName,
+              if (attachment.url != null) 'source': attachment.url,
+            },
+          },
+        };
+      case ChatContextAttachmentType.youtube:
+        // YouTube uses type 'text' with context 'full' for full transcript
+        return {
+          'type': 'text',
+          'name': attachment.url ?? attachment.displayName,
+          if (attachment.url != null) 'url': attachment.url,
+          'context': 'full',
+          if (attachment.collectionName != null)
+            'collection_name': attachment.collectionName,
+          'file': {
+            'data': {'content': attachment.content ?? ''},
+            'meta': {
+              'name': attachment.displayName,
+              if (attachment.url != null) 'source': attachment.url,
+            },
+          },
+        };
+      case ChatContextAttachmentType.knowledge:
+        // Knowledge base files use type 'file' with id for lookup
+        final map = <String, dynamic>{
+          'type': 'file',
+          'id': attachment.fileId ?? attachment.id,
+          'name': attachment.displayName,
+          'knowledge': true,
+          if (attachment.collectionName != null)
+            'collection_name': attachment.collectionName,
+          if (attachment.url != null) 'source': attachment.url,
+        };
+        return map;
+    }
+  }).toList();
+}
+
 // Regenerate message function that doesn't duplicate user message
 Future<void> regenerateMessage(
   dynamic ref,
@@ -1605,13 +1623,15 @@ Future<void> _sendMessageInternal(
   var activeConversation = ref.read(activeConversationProvider);
 
   // Create user message first
-  List<Map<String, dynamic>>? userFiles;
-  if (attachments != null &&
-      attachments.isNotEmpty &&
-      !reviewerMode &&
-      api != null) {
-    userFiles = await _buildFilesArrayFromAttachments(api, attachments);
-  }
+  // Note: We only store context attachments (web/youtube/knowledge) in msg.files.
+  // Uploaded files are tracked via attachmentIds and will be rebuilt by
+  // _buildMessagePayloadWithAttachments when constructing the API payload.
+  // This prevents uploaded files from being duplicated in the final message.
+  final contextAttachments = ref.read(contextAttachmentsProvider);
+  final contextFiles = _contextAttachmentsToFiles(contextAttachments);
+  final List<Map<String, dynamic>>? userFiles = contextFiles.isNotEmpty
+      ? contextFiles
+      : null;
 
   final userMessage = ChatMessage(
     id: const Uuid().v4(),
@@ -1768,10 +1788,23 @@ Future<void> _sendMessageInternal(
           cleanedText: cleaned,
           attachmentIds: ids,
         );
+        if (msg.files != null && msg.files!.isNotEmpty) {
+          messageMap['files'] = [
+            ...?messageMap['files'] as List<dynamic>?,
+            ...msg.files!,
+          ];
+        }
         conversationMessages.add(messageMap);
       } else {
         // Regular text-only message
-        conversationMessages.add({'role': msg.role, 'content': cleaned});
+        final Map<String, dynamic> messageMap = {
+          'role': msg.role,
+          'content': cleaned,
+        };
+        if (msg.files != null && msg.files!.isNotEmpty) {
+          messageMap['files'] = msg.files;
+        }
+        conversationMessages.add(messageMap);
       }
     }
   }
@@ -2110,6 +2143,13 @@ Future<void> _sendMessageInternal(
         activeStream.socketSubscriptions,
         onDispose: activeStream.disposeWatchdog,
       );
+
+    // Clear context attachments after successfully initiating the message send.
+    // This prevents stale attachments from being included in subsequent messages.
+    try {
+      ref.read(contextAttachmentsProvider.notifier).clear();
+    } catch (_) {}
+
     return;
   } catch (e) {
     // Handle error - remove the assistant message placeholder

lib/features/chat/providers/context_attachments_provider.dart

@@ -0,0 +1,83 @@
+import 'package:flutter_riverpod/flutter_riverpod.dart';
+import 'package:uuid/uuid.dart';
+
+import '../models/chat_context_attachment.dart';
+
+class ContextAttachmentsNotifier extends Notifier<List<ChatContextAttachment>> {
+  @override
+  List<ChatContextAttachment> build() => const [];
+
+  void add(ChatContextAttachment attachment) {
+    state = [...state, attachment];
+  }
+
+  void addWeb({
+    required String displayName,
+    required String content,
+    required String url,
+    String? collectionName,
+  }) {
+    final id = const Uuid().v4();
+    add(
+      ChatContextAttachment(
+        id: id,
+        type: ChatContextAttachmentType.web,
+        displayName: displayName,
+        url: url,
+        content: content,
+        collectionName: collectionName,
+      ),
+    );
+  }
+
+  void addYoutube({
+    required String displayName,
+    required String content,
+    required String url,
+    String? collectionName,
+  }) {
+    final id = const Uuid().v4();
+    add(
+      ChatContextAttachment(
+        id: id,
+        type: ChatContextAttachmentType.youtube,
+        displayName: displayName,
+        url: url,
+        content: content,
+        collectionName: collectionName,
+      ),
+    );
+  }
+
+  void addKnowledge({
+    required String displayName,
+    required String fileId,
+    String? collectionName,
+    String? url,
+  }) {
+    final id = const Uuid().v4();
+    add(
+      ChatContextAttachment(
+        id: id,
+        type: ChatContextAttachmentType.knowledge,
+        displayName: displayName,
+        fileId: fileId,
+        url: url,
+        collectionName: collectionName,
+      ),
+    );
+  }
+
+  void remove(String id) {
+    state = state.where((item) => item.id != id).toList();
+  }
+
+  void clear() {
+    state = const [];
+  }
+}
+
+final contextAttachmentsProvider =
+    NotifierProvider<ContextAttachmentsNotifier, List<ChatContextAttachment>>(
+      ContextAttachmentsNotifier.new,
+    );

lib/features/chat/providers/knowledge_cache_provider.dart

@@ -0,0 +1,179 @@
+import 'package:flutter_riverpod/flutter_riverpod.dart';
+
+import '../../../core/models/knowledge_base.dart';
+import '../../../core/services/api_service.dart';
+import '../../../core/services/cache_manager.dart';
+import '../../../core/providers/app_providers.dart';
+import '../../../core/utils/debug_logger.dart';
+
+/// Cache keys for knowledge base data.
+const String _basesKey = 'knowledge_bases';
+String _itemsKey(String baseId) => 'knowledge_items:$baseId';
+
+/// TTL for knowledge cache entries.
+const Duration _knowledgeCacheTtl = Duration(minutes: 10);
+
+/// Centralized cache manager for knowledge base data.
+///
+/// Uses the shared [CacheManager] pattern for TTL and LRU eviction.
+class KnowledgeCacheManager {
+  static final KnowledgeCacheManager _instance =
+      KnowledgeCacheManager._internal();
+  factory KnowledgeCacheManager() => _instance;
+  KnowledgeCacheManager._internal();
+
+  final CacheManager _cache = CacheManager(
+    defaultTtl: _knowledgeCacheTtl,
+    maxEntries: 64,
+  );
+
+  /// Returns cached knowledge bases, or null if not cached.
+  List<KnowledgeBase>? getCachedBases() {
+    final (hit: hit, value: bases) =
+        _cache.lookup<List<KnowledgeBase>>(_basesKey);
+    if (hit) {
+      DebugLogger.log('cache-hit', scope: 'knowledge/bases');
+    }
+    return hit ? bases : null;
+  }
+
+  /// Caches knowledge bases.
+  void cacheBases(List<KnowledgeBase> bases) {
+    _cache.write<List<KnowledgeBase>>(_basesKey, bases);
+    DebugLogger.log(
+      'cache-write',
+      scope: 'knowledge/bases',
+      data: {'count': bases.length},
+    );
+  }
+
+  /// Returns cached items for a knowledge base, or null if not cached.
+  List<KnowledgeBaseItem>? getCachedItems(String baseId) {
+    final (hit: hit, value: items) =
+        _cache.lookup<List<KnowledgeBaseItem>>(_itemsKey(baseId));
+    if (hit) {
+      DebugLogger.log('cache-hit', scope: 'knowledge/items', data: {'baseId': baseId});
+    }
+    return hit ? items : null;
+  }
+
+  /// Caches items for a knowledge base.
+  void cacheItems(String baseId, List<KnowledgeBaseItem> items) {
+    _cache.write<List<KnowledgeBaseItem>>(_itemsKey(baseId), items);
+    DebugLogger.log(
+      'cache-write',
+      scope: 'knowledge/items',
+      data: {'baseId': baseId, 'count': items.length},
+    );
+  }
+
+  /// Clears all knowledge cache entries.
+  void clear() {
+    _cache.invalidateMatching((key) => key.startsWith('knowledge'));
+    DebugLogger.log('cache-clear', scope: 'knowledge');
+  }
+
+  /// Returns cache statistics for debugging.
+  Map<String, dynamic> stats() => _cache.stats();
+}
+
+/// State for the knowledge cache provider.
+class KnowledgeCacheState {
+  const KnowledgeCacheState({
+    this.bases = const <KnowledgeBase>[],
+    this.items = const <String, List<KnowledgeBaseItem>>{},
+    this.isLoading = false,
+  });
+
+  final List<KnowledgeBase> bases;
+  final Map<String, List<KnowledgeBaseItem>> items;
+  final bool isLoading;
+
+  KnowledgeCacheState copyWith({
+    List<KnowledgeBase>? bases,
+    Map<String, List<KnowledgeBaseItem>>? items,
+    bool? isLoading,
+  }) {
+    return KnowledgeCacheState(
+      bases: bases ?? this.bases,
+      items: items ?? this.items,
+      isLoading: isLoading ?? this.isLoading,
+    );
+  }
+}
+
+/// Notifier that wraps [KnowledgeCacheManager] with Riverpod reactivity.
+class KnowledgeCacheNotifier extends Notifier<KnowledgeCacheState> {
+  final _cacheManager = KnowledgeCacheManager();
+
+  @override
+  KnowledgeCacheState build() {
+    // Initialize from cache if available
+    final cachedBases = _cacheManager.getCachedBases();
+    if (cachedBases != null && cachedBases.isNotEmpty) {
+      return KnowledgeCacheState(bases: cachedBases);
+    }
+    return const KnowledgeCacheState();
+  }
+
+  ApiService? get _api => ref.read(apiServiceProvider);
+
+  Future<void> ensureBases() async {
+    // Check if already loaded in state
+    if (state.bases.isNotEmpty) return;
+
+    // Check cache
+    final cached = _cacheManager.getCachedBases();
+    if (cached != null && cached.isNotEmpty) {
+      state = state.copyWith(bases: cached);
+      return;
+    }
+
+    if (_api == null) return;
+    state = state.copyWith(isLoading: true);
+    try {
+      final bases = await _api!.getKnowledgeBases();
+      _cacheManager.cacheBases(bases);
+      state = state.copyWith(bases: bases, isLoading: false);
+    } catch (_) {
+      state = state.copyWith(isLoading: false);
+    }
+  }
+
+  Future<void> fetchItemsForBase(String baseId) async {
+    // Check if already in state
+    if (state.items.containsKey(baseId)) return;
+
+    // Check cache
+    final cached = _cacheManager.getCachedItems(baseId);
+    if (cached != null) {
+      final next = Map<String, List<KnowledgeBaseItem>>.from(state.items);
+      next[baseId] = cached;
+      state = state.copyWith(items: next);
+      return;
+    }
+
+    if (_api == null) return;
+
+    final next = Map<String, List<KnowledgeBaseItem>>.from(state.items);
+    try {
+      final items = await _api!.getKnowledgeBaseItems(baseId);
+      _cacheManager.cacheItems(baseId, items);
+      next[baseId] = items;
+    } catch (_) {
+      next[baseId] = const <KnowledgeBaseItem>[];
+    }
+    state = state.copyWith(items: next);
+  }
+
+  /// Clears both in-memory state and persistent cache.
+  void clearCache() {
+    _cacheManager.clear();
+    state = const KnowledgeCacheState();
+  }
+}
+
+final knowledgeCacheProvider =
+    NotifierProvider<KnowledgeCacheNotifier, KnowledgeCacheState>(
+  KnowledgeCacheNotifier.new,
+);