ars/iiEsaywebUIapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: optimize regex patterns for image and markdown processing

Browse Source 
- Introduced pre-compiled regex patterns across various components, including streaming_helper, markdown_stream_formatter, and assistant_message_widget, to enhance performance during image extraction and markdown formatting.
- Updated the AssistantMessageWidget to utilize these optimized patterns for TTS sanitization and image processing, reducing unnecessary regex evaluations.
- Improved overall efficiency in handling markdown content by leveraging pre-compiled patterns for common markdown syntax detection.
This commit is contained in:
cogwheel0
2025-10-06 00:09:52 +05:30
parent 3af46b379b
commit a2e5f46d62
4 changed files with 94 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
lib/core/services/streaming_helper.dart
View File

@@ -20,6 +20,28 @@ import 'streaming_response_controller.dart';
// Keep local verbosity toggle for socket logs
const bool kSocketVerboseLogging = false;
// Pre-compiled regex patterns for image extraction (performance optimization)
final _base64ImagePattern = RegExp(
r'data:image/[^;\s]+;base64,[A-Za-z0-9+/]+=*',
);
final _urlImagePattern = RegExp(
r'https?://[^\s<>\"]+\.(jpg|jpeg|png|gif|webp)',
caseSensitive: false,
);
final _jsonImagePattern = RegExp(
r'\{[^}]*"url"[^}]*:[^}]*"(data:image/[^"]+|https?://[^"]+\.(jpg|jpeg|png|gif|webp))"[^}]*\}',
caseSensitive: false,
);
final _jsonUrlExtractPattern = RegExp(r'"url"[^:]*:[^"]*"([^"]+)"');
final _partialResultsPattern = RegExp(
r'(result|files)="([^"]*(?:data:image/[^"]*|https?://[^"]*\.(jpg|jpeg|png|gif|webp))[^"]*)"',
caseSensitive: false,
);
final _imageFilePattern = RegExp(
r'https?://[^\s]+\.(jpg|jpeg|png|gif|webp)$',
caseSensitive: false,
);
class ActiveSocketStream {
ActiveSocketStream({
required this.controller,
@@ -194,7 +216,8 @@ ActiveSocketStream attachUnifiedChunkedStreaming({
final collected = <Map<String, dynamic>>[];
if (content.contains('<details')) {
// Quick check: only parse tool calls if complete details blocks exist
if (content.contains('<details') && content.contains('</details>')) {
final parsed = ToolCallsParser.parse(content);
if (parsed != null) {
for (final entry in parsed.toolCalls) {
@@ -209,10 +232,8 @@ ActiveSocketStream attachUnifiedChunkedStreaming({
}
if (collected.isEmpty) {
final base64Pattern = RegExp(
r'data:image/[^;\s]+;base64,[A-Za-z0-9+/]+=*',
);
final base64Matches = base64Pattern.allMatches(content);
// Use pre-compiled patterns for better performance
final base64Matches = _base64ImagePattern.allMatches(content);
for (final match in base64Matches) {
final url = match.group(0);
if (url != null && url.isNotEmpty) {
@@ -220,11 +241,7 @@ ActiveSocketStream attachUnifiedChunkedStreaming({
}
}
final urlPattern = RegExp(
r'https?://[^\s<>\"]+\.(jpg|jpeg|png|gif|webp)',
caseSensitive: false,
);
final urlMatches = urlPattern.allMatches(content);
final urlMatches = _urlImagePattern.allMatches(content);
for (final match in urlMatches) {
final url = match.group(0);
if (url != null && url.isNotEmpty) {
@@ -232,25 +249,17 @@ ActiveSocketStream attachUnifiedChunkedStreaming({
}
}
final jsonPattern = RegExp(
r'\{[^}]*"url"[^}]*:[^}]*"(data:image/[^"]+|https?://[^"]+\.(jpg|jpeg|png|gif|webp))"[^}]*\}',
caseSensitive: false,
);
final jsonMatches = jsonPattern.allMatches(content);
final jsonMatches = _jsonImagePattern.allMatches(content);
for (final match in jsonMatches) {
final url = RegExp(
r'"url"[^:]*:[^"]*"([^"]+)"',
).firstMatch(match.group(0) ?? '')?.group(1);
final url = _jsonUrlExtractPattern
.firstMatch(match.group(0) ?? '')
?.group(1);
if (url != null && url.isNotEmpty) {
collected.add({'type': 'image', 'url': url});
}
}
final partialResultsPattern = RegExp(
r'(result|files)="([^"]*(?:data:image/[^"]*|https?://[^"]*\.(jpg|jpeg|png|gif|webp))[^"]*)"',
caseSensitive: false,
);
final partialMatches = partialResultsPattern.allMatches(content);
final partialMatches = _partialResultsPattern.allMatches(content);
for (final match in partialMatches) {
final attrValue = match.group(2);
if (attrValue != null) {
@@ -259,10 +268,7 @@ ActiveSocketStream attachUnifiedChunkedStreaming({
collected.addAll(_extractFilesFromResult(decoded));
} catch (_) {
if (attrValue.startsWith('data:image/') ||
RegExp(
r'https?://[^\s]+\.(jpg|jpeg|png|gif|webp)$',
caseSensitive: false,
).hasMatch(attrValue)) {
_imageFilePattern.hasMatch(attrValue)) {
collected.add({'type': 'image', 'url': attrValue});
}
}
@@ -410,14 +416,9 @@ ActiveSocketStream attachUnifiedChunkedStreaming({
: null;
if (name is String && name.isNotEmpty) {
final msgs = getMessages();
final exists =
(msgs.isNotEmpty) &&
RegExp(
r'<details\s+type=\"tool_calls\"[^>]*\bname=\"' +
RegExp.escape(name) +
r'\"',
multiLine: true,
).hasMatch(msgs.last.content);
// Quick string check before expensive regex
final exists = (msgs.isNotEmpty) &&
msgs.last.content.contains('name="$name"');
if (!exists) {
final status =
'\n<details type="tool_calls" done="false" name="$name"><summary>Executing...</summary>\n</details>\n';
@@ -517,14 +518,9 @@ ActiveSocketStream attachUnifiedChunkedStreaming({
: null;
if (name is String && name.isNotEmpty) {
final msgs = getMessages();
final exists =
(msgs.isNotEmpty) &&
RegExp(
r'<details\s+type=\"tool_calls\"[^>]*\bname=\"' +
RegExp.escape(name) +
r'\"',
multiLine: true,
).hasMatch(msgs.last.content);
// Quick string check before expensive regex
final exists = (msgs.isNotEmpty) &&
msgs.last.content.contains('name="$name"');
if (!exists) {
final status =
'\n<details type="tool_calls" done="false" name="$name"><summary>Executing...</summary>\n</details>\n';
@@ -552,14 +548,9 @@ ActiveSocketStream attachUnifiedChunkedStreaming({
: null;
if (name is String && name.isNotEmpty) {
final msgs = getMessages();
final exists =
(msgs.isNotEmpty) &&
RegExp(
r'<details\s+type=\"tool_calls\"[^>]*\bname=\"' +
RegExp.escape(name) +
r'\"',
multiLine: true,
).hasMatch(msgs.last.content);
// Quick string check before expensive regex
final exists = (msgs.isNotEmpty) &&
msgs.last.content.contains('name="$name"');
if (!exists) {
final status =
'\n<details type="tool_calls" done="false" name="$name"><summary>Executing...</summary>\n</details>\n';

8
lib/core/utils/markdown_stream_formatter.dart
View File

@@ -1,3 +1,7 @@
// Pre-compiled regex patterns for markdown syntax detection (performance optimization)
final _boldPattern = RegExp(r'\*\*');
final _italicPattern = RegExp(r'(?<!\*)\*(?!\*)');
/// Maintains a raw markdown buffer for streaming content and generates
/// preview-safe output by appending synthetic closing tokens when necessary.
class MarkdownStreamFormatter {
@@ -40,12 +44,12 @@ class MarkdownStreamFormatter {
buffer.writeln('```');
}
final boldCount = RegExp(r'\*\*').allMatches(content).length;
final boldCount = _boldPattern.allMatches(content).length;
if (boldCount.isOdd) {
buffer.write('**');
}
final italicCount = RegExp(r'(?<!\*)\*(?!\*)').allMatches(content).length;
final italicCount = _italicPattern.allMatches(content).length;
if (italicCount.isOdd) {
buffer.write('*');
}

67
lib/features/chat/widgets/assistant_message_widget.dart
View File

@@ -23,6 +23,25 @@ import '../../../core/utils/debug_logger.dart';
import 'sources/openwebui_sources.dart';
import '../providers/assistant_response_builder_provider.dart';
// Pre-compiled regex patterns for TTS sanitization (performance optimization)
final _ttsCodeBlockPattern = RegExp(r'```');
final _ttsInlineCodePattern = RegExp(r'`');
final _ttsImagePattern = RegExp(r'!\[(.*?)\]\((.*?)\)');
final _ttsLinkPattern = RegExp(r'\[(.*?)\]\((.*?)\)');
final _ttsBoldPattern1 = RegExp(r'\*\*');
final _ttsBoldPattern2 = RegExp(r'__');
final _ttsItalicPattern1 = RegExp(r'\*');
final _ttsItalicPattern2 = RegExp(r'_');
final _ttsStrikePattern = RegExp(r'~');
final _ttsListPattern = RegExp(r'^[-*+]\s+', multiLine: true);
final _ttsQuotePattern = RegExp(r'^>\s?', multiLine: true);
final _ttsMultiSpacePattern = RegExp(r'[ \t]{2,}');
final _ttsMultiNewlinePattern = RegExp(r'\n{3,}');
// Pre-compiled regex patterns for image processing (performance optimization)
final _base64ImagePattern = RegExp(r'data:image/[^;]+;base64,[A-Za-z0-9+/]+=*');
final _fileIdPattern = RegExp(r'/api/v1/files/([^/]+)/content');
class AssistantMessageWidget extends ConsumerStatefulWidget {
final dynamic message;
final bool isStreaming;
@@ -270,23 +289,24 @@ class _AssistantMessageWidgetState extends ConsumerState<AssistantMessageWidget>
}
var text = input;
text = text.replaceAll(RegExp(r'```'), ' ');
text = text.replaceAll(RegExp(r'`'), '');
text = text.replaceAll(RegExp(r'!\[(.*?)\]\((.*?)\)'), r'$1');
text = text.replaceAll(RegExp(r'\[(.*?)\]\((.*?)\)'), r'$1');
text = text.replaceAll(RegExp(r'\*\*'), '');
text = text.replaceAll(RegExp(r'__'), '');
text = text.replaceAll(RegExp(r'\*'), '');
text = text.replaceAll(RegExp(r'_'), '');
text = text.replaceAll(RegExp(r'~'), '');
text = text.replaceAll(RegExp(r'^[-*+]\s+', multiLine: true), '');
text = text.replaceAll(RegExp(r'^>\s?', multiLine: true), '');
// Use pre-compiled regex patterns for better performance
text = text.replaceAll(_ttsCodeBlockPattern, ' ');
text = text.replaceAll(_ttsInlineCodePattern, '');
text = text.replaceAll(_ttsImagePattern, r'$1');
text = text.replaceAll(_ttsLinkPattern, r'$1');
text = text.replaceAll(_ttsBoldPattern1, '');
text = text.replaceAll(_ttsBoldPattern2, '');
text = text.replaceAll(_ttsItalicPattern1, '');
text = text.replaceAll(_ttsItalicPattern2, '');
text = text.replaceAll(_ttsStrikePattern, '');
text = text.replaceAll(_ttsListPattern, '');
text = text.replaceAll(_ttsQuotePattern, '');
text = text.replaceAll('&nbsp;', ' ');
text = text.replaceAll('&amp;', '&');
text = text.replaceAll('&lt;', '<');
text = text.replaceAll('&gt;', '>');
text = text.replaceAll(RegExp(r'[ \t]{2,}'), ' ');
text = text.replaceAll(RegExp(r'\n{3,}'), '\n\n');
text = text.replaceAll(_ttsMultiSpacePattern, ' ');
text = text.replaceAll(_ttsMultiNewlinePattern, '\n\n');
return text.trim();
}
@@ -771,18 +791,17 @@ class _AssistantMessageWidgetState extends ConsumerState<AssistantMessageWidget>
// Check if content contains image markdown or base64 data URLs
// This ensures images generated by AI are properly formatted
// Pattern to detect base64 images that might not be in markdown format
final base64Pattern = RegExp(r'data:image/[^;]+;base64,[A-Za-z0-9+/]+=*');
// Quick check: only process if we have base64 images and no markdown
if (!content.contains('data:image/') || content.contains('![')) {
return content;
}
// If we find base64 images not wrapped in markdown, wrap them
if (base64Pattern.hasMatch(content) && !content.contains('![')) {
content = content.replaceAllMapped(base64Pattern, (match) {
if (_base64ImagePattern.hasMatch(content)) {
content = content.replaceAllMapped(_base64ImagePattern, (match) {
final imageData = match.group(0)!;
// Check if this image is already in markdown format
final markdownCheck = RegExp(
r'!\[.*?\]\(' + RegExp.escape(imageData) + r'\)',
);
if (!markdownCheck.hasMatch(content)) {
// Check if this image is already in markdown format (simple string check)
if (!content.contains('![$imageData)')) {
return '\n![Generated Image]($imageData)\n';
}
return imageData;
@@ -951,9 +970,7 @@ class _AssistantMessageWidgetState extends ConsumerState<AssistantMessageWidget>
String attachmentId = fileUrl;
if (fileUrl.contains('/api/v1/files/') &&
fileUrl.contains('/content')) {
final fileIdMatch = RegExp(
r'/api/v1/files/([^/]+)/content',
).firstMatch(fileUrl);
final fileIdMatch = _fileIdPattern.firstMatch(fileUrl);
if (fileIdMatch != null) {
attachmentId = fileIdMatch.group(1)!;
}

6
lib/shared/widgets/markdown/streaming_markdown_widget.dart
View File

@@ -4,6 +4,9 @@ import '../../theme/theme_extensions.dart';
import 'markdown_config.dart';
import 'markdown_preprocessor.dart';
// Pre-compiled regex for mermaid diagram detection (performance optimization)
final _mermaidRegex = RegExp(r'```mermaid\s*([\s\S]*?)```', multiLine: true);
class StreamingMarkdownWidget extends StatelessWidget {
const StreamingMarkdownWidget({
super.key,
@@ -23,8 +26,7 @@ class StreamingMarkdownWidget extends StatelessWidget {
}
final normalized = ConduitMarkdownPreprocessor.normalize(content);
final mermaidRegex = RegExp(r'```mermaid\s*([\s\S]*?)```', multiLine: true);
final matches = mermaidRegex.allMatches(normalized).toList();
final matches = _mermaidRegex.allMatches(normalized).toList();
Widget buildMarkdown(String data) {
return ConduitMarkdown.buildBlock(