import 'package:html_unescape/html_unescape.dart';
import 'package:markdown/markdown.dart' as md;
/// Content preprocessing, sanitization, and transformation for Markdown.
///
/// Provides:
/// - [normalize] - Prepares content for display (keeps reasoning blocks)
/// - [sanitize] - Cleans content for copy/API (removes reasoning blocks)
/// - [toPlainText] - Converts to plain text for TTS
/// - [softenInlineCode] - Breaks long inline code spans
class ConduitMarkdownPreprocessor {
const ConduitMarkdownPreprocessor._();
static final _htmlUnescape = HtmlUnescape();
// ============================================================
// Pre-compiled Patterns - Display/Sanitization
// ============================================================
static final _bulletFenceRegex = RegExp(
r'^(\s*(?:[*+-]|\d+\.)\s+)```([^\s`]*)\s*$',
multiLine: true,
);
static final _dedentOpenRegex = RegExp(
r'^[ \t]+```([^\n`]*)\s*$',
multiLine: true,
);
static final _dedentCloseRegex = RegExp(r'^[ \t]+```\s*$', multiLine: true);
static final _inlineClosingRegex =
RegExp(r'([^\r\n`])```(?=\s*(?:\r?\n|$))');
static final _labelThenDashRegex = RegExp(
r'^(\*\*[^\n*]+\*\*.*)\n(\s*-{3,}\s*$)',
multiLine: true,
);
static final _atxEnumRegex = RegExp(
r'^(\s{0,3}#{1,6}\s+\d+)\.(\s*)(\S)',
multiLine: true,
);
static final _fenceAtBolRegex = RegExp(r'^\s*```', multiLine: true);
static final _linkWithTrailingSpaces =
RegExp(r'\[[^\]]+\]\([^\)]+\)\s{2,}$');
static final _multipleNewlines = RegExp(r'\n{3,}');
/// Combined pattern for all reasoning/thinking blocks.
static final _reasoningBlocks = RegExp(
r']*>[\s\S]*? |'
r'<(?:think|thinking|reasoning)(?:\s[^>]*)?>[\s\S]*?(?:think|thinking|reasoning)>',
multiLine: true,
dotAll: true,
);
// ============================================================
// Pre-compiled Patterns - Plain Text (TTS)
// ============================================================
static final _codeBlock = RegExp(r'```[^\n]*\n[\s\S]*?```');
static final _inlineCode = RegExp(r'`([^`]+)`');
static final _image = RegExp(r'!\[[^\]]*\]\([^)]+\)');
static final _link = RegExp(r'\[([^\]]+)\]\([^)]+\)');
// Paired markdown formatting - only unambiguous markers for TTS
// Single * and _ are skipped as they're ambiguous (math, variable names)
static final _boldItalic = RegExp(r'\*\*\*([^*]+)\*\*\*');
static final _bold = RegExp(r'\*\*([^*]+)\*\*');
static final _strikethrough = RegExp(r'~~([^~]+)~~');
// Single asterisk italic: only at word boundaries (space or line start/end)
static final _italicAsterisk = RegExp(r'(?:^|\s)\*([^*\s]+)\*(?=\s|$)');
// Single underscore italic: only when surrounded by spaces (not in identifiers)
static final _italicUnderscore = RegExp(r'(?:^|\s)_([^_\s]+)_(?=\s|$)');
static final _heading = RegExp(r'^#{1,6}\s+', multiLine: true);
static final _listMarker = RegExp(r'^[\s]*(?:[-*+]|\d+\.)\s+', multiLine: true);
static final _blockquote = RegExp(r'^>\s*', multiLine: true);
static final _horizontalRule = RegExp(r'^[\s]*[-*_]{3,}[\s]*$', multiLine: true);
static final _htmlTag = RegExp(r'<[^>]+>');
/// Comprehensive emoji pattern for TTS cleanup.
static final _emoji = RegExp(
r'[\u{1F600}-\u{1F64F}]|' // Emoticons
r'[\u{1F300}-\u{1F5FF}]|' // Misc Symbols and Pictographs
r'[\u{1F680}-\u{1F6FF}]|' // Transport and Map
r'[\u{1F1E0}-\u{1F1FF}]|' // Flags
r'[\u{2600}-\u{26FF}]|' // Misc symbols
r'[\u{2700}-\u{27BF}]|' // Dingbats
r'[\u{1F900}-\u{1F9FF}]|' // Supplemental Symbols
r'[\u{1FA00}-\u{1FA6F}]|' // Chess, cards
r'[\u{1FA70}-\u{1FAFF}]|' // Symbols Extended-A
r'[\u{FE00}-\u{FE0F}]|' // Variation Selectors
r'[\u{1F018}-\u{1F270}]|' // Various
r'[\u{238C}-\u{2454}]|' // Misc Technical
r'[\u{20D0}-\u{20FF}]', // Combining Diacritical Marks
unicode: true,
);
static final _whitespace = RegExp(r'\s+');
// ============================================================
// Public API
// ============================================================
/// Normalizes content for Markdown display.
///
/// - Strips link reference definitions (including OpenAI annotations)
/// - Fixes common LLM fence issues
/// - Preserves reasoning blocks for collapsible UI rendering
static String normalize(String input) {
if (input.isEmpty) return input;
var output = input.replaceAll('\r\n', '\n');
// Strip link reference definitions using markdown package
output = _stripLinkReferenceDefinitions(output);
// Fix fence issues
output = _normalizeFences(output);
// Fix Setext heading false positives
output = output.replaceAllMapped(
_labelThenDashRegex,
(match) => '${match[1]}\n\n${match[2]}',
);
// Fix numeric heading parsing
output = output.replaceAllMapped(
_atxEnumRegex,
(match) => '${match[1]}.\u200C${match[2]}${match[3]}',
);
// Separate consecutive links
output = _separateConsecutiveLinks(output);
return output;
}
/// Sanitizes content for clipboard copy or API submission.
///
/// - Strips link reference definitions (including OpenAI annotations)
/// - Strips reasoning/thinking blocks
/// - Normalizes whitespace
static String sanitize(String input) {
if (input.isEmpty) return input;
return input
.replaceAll('\r\n', '\n')
.transform(_stripLinkReferenceDefinitions)
.replaceAll(_reasoningBlocks, '')
.replaceAll(_multipleNewlines, '\n\n')
.trim();
}
/// Converts markdown to plain text for text-to-speech.
static String toPlainText(String input) {
if (input.trim().isEmpty) return '';
return sanitize(input)
.replaceAll(_codeBlock, '') // Remove code blocks
.replaceAllMapped(_inlineCode, (m) => m[1] ?? '') // Keep code text
.replaceAll(_image, '') // Remove images
.replaceAllMapped(_link, (m) => m[1] ?? '') // Keep link text
// Strip paired markdown formatting (preserves lone * and _ in text)
.replaceAllMapped(_boldItalic, (m) => m[1] ?? '')
.replaceAllMapped(_bold, (m) => m[1] ?? '')
.replaceAllMapped(_strikethrough, (m) => m[1] ?? '')
.replaceAllMapped(_italicAsterisk, (m) => ' ${m[1] ?? ''}')
.replaceAllMapped(_italicUnderscore, (m) => ' ${m[1] ?? ''}')
.replaceAll(_heading, '') // Strip # markers
.replaceAll(_listMarker, '') // Strip list markers
.replaceAll(_blockquote, '') // Strip > markers
.replaceAll(_horizontalRule, '') // Remove ---
.replaceAll(_htmlTag, '') // Remove HTML
.transform(_htmlUnescape.convert) // Decode entities
.replaceAll(_emoji, '') // Remove emojis
.replaceAll(_whitespace, ' ') // Normalize whitespace
.trim();
}
/// Breaks long inline code spans for better wrapping.
static String softenInlineCode(String input, {int chunkSize = 24}) {
if (input.length <= chunkSize) return input;
final buffer = StringBuffer();
for (var i = 0; i < input.length; i++) {
buffer.write(input[i]);
if ((i + 1) % chunkSize == 0) {
buffer.write('\u200B');
}
}
return buffer.toString();
}
// ============================================================
// Private Helpers
// ============================================================
static String _normalizeFences(String input) {
var output = input;
// Move fences after list markers to new line
output = output.replaceAllMapped(
_bulletFenceRegex,
(match) => '${match[1]}\n```${match[2]}',
);
// Dedent opening fences
output = output.replaceAllMapped(
_dedentOpenRegex,
(match) => '```${match[1]}',
);
// Dedent closing fences
output = output.replaceAllMapped(_dedentCloseRegex, (_) => '```');
// Ensure closing fences stand alone
output = output.replaceAllMapped(
_inlineClosingRegex,
(match) => '${match[1]}\n```',
);
// Auto-close unmatched fence
final fenceCount = _fenceAtBolRegex.allMatches(output).length;
if (fenceCount.isOdd) {
if (!output.endsWith('\n')) output += '\n';
output += '```';
}
return output;
}
static String _separateConsecutiveLinks(String input) {
final lines = input.split('\n');
if (lines.length <= 1) return input;
final buffer = StringBuffer();
for (var i = 0; i < lines.length; i++) {
final line = lines[i];
buffer.write(line);
if (i < lines.length - 1) buffer.write('\n');
if (_linkWithTrailingSpaces.hasMatch(line)) buffer.write('\n');
}
return buffer.toString();
}
/// Strips link reference definitions using the `markdown` package.
static String _stripLinkReferenceDefinitions(String input) {
if (!input.contains('[')) return input;
final document = md.Document();
document.parseLines(input.split('\n'));
final refLabels = document.linkReferences.keys.toSet();
if (refLabels.isEmpty) return input;
final labelPatterns =
refLabels.map((label) => RegExp.escape(label)).join('|');
final refDefRegex = RegExp(
r'^[ ]{0,3}\[(?:' +
labelPatterns +
r')\]:[ \t]*(?:<[^>]*>|[^\s]*)(?:[ \t]+(?:"[^"]*"|' +
r"'[^']*'" +
r'|\([^)]*\)))?[ \t]*$',
multiLine: true,
caseSensitive: false,
);
return input
.replaceAll(refDefRegex, '')
.replaceAll(_multipleNewlines, '\n\n')
.trim();
}
}
/// Extension for chaining string transformations.
extension _StringTransform on String {
String transform(String Function(String) fn) => fn(this);
}