diff options
Diffstat (limited to 'Sources/NorgKit/Lexer/NorgLexer.swift')
| -rw-r--r-- | Sources/NorgKit/Lexer/NorgLexer.swift | 193 |
1 files changed, 193 insertions, 0 deletions
diff --git a/Sources/NorgKit/Lexer/NorgLexer.swift b/Sources/NorgKit/Lexer/NorgLexer.swift new file mode 100644 index 0000000..ad9d457 --- /dev/null +++ b/Sources/NorgKit/Lexer/NorgLexer.swift @@ -0,0 +1,193 @@ +/// Tokenizes Norg source into `NorgToken`s for syntax highlighting. +public enum NorgLexer { + + /// Tokenizes text for syntax highlighting. + public static func tokenize(_ text: String) -> [NorgToken] { + var tokens: [NorgToken] = [] + let lines = TextHelper.lineSlices(text) + var i = 0 + + while i < lines.count { + let raw = lines[i] + let trimmed = TextHelper.whitespaceTrimmed(raw) + + if trimmed.isEmpty { + i += 1 + continue + } + + if trimmed.first == "@" { + i = tagBlock(lines, at: i, into: &tokens) + continue + } + + if let delimiter = BlockScanner.delimiter(trimmed) { + let kind: NorgToken.Kind + switch delimiter { + case .weak: kind = .weakDelimiter + case .strong: kind = .strongDelimiter + case .rule: kind = .horizontalRule + } + tokens.append(NorgToken(kind: kind, range: trimmed.startIndex..<trimmed.endIndex)) + i += 1 + continue + } + + if let m = DetachedModifier.parse(raw, accepting: BlockScanner.rangeableMarkers) { + detachedTokens(m, into: &tokens) + } else if let m = DetachedModifier.parse(raw, accepting: BlockScanner.blockMarkers) { + detachedTokens(m, into: &tokens) + } else { + inlineTokens(in: raw, into: &tokens) + } + i += 1 + } + + return tokens + } + + // MARK: - Block-level tokens + + private static func tagBlock( + _ lines: [Substring], at i: Int, into tokens: inout [NorgToken] + ) -> Int { + tagHeaderTokens(lines[i], into: &tokens) + + let header = TextHelper.whitespaceTrimmed(TextHelper.whitespaceTrimmed(lines[i]).dropFirst()) + if header == "end" { return i + 1 } + + var j = i + 1 + while j < lines.count { + if TextHelper.whitespaceTrimmed(lines[j]) == "@end" { + tagHeaderTokens(lines[j], into: &tokens) + return j + 1 + } + let body = lines[j] + if body.startIndex < body.endIndex { + tokens.append(NorgToken(kind: .verbatimBlock, range: body.startIndex..<body.endIndex)) + } + j += 1 + } + return j + } + + private static func tagHeaderTokens(_ raw: Substring, into tokens: inout [NorgToken]) { + guard let at = raw.firstIndex(of: "@") else { return } + let afterAt = raw.index(after: at) + tokens.append(NorgToken(kind: .tagDelimiter, range: at..<afterAt)) + let header = TextHelper.whitespaceTrimmed(raw[afterAt...]) + if !header.isEmpty { + tokens.append(NorgToken(kind: .tagName, range: header.startIndex..<header.endIndex)) + } + } + + private static func detachedTokens(_ m: DetachedModifier, into tokens: inout [NorgToken]) { + if let kind = markerKind(m.marker, level: m.level) { + tokens.append(NorgToken(kind: kind, range: m.markerRange)) + } + if let status = m.status, let statusIndex = m.statusIndex { + let base = m.content.base + let open = base.index(before: statusIndex) + let close = base.index(after: base.index(after: statusIndex)) + tokens.append(NorgToken(kind: .taskStatus(status), range: open..<close)) + } + inlineTokens(in: m.content, into: &tokens) + } + + private static func markerKind(_ marker: Character, level: Int) -> NorgToken.Kind? { + switch marker { + case "*": return .heading(level: level) + case "-": return .unorderedList(level: level) + case "~": return .orderedList(level: level) + case ">": return .quote(level: level) + case "$": return .definition(level: level) + case "^": return .footnote(level: level) + case ":": return .tableCell(level: level) + default: return nil + } + } + + // MARK: - Inline tokens + + private static func inlineTokens(in slice: Substring, into tokens: inout [NorgToken]) { + if slice.isEmpty { return } + if !slice.utf8.contains(where: InlineScanner.significant.contains) { return } + + let chars = Array(slice.unicodeScalars) + var bound = Array(slice.unicodeScalars.indices) + bound.append(slice.endIndex) + + func append(_ kind: NorgToken.Kind, _ from: Int, _ to: Int) { + tokens.append(NorgToken(kind: kind, range: bound[from]..<bound[to])) + } + + func bracketed(_ b: InlineScanner.Bracketed, body kind: NorgToken.Kind) { + append(.linkDelimiter, b.open, b.open + 1) + if b.body.lowerBound < b.body.upperBound { + append(kind, b.body.lowerBound, b.body.upperBound) + } + append(.linkDelimiter, b.close, b.close + 1) + } + + func emit(_ lo: Int, _ hi: Int, style: InlineStyle) { + var runStart = -1 + + func flushRun(_ upTo: Int) { + if runStart >= 0 { + if !style.isEmpty { append(.styledText(style), runStart, upTo) } + runStart = -1 + } + } + + var i = lo + while i < hi { + guard let object = InlineScanner.object(in: chars, at: i, to: hi) else { + if runStart < 0 { runStart = i } + i += 1 + continue + } + flushRun(i) + + switch object { + case .escape(let escaped, let end): + append(.escape, i, i + 1) + if let escaped, !style.isEmpty { append(.styledText(style), escaped, end) } + + case .comment(let open, _, _, let end): + append(.comment, open, end) + + case .verbatim(let vstyle, let open, let body, let close, _): + let s = style.union(vstyle) + append(.modifierDelimiter(s), open, open + 1) + if body.lowerBound < body.upperBound { + append(.styledText(s), body.lowerBound, body.upperBound) + } + append(.modifierDelimiter(s), close, close + 1) + + case .modifier(let mstyle, let open, let body, let close, _): + let s = style.union(mstyle) + append(.modifierDelimiter(s), open, open + 1) + emit(body.lowerBound, body.upperBound, style: s) + append(.modifierDelimiter(s), close, close + 1) + + case .link(_, let target, let description, _): + bracketed(target, body: .linkTarget) + if let description { bracketed(description, body: .linkDescription) } + + case .anchor(_, let name, let suffix, _): + bracketed(name, body: .linkDescription) + switch suffix { + case .target(let b): bracketed(b, body: .linkTarget) + case .description(let b): bracketed(b, body: .linkDescription) + case nil: break + } + } + + i = object.end + } + flushRun(hi) + } + + emit(0, chars.count, style: []) + } +} |