2 files changed, 258 insertions, 0 deletions
diff --git a/Sources/NorgKit/Lexer/NorgLexer.swift b/Sources/NorgKit/Lexer/NorgLexer.swift
new file mode 100644
index 0000000..ad9d457
--- /dev/null
+++ b/Sources/NorgKit/Lexer/NorgLexer.swift
@@ -0,0 +1,193 @@
+/// Tokenizes Norg source into `NorgToken`s for syntax highlighting.
+public enum NorgLexer {
+
+  /// Tokenizes text for syntax highlighting.
+  public static func tokenize(_ text: String) -> [NorgToken] {
+    var tokens: [NorgToken] = []
+    let lines = TextHelper.lineSlices(text)
+    var i = 0
+
+    while i < lines.count {
+      let raw = lines[i]
+      let trimmed = TextHelper.whitespaceTrimmed(raw)
+
+      if trimmed.isEmpty {
+        i += 1
+        continue
+      }
+
+      if trimmed.first == "@" {
+        i = tagBlock(lines, at: i, into: &tokens)
+        continue
+      }
+
+      if let delimiter = BlockScanner.delimiter(trimmed) {
+        let kind: NorgToken.Kind
+        switch delimiter {
+        case .weak: kind = .weakDelimiter
+        case .strong: kind = .strongDelimiter
+        case .rule: kind = .horizontalRule
+        }
+        tokens.append(NorgToken(kind: kind, range: trimmed.startIndex..<trimmed.endIndex))
+        i += 1
+        continue
+      }
+
+      if let m = DetachedModifier.parse(raw, accepting: BlockScanner.rangeableMarkers) {
+        detachedTokens(m, into: &tokens)
+      } else if let m = DetachedModifier.parse(raw, accepting: BlockScanner.blockMarkers) {
+        detachedTokens(m, into: &tokens)
+      } else {
+        inlineTokens(in: raw, into: &tokens)
+      }
+      i += 1
+    }
+
+    return tokens
+  }
+
+  // MARK: - Block-level tokens
+
+  private static func tagBlock(
+    _ lines: [Substring], at i: Int, into tokens: inout [NorgToken]
+  ) -> Int {
+    tagHeaderTokens(lines[i], into: &tokens)
+
+    let header = TextHelper.whitespaceTrimmed(TextHelper.whitespaceTrimmed(lines[i]).dropFirst())
+    if header == "end" { return i + 1 }
+
+    var j = i + 1
+    while j < lines.count {
+      if TextHelper.whitespaceTrimmed(lines[j]) == "@end" {
+        tagHeaderTokens(lines[j], into: &tokens)
+        return j + 1
+      }
+      let body = lines[j]
+      if body.startIndex < body.endIndex {
+        tokens.append(NorgToken(kind: .verbatimBlock, range: body.startIndex..<body.endIndex))
+      }
+      j += 1
+    }
+    return j
+  }
+
+  private static func tagHeaderTokens(_ raw: Substring, into tokens: inout [NorgToken]) {
+    guard let at = raw.firstIndex(of: "@") else { return }
+    let afterAt = raw.index(after: at)
+    tokens.append(NorgToken(kind: .tagDelimiter, range: at..<afterAt))
+    let header = TextHelper.whitespaceTrimmed(raw[afterAt...])
+    if !header.isEmpty {
+      tokens.append(NorgToken(kind: .tagName, range: header.startIndex..<header.endIndex))
+    }
+  }
+
+  private static func detachedTokens(_ m: DetachedModifier, into tokens: inout [NorgToken]) {
+    if let kind = markerKind(m.marker, level: m.level) {
+      tokens.append(NorgToken(kind: kind, range: m.markerRange))
+    }
+    if let status = m.status, let statusIndex = m.statusIndex {
+      let base = m.content.base
+      let open = base.index(before: statusIndex)
+      let close = base.index(after: base.index(after: statusIndex))
+      tokens.append(NorgToken(kind: .taskStatus(status), range: open..<close))
+    }
+    inlineTokens(in: m.content, into: &tokens)
+  }
+
+  private static func markerKind(_ marker: Character, level: Int) -> NorgToken.Kind? {
+    switch marker {
+    case "*": return .heading(level: level)
+    case "-": return .unorderedList(level: level)
+    case "~": return .orderedList(level: level)
+    case ">": return .quote(level: level)
+    case "$": return .definition(level: level)
+    case "^": return .footnote(level: level)
+    case ":": return .tableCell(level: level)
+    default: return nil
+    }
+  }
+
+  // MARK: - Inline tokens
+
+  private static func inlineTokens(in slice: Substring, into tokens: inout [NorgToken]) {
+    if slice.isEmpty { return }
+    if !slice.utf8.contains(where: InlineScanner.significant.contains) { return }
+
+    let chars = Array(slice.unicodeScalars)
+    var bound = Array(slice.unicodeScalars.indices)
+    bound.append(slice.endIndex)
+
+    func append(_ kind: NorgToken.Kind, _ from: Int, _ to: Int) {
+      tokens.append(NorgToken(kind: kind, range: bound[from]..<bound[to]))
+    }
+
+    func bracketed(_ b: InlineScanner.Bracketed, body kind: NorgToken.Kind) {
+      append(.linkDelimiter, b.open, b.open + 1)
+      if b.body.lowerBound < b.body.upperBound {
+        append(kind, b.body.lowerBound, b.body.upperBound)
+      }
+      append(.linkDelimiter, b.close, b.close + 1)
+    }
+
+    func emit(_ lo: Int, _ hi: Int, style: InlineStyle) {
+      var runStart = -1
+
+      func flushRun(_ upTo: Int) {
+        if runStart >= 0 {
+          if !style.isEmpty { append(.styledText(style), runStart, upTo) }
+          runStart = -1
+        }
+      }
+
+      var i = lo
+      while i < hi {
+        guard let object = InlineScanner.object(in: chars, at: i, to: hi) else {
+          if runStart < 0 { runStart = i }
+          i += 1
+          continue
+        }
+        flushRun(i)
+
+        switch object {
+        case .escape(let escaped, let end):
+          append(.escape, i, i + 1)
+          if let escaped, !style.isEmpty { append(.styledText(style), escaped, end) }
+
+        case .comment(let open, _, _, let end):
+          append(.comment, open, end)
+
+        case .verbatim(let vstyle, let open, let body, let close, _):
+          let s = style.union(vstyle)
+          append(.modifierDelimiter(s), open, open + 1)
+          if body.lowerBound < body.upperBound {
+            append(.styledText(s), body.lowerBound, body.upperBound)
+          }
+          append(.modifierDelimiter(s), close, close + 1)
+
+        case .modifier(let mstyle, let open, let body, let close, _):
+          let s = style.union(mstyle)
+          append(.modifierDelimiter(s), open, open + 1)
+          emit(body.lowerBound, body.upperBound, style: s)
+          append(.modifierDelimiter(s), close, close + 1)
+
+        case .link(_, let target, let description, _):
+          bracketed(target, body: .linkTarget)
+          if let description { bracketed(description, body: .linkDescription) }
+
+        case .anchor(_, let name, let suffix, _):
+          bracketed(name, body: .linkDescription)
+          switch suffix {
+          case .target(let b): bracketed(b, body: .linkTarget)
+          case .description(let b): bracketed(b, body: .linkDescription)
+          case nil: break
+          }
+        }
+
+        i = object.end
+      }
+      flushRun(hi)
+    }
+
+    emit(0, chars.count, style: [])
+  }
+}
diff --git a/Sources/NorgKit/Lexer/NorgToken.swift b/Sources/NorgKit/Lexer/NorgToken.swift
new file mode 100644
index 0000000..050c37e
--- /dev/null
+++ b/Sources/NorgKit/Lexer/NorgToken.swift
@@ -0,0 +1,65 @@
+/// A lexical token over Norg source.
+public struct NorgToken: Equatable, Sendable {
+
+  /// What a token represents.
+  public enum Kind: Equatable, Sendable {
+
+    // MARK: Block / line level
+
+    /// A heading marker run (`*`…), carrying its level.
+    case heading(level: Int)
+    /// An unordered-list marker run (`-`…).
+    case unorderedList(level: Int)
+    /// An ordered-list marker run (`~`…).
+    case orderedList(level: Int)
+    /// A quote marker run (`>`…).
+    case quote(level: Int)
+    /// A definition marker run (`$`…).
+    case definition(level: Int)
+    /// A footnote marker run (`^`…).
+    case footnote(level: Int)
+    /// A table-cell marker run (`:`…).
+    case tableCell(level: Int)
+    /// A task status marker including its parentheses, eg. `(x)`.
+    case taskStatus(TaskStatus)
+    /// A weak delimiting line (`---`).
+    case weakDelimiter
+    /// A strong delimiting line (`===`).
+    case strongDelimiter
+    /// A horizontal rule (`___`).
+    case horizontalRule
+    /// A ranged-tag fence marker: the `@` of an opener and of `@end`.
+    case tagDelimiter
+    /// A ranged-tag header after `@` (name and parameters), eg. `code swift`.
+    case tagName
+    /// A raw body line inside a ranged tag (`@code` … `@end`).
+    case verbatimBlock
+
+    // MARK: Inline level
+
+    /// An attached- or verbatim-modifier delimiter (`*`, `/`, `` ` ``, `$`, …);
+    /// the style identifies which.
+    case modifierDelimiter(InlineStyle)
+    /// A run of text carrying a non-empty cumulative style (the content of one
+    /// or more nested modifiers).
+    case styledText(InlineStyle)
+    /// An inline comment, markers included (`%…%`).
+    case comment
+    /// An escape: the backslash of `\x` (the escaped character is not a token).
+    case escape
+    /// A link/anchor delimiter: `{`, `}`, `[`, or `]`.
+    case linkDelimiter
+    /// A link/anchor location (inside `{…}`).
+    case linkTarget
+    /// A link/anchor description or label (inside `[…]`).
+    case linkDescription
+  }
+
+  public let kind: Kind
+  public let range: Range<String.Index>
+
+  public init(kind: Kind, range: Range<String.Index>) {
+    self.kind = kind
+    self.range = range
+  }
+}