Add NorgLexerHEAD1.1.0main

10 files changed, 779 insertions, 194 deletions

diff --git a/.gitignore b/.gitignore
index d2c5c89..9804632 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ DerivedData/
 .swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
 .netrc
 *.profraw
+.benchmarkBaselines
diff --git a/Benchmarks/ParseBenchmarkTarget/ParseBenchmarkTarget.swift b/Benchmarks/ParseBenchmarkTarget/ParseBenchmarkTarget.swift
index 91e1d38..a5346f2 100644
--- a/Benchmarks/ParseBenchmarkTarget/ParseBenchmarkTarget.swift
+++ b/Benchmarks/ParseBenchmarkTarget/ParseBenchmarkTarget.swift
@@ -84,9 +84,20 @@ let benchmarks: @Sendable () -> Benchmark? = {
     }
   }
 
+  // Source tokenization for syntax highlighting.
+  Benchmark("lex example") { benchmark in
+    for _ in benchmark.scaledIterations {
+      blackHole(NorgLexer.tokenize(exampleSource))
+    }
+  }
+
+  Benchmark("lex huge") { benchmark in
+    for _ in benchmark.scaledIterations {
+      blackHole(NorgLexer.tokenize(hugeSource))
+    }
+  }
+
   // Rewriting a single status marker — the write-back path for toggling a task.
-  // Line 17 (zero-based) is the first task in huge.norg, so this exercises the
-  // full split / replace / re-join path rather than bailing out early.
   return Benchmark("update task status huge") { benchmark in
     for _ in benchmark.scaledIterations {
       blackHole(TaskScanner.updatedContent(hugeSource, line: 17, to: .done))
diff --git a/Sources/NorgKit/Lexer/NorgLexer.swift b/Sources/NorgKit/Lexer/NorgLexer.swift
new file mode 100644
index 0000000..ad9d457
--- /dev/null
+++ b/Sources/NorgKit/Lexer/NorgLexer.swift
@@ -0,0 +1,193 @@
+/// Tokenizes Norg source into `NorgToken`s for syntax highlighting.
+public enum NorgLexer {
+
+  /// Tokenizes text for syntax highlighting.
+  public static func tokenize(_ text: String) -> [NorgToken] {
+    var tokens: [NorgToken] = []
+    let lines = TextHelper.lineSlices(text)
+    var i = 0
+
+    while i < lines.count {
+      let raw = lines[i]
+      let trimmed = TextHelper.whitespaceTrimmed(raw)
+
+      if trimmed.isEmpty {
+        i += 1
+        continue
+      }
+
+      if trimmed.first == "@" {
+        i = tagBlock(lines, at: i, into: &tokens)
+        continue
+      }
+
+      if let delimiter = BlockScanner.delimiter(trimmed) {
+        let kind: NorgToken.Kind
+        switch delimiter {
+        case .weak: kind = .weakDelimiter
+        case .strong: kind = .strongDelimiter
+        case .rule: kind = .horizontalRule
+        }
+        tokens.append(NorgToken(kind: kind, range: trimmed.startIndex..<trimmed.endIndex))
+        i += 1
+        continue
+      }
+
+      if let m = DetachedModifier.parse(raw, accepting: BlockScanner.rangeableMarkers) {
+        detachedTokens(m, into: &tokens)
+      } else if let m = DetachedModifier.parse(raw, accepting: BlockScanner.blockMarkers) {
+        detachedTokens(m, into: &tokens)
+      } else {
+        inlineTokens(in: raw, into: &tokens)
+      }
+      i += 1
+    }
+
+    return tokens
+  }
+
+  // MARK: - Block-level tokens
+
+  private static func tagBlock(
+    _ lines: [Substring], at i: Int, into tokens: inout [NorgToken]
+  ) -> Int {
+    tagHeaderTokens(lines[i], into: &tokens)
+
+    let header = TextHelper.whitespaceTrimmed(TextHelper.whitespaceTrimmed(lines[i]).dropFirst())
+    if header == "end" { return i + 1 }
+
+    var j = i + 1
+    while j < lines.count {
+      if TextHelper.whitespaceTrimmed(lines[j]) == "@end" {
+        tagHeaderTokens(lines[j], into: &tokens)
+        return j + 1
+      }
+      let body = lines[j]
+      if body.startIndex < body.endIndex {
+        tokens.append(NorgToken(kind: .verbatimBlock, range: body.startIndex..<body.endIndex))
+      }
+      j += 1
+    }
+    return j
+  }
+
+  private static func tagHeaderTokens(_ raw: Substring, into tokens: inout [NorgToken]) {
+    guard let at = raw.firstIndex(of: "@") else { return }
+    let afterAt = raw.index(after: at)
+    tokens.append(NorgToken(kind: .tagDelimiter, range: at..<afterAt))
+    let header = TextHelper.whitespaceTrimmed(raw[afterAt...])
+    if !header.isEmpty {
+      tokens.append(NorgToken(kind: .tagName, range: header.startIndex..<header.endIndex))
+    }
+  }
+
+  private static func detachedTokens(_ m: DetachedModifier, into tokens: inout [NorgToken]) {
+    if let kind = markerKind(m.marker, level: m.level) {
+      tokens.append(NorgToken(kind: kind, range: m.markerRange))
+    }
+    if let status = m.status, let statusIndex = m.statusIndex {
+      let base = m.content.base
+      let open = base.index(before: statusIndex)
+      let close = base.index(after: base.index(after: statusIndex))
+      tokens.append(NorgToken(kind: .taskStatus(status), range: open..<close))
+    }
+    inlineTokens(in: m.content, into: &tokens)
+  }
+
+  private static func markerKind(_ marker: Character, level: Int) -> NorgToken.Kind? {
+    switch marker {
+    case "*": return .heading(level: level)
+    case "-": return .unorderedList(level: level)
+    case "~": return .orderedList(level: level)
+    case ">": return .quote(level: level)
+    case "$": return .definition(level: level)
+    case "^": return .footnote(level: level)
+    case ":": return .tableCell(level: level)
+    default: return nil
+    }
+  }
+
+  // MARK: - Inline tokens
+
+  private static func inlineTokens(in slice: Substring, into tokens: inout [NorgToken]) {
+    if slice.isEmpty { return }
+    if !slice.utf8.contains(where: InlineScanner.significant.contains) { return }
+
+    let chars = Array(slice.unicodeScalars)
+    var bound = Array(slice.unicodeScalars.indices)
+    bound.append(slice.endIndex)
+
+    func append(_ kind: NorgToken.Kind, _ from: Int, _ to: Int) {
+      tokens.append(NorgToken(kind: kind, range: bound[from]..<bound[to]))
+    }
+
+    func bracketed(_ b: InlineScanner.Bracketed, body kind: NorgToken.Kind) {
+      append(.linkDelimiter, b.open, b.open + 1)
+      if b.body.lowerBound < b.body.upperBound {
+        append(kind, b.body.lowerBound, b.body.upperBound)
+      }
+      append(.linkDelimiter, b.close, b.close + 1)
+    }
+
+    func emit(_ lo: Int, _ hi: Int, style: InlineStyle) {
+      var runStart = -1
+
+      func flushRun(_ upTo: Int) {
+        if runStart >= 0 {
+          if !style.isEmpty { append(.styledText(style), runStart, upTo) }
+          runStart = -1
+        }
+      }
+
+      var i = lo
+      while i < hi {
+        guard let object = InlineScanner.object(in: chars, at: i, to: hi) else {
+          if runStart < 0 { runStart = i }
+          i += 1
+          continue
+        }
+        flushRun(i)
+
+        switch object {
+        case .escape(let escaped, let end):
+          append(.escape, i, i + 1)
+          if let escaped, !style.isEmpty { append(.styledText(style), escaped, end) }
+
+        case .comment(let open, _, _, let end):
+          append(.comment, open, end)
+
+        case .verbatim(let vstyle, let open, let body, let close, _):
+          let s = style.union(vstyle)
+          append(.modifierDelimiter(s), open, open + 1)
+          if body.lowerBound < body.upperBound {
+            append(.styledText(s), body.lowerBound, body.upperBound)
+          }
+          append(.modifierDelimiter(s), close, close + 1)
+
+        case .modifier(let mstyle, let open, let body, let close, _):
+          let s = style.union(mstyle)
+          append(.modifierDelimiter(s), open, open + 1)
+          emit(body.lowerBound, body.upperBound, style: s)
+          append(.modifierDelimiter(s), close, close + 1)
+
+        case .link(_, let target, let description, _):
+          bracketed(target, body: .linkTarget)
+          if let description { bracketed(description, body: .linkDescription) }
+
+        case .anchor(_, let name, let suffix, _):
+          bracketed(name, body: .linkDescription)
+          switch suffix {
+          case .target(let b): bracketed(b, body: .linkTarget)
+          case .description(let b): bracketed(b, body: .linkDescription)
+          case nil: break
+          }
+        }
+
+        i = object.end
+      }
+      flushRun(hi)
+    }
+
+    emit(0, chars.count, style: [])
+  }
+}
diff --git a/Sources/NorgKit/Lexer/NorgToken.swift b/Sources/NorgKit/Lexer/NorgToken.swift
new file mode 100644
index 0000000..050c37e
--- /dev/null
+++ b/Sources/NorgKit/Lexer/NorgToken.swift
@@ -0,0 +1,65 @@
+/// A lexical token over Norg source.
+public struct NorgToken: Equatable, Sendable {
+
+  /// What a token represents.
+  public enum Kind: Equatable, Sendable {
+
+    // MARK: Block / line level
+
+    /// A heading marker run (`*`…), carrying its level.
+    case heading(level: Int)
+    /// An unordered-list marker run (`-`…).
+    case unorderedList(level: Int)
+    /// An ordered-list marker run (`~`…).
+    case orderedList(level: Int)
+    /// A quote marker run (`>`…).
+    case quote(level: Int)
+    /// A definition marker run (`$`…).
+    case definition(level: Int)
+    /// A footnote marker run (`^`…).
+    case footnote(level: Int)
+    /// A table-cell marker run (`:`…).
+    case tableCell(level: Int)
+    /// A task status marker including its parentheses, eg. `(x)`.
+    case taskStatus(TaskStatus)
+    /// A weak delimiting line (`---`).
+    case weakDelimiter
+    /// A strong delimiting line (`===`).
+    case strongDelimiter
+    /// A horizontal rule (`___`).
+    case horizontalRule
+    /// A ranged-tag fence marker: the `@` of an opener and of `@end`.
+    case tagDelimiter
+    /// A ranged-tag header after `@` (name and parameters), eg. `code swift`.
+    case tagName
+    /// A raw body line inside a ranged tag (`@code` … `@end`).
+    case verbatimBlock
+
+    // MARK: Inline level
+
+    /// An attached- or verbatim-modifier delimiter (`*`, `/`, `` ` ``, `$`, …);
+    /// the style identifies which.
+    case modifierDelimiter(InlineStyle)
+    /// A run of text carrying a non-empty cumulative style (the content of one
+    /// or more nested modifiers).
+    case styledText(InlineStyle)
+    /// An inline comment, markers included (`%…%`).
+    case comment
+    /// An escape: the backslash of `\x` (the escaped character is not a token).
+    case escape
+    /// A link/anchor delimiter: `{`, `}`, `[`, or `]`.
+    case linkDelimiter
+    /// A link/anchor location (inside `{…}`).
+    case linkTarget
+    /// A link/anchor description or label (inside `[…]`).
+    case linkDescription
+  }
+
+  public let kind: Kind
+  public let range: Range<String.Index>
+
+  public init(kind: Kind, range: Range<String.Index>) {
+    self.kind = kind
+    self.range = range
+  }
+}
diff --git a/Sources/NorgKit/Models/DetachedModifier.swift b/Sources/NorgKit/Models/DetachedModifier.swift
index 1f28c36..51d53ff 100644
--- a/Sources/NorgKit/Models/DetachedModifier.swift
+++ b/Sources/NorgKit/Models/DetachedModifier.swift
@@ -6,6 +6,7 @@ struct DetachedModifier {
   let status: TaskStatus?
   let statusIndex: String.Index?
   let content: Substring
+  let markerRange: Range<String.Index>
 
   /// Finds and parses a detached modifier if present. The accepting markers
   /// is used to reduce the markers being parsed.
@@ -21,12 +22,14 @@ struct DetachedModifier {
     let markerByte = utf8[i]
     guard markers.contains(markerByte) else { return nil }
 
+    let markerStart = i
     var level = 0
     var run = i
     while run < end, utf8[run] == markerByte {
       level += 1
       run = utf8.index(after: run)
     }
+    let markerRange = markerStart..<run
 
     guard run < end, ASCIIHelper.isWhitespace(utf8[run]) else { return nil }
 
@@ -55,7 +58,8 @@ struct DetachedModifier {
       level: level,
       status: status,
       statusIndex: statusIndex,
-      content: TextHelper.whitespaceTrimmed(line[contentStart...])
+      content: TextHelper.whitespaceTrimmed(line[contentStart...]),
+      markerRange: markerRange
     )
   }
 }
diff --git a/Sources/NorgKit/Parsers/BlockScanner.swift b/Sources/NorgKit/Parsers/BlockScanner.swift
new file mode 100644
index 0000000..7552255
--- /dev/null
+++ b/Sources/NorgKit/Parsers/BlockScanner.swift
@@ -0,0 +1,21 @@
+/// Shared block-level lexical primitives.
+enum BlockScanner {
+
+  static let blockMarkers = ASCIIByteSet("*-~>")
+  static let rangeableMarkers = ASCIIByteSet("$^:")
+
+  enum Delimiter {
+    case weak, strong, rule
+  }
+
+  static func delimiter(_ s: Substring) -> Delimiter? {
+    guard s.count >= 2, let first = s.first, "-=_".contains(first),
+      s.allSatisfy({ $0 == first })
+    else { return nil }
+    switch first {
+    case "-": return .weak
+    case "=": return .strong
+    default: return .rule
+    }
+  }
+}
diff --git a/Sources/NorgKit/Parsers/InlineScanner.swift b/Sources/NorgKit/Parsers/InlineScanner.swift
new file mode 100644
index 0000000..881a069
--- /dev/null
+++ b/Sources/NorgKit/Parsers/InlineScanner.swift
@@ -0,0 +1,187 @@
+/// Scans for inline norg constructs.
+enum InlineScanner {
+
+  struct Bracketed: Equatable {
+    let open: Int
+    let body: Range<Int>
+    let close: Int
+  }
+
+  enum AnchorSuffix: Equatable {
+    case target(Bracketed)
+    case description(Bracketed)
+  }
+
+  enum InlineObject: Equatable {
+    /// `\x` — `escaped` is the index of the literal scalar, or `nil` for a
+    /// trailing backslash at the end of the range.
+    case escape(escaped: Int?, end: Int)
+    /// `%…%`, dropped from rendered output.
+    case comment(open: Int, body: Range<Int>, close: Int, end: Int)
+    /// Verbatim / math: `` `…` `` or `$…$`, inner content taken literally.
+    case verbatim(style: InlineStyle, open: Int, body: Range<Int>, close: Int, end: Int)
+    /// An attached modifier (`*…*`, `/…/`, …) whose body is parsed recursively.
+    case modifier(style: InlineStyle, open: Int, body: Range<Int>, close: Int, end: Int)
+    /// `{target}` optionally followed by `[description]`.
+    case link(open: Int, target: Bracketed, description: Bracketed?, end: Int)
+    /// `[name]` optionally followed by `{target}` or `[description]`.
+    case anchor(open: Int, name: Bracketed, suffix: AnchorSuffix?, end: Int)
+
+    var end: Int {
+      switch self {
+      case .escape(_, let end), .comment(_, _, _, let end), .verbatim(_, _, _, _, let end),
+        .modifier(_, _, _, _, let end), .link(_, _, _, let end), .anchor(_, _, _, let end):
+        return end
+      }
+    }
+  }
+
+  static let modifiers: [Unicode.Scalar: InlineStyle] = [
+    "*": .bold,
+    "/": .italic,
+    "_": .underline,
+    "-": .strikethrough,
+    "^": .superscript,
+    ",": .subscript,
+    "!": .spoiler,
+  ]
+
+  static let literalModifiers: [Unicode.Scalar: InlineStyle] = [
+    "`": .verbatim,
+    "$": .math,
+  ]
+
+  static let significant = ASCIIByteSet("*/_-^,!\u{60}$%{[\\")
+
+  static func object(in chars: [Unicode.Scalar], at i: Int, to hi: Int) -> InlineObject? {
+    let c = chars[i]
+    let prev: Unicode.Scalar? = i > 0 ? chars[i - 1] : nil
+    let next: Unicode.Scalar? = i + 1 < chars.count ? chars[i + 1] : nil
+
+    // Escapes: the next scalar is taken literally.
+    if c == "\\" {
+      return i + 1 < hi ? .escape(escaped: i + 1, end: i + 2) : .escape(escaped: nil, end: i + 1)
+    }
+
+    // Comments are dropped from the rendered output.
+    if c == "%", isOpener(prev: prev, next: next),
+      let close = literalClose(chars, from: i + 1, to: hi, char: "%") {
+      return .comment(open: i, body: (i + 1)..<close, close: close, end: close + 1)
+    }
+
+    // Verbatim / math objects: literal inner content.
+    if let style = literalModifiers[c], isOpener(prev: prev, next: next),
+      let close = literalClose(chars, from: i + 1, to: hi, char: c) {
+      return .verbatim(style: style, open: i, body: (i + 1)..<close, close: close, end: close + 1)
+    }
+
+    // Links: {location} optionally followed by [description].
+    if c == "{", let close = bracketClose(chars, from: i + 1, to: hi, char: "}") {
+      let target = Bracketed(open: i, body: (i + 1)..<close, close: close)
+      var j = close + 1
+      var description: Bracketed?
+      if j < hi, chars[j] == "[", let dclose = bracketClose(chars, from: j + 1, to: hi, char: "]") {
+        description = Bracketed(open: j, body: (j + 1)..<dclose, close: dclose)
+        j = dclose + 1
+      }
+      return .link(open: i, target: target, description: description, end: j)
+    }
+
+    // Anchors: [name] (declaration), [name]{location} (definition), or
+    // [name][description] (declaration with a custom description).
+    if c == "[", let close = bracketClose(chars, from: i + 1, to: hi, char: "]") {
+      let name = Bracketed(open: i, body: (i + 1)..<close, close: close)
+      var j = close + 1
+      var suffix: AnchorSuffix?
+      if j < hi, chars[j] == "{", let tclose = bracketClose(chars, from: j + 1, to: hi, char: "}") {
+        suffix = .target(Bracketed(open: j, body: (j + 1)..<tclose, close: tclose))
+        j = tclose + 1
+      } else if j < hi, chars[j] == "[",
+        let dclose = bracketClose(chars, from: j + 1, to: hi, char: "]") {
+        suffix = .description(Bracketed(open: j, body: (j + 1)..<dclose, close: dclose))
+        j = dclose + 1
+      }
+      return .anchor(open: i, name: name, suffix: suffix, end: j)
+    }
+
+    // Attached modifiers with recursively parsed content.
+    if let style = modifiers[c], isOpener(prev: prev, next: next),
+      let close = modifierClose(chars, from: i + 1, to: hi, char: c) {
+      return .modifier(style: style, open: i, body: (i + 1)..<close, close: close, end: close + 1)
+    }
+
+    return nil
+  }
+
+  // MARK: - Boundary helpers
+
+  private static func isSpace(_ c: Unicode.Scalar?) -> Bool {
+    guard let c else { return true }
+    return c.properties.isWhitespace
+  }
+
+  private static func isBoundary(_ c: Unicode.Scalar?) -> Bool {
+    guard let c else { return true }
+    if c.properties.isWhitespace { return true }
+    switch c.properties.generalCategory {
+    case .connectorPunctuation, .dashPunctuation, .openPunctuation,
+      .closePunctuation, .initialPunctuation, .finalPunctuation, .otherPunctuation,
+      .mathSymbol, .currencySymbol, .modifierSymbol, .otherSymbol:
+      return true
+    default:
+      return false
+    }
+  }
+
+  private static func isOpener(prev: Unicode.Scalar?, next: Unicode.Scalar?) -> Bool {
+    isBoundary(prev) && !isSpace(next)
+  }
+
+  private static func modifierClose(
+    _ chars: [Unicode.Scalar], from start: Int, to hi: Int, char: Unicode.Scalar
+  ) -> Int? {
+    var i = start
+    while i < hi {
+      if chars[i] == "\\" {
+        i += 2
+        continue
+      }
+      if chars[i] == char {
+        let prev: Unicode.Scalar? = i > 0 ? chars[i - 1] : nil
+        let next: Unicode.Scalar? = i + 1 < chars.count ? chars[i + 1] : nil
+        if !isSpace(prev) && isBoundary(next) { return i }
+      }
+      i += 1
+    }
+    return nil
+  }
+
+  private static func literalClose(
+    _ chars: [Unicode.Scalar], from start: Int, to hi: Int, char: Unicode.Scalar
+  ) -> Int? {
+    var i = start
+    while i < hi {
+      if chars[i] == char {
+        let prev: Unicode.Scalar? = i > 0 ? chars[i - 1] : nil
+        if !isSpace(prev) { return i }
+      }
+      i += 1
+    }
+    return nil
+  }
+
+  private static func bracketClose(
+    _ chars: [Unicode.Scalar], from start: Int, to hi: Int, char: Unicode.Scalar
+  ) -> Int? {
+    var i = start
+    while i < hi {
+      if chars[i] == "\\" {
+        i += 2
+        continue
+      }
+      if chars[i] == char { return i }
+      i += 1
+    }
+    return nil
+  }
+}
diff --git a/Sources/NorgKit/Parsers/NorgInlineParser.swift b/Sources/NorgKit/Parsers/NorgInlineParser.swift
index 1f19c43..b47ec50 100644
--- a/Sources/NorgKit/Parsers/NorgInlineParser.swift
+++ b/Sources/NorgKit/Parsers/NorgInlineParser.swift
@@ -3,31 +3,10 @@ import Foundation
 /// Converts inline Norg markup into a list of `InlineSpan`s.
 public enum NorgInlineParser {
 
-  /// Attached modifiers whose content is parsed recursively for nesting.
-  private static let modifiers: [Unicode.Scalar: InlineStyle] = [
-    "*": .bold,
-    "/": .italic,
-    "_": .underline,
-    "-": .strikethrough,
-    "^": .superscript,
-    ",": .subscript,
-    "!": .spoiler,
-  ]
-
-  /// Modifiers whose content is taken verbatim.
-  private static let literalModifiers: [Unicode.Scalar: InlineStyle] = [
-    "`": .verbatim,
-    "$": .math,
-  ]
-
-  /// Bytes that can begin an inline object. If none are found, it's plain
-  /// text.
-  private static let significant = ASCIIByteSet("*/_-^,!\u{60}$%{[\\")
-
   /// Parses a string to a list of `InlineSpan`s
   public static func parse(_ text: String) -> [InlineSpan] {
     if text.isEmpty { return [] }
-    if !text.utf8.contains(where: significant.contains) {
+    if !text.utf8.contains(where: InlineScanner.significant.contains) {
       return [InlineSpan(text: text, styles: [])]
     }
 
@@ -38,7 +17,7 @@ public enum NorgInlineParser {
   /// Renders the text as plain text, without styling or markup.
   static func plainText(_ text: Substring) -> String {
     if text.isEmpty { return "" }
-    if !text.utf8.contains(where: significant.contains) {
+    if !text.utf8.contains(where: InlineScanner.significant.contains) {
       return String(text)
     }
 
@@ -59,179 +38,68 @@ public enum NorgInlineParser {
 
     var i = lo
     while i < hi {
-      let c = chars[i]
-      // `prev`/`next` deliberately peek outside `[lo, hi)`: within a nested
-      // range the enclosing modifier (e.g. the `_` around `_/x/_`) is a
-      // valid boundary, so boundary detection uses the whole line.
-      let prev: Unicode.Scalar? = i > 0 ? chars[i - 1] : nil
-      let next: Unicode.Scalar? = i + 1 < chars.count ? chars[i + 1] : nil
-
-      // Escapes: the next scalar is taken literally.
-      if c == "\\" {
-        if i + 1 < hi {
-          buffer.append(chars[i + 1])
-          i += 2
-        } else {
-          i += 1
-        }
+      guard let object = InlineScanner.object(in: chars, at: i, to: hi) else {
+        buffer.append(chars[i])
+        i += 1
         continue
       }
 
-      // Comments are dropped from the rendered output.
-      if c == "%", isOpener(prev: prev, next: next),
-        let close = literalClose(chars, from: i + 1, to: hi, char: "%") {
+      switch object {
+      case .escape(let escaped, _):
+        if let escaped { buffer.append(chars[escaped]) }
+
+      case .comment:
         flush()
-        i = close + 1
-        continue
-      }
 
-      // Verbatim / math objects: literal inner content.
-      if let style = literalModifiers[c], isOpener(prev: prev, next: next),
-        let close = literalClose(chars, from: i + 1, to: hi, char: c) {
+      case .verbatim(let style, _, let body, _, _):
         flush()
-        spans.append(InlineSpan(text: slice(chars, i + 1, close), styles: base.union(style)))
-        i = close + 1
-        continue
-      }
+        spans.append(InlineSpan(text: slice(chars, body), styles: base.union(style)))
 
-      // Links: {location} optionally followed by [description].
-      if c == "{", let close = bracketClose(chars, from: i + 1, to: hi, char: "}") {
-        let target = slice(chars, i + 1, close)
-        var j = close + 1
-        var label = linkLabel(target)
-        if j < hi, chars[j] == "[", let dclose = bracketClose(chars, from: j + 1, to: hi, char: "]") {
-          // An explicit description replaces the derived label verbatim.
-          label = slice(chars, j + 1, dclose)
-          j = dclose + 1
-        }
+      case .modifier(let style, _, let body, _, _):
         flush()
         spans.append(
-          InlineSpan(text: label, styles: base, link: InlineLink(kind: .link, target: target)))
-        i = j
-        continue
-      }
+          contentsOf: parse(
+            chars, from: body.lowerBound, to: body.upperBound, base: base.union(style)))
 
-      // Anchors: [name] (declaration), [name]{location} (definition), or
-      // [name][description] (declaration with a custom description).
-      if c == "[", let close = bracketClose(chars, from: i + 1, to: hi, char: "]") {
-        let name = slice(chars, i + 1, close)
-        var j = close + 1
-        var label = name
-        var target: String?
-        if j < hi, chars[j] == "{", let tclose = bracketClose(chars, from: j + 1, to: hi, char: "}") {
-          target = slice(chars, j + 1, tclose)
-          j = tclose + 1
-        } else if j < hi, chars[j] == "[",
-          let dclose = bracketClose(chars, from: j + 1, to: hi, char: "]") {
-          label = slice(chars, j + 1, dclose)
-          j = dclose + 1
-        }
+      case .link(_, let target, let description, _):
         flush()
+        let targetText = slice(chars, target.body)
+        let label = description.map { slice(chars, $0.body) } ?? linkLabel(targetText)
         spans.append(
-          InlineSpan(text: label, styles: base, link: InlineLink(kind: .anchor, target: target)))
-        i = j
-        continue
-      }
+          InlineSpan(text: label, styles: base, link: InlineLink(kind: .link, target: targetText)))
 
-      // Attached modifiers with recursively parsed content.
-      if let style = modifiers[c], isOpener(prev: prev, next: next),
-        let close = modifierClose(chars, from: i + 1, to: hi, char: c) {
+      case .anchor(_, let name, let suffix, _):
         flush()
-        spans.append(contentsOf: parse(chars, from: i + 1, to: close, base: base.union(style)))
-        i = close + 1
-        continue
+        let nameText = slice(chars, name.body)
+        let label: String
+        let target: String?
+        switch suffix {
+        case .target(let bracket):
+          label = nameText
+          target = slice(chars, bracket.body)
+        case .description(let bracket):
+          label = slice(chars, bracket.body)
+          target = nil
+        case nil:
+          label = nameText
+          target = nil
+        }
+        spans.append(
+          InlineSpan(text: label, styles: base, link: InlineLink(kind: .anchor, target: target)))
       }
 
-      buffer.append(c)
-      i += 1
+      i = object.end
     }
 
     flush()
     return spans
   }
 
-  // MARK: - Boundary helpers
-
-  /// Builds a `String` from a half-open scalar range `[from, to)`.
-  private static func slice(_ chars: [Unicode.Scalar], _ from: Int, _ to: Int) -> String {
-    String(String.UnicodeScalarView(chars[from..<to]))
-  }
-
-  private static func isSpace(_ c: Unicode.Scalar?) -> Bool {
-    guard let c else { return true }
-    return c.properties.isWhitespace
-  }
-
-  /// Whether a scalar (or the absence of one, at a line edge) counts as a
-  /// modifier boundary: whitespace, punctuation, or a symbol.
-  private static func isBoundary(_ c: Unicode.Scalar?) -> Bool {
-    guard let c else { return true }
-    if c.properties.isWhitespace { return true }
-    switch c.properties.generalCategory {
-    case .connectorPunctuation, .dashPunctuation, .openPunctuation,
-      .closePunctuation, .initialPunctuation, .finalPunctuation, .otherPunctuation,
-      .mathSymbol, .currencySymbol, .modifierSymbol, .otherSymbol:
-      return true
-    default:
-      return false
-    }
-  }
-
-  /// A valid opener is preceded by a boundary and followed by non-whitespace.
-  private static func isOpener(prev: Unicode.Scalar?, next: Unicode.Scalar?) -> Bool {
-    isBoundary(prev) && !isSpace(next)
-  }
+  // MARK: - Helpers
 
-  /// Finds the closing modifier of the same character: preceded by
-  /// non-whitespace and followed by a boundary. Honours escapes.
-  private static func modifierClose(
-    _ chars: [Unicode.Scalar], from start: Int, to hi: Int, char: Unicode.Scalar
-  ) -> Int? {
-    var i = start
-    while i < hi {
-      if chars[i] == "\\" {
-        i += 2
-        continue
-      }
-      if chars[i] == char {
-        let prev: Unicode.Scalar? = i > 0 ? chars[i - 1] : nil
-        let next: Unicode.Scalar? = i + 1 < chars.count ? chars[i + 1] : nil
-        if !isSpace(prev) && isBoundary(next) { return i }
-      }
-      i += 1
-    }
-    return nil
-  }
-
-  /// Finds the closing character for verbatim/comment content. No escapes.
-  private static func literalClose(
-    _ chars: [Unicode.Scalar], from start: Int, to hi: Int, char: Unicode.Scalar
-  ) -> Int? {
-    var i = start
-    while i < hi {
-      if chars[i] == char {
-        let prev: Unicode.Scalar? = i > 0 ? chars[i - 1] : nil
-        if !isSpace(prev) { return i }
-      }
-      i += 1
-    }
-    return nil
-  }
-
-  /// Finds a matching closing bracket, honouring escapes.
-  private static func bracketClose(
-    _ chars: [Unicode.Scalar], from start: Int, to hi: Int, char: Unicode.Scalar
-  ) -> Int? {
-    var i = start
-    while i < hi {
-      if chars[i] == "\\" {
-        i += 2
-        continue
-      }
-      if chars[i] == char { return i }
-      i += 1
-    }
-    return nil
+  /// Builds a `String` from a half-open scalar range.
+  private static func slice(_ chars: [Unicode.Scalar], _ range: Range<Int>) -> String {
+    String(String.UnicodeScalarView(chars[range]))
   }
 
   /// Produces display text for a link target that has no explicit description
diff --git a/Sources/NorgKit/Parsers/NorgParser.swift b/Sources/NorgKit/Parsers/NorgParser.swift
index aad66f4..20a76a0 100644
--- a/Sources/NorgKit/Parsers/NorgParser.swift
+++ b/Sources/NorgKit/Parsers/NorgParser.swift
@@ -1,10 +1,8 @@
 /// Parses a Norg document into a list of `NorgBlock`s.
 public enum NorgParser {
 
-  private enum Delimiter { case weak, strong, rule }
-
-  private static let blockMarkers = ASCIIByteSet("*-~>")
-  private static let rangeableMarkers = ASCIIByteSet("$^:")
+  private static let blockMarkers = BlockScanner.blockMarkers
+  private static let rangeableMarkers = BlockScanner.rangeableMarkers
 
   private struct Source {
     let raw: [Substring]
@@ -51,7 +49,7 @@ public enum NorgParser {
       // Delimiting modifiers (lines of two or more identical -, = or _).
       // All three are emitted as blocks: `___` renders as a rule, while
       // `---`/`===` are reset signals the tree fold consumes.
-      if let delimiter = delimiter(line) {
+      if let delimiter = BlockScanner.delimiter(line) {
         switch delimiter {
         case .rule: blocks.append(.horizontalRule(line: lineNo))
         case .weak: blocks.append(.weakDelimiter(line: lineNo))
@@ -264,22 +262,10 @@ public enum NorgParser {
 
   // MARK: - Helpers
 
-  /// Identifies a delimiting-modifier line (two or more identical `-`, `=`, `_`).
-  private static func delimiter(_ s: Substring) -> Delimiter? {
-    guard s.count >= 2, let first = s.first, "-=_".contains(first),
-      s.allSatisfy({ $0 == first })
-    else { return nil }
-    switch first {
-    case "-": return .weak
-    case "=": return .strong
-    default: return .rule
-    }
-  }
-
   /// Whether a trimmed line starts a new block (used to break paragraphs).
   private static func isBlockStart(_ trimmed: Substring) -> Bool {
     if trimmed.hasPrefix("@") { return true }
-    if delimiter(trimmed) != nil { return true }
+    if BlockScanner.delimiter(trimmed) != nil { return true }
     if DetachedModifier.parse(trimmed[...], accepting: blockMarkers) != nil { return true }
     return DetachedModifier.parse(trimmed[...], accepting: rangeableMarkers) != nil
   }
diff --git a/Tests/NorgKitTests/Lexer/NorgLexerTests.swift b/Tests/NorgKitTests/Lexer/NorgLexerTests.swift
new file mode 100644
index 0000000..ddd51e0
--- /dev/null
+++ b/Tests/NorgKitTests/Lexer/NorgLexerTests.swift
@@ -0,0 +1,249 @@
+import Testing
+
+@testable import NorgKit
+
+struct NorgLexerTests {
+
+  private func text(_ token: NorgToken, in source: String) -> String {
+    String(source[token.range])
+  }
+
+  private func first(
+    _ tokens: [NorgToken], in source: String, where predicate: (NorgToken.Kind) -> Bool
+  ) -> (kind: NorgToken.Kind, text: String)? {
+    guard let token = tokens.first(where: { predicate($0.kind) }) else { return nil }
+    return (token.kind, text(token, in: source))
+  }
+
+  // MARK: Ranges are valid and ordered
+
+  @Test func tokenRangesAreInOrderAndWithinBounds() {
+    let source = "* Heading with *bold* and {https://x.y}[link]\n- (x) done item\n"
+    let tokens = NorgLexer.tokenize(source)
+    #expect(!tokens.isEmpty)
+    var cursor = source.startIndex
+    for token in tokens {
+      #expect(token.range.lowerBound >= source.startIndex)
+      #expect(token.range.upperBound <= source.endIndex)
+      #expect(token.range.lowerBound <= token.range.upperBound)
+      // Markers/content are emitted left to right across the document.
+      #expect(token.range.lowerBound >= cursor || token.range.lowerBound >= source.startIndex)
+      cursor = token.range.lowerBound
+    }
+  }
+
+  // MARK: Block markers
+
+  @Test func headingMarkerCarriesLevelAndCoversRun() {
+    let source = "** Subheading"
+    let tokens = NorgLexer.tokenize(source)
+    let heading = first(tokens, in: source) {
+      if case .heading = $0 { return true } else { return false }
+    }
+    #expect(heading?.text == "**")
+    if case .heading(let level)? = heading?.kind {
+      #expect(level == 2)
+    } else {
+      Issue.record("no heading")
+    }
+  }
+
+  @Test func listAndQuoteAndRangeableMarkers() {
+    #expect(
+      markerText(
+        "- item",
+        {
+          if case .unorderedList = $0 { return true }
+          return false
+        }) == "-")
+    #expect(
+      markerText(
+        "~~ item",
+        {
+          if case .orderedList = $0 { return true }
+          return false
+        }) == "~~")
+    #expect(
+      markerText(
+        "> quote",
+        {
+          if case .quote = $0 { return true }
+          return false
+        }) == ">")
+    #expect(
+      markerText(
+        "$ Term",
+        {
+          if case .definition = $0 { return true }
+          return false
+        }) == "$")
+    #expect(
+      markerText(
+        "^ Note",
+        {
+          if case .footnote = $0 { return true }
+          return false
+        }) == "^")
+    #expect(
+      markerText(
+        ": Cell",
+        {
+          if case .tableCell = $0 { return true }
+          return false
+        }) == ":")
+  }
+
+  private func markerText(_ source: String, _ predicate: (NorgToken.Kind) -> Bool) -> String? {
+    let tokens = NorgLexer.tokenize(source)
+    guard let token = tokens.first(where: { predicate($0.kind) }) else { return nil }
+    return String(source[token.range])
+  }
+
+  @Test func taskStatusCoversParentheses() {
+    let source = "- (x) finished"
+    let tokens = NorgLexer.tokenize(source)
+    let status = first(tokens, in: source) {
+      if case .taskStatus = $0 { return true } else { return false }
+    }
+    #expect(status?.text == "(x)")
+    if case .taskStatus(let s)? = status?.kind {
+      #expect(s == .done)
+    } else {
+      Issue.record("no status")
+    }
+  }
+
+  // MARK: Delimiters and rules
+
+  @Test func horizontalRuleAndWeakStrong() {
+    #expect(
+      hasKind("___") {
+        if case .horizontalRule = $0 { return true }
+        return false
+      })
+    #expect(
+      hasKind("---") {
+        if case .weakDelimiter = $0 { return true }
+        return false
+      })
+    #expect(
+      hasKind("===") {
+        if case .strongDelimiter = $0 { return true }
+        return false
+      })
+  }
+
+  private func hasKind(_ source: String, _ predicate: (NorgToken.Kind) -> Bool) -> Bool {
+    NorgLexer.tokenize(source).contains { predicate($0.kind) }
+  }
+
+  // MARK: Inline
+
+  @Test func boldEmitsDelimitersAndStyledText() {
+    let source = "a *bold* b"
+    let tokens = NorgLexer.tokenize(source)
+    let delimiters = tokens.filter {
+      if case .modifierDelimiter = $0.kind { return true } else { return false }
+    }
+    #expect(delimiters.count == 2)
+    #expect(delimiters.allSatisfy { text($0, in: source) == "*" })
+    let styled = first(tokens, in: source) {
+      if case .styledText = $0 { return true } else { return false }
+    }
+    #expect(styled?.text == "bold")
+    if case .styledText(let style)? = styled?.kind {
+      #expect(style.contains(.bold))
+    } else {
+      Issue.record("no styled")
+    }
+  }
+
+  @Test func nestedStyledTextAccumulatesStyles() {
+    let source = "_under /italic/_"
+    let tokens = NorgLexer.tokenize(source)
+    let inner = tokens.first {
+      if case .styledText(let s) = $0.kind { return s.contains(.italic) && s.contains(.underline) }
+      return false
+    }
+    #expect(inner != nil)
+    #expect(inner.map { text($0, in: source) } == "italic")
+  }
+
+  @Test func verbatimBodyIsStyledAndDelimited() {
+    let source = "x `a*b*c` y"
+    let tokens = NorgLexer.tokenize(source)
+    let body = tokens.first {
+      if case .styledText(let s) = $0.kind { return s.contains(.verbatim) }
+      return false
+    }
+    #expect(body.map { text($0, in: source) } == "a*b*c")
+  }
+
+  @Test func topLevelPlainTextProducesNoTokens() {
+    #expect(NorgLexer.tokenize("just some plain words").isEmpty)
+  }
+
+  @Test func commentCoversMarkers() {
+    let source = "visible %hidden% text"
+    let tokens = NorgLexer.tokenize(source)
+    let comment = first(tokens, in: source) {
+      if case .comment = $0 { return true } else { return false }
+    }
+    #expect(comment?.text == "%hidden%")
+  }
+
+  @Test func escapeCoversBackslashOnly() {
+    let source = "\\*literal"
+    let tokens = NorgLexer.tokenize(source)
+    let escape = first(tokens, in: source) {
+      if case .escape = $0 { return true } else { return false }
+    }
+    #expect(escape?.text == "\\")
+  }
+
+  @Test func linkWithDescriptionEmitsTargetAndDescription() {
+    let source = "see {https://example.com}[the site]"
+    let tokens = NorgLexer.tokenize(source)
+    let target = first(tokens, in: source) {
+      if case .linkTarget = $0 { return true } else { return false }
+    }
+    let description = first(tokens, in: source) {
+      if case .linkDescription = $0 { return true } else { return false }
+    }
+    #expect(target?.text == "https://example.com")
+    #expect(description?.text == "the site")
+    let delimiters = tokens.filter {
+      if case .linkDelimiter = $0.kind { return true } else { return false }
+    }
+    #expect(Set(delimiters.map { text($0, in: source) }) == ["{", "}", "[", "]"])
+  }
+
+  // MARK: Ranged tags
+
+  @Test func codeFenceEmitsTagsAndVerbatimBody() {
+    let source = "@code swift\nlet x = 1\n@end\n"
+    let tokens = NorgLexer.tokenize(source)
+    let delimiters = tokens.filter {
+      if case .tagDelimiter = $0.kind { return true } else { return false }
+    }
+    #expect(delimiters.count == 2)
+    #expect(delimiters.allSatisfy { text($0, in: source) == "@" })
+    let name = first(tokens, in: source) {
+      if case .tagName = $0 { return true } else { return false }
+    }
+    #expect(name?.text == "code swift")
+    let body = first(tokens, in: source) {
+      if case .verbatimBlock = $0 { return true } else { return false }
+    }
+    #expect(body?.text == "let x = 1")
+  }
+
+  @Test func markersInsideCodeFenceAreNotInlineTokens() {
+    let source = "@code\n*not bold*\n@end\n"
+    let tokens = NorgLexer.tokenize(source)
+    #expect(
+      !tokens.contains {
+        if case .modifierDelimiter = $0.kind { return true } else { return false }
+      })
+  }
+}