Add NorgLexerHEAD1.1.0main

1 files changed, 249 insertions, 0 deletions

diff --git a/Tests/NorgKitTests/Lexer/NorgLexerTests.swift b/Tests/NorgKitTests/Lexer/NorgLexerTests.swift
new file mode 100644
index 0000000..ddd51e0
--- /dev/null
+++ b/Tests/NorgKitTests/Lexer/NorgLexerTests.swift
@@ -0,0 +1,249 @@
+import Testing
+
+@testable import NorgKit
+
+struct NorgLexerTests {
+
+  private func text(_ token: NorgToken, in source: String) -> String {
+    String(source[token.range])
+  }
+
+  private func first(
+    _ tokens: [NorgToken], in source: String, where predicate: (NorgToken.Kind) -> Bool
+  ) -> (kind: NorgToken.Kind, text: String)? {
+    guard let token = tokens.first(where: { predicate($0.kind) }) else { return nil }
+    return (token.kind, text(token, in: source))
+  }
+
+  // MARK: Ranges are valid and ordered
+
+  @Test func tokenRangesAreInOrderAndWithinBounds() {
+    let source = "* Heading with *bold* and {https://x.y}[link]\n- (x) done item\n"
+    let tokens = NorgLexer.tokenize(source)
+    #expect(!tokens.isEmpty)
+    var cursor = source.startIndex
+    for token in tokens {
+      #expect(token.range.lowerBound >= source.startIndex)
+      #expect(token.range.upperBound <= source.endIndex)
+      #expect(token.range.lowerBound <= token.range.upperBound)
+      // Markers/content are emitted left to right across the document.
+      #expect(token.range.lowerBound >= cursor || token.range.lowerBound >= source.startIndex)
+      cursor = token.range.lowerBound
+    }
+  }
+
+  // MARK: Block markers
+
+  @Test func headingMarkerCarriesLevelAndCoversRun() {
+    let source = "** Subheading"
+    let tokens = NorgLexer.tokenize(source)
+    let heading = first(tokens, in: source) {
+      if case .heading = $0 { return true } else { return false }
+    }
+    #expect(heading?.text == "**")
+    if case .heading(let level)? = heading?.kind {
+      #expect(level == 2)
+    } else {
+      Issue.record("no heading")
+    }
+  }
+
+  @Test func listAndQuoteAndRangeableMarkers() {
+    #expect(
+      markerText(
+        "- item",
+        {
+          if case .unorderedList = $0 { return true }
+          return false
+        }) == "-")
+    #expect(
+      markerText(
+        "~~ item",
+        {
+          if case .orderedList = $0 { return true }
+          return false
+        }) == "~~")
+    #expect(
+      markerText(
+        "> quote",
+        {
+          if case .quote = $0 { return true }
+          return false
+        }) == ">")
+    #expect(
+      markerText(
+        "$ Term",
+        {
+          if case .definition = $0 { return true }
+          return false
+        }) == "$")
+    #expect(
+      markerText(
+        "^ Note",
+        {
+          if case .footnote = $0 { return true }
+          return false
+        }) == "^")
+    #expect(
+      markerText(
+        ": Cell",
+        {
+          if case .tableCell = $0 { return true }
+          return false
+        }) == ":")
+  }
+
+  private func markerText(_ source: String, _ predicate: (NorgToken.Kind) -> Bool) -> String? {
+    let tokens = NorgLexer.tokenize(source)
+    guard let token = tokens.first(where: { predicate($0.kind) }) else { return nil }
+    return String(source[token.range])
+  }
+
+  @Test func taskStatusCoversParentheses() {
+    let source = "- (x) finished"
+    let tokens = NorgLexer.tokenize(source)
+    let status = first(tokens, in: source) {
+      if case .taskStatus = $0 { return true } else { return false }
+    }
+    #expect(status?.text == "(x)")
+    if case .taskStatus(let s)? = status?.kind {
+      #expect(s == .done)
+    } else {
+      Issue.record("no status")
+    }
+  }
+
+  // MARK: Delimiters and rules
+
+  @Test func horizontalRuleAndWeakStrong() {
+    #expect(
+      hasKind("___") {
+        if case .horizontalRule = $0 { return true }
+        return false
+      })
+    #expect(
+      hasKind("---") {
+        if case .weakDelimiter = $0 { return true }
+        return false
+      })
+    #expect(
+      hasKind("===") {
+        if case .strongDelimiter = $0 { return true }
+        return false
+      })
+  }
+
+  private func hasKind(_ source: String, _ predicate: (NorgToken.Kind) -> Bool) -> Bool {
+    NorgLexer.tokenize(source).contains { predicate($0.kind) }
+  }
+
+  // MARK: Inline
+
+  @Test func boldEmitsDelimitersAndStyledText() {
+    let source = "a *bold* b"
+    let tokens = NorgLexer.tokenize(source)
+    let delimiters = tokens.filter {
+      if case .modifierDelimiter = $0.kind { return true } else { return false }
+    }
+    #expect(delimiters.count == 2)
+    #expect(delimiters.allSatisfy { text($0, in: source) == "*" })
+    let styled = first(tokens, in: source) {
+      if case .styledText = $0 { return true } else { return false }
+    }
+    #expect(styled?.text == "bold")
+    if case .styledText(let style)? = styled?.kind {
+      #expect(style.contains(.bold))
+    } else {
+      Issue.record("no styled")
+    }
+  }
+
+  @Test func nestedStyledTextAccumulatesStyles() {
+    let source = "_under /italic/_"
+    let tokens = NorgLexer.tokenize(source)
+    let inner = tokens.first {
+      if case .styledText(let s) = $0.kind { return s.contains(.italic) && s.contains(.underline) }
+      return false
+    }
+    #expect(inner != nil)
+    #expect(inner.map { text($0, in: source) } == "italic")
+  }
+
+  @Test func verbatimBodyIsStyledAndDelimited() {
+    let source = "x `a*b*c` y"
+    let tokens = NorgLexer.tokenize(source)
+    let body = tokens.first {
+      if case .styledText(let s) = $0.kind { return s.contains(.verbatim) }
+      return false
+    }
+    #expect(body.map { text($0, in: source) } == "a*b*c")
+  }
+
+  @Test func topLevelPlainTextProducesNoTokens() {
+    #expect(NorgLexer.tokenize("just some plain words").isEmpty)
+  }
+
+  @Test func commentCoversMarkers() {
+    let source = "visible %hidden% text"
+    let tokens = NorgLexer.tokenize(source)
+    let comment = first(tokens, in: source) {
+      if case .comment = $0 { return true } else { return false }
+    }
+    #expect(comment?.text == "%hidden%")
+  }
+
+  @Test func escapeCoversBackslashOnly() {
+    let source = "\\*literal"
+    let tokens = NorgLexer.tokenize(source)
+    let escape = first(tokens, in: source) {
+      if case .escape = $0 { return true } else { return false }
+    }
+    #expect(escape?.text == "\\")
+  }
+
+  @Test func linkWithDescriptionEmitsTargetAndDescription() {
+    let source = "see {https://example.com}[the site]"
+    let tokens = NorgLexer.tokenize(source)
+    let target = first(tokens, in: source) {
+      if case .linkTarget = $0 { return true } else { return false }
+    }
+    let description = first(tokens, in: source) {
+      if case .linkDescription = $0 { return true } else { return false }
+    }
+    #expect(target?.text == "https://example.com")
+    #expect(description?.text == "the site")
+    let delimiters = tokens.filter {
+      if case .linkDelimiter = $0.kind { return true } else { return false }
+    }
+    #expect(Set(delimiters.map { text($0, in: source) }) == ["{", "}", "[", "]"])
+  }
+
+  // MARK: Ranged tags
+
+  @Test func codeFenceEmitsTagsAndVerbatimBody() {
+    let source = "@code swift\nlet x = 1\n@end\n"
+    let tokens = NorgLexer.tokenize(source)
+    let delimiters = tokens.filter {
+      if case .tagDelimiter = $0.kind { return true } else { return false }
+    }
+    #expect(delimiters.count == 2)
+    #expect(delimiters.allSatisfy { text($0, in: source) == "@" })
+    let name = first(tokens, in: source) {
+      if case .tagName = $0 { return true } else { return false }
+    }
+    #expect(name?.text == "code swift")
+    let body = first(tokens, in: source) {
+      if case .verbatimBlock = $0 { return true } else { return false }
+    }
+    #expect(body?.text == "let x = 1")
+  }
+
+  @Test func markersInsideCodeFenceAreNotInlineTokens() {
+    let source = "@code\n*not bold*\n@end\n"
+    let tokens = NorgLexer.tokenize(source)
+    #expect(
+      !tokens.contains {
+        if case .modifierDelimiter = $0.kind { return true } else { return false }
+      })
+  }
+}