import Testing @testable import NorgKit struct NorgLexerTests { private func text(_ token: NorgToken, in source: String) -> String { String(source[token.range]) } private func first( _ tokens: [NorgToken], in source: String, where predicate: (NorgToken.Kind) -> Bool ) -> (kind: NorgToken.Kind, text: String)? { guard let token = tokens.first(where: { predicate($0.kind) }) else { return nil } return (token.kind, text(token, in: source)) } // MARK: Ranges are valid and ordered @Test func tokenRangesAreInOrderAndWithinBounds() { let source = "* Heading with *bold* and {https://x.y}[link]\n- (x) done item\n" let tokens = NorgLexer.tokenize(source) #expect(!tokens.isEmpty) var cursor = source.startIndex for token in tokens { #expect(token.range.lowerBound >= source.startIndex) #expect(token.range.upperBound <= source.endIndex) #expect(token.range.lowerBound <= token.range.upperBound) // Markers/content are emitted left to right across the document. #expect(token.range.lowerBound >= cursor || token.range.lowerBound >= source.startIndex) cursor = token.range.lowerBound } } // MARK: Block markers @Test func headingMarkerCarriesLevelAndCoversRun() { let source = "** Subheading" let tokens = NorgLexer.tokenize(source) let heading = first(tokens, in: source) { if case .heading = $0 { return true } else { return false } } #expect(heading?.text == "**") if case .heading(let level)? = heading?.kind { #expect(level == 2) } else { Issue.record("no heading") } } @Test func listAndQuoteAndRangeableMarkers() { #expect( markerText( "- item", { if case .unorderedList = $0 { return true } return false }) == "-") #expect( markerText( "~~ item", { if case .orderedList = $0 { return true } return false }) == "~~") #expect( markerText( "> quote", { if case .quote = $0 { return true } return false }) == ">") #expect( markerText( "$ Term", { if case .definition = $0 { return true } return false }) == "$") #expect( markerText( "^ Note", { if case .footnote = $0 { return true } return false }) == "^") #expect( markerText( ": Cell", { if case .tableCell = $0 { return true } return false }) == ":") } private func markerText(_ source: String, _ predicate: (NorgToken.Kind) -> Bool) -> String? { let tokens = NorgLexer.tokenize(source) guard let token = tokens.first(where: { predicate($0.kind) }) else { return nil } return String(source[token.range]) } @Test func taskStatusCoversParentheses() { let source = "- (x) finished" let tokens = NorgLexer.tokenize(source) let status = first(tokens, in: source) { if case .taskStatus = $0 { return true } else { return false } } #expect(status?.text == "(x)") if case .taskStatus(let s)? = status?.kind { #expect(s == .done) } else { Issue.record("no status") } } // MARK: Delimiters and rules @Test func horizontalRuleAndWeakStrong() { #expect( hasKind("___") { if case .horizontalRule = $0 { return true } return false }) #expect( hasKind("---") { if case .weakDelimiter = $0 { return true } return false }) #expect( hasKind("===") { if case .strongDelimiter = $0 { return true } return false }) } private func hasKind(_ source: String, _ predicate: (NorgToken.Kind) -> Bool) -> Bool { NorgLexer.tokenize(source).contains { predicate($0.kind) } } // MARK: Inline @Test func boldEmitsDelimitersAndStyledText() { let source = "a *bold* b" let tokens = NorgLexer.tokenize(source) let delimiters = tokens.filter { if case .modifierDelimiter = $0.kind { return true } else { return false } } #expect(delimiters.count == 2) #expect(delimiters.allSatisfy { text($0, in: source) == "*" }) let styled = first(tokens, in: source) { if case .styledText = $0 { return true } else { return false } } #expect(styled?.text == "bold") if case .styledText(let style)? = styled?.kind { #expect(style.contains(.bold)) } else { Issue.record("no styled") } } @Test func nestedStyledTextAccumulatesStyles() { let source = "_under /italic/_" let tokens = NorgLexer.tokenize(source) let inner = tokens.first { if case .styledText(let s) = $0.kind { return s.contains(.italic) && s.contains(.underline) } return false } #expect(inner != nil) #expect(inner.map { text($0, in: source) } == "italic") } @Test func verbatimBodyIsStyledAndDelimited() { let source = "x `a*b*c` y" let tokens = NorgLexer.tokenize(source) let body = tokens.first { if case .styledText(let s) = $0.kind { return s.contains(.verbatim) } return false } #expect(body.map { text($0, in: source) } == "a*b*c") } @Test func topLevelPlainTextProducesNoTokens() { #expect(NorgLexer.tokenize("just some plain words").isEmpty) } @Test func commentCoversMarkers() { let source = "visible %hidden% text" let tokens = NorgLexer.tokenize(source) let comment = first(tokens, in: source) { if case .comment = $0 { return true } else { return false } } #expect(comment?.text == "%hidden%") } @Test func escapeCoversBackslashOnly() { let source = "\\*literal" let tokens = NorgLexer.tokenize(source) let escape = first(tokens, in: source) { if case .escape = $0 { return true } else { return false } } #expect(escape?.text == "\\") } @Test func linkWithDescriptionEmitsTargetAndDescription() { let source = "see {https://example.com}[the site]" let tokens = NorgLexer.tokenize(source) let target = first(tokens, in: source) { if case .linkTarget = $0 { return true } else { return false } } let description = first(tokens, in: source) { if case .linkDescription = $0 { return true } else { return false } } #expect(target?.text == "https://example.com") #expect(description?.text == "the site") let delimiters = tokens.filter { if case .linkDelimiter = $0.kind { return true } else { return false } } #expect(Set(delimiters.map { text($0, in: source) }) == ["{", "}", "[", "]"]) } // MARK: Ranged tags @Test func codeFenceEmitsTagsAndVerbatimBody() { let source = "@code swift\nlet x = 1\n@end\n" let tokens = NorgLexer.tokenize(source) let delimiters = tokens.filter { if case .tagDelimiter = $0.kind { return true } else { return false } } #expect(delimiters.count == 2) #expect(delimiters.allSatisfy { text($0, in: source) == "@" }) let name = first(tokens, in: source) { if case .tagName = $0 { return true } else { return false } } #expect(name?.text == "code swift") let body = first(tokens, in: source) { if case .verbatimBlock = $0 { return true } else { return false } } #expect(body?.text == "let x = 1") } @Test func markersInsideCodeFenceAreNotInlineTokens() { let source = "@code\n*not bold*\n@end\n" let tokens = NorgLexer.tokenize(source) #expect( !tokens.contains { if case .modifierDelimiter = $0.kind { return true } else { return false } }) } }