1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
|
/// Tokenizes Norg source into `NorgToken`s for syntax highlighting.
public enum NorgLexer {
/// Tokenizes text for syntax highlighting.
public static func tokenize(_ text: String) -> [NorgToken] {
var tokens: [NorgToken] = []
let lines = TextHelper.lineSlices(text)
var i = 0
while i < lines.count {
let raw = lines[i]
let trimmed = TextHelper.whitespaceTrimmed(raw)
if trimmed.isEmpty {
i += 1
continue
}
if trimmed.first == "@" {
i = tagBlock(lines, at: i, into: &tokens)
continue
}
if let delimiter = BlockScanner.delimiter(trimmed) {
let kind: NorgToken.Kind
switch delimiter {
case .weak: kind = .weakDelimiter
case .strong: kind = .strongDelimiter
case .rule: kind = .horizontalRule
}
tokens.append(NorgToken(kind: kind, range: trimmed.startIndex..<trimmed.endIndex))
i += 1
continue
}
if let m = DetachedModifier.parse(raw, accepting: BlockScanner.rangeableMarkers) {
detachedTokens(m, into: &tokens)
} else if let m = DetachedModifier.parse(raw, accepting: BlockScanner.blockMarkers) {
detachedTokens(m, into: &tokens)
} else {
inlineTokens(in: raw, into: &tokens)
}
i += 1
}
return tokens
}
// MARK: - Block-level tokens
private static func tagBlock(
_ lines: [Substring], at i: Int, into tokens: inout [NorgToken]
) -> Int {
tagHeaderTokens(lines[i], into: &tokens)
let header = TextHelper.whitespaceTrimmed(TextHelper.whitespaceTrimmed(lines[i]).dropFirst())
if header == "end" { return i + 1 }
var j = i + 1
while j < lines.count {
if TextHelper.whitespaceTrimmed(lines[j]) == "@end" {
tagHeaderTokens(lines[j], into: &tokens)
return j + 1
}
let body = lines[j]
if body.startIndex < body.endIndex {
tokens.append(NorgToken(kind: .verbatimBlock, range: body.startIndex..<body.endIndex))
}
j += 1
}
return j
}
private static func tagHeaderTokens(_ raw: Substring, into tokens: inout [NorgToken]) {
guard let at = raw.firstIndex(of: "@") else { return }
let afterAt = raw.index(after: at)
tokens.append(NorgToken(kind: .tagDelimiter, range: at..<afterAt))
let header = TextHelper.whitespaceTrimmed(raw[afterAt...])
if !header.isEmpty {
tokens.append(NorgToken(kind: .tagName, range: header.startIndex..<header.endIndex))
}
}
private static func detachedTokens(_ m: DetachedModifier, into tokens: inout [NorgToken]) {
if let kind = markerKind(m.marker, level: m.level) {
tokens.append(NorgToken(kind: kind, range: m.markerRange))
}
if let status = m.status, let statusIndex = m.statusIndex {
let base = m.content.base
let open = base.index(before: statusIndex)
let close = base.index(after: base.index(after: statusIndex))
tokens.append(NorgToken(kind: .taskStatus(status), range: open..<close))
}
inlineTokens(in: m.content, into: &tokens)
}
private static func markerKind(_ marker: Character, level: Int) -> NorgToken.Kind? {
switch marker {
case "*": return .heading(level: level)
case "-": return .unorderedList(level: level)
case "~": return .orderedList(level: level)
case ">": return .quote(level: level)
case "$": return .definition(level: level)
case "^": return .footnote(level: level)
case ":": return .tableCell(level: level)
default: return nil
}
}
// MARK: - Inline tokens
private static func inlineTokens(in slice: Substring, into tokens: inout [NorgToken]) {
if slice.isEmpty { return }
if !slice.utf8.contains(where: InlineScanner.significant.contains) { return }
let chars = Array(slice.unicodeScalars)
var bound = Array(slice.unicodeScalars.indices)
bound.append(slice.endIndex)
func append(_ kind: NorgToken.Kind, _ from: Int, _ to: Int) {
tokens.append(NorgToken(kind: kind, range: bound[from]..<bound[to]))
}
func bracketed(_ b: InlineScanner.Bracketed, body kind: NorgToken.Kind) {
append(.linkDelimiter, b.open, b.open + 1)
if b.body.lowerBound < b.body.upperBound {
append(kind, b.body.lowerBound, b.body.upperBound)
}
append(.linkDelimiter, b.close, b.close + 1)
}
func emit(_ lo: Int, _ hi: Int, style: InlineStyle) {
var runStart = -1
func flushRun(_ upTo: Int) {
if runStart >= 0 {
if !style.isEmpty { append(.styledText(style), runStart, upTo) }
runStart = -1
}
}
var i = lo
while i < hi {
guard let object = InlineScanner.object(in: chars, at: i, to: hi) else {
if runStart < 0 { runStart = i }
i += 1
continue
}
flushRun(i)
switch object {
case .escape(let escaped, let end):
append(.escape, i, i + 1)
if let escaped, !style.isEmpty { append(.styledText(style), escaped, end) }
case .comment(let open, _, _, let end):
append(.comment, open, end)
case .verbatim(let vstyle, let open, let body, let close, _):
let s = style.union(vstyle)
append(.modifierDelimiter(s), open, open + 1)
if body.lowerBound < body.upperBound {
append(.styledText(s), body.lowerBound, body.upperBound)
}
append(.modifierDelimiter(s), close, close + 1)
case .modifier(let mstyle, let open, let body, let close, _):
let s = style.union(mstyle)
append(.modifierDelimiter(s), open, open + 1)
emit(body.lowerBound, body.upperBound, style: s)
append(.modifierDelimiter(s), close, close + 1)
case .link(_, let target, let description, _):
bracketed(target, body: .linkTarget)
if let description { bracketed(description, body: .linkDescription) }
case .anchor(_, let name, let suffix, _):
bracketed(name, body: .linkDescription)
switch suffix {
case .target(let b): bracketed(b, body: .linkTarget)
case .description(let b): bracketed(b, body: .linkDescription)
case nil: break
}
}
i = object.end
}
flushRun(hi)
}
emit(0, chars.count, style: [])
}
}
|
