Skip to content

Commit 4e4300d

Browse files
iafaniafan
committed
Normalize (consolidate) tokens of the same kind
This makes the output smaller and allows for font ligatures (like `<=`, `!=`, `==` and so on) to be displayed correctly
1 parent bd320f5 commit 4e4300d

20 files changed

+71
-35
lines changed

‎example_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ function b() {
3030
// <span class="kwd">var</span> <span class="pln">a</span> <span class="pun">=</span> <span class="dec">3</span><span class="pun">;</span>
3131
//
3232
// <span class="com">// b is a cool function</span>
33-
// <span class="kwd">function</span> <span class="pln">b</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span>
33+
// <span class="kwd">function</span> <span class="pln">b</span><span class="pun">()</span> <span class="pun">{</span>
3434
// <span class="kwd">return</span> <span class="dec">7</span><span class="pun">;</span>
3535
// <span class="pun">}</span>
3636
}

‎highlight.go

+42-10
Original file line numberDiff line numberDiff line change
@@ -194,17 +194,31 @@ var DefaultHTMLConfig = HTMLConfig{
194194
}
195195

196196
func Print(s *scanner.Scanner, w io.Writer, p Printer) error {
197+
accum := ""
198+
lastKind := Whitespace
197199
tok := s.Scan()
198200
for tok != scanner.EOF {
199-
tokText := s.TokenText()
200-
err := p.Print(w, tokenKind(tok, tokText), tokText)
201-
if err != nil {
202-
return err
201+
text := s.TokenText()
202+
kind := tokenKind(tok, text)
203+
if kind != lastKind {
204+
if accum != "" {
205+
if err := p.Print(w, lastKind, accum); err != nil {
206+
return err
207+
}
208+
accum = ""
209+
}
210+
lastKind = kind
203211
}
204-
212+
accum += text
205213
tok = s.Scan()
206214
}
207215

216+
if accum != "" {
217+
if err := p.Print(w, lastKind, accum); err != nil {
218+
return err
219+
}
220+
}
221+
208222
return nil
209223
}
210224

@@ -214,20 +228,38 @@ func Annotate(src []byte, a Annotator) (annotate.Annotations, error) {
214228
var anns annotate.Annotations
215229
read := 0
216230

231+
accum := ""
232+
lastKind := Whitespace
217233
tok := s.Scan()
218234
for tok != scanner.EOF {
219-
tokText := s.TokenText()
235+
text := s.TokenText()
236+
kind := tokenKind(tok, text)
237+
if kind != lastKind {
238+
if accum != "" {
239+
ann, err := a.Annotate(read, lastKind, accum)
240+
if err != nil {
241+
return nil, err
242+
}
243+
read += len(accum)
244+
if ann != nil {
245+
anns = append(anns, ann)
246+
}
247+
accum = ""
248+
}
249+
lastKind = kind
250+
}
251+
accum += text
252+
tok = s.Scan()
253+
}
220254

221-
ann, err := a.Annotate(read, tokenKind(tok, tokText), tokText)
255+
if accum != "" {
256+
ann, err := a.Annotate(read, lastKind, accum)
222257
if err != nil {
223258
return nil, err
224259
}
225-
read += len(tokText)
226260
if ann != nil {
227261
anns = append(anns, ann)
228262
}
229-
230-
tok = s.Scan()
231263
}
232264

233265
return anns, nil

‎highlight_test.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,7 @@ func TestAnnotate(t *testing.T) {
8888
src := []byte(`a:=2`)
8989
want := annotate.Annotations{
9090
{Start: 0, End: 1, Left: []byte(`<span class="pln">`), Right: []byte("</span>")},
91-
{Start: 1, End: 2, Left: []byte(`<span class="pun">`), Right: []byte("</span>")},
92-
{Start: 2, End: 3, Left: []byte(`<span class="pun">`), Right: []byte("</span>")},
91+
{Start: 1, End: 3, Left: []byte(`<span class="pun">`), Right: []byte("</span>")},
9392
{Start: 3, End: 4, Left: []byte(`<span class="dec">`), Right: []byte("</span>")},
9493
}
9594
got, err := Annotate(src, HTMLAnnotator(DefaultHTMLConfig))

‎testdata/consolidate_tokens.js

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
>=

‎testdata/consolidate_tokens.js.html

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<span class="pun">&gt;=</span>

‎testdata/consolidate_tokens.js.ol.html

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
<ol>
2+
<li><span class="pun">&gt;=</span></li>
3+
</ol>

‎testdata/simple.c.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22

33
<span class="kwd">int</span> <span class="pln">main</span><span class="pun">(</span><span class="kwd">void</span><span class="pun">)</span>
44
<span class="pun">{</span>
5-
<span class="pln">printf</span><span class="pun">(</span><span class="str">&#34;hello, world\n&#34;</span><span class="pun">)</span><span class="pun">;</span>
5+
<span class="pln">printf</span><span class="pun">(</span><span class="str">&#34;hello, world\n&#34;</span><span class="pun">);</span>
66
<span class="pun">}</span>

‎testdata/simple.c.ol.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<li> </li>
44
<li><span class="kwd">int</span> <span class="pln">main</span><span class="pun">(</span><span class="kwd">void</span><span class="pun">)</span></li>
55
<li><span class="pun">{</span></li>
6-
<li> <span class="pln">printf</span><span class="pun">(</span><span class="str">&#34;hello, world\n&#34;</span><span class="pun">)</span><span class="pun">;</span></li>
6+
<li> <span class="pln">printf</span><span class="pun">(</span><span class="str">&#34;hello, world\n&#34;</span><span class="pun">);</span></li>
77
<li><span class="pun">}</span></li>
88
<li></li>
99
</ol>

‎testdata/simple.go.html

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
<span class="com">// +build ignore</span>
22
<span class="kwd">package</span> <span class="pln">foo</span>
33

4-
<span class="kwd">func</span> <span class="typ">Bar</span><span class="pun">(</span><span class="pln">baz</span> <span class="pln">string</span><span class="pun">,</span> <span class="pln">qux</span> <span class="pun">*</span><span class="typ">Zip</span><span class="pun">)</span> <span class="pun">(</span><span class="pun">*</span><span class="typ">Zap</span><span class="pun">,</span> <span class="typ">Zop</span><span class="pun">)</span> <span class="pun">{</span>
5-
<span class="pln">ziz</span> <span class="pun">:</span><span class="pun">=</span> <span class="pln">mop</span><span class="pun">(</span><span class="dec">3</span><span class="pun">,</span> <span class="str">&#34;hello world&#34;</span><span class="pun">)</span>
4+
<span class="kwd">func</span> <span class="typ">Bar</span><span class="pun">(</span><span class="pln">baz</span> <span class="pln">string</span><span class="pun">,</span> <span class="pln">qux</span> <span class="pun">*</span><span class="typ">Zip</span><span class="pun">)</span> <span class="pun">(*</span><span class="typ">Zap</span><span class="pun">,</span> <span class="typ">Zop</span><span class="pun">)</span> <span class="pun">{</span>
5+
<span class="pln">ziz</span> <span class="pun">:=</span> <span class="pln">mop</span><span class="pun">(</span><span class="dec">3</span><span class="pun">,</span> <span class="str">&#34;hello world&#34;</span><span class="pun">)</span>
66
<span class="pun">}</span>
77

88
<span class="kwd">type</span> <span class="typ">Qaz</span> <span class="kwd">struct</span> <span class="pun">{</span>

‎testdata/simple.go.ol.html

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
<li><span class="com">// +build ignore</span></li>
33
<li><span class="kwd">package</span> <span class="pln">foo</span></li>
44
<li></li>
5-
<li><span class="kwd">func</span> <span class="typ">Bar</span><span class="pun">(</span><span class="pln">baz</span> <span class="pln">string</span><span class="pun">,</span> <span class="pln">qux</span> <span class="pun">*</span><span class="typ">Zip</span><span class="pun">)</span> <span class="pun">(</span><span class="pun">*</span><span class="typ">Zap</span><span class="pun">,</span> <span class="typ">Zop</span><span class="pun">)</span> <span class="pun">{</span></li>
6-
<li> <span class="pln">ziz</span> <span class="pun">:</span><span class="pun">=</span> <span class="pln">mop</span><span class="pun">(</span><span class="dec">3</span><span class="pun">,</span> <span class="str">&#34;hello world&#34;</span><span class="pun">)</span></li>
5+
<li><span class="kwd">func</span> <span class="typ">Bar</span><span class="pun">(</span><span class="pln">baz</span> <span class="pln">string</span><span class="pun">,</span> <span class="pln">qux</span> <span class="pun">*</span><span class="typ">Zip</span><span class="pun">)</span> <span class="pun">(*</span><span class="typ">Zap</span><span class="pun">,</span> <span class="typ">Zop</span><span class="pun">)</span> <span class="pun">{</span></li>
6+
<li> <span class="pln">ziz</span> <span class="pun">:=</span> <span class="pln">mop</span><span class="pun">(</span><span class="dec">3</span><span class="pun">,</span> <span class="str">&#34;hello world&#34;</span><span class="pun">)</span></li>
77
<li><span class="pun">}</span></li>
88
<li></li>
99
<li><span class="kwd">type</span> <span class="typ">Qaz</span> <span class="kwd">struct</span> <span class="pun">{</span></li>

‎testdata/simple.js.html

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
<span class="com">// foo is a cool function</span>
2-
<span class="kwd">function</span> <span class="pln">foo</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span><span class="pun">}</span>
2+
<span class="kwd">function</span> <span class="pln">foo</span><span class="pun">()</span> <span class="pun">{}</span>
33

44
<span class="com">/* bar is a cool var */</span>
55
<span class="kwd">var</span> <span class="pln">bar</span> <span class="pun">=</span> <span class="dec">3</span><span class="pun">;</span>
66

7-
<span class="typ">A</span><span class="pun">.</span><span class="pln">prototype</span><span class="pun">.</span><span class="pln">foo</span> <span class="pun">=</span> <span class="kwd">function</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span>
8-
<span class="kwd">this</span><span class="pun">.</span><span class="pln">noise</span> <span class="pun">|</span><span class="pun">|</span> <span class="str">&#39;&lt;chirp&gt;&#39;</span><span class="pun">;</span>
7+
<span class="typ">A</span><span class="pun">.</span><span class="pln">prototype</span><span class="pun">.</span><span class="pln">foo</span> <span class="pun">=</span> <span class="kwd">function</span><span class="pun">()</span> <span class="pun">{</span>
8+
<span class="kwd">this</span><span class="pun">.</span><span class="pln">noise</span> <span class="pun">||</span> <span class="str">&#39;&lt;chirp&gt;&#39;</span><span class="pun">;</span>
99
<span class="kwd">return</span> <span class="str">&#39;Hello from &#39;</span> <span class="pun">+</span> <span class="kwd">this</span><span class="pun">.</span><span class="pln">name</span><span class="pun">;</span>
1010
<span class="pun">}</span>

‎testdata/simple.js.ol.html

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
<ol>
22
<li><span class="com">// foo is a cool function</span></li>
3-
<li><span class="kwd">function</span> <span class="pln">foo</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span><span class="pun">}</span></li>
3+
<li><span class="kwd">function</span> <span class="pln">foo</span><span class="pun">()</span> <span class="pun">{}</span></li>
44
<li></li>
55
<li><span class="com">/* bar is a cool var */</span></li>
66
<li><span class="kwd">var</span> <span class="pln">bar</span> <span class="pun">=</span> <span class="dec">3</span><span class="pun">;</span></li>
77
<li></li>
8-
<li><span class="typ">A</span><span class="pun">.</span><span class="pln">prototype</span><span class="pun">.</span><span class="pln">foo</span> <span class="pun">=</span> <span class="kwd">function</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span></li>
9-
<li> <span class="kwd">this</span><span class="pun">.</span><span class="pln">noise</span> <span class="pun">|</span><span class="pun">|</span> <span class="str">&#39;&lt;chirp&gt;&#39;</span><span class="pun">;</span></li>
8+
<li><span class="typ">A</span><span class="pun">.</span><span class="pln">prototype</span><span class="pun">.</span><span class="pln">foo</span> <span class="pun">=</span> <span class="kwd">function</span><span class="pun">()</span> <span class="pun">{</span></li>
9+
<li> <span class="kwd">this</span><span class="pun">.</span><span class="pln">noise</span> <span class="pun">||</span> <span class="str">&#39;&lt;chirp&gt;&#39;</span><span class="pun">;</span></li>
1010
<li> <span class="kwd">return</span> <span class="str">&#39;Hello from &#39;</span> <span class="pun">+</span> <span class="kwd">this</span><span class="pun">.</span><span class="pln">name</span><span class="pun">;</span></li>
1111
<li><span class="pun">}</span></li>
1212
<li></li>

‎testdata/simple.py.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
<span class="kwd">from</span> <span class="pln">foo</span> <span class="kwd">import</span> <span class="pln">bar</span>
22

3-
<span class="kwd">def</span> <span class="pln">f</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span> <span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">)</span><span class="pun">:</span>
3+
<span class="kwd">def</span> <span class="pln">f</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span> <span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">):</span>
44
<span class="kwd">print</span><span class="pun">(</span><span class="str">&#39;hello!&#39;</span><span class="pun">)</span>

‎testdata/simple.py.ol.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<ol>
22
<li><span class="kwd">from</span> <span class="pln">foo</span> <span class="kwd">import</span> <span class="pln">bar</span></li>
33
<li></li>
4-
<li><span class="kwd">def</span> <span class="pln">f</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span> <span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">)</span><span class="pun">:</span></li>
4+
<li><span class="kwd">def</span> <span class="pln">f</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span> <span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">):</span></li>
55
<li> <span class="kwd">print</span><span class="pun">(</span><span class="str">&#39;hello!&#39;</span><span class="pun">)</span></li>
66
<li></li>
77
</ol>

‎testdata/simple.rb.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,5 @@
88

99
<span class="kwd">def</span> <span class="pln">foo</span><span class="pun">(</span><span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">)</span>
1010
<span class="pln">puts</span> <span class="pln">a</span>
11-
<span class="typ">A</span><span class="pun">:</span><span class="pun">:</span><span class="typ">B</span>
11+
<span class="typ">A</span><span class="pun">::</span><span class="typ">B</span>
1212
<span class="kwd">end</span>

‎testdata/simple.rb.ol.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<li></li>
1010
<li><span class="kwd">def</span> <span class="pln">foo</span><span class="pun">(</span><span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">)</span></li>
1111
<li> <span class="pln">puts</span> <span class="pln">a</span></li>
12-
<li> <span class="typ">A</span><span class="pun">:</span><span class="pun">:</span><span class="typ">B</span></li>
12+
<li> <span class="typ">A</span><span class="pun">::</span><span class="typ">B</span></li>
1313
<li><span class="kwd">end</span></li>
1414
<li></li>
1515
</ol>

‎testdata/underscore.go.html

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<span class="kwd">package</span> <span class="pln">foo_bar</span>
44

5-
<span class="kwd">func</span> <span class="pln">foo</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span>
6-
<span class="kwd">for</span> <span class="pln">_</span><span class="pun">,</span> <span class="pln">a</span> <span class="pun">:</span><span class="pun">=</span> <span class="pln">range</span> <span class="pln">foo</span> <span class="pun">{</span>
5+
<span class="kwd">func</span> <span class="pln">foo</span><span class="pun">()</span> <span class="pun">{</span>
6+
<span class="kwd">for</span> <span class="pln">_</span><span class="pun">,</span> <span class="pln">a</span> <span class="pun">:=</span> <span class="pln">range</span> <span class="pln">foo</span> <span class="pun">{</span>
77
<span class="pun">}</span>
88
<span class="pun">}</span>

‎testdata/underscore.go.ol.html

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
<li></li>
44
<li><span class="kwd">package</span> <span class="pln">foo_bar</span></li>
55
<li></li>
6-
<li><span class="kwd">func</span> <span class="pln">foo</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span></li>
7-
<li> <span class="kwd">for</span> <span class="pln">_</span><span class="pun">,</span> <span class="pln">a</span> <span class="pun">:</span><span class="pun">=</span> <span class="pln">range</span> <span class="pln">foo</span> <span class="pun">{</span></li>
6+
<li><span class="kwd">func</span> <span class="pln">foo</span><span class="pun">()</span> <span class="pun">{</span></li>
7+
<li> <span class="kwd">for</span> <span class="pln">_</span><span class="pun">,</span> <span class="pln">a</span> <span class="pun">:=</span> <span class="pln">range</span> <span class="pln">foo</span> <span class="pun">{</span></li>
88
<li> <span class="pun">}</span></li>
99
<li><span class="pun">}</span></li>
1010
<li></li>

‎testdata/utf8.go.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,6 @@
1111
<span class="com">// ᚷᛁᚠ᛫ᚻᛖ᛫ᚹᛁᛚᛖ᛫ᚠᚩᚱ᛫ᛞᚱᛁᚻᛏᚾᛖ᛫ᛞᚩᛗᛖᛋ᛫ᚻᛚᛇᛏᚪᚾ᛬</span>
1212
<span class="kwd">var</span> <span class="typ">B</span> <span class="pun">=</span> <span class="str">&#34;Τὴ γλῶσσα μοῦ ἔδωσαν ἑλληνικὴ&#34;</span>
1313

14-
<span class="kwd">func</span> <span class="typ">F</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span>
14+
<span class="kwd">func</span> <span class="typ">F</span><span class="pun">()</span> <span class="pun">{</span>
1515
<span class="pln">fmt</span><span class="pun">.</span><span class="typ">Println</span><span class="pun">(</span><span class="typ">A</span><span class="pun">,</span> <span class="typ">B</span><span class="pun">)</span>
1616
<span class="pun">}</span>

‎testdata/utf8.go.ol.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
<li><span class="com">// ᚷᛁᚠ᛫ᚻᛖ᛫ᚹᛁᛚᛖ᛫ᚠᚩᚱ᛫ᛞᚱᛁᚻᛏᚾᛖ᛫ᛞᚩᛗᛖᛋ᛫ᚻᛚᛇᛏᚪᚾ᛬</span></li>
1313
<li><span class="kwd">var</span> <span class="typ">B</span> <span class="pun">=</span> <span class="str">&#34;Τὴ γλῶσσα μοῦ ἔδωσαν ἑλληνικὴ&#34;</span></li>
1414
<li></li>
15-
<li><span class="kwd">func</span> <span class="typ">F</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span></li>
15+
<li><span class="kwd">func</span> <span class="typ">F</span><span class="pun">()</span> <span class="pun">{</span></li>
1616
<li> <span class="pln">fmt</span><span class="pun">.</span><span class="typ">Println</span><span class="pun">(</span><span class="typ">A</span><span class="pun">,</span> <span class="typ">B</span><span class="pun">)</span></li>
1717
<li><span class="pun">}</span></li>
1818
<li></li>

0 commit comments

Comments
 (0)