Normalize (consolidate) tokens of the same kind

example_test.go

@@ -30,7 +30,7 @@ function b() {
 	// <span class="kwd">var</span> <span class="pln">a</span> <span class="pun">=</span> <span class="dec">3</span><span class="pun">;</span>
 	//
 	// <span class="com">// b is a cool function</span>
-	// <span class="kwd">function</span> <span class="pln">b</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span>
+	// <span class="kwd">function</span> <span class="pln">b</span><span class="pun">()</span> <span class="pun">{</span>
 	//   <span class="kwd">return</span> <span class="dec">7</span><span class="pun">;</span>
 	// <span class="pun">}</span>
 }

highlight.go

@@ -194,17 +194,31 @@ var DefaultHTMLConfig = HTMLConfig{
 }
 
 func Print(s *scanner.Scanner, w io.Writer, p Printer) error {
+	accum := ""
+	lastKind := Whitespace
 	tok := s.Scan()
 	for tok != scanner.EOF {
-		tokText := s.TokenText()
-		err := p.Print(w, tokenKind(tok, tokText), tokText)
-		if err != nil {
-			return err
+		text := s.TokenText()
+		kind := tokenKind(tok, text)
+		if kind != lastKind {
+			if accum != "" {
+				if err := p.Print(w, lastKind, accum); err != nil {
+					return err
+				}
+				accum = ""
+			}
+			lastKind = kind
 		}
-
+		accum += text
 		tok = s.Scan()
 	}
 
+	if accum != "" {
+		if err := p.Print(w, lastKind, accum); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -214,20 +228,38 @@ func Annotate(src []byte, a Annotator) (annotate.Annotations, error) {
 	var anns annotate.Annotations
 	read := 0
 
+	accum := ""
+	lastKind := Whitespace
 	tok := s.Scan()
 	for tok != scanner.EOF {
-		tokText := s.TokenText()
+		text := s.TokenText()
+		kind := tokenKind(tok, text)
+		if kind != lastKind {
+			if accum != "" {
+				ann, err := a.Annotate(read, lastKind, accum)
+				if err != nil {
+					return nil, err
+				}
+				read += len(accum)
+				if ann != nil {
+					anns = append(anns, ann)
+				}
+				accum = ""
+			}
+			lastKind = kind
+		}
+		accum += text
+		tok = s.Scan()
+	}
 
-		ann, err := a.Annotate(read, tokenKind(tok, tokText), tokText)
+	if accum != "" {
+		ann, err := a.Annotate(read, lastKind, accum)
 		if err != nil {
 			return nil, err
 		}
-		read += len(tokText)
 		if ann != nil {
 			anns = append(anns, ann)
 		}
-
-		tok = s.Scan()
 	}
 
 	return anns, nil

highlight_test.go

@@ -88,8 +88,7 @@ func TestAnnotate(t *testing.T) {
 	src := []byte(`a:=2`)
 	want := annotate.Annotations{
 		{Start: 0, End: 1, Left: []byte(`<span class="pln">`), Right: []byte("</span>")},
-		{Start: 1, End: 2, Left: []byte(`<span class="pun">`), Right: []byte("</span>")},
-		{Start: 2, End: 3, Left: []byte(`<span class="pun">`), Right: []byte("</span>")},
+		{Start: 1, End: 3, Left: []byte(`<span class="pun">`), Right: []byte("</span>")},
 		{Start: 3, End: 4, Left: []byte(`<span class="dec">`), Right: []byte("</span>")},
 	}
 	got, err := Annotate(src, HTMLAnnotator(DefaultHTMLConfig))

testdata/consolidate_tokens.js

@@ -0,0 +1 @@
+>=

testdata/consolidate_tokens.js.html

@@ -0,0 +1 @@
+<span class="pun">&gt;=</span>

testdata/consolidate_tokens.js.ol.html

@@ -0,0 +1,3 @@
+<ol>
+<li><span class="pun">&gt;=</span></li>
+</ol>

testdata/simple.c.html

@@ -2,5 +2,5 @@
 
 <span class="kwd">int</span> <span class="pln">main</span><span class="pun">(</span><span class="kwd">void</span><span class="pun">)</span>
 <span class="pun">{</span>
-    <span class="pln">printf</span><span class="pun">(</span><span class="str">&#34;hello, world\n&#34;</span><span class="pun">)</span><span class="pun">;</span>
+    <span class="pln">printf</span><span class="pun">(</span><span class="str">&#34;hello, world\n&#34;</span><span class="pun">);</span>
 <span class="pun">}</span>

testdata/simple.c.ol.html

@@ -3,7 +3,7 @@
 <li> </li>
 <li><span class="kwd">int</span> <span class="pln">main</span><span class="pun">(</span><span class="kwd">void</span><span class="pun">)</span></li>
 <li><span class="pun">{</span></li>
-<li>    <span class="pln">printf</span><span class="pun">(</span><span class="str">&#34;hello, world\n&#34;</span><span class="pun">)</span><span class="pun">;</span></li>
+<li>    <span class="pln">printf</span><span class="pun">(</span><span class="str">&#34;hello, world\n&#34;</span><span class="pun">);</span></li>
 <li><span class="pun">}</span></li>
 <li></li>
 </ol>

testdata/simple.go.html

@@ -1,8 +1,8 @@
 <span class="com">// +build ignore</span>
 <span class="kwd">package</span> <span class="pln">foo</span>
 
-<span class="kwd">func</span> <span class="typ">Bar</span><span class="pun">(</span><span class="pln">baz</span> <span class="pln">string</span><span class="pun">,</span> <span class="pln">qux</span> <span class="pun">*</span><span class="typ">Zip</span><span class="pun">)</span> <span class="pun">(</span><span class="pun">*</span><span class="typ">Zap</span><span class="pun">,</span> <span class="typ">Zop</span><span class="pun">)</span> <span class="pun">{</span>
-	<span class="pln">ziz</span> <span class="pun">:</span><span class="pun">=</span> <span class="pln">mop</span><span class="pun">(</span><span class="dec">3</span><span class="pun">,</span> <span class="str">&#34;hello world&#34;</span><span class="pun">)</span>
+<span class="kwd">func</span> <span class="typ">Bar</span><span class="pun">(</span><span class="pln">baz</span> <span class="pln">string</span><span class="pun">,</span> <span class="pln">qux</span> <span class="pun">*</span><span class="typ">Zip</span><span class="pun">)</span> <span class="pun">(*</span><span class="typ">Zap</span><span class="pun">,</span> <span class="typ">Zop</span><span class="pun">)</span> <span class="pun">{</span>
+	<span class="pln">ziz</span> <span class="pun">:=</span> <span class="pln">mop</span><span class="pun">(</span><span class="dec">3</span><span class="pun">,</span> <span class="str">&#34;hello world&#34;</span><span class="pun">)</span>
 <span class="pun">}</span>
 
 <span class="kwd">type</span> <span class="typ">Qaz</span> <span class="kwd">struct</span> <span class="pun">{</span>

testdata/simple.go.ol.html

@@ -2,8 +2,8 @@
 <li><span class="com">// +build ignore</span></li>
 <li><span class="kwd">package</span> <span class="pln">foo</span></li>
 <li></li>
-<li><span class="kwd">func</span> <span class="typ">Bar</span><span class="pun">(</span><span class="pln">baz</span> <span class="pln">string</span><span class="pun">,</span> <span class="pln">qux</span> <span class="pun">*</span><span class="typ">Zip</span><span class="pun">)</span> <span class="pun">(</span><span class="pun">*</span><span class="typ">Zap</span><span class="pun">,</span> <span class="typ">Zop</span><span class="pun">)</span> <span class="pun">{</span></li>
-<li>	<span class="pln">ziz</span> <span class="pun">:</span><span class="pun">=</span> <span class="pln">mop</span><span class="pun">(</span><span class="dec">3</span><span class="pun">,</span> <span class="str">&#34;hello world&#34;</span><span class="pun">)</span></li>
+<li><span class="kwd">func</span> <span class="typ">Bar</span><span class="pun">(</span><span class="pln">baz</span> <span class="pln">string</span><span class="pun">,</span> <span class="pln">qux</span> <span class="pun">*</span><span class="typ">Zip</span><span class="pun">)</span> <span class="pun">(*</span><span class="typ">Zap</span><span class="pun">,</span> <span class="typ">Zop</span><span class="pun">)</span> <span class="pun">{</span></li>
+<li>	<span class="pln">ziz</span> <span class="pun">:=</span> <span class="pln">mop</span><span class="pun">(</span><span class="dec">3</span><span class="pun">,</span> <span class="str">&#34;hello world&#34;</span><span class="pun">)</span></li>
 <li><span class="pun">}</span></li>
 <li></li>
 <li><span class="kwd">type</span> <span class="typ">Qaz</span> <span class="kwd">struct</span> <span class="pun">{</span></li>

testdata/simple.js.html

@@ -1,10 +1,10 @@
 <span class="com">// foo is a cool function</span>
-<span class="kwd">function</span> <span class="pln">foo</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span><span class="pun">}</span>
+<span class="kwd">function</span> <span class="pln">foo</span><span class="pun">()</span> <span class="pun">{}</span>
 
 <span class="com">/* bar is a cool var */</span>
 <span class="kwd">var</span> <span class="pln">bar</span> <span class="pun">=</span> <span class="dec">3</span><span class="pun">;</span>
 
-<span class="typ">A</span><span class="pun">.</span><span class="pln">prototype</span><span class="pun">.</span><span class="pln">foo</span> <span class="pun">=</span> <span class="kwd">function</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span>
-  <span class="kwd">this</span><span class="pun">.</span><span class="pln">noise</span> <span class="pun">|</span><span class="pun">|</span> <span class="str">&#39;&lt;chirp&gt;&#39;</span><span class="pun">;</span>
+<span class="typ">A</span><span class="pun">.</span><span class="pln">prototype</span><span class="pun">.</span><span class="pln">foo</span> <span class="pun">=</span> <span class="kwd">function</span><span class="pun">()</span> <span class="pun">{</span>
+  <span class="kwd">this</span><span class="pun">.</span><span class="pln">noise</span> <span class="pun">||</span> <span class="str">&#39;&lt;chirp&gt;&#39;</span><span class="pun">;</span>
   <span class="kwd">return</span> <span class="str">&#39;Hello from &#39;</span> <span class="pun">+</span> <span class="kwd">this</span><span class="pun">.</span><span class="pln">name</span><span class="pun">;</span>
 <span class="pun">}</span>

testdata/simple.js.ol.html

@@ -1,12 +1,12 @@
 <ol>
 <li><span class="com">// foo is a cool function</span></li>
-<li><span class="kwd">function</span> <span class="pln">foo</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span><span class="pun">}</span></li>
+<li><span class="kwd">function</span> <span class="pln">foo</span><span class="pun">()</span> <span class="pun">{}</span></li>
 <li></li>
 <li><span class="com">/* bar is a cool var */</span></li>
 <li><span class="kwd">var</span> <span class="pln">bar</span> <span class="pun">=</span> <span class="dec">3</span><span class="pun">;</span></li>
 <li></li>
-<li><span class="typ">A</span><span class="pun">.</span><span class="pln">prototype</span><span class="pun">.</span><span class="pln">foo</span> <span class="pun">=</span> <span class="kwd">function</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span></li>
-<li>  <span class="kwd">this</span><span class="pun">.</span><span class="pln">noise</span> <span class="pun">|</span><span class="pun">|</span> <span class="str">&#39;&lt;chirp&gt;&#39;</span><span class="pun">;</span></li>
+<li><span class="typ">A</span><span class="pun">.</span><span class="pln">prototype</span><span class="pun">.</span><span class="pln">foo</span> <span class="pun">=</span> <span class="kwd">function</span><span class="pun">()</span> <span class="pun">{</span></li>
+<li>  <span class="kwd">this</span><span class="pun">.</span><span class="pln">noise</span> <span class="pun">||</span> <span class="str">&#39;&lt;chirp&gt;&#39;</span><span class="pun">;</span></li>
 <li>  <span class="kwd">return</span> <span class="str">&#39;Hello from &#39;</span> <span class="pun">+</span> <span class="kwd">this</span><span class="pun">.</span><span class="pln">name</span><span class="pun">;</span></li>
 <li><span class="pun">}</span></li>
 <li></li>

testdata/simple.py.html

@@ -1,4 +1,4 @@
 <span class="kwd">from</span> <span class="pln">foo</span> <span class="kwd">import</span> <span class="pln">bar</span>
 
-<span class="kwd">def</span> <span class="pln">f</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span> <span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">)</span><span class="pun">:</span>
+<span class="kwd">def</span> <span class="pln">f</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span> <span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">):</span>
     <span class="kwd">print</span><span class="pun">(</span><span class="str">&#39;hello!&#39;</span><span class="pun">)</span>

testdata/simple.py.ol.html

@@ -1,7 +1,7 @@
 <ol>
 <li><span class="kwd">from</span> <span class="pln">foo</span> <span class="kwd">import</span> <span class="pln">bar</span></li>
 <li></li>
-<li><span class="kwd">def</span> <span class="pln">f</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span> <span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">)</span><span class="pun">:</span></li>
+<li><span class="kwd">def</span> <span class="pln">f</span><span class="pun">(</span><span class="kwd">self</span><span class="pun">,</span> <span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">):</span></li>
 <li>    <span class="kwd">print</span><span class="pun">(</span><span class="str">&#39;hello!&#39;</span><span class="pun">)</span></li>
 <li></li>
 </ol>

testdata/simple.rb.html

@@ -8,5 +8,5 @@
 
 <span class="kwd">def</span> <span class="pln">foo</span><span class="pun">(</span><span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">)</span>
   <span class="pln">puts</span> <span class="pln">a</span>
-  <span class="typ">A</span><span class="pun">:</span><span class="pun">:</span><span class="typ">B</span>
+  <span class="typ">A</span><span class="pun">::</span><span class="typ">B</span>
 <span class="kwd">end</span>

testdata/simple.rb.ol.html

@@ -9,7 +9,7 @@
 <li></li>
 <li><span class="kwd">def</span> <span class="pln">foo</span><span class="pun">(</span><span class="pln">a</span><span class="pun">,</span> <span class="pln">b</span><span class="pun">)</span></li>
 <li>  <span class="pln">puts</span> <span class="pln">a</span></li>
-<li>  <span class="typ">A</span><span class="pun">:</span><span class="pun">:</span><span class="typ">B</span></li>
+<li>  <span class="typ">A</span><span class="pun">::</span><span class="typ">B</span></li>
 <li><span class="kwd">end</span></li>
 <li></li>
 </ol>

testdata/underscore.go.html

@@ -2,7 +2,7 @@
 
 <span class="kwd">package</span> <span class="pln">foo_bar</span>
 
-<span class="kwd">func</span> <span class="pln">foo</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span>
-	<span class="kwd">for</span> <span class="pln">_</span><span class="pun">,</span> <span class="pln">a</span> <span class="pun">:</span><span class="pun">=</span> <span class="pln">range</span> <span class="pln">foo</span> <span class="pun">{</span>
+<span class="kwd">func</span> <span class="pln">foo</span><span class="pun">()</span> <span class="pun">{</span>
+	<span class="kwd">for</span> <span class="pln">_</span><span class="pun">,</span> <span class="pln">a</span> <span class="pun">:=</span> <span class="pln">range</span> <span class="pln">foo</span> <span class="pun">{</span>
 	<span class="pun">}</span>
 <span class="pun">}</span>

testdata/underscore.go.ol.html

@@ -3,8 +3,8 @@
 <li></li>
 <li><span class="kwd">package</span> <span class="pln">foo_bar</span></li>
 <li></li>
-<li><span class="kwd">func</span> <span class="pln">foo</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span></li>
-<li>	<span class="kwd">for</span> <span class="pln">_</span><span class="pun">,</span> <span class="pln">a</span> <span class="pun">:</span><span class="pun">=</span> <span class="pln">range</span> <span class="pln">foo</span> <span class="pun">{</span></li>
+<li><span class="kwd">func</span> <span class="pln">foo</span><span class="pun">()</span> <span class="pun">{</span></li>
+<li>	<span class="kwd">for</span> <span class="pln">_</span><span class="pun">,</span> <span class="pln">a</span> <span class="pun">:=</span> <span class="pln">range</span> <span class="pln">foo</span> <span class="pun">{</span></li>
 <li>	<span class="pun">}</span></li>
 <li><span class="pun">}</span></li>
 <li></li>

testdata/utf8.go.html

@@ -11,6 +11,6 @@
 <span class="com">// ᚷᛁᚠ᛫ᚻᛖ᛫ᚹᛁᛚᛖ᛫ᚠᚩᚱ᛫ᛞᚱᛁᚻᛏᚾᛖ᛫ᛞᚩᛗᛖᛋ᛫ᚻᛚᛇᛏᚪᚾ᛬</span>
 <span class="kwd">var</span> <span class="typ">B</span> <span class="pun">=</span> <span class="str">&#34;Τὴ γλῶσσα μοῦ ἔδωσαν ἑλληνικὴ&#34;</span>
 
-<span class="kwd">func</span> <span class="typ">F</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span>
+<span class="kwd">func</span> <span class="typ">F</span><span class="pun">()</span> <span class="pun">{</span>
 	<span class="pln">fmt</span><span class="pun">.</span><span class="typ">Println</span><span class="pun">(</span><span class="typ">A</span><span class="pun">,</span> <span class="typ">B</span><span class="pun">)</span>
 <span class="pun">}</span>

testdata/utf8.go.ol.html

@@ -12,7 +12,7 @@
 <li><span class="com">// ᚷᛁᚠ᛫ᚻᛖ᛫ᚹᛁᛚᛖ᛫ᚠᚩᚱ᛫ᛞᚱᛁᚻᛏᚾᛖ᛫ᛞᚩᛗᛖᛋ᛫ᚻᛚᛇᛏᚪᚾ᛬</span></li>
 <li><span class="kwd">var</span> <span class="typ">B</span> <span class="pun">=</span> <span class="str">&#34;Τὴ γλῶσσα μοῦ ἔδωσαν ἑλληνικὴ&#34;</span></li>
 <li></li>
-<li><span class="kwd">func</span> <span class="typ">F</span><span class="pun">(</span><span class="pun">)</span> <span class="pun">{</span></li>
+<li><span class="kwd">func</span> <span class="typ">F</span><span class="pun">()</span> <span class="pun">{</span></li>
 <li>	<span class="pln">fmt</span><span class="pun">.</span><span class="typ">Println</span><span class="pun">(</span><span class="typ">A</span><span class="pun">,</span> <span class="typ">B</span><span class="pun">)</span></li>
 <li><span class="pun">}</span></li>
 <li></li>