go.text/language: added function to parse the value of an HTTP Accept-Language

header.
It supports a few non-standard language tags that appear relatively frequently
in the Accept-Language headers.

R=r
CC=golang-dev, nigeltao
https://golang.org/cl/13974043

Author: mpvl

language/maketables.go

@@ -673,7 +673,7 @@ func (b *builder) writeLanguage() {
 	meta := b.supp.Metadata
 
 	b.writeConst("nonCanonicalUnd", b.lang.index("und"))
-	b.writeConsts("lang_", b.lang.index, "de", "en", "fil", "mo", "nb", "no", "sh", "sr", "tl")
+	b.writeConsts("lang_", b.lang.index, "de", "en", "fil", "fr", "it", "mo", "mul", "nb", "no", "sh", "sr", "tl")
 	b.writeConst("langPrivateStart", b.langIndex("qaa"))
 	b.writeConst("langPrivateEnd", b.langIndex("qtz"))
 

language/parse.go

@@ -8,7 +8,9 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 )
 
@@ -550,3 +552,77 @@ func nextExtension(s string, p int) int {
 	}
 	return len(s)
 }
+
+var (
+	acceptErr = errors.New("ParseAcceptLanguage: syntax error")
+	acceptRe  = regexp.MustCompile(`^ *(?:([\w-]+|\*)(?: *; *q *= *([0-9\.]+))?)? *$`)
+)
+
+// ParseAcceptLanguage parses the contents of a Accept-Language header as
+// defined in http://www.google.com/url?q=http://www.ietf.org/rfc/rfc2616.txt
+// and returns a list of Tags and a list of corresponding quality weights.
+// The Tags will be sorted by highest weight first and then by first occurrence.
+// Tags with a weight of zero will be dropped. An error will be returned if the
+// input could not be parsed.
+func ParseAcceptLanguage(s string) (tag []Tag, q []float32, err error) {
+	for start, end := 0, 0; start < len(s); start = end + 1 {
+		for end = start; end < len(s) && s[end] != ','; end++ {
+		}
+		m := acceptRe.FindStringSubmatch(s[start:end])
+		if m == nil {
+			return nil, nil, acceptErr
+		}
+		if len(m[1]) > 0 {
+			w := 1.0
+			if len(m[2]) > 0 {
+				if w, err = strconv.ParseFloat(m[2], 32); err != nil {
+					return nil, nil, err
+				}
+				// Drop tags with a quality weight of 0.
+				if w <= 0 {
+					continue
+				}
+			}
+			t, err := Parse(m[1])
+			if err != nil {
+				id, ok := acceptFallback[m[1]]
+				if !ok {
+					return nil, nil, err
+				}
+				t = Tag{lang: id}
+			}
+			tag = append(tag, t)
+			q = append(q, float32(w))
+		}
+	}
+	sort.Stable(&tagSort{tag, q})
+	return tag, q, nil
+}
+
+// Add hack mapping to deal with a small number of cases that that occur
+// in Accept-Language (with reasonable frequency).
+var acceptFallback = map[string]langID{
+	"english": lang_en,
+	"deutsch": lang_de,
+	"italian": lang_it,
+	"french":  lang_fr,
+	"*":       lang_mul, // defined in the spec to match all languages.
+}
+
+type tagSort struct {
+	tag []Tag
+	q   []float32
+}
+
+func (s *tagSort) Len() int {
+	return len(s.q)
+}
+
+func (s *tagSort) Less(i, j int) bool {
+	return s.q[i] > s.q[j]
+}
+
+func (s *tagSort) Swap(i, j int) {
+	s.tag[i], s.tag[j] = s.tag[j], s.tag[i]
+	s.q[i], s.q[j] = s.q[j], s.q[i]
+}

language/parse_test.go

@@ -354,3 +354,112 @@ func TestCompose2(t *testing.T) {
 		}
 	})
 }
+
+func mk(s string) Tag {
+	id, _ := Parse(s)
+	return id
+}
+
+func TestParseAcceptLanguage(t *testing.T) {
+	type res struct {
+		t Tag
+		q float32
+	}
+	en := []res{{mk("en"), 1.0}}
+	tests := []struct {
+		out []res
+		in  string
+		ok  bool
+	}{
+		{en, "en", true},
+		{en, "   en", true},
+		{en, "en   ", true},
+		{en, "  en  ", true},
+		{en, "en,", true},
+		{en, ",en", true},
+		{en, ",,,en,,,", true},
+		{en, ",en;q=1", true},
+		// We allow an empty input, contrary to spec.
+		{nil, "", true},
+
+		// errors
+		{nil, ";", false},
+		{nil, "$", false},
+		{nil, "e;", false},
+		{nil, "x;", false},
+		{nil, "x", false},
+		{nil, "ac", false}, // non-existing language
+		{nil, "aa;", false},
+		{nil, "aa;q", false},
+		{nil, "aa;q=", false},
+		{nil, "aa;q=.", false},
+
+		// odd fallbacks
+		{
+			[]res{{mk("en"), 0.1}},
+			" english ;q=.1",
+			true,
+		},
+		{
+			[]res{{mk("it"), 1.0}, {mk("de"), 1.0}, {mk("fr"), 1.0}},
+			" italian, deutsch, french",
+			true,
+		},
+
+		// lists
+		{
+			[]res{{mk("en"), 0.1}},
+			"en;q=.1",
+			true,
+		},
+		{
+			[]res{{mk("mul"), 1.0}},
+			"*",
+			true,
+		},
+		{
+			[]res{{mk("en"), 1.0}, {mk("de"), 1.0}},
+			"en,de",
+			true,
+		},
+		{
+			[]res{{mk("en"), 1.0}, {mk("de"), .5}},
+			"en,de;q=0.5",
+			true,
+		},
+		{
+			[]res{{mk("de"), 0.8}, {mk("en"), 0.5}},
+			"  en ;   q    =   0.5    ,  , de;q=0.8",
+			true,
+		},
+		{
+			[]res{{mk("en"), 1.0}, {mk("de"), 1.0}, {mk("fr"), 1.0}, {mk("tlh"), 1.0}},
+			"en,de,fr,i-klingon",
+			true,
+		},
+		// sorting
+		{
+			[]res{{mk("tlh"), 0.4}, {mk("de"), 0.2}, {mk("fr"), 0.2}, {mk("en"), 0.1}},
+			"en;q=0.1,de;q=0.2,fr;q=0.2,i-klingon;q=0.4",
+			true,
+		},
+		// dropping
+		{
+			[]res{{mk("fr"), 0.2}, {mk("en"), 0.1}},
+			"en;q=0.1,de;q=0,fr;q=0.2,i-klingon;q=0.0",
+			true,
+		},
+	}
+	for i, tt := range tests {
+		tags, qs, e := ParseAcceptLanguage(tt.in)
+		if e == nil != tt.ok {
+			t.Errorf("%d:%s:err: was %v; want %v", i, tt.in, e == nil, tt.ok)
+		}
+		for j, tag := range tags {
+			if out := tt.out[j]; !tag.equalTags(out.t) || qs[j] != out.q {
+				t.Errorf("%d:%s: was %s, %1f; want %s, %1f", i, tt.in, tag, qs[j], out.t, out.q)
+				break
+			}
+		}
+	}
+}