Add BISON-style location tracking

Author: danswartzendruber

cmd/eg/eg.go

@@ -10,16 +10,18 @@ package main // import "golang.org/x/tools/cmd/eg"
 import (
 	"flag"
 	"fmt"
-	"go/build"
+	"go/ast"
 	"go/format"
 	"go/parser"
 	"go/token"
-	exec "golang.org/x/sys/execabs"
+	"go/types"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 
-	"golang.org/x/tools/go/buildutil"
-	"golang.org/x/tools/go/loader"
+	exec "golang.org/x/sys/execabs"
+	"golang.org/x/tools/go/packages"
 	"golang.org/x/tools/refactor/eg"
 )
 
@@ -32,13 +34,9 @@ var (
 	verboseFlag    = flag.Bool("v", false, "show verbose matcher diagnostics")
 )
 
-func init() {
-	flag.Var((*buildutil.TagsFlag)(&build.Default.BuildTags), "tags", buildutil.TagsFlagDoc)
-}
-
 const usage = `eg: an example-based refactoring tool.
 
-Usage: eg -t template.go [-w] [-transitive] <args>...
+Usage: eg -t template.go [-w] [-transitive] <packages>
 
 -help            show detailed help message
 -t template.go	 specifies the template file (use -help to see explanation)
@@ -47,7 +45,7 @@ Usage: eg -t template.go [-w] [-transitive] <args>...
 -v               show verbose matcher diagnostics
 -beforeedit cmd  a command to exec before each file is modified.
                  "{}" represents the name of the file.
-` + loader.FromArgsUsage
+`
 
 func main() {
 	if err := doMain(); err != nil {
@@ -74,51 +72,73 @@ func doMain() error {
 		return fmt.Errorf("no -t template.go file specified")
 	}
 
-	conf := loader.Config{
-		Fset:       token.NewFileSet(),
-		ParserMode: parser.ParseComments,
+	tAbs, err := filepath.Abs(*templateFlag)
+	if err != nil {
+		return err
+	}
+	template, err := ioutil.ReadFile(tAbs)
+	if err != nil {
+		return err
 	}
 
-	// The first Created package is the template.
-	conf.CreateFromFilenames("template", *templateFlag)
+	cfg := &packages.Config{
+		Fset:  token.NewFileSet(),
+		Mode:  packages.NeedTypesInfo | packages.NeedName | packages.NeedTypes | packages.NeedSyntax | packages.NeedImports | packages.NeedDeps | packages.NeedCompiledGoFiles,
+		Tests: true,
+	}
 
-	if _, err := conf.FromArgs(args, true); err != nil {
+	pkgs, err := packages.Load(cfg, args...)
+	if err != nil {
 		return err
 	}
 
-	// Load, parse and type-check the whole program.
-	iprog, err := conf.Load()
+	tFile, err := parser.ParseFile(cfg.Fset, tAbs, template, parser.ParseComments)
+	if err != nil {
+		return err
+	}
+
+	// Type-check the template.
+	tInfo := types.Info{
+		Types:      make(map[ast.Expr]types.TypeAndValue),
+		Defs:       make(map[*ast.Ident]types.Object),
+		Uses:       make(map[*ast.Ident]types.Object),
+		Implicits:  make(map[ast.Node]types.Object),
+		Selections: make(map[*ast.SelectorExpr]*types.Selection),
+		Scopes:     make(map[ast.Node]*types.Scope),
+	}
+	conf := types.Config{
+		Importer: pkgsImporter(pkgs),
+	}
+	tPkg, err := conf.Check("egtemplate", cfg.Fset, []*ast.File{tFile}, &tInfo)
 	if err != nil {
 		return err
 	}
 
 	// Analyze the template.
-	template := iprog.Created[0]
-	xform, err := eg.NewTransformer(iprog.Fset, template.Pkg, template.Files[0], &template.Info, *verboseFlag)
+	xform, err := eg.NewTransformer(cfg.Fset, tPkg, tFile, &tInfo, *verboseFlag)
 	if err != nil {
 		return err
 	}
 
 	// Apply it to the input packages.
-	var pkgs []*loader.PackageInfo
+	var all []*packages.Package
 	if *transitiveFlag {
-		for _, info := range iprog.AllPackages {
-			pkgs = append(pkgs, info)
-		}
+		packages.Visit(pkgs, nil, func(p *packages.Package) { all = append(all, p) })
 	} else {
-		pkgs = iprog.InitialPackages()
+		all = pkgs
 	}
 	var hadErrors bool
 	for _, pkg := range pkgs {
-		if pkg == template {
-			continue
-		}
-		for _, file := range pkg.Files {
-			n := xform.Transform(&pkg.Info, pkg.Pkg, file)
+		for i, filename := range pkg.CompiledGoFiles {
+			if filename == tAbs {
+				// Don't rewrite the template file.
+				continue
+			}
+			file := pkg.Syntax[i]
+			n := xform.Transform(pkg.TypesInfo, pkg.Types, file)
 			if n == 0 {
 				continue
 			}
-			filename := iprog.Fset.File(file.Pos()).Name()
 			fmt.Fprintf(os.Stderr, "=== %s (%d matches)\n", filename, n)
 			if *writeFlag {
 				// Run the before-edit command (e.g. "chmod +w",  "checkout") if any.
@@ -138,17 +158,34 @@ func doMain() error {
 							args, err)
 					}
 				}
-				if err := eg.WriteAST(iprog.Fset, filename, file); err != nil {
+				if err := eg.WriteAST(cfg.Fset, filename, file); err != nil {
 					fmt.Fprintf(os.Stderr, "eg: %s\n", err)
 					hadErrors = true
 				}
 			} else {
-				format.Node(os.Stdout, iprog.Fset, file)
+				format.Node(os.Stdout, cfg.Fset, file)
 			}
 		}
 	}
 	if hadErrors {
 		os.Exit(1)
 	}
+
 	return nil
 }
+
+type pkgsImporter []*packages.Package
+
+func (p pkgsImporter) Import(path string) (tpkg *types.Package, err error) {
+	packages.Visit([]*packages.Package(p), func(pkg *packages.Package) bool {
+		if pkg.PkgPath == path {
+			tpkg = pkg.Types
+			return false
+		}
+		return true
+	}, nil)
+	if tpkg != nil {
+		return tpkg, nil
+	}
+	return nil, fmt.Errorf("package %q not found", path)
+}

cmd/godoc/handlers.go

@@ -14,11 +14,9 @@ import (
 	"golang.org/x/tools/godoc"
 	"golang.org/x/tools/godoc/redirect"
 	"golang.org/x/tools/godoc/vfs"
-)
 
-// This package registers "/compile" and "/share" handlers
-// that redirect to the golang.org playground.
-import _ "golang.org/x/tools/playground"
+	_ "golang.org/x/tools/playground" // register "/compile" playground redirect
+)
 
 var (
 	pres *godoc.Presentation

cmd/godoc/main.go

@@ -192,8 +192,10 @@ func main() {
 	}
 	if *templateDir != "" {
 		fs.Bind("/lib/godoc", vfs.OS(*templateDir), "/", vfs.BindBefore)
+		fs.Bind("/favicon.ico", vfs.OS(*templateDir), "/favicon.ico", vfs.BindReplace)
 	} else {
 		fs.Bind("/lib/godoc", mapfs.New(static.Files), "/", vfs.BindReplace)
+		fs.Bind("/favicon.ico", mapfs.New(static.Files), "/favicon.ico", vfs.BindReplace)
 	}
 
 	// Get the GOMOD value, use it to determine if godoc is being invoked in module mode.

cmd/goyacc/doc.go

@@ -33,6 +33,25 @@ to be given an argument that conforms to the following interface:
 		Error(e string)
 	}
 
+    The parser will also define the following
+
+	type yyLexerErrLoc interface {
+		Lex(lval *yySymType) int
+		Error(e string)
+		ErrorLoc(e string, loc *yySymLoc)
+	}
+
+If token location tracking in error messages is desired, the lexer
+should also define the following method, in addition to Error:
+
+func (yylex *Lexer) ErrorLoc(e string, errLoc *yySymLoc) {
+...
+}
+
+If this method is present, the generated parser will call the ErrorLoc
+method, passing it the error string, as well as the token location
+information for the offending token.
+
 Lex should return the token identifier, and place other token
 information in lval (which replaces the usual yylval).
 Error is equivalent to yyerror in the original yacc.
@@ -66,5 +85,44 @@ symbols, including types, the parser, and the lexer, generated and
 referenced by yacc's generated code.  Setting it to distinct values
 allows multiple grammars to be placed in a single package.
 
+goyacc will generate a parser compatible with bison's token location
+tracking semantics.  For more details:
+
+https://www.gnu.org/software/bison/manual/html_node/Tracking-Locations.html
+https://www.gnu.org/software/bison/manual/html_node/Token-Locations.html
+
+The generated Go parser will define two types:
+
+type yyPos struct {
+        line   int
+        column int
+}
+
+type yySymLoc struct {
+        pos yyPos
+        end yyPos
+}
+
+The pos field refers to the beginning of the token in question,
+and the end field to the end it.  To avoid having to change the
+definition of the lexer's Error method, just before the parser calls
+the Error method, it will set a global variable, yyErrLoc to the
+address of the problematic token's yySymLoc structure.  Since the
+lexer provides location information to the parser, and in turn is
+provided it, if needed, it's up to the lexer to do so consistently.
+
+As in the above-cited BISON web pages, goyacc will support the use
+of @N, where N is an integer from 1 to 9, and will be expanded in
+the generated parser to the appropriate variable. If an action rule
+wants to print a specific error message, the lexer should be written
+to provide one to the parser.
+
+If goyacc was invoked with an explicit prefix, via the '-p' switch,
+the above types and variables will have the appropriate prefix.
+
+The token tracking structure yySymLoc is stored inside the yySymType
+structure.  This simplifies the changes, since goyacc already has to
+copy the structure in question.
+
 */
 package main

cmd/goyacc/yacc.go

@@ -386,7 +386,7 @@ func setup() {
 		fmt.Fprintf(stderr, "yacc: stack size too small\n")
 		usage()
 	}
-	yaccpar = strings.Replace(yaccpartext, "$$", prefix, -1)
+	yaccpar = strings.ReplaceAll(yaccpartext, "$$", prefix)
 	openup()
 
 	fmt.Fprintf(ftable, "// Code generated by goyacc %s. DO NOT EDIT.\n", strings.Join(os.Args[1:], " "))
@@ -1065,6 +1065,15 @@ func chfind(t int, s string) int {
 // copy the union declaration to the output, and the define file if present
 //
 func cpyunion() {
+	fmt.Fprintf(ftable, "\ntype %sPos struct {", prefix)
+	fmt.Fprintf(ftable, "\n\tline\tint")
+	fmt.Fprintf(ftable, "\n\tcolumn\tint")
+	fmt.Fprintf(ftable, "\n}\n")
+
+	fmt.Fprintf(ftable, "\ntype %sSymLoc struct {", prefix)
+	fmt.Fprintf(ftable, "\n\tpos %sPos", prefix)
+	fmt.Fprintf(ftable, "\n\tend %sPos", prefix)
+	fmt.Fprintf(ftable, "\n}\n")
 
 	if !lflag {
 		fmt.Fprintf(ftable, "\n//line %v:%v\n", infile, lineno)
@@ -1085,7 +1094,8 @@ out:
 			lineno++
 		case '{':
 			if level == 0 {
-				fmt.Fprintf(ftable, "\n\tyys int")
+				fmt.Fprintf(ftable, "\n\tyys\tint")
+				fmt.Fprintf(ftable, "\n\tsymLoc\t%sSymLoc", prefix)
 			}
 			level++
 		case '}':
@@ -1308,6 +1318,29 @@ loop:
 		case '{':
 			brac++
 
+		case '@':
+			n := 0
+			c2 := getrune(finput)
+			if isdigit(c2) {
+				for isdigit(c2) {
+					n = n*10 + int(c2-'0')
+					c2 = getrune(finput)
+				}
+				ungetrune(finput, c2)
+
+				if n < 1 || n >= max {
+					errorf("Illegal use of @%v", n)
+				}
+
+				fmt.Fprintf(fcode, "%sDollar[%d].symLoc", prefix, n)
+				continue loop
+			} else {
+				ungetrune(finput, c2)
+				if gettok() != NUMBER {
+					errorf("@ must be followed by number")
+				}
+			}
+
 		case '$':
 			s := 1
 			tok := -1
@@ -3287,6 +3320,12 @@ type $$Lexer interface {
 	Error(s string)
 }
 
+type $$LexerErrLoc interface {
+	Lex(lval *$$SymType) int
+	Error(s string)
+	ErrorLoc(s string, loc *$$SymLoc)
+}
+
 type $$Parser interface {
 	Parse($$Lexer) int
 	Lookahead() int
@@ -3531,7 +3570,13 @@ $$default:
 		/* error ... attempt to resume parsing */
 		switch Errflag {
 		case 0: /* brand new error */
-			$$lex.Error($$ErrorMessage($$state, $$token))
+            errMsg := $$ErrorMessage($$state, $$token)
+            if errLoc, ok := $$lex.($$LexerErrLoc); ok {
+               errLoc.ErrorLoc(errMsg, &$$rcvr.lval.symLoc)
+            } else {
+               $$lex.Error(errMsg)
+            }
+
 			Nerrs++
 			if $$Debug >= 1 {
 				__yyfmt__.Printf("%s", $$Statname($$state))
@@ -3593,6 +3638,11 @@ $$default:
 	}
 	$$VAL = $$S[$$p+1]
 
+	$$VAL.symLoc.pos.line = $$S[$$p+1].symLoc.pos.line
+	$$VAL.symLoc.pos.column = $$S[$$p+1].symLoc.pos.column
+	$$VAL.symLoc.end.line = $$S[$$pt].symLoc.end.line
+	$$VAL.symLoc.end.column = $$S[$$pt].symLoc.end.column
+
 	/* consult goto table to find next state */
 	$$n = $$R1[$$n]
 	$$g := $$Pgo[$$n]