12. Token
// Token is the set of lexical tokens of the Go programming language.
type Token int
// The list of tokens.
const (
// Special tokens
ILLEGAL Token = iota
EOF
COMMENT
literal_beg
// Identifiers and basic type literals
// (these tokens stand for classes of literals)
IDENT // main
INT // 12345
FLOAT // 123.45
IMAG // 123.45i
CHAR // 'a'
STRING // "abc"
literal_end
...
16. Position!and!Pos
// Position describes an arbitrary source position
// including the file, line, and column location.
// A Position is valid if the line number is > 0.
type Position struct {
// filename, if any
Filename string
// offset, starting at 0
Offset int
// line number, starting at 1
Line int
// column number, starting at 1 (character count)
Column int
}
// Offset in FileSet
type Pos int
17. File
// A File is a handle for a file belonging to a FileSet.
// A File has a name, size, and line offset table.
type File struct {
set *FileSet
// file name as provided to AddFile
name string
// Pos value range for this file is
// [base...base+size]
base int
// file size as provided to AddFile
size int
// lines and infos are protected by set.mutex
// lines contains the offset of the first character
// for each line (the first entry is always 0)
lines []int
infos []lineInfo
}
18. FileSet
// A FileSet represents a set of source files.
// Methods of file sets are synchronized; multiple
// goroutines may invoke them concurrently.
type FileSet struct {
// protects the file set
mutex sync.RWMutex
// base offset for the next file
base int
// list of files in the order added to the set
files []*File
// cache of last file looked up
last *File
}
19. FileSet!methods
// AddFile adds a new file with a given filename, base offset,
// and file size to the file set s and returns the file.
func (s *FileSet) AddFile(filename string, base, size int) *File
// Read calls decode to deserialize a file set into s; s must
// not be nil.
func (s *FileSet) Read(decode func(interface{}) error) error
// Write calls encode to serialize the file set s.
func (s *FileSet) Write(encode func(interface{}) error) error
21. Scanner
type Scanner struct {
// immutable state
file *token.File // source file handle
dir string // directory portion of file.Name()
src []byte // source
err ErrorHandler // error reporting; or nil
mode Mode // scanning mode
// scanning state
ch rune // current character
offset int // character offset
rdOffset int // reading offset (position after current character)
lineOffset int // current line offset
insertSemi bool // insert a semicolon before next newline
// public state - ok to modify
ErrorCount int // number of errors encountered
}
22. Scanner!methods
func (s *Scanner) Init(file *token.File,
src []byte,
err ErrorHandler,
mode Mode)
// Scan scans the next token and returns the token
// position, the token, and its literal string if
// applicable. The source end is indicated by
// token.EOF.
func (s *Scanner) Scan() (pos token.Pos,
tok token.Token,
lit string)
23. Mode
type Mode uint
const (
// return comments as COMMENT tokens
ScanComments Mode = 1 << iota
// do not automatically insert semicolons
// - for testing only
dontInsertSemis
)
24. ErrorHandler,"Error"and"
ErrorList
// An ErrorHandler may be provided to Scanner.Init. If a
// syntax error is encountered and a handler was installed,
// the handler is called with a position and an error
// message. The position points to the beginning of the
// offending token.
type ErrorHandler func(pos token.Position, msg string)
type Error struct {
Pos token.Position
Msg string
}
type ErrorList []*Error
25. Usage&example&in&
scanner_test.go
func ExampleScanner_Scan() {
// src is the input that we want to tokenize.
src := []byte("cos(x) + 1i*sin(x) // Euler")
// Initialize the scanner.
var s scanner.Scanner
fset := token.NewFileSet() // positions are relative to fset
file := fset.AddFile("", fset.Base(), len(src)) // register input "file"
s.Init(file, src, nil /* no error handler */, scanner.ScanComments)
// Repeated calls to Scan yield the token sequence found in the input.
for {
pos, tok, lit := s.Scan()
if tok == token.EOF { break }
fmt.Printf("%st%st%qn", fset.Position(pos), tok, lit)
}
}
30. Interfaces
// All node types implement the Node interface.
type Node interface {
Pos() token.Pos // position of first character belonging to the node
End() token.Pos // position of first character immediately after the node
}
// All expression nodes implement the Expr interface.
type Expr interface {
Node
exprNode()
}
// All statement nodes implement the Stmt interface.
type Stmt interface {
Node
stmtNode()
}
// All declaration nodes implement the Decl interface.
type Decl interface {
Node
declNode()
}
35. Comment!and!CommentGroup
// A Comment node represents a single //-style or /*-style comment.
type Comment struct {
// position of "/" starting the comment
Slash token.Pos
// comment text (excluding 'n' for //-style comments)
Text string
}
// A CommentGroup represents a sequence of comments
// with no other tokens and no empty lines between.
//
type CommentGroup struct {
List []*Comment // len(List) > 0
}
37. Filed!and!FieldList
// A Field represents a Field declaration list in a struct type,
// a method list in an interface type, or a parameter/result
// declaration in a signature.
type Field struct {
Doc *CommentGroup // associated documentation; or nil
Names []*Ident // field/method/parameter names; or nil if anonymous field
Type Expr // field/method/parameter type
Tag *BasicLit // field tag; or nil
Comment *CommentGroup // line comments; or nil
}
// A FieldList represents a list of Fields, enclosed
// by parentheses or braces.
type FieldList struct {
Opening token.Pos // position of opening parenthesis/brace, if any
List []*Field // field list; or nil
Closing token.Pos // position of closing parenthesis/brace, if any
}
39. Types
// The direction of a channel type is indicated by one
// of the following constants.
type ChanDir int
const (
SEND ChanDir = 1 << iota
RECV
)
type (
ArrayType struct { ... }
StructType struct { ... }
FuncType struct { ... }
InterfaceType struct { ... }
MapType struct { ... }
ChanType struct { ... }
)
45. File!and!Package
type File struct {
Doc *CommentGroup // associated documentation; or nil
Package token.Pos // position of "package" keyword
Name *Ident // package name
Decls []Decl // top-level declarations; or nil
Scope *Scope // package scope (this file only)
Imports []*ImportSpec // imports in this file
Unresolved []*Ident // unresolved identifiers in this file
Comments []*CommentGroup // list of all comments in the source file
}
type Package struct {
Name string // package name
Scope *Scope // package scope across all files
Imports map[string]*Object // map of package id -> package object
Files map[string]*File // Go source files by filename
}
48. parser
// The parser structure holds the parser's internal state.
type parser struct {
file *token.File
errors scanner.ErrorList
scanner scanner.Scanner
...
// Next token
pos token.Pos // token position
tok token.Token // one token look-ahead
lit string // token literal
...
// Ordinary identifier scopes
pkgScope *ast.Scope // pkgScope.Outer == nil
topScope *ast.Scope // top-most scope; may be pkgScope
unresolved []*ast.Ident // unresolved identifiers
imports []*ast.ImportSpec // list of imports
// Label scopes
// (maintained by open/close LabelScope)
labelScope *ast.Scope // label scope for current function
targetStack [][]*ast.Ident // stack of unresolved labels
}
49. Parsing(func,ons
func ParseFile(fset *token.FileSet, filename string,
src interface{}, mode Mode) (f *ast.File, err error)
func ParseDir(fset *token.FileSet, path string,
filter func(os.FileInfo) bool,
mode Mode) (pkgs map[string]*ast.Package, first error)
// ParseExpr is a convenience function for obtaining
// the AST of an expression x.
func ParseExpr(x string) (ast.Expr, error)
50. Mode
type Mode uint
const (
// stop parsing after package clause
PackageClauseOnly Mode = 1 << iota
// stop parsing after import declarations
ImportsOnly
// parse comments and add them to AST
ParseComments
// print a trace of parsed productions
Trace
// report declaration errors
DeclarationErrors
// same as AllErrors, for backward-compatibility
SpuriousErrors
// report all errors (not just the first 10 on different lines)
AllErrors = SpuriousErrors
)
51. Usage&example&in&
example_test.go
var fset = token.NewFileSet()
var validFiles = []string{
"parser.go",
"parser_test.go",
"error_test.go",
"short_test.go",
}
func TestParse(t *testing.T) {
for _, filename := range validFiles {
_, err := ParseFile(fset, filename, nil, DeclarationErrors)
if err != nil {
t.Fatalf("ParseFile(%s): %v", filename, err)
}
}
}
52. Usage&example&in&
example_test.go
// This example shows what an AST looks like when printed for debugging.
func ExamplePrint() {
// src is the input for which we want to print the AST.
src := `
package main
func main() {
println("Hello, World!")
}
`
// Create the AST by parsing src.
fset := token.NewFileSet() // positions are relative to fset
f, err := parser.ParseFile(fset, "", src, 0)
if err != nil {
panic(err)
}
// Print the AST.
ast.Print(fset, f)
}
56. Fprint
// Fprint "pretty-prints" an AST node to output.
// It calls Config.Fprint with default settings.
func Fprint(output io.Writer,
fset *token.FileSet,
node interface{}) error
57. Config!and!Mode
// A Config node controls the output of Fprint.
type Config struct {
Mode Mode // default: 0
Tabwidth int // default: 8
// default: 0 (all code is indented at least by this much)
Indent int
}
func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet,
node interface{}) error
// A Mode value is a set of flags (or 0). They control printing.
type Mode uint
const (
// do not use a tabwriter; if set, UseSpaces is ignored
RawFormat Mode = 1 << iota
// use tabs for indentation independent of UseSpaces
TabIndent
// use spaces instead of tabs for alignment
UseSpaces
// emit //line comments to preserve original source positions
SourcePos
)
58. CommentedNode
// A CommentedNode bundles an AST node and corresponding comments.
// It may be provided as argument to any of the Fprint functions.
type CommentedNode struct {
// *ast.File, or ast.Expr, ast.Decl, ast.Spec, or ast.Stmt
Node interface{}
Comments []*ast.CommentGroup
}
60. doc
Types&of&doc
type Func struct { ... } // func foo() { ... }
type Note struct { ... } // TODO(tfukushima): ...
type Package struct { ... } // package bar
type Type struct { ... } // type
type Value struct { ... } // var, const
61.
62. Package!methods
func New(pkg *ast.Package, importPath string, mode Mode) *Package
func (p *Package) Filter(f Filter)
type Filter func(string) bool
// Mode values control the operation of New.
type Mode int
const (
// extract documentation for all package-level
// declarations, not just exported ones
AllDecls Mode = 1 << iota
// show all embedded methods, not just the ones of
// invisible (unexported) anonymous fields
AllMethods
)
64. Example
// An Example represents an example function found in a source files.
type Example struct {
Name string // name of the item being exemplified
Doc string // example function doc string
Code ast.Node
Play *ast.File // a whole program version of the example
Comments []*ast.CommentGroup
Output string // expected output
EmptyOutput bool // expect empty output
Order int // original source code order
}
// Examples returns the examples found in the files, sorted by
// Name field.
func Examples(files ...*ast.File) []*Example
65. Synopsis
// Synopsis returns a cleaned version of the first sentence in s.
func Synopsis(s string) string
var IllegalPrefixes = []string{
"copyright",
"all rights",
"author",
}
66. ToHTML!and!ToText
// ToHTML converts comment text to formatted HTML.
func ToHTML(w io.Writer, text string,
words map[string]string)
// ToText prepares comment text for presentation in textual
// output.
func ToText(w io.Writer, text string, indent,
preIndent string, width int)
68. Node!and!Source
// Node formats node in canonical gofmt style and writes the result
// to dst.
func Node(dst io.Writer, fset *token.FileSet, node interface{}) error
// Source formats src in canonical gofmt style and returns the result
// or an (I/O or syntax) error.
func Source(src []byte) ([]byte, error)
74. Build&constraints
During'a'par*cular'build,'the'following'words'are'
sa*sfied:
- the target operating system, as spelled by runtime.GOOS
- the target architecture, as spelled by runtime.GOARCH
- the compiler being used, either "gc" or "gccgo"
- "cgo", if ctxt.CgoEnabled is true
- "go1.1", from Go version 1.1 onward
- "go1.2", from Go version 1.2 onward
- "go1.3", from Go version 1.3 onward
- any additional words listed in ctxt.BuildTags
76. Package
// A Package describes the Go package found in a directory.
type Package struct {
Dir string // directory containing package sources
Name string // package name
Doc string // documentation synopsis
ImportPath string // import path of package ("" if unknown)
Root string // root of Go tree where this package lives
SrcRoot string // package source root directory ("" if unknown)
PkgRoot string // package install root directory ("" if unknown)
BinDir string // command install directory ("" if unknown)
Goroot bool // package found in Go root
PkgObj string // installed .a file
AllTags []string // tags that can influence file selection in this directory
ConflictDir string // this directory shadows Dir in $GOPATH
...
}
// IsCommand reports whether the package is considered a command to be installed
// (not just a library). Packages named "main" are treated as commands.
func (p *Package) IsCommand() bool {
return p.Name == "main"
}
77. Context
// A Context specifies the supporting context for a build.
type Context struct {
GOARCH string // target architecture
GOOS string // target operating system
GOROOT string // Go root
GOPATH string // Go path
CgoEnabled bool // whether cgo can be used
UseAllFiles bool // use files regardless of +build lines, file names
Compiler string // compiler to assume when computing target paths
...
}
// Default is the default Context for builds. It uses the GOARCH, GOOS,
// GOROOT, and GOPATH environment variables if set, or else the compiled
// code's GOARCH, GOOS, and GOROOT.
var Default Context = defaultContext()
78. Context!methods
func (ctxt *Context) Import(path string, srcDir string,
mode ImportMode) (*Package, error)
func (ctxt *Context) ImportDir(dir string,
mode ImportMode) (*Package, error)
func (ctxt *Context) MatchFile(dir, name string) (match bool, err error)
// SrcDirs returns a list of package source root directories. It
// draws from the current Go root and Go path but omits directories
// that do not exist.
func (ctxt *Context) SrcDirs() []string
79. ImportMode
type ImportMode uint
const (
// If FindOnly is set, Import stops after locating the directory
// that should contain the sources for a package. It does not
// read any files in the directory.
FindOnly ImportMode = 1 << iota
// If AllowBinary is set, Import can be satisfied by a compiled
// package object without corresponding sources.
AllowBinary
)
80. NoGoError
// NoGoError is the error used by Import to describe a directory
// containing no buildable Go source files. (It may still contain
// test files, files hidden by build tags, and so on.)
type NoGoError struct {
Dir string
}
func (e *NoGoError) Error() string {
return "no buildable Go source files in " + e.Dir
}
81. Import!and!ImportDir
// Import is shorthand for Default.Import.
func Import(path, srcDir string, mode ImportMode) (*Package, error)
// Import is shorthand for Default.ImportDir.
func ImportDir(dir string, mode ImportMode) (*Package, error)
82. Misc
// ToolDir is the directory containing build tools.
var ToolDir = filepath.Join(runtime.GOROOT(),
"pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH)
// ArchChar returns the architecture character for the given goarch.
// For example, ArchChar("amd64") returns "6".
func ArchChar(goarch string) (string, error)
// IsLocalImport reports whether the import path is
// a local import path, like ".", "..", "./foo", or "../foo".
func IsLocalImport(path string) bool