Gunosy.go #4 go

890 views
774 views

Published on

This is slides for Gunosy.go#4.

http://gunosygo.connpass.com/event/7290/

Published in: Engineering, Technology

Gunosy.go #4 go

  1. 1. go#package @takufukushima
  2. 2. go!package!hierarchy • ast • build • doc • format • parser • printer • scanner • token
  3. 3. go!package:!Sta+c!analysis • ast • build • doc • format • parser • printer • scanner • token
  4. 4. go!package:!U*li*es • ast • build • doc • format • parser • printer • scanner • token
  5. 5. Sta$c&analysis
  6. 6. Typical(compiler 1. Lexical(analysis((scanning;(tokenizing) 2. Parsing • Concrete,syntax,tree,(CST,,parse,tree) 3. Seman8c(analysis • Abstract,syntax,tree,(AST) 4. Code(genera8on 5. Code,op=miza=on
  7. 7. go!package 1. Lexical(analysis((scanning;(tokenizing) 2. Parsing • Concrete,syntax,tree,(CST,,parse,tree) 3. Seman8c(analysis,(Included,in,the,parser) • Abstract,syntax,tree,(AST) 4. ___(genera8on
  8. 8. ___"genera(on • Documenta+on,from,the,comment • String,representa+on,of,AST • Forma8ed,code • Package,informa+on,for,the,build
  9. 9. Sta$c&analysis 1. token 2. scanner 3. ast 4. parser
  10. 10. token 1. token 2. scanner 3. ast 4. parser
  11. 11. Token // Token is the set of lexical tokens of the Go programming language. type Token int // The list of tokens. const ( // Special tokens ILLEGAL Token = iota EOF COMMENT literal_beg // Identifiers and basic type literals // (these tokens stand for classes of literals) IDENT // main INT // 12345 FLOAT // 123.45 IMAG // 123.45i CHAR // 'a' STRING // "abc" literal_end ...
  12. 12. Tokens var tokens = [...]string{ ILLEGAL: "ILLEGAL", EOF: "EOF", COMMENT: "COMMENT", IDENT: "IDENT", INT: "INT", FLOAT: "FLOAT", IMAG: "IMAG", CHAR: "CHAR", STRING: "STRING", ADD: "+", SUB: "-", MUL: "*", QUO: "/", REM: "%", ...
  13. 13. Tokens'in'BNF AST Expression = UnaryExpr | Expression binary_op UnaryExpr . UnaryExpr = PrimaryExpr | unary_op UnaryExpr . Tokens binary_op = "||" | "&&" | rel_op | add_op | mul_op . rel_op = "==" | "!=" | "<" | "<=" | ">" | ">=" . add_op = "+" | "-" | "|" | "^" . mul_op = "*" | "/" | "%" | "<<" | ">>" | "&" | "&^" . unary_op = "+" | "-" | "!" | "^" | "*" | "&" | "<-" .
  14. 14. token!package!provides!basic!data! structures!for!scanner • Position • Pos • File • FileSet
  15. 15. Position!and!Pos // Position describes an arbitrary source position // including the file, line, and column location. // A Position is valid if the line number is > 0. type Position struct { // filename, if any Filename string // offset, starting at 0 Offset int // line number, starting at 1 Line int // column number, starting at 1 (character count) Column int } // Offset in FileSet type Pos int
  16. 16. File // A File is a handle for a file belonging to a FileSet. // A File has a name, size, and line offset table. type File struct { set *FileSet // file name as provided to AddFile name string // Pos value range for this file is // [base...base+size] base int // file size as provided to AddFile size int // lines and infos are protected by set.mutex // lines contains the offset of the first character // for each line (the first entry is always 0) lines []int infos []lineInfo }
  17. 17. FileSet // A FileSet represents a set of source files. // Methods of file sets are synchronized; multiple // goroutines may invoke them concurrently. type FileSet struct { // protects the file set mutex sync.RWMutex // base offset for the next file base int // list of files in the order added to the set files []*File // cache of last file looked up last *File }
  18. 18. FileSet!methods // AddFile adds a new file with a given filename, base offset, // and file size to the file set s and returns the file. func (s *FileSet) AddFile(filename string, base, size int) *File // Read calls decode to deserialize a file set into s; s must // not be nil. func (s *FileSet) Read(decode func(interface{}) error) error // Write calls encode to serialize the file set s. func (s *FileSet) Write(encode func(interface{}) error) error
  19. 19. scanner 1. token 2. scanner 3. ast 4. parser
  20. 20. Scanner type Scanner struct { // immutable state file *token.File // source file handle dir string // directory portion of file.Name() src []byte // source err ErrorHandler // error reporting; or nil mode Mode // scanning mode // scanning state ch rune // current character offset int // character offset rdOffset int // reading offset (position after current character) lineOffset int // current line offset insertSemi bool // insert a semicolon before next newline // public state - ok to modify ErrorCount int // number of errors encountered }
  21. 21. Scanner!methods func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) // Scan scans the next token and returns the token // position, the token, and its literal string if // applicable. The source end is indicated by // token.EOF. func (s *Scanner) Scan() (pos token.Pos, tok token.Token, lit string)
  22. 22. Mode type Mode uint const ( // return comments as COMMENT tokens ScanComments Mode = 1 << iota // do not automatically insert semicolons // - for testing only dontInsertSemis )
  23. 23. ErrorHandler,"Error"and" ErrorList // An ErrorHandler may be provided to Scanner.Init. If a // syntax error is encountered and a handler was installed, // the handler is called with a position and an error // message. The position points to the beginning of the // offending token. type ErrorHandler func(pos token.Position, msg string) type Error struct { Pos token.Position Msg string } type ErrorList []*Error
  24. 24. Usage&example&in& scanner_test.go func ExampleScanner_Scan() { // src is the input that we want to tokenize. src := []byte("cos(x) + 1i*sin(x) // Euler") // Initialize the scanner. var s scanner.Scanner fset := token.NewFileSet() // positions are relative to fset file := fset.AddFile("", fset.Base(), len(src)) // register input "file" s.Init(file, src, nil /* no error handler */, scanner.ScanComments) // Repeated calls to Scan yield the token sequence found in the input. for { pos, tok, lit := s.Scan() if tok == token.EOF { break } fmt.Printf("%st%st%qn", fset.Position(pos), tok, lit) } }
  25. 25. Result'of'the'usage'example // output: // 1:1 IDENT "cos" // 1:4 ( "" // 1:5 IDENT "x" // 1:6 ) "" // 1:8 + "" // 1:10 IMAG "1i" // 1:12 * "" // 1:13 IDENT "sin" // 1:16 ( "" // 1:17 IDENT "x" // 1:18 ) "" // 1:20 ; "n" // 1:20 COMMENT "// Euler"
  26. 26. ast 1. token 2. scanner 3. ast 4. parser
  27. 27. Syntax Read%the%spec:"h$p://golang.org/ref/spec
  28. 28. Interfaces // All node types implement the Node interface. type Node interface { Pos() token.Pos // position of first character belonging to the node End() token.Pos // position of first character immediately after the node } // All expression nodes implement the Expr interface. type Expr interface { Node exprNode() } // All statement nodes implement the Stmt interface. type Stmt interface { Node stmtNode() } // All declaration nodes implement the Decl interface. type Decl interface { Node declNode() }
  29. 29. Interfaces • Node • Expression • Statement • Declara4on • Comment6and6CommentGroup • File6and6Package
  30. 30. U"lity'func"ons func Walk(v Visitor, node Node) func Fprint(w io.Writer, fset *token.FileSet, x interface{}, f FieldFilter) (err error) func Print(fset *token.FileSet, x interface{}) error func FilterDecl(decl Decl, f Filter) bool func FilterFile(src *File, f Filter) bool func FilterPackage(pkg *Package, f Filter) bool func NotNilFilter(_ string, v reflect.Value) bool func Inspect(node Node, f func(Node) bool) func FileExports(src *File) bool func IsExported(name string) bool func PackageExports(pkg *Package) bool func SortImports(fset *token.FileSet, f *File)
  31. 31. Node%methods For$node$N$(e.g.,$N$=$Comment,$Field,$IdentExpr,$ SwitchStmt$and$so$on) // position of first character belonging to the node func (x *N) End() token.Pos // position of first character immediately after the node func (x *N) Pos() token.Pos and$for$CommentGroup // Text returns the text of the comment. func (g *CommentGroup) Text() string
  32. 32. Comment'and'CommentGroup • Node • Expression • Statement • Declara4on • Comment'and'CommentGroup • File6and6Package
  33. 33. Comment!and!CommentGroup // A Comment node represents a single //-style or /*-style comment. type Comment struct { // position of "/" starting the comment Slash token.Pos // comment text (excluding 'n' for //-style comments) Text string } // A CommentGroup represents a sequence of comments // with no other tokens and no empty lines between. // type CommentGroup struct { List []*Comment // len(List) > 0 }
  34. 34. Expression*and*types • Node • Expression • Statement • Declara/on • Comment1and1CommentGroup • File1and1Package
  35. 35. Filed!and!FieldList // A Field represents a Field declaration list in a struct type, // a method list in an interface type, or a parameter/result // declaration in a signature. type Field struct { Doc *CommentGroup // associated documentation; or nil Names []*Ident // field/method/parameter names; or nil if anonymous field Type Expr // field/method/parameter type Tag *BasicLit // field tag; or nil Comment *CommentGroup // line comments; or nil } // A FieldList represents a list of Fields, enclosed // by parentheses or braces. type FieldList struct { Opening token.Pos // position of opening parenthesis/brace, if any List []*Field // field list; or nil Closing token.Pos // position of closing parenthesis/brace, if any }
  36. 36. Expressions type ( BadExpr struct { ... } Ident struct { ... } Ellipsis struct { ... } BasicLit struct { ... } FuncLit struct { ... } CompositeLit struct { ... } ParenExpr struct { ... } SelectorExpr struct { ... } IndexExpr struct { ... } SliceExpr struct { ... } TypeAssertExpr struct { ... } CallExpr struct { .... } StarExpr struct { ... } UnaryExpr struct { ... } BinaryExpr struct { ... } KeyValueExpr struct { ... } )
  37. 37. Types // The direction of a channel type is indicated by one // of the following constants. type ChanDir int const ( SEND ChanDir = 1 << iota RECV ) type ( ArrayType struct { ... } StructType struct { ... } FuncType struct { ... } InterfaceType struct { ... } MapType struct { ... } ChanType struct { ... } )
  38. 38. Statement • Node • Expression • Statement • Declara1on • Comment5and5CommentGroup • File5and5Package
  39. 39. Statements type ( BadStmt struct { ... } DeclStmt struct { ... } EmptyStmt struct { ... } LabeledStmt struct { ... } ExprStmt struct { ... } SendStmt struct { ... } IncDecStmt struct { ... } AssignStmt struct { ... } GoStmt struct { ... } DeferStmt struct { ... } ReturnStmt struct { ... } BranchStmt struct { ... } BlockStmt struct {... } IfStmt struct { ... } CaseClause struct { ... } SwitchStmt struct { ... } TypeSwitchStmt struct { ... } CommClause struct { ... } SelectStmt struct { ... } ForStmt struct { ... } RangeStmt struct { ... } )
  40. 40. Declara'on • Node • Expression • Statement • Declara'on • Comment2and2CommentGroup • File2and2Package
  41. 41. Declara'on type ( // The Spec type stands for any of *ImportSpec, // *ValueSpec, and *TypeSpec. Spec interface { ... } ImportSpec struct { ... } ValueSpec struct { ... } TypeSpec struct { ... } ) type ( BadDecl struct { ... } GenDecl struct { ... } // General declaration node FuncDecl struct { ... } )
  42. 42. File%and%Package • Node • Expression • Statement • Declara4on • Comment6and6CommentGroup • File%and%Package
  43. 43. File!and!Package type File struct { Doc *CommentGroup // associated documentation; or nil Package token.Pos // position of "package" keyword Name *Ident // package name Decls []Decl // top-level declarations; or nil Scope *Scope // package scope (this file only) Imports []*ImportSpec // imports in this file Unresolved []*Ident // unresolved identifiers in this file Comments []*CommentGroup // list of all comments in the source file } type Package struct { Name string // package name Scope *Scope // package scope across all files Imports map[string]*Object // map of package id -> package object Files map[string]*File // Go source files by filename }
  44. 44. parser 1. token 2. scanner 3. ast 4. parser
  45. 45. parser // The parser structure holds the parser's internal state. type parser struct { file *token.File errors scanner.ErrorList scanner scanner.Scanner ... // Next token pos token.Pos // token position tok token.Token // one token look-ahead lit string // token literal ... // Ordinary identifier scopes pkgScope *ast.Scope // pkgScope.Outer == nil topScope *ast.Scope // top-most scope; may be pkgScope unresolved []*ast.Ident // unresolved identifiers imports []*ast.ImportSpec // list of imports // Label scopes // (maintained by open/close LabelScope) labelScope *ast.Scope // label scope for current function targetStack [][]*ast.Ident // stack of unresolved labels }
  46. 46. Parsing(func,ons func ParseFile(fset *token.FileSet, filename string, src interface{}, mode Mode) (f *ast.File, err error) func ParseDir(fset *token.FileSet, path string, filter func(os.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error) // ParseExpr is a convenience function for obtaining // the AST of an expression x. func ParseExpr(x string) (ast.Expr, error)
  47. 47. Mode type Mode uint const ( // stop parsing after package clause PackageClauseOnly Mode = 1 << iota // stop parsing after import declarations ImportsOnly // parse comments and add them to AST ParseComments // print a trace of parsed productions Trace // report declaration errors DeclarationErrors // same as AllErrors, for backward-compatibility SpuriousErrors // report all errors (not just the first 10 on different lines) AllErrors = SpuriousErrors )
  48. 48. Usage&example&in& example_test.go var fset = token.NewFileSet() var validFiles = []string{ "parser.go", "parser_test.go", "error_test.go", "short_test.go", } func TestParse(t *testing.T) { for _, filename := range validFiles { _, err := ParseFile(fset, filename, nil, DeclarationErrors) if err != nil { t.Fatalf("ParseFile(%s): %v", filename, err) } } }
  49. 49. Usage&example&in& example_test.go // This example shows what an AST looks like when printed for debugging. func ExamplePrint() { // src is the input for which we want to print the AST. src := ` package main func main() { println("Hello, World!") } ` // Create the AST by parsing src. fset := token.NewFileSet() // positions are relative to fset f, err := parser.ParseFile(fset, "", src, 0) if err != nil { panic(err) } // Print the AST. ast.Print(fset, f) }
  50. 50. U"li"es
  51. 51. U"li"es 1. printer 2. doc 3. format 4. build
  52. 52. printer 1. printer 2. doc 3. format 4. build
  53. 53. Fprint // Fprint "pretty-prints" an AST node to output. // It calls Config.Fprint with default settings. func Fprint(output io.Writer, fset *token.FileSet, node interface{}) error
  54. 54. Config!and!Mode // A Config node controls the output of Fprint. type Config struct { Mode Mode // default: 0 Tabwidth int // default: 8 // default: 0 (all code is indented at least by this much) Indent int } func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{}) error // A Mode value is a set of flags (or 0). They control printing. type Mode uint const ( // do not use a tabwriter; if set, UseSpaces is ignored RawFormat Mode = 1 << iota // use tabs for indentation independent of UseSpaces TabIndent // use spaces instead of tabs for alignment UseSpaces // emit //line comments to preserve original source positions SourcePos )
  55. 55. CommentedNode // A CommentedNode bundles an AST node and corresponding comments. // It may be provided as argument to any of the Fprint functions. type CommentedNode struct { // *ast.File, or ast.Expr, ast.Decl, ast.Spec, or ast.Stmt Node interface{} Comments []*ast.CommentGroup }
  56. 56. doc 1. printer 2. doc 3. format 4. build
  57. 57. doc Types&of&doc type Func struct { ... } // func foo() { ... } type Note struct { ... } // TODO(tfukushima): ... type Package struct { ... } // package bar type Type struct { ... } // type type Value struct { ... } // var, const
  58. 58. Package!methods func New(pkg *ast.Package, importPath string, mode Mode) *Package func (p *Package) Filter(f Filter) type Filter func(string) bool // Mode values control the operation of New. type Mode int const ( // extract documentation for all package-level // declarations, not just exported ones AllDecls Mode = 1 << iota // show all embedded methods, not just the ones of // invisible (unexported) anonymous fields AllMethods )
  59. 59. Every&exported&(capitalized)&name&in&a& program&should&have&a&doc&comment Python def _this_is_an_unexported_function(): ... def this_is_an_exported_function(): """This is a Docstring. Lisp has it, too. """ Go func thisIsAnUnexportedFunction() { ... } // This should have a doc comment. func ThisIsAnExportedFunction() { ... }
  60. 60. Example // An Example represents an example function found in a source files. type Example struct { Name string // name of the item being exemplified Doc string // example function doc string Code ast.Node Play *ast.File // a whole program version of the example Comments []*ast.CommentGroup Output string // expected output EmptyOutput bool // expect empty output Order int // original source code order } // Examples returns the examples found in the files, sorted by // Name field. func Examples(files ...*ast.File) []*Example
  61. 61. Synopsis // Synopsis returns a cleaned version of the first sentence in s. func Synopsis(s string) string var IllegalPrefixes = []string{ "copyright", "all rights", "author", }
  62. 62. ToHTML!and!ToText // ToHTML converts comment text to formatted HTML. func ToHTML(w io.Writer, text string, words map[string]string) // ToText prepares comment text for presentation in textual // output. func ToText(w io.Writer, text string, indent, preIndent string, width int)
  63. 63. format 1. printer 2. doc 3. format 4. build
  64. 64. Node!and!Source // Node formats node in canonical gofmt style and writes the result // to dst. func Node(dst io.Writer, fset *token.FileSet, node interface{}) error // Source formats src in canonical gofmt style and returns the result // or an (I/O or syntax) error. func Source(src []byte) ([]byte, error)
  65. 65. Usage&Example:&My&go fmt import ( "go/format" "io/ioutil" "os" ) const fileName = "hello.go" func formatFile(fileName string) { src, err := ioutil.ReadFile(fileName) if err != nil { panic(err) } res, err := format.Source(src) if err != nil { panic(err) } ioutil.WriteFile(fileName, res, os.ModeType) } func main() { formatFile(fileName) }
  66. 66. Resulf'of'the'usage'example /Users/tfukushima/go/src/github.com/tfukushima/mygofmt% cat hello.go package main import ("fmt" "github.com/tfukushima/string") func main(){fmt.Println(string.Reverse("Hello, new gopher!"))} /Users/tfukushima/go/src/github.com/tfukushima/mygofmt% go run mygofmt.go /Users/tfukushima/go/src/github.com/tfukushima/mygofmt% cat hello.go package main import ( "fmt" "github.com/tfukushima/string" ) func main() { fmt.Println(string.Reverse("Hello, new gopher!")) }
  67. 67. build 1. printer 2. doc 3. format 4. build
  68. 68. Go#Path GOPATH=/home/user/gocode /home/user/gocode/ src/ foo/ bar/ (go code in package bar) x.go quux/ (go code in package main) y.go bin/ quux (installed command) pkg/ linux_amd64/ foo/ bar.a (installed package object)
  69. 69. Build&constraints To#build#only#on#the#specific#OS#and/or#arch: // +build linux,386 darwin,!cgo (linux AND 386) OR (darwin AND (NOT cgo)) // +build linux darwin // +build 386 (linux OR darwin) AND 386 To#keep#a#file#from#being#considered#for#the#build: // +build ignore // +build unsatisfied_word
  70. 70. Build&constraints During'a'par*cular'build,'the'following'words'are' sa*sfied: - the target operating system, as spelled by runtime.GOOS - the target architecture, as spelled by runtime.GOARCH - the compiler being used, either "gc" or "gccgo" - "cgo", if ctxt.CgoEnabled is true - "go1.1", from Go version 1.1 onward - "go1.2", from Go version 1.2 onward - "go1.3", from Go version 1.3 onward - any additional words listed in ctxt.BuildTags
  71. 71. Build&constraints&by&file&names - source_windows_arm64.go - source_windows_arm64_test.go GOOS=windows GOARCH=arm64
  72. 72. Package // A Package describes the Go package found in a directory. type Package struct { Dir string // directory containing package sources Name string // package name Doc string // documentation synopsis ImportPath string // import path of package ("" if unknown) Root string // root of Go tree where this package lives SrcRoot string // package source root directory ("" if unknown) PkgRoot string // package install root directory ("" if unknown) BinDir string // command install directory ("" if unknown) Goroot bool // package found in Go root PkgObj string // installed .a file AllTags []string // tags that can influence file selection in this directory ConflictDir string // this directory shadows Dir in $GOPATH ... } // IsCommand reports whether the package is considered a command to be installed // (not just a library). Packages named "main" are treated as commands. func (p *Package) IsCommand() bool { return p.Name == "main" }
  73. 73. Context // A Context specifies the supporting context for a build. type Context struct { GOARCH string // target architecture GOOS string // target operating system GOROOT string // Go root GOPATH string // Go path CgoEnabled bool // whether cgo can be used UseAllFiles bool // use files regardless of +build lines, file names Compiler string // compiler to assume when computing target paths ... } // Default is the default Context for builds. It uses the GOARCH, GOOS, // GOROOT, and GOPATH environment variables if set, or else the compiled // code's GOARCH, GOOS, and GOROOT. var Default Context = defaultContext()
  74. 74. Context!methods func (ctxt *Context) Import(path string, srcDir string, mode ImportMode) (*Package, error) func (ctxt *Context) ImportDir(dir string, mode ImportMode) (*Package, error) func (ctxt *Context) MatchFile(dir, name string) (match bool, err error) // SrcDirs returns a list of package source root directories. It // draws from the current Go root and Go path but omits directories // that do not exist. func (ctxt *Context) SrcDirs() []string
  75. 75. ImportMode type ImportMode uint const ( // If FindOnly is set, Import stops after locating the directory // that should contain the sources for a package. It does not // read any files in the directory. FindOnly ImportMode = 1 << iota // If AllowBinary is set, Import can be satisfied by a compiled // package object without corresponding sources. AllowBinary )
  76. 76. NoGoError // NoGoError is the error used by Import to describe a directory // containing no buildable Go source files. (It may still contain // test files, files hidden by build tags, and so on.) type NoGoError struct { Dir string } func (e *NoGoError) Error() string { return "no buildable Go source files in " + e.Dir }
  77. 77. Import!and!ImportDir // Import is shorthand for Default.Import. func Import(path, srcDir string, mode ImportMode) (*Package, error) // Import is shorthand for Default.ImportDir. func ImportDir(dir string, mode ImportMode) (*Package, error)
  78. 78. Misc // ToolDir is the directory containing build tools. var ToolDir = filepath.Join(runtime.GOROOT(), "pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH) // ArchChar returns the architecture character for the given goarch. // For example, ArchChar("amd64") returns "6". func ArchChar(goarch string) (string, error) // IsLocalImport reports whether the import path is // a local import path, like ".", "..", "./foo", or "../foo". func IsLocalImport(path string) bool
  79. 79. Wrap%up
  80. 80. go!package!hierarchy • ast • build • doc • format • parser • printer • scanner • token
  81. 81. go!package:!Sta+c!analysis • ast • build • doc • format • parser • printer • scanner • token
  82. 82. go!package:!U*li*es • ast • build • doc • format • parser • printer • scanner • token
  83. 83. The$end$of$slides.$Any$ ques1on?

×