diff --git a/Documentation/cli/README.md b/Documentation/cli/README.md index 8fd53224..2845013a 100644 --- a/Documentation/cli/README.md +++ b/Documentation/cli/README.md @@ -346,6 +346,10 @@ See [Documentation/cli/expr.md](//github.com/go-delve/delve/tree/master/Document Executes a file containing a list of delve commands source + +If path ends with the .star extension it will be interpreted as a starlark script. See [Documentation/cli/starlark.md](//github.com/go-delve/delve/tree/master/Documentation/cli/starlark.md) for the syntax. + +If path is a single '-' character an interactive starlark interpreter will start instead. Type 'exit' to exit. ## sources diff --git a/Documentation/cli/starlark.md b/Documentation/cli/starlark.md new file mode 100644 index 00000000..02e1eba1 --- /dev/null +++ b/Documentation/cli/starlark.md @@ -0,0 +1,263 @@ +# Introduction + +Passing a file with the .star extension to the `source` command will cause delve to interpret it as a starlark script. + +Starlark is a dialect of python, a [specification of its syntax can be found here](https://github.com/google/starlark-go/blob/master/doc/spec.md). + +In addition to the normal starlark built-ins delve defines [a number of global functions](#Starlark-built-ins) that can be used to interact with the debugger. + +After the file has been evaluated delve will bind any function starting with `command_` to a command-line command: for example `command_goroutines_wait_reason` will be bound to `goroutines_wait_reason`. + +Then if a function named `main` exists it will be executed. + +Global functions with a name that begins with a capital letter will be available to other scripts. + +# Starlark built-ins + + +Function | API Call +---------|--------- +amend_breakpoint(Breakpoint) | Equivalent to API call [AmendBreakpoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.AmendBreakpoint) +ancestors(GoroutineID, NumAncestors, Depth) | Equivalent to API call [Ancestors](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Ancestors) +attached_to_existing_process() | Equivalent to API call [AttachedToExistingProcess](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.AttachedToExistingProcess) +cancel_next() | Equivalent to API call [CancelNext](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.CancelNext) +checkpoint(Where) | Equivalent to API call [Checkpoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Checkpoint) +clear_breakpoint(Id, Name) | Equivalent to API call [ClearBreakpoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ClearBreakpoint) +clear_checkpoint(ID) | Equivalent to API call [ClearCheckpoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ClearCheckpoint) +raw_command(Name, ThreadID, GoroutineID, ReturnInfoLoadConfig, Expr, UnsafeCall) | Equivalent to API call [Command](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Command) +create_breakpoint(Breakpoint) | Equivalent to API call [CreateBreakpoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.CreateBreakpoint) +detach(Kill) | Equivalent to API call [Detach](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Detach) +disassemble(Scope, StartPC, EndPC, Flavour) | Equivalent to API call [Disassemble](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Disassemble) +eval(Scope, Expr, Cfg) | Equivalent to API call [Eval](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Eval) +find_location(Scope, Loc) | Equivalent to API call [FindLocation](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.FindLocation) +function_return_locations(FnName) | Equivalent to API call [FunctionReturnLocations](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.FunctionReturnLocations) +get_breakpoint(Id, Name) | Equivalent to API call [GetBreakpoint](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.GetBreakpoint) +get_thread(Id) | Equivalent to API call [GetThread](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.GetThread) +is_multiclient() | Equivalent to API call [IsMulticlient](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.IsMulticlient) +last_modified() | Equivalent to API call [LastModified](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.LastModified) +breakpoints() | Equivalent to API call [ListBreakpoints](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListBreakpoints) +checkpoints() | Equivalent to API call [ListCheckpoints](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListCheckpoints) +dynamic_libraries() | Equivalent to API call [ListDynamicLibraries](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListDynamicLibraries) +function_args(Scope, Cfg) | Equivalent to API call [ListFunctionArgs](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctionArgs) +functions(Filter) | Equivalent to API call [ListFunctions](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctions) +goroutines(Start, Count) | Equivalent to API call [ListGoroutines](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListGoroutines) +local_vars(Scope, Cfg) | Equivalent to API call [ListLocalVars](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListLocalVars) +package_vars(Filter, Cfg) | Equivalent to API call [ListPackageVars](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListPackageVars) +registers(ThreadID, IncludeFp) | Equivalent to API call [ListRegisters](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListRegisters) +sources(Filter) | Equivalent to API call [ListSources](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListSources) +threads() | Equivalent to API call [ListThreads](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListThreads) +types(Filter) | Equivalent to API call [ListTypes](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListTypes) +process_pid() | Equivalent to API call [ProcessPid](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ProcessPid) +recorded() | Equivalent to API call [Recorded](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Recorded) +restart(Position, ResetArgs, NewArgs) | Equivalent to API call [Restart](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Restart) +set_expr(Scope, Symbol, Value) | Equivalent to API call [Set](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Set) +stacktrace(Id, Depth, Full, Defers, Cfg) | Equivalent to API call [Stacktrace](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.Stacktrace) +state(NonBlocking) | Equivalent to API call [State](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.State) +dlv_command(command) | Executes the specified command as if typed at the dlv_prompt +read_file(path) | Reads the file as a string +write_file(path, contents) | Writes string to a file +cur_scope() | Returns the current evaluation scope +default_load_config() | Returns the current default load configuration + + +## Should I use raw_command or dlv_command? + +There are two ways to resume the execution of the target program: + + raw_command("continue") + dlv_command("continue") + +The first one maps to the API call [Command](https://godoc.org/github.com/derekparker/delve/service/rpc2#RPCServer.Command). As such all the caveats explained in the [Client HowTo](../api/ClientHowto.md). + +The latter is equivalent to typing `continue` to the `(dlv)` command line and should do what you expect. + +In general `dlv_command("continue")` should be preferred, unless the behavior you wish to produces diverges significantly from that of the command line's `continue`. + +# Creating new commands + +Any global function with a name starting with `command_` will be made available as a command line command. If the function has a single argument named `args` all arguments passed on the command line will be passed to the function as a single string. + +Otherwise arguments passed on the command line are interpreted as starlark expressions. See the [expression arguments](#expression-arguments) example. + +If the command function has a doc string it will be used as a help message. + +# Working with variables + +Variables of the target program can be accessed using `local_vars`, `function_args` or the `eval` functions. Each variable will be returned as a [Variable](https://godoc.org/github.com/go-delve/delve/service/api#Variable) struct, with one special field: `Value`. + +## Variable.Value + +The `Value` field will return the value of the target variable converted to a starlark value: + +* integers, floating point numbers and strings are represented by equivalent starlark values +* structs are represented as starlark dictionaries +* slices and arrays are represented by starlark lists +* maps are represented by starlark dicts +* pointers and interfaces are represented by a one-element starlark list containing the value they point to + +For example, given this variable in the target program: + +``` +type astruct struct { + A int + B int +} + +s2 := []astruct{{1, 2}, {3, 4}, {5, 6}, {7, 8}, {9, 10}, {11, 12}, {13, 14}, {15, 16}} +``` + +The following is possible: + +``` +>>> s2 = eval(None, "s2").Variable +>>> s2.Value[0] # access of a slice item by index +main.astruct {A: 1, B: 2} +>>> a = s2.Value[1] +>>> a.Value.A # access to a struct field +3 +>>> a.Value.A + 10 # arithmetic on the value of s2[1].X +13 +>>> a.Value["B"] # access to a struct field, using dictionary syntax +4 +``` + +For more examples see the [linked list example](#Print-all-elements-of-a-linked-list) below. + +# Examples + +## Listing goroutines and making custom commands + +Create a `goroutine_start_line` command that prints the starting line of each goroutine, sets `gsl` as an alias: + +``` +def command_goroutine_start_line(args): + gs = goroutines().Goroutines + for g in gs: + line = read_file(g.StartLoc.File).splitlines()[g.StartLoc.Line-1].strip() + print(g.ID, "\t", g.StartLoc.File + ":" + str(g.StartLoc.Line), "\t", line) + +def main(): + dlv_command("config alias goroutine_start_line gsl") +``` + +Use it like this: + +``` +(dlv) source goroutine_start_line.start +(dlv) goroutine_start_line +1 /usr/local/go/src/runtime/proc.go:110 func main() { +2 /usr/local/go/src/runtime/proc.go:242 func forcegchelper() { +17 /usr/local/go/src/runtime/mgcsweep.go:64 func bgsweep(c chan int) { +18 /usr/local/go/src/runtime/mfinal.go:161 func runfinq() { +(dlv) gsl +1 /usr/local/go/src/runtime/proc.go:110 func main() { +2 /usr/local/go/src/runtime/proc.go:242 func forcegchelper() { +17 /usr/local/go/src/runtime/mgcsweep.go:64 func bgsweep(c chan int) { +18 /usr/local/go/src/runtime/mfinal.go:161 func runfinq() { +``` + +## Expression arguments + +After evaluating this script: + +``` +def command_echo(args): + print(args) + +def command_echo_expr(a, b, c): + print("a", a, "b", b, "c", c) +``` + +The first commnad, `echo`, takes its arguments as a single string, while for `echo_expr` it will be possible to pass starlark expression as arguments: + +``` +(dlv) echo 2+2, 2-1, 2*3 +"2+2, 2-1, 2*3" +(dlv) echo_expr 2+2, 2-1, 2*3 +a 4 b 1 c 6 +``` + +## Creating breakpoints + +Set a breakpoint on all private methods of package `main`: + +``` +def main(): + for f in functions().Funcs: + v = f.split('.') + if len(v) != 2: + continue + if v[0] != "main": + continue + if v[1][0] >= 'a' and v[1][0] <= 'z': + create_breakpoint({ "FunctionName": f, "Line": -1 }) # see documentation of RPCServer.CreateBreakpoint +``` + +## Switching goroutines + +Create a command, `switch_to_main_goroutine`, that searches for a goroutine running a function in the main package and switches to it: + +``` +def command_switch_to_main_goroutine(args): + for g in goroutines().Goroutines: + if g.currentLoc.function != None and g.currentLoc.function.name.startswith("main."): + print("switching to:", g.id) + raw_command("switchGoroutine", GoroutineID=g.id) + break +``` + +## Repeatedly executing the target until a breakpoint is hit. + +Repeatedly call continue and restart until the target hits a breakpoint. + +``` +def command_flaky(args): + "Repeatedly runs program until a breakpoint is hit" + while True: + if dlv_command("continue") == None: + break + dlv_command("restart") +``` + +## Print all elements of a linked list + +``` +def command_linked_list(args): + """Prints the contents of a linked list. + + linked_list + +Prints up to max_depth elements of the linked list variable 'var_name' using 'next_field_name' as the name of the link field. +""" + var_name, next_field_name, max_depth = args.split(" ") + max_depth = int(max_depth) + next_name = var_name + v = eval(None, var_name).Variable.Value + for i in range(0, max_depth): + print(str(i)+":",v) + if v[0] == None: + break + v = v[next_field_name] +``` + +## Find an array element matching a predicate + +``` +def command_find_array(arr, pred): + """Calls pred for each element of the array or slice 'arr' returns the index of the first element for which pred returns true. + + find_arr + +Example use (find the first element of slice 's2' with field A equal to 5): + + find_arr "s2", lambda x: x.A == 5 +""" + arrv = eval(None, arr).Variable + for i in range(0, arrv.Len): + v = arrv.Value[i] + if pred(v): + print("found", i) + return + + print("not found") +``` diff --git a/_fixtures/create_breakpoint_main.star b/_fixtures/create_breakpoint_main.star new file mode 100644 index 00000000..a29d1d71 --- /dev/null +++ b/_fixtures/create_breakpoint_main.star @@ -0,0 +1,9 @@ +def main(): + for f in functions().Funcs: + v = f.split('.') + if len(v) != 2: + continue + if v[0] != "main": + continue + if v[1][0] >= 'a' and v[1][0] <= 'z': + create_breakpoint({ "FunctionName": f, "Line": -1 }) # see documentation of RPCServer.CreateBreakpoint diff --git a/_fixtures/echo_expr.star b/_fixtures/echo_expr.star new file mode 100644 index 00000000..dd6bdba9 --- /dev/null +++ b/_fixtures/echo_expr.star @@ -0,0 +1,2 @@ +def command_echo_expr(a, b, c): + print("a", a, "b", b, "c", c) diff --git a/_fixtures/find_array.star b/_fixtures/find_array.star new file mode 100644 index 00000000..ab13708c --- /dev/null +++ b/_fixtures/find_array.star @@ -0,0 +1,17 @@ +def command_find_array(arr, pred): + """Calls pred for each element of the array or slice 'arr' returns the index of the first element for which pred returns true. + + find_arr , + +Example use: + + find_arr "s2", lambda x: x.A == 5 +""" + arrv = eval(None, arr).Variable + for i in range(0, arrv.Len): + v = arrv.Value[i] + if pred(v): + print("found", i) + return + + print("not found") diff --git a/_fixtures/goroutine_start_line.star b/_fixtures/goroutine_start_line.star new file mode 100644 index 00000000..45be53d3 --- /dev/null +++ b/_fixtures/goroutine_start_line.star @@ -0,0 +1,9 @@ +def command_goroutine_start_line(args): + "prints the line of source code that started each currently running goroutine" + gs = goroutines().Goroutines + for g in gs: + line = read_file(g.StartLoc.File).splitlines()[g.StartLoc.Line-1].strip() + print(g.ID, "\t", g.StartLoc.File + ":" + str(g.StartLoc.Line), "\t", line) + +def main(): + dlv_command("config alias goroutine_start_line gsl") diff --git a/_fixtures/linked_list.star b/_fixtures/linked_list.star new file mode 100644 index 00000000..42da7a04 --- /dev/null +++ b/_fixtures/linked_list.star @@ -0,0 +1,16 @@ +def command_linked_list(args): + """Prints the contents of a linked list. + + linked_list + +Prints up to max_depth elements of the linked list variable 'var_name' using 'next_field_name' as the name of the link field. +""" + var_name, next_field_name, max_depth = args.split(" ") + max_depth = int(max_depth) + next_name = var_name + v = eval(None, var_name).Variable.Value + for i in range(0, max_depth): + print(str(i)+":",v) + if v[0] == None: + break + v = v[next_field_name] diff --git a/_fixtures/starlark_map_iteration.star b/_fixtures/starlark_map_iteration.star new file mode 100644 index 00000000..17c535e3 --- /dev/null +++ b/_fixtures/starlark_map_iteration.star @@ -0,0 +1,6 @@ +v = eval(None, "m1").Variable +n = 0 +for k in v.Value: + n = n + 1 + print(k) +print(n) diff --git a/_fixtures/switch_to_main_goroutine.star b/_fixtures/switch_to_main_goroutine.star new file mode 100644 index 00000000..99b16ac3 --- /dev/null +++ b/_fixtures/switch_to_main_goroutine.star @@ -0,0 +1,6 @@ +def command_switch_to_main_goroutine(args): + for g in goroutines().Goroutines: + if g.CurrentLoc.Function != None and g.CurrentLoc.Function.Name_.startswith("main."): + print("switching to:", g.ID) + raw_command("switchGoroutine", GoroutineID=g.ID) + break diff --git a/_fixtures/testvariables2.go b/_fixtures/testvariables2.go index d3bea362..8679f54e 100644 --- a/_fixtures/testvariables2.go +++ b/_fixtures/testvariables2.go @@ -101,6 +101,11 @@ type Block struct { cache *Cache } +type List struct { + N int + Next *List +} + func main() { i1 := 1 i2 := 2 @@ -275,6 +280,8 @@ func main() { }, } + ll := &List{0, &List{1, &List{2, &List{3, &List{4, nil}}}}} + var amb1 = 1 runtime.Breakpoint() for amb1 := 0; amb1 < 10; amb1++ { @@ -282,5 +289,5 @@ func main() { } runtime.Breakpoint() - fmt.Println(i1, i2, i3, p1, amb1, s1, s3, a1, p2, p3, s2, as1, str1, f1, fn1, fn2, nilslice, nilptr, ch1, chnil, m1, mnil, m2, m3, up1, i4, i5, i6, err1, err2, errnil, iface1, iface2, ifacenil, arr1, parr, cpx1, const1, iface3, iface4, recursive1, recursive1.x, iface5, iface2fn1, iface2fn2, bencharr, benchparr, mapinf, mainMenu, b, b2, sd, anonstruct1, anonstruct2, anoniface1, anonfunc, mapanonstruct1, ifacearr, efacearr, ni8, ni16, ni32, pinf, ninf, nan, zsvmap, zsslice, zsvar, tm, errtypednil, emptyslice, emptymap, byteslice, runeslice, bytearray, runearray, longstr, nilstruct, as2, as2.NonPointerRecieverMethod, s4, iface2map, issue1578) + fmt.Println(i1, i2, i3, p1, amb1, s1, s3, a1, p2, p3, s2, as1, str1, f1, fn1, fn2, nilslice, nilptr, ch1, chnil, m1, mnil, m2, m3, up1, i4, i5, i6, err1, err2, errnil, iface1, iface2, ifacenil, arr1, parr, cpx1, const1, iface3, iface4, recursive1, recursive1.x, iface5, iface2fn1, iface2fn2, bencharr, benchparr, mapinf, mainMenu, b, b2, sd, anonstruct1, anonstruct2, anoniface1, anonfunc, mapanonstruct1, ifacearr, efacearr, ni8, ni16, ni32, pinf, ninf, nan, zsvmap, zsslice, zsvar, tm, errtypednil, emptyslice, emptymap, byteslice, runeslice, bytearray, runearray, longstr, nilstruct, as2, as2.NonPointerRecieverMethod, s4, iface2map, issue1578, ll) } diff --git a/cmd/dlv/dlv_test.go b/cmd/dlv/dlv_test.go index 917bb683..8767c16c 100644 --- a/cmd/dlv/dlv_test.go +++ b/cmd/dlv/dlv_test.go @@ -247,8 +247,7 @@ func TestGeneratedDoc(t *testing.T) { var generatedBuf bytes.Buffer commands := terminal.DebugCommands(nil) commands.WriteMarkdown(&generatedBuf) - cliDocFilename := "Documentation/cli/README.md" - checkAutogenDoc(t, cliDocFilename, "scripts/gen-cli-docs.go", generatedBuf.Bytes()) + checkAutogenDoc(t, "Documentation/cli/README.md", "scripts/gen-cli-docs.go", generatedBuf.Bytes()) // Checks gen-usage-docs.go tempDir, err := ioutil.TempDir(os.TempDir(), "test-gen-doc") @@ -261,6 +260,21 @@ func TestGeneratedDoc(t *testing.T) { docFilename := "Documentation/usage/" + doc.Name() checkAutogenDoc(t, docFilename, "scripts/gen-usage-docs.go", slurpFile(t, tempDir+"/"+doc.Name())) } + + runScript := func(args ...string) []byte { + a := []string{"run"} + a = append(a, args...) + cmd := exec.Command("go", a...) + cmd.Dir = projectRoot() + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("could not run script %v: %v (output: %q)", args, err, string(out)) + } + return out + } + + checkAutogenDoc(t, "pkg/terminal/starbind/starlark_mapping.go", "'go generate' inside pkg/terminal/starbind", runScript("scripts/gen-starlark-bindings.go", "go", "-")) + checkAutogenDoc(t, "Documentation/cli/starlark.md", "'go generate' inside pkg/terminal/starbind", runScript("scripts/gen-starlark-bindings.go", "doc/dummy", "Documentation/cli/starlark.md")) } func TestExitInInit(t *testing.T) { diff --git a/go.mod b/go.mod index 5e756d68..2cfc8cbb 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/go-delve/delve require ( + github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/cosiner/argv v0.0.0-20170225145430-13bacc38a0a5 github.com/cpuguy83/go-md2man v1.0.8 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -12,6 +13,7 @@ require ( github.com/sirupsen/logrus v0.0.0-20180523074243-ea8897e79973 github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372 github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1 // indirect + go.starlark.net v0.0.0-20190225160109-1174b2613e82 golang.org/x/arch v0.0.0-20171004143515-077ac972c2e4 golang.org/x/crypto v0.0.0-20180614174826-fd5f17ee7299 // indirect golang.org/x/sys v0.0.0-20180614134839-8883426083c0 diff --git a/go.sum b/go.sum index 21cd1497..74db2d07 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/cosiner/argv v0.0.0-20170225145430-13bacc38a0a5 h1:rIXlvz2IWiupMFlC45cZCXZFvKX/ExBcSLrDy2G0Lp8= github.com/cosiner/argv v0.0.0-20170225145430-13bacc38a0a5/go.mod h1:p/NrK5tF6ICIly4qwEDsf6VDirFiWWz0FenfYBwJaKQ= github.com/cpuguy83/go-md2man v1.0.8 h1:DwoNytLphI8hzS2Af4D0dfaEaiSq2bN05mEm4R6vf8M= @@ -20,6 +22,16 @@ github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372 h1:eRfW1vRS4th8IX2iQey github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1 h1:7bozMfSdo41n2NOc0GsVTTVUiA+Ncaj6pXNpm4UHKys= github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +go.starlark.net v0.0.0-20181108041844-f4938bde4080 h1:PynO3TmUXWWlWQ1FHArWPoFcoQR3oCaMm0l+d6rbjeo= +go.starlark.net v0.0.0-20181108041844-f4938bde4080/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= +go.starlark.net v0.0.0-20181213015430-66ac3a2d309f h1:8Fs2MRAY3gcipeW5YuV7PR0PYZ30uyRurWJSON4km/I= +go.starlark.net v0.0.0-20181213015430-66ac3a2d309f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= +go.starlark.net v0.0.0-20181213164318-58464401628a h1:zSGIMfylaSMaCEjwVKLMO8Hb/YXDHik8Z+j6Yb1/MJc= +go.starlark.net v0.0.0-20181213164318-58464401628a/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= +go.starlark.net v0.0.0-20190106165729-93b8d146431e h1:iAWNmlXeb8B9A+QSPlujsxdd5paZFxRA2RSGQE8qvJ4= +go.starlark.net v0.0.0-20190106165729-93b8d146431e/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= +go.starlark.net v0.0.0-20190225160109-1174b2613e82 h1:vAHCTDREx7LqPeWqNeTvCKWcURJztP3wBKnF2Q0sW7Q= +go.starlark.net v0.0.0-20190225160109-1174b2613e82/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= golang.org/x/arch v0.0.0-20171004143515-077ac972c2e4 h1:TP7YcWHbnFq4v8/3wM2JwgM0SRRtsYJ7Z6Oj0arz2bs= golang.org/x/arch v0.0.0-20171004143515-077ac972c2e4/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/crypto v0.0.0-20180614174826-fd5f17ee7299 h1:zxP+xTjjk4kD+M5IFPweL7/4851FUhYkzbDqbzkN1JE= diff --git a/pkg/terminal/command.go b/pkg/terminal/command.go index 45c8fd94..b10cb22a 100644 --- a/pkg/terminal/command.go +++ b/pkg/terminal/command.go @@ -12,6 +12,7 @@ import ( "math" "os" "os/exec" + "path/filepath" "reflect" "regexp" "sort" @@ -297,7 +298,11 @@ Move the current frame down by . The second form runs the command on the give Executes the specified command (print, args, locals) in the context of the n-th deferred call in the current frame.`}, {aliases: []string{"source"}, cmdFn: c.sourceCommand, helpMsg: `Executes a file containing a list of delve commands - source `}, + source + +If path ends with the .star extension it will be interpreted as a starlark script. See $GOPATH/src/github.com/go-delve/delve/Documentation/cli/starlark.md for the syntax. + +If path is a single '-' character an interactive starlark interpreter will start instead. Type 'exit' to exit.`}, {aliases: []string{"disassemble", "disass"}, cmdFn: disassCommand, helpMsg: `Disassembler. [goroutine ] [frame ] disassemble [-a ] [-l ] @@ -1566,6 +1571,15 @@ func (c *Commands) sourceCommand(t *Term, ctx callContext, args string) error { return fmt.Errorf("wrong number of arguments: source ") } + if filepath.Ext(args) == ".star" { + _, err := t.starlarkEnv.Execute(args, nil, "main", nil) + return err + } + + if args == "-" { + return t.starlarkEnv.REPL() + } + return c.executeFile(t, args) } diff --git a/pkg/terminal/command_test.go b/pkg/terminal/command_test.go index 23e79169..8b88be44 100644 --- a/pkg/terminal/command_test.go +++ b/pkg/terminal/command_test.go @@ -17,6 +17,7 @@ import ( "github.com/go-delve/delve/pkg/config" "github.com/go-delve/delve/pkg/goversion" + "github.com/go-delve/delve/pkg/logflags" "github.com/go-delve/delve/pkg/proc/test" "github.com/go-delve/delve/service" "github.com/go-delve/delve/service/api" @@ -29,12 +30,15 @@ var testBackend, buildMode string func TestMain(m *testing.M) { flag.StringVar(&testBackend, "backend", "", "selects backend") flag.StringVar(&buildMode, "test-buildmode", "", "selects build mode") + var logConf string + flag.StringVar(&logConf, "log", "", "configures logging") flag.Parse() test.DefaultTestBackend(&testBackend) if buildMode != "" && buildMode != "pie" { fmt.Fprintf(os.Stderr, "unknown build mode %q", buildMode) os.Exit(1) } + logflags.Setup(logConf != "", logConf, "") os.Exit(test.RunTestsWithFixtures(m)) } @@ -70,14 +74,49 @@ func (ft *FakeTerminal) Exec(cmdstr string) (outstr string, err error) { return } +func (ft *FakeTerminal) ExecStarlark(starlarkProgram string) (outstr string, err error) { + outfh, err := ioutil.TempFile("", "cmdtestout") + if err != nil { + ft.t.Fatalf("could not create temporary file: %v", err) + } + + stdout, stderr, termstdout := os.Stdout, os.Stderr, ft.Term.stdout + os.Stdout, os.Stderr, ft.Term.stdout = outfh, outfh, outfh + defer func() { + os.Stdout, os.Stderr, ft.Term.stdout = stdout, stderr, termstdout + outfh.Close() + outbs, err1 := ioutil.ReadFile(outfh.Name()) + if err1 != nil { + ft.t.Fatalf("could not read temporary output file: %v", err) + } + outstr = string(outbs) + if logCommandOutput { + ft.t.Logf("command %q -> %q", starlarkProgram, outstr) + } + os.Remove(outfh.Name()) + }() + _, err = ft.Term.starlarkEnv.Execute("", starlarkProgram, "main", nil) + return +} + func (ft *FakeTerminal) MustExec(cmdstr string) string { outstr, err := ft.Exec(cmdstr) if err != nil { + ft.t.Errorf("output of %q: %q", cmdstr, outstr) ft.t.Fatalf("Error executing <%s>: %v", cmdstr, err) } return outstr } +func (ft *FakeTerminal) MustExecStarlark(starlarkProgram string) string { + outstr, err := ft.ExecStarlark(starlarkProgram) + if err != nil { + ft.t.Errorf("output of %q: %q", starlarkProgram, outstr) + ft.t.Fatalf("Error executing <%s>: %v", starlarkProgram, err) + } + return outstr +} + func (ft *FakeTerminal) AssertExec(cmdstr, tgt string) { out := ft.MustExec(cmdstr) if out != tgt { @@ -870,3 +909,7 @@ func TestIssue1493(t *testing.T) { } }) } + +func findStarFile(name string) string { + return filepath.Join(test.FindFixturesDir(), name+".star") +} diff --git a/pkg/terminal/starbind/conv.go b/pkg/terminal/starbind/conv.go new file mode 100644 index 00000000..d0d91763 --- /dev/null +++ b/pkg/terminal/starbind/conv.go @@ -0,0 +1,695 @@ +package starbind + +import ( + "errors" + "fmt" + "math" + "reflect" + "strconv" + + "go.starlark.net/starlark" + + "github.com/go-delve/delve/service/api" +) + +// autoLoadConfig is the load configuration used to automatically load more from a variable +var autoLoadConfig = api.LoadConfig{false, 1, 1024, 64, -1} + +// interfaceToStarlarkValue converts an interface{} variable (produced by +// decoding JSON) into a starlark.Value. +func (env *Env) interfaceToStarlarkValue(v interface{}) starlark.Value { + switch v := v.(type) { + case uint8: + return starlark.MakeUint64(uint64(v)) + case uint16: + return starlark.MakeUint64(uint64(v)) + case uint32: + return starlark.MakeUint64(uint64(v)) + case uint64: + return starlark.MakeUint64(v) + case uintptr: + return starlark.MakeUint64(uint64(v)) + case uint: + return starlark.MakeUint64(uint64(v)) + case int8: + return starlark.MakeInt64(int64(v)) + case int16: + return starlark.MakeInt64(int64(v)) + case int32: + return starlark.MakeInt64(int64(v)) + case int64: + return starlark.MakeInt64(v) + case int: + return starlark.MakeInt64(int64(v)) + case string: + return starlark.String(v) + case map[string]uint64: + // this is the only map type that we use in the api, if we ever want to + // add more maps to the api a more general approach will be necessary. + var r starlark.Dict + for k, v := range v { + r.SetKey(starlark.String(k), starlark.MakeUint64(v)) + } + return &r + case nil: + return starlark.None + case error: + return starlark.String(v.Error()) + default: + vval := reflect.ValueOf(v) + switch vval.Type().Kind() { + case reflect.Ptr: + if vval.IsNil() { + return starlark.None + } + vval = vval.Elem() + if vval.Type().Kind() == reflect.Struct { + return structAsStarlarkValue{vval, env} + } + case reflect.Struct: + return structAsStarlarkValue{vval, env} + case reflect.Slice: + return sliceAsStarlarkValue{vval, env} + } + return starlark.String(fmt.Sprintf("%v", v)) + } +} + +// sliceAsStarlarkValue converts a reflect.Value containing a slice +// into a starlark value. +// The public methods of sliceAsStarlarkValue implement the Indexable and +// Sequence starlark interfaces. +type sliceAsStarlarkValue struct { + v reflect.Value + env *Env +} + +var _ starlark.Indexable = sliceAsStarlarkValue{} +var _ starlark.Sequence = sliceAsStarlarkValue{} + +func (v sliceAsStarlarkValue) Freeze() { +} + +func (v sliceAsStarlarkValue) Hash() (uint32, error) { + return 0, fmt.Errorf("not hashable") +} + +func (v sliceAsStarlarkValue) String() string { + return fmt.Sprintf("%#v", v.v) +} + +func (v sliceAsStarlarkValue) Truth() starlark.Bool { + return v.v.Len() != 0 +} + +func (v sliceAsStarlarkValue) Type() string { + return v.v.Type().String() +} + +func (v sliceAsStarlarkValue) Index(i int) starlark.Value { + if i >= v.v.Len() { + return nil + } + return v.env.interfaceToStarlarkValue(v.v.Index(i).Interface()) +} + +func (v sliceAsStarlarkValue) Len() int { + return v.v.Len() +} + +func (v sliceAsStarlarkValue) Iterate() starlark.Iterator { + return &sliceAsStarlarkValueIterator{0, v.v, v.env} +} + +type sliceAsStarlarkValueIterator struct { + cur int + v reflect.Value + env *Env +} + +func (it *sliceAsStarlarkValueIterator) Done() { +} + +func (it *sliceAsStarlarkValueIterator) Next(p *starlark.Value) bool { + if it.cur >= it.v.Len() { + return false + } + *p = it.env.interfaceToStarlarkValue(it.v.Index(it.cur).Interface()) + it.cur++ + return true +} + +// structAsStarlarkValue converts any Go struct into a starlark.Value. +// The public methods of structAsStarlarkValue implement the +// starlark.HasAttrs interface. +type structAsStarlarkValue struct { + v reflect.Value + env *Env +} + +var _ starlark.HasAttrs = structAsStarlarkValue{} + +func (v structAsStarlarkValue) Freeze() { +} + +func (v structAsStarlarkValue) Hash() (uint32, error) { + return 0, fmt.Errorf("not hashable") +} + +func (v structAsStarlarkValue) String() string { + if vv, ok := v.v.Interface().(api.Variable); ok { + return fmt.Sprintf("Variable<%s>", vv.SinglelineString()) + } + return fmt.Sprintf("%#v", v.v) +} + +func (v structAsStarlarkValue) Truth() starlark.Bool { + return true +} + +func (v structAsStarlarkValue) Type() string { + if vv, ok := v.v.Interface().(api.Variable); ok { + return fmt.Sprintf("Variable<%s>", vv.Type) + } + return v.v.Type().String() +} + +func (v structAsStarlarkValue) Attr(name string) (starlark.Value, error) { + if r, err := v.valueAttr(name); err != nil || r != nil { + return r, err + } + r := v.v.FieldByName(name) + if r == (reflect.Value{}) { + return starlark.None, fmt.Errorf("no field named %q in %T", name, v.v.Interface()) + } + return v.env.interfaceToStarlarkValue(r.Interface()), nil +} + +func (v structAsStarlarkValue) valueAttr(name string) (starlark.Value, error) { + if v.v.Type().Name() != "Variable" || (name != "Value" && name != "Expr") { + return nil, nil + } + v2 := v.v.Interface().(api.Variable) + + if name == "Expr" { + return starlark.String(varAddrExpr(&v2)), nil + } + + return v.env.variableValueToStarlarkValue(&v2, true) +} + +func varAddrExpr(v *api.Variable) string { + return fmt.Sprintf("(*(*%q)(%#x))", v.Type, v.Addr) +} + +func (env *Env) variableValueToStarlarkValue(v *api.Variable, top bool) (starlark.Value, error) { + if !top && v.Addr == 0 && v.Value == "" { + return starlark.None, nil + } + + switch v.Kind { + case reflect.Struct: + if v.Len != 0 && len(v.Children) == 0 { + return starlark.None, errors.New("value not loaded") + } + return structVariableAsStarlarkValue{v, env}, nil + case reflect.Slice, reflect.Array: + if v.Len != 0 && len(v.Children) == 0 { + return starlark.None, errors.New("value not loaded") + } + return sliceVariableAsStarlarkValue{v, env}, nil + case reflect.Map: + if v.Len != 0 && len(v.Children) == 0 { + return starlark.None, errors.New("value not loaded") + } + return mapVariableAsStarlarkValue{v, env}, nil + case reflect.String: + return starlark.String(v.Value), nil + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + n, _ := strconv.ParseInt(v.Value, 0, 64) + return starlark.MakeInt64(n), nil + case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint, reflect.Uintptr: + n, _ := strconv.ParseUint(v.Value, 0, 64) + return starlark.MakeUint64(n), nil + case reflect.Float32, reflect.Float64: + switch v.Value { + case "+Inf": + return starlark.Float(math.Inf(+1)), nil + case "-Inf": + return starlark.Float(math.Inf(-1)), nil + case "NaN": + return starlark.Float(math.NaN()), nil + default: + n, _ := strconv.ParseFloat(v.Value, 0) + return starlark.Float(n), nil + } + case reflect.Ptr, reflect.Interface: + if len(v.Children) > 0 { + v.Children[0] = *env.autoLoad(varAddrExpr(&v.Children[0])) + } + return ptrVariableAsStarlarkValue{v, env}, nil + } + return nil, nil +} + +func (env *Env) autoLoad(expr string) *api.Variable { + v, err := env.ctx.Client().EvalVariable(api.EvalScope{-1, 0, 0}, expr, autoLoadConfig) + if err != nil { + return &api.Variable{Unreadable: err.Error()} + } + return v +} + +func (v structAsStarlarkValue) AttrNames() []string { + typ := v.v.Type() + r := make([]string, 0, typ.NumField()+1) + for i := 0; i < typ.NumField(); i++ { + r = append(r, typ.Field(i).Name) + } + return r +} + +// structVariableAsStarlarkValue converts an api.Variable representing a +// struct variable (in the target process) into a starlark.Value. +// The public methods of structVariableAsStarlarkValue implement the +// starlark.HasAttrs and starlark.Mapping interfaces. +type structVariableAsStarlarkValue struct { + v *api.Variable + env *Env +} + +var _ starlark.HasAttrs = structVariableAsStarlarkValue{} +var _ starlark.Mapping = structVariableAsStarlarkValue{} + +func (v structVariableAsStarlarkValue) Freeze() { +} + +func (v structVariableAsStarlarkValue) Hash() (uint32, error) { + return 0, fmt.Errorf("not hashable") +} + +func (v structVariableAsStarlarkValue) String() string { + return fmt.Sprintf("%s", v.v.SinglelineString()) +} + +func (v structVariableAsStarlarkValue) Truth() starlark.Bool { + return true +} + +func (v structVariableAsStarlarkValue) Type() string { + return v.v.Type +} + +func (v structVariableAsStarlarkValue) Attr(name string) (starlark.Value, error) { + for i := range v.v.Children { + if v.v.Children[i].Name == name { + v2 := v.env.autoLoad(varAddrExpr(&v.v.Children[i])) + return v.env.variableValueToStarlarkValue(v2, false) + } + } + return nil, nil // no such field or method +} + +func (v structVariableAsStarlarkValue) AttrNames() []string { + r := make([]string, len(v.v.Children)) + for i := range v.v.Children { + r[i] = v.v.Children[i].Name + } + return r +} + +func (v structVariableAsStarlarkValue) Get(key starlark.Value) (starlark.Value, bool, error) { + skey, ok := key.(starlark.String) + if !ok { + return starlark.None, false, nil + } + r, err := v.Attr(string(skey)) + if r == nil && err == nil { + return starlark.None, false, nil + } + if err != nil { + return starlark.None, false, err + } + return r, true, nil +} + +type sliceVariableAsStarlarkValue struct { + v *api.Variable + env *Env +} + +var _ starlark.Indexable = sliceVariableAsStarlarkValue{} +var _ starlark.Sequence = sliceVariableAsStarlarkValue{} + +func (v sliceVariableAsStarlarkValue) Freeze() { +} + +func (v sliceVariableAsStarlarkValue) Hash() (uint32, error) { + return 0, fmt.Errorf("not hashable") +} + +func (v sliceVariableAsStarlarkValue) String() string { + return fmt.Sprintf("%s", v.v.SinglelineString()) +} + +func (v sliceVariableAsStarlarkValue) Truth() starlark.Bool { + return v.v.Len != 0 +} + +func (v sliceVariableAsStarlarkValue) Type() string { + return v.v.Type +} + +func (v sliceVariableAsStarlarkValue) Index(i int) starlark.Value { + if i >= v.Len() { + return nil + } + v2 := v.env.autoLoad(fmt.Sprintf("%s[%d]", varAddrExpr(v.v), i)) + r, err := v.env.variableValueToStarlarkValue(v2, false) + if err != nil { + return starlark.String(err.Error()) + } + return r +} + +func (v sliceVariableAsStarlarkValue) Len() int { + return int(v.v.Len) +} + +func (v sliceVariableAsStarlarkValue) Iterate() starlark.Iterator { + return &sliceVariableAsStarlarkValueIterator{0, v.v, v.env} +} + +type sliceVariableAsStarlarkValueIterator struct { + cur int64 + v *api.Variable + env *Env +} + +func (it *sliceVariableAsStarlarkValueIterator) Done() { +} + +func (it *sliceVariableAsStarlarkValueIterator) Next(p *starlark.Value) bool { + if it.cur >= it.v.Len { + return false + } + s := sliceVariableAsStarlarkValue{it.v, it.env} + *p = s.Index(int(it.cur)) + it.cur++ + return true +} + +type ptrVariableAsStarlarkValue struct { + v *api.Variable + env *Env +} + +var _ starlark.HasAttrs = ptrVariableAsStarlarkValue{} +var _ starlark.Mapping = ptrVariableAsStarlarkValue{} + +func (v ptrVariableAsStarlarkValue) Freeze() { +} + +func (v ptrVariableAsStarlarkValue) Hash() (uint32, error) { + return 0, fmt.Errorf("not hashable") +} + +func (v ptrVariableAsStarlarkValue) String() string { + return fmt.Sprintf("%s", v.v.SinglelineString()) +} + +func (v ptrVariableAsStarlarkValue) Truth() starlark.Bool { + return true +} + +func (v ptrVariableAsStarlarkValue) Type() string { + return v.v.Type +} + +func (v ptrVariableAsStarlarkValue) Attr(name string) (starlark.Value, error) { + if len(v.v.Children) == 0 { + return nil, nil // no such field or method + } + if v.v.Children[0].Kind == reflect.Struct { + // autodereference pointers to structs + x := structVariableAsStarlarkValue{&v.v.Children[0], v.env} + return x.Attr(name) + } else if v.v.Kind == reflect.Interface && v.v.Children[0].Kind == reflect.Ptr { + // allow double-autodereference for iface to ptr to struct + vchild := &v.v.Children[0] + if len(vchild.Children) > 0 { + vchild.Children[0] = *v.env.autoLoad(varAddrExpr(&vchild.Children[0])) + } + v2 := ptrVariableAsStarlarkValue{vchild, v.env} + return v2.Attr(name) + } + + return nil, nil +} + +func (v ptrVariableAsStarlarkValue) AttrNames() []string { + if v.v.Children[0].Kind != reflect.Struct { + return nil + } + // autodereference + x := structVariableAsStarlarkValue{&v.v.Children[0], v.env} + return x.AttrNames() +} + +func (v ptrVariableAsStarlarkValue) Get(key starlark.Value) (starlark.Value, bool, error) { + if ikey, ok := key.(starlark.Int); ok { + if len(v.v.Children) == 0 { + return starlark.None, true, nil + } + if idx, _ := ikey.Int64(); idx == 0 { + r, err := v.env.variableValueToStarlarkValue(&v.v.Children[0], false) + if err != nil { + return starlark.String(err.Error()), true, nil + } + return r, true, nil + } + return starlark.None, false, nil + } + + if len(v.v.Children) == 0 || v.v.Children[0].Kind != reflect.Struct { + return starlark.None, false, nil + } + // autodereference + x := structVariableAsStarlarkValue{&v.v.Children[0], v.env} + return x.Get(key) +} + +type mapVariableAsStarlarkValue struct { + v *api.Variable + env *Env +} + +var _ starlark.IterableMapping = mapVariableAsStarlarkValue{} + +func (v mapVariableAsStarlarkValue) Freeze() { +} + +func (v mapVariableAsStarlarkValue) Hash() (uint32, error) { + return 0, fmt.Errorf("not hashable") +} + +func (v mapVariableAsStarlarkValue) String() string { + return fmt.Sprintf("%s", v.v.SinglelineString()) +} + +func (v mapVariableAsStarlarkValue) Truth() starlark.Bool { + return true +} + +func (v mapVariableAsStarlarkValue) Type() string { + return v.v.Type +} + +func (v mapVariableAsStarlarkValue) Get(key starlark.Value) (starlark.Value, bool, error) { + var keyExpr string + switch key := key.(type) { + case starlark.Int: + keyExpr = key.String() + case starlark.Float: + keyExpr = fmt.Sprintf("%g", float64(key)) + case starlark.String: + keyExpr = fmt.Sprintf("%q", string(key)) + case starlark.Bool: + keyExpr = fmt.Sprintf("%v", bool(key)) + case structVariableAsStarlarkValue: + keyExpr = varAddrExpr(key.v) + default: + return starlark.None, false, fmt.Errorf("key type not supported %T", key) + } + + v2 := v.env.autoLoad(fmt.Sprintf("%s[%s]", varAddrExpr(v.v), keyExpr)) + r, err := v.env.variableValueToStarlarkValue(v2, false) + if err != nil { + if err.Error() == "key not found" { + return starlark.None, false, nil + } + return starlark.None, false, err + } + return r, true, nil +} + +func (v mapVariableAsStarlarkValue) Items() []starlark.Tuple { + r := make([]starlark.Tuple, 0, len(v.v.Children)/2) + for i := 0; i < len(v.v.Children); i += 2 { + r = append(r, mapStarlarkTupleAt(v.v, v.env, i)) + } + return r +} + +func mapStarlarkTupleAt(v *api.Variable, env *Env, i int) starlark.Tuple { + keyv := env.autoLoad(varAddrExpr(&v.Children[i])) + key, err := env.variableValueToStarlarkValue(keyv, false) + if err != nil { + key = starlark.None + } + valv := env.autoLoad(varAddrExpr(&v.Children[i+1])) + val, err := env.variableValueToStarlarkValue(valv, false) + if err != nil { + val = starlark.None + } + return starlark.Tuple{key, val} +} + +func (v mapVariableAsStarlarkValue) Iterate() starlark.Iterator { + return &mapVariableAsStarlarkValueIterator{0, v.v, v.env} +} + +type mapVariableAsStarlarkValueIterator struct { + cur int + v *api.Variable + env *Env +} + +func (it *mapVariableAsStarlarkValueIterator) Done() { +} + +func (it *mapVariableAsStarlarkValueIterator) Next(p *starlark.Value) bool { + if it.cur >= 2*int(it.v.Len) { + return false + } + if it.cur >= len(it.v.Children) { + v2 := it.env.autoLoad(fmt.Sprintf("%s[%d:]", varAddrExpr(it.v), len(it.v.Children))) + it.v.Children = append(it.v.Children, v2.Children...) + } + if it.cur >= len(it.v.Children) { + return false + } + + keyv := it.env.autoLoad(varAddrExpr(&it.v.Children[it.cur])) + key, err := it.env.variableValueToStarlarkValue(keyv, false) + if err != nil { + key = starlark.None + } + *p = key + + it.cur += 2 + return true +} + +// unmarshalStarlarkValue unmarshals a starlark.Value 'val' into a Go variable 'dst'. +// This works similarly to encoding/json.Unmarshal and similar functions, +// but instead of getting its input from a byte buffer, it uses a +// starlark.Value. +func unmarshalStarlarkValue(val starlark.Value, dst interface{}, path string) error { + return unmarshalStarlarkValueIntl(val, reflect.ValueOf(dst), path) +} + +func unmarshalStarlarkValueIntl(val starlark.Value, dst reflect.Value, path string) (err error) { + defer func() { + // catches reflect panics + ierr := recover() + if ierr != nil { + err = fmt.Errorf("error setting argument %q to %s: %v", path, val, ierr) + } + }() + + converr := func(args ...string) error { + if len(args) > 0 { + return fmt.Errorf("error setting argument %q: can not convert %s to %s: %s", path, val, dst.Type().String(), args[0]) + } + return fmt.Errorf("error setting argument %q: can not convert %s to %s", path, val, dst.Type().String()) + } + + if _, isnone := val.(starlark.NoneType); isnone { + return nil + } + + for dst.Kind() == reflect.Ptr { + if dst.IsNil() { + dst.Set(reflect.New(dst.Type().Elem())) + } + dst = dst.Elem() + } + + switch val := val.(type) { + case starlark.Bool: + dst.SetBool(bool(val)) + case starlark.Int: + switch dst.Kind() { + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + n, ok := val.Uint64() + if !ok { + return converr() + } + dst.SetUint(n) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n, ok := val.Int64() + if !ok { + return converr() + } + dst.SetInt(n) + default: + return converr() + } + case starlark.Float: + dst.SetFloat(float64(val)) + case starlark.String: + dst.SetString(string(val)) + case *starlark.List: + if dst.Kind() != reflect.Slice { + return converr() + } + r := []reflect.Value{} + for i := 0; i < val.Len(); i++ { + cur := reflect.New(dst.Type().Elem()) + err := unmarshalStarlarkValueIntl(val.Index(i), cur, path) + if err != nil { + return err + } + r = append(r, cur) + } + case *starlark.Dict: + if dst.Kind() != reflect.Struct { + return converr() + } + for _, k := range val.Keys() { + if _, ok := k.(starlark.String); !ok { + return converr(fmt.Sprintf("non-string key %q", k.String())) + } + fieldName := string(k.(starlark.String)) + dstfield := dst.FieldByName(fieldName) + if dstfield == (reflect.Value{}) { + return converr(fmt.Sprintf("unknown field %s", fieldName)) + } + valfield, _, _ := val.Get(starlark.String(fieldName)) + err := unmarshalStarlarkValueIntl(valfield, dstfield, path+"."+fieldName) + if err != nil { + return err + } + } + case structAsStarlarkValue: + rv := val.v + if rv.Kind() == reflect.Ptr { + rv = rv.Elem() + } + dst.Set(rv) + default: + return converr() + } + return nil +} diff --git a/pkg/terminal/starbind/repl.go b/pkg/terminal/starbind/repl.go new file mode 100644 index 00000000..1ecede4e --- /dev/null +++ b/pkg/terminal/starbind/repl.go @@ -0,0 +1,203 @@ +package starbind + +// Code in this file is derived from go.starlark.net/repl/repl.go +// Which is licensed under the following copyright: +// +// Copyright (c) 2017 The Bazel Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the +// distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import ( + "fmt" + "io" + "os" + + "go.starlark.net/starlark" + "go.starlark.net/syntax" + + "github.com/peterh/liner" +) + +// REPL executes a read, eval, print loop. +func (env *Env) REPL() error { + thread := env.newThread() + globals := starlark.StringDict{} + for k, v := range env.env { + globals[k] = v + } + + rl := liner.NewLiner() + defer rl.Close() + for { + if err := isCancelled(thread); err != nil { + return err + } + if err := rep(rl, thread, globals); err != nil { + if err == io.EOF { + break + } + return err + } + } + fmt.Println() + return env.exportGlobals(globals) +} + +const ( + normalPrompt = ">>> " + extraPrompt = "... " + + exitCommand = "exit" +) + +// rep reads, evaluates, and prints one item. +// +// It returns an error (possibly readline.ErrInterrupt) +// only if readline failed. Starlark errors are printed. +func rep(rl *liner.State, thread *starlark.Thread, globals starlark.StringDict) error { + eof := false + + prompt := normalPrompt + readline := func() ([]byte, error) { + line, err := rl.Prompt(prompt) + if line == exitCommand { + eof = true + return nil, io.EOF + } + rl.AppendHistory(line) + prompt = extraPrompt + if err != nil { + if err == io.EOF { + eof = true + } + return nil, err + } + return []byte(line + "\n"), nil + } + + // parse + f, err := syntax.ParseCompoundStmt("", readline) + if err != nil { + if eof { + return io.EOF + } + printError(err) + return nil + } + + if expr := soleExpr(f); expr != nil { + //TODO: check for 'exit' + // eval + v, err := starlark.EvalExpr(thread, expr, globals) + if err != nil { + printError(err) + return nil + } + + // print + if v != starlark.None { + fmt.Println(v) + } + } else { + // compile + prog, err := starlark.FileProgram(f, globals.Has) + if err != nil { + printError(err) + return nil + } + + // execute (but do not freeze) + res, err := prog.Init(thread, globals) + if err != nil { + printError(err) + } + + // The global names from the previous call become + // the predeclared names of this call. + // If execution failed, some globals may be undefined. + for k, v := range res { + globals[k] = v + } + } + + return nil +} + +func soleExpr(f *syntax.File) syntax.Expr { + if len(f.Stmts) == 1 { + if stmt, ok := f.Stmts[0].(*syntax.ExprStmt); ok { + return stmt.X + } + } + return nil +} + +// PrintError prints the error to stderr, +// or its backtrace if it is a Starlark evaluation error. +func printError(err error) { + if evalErr, ok := err.(*starlark.EvalError); ok { + fmt.Fprintln(os.Stderr, evalErr.Backtrace()) + } else { + fmt.Fprintln(os.Stderr, err) + } +} + +// MakeLoad returns a simple sequential implementation of module loading +// suitable for use in the REPL. +// Each function returned by MakeLoad accesses a distinct private cache. +func MakeLoad() func(thread *starlark.Thread, module string) (starlark.StringDict, error) { + type entry struct { + globals starlark.StringDict + err error + } + + var cache = make(map[string]*entry) + + return func(thread *starlark.Thread, module string) (starlark.StringDict, error) { + e, ok := cache[module] + if e == nil { + if ok { + // request for package whose loading is in progress + return nil, fmt.Errorf("cycle in load graph") + } + + // Add a placeholder to indicate "load in progress". + cache[module] = nil + + // Load it. + thread := &starlark.Thread{Name: "exec " + module, Load: thread.Load} + globals, err := starlark.ExecFile(thread, module, nil, nil) + e = &entry{globals, err} + + // Update the cache. + cache[module] = e + } + return e.globals, e.err + } +} diff --git a/pkg/terminal/starbind/starlark.go b/pkg/terminal/starbind/starlark.go new file mode 100644 index 00000000..910e56df --- /dev/null +++ b/pkg/terminal/starbind/starlark.go @@ -0,0 +1,297 @@ +package starbind + +import ( + "context" + "fmt" + "io/ioutil" + "runtime" + "strings" + "sync" + + "go.starlark.net/resolve" + "go.starlark.net/starlark" + + "github.com/go-delve/delve/service" + "github.com/go-delve/delve/service/api" +) + +//go:generate go run ../../../scripts/gen-starlark-bindings.go go ./starlark_mapping.go +//go:generate go run ../../../scripts/gen-starlark-bindings.go doc ../../../Documentation/cli/starlark.md + +const ( + dlvCommandBuiltinName = "dlv_command" + readFileBuiltinName = "read_file" + writeFileBuiltinName = "write_file" + commandPrefix = "command_" + dlvContextName = "dlv_context" + curScopeBuiltinName = "cur_scope" + defaultLoadConfigBuiltinName = "default_load_config" +) + +func init() { + resolve.AllowNestedDef = true + resolve.AllowLambda = true + resolve.AllowFloat = true + resolve.AllowSet = true + resolve.AllowBitwise = true + resolve.AllowRecursion = true + resolve.AllowGlobalReassign = true +} + +// Context is the context in which starlark scripts are evaluated. +// It contains methods to call API functions, command line commands, etc. +type Context interface { + Client() service.Client + RegisterCommand(name, helpMsg string, cmdfn func(args string) error) + CallCommand(cmdstr string) error + Scope() api.EvalScope + LoadConfig() api.LoadConfig +} + +// Env is the environment used to evaluate starlark scripts. +type Env struct { + env starlark.StringDict + contextMu sync.Mutex + cancelfn context.CancelFunc + + ctx Context +} + +// New creates a new starlark binding environment. +func New(ctx Context) *Env { + env := &Env{} + + env.ctx = ctx + + env.env = env.starlarkPredeclare() + env.env[dlvCommandBuiltinName] = starlark.NewBuiltin(dlvCommandBuiltinName, func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, err + } + argstrs := make([]string, len(args)) + for i := range args { + a, ok := args[i].(starlark.String) + if !ok { + return nil, fmt.Errorf("argument of dlv_command is not a string") + } + argstrs[i] = string(a) + } + err := env.ctx.CallCommand(strings.Join(argstrs, " ")) + if err != nil && strings.Contains(err.Error(), " has exited with status ") { + return env.interfaceToStarlarkValue(err), nil + } + return starlark.None, decorateError(thread, err) + }) + env.env[readFileBuiltinName] = starlark.NewBuiltin(readFileBuiltinName, func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if len(args) != 1 { + return nil, decorateError(thread, fmt.Errorf("wrong number of arguments")) + } + path, ok := args[0].(starlark.String) + if !ok { + return nil, decorateError(thread, fmt.Errorf("argument of read_file was not a string")) + } + buf, err := ioutil.ReadFile(string(path)) + if err != nil { + return nil, decorateError(thread, err) + } + return starlark.String(string(buf)), nil + }) + env.env[writeFileBuiltinName] = starlark.NewBuiltin(writeFileBuiltinName, func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if len(args) != 2 { + return nil, decorateError(thread, fmt.Errorf("wrong number of arguments")) + } + path, ok := args[0].(starlark.String) + if !ok { + return nil, decorateError(thread, fmt.Errorf("first argument of write_file was not a string")) + } + err := ioutil.WriteFile(string(path), []byte(args[1].String()), 0640) + return starlark.None, decorateError(thread, err) + }) + env.env[curScopeBuiltinName] = starlark.NewBuiltin(curScopeBuiltinName, func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + return env.interfaceToStarlarkValue(env.ctx.Scope()), nil + }) + env.env[defaultLoadConfigBuiltinName] = starlark.NewBuiltin(defaultLoadConfigBuiltinName, func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + return env.interfaceToStarlarkValue(env.ctx.LoadConfig()), nil + }) + return env +} + +// Execute executes a script. Path is the name of the file to execute and +// source is the source code to execute. +// Source can be either a []byte, a string or a io.Reader. If source is nil +// Execute will execute the file specified by 'path'. +// After the file is executed if a function named mainFnName exists it will be called, passing args to it. +func (env *Env) Execute(path string, source interface{}, mainFnName string, args []interface{}) (starlark.Value, error) { + defer func() { + err := recover() + if err == nil { + return + } + fmt.Printf("panic executing starlark script: %v\n", err) + for i := 0; ; i++ { + pc, file, line, ok := runtime.Caller(i) + if !ok { + break + } + fname := "" + fn := runtime.FuncForPC(pc) + if fn != nil { + fname = fn.Name() + } + fmt.Printf("%s\n\tin %s:%d\n", fname, file, line) + } + }() + + thread := env.newThread() + globals, err := starlark.ExecFile(thread, path, source, env.env) + if err != nil { + return starlark.None, err + } + + err = env.exportGlobals(globals) + if err != nil { + return starlark.None, err + } + + return env.callMain(thread, globals, mainFnName, args) +} + +// exportGlobals saves globals with a name starting with a capital letter +// into the environment and creates commands from globals with a name +// starting with "command_" +func (env *Env) exportGlobals(globals starlark.StringDict) error { + for name, val := range globals { + switch { + case strings.HasPrefix(name, commandPrefix): + err := env.createCommand(name, val) + if err != nil { + return err + } + case name[0] >= 'A' && name[0] <= 'Z': + env.env[name] = val + } + } + return nil +} + +// Cancel cancels the execution of a currently running script or function. +func (env *Env) Cancel() { + if env == nil { + return + } + env.contextMu.Lock() + if env.cancelfn != nil { + env.cancelfn() + env.cancelfn = nil + } + env.contextMu.Unlock() +} + +func (env *Env) newThread() *starlark.Thread { + thread := &starlark.Thread{ + Print: func(_ *starlark.Thread, msg string) { fmt.Println(msg) }, + } + env.contextMu.Lock() + var ctx context.Context + ctx, env.cancelfn = context.WithCancel(context.Background()) + env.contextMu.Unlock() + thread.SetLocal(dlvContextName, ctx) + return thread +} + +func (env *Env) createCommand(name string, val starlark.Value) error { + fnval, ok := val.(*starlark.Function) + if !ok { + return nil + } + + name = name[len(commandPrefix):] + + helpMsg := fnval.Doc() + if helpMsg == "" { + helpMsg = "user defined" + } + + if fnval.NumParams() == 1 { + if p0, _ := fnval.Param(0); p0 == "args" { + env.ctx.RegisterCommand(name, helpMsg, func(args string) error { + _, err := starlark.Call(env.newThread(), fnval, starlark.Tuple{starlark.String(args)}, nil) + return err + }) + return nil + } + } + + env.ctx.RegisterCommand(name, helpMsg, func(args string) error { + thread := env.newThread() + argval, err := starlark.Eval(thread, "", "("+args+")", env.env) + if err != nil { + return err + } + argtuple, ok := argval.(starlark.Tuple) + if !ok { + argtuple = starlark.Tuple{argval} + } + _, err = starlark.Call(thread, fnval, argtuple, nil) + return err + }) + return nil +} + +// callMain calls the main function in globals, if one was defined. +func (env *Env) callMain(thread *starlark.Thread, globals starlark.StringDict, mainFnName string, args []interface{}) (starlark.Value, error) { + if mainFnName == "" { + return starlark.None, nil + } + mainval := globals[mainFnName] + if mainval == nil { + return starlark.None, nil + } + mainfn, ok := mainval.(*starlark.Function) + if !ok { + return starlark.None, fmt.Errorf("%s is not a function", mainFnName) + } + if mainfn.NumParams() != len(args) { + return starlark.None, fmt.Errorf("wrong number of arguments for %s", mainFnName) + } + argtuple := make(starlark.Tuple, len(args)) + for i := range args { + argtuple[i] = env.interfaceToStarlarkValue(args[i]) + } + return starlark.Call(thread, mainfn, argtuple, nil) +} + +type argument struct { + name string + defaultValue defaultValue +} + +type defaultValue uint8 + +const ( + defaultNone = iota + defaultScope + defaultLoadConfig +) + +func isCancelled(thread *starlark.Thread) error { + if ctx, ok := thread.Local(dlvContextName).(context.Context); ok { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + } + return nil +} + +func decorateError(thread *starlark.Thread, err error) error { + if err == nil { + return nil + } + pos := thread.Caller().Position() + if pos.Col > 0 { + return fmt.Errorf("%s:%d:%d: %v", pos.Filename(), pos.Line, pos.Col, err) + } + return fmt.Errorf("%s:%d: %v", pos.Filename(), pos.Line, err) +} diff --git a/pkg/terminal/starbind/starlark_mapping.go b/pkg/terminal/starbind/starlark_mapping.go new file mode 100644 index 00000000..bd93d339 --- /dev/null +++ b/pkg/terminal/starbind/starlark_mapping.go @@ -0,0 +1,1164 @@ +// DO NOT EDIT: auto-generated using scripts/gen-starlark-bindings.go + +package starbind + +import ( + "fmt" + "github.com/go-delve/delve/service/api" + "github.com/go-delve/delve/service/rpc2" + "go.starlark.net/starlark" +) + +func (env *Env) starlarkPredeclare() starlark.StringDict { + r := starlark.StringDict{} + + r["amend_breakpoint"] = starlark.NewBuiltin("amend_breakpoint", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.AmendBreakpointIn + var rpcRet rpc2.AmendBreakpointOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Breakpoint, "Breakpoint") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Breakpoint": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Breakpoint, "Breakpoint") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("AmendBreakpoint", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["ancestors"] = starlark.NewBuiltin("ancestors", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.AncestorsIn + var rpcRet rpc2.AncestorsOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.GoroutineID, "GoroutineID") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.NumAncestors, "NumAncestors") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 2 && args[2] != starlark.None { + err := unmarshalStarlarkValue(args[2], &rpcArgs.Depth, "Depth") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "GoroutineID": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.GoroutineID, "GoroutineID") + case "NumAncestors": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.NumAncestors, "NumAncestors") + case "Depth": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Depth, "Depth") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("Ancestors", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["attached_to_existing_process"] = starlark.NewBuiltin("attached_to_existing_process", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.AttachedToExistingProcessIn + var rpcRet rpc2.AttachedToExistingProcessOut + err := env.ctx.Client().CallAPI("AttachedToExistingProcess", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["cancel_next"] = starlark.NewBuiltin("cancel_next", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.CancelNextIn + var rpcRet rpc2.CancelNextOut + err := env.ctx.Client().CallAPI("CancelNext", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["checkpoint"] = starlark.NewBuiltin("checkpoint", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.CheckpointIn + var rpcRet rpc2.CheckpointOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Where, "Where") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Where": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Where, "Where") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("Checkpoint", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["clear_breakpoint"] = starlark.NewBuiltin("clear_breakpoint", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ClearBreakpointIn + var rpcRet rpc2.ClearBreakpointOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Id, "Id") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.Name, "Name") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Id": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Id, "Id") + case "Name": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Name, "Name") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("ClearBreakpoint", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["clear_checkpoint"] = starlark.NewBuiltin("clear_checkpoint", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ClearCheckpointIn + var rpcRet rpc2.ClearCheckpointOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.ID, "ID") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "ID": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.ID, "ID") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("ClearCheckpoint", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["raw_command"] = starlark.NewBuiltin("raw_command", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs api.DebuggerCommand + var rpcRet rpc2.CommandOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Name, "Name") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.ThreadID, "ThreadID") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 2 && args[2] != starlark.None { + err := unmarshalStarlarkValue(args[2], &rpcArgs.GoroutineID, "GoroutineID") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 3 && args[3] != starlark.None { + err := unmarshalStarlarkValue(args[3], &rpcArgs.ReturnInfoLoadConfig, "ReturnInfoLoadConfig") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + cfg := env.ctx.LoadConfig() + rpcArgs.ReturnInfoLoadConfig = &cfg + } + if len(args) > 4 && args[4] != starlark.None { + err := unmarshalStarlarkValue(args[4], &rpcArgs.Expr, "Expr") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 5 && args[5] != starlark.None { + err := unmarshalStarlarkValue(args[5], &rpcArgs.UnsafeCall, "UnsafeCall") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Name": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Name, "Name") + case "ThreadID": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.ThreadID, "ThreadID") + case "GoroutineID": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.GoroutineID, "GoroutineID") + case "ReturnInfoLoadConfig": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.ReturnInfoLoadConfig, "ReturnInfoLoadConfig") + case "Expr": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Expr, "Expr") + case "UnsafeCall": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.UnsafeCall, "UnsafeCall") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("Command", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["create_breakpoint"] = starlark.NewBuiltin("create_breakpoint", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.CreateBreakpointIn + var rpcRet rpc2.CreateBreakpointOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Breakpoint, "Breakpoint") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Breakpoint": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Breakpoint, "Breakpoint") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("CreateBreakpoint", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["detach"] = starlark.NewBuiltin("detach", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.DetachIn + var rpcRet rpc2.DetachOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Kill, "Kill") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Kill": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Kill, "Kill") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("Detach", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["disassemble"] = starlark.NewBuiltin("disassemble", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.DisassembleIn + var rpcRet rpc2.DisassembleOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Scope, "Scope") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + rpcArgs.Scope = env.ctx.Scope() + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.StartPC, "StartPC") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 2 && args[2] != starlark.None { + err := unmarshalStarlarkValue(args[2], &rpcArgs.EndPC, "EndPC") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 3 && args[3] != starlark.None { + err := unmarshalStarlarkValue(args[3], &rpcArgs.Flavour, "Flavour") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Scope": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Scope, "Scope") + case "StartPC": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.StartPC, "StartPC") + case "EndPC": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.EndPC, "EndPC") + case "Flavour": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Flavour, "Flavour") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("Disassemble", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["eval"] = starlark.NewBuiltin("eval", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.EvalIn + var rpcRet rpc2.EvalOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Scope, "Scope") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + rpcArgs.Scope = env.ctx.Scope() + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.Expr, "Expr") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 2 && args[2] != starlark.None { + err := unmarshalStarlarkValue(args[2], &rpcArgs.Cfg, "Cfg") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + cfg := env.ctx.LoadConfig() + rpcArgs.Cfg = &cfg + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Scope": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Scope, "Scope") + case "Expr": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Expr, "Expr") + case "Cfg": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Cfg, "Cfg") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("Eval", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["find_location"] = starlark.NewBuiltin("find_location", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.FindLocationIn + var rpcRet rpc2.FindLocationOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Scope, "Scope") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + rpcArgs.Scope = env.ctx.Scope() + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.Loc, "Loc") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Scope": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Scope, "Scope") + case "Loc": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Loc, "Loc") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("FindLocation", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["function_return_locations"] = starlark.NewBuiltin("function_return_locations", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.FunctionReturnLocationsIn + var rpcRet rpc2.FunctionReturnLocationsOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.FnName, "FnName") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "FnName": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.FnName, "FnName") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("FunctionReturnLocations", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["get_breakpoint"] = starlark.NewBuiltin("get_breakpoint", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.GetBreakpointIn + var rpcRet rpc2.GetBreakpointOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Id, "Id") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.Name, "Name") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Id": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Id, "Id") + case "Name": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Name, "Name") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("GetBreakpoint", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["get_thread"] = starlark.NewBuiltin("get_thread", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.GetThreadIn + var rpcRet rpc2.GetThreadOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Id, "Id") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Id": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Id, "Id") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("GetThread", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["is_multiclient"] = starlark.NewBuiltin("is_multiclient", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.IsMulticlientIn + var rpcRet rpc2.IsMulticlientOut + err := env.ctx.Client().CallAPI("IsMulticlient", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["last_modified"] = starlark.NewBuiltin("last_modified", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.LastModifiedIn + var rpcRet rpc2.LastModifiedOut + err := env.ctx.Client().CallAPI("LastModified", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["breakpoints"] = starlark.NewBuiltin("breakpoints", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListBreakpointsIn + var rpcRet rpc2.ListBreakpointsOut + err := env.ctx.Client().CallAPI("ListBreakpoints", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["checkpoints"] = starlark.NewBuiltin("checkpoints", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListCheckpointsIn + var rpcRet rpc2.ListCheckpointsOut + err := env.ctx.Client().CallAPI("ListCheckpoints", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["dynamic_libraries"] = starlark.NewBuiltin("dynamic_libraries", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListDynamicLibrariesIn + var rpcRet rpc2.ListDynamicLibrariesOut + err := env.ctx.Client().CallAPI("ListDynamicLibraries", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["function_args"] = starlark.NewBuiltin("function_args", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListFunctionArgsIn + var rpcRet rpc2.ListFunctionArgsOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Scope, "Scope") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + rpcArgs.Scope = env.ctx.Scope() + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.Cfg, "Cfg") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + rpcArgs.Cfg = env.ctx.LoadConfig() + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Scope": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Scope, "Scope") + case "Cfg": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Cfg, "Cfg") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("ListFunctionArgs", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["functions"] = starlark.NewBuiltin("functions", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListFunctionsIn + var rpcRet rpc2.ListFunctionsOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Filter, "Filter") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Filter": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Filter, "Filter") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("ListFunctions", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["goroutines"] = starlark.NewBuiltin("goroutines", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListGoroutinesIn + var rpcRet rpc2.ListGoroutinesOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Start, "Start") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.Count, "Count") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Start": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Start, "Start") + case "Count": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Count, "Count") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("ListGoroutines", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["local_vars"] = starlark.NewBuiltin("local_vars", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListLocalVarsIn + var rpcRet rpc2.ListLocalVarsOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Scope, "Scope") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + rpcArgs.Scope = env.ctx.Scope() + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.Cfg, "Cfg") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + rpcArgs.Cfg = env.ctx.LoadConfig() + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Scope": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Scope, "Scope") + case "Cfg": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Cfg, "Cfg") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("ListLocalVars", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["package_vars"] = starlark.NewBuiltin("package_vars", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListPackageVarsIn + var rpcRet rpc2.ListPackageVarsOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Filter, "Filter") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.Cfg, "Cfg") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + rpcArgs.Cfg = env.ctx.LoadConfig() + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Filter": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Filter, "Filter") + case "Cfg": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Cfg, "Cfg") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("ListPackageVars", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["registers"] = starlark.NewBuiltin("registers", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListRegistersIn + var rpcRet rpc2.ListRegistersOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.ThreadID, "ThreadID") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.IncludeFp, "IncludeFp") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "ThreadID": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.ThreadID, "ThreadID") + case "IncludeFp": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.IncludeFp, "IncludeFp") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("ListRegisters", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["sources"] = starlark.NewBuiltin("sources", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListSourcesIn + var rpcRet rpc2.ListSourcesOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Filter, "Filter") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Filter": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Filter, "Filter") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("ListSources", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["threads"] = starlark.NewBuiltin("threads", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListThreadsIn + var rpcRet rpc2.ListThreadsOut + err := env.ctx.Client().CallAPI("ListThreads", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["types"] = starlark.NewBuiltin("types", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ListTypesIn + var rpcRet rpc2.ListTypesOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Filter, "Filter") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Filter": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Filter, "Filter") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("ListTypes", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["process_pid"] = starlark.NewBuiltin("process_pid", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.ProcessPidIn + var rpcRet rpc2.ProcessPidOut + err := env.ctx.Client().CallAPI("ProcessPid", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["recorded"] = starlark.NewBuiltin("recorded", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.RecordedIn + var rpcRet rpc2.RecordedOut + err := env.ctx.Client().CallAPI("Recorded", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["restart"] = starlark.NewBuiltin("restart", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.RestartIn + var rpcRet rpc2.RestartOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Position, "Position") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.ResetArgs, "ResetArgs") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 2 && args[2] != starlark.None { + err := unmarshalStarlarkValue(args[2], &rpcArgs.NewArgs, "NewArgs") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Position": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Position, "Position") + case "ResetArgs": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.ResetArgs, "ResetArgs") + case "NewArgs": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.NewArgs, "NewArgs") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("Restart", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["set_expr"] = starlark.NewBuiltin("set_expr", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.SetIn + var rpcRet rpc2.SetOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Scope, "Scope") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } else { + rpcArgs.Scope = env.ctx.Scope() + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.Symbol, "Symbol") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 2 && args[2] != starlark.None { + err := unmarshalStarlarkValue(args[2], &rpcArgs.Value, "Value") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Scope": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Scope, "Scope") + case "Symbol": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Symbol, "Symbol") + case "Value": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Value, "Value") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("Set", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["stacktrace"] = starlark.NewBuiltin("stacktrace", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.StacktraceIn + var rpcRet rpc2.StacktraceOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.Id, "Id") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 1 && args[1] != starlark.None { + err := unmarshalStarlarkValue(args[1], &rpcArgs.Depth, "Depth") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 2 && args[2] != starlark.None { + err := unmarshalStarlarkValue(args[2], &rpcArgs.Full, "Full") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 3 && args[3] != starlark.None { + err := unmarshalStarlarkValue(args[3], &rpcArgs.Defers, "Defers") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + if len(args) > 4 && args[4] != starlark.None { + err := unmarshalStarlarkValue(args[4], &rpcArgs.Cfg, "Cfg") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "Id": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Id, "Id") + case "Depth": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Depth, "Depth") + case "Full": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Full, "Full") + case "Defers": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Defers, "Defers") + case "Cfg": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.Cfg, "Cfg") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("Stacktrace", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + r["state"] = starlark.NewBuiltin("state", func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := isCancelled(thread); err != nil { + return starlark.None, decorateError(thread, err) + } + var rpcArgs rpc2.StateIn + var rpcRet rpc2.StateOut + if len(args) > 0 && args[0] != starlark.None { + err := unmarshalStarlarkValue(args[0], &rpcArgs.NonBlocking, "NonBlocking") + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + for _, kv := range kwargs { + var err error + switch kv[0].(starlark.String) { + case "NonBlocking": + err = unmarshalStarlarkValue(kv[1], &rpcArgs.NonBlocking, "NonBlocking") + default: + err = fmt.Errorf("unknown argument %q", kv[0]) + } + if err != nil { + return starlark.None, decorateError(thread, err) + } + } + err := env.ctx.Client().CallAPI("State", &rpcArgs, &rpcRet) + if err != nil { + return starlark.None, err + } + return env.interfaceToStarlarkValue(rpcRet), nil + }) + return r +} diff --git a/pkg/terminal/starlark.go b/pkg/terminal/starlark.go new file mode 100644 index 00000000..85a1ab98 --- /dev/null +++ b/pkg/terminal/starlark.go @@ -0,0 +1,59 @@ +package terminal + +import ( + "github.com/go-delve/delve/pkg/terminal/starbind" + "github.com/go-delve/delve/service" + "github.com/go-delve/delve/service/api" +) + +type starlarkContext struct { + term *Term +} + +var _ starbind.Context = starlarkContext{} + +func (ctx starlarkContext) Client() service.Client { + return ctx.term.client +} + +func (ctx starlarkContext) RegisterCommand(name, helpMsg string, fn func(args string) error) { + cmdfn := func(t *Term, ctx callContext, args string) error { + return fn(args) + } + + found := false + for i := range ctx.term.cmds.cmds { + cmd := &ctx.term.cmds.cmds[i] + for _, alias := range cmd.aliases { + if alias == name { + cmd.cmdFn = cmdfn + cmd.helpMsg = helpMsg + found = true + break + } + } + if found { + break + } + } + if !found { + newcmd := command{ + aliases: []string{name}, + helpMsg: helpMsg, + cmdFn: cmdfn, + } + ctx.term.cmds.cmds = append(ctx.term.cmds.cmds, newcmd) + } +} + +func (ctx starlarkContext) CallCommand(cmdstr string) error { + return ctx.term.cmds.Call(cmdstr, ctx.term) +} + +func (ctx starlarkContext) Scope() api.EvalScope { + return api.EvalScope{-1, ctx.term.cmds.frame, 0} +} + +func (ctx starlarkContext) LoadConfig() api.LoadConfig { + return ctx.term.loadConfig() +} diff --git a/pkg/terminal/starlark_test.go b/pkg/terminal/starlark_test.go new file mode 100644 index 00000000..665c5b4c --- /dev/null +++ b/pkg/terminal/starlark_test.go @@ -0,0 +1,157 @@ +package terminal + +import ( + "fmt" + "strings" + "testing" +) + +func TestStarlarkExamples(t *testing.T) { + withTestTerminal("testvariables2", t, func(term *FakeTerminal) { + term.MustExec("continue") + t.Run("goroutine_start_line", func(t *testing.T) { testStarlarkExampleGoroutineStartLine(t, term) }) + t.Run("create_breakpoint_main", func(t *testing.T) { testStarlarkExampleCreateBreakpointmain(t, term) }) + t.Run("switch_to_main_goroutine", func(t *testing.T) { testStarlarkExampleSwitchToMainGoroutine(t, term) }) + t.Run("linked_list", func(t *testing.T) { testStarlarkExampleLinkedList(t, term) }) + t.Run("echo_expr", func(t *testing.T) { testStarlarkEchoExpr(t, term) }) + t.Run("find_array", func(t *testing.T) { testStarlarkFindArray(t, term) }) + t.Run("map_iteartion", func(t *testing.T) { testStarlarkMapIteration(t, term) }) + }) +} + +func testStarlarkExampleGoroutineStartLine(t *testing.T, term *FakeTerminal) { + term.MustExec("source " + findStarFile("goroutine_start_line")) + out1 := term.MustExec("gsl") + t.Logf("gsl: %q", out1) + if !strings.Contains(out1, "func main() {") { + t.Fatalf("goroutine_start_line example failed") + } +} + +func testStarlarkExampleCreateBreakpointmain(t *testing.T, term *FakeTerminal) { + out2p1 := term.MustExec("source " + findStarFile("create_breakpoint_main")) + t.Logf("create_breakpoint_main: %s", out2p1) + out2p2 := term.MustExec("breakpoints") + t.Logf("breakpoints: %q", out2p2) + if !strings.Contains(out2p2, "main.afunc1") { + t.Fatalf("create_breakpoint_runtime_func example failed") + } +} + +func testStarlarkExampleSwitchToMainGoroutine(t *testing.T, term *FakeTerminal) { + gs, _, err := term.client.ListGoroutines(0, 0) + if err != nil { + t.Fatal(err) + } + for _, g := range gs { + if g.ID != 1 { + t.Logf("switching to goroutine %d\n", g.ID) + term.MustExec(fmt.Sprintf("goroutine %d", g.ID)) + break + } + } + term.MustExec("source " + findStarFile("switch_to_main_goroutine")) + out3p1 := term.MustExec("switch_to_main_goroutine") + out3p2 := term.MustExec("goroutine") + t.Logf("%q\n", out3p1) + t.Logf("%q\n", out3p2) + if !strings.Contains(out3p2, "Goroutine 1:\n") { + t.Fatalf("switch_to_main_goroutine example failed") + } +} + +func testStarlarkExampleLinkedList(t *testing.T, term *FakeTerminal) { + term.MustExec("source " + findStarFile("linked_list")) + out := term.MustExec("linked_list ll Next 3") + t.Logf("%q\n", out) + if n := len(strings.Split(strings.TrimRight(out, "\n"), "\n")); n != 3 { + t.Fatalf("wrong number of lines in output %d", n) + } + + out = term.MustExec("linked_list ll Next 100") + t.Logf("%q\n", out) + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + for i, line := range lines { + if i == 5 { + if line != "5: *main.List nil" { + t.Errorf("mismatched line %d %q", i, line) + } + } else { + if !strings.HasPrefix(line, fmt.Sprintf("%d: *main.List {N: %d, Next: ", i, i)) { + t.Errorf("mismatched line %d %q", i, line) + } + } + } + if len(lines) != 6 { + t.Fatalf("wrong number of output lines %d", len(lines)) + } +} + +func testStarlarkEchoExpr(t *testing.T, term *FakeTerminal) { + term.MustExec("source " + findStarFile("echo_expr")) + out := term.MustExec("echo_expr 2+2, 1-1, 2*3") + t.Logf("echo_expr %q", out) + if out != "a 4 b 0 c 6\n" { + t.Error("output mismatch") + } +} + +func testStarlarkFindArray(t *testing.T, term *FakeTerminal) { + term.MustExec("source " + findStarFile("find_array")) + out := term.MustExec(`find_array "s2", lambda x: x.A == 5`) + t.Logf("find_array (1) %q", out) + if out != "found 2\n" { + t.Error("output mismatch") + } + out = term.MustExec(`find_array "s2", lambda x: x.A == 20`) + t.Logf("find_array (2) %q", out) + if out != "not found\n" { + t.Error("output mismatch") + } +} + +func testStarlarkMapIteration(t *testing.T, term *FakeTerminal) { + out := term.MustExec("source " + findStarFile("starlark_map_iteration")) + t.Logf("%s", out) +} + +func TestStarlarkVariable(t *testing.T) { + withTestTerminal("testvariables2", t, func(term *FakeTerminal) { + term.MustExec("continue") + for _, tc := range []struct{ expr, tgt string }{ + {`v = eval(None, "i1").Variable; print(v.Value)`, "1"}, + {`v = eval(None, "f1").Variable; print(v.Value)`, "3"}, + {`v = eval(None, "as1").Variable; print(v.Value.A)`, "1"}, + {`v = eval(None, "as1").Variable; print(v.Value.B)`, "1"}, + {`v = eval(None, "as1").Variable; print(v.Value["A"])`, "1"}, + {`v = eval(None, "s1").Variable; print(v.Value[0])`, "one"}, + {`v = eval(None, "nilstruct").Variable; print(v.Value)`, "*main.astruct nil"}, + {`v = eval(None, "nilstruct").Variable; print(v.Value[0])`, "None"}, + {`v = eval(None, 'm1["Malone"]').Variable; print(v.Value)`, "main.astruct {A: 2, B: 3}"}, + {`v = eval(None, "m1").Variable; print(v.Value["Malone"])`, "main.astruct {A: 2, B: 3}"}, + {`v = eval(None, "m2").Variable; print(v.Value[1])`, "*main.astruct {A: 10, B: 11}"}, + {`v = eval(None, "c1.pb").Variable; print(v.Value)`, "*main.bstruct {a: main.astruct {A: 1, B: 2}}"}, + {`v = eval(None, "c1.pb").Variable; print(v.Value[0])`, "main.bstruct {a: main.astruct {A: 1, B: 2}}"}, + {`v = eval(None, "c1.pb").Variable; print(v.Value.a.B)`, "2"}, + {`v = eval(None, "iface1").Variable; print(v.Value[0])`, "*main.astruct {A: 1, B: 2}"}, + {`v = eval(None, "iface1").Variable; print(v.Value[0][0])`, "main.astruct {A: 1, B: 2}"}, + {`v = eval(None, "iface1").Variable; print(v.Value.A)`, "1"}, + + {`v = eval(None, "as1", {"MaxStructFields": -1}).Variable; print(v.Value.A)`, "1"}, + {`v = eval({"GoroutineID": -1}, "as1").Variable; print(v.Value.A)`, "1"}, + {`v = eval(cur_scope(), "as1").Variable; print(v.Value.A)`, "1"}, + {`v = eval(None, "as1", default_load_config()).Variable; print(v.Value.A)`, "1"}, + + // From starlark.md's examples + {`v = eval(None, "s2").Variable; print(v.Value[0])`, "main.astruct {A: 1, B: 2}"}, + {`v = eval(None, "s2").Variable; print(v.Value[1].A)`, "3"}, + {`v = eval(None, "s2").Variable; print(v.Value[1].A + 10)`, "13"}, + {`v = eval(None, "s2").Variable; print(v.Value[1]["B"])`, "4"}, + } { + out := strings.TrimSpace(term.MustExecStarlark(tc.expr)) + if out != tc.tgt { + t.Errorf("for %q\nexpected %q\ngot %q", tc.expr, tc.tgt, out) + } + } + }) +} diff --git a/pkg/terminal/terminal.go b/pkg/terminal/terminal.go index 5e6ecbce..251260fa 100644 --- a/pkg/terminal/terminal.go +++ b/pkg/terminal/terminal.go @@ -8,12 +8,12 @@ import ( "runtime" "strings" "sync" - "syscall" "github.com/peterh/liner" "github.com/go-delve/delve/pkg/config" + "github.com/go-delve/delve/pkg/terminal/starbind" "github.com/go-delve/delve/service" "github.com/go-delve/delve/service/api" ) @@ -54,6 +54,8 @@ type Term struct { stdout io.Writer InitFile string + starlarkEnv *starbind.Env + // quitContinue is set to true by exitCommand to signal that the process // should be resumed before quitting. quitContinue bool @@ -93,7 +95,7 @@ func New(client service.Client, conf *config.Config) *Term { conf.SourceListLineColor = ansiBlue } - return &Term{ + t := &Term{ client: client, conf: conf, prompt: "(dlv) ", @@ -102,6 +104,9 @@ func New(client service.Client, conf *config.Config) *Term { dumb: dumb, stdout: w, } + + t.starlarkEnv = starbind.New(starlarkContext{t}) + return t } // Close returns the terminal to its previous mode. @@ -111,6 +116,7 @@ func (t *Term) Close() { func (t *Term) sigintGuard(ch <-chan os.Signal, multiClient bool) { for range ch { + t.starlarkEnv.Cancel() if multiClient { answer, err := t.line.Prompt("Would you like to [s]top the target or [q]uit this client, leaving the target running [s/q]? ") if err != nil { diff --git a/scripts/gen-starlark-bindings.go b/scripts/gen-starlark-bindings.go new file mode 100644 index 00000000..89849943 --- /dev/null +++ b/scripts/gen-starlark-bindings.go @@ -0,0 +1,335 @@ +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/token" + "go/types" + "io/ioutil" + "log" + "os" + "strings" + "unicode" + + "golang.org/x/tools/go/packages" +) + +// getSuitableMethods returns the list of methods of service/rpc2.RPCServer that are exported as API calls +func getSuitableMethods(pkg *types.Package, typename string) []*types.Func { + r := []*types.Func{} + mset := types.NewMethodSet(types.NewPointer(pkg.Scope().Lookup(typename).Type())) + for i := 0; i < mset.Len(); i++ { + fn := mset.At(i).Obj().(*types.Func) + + if !fn.Exported() { + continue + } + + if fn.Name() == "Command" { + r = append(r, fn) + continue + } + + sig, ok := fn.Type().(*types.Signature) + if !ok { + continue + } + + // arguments must be (args, *reply) + if sig.Params().Len() != 2 { + continue + } + if ntyp, isname := sig.Params().At(0).Type().(*types.Named); !isname { + continue + } else if _, isstr := ntyp.Underlying().(*types.Struct); !isstr { + continue + } + if _, isptr := sig.Params().At(1).Type().(*types.Pointer); !isptr { + continue + } + + // return values must be (error) + if sig.Results().Len() != 1 { + continue + } + if sig.Results().At(0).Type().String() != "error" { + continue + } + + r = append(r, fn) + } + return r +} + +func fieldsOfStruct(typ types.Type) (fieldNames, fieldTypes []string) { + styp := typ.(*types.Named).Underlying().(*types.Struct) + for i := 0; i < styp.NumFields(); i++ { + fieldNames = append(fieldNames, styp.Field(i).Name()) + fieldTypes = append(fieldTypes, styp.Field(i).Type().String()) + } + return fieldNames, fieldTypes +} + +func camelToDash(in string) string { + out := []rune{} + for i, ch := range in { + if ch < 'A' || ch > 'Z' { + out = append(out, ch) + continue + } + + if i != 0 { + out = append(out, '_') + } + out = append(out, unicode.ToLower(ch)) + } + return string(out) +} + +type binding struct { + name string + fn *types.Func + + argType, retType string + + argNames []string + argTypes []string +} + +func processServerMethods(serverMethods []*types.Func) []binding { + bindings := make([]binding, len(serverMethods)) + for i, fn := range serverMethods { + sig, _ := fn.Type().(*types.Signature) + argNames, argTypes := fieldsOfStruct(sig.Params().At(0).Type()) + + name := camelToDash(fn.Name()) + + switch name { + case "set": + // avoid collision with builtin that already exists in starlark + name = "set_expr" + case "command": + name = "raw_command" + default: + // remove list_ prefix, it looks better + const listPrefix = "list_" + if strings.HasPrefix(name, listPrefix) { + name = name[len(listPrefix):] + } + } + + retType := sig.Params().At(1).Type().String() + if fn.Name() == "Command" { + retType = "rpc2.CommandOut" + } + + bindings[i] = binding{ + name: name, + fn: fn, + argType: sig.Params().At(0).Type().String(), + retType: retType, + argNames: argNames, + argTypes: argTypes, + } + } + return bindings +} + +func removePackagePath(typePath string) string { + lastSlash := strings.LastIndex(typePath, "/") + if lastSlash < 0 { + return typePath + } + return typePath[lastSlash+1:] +} + +func genMapping(bindings []binding) []byte { + buf := bytes.NewBuffer([]byte{}) + + fmt.Fprintf(buf, "// DO NOT EDIT: auto-generated using scripts/gen-starlark-bindings.go\n\n") + fmt.Fprintf(buf, "package starbind\n\n") + fmt.Fprintf(buf, "import ( \"go.starlark.net/starlark\" \n \"github.com/go-delve/delve/service/api\" \n \"github.com/go-delve/delve/service/rpc2\" \n \"fmt\" )\n\n") + fmt.Fprintf(buf, "func (env *Env) starlarkPredeclare() starlark.StringDict {\n") + fmt.Fprintf(buf, "r := starlark.StringDict{}\n\n") + + for _, binding := range bindings { + fmt.Fprintf(buf, "r[%q] = starlark.NewBuiltin(%q, func(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {", binding.name, binding.name) + fmt.Fprintf(buf, "if err := isCancelled(thread); err != nil { return starlark.None, decorateError(thread, err) }\n") + fmt.Fprintf(buf, "var rpcArgs %s\n", removePackagePath(binding.argType)) + fmt.Fprintf(buf, "var rpcRet %s\n", removePackagePath(binding.retType)) + + // unmarshal normal unnamed arguments + for i := range binding.argNames { + fmt.Fprintf(buf, "if len(args) > %d && args[%d] != starlark.None { err := unmarshalStarlarkValue(args[%d], &rpcArgs.%s, %q); if err != nil { return starlark.None, decorateError(thread, err) } }", i, i, i, binding.argNames[i], binding.argNames[i]) + + switch binding.argTypes[i] { + case "*github.com/go-delve/delve/service/api.LoadConfig": + if binding.fn.Name() != "Stacktrace" { + fmt.Fprintf(buf, "else { cfg := env.ctx.LoadConfig(); rpcArgs.%s = &cfg }", binding.argNames[i]) + } + case "github.com/go-delve/delve/service/api.LoadConfig": + fmt.Fprintf(buf, "else { rpcArgs.%s = env.ctx.LoadConfig() }", binding.argNames[i]) + case "*github.com/go-delve/delve/service/api.EvalScope": + fmt.Fprintf(buf, "else { scope := env.ctx.Scope(); rpcArgs.%s = &scope }", binding.argNames[i]) + case "github.com/go-delve/delve/service/api.EvalScope": + fmt.Fprintf(buf, "else { rpcArgs.%s = env.ctx.Scope() }", binding.argNames[i]) + } + + fmt.Fprintf(buf, "\n") + + } + + // unmarshal keyword arguments + if len(binding.argNames) > 0 { + fmt.Fprintf(buf, "for _, kv := range kwargs {\n") + fmt.Fprintf(buf, "var err error\n") + fmt.Fprintf(buf, "switch kv[0].(starlark.String) {\n") + for i := range binding.argNames { + fmt.Fprintf(buf, "case %q: ", binding.argNames[i]) + fmt.Fprintf(buf, "err = unmarshalStarlarkValue(kv[1], &rpcArgs.%s, %q)\n", binding.argNames[i], binding.argNames[i]) + } + fmt.Fprintf(buf, "default: err = fmt.Errorf(\"unknown argument %%q\", kv[0])") + fmt.Fprintf(buf, "}\n") + fmt.Fprintf(buf, "if err != nil { return starlark.None, decorateError(thread, err) }\n") + fmt.Fprintf(buf, "}\n") + } + + fmt.Fprintf(buf, "err := env.ctx.Client().CallAPI(%q, &rpcArgs, &rpcRet)\n", binding.fn.Name()) + fmt.Fprintf(buf, "if err != nil { return starlark.None, err }\n") + fmt.Fprintf(buf, "return env.interfaceToStarlarkValue(rpcRet), nil\n") + + fmt.Fprintf(buf, "})\n") + } + + fmt.Fprintf(buf, "return r\n") + fmt.Fprintf(buf, "}\n") + + return buf.Bytes() +} + +func genDocs(bindings []binding) []byte { + var buf bytes.Buffer + + fmt.Fprintf(&buf, "Function | API Call\n") + fmt.Fprintf(&buf, "---------|---------\n") + + for _, binding := range bindings { + argNames := strings.Join(binding.argNames, ", ") + fmt.Fprintf(&buf, "%s(%s) | Equivalent to API call [%s](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.%s)\n", binding.name, argNames, binding.fn.Name(), binding.fn.Name()) + } + + fmt.Fprintf(&buf, "dlv_command(command) | Executes the specified command as if typed at the dlv_prompt\n") + fmt.Fprintf(&buf, "read_file(path) | Reads the file as a string\n") + fmt.Fprintf(&buf, "write_file(path, contents) | Writes string to a file\n") + fmt.Fprintf(&buf, "cur_scope() | Returns the current evaluation scope\n") + fmt.Fprintf(&buf, "default_load_config() | Returns the current default load configuration\n") + + return buf.Bytes() +} + +const ( + startOfMappingTable = "" + endOfMappingTable = "" +) + +func spliceDocs(docpath string, docs []byte, outpath string) { + docbuf, err := ioutil.ReadFile(docpath) + if err != nil { + log.Fatalf("could not read doc file: %v", err) + } + + v := strings.Split(string(docbuf), startOfMappingTable) + if len(v) != 2 { + log.Fatal("could not find start of mapping table") + } + header := v[0] + v = strings.Split(v[1], endOfMappingTable) + if len(v) != 2 { + log.Fatal("could not find end of mapping table") + } + footer := v[1] + + outbuf := make([]byte, 0, len(header)+len(docs)+len(footer)+len(startOfMappingTable)+len(endOfMappingTable)+1) + outbuf = append(outbuf, []byte(header)...) + outbuf = append(outbuf, []byte(startOfMappingTable)...) + outbuf = append(outbuf, '\n') + outbuf = append(outbuf, docs...) + outbuf = append(outbuf, []byte(endOfMappingTable)...) + outbuf = append(outbuf, []byte(footer)...) + + if outpath != "-" { + err = ioutil.WriteFile(outpath, outbuf, 0664) + if err != nil { + log.Fatalf("could not write documentation file: %v", err) + } + } else { + os.Stdout.Write(outbuf) + } +} + +func usage() { + fmt.Fprintf(os.Stderr, "gen-starlark-bindings [doc|doc/dummy|go] \n\n") + fmt.Fprintf(os.Stderr, "Writes starlark documentation (doc) or mapping file (go) to . Specify doc/dummy to generated documentation without overwriting the destination file.\n") + os.Exit(1) +} + +func main() { + if len(os.Args) != 3 { + usage() + } + kind := os.Args[1] + path := os.Args[2] + + fset := &token.FileSet{} + cfg := &packages.Config{ + Mode: packages.LoadSyntax, + Fset: fset, + } + pkgs, err := packages.Load(cfg, "github.com/go-delve/delve/service/rpc2") + if err != nil { + log.Fatalf("could not load packages: %v", err) + } + + var serverMethods []*types.Func + packages.Visit(pkgs, func(pkg *packages.Package) bool { + if pkg.PkgPath == "github.com/go-delve/delve/service/rpc2" { + serverMethods = getSuitableMethods(pkg.Types, "RPCServer") + } + return true + }, nil) + + bindings := processServerMethods(serverMethods) + + switch kind { + case "go": + mapping := genMapping(bindings) + + outfh := os.Stdout + if path != "-" { + outfh, err = os.Create(path) + if err != nil { + log.Fatalf("could not create output file: %v", err) + } + defer outfh.Close() + } + + src, err := format.Source(mapping) + if err != nil { + fmt.Fprintf(os.Stderr, "%s", string(mapping)) + log.Fatal(err) + } + outfh.Write(src) + + case "doc": + docs := genDocs(bindings) + spliceDocs(path, docs, path) + + case "doc/dummy": + docs := genDocs(bindings) + spliceDocs(path, docs, "-") + + default: + usage() + } +} diff --git a/service/client.go b/service/client.go index 03e0eb9d..f7bed988 100644 --- a/service/client.go +++ b/service/client.go @@ -147,4 +147,7 @@ type Client interface { // Disconnect closes the connection to the server without sending a Detach request first. // If cont is true a continue command will be sent instead. Disconnect(cont bool) error + + // CallAPI allows calling an arbitrary rpc method (used by starlark bindings) + CallAPI(method string, args, reply interface{}) error } diff --git a/service/rpc2/client.go b/service/rpc2/client.go index 2b8da0f8..169f35fd 100644 --- a/service/rpc2/client.go +++ b/service/rpc2/client.go @@ -410,3 +410,7 @@ func (c *RPCClient) ListDynamicLibraries() ([]api.Image, error) { func (c *RPCClient) call(method string, args, reply interface{}) error { return c.client.Call("RPCServer."+method, args, reply) } + +func (c *RPCClient) CallAPI(method string, args, reply interface{}) error { + return c.call(method, args, reply) +} diff --git a/vendor/go.starlark.net/LICENSE b/vendor/go.starlark.net/LICENSE new file mode 100644 index 00000000..a6609a14 --- /dev/null +++ b/vendor/go.starlark.net/LICENSE @@ -0,0 +1,29 @@ +Copyright (c) 2017 The Bazel Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/go.starlark.net/internal/compile/compile.go b/vendor/go.starlark.net/internal/compile/compile.go new file mode 100644 index 00000000..1aee10c3 --- /dev/null +++ b/vendor/go.starlark.net/internal/compile/compile.go @@ -0,0 +1,1790 @@ +// Package compile defines the Starlark bytecode compiler. +// It is an internal package of the Starlark interpreter and is not directly accessible to clients. +// +// The compiler generates byte code with optional uint32 operands for a +// virtual machine with the following components: +// - a program counter, which is an index into the byte code array. +// - an operand stack, whose maximum size is computed for each function by the compiler. +// - an stack of active iterators. +// - an array of local variables. +// The number of local variables and their indices are computed by the resolver. +// - an array of free variables, for nested functions. +// As with locals, these are computed by the resolver. +// - an array of global variables, shared among all functions in the same module. +// All elements are initially nil. +// - two maps of predeclared and universal identifiers. +// +// A line number table maps each program counter value to a source position; +// these source positions do not currently record column information. +// +// Operands, logically uint32s, are encoded using little-endian 7-bit +// varints, the top bit indicating that more bytes follow. +// +package compile // import "go.starlark.net/internal/compile" + +import ( + "bytes" + "fmt" + "log" + "os" + "path/filepath" + "strconv" + + "go.starlark.net/resolve" + "go.starlark.net/syntax" +) + +const debug = false // TODO(adonovan): use a bitmap of options; and regexp to match files + +// Increment this to force recompilation of saved bytecode files. +const Version = 7 + +type Opcode uint8 + +// "x DUP x x" is a "stack picture" that describes the state of the +// stack before and after execution of the instruction. +// +// OP indicates an immediate operand that is an index into the +// specified table: locals, names, freevars, constants. +const ( + NOP Opcode = iota // - NOP - + + // stack operations + DUP // x DUP x x + DUP2 // x y DUP2 x y x y + POP // x POP - + EXCH // x y EXCH y x + + // binary comparisons + // (order must match Token) + LT + GT + GE + LE + EQL + NEQ + + // binary arithmetic + // (order must match Token) + PLUS + MINUS + STAR + SLASH + SLASHSLASH + PERCENT + AMP + PIPE + CIRCUMFLEX + LTLT + GTGT + + IN + + // unary operators + UPLUS // x UPLUS x + UMINUS // x UMINUS -x + TILDE // x TILDE ~x + + NONE // - NONE None + TRUE // - TRUE True + FALSE // - FALSE False + MANDATORY // - MANDATORY Mandatory [sentinel value for required kwonly args] + + ITERPUSH // iterable ITERPUSH - [pushes the iterator stack] + ITERPOP // - ITERPOP - [pops the iterator stack] + NOT // value NOT bool + RETURN // value RETURN - + SETINDEX // a i new SETINDEX - + INDEX // a i INDEX elem + SETDICT // dict key value SETDICT - + SETDICTUNIQ // dict key value SETDICTUNIQ - + APPEND // list elem APPEND - + SLICE // x lo hi step SLICE slice + INPLACE_ADD // x y INPLACE_ADD z where z is x+y or x.extend(y) + MAKEDICT // - MAKEDICT dict + + // --- opcodes with an argument must go below this line --- + + // control flow + JMP // - JMP - + CJMP // cond CJMP - + ITERJMP // - ITERJMP elem (and fall through) [acts on topmost iterator] + // or: - ITERJMP - (and jump) + + CONSTANT // - CONSTANT value + MAKETUPLE // x1 ... xn MAKETUPLE tuple + MAKELIST // x1 ... xn MAKELIST list + MAKEFUNC // defaults freevars MAKEFUNC fn + LOAD // from1 ... fromN module LOAD v1 ... vN + SETLOCAL // value SETLOCAL - + SETGLOBAL // value SETGLOBAL - + LOCAL // - LOCAL value + FREE // - FREE value + GLOBAL // - GLOBAL value + PREDECLARED // - PREDECLARED value + UNIVERSAL // - UNIVERSAL value + ATTR // x ATTR y y = x.name + SETFIELD // x y SETFIELD - x.name = y + UNPACK // iterable UNPACK vn ... v1 + + // n>>8 is #positional args and n&0xff is #named args (pairs). + CALL // fn positional named CALL result + CALL_VAR // fn positional named *args CALL_VAR result + CALL_KW // fn positional named **kwargs CALL_KW result + CALL_VAR_KW // fn positional named *args **kwargs CALL_VAR_KW result + + OpcodeArgMin = JMP + OpcodeMax = CALL_VAR_KW +) + +// TODO(adonovan): add dynamic checks for missing opcodes in the tables below. + +var opcodeNames = [...]string{ + AMP: "amp", + APPEND: "append", + ATTR: "attr", + CALL: "call", + CALL_KW: "call_kw ", + CALL_VAR: "call_var", + CALL_VAR_KW: "call_var_kw", + CIRCUMFLEX: "circumflex", + CJMP: "cjmp", + CONSTANT: "constant", + DUP2: "dup2", + DUP: "dup", + EQL: "eql", + EXCH: "exch", + FALSE: "false", + FREE: "free", + GE: "ge", + GLOBAL: "global", + GT: "gt", + GTGT: "gtgt", + IN: "in", + INDEX: "index", + INPLACE_ADD: "inplace_add", + ITERJMP: "iterjmp", + ITERPOP: "iterpop", + ITERPUSH: "iterpush", + JMP: "jmp", + LE: "le", + LOAD: "load", + LOCAL: "local", + LT: "lt", + LTLT: "ltlt", + MAKEDICT: "makedict", + MAKEFUNC: "makefunc", + MAKELIST: "makelist", + MAKETUPLE: "maketuple", + MANDATORY: "mandatory", + MINUS: "minus", + NEQ: "neq", + NONE: "none", + NOP: "nop", + NOT: "not", + PERCENT: "percent", + PIPE: "pipe", + PLUS: "plus", + POP: "pop", + PREDECLARED: "predeclared", + RETURN: "return", + SETDICT: "setdict", + SETDICTUNIQ: "setdictuniq", + SETFIELD: "setfield", + SETGLOBAL: "setglobal", + SETINDEX: "setindex", + SETLOCAL: "setlocal", + SLASH: "slash", + SLASHSLASH: "slashslash", + SLICE: "slice", + STAR: "star", + TILDE: "tilde", + TRUE: "true", + UMINUS: "uminus", + UNIVERSAL: "universal", + UNPACK: "unpack", + UPLUS: "uplus", +} + +const variableStackEffect = 0x7f + +// stackEffect records the effect on the size of the operand stack of +// each kind of instruction. For some instructions this requires computation. +var stackEffect = [...]int8{ + AMP: -1, + APPEND: -2, + ATTR: 0, + CALL: variableStackEffect, + CALL_KW: variableStackEffect, + CALL_VAR: variableStackEffect, + CALL_VAR_KW: variableStackEffect, + CIRCUMFLEX: -1, + CJMP: -1, + CONSTANT: +1, + DUP2: +2, + DUP: +1, + EQL: -1, + FALSE: +1, + FREE: +1, + GE: -1, + GLOBAL: +1, + GT: -1, + GTGT: -1, + IN: -1, + INDEX: -1, + INPLACE_ADD: -1, + ITERJMP: variableStackEffect, + ITERPOP: 0, + ITERPUSH: -1, + JMP: 0, + LE: -1, + LOAD: -1, + LOCAL: +1, + LT: -1, + LTLT: -1, + MAKEDICT: +1, + MAKEFUNC: -1, + MAKELIST: variableStackEffect, + MAKETUPLE: variableStackEffect, + MANDATORY: +1, + MINUS: -1, + NEQ: -1, + NONE: +1, + NOP: 0, + NOT: 0, + PERCENT: -1, + PIPE: -1, + PLUS: -1, + POP: -1, + PREDECLARED: +1, + RETURN: -1, + SETDICT: -3, + SETDICTUNIQ: -3, + SETFIELD: -2, + SETGLOBAL: -1, + SETINDEX: -3, + SETLOCAL: -1, + SLASH: -1, + SLASHSLASH: -1, + SLICE: -3, + STAR: -1, + TRUE: +1, + UMINUS: 0, + UNIVERSAL: +1, + UNPACK: variableStackEffect, + UPLUS: 0, +} + +func (op Opcode) String() string { + if op < OpcodeMax { + if name := opcodeNames[op]; name != "" { + return name + } + } + return fmt.Sprintf("illegal op (%d)", op) +} + +// A Program is a Starlark file in executable form. +// +// Programs are serialized by the gobProgram function, +// which must be updated whenever this declaration is changed. +type Program struct { + Loads []Ident // name (really, string) and position of each load stmt + Names []string // names of attributes and predeclared variables + Constants []interface{} // = string | int64 | float64 | *big.Int + Functions []*Funcode + Globals []Ident // for error messages and tracing + Toplevel *Funcode // module initialization function +} + +// A Funcode is the code of a compiled Starlark function. +// +// Funcodes are serialized by the gobFunc function, +// which must be updated whenever this declaration is changed. +type Funcode struct { + Prog *Program + Pos syntax.Position // position of def or lambda token + Name string // name of this function + Doc string // docstring of this function + Code []byte // the byte code + pclinetab []uint16 // mapping from pc to linenum + Locals []Ident // locals, parameters first + Freevars []Ident // for tracing + MaxStack int + NumParams int + NumKwonlyParams int + HasVarargs, HasKwargs bool +} + +// An Ident is the name and position of an identifier. +type Ident struct { + Name string + Pos syntax.Position +} + +// A pcomp holds the compiler state for a Program. +type pcomp struct { + prog *Program // what we're building + + names map[string]uint32 + constants map[interface{}]uint32 + functions map[*Funcode]uint32 +} + +// An fcomp holds the compiler state for a Funcode. +type fcomp struct { + fn *Funcode // what we're building + + pcomp *pcomp + pos syntax.Position // current position of generated code + loops []loop + block *block +} + +type loop struct { + break_, continue_ *block +} + +type block struct { + insns []insn + + // If the last insn is a RETURN, jmp and cjmp are nil. + // If the last insn is a CJMP or ITERJMP, + // cjmp and jmp are the "true" and "false" successors. + // Otherwise, jmp is the sole successor. + jmp, cjmp *block + + initialstack int // for stack depth computation + + // Used during encoding + index int // -1 => not encoded yet + addr uint32 +} + +type insn struct { + op Opcode + arg uint32 + line int32 +} + +func (fn *Funcode) Position(pc uint32) syntax.Position { + // Conceptually the table contains rows of the form (pc uint32, + // line int32). Since the pc always increases, usually by a + // small amount, and the line number typically also does too + // although it may decrease, again typically by a small amount, + // we use delta encoding, starting from {pc: 0, line: 0}. + // + // Each entry is encoded in 16 bits. + // The top 8 bits are the unsigned delta pc; the next 7 bits are + // the signed line number delta; and the bottom bit indicates + // that more rows follow because one of the deltas was maxed out. + // + // TODO(adonovan): opt: improve the encoding; include the column. + + pos := fn.Pos // copy the (annoyingly inaccessible) filename + pos.Line = 0 + pos.Col = 0 + + // Position returns the record for the + // largest PC value not greater than 'pc'. + var prevpc uint32 + complete := true + for _, x := range fn.pclinetab { + nextpc := prevpc + uint32(x>>8) + if complete && nextpc > pc { + return pos + } + prevpc = nextpc + pos.Line += int32(int8(x) >> 1) // sign extend Δline from 7 to 32 bits + complete = (x & 1) == 0 + } + return pos +} + +// idents convert syntactic identifiers to compiled form. +func idents(ids []*syntax.Ident) []Ident { + res := make([]Ident, len(ids)) + for i, id := range ids { + res[i].Name = id.Name + res[i].Pos = id.NamePos + } + return res +} + +// Expr compiles an expression to a program consisting of a single toplevel function. +func Expr(expr syntax.Expr, name string, locals []*syntax.Ident) *Funcode { + pos := syntax.Start(expr) + stmts := []syntax.Stmt{&syntax.ReturnStmt{Result: expr}} + return File(stmts, pos, name, locals, nil).Toplevel +} + +// File compiles the statements of a file into a program. +func File(stmts []syntax.Stmt, pos syntax.Position, name string, locals, globals []*syntax.Ident) *Program { + pcomp := &pcomp{ + prog: &Program{ + Globals: idents(globals), + }, + names: make(map[string]uint32), + constants: make(map[interface{}]uint32), + functions: make(map[*Funcode]uint32), + } + pcomp.prog.Toplevel = pcomp.function(name, pos, stmts, locals, nil) + + return pcomp.prog +} + +func (pcomp *pcomp) function(name string, pos syntax.Position, stmts []syntax.Stmt, locals, freevars []*syntax.Ident) *Funcode { + fcomp := &fcomp{ + pcomp: pcomp, + pos: pos, + fn: &Funcode{ + Prog: pcomp.prog, + Pos: pos, + Name: name, + Doc: docStringFromBody(stmts), + Locals: idents(locals), + Freevars: idents(freevars), + }, + } + + if debug { + fmt.Fprintf(os.Stderr, "start function(%s @ %s)\n", name, pos) + } + + // Convert AST to a CFG of instructions. + entry := fcomp.newBlock() + fcomp.block = entry + fcomp.stmts(stmts) + if fcomp.block != nil { + fcomp.emit(NONE) + fcomp.emit(RETURN) + } + + var oops bool // something bad happened + + setinitialstack := func(b *block, depth int) { + if b.initialstack == -1 { + b.initialstack = depth + } else if b.initialstack != depth { + fmt.Fprintf(os.Stderr, "%d: setinitialstack: depth mismatch: %d vs %d\n", + b.index, b.initialstack, depth) + oops = true + } + } + + // Linearize the CFG: + // compute order, address, and initial + // stack depth of each reachable block. + var pc uint32 + var blocks []*block + var maxstack int + var visit func(b *block) + visit = func(b *block) { + if b.index >= 0 { + return // already visited + } + b.index = len(blocks) + b.addr = pc + blocks = append(blocks, b) + + stack := b.initialstack + if debug { + fmt.Fprintf(os.Stderr, "%s block %d: (stack = %d)\n", name, b.index, stack) + } + var cjmpAddr *uint32 + var isiterjmp int + for i, insn := range b.insns { + pc++ + + // Compute size of argument. + if insn.op >= OpcodeArgMin { + switch insn.op { + case ITERJMP: + isiterjmp = 1 + fallthrough + case CJMP: + cjmpAddr = &b.insns[i].arg + pc += 4 + default: + pc += uint32(argLen(insn.arg)) + } + } + + // Compute effect on stack. + se := insn.stackeffect() + if debug { + fmt.Fprintln(os.Stderr, "\t", insn.op, stack, stack+se) + } + stack += se + if stack < 0 { + fmt.Fprintf(os.Stderr, "After pc=%d: stack underflow\n", pc) + oops = true + } + if stack+isiterjmp > maxstack { + maxstack = stack + isiterjmp + } + } + + if debug { + fmt.Fprintf(os.Stderr, "successors of block %d (start=%d):\n", + b.addr, b.index) + if b.jmp != nil { + fmt.Fprintf(os.Stderr, "jmp to %d\n", b.jmp.index) + } + if b.cjmp != nil { + fmt.Fprintf(os.Stderr, "cjmp to %d\n", b.cjmp.index) + } + } + + // Place the jmp block next. + if b.jmp != nil { + // jump threading (empty cycles are impossible) + for b.jmp.insns == nil { + b.jmp = b.jmp.jmp + } + + setinitialstack(b.jmp, stack+isiterjmp) + if b.jmp.index < 0 { + // Successor is not yet visited: + // place it next and fall through. + visit(b.jmp) + } else { + // Successor already visited; + // explicit backward jump required. + pc += 5 + } + } + + // Then the cjmp block. + if b.cjmp != nil { + // jump threading (empty cycles are impossible) + for b.cjmp.insns == nil { + b.cjmp = b.cjmp.jmp + } + + setinitialstack(b.cjmp, stack) + visit(b.cjmp) + + // Patch the CJMP/ITERJMP, if present. + if cjmpAddr != nil { + *cjmpAddr = b.cjmp.addr + } + } + } + setinitialstack(entry, 0) + visit(entry) + + fn := fcomp.fn + fn.MaxStack = maxstack + + // Emit bytecode (and position table). + if debug { + fmt.Fprintf(os.Stderr, "Function %s: (%d blocks, %d bytes)\n", name, len(blocks), pc) + } + fcomp.generate(blocks, pc) + + if debug { + fmt.Fprintf(os.Stderr, "code=%d maxstack=%d\n", fn.Code, fn.MaxStack) + } + + // Don't panic until we've completed printing of the function. + if oops { + panic("internal error") + } + + if debug { + fmt.Fprintf(os.Stderr, "end function(%s @ %s)\n", name, pos) + } + + return fn +} + +func docStringFromBody(body []syntax.Stmt) string { + if len(body) == 0 { + return "" + } + expr, ok := body[0].(*syntax.ExprStmt) + if !ok { + return "" + } + lit, ok := expr.X.(*syntax.Literal) + if !ok { + return "" + } + if lit.Token != syntax.STRING { + return "" + } + return lit.Value.(string) +} + +func (insn *insn) stackeffect() int { + se := int(stackEffect[insn.op]) + if se == variableStackEffect { + arg := int(insn.arg) + switch insn.op { + case CALL, CALL_KW, CALL_VAR, CALL_VAR_KW: + se = -int(2*(insn.arg&0xff) + insn.arg>>8) + if insn.op != CALL { + se-- + } + if insn.op == CALL_VAR_KW { + se-- + } + case ITERJMP: + // Stack effect differs by successor: + // +1 for jmp/false/ok + // 0 for cjmp/true/exhausted + // Handled specially in caller. + se = 0 + case MAKELIST, MAKETUPLE: + se = 1 - arg + case UNPACK: + se = arg - 1 + default: + panic(insn.op) + } + } + return se +} + +// generate emits the linear instruction stream from the CFG, +// and builds the PC-to-line number table. +func (fcomp *fcomp) generate(blocks []*block, codelen uint32) { + code := make([]byte, 0, codelen) + var pclinetab []uint16 + var prev struct { + pc uint32 + line int32 + } + + for _, b := range blocks { + if debug { + fmt.Fprintf(os.Stderr, "%d:\n", b.index) + } + pc := b.addr + for _, insn := range b.insns { + if insn.line != 0 { + // Instruction has a source position. Delta-encode it. + // See Funcode.Position for the encoding. + for { + var incomplete uint16 + + deltapc := pc - prev.pc + if deltapc > 0xff { + deltapc = 0xff + incomplete = 1 + } + prev.pc += deltapc + + deltaline := insn.line - prev.line + if deltaline > 0x3f { + deltaline = 0x3f + incomplete = 1 + } else if deltaline < -0x40 { + deltaline = -0x40 + incomplete = 1 + } + prev.line += deltaline + + entry := uint16(deltapc<<8) | uint16(uint8(deltaline<<1)) | incomplete + pclinetab = append(pclinetab, entry) + if incomplete == 0 { + break + } + } + + if debug { + fmt.Fprintf(os.Stderr, "\t\t\t\t\t; %s %d\n", + filepath.Base(fcomp.fn.Pos.Filename()), insn.line) + } + } + if debug { + PrintOp(fcomp.fn, pc, insn.op, insn.arg) + } + code = append(code, byte(insn.op)) + pc++ + if insn.op >= OpcodeArgMin { + if insn.op == CJMP || insn.op == ITERJMP { + code = addUint32(code, insn.arg, 4) // pad arg to 4 bytes + } else { + code = addUint32(code, insn.arg, 0) + } + pc = uint32(len(code)) + } + } + + if b.jmp != nil && b.jmp.index != b.index+1 { + addr := b.jmp.addr + if debug { + fmt.Fprintf(os.Stderr, "\t%d\tjmp\t\t%d\t; block %d\n", + pc, addr, b.jmp.index) + } + code = append(code, byte(JMP)) + code = addUint32(code, addr, 4) + } + } + if len(code) != int(codelen) { + panic("internal error: wrong code length") + } + + fcomp.fn.pclinetab = pclinetab + fcomp.fn.Code = code +} + +// addUint32 encodes x as 7-bit little-endian varint. +// TODO(adonovan): opt: steal top two bits of opcode +// to encode the number of complete bytes that follow. +func addUint32(code []byte, x uint32, min int) []byte { + end := len(code) + min + for x >= 0x80 { + code = append(code, byte(x)|0x80) + x >>= 7 + } + code = append(code, byte(x)) + // Pad the operand with NOPs to exactly min bytes. + for len(code) < end { + code = append(code, byte(NOP)) + } + return code +} + +func argLen(x uint32) int { + n := 0 + for x >= 0x80 { + n++ + x >>= 7 + } + return n + 1 +} + +// PrintOp prints an instruction. +// It is provided for debugging. +func PrintOp(fn *Funcode, pc uint32, op Opcode, arg uint32) { + if op < OpcodeArgMin { + fmt.Fprintf(os.Stderr, "\t%d\t%s\n", pc, op) + return + } + + var comment string + switch op { + case CONSTANT: + switch x := fn.Prog.Constants[arg].(type) { + case string: + comment = strconv.Quote(x) + default: + comment = fmt.Sprint(x) + } + case MAKEFUNC: + comment = fn.Prog.Functions[arg].Name + case SETLOCAL, LOCAL: + comment = fn.Locals[arg].Name + case SETGLOBAL, GLOBAL: + comment = fn.Prog.Globals[arg].Name + case ATTR, SETFIELD, PREDECLARED, UNIVERSAL: + comment = fn.Prog.Names[arg] + case FREE: + comment = fn.Freevars[arg].Name + case CALL, CALL_VAR, CALL_KW, CALL_VAR_KW: + comment = fmt.Sprintf("%d pos, %d named", arg>>8, arg&0xff) + default: + // JMP, CJMP, ITERJMP, MAKETUPLE, MAKELIST, LOAD, UNPACK: + // arg is just a number + } + var buf bytes.Buffer + fmt.Fprintf(&buf, "\t%d\t%-10s\t%d", pc, op, arg) + if comment != "" { + fmt.Fprint(&buf, "\t; ", comment) + } + fmt.Fprintln(&buf) + os.Stderr.Write(buf.Bytes()) +} + +// newBlock returns a new block. +func (fcomp) newBlock() *block { + return &block{index: -1, initialstack: -1} +} + +// emit emits an instruction to the current block. +func (fcomp *fcomp) emit(op Opcode) { + if op >= OpcodeArgMin { + panic("missing arg: " + op.String()) + } + insn := insn{op: op, line: fcomp.pos.Line} + fcomp.block.insns = append(fcomp.block.insns, insn) + fcomp.pos.Line = 0 +} + +// emit1 emits an instruction with an immediate operand. +func (fcomp *fcomp) emit1(op Opcode, arg uint32) { + if op < OpcodeArgMin { + panic("unwanted arg: " + op.String()) + } + insn := insn{op: op, arg: arg, line: fcomp.pos.Line} + fcomp.block.insns = append(fcomp.block.insns, insn) + fcomp.pos.Line = 0 +} + +// jump emits a jump to the specified block. +// On return, the current block is unset. +func (fcomp *fcomp) jump(b *block) { + if b == fcomp.block { + panic("self-jump") // unreachable: Starlark has no arbitrary looping constructs + } + fcomp.block.jmp = b + fcomp.block = nil +} + +// condjump emits a conditional jump (CJMP or ITERJMP) +// to the specified true/false blocks. +// (For ITERJMP, the cases are jmp/f/ok and cjmp/t/exhausted.) +// On return, the current block is unset. +func (fcomp *fcomp) condjump(op Opcode, t, f *block) { + if !(op == CJMP || op == ITERJMP) { + panic("not a conditional jump: " + op.String()) + } + fcomp.emit1(op, 0) // fill in address later + fcomp.block.cjmp = t + fcomp.jump(f) +} + +// nameIndex returns the index of the specified name +// within the name pool, adding it if necessary. +func (pcomp *pcomp) nameIndex(name string) uint32 { + index, ok := pcomp.names[name] + if !ok { + index = uint32(len(pcomp.prog.Names)) + pcomp.names[name] = index + pcomp.prog.Names = append(pcomp.prog.Names, name) + } + return index +} + +// constantIndex returns the index of the specified constant +// within the constant pool, adding it if necessary. +func (pcomp *pcomp) constantIndex(v interface{}) uint32 { + index, ok := pcomp.constants[v] + if !ok { + index = uint32(len(pcomp.prog.Constants)) + pcomp.constants[v] = index + pcomp.prog.Constants = append(pcomp.prog.Constants, v) + } + return index +} + +// functionIndex returns the index of the specified function +// AST the nestedfun pool, adding it if necessary. +func (pcomp *pcomp) functionIndex(fn *Funcode) uint32 { + index, ok := pcomp.functions[fn] + if !ok { + index = uint32(len(pcomp.prog.Functions)) + pcomp.functions[fn] = index + pcomp.prog.Functions = append(pcomp.prog.Functions, fn) + } + return index +} + +// string emits code to push the specified string. +func (fcomp *fcomp) string(s string) { + fcomp.emit1(CONSTANT, fcomp.pcomp.constantIndex(s)) +} + +// setPos sets the current source position. +// It should be called prior to any operation that can fail dynamically. +// All positions are assumed to belong to the same file. +func (fcomp *fcomp) setPos(pos syntax.Position) { + fcomp.pos = pos +} + +// set emits code to store the top-of-stack value +// to the specified local or global variable. +func (fcomp *fcomp) set(id *syntax.Ident) { + switch resolve.Scope(id.Scope) { + case resolve.Local: + fcomp.emit1(SETLOCAL, uint32(id.Index)) + case resolve.Global: + fcomp.emit1(SETGLOBAL, uint32(id.Index)) + default: + log.Fatalf("%s: set(%s): neither global nor local (%d)", id.NamePos, id.Name, id.Scope) + } +} + +// lookup emits code to push the value of the specified variable. +func (fcomp *fcomp) lookup(id *syntax.Ident) { + switch resolve.Scope(id.Scope) { + case resolve.Local: + fcomp.setPos(id.NamePos) + fcomp.emit1(LOCAL, uint32(id.Index)) + case resolve.Free: + fcomp.emit1(FREE, uint32(id.Index)) + case resolve.Global: + fcomp.setPos(id.NamePos) + fcomp.emit1(GLOBAL, uint32(id.Index)) + case resolve.Predeclared: + fcomp.setPos(id.NamePos) + fcomp.emit1(PREDECLARED, fcomp.pcomp.nameIndex(id.Name)) + case resolve.Universal: + fcomp.emit1(UNIVERSAL, fcomp.pcomp.nameIndex(id.Name)) + default: + log.Fatalf("%s: compiler.lookup(%s): scope = %d", id.NamePos, id.Name, id.Scope) + } +} + +func (fcomp *fcomp) stmts(stmts []syntax.Stmt) { + for _, stmt := range stmts { + fcomp.stmt(stmt) + } +} + +func (fcomp *fcomp) stmt(stmt syntax.Stmt) { + switch stmt := stmt.(type) { + case *syntax.ExprStmt: + if _, ok := stmt.X.(*syntax.Literal); ok { + // Opt: don't compile doc comments only to pop them. + return + } + fcomp.expr(stmt.X) + fcomp.emit(POP) + + case *syntax.BranchStmt: + // Resolver invariant: break/continue appear only within loops. + switch stmt.Token { + case syntax.PASS: + // no-op + case syntax.BREAK: + b := fcomp.loops[len(fcomp.loops)-1].break_ + fcomp.jump(b) + fcomp.block = fcomp.newBlock() // dead code + case syntax.CONTINUE: + b := fcomp.loops[len(fcomp.loops)-1].continue_ + fcomp.jump(b) + fcomp.block = fcomp.newBlock() // dead code + } + + case *syntax.IfStmt: + // Keep consistent with CondExpr. + t := fcomp.newBlock() + f := fcomp.newBlock() + done := fcomp.newBlock() + + fcomp.ifelse(stmt.Cond, t, f) + + fcomp.block = t + fcomp.stmts(stmt.True) + fcomp.jump(done) + + fcomp.block = f + fcomp.stmts(stmt.False) + fcomp.jump(done) + + fcomp.block = done + + case *syntax.AssignStmt: + switch stmt.Op { + case syntax.EQ: + // simple assignment: x = y + fcomp.expr(stmt.RHS) + fcomp.assign(stmt.OpPos, stmt.LHS) + + case syntax.PLUS_EQ, + syntax.MINUS_EQ, + syntax.STAR_EQ, + syntax.SLASH_EQ, + syntax.SLASHSLASH_EQ, + syntax.PERCENT_EQ, + syntax.AMP_EQ, + syntax.PIPE_EQ, + syntax.CIRCUMFLEX_EQ, + syntax.LTLT_EQ, + syntax.GTGT_EQ: + // augmented assignment: x += y + + var set func() + + // Evaluate "address" of x exactly once to avoid duplicate side-effects. + switch lhs := unparen(stmt.LHS).(type) { + case *syntax.Ident: + // x = ... + fcomp.lookup(lhs) + set = func() { + fcomp.set(lhs) + } + + case *syntax.IndexExpr: + // x[y] = ... + fcomp.expr(lhs.X) + fcomp.expr(lhs.Y) + fcomp.emit(DUP2) + fcomp.setPos(lhs.Lbrack) + fcomp.emit(INDEX) + set = func() { + fcomp.setPos(lhs.Lbrack) + fcomp.emit(SETINDEX) + } + + case *syntax.DotExpr: + // x.f = ... + fcomp.expr(lhs.X) + fcomp.emit(DUP) + name := fcomp.pcomp.nameIndex(lhs.Name.Name) + fcomp.setPos(lhs.Dot) + fcomp.emit1(ATTR, name) + set = func() { + fcomp.setPos(lhs.Dot) + fcomp.emit1(SETFIELD, name) + } + + default: + panic(lhs) + } + + fcomp.expr(stmt.RHS) + + if stmt.Op == syntax.PLUS_EQ { + // Allow the runtime to optimize list += iterable. + fcomp.setPos(stmt.OpPos) + fcomp.emit(INPLACE_ADD) + } else { + fcomp.binop(stmt.OpPos, stmt.Op-syntax.PLUS_EQ+syntax.PLUS) + } + set() + } + + case *syntax.DefStmt: + fcomp.function(stmt.Def, stmt.Name.Name, &stmt.Function) + fcomp.set(stmt.Name) + + case *syntax.ForStmt: + // Keep consistent with ForClause. + head := fcomp.newBlock() + body := fcomp.newBlock() + tail := fcomp.newBlock() + + fcomp.expr(stmt.X) + fcomp.setPos(stmt.For) + fcomp.emit(ITERPUSH) + fcomp.jump(head) + + fcomp.block = head + fcomp.condjump(ITERJMP, tail, body) + + fcomp.block = body + fcomp.assign(stmt.For, stmt.Vars) + fcomp.loops = append(fcomp.loops, loop{break_: tail, continue_: head}) + fcomp.stmts(stmt.Body) + fcomp.loops = fcomp.loops[:len(fcomp.loops)-1] + fcomp.jump(head) + + fcomp.block = tail + fcomp.emit(ITERPOP) + + case *syntax.WhileStmt: + head := fcomp.newBlock() + body := fcomp.newBlock() + done := fcomp.newBlock() + + fcomp.jump(head) + fcomp.block = head + fcomp.ifelse(stmt.Cond, body, done) + + fcomp.block = body + fcomp.loops = append(fcomp.loops, loop{break_: done, continue_: head}) + fcomp.stmts(stmt.Body) + fcomp.loops = fcomp.loops[:len(fcomp.loops)-1] + fcomp.jump(head) + + fcomp.block = done + + case *syntax.ReturnStmt: + if stmt.Result != nil { + fcomp.expr(stmt.Result) + } else { + fcomp.emit(NONE) + } + fcomp.emit(RETURN) + fcomp.block = fcomp.newBlock() // dead code + + case *syntax.LoadStmt: + for i := range stmt.From { + fcomp.string(stmt.From[i].Name) + } + module := stmt.Module.Value.(string) + fcomp.pcomp.prog.Loads = append(fcomp.pcomp.prog.Loads, Ident{ + Name: module, + Pos: stmt.Module.TokenPos, + }) + fcomp.string(module) + fcomp.setPos(stmt.Load) + fcomp.emit1(LOAD, uint32(len(stmt.From))) + for i := range stmt.To { + fcomp.emit1(SETGLOBAL, uint32(stmt.To[len(stmt.To)-1-i].Index)) + } + + default: + start, _ := stmt.Span() + log.Fatalf("%s: exec: unexpected statement %T", start, stmt) + } +} + +// assign implements lhs = rhs for arbitrary expressions lhs. +// RHS is on top of stack, consumed. +func (fcomp *fcomp) assign(pos syntax.Position, lhs syntax.Expr) { + switch lhs := lhs.(type) { + case *syntax.ParenExpr: + // (lhs) = rhs + fcomp.assign(pos, lhs.X) + + case *syntax.Ident: + // x = rhs + fcomp.set(lhs) + + case *syntax.TupleExpr: + // x, y = rhs + fcomp.assignSequence(pos, lhs.List) + + case *syntax.ListExpr: + // [x, y] = rhs + fcomp.assignSequence(pos, lhs.List) + + case *syntax.IndexExpr: + // x[y] = rhs + fcomp.expr(lhs.X) + fcomp.emit(EXCH) + fcomp.expr(lhs.Y) + fcomp.emit(EXCH) + fcomp.setPos(lhs.Lbrack) + fcomp.emit(SETINDEX) + + case *syntax.DotExpr: + // x.f = rhs + fcomp.expr(lhs.X) + fcomp.emit(EXCH) + fcomp.setPos(lhs.Dot) + fcomp.emit1(SETFIELD, fcomp.pcomp.nameIndex(lhs.Name.Name)) + + default: + panic(lhs) + } +} + +func (fcomp *fcomp) assignSequence(pos syntax.Position, lhs []syntax.Expr) { + fcomp.setPos(pos) + fcomp.emit1(UNPACK, uint32(len(lhs))) + for i := range lhs { + fcomp.assign(pos, lhs[i]) + } +} + +func (fcomp *fcomp) expr(e syntax.Expr) { + switch e := e.(type) { + case *syntax.ParenExpr: + fcomp.expr(e.X) + + case *syntax.Ident: + fcomp.lookup(e) + + case *syntax.Literal: + // e.Value is int64, float64, *bigInt, or string. + fcomp.emit1(CONSTANT, fcomp.pcomp.constantIndex(e.Value)) + + case *syntax.ListExpr: + for _, x := range e.List { + fcomp.expr(x) + } + fcomp.emit1(MAKELIST, uint32(len(e.List))) + + case *syntax.CondExpr: + // Keep consistent with IfStmt. + t := fcomp.newBlock() + f := fcomp.newBlock() + done := fcomp.newBlock() + + fcomp.ifelse(e.Cond, t, f) + + fcomp.block = t + fcomp.expr(e.True) + fcomp.jump(done) + + fcomp.block = f + fcomp.expr(e.False) + fcomp.jump(done) + + fcomp.block = done + + case *syntax.IndexExpr: + fcomp.expr(e.X) + fcomp.expr(e.Y) + fcomp.setPos(e.Lbrack) + fcomp.emit(INDEX) + + case *syntax.SliceExpr: + fcomp.setPos(e.Lbrack) + fcomp.expr(e.X) + if e.Lo != nil { + fcomp.expr(e.Lo) + } else { + fcomp.emit(NONE) + } + if e.Hi != nil { + fcomp.expr(e.Hi) + } else { + fcomp.emit(NONE) + } + if e.Step != nil { + fcomp.expr(e.Step) + } else { + fcomp.emit(NONE) + } + fcomp.emit(SLICE) + + case *syntax.Comprehension: + if e.Curly { + fcomp.emit(MAKEDICT) + } else { + fcomp.emit1(MAKELIST, 0) + } + fcomp.comprehension(e, 0) + + case *syntax.TupleExpr: + fcomp.tuple(e.List) + + case *syntax.DictExpr: + fcomp.emit(MAKEDICT) + for _, entry := range e.List { + entry := entry.(*syntax.DictEntry) + fcomp.emit(DUP) + fcomp.expr(entry.Key) + fcomp.expr(entry.Value) + fcomp.setPos(entry.Colon) + fcomp.emit(SETDICTUNIQ) + } + + case *syntax.UnaryExpr: + fcomp.expr(e.X) + fcomp.setPos(e.OpPos) + switch e.Op { + case syntax.MINUS: + fcomp.emit(UMINUS) + case syntax.PLUS: + fcomp.emit(UPLUS) + case syntax.NOT: + fcomp.emit(NOT) + case syntax.TILDE: + fcomp.emit(TILDE) + default: + log.Fatalf("%s: unexpected unary op: %s", e.OpPos, e.Op) + } + + case *syntax.BinaryExpr: + switch e.Op { + // short-circuit operators + // TODO(adonovan): use ifelse to simplify conditions. + case syntax.OR: + // x or y => if x then x else y + done := fcomp.newBlock() + y := fcomp.newBlock() + + fcomp.expr(e.X) + fcomp.emit(DUP) + fcomp.condjump(CJMP, done, y) + + fcomp.block = y + fcomp.emit(POP) // discard X + fcomp.expr(e.Y) + fcomp.jump(done) + + fcomp.block = done + + case syntax.AND: + // x and y => if x then y else x + done := fcomp.newBlock() + y := fcomp.newBlock() + + fcomp.expr(e.X) + fcomp.emit(DUP) + fcomp.condjump(CJMP, y, done) + + fcomp.block = y + fcomp.emit(POP) // discard X + fcomp.expr(e.Y) + fcomp.jump(done) + + fcomp.block = done + + case syntax.PLUS: + fcomp.plus(e) + + default: + // all other strict binary operator (includes comparisons) + fcomp.expr(e.X) + fcomp.expr(e.Y) + fcomp.binop(e.OpPos, e.Op) + } + + case *syntax.DotExpr: + fcomp.expr(e.X) + fcomp.setPos(e.Dot) + fcomp.emit1(ATTR, fcomp.pcomp.nameIndex(e.Name.Name)) + + case *syntax.CallExpr: + fcomp.call(e) + + case *syntax.LambdaExpr: + fcomp.function(e.Lambda, "lambda", &e.Function) + + default: + start, _ := e.Span() + log.Fatalf("%s: unexpected expr %T", start, e) + } +} + +type summand struct { + x syntax.Expr + plusPos syntax.Position +} + +// plus emits optimized code for ((a+b)+...)+z that avoids naive +// quadratic behavior for strings, tuples, and lists, +// and folds together adjacent literals of the same type. +func (fcomp *fcomp) plus(e *syntax.BinaryExpr) { + // Gather all the right operands of the left tree of plusses. + // A tree (((a+b)+c)+d) becomes args=[a +b +c +d]. + args := make([]summand, 0, 2) // common case: 2 operands + for plus := e; ; { + args = append(args, summand{unparen(plus.Y), plus.OpPos}) + left := unparen(plus.X) + x, ok := left.(*syntax.BinaryExpr) + if !ok || x.Op != syntax.PLUS { + args = append(args, summand{x: left}) + break + } + plus = x + } + // Reverse args to syntactic order. + for i, n := 0, len(args)/2; i < n; i++ { + j := len(args) - 1 - i + args[i], args[j] = args[j], args[i] + } + + // Fold sums of adjacent literals of the same type: ""+"", []+[], ()+(). + out := args[:0] // compact in situ + for i := 0; i < len(args); { + j := i + 1 + if code := addable(args[i].x); code != 0 { + for j < len(args) && addable(args[j].x) == code { + j++ + } + if j > i+1 { + args[i].x = add(code, args[i:j]) + } + } + out = append(out, args[i]) + i = j + } + args = out + + // Emit code for an n-ary sum (n > 0). + fcomp.expr(args[0].x) + for _, summand := range args[1:] { + fcomp.expr(summand.x) + fcomp.setPos(summand.plusPos) + fcomp.emit(PLUS) + } + + // If len(args) > 2, use of an accumulator instead of a chain of + // PLUS operations may be more efficient. + // However, no gain was measured on a workload analogous to Bazel loading; + // TODO(adonovan): opt: re-evaluate on a Bazel analysis-like workload. + // + // We cannot use a single n-ary SUM operation + // a b c SUM<3> + // because we need to report a distinct error for each + // individual '+' operation, so three additional operations are + // needed: + // + // ACCSTART => create buffer and append to it + // ACCUM => append to buffer + // ACCEND => get contents of buffer + // + // For string, list, and tuple values, the interpreter can + // optimize these operations by using a mutable buffer. + // For all other types, ACCSTART and ACCEND would behave like + // the identity function and ACCUM behaves like PLUS. + // ACCUM must correctly support user-defined operations + // such as list+foo. + // + // fcomp.emit(ACCSTART) + // for _, summand := range args[1:] { + // fcomp.expr(summand.x) + // fcomp.setPos(summand.plusPos) + // fcomp.emit(ACCUM) + // } + // fcomp.emit(ACCEND) +} + +// addable reports whether e is a statically addable +// expression: a [s]tring, [l]ist, or [t]uple. +func addable(e syntax.Expr) rune { + switch e := e.(type) { + case *syntax.Literal: + // TODO(adonovan): opt: support INT/FLOAT/BIGINT constant folding. + switch e.Token { + case syntax.STRING: + return 's' + } + case *syntax.ListExpr: + return 'l' + case *syntax.TupleExpr: + return 't' + } + return 0 +} + +// add returns an expression denoting the sum of args, +// which are all addable values of the type indicated by code. +// The resulting syntax is degenerate, lacking position, etc. +func add(code rune, args []summand) syntax.Expr { + switch code { + case 's': + var buf bytes.Buffer + for _, arg := range args { + buf.WriteString(arg.x.(*syntax.Literal).Value.(string)) + } + return &syntax.Literal{Token: syntax.STRING, Value: buf.String()} + case 'l': + var elems []syntax.Expr + for _, arg := range args { + elems = append(elems, arg.x.(*syntax.ListExpr).List...) + } + return &syntax.ListExpr{List: elems} + case 't': + var elems []syntax.Expr + for _, arg := range args { + elems = append(elems, arg.x.(*syntax.TupleExpr).List...) + } + return &syntax.TupleExpr{List: elems} + } + panic(code) +} + +func unparen(e syntax.Expr) syntax.Expr { + if p, ok := e.(*syntax.ParenExpr); ok { + return unparen(p.X) + } + return e +} + +func (fcomp *fcomp) binop(pos syntax.Position, op syntax.Token) { + // TODO(adonovan): simplify by assuming syntax and compiler constants align. + fcomp.setPos(pos) + switch op { + // arithmetic + case syntax.PLUS: + fcomp.emit(PLUS) + case syntax.MINUS: + fcomp.emit(MINUS) + case syntax.STAR: + fcomp.emit(STAR) + case syntax.SLASH: + fcomp.emit(SLASH) + case syntax.SLASHSLASH: + fcomp.emit(SLASHSLASH) + case syntax.PERCENT: + fcomp.emit(PERCENT) + case syntax.AMP: + fcomp.emit(AMP) + case syntax.PIPE: + fcomp.emit(PIPE) + case syntax.CIRCUMFLEX: + fcomp.emit(CIRCUMFLEX) + case syntax.LTLT: + fcomp.emit(LTLT) + case syntax.GTGT: + fcomp.emit(GTGT) + case syntax.IN: + fcomp.emit(IN) + case syntax.NOT_IN: + fcomp.emit(IN) + fcomp.emit(NOT) + + // comparisons + case syntax.EQL, + syntax.NEQ, + syntax.GT, + syntax.LT, + syntax.LE, + syntax.GE: + fcomp.emit(Opcode(op-syntax.EQL) + EQL) + + default: + log.Fatalf("%s: unexpected binary op: %s", pos, op) + } +} + +func (fcomp *fcomp) call(call *syntax.CallExpr) { + // TODO(adonovan): opt: Use optimized path for calling methods + // of built-ins: x.f(...) to avoid materializing a closure. + // if dot, ok := call.Fcomp.(*syntax.DotExpr); ok { + // fcomp.expr(dot.X) + // fcomp.args(call) + // fcomp.emit1(CALL_ATTR, fcomp.name(dot.Name.Name)) + // return + // } + + // usual case + fcomp.expr(call.Fn) + op, arg := fcomp.args(call) + fcomp.setPos(call.Lparen) + fcomp.emit1(op, arg) +} + +// args emits code to push a tuple of positional arguments +// and a tuple of named arguments containing alternating keys and values. +// Either or both tuples may be empty (TODO(adonovan): optimize). +func (fcomp *fcomp) args(call *syntax.CallExpr) (op Opcode, arg uint32) { + var callmode int + // Compute the number of each kind of parameter. + var p, n int // number of positional, named arguments + var varargs, kwargs syntax.Expr + for _, arg := range call.Args { + if binary, ok := arg.(*syntax.BinaryExpr); ok && binary.Op == syntax.EQ { + + // named argument (name, value) + fcomp.string(binary.X.(*syntax.Ident).Name) + fcomp.expr(binary.Y) + n++ + continue + } + if unary, ok := arg.(*syntax.UnaryExpr); ok { + if unary.Op == syntax.STAR { + callmode |= 1 + varargs = unary.X + continue + } else if unary.Op == syntax.STARSTAR { + callmode |= 2 + kwargs = unary.X + continue + } + } + + // positional argument + fcomp.expr(arg) + p++ + } + + // Python2 and Python3 both permit named arguments + // to appear both before and after a *args argument: + // f(1, 2, x=3, *[4], y=5, **dict(z=6)) + // + // They also differ in their evaluation order: + // Python2: 1 2 3 5 4 6 (*args and **kwargs evaluated last) + // Python3: 1 2 4 3 5 6 (positional args evaluated before named args) + // Starlark-in-Java historically used a third order: + // Lexical: 1 2 3 4 5 6 (all args evaluated left-to-right) + // + // After discussion in github.com/bazelbuild/starlark#13, the + // spec now requires Starlark to statically reject named + // arguments after *args (e.g. y=5), and to use Python2-style + // evaluation order. This is both easy to implement and + // consistent with lexical order: + // + // f(1, 2, x=3, *[4], **dict(z=6)) # 1 2 3 4 6 + + // *args + if varargs != nil { + fcomp.expr(varargs) + } + + // **kwargs + if kwargs != nil { + fcomp.expr(kwargs) + } + + // TODO(adonovan): avoid this with a more flexible encoding. + if p >= 256 || n >= 256 { + // resolve already checked this; should be unreachable + panic("too many arguments in call") + } + + return CALL + Opcode(callmode), uint32(p<<8 | n) +} + +func (fcomp *fcomp) tuple(elems []syntax.Expr) { + for _, elem := range elems { + fcomp.expr(elem) + } + fcomp.emit1(MAKETUPLE, uint32(len(elems))) +} + +func (fcomp *fcomp) comprehension(comp *syntax.Comprehension, clauseIndex int) { + if clauseIndex == len(comp.Clauses) { + fcomp.emit(DUP) // accumulator + if comp.Curly { + // dict: {k:v for ...} + // Parser ensures that body is of form k:v. + // Python-style set comprehensions {body for vars in x} + // are not supported. + entry := comp.Body.(*syntax.DictEntry) + fcomp.expr(entry.Key) + fcomp.expr(entry.Value) + fcomp.setPos(entry.Colon) + fcomp.emit(SETDICT) + } else { + // list: [body for vars in x] + fcomp.expr(comp.Body) + fcomp.emit(APPEND) + } + return + } + + clause := comp.Clauses[clauseIndex] + switch clause := clause.(type) { + case *syntax.IfClause: + t := fcomp.newBlock() + done := fcomp.newBlock() + fcomp.ifelse(clause.Cond, t, done) + + fcomp.block = t + fcomp.comprehension(comp, clauseIndex+1) + fcomp.jump(done) + + fcomp.block = done + return + + case *syntax.ForClause: + // Keep consistent with ForStmt. + head := fcomp.newBlock() + body := fcomp.newBlock() + tail := fcomp.newBlock() + + fcomp.expr(clause.X) + fcomp.setPos(clause.For) + fcomp.emit(ITERPUSH) + fcomp.jump(head) + + fcomp.block = head + fcomp.condjump(ITERJMP, tail, body) + + fcomp.block = body + fcomp.assign(clause.For, clause.Vars) + fcomp.comprehension(comp, clauseIndex+1) + fcomp.jump(head) + + fcomp.block = tail + fcomp.emit(ITERPOP) + return + } + + start, _ := clause.Span() + log.Fatalf("%s: unexpected comprehension clause %T", start, clause) +} + +func (fcomp *fcomp) function(pos syntax.Position, name string, f *syntax.Function) { + // Evalution of the elements of both MAKETUPLEs may fail, + // so record the position. + fcomp.setPos(pos) + + // Generate tuple of parameter defaults. For: + // def f(p1, p2=dp2, p3=dp3, *, k1, k2=dk2, k3, **kwargs) + // the tuple is: + // (dp2, dp3, MANDATORY, dk2, MANDATORY). + n := 0 + seenStar := false + for _, param := range f.Params { + switch param := param.(type) { + case *syntax.BinaryExpr: + fcomp.expr(param.Y) + n++ + case *syntax.UnaryExpr: + seenStar = true // * or *args (also **kwargs) + case *syntax.Ident: + if seenStar { + fcomp.emit(MANDATORY) + n++ + } + } + } + fcomp.emit1(MAKETUPLE, uint32(n)) + + // Capture the values of the function's + // free variables from the lexical environment. + for _, freevar := range f.FreeVars { + fcomp.lookup(freevar) + } + fcomp.emit1(MAKETUPLE, uint32(len(f.FreeVars))) + + funcode := fcomp.pcomp.function(name, pos, f.Body, f.Locals, f.FreeVars) + + if debug { + // TODO(adonovan): do compilations sequentially not as a tree, + // to make the log easier to read. + // Simplify by identifying Toplevel and functionIndex 0. + fmt.Fprintf(os.Stderr, "resuming %s @ %s\n", fcomp.fn.Name, fcomp.pos) + } + + // def f(a, *, b=1) has only 2 parameters. + numParams := len(f.Params) + if f.NumKwonlyParams > 0 && !f.HasVarargs { + numParams-- + } + + funcode.NumParams = numParams + funcode.NumKwonlyParams = f.NumKwonlyParams + funcode.HasVarargs = f.HasVarargs + funcode.HasKwargs = f.HasKwargs + fcomp.emit1(MAKEFUNC, fcomp.pcomp.functionIndex(funcode)) +} + +// ifelse emits a Boolean control flow decision. +// On return, the current block is unset. +func (fcomp *fcomp) ifelse(cond syntax.Expr, t, f *block) { + switch cond := cond.(type) { + case *syntax.UnaryExpr: + if cond.Op == syntax.NOT { + // if not x then goto t else goto f + // => + // if x then goto f else goto t + fcomp.ifelse(cond.X, f, t) + return + } + + case *syntax.BinaryExpr: + switch cond.Op { + case syntax.AND: + // if x and y then goto t else goto f + // => + // if x then ifelse(y, t, f) else goto f + fcomp.expr(cond.X) + y := fcomp.newBlock() + fcomp.condjump(CJMP, y, f) + + fcomp.block = y + fcomp.ifelse(cond.Y, t, f) + return + + case syntax.OR: + // if x or y then goto t else goto f + // => + // if x then goto t else ifelse(y, t, f) + fcomp.expr(cond.X) + y := fcomp.newBlock() + fcomp.condjump(CJMP, t, y) + + fcomp.block = y + fcomp.ifelse(cond.Y, t, f) + return + case syntax.NOT_IN: + // if x not in y then goto t else goto f + // => + // if x in y then goto f else goto t + copy := *cond + copy.Op = syntax.IN + fcomp.expr(©) + fcomp.condjump(CJMP, f, t) + return + } + } + + // general case + fcomp.expr(cond) + fcomp.condjump(CJMP, t, f) +} diff --git a/vendor/go.starlark.net/internal/compile/serial.go b/vendor/go.starlark.net/internal/compile/serial.go new file mode 100644 index 00000000..6056ea73 --- /dev/null +++ b/vendor/go.starlark.net/internal/compile/serial.go @@ -0,0 +1,373 @@ +package compile + +// This file defines functions to read and write a compile.Program to a file. +// +// It is the client's responsibility to avoid version skew between the +// compiler used to produce a file and the interpreter that consumes it. +// The version number is provided as a constant. +// Incompatible protocol changes should also increment the version number. +// +// Encoding +// +// Program: +// "sky!" [4]byte # magic number +// str uint32le # offset of section +// version varint # must match Version +// filename string +// numloads varint +// loads []Ident +// numnames varint +// names []string +// numconsts varint +// consts []Constant +// numglobals varint +// globals []Ident +// toplevel Funcode +// numfuncs varint +// funcs []Funcode +// []byte # concatenation of all referenced strings +// EOF +// +// Funcode: +// id Ident +// code []byte +// pclinetablen varint +// pclinetab []varint +// numlocals varint +// locals []Ident +// numfreevars varint +// freevar []Ident +// maxstack varint +// numparams varint +// numkwonlyparams varint +// hasvarargs varint (0 or 1) +// haskwargs varint (0 or 1) +// +// Ident: +// filename string +// line, col varint +// +// Constant: # type data +// type varint # 0=string string +// data ... # 1=int varint +// # 2=float varint (bits as uint64) +// # 3=bigint string (decimal ASCII text) +// +// The encoding starts with a four-byte magic number. +// The next four bytes are a little-endian uint32 +// that provides the offset of the string section +// at the end of the file, which contains the ordered +// concatenation of all strings referenced by the +// program. This design permits the decoder to read +// the first and second parts of the file into different +// memory allocations: the first (the encoded program) +// is transient, but the second (the strings) persists +// for the life of the Program. +// +// Within the encoded program, all strings are referred +// to by their length. As the encoder and decoder process +// the entire file sequentially, they are in lock step, +// so the start offset of each string is implicit. +// +// Program.Code is represented as a []byte slice to permit +// modification when breakpoints are set. All other strings +// are represented as strings. They all (unsafely) share the +// same backing byte slice. +// +// Aside from the str field, all integers are encoded as varints. + +import ( + "encoding/binary" + "fmt" + "math" + "math/big" + debugpkg "runtime/debug" + "unsafe" + + "go.starlark.net/syntax" +) + +const magic = "!sky" + +// Encode encodes a compiled Starlark program. +func (prog *Program) Encode() []byte { + var e encoder + e.p = append(e.p, magic...) + e.p = append(e.p, "????"...) // string data offset; filled in later + e.int(Version) + e.string(prog.Toplevel.Pos.Filename()) + e.idents(prog.Loads) + e.int(len(prog.Names)) + for _, name := range prog.Names { + e.string(name) + } + e.int(len(prog.Constants)) + for _, c := range prog.Constants { + switch c := c.(type) { + case string: + e.int(0) + e.string(c) + case int64: + e.int(1) + e.int64(c) + case float64: + e.int(2) + e.uint64(math.Float64bits(c)) + case *big.Int: + e.int(3) + e.string(c.Text(10)) + } + } + e.idents(prog.Globals) + e.function(prog.Toplevel) + e.int(len(prog.Functions)) + for _, fn := range prog.Functions { + e.function(fn) + } + + // Patch in the offset of the string data section. + binary.LittleEndian.PutUint32(e.p[4:8], uint32(len(e.p))) + + return append(e.p, e.s...) +} + +type encoder struct { + p []byte // encoded program + s []byte // strings + tmp [binary.MaxVarintLen64]byte +} + +func (e *encoder) int(x int) { + e.int64(int64(x)) +} + +func (e *encoder) int64(x int64) { + n := binary.PutVarint(e.tmp[:], x) + e.p = append(e.p, e.tmp[:n]...) +} + +func (e *encoder) uint64(x uint64) { + n := binary.PutUvarint(e.tmp[:], x) + e.p = append(e.p, e.tmp[:n]...) +} + +func (e *encoder) string(s string) { + e.int(len(s)) + e.s = append(e.s, s...) +} + +func (e *encoder) bytes(b []byte) { + e.int(len(b)) + e.s = append(e.s, b...) +} + +func (e *encoder) ident(id Ident) { + e.string(id.Name) + e.int(int(id.Pos.Line)) + e.int(int(id.Pos.Col)) +} + +func (e *encoder) idents(ids []Ident) { + e.int(len(ids)) + for _, id := range ids { + e.ident(id) + } +} + +func (e *encoder) function(fn *Funcode) { + e.ident(Ident{fn.Name, fn.Pos}) + e.string(fn.Doc) + e.bytes(fn.Code) + e.int(len(fn.pclinetab)) + for _, x := range fn.pclinetab { + e.int64(int64(x)) + } + e.idents(fn.Locals) + e.idents(fn.Freevars) + e.int(fn.MaxStack) + e.int(fn.NumParams) + e.int(fn.NumKwonlyParams) + e.int(b2i(fn.HasVarargs)) + e.int(b2i(fn.HasKwargs)) +} + +func b2i(b bool) int { + if b { + return 1 + } else { + return 0 + } +} + +// DecodeProgram decodes a compiled Starlark program from data. +func DecodeProgram(data []byte) (_ *Program, err error) { + if len(data) < len(magic) { + return nil, fmt.Errorf("not a compiled module: no magic number") + } + if got := string(data[:4]); got != magic { + return nil, fmt.Errorf("not a compiled module: got magic number %q, want %q", + got, magic) + } + defer func() { + if x := recover(); x != nil { + debugpkg.PrintStack() + err = fmt.Errorf("internal error while decoding program: %v", x) + } + }() + + offset := binary.LittleEndian.Uint32(data[4:8]) + d := decoder{ + p: data[8:offset], + s: append([]byte(nil), data[offset:]...), // allocate a copy, which will persist + } + + if v := d.int(); v != Version { + return nil, fmt.Errorf("version mismatch: read %d, want %d", v, Version) + } + + filename := d.string() + d.filename = &filename + + loads := d.idents() + + names := make([]string, d.int()) + for i := range names { + names[i] = d.string() + } + + // constants + constants := make([]interface{}, d.int()) + for i := range constants { + var c interface{} + switch d.int() { + case 0: + c = d.string() + case 1: + c = d.int64() + case 2: + c = math.Float64frombits(d.uint64()) + case 3: + c, _ = new(big.Int).SetString(d.string(), 10) + } + constants[i] = c + } + + globals := d.idents() + toplevel := d.function() + funcs := make([]*Funcode, d.int()) + for i := range funcs { + funcs[i] = d.function() + } + + prog := &Program{ + Loads: loads, + Names: names, + Constants: constants, + Globals: globals, + Functions: funcs, + Toplevel: toplevel, + } + toplevel.Prog = prog + for _, f := range funcs { + f.Prog = prog + } + + if len(d.p)+len(d.s) > 0 { + return nil, fmt.Errorf("internal error: unconsumed data during decoding") + } + + return prog, nil +} + +type decoder struct { + p []byte // encoded program + s []byte // strings + filename *string // (indirect to avoid keeping decoder live) +} + +func (d *decoder) int() int { + return int(d.int64()) +} + +func (d *decoder) int64() int64 { + x, len := binary.Varint(d.p[:]) + d.p = d.p[len:] + return x +} + +func (d *decoder) uint64() uint64 { + x, len := binary.Uvarint(d.p[:]) + d.p = d.p[len:] + return x +} + +func (d *decoder) string() (s string) { + if slice := d.bytes(); len(slice) > 0 { + // Avoid a memory allocation for each string + // by unsafely aliasing slice. + type string struct { + data *byte + len int + } + ptr := (*string)(unsafe.Pointer(&s)) + ptr.data = &slice[0] + ptr.len = len(slice) + } + return s +} + +func (d *decoder) bytes() []byte { + len := d.int() + r := d.s[:len:len] + d.s = d.s[len:] + return r +} + +func (d *decoder) ident() Ident { + name := d.string() + line := int32(d.int()) + col := int32(d.int()) + return Ident{Name: name, Pos: syntax.MakePosition(d.filename, line, col)} +} + +func (d *decoder) idents() []Ident { + idents := make([]Ident, d.int()) + for i := range idents { + idents[i] = d.ident() + } + return idents +} + +func (d *decoder) bool() bool { return d.int() != 0 } + +func (d *decoder) function() *Funcode { + id := d.ident() + doc := d.string() + code := d.bytes() + pclinetab := make([]uint16, d.int()) + for i := range pclinetab { + pclinetab[i] = uint16(d.int()) + } + locals := d.idents() + freevars := d.idents() + maxStack := d.int() + numParams := d.int() + numKwonlyParams := d.int() + hasVarargs := d.int() != 0 + hasKwargs := d.int() != 0 + return &Funcode{ + // Prog is filled in later. + Pos: id.Pos, + Name: id.Name, + Doc: doc, + Code: code, + pclinetab: pclinetab, + Locals: locals, + Freevars: freevars, + MaxStack: maxStack, + NumParams: numParams, + NumKwonlyParams: numKwonlyParams, + HasVarargs: hasVarargs, + HasKwargs: hasKwargs, + } +} diff --git a/vendor/go.starlark.net/internal/spell/spell.go b/vendor/go.starlark.net/internal/spell/spell.go new file mode 100644 index 00000000..7739faba --- /dev/null +++ b/vendor/go.starlark.net/internal/spell/spell.go @@ -0,0 +1,115 @@ +// Package spell file defines a simple spelling checker for use in attribute errors +// such as "no such field .foo; did you mean .food?". +package spell + +import ( + "strings" + "unicode" +) + +// Nearest returns the element of candidates +// nearest to x using the Levenshtein metric, +// or "" if none were promising. +func Nearest(x string, candidates []string) string { + // Ignore underscores and case when matching. + fold := func(s string) string { + return strings.Map(func(r rune) rune { + if r == '_' { + return -1 + } + return unicode.ToLower(r) + }, s) + } + + x = fold(x) + + var best string + bestD := (len(x) + 1) / 2 // allow up to 50% typos + for _, c := range candidates { + d := levenshtein(x, fold(c), bestD) + if d < bestD { + bestD = d + best = c + } + } + return best +} + +// levenshtein returns the non-negative Levenshtein edit distance +// between the byte strings x and y. +// +// If the computed distance exceeds max, +// the function may return early with an approximate value > max. +func levenshtein(x, y string, max int) int { + // This implementation is derived from one by Laurent Le Brun in + // Bazel that uses the single-row space efficiency trick + // described at bitbucket.org/clearer/iosifovich. + + // Let x be the shorter string. + if len(x) > len(y) { + x, y = y, x + } + + // Remove common prefix. + for i := 0; i < len(x); i++ { + if x[i] != y[i] { + x = x[i:] + y = y[i:] + break + } + } + if x == "" { + return len(y) + } + + if d := abs(len(x) - len(y)); d > max { + return d // excessive length divergence + } + + row := make([]int, len(y)+1) + for i := range row { + row[i] = i + } + + for i := 1; i <= len(x); i++ { + row[0] = i + best := i + prev := i - 1 + for j := 1; j <= len(y); j++ { + a := prev + b2i(x[i-1] != y[j-1]) // substitution + b := 1 + row[j-1] // deletion + c := 1 + row[j] // insertion + k := min(a, min(b, c)) + prev, row[j] = row[j], k + best = min(best, k) + } + if best > max { + return best + } + } + return row[len(y)] +} + +func b2i(b bool) int { + if b { + return 1 + } else { + return 0 + } +} + +func min(x, y int) int { + if x < y { + return x + } else { + return y + } +} + +func abs(x int) int { + if x >= 0 { + return x + } else { + return -x + } +} diff --git a/vendor/go.starlark.net/resolve/resolve.go b/vendor/go.starlark.net/resolve/resolve.go new file mode 100644 index 00000000..a5190066 --- /dev/null +++ b/vendor/go.starlark.net/resolve/resolve.go @@ -0,0 +1,922 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package resolve defines a name-resolution pass for Starlark abstract +// syntax trees. +// +// The resolver sets the Locals and FreeVars arrays of each DefStmt and +// the LocalIndex field of each syntax.Ident that refers to a local or +// free variable. It also sets the Locals array of a File for locals +// bound by comprehensions outside any function. Identifiers for global +// variables do not get an index. +package resolve // import "go.starlark.net/resolve" + +// All references to names are statically resolved. Names may be +// predeclared, global, or local to a function or module-level comprehension. +// The resolver maps each global name to a small integer and each local +// name to a small integer; these integers enable a fast and compact +// representation of globals and locals in the evaluator. +// +// As an optimization, the resolver classifies each predeclared name as +// either universal (e.g. None, len) or per-module (e.g. glob in Bazel's +// build language), enabling the evaluator to share the representation +// of the universal environment across all modules. +// +// The lexical environment is a tree of blocks with the module block at +// its root. The module's child blocks may be of two kinds: functions +// and comprehensions, and these may have further children of either +// kind. +// +// Python-style resolution requires multiple passes because a name is +// determined to be local to a function only if the function contains a +// "binding" use of it; similarly, a name is determined be global (as +// opposed to predeclared) if the module contains a binding use at the +// top level. For both locals and globals, a non-binding use may +// lexically precede the binding to which it is resolved. +// In the first pass, we inspect each function, recording in +// 'uses' each identifier and the environment block in which it occurs. +// If a use of a name is binding, such as a function parameter or +// assignment, we add the name to the block's bindings mapping and add a +// local variable to the enclosing function. +// +// As we finish resolving each function, we inspect all the uses within +// that function and discard ones that were found to be local. The +// remaining ones must be either free (local to some lexically enclosing +// function), or nonlocal (global or predeclared), but we cannot tell +// which until we have finished inspecting the outermost enclosing +// function. At that point, we can distinguish local from global names +// (and this is when Python would compute free variables). +// +// However, Starlark additionally requires that all references to global +// names are satisfied by some declaration in the current module; +// Starlark permits a function to forward-reference a global that has not +// been declared yet so long as it is declared before the end of the +// module. So, instead of re-resolving the unresolved references after +// each top-level function, we defer this until the end of the module +// and ensure that all such references are satisfied by some definition. +// +// At the end of the module, we visit each of the nested function blocks +// in bottom-up order, doing a recursive lexical lookup for each +// unresolved name. If the name is found to be local to some enclosing +// function, we must create a DefStmt.FreeVar (capture) parameter for +// each intervening function. We enter these synthetic bindings into +// the bindings map so that we create at most one freevar per name. If +// the name was not local, we check that it was defined at module level. +// +// We resolve all uses of locals in the module (due to comprehensions) +// in a similar way and compute the set of its local variables. +// +// Starlark enforces that all global names are assigned at most once on +// all control flow paths by forbidding if/else statements and loops at +// top level. A global may be used before it is defined, leading to a +// dynamic error. +// +// TODO(adonovan): opt: reuse local slots once locals go out of scope. + +import ( + "fmt" + "log" + "sort" + "strings" + + "go.starlark.net/internal/spell" + "go.starlark.net/syntax" +) + +const debug = false +const doesnt = "this Starlark dialect does not " + +// global options +// These features are either not standard Starlark (yet), or deprecated +// features of the BUILD language, so we put them behind flags. +var ( + AllowNestedDef = false // allow def statements within function bodies + AllowLambda = false // allow lambda expressions + AllowFloat = false // allow floating point literals, the 'float' built-in, and x / y + AllowSet = false // allow the 'set' built-in + AllowGlobalReassign = false // allow reassignment to globals declared in same file (deprecated) + AllowRecursion = false // allow while statements and recursive functions + + AllowBitwise = true // obsolete; bitwise operations (&, |, ^, ~, <<, and >>) are always enabled +) + +// File resolves the specified file. +// +// The isPredeclared and isUniversal predicates report whether a name is +// a pre-declared identifier (visible in the current module) or a +// universal identifier (visible in every module). +// Clients should typically pass predeclared.Has for the first and +// starlark.Universe.Has for the second, where predeclared is the +// module's StringDict of predeclared names and starlark.Universe is the +// standard set of built-ins. +// The isUniverse predicate is supplied a parameter to avoid a cyclic +// dependency upon starlark.Universe, not because users should ever need +// to redefine it. +func File(file *syntax.File, isPredeclared, isUniversal func(name string) bool) error { + r := newResolver(isPredeclared, isUniversal) + r.stmts(file.Stmts) + + r.env.resolveLocalUses() + + // At the end of the module, resolve all non-local variable references, + // computing closures. + // Function bodies may contain forward references to later global declarations. + r.resolveNonLocalUses(r.env) + + file.Locals = r.moduleLocals + file.Globals = r.moduleGlobals + + if len(r.errors) > 0 { + return r.errors + } + return nil +} + +// Expr resolves the specified expression. +// It returns the local variables bound within the expression. +// +// The isPredeclared and isUniversal predicates behave as for the File function. +func Expr(expr syntax.Expr, isPredeclared, isUniversal func(name string) bool) ([]*syntax.Ident, error) { + r := newResolver(isPredeclared, isUniversal) + r.expr(expr) + r.env.resolveLocalUses() + r.resolveNonLocalUses(r.env) // globals & universals + if len(r.errors) > 0 { + return nil, r.errors + } + return r.moduleLocals, nil +} + +// An ErrorList is a non-empty list of resolver error messages. +type ErrorList []Error // len > 0 + +func (e ErrorList) Error() string { return e[0].Error() } + +// An Error describes the nature and position of a resolver error. +type Error struct { + Pos syntax.Position + Msg string +} + +func (e Error) Error() string { return e.Pos.String() + ": " + e.Msg } + +// The Scope of a syntax.Ident indicates what kind of scope it has. +type Scope uint8 + +const ( + Undefined Scope = iota // name is not defined + Local // name is local to its function + Free // name is local to some enclosing function + Global // name is global to module + Predeclared // name is predeclared for this module (e.g. glob) + Universal // name is universal (e.g. len) +) + +var scopeNames = [...]string{ + Undefined: "undefined", + Local: "local", + Free: "free", + Global: "global", + Predeclared: "predeclared", + Universal: "universal", +} + +func (scope Scope) String() string { return scopeNames[scope] } + +func newResolver(isPredeclared, isUniversal func(name string) bool) *resolver { + return &resolver{ + env: new(block), // module block + isPredeclared: isPredeclared, + isUniversal: isUniversal, + globals: make(map[string]*syntax.Ident), + } +} + +type resolver struct { + // env is the current local environment: + // a linked list of blocks, innermost first. + // The tail of the list is the module block. + env *block + + // moduleLocals contains the local variables of the module + // (due to comprehensions outside any function). + // moduleGlobals contains the global variables of the module. + moduleLocals []*syntax.Ident + moduleGlobals []*syntax.Ident + + // globals maps each global name in the module + // to its first binding occurrence. + globals map[string]*syntax.Ident + + // These predicates report whether a name is + // pre-declared, either in this module or universally. + isPredeclared, isUniversal func(name string) bool + + loops int // number of enclosing for loops + + errors ErrorList +} + +// container returns the innermost enclosing "container" block: +// a function (function != nil) or module (function == nil). +// Container blocks accumulate local variable bindings. +func (r *resolver) container() *block { + for b := r.env; ; b = b.parent { + if b.function != nil || b.isModule() { + return b + } + } +} + +func (r *resolver) push(b *block) { + r.env.children = append(r.env.children, b) + b.parent = r.env + r.env = b +} + +func (r *resolver) pop() { r.env = r.env.parent } + +type block struct { + parent *block // nil for module block + + // In the module (root) block, both these fields are nil. + function *syntax.Function // only for function blocks + comp *syntax.Comprehension // only for comprehension blocks + + // bindings maps a name to its binding. + // A local binding has an index into its innermost enclosing container's locals array. + // A free binding has an index into its innermost enclosing function's freevars array. + bindings map[string]binding + + // children records the child blocks of the current one. + children []*block + + // uses records all identifiers seen in this container (function or module), + // and a reference to the environment in which they appear. + // As we leave each container block, we resolve them, + // so that only free and global ones remain. + // At the end of each top-level function we compute closures. + uses []use +} + +type binding struct { + scope Scope + index int +} + +func (b *block) isModule() bool { return b.parent == nil } + +func (b *block) bind(name string, bind binding) { + if b.bindings == nil { + b.bindings = make(map[string]binding) + } + b.bindings[name] = bind +} + +func (b *block) String() string { + if b.function != nil { + return "function block at " + fmt.Sprint(b.function.Span()) + } + if b.comp != nil { + return "comprehension block at " + fmt.Sprint(b.comp.Span()) + } + return "module block" +} + +func (r *resolver) errorf(posn syntax.Position, format string, args ...interface{}) { + r.errors = append(r.errors, Error{posn, fmt.Sprintf(format, args...)}) +} + +// A use records an identifier and the environment in which it appears. +type use struct { + id *syntax.Ident + env *block +} + +// bind creates a binding for id in the current block, +// if there is not one already, and reports an error if +// a global was re-bound. +// It returns whether a binding already existed. +func (r *resolver) bind(id *syntax.Ident) bool { + // Binding outside any local (comprehension/function) block? + if r.env.isModule() { + id.Scope = uint8(Global) + prev, ok := r.globals[id.Name] + if ok { + if !AllowGlobalReassign { + r.errorf(id.NamePos, "cannot reassign global %s declared at %s", id.Name, prev.NamePos) + } + id.Index = prev.Index + } else { + // first global binding of this name + r.globals[id.Name] = id + id.Index = len(r.moduleGlobals) + r.moduleGlobals = append(r.moduleGlobals, id) + } + return ok + } + + // Mark this name as local to current block. + // Assign it a new local (positive) index in the current container. + _, ok := r.env.bindings[id.Name] + if !ok { + var locals *[]*syntax.Ident + if fn := r.container().function; fn != nil { + locals = &fn.Locals + } else { + locals = &r.moduleLocals + } + r.env.bind(id.Name, binding{Local, len(*locals)}) + *locals = append(*locals, id) + } + + r.use(id) + return ok +} + +func (r *resolver) use(id *syntax.Ident) { + use := use{id, r.env} + + // The spec says that if there is a global binding of a name + // then all references to that name in that block refer to the + // global, even if the use precedes the def---just as for locals. + // For example, in this code, + // + // print(len); len=1; print(len) + // + // both occurrences of len refer to the len=1 binding, which + // completely shadows the predeclared len function. + // + // The rationale for these semantics, which differ from Python, + // is that the static meaning of len (a reference to a global) + // does not change depending on where it appears in the file. + // Of course, its dynamic meaning does change, from an error + // into a valid reference, so it's not clear these semantics + // have any practical advantage. + // + // In any case, the Bazel implementation lags behind the spec + // and follows Python behavior, so the first use of len refers + // to the predeclared function. This typically used in a BUILD + // file that redefines a predeclared name half way through, + // for example: + // + // proto_library(...) # built-in rule + // load("myproto.bzl", "proto_library") + // proto_library(...) # user-defined rule + // + // We will piggyback support for the legacy semantics on the + // AllowGlobalReassign flag, which is loosely related and also + // required for Bazel. + if AllowGlobalReassign && r.env.isModule() { + r.useGlobal(use) + return + } + + b := r.container() + b.uses = append(b.uses, use) +} + +// useGlobals resolves use.id as a reference to a global. +// The use.env field captures the original environment for error reporting. +func (r *resolver) useGlobal(use use) binding { + id := use.id + var scope Scope + if prev, ok := r.globals[id.Name]; ok { + scope = Global // use of global declared by module + id.Index = prev.Index + } else if r.isPredeclared(id.Name) { + scope = Predeclared // use of pre-declared + } else if r.isUniversal(id.Name) { + scope = Universal // use of universal name + if !AllowFloat && id.Name == "float" { + r.errorf(id.NamePos, doesnt+"support floating point") + } + if !AllowSet && id.Name == "set" { + r.errorf(id.NamePos, doesnt+"support sets") + } + } else { + scope = Undefined + var hint string + if n := r.spellcheck(use); n != "" { + hint = fmt.Sprintf(" (did you mean %s?)", n) + } + r.errorf(id.NamePos, "undefined: %s%s", id.Name, hint) + } + id.Scope = uint8(scope) + return binding{scope, id.Index} +} + +// spellcheck returns the most likely misspelling of +// the name use.id in the environment use.env. +func (r *resolver) spellcheck(use use) string { + var names []string + + // locals + for b := use.env; b != nil; b = b.parent { + for name := range b.bindings { + names = append(names, name) + } + } + + // globals + // + // We have no way to enumerate predeclared/universe, + // which includes prior names in the REPL session. + for _, id := range r.moduleGlobals { + names = append(names, id.Name) + } + + sort.Strings(names) + return spell.Nearest(use.id.Name, names) +} + +// resolveLocalUses is called when leaving a container (function/module) +// block. It resolves all uses of locals within that block. +func (b *block) resolveLocalUses() { + unresolved := b.uses[:0] + for _, use := range b.uses { + if bind := lookupLocal(use); bind.scope == Local { + use.id.Scope = uint8(bind.scope) + use.id.Index = bind.index + } else { + unresolved = append(unresolved, use) + } + } + b.uses = unresolved +} + +func (r *resolver) stmts(stmts []syntax.Stmt) { + for _, stmt := range stmts { + r.stmt(stmt) + } +} + +func (r *resolver) stmt(stmt syntax.Stmt) { + switch stmt := stmt.(type) { + case *syntax.ExprStmt: + r.expr(stmt.X) + + case *syntax.BranchStmt: + if r.loops == 0 && (stmt.Token == syntax.BREAK || stmt.Token == syntax.CONTINUE) { + r.errorf(stmt.TokenPos, "%s not in a loop", stmt.Token) + } + + case *syntax.IfStmt: + if !AllowGlobalReassign && r.container().function == nil { + r.errorf(stmt.If, "if statement not within a function") + } + r.expr(stmt.Cond) + r.stmts(stmt.True) + r.stmts(stmt.False) + + case *syntax.AssignStmt: + r.expr(stmt.RHS) + isAugmented := stmt.Op != syntax.EQ + r.assign(stmt.LHS, isAugmented) + + case *syntax.DefStmt: + if !AllowNestedDef && r.container().function != nil { + r.errorf(stmt.Def, doesnt+"support nested def") + } + r.bind(stmt.Name) + r.function(stmt.Def, stmt.Name.Name, &stmt.Function) + + case *syntax.ForStmt: + if !AllowGlobalReassign && r.container().function == nil { + r.errorf(stmt.For, "for loop not within a function") + } + r.expr(stmt.X) + const isAugmented = false + r.assign(stmt.Vars, isAugmented) + r.loops++ + r.stmts(stmt.Body) + r.loops-- + + case *syntax.WhileStmt: + if !AllowRecursion { + r.errorf(stmt.While, doesnt+"support while loops") + } + if !AllowGlobalReassign && r.container().function == nil { + r.errorf(stmt.While, "while loop not within a function") + } + r.expr(stmt.Cond) + r.loops++ + r.stmts(stmt.Body) + r.loops-- + + case *syntax.ReturnStmt: + if r.container().function == nil { + r.errorf(stmt.Return, "return statement not within a function") + } + if stmt.Result != nil { + r.expr(stmt.Result) + } + + case *syntax.LoadStmt: + if r.container().function != nil { + r.errorf(stmt.Load, "load statement within a function") + } + + for i, from := range stmt.From { + if from.Name == "" { + r.errorf(from.NamePos, "load: empty identifier") + continue + } + if from.Name[0] == '_' { + r.errorf(from.NamePos, "load: names with leading underscores are not exported: %s", from.Name) + } + r.bind(stmt.To[i]) + } + + default: + log.Fatalf("unexpected stmt %T", stmt) + } +} + +func (r *resolver) assign(lhs syntax.Expr, isAugmented bool) { + switch lhs := lhs.(type) { + case *syntax.Ident: + // x = ... + r.bind(lhs) + + case *syntax.IndexExpr: + // x[i] = ... + r.expr(lhs.X) + r.expr(lhs.Y) + + case *syntax.DotExpr: + // x.f = ... + r.expr(lhs.X) + + case *syntax.TupleExpr: + // (x, y) = ... + if len(lhs.List) == 0 { + r.errorf(syntax.Start(lhs), "can't assign to ()") + } + if isAugmented { + r.errorf(syntax.Start(lhs), "can't use tuple expression in augmented assignment") + } + for _, elem := range lhs.List { + r.assign(elem, isAugmented) + } + + case *syntax.ListExpr: + // [x, y, z] = ... + if len(lhs.List) == 0 { + r.errorf(syntax.Start(lhs), "can't assign to []") + } + if isAugmented { + r.errorf(syntax.Start(lhs), "can't use list expression in augmented assignment") + } + for _, elem := range lhs.List { + r.assign(elem, isAugmented) + } + + case *syntax.ParenExpr: + r.assign(lhs.X, isAugmented) + + default: + name := strings.ToLower(strings.TrimPrefix(fmt.Sprintf("%T", lhs), "*syntax.")) + r.errorf(syntax.Start(lhs), "can't assign to %s", name) + } +} + +func (r *resolver) expr(e syntax.Expr) { + switch e := e.(type) { + case *syntax.Ident: + r.use(e) + + case *syntax.Literal: + if !AllowFloat && e.Token == syntax.FLOAT { + r.errorf(e.TokenPos, doesnt+"support floating point") + } + + case *syntax.ListExpr: + for _, x := range e.List { + r.expr(x) + } + + case *syntax.CondExpr: + r.expr(e.Cond) + r.expr(e.True) + r.expr(e.False) + + case *syntax.IndexExpr: + r.expr(e.X) + r.expr(e.Y) + + case *syntax.DictEntry: + r.expr(e.Key) + r.expr(e.Value) + + case *syntax.SliceExpr: + r.expr(e.X) + if e.Lo != nil { + r.expr(e.Lo) + } + if e.Hi != nil { + r.expr(e.Hi) + } + if e.Step != nil { + r.expr(e.Step) + } + + case *syntax.Comprehension: + // The 'in' operand of the first clause (always a ForClause) + // is resolved in the outer block; consider: [x for x in x]. + clause := e.Clauses[0].(*syntax.ForClause) + r.expr(clause.X) + + // A list/dict comprehension defines a new lexical block. + // Locals defined within the block will be allotted + // distinct slots in the locals array of the innermost + // enclosing container (function/module) block. + r.push(&block{comp: e}) + + const isAugmented = false + r.assign(clause.Vars, isAugmented) + + for _, clause := range e.Clauses[1:] { + switch clause := clause.(type) { + case *syntax.IfClause: + r.expr(clause.Cond) + case *syntax.ForClause: + r.assign(clause.Vars, isAugmented) + r.expr(clause.X) + } + } + r.expr(e.Body) // body may be *DictEntry + r.pop() + + case *syntax.TupleExpr: + for _, x := range e.List { + r.expr(x) + } + + case *syntax.DictExpr: + for _, entry := range e.List { + entry := entry.(*syntax.DictEntry) + r.expr(entry.Key) + r.expr(entry.Value) + } + + case *syntax.UnaryExpr: + r.expr(e.X) + + case *syntax.BinaryExpr: + if !AllowFloat && e.Op == syntax.SLASH { + r.errorf(e.OpPos, doesnt+"support floating point (use //)") + } + r.expr(e.X) + r.expr(e.Y) + + case *syntax.DotExpr: + r.expr(e.X) + // ignore e.Name + + case *syntax.CallExpr: + r.expr(e.Fn) + var seenVarargs, seenKwargs bool + var seenName map[string]bool + var n, p int + for _, arg := range e.Args { + pos, _ := arg.Span() + if unop, ok := arg.(*syntax.UnaryExpr); ok && unop.Op == syntax.STARSTAR { + // **kwargs + if seenKwargs { + r.errorf(pos, "multiple **kwargs not allowed") + } + seenKwargs = true + r.expr(arg) + } else if ok && unop.Op == syntax.STAR { + // *args + if seenKwargs { + r.errorf(pos, "*args may not follow **kwargs") + } else if seenVarargs { + r.errorf(pos, "multiple *args not allowed") + } + seenVarargs = true + r.expr(arg) + } else if binop, ok := arg.(*syntax.BinaryExpr); ok && binop.Op == syntax.EQ { + // k=v + n++ + if seenKwargs { + r.errorf(pos, "argument may not follow **kwargs") + } + x := binop.X.(*syntax.Ident) + if seenName[x.Name] { + r.errorf(x.NamePos, "keyword argument %s repeated", x.Name) + } else { + if seenName == nil { + seenName = make(map[string]bool) + } + seenName[x.Name] = true + } + r.expr(binop.Y) + } else { + // positional argument + p++ + if seenVarargs { + r.errorf(pos, "argument may not follow *args") + } else if seenKwargs { + r.errorf(pos, "argument may not follow **kwargs") + } else if len(seenName) > 0 { + r.errorf(pos, "positional argument may not follow named") + } + r.expr(arg) + } + } + + // Fail gracefully if compiler-imposed limit is exceeded. + if p >= 256 { + pos, _ := e.Span() + r.errorf(pos, "%v positional arguments in call, limit is 255", p) + } + if n >= 256 { + pos, _ := e.Span() + r.errorf(pos, "%v keyword arguments in call, limit is 255", n) + } + + case *syntax.LambdaExpr: + if !AllowLambda { + r.errorf(e.Lambda, doesnt+"support lambda") + } + r.function(e.Lambda, "lambda", &e.Function) + + case *syntax.ParenExpr: + r.expr(e.X) + + default: + log.Fatalf("unexpected expr %T", e) + } +} + +func (r *resolver) function(pos syntax.Position, name string, function *syntax.Function) { + // Resolve defaults in enclosing environment. + for _, param := range function.Params { + if binary, ok := param.(*syntax.BinaryExpr); ok { + r.expr(binary.Y) + } + } + + // Enter function block. + b := &block{function: function} + r.push(b) + + var seenOptional bool + var star *syntax.UnaryExpr // * or *args param + var starStar *syntax.Ident // **kwargs ident + var numKwonlyParams int + for _, param := range function.Params { + switch param := param.(type) { + case *syntax.Ident: + // e.g. x + if starStar != nil { + r.errorf(param.NamePos, "required parameter may not follow **%s", starStar.Name) + } else if star != nil { + numKwonlyParams++ + } else if seenOptional { + r.errorf(param.NamePos, "required parameter may not follow optional") + } + if r.bind(param) { + r.errorf(param.NamePos, "duplicate parameter: %s", param.Name) + } + + case *syntax.BinaryExpr: + // e.g. y=dflt + if starStar != nil { + r.errorf(param.OpPos, "optional parameter may not follow **%s", starStar.Name) + } else if star != nil { + numKwonlyParams++ + } + if id := param.X.(*syntax.Ident); r.bind(id) { + r.errorf(param.OpPos, "duplicate parameter: %s", id.Name) + } + seenOptional = true + + case *syntax.UnaryExpr: + // * or *args or **kwargs + if param.Op == syntax.STAR { + if starStar != nil { + r.errorf(param.OpPos, "* parameter may not follow **%s", starStar.Name) + } else if star != nil { + r.errorf(param.OpPos, "multiple * parameters not allowed") + } else { + star = param + } + } else { + if starStar != nil { + r.errorf(param.OpPos, "multiple ** parameters not allowed") + } + starStar = param.X.(*syntax.Ident) + } + } + } + + // Bind the *args and **kwargs parameters at the end, + // so that regular parameters a/b/c are contiguous and + // there is no hole for the "*": + // def f(a, b, *args, c=0, **kwargs) + // def f(a, b, *, c=0, **kwargs) + if star != nil { + if id, _ := star.X.(*syntax.Ident); id != nil { + // *args + if r.bind(id) { + r.errorf(id.NamePos, "duplicate parameter: %s", id.Name) + } + function.HasVarargs = true + } else if numKwonlyParams == 0 { + r.errorf(star.OpPos, "bare * must be followed by keyword-only parameters") + } + } + if starStar != nil { + if r.bind(starStar) { + r.errorf(starStar.NamePos, "duplicate parameter: %s", starStar.Name) + } + function.HasKwargs = true + } + + function.NumKwonlyParams = numKwonlyParams + r.stmts(function.Body) + + // Resolve all uses of this function's local vars, + // and keep just the remaining uses of free/global vars. + b.resolveLocalUses() + + // Leave function block. + r.pop() + + // References within the function body to globals are not + // resolved until the end of the module. +} + +func (r *resolver) resolveNonLocalUses(b *block) { + // First resolve inner blocks. + for _, child := range b.children { + r.resolveNonLocalUses(child) + } + for _, use := range b.uses { + bind := r.lookupLexical(use, use.env) + use.id.Scope = uint8(bind.scope) + use.id.Index = bind.index + } +} + +// lookupLocal looks up an identifier within its immediately enclosing function. +func lookupLocal(use use) binding { + for env := use.env; env != nil; env = env.parent { + if bind, ok := env.bindings[use.id.Name]; ok { + if bind.scope == Free { + // shouldn't exist till later + log.Fatalf("%s: internal error: %s, %d", use.id.NamePos, use.id.Name, bind) + } + return bind // found + } + if env.function != nil { + break + } + } + return binding{} // not found in this function +} + +// lookupLexical looks up an identifier use.id within its lexically enclosing environment. +// The use.env field captures the original environment for error reporting. +func (r *resolver) lookupLexical(use use, env *block) (bind binding) { + if debug { + fmt.Printf("lookupLexical %s in %s = ...\n", use.id.Name, env) + defer func() { fmt.Printf("= %d\n", bind) }() + } + + // Is this the module block? + if env.isModule() { + return r.useGlobal(use) // global, predeclared, or not found + } + + // Defined in this block? + bind, ok := env.bindings[use.id.Name] + if !ok { + // Defined in parent block? + bind = r.lookupLexical(use, env.parent) + if env.function != nil && (bind.scope == Local || bind.scope == Free) { + // Found in parent block, which belongs to enclosing function. + id := &syntax.Ident{ + Name: use.id.Name, + Scope: uint8(bind.scope), + Index: bind.index, + } + bind.scope = Free + bind.index = len(env.function.FreeVars) + env.function.FreeVars = append(env.function.FreeVars, id) + if debug { + fmt.Printf("creating freevar %v in function at %s: %s\n", + len(env.function.FreeVars), fmt.Sprint(env.function.Span()), id.Name) + } + } + + // Memoize, to avoid duplicate free vars + // and redundant global (failing) lookups. + env.bind(use.id.Name, bind) + } + return bind +} diff --git a/vendor/go.starlark.net/starlark/debug.go b/vendor/go.starlark.net/starlark/debug.go new file mode 100644 index 00000000..c08197ca --- /dev/null +++ b/vendor/go.starlark.net/starlark/debug.go @@ -0,0 +1,18 @@ +package starlark + +// This file defines an experimental API for the debugging tools. +// Some of these declarations expose details of internal packages. +// (The debugger makes liberal use of exported fields of unexported types.) +// Breaking changes may occur without notice. + +// Local returns the value of the i'th local variable. +// It may be nil if not yet assigned. +// +// Local may be called only for frames whose Callable is a *Function (a +// function defined by Starlark source code), and only while the frame +// is active; it will panic otherwise. +// +// This function is provided only for debugging tools. +// +// THIS API IS EXPERIMENTAL AND MAY CHANGE WITHOUT NOTICE. +func (fr *Frame) Local(i int) Value { return fr.locals[i] } diff --git a/vendor/go.starlark.net/starlark/empty.s b/vendor/go.starlark.net/starlark/empty.s new file mode 100644 index 00000000..3b821699 --- /dev/null +++ b/vendor/go.starlark.net/starlark/empty.s @@ -0,0 +1,3 @@ +// The presence of this file allows the package to use the +// "go:linkname" hack to call non-exported functions in the +// Go runtime, such as hardware-accelerated string hashing. diff --git a/vendor/go.starlark.net/starlark/eval.go b/vendor/go.starlark.net/starlark/eval.go new file mode 100644 index 00000000..82682703 --- /dev/null +++ b/vendor/go.starlark.net/starlark/eval.go @@ -0,0 +1,1438 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package starlark + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "math/big" + "sort" + "strings" + "unicode" + "unicode/utf8" + + "go.starlark.net/internal/compile" + "go.starlark.net/internal/spell" + "go.starlark.net/resolve" + "go.starlark.net/syntax" +) + +// A Thread contains the state of a Starlark thread, +// such as its call stack and thread-local storage. +// The Thread is threaded throughout the evaluator. +type Thread struct { + // Name is an optional name that describes the thread, for debugging. + Name string + + // frame is the current Starlark execution frame. + frame *Frame + + // Print is the client-supplied implementation of the Starlark + // 'print' function. If nil, fmt.Fprintln(os.Stderr, msg) is + // used instead. + Print func(thread *Thread, msg string) + + // Load is the client-supplied implementation of module loading. + // Repeated calls with the same module name must return the same + // module environment or error. + // The error message need not include the module name. + // + // See example_test.go for some example implementations of Load. + Load func(thread *Thread, module string) (StringDict, error) + + // locals holds arbitrary "thread-local" Go values belonging to the client. + // They are accessible to the client but not to any Starlark program. + locals map[string]interface{} +} + +// SetLocal sets the thread-local value associated with the specified key. +// It must not be called after execution begins. +func (thread *Thread) SetLocal(key string, value interface{}) { + if thread.locals == nil { + thread.locals = make(map[string]interface{}) + } + thread.locals[key] = value +} + +// Local returns the thread-local value associated with the specified key. +func (thread *Thread) Local(key string) interface{} { + return thread.locals[key] +} + +// Caller returns the frame of the caller of the current function. +// It should only be used in built-ins called from Starlark code. +func (thread *Thread) Caller() *Frame { return thread.frame.parent } + +// TopFrame returns the topmost stack frame. +func (thread *Thread) TopFrame() *Frame { return thread.frame } + +// A StringDict is a mapping from names to values, and represents +// an environment such as the global variables of a module. +// It is not a true starlark.Value. +type StringDict map[string]Value + +// Keys returns a new sorted slice of d's keys. +func (d StringDict) Keys() []string { + names := make([]string, 0, len(d)) + for name := range d { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func (d StringDict) String() string { + var buf bytes.Buffer + buf.WriteByte('{') + sep := "" + for _, name := range d.Keys() { + buf.WriteString(sep) + buf.WriteString(name) + buf.WriteString(": ") + writeValue(&buf, d[name], nil) + sep = ", " + } + buf.WriteByte('}') + return buf.String() +} + +func (d StringDict) Freeze() { + for _, v := range d { + v.Freeze() + } +} + +// Has reports whether the dictionary contains the specified key. +func (d StringDict) Has(key string) bool { _, ok := d[key]; return ok } + +// A Frame records a call to a Starlark function (including module toplevel) +// or a built-in function or method. +type Frame struct { + parent *Frame // caller's frame (or nil) + callable Callable // current function (or toplevel) or built-in + posn syntax.Position // source position of PC, set during error + callpc uint32 // PC of position of active call, set during call + locals []Value // local variables, for debugger +} + +// NewFrame returns a new frame with the specified parent and callable. +// +// It may be used to synthesize stack frames for error messages. +// Few clients should need to use this function. +func NewFrame(parent *Frame, callable Callable) *Frame { + return &Frame{parent: parent, callable: callable} +} + +// The Frames of a thread are structured as a spaghetti stack, not a +// slice, so that an EvalError can copy a stack efficiently and immutably. +// In hindsight using a slice would have led to a more convenient API. + +func (fr *Frame) errorf(posn syntax.Position, format string, args ...interface{}) *EvalError { + fr.posn = posn + msg := fmt.Sprintf(format, args...) + return &EvalError{Msg: msg, Frame: fr} +} + +// Position returns the source position of the current point of execution in this frame. +func (fr *Frame) Position() syntax.Position { + if fr.posn.IsValid() { + return fr.posn // leaf frame only (the error) + } + switch c := fr.callable.(type) { + case *Function: + // Starlark function + return c.funcode.Position(fr.callpc) // position of active call + case interface{ Position() syntax.Position }: + // If a built-in Callable defines + // a Position method, use it. + return c.Position() + } + return syntax.MakePosition(&builtinFilename, 1, 0) +} + +var builtinFilename = "" + +// Function returns the frame's function or built-in. +func (fr *Frame) Callable() Callable { return fr.callable } + +// Parent returns the frame of the enclosing function call, if any. +func (fr *Frame) Parent() *Frame { return fr.parent } + +// An EvalError is a Starlark evaluation error and its associated call stack. +type EvalError struct { + Msg string + Frame *Frame +} + +func (e *EvalError) Error() string { return e.Msg } + +// Backtrace returns a user-friendly error message describing the stack +// of calls that led to this error. +func (e *EvalError) Backtrace() string { + var buf bytes.Buffer + e.Frame.WriteBacktrace(&buf) + fmt.Fprintf(&buf, "Error: %s", e.Msg) + return buf.String() +} + +// WriteBacktrace writes a user-friendly description of the stack to buf. +func (fr *Frame) WriteBacktrace(out *bytes.Buffer) { + fmt.Fprintf(out, "Traceback (most recent call last):\n") + var print func(fr *Frame) + print = func(fr *Frame) { + if fr != nil { + print(fr.parent) + fmt.Fprintf(out, " %s: in %s\n", fr.Position(), fr.Callable().Name()) + } + } + print(fr) +} + +// Stack returns the stack of frames, innermost first. +func (e *EvalError) Stack() []*Frame { + var stack []*Frame + for fr := e.Frame; fr != nil; fr = fr.parent { + stack = append(stack, fr) + } + return stack +} + +// A Program is a compiled Starlark program. +// +// Programs are immutable, and contain no Values. +// A Program may be created by parsing a source file (see SourceProgram) +// or by loading a previously saved compiled program (see CompiledProgram). +type Program struct { + compiled *compile.Program +} + +// CompilerVersion is the version number of the protocol for compiled +// files. Applications must not run programs compiled by one version +// with an interpreter at another version, and should thus incorporate +// the compiler version into the cache key when reusing compiled code. +const CompilerVersion = compile.Version + +// Filename returns the name of the file from which this program was loaded. +func (prog *Program) Filename() string { return prog.compiled.Toplevel.Pos.Filename() } + +func (prog *Program) String() string { return prog.Filename() } + +// NumLoads returns the number of load statements in the compiled program. +func (prog *Program) NumLoads() int { return len(prog.compiled.Loads) } + +// Load(i) returns the name and position of the i'th module directly +// loaded by this one, where 0 <= i < NumLoads(). +// The name is unresolved---exactly as it appears in the source. +func (prog *Program) Load(i int) (string, syntax.Position) { + id := prog.compiled.Loads[i] + return id.Name, id.Pos +} + +// WriteTo writes the compiled module to the specified output stream. +func (prog *Program) Write(out io.Writer) error { + data := prog.compiled.Encode() + _, err := out.Write(data) + return err +} + +// ExecFile parses, resolves, and executes a Starlark file in the +// specified global environment, which may be modified during execution. +// +// Thread is the state associated with the Starlark thread. +// +// The filename and src parameters are as for syntax.Parse: +// filename is the name of the file to execute, +// and the name that appears in error messages; +// src is an optional source of bytes to use +// instead of filename. +// +// predeclared defines the predeclared names specific to this module. +// Execution does not modify this dictionary, though it may mutate +// its values. +// +// If ExecFile fails during evaluation, it returns an *EvalError +// containing a backtrace. +func ExecFile(thread *Thread, filename string, src interface{}, predeclared StringDict) (StringDict, error) { + // Parse, resolve, and compile a Starlark source file. + _, mod, err := SourceProgram(filename, src, predeclared.Has) + if err != nil { + return nil, err + } + + g, err := mod.Init(thread, predeclared) + g.Freeze() + return g, err +} + +// SourceProgram produces a new program by parsing, resolving, +// and compiling a Starlark source file. +// On success, it returns the parsed file and the compiled program. +// The filename and src parameters are as for syntax.Parse. +// +// The isPredeclared predicate reports whether a name is +// a pre-declared identifier of the current module. +// Its typical value is predeclared.Has, +// where predeclared is a StringDict of pre-declared values. +func SourceProgram(filename string, src interface{}, isPredeclared func(string) bool) (*syntax.File, *Program, error) { + f, err := syntax.Parse(filename, src, 0) + if err != nil { + return nil, nil, err + } + prog, err := FileProgram(f, isPredeclared) + return f, prog, err +} + +// FileProgram produces a new program by resolving, +// and compiling the Starlark source file syntax tree. +// On success, it returns the compiled program. +// +// Resolving a syntax tree mutates it. +// Do not call FileProgram more than once on the same file. +// +// The isPredeclared predicate reports whether a name is +// a pre-declared identifier of the current module. +// Its typical value is predeclared.Has, +// where predeclared is a StringDict of pre-declared values. +func FileProgram(f *syntax.File, isPredeclared func(string) bool) (*Program, error) { + if err := resolve.File(f, isPredeclared, Universe.Has); err != nil { + return nil, err + } + + var pos syntax.Position + if len(f.Stmts) > 0 { + pos = syntax.Start(f.Stmts[0]) + } else { + pos = syntax.MakePosition(&f.Path, 1, 1) + } + + compiled := compile.File(f.Stmts, pos, "", f.Locals, f.Globals) + + return &Program{compiled}, nil +} + +// CompiledProgram produces a new program from the representation +// of a compiled program previously saved by Program.Write. +func CompiledProgram(in io.Reader) (*Program, error) { + data, err := ioutil.ReadAll(in) + if err != nil { + return nil, err + } + compiled, err := compile.DecodeProgram(data) + if err != nil { + return nil, err + } + return &Program{compiled}, nil +} + +// Init creates a set of global variables for the program, +// executes the toplevel code of the specified program, +// and returns a new, unfrozen dictionary of the globals. +func (prog *Program) Init(thread *Thread, predeclared StringDict) (StringDict, error) { + toplevel := makeToplevelFunction(prog.compiled.Toplevel, predeclared) + + _, err := Call(thread, toplevel, nil, nil) + + // Convert the global environment to a map. + // We return a (partial) map even in case of error. + return toplevel.Globals(), err +} + +func makeToplevelFunction(funcode *compile.Funcode, predeclared StringDict) *Function { + // Create the Starlark value denoted by each program constant c. + constants := make([]Value, len(funcode.Prog.Constants)) + for i, c := range funcode.Prog.Constants { + var v Value + switch c := c.(type) { + case int64: + v = MakeInt64(c) + case *big.Int: + v = MakeBigInt(c) + case string: + v = String(c) + case float64: + v = Float(c) + default: + log.Fatalf("unexpected constant %T: %v", c, c) + } + constants[i] = v + } + + return &Function{ + funcode: funcode, + predeclared: predeclared, + globals: make([]Value, len(funcode.Prog.Globals)), + constants: constants, + } +} + +// Eval parses, resolves, and evaluates an expression within the +// specified (predeclared) environment. +// +// Evaluation cannot mutate the environment dictionary itself, +// though it may modify variables reachable from the dictionary. +// +// The filename and src parameters are as for syntax.Parse. +// +// If Eval fails during evaluation, it returns an *EvalError +// containing a backtrace. +func Eval(thread *Thread, filename string, src interface{}, env StringDict) (Value, error) { + expr, err := syntax.ParseExpr(filename, src, 0) + if err != nil { + return nil, err + } + f, err := makeExprFunc(expr, env) + if err != nil { + return nil, err + } + return Call(thread, f, nil, nil) +} + +// EvalExpr resolves and evaluates an expression within the +// specified (predeclared) environment. +// +// Resolving an expression mutates it. +// Do not call EvalExpr more than once for the same expression. +// +// Evaluation cannot mutate the environment dictionary itself, +// though it may modify variables reachable from the dictionary. +// +// If Eval fails during evaluation, it returns an *EvalError +// containing a backtrace. +func EvalExpr(thread *Thread, expr syntax.Expr, env StringDict) (Value, error) { + fn, err := makeExprFunc(expr, env) + if err != nil { + return nil, err + } + return Call(thread, fn, nil, nil) +} + +// ExprFunc returns a no-argument function +// that evaluates the expression whose source is src. +func ExprFunc(filename string, src interface{}, env StringDict) (*Function, error) { + expr, err := syntax.ParseExpr(filename, src, 0) + if err != nil { + return nil, err + } + return makeExprFunc(expr, env) +} + +// makeExprFunc returns a no-argument function whose body is expr. +func makeExprFunc(expr syntax.Expr, env StringDict) (*Function, error) { + locals, err := resolve.Expr(expr, env.Has, Universe.Has) + if err != nil { + return nil, err + } + + return makeToplevelFunction(compile.Expr(expr, "", locals), env), nil +} + +// The following functions are primitive operations of the byte code interpreter. + +// list += iterable +func listExtend(x *List, y Iterable) { + if ylist, ok := y.(*List); ok { + // fast path: list += list + x.elems = append(x.elems, ylist.elems...) + } else { + iter := y.Iterate() + defer iter.Done() + var z Value + for iter.Next(&z) { + x.elems = append(x.elems, z) + } + } +} + +// getAttr implements x.dot. +func getAttr(x Value, name string) (Value, error) { + hasAttr, ok := x.(HasAttrs) + if !ok { + return nil, fmt.Errorf("%s has no .%s field or method", x.Type(), name) + } + + var errmsg string + v, err := hasAttr.Attr(name) + if err == nil { + if v != nil { + return v, nil // success + } + // (nil, nil) => generic error + errmsg = fmt.Sprintf("%s has no .%s field or method", x.Type(), name) + } else if nsa, ok := err.(NoSuchAttrError); ok { + errmsg = string(nsa) + } else { + return nil, err // return error as is + } + + // add spelling hint + if n := spell.Nearest(name, hasAttr.AttrNames()); n != "" { + errmsg = fmt.Sprintf("%s (did you mean .%s?)", errmsg, n) + } + + return nil, fmt.Errorf("%s", errmsg) +} + +// setField implements x.name = y. +func setField(x Value, name string, y Value) error { + if x, ok := x.(HasSetField); ok { + err := x.SetField(name, y) + if _, ok := err.(NoSuchAttrError); ok { + // No such field: check spelling. + if n := spell.Nearest(name, x.AttrNames()); n != "" { + err = fmt.Errorf("%s (did you mean .%s?)", err, n) + } + } + return err + } + + return fmt.Errorf("can't assign to .%s field of %s", name, x.Type()) +} + +// getIndex implements x[y]. +func getIndex(x, y Value) (Value, error) { + switch x := x.(type) { + case Mapping: // dict + z, found, err := x.Get(y) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("key %v not in %s", y, x.Type()) + } + return z, nil + + case Indexable: // string, list, tuple + n := x.Len() + i, err := AsInt32(y) + if err != nil { + return nil, fmt.Errorf("%s index: %s", x.Type(), err) + } + origI := i + if i < 0 { + i += n + } + if i < 0 || i >= n { + return nil, outOfRange(origI, n, x) + } + return x.Index(i), nil + } + return nil, fmt.Errorf("unhandled index operation %s[%s]", x.Type(), y.Type()) +} + +func outOfRange(i, n int, x Value) error { + if n == 0 { + return fmt.Errorf("index %d out of range: empty %s", i, x.Type()) + } else { + return fmt.Errorf("%s index %d out of range [%d:%d]", x.Type(), i, -n, n-1) + } +} + +// setIndex implements x[y] = z. +func setIndex(x, y, z Value) error { + switch x := x.(type) { + case HasSetKey: + if err := x.SetKey(y, z); err != nil { + return err + } + + case HasSetIndex: + n := x.Len() + i, err := AsInt32(y) + if err != nil { + return err + } + origI := i + if i < 0 { + i += n + } + if i < 0 || i >= n { + return outOfRange(origI, n, x) + } + return x.SetIndex(i, z) + + default: + return fmt.Errorf("%s value does not support item assignment", x.Type()) + } + return nil +} + +// Unary applies a unary operator (+, -, ~, not) to its operand. +func Unary(op syntax.Token, x Value) (Value, error) { + // The NOT operator is not customizable. + if op == syntax.NOT { + return !x.Truth(), nil + } + + // Int, Float, and user-defined types + if x, ok := x.(HasUnary); ok { + // (nil, nil) => unhandled + y, err := x.Unary(op) + if y != nil || err != nil { + return y, err + } + } + + return nil, fmt.Errorf("unknown unary op: %s %s", op, x.Type()) +} + +// Binary applies a strict binary operator (not AND or OR) to its operands. +// For equality tests or ordered comparisons, use Compare instead. +func Binary(op syntax.Token, x, y Value) (Value, error) { + switch op { + case syntax.PLUS: + switch x := x.(type) { + case String: + if y, ok := y.(String); ok { + return x + y, nil + } + case Int: + switch y := y.(type) { + case Int: + return x.Add(y), nil + case Float: + return x.Float() + y, nil + } + case Float: + switch y := y.(type) { + case Float: + return x + y, nil + case Int: + return x + y.Float(), nil + } + case *List: + if y, ok := y.(*List); ok { + z := make([]Value, 0, x.Len()+y.Len()) + z = append(z, x.elems...) + z = append(z, y.elems...) + return NewList(z), nil + } + case Tuple: + if y, ok := y.(Tuple); ok { + z := make(Tuple, 0, len(x)+len(y)) + z = append(z, x...) + z = append(z, y...) + return z, nil + } + } + + case syntax.MINUS: + switch x := x.(type) { + case Int: + switch y := y.(type) { + case Int: + return x.Sub(y), nil + case Float: + return x.Float() - y, nil + } + case Float: + switch y := y.(type) { + case Float: + return x - y, nil + case Int: + return x - y.Float(), nil + } + } + + case syntax.STAR: + switch x := x.(type) { + case Int: + switch y := y.(type) { + case Int: + return x.Mul(y), nil + case Float: + return x.Float() * y, nil + case String: + return stringRepeat(y, x) + case *List: + elems, err := tupleRepeat(Tuple(y.elems), x) + if err != nil { + return nil, err + } + return NewList(elems), nil + case Tuple: + return tupleRepeat(y, x) + } + case Float: + switch y := y.(type) { + case Float: + return x * y, nil + case Int: + return x * y.Float(), nil + } + case String: + if y, ok := y.(Int); ok { + return stringRepeat(x, y) + } + case *List: + if y, ok := y.(Int); ok { + elems, err := tupleRepeat(Tuple(x.elems), y) + if err != nil { + return nil, err + } + return NewList(elems), nil + } + case Tuple: + if y, ok := y.(Int); ok { + return tupleRepeat(x, y) + } + + } + + case syntax.SLASH: + switch x := x.(type) { + case Int: + switch y := y.(type) { + case Int: + yf := y.Float() + if yf == 0.0 { + return nil, fmt.Errorf("real division by zero") + } + return x.Float() / yf, nil + case Float: + if y == 0.0 { + return nil, fmt.Errorf("real division by zero") + } + return x.Float() / y, nil + } + case Float: + switch y := y.(type) { + case Float: + if y == 0.0 { + return nil, fmt.Errorf("real division by zero") + } + return x / y, nil + case Int: + yf := y.Float() + if yf == 0.0 { + return nil, fmt.Errorf("real division by zero") + } + return x / yf, nil + } + } + + case syntax.SLASHSLASH: + switch x := x.(type) { + case Int: + switch y := y.(type) { + case Int: + if y.Sign() == 0 { + return nil, fmt.Errorf("floored division by zero") + } + return x.Div(y), nil + case Float: + if y == 0.0 { + return nil, fmt.Errorf("floored division by zero") + } + return floor((x.Float() / y)), nil + } + case Float: + switch y := y.(type) { + case Float: + if y == 0.0 { + return nil, fmt.Errorf("floored division by zero") + } + return floor(x / y), nil + case Int: + yf := y.Float() + if yf == 0.0 { + return nil, fmt.Errorf("floored division by zero") + } + return floor(x / yf), nil + } + } + + case syntax.PERCENT: + switch x := x.(type) { + case Int: + switch y := y.(type) { + case Int: + if y.Sign() == 0 { + return nil, fmt.Errorf("integer modulo by zero") + } + return x.Mod(y), nil + case Float: + if y == 0 { + return nil, fmt.Errorf("float modulo by zero") + } + return x.Float().Mod(y), nil + } + case Float: + switch y := y.(type) { + case Float: + if y == 0.0 { + return nil, fmt.Errorf("float modulo by zero") + } + return Float(math.Mod(float64(x), float64(y))), nil + case Int: + if y.Sign() == 0 { + return nil, fmt.Errorf("float modulo by zero") + } + return x.Mod(y.Float()), nil + } + case String: + return interpolate(string(x), y) + } + + case syntax.NOT_IN: + z, err := Binary(syntax.IN, x, y) + if err != nil { + return nil, err + } + return !z.Truth(), nil + + case syntax.IN: + switch y := y.(type) { + case *List: + for _, elem := range y.elems { + if eq, err := Equal(elem, x); err != nil { + return nil, err + } else if eq { + return True, nil + } + } + return False, nil + case Tuple: + for _, elem := range y { + if eq, err := Equal(elem, x); err != nil { + return nil, err + } else if eq { + return True, nil + } + } + return False, nil + case Mapping: // e.g. dict + // Ignore error from Get as we cannot distinguish true + // errors (value cycle, type error) from "key not found". + _, found, _ := y.Get(x) + return Bool(found), nil + case *Set: + ok, err := y.Has(x) + return Bool(ok), err + case String: + needle, ok := x.(String) + if !ok { + return nil, fmt.Errorf("'in ' requires string as left operand, not %s", x.Type()) + } + return Bool(strings.Contains(string(y), string(needle))), nil + case rangeValue: + i, err := NumberToInt(x) + if err != nil { + return nil, fmt.Errorf("'in ' requires integer as left operand, not %s", x.Type()) + } + return Bool(y.contains(i)), nil + } + + case syntax.PIPE: + switch x := x.(type) { + case Int: + if y, ok := y.(Int); ok { + return x.Or(y), nil + } + case *Set: // union + if y, ok := y.(*Set); ok { + iter := Iterate(y) + defer iter.Done() + return x.Union(iter) + } + } + + case syntax.AMP: + switch x := x.(type) { + case Int: + if y, ok := y.(Int); ok { + return x.And(y), nil + } + case *Set: // intersection + if y, ok := y.(*Set); ok { + set := new(Set) + if x.Len() > y.Len() { + x, y = y, x // opt: range over smaller set + } + for _, xelem := range x.elems() { + // Has, Insert cannot fail here. + if found, _ := y.Has(xelem); found { + set.Insert(xelem) + } + } + return set, nil + } + } + + case syntax.CIRCUMFLEX: + switch x := x.(type) { + case Int: + if y, ok := y.(Int); ok { + return x.Xor(y), nil + } + case *Set: // symmetric difference + if y, ok := y.(*Set); ok { + set := new(Set) + for _, xelem := range x.elems() { + if found, _ := y.Has(xelem); !found { + set.Insert(xelem) + } + } + for _, yelem := range y.elems() { + if found, _ := x.Has(yelem); !found { + set.Insert(yelem) + } + } + return set, nil + } + } + + case syntax.LTLT, syntax.GTGT: + if x, ok := x.(Int); ok { + y, err := AsInt32(y) + if err != nil { + return nil, err + } + if y < 0 { + return nil, fmt.Errorf("negative shift count: %v", y) + } + if op == syntax.LTLT { + if y >= 512 { + return nil, fmt.Errorf("shift count too large: %v", y) + } + return x.Lsh(uint(y)), nil + } else { + return x.Rsh(uint(y)), nil + } + } + + default: + // unknown operator + goto unknown + } + + // user-defined types + // (nil, nil) => unhandled + if x, ok := x.(HasBinary); ok { + z, err := x.Binary(op, y, Left) + if z != nil || err != nil { + return z, err + } + } + if y, ok := y.(HasBinary); ok { + z, err := y.Binary(op, x, Right) + if z != nil || err != nil { + return z, err + } + } + + // unsupported operand types +unknown: + return nil, fmt.Errorf("unknown binary op: %s %s %s", x.Type(), op, y.Type()) +} + +// It's always possible to overeat in small bites but we'll +// try to stop someone swallowing the world in one gulp. +const maxAlloc = 1 << 30 + +func tupleRepeat(elems Tuple, n Int) (Tuple, error) { + if len(elems) == 0 { + return nil, nil + } + i, err := AsInt32(n) + if err != nil { + return nil, fmt.Errorf("repeat count %s too large", n) + } + if i < 1 { + return nil, nil + } + // Inv: i > 0, len > 0 + sz := len(elems) * i + if sz < 0 || sz >= maxAlloc { // sz < 0 => overflow + return nil, fmt.Errorf("excessive repeat (%d elements)", sz) + } + res := make([]Value, sz) + // copy elems into res, doubling each time + x := copy(res, elems) + for x < len(res) { + copy(res[x:], res[:x]) + x *= 2 + } + return res, nil +} + +func stringRepeat(s String, n Int) (String, error) { + if s == "" { + return "", nil + } + i, err := AsInt32(n) + if err != nil { + return "", fmt.Errorf("repeat count %s too large", n) + } + if i < 1 { + return "", nil + } + // Inv: i > 0, len > 0 + sz := len(s) * i + if sz < 0 || sz >= maxAlloc { // sz < 0 => overflow + return "", fmt.Errorf("excessive repeat (%d elements)", sz) + } + return String(strings.Repeat(string(s), i)), nil +} + +// Call calls the function fn with the specified positional and keyword arguments. +func Call(thread *Thread, fn Value, args Tuple, kwargs []Tuple) (Value, error) { + c, ok := fn.(Callable) + if !ok { + return nil, fmt.Errorf("invalid call of non-function (%s)", fn.Type()) + } + + thread.frame = NewFrame(thread.frame, c) + result, err := c.CallInternal(thread, args, kwargs) + thread.frame = thread.frame.parent + + // Sanity check: nil is not a valid Starlark value. + if result == nil && err == nil { + return nil, fmt.Errorf("internal error: nil (not None) returned from %s", fn) + } + + return result, err +} + +func slice(x, lo, hi, step_ Value) (Value, error) { + sliceable, ok := x.(Sliceable) + if !ok { + return nil, fmt.Errorf("invalid slice operand %s", x.Type()) + } + + n := sliceable.Len() + step := 1 + if step_ != None { + var err error + step, err = AsInt32(step_) + if err != nil { + return nil, fmt.Errorf("got %s for slice step, want int", step_.Type()) + } + if step == 0 { + return nil, fmt.Errorf("zero is not a valid slice step") + } + } + + // TODO(adonovan): opt: preallocate result array. + + var start, end int + if step > 0 { + // positive stride + // default indices are [0:n]. + var err error + start, end, err = indices(lo, hi, n) + if err != nil { + return nil, err + } + + if end < start { + end = start // => empty result + } + } else { + // negative stride + // default indices are effectively [n-1:-1], though to + // get this effect using explicit indices requires + // [n-1:-1-n:-1] because of the treatment of -ve values. + start = n - 1 + if err := asIndex(lo, n, &start); err != nil { + return nil, fmt.Errorf("invalid start index: %s", err) + } + if start >= n { + start = n - 1 + } + + end = -1 + if err := asIndex(hi, n, &end); err != nil { + return nil, fmt.Errorf("invalid end index: %s", err) + } + if end < -1 { + end = -1 + } + + if start < end { + start = end // => empty result + } + } + + return sliceable.Slice(start, end, step), nil +} + +// From Hacker's Delight, section 2.8. +func signum64(x int64) int { return int(uint64(x>>63) | uint64(-x)>>63) } +func signum(x int) int { return signum64(int64(x)) } + +// indices converts start_ and end_ to indices in the range [0:len]. +// The start index defaults to 0 and the end index defaults to len. +// An index -len < i < 0 is treated like i+len. +// All other indices outside the range are clamped to the nearest value in the range. +// Beware: start may be greater than end. +// This function is suitable only for slices with positive strides. +func indices(start_, end_ Value, len int) (start, end int, err error) { + start = 0 + if err := asIndex(start_, len, &start); err != nil { + return 0, 0, fmt.Errorf("invalid start index: %s", err) + } + // Clamp to [0:len]. + if start < 0 { + start = 0 + } else if start > len { + start = len + } + + end = len + if err := asIndex(end_, len, &end); err != nil { + return 0, 0, fmt.Errorf("invalid end index: %s", err) + } + // Clamp to [0:len]. + if end < 0 { + end = 0 + } else if end > len { + end = len + } + + return start, end, nil +} + +// asIndex sets *result to the integer value of v, adding len to it +// if it is negative. If v is nil or None, *result is unchanged. +func asIndex(v Value, len int, result *int) error { + if v != nil && v != None { + var err error + *result, err = AsInt32(v) + if err != nil { + return fmt.Errorf("got %s, want int", v.Type()) + } + if *result < 0 { + *result += len + } + } + return nil +} + +// setArgs sets the values of the formal parameters of function fn in +// based on the actual parameter values in args and kwargs. +func setArgs(locals []Value, fn *Function, args Tuple, kwargs []Tuple) error { + + // This is the general schema of a function: + // + // def f(p1, p2=dp2, p3=dp3, *args, k1, k2=dk2, k3, **kwargs) + // + // The p parameters are non-kwonly, and may be specified positionally. + // The k parameters are kwonly, and must be specified by name. + // The defaults tuple is (dp2, dp3, mandatory, dk2, mandatory). + // + // Arguments are processed as follows: + // - positional arguments are bound to a prefix of [p1, p2, p3]. + // - surplus positional arguments are bound to *args. + // - keyword arguments are bound to any of {p1, p2, p3, k1, k2, k3}; + // duplicate bindings are rejected. + // - surplus keyword arguments are bound to **kwargs. + // - defaults are bound to each parameter from p2 to k3 if no value was set. + // default values come from the tuple above. + // It is an error if the tuple entry for an unset parameter is 'mandatory'. + + // Nullary function? + if fn.NumParams() == 0 { + if nactual := len(args) + len(kwargs); nactual > 0 { + return fmt.Errorf("function %s accepts no arguments (%d given)", fn.Name(), nactual) + } + return nil + } + + cond := func(x bool, y, z interface{}) interface{} { + if x { + return y + } + return z + } + + // nparams is the number of ordinary parameters (sans *args and **kwargs). + nparams := fn.NumParams() + var kwdict *Dict + if fn.HasKwargs() { + nparams-- + kwdict = new(Dict) + locals[nparams] = kwdict + } + if fn.HasVarargs() { + nparams-- + } + + // nonkwonly is the number of non-kwonly parameters. + nonkwonly := nparams - fn.NumKwonlyParams() + + // Too many positional args? + n := len(args) + if len(args) > nonkwonly { + if !fn.HasVarargs() { + return fmt.Errorf("function %s accepts %s%d positional argument%s (%d given)", + fn.Name(), + cond(len(fn.defaults) > fn.NumKwonlyParams(), "at most ", ""), + nonkwonly, + cond(nonkwonly == 1, "", "s"), + len(args)) + } + n = nonkwonly + } + + // Bind positional arguments to non-kwonly parameters. + for i := 0; i < n; i++ { + locals[i] = args[i] + } + + // Bind surplus positional arguments to *args parameter. + if fn.HasVarargs() { + tuple := make(Tuple, len(args)-n) + for i := n; i < len(args); i++ { + tuple[i-n] = args[i] + } + locals[nparams] = tuple + } + + // Bind keyword arguments to parameters. + paramIdents := fn.funcode.Locals[:nparams] + for _, pair := range kwargs { + k, v := pair[0].(String), pair[1] + if i := findParam(paramIdents, string(k)); i >= 0 { + if locals[i] != nil { + return fmt.Errorf("function %s got multiple values for parameter %s", fn.Name(), k) + } + locals[i] = v + continue + } + if kwdict == nil { + return fmt.Errorf("function %s got an unexpected keyword argument %s", fn.Name(), k) + } + oldlen := kwdict.Len() + kwdict.SetKey(k, v) + if kwdict.Len() == oldlen { + return fmt.Errorf("function %s got multiple values for parameter %s", fn.Name(), k) + } + } + + // Are defaults required? + if n < nparams || fn.NumKwonlyParams() > 0 { + m := nparams - len(fn.defaults) // first default + + // Report errors for missing required arguments. + var missing []string + var i int + for i = n; i < m; i++ { + if locals[i] == nil { + missing = append(missing, paramIdents[i].Name) + } + } + + // Bind default values to parameters. + for ; i < nparams; i++ { + if locals[i] == nil { + dflt := fn.defaults[i-m] + if _, ok := dflt.(mandatory); ok { + missing = append(missing, paramIdents[i].Name) + continue + } + locals[i] = dflt + } + } + + if missing != nil { + return fmt.Errorf("function %s missing %d argument%s (%s)", + fn.Name(), len(missing), cond(len(missing) > 1, "s", ""), strings.Join(missing, ", ")) + } + } + return nil +} + +func findParam(params []compile.Ident, name string) int { + for i, param := range params { + if param.Name == name { + return i + } + } + return -1 +} + +type intset struct { + small uint64 // bitset, used if n < 64 + large map[int]bool // set, used if n >= 64 +} + +func (is *intset) init(n int) { + if n >= 64 { + is.large = make(map[int]bool) + } +} + +func (is *intset) set(i int) (prev bool) { + if is.large == nil { + prev = is.small&(1<= nargs { + return nil, fmt.Errorf("not enough arguments for format string") + } + if tuple, ok := x.(Tuple); ok { + arg = tuple[index] + } else { + arg = x + } + } + + // NOTE: Starlark does not support any of these optional Python features: + // - optional conversion flags: [#0- +], etc. + // - optional minimum field width (number or *). + // - optional precision (.123 or *) + // - optional length modifier + + // conversion type + if format == "" { + return nil, fmt.Errorf("incomplete format") + } + switch c := format[0]; c { + case 's', 'r': + if str, ok := AsString(arg); ok && c == 's' { + buf.WriteString(str) + } else { + writeValue(&buf, arg, nil) + } + case 'd', 'i', 'o', 'x', 'X': + i, err := NumberToInt(arg) + if err != nil { + return nil, fmt.Errorf("%%%c format requires integer: %v", c, err) + } + switch c { + case 'd', 'i': + fmt.Fprintf(&buf, "%d", i) + case 'o': + fmt.Fprintf(&buf, "%o", i) + case 'x': + fmt.Fprintf(&buf, "%x", i) + case 'X': + fmt.Fprintf(&buf, "%X", i) + } + case 'e', 'f', 'g', 'E', 'F', 'G': + f, ok := AsFloat(arg) + if !ok { + return nil, fmt.Errorf("%%%c format requires float, not %s", c, arg.Type()) + } + switch c { + case 'e': + fmt.Fprintf(&buf, "%e", f) + case 'f': + fmt.Fprintf(&buf, "%f", f) + case 'g': + fmt.Fprintf(&buf, "%g", f) + case 'E': + fmt.Fprintf(&buf, "%E", f) + case 'F': + fmt.Fprintf(&buf, "%F", f) + case 'G': + fmt.Fprintf(&buf, "%G", f) + } + case 'c': + switch arg := arg.(type) { + case Int: + // chr(int) + r, err := AsInt32(arg) + if err != nil || r < 0 || r > unicode.MaxRune { + return nil, fmt.Errorf("%%c format requires a valid Unicode code point, got %s", arg) + } + buf.WriteRune(rune(r)) + case String: + r, size := utf8.DecodeRuneInString(string(arg)) + if size != len(arg) || len(arg) == 0 { + return nil, fmt.Errorf("%%c format requires a single-character string") + } + buf.WriteRune(r) + default: + return nil, fmt.Errorf("%%c format requires int or single-character string, not %s", arg.Type()) + } + case '%': + buf.WriteByte('%') + default: + return nil, fmt.Errorf("unknown conversion %%%c", c) + } + format = format[1:] + index++ + } + + if index < nargs { + return nil, fmt.Errorf("too many arguments for format string") + } + + return String(buf.String()), nil +} diff --git a/vendor/go.starlark.net/starlark/hashtable.go b/vendor/go.starlark.net/starlark/hashtable.go new file mode 100644 index 00000000..f63f5c54 --- /dev/null +++ b/vendor/go.starlark.net/starlark/hashtable.go @@ -0,0 +1,358 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package starlark + +import ( + "fmt" + _ "unsafe" // for go:linkname hack +) + +// hashtable is used to represent Starlark dict and set values. +// It is a hash table whose key/value entries form a doubly-linked list +// in the order the entries were inserted. +type hashtable struct { + table []bucket // len is zero or a power of two + bucket0 [1]bucket // inline allocation for small maps. + len uint32 + itercount uint32 // number of active iterators (ignored if frozen) + head *entry // insertion order doubly-linked list; may be nil + tailLink **entry // address of nil link at end of list (perhaps &head) + frozen bool +} + +const bucketSize = 8 + +type bucket struct { + entries [bucketSize]entry + next *bucket // linked list of buckets +} + +type entry struct { + hash uint32 // nonzero => in use + key, value Value + next *entry // insertion order doubly-linked list; may be nil + prevLink **entry // address of link to this entry (perhaps &head) +} + +func (ht *hashtable) freeze() { + if !ht.frozen { + ht.frozen = true + for i := range ht.table { + for p := &ht.table[i]; p != nil; p = p.next { + for i := range p.entries { + e := &p.entries[i] + if e.hash != 0 { + e.key.Freeze() + e.value.Freeze() + } + } + } + } + } +} + +func (ht *hashtable) insert(k, v Value) error { + if ht.frozen { + return fmt.Errorf("cannot insert into frozen hash table") + } + if ht.itercount > 0 { + return fmt.Errorf("cannot insert into hash table during iteration") + } + if ht.table == nil { + ht.table = ht.bucket0[:1] + ht.tailLink = &ht.head + } + h, err := k.Hash() + if err != nil { + return err + } + if h == 0 { + h = 1 // zero is reserved + } + +retry: + var insert *entry + + // Inspect each bucket in the bucket list. + p := &ht.table[h&(uint32(len(ht.table)-1))] + for { + for i := range p.entries { + e := &p.entries[i] + if e.hash != h { + if e.hash == 0 { + // Found empty entry; make a note. + insert = e + } + continue + } + if eq, err := Equal(k, e.key); err != nil { + return err // e.g. excessively recursive tuple + } else if !eq { + continue + } + // Key already present; update value. + e.value = v + return nil + } + if p.next == nil { + break + } + p = p.next + } + + // Key not found. p points to the last bucket. + + // Does the number of elements exceed the buckets' load factor? + if overloaded(int(ht.len), len(ht.table)) { + ht.grow() + goto retry + } + + if insert == nil { + // No space in existing buckets. Add a new one to the bucket list. + b := new(bucket) + p.next = b + insert = &b.entries[0] + } + + // Insert key/value pair. + insert.hash = h + insert.key = k + insert.value = v + + // Append entry to doubly-linked list. + insert.prevLink = ht.tailLink + *ht.tailLink = insert + ht.tailLink = &insert.next + + ht.len++ + + return nil +} + +func overloaded(elems, buckets int) bool { + const loadFactor = 6.5 // just a guess + return elems >= bucketSize && float64(elems) >= loadFactor*float64(buckets) +} + +func (ht *hashtable) grow() { + // Double the number of buckets and rehash. + // TODO(adonovan): opt: + // - avoid reentrant calls to ht.insert, and specialize it. + // e.g. we know the calls to Equals will return false since + // there are no duplicates among the old keys. + // - saving the entire hash in the bucket would avoid the need to + // recompute the hash. + // - save the old buckets on a free list. + ht.table = make([]bucket, len(ht.table)<<1) + oldhead := ht.head + ht.head = nil + ht.tailLink = &ht.head + ht.len = 0 + for e := oldhead; e != nil; e = e.next { + ht.insert(e.key, e.value) + } + ht.bucket0[0] = bucket{} // clear out unused initial bucket +} + +func (ht *hashtable) lookup(k Value) (v Value, found bool, err error) { + h, err := k.Hash() + if err != nil { + return nil, false, err // unhashable + } + if h == 0 { + h = 1 // zero is reserved + } + if ht.table == nil { + return None, false, nil // empty + } + + // Inspect each bucket in the bucket list. + for p := &ht.table[h&(uint32(len(ht.table)-1))]; p != nil; p = p.next { + for i := range p.entries { + e := &p.entries[i] + if e.hash == h { + if eq, err := Equal(k, e.key); err != nil { + return nil, false, err // e.g. excessively recursive tuple + } else if eq { + return e.value, true, nil // found + } + } + } + } + return None, false, nil // not found +} + +// Items returns all the items in the map (as key/value pairs) in insertion order. +func (ht *hashtable) items() []Tuple { + items := make([]Tuple, 0, ht.len) + array := make([]Value, ht.len*2) // allocate a single backing array + for e := ht.head; e != nil; e = e.next { + pair := Tuple(array[:2:2]) + array = array[2:] + pair[0] = e.key + pair[1] = e.value + items = append(items, pair) + } + return items +} + +func (ht *hashtable) first() (Value, bool) { + if ht.head != nil { + return ht.head.key, true + } + return None, false +} + +func (ht *hashtable) keys() []Value { + keys := make([]Value, 0, ht.len) + for e := ht.head; e != nil; e = e.next { + keys = append(keys, e.key) + } + return keys +} + +func (ht *hashtable) delete(k Value) (v Value, found bool, err error) { + if ht.frozen { + return nil, false, fmt.Errorf("cannot delete from frozen hash table") + } + if ht.itercount > 0 { + return nil, false, fmt.Errorf("cannot delete from hash table during iteration") + } + if ht.table == nil { + return None, false, nil // empty + } + h, err := k.Hash() + if err != nil { + return nil, false, err // unhashable + } + if h == 0 { + h = 1 // zero is reserved + } + + // Inspect each bucket in the bucket list. + for p := &ht.table[h&(uint32(len(ht.table)-1))]; p != nil; p = p.next { + for i := range p.entries { + e := &p.entries[i] + if e.hash == h { + if eq, err := Equal(k, e.key); err != nil { + return nil, false, err + } else if eq { + // Remove e from doubly-linked list. + *e.prevLink = e.next + if e.next == nil { + ht.tailLink = e.prevLink // deletion of last entry + } else { + e.next.prevLink = e.prevLink + } + + v := e.value + *e = entry{} + ht.len-- + return v, true, nil // found + } + } + } + } + + // TODO(adonovan): opt: remove completely empty bucket from bucket list. + + return None, false, nil // not found +} + +func (ht *hashtable) clear() error { + if ht.frozen { + return fmt.Errorf("cannot clear frozen hash table") + } + if ht.itercount > 0 { + return fmt.Errorf("cannot clear hash table during iteration") + } + if ht.table != nil { + for i := range ht.table { + ht.table[i] = bucket{} + } + } + ht.head = nil + ht.tailLink = &ht.head + ht.len = 0 + return nil +} + +// dump is provided as an aid to debugging. +func (ht *hashtable) dump() { + fmt.Printf("hashtable %p len=%d head=%p tailLink=%p", + ht, ht.len, ht.head, ht.tailLink) + if ht.tailLink != nil { + fmt.Printf(" *tailLink=%p", *ht.tailLink) + } + fmt.Println() + for j := range ht.table { + fmt.Printf("bucket chain %d\n", j) + for p := &ht.table[j]; p != nil; p = p.next { + fmt.Printf("bucket %p\n", p) + for i := range p.entries { + e := &p.entries[i] + fmt.Printf("\tentry %d @ %p hash=%d key=%v value=%v\n", + i, e, e.hash, e.key, e.value) + fmt.Printf("\t\tnext=%p &next=%p prev=%p", + e.next, &e.next, e.prevLink) + if e.prevLink != nil { + fmt.Printf(" *prev=%p", *e.prevLink) + } + fmt.Println() + } + } + } +} + +func (ht *hashtable) iterate() *keyIterator { + if !ht.frozen { + ht.itercount++ + } + return &keyIterator{ht: ht, e: ht.head} +} + +type keyIterator struct { + ht *hashtable + e *entry +} + +func (it *keyIterator) Next(k *Value) bool { + if it.e != nil { + *k = it.e.key + it.e = it.e.next + return true + } + return false +} + +func (it *keyIterator) Done() { + if !it.ht.frozen { + it.ht.itercount-- + } +} + +// hashString computes the hash of s. +func hashString(s string) uint32 { + if len(s) >= 12 { + // Call the Go runtime's optimized hash implementation, + // which uses the AESENC instruction on amd64 machines. + return uint32(goStringHash(s, 0)) + } + return softHashString(s) +} + +//go:linkname goStringHash runtime.stringHash +func goStringHash(s string, seed uintptr) uintptr + +// softHashString computes the FNV hash of s in software. +func softHashString(s string) uint32 { + var h uint32 + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= 16777619 + } + return h +} diff --git a/vendor/go.starlark.net/starlark/int.go b/vendor/go.starlark.net/starlark/int.go new file mode 100644 index 00000000..35bd42b3 --- /dev/null +++ b/vendor/go.starlark.net/starlark/int.go @@ -0,0 +1,350 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package starlark + +import ( + "fmt" + "math" + "math/big" + "strconv" + + "go.starlark.net/syntax" +) + +// Int is the type of a Starlark int. +type Int struct { + // We use only the signed 32 bit range of small to ensure + // that small+small and small*small do not overflow. + + small int64 // minint32 <= small <= maxint32 + big *big.Int // big != nil <=> value is not representable as int32 +} + +// newBig allocates a new big.Int. +func newBig(x int64) *big.Int { + if 0 <= x && int64(big.Word(x)) == x { + // x is guaranteed to fit into a single big.Word. + // Most starlark ints are small, + // but math/big assumes that since you've chosen to use math/big, + // your big.Ints will probably grow, so it over-allocates. + // Avoid that over-allocation by manually constructing a single-word slice. + // See https://golang.org/cl/150999, which will hopefully land in Go 1.13. + return new(big.Int).SetBits([]big.Word{big.Word(x)}) + } + return big.NewInt(x) +} + +// MakeInt returns a Starlark int for the specified signed integer. +func MakeInt(x int) Int { return MakeInt64(int64(x)) } + +// MakeInt64 returns a Starlark int for the specified int64. +func MakeInt64(x int64) Int { + if math.MinInt32 <= x && x <= math.MaxInt32 { + return Int{small: x} + } + return Int{big: newBig(x)} +} + +// MakeUint returns a Starlark int for the specified unsigned integer. +func MakeUint(x uint) Int { return MakeUint64(uint64(x)) } + +// MakeUint64 returns a Starlark int for the specified uint64. +func MakeUint64(x uint64) Int { + if x <= math.MaxInt32 { + return Int{small: int64(x)} + } + if uint64(big.Word(x)) == x { + // See comment in newBig for an explanation of this optimization. + return Int{big: new(big.Int).SetBits([]big.Word{big.Word(x)})} + } + return Int{big: new(big.Int).SetUint64(x)} +} + +// MakeBigInt returns a Starlark int for the specified big.Int. +// The caller must not subsequently modify x. +func MakeBigInt(x *big.Int) Int { + if n := x.BitLen(); n < 32 || n == 32 && x.Int64() == math.MinInt32 { + return Int{small: x.Int64()} + } + return Int{big: x} +} + +var ( + zero, one = Int{small: 0}, Int{small: 1} + oneBig = newBig(1) + + _ HasUnary = Int{} +) + +// Unary implements the operations +int, -int, and ~int. +func (i Int) Unary(op syntax.Token) (Value, error) { + switch op { + case syntax.MINUS: + return zero.Sub(i), nil + case syntax.PLUS: + return i, nil + case syntax.TILDE: + return i.Not(), nil + } + return nil, nil +} + +// Int64 returns the value as an int64. +// If it is not exactly representable the result is undefined and ok is false. +func (i Int) Int64() (_ int64, ok bool) { + if i.big != nil { + x, acc := bigintToInt64(i.big) + if acc != big.Exact { + return // inexact + } + return x, true + } + return i.small, true +} + +// BigInt returns the value as a big.Int. +// The returned variable must not be modified by the client. +func (i Int) BigInt() *big.Int { + if i.big != nil { + return i.big + } + return newBig(i.small) +} + +// Uint64 returns the value as a uint64. +// If it is not exactly representable the result is undefined and ok is false. +func (i Int) Uint64() (_ uint64, ok bool) { + if i.big != nil { + x, acc := bigintToUint64(i.big) + if acc != big.Exact { + return // inexact + } + return x, true + } + if i.small < 0 { + return // inexact + } + return uint64(i.small), true +} + +// The math/big API should provide this function. +func bigintToInt64(i *big.Int) (int64, big.Accuracy) { + sign := i.Sign() + if sign > 0 { + if i.Cmp(maxint64) > 0 { + return math.MaxInt64, big.Below + } + } else if sign < 0 { + if i.Cmp(minint64) < 0 { + return math.MinInt64, big.Above + } + } + return i.Int64(), big.Exact +} + +// The math/big API should provide this function. +func bigintToUint64(i *big.Int) (uint64, big.Accuracy) { + sign := i.Sign() + if sign > 0 { + if i.BitLen() > 64 { + return math.MaxUint64, big.Below + } + } else if sign < 0 { + return 0, big.Above + } + return i.Uint64(), big.Exact +} + +var ( + minint64 = new(big.Int).SetInt64(math.MinInt64) + maxint64 = new(big.Int).SetInt64(math.MaxInt64) +) + +func (i Int) Format(s fmt.State, ch rune) { + if i.big != nil { + i.big.Format(s, ch) + return + } + newBig(i.small).Format(s, ch) +} +func (i Int) String() string { + if i.big != nil { + return i.big.Text(10) + } + return strconv.FormatInt(i.small, 10) +} +func (i Int) Type() string { return "int" } +func (i Int) Freeze() {} // immutable +func (i Int) Truth() Bool { return i.Sign() != 0 } +func (i Int) Hash() (uint32, error) { + var lo big.Word + if i.big != nil { + lo = i.big.Bits()[0] + } else { + lo = big.Word(i.small) + } + return 12582917 * uint32(lo+3), nil +} +func (x Int) CompareSameType(op syntax.Token, v Value, depth int) (bool, error) { + y := v.(Int) + if x.big != nil || y.big != nil { + return threeway(op, x.BigInt().Cmp(y.BigInt())), nil + } + return threeway(op, signum64(x.small-y.small)), nil +} + +// Float returns the float value nearest i. +func (i Int) Float() Float { + if i.big != nil { + f, _ := new(big.Float).SetInt(i.big).Float64() + return Float(f) + } + return Float(i.small) +} + +func (x Int) Sign() int { + if x.big != nil { + return x.big.Sign() + } + return signum64(x.small) +} + +func (x Int) Add(y Int) Int { + if x.big != nil || y.big != nil { + return MakeBigInt(new(big.Int).Add(x.BigInt(), y.BigInt())) + } + return MakeInt64(x.small + y.small) +} +func (x Int) Sub(y Int) Int { + if x.big != nil || y.big != nil { + return MakeBigInt(new(big.Int).Sub(x.BigInt(), y.BigInt())) + } + return MakeInt64(x.small - y.small) +} +func (x Int) Mul(y Int) Int { + if x.big != nil || y.big != nil { + return MakeBigInt(new(big.Int).Mul(x.BigInt(), y.BigInt())) + } + return MakeInt64(x.small * y.small) +} +func (x Int) Or(y Int) Int { + if x.big != nil || y.big != nil { + return Int{big: new(big.Int).Or(x.BigInt(), y.BigInt())} + } + return Int{small: x.small | y.small} +} +func (x Int) And(y Int) Int { + if x.big != nil || y.big != nil { + return MakeBigInt(new(big.Int).And(x.BigInt(), y.BigInt())) + } + return Int{small: x.small & y.small} +} +func (x Int) Xor(y Int) Int { + if x.big != nil || y.big != nil { + return MakeBigInt(new(big.Int).Xor(x.BigInt(), y.BigInt())) + } + return Int{small: x.small ^ y.small} +} +func (x Int) Not() Int { + if x.big != nil { + return MakeBigInt(new(big.Int).Not(x.big)) + } + return Int{small: ^x.small} +} +func (x Int) Lsh(y uint) Int { return MakeBigInt(new(big.Int).Lsh(x.BigInt(), y)) } +func (x Int) Rsh(y uint) Int { return MakeBigInt(new(big.Int).Rsh(x.BigInt(), y)) } + +// Precondition: y is nonzero. +func (x Int) Div(y Int) Int { + // http://python-history.blogspot.com/2010/08/why-pythons-integer-division-floors.html + if x.big != nil || y.big != nil { + xb, yb := x.BigInt(), y.BigInt() + + var quo, rem big.Int + quo.QuoRem(xb, yb, &rem) + if (xb.Sign() < 0) != (yb.Sign() < 0) && rem.Sign() != 0 { + quo.Sub(&quo, oneBig) + } + return MakeBigInt(&quo) + } + quo := x.small / y.small + rem := x.small % y.small + if (x.small < 0) != (y.small < 0) && rem != 0 { + quo -= 1 + } + return MakeInt64(quo) +} + +// Precondition: y is nonzero. +func (x Int) Mod(y Int) Int { + if x.big != nil || y.big != nil { + xb, yb := x.BigInt(), y.BigInt() + + var quo, rem big.Int + quo.QuoRem(xb, yb, &rem) + if (xb.Sign() < 0) != (yb.Sign() < 0) && rem.Sign() != 0 { + rem.Add(&rem, yb) + } + return MakeBigInt(&rem) + } + rem := x.small % y.small + if (x.small < 0) != (y.small < 0) && rem != 0 { + rem += y.small + } + return Int{small: rem} +} + +func (i Int) rational() *big.Rat { + if i.big != nil { + return new(big.Rat).SetInt(i.big) + } + return new(big.Rat).SetInt64(i.small) +} + +// AsInt32 returns the value of x if is representable as an int32. +func AsInt32(x Value) (int, error) { + i, ok := x.(Int) + if !ok { + return 0, fmt.Errorf("got %s, want int", x.Type()) + } + if i.big != nil { + return 0, fmt.Errorf("%s out of range", i) + } + return int(i.small), nil +} + +// NumberToInt converts a number x to an integer value. +// An int is returned unchanged, a float is truncated towards zero. +// NumberToInt reports an error for all other values. +func NumberToInt(x Value) (Int, error) { + switch x := x.(type) { + case Int: + return x, nil + case Float: + f := float64(x) + if math.IsInf(f, 0) { + return zero, fmt.Errorf("cannot convert float infinity to integer") + } else if math.IsNaN(f) { + return zero, fmt.Errorf("cannot convert float NaN to integer") + } + return finiteFloatToInt(x), nil + + } + return zero, fmt.Errorf("cannot convert %s to int", x.Type()) +} + +// finiteFloatToInt converts f to an Int, truncating towards zero. +// f must be finite. +func finiteFloatToInt(f Float) Int { + if math.MinInt64 <= f && f <= math.MaxInt64 { + // small values + return MakeInt64(int64(f)) + } + rat := f.rational() + if rat == nil { + panic(f) // non-finite + } + return MakeBigInt(new(big.Int).Div(rat.Num(), rat.Denom())) +} diff --git a/vendor/go.starlark.net/starlark/interp.go b/vendor/go.starlark.net/starlark/interp.go new file mode 100644 index 00000000..97ef66f1 --- /dev/null +++ b/vendor/go.starlark.net/starlark/interp.go @@ -0,0 +1,575 @@ +package starlark + +// This file defines the bytecode interpreter. + +import ( + "fmt" + "os" + + "go.starlark.net/internal/compile" + "go.starlark.net/resolve" + "go.starlark.net/syntax" +) + +const vmdebug = false // TODO(adonovan): use a bitfield of specific kinds of error. + +// TODO(adonovan): +// - optimize position table. +// - opt: reduce allocations by preallocating a large stack, saving it +// in the thread, and slicing it. +// - opt: record MaxIterStack during compilation and preallocate the stack. + +func (fn *Function) CallInternal(thread *Thread, args Tuple, kwargs []Tuple) (Value, error) { + if !resolve.AllowRecursion { + // detect recursion + for fr := thread.frame.parent; fr != nil; fr = fr.parent { + // We look for the same function code, + // not function value, otherwise the user could + // defeat the check by writing the Y combinator. + if frfn, ok := fr.Callable().(*Function); ok && frfn.funcode == fn.funcode { + return nil, fmt.Errorf("function %s called recursively", fn.Name()) + } + } + } + + f := fn.funcode + fr := thread.frame + nlocals := len(f.Locals) + stack := make([]Value, nlocals+f.MaxStack) + locals := stack[:nlocals:nlocals] // local variables, starting with parameters + stack = stack[nlocals:] + + err := setArgs(locals, fn, args, kwargs) + if err != nil { + return nil, fr.errorf(fr.Position(), "%v", err) + } + + fr.locals = locals // for debugger + + if vmdebug { + fmt.Printf("Entering %s @ %s\n", f.Name, f.Position(0)) + fmt.Printf("%d stack, %d locals\n", len(stack), len(locals)) + defer fmt.Println("Leaving ", f.Name) + } + + // TODO(adonovan): add static check that beneath this point + // - there is exactly one return statement + // - there is no redefinition of 'err'. + + var iterstack []Iterator // stack of active iterators + + sp := 0 + var pc, savedpc uint32 + var result Value + code := f.Code +loop: + for { + savedpc = pc + + op := compile.Opcode(code[pc]) + pc++ + var arg uint32 + if op >= compile.OpcodeArgMin { + // TODO(adonovan): opt: profile this. + // Perhaps compiling big endian would be less work to decode? + for s := uint(0); ; s += 7 { + b := code[pc] + pc++ + arg |= uint32(b&0x7f) << s + if b < 0x80 { + break + } + } + } + if vmdebug { + fmt.Fprintln(os.Stderr, stack[:sp]) // very verbose! + compile.PrintOp(f, savedpc, op, arg) + } + + switch op { + case compile.NOP: + // nop + + case compile.DUP: + stack[sp] = stack[sp-1] + sp++ + + case compile.DUP2: + stack[sp] = stack[sp-2] + stack[sp+1] = stack[sp-1] + sp += 2 + + case compile.POP: + sp-- + + case compile.EXCH: + stack[sp-2], stack[sp-1] = stack[sp-1], stack[sp-2] + + case compile.EQL, compile.NEQ, compile.GT, compile.LT, compile.LE, compile.GE: + op := syntax.Token(op-compile.EQL) + syntax.EQL + y := stack[sp-1] + x := stack[sp-2] + sp -= 2 + ok, err2 := Compare(op, x, y) + if err2 != nil { + err = err2 + break loop + } + stack[sp] = Bool(ok) + sp++ + + case compile.PLUS, + compile.MINUS, + compile.STAR, + compile.SLASH, + compile.SLASHSLASH, + compile.PERCENT, + compile.AMP, + compile.PIPE, + compile.CIRCUMFLEX, + compile.LTLT, + compile.GTGT, + compile.IN: + binop := syntax.Token(op-compile.PLUS) + syntax.PLUS + if op == compile.IN { + binop = syntax.IN // IN token is out of order + } + y := stack[sp-1] + x := stack[sp-2] + sp -= 2 + z, err2 := Binary(binop, x, y) + if err2 != nil { + err = err2 + break loop + } + stack[sp] = z + sp++ + + case compile.UPLUS, compile.UMINUS, compile.TILDE: + var unop syntax.Token + if op == compile.TILDE { + unop = syntax.TILDE + } else { + unop = syntax.Token(op-compile.UPLUS) + syntax.PLUS + } + x := stack[sp-1] + y, err2 := Unary(unop, x) + if err2 != nil { + err = err2 + break loop + } + stack[sp-1] = y + + case compile.INPLACE_ADD: + y := stack[sp-1] + x := stack[sp-2] + sp -= 2 + + // It's possible that y is not Iterable but + // nonetheless defines x+y, in which case we + // should fall back to the general case. + var z Value + if xlist, ok := x.(*List); ok { + if yiter, ok := y.(Iterable); ok { + if err = xlist.checkMutable("apply += to"); err != nil { + break loop + } + listExtend(xlist, yiter) + z = xlist + } + } + if z == nil { + z, err = Binary(syntax.PLUS, x, y) + if err != nil { + break loop + } + } + + stack[sp] = z + sp++ + + case compile.NONE: + stack[sp] = None + sp++ + + case compile.TRUE: + stack[sp] = True + sp++ + + case compile.FALSE: + stack[sp] = False + sp++ + + case compile.MANDATORY: + stack[sp] = mandatory{} + sp++ + + case compile.JMP: + pc = arg + + case compile.CALL, compile.CALL_VAR, compile.CALL_KW, compile.CALL_VAR_KW: + var kwargs Value + if op == compile.CALL_KW || op == compile.CALL_VAR_KW { + kwargs = stack[sp-1] + sp-- + } + + var args Value + if op == compile.CALL_VAR || op == compile.CALL_VAR_KW { + args = stack[sp-1] + sp-- + } + + // named args (pairs) + var kvpairs []Tuple + if nkvpairs := int(arg & 0xff); nkvpairs > 0 { + kvpairs = make([]Tuple, 0, nkvpairs) + kvpairsAlloc := make(Tuple, 2*nkvpairs) // allocate a single backing array + sp -= 2 * nkvpairs + for i := 0; i < nkvpairs; i++ { + pair := kvpairsAlloc[:2:2] + kvpairsAlloc = kvpairsAlloc[2:] + pair[0] = stack[sp+2*i] // name + pair[1] = stack[sp+2*i+1] // value + kvpairs = append(kvpairs, pair) + } + } + if kwargs != nil { + // Add key/value items from **kwargs dictionary. + dict, ok := kwargs.(IterableMapping) + if !ok { + err = fmt.Errorf("argument after ** must be a mapping, not %s", kwargs.Type()) + break loop + } + items := dict.Items() + for _, item := range items { + if _, ok := item[0].(String); !ok { + err = fmt.Errorf("keywords must be strings, not %s", item[0].Type()) + break loop + } + } + if len(kvpairs) == 0 { + kvpairs = items + } else { + kvpairs = append(kvpairs, items...) + } + } + + // positional args + var positional Tuple + if npos := int(arg >> 8); npos > 0 { + positional = make(Tuple, npos) + sp -= npos + copy(positional, stack[sp:]) + } + if args != nil { + // Add elements from *args sequence. + iter := Iterate(args) + if iter == nil { + err = fmt.Errorf("argument after * must be iterable, not %s", args.Type()) + break loop + } + var elem Value + for iter.Next(&elem) { + positional = append(positional, elem) + } + iter.Done() + } + + function := stack[sp-1] + + if vmdebug { + fmt.Printf("VM call %s args=%s kwargs=%s @%s\n", + function, positional, kvpairs, f.Position(fr.callpc)) + } + + fr.callpc = savedpc + z, err2 := Call(thread, function, positional, kvpairs) + if err2 != nil { + err = err2 + break loop + } + if vmdebug { + fmt.Printf("Resuming %s @ %s\n", f.Name, f.Position(0)) + } + stack[sp-1] = z + + case compile.ITERPUSH: + x := stack[sp-1] + sp-- + iter := Iterate(x) + if iter == nil { + err = fmt.Errorf("%s value is not iterable", x.Type()) + break loop + } + iterstack = append(iterstack, iter) + + case compile.ITERJMP: + iter := iterstack[len(iterstack)-1] + if iter.Next(&stack[sp]) { + sp++ + } else { + pc = arg + } + + case compile.ITERPOP: + n := len(iterstack) - 1 + iterstack[n].Done() + iterstack = iterstack[:n] + + case compile.NOT: + stack[sp-1] = !stack[sp-1].Truth() + + case compile.RETURN: + result = stack[sp-1] + break loop + + case compile.SETINDEX: + z := stack[sp-1] + y := stack[sp-2] + x := stack[sp-3] + sp -= 3 + err = setIndex(x, y, z) + if err != nil { + break loop + } + + case compile.INDEX: + y := stack[sp-1] + x := stack[sp-2] + sp -= 2 + z, err2 := getIndex(x, y) + if err2 != nil { + err = err2 + break loop + } + stack[sp] = z + sp++ + + case compile.ATTR: + x := stack[sp-1] + name := f.Prog.Names[arg] + y, err2 := getAttr(x, name) + if err2 != nil { + err = err2 + break loop + } + stack[sp-1] = y + + case compile.SETFIELD: + y := stack[sp-1] + x := stack[sp-2] + sp -= 2 + name := f.Prog.Names[arg] + if err2 := setField(x, name, y); err2 != nil { + err = err2 + break loop + } + + case compile.MAKEDICT: + stack[sp] = new(Dict) + sp++ + + case compile.SETDICT, compile.SETDICTUNIQ: + dict := stack[sp-3].(*Dict) + k := stack[sp-2] + v := stack[sp-1] + sp -= 3 + oldlen := dict.Len() + if err2 := dict.SetKey(k, v); err2 != nil { + err = err2 + break loop + } + if op == compile.SETDICTUNIQ && dict.Len() == oldlen { + err = fmt.Errorf("duplicate key: %v", k) + break loop + } + + case compile.APPEND: + elem := stack[sp-1] + list := stack[sp-2].(*List) + sp -= 2 + list.elems = append(list.elems, elem) + + case compile.SLICE: + x := stack[sp-4] + lo := stack[sp-3] + hi := stack[sp-2] + step := stack[sp-1] + sp -= 4 + res, err2 := slice(x, lo, hi, step) + if err2 != nil { + err = err2 + break loop + } + stack[sp] = res + sp++ + + case compile.UNPACK: + n := int(arg) + iterable := stack[sp-1] + sp-- + iter := Iterate(iterable) + if iter == nil { + err = fmt.Errorf("got %s in sequence assignment", iterable.Type()) + break loop + } + i := 0 + sp += n + for i < n && iter.Next(&stack[sp-1-i]) { + i++ + } + var dummy Value + if iter.Next(&dummy) { + // NB: Len may return -1 here in obscure cases. + err = fmt.Errorf("too many values to unpack (got %d, want %d)", Len(iterable), n) + break loop + } + iter.Done() + if i < n { + err = fmt.Errorf("too few values to unpack (got %d, want %d)", i, n) + break loop + } + + case compile.CJMP: + if stack[sp-1].Truth() { + pc = arg + } + sp-- + + case compile.CONSTANT: + stack[sp] = fn.constants[arg] + sp++ + + case compile.MAKETUPLE: + n := int(arg) + tuple := make(Tuple, n) + sp -= n + copy(tuple, stack[sp:]) + stack[sp] = tuple + sp++ + + case compile.MAKELIST: + n := int(arg) + elems := make([]Value, n) + sp -= n + copy(elems, stack[sp:]) + stack[sp] = NewList(elems) + sp++ + + case compile.MAKEFUNC: + funcode := f.Prog.Functions[arg] + freevars := stack[sp-1].(Tuple) + defaults := stack[sp-2].(Tuple) + sp -= 2 + stack[sp] = &Function{ + funcode: funcode, + predeclared: fn.predeclared, + globals: fn.globals, + constants: fn.constants, + defaults: defaults, + freevars: freevars, + } + sp++ + + case compile.LOAD: + n := int(arg) + module := string(stack[sp-1].(String)) + sp-- + + if thread.Load == nil { + err = fmt.Errorf("load not implemented by this application") + break loop + } + + dict, err2 := thread.Load(thread, module) + if err2 != nil { + err = fmt.Errorf("cannot load %s: %v", module, err2) + break loop + } + + for i := 0; i < n; i++ { + from := string(stack[sp-1-i].(String)) + v, ok := dict[from] + if !ok { + err = fmt.Errorf("load: name %s not found in module %s", from, module) + break loop + } + stack[sp-1-i] = v + } + + case compile.SETLOCAL: + locals[arg] = stack[sp-1] + sp-- + + case compile.SETGLOBAL: + fn.globals[arg] = stack[sp-1] + sp-- + + case compile.LOCAL: + x := locals[arg] + if x == nil { + err = fmt.Errorf("local variable %s referenced before assignment", f.Locals[arg].Name) + break loop + } + stack[sp] = x + sp++ + + case compile.FREE: + stack[sp] = fn.freevars[arg] + sp++ + + case compile.GLOBAL: + x := fn.globals[arg] + if x == nil { + err = fmt.Errorf("global variable %s referenced before assignment", f.Prog.Globals[arg].Name) + break loop + } + stack[sp] = x + sp++ + + case compile.PREDECLARED: + name := f.Prog.Names[arg] + x := fn.predeclared[name] + if x == nil { + err = fmt.Errorf("internal error: predeclared variable %s is uninitialized", name) + break loop + } + stack[sp] = x + sp++ + + case compile.UNIVERSAL: + stack[sp] = Universe[f.Prog.Names[arg]] + sp++ + + default: + err = fmt.Errorf("unimplemented: %s", op) + break loop + } + } + + // ITERPOP the rest of the iterator stack. + for _, iter := range iterstack { + iter.Done() + } + + if err != nil { + if _, ok := err.(*EvalError); !ok { + err = fr.errorf(f.Position(savedpc), "%s", err.Error()) + } + } + + fr.locals = nil + + return result, err +} + +// mandatory is a sentinel value used in a function's defaults tuple +// to indicate that a (keyword-only) parameter is mandatory. +type mandatory struct{} + +func (mandatory) String() string { return "mandatory" } +func (mandatory) Type() string { return "mandatory" } +func (mandatory) Freeze() {} // immutable +func (mandatory) Truth() Bool { return False } +func (mandatory) Hash() (uint32, error) { return 0, nil } diff --git a/vendor/go.starlark.net/starlark/library.go b/vendor/go.starlark.net/starlark/library.go new file mode 100644 index 00000000..5b7faad9 --- /dev/null +++ b/vendor/go.starlark.net/starlark/library.go @@ -0,0 +1,2250 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package starlark + +// This file defines the library of built-ins. +// +// Built-ins must explicitly check the "frozen" flag before updating +// mutable types such as lists and dicts. + +import ( + "bytes" + "fmt" + "log" + "math/big" + "os" + "reflect" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "go.starlark.net/syntax" +) + +// Universe defines the set of universal built-ins, such as None, True, and len. +// +// The Go application may add or remove items from the +// universe dictionary before Starlark evaluation begins. +// All values in the dictionary must be immutable. +// Starlark programs cannot modify the dictionary. +var Universe StringDict + +func init() { + // https://github.com/google/starlark-go/blob/master/doc/spec.md#built-in-constants-and-functions + Universe = StringDict{ + "None": None, + "True": True, + "False": False, + "any": NewBuiltin("any", any), + "all": NewBuiltin("all", all), + "bool": NewBuiltin("bool", bool_), + "chr": NewBuiltin("chr", chr), + "dict": NewBuiltin("dict", dict), + "dir": NewBuiltin("dir", dir), + "enumerate": NewBuiltin("enumerate", enumerate), + "float": NewBuiltin("float", float), // requires resolve.AllowFloat + "getattr": NewBuiltin("getattr", getattr), + "hasattr": NewBuiltin("hasattr", hasattr), + "hash": NewBuiltin("hash", hash), + "int": NewBuiltin("int", int_), + "len": NewBuiltin("len", len_), + "list": NewBuiltin("list", list), + "max": NewBuiltin("max", minmax), + "min": NewBuiltin("min", minmax), + "ord": NewBuiltin("ord", ord), + "print": NewBuiltin("print", print), + "range": NewBuiltin("range", range_), + "repr": NewBuiltin("repr", repr), + "reversed": NewBuiltin("reversed", reversed), + "set": NewBuiltin("set", set), // requires resolve.AllowSet + "sorted": NewBuiltin("sorted", sorted), + "str": NewBuiltin("str", str), + "tuple": NewBuiltin("tuple", tuple), + "type": NewBuiltin("type", type_), + "zip": NewBuiltin("zip", zip), + } +} + +type builtinMethod func(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) + +// methods of built-in types +// https://github.com/google/starlark-go/blob/master/doc/spec.md#built-in-methods +var ( + dictMethods = map[string]builtinMethod{ + "clear": dict_clear, + "get": dict_get, + "items": dict_items, + "keys": dict_keys, + "pop": dict_pop, + "popitem": dict_popitem, + "setdefault": dict_setdefault, + "update": dict_update, + "values": dict_values, + } + + listMethods = map[string]builtinMethod{ + "append": list_append, + "clear": list_clear, + "extend": list_extend, + "index": list_index, + "insert": list_insert, + "pop": list_pop, + "remove": list_remove, + } + + stringMethods = map[string]builtinMethod{ + "capitalize": string_capitalize, + "codepoint_ords": string_iterable, + "codepoints": string_iterable, // sic + "count": string_count, + "elem_ords": string_iterable, + "elems": string_iterable, // sic + "endswith": string_startswith, // sic + "find": string_find, + "format": string_format, + "index": string_index, + "isalnum": string_isalnum, + "isalpha": string_isalpha, + "isdigit": string_isdigit, + "islower": string_islower, + "isspace": string_isspace, + "istitle": string_istitle, + "isupper": string_isupper, + "join": string_join, + "lower": string_lower, + "lstrip": string_strip, // sic + "partition": string_partition, + "replace": string_replace, + "rfind": string_rfind, + "rindex": string_rindex, + "rpartition": string_partition, // sic + "rsplit": string_split, // sic + "rstrip": string_strip, // sic + "split": string_split, + "splitlines": string_splitlines, + "startswith": string_startswith, + "strip": string_strip, + "title": string_title, + "upper": string_upper, + } + + setMethods = map[string]builtinMethod{ + "union": set_union, + } +) + +func builtinAttr(recv Value, name string, methods map[string]builtinMethod) (Value, error) { + method := methods[name] + if method == nil { + return nil, nil // no such method + } + + // Allocate a closure over 'method'. + impl := func(thread *Thread, b *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + return method(b.Name(), b.Receiver(), args, kwargs) + } + return NewBuiltin(name, impl).BindReceiver(recv), nil +} + +func builtinAttrNames(methods map[string]builtinMethod) []string { + names := make([]string, 0, len(methods)) + for name := range methods { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// UnpackArgs unpacks the positional and keyword arguments into the +// supplied parameter variables. pairs is an alternating list of names +// and pointers to variables. +// +// If the variable is a bool, int, string, *List, *Dict, Callable, +// Iterable, or user-defined implementation of Value, +// UnpackArgs performs the appropriate type check. +// An int uses the AsInt32 check. +// If the parameter name ends with "?", +// it and all following parameters are optional. +// +// If the variable implements Value, UnpackArgs may call +// its Type() method while constructing the error message. +// +// Beware: an optional *List, *Dict, Callable, Iterable, or Value variable that is +// not assigned is not a valid Starlark Value, so the caller must +// explicitly handle such cases by interpreting nil as None or some +// computed default. +func UnpackArgs(fnname string, args Tuple, kwargs []Tuple, pairs ...interface{}) error { + nparams := len(pairs) / 2 + var defined intset + defined.init(nparams) + + // positional arguments + if len(args) > nparams { + return fmt.Errorf("%s: got %d arguments, want at most %d", + fnname, len(args), nparams) + } + for i, arg := range args { + defined.set(i) + if err := unpackOneArg(arg, pairs[2*i+1]); err != nil { + return fmt.Errorf("%s: for parameter %d: %s", fnname, i+1, err) + } + } + + // keyword arguments +kwloop: + for _, item := range kwargs { + name, arg := item[0].(String), item[1] + for i := 0; i < nparams; i++ { + paramName := pairs[2*i].(string) + if paramName[len(paramName)-1] == '?' { + paramName = paramName[:len(paramName)-1] + } + if paramName == string(name) { + // found it + if defined.set(i) { + return fmt.Errorf("%s: got multiple values for keyword argument %s", + fnname, name) + } + ptr := pairs[2*i+1] + if err := unpackOneArg(arg, ptr); err != nil { + return fmt.Errorf("%s: for parameter %s: %s", fnname, name, err) + } + continue kwloop + } + } + return fmt.Errorf("%s: unexpected keyword argument %s", fnname, name) + } + + // Check that all non-optional parameters are defined. + // (We needn't check the first len(args).) + for i := len(args); i < nparams; i++ { + name := pairs[2*i].(string) + if strings.HasSuffix(name, "?") { + break // optional + } + if !defined.get(i) { + return fmt.Errorf("%s: missing argument for %s", fnname, name) + } + } + + return nil +} + +// UnpackPositionalArgs unpacks the positional arguments into +// corresponding variables. Each element of vars is a pointer; see +// UnpackArgs for allowed types and conversions. +// +// UnpackPositionalArgs reports an error if the number of arguments is +// less than min or greater than len(vars), if kwargs is nonempty, or if +// any conversion fails. +func UnpackPositionalArgs(fnname string, args Tuple, kwargs []Tuple, min int, vars ...interface{}) error { + if len(kwargs) > 0 { + return fmt.Errorf("%s: unexpected keyword arguments", fnname) + } + max := len(vars) + if len(args) < min { + var atleast string + if min < max { + atleast = "at least " + } + return fmt.Errorf("%s: got %d arguments, want %s%d", fnname, len(args), atleast, min) + } + if len(args) > max { + var atmost string + if max > min { + atmost = "at most " + } + return fmt.Errorf("%s: got %d arguments, want %s%d", fnname, len(args), atmost, max) + } + for i, arg := range args { + if err := unpackOneArg(arg, vars[i]); err != nil { + return fmt.Errorf("%s: for parameter %d: %s", fnname, i+1, err) + } + } + return nil +} + +func unpackOneArg(v Value, ptr interface{}) error { + // On failure, don't clobber *ptr. + switch ptr := ptr.(type) { + case *Value: + *ptr = v + case *string: + s, ok := AsString(v) + if !ok { + return fmt.Errorf("got %s, want string", v.Type()) + } + *ptr = s + case *bool: + b, ok := v.(Bool) + if !ok { + return fmt.Errorf("got %s, want bool", v.Type()) + } + *ptr = bool(b) + case *int: + i, err := AsInt32(v) + if err != nil { + return err + } + *ptr = i + case **List: + list, ok := v.(*List) + if !ok { + return fmt.Errorf("got %s, want list", v.Type()) + } + *ptr = list + case **Dict: + dict, ok := v.(*Dict) + if !ok { + return fmt.Errorf("got %s, want dict", v.Type()) + } + *ptr = dict + case *Callable: + f, ok := v.(Callable) + if !ok { + return fmt.Errorf("got %s, want callable", v.Type()) + } + *ptr = f + case *Iterable: + it, ok := v.(Iterable) + if !ok { + return fmt.Errorf("got %s, want iterable", v.Type()) + } + *ptr = it + default: + // v must have type *V, where V is some subtype of starlark.Value. + ptrv := reflect.ValueOf(ptr) + if ptrv.Kind() != reflect.Ptr { + log.Panicf("internal error: not a pointer: %T", ptr) + } + paramVar := ptrv.Elem() + if !reflect.TypeOf(v).AssignableTo(paramVar.Type()) { + // The value is not assignable to the variable. + + // Detect a possible bug in the Go program that called Unpack: + // If the variable *ptr is not a subtype of Value, + // no value of v can possibly work. + if !paramVar.Type().AssignableTo(reflect.TypeOf(new(Value)).Elem()) { + log.Panicf("pointer element type does not implement Value: %T", ptr) + } + + // Report Starlark dynamic type error. + // + // We prefer the Starlark Value.Type name over + // its Go reflect.Type name, but calling the + // Value.Type method on the variable is not safe + // in general. If the variable is an interface, + // the call will fail. Even if the variable has + // a concrete type, it might not be safe to call + // Type() on a zero instance. Thus we must use + // recover. + + // Default to Go reflect.Type name + paramType := paramVar.Type().String() + + // Attempt to call Value.Type method. + func() { + defer func() { recover() }() + paramType = paramVar.MethodByName("Type").Call(nil)[0].String() + }() + return fmt.Errorf("got %s, want %s", v.Type(), paramType) + } + paramVar.Set(reflect.ValueOf(v)) + } + return nil +} + +// ---- built-in functions ---- + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#all +func all(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var iterable Iterable + if err := UnpackPositionalArgs("all", args, kwargs, 1, &iterable); err != nil { + return nil, err + } + iter := iterable.Iterate() + defer iter.Done() + var x Value + for iter.Next(&x) { + if !x.Truth() { + return False, nil + } + } + return True, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#any +func any(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var iterable Iterable + if err := UnpackPositionalArgs("any", args, kwargs, 1, &iterable); err != nil { + return nil, err + } + iter := iterable.Iterate() + defer iter.Done() + var x Value + for iter.Next(&x) { + if x.Truth() { + return True, nil + } + } + return False, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#bool +func bool_(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var x Value = False + if err := UnpackPositionalArgs("bool", args, kwargs, 0, &x); err != nil { + return nil, err + } + return x.Truth(), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#chr +func chr(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if len(kwargs) > 0 { + return nil, fmt.Errorf("chr does not accept keyword arguments") + } + if len(args) != 1 { + return nil, fmt.Errorf("chr: got %d arguments, want 1", len(args)) + } + i, err := AsInt32(args[0]) + if err != nil { + return nil, fmt.Errorf("chr: got %s, want int", args[0].Type()) + } + if i < 0 { + return nil, fmt.Errorf("chr: Unicode code point %d out of range (<0)", i) + } + if i > unicode.MaxRune { + return nil, fmt.Errorf("chr: Unicode code point U+%X out of range (>0x10FFFF)", i) + } + return String(string(i)), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dict +func dict(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if len(args) > 1 { + return nil, fmt.Errorf("dict: got %d arguments, want at most 1", len(args)) + } + dict := new(Dict) + if err := updateDict(dict, args, kwargs); err != nil { + return nil, fmt.Errorf("dict: %v", err) + } + return dict, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dir +func dir(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if len(kwargs) > 0 { + return nil, fmt.Errorf("dir does not accept keyword arguments") + } + if len(args) != 1 { + return nil, fmt.Errorf("dir: got %d arguments, want 1", len(args)) + } + + var names []string + if x, ok := args[0].(HasAttrs); ok { + names = x.AttrNames() + } + elems := make([]Value, len(names)) + for i, name := range names { + elems[i] = String(name) + } + return NewList(elems), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#enumerate +func enumerate(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var iterable Iterable + var start int + if err := UnpackPositionalArgs("enumerate", args, kwargs, 1, &iterable, &start); err != nil { + return nil, err + } + + iter := iterable.Iterate() + if iter == nil { + return nil, fmt.Errorf("enumerate: got %s, want iterable", iterable.Type()) + } + defer iter.Done() + + var pairs []Value + var x Value + + if n := Len(iterable); n >= 0 { + // common case: known length + pairs = make([]Value, 0, n) + array := make(Tuple, 2*n) // allocate a single backing array + for i := 0; iter.Next(&x); i++ { + pair := array[:2:2] + array = array[2:] + pair[0] = MakeInt(start + i) + pair[1] = x + pairs = append(pairs, pair) + } + } else { + // non-sequence (unknown length) + for i := 0; iter.Next(&x); i++ { + pair := Tuple{MakeInt(start + i), x} + pairs = append(pairs, pair) + } + } + + return NewList(pairs), nil +} + +func float(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if len(kwargs) > 0 { + return nil, fmt.Errorf("float does not accept keyword arguments") + } + if len(args) == 0 { + return Float(0.0), nil + } + if len(args) != 1 { + return nil, fmt.Errorf("float got %d arguments, wants 1", len(args)) + } + switch x := args[0].(type) { + case Bool: + if x { + return Float(1.0), nil + } else { + return Float(0.0), nil + } + case Int: + return x.Float(), nil + case Float: + return x, nil + case String: + f, err := strconv.ParseFloat(string(x), 64) + if err != nil { + return nil, err + } + return Float(f), nil + default: + return nil, fmt.Errorf("float got %s, want number or string", x.Type()) + } +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#getattr +func getattr(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var object, dflt Value + var name string + if err := UnpackPositionalArgs("getattr", args, kwargs, 2, &object, &name, &dflt); err != nil { + return nil, err + } + if object, ok := object.(HasAttrs); ok { + v, err := object.Attr(name) + if err != nil { + // An error could mean the field doesn't exist, + // or it exists but could not be computed. + if dflt != nil { + return dflt, nil + } + return nil, err + } + if v != nil { + return v, nil + } + // (nil, nil) => no such field + } + if dflt != nil { + return dflt, nil + } + return nil, fmt.Errorf("%s has no .%s field or method", object.Type(), name) +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#hasattr +func hasattr(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var object Value + var name string + if err := UnpackPositionalArgs("hasattr", args, kwargs, 2, &object, &name); err != nil { + return nil, err + } + if object, ok := object.(HasAttrs); ok { + v, err := object.Attr(name) + if err == nil { + return Bool(v != nil), nil + } + + // An error does not conclusively indicate presence or + // absence of a field: it could occur while computing + // the value of a present attribute, or it could be a + // "no such attribute" error with details. + for _, x := range object.AttrNames() { + if x == name { + return True, nil + } + } + } + return False, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#hash +func hash(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var x Value + if err := UnpackPositionalArgs("hash", args, kwargs, 1, &x); err != nil { + return nil, err + } + h, err := x.Hash() + return MakeUint(uint(h)), err +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#int +func int_(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var x Value = zero + var base Value + if err := UnpackArgs("int", args, kwargs, "x", &x, "base?", &base); err != nil { + return nil, err + } + + // "If x is not a number or base is given, x must be a string." + if s, ok := AsString(x); ok { + b := 10 + if base != nil { + var err error + b, err = AsInt32(base) + if err != nil || b != 0 && (b < 2 || b > 36) { + return nil, fmt.Errorf("int: base must be an integer >= 2 && <= 36") + } + } + + orig := s // save original for error message + + // remove sign + var neg bool + if s != "" { + if s[0] == '+' { + s = s[1:] + } else if s[0] == '-' { + neg = true + s = s[1:] + } + } + + // remove base prefix + baseprefix := 0 + if len(s) > 1 && s[0] == '0' { + if len(s) > 2 { + switch s[1] { + case 'o', 'O': + s = s[2:] + baseprefix = 8 + case 'x', 'X': + s = s[2:] + baseprefix = 16 + case 'b', 'B': + s = s[2:] + baseprefix = 2 + } + } + + // For automatic base detection, + // a string starting with zero + // must be all zeros. + // Thus we reject int("0755", 0). + if baseprefix == 0 && b == 0 { + for i := 1; i < len(s); i++ { + if s[i] != '0' { + goto invalid + } + } + return zero, nil + } + + if b != 0 && baseprefix != 0 && baseprefix != b { + // Explicit base doesn't match prefix, + // e.g. int("0o755", 16). + goto invalid + } + } + + // select base + if b == 0 { + if baseprefix != 0 { + b = baseprefix + } else { + b = 10 + } + } + + // we explicitly handled sign above. + // if a sign remains, it is invalid. + if s != "" && (s[0] == '-' || s[0] == '+') { + goto invalid + } + + // s has no sign or base prefix. + // + // int(x) permits arbitrary precision, unlike the scanner. + if i, ok := new(big.Int).SetString(s, b); ok { + res := MakeBigInt(i) + if neg { + res = zero.Sub(res) + } + return res, nil + } + + invalid: + return nil, fmt.Errorf("int: invalid literal with base %d: %s", b, orig) + } + + if base != nil { + return nil, fmt.Errorf("int: can't convert non-string with explicit base") + } + + if b, ok := x.(Bool); ok { + if b { + return one, nil + } else { + return zero, nil + } + } + + i, err := NumberToInt(x) + if err != nil { + return nil, fmt.Errorf("int: %s", err) + } + return i, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#len +func len_(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var x Value + if err := UnpackPositionalArgs("len", args, kwargs, 1, &x); err != nil { + return nil, err + } + len := Len(x) + if len < 0 { + return nil, fmt.Errorf("value of type %s has no len", x.Type()) + } + return MakeInt(len), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#list +func list(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var iterable Iterable + if err := UnpackPositionalArgs("list", args, kwargs, 0, &iterable); err != nil { + return nil, err + } + var elems []Value + if iterable != nil { + iter := iterable.Iterate() + defer iter.Done() + if n := Len(iterable); n > 0 { + elems = make([]Value, 0, n) // preallocate if length known + } + var x Value + for iter.Next(&x) { + elems = append(elems, x) + } + } + return NewList(elems), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#min +func minmax(thread *Thread, fn *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if len(args) == 0 { + return nil, fmt.Errorf("%s requires at least one positional argument", fn.Name()) + } + var keyFunc Callable + if err := UnpackArgs(fn.Name(), nil, kwargs, "key?", &keyFunc); err != nil { + return nil, err + } + var op syntax.Token + if fn.Name() == "max" { + op = syntax.GT + } else { + op = syntax.LT + } + var iterable Value + if len(args) == 1 { + iterable = args[0] + } else { + iterable = args + } + iter := Iterate(iterable) + if iter == nil { + return nil, fmt.Errorf("%s: %s value is not iterable", fn.Name(), iterable.Type()) + } + defer iter.Done() + var extremum Value + if !iter.Next(&extremum) { + return nil, fmt.Errorf("%s: argument is an empty sequence", fn.Name()) + } + + var extremeKey Value + var keyargs Tuple + if keyFunc == nil { + extremeKey = extremum + } else { + keyargs = Tuple{extremum} + res, err := Call(thread, keyFunc, keyargs, nil) + if err != nil { + return nil, err + } + extremeKey = res + } + + var x Value + for iter.Next(&x) { + var key Value + if keyFunc == nil { + key = x + } else { + keyargs[0] = x + res, err := Call(thread, keyFunc, keyargs, nil) + if err != nil { + return nil, err + } + key = res + } + + if ok, err := Compare(op, key, extremeKey); err != nil { + return nil, err + } else if ok { + extremum = x + extremeKey = key + } + } + return extremum, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#ord +func ord(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if len(kwargs) > 0 { + return nil, fmt.Errorf("ord does not accept keyword arguments") + } + if len(args) != 1 { + return nil, fmt.Errorf("ord: got %d arguments, want 1", len(args)) + } + s, ok := AsString(args[0]) + if !ok { + return nil, fmt.Errorf("ord: got %s, want string", args[0].Type()) + } + r, sz := utf8.DecodeRuneInString(s) + if sz == 0 || sz != len(s) { + n := utf8.RuneCountInString(s) + return nil, fmt.Errorf("ord: string encodes %d Unicode code points, want 1", n) + } + return MakeInt(int(r)), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#print +func print(thread *Thread, fn *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + sep := " " + if err := UnpackArgs("print", nil, kwargs, "sep?", &sep); err != nil { + return nil, err + } + var buf bytes.Buffer + for i, v := range args { + if i > 0 { + buf.WriteString(sep) + } + if s, ok := AsString(v); ok { + buf.WriteString(s) + } else { + writeValue(&buf, v, nil) + } + } + + if thread.Print != nil { + thread.Print(thread, buf.String()) + } else { + fmt.Fprintln(os.Stderr, &buf) + } + return None, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#range +func range_(thread *Thread, fn *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var start, stop, step int + step = 1 + if err := UnpackPositionalArgs("range", args, kwargs, 1, &start, &stop, &step); err != nil { + return nil, err + } + + // TODO(adonovan): analyze overflow/underflows cases for 32-bit implementations. + if len(args) == 1 { + // range(stop) + start, stop = 0, start + } + if step == 0 { + // we were given range(start, stop, 0) + return nil, fmt.Errorf("range: step argument must not be zero") + } + + return rangeValue{start: start, stop: stop, step: step, len: rangeLen(start, stop, step)}, nil +} + +// A rangeValue is a comparable, immutable, indexable sequence of integers +// defined by the three parameters to a range(...) call. +// Invariant: step != 0. +type rangeValue struct{ start, stop, step, len int } + +var ( + _ Indexable = rangeValue{} + _ Sequence = rangeValue{} + _ Comparable = rangeValue{} + _ Sliceable = rangeValue{} +) + +func (r rangeValue) Len() int { return r.len } +func (r rangeValue) Index(i int) Value { return MakeInt(r.start + i*r.step) } +func (r rangeValue) Iterate() Iterator { return &rangeIterator{r, 0} } + +// rangeLen calculates the length of a range with the provided start, stop, and step. +// caller must ensure that step is non-zero. +func rangeLen(start, stop, step int) int { + switch { + case step > 0: + if stop > start { + return (stop-1-start)/step + 1 + } + case step < 0: + if start > stop { + return (start-1-stop)/-step + 1 + } + default: + panic("rangeLen: zero step") + } + return 0 +} + +func (r rangeValue) Slice(start, end, step int) Value { + newStart := r.start + r.step*start + newStop := r.start + r.step*end + newStep := r.step * step + return rangeValue{ + start: newStart, + stop: newStop, + step: newStep, + len: rangeLen(newStart, newStop, newStep), + } +} + +func (r rangeValue) Freeze() {} // immutable +func (r rangeValue) String() string { + if r.step != 1 { + return fmt.Sprintf("range(%d, %d, %d)", r.start, r.stop, r.step) + } else if r.start != 0 { + return fmt.Sprintf("range(%d, %d)", r.start, r.stop) + } else { + return fmt.Sprintf("range(%d)", r.stop) + } +} +func (r rangeValue) Type() string { return "range" } +func (r rangeValue) Truth() Bool { return r.len > 0 } +func (r rangeValue) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: range") } + +func (x rangeValue) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { + y := y_.(rangeValue) + switch op { + case syntax.EQL: + return rangeEqual(x, y), nil + case syntax.NEQ: + return !rangeEqual(x, y), nil + default: + return false, fmt.Errorf("%s %s %s not implemented", x.Type(), op, y.Type()) + } +} + +func rangeEqual(x, y rangeValue) bool { + // Two ranges compare equal if they denote the same sequence. + if x.len != y.len { + return false // sequences differ in length + } + if x.len == 0 { + return true // both sequences are empty + } + if x.start != y.start { + return false // first element differs + } + return x.len == 1 || x.step == y.step +} + +func (r rangeValue) contains(x Int) bool { + x32, err := AsInt32(x) + if err != nil { + return false // out of range + } + delta := x32 - r.start + quo, rem := delta/r.step, delta%r.step + return rem == 0 && 0 <= quo && quo < r.len +} + +type rangeIterator struct { + r rangeValue + i int +} + +func (it *rangeIterator) Next(p *Value) bool { + if it.i < it.r.len { + *p = it.r.Index(it.i) + it.i++ + return true + } + return false +} +func (*rangeIterator) Done() {} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#repr +func repr(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var x Value + if err := UnpackPositionalArgs("repr", args, kwargs, 1, &x); err != nil { + return nil, err + } + return String(x.String()), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#reversed +func reversed(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var iterable Iterable + if err := UnpackPositionalArgs("reversed", args, kwargs, 1, &iterable); err != nil { + return nil, err + } + iter := iterable.Iterate() + defer iter.Done() + var elems []Value + if n := Len(args[0]); n >= 0 { + elems = make([]Value, 0, n) // preallocate if length known + } + var x Value + for iter.Next(&x) { + elems = append(elems, x) + } + n := len(elems) + for i := 0; i < n>>1; i++ { + elems[i], elems[n-1-i] = elems[n-1-i], elems[i] + } + return NewList(elems), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set +func set(thread *Thread, fn *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var iterable Iterable + if err := UnpackPositionalArgs("set", args, kwargs, 0, &iterable); err != nil { + return nil, err + } + set := new(Set) + if iterable != nil { + iter := iterable.Iterate() + defer iter.Done() + var x Value + for iter.Next(&x) { + if err := set.Insert(x); err != nil { + return nil, err + } + } + } + return set, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#sorted +func sorted(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var iterable Iterable + var key Callable + var reverse bool + if err := UnpackArgs("sorted", args, kwargs, + "iterable", &iterable, + "key?", &key, + "reverse?", &reverse, + ); err != nil { + return nil, err + } + + iter := iterable.Iterate() + defer iter.Done() + var values []Value + if n := Len(iterable); n > 0 { + values = make(Tuple, 0, n) // preallocate if length is known + } + var x Value + for iter.Next(&x) { + values = append(values, x) + } + + // Derive keys from values by applying key function. + var keys []Value + if key != nil { + keys = make([]Value, len(values)) + for i, v := range values { + k, err := Call(thread, key, Tuple{v}, nil) + if err != nil { + return nil, err // to preserve backtrace, don't modify error + } + keys[i] = k + } + } + + slice := &sortSlice{keys: keys, values: values} + if reverse { + sort.Stable(sort.Reverse(slice)) + } else { + sort.Stable(slice) + } + return NewList(slice.values), slice.err +} + +type sortSlice struct { + keys []Value // nil => values[i] is key + values []Value + err error +} + +func (s *sortSlice) Len() int { return len(s.values) } +func (s *sortSlice) Less(i, j int) bool { + keys := s.keys + if s.keys == nil { + keys = s.values + } + ok, err := Compare(syntax.LT, keys[i], keys[j]) + if err != nil { + s.err = err + } + return ok +} +func (s *sortSlice) Swap(i, j int) { + if s.keys != nil { + s.keys[i], s.keys[j] = s.keys[j], s.keys[i] + } + s.values[i], s.values[j] = s.values[j], s.values[i] +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#str +func str(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if len(kwargs) > 0 { + return nil, fmt.Errorf("str does not accept keyword arguments") + } + if len(args) != 1 { + return nil, fmt.Errorf("str: got %d arguments, want exactly 1", len(args)) + } + x := args[0] + if _, ok := AsString(x); !ok { + x = String(x.String()) + } + return x, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#tuple +func tuple(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + var iterable Iterable + if err := UnpackPositionalArgs("tuple", args, kwargs, 0, &iterable); err != nil { + return nil, err + } + if len(args) == 0 { + return Tuple(nil), nil + } + iter := iterable.Iterate() + defer iter.Done() + var elems Tuple + if n := Len(iterable); n > 0 { + elems = make(Tuple, 0, n) // preallocate if length is known + } + var x Value + for iter.Next(&x) { + elems = append(elems, x) + } + return elems, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#type +func type_(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if len(kwargs) > 0 { + return nil, fmt.Errorf("type does not accept keyword arguments") + } + if len(args) != 1 { + return nil, fmt.Errorf("type: got %d arguments, want exactly 1", len(args)) + } + return String(args[0].Type()), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#zip +func zip(thread *Thread, _ *Builtin, args Tuple, kwargs []Tuple) (Value, error) { + if len(kwargs) > 0 { + return nil, fmt.Errorf("zip does not accept keyword arguments") + } + rows, cols := 0, len(args) + iters := make([]Iterator, cols) + defer func() { + for _, iter := range iters { + if iter != nil { + iter.Done() + } + } + }() + for i, seq := range args { + it := Iterate(seq) + if it == nil { + return nil, fmt.Errorf("zip: argument #%d is not iterable: %s", i+1, seq.Type()) + } + iters[i] = it + n := Len(seq) + if i == 0 || n < rows { + rows = n // possibly -1 + } + } + var result []Value + if rows >= 0 { + // length known + result = make([]Value, rows) + array := make(Tuple, cols*rows) // allocate a single backing array + for i := 0; i < rows; i++ { + tuple := array[:cols:cols] + array = array[cols:] + for j, iter := range iters { + iter.Next(&tuple[j]) + } + result[i] = tuple + } + } else { + // length not known + outer: + for { + tuple := make(Tuple, cols) + for i, iter := range iters { + if !iter.Next(&tuple[i]) { + break outer + } + } + result = append(result, tuple) + } + } + return NewList(result), nil +} + +// ---- methods of built-in types --- + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·get +func dict_get(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + var key, dflt Value + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &key, &dflt); err != nil { + return nil, err + } + if v, ok, err := recv.(*Dict).Get(key); err != nil { + return nil, err + } else if ok { + return v, nil + } else if dflt != nil { + return dflt, nil + } + return None, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·clear +func dict_clear(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + return None, recv.(*Dict).Clear() +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·items +func dict_items(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + items := recv.(*Dict).Items() + res := make([]Value, len(items)) + for i, item := range items { + res[i] = item // convert [2]Value to Value + } + return NewList(res), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·keys +func dict_keys(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + return NewList(recv.(*Dict).Keys()), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·pop +func dict_pop(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := recv_.(*Dict) + var k, d Value + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &k, &d); err != nil { + return nil, err + } + if v, found, err := recv.Delete(k); err != nil { + return nil, err // dict is frozen or key is unhashable + } else if found { + return v, nil + } else if d != nil { + return d, nil + } + return nil, fmt.Errorf("pop: missing key") +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·popitem +func dict_popitem(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + recv := recv_.(*Dict) + k, ok := recv.ht.first() + if !ok { + return nil, fmt.Errorf("popitem: empty dict") + } + v, _, err := recv.Delete(k) + if err != nil { + return nil, err // dict is frozen + } + return Tuple{k, v}, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·setdefault +func dict_setdefault(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + var key, dflt Value = nil, None + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &key, &dflt); err != nil { + return nil, err + } + dict := recv.(*Dict) + if v, ok, err := dict.Get(key); err != nil { + return nil, err + } else if ok { + return v, nil + } else { + return dflt, dict.SetKey(key, dflt) + } +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·update +func dict_update(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + if len(args) > 1 { + return nil, fmt.Errorf("update: got %d arguments, want at most 1", len(args)) + } + if err := updateDict(recv.(*Dict), args, kwargs); err != nil { + return nil, fmt.Errorf("update: %v", err) + } + return None, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·update +func dict_values(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + items := recv.(*Dict).Items() + res := make([]Value, len(items)) + for i, item := range items { + res[i] = item[1] + } + return NewList(res), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#list·append +func list_append(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := recv_.(*List) + var object Value + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &object); err != nil { + return nil, err + } + if err := recv.checkMutable("append to"); err != nil { + return nil, err + } + recv.elems = append(recv.elems, object) + return None, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#list·clear +func list_clear(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + return None, recv_.(*List).Clear() +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#list·extend +func list_extend(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := recv_.(*List) + var iterable Iterable + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &iterable); err != nil { + return nil, err + } + if err := recv.checkMutable("extend"); err != nil { + return nil, err + } + listExtend(recv, iterable) + return None, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#list·index +func list_index(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := recv_.(*List) + var value, start_, end_ Value + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &value, &start_, &end_); err != nil { + return nil, err + } + + start, end, err := indices(start_, end_, recv.Len()) + if err != nil { + return nil, fmt.Errorf("%s: %s", fnname, err) + } + + for i := start; i < end; i++ { + if eq, err := Equal(recv.elems[i], value); err != nil { + return nil, fmt.Errorf("index: %s", err) + } else if eq { + return MakeInt(i), nil + } + } + return nil, fmt.Errorf("index: value not in list") +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#list·insert +func list_insert(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := recv_.(*List) + var index int + var object Value + if err := UnpackPositionalArgs(fnname, args, kwargs, 2, &index, &object); err != nil { + return nil, err + } + if err := recv.checkMutable("insert into"); err != nil { + return nil, err + } + + if index < 0 { + index += recv.Len() + } + + if index >= recv.Len() { + // end + recv.elems = append(recv.elems, object) + } else { + if index < 0 { + index = 0 // start + } + recv.elems = append(recv.elems, nil) + copy(recv.elems[index+1:], recv.elems[index:]) // slide up one + recv.elems[index] = object + } + return None, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#list·remove +func list_remove(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := recv_.(*List) + var value Value + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &value); err != nil { + return nil, err + } + if err := recv.checkMutable("remove from"); err != nil { + return nil, err + } + for i, elem := range recv.elems { + if eq, err := Equal(elem, value); err != nil { + return nil, fmt.Errorf("remove: %v", err) + } else if eq { + recv.elems = append(recv.elems[:i], recv.elems[i+1:]...) + return None, nil + } + } + return nil, fmt.Errorf("remove: element not found") +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#list·pop +func list_pop(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + list := recv.(*List) + n := list.Len() + i := n - 1 + if err := UnpackPositionalArgs(fnname, args, kwargs, 0, &i); err != nil { + return nil, err + } + origI := i + if i < 0 { + i += n + } + if i < 0 || i >= n { + return nil, outOfRange(origI, n, list) + } + if err := list.checkMutable("pop from"); err != nil { + return nil, err + } + res := list.elems[i] + list.elems = append(list.elems[:i], list.elems[i+1:]...) + return res, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·capitalize +func string_capitalize(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + s := string(recv.(String)) + var res bytes.Buffer + res.Grow(len(s)) + for i, r := range s { + if i == 0 { + r = unicode.ToTitle(r) + } else { + r = unicode.ToLower(r) + } + res.WriteRune(r) + } + return String(res.String()), nil +} + +// string_iterable returns an unspecified iterable value whose iterator yields: +// - elems: successive 1-byte substrings +// - codepoints: successive substrings that encode a single Unicode code point. +// - elem_ords: numeric values of successive bytes +// - codepoint_ords: numeric values of successive Unicode code points +func string_iterable(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + return stringIterable{ + s: recv.(String), + ords: fnname[len(fnname)-2] == 'd', + codepoints: fnname[0] == 'c', + }, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·count +func string_count(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := string(recv_.(String)) + + var sub string + var start_, end_ Value + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &sub, &start_, &end_); err != nil { + return nil, err + } + + start, end, err := indices(start_, end_, len(recv)) + if err != nil { + return nil, fmt.Errorf("%s: %s", fnname, err) + } + + var slice string + if start < end { + slice = recv[start:end] + } + return MakeInt(strings.Count(slice, sub)), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isalnum +func string_isalnum(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + recv := string(recv_.(String)) + for _, r := range recv { + if !unicode.IsLetter(r) && !unicode.IsDigit(r) { + return False, nil + } + } + return Bool(recv != ""), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isalpha +func string_isalpha(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + recv := string(recv_.(String)) + for _, r := range recv { + if !unicode.IsLetter(r) { + return False, nil + } + } + return Bool(recv != ""), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isdigit +func string_isdigit(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + recv := string(recv_.(String)) + for _, r := range recv { + if !unicode.IsDigit(r) { + return False, nil + } + } + return Bool(recv != ""), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·islower +func string_islower(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + recv := string(recv_.(String)) + return Bool(isCasedString(recv) && recv == strings.ToLower(recv)), nil +} + +// isCasedString reports whether its argument contains any cased code points. +func isCasedString(s string) bool { + for _, r := range s { + if isCasedRune(r) { + return true + } + } + return false +} + +func isCasedRune(r rune) bool { + // It's unclear what the correct behavior is for a rune such as 'ffi', + // a lowercase letter with no upper or title case and no SimpleFold. + return 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || unicode.SimpleFold(r) != r +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isspace +func string_isspace(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + recv := string(recv_.(String)) + for _, r := range recv { + if !unicode.IsSpace(r) { + return False, nil + } + } + return Bool(recv != ""), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·istitle +func string_istitle(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + recv := string(recv_.(String)) + + // Python semantics differ from x==strings.{To,}Title(x) in Go: + // "uppercase characters may only follow uncased characters and + // lowercase characters only cased ones." + var cased, prevCased bool + for _, r := range recv { + if 'A' <= r && r <= 'Z' || unicode.IsTitle(r) { // e.g. "Ç…" + if prevCased { + return False, nil + } + prevCased = true + cased = true + } else if unicode.IsLower(r) { + if !prevCased { + return False, nil + } + prevCased = true + cased = true + } else if unicode.IsUpper(r) { + return False, nil + } else { + prevCased = false + } + } + return Bool(cased), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isupper +func string_isupper(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + recv := string(recv_.(String)) + return Bool(isCasedString(recv) && recv == strings.ToUpper(recv)), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·find +func string_find(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + return string_find_impl(fnname, string(recv.(String)), args, kwargs, true, false) +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·format +func string_format(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + format := string(recv_.(String)) + var auto, manual bool // kinds of positional indexing used + var buf bytes.Buffer + index := 0 + for { + literal := format + i := strings.IndexByte(format, '{') + if i >= 0 { + literal = format[:i] + } + + // Replace "}}" with "}" in non-field portion, rejecting a lone '}'. + for { + j := strings.IndexByte(literal, '}') + if j < 0 { + buf.WriteString(literal) + break + } + if len(literal) == j+1 || literal[j+1] != '}' { + return nil, fmt.Errorf("single '}' in format") + } + buf.WriteString(literal[:j+1]) + literal = literal[j+2:] + } + + if i < 0 { + break // end of format string + } + + if i+1 < len(format) && format[i+1] == '{' { + // "{{" means a literal '{' + buf.WriteByte('{') + format = format[i+2:] + continue + } + + format = format[i+1:] + i = strings.IndexByte(format, '}') + if i < 0 { + return nil, fmt.Errorf("unmatched '{' in format") + } + + var arg Value + conv := "s" + var spec string + + field := format[:i] + format = format[i+1:] + + var name string + if i := strings.IndexByte(field, '!'); i < 0 { + // "name" or "name:spec" + if i := strings.IndexByte(field, ':'); i < 0 { + name = field + } else { + name = field[:i] + spec = field[i+1:] + } + } else { + // "name!conv" or "name!conv:spec" + name = field[:i] + field = field[i+1:] + // "conv" or "conv:spec" + if i := strings.IndexByte(field, ':'); i < 0 { + conv = field + } else { + conv = field[:i] + spec = field[i+1:] + } + } + + if name == "" { + // "{}": automatic indexing + if manual { + return nil, fmt.Errorf("cannot switch from manual field specification to automatic field numbering") + } + auto = true + if index >= len(args) { + return nil, fmt.Errorf("tuple index out of range") + } + arg = args[index] + index++ + } else if num, ok := decimal(name); ok { + // positional argument + if auto { + return nil, fmt.Errorf("cannot switch from automatic field numbering to manual field specification") + } + manual = true + if num >= len(args) { + return nil, fmt.Errorf("tuple index out of range") + } else { + arg = args[num] + } + } else { + // keyword argument + for _, kv := range kwargs { + if string(kv[0].(String)) == name { + arg = kv[1] + break + } + } + if arg == nil { + // Starlark does not support Python's x.y or a[i] syntaxes, + // or nested use of {...}. + if strings.Contains(name, ".") { + return nil, fmt.Errorf("attribute syntax x.y is not supported in replacement fields: %s", name) + } + if strings.Contains(name, "[") { + return nil, fmt.Errorf("element syntax a[i] is not supported in replacement fields: %s", name) + } + if strings.Contains(name, "{") { + return nil, fmt.Errorf("nested replacement fields not supported") + } + return nil, fmt.Errorf("keyword %s not found", name) + } + } + + if spec != "" { + // Starlark does not support Python's format_spec features. + return nil, fmt.Errorf("format spec features not supported in replacement fields: %s", spec) + } + + switch conv { + case "s": + if str, ok := AsString(arg); ok { + buf.WriteString(str) + } else { + writeValue(&buf, arg, nil) + } + case "r": + writeValue(&buf, arg, nil) + default: + return nil, fmt.Errorf("unknown conversion %q", conv) + } + } + return String(buf.String()), nil +} + +// decimal interprets s as a sequence of decimal digits. +func decimal(s string) (x int, ok bool) { + n := len(s) + for i := 0; i < n; i++ { + digit := s[i] - '0' + if digit > 9 { + return 0, false + } + x = x*10 + int(digit) + if x < 0 { + return 0, false // underflow + } + } + return x, true +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·index +func string_index(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + return string_find_impl(fnname, string(recv.(String)), args, kwargs, false, false) +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·join +func string_join(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := string(recv_.(String)) + var iterable Iterable + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &iterable); err != nil { + return nil, err + } + iter := iterable.Iterate() + defer iter.Done() + var buf bytes.Buffer + var x Value + for i := 0; iter.Next(&x); i++ { + if i > 0 { + buf.WriteString(recv) + } + s, ok := AsString(x) + if !ok { + return nil, fmt.Errorf("in list, want string, got %s", x.Type()) + } + buf.WriteString(s) + } + return String(buf.String()), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·lower +func string_lower(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + return String(strings.ToLower(string(recv.(String)))), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·partition +func string_partition(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := string(recv_.(String)) + var sep string + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &sep); err != nil { + return nil, err + } + if sep == "" { + return nil, fmt.Errorf("%s: empty separator", fnname) + } + var i int + if fnname[0] == 'p' { + i = strings.Index(recv, sep) // partition + } else { + i = strings.LastIndex(recv, sep) // rpartition + } + tuple := make(Tuple, 0, 3) + if i < 0 { + if fnname[0] == 'p' { + tuple = append(tuple, String(recv), String(""), String("")) + } else { + tuple = append(tuple, String(""), String(""), String(recv)) + } + } else { + tuple = append(tuple, String(recv[:i]), String(sep), String(recv[i+len(sep):])) + } + return tuple, nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·replace +func string_replace(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := string(recv_.(String)) + var old, new string + count := -1 + if err := UnpackPositionalArgs(fnname, args, kwargs, 2, &old, &new, &count); err != nil { + return nil, err + } + return String(strings.Replace(recv, old, new, count)), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rfind +func string_rfind(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + return string_find_impl(fnname, string(recv.(String)), args, kwargs, true, true) +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rindex +func string_rindex(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + return string_find_impl(fnname, string(recv.(String)), args, kwargs, false, true) +} + +// https://github.com/google/starlark-go/starlark/blob/master/doc/spec.md#string·startswith +// https://github.com/google/starlark-go/starlark/blob/master/doc/spec.md#string·endswith +func string_startswith(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + var x Value + var start, end Value = None, None + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &x, &start, &end); err != nil { + return nil, err + } + + // compute effective substring. + s := string(recv_.(String)) + if start, end, err := indices(start, end, len(s)); err != nil { + return nil, err + } else { + if end < start { + end = start // => empty result + } + s = s[start:end] + } + + f := strings.HasPrefix + if fnname[0] == 'e' { // endswith + f = strings.HasSuffix + } + + switch x := x.(type) { + case Tuple: + for i, x := range x { + prefix, ok := AsString(x) + if !ok { + return nil, fmt.Errorf("%s: want string, got %s, for element %d", + fnname, x.Type(), i) + } + if f(s, prefix) { + return True, nil + } + } + return False, nil + case String: + return Bool(f(s, string(x))), nil + } + return nil, fmt.Errorf("%s: got %s, want string or tuple of string", fnname, x.Type()) +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·strip +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·lstrip +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rstrip +func string_strip(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + var chars string + if err := UnpackPositionalArgs(fnname, args, kwargs, 0, &chars); err != nil { + return nil, err + } + recv := string(recv_.(String)) + var s string + switch fnname[0] { + case 's': // strip + if chars != "" { + s = strings.Trim(recv, chars) + } else { + s = strings.TrimSpace(recv) + } + case 'l': // lstrip + if chars != "" { + s = strings.TrimLeft(recv, chars) + } else { + s = strings.TrimLeftFunc(recv, unicode.IsSpace) + } + case 'r': // rstrip + if chars != "" { + s = strings.TrimRight(recv, chars) + } else { + s = strings.TrimRightFunc(recv, unicode.IsSpace) + } + } + return String(s), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·title +func string_title(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + + s := string(recv.(String)) + + // Python semantics differ from x==strings.{To,}Title(x) in Go: + // "uppercase characters may only follow uncased characters and + // lowercase characters only cased ones." + var buf bytes.Buffer + buf.Grow(len(s)) + var prevCased bool + for _, r := range s { + if prevCased { + r = unicode.ToLower(r) + } else { + r = unicode.ToTitle(r) + } + prevCased = isCasedRune(r) + buf.WriteRune(r) + } + return String(buf.String()), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·upper +func string_upper(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + if err := UnpackPositionalArgs(fnname, args, kwargs, 0); err != nil { + return nil, err + } + return String(strings.ToUpper(string(recv.(String)))), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·split +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rsplit +func string_split(fnname string, recv_ Value, args Tuple, kwargs []Tuple) (Value, error) { + recv := string(recv_.(String)) + var sep_ Value + maxsplit := -1 + if err := UnpackPositionalArgs(fnname, args, kwargs, 0, &sep_, &maxsplit); err != nil { + return nil, err + } + + var res []string + + if sep_ == nil || sep_ == None { + // special case: split on whitespace + if maxsplit < 0 { + res = strings.Fields(recv) + } else if fnname == "split" { + res = splitspace(recv, maxsplit) + } else { // rsplit + res = rsplitspace(recv, maxsplit) + } + + } else if sep, ok := AsString(sep_); ok { + if sep == "" { + return nil, fmt.Errorf("split: empty separator") + } + // usual case: split on non-empty separator + if maxsplit < 0 { + res = strings.Split(recv, sep) + } else if fnname == "split" { + res = strings.SplitN(recv, sep, maxsplit+1) + } else { // rsplit + res = strings.Split(recv, sep) + if excess := len(res) - maxsplit; excess > 0 { + res[0] = strings.Join(res[:excess], sep) + res = append(res[:1], res[excess:]...) + } + } + + } else { + return nil, fmt.Errorf("split: got %s for separator, want string", sep_.Type()) + } + + list := make([]Value, len(res)) + for i, x := range res { + list[i] = String(x) + } + return NewList(list), nil +} + +// Precondition: max >= 0. +func rsplitspace(s string, max int) []string { + res := make([]string, 0, max+1) + end := -1 // index of field end, or -1 in a region of spaces. + for i := len(s); i > 0; { + r, sz := utf8.DecodeLastRuneInString(s[:i]) + if unicode.IsSpace(r) { + if end >= 0 { + if len(res) == max { + break // let this field run to the start + } + res = append(res, s[i:end]) + end = -1 + } + } else if end < 0 { + end = i + } + i -= sz + } + if end >= 0 { + res = append(res, s[:end]) + } + + resLen := len(res) + for i := 0; i < resLen/2; i++ { + res[i], res[resLen-1-i] = res[resLen-1-i], res[i] + } + + return res +} + +// Precondition: max >= 0. +func splitspace(s string, max int) []string { + var res []string + start := -1 // index of field start, or -1 in a region of spaces + for i, r := range s { + if unicode.IsSpace(r) { + if start >= 0 { + if len(res) == max { + break // let this field run to the end + } + res = append(res, s[start:i]) + start = -1 + } + } else if start == -1 { + start = i + } + } + if start >= 0 { + res = append(res, s[start:]) + } + return res +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#string·splitlines +func string_splitlines(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + var keepends bool + if err := UnpackPositionalArgs(fnname, args, kwargs, 0, &keepends); err != nil { + return nil, err + } + var lines []string + if s := string(recv.(String)); s != "" { + // TODO(adonovan): handle CRLF correctly. + if keepends { + lines = strings.SplitAfter(s, "\n") + } else { + lines = strings.Split(s, "\n") + } + if strings.HasSuffix(s, "\n") { + lines = lines[:len(lines)-1] + } + } + list := make([]Value, len(lines)) + for i, x := range lines { + list[i] = String(x) + } + return NewList(list), nil +} + +// https://github.com/google/starlark-go/blob/master/doc/spec.md#set·union. +func set_union(fnname string, recv Value, args Tuple, kwargs []Tuple) (Value, error) { + var iterable Iterable + if err := UnpackPositionalArgs(fnname, args, kwargs, 0, &iterable); err != nil { + return nil, err + } + iter := iterable.Iterate() + defer iter.Done() + union, err := recv.(*Set).Union(iter) + if err != nil { + return nil, fmt.Errorf("union: %v", err) + } + return union, nil +} + +// Common implementation of string_{r}{find,index}. +func string_find_impl(fnname string, s string, args Tuple, kwargs []Tuple, allowError, last bool) (Value, error) { + var sub string + var start_, end_ Value + if err := UnpackPositionalArgs(fnname, args, kwargs, 1, &sub, &start_, &end_); err != nil { + return nil, err + } + + start, end, err := indices(start_, end_, len(s)) + if err != nil { + return nil, fmt.Errorf("%s: %s", fnname, err) + } + var slice string + if start < end { + slice = s[start:end] + } + + var i int + if last { + i = strings.LastIndex(slice, sub) + } else { + i = strings.Index(slice, sub) + } + if i < 0 { + if !allowError { + return nil, fmt.Errorf("substring not found") + } + return MakeInt(-1), nil + } + return MakeInt(i + start), nil +} + +// Common implementation of builtin dict function and dict.update method. +// Precondition: len(updates) == 0 or 1. +func updateDict(dict *Dict, updates Tuple, kwargs []Tuple) error { + if len(updates) == 1 { + switch updates := updates[0].(type) { + case IterableMapping: + // Iterate over dict's key/value pairs, not just keys. + for _, item := range updates.Items() { + if err := dict.SetKey(item[0], item[1]); err != nil { + return err // dict is frozen + } + } + default: + // all other sequences + iter := Iterate(updates) + if iter == nil { + return fmt.Errorf("got %s, want iterable", updates.Type()) + } + defer iter.Done() + var pair Value + for i := 0; iter.Next(&pair); i++ { + iter2 := Iterate(pair) + if iter2 == nil { + return fmt.Errorf("dictionary update sequence element #%d is not iterable (%s)", i, pair.Type()) + + } + defer iter2.Done() + len := Len(pair) + if len < 0 { + return fmt.Errorf("dictionary update sequence element #%d has unknown length (%s)", i, pair.Type()) + } else if len != 2 { + return fmt.Errorf("dictionary update sequence element #%d has length %d, want 2", i, len) + } + var k, v Value + iter2.Next(&k) + iter2.Next(&v) + if err := dict.SetKey(k, v); err != nil { + return err + } + } + } + } + + // Then add the kwargs. + before := dict.Len() + for _, pair := range kwargs { + if err := dict.SetKey(pair[0], pair[1]); err != nil { + return err // dict is frozen + } + } + // In the common case, each kwarg will add another dict entry. + // If that's not so, check whether it is because there was a duplicate kwarg. + if dict.Len() < before+len(kwargs) { + keys := make(map[String]bool, len(kwargs)) + for _, kv := range kwargs { + k := kv[0].(String) + if keys[k] { + return fmt.Errorf("duplicate keyword arg: %v", k) + } + keys[k] = true + } + } + + return nil +} diff --git a/vendor/go.starlark.net/starlark/value.go b/vendor/go.starlark.net/starlark/value.go new file mode 100644 index 00000000..f6974427 --- /dev/null +++ b/vendor/go.starlark.net/starlark/value.go @@ -0,0 +1,1252 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package starlark provides a Starlark interpreter. +// +// Starlark values are represented by the Value interface. +// The following built-in Value types are known to the evaluator: +// +// NoneType -- NoneType +// Bool -- bool +// Int -- int +// Float -- float +// String -- string +// *List -- list +// Tuple -- tuple +// *Dict -- dict +// *Set -- set +// *Function -- function (implemented in Starlark) +// *Builtin -- builtin_function_or_method (function or method implemented in Go) +// +// Client applications may define new data types that satisfy at least +// the Value interface. Such types may provide additional operations by +// implementing any of these optional interfaces: +// +// Callable -- value is callable like a function +// Comparable -- value defines its own comparison operations +// Iterable -- value is iterable using 'for' loops +// Sequence -- value is iterable sequence of known length +// Indexable -- value is sequence with efficient random access +// Mapping -- value maps from keys to values, like a dictionary +// HasBinary -- value defines binary operations such as * and + +// HasAttrs -- value has readable fields or methods x.f +// HasSetField -- value has settable fields x.f +// HasSetIndex -- value supports element update using x[i]=y +// HasSetKey -- value supports map update using x[k]=v +// HasUnary -- value defines unary operations such as + and - +// +// Client applications may also define domain-specific functions in Go +// and make them available to Starlark programs. Use NewBuiltin to +// construct a built-in value that wraps a Go function. The +// implementation of the Go function may use UnpackArgs to make sense of +// the positional and keyword arguments provided by the caller. +// +// Starlark's None value is not equal to Go's nil, but nil may be +// assigned to a Starlark Value. Be careful to avoid allowing Go nil +// values to leak into Starlark data structures. +// +// The Compare operation requires two arguments of the same +// type, but this constraint cannot be expressed in Go's type system. +// (This is the classic "binary method problem".) +// So, each Value type's CompareSameType method is a partial function +// that compares a value only against others of the same type. +// Use the package's standalone Compare (or Equal) function to compare +// an arbitrary pair of values. +// +// To parse and evaluate a Starlark source file, use ExecFile. The Eval +// function evaluates a single expression. All evaluator functions +// require a Thread parameter which defines the "thread-local storage" +// of a Starlark thread and may be used to plumb application state +// through Sklyark code and into callbacks. When evaluation fails it +// returns an EvalError from which the application may obtain a +// backtrace of active Starlark calls. +// +package starlark // import "go.starlark.net/starlark" + +// This file defines the data types of Starlark and their basic operations. + +import ( + "bytes" + "fmt" + "math" + "math/big" + "reflect" + "strconv" + "strings" + "unicode/utf8" + + "go.starlark.net/internal/compile" + "go.starlark.net/syntax" +) + +// Value is a value in the Starlark interpreter. +type Value interface { + // String returns the string representation of the value. + // Starlark string values are quoted as if by Python's repr. + String() string + + // Type returns a short string describing the value's type. + Type() string + + // Freeze causes the value, and all values transitively + // reachable from it through collections and closures, to be + // marked as frozen. All subsequent mutations to the data + // structure through this API will fail dynamically, making the + // data structure immutable and safe for publishing to other + // Starlark interpreters running concurrently. + Freeze() + + // Truth returns the truth value of an object. + Truth() Bool + + // Hash returns a function of x such that Equals(x, y) => Hash(x) == Hash(y). + // Hash may fail if the value's type is not hashable, or if the value + // contains a non-hashable value. + Hash() (uint32, error) +} + +// A Comparable is a value that defines its own equivalence relation and +// perhaps ordered comparisons. +type Comparable interface { + Value + // CompareSameType compares one value to another of the same Type(). + // The comparison operation must be one of EQL, NEQ, LT, LE, GT, or GE. + // CompareSameType returns an error if an ordered comparison was + // requested for a type that does not support it. + // + // Implementations that recursively compare subcomponents of + // the value should use the CompareDepth function, not Compare, to + // avoid infinite recursion on cyclic structures. + // + // The depth parameter is used to bound comparisons of cyclic + // data structures. Implementations should decrement depth + // before calling CompareDepth and should return an error if depth + // < 1. + // + // Client code should not call this method. Instead, use the + // standalone Compare or Equals functions, which are defined for + // all pairs of operands. + CompareSameType(op syntax.Token, y Value, depth int) (bool, error) +} + +var ( + _ Comparable = None + _ Comparable = Int{} + _ Comparable = False + _ Comparable = Float(0) + _ Comparable = String("") + _ Comparable = (*Dict)(nil) + _ Comparable = (*List)(nil) + _ Comparable = Tuple(nil) + _ Comparable = (*Set)(nil) +) + +// A Callable value f may be the operand of a function call, f(x). +// +// Clients should use the Call function, never the CallInternal method. +type Callable interface { + Value + Name() string + CallInternal(thread *Thread, args Tuple, kwargs []Tuple) (Value, error) +} + +var ( + _ Callable = (*Builtin)(nil) + _ Callable = (*Function)(nil) +) + +// An Iterable abstracts a sequence of values. +// An iterable value may be iterated over by a 'for' loop or used where +// any other Starlark iterable is allowed. Unlike a Sequence, the length +// of an Iterable is not necessarily known in advance of iteration. +type Iterable interface { + Value + Iterate() Iterator // must be followed by call to Iterator.Done +} + +// A Sequence is a sequence of values of known length. +type Sequence interface { + Iterable + Len() int +} + +var ( + _ Sequence = (*Dict)(nil) + _ Sequence = (*Set)(nil) +) + +// An Indexable is a sequence of known length that supports efficient random access. +// It is not necessarily iterable. +type Indexable interface { + Value + Index(i int) Value // requires 0 <= i < Len() + Len() int +} + +// A Sliceable is a sequence that can be cut into pieces with the slice operator (x[i:j:step]). +// +// All native indexable objects are sliceable. +// This is a separate interface for backwards-compatibility. +type Sliceable interface { + Indexable + // For positive strides (step > 0), 0 <= start <= end <= n. + // For negative strides (step < 0), -1 <= end <= start < n. + // The caller must ensure that the start and end indices are valid + // and that step is non-zero. + Slice(start, end, step int) Value +} + +// A HasSetIndex is an Indexable value whose elements may be assigned (x[i] = y). +// +// The implementation should not add Len to a negative index as the +// evaluator does this before the call. +type HasSetIndex interface { + Indexable + SetIndex(index int, v Value) error +} + +var ( + _ HasSetIndex = (*List)(nil) + _ Indexable = Tuple(nil) + _ Indexable = String("") + _ Sliceable = Tuple(nil) + _ Sliceable = String("") + _ Sliceable = (*List)(nil) +) + +// An Iterator provides a sequence of values to the caller. +// +// The caller must call Done when the iterator is no longer needed. +// Operations that modify a sequence will fail if it has active iterators. +// +// Example usage: +// +// iter := iterable.Iterator() +// defer iter.Done() +// var x Value +// for iter.Next(&x) { +// ... +// } +// +type Iterator interface { + // If the iterator is exhausted, Next returns false. + // Otherwise it sets *p to the current element of the sequence, + // advances the iterator, and returns true. + Next(p *Value) bool + Done() +} + +// A Mapping is a mapping from keys to values, such as a dictionary. +// +// If a type satisfies both Mapping and Iterable, the iterator yields +// the keys of the mapping. +type Mapping interface { + Value + // Get returns the value corresponding to the specified key, + // or !found if the mapping does not contain the key. + // + // Get also defines the behavior of "v in mapping". + // The 'in' operator reports the 'found' component, ignoring errors. + Get(Value) (v Value, found bool, err error) +} + +// An IterableMapping is a mapping that supports key enumeration. +type IterableMapping interface { + Mapping + Iterate() Iterator // see Iterable interface + Items() []Tuple // a new slice containing all key/value pairs +} + +var _ IterableMapping = (*Dict)(nil) + +// A HasSetKey supports map update using x[k]=v syntax, like a dictionary. +type HasSetKey interface { + Mapping + SetKey(k, v Value) error +} + +var _ HasSetKey = (*Dict)(nil) + +// A HasBinary value may be used as either operand of these binary operators: +// + - * / // % in not in | & ^ << >> +// +// The Side argument indicates whether the receiver is the left or right operand. +// +// An implementation may decline to handle an operation by returning (nil, nil). +// For this reason, clients should always call the standalone Binary(op, x, y) +// function rather than calling the method directly. +type HasBinary interface { + Value + Binary(op syntax.Token, y Value, side Side) (Value, error) +} + +type Side bool + +const ( + Left Side = false + Right Side = true +) + +// A HasUnary value may be used as the operand of these unary operators: +// + - ~ +// +// An implementation may decline to handle an operation by returning (nil, nil). +// For this reason, clients should always call the standalone Unary(op, x) +// function rather than calling the method directly. +type HasUnary interface { + Value + Unary(op syntax.Token) (Value, error) +} + +// A HasAttrs value has fields or methods that may be read by a dot expression (y = x.f). +// Attribute names may be listed using the built-in 'dir' function. +// +// For implementation convenience, a result of (nil, nil) from Attr is +// interpreted as a "no such field or method" error. Implementations are +// free to return a more precise error. +type HasAttrs interface { + Value + Attr(name string) (Value, error) // returns (nil, nil) if attribute not present + AttrNames() []string // callers must not modify the result. +} + +var ( + _ HasAttrs = String("") + _ HasAttrs = new(List) + _ HasAttrs = new(Dict) + _ HasAttrs = new(Set) +) + +// A HasSetField value has fields that may be written by a dot expression (x.f = y). +// +// An implementation of SetField may return a NoSuchAttrError, +// in which case the runtime may augment the error message to +// warn of possible misspelling. +type HasSetField interface { + HasAttrs + SetField(name string, val Value) error +} + +// A NoSuchAttrError may be returned by an implementation of +// HasAttrs.Attr or HasSetField.SetField to indicate that no such field +// exists. In that case the runtime may augment the error message to +// warn of possible misspelling. +type NoSuchAttrError string + +func (e NoSuchAttrError) Error() string { return string(e) } + +// NoneType is the type of None. Its only legal value is None. +// (We represent it as a number, not struct{}, so that None may be constant.) +type NoneType byte + +const None = NoneType(0) + +func (NoneType) String() string { return "None" } +func (NoneType) Type() string { return "NoneType" } +func (NoneType) Freeze() {} // immutable +func (NoneType) Truth() Bool { return False } +func (NoneType) Hash() (uint32, error) { return 0, nil } +func (NoneType) CompareSameType(op syntax.Token, y Value, depth int) (bool, error) { + return threeway(op, 0), nil +} + +// Bool is the type of a Starlark bool. +type Bool bool + +const ( + False Bool = false + True Bool = true +) + +func (b Bool) String() string { + if b { + return "True" + } else { + return "False" + } +} +func (b Bool) Type() string { return "bool" } +func (b Bool) Freeze() {} // immutable +func (b Bool) Truth() Bool { return b } +func (b Bool) Hash() (uint32, error) { return uint32(b2i(bool(b))), nil } +func (x Bool) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { + y := y_.(Bool) + return threeway(op, b2i(bool(x))-b2i(bool(y))), nil +} + +// Float is the type of a Starlark float. +type Float float64 + +func (f Float) String() string { return strconv.FormatFloat(float64(f), 'g', 6, 64) } +func (f Float) Type() string { return "float" } +func (f Float) Freeze() {} // immutable +func (f Float) Truth() Bool { return f != 0.0 } +func (f Float) Hash() (uint32, error) { + // Equal float and int values must yield the same hash. + // TODO(adonovan): opt: if f is non-integral, and thus not equal + // to any Int, we can avoid the Int conversion and use a cheaper hash. + if isFinite(float64(f)) { + return finiteFloatToInt(f).Hash() + } + return 1618033, nil // NaN, +/-Inf +} + +func floor(f Float) Float { return Float(math.Floor(float64(f))) } + +// isFinite reports whether f represents a finite rational value. +// It is equivalent to !math.IsNan(f) && !math.IsInf(f, 0). +func isFinite(f float64) bool { + return math.Abs(f) <= math.MaxFloat64 +} + +func (x Float) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { + y := y_.(Float) + switch op { + case syntax.EQL: + return x == y, nil + case syntax.NEQ: + return x != y, nil + case syntax.LE: + return x <= y, nil + case syntax.LT: + return x < y, nil + case syntax.GE: + return x >= y, nil + case syntax.GT: + return x > y, nil + } + panic(op) +} + +func (f Float) rational() *big.Rat { return new(big.Rat).SetFloat64(float64(f)) } + +// AsFloat returns the float64 value closest to x. +// The f result is undefined if x is not a float or int. +func AsFloat(x Value) (f float64, ok bool) { + switch x := x.(type) { + case Float: + return float64(x), true + case Int: + return float64(x.Float()), true + } + return 0, false +} + +func (x Float) Mod(y Float) Float { return Float(math.Mod(float64(x), float64(y))) } + +// Unary implements the operations +float and -float. +func (f Float) Unary(op syntax.Token) (Value, error) { + switch op { + case syntax.MINUS: + return -f, nil + case syntax.PLUS: + return +f, nil + } + return nil, nil +} + +// String is the type of a Starlark string. +// +// A String encapsulates an an immutable sequence of bytes, +// but strings are not directly iterable. Instead, iterate +// over the result of calling one of these four methods: +// codepoints, codepoint_ords, elems, elem_ords. +// +// Warning: the contract of the Value interface's String method is that +// it returns the value printed in Starlark notation, +// so s.String() or fmt.Sprintf("%s", s) returns a quoted string. +// Use string(s) or s.GoString() or fmt.Sprintf("%#v", s) to obtain the raw contents +// of a Starlark string as a Go string. +type String string + +func (s String) String() string { return strconv.Quote(string(s)) } +func (s String) GoString() string { return string(s) } +func (s String) Type() string { return "string" } +func (s String) Freeze() {} // immutable +func (s String) Truth() Bool { return len(s) > 0 } +func (s String) Hash() (uint32, error) { return hashString(string(s)), nil } +func (s String) Len() int { return len(s) } // bytes +func (s String) Index(i int) Value { return s[i : i+1] } + +func (s String) Slice(start, end, step int) Value { + if step == 1 { + return s[start:end] + } + + sign := signum(step) + var str []byte + for i := start; signum(end-i) == sign; i += step { + str = append(str, s[i]) + } + return String(str) +} + +func (s String) Attr(name string) (Value, error) { return builtinAttr(s, name, stringMethods) } +func (s String) AttrNames() []string { return builtinAttrNames(stringMethods) } + +func (x String) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { + y := y_.(String) + return threeway(op, strings.Compare(string(x), string(y))), nil +} + +func AsString(x Value) (string, bool) { v, ok := x.(String); return string(v), ok } + +// A stringIterable is an iterable whose iterator yields a sequence of +// either Unicode code points or elements (bytes), +// either numerically or as successive substrings. +type stringIterable struct { + s String + ords bool + codepoints bool +} + +var _ Iterable = (*stringIterable)(nil) + +func (si stringIterable) String() string { + var etype string + if si.codepoints { + etype = "codepoint" + } else { + etype = "elem" + } + if si.ords { + return si.s.String() + "." + etype + "_ords()" + } else { + return si.s.String() + "." + etype + "s()" + } +} +func (si stringIterable) Type() string { + if si.codepoints { + return "codepoints" + } else { + return "elems" + } +} +func (si stringIterable) Freeze() {} // immutable +func (si stringIterable) Truth() Bool { return True } +func (si stringIterable) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: %s", si.Type()) } +func (si stringIterable) Iterate() Iterator { return &stringIterator{si, 0} } + +type stringIterator struct { + si stringIterable + i int +} + +func (it *stringIterator) Next(p *Value) bool { + s := it.si.s[it.i:] + if s == "" { + return false + } + if it.si.codepoints { + r, sz := utf8.DecodeRuneInString(string(s)) + if !it.si.ords { + *p = s[:sz] + } else { + *p = MakeInt(int(r)) + } + it.i += sz + } else { + b := int(s[0]) + if !it.si.ords { + *p = s[:1] + } else { + *p = MakeInt(b) + } + it.i += 1 + } + return true +} + +func (*stringIterator) Done() {} + +// A Function is a function defined by a Starlark def statement or lambda expression. +// The initialization behavior of a Starlark module is also represented by a Function. +type Function struct { + funcode *compile.Funcode + defaults Tuple + freevars Tuple + + // These fields are shared by all functions in a module. + predeclared StringDict + globals []Value + constants []Value +} + +func (fn *Function) Name() string { return fn.funcode.Name } // "lambda" for anonymous functions +func (fn *Function) Doc() string { return fn.funcode.Doc } +func (fn *Function) Hash() (uint32, error) { return hashString(fn.funcode.Name), nil } +func (fn *Function) Freeze() { fn.defaults.Freeze(); fn.freevars.Freeze() } +func (fn *Function) String() string { return toString(fn) } +func (fn *Function) Type() string { return "function" } +func (fn *Function) Truth() Bool { return true } + +// Globals returns a new, unfrozen StringDict containing all global +// variables so far defined in the function's module. +func (fn *Function) Globals() StringDict { + m := make(StringDict, len(fn.funcode.Prog.Globals)) + for i, id := range fn.funcode.Prog.Globals { + if v := fn.globals[i]; v != nil { + m[id.Name] = v + } + } + return m +} + +func (fn *Function) Position() syntax.Position { return fn.funcode.Pos } +func (fn *Function) NumParams() int { return fn.funcode.NumParams } +func (fn *Function) NumKwonlyParams() int { return fn.funcode.NumKwonlyParams } + +// Param returns the name and position of the ith parameter, +// where 0 <= i < NumParams(). +// The *args and **kwargs parameters are at the end +// even if there were optional parameters after *args. +func (fn *Function) Param(i int) (string, syntax.Position) { + id := fn.funcode.Locals[i] + return id.Name, id.Pos +} +func (fn *Function) HasVarargs() bool { return fn.funcode.HasVarargs } +func (fn *Function) HasKwargs() bool { return fn.funcode.HasKwargs } + +// A Builtin is a function implemented in Go. +type Builtin struct { + name string + fn func(thread *Thread, fn *Builtin, args Tuple, kwargs []Tuple) (Value, error) + recv Value // for bound methods (e.g. "".startswith) +} + +func (b *Builtin) Name() string { return b.name } +func (b *Builtin) Freeze() { + if b.recv != nil { + b.recv.Freeze() + } +} +func (b *Builtin) Hash() (uint32, error) { + h := hashString(b.name) + if b.recv != nil { + h ^= 5521 + } + return h, nil +} +func (b *Builtin) Receiver() Value { return b.recv } +func (b *Builtin) String() string { return toString(b) } +func (b *Builtin) Type() string { return "builtin_function_or_method" } +func (b *Builtin) CallInternal(thread *Thread, args Tuple, kwargs []Tuple) (Value, error) { + return b.fn(thread, b, args, kwargs) +} +func (b *Builtin) Truth() Bool { return true } + +// NewBuiltin returns a new 'builtin_function_or_method' value with the specified name +// and implementation. It compares unequal with all other values. +func NewBuiltin(name string, fn func(thread *Thread, fn *Builtin, args Tuple, kwargs []Tuple) (Value, error)) *Builtin { + return &Builtin{name: name, fn: fn} +} + +// BindReceiver returns a new Builtin value representing a method +// closure, that is, a built-in function bound to a receiver value. +// +// In the example below, the value of f is the string.index +// built-in method bound to the receiver value "abc": +// +// f = "abc".index; f("a"); f("b") +// +// In the common case, the receiver is bound only during the call, +// but this still results in the creation of a temporary method closure: +// +// "abc".index("a") +// +func (b *Builtin) BindReceiver(recv Value) *Builtin { + return &Builtin{name: b.name, fn: b.fn, recv: recv} +} + +// A *Dict represents a Starlark dictionary. +type Dict struct { + ht hashtable +} + +func (d *Dict) Clear() error { return d.ht.clear() } +func (d *Dict) Delete(k Value) (v Value, found bool, err error) { return d.ht.delete(k) } +func (d *Dict) Get(k Value) (v Value, found bool, err error) { return d.ht.lookup(k) } +func (d *Dict) Items() []Tuple { return d.ht.items() } +func (d *Dict) Keys() []Value { return d.ht.keys() } +func (d *Dict) Len() int { return int(d.ht.len) } +func (d *Dict) Iterate() Iterator { return d.ht.iterate() } +func (d *Dict) SetKey(k, v Value) error { return d.ht.insert(k, v) } +func (d *Dict) String() string { return toString(d) } +func (d *Dict) Type() string { return "dict" } +func (d *Dict) Freeze() { d.ht.freeze() } +func (d *Dict) Truth() Bool { return d.Len() > 0 } +func (d *Dict) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable type: dict") } + +func (d *Dict) Attr(name string) (Value, error) { return builtinAttr(d, name, dictMethods) } +func (d *Dict) AttrNames() []string { return builtinAttrNames(dictMethods) } + +func (x *Dict) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { + y := y_.(*Dict) + switch op { + case syntax.EQL: + ok, err := dictsEqual(x, y, depth) + return ok, err + case syntax.NEQ: + ok, err := dictsEqual(x, y, depth) + return !ok, err + default: + return false, fmt.Errorf("%s %s %s not implemented", x.Type(), op, y.Type()) + } +} + +func dictsEqual(x, y *Dict, depth int) (bool, error) { + if x.Len() != y.Len() { + return false, nil + } + for _, xitem := range x.Items() { + key, xval := xitem[0], xitem[1] + + if yval, found, _ := y.Get(key); !found { + return false, nil + } else if eq, err := EqualDepth(xval, yval, depth-1); err != nil { + return false, err + } else if !eq { + return false, nil + } + } + return true, nil +} + +// A *List represents a Starlark list value. +type List struct { + elems []Value + frozen bool + itercount uint32 // number of active iterators (ignored if frozen) +} + +// NewList returns a list containing the specified elements. +// Callers should not subsequently modify elems. +func NewList(elems []Value) *List { return &List{elems: elems} } + +func (l *List) Freeze() { + if !l.frozen { + l.frozen = true + for _, elem := range l.elems { + elem.Freeze() + } + } +} + +// checkMutable reports an error if the list should not be mutated. +// verb+" list" should describe the operation. +func (l *List) checkMutable(verb string) error { + if l.frozen { + return fmt.Errorf("cannot %s frozen list", verb) + } + if l.itercount > 0 { + return fmt.Errorf("cannot %s list during iteration", verb) + } + return nil +} + +func (l *List) String() string { return toString(l) } +func (l *List) Type() string { return "list" } +func (l *List) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable type: list") } +func (l *List) Truth() Bool { return l.Len() > 0 } +func (l *List) Len() int { return len(l.elems) } +func (l *List) Index(i int) Value { return l.elems[i] } + +func (l *List) Slice(start, end, step int) Value { + if step == 1 { + elems := append([]Value{}, l.elems[start:end]...) + return NewList(elems) + } + + sign := signum(step) + var list []Value + for i := start; signum(end-i) == sign; i += step { + list = append(list, l.elems[i]) + } + return NewList(list) +} + +func (l *List) Attr(name string) (Value, error) { return builtinAttr(l, name, listMethods) } +func (l *List) AttrNames() []string { return builtinAttrNames(listMethods) } + +func (l *List) Iterate() Iterator { + if !l.frozen { + l.itercount++ + } + return &listIterator{l: l} +} + +func (x *List) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { + y := y_.(*List) + // It's tempting to check x == y as an optimization here, + // but wrong because a list containing NaN is not equal to itself. + return sliceCompare(op, x.elems, y.elems, depth) +} + +func sliceCompare(op syntax.Token, x, y []Value, depth int) (bool, error) { + // Fast path: check length. + if len(x) != len(y) && (op == syntax.EQL || op == syntax.NEQ) { + return op == syntax.NEQ, nil + } + + // Find first element that is not equal in both lists. + for i := 0; i < len(x) && i < len(y); i++ { + if eq, err := EqualDepth(x[i], y[i], depth-1); err != nil { + return false, err + } else if !eq { + switch op { + case syntax.EQL: + return false, nil + case syntax.NEQ: + return true, nil + default: + return CompareDepth(op, x[i], y[i], depth-1) + } + } + } + + return threeway(op, len(x)-len(y)), nil +} + +type listIterator struct { + l *List + i int +} + +func (it *listIterator) Next(p *Value) bool { + if it.i < it.l.Len() { + *p = it.l.elems[it.i] + it.i++ + return true + } + return false +} + +func (it *listIterator) Done() { + if !it.l.frozen { + it.l.itercount-- + } +} + +func (l *List) SetIndex(i int, v Value) error { + if err := l.checkMutable("assign to element of"); err != nil { + return err + } + l.elems[i] = v + return nil +} + +func (l *List) Append(v Value) error { + if err := l.checkMutable("append to"); err != nil { + return err + } + l.elems = append(l.elems, v) + return nil +} + +func (l *List) Clear() error { + if err := l.checkMutable("clear"); err != nil { + return err + } + for i := range l.elems { + l.elems[i] = nil // aid GC + } + l.elems = l.elems[:0] + return nil +} + +// A Tuple represents a Starlark tuple value. +type Tuple []Value + +func (t Tuple) Len() int { return len(t) } +func (t Tuple) Index(i int) Value { return t[i] } + +func (t Tuple) Slice(start, end, step int) Value { + if step == 1 { + return t[start:end] + } + + sign := signum(step) + var tuple Tuple + for i := start; signum(end-i) == sign; i += step { + tuple = append(tuple, t[i]) + } + return tuple +} + +func (t Tuple) Iterate() Iterator { return &tupleIterator{elems: t} } +func (t Tuple) Freeze() { + for _, elem := range t { + elem.Freeze() + } +} +func (t Tuple) String() string { return toString(t) } +func (t Tuple) Type() string { return "tuple" } +func (t Tuple) Truth() Bool { return len(t) > 0 } + +func (x Tuple) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { + y := y_.(Tuple) + return sliceCompare(op, x, y, depth) +} + +func (t Tuple) Hash() (uint32, error) { + // Use same algorithm as Python. + var x, mult uint32 = 0x345678, 1000003 + for _, elem := range t { + y, err := elem.Hash() + if err != nil { + return 0, err + } + x = x ^ y*mult + mult += 82520 + uint32(len(t)+len(t)) + } + return x, nil +} + +type tupleIterator struct{ elems Tuple } + +func (it *tupleIterator) Next(p *Value) bool { + if len(it.elems) > 0 { + *p = it.elems[0] + it.elems = it.elems[1:] + return true + } + return false +} + +func (it *tupleIterator) Done() {} + +// A Set represents a Starlark set value. +type Set struct { + ht hashtable // values are all None +} + +func (s *Set) Delete(k Value) (found bool, err error) { _, found, err = s.ht.delete(k); return } +func (s *Set) Clear() error { return s.ht.clear() } +func (s *Set) Has(k Value) (found bool, err error) { _, found, err = s.ht.lookup(k); return } +func (s *Set) Insert(k Value) error { return s.ht.insert(k, None) } +func (s *Set) Len() int { return int(s.ht.len) } +func (s *Set) Iterate() Iterator { return s.ht.iterate() } +func (s *Set) String() string { return toString(s) } +func (s *Set) Type() string { return "set" } +func (s *Set) elems() []Value { return s.ht.keys() } +func (s *Set) Freeze() { s.ht.freeze() } +func (s *Set) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable type: set") } +func (s *Set) Truth() Bool { return s.Len() > 0 } + +func (s *Set) Attr(name string) (Value, error) { return builtinAttr(s, name, setMethods) } +func (s *Set) AttrNames() []string { return builtinAttrNames(setMethods) } + +func (x *Set) CompareSameType(op syntax.Token, y_ Value, depth int) (bool, error) { + y := y_.(*Set) + switch op { + case syntax.EQL: + ok, err := setsEqual(x, y, depth) + return ok, err + case syntax.NEQ: + ok, err := setsEqual(x, y, depth) + return !ok, err + default: + return false, fmt.Errorf("%s %s %s not implemented", x.Type(), op, y.Type()) + } +} + +func setsEqual(x, y *Set, depth int) (bool, error) { + if x.Len() != y.Len() { + return false, nil + } + for _, elem := range x.elems() { + if found, _ := y.Has(elem); !found { + return false, nil + } + } + return true, nil +} + +func (s *Set) Union(iter Iterator) (Value, error) { + set := new(Set) + for _, elem := range s.elems() { + set.Insert(elem) // can't fail + } + var x Value + for iter.Next(&x) { + if err := set.Insert(x); err != nil { + return nil, err + } + } + return set, nil +} + +// toString returns the string form of value v. +// It may be more efficient than v.String() for larger values. +func toString(v Value) string { + var buf bytes.Buffer + writeValue(&buf, v, nil) + return buf.String() +} + +// writeValue writes x to out. +// +// path is used to detect cycles. +// It contains the list of *List and *Dict values we're currently printing. +// (These are the only potentially cyclic structures.) +// Callers should generally pass a temporary slice of length zero but non-zero capacity. +// It is safe to re-use the same path slice for multiple calls. +func writeValue(out *bytes.Buffer, x Value, path []Value) { + switch x := x.(type) { + case nil: + out.WriteString("") // indicates a bug + + case NoneType: + out.WriteString("None") + + case Int: + out.WriteString(x.String()) + + case Bool: + if x { + out.WriteString("True") + } else { + out.WriteString("False") + } + + case String: + fmt.Fprintf(out, "%q", string(x)) + + case *List: + out.WriteByte('[') + if pathContains(path, x) { + out.WriteString("...") // list contains itself + } else { + for i, elem := range x.elems { + if i > 0 { + out.WriteString(", ") + } + writeValue(out, elem, append(path, x)) + } + } + out.WriteByte(']') + + case Tuple: + out.WriteByte('(') + for i, elem := range x { + if i > 0 { + out.WriteString(", ") + } + writeValue(out, elem, path) + } + if len(x) == 1 { + out.WriteByte(',') + } + out.WriteByte(')') + + case *Function: + fmt.Fprintf(out, "", x.Name()) + + case *Builtin: + if x.recv != nil { + fmt.Fprintf(out, "", x.Name(), x.recv.Type()) + } else { + fmt.Fprintf(out, "", x.Name()) + } + + case *Dict: + out.WriteByte('{') + if pathContains(path, x) { + out.WriteString("...") // dict contains itself + } else { + sep := "" + for _, item := range x.Items() { + k, v := item[0], item[1] + out.WriteString(sep) + writeValue(out, k, path) + out.WriteString(": ") + writeValue(out, v, append(path, x)) // cycle check + sep = ", " + } + } + out.WriteByte('}') + + case *Set: + out.WriteString("set([") + for i, elem := range x.elems() { + if i > 0 { + out.WriteString(", ") + } + writeValue(out, elem, path) + } + out.WriteString("])") + + default: + out.WriteString(x.String()) + } +} + +func pathContains(path []Value, x Value) bool { + for _, y := range path { + if x == y { + return true + } + } + return false +} + +const maxdepth = 10 + +// Equal reports whether two Starlark values are equal. +func Equal(x, y Value) (bool, error) { + if x, ok := x.(String); ok { + return x == y, nil // fast path for an important special case + } + return EqualDepth(x, y, maxdepth) +} + +// EqualDepth reports whether two Starlark values are equal. +// +// Recursive comparisons by implementations of Value.CompareSameType +// should use EqualDepth to prevent infinite recursion. +func EqualDepth(x, y Value, depth int) (bool, error) { + return CompareDepth(syntax.EQL, x, y, depth) +} + +// Compare compares two Starlark values. +// The comparison operation must be one of EQL, NEQ, LT, LE, GT, or GE. +// Compare returns an error if an ordered comparison was +// requested for a type that does not support it. +// +// Recursive comparisons by implementations of Value.CompareSameType +// should use CompareDepth to prevent infinite recursion. +func Compare(op syntax.Token, x, y Value) (bool, error) { + return CompareDepth(op, x, y, maxdepth) +} + +// CompareDepth compares two Starlark values. +// The comparison operation must be one of EQL, NEQ, LT, LE, GT, or GE. +// CompareDepth returns an error if an ordered comparison was +// requested for a pair of values that do not support it. +// +// The depth parameter limits the maximum depth of recursion +// in cyclic data structures. +func CompareDepth(op syntax.Token, x, y Value, depth int) (bool, error) { + if depth < 1 { + return false, fmt.Errorf("comparison exceeded maximum recursion depth") + } + if sameType(x, y) { + if xcomp, ok := x.(Comparable); ok { + return xcomp.CompareSameType(op, y, depth) + } + + // use identity comparison + switch op { + case syntax.EQL: + return x == y, nil + case syntax.NEQ: + return x != y, nil + } + return false, fmt.Errorf("%s %s %s not implemented", x.Type(), op, y.Type()) + } + + // different types + + // int/float ordered comparisons + switch x := x.(type) { + case Int: + if y, ok := y.(Float); ok { + if y != y { + return false, nil // y is NaN + } + var cmp int + if !math.IsInf(float64(y), 0) { + cmp = x.rational().Cmp(y.rational()) // y is finite + } else if y > 0 { + cmp = -1 // y is +Inf + } else { + cmp = +1 // y is -Inf + } + return threeway(op, cmp), nil + } + case Float: + if y, ok := y.(Int); ok { + if x != x { + return false, nil // x is NaN + } + var cmp int + if !math.IsInf(float64(x), 0) { + cmp = x.rational().Cmp(y.rational()) // x is finite + } else if x > 0 { + cmp = -1 // x is +Inf + } else { + cmp = +1 // x is -Inf + } + return threeway(op, cmp), nil + } + } + + // All other values of different types compare unequal. + switch op { + case syntax.EQL: + return false, nil + case syntax.NEQ: + return true, nil + } + return false, fmt.Errorf("%s %s %s not implemented", x.Type(), op, y.Type()) +} + +func sameType(x, y Value) bool { + return reflect.TypeOf(x) == reflect.TypeOf(y) || x.Type() == y.Type() +} + +// threeway interprets a three-way comparison value cmp (-1, 0, +1) +// as a boolean comparison (e.g. x < y). +func threeway(op syntax.Token, cmp int) bool { + switch op { + case syntax.EQL: + return cmp == 0 + case syntax.NEQ: + return cmp != 0 + case syntax.LE: + return cmp <= 0 + case syntax.LT: + return cmp < 0 + case syntax.GE: + return cmp >= 0 + case syntax.GT: + return cmp > 0 + } + panic(op) +} + +func b2i(b bool) int { + if b { + return 1 + } else { + return 0 + } +} + +// Len returns the length of a string or sequence value, +// and -1 for all others. +// +// Warning: Len(x) >= 0 does not imply Iterate(x) != nil. +// A string has a known length but is not directly iterable. +func Len(x Value) int { + switch x := x.(type) { + case String: + return x.Len() + case Sequence: + return x.Len() + } + return -1 +} + +// Iterate return a new iterator for the value if iterable, nil otherwise. +// If the result is non-nil, the caller must call Done when finished with it. +// +// Warning: Iterate(x) != nil does not imply Len(x) >= 0. +// Some iterables may have unknown length. +func Iterate(x Value) Iterator { + if x, ok := x.(Iterable); ok { + return x.Iterate() + } + return nil +} diff --git a/vendor/go.starlark.net/syntax/parse.go b/vendor/go.starlark.net/syntax/parse.go new file mode 100644 index 00000000..65a18529 --- /dev/null +++ b/vendor/go.starlark.net/syntax/parse.go @@ -0,0 +1,1048 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +// This file defines a recursive-descent parser for Starlark. +// The LL(1) grammar of Starlark and the names of many productions follow Python 2.7. +// +// TODO(adonovan): use syntax.Error more systematically throughout the +// package. Verify that error positions are correct using the +// chunkedfile mechanism. + +import "log" + +// Enable this flag to print the token stream and log.Fatal on the first error. +const debug = false + +// A Mode value is a set of flags (or 0) that controls optional parser functionality. +type Mode uint + +const ( + RetainComments Mode = 1 << iota // retain comments in AST; see Node.Comments +) + +// Parse parses the input data and returns the corresponding parse tree. +// +// If src != nil, ParseFile parses the source from src and the filename +// is only used when recording position information. +// The type of the argument for the src parameter must be string, +// []byte, or io.Reader. +// If src == nil, ParseFile parses the file specified by filename. +func Parse(filename string, src interface{}, mode Mode) (f *File, err error) { + in, err := newScanner(filename, src, mode&RetainComments != 0) + if err != nil { + return nil, err + } + p := parser{in: in} + defer p.in.recover(&err) + + p.nextToken() // read first lookahead token + f = p.parseFile() + if f != nil { + f.Path = filename + } + p.assignComments(f) + return f, nil +} + +// ParseCompoundStmt parses a single compound statement: +// a blank line, a def, for, while, or if statement, or a +// semicolon-separated list of simple statements followed +// by a newline. These are the units on which the REPL operates. +// ParseCompoundStmt does not consume any following input. +// The parser calls the readline function each +// time it needs a new line of input. +func ParseCompoundStmt(filename string, readline func() ([]byte, error)) (f *File, err error) { + in, err := newScanner(filename, readline, false) + if err != nil { + return nil, err + } + + p := parser{in: in} + defer p.in.recover(&err) + + p.nextToken() // read first lookahead token + + var stmts []Stmt + switch p.tok { + case DEF, IF, FOR, WHILE: + stmts = p.parseStmt(stmts) + case NEWLINE: + // blank line + default: + stmts = p.parseSimpleStmt(stmts, false) + // Require but don't consume newline, to avoid blocking again. + if p.tok != NEWLINE { + p.in.errorf(p.in.pos, "invalid syntax") + } + } + + return &File{Path: filename, Stmts: stmts}, nil +} + +// ParseExpr parses a Starlark expression. +// See Parse for explanation of parameters. +func ParseExpr(filename string, src interface{}, mode Mode) (expr Expr, err error) { + in, err := newScanner(filename, src, mode&RetainComments != 0) + if err != nil { + return nil, err + } + p := parser{in: in} + defer p.in.recover(&err) + + p.nextToken() // read first lookahead token + + // TODO(adonovan): Python's eval would use the equivalent of + // parseExpr here, which permits an unparenthesized tuple. + // We should too. + expr = p.parseTest() + + // A following newline (e.g. "f()\n") appears outside any brackets, + // on a non-blank line, and thus results in a NEWLINE token. + if p.tok == NEWLINE { + p.nextToken() + } + + if p.tok != EOF { + p.in.errorf(p.in.pos, "got %#v after expression, want EOF", p.tok) + } + p.assignComments(expr) + return expr, nil +} + +type parser struct { + in *scanner + tok Token + tokval tokenValue +} + +// nextToken advances the scanner and returns the position of the +// previous token. +func (p *parser) nextToken() Position { + oldpos := p.tokval.pos + p.tok = p.in.nextToken(&p.tokval) + // enable to see the token stream + if debug { + log.Printf("nextToken: %-20s%+v\n", p.tok, p.tokval.pos) + } + return oldpos +} + +// file_input = (NEWLINE | stmt)* EOF +func (p *parser) parseFile() *File { + var stmts []Stmt + for p.tok != EOF { + if p.tok == NEWLINE { + p.nextToken() + continue + } + stmts = p.parseStmt(stmts) + } + return &File{Stmts: stmts} +} + +func (p *parser) parseStmt(stmts []Stmt) []Stmt { + if p.tok == DEF { + return append(stmts, p.parseDefStmt()) + } else if p.tok == IF { + return append(stmts, p.parseIfStmt()) + } else if p.tok == FOR { + return append(stmts, p.parseForStmt()) + } else if p.tok == WHILE { + return append(stmts, p.parseWhileStmt()) + } + return p.parseSimpleStmt(stmts, true) +} + +func (p *parser) parseDefStmt() Stmt { + defpos := p.nextToken() // consume DEF + id := p.parseIdent() + p.consume(LPAREN) + params := p.parseParams() + p.consume(RPAREN) + p.consume(COLON) + body := p.parseSuite() + return &DefStmt{ + Def: defpos, + Name: id, + Function: Function{ + StartPos: defpos, + Params: params, + Body: body, + }, + } +} + +func (p *parser) parseIfStmt() Stmt { + ifpos := p.nextToken() // consume IF + cond := p.parseTest() + p.consume(COLON) + body := p.parseSuite() + ifStmt := &IfStmt{ + If: ifpos, + Cond: cond, + True: body, + } + tail := ifStmt + for p.tok == ELIF { + elifpos := p.nextToken() // consume ELIF + cond := p.parseTest() + p.consume(COLON) + body := p.parseSuite() + elif := &IfStmt{ + If: elifpos, + Cond: cond, + True: body, + } + tail.ElsePos = elifpos + tail.False = []Stmt{elif} + tail = elif + } + if p.tok == ELSE { + tail.ElsePos = p.nextToken() // consume ELSE + p.consume(COLON) + tail.False = p.parseSuite() + } + return ifStmt +} + +func (p *parser) parseForStmt() Stmt { + forpos := p.nextToken() // consume FOR + vars := p.parseForLoopVariables() + p.consume(IN) + x := p.parseExpr(false) + p.consume(COLON) + body := p.parseSuite() + return &ForStmt{ + For: forpos, + Vars: vars, + X: x, + Body: body, + } +} + +func (p *parser) parseWhileStmt() Stmt { + whilepos := p.nextToken() // consume WHILE + cond := p.parseTest() + p.consume(COLON) + body := p.parseSuite() + return &WhileStmt{ + While: whilepos, + Cond: cond, + Body: body, + } +} + +// Equivalent to 'exprlist' production in Python grammar. +// +// loop_variables = primary_with_suffix (COMMA primary_with_suffix)* COMMA? +func (p *parser) parseForLoopVariables() Expr { + // Avoid parseExpr because it would consume the IN token + // following x in "for x in y: ...". + v := p.parsePrimaryWithSuffix() + if p.tok != COMMA { + return v + } + + list := []Expr{v} + for p.tok == COMMA { + p.nextToken() + if terminatesExprList(p.tok) { + break + } + list = append(list, p.parsePrimaryWithSuffix()) + } + return &TupleExpr{List: list} +} + +// simple_stmt = small_stmt (SEMI small_stmt)* SEMI? NEWLINE +// In REPL mode, it does not consume the NEWLINE. +func (p *parser) parseSimpleStmt(stmts []Stmt, consumeNL bool) []Stmt { + for { + stmts = append(stmts, p.parseSmallStmt()) + if p.tok != SEMI { + break + } + p.nextToken() // consume SEMI + if p.tok == NEWLINE || p.tok == EOF { + break + } + } + // EOF without NEWLINE occurs in `if x: pass`, for example. + if p.tok != EOF && consumeNL { + p.consume(NEWLINE) + } + + return stmts +} + +// small_stmt = RETURN expr? +// | PASS | BREAK | CONTINUE +// | LOAD ... +// | expr ('=' | '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=') expr // assign +// | expr +func (p *parser) parseSmallStmt() Stmt { + switch p.tok { + case RETURN: + pos := p.nextToken() // consume RETURN + var result Expr + if p.tok != EOF && p.tok != NEWLINE && p.tok != SEMI { + result = p.parseExpr(false) + } + return &ReturnStmt{Return: pos, Result: result} + + case BREAK, CONTINUE, PASS: + tok := p.tok + pos := p.nextToken() // consume it + return &BranchStmt{Token: tok, TokenPos: pos} + + case LOAD: + return p.parseLoadStmt() + } + + // Assignment + x := p.parseExpr(false) + switch p.tok { + case EQ, PLUS_EQ, MINUS_EQ, STAR_EQ, SLASH_EQ, SLASHSLASH_EQ, PERCENT_EQ, AMP_EQ, PIPE_EQ, CIRCUMFLEX_EQ, LTLT_EQ, GTGT_EQ: + op := p.tok + pos := p.nextToken() // consume op + rhs := p.parseExpr(false) + return &AssignStmt{OpPos: pos, Op: op, LHS: x, RHS: rhs} + } + + // Expression statement (e.g. function call, doc string). + return &ExprStmt{X: x} +} + +// stmt = LOAD '(' STRING {',' (IDENT '=')? STRING} [','] ')' +func (p *parser) parseLoadStmt() *LoadStmt { + loadPos := p.nextToken() // consume LOAD + lparen := p.consume(LPAREN) + + if p.tok != STRING { + p.in.errorf(p.in.pos, "first operand of load statement must be a string literal") + } + module := p.parsePrimary().(*Literal) + + var from, to []*Ident + for p.tok != RPAREN && p.tok != EOF { + p.consume(COMMA) + if p.tok == RPAREN { + break // allow trailing comma + } + switch p.tok { + case STRING: + // load("module", "id") + // To name is same as original. + lit := p.parsePrimary().(*Literal) + id := &Ident{ + NamePos: lit.TokenPos.add(`"`), + Name: lit.Value.(string), + } + to = append(to, id) + from = append(from, id) + + case IDENT: + // load("module", to="from") + id := p.parseIdent() + to = append(to, id) + if p.tok != EQ { + p.in.errorf(p.in.pos, `load operand must be "%[1]s" or %[1]s="originalname" (want '=' after %[1]s)`, id.Name) + } + p.consume(EQ) + if p.tok != STRING { + p.in.errorf(p.in.pos, `original name of loaded symbol must be quoted: %s="originalname"`, id.Name) + } + lit := p.parsePrimary().(*Literal) + from = append(from, &Ident{ + NamePos: lit.TokenPos.add(`"`), + Name: lit.Value.(string), + }) + + case RPAREN: + p.in.errorf(p.in.pos, "trailing comma in load statement") + + default: + p.in.errorf(p.in.pos, `load operand must be "name" or localname="name" (got %#v)`, p.tok) + } + } + rparen := p.consume(RPAREN) + + if len(to) == 0 { + p.in.errorf(lparen, "load statement must import at least 1 symbol") + } + return &LoadStmt{ + Load: loadPos, + Module: module, + To: to, + From: from, + Rparen: rparen, + } +} + +// suite is typically what follows a COLON (e.g. after DEF or FOR). +// suite = simple_stmt | NEWLINE INDENT stmt+ OUTDENT +func (p *parser) parseSuite() []Stmt { + if p.tok == NEWLINE { + p.nextToken() // consume NEWLINE + p.consume(INDENT) + var stmts []Stmt + for p.tok != OUTDENT && p.tok != EOF { + stmts = p.parseStmt(stmts) + } + p.consume(OUTDENT) + return stmts + } + + return p.parseSimpleStmt(nil, true) +} + +func (p *parser) parseIdent() *Ident { + if p.tok != IDENT { + p.in.error(p.in.pos, "not an identifier") + } + id := &Ident{ + NamePos: p.tokval.pos, + Name: p.tokval.raw, + } + p.nextToken() + return id +} + +func (p *parser) consume(t Token) Position { + if p.tok != t { + p.in.errorf(p.in.pos, "got %#v, want %#v", p.tok, t) + } + return p.nextToken() +} + +// params = (param COMMA)* param +// | +// +// param = IDENT +// | IDENT EQ test +// | STAR +// | STAR IDENT +// | STARSTAR IDENT +// +// parseParams parses a parameter list. The resulting expressions are of the form: +// +// *Ident x +// *Binary{Op: EQ, X: *Ident, Y: Expr} x=y +// *Unary{Op: STAR} * +// *Unary{Op: STAR, X: *Ident} *args +// *Unary{Op: STARSTAR, X: *Ident} **kwargs +func (p *parser) parseParams() []Expr { + var params []Expr + stars := false + for p.tok != RPAREN && p.tok != COLON && p.tok != EOF { + if len(params) > 0 { + p.consume(COMMA) + } + if p.tok == RPAREN { + // list can end with a COMMA if there is neither * nor ** + if stars { + p.in.errorf(p.in.pos, "got %#v, want parameter", p.tok) + } + break + } + + // * or *args or **kwargs + if p.tok == STAR || p.tok == STARSTAR { + stars = true + op := p.tok + pos := p.nextToken() + var x Expr + if op == STARSTAR || p.tok == IDENT { + x = p.parseIdent() + } + params = append(params, &UnaryExpr{ + OpPos: pos, + Op: op, + X: x, + }) + continue + } + + // IDENT + // IDENT = test + id := p.parseIdent() + if p.tok == EQ { // default value + eq := p.nextToken() + dflt := p.parseTest() + params = append(params, &BinaryExpr{ + X: id, + OpPos: eq, + Op: EQ, + Y: dflt, + }) + continue + } + + params = append(params, id) + } + return params +} + +// parseExpr parses an expression, possible consisting of a +// comma-separated list of 'test' expressions. +// +// In many cases we must use parseTest to avoid ambiguity such as +// f(x, y) vs. f((x, y)). +func (p *parser) parseExpr(inParens bool) Expr { + x := p.parseTest() + if p.tok != COMMA { + return x + } + + // tuple + exprs := p.parseExprs([]Expr{x}, inParens) + return &TupleExpr{List: exprs} +} + +// parseExprs parses a comma-separated list of expressions, starting with the comma. +// It is used to parse tuples and list elements. +// expr_list = (',' expr)* ','? +func (p *parser) parseExprs(exprs []Expr, allowTrailingComma bool) []Expr { + for p.tok == COMMA { + pos := p.nextToken() + if terminatesExprList(p.tok) { + if !allowTrailingComma { + p.in.error(pos, "unparenthesized tuple with trailing comma") + } + break + } + exprs = append(exprs, p.parseTest()) + } + return exprs +} + +// parseTest parses a 'test', a single-component expression. +func (p *parser) parseTest() Expr { + if p.tok == LAMBDA { + return p.parseLambda(true) + } + + x := p.parseTestPrec(0) + + // conditional expression (t IF cond ELSE f) + if p.tok == IF { + ifpos := p.nextToken() + cond := p.parseTestPrec(0) + if p.tok != ELSE { + p.in.error(ifpos, "conditional expression without else clause") + } + elsepos := p.nextToken() + else_ := p.parseTest() + return &CondExpr{If: ifpos, Cond: cond, True: x, ElsePos: elsepos, False: else_} + } + + return x +} + +// parseTestNoCond parses a a single-component expression without +// consuming a trailing 'if expr else expr'. +func (p *parser) parseTestNoCond() Expr { + if p.tok == LAMBDA { + return p.parseLambda(false) + } + return p.parseTestPrec(0) +} + +// parseLambda parses a lambda expression. +// The allowCond flag allows the body to be an 'a if b else c' conditional. +func (p *parser) parseLambda(allowCond bool) Expr { + lambda := p.nextToken() + var params []Expr + if p.tok != COLON { + params = p.parseParams() + } + p.consume(COLON) + + var body Expr + if allowCond { + body = p.parseTest() + } else { + body = p.parseTestNoCond() + } + + return &LambdaExpr{ + Lambda: lambda, + Function: Function{ + StartPos: lambda, + Params: params, + Body: []Stmt{&ReturnStmt{Result: body}}, + }, + } +} + +func (p *parser) parseTestPrec(prec int) Expr { + if prec >= len(preclevels) { + return p.parsePrimaryWithSuffix() + } + + // expr = NOT expr + if p.tok == NOT && prec == int(precedence[NOT]) { + pos := p.nextToken() + x := p.parseTestPrec(prec) + return &UnaryExpr{ + OpPos: pos, + Op: NOT, + X: x, + } + } + + return p.parseBinopExpr(prec) +} + +// expr = test (OP test)* +// Uses precedence climbing; see http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm#climbing. +func (p *parser) parseBinopExpr(prec int) Expr { + x := p.parseTestPrec(prec + 1) + for first := true; ; first = false { + if p.tok == NOT { + p.nextToken() // consume NOT + // In this context, NOT must be followed by IN. + // Replace NOT IN by a single NOT_IN token. + if p.tok != IN { + p.in.errorf(p.in.pos, "got %#v, want in", p.tok) + } + p.tok = NOT_IN + } + + // Binary operator of specified precedence? + opprec := int(precedence[p.tok]) + if opprec < prec { + return x + } + + // Comparisons are non-associative. + if !first && opprec == int(precedence[EQL]) { + p.in.errorf(p.in.pos, "%s does not associate with %s (use parens)", + x.(*BinaryExpr).Op, p.tok) + } + + op := p.tok + pos := p.nextToken() + y := p.parseTestPrec(opprec + 1) + x = &BinaryExpr{OpPos: pos, Op: op, X: x, Y: y} + } +} + +// precedence maps each operator to its precedence (0-7), or -1 for other tokens. +var precedence [maxToken]int8 + +// preclevels groups operators of equal precedence. +// Comparisons are nonassociative; other binary operators associate to the left. +// Unary MINUS, unary PLUS, and TILDE have higher precedence so are handled in parsePrimary. +// See https://github.com/google/starlark-go/blob/master/doc/spec.md#binary-operators +var preclevels = [...][]Token{ + {OR}, // or + {AND}, // and + {NOT}, // not (unary) + {EQL, NEQ, LT, GT, LE, GE, IN, NOT_IN}, // == != < > <= >= in not in + {PIPE}, // | + {CIRCUMFLEX}, // ^ + {AMP}, // & + {LTLT, GTGT}, // << >> + {MINUS, PLUS}, // - + {STAR, PERCENT, SLASH, SLASHSLASH}, // * % / // +} + +func init() { + // populate precedence table + for i := range precedence { + precedence[i] = -1 + } + for level, tokens := range preclevels { + for _, tok := range tokens { + precedence[tok] = int8(level) + } + } +} + +// primary_with_suffix = primary +// | primary '.' IDENT +// | primary slice_suffix +// | primary call_suffix +func (p *parser) parsePrimaryWithSuffix() Expr { + x := p.parsePrimary() + for { + switch p.tok { + case DOT: + dot := p.nextToken() + id := p.parseIdent() + x = &DotExpr{Dot: dot, X: x, Name: id} + case LBRACK: + x = p.parseSliceSuffix(x) + case LPAREN: + x = p.parseCallSuffix(x) + default: + return x + } + } +} + +// slice_suffix = '[' expr? ':' expr? ':' expr? ']' +func (p *parser) parseSliceSuffix(x Expr) Expr { + lbrack := p.nextToken() + var lo, hi, step Expr + if p.tok != COLON { + y := p.parseExpr(false) + + // index x[y] + if p.tok == RBRACK { + rbrack := p.nextToken() + return &IndexExpr{X: x, Lbrack: lbrack, Y: y, Rbrack: rbrack} + } + + lo = y + } + + // slice or substring x[lo:hi:step] + if p.tok == COLON { + p.nextToken() + if p.tok != COLON && p.tok != RBRACK { + hi = p.parseTest() + } + } + if p.tok == COLON { + p.nextToken() + if p.tok != RBRACK { + step = p.parseTest() + } + } + rbrack := p.consume(RBRACK) + return &SliceExpr{X: x, Lbrack: lbrack, Lo: lo, Hi: hi, Step: step, Rbrack: rbrack} +} + +// call_suffix = '(' arg_list? ')' +func (p *parser) parseCallSuffix(fn Expr) Expr { + lparen := p.consume(LPAREN) + var rparen Position + var args []Expr + if p.tok == RPAREN { + rparen = p.nextToken() + } else { + args = p.parseArgs() + rparen = p.consume(RPAREN) + } + return &CallExpr{Fn: fn, Lparen: lparen, Args: args, Rparen: rparen} +} + +// parseArgs parses a list of actual parameter values (arguments). +// It mirrors the structure of parseParams. +// arg_list = ((arg COMMA)* arg COMMA?)? +func (p *parser) parseArgs() []Expr { + var args []Expr + stars := false + for p.tok != RPAREN && p.tok != EOF { + if len(args) > 0 { + p.consume(COMMA) + } + if p.tok == RPAREN { + // list can end with a COMMA if there is neither * nor ** + if stars { + p.in.errorf(p.in.pos, `got %#v, want argument`, p.tok) + } + break + } + + // *args or **kwargs + if p.tok == STAR || p.tok == STARSTAR { + stars = true + op := p.tok + pos := p.nextToken() + x := p.parseTest() + args = append(args, &UnaryExpr{ + OpPos: pos, + Op: op, + X: x, + }) + continue + } + + // We use a different strategy from Bazel here to stay within LL(1). + // Instead of looking ahead two tokens (IDENT, EQ) we parse + // 'test = test' then check that the first was an IDENT. + x := p.parseTest() + + if p.tok == EQ { + // name = value + if _, ok := x.(*Ident); !ok { + p.in.errorf(p.in.pos, "keyword argument must have form name=expr") + } + eq := p.nextToken() + y := p.parseTest() + x = &BinaryExpr{ + X: x, + OpPos: eq, + Op: EQ, + Y: y, + } + } + + args = append(args, x) + } + return args +} + +// primary = IDENT +// | INT | FLOAT +// | STRING +// | '[' ... // list literal or comprehension +// | '{' ... // dict literal or comprehension +// | '(' ... // tuple or parenthesized expression +// | ('-'|'+'|'~') primary_with_suffix +func (p *parser) parsePrimary() Expr { + switch p.tok { + case IDENT: + return p.parseIdent() + + case INT, FLOAT, STRING: + var val interface{} + tok := p.tok + switch tok { + case INT: + if p.tokval.bigInt != nil { + val = p.tokval.bigInt + } else { + val = p.tokval.int + } + case FLOAT: + val = p.tokval.float + case STRING: + val = p.tokval.string + } + raw := p.tokval.raw + pos := p.nextToken() + return &Literal{Token: tok, TokenPos: pos, Raw: raw, Value: val} + + case LBRACK: + return p.parseList() + + case LBRACE: + return p.parseDict() + + case LPAREN: + lparen := p.nextToken() + if p.tok == RPAREN { + // empty tuple + rparen := p.nextToken() + return &TupleExpr{Lparen: lparen, Rparen: rparen} + } + e := p.parseExpr(true) // allow trailing comma + rparen := p.consume(RPAREN) + return &ParenExpr{ + Lparen: lparen, + X: e, + Rparen: rparen, + } + + case MINUS, PLUS, TILDE: // unary + tok := p.tok + pos := p.nextToken() + x := p.parsePrimaryWithSuffix() + return &UnaryExpr{ + OpPos: pos, + Op: tok, + X: x, + } + } + p.in.errorf(p.in.pos, "got %#v, want primary expression", p.tok) + panic("unreachable") +} + +// list = '[' ']' +// | '[' expr ']' +// | '[' expr expr_list ']' +// | '[' expr (FOR loop_variables IN expr)+ ']' +func (p *parser) parseList() Expr { + lbrack := p.nextToken() + if p.tok == RBRACK { + // empty List + rbrack := p.nextToken() + return &ListExpr{Lbrack: lbrack, Rbrack: rbrack} + } + + x := p.parseTest() + + if p.tok == FOR { + // list comprehension + return p.parseComprehensionSuffix(lbrack, x, RBRACK) + } + + exprs := []Expr{x} + if p.tok == COMMA { + // multi-item list literal + exprs = p.parseExprs(exprs, true) // allow trailing comma + } + + rbrack := p.consume(RBRACK) + return &ListExpr{Lbrack: lbrack, List: exprs, Rbrack: rbrack} +} + +// dict = '{' '}' +// | '{' dict_entry_list '}' +// | '{' dict_entry FOR loop_variables IN expr '}' +func (p *parser) parseDict() Expr { + lbrace := p.nextToken() + if p.tok == RBRACE { + // empty dict + rbrace := p.nextToken() + return &DictExpr{Lbrace: lbrace, Rbrace: rbrace} + } + + x := p.parseDictEntry() + + if p.tok == FOR { + // dict comprehension + return p.parseComprehensionSuffix(lbrace, x, RBRACE) + } + + entries := []Expr{x} + for p.tok == COMMA { + p.nextToken() + if p.tok == RBRACE { + break + } + entries = append(entries, p.parseDictEntry()) + } + + rbrace := p.consume(RBRACE) + return &DictExpr{Lbrace: lbrace, List: entries, Rbrace: rbrace} +} + +// dict_entry = test ':' test +func (p *parser) parseDictEntry() *DictEntry { + k := p.parseTest() + colon := p.consume(COLON) + v := p.parseTest() + return &DictEntry{Key: k, Colon: colon, Value: v} +} + +// comp_suffix = FOR loopvars IN expr comp_suffix +// | IF expr comp_suffix +// | ']' or ')' (end) +// +// There can be multiple FOR/IF clauses; the first is always a FOR. +func (p *parser) parseComprehensionSuffix(lbrace Position, body Expr, endBrace Token) Expr { + var clauses []Node + for p.tok != endBrace { + if p.tok == FOR { + pos := p.nextToken() + vars := p.parseForLoopVariables() + in := p.consume(IN) + // Following Python 3, the operand of IN cannot be: + // - a conditional expression ('x if y else z'), + // due to conflicts in Python grammar + // ('if' is used by the comprehension); + // - a lambda expression + // - an unparenthesized tuple. + x := p.parseTestPrec(0) + clauses = append(clauses, &ForClause{For: pos, Vars: vars, In: in, X: x}) + } else if p.tok == IF { + pos := p.nextToken() + cond := p.parseTestNoCond() + clauses = append(clauses, &IfClause{If: pos, Cond: cond}) + } else { + p.in.errorf(p.in.pos, "got %#v, want '%s', for, or if", p.tok, endBrace) + } + } + rbrace := p.nextToken() + + return &Comprehension{ + Curly: endBrace == RBRACE, + Lbrack: lbrace, + Body: body, + Clauses: clauses, + Rbrack: rbrace, + } +} + +func terminatesExprList(tok Token) bool { + switch tok { + case EOF, NEWLINE, EQ, RBRACE, RBRACK, RPAREN, SEMI: + return true + } + return false +} + +// Comment assignment. +// We build two lists of all subnodes, preorder and postorder. +// The preorder list is ordered by start location, with outer nodes first. +// The postorder list is ordered by end location, with outer nodes last. +// We use the preorder list to assign each whole-line comment to the syntax +// immediately following it, and we use the postorder list to assign each +// end-of-line comment to the syntax immediately preceding it. + +// flattenAST returns the list of AST nodes, both in prefix order and in postfix +// order. +func flattenAST(root Node) (pre, post []Node) { + stack := []Node{} + Walk(root, func(n Node) bool { + if n != nil { + pre = append(pre, n) + stack = append(stack, n) + } else { + post = append(post, stack[len(stack)-1]) + stack = stack[:len(stack)-1] + } + return true + }) + return pre, post +} + +// assignComments attaches comments to nearby syntax. +func (p *parser) assignComments(n Node) { + // Leave early if there are no comments + if len(p.in.lineComments)+len(p.in.suffixComments) == 0 { + return + } + + pre, post := flattenAST(n) + + // Assign line comments to syntax immediately following. + line := p.in.lineComments + for _, x := range pre { + start, _ := x.Span() + + switch x.(type) { + case *File: + continue + } + + for len(line) > 0 && !start.isBefore(line[0].Start) { + x.AllocComments() + x.Comments().Before = append(x.Comments().Before, line[0]) + line = line[1:] + } + } + + // Remaining line comments go at end of file. + if len(line) > 0 { + n.AllocComments() + n.Comments().After = append(n.Comments().After, line...) + } + + // Assign suffix comments to syntax immediately before. + suffix := p.in.suffixComments + for i := len(post) - 1; i >= 0; i-- { + x := post[i] + + // Do not assign suffix comments to file + switch x.(type) { + case *File: + continue + } + + _, end := x.Span() + if len(suffix) > 0 && end.isBefore(suffix[len(suffix)-1].Start) { + x.AllocComments() + x.Comments().Suffix = append(x.Comments().Suffix, suffix[len(suffix)-1]) + suffix = suffix[:len(suffix)-1] + } + } +} diff --git a/vendor/go.starlark.net/syntax/quote.go b/vendor/go.starlark.net/syntax/quote.go new file mode 100644 index 00000000..d31eb2a7 --- /dev/null +++ b/vendor/go.starlark.net/syntax/quote.go @@ -0,0 +1,270 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +// Starlark quoted string utilities. + +import ( + "bytes" + "fmt" + "strconv" + "strings" +) + +// unesc maps single-letter chars following \ to their actual values. +var unesc = [256]byte{ + 'a': '\a', + 'b': '\b', + 'f': '\f', + 'n': '\n', + 'r': '\r', + 't': '\t', + 'v': '\v', + '\\': '\\', + '\'': '\'', + '"': '"', +} + +// esc maps escape-worthy bytes to the char that should follow \. +var esc = [256]byte{ + '\a': 'a', + '\b': 'b', + '\f': 'f', + '\n': 'n', + '\r': 'r', + '\t': 't', + '\v': 'v', + '\\': '\\', + '\'': '\'', + '"': '"', +} + +// notEsc is a list of characters that can follow a \ in a string value +// without having to escape the \. That is, since ( is in this list, we +// quote the Go string "foo\\(bar" as the Python literal "foo\(bar". +// This really does happen in BUILD files, especially in strings +// being used as shell arguments containing regular expressions. +const notEsc = " !#$%&()*+,-./:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~" + +// unquote unquotes the quoted string, returning the actual +// string value, whether the original was triple-quoted, and +// an error describing invalid input. +func unquote(quoted string) (s string, triple bool, err error) { + // Check for raw prefix: means don't interpret the inner \. + raw := false + if strings.HasPrefix(quoted, "r") { + raw = true + quoted = quoted[1:] + } + + if len(quoted) < 2 { + err = fmt.Errorf("string literal too short") + return + } + + if quoted[0] != '"' && quoted[0] != '\'' || quoted[0] != quoted[len(quoted)-1] { + err = fmt.Errorf("string literal has invalid quotes") + return + } + + // Check for triple quoted string. + quote := quoted[0] + if len(quoted) >= 6 && quoted[1] == quote && quoted[2] == quote && quoted[:3] == quoted[len(quoted)-3:] { + triple = true + quoted = quoted[3 : len(quoted)-3] + } else { + quoted = quoted[1 : len(quoted)-1] + } + + // Now quoted is the quoted data, but no quotes. + // If we're in raw mode or there are no escapes or + // carriage returns, we're done. + var unquoteChars string + if raw { + unquoteChars = "\r" + } else { + unquoteChars = "\\\r" + } + if !strings.ContainsAny(quoted, unquoteChars) { + s = quoted + return + } + + // Otherwise process quoted string. + // Each iteration processes one escape sequence along with the + // plain text leading up to it. + var buf bytes.Buffer + for { + // Remove prefix before escape sequence. + i := strings.IndexAny(quoted, unquoteChars) + if i < 0 { + i = len(quoted) + } + buf.WriteString(quoted[:i]) + quoted = quoted[i:] + + if len(quoted) == 0 { + break + } + + // Process carriage return. + if quoted[0] == '\r' { + buf.WriteByte('\n') + if len(quoted) > 1 && quoted[1] == '\n' { + quoted = quoted[2:] + } else { + quoted = quoted[1:] + } + continue + } + + // Process escape sequence. + if len(quoted) == 1 { + err = fmt.Errorf(`truncated escape sequence \`) + return + } + + switch quoted[1] { + default: + // In Python, if \z (for some byte z) is not a known escape sequence + // then it appears as literal text in the string. + buf.WriteString(quoted[:2]) + quoted = quoted[2:] + + case '\n': + // Ignore the escape and the line break. + quoted = quoted[2:] + + case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', '\'', '"': + // One-char escape + buf.WriteByte(unesc[quoted[1]]) + quoted = quoted[2:] + + case '0', '1', '2', '3', '4', '5', '6', '7': + // Octal escape, up to 3 digits. + n := int(quoted[1] - '0') + quoted = quoted[2:] + for i := 1; i < 3; i++ { + if len(quoted) == 0 || quoted[0] < '0' || '7' < quoted[0] { + break + } + n = n*8 + int(quoted[0]-'0') + quoted = quoted[1:] + } + if n >= 256 { + // NOTE: Python silently discards the high bit, + // so that '\541' == '\141' == 'a'. + // Let's see if we can avoid doing that in BUILD files. + err = fmt.Errorf(`invalid escape sequence \%03o`, n) + return + } + buf.WriteByte(byte(n)) + + case 'x': + // Hexadecimal escape, exactly 2 digits. + if len(quoted) < 4 { + err = fmt.Errorf(`truncated escape sequence %s`, quoted) + return + } + n, err1 := strconv.ParseUint(quoted[2:4], 16, 0) + if err1 != nil { + err = fmt.Errorf(`invalid escape sequence %s`, quoted[:4]) + return + } + buf.WriteByte(byte(n)) + quoted = quoted[4:] + } + } + + s = buf.String() + return +} + +// indexByte returns the index of the first instance of b in s, or else -1. +func indexByte(s string, b byte) int { + for i := 0; i < len(s); i++ { + if s[i] == b { + return i + } + } + return -1 +} + +// hex is a list of the hexadecimal digits, for use in quoting. +// We always print lower-case hexadecimal. +const hex = "0123456789abcdef" + +// quote returns the quoted form of the string value "x". +// If triple is true, quote uses the triple-quoted form """x""". +func quote(unquoted string, triple bool) string { + q := `"` + if triple { + q = `"""` + } + + var buf bytes.Buffer + buf.WriteString(q) + + for i := 0; i < len(unquoted); i++ { + c := unquoted[i] + if c == '"' && triple && (i+1 < len(unquoted) && unquoted[i+1] != '"' || i+2 < len(unquoted) && unquoted[i+2] != '"') { + // Can pass up to two quotes through, because they are followed by a non-quote byte. + buf.WriteByte(c) + if i+1 < len(unquoted) && unquoted[i+1] == '"' { + buf.WriteByte(c) + i++ + } + continue + } + if triple && c == '\n' { + // Can allow newline in triple-quoted string. + buf.WriteByte(c) + continue + } + if c == '\'' { + // Can allow ' since we always use ". + buf.WriteByte(c) + continue + } + if c == '\\' { + if i+1 < len(unquoted) && indexByte(notEsc, unquoted[i+1]) >= 0 { + // Can pass \ through when followed by a byte that + // known not to be a valid escape sequence and also + // that does not trigger an escape sequence of its own. + // Use this, because various BUILD files do. + buf.WriteByte('\\') + buf.WriteByte(unquoted[i+1]) + i++ + continue + } + } + if esc[c] != 0 { + buf.WriteByte('\\') + buf.WriteByte(esc[c]) + continue + } + if c < 0x20 || c >= 0x80 { + // BUILD files are supposed to be Latin-1, so escape all control and high bytes. + // I'd prefer to use \x here, but Blaze does not implement + // \x in quoted strings (b/7272572). + buf.WriteByte('\\') + buf.WriteByte(hex[c>>6]) // actually octal but reusing hex digits 0-7. + buf.WriteByte(hex[(c>>3)&7]) + buf.WriteByte(hex[c&7]) + /* + buf.WriteByte('\\') + buf.WriteByte('x') + buf.WriteByte(hex[c>>4]) + buf.WriteByte(hex[c&0xF]) + */ + continue + } + buf.WriteByte(c) + continue + } + + buf.WriteString(q) + return buf.String() +} diff --git a/vendor/go.starlark.net/syntax/scan.go b/vendor/go.starlark.net/syntax/scan.go new file mode 100644 index 00000000..7fbc5d68 --- /dev/null +++ b/vendor/go.starlark.net/syntax/scan.go @@ -0,0 +1,1092 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +// A lexical scanner for Starlark. + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "math/big" + "os" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// A Token represents a Starlark lexical token. +type Token int8 + +const ( + ILLEGAL Token = iota + EOF + + NEWLINE + INDENT + OUTDENT + + // Tokens with values + IDENT // x + INT // 123 + FLOAT // 1.23e45 + STRING // "foo" or 'foo' or '''foo''' or r'foo' or r"foo" + + // Punctuation + PLUS // + + MINUS // - + STAR // * + SLASH // / + SLASHSLASH // // + PERCENT // % + AMP // & + PIPE // | + CIRCUMFLEX // ^ + LTLT // << + GTGT // >> + TILDE // ~ + DOT // . + COMMA // , + EQ // = + SEMI // ; + COLON // : + LPAREN // ( + RPAREN // ) + LBRACK // [ + RBRACK // ] + LBRACE // { + RBRACE // } + LT // < + GT // > + GE // >= + LE // <= + EQL // == + NEQ // != + PLUS_EQ // += (keep order consistent with PLUS..GTGT) + MINUS_EQ // -= + STAR_EQ // *= + SLASH_EQ // /= + SLASHSLASH_EQ // //= + PERCENT_EQ // %= + AMP_EQ // &= + PIPE_EQ // |= + CIRCUMFLEX_EQ // ^= + LTLT_EQ // <<= + GTGT_EQ // >>= + STARSTAR // ** + + // Keywords + AND + BREAK + CONTINUE + DEF + ELIF + ELSE + FOR + IF + IN + LAMBDA + LOAD + NOT + NOT_IN // synthesized by parser from NOT IN + OR + PASS + RETURN + WHILE + + maxToken +) + +func (tok Token) String() string { return tokenNames[tok] } + +// GoString is like String but quotes punctuation tokens. +// Use Sprintf("%#v", tok) when constructing error messages. +func (tok Token) GoString() string { + if tok >= PLUS && tok <= STARSTAR { + return "'" + tokenNames[tok] + "'" + } + return tokenNames[tok] +} + +var tokenNames = [...]string{ + ILLEGAL: "illegal token", + EOF: "end of file", + NEWLINE: "newline", + INDENT: "indent", + OUTDENT: "outdent", + IDENT: "identifier", + INT: "int literal", + FLOAT: "float literal", + STRING: "string literal", + PLUS: "+", + MINUS: "-", + STAR: "*", + SLASH: "/", + SLASHSLASH: "//", + PERCENT: "%", + AMP: "&", + PIPE: "|", + CIRCUMFLEX: "^", + LTLT: "<<", + GTGT: ">>", + TILDE: "~", + DOT: ".", + COMMA: ",", + EQ: "=", + SEMI: ";", + COLON: ":", + LPAREN: "(", + RPAREN: ")", + LBRACK: "[", + RBRACK: "]", + LBRACE: "{", + RBRACE: "}", + LT: "<", + GT: ">", + GE: ">=", + LE: "<=", + EQL: "==", + NEQ: "!=", + PLUS_EQ: "+=", + MINUS_EQ: "-=", + STAR_EQ: "*=", + SLASH_EQ: "/=", + SLASHSLASH_EQ: "//=", + PERCENT_EQ: "%=", + AMP_EQ: "&=", + PIPE_EQ: "|=", + CIRCUMFLEX_EQ: "^=", + LTLT_EQ: "<<=", + GTGT_EQ: ">>=", + STARSTAR: "**", + AND: "and", + BREAK: "break", + CONTINUE: "continue", + DEF: "def", + ELIF: "elif", + ELSE: "else", + FOR: "for", + IF: "if", + IN: "in", + LAMBDA: "lambda", + LOAD: "load", + NOT: "not", + NOT_IN: "not in", + OR: "or", + PASS: "pass", + RETURN: "return", + WHILE: "while", +} + +// A Position describes the location of a rune of input. +type Position struct { + file *string // filename (indirect for compactness) + Line int32 // 1-based line number + Col int32 // 1-based column number (strictly: rune) +} + +// IsValid reports whether the position is valid. +func (p Position) IsValid() bool { + return p.Line >= 1 +} + +// Filename returns the name of the file containing this position. +func (p Position) Filename() string { + if p.file != nil { + return *p.file + } + return "" +} + +// MakePosition returns position with the specified components. +func MakePosition(file *string, line, col int32) Position { return Position{file, line, col} } + +// add returns the position at the end of s, assuming it starts at p. +func (p Position) add(s string) Position { + if n := strings.Count(s, "\n"); n > 0 { + p.Line += int32(n) + s = s[strings.LastIndex(s, "\n")+1:] + p.Col = 1 + } + p.Col += int32(utf8.RuneCountInString(s)) + return p +} + +func (p Position) String() string { + if p.Col > 0 { + return fmt.Sprintf("%s:%d:%d", p.Filename(), p.Line, p.Col) + } + return fmt.Sprintf("%s:%d", p.Filename(), p.Line) +} + +func (p Position) isBefore(q Position) bool { + if p.Line != q.Line { + return p.Line < q.Line + } + return p.Col < q.Col +} + +// An scanner represents a single input file being parsed. +type scanner struct { + rest []byte // rest of input (in REPL, a line of input) + token []byte // token being scanned + pos Position // current input position + depth int // nesting of [ ] { } ( ) + indentstk []int // stack of indentation levels + dents int // number of saved INDENT (>0) or OUTDENT (<0) tokens to return + lineStart bool // after NEWLINE; convert spaces to indentation tokens + keepComments bool // accumulate comments in slice + lineComments []Comment // list of full line comments (if keepComments) + suffixComments []Comment // list of suffix comments (if keepComments) + + readline func() ([]byte, error) // read next line of input (REPL only) +} + +func newScanner(filename string, src interface{}, keepComments bool) (*scanner, error) { + sc := &scanner{ + pos: Position{file: &filename, Line: 1, Col: 1}, + indentstk: make([]int, 1, 10), // []int{0} + spare capacity + lineStart: true, + keepComments: keepComments, + } + sc.readline, _ = src.(func() ([]byte, error)) // REPL only + if sc.readline == nil { + data, err := readSource(filename, src) + if err != nil { + return nil, err + } + sc.rest = data + } + return sc, nil +} + +func readSource(filename string, src interface{}) ([]byte, error) { + switch src := src.(type) { + case string: + return []byte(src), nil + case []byte: + return src, nil + case io.Reader: + data, err := ioutil.ReadAll(src) + if err != nil { + err = &os.PathError{Op: "read", Path: filename, Err: err} + } + return data, nil + case nil: + return ioutil.ReadFile(filename) + default: + return nil, fmt.Errorf("invalid source: %T", src) + } +} + +// An Error describes the nature and position of a scanner or parser error. +type Error struct { + Pos Position + Msg string +} + +func (e Error) Error() string { return e.Pos.String() + ": " + e.Msg } + +// errorf is called to report an error. +// errorf does not return: it panics. +func (sc *scanner) error(pos Position, s string) { + panic(Error{pos, s}) +} + +func (sc *scanner) errorf(pos Position, format string, args ...interface{}) { + sc.error(pos, fmt.Sprintf(format, args...)) +} + +func (sc *scanner) recover(err *error) { + // The scanner and parser panic both for routine errors like + // syntax errors and for programmer bugs like array index + // errors. Turn both into error returns. Catching bug panics + // is especially important when processing many files. + switch e := recover().(type) { + case nil: + // no panic + case Error: + *err = e + default: + *err = Error{sc.pos, fmt.Sprintf("internal error: %v", e)} + if debug { + log.Fatal(*err) + } + } +} + +// eof reports whether the input has reached end of file. +func (sc *scanner) eof() bool { + return len(sc.rest) == 0 && !sc.readLine() +} + +// readLine attempts to read another line of input. +// Precondition: len(sc.rest)==0. +func (sc *scanner) readLine() bool { + if sc.readline != nil { + var err error + sc.rest, err = sc.readline() + if err != nil { + sc.errorf(sc.pos, "%v", err) // EOF or ErrInterrupt + } + return len(sc.rest) > 0 + } + return false +} + +// peekRune returns the next rune in the input without consuming it. +// Newlines in Unix, DOS, or Mac format are treated as one rune, '\n'. +func (sc *scanner) peekRune() rune { + // TODO(adonovan): opt: measure and perhaps inline eof. + if sc.eof() { + return 0 + } + + // fast path: ASCII + if b := sc.rest[0]; b < utf8.RuneSelf { + if b == '\r' { + return '\n' + } + return rune(b) + } + + r, _ := utf8.DecodeRune(sc.rest) + return r +} + +// readRune consumes and returns the next rune in the input. +// Newlines in Unix, DOS, or Mac format are treated as one rune, '\n'. +func (sc *scanner) readRune() rune { + // eof() has been inlined here, both to avoid a call + // and to establish len(rest)>0 to avoid a bounds check. + if len(sc.rest) == 0 { + if !sc.readLine() { + sc.error(sc.pos, "internal scanner error: readRune at EOF") + } + // Redundant, but eliminates the bounds-check below. + if len(sc.rest) == 0 { + return 0 + } + } + + // fast path: ASCII + if b := sc.rest[0]; b < utf8.RuneSelf { + r := rune(b) + sc.rest = sc.rest[1:] + if r == '\r' { + if len(sc.rest) > 0 && sc.rest[0] == '\n' { + sc.rest = sc.rest[1:] + } + r = '\n' + } + if r == '\n' { + sc.pos.Line++ + sc.pos.Col = 1 + } else { + sc.pos.Col++ + } + return r + } + + r, size := utf8.DecodeRune(sc.rest) + sc.rest = sc.rest[size:] + sc.pos.Col++ + return r +} + +// tokenValue records the position and value associated with each token. +type tokenValue struct { + raw string // raw text of token + int int64 // decoded int + bigInt *big.Int // decoded integers > int64 + float float64 // decoded float + string string // decoded string + pos Position // start position of token +} + +// startToken marks the beginning of the next input token. +// It must be followed by a call to endToken once the token has +// been consumed using readRune. +func (sc *scanner) startToken(val *tokenValue) { + sc.token = sc.rest + val.raw = "" + val.pos = sc.pos +} + +// endToken marks the end of an input token. +// It records the actual token string in val.raw if the caller +// has not done that already. +func (sc *scanner) endToken(val *tokenValue) { + if val.raw == "" { + val.raw = string(sc.token[:len(sc.token)-len(sc.rest)]) + } +} + +// nextToken is called by the parser to obtain the next input token. +// It returns the token value and sets val to the data associated with +// the token. +// +// For all our input tokens, the associated data is val.pos (the +// position where the token begins), val.raw (the input string +// corresponding to the token). For string and int tokens, the string +// and int fields additionally contain the token's interpreted value. +func (sc *scanner) nextToken(val *tokenValue) Token { + + // The following distribution of tokens guides case ordering: + // + // COMMA 27 % + // STRING 23 % + // IDENT 15 % + // EQL 11 % + // LBRACK 5.5 % + // RBRACK 5.5 % + // NEWLINE 3 % + // LPAREN 2.9 % + // RPAREN 2.9 % + // INT 2 % + // others < 1 % + // + // Although NEWLINE tokens are infrequent, and lineStart is + // usually (~97%) false on entry, skipped newlines account for + // about 50% of all iterations of the 'start' loop. + +start: + var c rune + + // Deal with leading spaces and indentation. + blank := false + savedLineStart := sc.lineStart + if sc.lineStart { + sc.lineStart = false + col := 0 + for { + c = sc.peekRune() + if c == ' ' { + col++ + sc.readRune() + } else if c == '\t' { + const tab = 8 + col += int(tab - (sc.pos.Col-1)%tab) + sc.readRune() + } else { + break + } + } + + // The third clause matches EOF. + if c == '#' || c == '\n' || c == 0 { + blank = true + } + + // Compute indentation level for non-blank lines not + // inside an expression. This is not the common case. + if !blank && sc.depth == 0 { + cur := sc.indentstk[len(sc.indentstk)-1] + if col > cur { + // indent + sc.dents++ + sc.indentstk = append(sc.indentstk, col) + } else if col < cur { + // outdent(s) + for len(sc.indentstk) > 0 && col < sc.indentstk[len(sc.indentstk)-1] { + sc.dents-- + sc.indentstk = sc.indentstk[:len(sc.indentstk)-1] // pop + } + if col != sc.indentstk[len(sc.indentstk)-1] { + sc.error(sc.pos, "unindent does not match any outer indentation level") + } + } + } + } + + // Return saved indentation tokens. + if sc.dents != 0 { + sc.startToken(val) + sc.endToken(val) + if sc.dents < 0 { + sc.dents++ + return OUTDENT + } else { + sc.dents-- + return INDENT + } + } + + // start of line proper + c = sc.peekRune() + + // Skip spaces. + for c == ' ' || c == '\t' { + sc.readRune() + c = sc.peekRune() + } + + // comment + if c == '#' { + if sc.keepComments { + sc.startToken(val) + } + // Consume up to newline (included). + for c != 0 && c != '\n' { + sc.readRune() + c = sc.peekRune() + } + if sc.keepComments { + sc.endToken(val) + if blank { + sc.lineComments = append(sc.lineComments, Comment{val.pos, val.raw}) + } else { + sc.suffixComments = append(sc.suffixComments, Comment{val.pos, val.raw}) + } + } + } + + // newline + if c == '\n' { + sc.lineStart = true + + // Ignore newlines within expressions (common case). + if sc.depth > 0 { + sc.readRune() + goto start + } + + // Ignore blank lines, except in the REPL, + // where they emit OUTDENTs and NEWLINE. + if blank { + if sc.readline == nil { + sc.readRune() + goto start + } else if len(sc.indentstk) > 1 { + sc.dents = 1 - len(sc.indentstk) + sc.indentstk = sc.indentstk[:1] + goto start + } + } + + // At top-level (not in an expression). + sc.startToken(val) + sc.readRune() + val.raw = "\n" + return NEWLINE + } + + // end of file + if c == 0 { + // Emit OUTDENTs for unfinished indentation, + // preceded by a NEWLINE if we haven't just emitted one. + if len(sc.indentstk) > 1 { + if savedLineStart { + sc.dents = 1 - len(sc.indentstk) + sc.indentstk = sc.indentstk[:1] + goto start + } else { + sc.lineStart = true + sc.startToken(val) + val.raw = "\n" + return NEWLINE + } + } + + sc.startToken(val) + sc.endToken(val) + return EOF + } + + // line continuation + if c == '\\' { + sc.readRune() + if sc.peekRune() != '\n' { + sc.errorf(sc.pos, "stray backslash in program") + } + sc.readRune() + goto start + } + + // start of the next token + sc.startToken(val) + + // comma (common case) + if c == ',' { + sc.readRune() + sc.endToken(val) + return COMMA + } + + // string literal + if c == '"' || c == '\'' { + return sc.scanString(val, c) + } + + // identifier or keyword + if isIdentStart(c) { + // raw string literal + if c == 'r' && len(sc.rest) > 1 && (sc.rest[1] == '"' || sc.rest[1] == '\'') { + sc.readRune() + c = sc.peekRune() + return sc.scanString(val, c) + } + + for isIdent(c) { + sc.readRune() + c = sc.peekRune() + } + sc.endToken(val) + if k, ok := keywordToken[val.raw]; ok { + return k + } + + return IDENT + } + + // brackets + switch c { + case '[', '(', '{': + sc.depth++ + sc.readRune() + sc.endToken(val) + switch c { + case '[': + return LBRACK + case '(': + return LPAREN + case '{': + return LBRACE + } + panic("unreachable") + + case ']', ')', '}': + if sc.depth == 0 { + sc.errorf(sc.pos, "unexpected %q", c) + } else { + sc.depth-- + } + sc.readRune() + sc.endToken(val) + switch c { + case ']': + return RBRACK + case ')': + return RPAREN + case '}': + return RBRACE + } + panic("unreachable") + } + + // int or float literal, or period + if isdigit(c) || c == '.' { + return sc.scanNumber(val, c) + } + + // other punctuation + defer sc.endToken(val) + switch c { + case '=', '<', '>', '!', '+', '-', '%', '/', '&', '|', '^': // possibly followed by '=' + start := sc.pos + sc.readRune() + if sc.peekRune() == '=' { + sc.readRune() + switch c { + case '<': + return LE + case '>': + return GE + case '=': + return EQL + case '!': + return NEQ + case '+': + return PLUS_EQ + case '-': + return MINUS_EQ + case '/': + return SLASH_EQ + case '%': + return PERCENT_EQ + case '&': + return AMP_EQ + case '|': + return PIPE_EQ + case '^': + return CIRCUMFLEX_EQ + } + } + switch c { + case '=': + return EQ + case '<': + if sc.peekRune() == '<' { + sc.readRune() + if sc.peekRune() == '=' { + sc.readRune() + return LTLT_EQ + } else { + return LTLT + } + } + return LT + case '>': + if sc.peekRune() == '>' { + sc.readRune() + if sc.peekRune() == '=' { + sc.readRune() + return GTGT_EQ + } else { + return GTGT + } + } + return GT + case '!': + sc.error(start, "unexpected input character '!'") + case '+': + return PLUS + case '-': + return MINUS + case '/': + if sc.peekRune() == '/' { + sc.readRune() + if sc.peekRune() == '=' { + sc.readRune() + return SLASHSLASH_EQ + } else { + return SLASHSLASH + } + } + return SLASH + case '%': + return PERCENT + case '&': + return AMP + case '|': + return PIPE + case '^': + return CIRCUMFLEX + } + panic("unreachable") + + case ':', ';', '~': // single-char tokens (except comma) + sc.readRune() + switch c { + case ':': + return COLON + case ';': + return SEMI + case '~': + return TILDE + } + panic("unreachable") + + case '*': // possibly followed by '*' or '=' + sc.readRune() + switch sc.peekRune() { + case '*': + sc.readRune() + return STARSTAR + case '=': + sc.readRune() + return STAR_EQ + } + return STAR + } + + sc.errorf(sc.pos, "unexpected input character %#q", c) + panic("unreachable") +} + +func (sc *scanner) scanString(val *tokenValue, quote rune) Token { + start := sc.pos + triple := len(sc.rest) >= 3 && sc.rest[0] == byte(quote) && sc.rest[1] == byte(quote) && sc.rest[2] == byte(quote) + sc.readRune() + if !triple { + // Precondition: startToken was already called. + for { + if sc.eof() { + sc.error(val.pos, "unexpected EOF in string") + } + c := sc.readRune() + if c == quote { + break + } + if c == '\n' { + sc.error(val.pos, "unexpected newline in string") + } + if c == '\\' { + if sc.eof() { + sc.error(val.pos, "unexpected EOF in string") + } + sc.readRune() + } + } + sc.endToken(val) + } else { + // triple-quoted string literal + sc.readRune() + sc.readRune() + + // A triple-quoted string literal may span multiple + // gulps of REPL input; it is the only such token. + // Thus we must avoid {start,end}Token. + var raw bytes.Buffer + + // Copy the prefix, e.g. r''' or """ (see startToken). + raw.Write(sc.token[:len(sc.token)-len(sc.rest)]) + + quoteCount := 0 + for { + if sc.eof() { + sc.error(val.pos, "unexpected EOF in string") + } + c := sc.readRune() + raw.WriteRune(c) + if c == quote { + quoteCount++ + if quoteCount == 3 { + break + } + } else { + quoteCount = 0 + } + if c == '\\' { + if sc.eof() { + sc.error(val.pos, "unexpected EOF in string") + } + c = sc.readRune() + raw.WriteRune(c) + } + } + val.raw = raw.String() + } + + s, _, err := unquote(val.raw) + if err != nil { + sc.error(start, err.Error()) + } + val.string = s + return STRING +} + +func (sc *scanner) scanNumber(val *tokenValue, c rune) Token { + // https://github.com/google/starlark-go/blob/master/doc/spec.md#lexical-elements + // + // Python features not supported: + // - integer literals of >64 bits of precision + // - 123L or 123l long suffix + // - traditional octal: 0755 + // https://docs.python.org/2/reference/lexical_analysis.html#integer-and-long-integer-literals + + start := sc.pos + fraction, exponent := false, false + + if c == '.' { + // dot or start of fraction + sc.readRune() + c = sc.peekRune() + if !isdigit(c) { + sc.endToken(val) + return DOT + } + fraction = true + } else if c == '0' { + // hex, octal, binary or float + sc.readRune() + c = sc.peekRune() + + if c == '.' { + fraction = true + } else if c == 'x' || c == 'X' { + // hex + sc.readRune() + c = sc.peekRune() + if !isxdigit(c) { + sc.error(start, "invalid hex literal") + } + for isxdigit(c) { + sc.readRune() + c = sc.peekRune() + } + } else if c == 'o' || c == 'O' { + // octal + sc.readRune() + c = sc.peekRune() + if !isodigit(c) { + sc.error(sc.pos, "invalid octal literal") + } + for isodigit(c) { + sc.readRune() + c = sc.peekRune() + } + } else if c == 'b' || c == 'B' { + // binary + sc.readRune() + c = sc.peekRune() + if !isbdigit(c) { + sc.error(sc.pos, "invalid binary literal") + } + for isbdigit(c) { + sc.readRune() + c = sc.peekRune() + } + } else { + // float (or obsolete octal "0755") + allzeros, octal := true, true + for isdigit(c) { + if c != '0' { + allzeros = false + } + if c > '7' { + octal = false + } + sc.readRune() + c = sc.peekRune() + } + if c == '.' { + fraction = true + } else if c == 'e' || c == 'E' { + exponent = true + } else if octal && !allzeros { + // We must support old octal until the Java + // implementation groks the new one. + // TODO(adonovan): reenable the check. + if false { + sc.endToken(val) + sc.errorf(sc.pos, "obsolete form of octal literal; use 0o%s", val.raw[1:]) + } + } + } + } else { + // decimal + for isdigit(c) { + sc.readRune() + c = sc.peekRune() + } + + if c == '.' { + fraction = true + } else if c == 'e' || c == 'E' { + exponent = true + } + } + + if fraction { + sc.readRune() // consume '.' + c = sc.peekRune() + for isdigit(c) { + sc.readRune() + c = sc.peekRune() + } + + if c == 'e' || c == 'E' { + exponent = true + } + } + + if exponent { + sc.readRune() // consume [eE] + c = sc.peekRune() + if c == '+' || c == '-' { + sc.readRune() + c = sc.peekRune() + if !isdigit(c) { + sc.error(sc.pos, "invalid float literal") + } + } + for isdigit(c) { + sc.readRune() + c = sc.peekRune() + } + } + + sc.endToken(val) + if fraction || exponent { + var err error + val.float, err = strconv.ParseFloat(val.raw, 64) + if err != nil { + sc.error(sc.pos, "invalid float literal") + } + return FLOAT + } else { + var err error + s := val.raw + val.bigInt = nil + if len(s) > 2 && s[0] == '0' && (s[1] == 'o' || s[1] == 'O') { + val.int, err = strconv.ParseInt(s[2:], 8, 64) + } else if len(s) > 2 && s[0] == '0' && (s[1] == 'b' || s[1] == 'B') { + val.int, err = strconv.ParseInt(s[2:], 2, 64) + } else { + val.int, err = strconv.ParseInt(s, 0, 64) + if err != nil { + num := new(big.Int) + var ok bool = true + val.bigInt, ok = num.SetString(s, 0) + if ok { + err = nil + } + } + } + if err != nil { + sc.error(start, "invalid int literal") + } + return INT + } +} + +// isIdent reports whether c is an identifier rune. +func isIdent(c rune) bool { + return isdigit(c) || isIdentStart(c) +} + +func isIdentStart(c rune) bool { + return 'a' <= c && c <= 'z' || + 'A' <= c && c <= 'Z' || + c == '_' || + unicode.IsLetter(c) +} + +func isdigit(c rune) bool { return '0' <= c && c <= '9' } +func isodigit(c rune) bool { return '0' <= c && c <= '7' } +func isxdigit(c rune) bool { return isdigit(c) || 'A' <= c && c <= 'F' || 'a' <= c && c <= 'f' } +func isbdigit(c rune) bool { return '0' == c || c == '1' } + +// keywordToken records the special tokens for +// strings that should not be treated as ordinary identifiers. +var keywordToken = map[string]Token{ + "and": AND, + "break": BREAK, + "continue": CONTINUE, + "def": DEF, + "elif": ELIF, + "else": ELSE, + "for": FOR, + "if": IF, + "in": IN, + "lambda": LAMBDA, + "load": LOAD, + "not": NOT, + "or": OR, + "pass": PASS, + "return": RETURN, + "while": WHILE, + + // reserved words: + "as": ILLEGAL, + // "assert": ILLEGAL, // heavily used by our tests + "class": ILLEGAL, + "del": ILLEGAL, + "except": ILLEGAL, + "finally": ILLEGAL, + "from": ILLEGAL, + "global": ILLEGAL, + "import": ILLEGAL, + "is": ILLEGAL, + "nonlocal": ILLEGAL, + "raise": ILLEGAL, + "try": ILLEGAL, + "with": ILLEGAL, + "yield": ILLEGAL, +} diff --git a/vendor/go.starlark.net/syntax/syntax.go b/vendor/go.starlark.net/syntax/syntax.go new file mode 100644 index 00000000..4ed40668 --- /dev/null +++ b/vendor/go.starlark.net/syntax/syntax.go @@ -0,0 +1,548 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package syntax provides a Starlark parser and abstract syntax tree. +package syntax // import "go.starlark.net/syntax" + +// A Node is a node in a Starlark syntax tree. +type Node interface { + // Span returns the start and end position of the expression. + Span() (start, end Position) + + // Comments returns the comments associated with this node. + // It returns nil if RetainComments was not specified during parsing, + // or if AllocComments was not called. + Comments() *Comments + + // AllocComments allocates a new Comments node if there was none. + // This makes possible to add new comments using Comments() method. + AllocComments() +} + +// A Comment represents a single # comment. +type Comment struct { + Start Position + Text string // without trailing newline +} + +// Comments collects the comments associated with an expression. +type Comments struct { + Before []Comment // whole-line comments before this expression + Suffix []Comment // end-of-line comments after this expression (up to 1) + + // For top-level expressions only, After lists whole-line + // comments following the expression. + After []Comment +} + +// A commentsRef is a possibly-nil reference to a set of comments. +// A commentsRef is embedded in each type of syntax node, +// and provides its Comments and AllocComments methods. +type commentsRef struct{ ref *Comments } + +// Comments returns the comments associated with a syntax node, +// or nil if AllocComments has not yet been called. +func (cr commentsRef) Comments() *Comments { return cr.ref } + +// AllocComments enables comments to be associated with a syntax node. +func (cr *commentsRef) AllocComments() { + if cr.ref == nil { + cr.ref = new(Comments) + } +} + +// Start returns the start position of the expression. +func Start(n Node) Position { + start, _ := n.Span() + return start +} + +// End returns the end position of the expression. +func End(n Node) Position { + _, end := n.Span() + return end +} + +// A File represents a Starlark file. +type File struct { + commentsRef + Path string + Stmts []Stmt + + // set by resolver: + Locals []*Ident // this file's (comprehension-)local variables + Globals []*Ident // this file's global variables +} + +func (x *File) Span() (start, end Position) { + if len(x.Stmts) == 0 { + return + } + start, _ = x.Stmts[0].Span() + _, end = x.Stmts[len(x.Stmts)-1].Span() + return start, end +} + +// A Stmt is a Starlark statement. +type Stmt interface { + Node + stmt() +} + +func (*AssignStmt) stmt() {} +func (*BranchStmt) stmt() {} +func (*DefStmt) stmt() {} +func (*ExprStmt) stmt() {} +func (*ForStmt) stmt() {} +func (*WhileStmt) stmt() {} +func (*IfStmt) stmt() {} +func (*LoadStmt) stmt() {} +func (*ReturnStmt) stmt() {} + +// An AssignStmt represents an assignment: +// x = 0 +// x, y = y, x +// x += 1 +type AssignStmt struct { + commentsRef + OpPos Position + Op Token // = EQ | {PLUS,MINUS,STAR,PERCENT}_EQ + LHS Expr + RHS Expr +} + +func (x *AssignStmt) Span() (start, end Position) { + start, _ = x.LHS.Span() + _, end = x.RHS.Span() + return +} + +// A Function represents the common parts of LambdaExpr and DefStmt. +type Function struct { + commentsRef + StartPos Position // position of DEF or LAMBDA token + Params []Expr // param = ident | ident=expr | * | *ident | **ident + Body []Stmt + + // set by resolver: + HasVarargs bool // whether params includes *args (convenience) + HasKwargs bool // whether params includes **kwargs (convenience) + NumKwonlyParams int // number of keyword-only optional parameters + Locals []*Ident // this function's local variables, parameters first + FreeVars []*Ident // enclosing local variables to capture in closure +} + +func (x *Function) Span() (start, end Position) { + _, end = x.Body[len(x.Body)-1].Span() + return x.StartPos, end +} + +// A DefStmt represents a function definition. +type DefStmt struct { + commentsRef + Def Position + Name *Ident + Function +} + +func (x *DefStmt) Span() (start, end Position) { + _, end = x.Function.Body[len(x.Body)-1].Span() + return x.Def, end +} + +// An ExprStmt is an expression evaluated for side effects. +type ExprStmt struct { + commentsRef + X Expr +} + +func (x *ExprStmt) Span() (start, end Position) { + return x.X.Span() +} + +// An IfStmt is a conditional: If Cond: True; else: False. +// 'elseif' is desugared into a chain of IfStmts. +type IfStmt struct { + commentsRef + If Position // IF or ELIF + Cond Expr + True []Stmt + ElsePos Position // ELSE or ELIF + False []Stmt // optional +} + +func (x *IfStmt) Span() (start, end Position) { + body := x.False + if body == nil { + body = x.True + } + _, end = body[len(body)-1].Span() + return x.If, end +} + +// A LoadStmt loads another module and binds names from it: +// load(Module, "x", y="foo"). +// +// The AST is slightly unfaithful to the concrete syntax here because +// Starlark's load statement, so that it can be implemented in Python, +// binds some names (like y above) with an identifier and some (like x) +// without. For consistency we create fake identifiers for all the +// strings. +type LoadStmt struct { + commentsRef + Load Position + Module *Literal // a string + From []*Ident // name defined in loading module + To []*Ident // name in loaded module + Rparen Position +} + +func (x *LoadStmt) Span() (start, end Position) { + return x.Load, x.Rparen +} + +// ModuleName returns the name of the module loaded by this statement. +func (x *LoadStmt) ModuleName() string { return x.Module.Value.(string) } + +// A BranchStmt changes the flow of control: break, continue, pass. +type BranchStmt struct { + commentsRef + Token Token // = BREAK | CONTINUE | PASS + TokenPos Position +} + +func (x *BranchStmt) Span() (start, end Position) { + return x.TokenPos, x.TokenPos.add(x.Token.String()) +} + +// A ReturnStmt returns from a function. +type ReturnStmt struct { + commentsRef + Return Position + Result Expr // may be nil +} + +func (x *ReturnStmt) Span() (start, end Position) { + if x.Result == nil { + return x.Return, x.Return.add("return") + } + _, end = x.Result.Span() + return x.Return, end +} + +// An Expr is a Starlark expression. +type Expr interface { + Node + expr() +} + +func (*BinaryExpr) expr() {} +func (*CallExpr) expr() {} +func (*Comprehension) expr() {} +func (*CondExpr) expr() {} +func (*DictEntry) expr() {} +func (*DictExpr) expr() {} +func (*DotExpr) expr() {} +func (*Ident) expr() {} +func (*IndexExpr) expr() {} +func (*LambdaExpr) expr() {} +func (*ListExpr) expr() {} +func (*Literal) expr() {} +func (*ParenExpr) expr() {} +func (*SliceExpr) expr() {} +func (*TupleExpr) expr() {} +func (*UnaryExpr) expr() {} + +// An Ident represents an identifier. +type Ident struct { + commentsRef + NamePos Position + Name string + + // set by resolver: + + Scope uint8 // see type resolve.Scope + Index int // index into enclosing {DefStmt,File}.Locals (if scope==Local) or DefStmt.FreeVars (if scope==Free) or File.Globals (if scope==Global) +} + +func (x *Ident) Span() (start, end Position) { + return x.NamePos, x.NamePos.add(x.Name) +} + +// A Literal represents a literal string or number. +type Literal struct { + commentsRef + Token Token // = STRING | INT + TokenPos Position + Raw string // uninterpreted text + Value interface{} // = string | int64 | *big.Int +} + +func (x *Literal) Span() (start, end Position) { + return x.TokenPos, x.TokenPos.add(x.Raw) +} + +// A ParenExpr represents a parenthesized expression: (X). +type ParenExpr struct { + commentsRef + Lparen Position + X Expr + Rparen Position +} + +func (x *ParenExpr) Span() (start, end Position) { + return x.Lparen, x.Rparen.add(")") +} + +// A CallExpr represents a function call expression: Fn(Args). +type CallExpr struct { + commentsRef + Fn Expr + Lparen Position + Args []Expr // arg = expr | ident=expr | *expr | **expr + Rparen Position +} + +func (x *CallExpr) Span() (start, end Position) { + start, _ = x.Fn.Span() + return start, x.Rparen.add(")") +} + +// A DotExpr represents a field or method selector: X.Name. +type DotExpr struct { + commentsRef + X Expr + Dot Position + NamePos Position + Name *Ident +} + +func (x *DotExpr) Span() (start, end Position) { + start, _ = x.X.Span() + _, end = x.Name.Span() + return +} + +// A Comprehension represents a list or dict comprehension: +// [Body for ... if ...] or {Body for ... if ...} +type Comprehension struct { + commentsRef + Curly bool // {x:y for ...} or {x for ...}, not [x for ...] + Lbrack Position + Body Expr + Clauses []Node // = *ForClause | *IfClause + Rbrack Position +} + +func (x *Comprehension) Span() (start, end Position) { + return x.Lbrack, x.Rbrack.add("]") +} + +// A ForStmt represents a loop: for Vars in X: Body. +type ForStmt struct { + commentsRef + For Position + Vars Expr // name, or tuple of names + X Expr + Body []Stmt +} + +func (x *ForStmt) Span() (start, end Position) { + _, end = x.Body[len(x.Body)-1].Span() + return x.For, end +} + +// A WhileStmt represents a while loop: while X: Body. +type WhileStmt struct { + commentsRef + While Position + Cond Expr + Body []Stmt +} + +func (x *WhileStmt) Span() (start, end Position) { + _, end = x.Body[len(x.Body)-1].Span() + return x.While, end +} + +// A ForClause represents a for clause in a list comprehension: for Vars in X. +type ForClause struct { + commentsRef + For Position + Vars Expr // name, or tuple of names + In Position + X Expr +} + +func (x *ForClause) Span() (start, end Position) { + _, end = x.X.Span() + return x.For, end +} + +// An IfClause represents an if clause in a list comprehension: if Cond. +type IfClause struct { + commentsRef + If Position + Cond Expr +} + +func (x *IfClause) Span() (start, end Position) { + _, end = x.Cond.Span() + return x.If, end +} + +// A DictExpr represents a dictionary literal: { List }. +type DictExpr struct { + commentsRef + Lbrace Position + List []Expr // all *DictEntrys + Rbrace Position +} + +func (x *DictExpr) Span() (start, end Position) { + return x.Lbrace, x.Rbrace.add("}") +} + +// A DictEntry represents a dictionary entry: Key: Value. +// Used only within a DictExpr. +type DictEntry struct { + commentsRef + Key Expr + Colon Position + Value Expr +} + +func (x *DictEntry) Span() (start, end Position) { + start, _ = x.Key.Span() + _, end = x.Value.Span() + return start, end +} + +// A LambdaExpr represents an inline function abstraction. +// +// Although they may be added in future, lambda expressions are not +// currently part of the Starlark spec, so their use is controlled by the +// resolver.AllowLambda flag. +type LambdaExpr struct { + commentsRef + Lambda Position + Function +} + +func (x *LambdaExpr) Span() (start, end Position) { + _, end = x.Function.Body[len(x.Body)-1].Span() + return x.Lambda, end +} + +// A ListExpr represents a list literal: [ List ]. +type ListExpr struct { + commentsRef + Lbrack Position + List []Expr + Rbrack Position +} + +func (x *ListExpr) Span() (start, end Position) { + return x.Lbrack, x.Rbrack.add("]") +} + +// CondExpr represents the conditional: X if COND else ELSE. +type CondExpr struct { + commentsRef + If Position + Cond Expr + True Expr + ElsePos Position + False Expr +} + +func (x *CondExpr) Span() (start, end Position) { + start, _ = x.True.Span() + _, end = x.False.Span() + return start, end +} + +// A TupleExpr represents a tuple literal: (List). +type TupleExpr struct { + commentsRef + Lparen Position // optional (e.g. in x, y = 0, 1), but required if List is empty + List []Expr + Rparen Position +} + +func (x *TupleExpr) Span() (start, end Position) { + if x.Lparen.IsValid() { + return x.Lparen, x.Rparen + } else { + return Start(x.List[0]), End(x.List[len(x.List)-1]) + } +} + +// A UnaryExpr represents a unary expression: Op X. +// +// As a special case, UnaryOp{Op:Star} may also represent +// the star parameter in def f(*args) or def f(*, x). +type UnaryExpr struct { + commentsRef + OpPos Position + Op Token + X Expr // may be nil if Op==STAR +} + +func (x *UnaryExpr) Span() (start, end Position) { + if x.X != nil { + _, end = x.X.Span() + } else { + end = x.OpPos.add("*") + } + return x.OpPos, end +} + +// A BinaryExpr represents a binary expression: X Op Y. +// +// As a special case, BinaryExpr{Op:EQ} may also +// represent a named argument in a call f(k=v) +// or a named parameter in a function declaration +// def f(param=default). +type BinaryExpr struct { + commentsRef + X Expr + OpPos Position + Op Token + Y Expr +} + +func (x *BinaryExpr) Span() (start, end Position) { + start, _ = x.X.Span() + _, end = x.Y.Span() + return start, end +} + +// A SliceExpr represents a slice or substring expression: X[Lo:Hi:Step]. +type SliceExpr struct { + commentsRef + X Expr + Lbrack Position + Lo, Hi, Step Expr // all optional + Rbrack Position +} + +func (x *SliceExpr) Span() (start, end Position) { + start, _ = x.X.Span() + return start, x.Rbrack +} + +// An IndexExpr represents an index expression: X[Y]. +type IndexExpr struct { + commentsRef + X Expr + Lbrack Position + Y Expr + Rbrack Position +} + +func (x *IndexExpr) Span() (start, end Position) { + start, _ = x.X.Span() + return start, x.Rbrack +} diff --git a/vendor/go.starlark.net/syntax/walk.go b/vendor/go.starlark.net/syntax/walk.go new file mode 100644 index 00000000..ad2aa889 --- /dev/null +++ b/vendor/go.starlark.net/syntax/walk.go @@ -0,0 +1,163 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package syntax + +// Walk traverses a syntax tree in depth-first order. +// It starts by calling f(n); n must not be nil. +// If f returns true, Walk calls itself +// recursively for each non-nil child of n. +// Walk then calls f(nil). +func Walk(n Node, f func(Node) bool) { + if n == nil { + panic("nil") + } + if !f(n) { + return + } + + // TODO(adonovan): opt: order cases using profile data. + switch n := n.(type) { + case *File: + walkStmts(n.Stmts, f) + + case *ExprStmt: + Walk(n.X, f) + + case *BranchStmt: + // no-op + + case *IfStmt: + Walk(n.Cond, f) + walkStmts(n.True, f) + walkStmts(n.False, f) + + case *AssignStmt: + Walk(n.LHS, f) + Walk(n.RHS, f) + + case *DefStmt: + Walk(n.Name, f) + for _, param := range n.Function.Params { + Walk(param, f) + } + walkStmts(n.Function.Body, f) + + case *ForStmt: + Walk(n.Vars, f) + Walk(n.X, f) + walkStmts(n.Body, f) + + case *ReturnStmt: + if n.Result != nil { + Walk(n.Result, f) + } + + case *LoadStmt: + Walk(n.Module, f) + for _, from := range n.From { + Walk(from, f) + } + for _, to := range n.To { + Walk(to, f) + } + + case *Ident, *Literal: + // no-op + + case *ListExpr: + for _, x := range n.List { + Walk(x, f) + } + + case *ParenExpr: + Walk(n.X, f) + + case *CondExpr: + Walk(n.Cond, f) + Walk(n.True, f) + Walk(n.False, f) + + case *IndexExpr: + Walk(n.X, f) + Walk(n.Y, f) + + case *DictEntry: + Walk(n.Key, f) + Walk(n.Value, f) + + case *SliceExpr: + Walk(n.X, f) + if n.Lo != nil { + Walk(n.Lo, f) + } + if n.Hi != nil { + Walk(n.Hi, f) + } + if n.Step != nil { + Walk(n.Step, f) + } + + case *Comprehension: + Walk(n.Body, f) + for _, clause := range n.Clauses { + Walk(clause, f) + } + + case *IfClause: + Walk(n.Cond, f) + + case *ForClause: + Walk(n.Vars, f) + Walk(n.X, f) + + case *TupleExpr: + for _, x := range n.List { + Walk(x, f) + } + + case *DictExpr: + for _, entry := range n.List { + entry := entry.(*DictEntry) + Walk(entry.Key, f) + Walk(entry.Value, f) + } + + case *UnaryExpr: + if n.X != nil { + Walk(n.X, f) + } + + case *BinaryExpr: + Walk(n.X, f) + Walk(n.Y, f) + + case *DotExpr: + Walk(n.X, f) + Walk(n.Name, f) + + case *CallExpr: + Walk(n.Fn, f) + for _, arg := range n.Args { + Walk(arg, f) + } + + case *LambdaExpr: + for _, param := range n.Function.Params { + Walk(param, f) + } + walkStmts(n.Function.Body, f) + + default: + panic(n) + } + + f(nil) +} + +func walkStmts(stmts []Stmt, f func(Node) bool) { + for _, stmt := range stmts { + Walk(stmt, f) + } +}