// Copyright 2018 The CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package parser

import (
	"fmt"
	"slices"
	"strings"
	"unicode"

	"cuelang.org/go/cue/ast"
	"cuelang.org/go/cue/errors"
	"cuelang.org/go/cue/literal"
	"cuelang.org/go/cue/scanner"
	"cuelang.org/go/cue/token"
	"cuelang.org/go/internal"
	"cuelang.org/go/internal/cueexperiment"
)

// The parser structure holds the parser's internal state.
type parser struct {
	file    *token.File
	errors  errors.Error
	scanner scanner.Scanner

	expList     []string // list of experiments to enable
	experiments *cueexperiment.File

	// Tracing/debugging
	cfg       Config
	trace     bool // == (cfg.Mode & Trace != 0)
	panicking bool // set if we are bailing out due to too many errors.
	indent    int  // indentation used for tracing output

	// Comments
	leadComment  *ast.CommentGroup
	comments     *commentState
	commentStack []*commentState // to reuse [commentState] allocations

	// Next token, filled by [parser.next0].
	pos token.Pos   // token position
	tok token.Token // one token look-ahead
	lit string      // token literal

	// Token after next, filled by [parser.peek].
	peekToken struct {
		scanned bool

		pos token.Pos
		tok token.Token
		lit string
	}

	// Error recovery
	// (used to limit the number of calls to sync... functions
	// w/o making scanning progress - avoids potential endless
	// loops across multiple parser functions during error recovery)
	syncPos token.Pos // last synchronization position
	syncCnt int       // number of calls to sync... functions without progress

	// Non-syntactic parser control
	exprLev int // < 0: in control clause, >= 0: in expression

	imports []*ast.ImportSpec // list of imports
}

func (p *parser) init(filename string, src []byte, opts []Option) {
	p.cfg = NewConfig().Apply(opts...)
	p.file = token.NewFile(filename, -1, len(src))

	var m scanner.Mode
	if p.cfg.Mode&ParseComments != 0 {
		m = scanner.ScanComments
	}
	eh := func(pos token.Pos, msg string, args []interface{}) {
		p.errors = errors.Append(p.errors, errors.Newf(pos, msg, args...))
	}
	p.scanner.Init(p.file, src, eh, m)

	p.trace = p.cfg.Mode&Trace != 0 // for convenience (p.trace is used frequently)

	p.comments = &commentState{pos: -1}

	p.next()
}

type commentState struct {
	parent *commentState
	pos    int8
	groups []*ast.CommentGroup

	// lists are not attached to nodes themselves. Enclosed expressions may
	// miss a comment due to commas and line termination. closeLists ensures
	// that comments will be passed to someone.
	isList    int
	lastChild ast.Node
	lastPos   int8
}

func (p *parser) allocCommentState() *commentState {
	if n := len(p.commentStack); n > 0 {
		c := p.commentStack[n-1]
		p.commentStack = p.commentStack[:n-1]
		return c
	}
	return &commentState{}
}

func (p *parser) freeCommentState(c *commentState) {
	// Ensure no pointers remain, which can hold onto memory.
	// We only reuse the groups slice capacity.
	*c = commentState{groups: c.groups[:0]}
	p.commentStack = append(p.commentStack, c)
}

// openComments reserves the next doc comment for the caller and flushes
func (p *parser) openComments() *commentState {
	child := p.allocCommentState()
	child.parent = p.comments
	if c := p.comments; c != nil && c.isList > 0 {
		if c.lastChild != nil {
			var groups []*ast.CommentGroup
			for _, cg := range c.groups {
				if cg.Position == 0 {
					groups = append(groups, cg)
				}
			}
			groups = append(groups, ast.Comments(c.lastChild)...)
			for _, cg := range c.groups {
				if cg.Position != 0 {
					cg.Position = c.lastPos
					groups = append(groups, cg)
				}
			}
			ast.SetComments(c.lastChild, groups)
		} else {
			// attach before next
			for _, cg := range c.groups {
				cg.Position = 0
			}
			child.groups = append(child.groups, c.groups...)
		}
		c.groups = c.groups[:0]
	}
	if p.leadComment != nil {
		child.groups = append(child.groups, p.leadComment)
		p.leadComment = nil
	}
	p.comments = child
	return child
}

// openList is used to treat a list of comments as a single comment
// position in a production.
func (p *parser) openList() {
	if p.comments.isList > 0 {
		p.comments.isList++
		return
	}
	c := p.allocCommentState()
	c.parent = p.comments
	c.isList = 1
	p.comments = c
}

func (c *commentState) add(g *ast.CommentGroup) {
	g.Position = c.pos
	c.groups = append(c.groups, g)
}

func (p *parser) closeList() {
	c := p.comments
	if c.lastChild != nil {
		for _, cg := range c.groups {
			cg.Position = c.lastPos
			ast.AddComment(c.lastChild, cg)
		}
		c.groups = c.groups[:0]
	}
	switch c.isList--; {
	case c.isList < 0:
		if !p.panicking {
			err := errors.Newf(p.pos, "unmatched close list")
			p.errors = errors.Append(p.errors, err)
			p.panicking = true
			panic(err)
		}
	case c.isList == 0:
		parent := c.parent
		if len(c.groups) > 0 {
			parent.groups = append(parent.groups, c.groups...)
		}
		parent.pos++
		p.comments = parent
		p.freeCommentState(c)
	}
}

func (c *commentState) closeNode(p *parser, n ast.Node) ast.Node {
	if p.comments != c {
		if !p.panicking {
			err := errors.Newf(p.pos, "unmatched comments")
			p.errors = errors.Append(p.errors, err)
			p.panicking = true
			panic(err)
		}
		return n
	}
	p.comments = c.parent
	if c.parent != nil {
		c.parent.lastChild = n
		c.parent.lastPos = c.pos
		c.parent.pos++
	}
	for _, cg := range c.groups {
		if n != nil {
			if cg != nil {
				ast.AddComment(n, cg)
			}
		}
	}
	p.freeCommentState(c)
	return n
}

func (c *commentState) closeExpr(p *parser, n ast.Expr) ast.Expr {
	c.closeNode(p, n)
	return n
}

func (c *commentState) closeClause(p *parser, n ast.Clause) ast.Clause {
	c.closeNode(p, n)
	return n
}

// ----------------------------------------------------------------------------
// Parsing support

func (p *parser) printTrace(a ...interface{}) {
	const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
	const n = len(dots)
	pos := p.file.Position(p.pos)
	fmt.Printf("%5d:%3d: ", pos.Line, pos.Column)
	i := 2 * p.indent
	for i > n {
		fmt.Print(dots)
		i -= n
	}
	// i <= n
	fmt.Print(dots[0:i])
	fmt.Println(a...)
}

func trace(p *parser, msg string) *parser {
	p.printTrace(msg, "(")
	p.indent++
	return p
}

// Usage pattern: defer un(trace(p, "..."))
func un(p *parser) {
	p.indent--
	p.printTrace(")")
}

// Advance to the next
func (p *parser) next0() {
	// Because of one-token look-ahead, print the previous token
	// when tracing as it provides a more readable output. The
	// very first token (!p.pos.IsValid()) is not initialized
	// (it is ILLEGAL), so don't print it .
	if p.trace && p.pos.IsValid() {
		s := p.tok.String()
		switch {
		case p.tok.IsLiteral():
			p.printTrace(s, p.lit)
		case p.tok.IsOperator(), p.tok.IsKeyword():
			p.printTrace("\"" + s + "\"")
		default:
			p.printTrace(s)
		}
	}

	// We had peeked one token, effectively scanning it early; use it now.
	if p.peekToken.scanned {
		p.pos, p.tok, p.lit = p.peekToken.pos, p.peekToken.tok, p.peekToken.lit
		p.peekToken.scanned = false
		return
	}

	p.pos, p.tok, p.lit = p.scanner.Scan()
}

// peek scans one more token as a look-ahead and stores it in [parser.peekToken].
// Peeking multiple tokens ahead is not supported.
func (p *parser) peek() {
	if p.peekToken.scanned {
		panic("can only peek one token at a time")
	}
	p.peekToken.pos, p.peekToken.tok, p.peekToken.lit = p.scanner.Scan()
	p.peekToken.scanned = true
}

// Consume a comment and return it and the line on which it ends.
func (p *parser) consumeComment() (comment *ast.Comment, endline int) {
	endline = p.file.Line(p.pos)

	comment = &ast.Comment{Slash: p.pos, Text: p.lit}
	p.next0()

	return
}

// Consume a group of adjacent comments, add it to the parser's
// comments list, and return it together with the line at which
// the last comment in the group ends. A non-comment token or n
// empty lines terminate a comment group.
func (p *parser) consumeCommentGroup(prevLine, n int) (comments *ast.CommentGroup, endline int) {
	var list []*ast.Comment
	var rel token.RelPos
	endline = p.file.Line(p.pos)
	switch endline - prevLine {
	case 0:
		rel = token.Blank
	case 1:
		rel = token.Newline
	default:
		rel = token.NewSection
	}
	for p.tok == token.COMMENT && p.file.Line(p.pos) <= endline+n {
		var comment *ast.Comment
		comment, endline = p.consumeComment()
		list = append(list, comment)
	}

	cg := &ast.CommentGroup{List: list}
	ast.SetRelPos(cg, rel)
	comments = cg
	return
}

// Advance to the next non-comment  In the process, collect
// any comment groups encountered, and refield the last lead and
// line comments.
//
// A lead comment is a comment group that starts and ends in a
// line without any other tokens and that is followed by a non-comment
// token on the line immediately after the comment group.
//
// A line comment is a comment group that follows a non-comment
// token on the same line, and that has no tokens after it on the line
// where it ends.
//
// Lead and line comments may be considered documentation that is
// stored in the AST.
func (p *parser) next() {
	// A leadComment may not be consumed if it leads an inner token of a node.
	if p.leadComment != nil {
		p.comments.add(p.leadComment)
	}
	p.leadComment = nil
	prev := p.pos
	p.next0()
	p.comments.pos++

	if p.tok == token.COMMENT {
		var comment *ast.CommentGroup
		var endline int

		currentLine := p.file.Line(p.pos)
		prevLine := p.file.Line(prev)
		if prevLine == currentLine {
			// The comment is on same line as the previous token; it
			// cannot be a lead comment but may be a line comment.
			comment, endline = p.consumeCommentGroup(prevLine, 0)
			if p.file.Line(p.pos) != endline {
				// The next token is on a different line, thus
				// the last comment group is a line comment.
				comment.Line = true
			}
		}

		// consume successor comments, if any
		endline = -1
		for p.tok == token.COMMENT {
			if comment != nil {
				p.comments.add(comment)
			}
			comment, endline = p.consumeCommentGroup(prevLine, 1)
			prevLine = currentLine
			currentLine = p.file.Line(p.pos)

		}

		if endline+1 == p.file.Line(p.pos) && p.tok != token.EOF {
			// The next token is following on the line immediately after the
			// comment group, thus the last comment group is a lead comment.
			comment.Doc = true
			p.leadComment = comment
		} else {
			p.comments.add(comment)
		}
	}
}

func (p *parser) errf(pos token.Pos, msg string, args ...interface{}) {
	// ePos := p.file.Position(pos)
	ePos := pos

	// If AllErrors is not set, discard errors reported on the same line
	// as the last recorded error and stop parsing if there are more than
	// 10 errors.
	if p.cfg.Mode&AllErrors == 0 {
		errors := errors.Errors(p.errors)
		n := len(errors)
		if n > 0 && errors[n-1].Position().Line() == ePos.Line() {
			return // discard - likely a spurious error
		}
		if n > 10 {
			p.panicking = true
			panic("too many errors")
		}
	}

	p.errors = errors.Append(p.errors, errors.Newf(ePos, msg, args...))
}

func (p *parser) errorExpected(pos token.Pos, obj string) {
	if pos != p.pos {
		p.errf(pos, "expected %s", obj)
		return
	}
	// the error happened at the current position;
	// make the error message more specific
	if p.tok == token.COMMA && p.lit == "\n" {
		p.errf(pos, "expected %s, found newline", obj)
		return
	}

	if p.tok.IsLiteral() {
		p.errf(pos, "expected %s, found '%s' %s", obj, p.tok, p.lit)
	} else {
		p.errf(pos, "expected %s, found '%s'", obj, p.tok)
	}
}

func (p *parser) expect(tok token.Token) token.Pos {
	pos := p.pos
	if p.tok != tok {
		p.errorExpected(pos, "'"+tok.String()+"'")
		pos = token.NoPos
	}
	p.next() // make progress
	return pos
}

// expectClosing is like expect but provides a better error message
// for the common case of a missing comma before a newline.
func (p *parser) expectClosing(tok token.Token, context string) token.Pos {
	if p.tok != tok && p.tok == token.COMMA && p.lit == "\n" {
		p.errf(p.pos, "missing ',' before newline in %s", context)
		p.next()
	}
	return p.expect(tok)
}

func (p *parser) expectComma() {
	switch p.tok {
	case token.COMMA:
		p.next()
	// the comma is optional before a closing ')', ']', '}', or newline
	case token.RPAREN, token.RBRACE, token.EOF:
	default:
		p.errorExpected(p.pos, "','")
		syncExpr(p)
	}
}

func (p *parser) atComma(context string, follow ...token.Token) bool {
	if p.tok == token.COMMA {
		return true
	}
	if slices.Contains(follow, p.tok) {
		return false
	}
	// TODO: find a way to detect crossing lines now we don't have a semi.
	if p.lit == "\n" {
		p.errf(p.pos, "missing ',' before newline")
	} else {
		p.errf(p.pos, "missing ',' in %s", context)
	}
	return true // "insert" comma and continue
}

// syncExpr advances to the next field in a field list.
// Used for synchronization after an error.
func syncExpr(p *parser) {
	for {
		switch p.tok {
		case token.COMMA:
			// Return only if parser made some progress since last
			// sync or if it has not reached 10 sync calls without
			// progress. Otherwise consume at least one token to
			// avoid an endless parser loop (it is possible that
			// both parseOperand and parseStmt call syncStmt and
			// correctly do not advance, thus the need for the
			// invocation limit p.syncCnt).
			if p.pos == p.syncPos && p.syncCnt < 10 {
				p.syncCnt++
				return
			}
			if p.syncPos.Compare(p.pos) < 0 {
				p.syncPos = p.pos
				p.syncCnt = 0
				return
			}
			// Reaching here indicates a parser bug, likely an
			// incorrect token list in this function, but it only
			// leads to skipping of possibly correct code if a
			// previous error is present, and thus is preferred
			// over a non-terminating parse.
		case token.EOF:
			return
		}
		p.next()
	}
}

// safePos returns a valid file position for a given position: If pos
// is valid to begin with, safePos returns pos. If pos is out-of-range,
// safePos returns the EOF position.
//
// This is hack to work around "artificial" end positions in the AST which
// are computed by adding 1 to (presumably valid) token positions. If the
// token positions are invalid due to parse errors, the resulting end position
// may be past the file's EOF position, which would lead to panics if used
// later on.
func (p *parser) safePos(pos token.Pos) (res token.Pos) {
	defer func() {
		if recover() != nil {
			res = p.file.Pos(p.file.Base()+p.file.Size(), pos.RelPos()) // EOF position
		}
	}()
	_ = p.file.Offset(pos) // trigger a panic if position is out-of-range
	return pos
}

// ----------------------------------------------------------------------------
// Identifiers

func (p *parser) parseIdent() *ast.Ident {
	c := p.openComments()
	name := p.lit
	pos := p.expect(token.IDENT)
	if !pos.IsValid() {
		name = "_"
	}
	ident := &ast.Ident{NamePos: pos, Name: name}
	c.closeNode(p, ident)
	return ident
}

// checkDeclIdent validates that an identifier is not a reserved
// double-underscore identifier. Use this when an identifier is being declared.
func (p *parser) checkDeclIdent(ident *ast.Ident) {
	if strings.HasPrefix(ident.Name, "__") {
		p.errf(ident.NamePos, "identifiers starting with '__' are reserved")
	}
}

// parseIdentDecl parses an identifier and validates that it's not a reserved
// double-underscore identifier. Use this for identifier declarations.
func (p *parser) parseIdentDecl() *ast.Ident {
	ident := p.parseIdent()
	p.checkDeclIdent(ident)
	return ident
}

func (p *parser) parseKeyIdent() *ast.Ident {
	c := p.openComments()
	pos := p.pos
	name := p.lit
	p.next()
	ident := &ast.Ident{NamePos: pos, Name: name}
	c.closeNode(p, ident)
	return ident
}

// ----------------------------------------------------------------------------
// Expressions

// parseOperand returns an expression.
// Callers must verify the result.
func (p *parser) parseOperand() (expr ast.Expr) {
	if p.trace {
		defer un(trace(p, "Operand"))
	}

	switch p.tok {
	case token.IDENT:
		ident := p.parseIdent()
		// Check for optional reference marker (?)
		// Don't consume ? if it's followed by : (that's a field constraint, not optional reference)
		if p.tok == token.OPTION {
			// Peek ahead to see if this is a field constraint (foo?: value)
			p.peek()
			if p.peekToken.tok != token.COLON {
				return p.wrapOptional(ident, p.tok)
			}
		}
		return ident

	case token.LBRACE:
		return p.parseStruct()

	case token.LBRACK:
		return p.parseList()

	case token.FUNC:
		if p.cfg.Mode&ParseFuncs != 0 {
			return p.parseFunc()
		} else {
			return p.parseKeyIdent()
		}

	case token.BOTTOM:
		c := p.openComments()
		x := &ast.BottomLit{Bottom: p.pos}
		p.next()
		return c.closeExpr(p, x)

	case token.NULL, token.TRUE, token.FALSE, token.INT, token.FLOAT, token.STRING:
		c := p.openComments()
		x := &ast.BasicLit{ValuePos: p.pos, Kind: p.tok, Value: p.lit}
		p.next()
		return c.closeExpr(p, x)

	case token.INTERPOLATION:
		return p.parseInterpolation()

	case token.LPAREN:
		c := p.openComments()
		defer func() { c.closeNode(p, expr) }()
		lparen := p.pos
		p.next()
		p.exprLev++
		p.openList()
		x := p.parseRHS() // types may be parenthesized: (some type)
		p.closeList()
		p.exprLev--
		rparen := p.expect(token.RPAREN)
		return &ast.ParenExpr{
			Lparen: lparen,
			X:      x,
			Rparen: rparen,
		}

	default:
		if p.tok.IsKeyword() {
			return p.parseKeyIdent()
		}
	}

	// we have an error
	c := p.openComments()
	pos := p.pos
	p.errorExpected(pos, "operand")
	syncExpr(p)
	return c.closeExpr(p, &ast.BadExpr{From: pos, To: p.pos})
}

func (p *parser) parseIndexOrSlice(x ast.Expr) (expr ast.Expr) {
	if p.trace {
		defer un(trace(p, "IndexOrSlice"))
	}

	c := p.openComments()
	defer func() { c.closeNode(p, expr) }()
	c.pos = 1

	const N = 2
	lbrack := p.expect(token.LBRACK)

	p.exprLev++
	var index [N]ast.Expr
	var colons [N - 1]token.Pos
	if p.tok != token.COLON {
		index[0] = p.parseRHS()
	}
	nColons := 0
	for p.tok == token.COLON && nColons < len(colons) {
		colons[nColons] = p.pos
		nColons++
		p.next()
		if p.tok != token.COLON && p.tok != token.RBRACK && p.tok != token.EOF {
			index[nColons] = p.parseRHS()
		}
	}
	p.exprLev--
	rbrack := p.expect(token.RBRACK)

	if nColons > 0 {
		return &ast.SliceExpr{
			X:      x,
			Lbrack: lbrack,
			Low:    index[0],
			High:   index[1],
			Rbrack: rbrack,
		}
	}

	result := &ast.IndexExpr{
		X:      x,
		Lbrack: lbrack,
		Index:  index[0],
		Rbrack: rbrack,
	}
	return p.wrapOptional(result, p.tok)
}

func (p *parser) wrapOptional(x ast.Expr, tok token.Token) ast.Expr {
	if tok == token.OPTION {
		pos := p.pos
		p.next()
		return &ast.PostfixExpr{X: x, Op: token.OPTION, OpPos: pos}
	}
	return x
}

func (p *parser) parseCallOrConversion(fun ast.Expr) (expr *ast.CallExpr) {
	if p.trace {
		defer un(trace(p, "CallOrConversion"))
	}
	c := p.openComments()
	defer func() { c.closeNode(p, expr) }()

	p.openList()
	defer p.closeList()

	lparen := p.expect(token.LPAREN)

	p.exprLev++
	var list []ast.Expr
	for p.tok != token.RPAREN && p.tok != token.EOF {
		list = append(list, p.parseRHS()) // builtins may expect a type: make(some type, ...)
		if !p.atComma("argument list", token.RPAREN) {
			break
		}
		p.next()
	}
	p.exprLev--
	rparen := p.expectClosing(token.RPAREN, "argument list")

	return &ast.CallExpr{
		Fun:    fun,
		Lparen: lparen,
		Args:   list,
		Rparen: rparen,
	}
}

// TODO: inline this function in parseFieldList once we no longer user comment
// position information in parsing.
func (p *parser) consumeDeclComma() {
	if p.atComma("struct literal", token.RBRACE, token.EOF) {
		p.next()
	}
}

func (p *parser) parseFieldList() (list []ast.Decl) {
	if p.trace {
		defer un(trace(p, "FieldList"))
	}
	p.openList()
	defer p.closeList()

	for p.tok != token.RBRACE && p.tok != token.EOF {
		switch p.tok {
		case token.ATTRIBUTE:
			list = append(list, p.parseAttribute(false))
			p.consumeDeclComma()

		case token.ELLIPSIS:
			c := p.openComments()
			ellipsis := &ast.Ellipsis{Ellipsis: p.pos}
			p.next()
			c.closeNode(p, ellipsis)
			list = append(list, ellipsis)
			p.consumeDeclComma()

		default:
			list = append(list, p.parseField())
		}

		// TODO: handle next comma here, after disallowing non-colon separator
		// and we have eliminated the need comment positions.
	}

	return
}

func (p *parser) parseLetDecl() (decl ast.Decl, ident *ast.Ident) {
	if p.trace {
		defer un(trace(p, "Field"))
	}

	c := p.openComments()

	letPos := p.expect(token.LET)
	if p.tok != token.IDENT {
		c.closeNode(p, ident)
		return nil, &ast.Ident{
			NamePos: letPos,
			Name:    "let",
		}
	}
	defer func() { c.closeNode(p, decl) }()

	ident = p.parseIdentDecl()
	assign := p.expect(token.BIND)
	expr := p.parseRHS()

	p.consumeDeclComma()

	return &ast.LetClause{
		Let:   letPos,
		Ident: ident,
		Equal: assign,
		Expr:  expr,
	}, nil
}

func (p *parser) parseComprehension() (decl ast.Decl, ident *ast.Ident) {
	if p.trace {
		defer un(trace(p, "Comprehension"))
	}

	c := p.openComments()
	defer func() { c.closeNode(p, decl) }()

	tok := p.tok
	pos := p.pos
	clauses, fc := p.parseComprehensionClauses()
	if fc != nil {
		ident = &ast.Ident{
			NamePos: pos,
			Name:    tok.String(),
		}
		fc.closeNode(p, ident)
		return nil, ident
	}

	sc := p.openComments()
	expr := p.parseStruct()
	sc.closeExpr(p, expr)

	var fallbackClause *ast.FallbackClause
	if p.tok == token.ELSE || p.tok == token.FALLBACK {
		fallbackClause = p.parseFallbackClause(clauses)
	}

	if p.atComma("struct literal", token.RBRACE) { // TODO: may be EOF
		p.next()
	}

	return &ast.Comprehension{
		Clauses:  clauses,
		Value:    expr,
		Fallback: fallbackClause,
	}, nil
}

func (p *parser) parseField() (decl ast.Decl) {
	if p.trace {
		defer un(trace(p, "Field"))
	}

	c := p.openComments()
	defer func() { c.closeNode(p, decl) }()

	pos := p.pos
	tok := p.tok

	label, expr, decl, ok := p.parseLabel(false)
	if decl != nil {
		return decl
	}

	if !ok {
		if expr == nil {
			expr = p.parseRHS()
		}
		if a, ok := expr.(*ast.Alias); ok {
			p.errf(a.Pos(), `pre-v0.2 alias; use "let X = expr" instead`)
			p.consumeDeclComma()
			return a
		}
		e := &ast.EmbedDecl{Expr: expr}
		p.consumeDeclComma()
		return e
	}

	this := &ast.Field{}
	m := this
	m.Label = label

	// Parse postfix alias if present
	m.Alias = p.parsePostfixAlias()

	switch p.tok {
	case token.OPTION, token.NOT:
		m.Constraint = p.tok
		p.next()
	}

	// TODO: consider disallowing comprehensions with more than one label.
	// This can be a bit awkward in some cases, but it would naturally
	// enforce the proper style that a comprehension be defined in the
	// smallest possible scope.
	// allowComprehension = false

	switch p.tok {
	case token.COLON:
		// Now we know it's being used as a label, validate double-underscore
		if ident, ok := label.(*ast.Ident); ok {
			p.checkDeclIdent(ident)
		}
	case token.COMMA:
		p.expectComma() // sync parser.
		fallthrough

	case token.RBRACE, token.EOF:
		if a, ok := expr.(*ast.Alias); ok {
			p.errf(a.Pos(), `pre-v0.2 alias; use "let X = expr" instead`)
			return a
		}
		switch tok {
		case token.IDENT, token.LBRACK, token.LPAREN,
			token.STRING, token.INTERPOLATION,
			token.NULL, token.TRUE, token.FALSE,
			token.FOR, token.IF, token.LET, token.IN,
			token.TRY, token.ELSE, token.FALLBACK:
			return &ast.EmbedDecl{Expr: expr}
		}
		fallthrough

	default:
		p.errorExpected(p.pos, "label or ':'")
		return &ast.BadDecl{From: pos, To: p.pos}
	}

	m.TokenPos = p.pos
	p.expect(token.COLON)

	for {
		if l, ok := m.Label.(*ast.ListLit); ok && len(l.Elts) != 1 {
			p.errf(l.Pos(), "square bracket must have exactly one element")
		}

		label, expr, _, ok := p.parseLabel(true)
		if !ok || (p.tok != token.COLON && p.tok != token.OPTION && p.tok != token.NOT && p.tok != token.TILDE) {
			if expr == nil {
				expr = p.parseRHS()
			}
			m.Value = expr
			break
		}
		field := &ast.Field{Label: label}
		m.Value = &ast.StructLit{Elts: []ast.Decl{field}}
		m = field

		// Parse postfix alias if present
		m.Alias = p.parsePostfixAlias()

		switch p.tok {
		case token.OPTION, token.NOT:
			m.Constraint = p.tok
			p.next()
		}

		m.TokenPos = p.pos
		p.expect(token.COLON)
	}

	if attrs := p.parseAttributes(); attrs != nil {
		m.Attrs = attrs
	}

	p.consumeDeclComma()

	return this
}

func (p *parser) parseAttributes() (attrs []*ast.Attribute) {
	p.openList()
	for p.tok == token.ATTRIBUTE {
		attrs = append(attrs, p.parseAttribute(false))
	}
	p.closeList()
	return attrs
}

func (p *parser) parseAttribute(inPreamble bool) *ast.Attribute {
	c := p.openComments()
	a := &ast.Attribute{At: p.pos, Text: p.lit}

	if inPreamble {
		key, body := a.Split()
		if key == "experiment" {
			p.expList = append(p.expList, body)
		}
	}
	p.next()
	c.closeNode(p, a)
	return a
}

func (p *parser) parseLabel(rhs bool) (label ast.Label, expr ast.Expr, decl ast.Decl, ok bool) {
	tok := p.tok
	switch tok {

	case token.FOR, token.IF, token.TRY:
		if rhs {
			expr = p.parseExpr()
			break
		}
		comp, ident := p.parseComprehension()
		if comp != nil {
			return nil, nil, comp, false
		}
		expr = ident

	case token.ELSE, token.FALLBACK:
		// These keywords can be used as field labels
		expr = p.parseExpr()

	case token.LET:
		let, ident := p.parseLetDecl()
		if let != nil {
			return nil, nil, let, false
		}
		expr = ident

	case token.IDENT, token.STRING, token.INTERPOLATION, token.LPAREN,
		token.NULL, token.TRUE, token.FALSE, token.IN, token.FUNC:
		expr = p.parseExpr()

	case token.LBRACK:
		expr = p.parseRHS()
		switch x := expr.(type) {
		case *ast.ListLit:
			// Note: caller must verify this list is suitable as a label.
			label, ok = x, true
		}
	}

	switch x := expr.(type) {
	case *ast.BasicLit:
		switch x.Kind {
		case token.STRING, token.NULL, token.TRUE, token.FALSE, token.FUNC:
			// Keywords that represent operands.

			// Allowing keywords to be used as a labels should not interfere with
			// generating good errors: any keyword can only appear on the RHS of a
			// field (after a ':'), whereas labels always appear on the LHS.

			label, ok = x, true
		}

	case *ast.Ident:
		expr = p.parseAlias(x)
		if a, ok := expr.(*ast.Alias); ok {
			if _, ok = a.Expr.(ast.Label); !ok {
				break
			}
			label = a
		} else {
			label = x
		}
		ok = true

	case ast.Label:
		label, ok = x, true
	}
	return label, expr, nil, ok
}

func (p *parser) parseStruct() (expr ast.Expr) {
	lbrace := p.expect(token.LBRACE)

	if p.trace {
		defer un(trace(p, "StructLit"))
	}

	elts := p.parseStructBody()
	rbrace := p.expectClosing(token.RBRACE, "struct literal")
	return &ast.StructLit{
		Lbrace: lbrace,
		Elts:   elts,
		Rbrace: rbrace,
	}
}

func (p *parser) parseStructBody() []ast.Decl {
	if p.trace {
		defer un(trace(p, "StructBody"))
	}

	p.exprLev++
	var elts []ast.Decl

	// TODO: consider "stealing" non-lead comments.
	// for _, cg := range p.comments.groups {
	// 	if cg != nil {
	// 		elts = append(elts, cg)
	// 	}
	// }
	// p.comments.groups = p.comments.groups[:0]

	if p.tok != token.RBRACE {
		elts = p.parseFieldList()
	}
	p.exprLev--

	return elts
}

// parseComprehensionClauses parses comprehension clauses.
// Should we now disallow keywords as identifiers? If not, we need to
// return a list of discovered labels as the alternative.
func (p *parser) parseComprehensionClauses() (clauses []ast.Clause, c *commentState) {
	// TODO: reuse Template spec, which is possible if it doesn't check the
	// first is an identifier.
	first := true
	for {
		switch p.tok {
		case token.FOR:
			c := p.openComments()
			forPos := p.expect(token.FOR)
			if first {
				switch p.tok {
				case token.COLON, token.BIND, token.OPTION, token.NOT,
					token.COMMA, token.EOF:
					return nil, c
				}
			}

			var key, value *ast.Ident
			var colon token.Pos
			value = p.parseIdentDecl()
			if p.tok == token.COMMA {
				colon = p.expect(token.COMMA)
				key = value
				value = p.parseIdentDecl()
			}
			c.pos = 4
			// params := p.parseParams(nil, ARROW)
			clauses = append(clauses, c.closeClause(p, &ast.ForClause{
				For:    forPos,
				Key:    key,
				Colon:  colon,
				Value:  value,
				In:     p.expect(token.IN),
				Source: p.parseRHS(),
			}))

		case token.IF:
			c := p.openComments()
			ifPos := p.expect(token.IF)
			if first {
				switch p.tok {
				case token.COLON, token.BIND, token.OPTION,
					token.COMMA, token.EOF:
					return nil, c
				case token.NOT:
					p.peek()
					if p.peekToken.tok == token.COLON {
						return nil, c
					}
				}
			}

			clauses = append(clauses, c.closeClause(p, &ast.IfClause{
				If:        ifPos,
				Condition: p.parseRHS(),
			}))

		case token.LET:
			c := p.openComments()
			letPos := p.expect(token.LET)

			ident := p.parseIdentDecl()
			assign := p.expect(token.BIND)
			expr := p.parseRHS()

			clauses = append(clauses, c.closeClause(p, &ast.LetClause{
				Let:   letPos,
				Ident: ident,
				Equal: assign,
				Expr:  expr,
			}))

		case token.TRY:
			c := p.openComments()

			tc := &ast.TryClause{Try: p.expect(token.TRY)}

			// Check for assignment form: try x = expr
			if p.tok == token.IDENT {
				tc.Ident = p.parseIdent()
				tc.Equal = p.expect(token.BIND)
				tc.Expr = p.parseRHS()
			}

			clauses = append(clauses, c.closeClause(p, tc))

		default:
			return clauses, nil
		}
		if p.tok == token.COMMA {
			p.next()
		}
		first = false
	}
}

// parseFallbackClause parses an else or fallback clause in a comprehension.
// It determines the appropriate keyword based on the clause composition:
// - Single if or try clause: expects ELSE, errors on FALLBACK
// - Everything else: expects FALLBACK, errors on ELSE
func (p *parser) parseFallbackClause(clauses []ast.Clause) *ast.FallbackClause {
	if p.trace {
		defer un(trace(p, "FallbackClause"))
	}
	c := p.openComments()

	if p.experiments == nil || !p.experiments.Try {
		p.errf(p.pos, "%s requires @experiment(try)", p.tok)
	}

	// Determine if this is a single if or try clause
	isSingleGuard := len(clauses) == 1
	if isSingleGuard {
		switch clauses[0].(type) {
		case *ast.IfClause, *ast.TryClause:
			// Single if or try: use else
		default:
			isSingleGuard = false
		}
	}

	var pos token.Pos
	if isSingleGuard {
		// Single if/try clause: must use else
		if p.tok == token.FALLBACK {
			p.errf(p.pos, "use 'else' with single 'if' or 'try' clause")
			pos = p.pos
			p.next()
		} else {
			pos = p.expect(token.ELSE)
		}
	} else {
		// Everything else: must use fallback
		if p.tok == token.ELSE {
			p.errf(p.pos, "use 'fallback' for comprehensions with multiple clauses or 'for' clauses")
			pos = p.pos
			p.next()
		} else {
			pos = p.expect(token.FALLBACK)
		}
	}
	body := p.parseStruct()
	return c.closeClause(p, &ast.FallbackClause{
		Fallback: pos,
		Body:     body.(*ast.StructLit),
	}).(*ast.FallbackClause)
}

func (p *parser) parseFunc() (expr ast.Expr) {
	if p.trace {
		defer un(trace(p, "Func"))
	}
	tok := p.tok
	pos := p.pos
	fun := p.expect(token.FUNC)

	// "func" might be used as an identifier, in which case bail out early.
	switch p.tok {
	case token.COLON, token.BIND, token.OPTION,
		token.COMMA, token.EOF:

		return &ast.Ident{
			NamePos: pos,
			Name:    tok.String(),
		}
	}

	p.expect(token.LPAREN)
	args := p.parseFuncArgs()
	p.expectClosing(token.RPAREN, "argument type list")

	p.expect(token.COLON)
	ret := p.parseExpr()

	return &ast.Func{
		Func: fun,
		Args: args,
		Ret:  ret,
	}
}

func (p *parser) parseFuncArgs() (list []ast.Expr) {
	if p.trace {
		defer un(trace(p, "FuncArgs"))
	}
	p.openList()
	defer p.closeList()

	for p.tok != token.RPAREN && p.tok != token.EOF {
		list = append(list, p.parseFuncArg())
		p.expectComma() // skip over a trailing comma or newline
	}

	return list
}

func (p *parser) parseFuncArg() (expr ast.Expr) {
	if p.trace {
		defer un(trace(p, "FuncArg"))
	}
	return p.parseExpr()
}

func (p *parser) parseList() (expr ast.Expr) {
	lbrack := p.expect(token.LBRACK)

	if p.trace {
		defer un(trace(p, "ListLiteral"))
	}

	elts := p.parseListElements()

	if p.tok == token.ELLIPSIS {
		ellipsis := &ast.Ellipsis{
			Ellipsis: p.pos,
		}
		elts = append(elts, ellipsis)
		p.next()
		if p.tok != token.COMMA && p.tok != token.RBRACK {
			ellipsis.Type = p.parseRHS()
		}
		if p.atComma("list literal", token.RBRACK) {
			p.next()
		}
	}

	rbrack := p.expectClosing(token.RBRACK, "list literal")
	return &ast.ListLit{
		Lbrack: lbrack,
		Elts:   elts,
		Rbrack: rbrack,
	}
}

func (p *parser) parseListElements() (list []ast.Expr) {
	if p.trace {
		defer un(trace(p, "ListElements"))
	}
	p.openList()
	defer p.closeList()

	for p.tok != token.RBRACK && p.tok != token.ELLIPSIS && p.tok != token.EOF {
		expr, ok := p.parseListElement()
		list = append(list, expr)
		if !ok {
			break
		}
	}

	return
}

func (p *parser) parseListElement() (expr ast.Expr, ok bool) {
	if p.trace {
		defer un(trace(p, "ListElement"))
	}
	c := p.openComments()
	defer func() { c.closeNode(p, expr) }()

	switch p.tok {
	case token.FOR, token.IF:
		tok := p.tok
		pos := p.pos
		clauses, fc := p.parseComprehensionClauses()
		if clauses != nil {
			sc := p.openComments()
			expr := p.parseStruct()
			sc.closeExpr(p, expr)

			var fallbackClause *ast.FallbackClause
			if p.tok == token.ELSE || p.tok == token.FALLBACK {
				fallbackClause = p.parseFallbackClause(clauses)
			}

			if p.atComma("list literal", token.RBRACK) { // TODO: may be EOF
				p.next()
			}

			return &ast.Comprehension{
				Clauses:  clauses,
				Value:    expr,
				Fallback: fallbackClause,
			}, true
		}

		expr = &ast.Ident{
			NamePos: pos,
			Name:    tok.String(),
		}
		fc.closeNode(p, expr)

	default:
		expr = p.parseUnaryExpr()
	}

	expr = p.parseBinaryExprTail(token.LowestPrec+1, expr)
	expr = p.parseAlias(expr)

	// Enforce there is an explicit comma. We could also allow the
	// omission of commas in lists, but this gives rise to some ambiguities
	// with list comprehensions.
	if p.tok == token.COMMA && p.lit != "," {
		p.next()
		// Allow missing comma for last element, though, to be compliant
		// with JSON.
		if p.tok == token.RBRACK || p.tok == token.FOR || p.tok == token.IF {
			return expr, false
		}
		p.errf(p.pos, "missing ',' before newline in list literal")
	} else if !p.atComma("list literal", token.RBRACK, token.FOR, token.IF) {
		return expr, false
	}
	p.next()

	return expr, true
}

// parseAlias turns an expression into an alias.
func (p *parser) parseAlias(lhs ast.Expr) (expr ast.Expr) {
	if p.tok != token.BIND {
		return lhs
	}
	pos := p.pos

	// Check if old-style aliases are disallowed
	if p.experiments != nil && p.experiments.AliasV2 {
		p.errf(pos, "old-style alias syntax (=) is not allowed with @experiment(aliasv2); use postfix syntax (~X or ~(K,V))")
		p.next()
		expr = p.parseRHS()
		return expr
	}

	p.next()
	expr = p.parseRHS()
	if expr == nil {
		panic("empty return")
	}
	switch x := lhs.(type) {
	case *ast.Ident:
		p.checkDeclIdent(x)
		return &ast.Alias{Ident: x, Equal: pos, Expr: expr}
	}
	p.errorExpected(p.pos, "identifier for alias")
	return expr
}

// parsePostfixAlias parses the postfix alias syntax: ~X or ~(K,V)
// Returns nil if no alias is present.
func (p *parser) parsePostfixAlias() *ast.PostfixAlias {
	if p.tok != token.TILDE {
		return nil
	}

	pos := p.pos

	// Check if postfix alias syntax requires experiment
	if p.experiments == nil || !p.experiments.AliasV2 {
		p.errf(pos, "postfix alias syntax requires @experiment(aliasv2)")
	}

	p.next()

	switch p.tok {
	case token.LPAREN:
		// Dual form: ~(K,V)
		lparen := p.pos
		p.next()

		if p.tok != token.IDENT {
			p.errorExpected(p.pos, "identifier for label alias")
			return nil
		}
		k := p.parseIdent()

		comma := p.expect(token.COMMA)
		if !comma.IsValid() {
			// Recovery: treat as simple form with just K
			return &ast.PostfixAlias{
				Tilde: pos,
				Field: k,
			}
		}

		if p.tok != token.IDENT {
			p.errorExpected(p.pos, "identifier for field alias")
			// Recovery: return what we have
			return &ast.PostfixAlias{
				Tilde:  pos,
				Lparen: lparen,
				Label:  k,
				Comma:  comma,
				Field:  k, // Use K as field too for recovery
			}
		}
		v := p.parseIdent()

		rparen := p.expect(token.RPAREN)

		return &ast.PostfixAlias{
			Tilde:  pos,
			Lparen: lparen,
			Label:  k,
			Comma:  comma,
			Field:  v,
			Rparen: rparen,
		}

	case token.IDENT:
		// Simple form: ~X
		ident := p.parseIdent()
		return &ast.PostfixAlias{
			Tilde: pos,
			Field: ident,
		}

	default:
		p.errorExpected(p.pos, "identifier or '('")
		return nil
	}
}

// checkExpr checks that x is an expression (and not a type).
func (p *parser) checkExpr(x ast.Expr) ast.Expr {
	switch unparen(x).(type) {
	case *ast.BadExpr:
	case *ast.BottomLit:
	case *ast.Ident:
	case *ast.BasicLit:
	case *ast.Interpolation:
	case *ast.Func:
	case *ast.StructLit:
	case *ast.ListLit:
	case *ast.ParenExpr:
		panic("unreachable")
	case *ast.SelectorExpr:
	case *ast.IndexExpr:
	case *ast.SliceExpr:
	case *ast.CallExpr:
	case *ast.UnaryExpr:
	case *ast.BinaryExpr:
	case *ast.PostfixExpr:
	default:
		// all other nodes are not proper expressions
		p.errorExpected(x.Pos(), "expression")
		x = &ast.BadExpr{
			From: x.Pos(), To: p.safePos(x.End()),
		}
	}
	return x
}

// If x is of the form (T), unparen returns unparen(T), otherwise it returns x.
func unparen(x ast.Expr) ast.Expr {
	if p, isParen := x.(*ast.ParenExpr); isParen {
		x = unparen(p.X)
	}
	return x
}

// If lhs is set and the result is an identifier, it is not resolved.
func (p *parser) parsePrimaryExpr() ast.Expr {
	if p.trace {
		defer un(trace(p, "PrimaryExpr"))
	}

	return p.parsePrimaryExprTail(p.parseOperand())
}

func (p *parser) parsePrimaryExprTail(operand ast.Expr) ast.Expr {
	x := operand
L:
	for {
		switch p.tok {
		case token.PERIOD:
			period := p.pos
			c := p.openComments()
			c.pos = 1
			p.next()
			switch p.tok {
			case token.IDENT:
				x = &ast.SelectorExpr{
					X:      p.checkExpr(x),
					Period: period,
					Sel:    p.parseIdent(),
				}
				x = p.wrapOptional(x, p.tok)
			case token.STRING:
				if strings.HasPrefix(p.lit, `"`) && !strings.HasPrefix(p.lit, `""`) {
					str := &ast.BasicLit{
						ValuePos: p.pos,
						Kind:     token.STRING,
						Value:    p.lit,
					}
					p.next()
					x = &ast.SelectorExpr{
						X:      p.checkExpr(x),
						Period: period,
						Sel:    str,
					}
					x = p.wrapOptional(x, p.tok)
					break
				}
				fallthrough
			default:
				if p.tok.IsKeyword() {
					x = &ast.SelectorExpr{
						X:      p.checkExpr(x),
						Period: period,
						Sel:    p.parseKeyIdent(),
					}
					x = p.wrapOptional(x, p.tok)
					break
				}

				pos := p.pos
				p.errorExpected(pos, "selector")
				p.next() // make progress
				x = &ast.SelectorExpr{
					X:      x,
					Period: period,
					Sel:    &ast.Ident{NamePos: pos, Name: "_"},
				}
			}
			c.closeNode(p, x)
		case token.LBRACK:
			x = p.parseIndexOrSlice(p.checkExpr(x))
		case token.LPAREN:
			x = p.parseCallOrConversion(p.checkExpr(x))
		case token.ELLIPSIS:
			if p.experiments.ExplicitOpen {
				pos := p.pos
				c := p.openComments()
				p.next()
				x = c.closeExpr(p, &ast.PostfixExpr{
					X:     p.checkExpr(x),
					Op:    token.ELLIPSIS,
					OpPos: pos,
				})
			} else {
				// Consume the token and give a clear error
				pos := p.pos
				p.next()
				err := errors.Newf(pos, "postfix ... operator requires @experiment(explicitopen)")
				p.errors = errors.Append(p.errors, err)
				// Return a BadExpr to continue parsing
				x = &ast.BadExpr{From: pos, To: p.pos}
			}
		default:
			break L
		}
	}

	return x
}

// If lhs is set and the result is an identifier, it is not resolved.
func (p *parser) parseUnaryExpr() ast.Expr {
	if p.trace {
		defer un(trace(p, "UnaryExpr"))
	}

	switch p.tok {
	case token.EQL:
		if !p.experiments.StructCmp {
			break
		}
		fallthrough
	case token.ADD, token.SUB, token.NOT, token.MUL,
		token.LSS, token.LEQ, token.GEQ, token.GTR,
		token.NEQ, token.MAT, token.NMAT:
		pos, op := p.pos, p.tok
		c := p.openComments()
		p.next()
		return c.closeExpr(p, &ast.UnaryExpr{
			OpPos: pos,
			Op:    op,
			X:     p.checkExpr(p.parseUnaryExpr()),
		})
	}

	return p.parsePrimaryExpr()
}

func (p *parser) tokPrec() (token.Token, int) {
	tok := p.tok
	if tok == token.IDENT {
		switch p.lit {
		case "quo":
			return token.IQUO, 7
		case "rem":
			return token.IREM, 7
		case "div":
			return token.IDIV, 7
		case "mod":
			return token.IMOD, 7
		default:
			return tok, 0
		}
	}
	return tok, tok.Precedence()
}

// If lhs is set and the result is an identifier, it is not resolved.
func (p *parser) parseBinaryExpr(prec1 int) ast.Expr {
	if p.trace {
		defer un(trace(p, "BinaryExpr"))
	}
	p.openList()
	defer p.closeList()

	return p.parseBinaryExprTail(prec1, p.parseUnaryExpr())
}

func (p *parser) parseBinaryExprTail(prec1 int, x ast.Expr) ast.Expr {
	for {
		op, prec := p.tokPrec()
		if prec < prec1 {
			return x
		}
		c := p.openComments()
		c.pos = 1
		pos := p.expect(p.tok)
		x = c.closeExpr(p, &ast.BinaryExpr{
			X:     p.checkExpr(x),
			OpPos: pos,
			Op:    op,
			// Treat nested expressions as RHS.
			Y: p.checkExpr(p.parseBinaryExpr(prec + 1)),
		})
	}
}

func (p *parser) parseInterpolation() (expr ast.Expr) {
	c := p.openComments()
	defer func() { c.closeNode(p, expr) }()

	p.openList()
	defer p.closeList()

	cc := p.openComments()

	lit := p.lit
	pos := p.pos
	p.next()
	last := &ast.BasicLit{ValuePos: pos, Kind: token.STRING, Value: lit}
	exprs := []ast.Expr{last}

	// Note: we can only tell if the string returned by ResumeInterpolation
	// starts a new interpolated expression by whether it ends in a parenthesis.
	for strings.HasSuffix(last.Value, "(") {
		c.pos = 1
		p.expect(token.LPAREN)
		cc.closeExpr(p, last)

		exprs = append(exprs, p.parseRHS())

		cc = p.openComments()
		if p.tok != token.RPAREN {
			p.errorExpected(p.pos, "')' for string interpolation")
		}
		lit = p.scanner.ResumeInterpolation()
		pos = p.pos
		p.next()
		last = &ast.BasicLit{
			ValuePos: pos,
			Kind:     token.STRING,
			Value:    lit,
		}
		exprs = append(exprs, last)
	}
	cc.closeExpr(p, last)
	return &ast.Interpolation{Elts: exprs}
}

// Callers must check the result (using checkExpr), depending on context.
func (p *parser) parseExpr() (expr ast.Expr) {
	if p.trace {
		defer un(trace(p, "Expression"))
	}

	c := p.openComments()
	defer func() { c.closeExpr(p, expr) }()

	return p.parseBinaryExpr(token.LowestPrec + 1)
}

func (p *parser) parseRHS() ast.Expr {
	x := p.checkExpr(p.parseExpr())
	return x
}

// ----------------------------------------------------------------------------
// Declarations

func isValidImport(lit string) bool {
	const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD"
	s, _ := literal.Unquote(lit) // go/scanner returns a legal string literal
	if p := strings.LastIndexByte(s, ':'); p >= 0 {
		s = s[:p]
	}
	for _, r := range s {
		if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) {
			return false
		}
	}
	return s != ""
}

func (p *parser) parseImportSpec(_ int) *ast.ImportSpec {
	if p.trace {
		defer un(trace(p, "ImportSpec"))
	}

	c := p.openComments()

	var ident *ast.Ident
	if p.tok == token.IDENT {
		ident = p.parseIdentDecl()
		if internal.IsDef(ident.Name) {
			p.errf(p.pos, "cannot import package as definition identifier")
		}
	}

	path := p.lit
	pos := p.expect(token.STRING)
	if pos.IsValid() && !isValidImport(path) {
		p.errf(pos, "invalid import path: %s", path)
	}
	p.expectComma() // skip over a comma or newline
	// collect imports
	spec := &ast.ImportSpec{
		Name: ident,
		Path: &ast.BasicLit{ValuePos: pos, Kind: token.STRING, Value: path},
	}
	c.closeNode(p, spec)
	p.imports = append(p.imports, spec)

	return spec
}

func (p *parser) parseImports() *ast.ImportDecl {
	if p.trace {
		defer un(trace(p, "Imports"))
	}
	c := p.openComments()

	ident := p.parseIdent()
	var lparen, rparen token.Pos
	var list []*ast.ImportSpec
	if p.tok == token.LPAREN {
		lparen = p.pos
		p.next()
		p.openList()
		for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ {
			list = append(list, p.parseImportSpec(iota))
		}
		p.closeList()
		rparen = p.expect(token.RPAREN)
		p.expectComma() // skip over a comma or newline
	} else {
		list = append(list, p.parseImportSpec(0))
	}

	d := &ast.ImportDecl{
		Import: ident.Pos(),
		Lparen: lparen,
		Specs:  list,
		Rparen: rparen,
	}
	c.closeNode(p, d)
	return d
}

// ----------------------------------------------------------------------------
// Source files

func (p *parser) parseFile() *ast.File {
	if p.trace {
		defer un(trace(p, "File"))
	}

	c := p.comments

	// Don't bother parsing the rest if we had errors scanning the first
	// Likely not a CUE source file at all.
	if p.errors != nil {
		return nil
	}
	p.openList()

	var decls []ast.Decl

	for p.tok == token.ATTRIBUTE {
		decls = append(decls, p.parseAttribute(true))
		p.consumeDeclComma()
	}

	v := p.cfg.Version
	exp, err := cueexperiment.NewFile(v, p.expList...)
	if err != nil {
		e := errors.Wrapf(err, p.pos, "parsing experiments for version %q", v)
		p.errors = errors.Append(p.errors, e)
		// Do not proceed without setting p.experiments.
		return nil
	}
	p.experiments = exp
	p.file.SetExperiments(exp)

	// The package clause is not a declaration: it does not appear in any
	// scope.
	if p.tok == token.IDENT && p.lit == "package" {
		c := p.openComments()

		pos := p.pos
		var name *ast.Ident
		p.expect(token.IDENT)
		name = p.parseIdent()
		if name.Name == "_" && p.cfg.Mode&DeclarationErrors != 0 {
			p.errf(p.pos, "invalid package name _")
		}
		if internal.IsDef(name.Name) {
			p.errf(p.pos, "invalid package name %s", name.Name)
		}
		pkg := &ast.Package{
			PackagePos: pos,
			Name:       name,
		}
		decls = append(decls, pkg)
		p.expectComma() // skip over a comma or newline
		c.closeNode(p, pkg)
	}

	for p.tok == token.ATTRIBUTE {
		decls = append(decls, p.parseAttribute(false))
		p.consumeDeclComma()
	}

	if p.cfg.Mode&PackageClauseOnly == 0 {
		// import decls
		for p.tok == token.IDENT && p.lit == "import" {
			decls = append(decls, p.parseImports())
		}

		if p.cfg.Mode&ImportsOnly == 0 {
			// rest of package decls
			// TODO: loop and allow multiple expressions.
			decls = append(decls, p.parseFieldList()...)
			p.expect(token.EOF)
		}
	}
	p.closeList()

	f := &ast.File{
		Imports:         p.imports,
		Decls:           decls,
		LanguageVersion: p.cfg.Version,
	}
	c.closeNode(p, f)
	return f
}
