% Copyright 2012-2024, Alexander Shibakov
% This file is part of SPLinT
%
% SPLinT is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% SPLinT is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with SPLinT.  If not, see <http://www.gnu.org/licenses/>.

% prototypes of all the macros produced by the parser
% we will follow the convention: 12string is a string of category 12 characters and spaces, tex_string: a string
% of TeX tokens; sptr is a pointer to the stash stream, fptr is a pointer to the format stream

\def\yyuniontag{\yyunion}
\def\parserstrictnamespace{parser-strict}
\def\parserprototypesnamespace{parser-strict:headers}
\def\parserdebugnamespace{parser-debug}

\def\yyunion{\currentyyunionnamespace}
\def\currentyyunionnamespace{generic}

% types inserted by the input routines: these should not be tampered with randomly
% therefore they are not included in the yyunion list and only defined once below and redefined later
% permanently

\def\cwebstreamchars{% characters seen by the input routine
    \|\`\\\{\}\~\ \_\&\^\$\#\n% the last one is inserted by the preprocessing scripts
}

% insidemost (level 0)

\long\def\stashed#1{} % stashed material (usually \Cee\ code) :: \stashed{tex_string}
\long\def\sflush#1#2{} % stash flushing marker :: \sflush{tex_string}
\def\format#1{} % format material triggered by braces and the start of production :: \format{tex_string}
\def\formatlocal#1{} % format material triggered by every term :: \formatlocal{tex_string}

% level 1---atoms minimally processed by the parser

% types returned by the lexer; the integer type (hex vs. decimal) is determined at the lexer level so
% these are tagged by the lexer as well

\defp\anint#1#2#3{} % INT :: \anint{digits}{fptr}{sptr}
\defp\hexint#1#2#3{} % INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}

% types inserted by the parser (* marks the types that are essentially returned by the lexer
% and `tagged' by the parser)

\defp\idit#1#2#3#4{} % *ID :: \idit{12string}{tex_string}{fptr}{sptr}
\defp\charit#1#2#3#4{} % *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
\defp\stringify#1#2#3#4{} % *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
\defp\tagit#1#2#3#4{} % *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
\defp\braceit#1#2#3{} % *BRACED_CODE :: \braceit{tex_string}{fptr}{sptr}
\defp\midf#1#2{} % *rule separator :: \midf{fptr}{sptr}
\defp\preckind#1#2#3{} % *precedence operator :: \preckind{12string}{fptr}{sptr}
%
\defp\termname#1#2{} % a production term :: \termname{\idit|\stringify|\charit}{\idit}
\defp\mergeop#1#2#3{} % merge directive :: \mergeop{\tagit}{fptr}{sptr}
\defp\dprecop#1#2#3{} % dprec directive :: \dprecop{\anint|\hexint}{fptr}{sptr}
\defp\sprecop#1#2#3{} % prec directive :: \sprecop{\idit|\stringify|\charit}{fptr}{sptr}
\defp\rrhssep#1#2{} % separator :: \rrhssep{fptr}{sptr}
\defp\arhssep#1#2{} % separator (between the action and the right hand side) :: \arhssep{fptr}{sptr}
\defp\rarhssep#1#2{} % separator :: \rarhssep{fptr}{sptr}
\defp\hspace#1#2{} % a space has to take two parameters just like any connector :: \hspace{fptr}{sptr}
\defp\emptyterm{} % implicit empty right hand side :: \emptyterm
\defp\onesymbol#1#2#3{} % a symbol definition :: \onesymbol{\idit}{\anint|\hexint}{\stringify}
\defp\bracedvalue#1#2#3{} % a braced variable value :: \bracedvalue{tex_string}{fptr}{sptr}

% level 2---the structures built by the parser

\defp\prodheader#1#2#3#4{} % a `left-hand side' of a production :: \prodheader{\idit}{\idit}{fptr}{sptr}
\defp\rhs#1#2#3{} % a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
\defp\actbraces#1#2#3#4#5\bdend{} % action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
\defp\bpredicate#1#2#3#4#5\bdend{} % predicate :: \bpredicate{tex_string}{<nothing>}{fptr}{sptr}directive*\bdend

% level 3---the structures composed of structures above

\defp\rules#1{} % a complete right-hand side :: \rules{[\rhs|\rrhssep]}
\defp\symbolprec#1#2{} % symbol precedence :: \symbolprec{\idit|\stringify|\charit}{\anint|\hexint}
%\def\pcluster#1#2{} % a production cluster :: \pcluster{\prodheader}{\termname|...} should never appear in the final output

% level 4---larger structures and top level constructions

\defp\oneproduction#1#2#3{} % a complete production :: \oneproduction{\prodheader\rules}{fptr}{sptr}
\defp\prodprodsepsmall{} % production separator
\defp\prodprodsep{} % production separator
\defp\prodprodseplarge{} % production separator
\defp\optoptsepsmall{} % separator between options
\defp\optoptsepnone{} % (trivial) separator between options
\defp\posmark#1#2{} % the position marker :: \posmark{fptr}{sptr}
\defp\tokendecls#1#2#3{} % token declarations :: \tokendecls{[\onesymbol]+}{fptr}{sptr}
\defp\ntermdecls#1#2#3{} % nterm declarations :: \ntermdecls{[\onesymbol]+}{fptr}{sptr}
\defp\typedecls#1#2#3#4{} % type declarations :: \typedecls{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
\defp\precdecls#1#2#3#4#5{} % precedence declarations :: \precdecls{left|right|nonassoc|precedence}{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
\defp\oneparametricoption#1#2#3#4{} % option :: \oneparametricoption{option-name}{\stringify|\idit}{fptr}{sptr}
\defp\optionflag#1#2#3#4{} % option flag :: \optionflag{flag name}{abbreviated name}{fptr}{sptr}
\defp\paramdef#1#2#3#4{} % parameters :: \paramdef{\braceit}{parameter domain}{fptr}{sptr}
\defp\codepropstype#1#2#3#4#5#6#7{} % term method :: \codepropstype{printer|destructor}{tex_string}{\stringit|\idit}{fptr}{sptr}{fptr}{sptr}
\defp\initaction#1#2#3{} % initial action :: \initaction{tex_string}{fptr}{sptr}
\defp\codeassoc#1#2#3#4#5#6#7{} % code association :: \codeassoc{code kind}{|\idit}{tex_string}{fptr}{sptr}{fptr}{sptr}
\defp\vardef#1#2#3#4{} % variable definition :: \vardef{\idit|\stringify}{|\idit|\stringify|\bracedvalue}{fptr}{sptr}
\defp\prologuecode#1#2#3#4#5{} % prologue code :: \prologuecode{tex_string}{fptr}{sptr}{fptr}{sptr}

\savecslist{parser-prototypes}\yyunion

% table construction: the macros run to build the contents of the table of productions to be
% typeset.

\newtoks\table
\newtoks\tokdectoks
\newtoks\opttable
\newtoks\typestable
\newtoks\prectable

\newtoks\tokstoks
\newtoks\typesset

\let\rhscont\getsecond
\let\rhscnct\getthird
\let\rhsbool\getfourth

\newif\ifrhsfull

\newtoks\yystashlocal
\newtoks\yyformatlocal

\def\astappend{\appendtolist\aststream}
\def\astappendx{\appendtolistx\aststream}
\def\astisempty{\listisempty\aststream}

% index implementation
% note that the rank and domain parameters are handed down by the
% `outer' macro while the `qualifier' (\term...string) is decided
% at the point of invocation; some auxilary sequences first

% a macro that always produces the current value of the register

\def\thewrap#1#2{%
    \noexpand\thewrap{\noexpand#1}{\the#1}%
}

% a macro that always produces the current page number.

\def\selfpageno#1{%
   \noexpand\selfpageno{\the\pageno}%
}

% a macro that keeps the (single) argument intact

\def\stickycs#1{\noexpand\stickycs\noexpand#1}

% for finer control over indexing, there are two levels on which
% the programmer can disable the production of a whatsit node (\write):
% (1) by setting the \iftermindex conditional to \iffalse, and (2) by defining
% the \termindex macro to ignore its argument; both of these are expanded
% at the point where the index entry is made; the intention is that
% the conditional controls indexing at large (say, the index entries
% inside the index should not be indexed again), while redefining the macro
% provides a finer control over the indexing style

\newif\iftermindex

\let\termindex\eatone

% the index is split into different domains: say, the \bison\ domain contains
% the entries for the terms of a grammar;

\def\bisonidxdomain{B} % marker for the domain (section of the index)

\def\defidxrank{0} % index rank of definitions is lowest so they are listed first
\def\headeridxrank{1} % index rank of lhs
\def\termidxrank{2} % ordinary term rank

% the expansion of the various components of an index entry must be carefully
% controlled: it is not possible to predict how many times the entry will be expanded;
% there are four types of expandable entries in general:
% o the entries that must be expanded at the point of invocation (say, \secno)
% o the entries that must be expanded only when the \write is inside a \box that is
%   being shipped out (say, the \pageno)
% o the `lazy' entries that can be expanded any time (although at least one expansion
%   must take place)
% o finally, the entries that must be expanded at a specific point between the point
%   of invocation and the \shipout
% the indexing macro below treats the first three cases for the entries that the macros
% below produce; the last case can be implemented by redefining the \stickycs macro
% at the appropriate point

\def\indxe#1#2{% index entry:
               % #1 is the output stream
               % #2 is the entry (see below for the proper form)
    \expandafter\ind@@\expandafter#2\end{#1}%          
}

\def\ind@@#1\end#2{%
    \ind@e{#2}{#1}%
}

\def\ind@e#1#2{% #1 is the output stream
               % #2 is a 8-parameter sequence, see below
    \write#1{\in@@e#2\in@@ecomment#2}%
}

\def\in@@e#1#2#3#4#5#6#7#8{% #1: early expandable (like \secno)
                           % #2: auxilary info (expandable, invariant)
                           % #3: domain
                           % #4: rank
                           % #5: type1
                           % #6: type2
                           % #7: key1
                           % #8: key2
    \i@@@e{#1}{\errmessage{\noexpand\\in@@e error: expanded page number placeholder}}%
        {#2}{#3}{#4}{#5}{#6}{\charstonumberse#7\end}{\charstonumberse#8\end}%
}

\def\in@@ecomment#1#2#3#4#5#6#7#8{% #1: early expandable (like \secno)
                                  % #2: auxilary info (expandable, invariant)
                                  % #3: domain
                                  % #4: rank
                                  % #5: type1
                                  % #6: type2
                                  % #7: key1
                                  % #8: key2
    \harmlesscomment\space\charstocharsx#7\end :: \charstocharsx#8\end%
}

\def\i@@@e#1#2#3#4#5#6#7#8#9{% #1: early expandable (like \secno)
                             % #2: page number placeholder
                             % #3: auxilary info
                             % #4: domain
                             % #5: rank
                             % #6: type1
                             % #7: type2
                             % #8: key1
                             % #9: key2
    \noexpand\i@@@e{#1}{\the\pageno}{#3}{#4}{#5}{\noexpand#6}{\noexpand#7}{#8}{#9}%
}

\def\appendfitotoksa{\toksa\expandafter{\the\toksa\else<>\fi}} % hide \fi inside the loop

% defining expandable macros for string transformations

\tempca\@ne
\toksa{\ifcase#1<NUL>}
\tempcb=\uccode`.

% a loop that builds a macro to produce a category 12 character with the given character code

\bloop
\uccode`.=\tempca
\uppercase{\toksb{.}}%
\toksa\expandafter\expandafter\expandafter{\expandafter\the\expandafter\toksa\expandafter\or\the\toksb}%
\advance\tempca by\@ne
\ifnum\tempca>"FF
    \appendfitotoksa
\else
\repeat

% defining the macro that was built above

\edef\next{\toksa{\def\noexpand\asciiprint##1{\the\toksa}}}\next
\the\toksa

% return . to its regular state

\uccode`.=\tempcb

\def\charstocharsx#1\end{% expandable version of a `safety' macro, ascii characters only, no tricky space characters
    \yystartsinspace{#1.}{%
        \space\ch@rstocharsx#1\end
    }{%
        \yystringempty{#1}{}{%
            \ch@rst@charsx#1\end
        }%
    }%
}

\def\ch@rstocharsx#1 #2\end{\charstocharsx#2\end}

\def\ch@rst@charsx#1#2\end{\asciiprint{`#1}\charstocharsx#2\end}

\def\writeidxhentry#1{%
      \indxe\gindex{{\secno}{{\currentrulecontext}{\hostparsernamespace}}{\bisonidxdomain}{\headeridxrank}#1}%
}

\def\writeidxentry#1{%
      \indxe\gindex{{\secno}{{\currentrulecontext}{\hostparsernamespace}}{\bisonidxdomain}{\termidxrank}#1}%
}

\def\writetokenidxentry#1{%
      \indxe\gindex{{\secno}{{\currentrulecontext}{\hostparsernamespace}}{\bisonidxdomain}{\defidxrank}#1}%
}

\def\gidxentry#1#2#3{% a generic index entry:
                     % #1 is the type processor,
                     % #2 is the key
                     % #3 is the visual key
    \iftermindex
        \termindex{{}{#1}{#2}{#3}}%
    \fi
}

\def\ridxentry#1#2#3{% index entry that allows raw input (see 
    \gidxentry{#1}{\end0 \vend{32}\end\toksa{#2}\noexpand\noexpand\noexpand\eatone\vend\eatone}{#3}%
    % the first \end and \eatone will be removed by \charsto..., the next (innermost) by \numbersto... inside \FI
}

\def\didxentry#1#2{% (another) index entry that allows raw input (see \csname flex@ption_other\endcsname below)
    \gidxentryxb{\termraw}{\number#1}{#2}%
}

\def\gtextidx#1#2#3{% #1 is the entry
                    % #2 is the visual key
                    % #3 is the domain
    \expandafter\ifx\csname indexterms<count>\endcsname\relax
    \else
    {%
        \storerawidxentry{#1}%
        \edef\next{%
            \noexpand\indxe\gindex{{\secno}{{}{\indexpseudonamespace}}{#3}{\termidxrank}{}%
                {\noexpand\termraw}{\number\getstackpointer\indexterms}{#2}}%
        }\next
        \edef\next{\next}\next
    }%
    \fi
}

% a shortcut to simplify using the macro above

\def\storerawidxentry#1{%
    {%
        \toksa{#1}%
        \let\parsernamespace\empty\yypushg{\the\toksa}\on\indexterms
    }%
}

% a convenient shortcut, as many visual keys come in a token register

\def\gidxentryxv#1#2#3{% a generic index entry, e(x)pand (v)isual key:
                       % #1 is the type processor,
                       % #2 is the key
                       % #3 is the visual key
    \expandafter\gidx@ntryxv\expandafter{#3}{#1}{#2}%
}

\def\gidxentryxb#1#2#3{% a generic index entry, e(x)pand (b)oth keys:
                       % #1 is the type processor,
                       % #2 is the key
                       % #3 is the visual key
    \expandafter\gidx@ntryxb\expandafter{#2}{#3}{#1}%
}

\def\gidx@ntryxb#1#2#3{%
    \expandafter\gidx@ntryxv\expandafter{#2}{#3}{#1}%
}

\def\gidx@ntryxv#1#2#3{%
    \gidxentry{#2}{#3}{#1}%
}

\def\tidxentry#1#2{% a text index entry, TODO: expand \currentrulecontext 
    \iftermindex
        \indxe\gindex{{\secno}{{\currentrulecontext}{\indexpseudonamespace}}{\bisonidxdomain}{\termidxrank}{}{#1}{#2}{}}%
    \fi
}

\def\idxinline#1{% index the term in text
    \edef\next{\tidxentry{\termidstring}{#1}}\next
}

% stash processing wrapper macros

\def\yyreadstash#1\to#2\in#3\with#4{%
    \consumelistupto{#1}{#2}\to{#3}%
    #4{#3}%
    \mathsplittoks{#3}{\yyr@adstash}{\eatthree}{#1}{#2}{#3}%
}

% this macro is based on the assumption that all the math mode
% changes are introduced by unadulterated $ tokens (as is done in 
% \CWEB); \CWEB's handling of math mode is a mistake so there is
% no need to introduce another one by using clever math switches;
% note that this is not affected by \inmath (see somewhere in this file),
% since (as long as!) it is only inserted in the verbatim mode ...
% ... or at least it should, so never do something like 
%    @t}\inmath{{@> ... @t}}@>
% unless you really know what you are doing (hint: it is only ok
% to do this if there are no intervening \vb{...} groups inside
% the ... portion above.

\def\mathsplittoks#1#2#3{% execute actions depending on whether 
                         % token register #1 contains
                         % odd (then insert #2)
                         % or even (insert #3) number of $'s
    \expandafter\evenmathify\the#1$%
        \evenmather\end$\evenmather\end$\evenmather\end$%
        \evenmather\end$\evenmather\end$\evenmather\end$%
        \emo$\eme$\emo$\eme$\emo$\eme$\emo$\eme$\emo$\end
        {#2}{#3}%
}

\def\evenmathcontainer{\evenmather\end}

\def\evenmather#1$#2$#3$#4$#5$#6${}

\def\emo#1\end{\yyfirstoftwo}

\def\eme#1\end{\yysecondoftwo}

\def\evenmathify#1$#2$#3$#4$#5$#6${%
    {% make all changes to \next local
        \def\next{#6}%
        \ifx\next\evenmathcontainer
            \yybreak{}%
        \else
            \yybreak{\aftergroup\evenmathify}%
        \yycontinue
    }%
}

\def\yyr@adstash#1#2#3{%
    #3\expandafter{\the#3{}$}% $upply the math mode switch
    {%
        \tempca#2\relax
        \advance\tempca\@ne
        \edef\next{%
            \noexpand\changelistelem{#1}{#2}{}{{}$}% change the consumed stash, so it can be read again
            \noexpand\changelistelem{#1}{\the\tempca}{{}${}}{}%
        }\expandafter
    }\next
}

\def\readstash#1{%
    \yyreadstash\yystash\to#1\in\yystashlocal\with\eatone
}

\def\readstashwredirect#1#2#3{% helper macro to read the stash when the stash and the format pointers appear together
                              % #1(the format pointer) is ignored
                              % #2 is the stash marker
                              % #3 is the local stash register
    \yyreadstash\yystash\to#2\in#3\with\eatone
}

% format processing wrapper macros

\def\readformat#1{%
    \let\listiterator\listiteratorsimple
    \consumelistupto\yyformat{#1}\to\yyformatlocal
}

\def\readformatlocalonly#1{%
    \let\listiterator\listiteratorfmtloc
    \let\wrapupfmtlocal\wrapupyyformatlocal
    \yyformatlocal{}%
    \evallistbetween\yyformat{\showlistpointer\yyformat}{#1}%
}

% iterator to extract \formatlocal

\def\listiteratorfmtloc#1#2{%
    \lookforfmtloc{#1}{}{}.#2\formatlocal{\formatlocal}%
}

\def\lookforfmtl@c#1#2#3#4#5{%
    \isthisfmtlocal#5.\formatlocal{%
        \wrapupfmtlocal{#2}{#3#1}{#4}%
    }{%
        \lookforfmtloc{#2}{#3#1}{#4\formatlocal{#5}}.%
    }%
}

\def\lookforfmtloc#1#2#3#4\formatlocal#5{% remove the guard .
    \expandafter\lookforfmtl@c\expandafter{\eatone#4}{#1}{#2}{#3}{#5}%
}


\def\isthisfmtlocal#1\formatlocal{%
    \yystringempty{#1}{% a match, clean up the inserted \formatlocal
        \endoffmtlocal
    }{% not a match yet, \lookforfmtloc again
        \yysecondoftwo
    }%  
}

\def\endoffmtlocal#1\formatlocal{\yyfirstoftwo}

\def\wrapupfmtlocal#1#2#3{% a postprocessing macro, to be changed by the one that
                          % invokes the iterator
    \fdef#1{#2}%
}

\def\wrapupyyformatlocal#1#2#3{% macro used to save the \formatlocal contents in \yyformatlocal
    \fdef#1{#2}\yyformatlocal\expandafter{\the\yyformatlocal#3}%
}

\let\wrapupfmtlocal\wrapupyyformatlocal

% use example for the macros above:
%\initlist{table}
%\toksa{#}
%\let\listiterator\listiteratorfmtloc
%\appendtolisti{table}{..}
%\appendtolisti{table}{a,}
%\appendtolisti{table}{\formatlocal{}{b}\formatlocal{k} , }
%\appendtolisti{table}{{c},}
%\appendtolisti{table}{ \space d,}
%\appendtolisti{table}{e\formatlocal{d},}
%\appendtolisti{table}{\format{}{f}\formatlocal{j}}
%\appendtolisti{table}{.}
%\appendtolisti{table}{\end}
%\finishlist{table}
%\storelist{table}\to\toksa
%\the\toksa
%\storelist{table}\to\toksa
%\message{full list: \the\toksa}
%\errmessage{separated: \the\yyformatlocal}

% parser invocation

\def\newparserstateextra{}

\def\d@parse#1{\yyparse#1\yyeof\yyeof\endparseinput\endparse}

% as an alternative to lists, stacks (as arrays) can be used to speed up the parser;
% the following setup (along with a convenient shortcut to reference the most
% recent store) can be employed if desired; unlike lists, stacks do not have
% a built-in mechanism for slice expansion.
%\yyinitstack\astarray % flattened AST to (possibly) speed up the parser
%\def\astarraylastcs{\expandafter\noexpand\romannumeral\gettopofstackcsx\astarray}

\long\def\inmath#1{$#1$}

%\long\def\stashed#1{\toksa\expandafter{\the\toksa#1}} % to be used with unpackaged stash
%\long\def\stashed#1{\toksa\expandafter\expandafter\expandafter{\expandafter\the\expandafter\toksa#1}} % to use with AST array
\long\def\stashed#1{\errmessage{making \noexpand\stashed\space expandable is deprecated}}

\long\def\sflush#1#2{}

\def\ntt{\noexpand\ntt@} % a convenient shortcut
\def\ntt@{\tt\let\_\UL}

% macros that process the bison grammar terms;
% the higher level macros (like the \prodheader below) set up the sequence in
% which the processing takes place; the actual typesetting (or whatever other
% task is desired, like reference creation) is done by the lower level macros
% like \prodhead@r in this case.

\defc\prodheader{% a `left-hand side' of a production :: \prodheader{\idit}{\idit}{fptr}{sptr}
    \readformat{#3}\the\yyformatlocal
    \let\termindex\writeidxhentry
    \prodhead@r{#1}{#2}%
    \let\termindex\eatone
}

\def\prodhead@r#1#2{%
    \let\idit\prodhdridit
    #1\relax\astappendx{\the\toksa}\hrhss@p
}

\def\hrhss@p{\toksa{&\omit\hfil&\omit\hfil&\omit\hfil\cr\noalign{\penalty100}\omit\hfil&\termspostformat}\astappendx{\the\toksa}}

\defc\rrhssep{% separator :: \rrhssep{fptr}{sptr}
    \readformatlocalonly{#1}%
    \the\yyformatlocal % this will have a chance to change the definition of \rrhss@p
    \rrhss@p
}

\def\rrhss@p{\toksa{\omit\hfil&}\astappendx{\the\toksa}}

\defc\arhssep{% separator (between the action and the right hand side) :: \arhssep{fptr}{sptr}
    \readformat{#1}%
    \the\yyformatlocal % this will have a chance to change the definition of \arhss@p
    \arhss@p
}

%% between the inline action and next term:

\def\arhss@p{\toksa{\omit\hfil&\hbox to 2em{\hfil}}\astappendx{\the\toksa}}

\defc\rarhssep{% separator :: \rarhssep{fptr}{sptr}
    \readformat{#1}%
    \the\yyformatlocal % this will have a chance to change the definition of \rarhss@p
    \rarhss@p
}

%% between a term and the following action

\def\rarhss@p{\astappend&}

\defc\hspace{% a space has to take two parameters just like any connector :: \hspace{fptr}{sptr}
    \readformatlocalonly{#1}%
    \the\yyformatlocal % may change \hspac@
    \hspac@
}

\def\hspac@{%
    \astappend{ }%
}

\defc\rhs{#1}% a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}

\def\supplybdirective#1#2{%
    \edef\next{\nx\s@pplybdirective#1{\the#2}\the#1\nx\bdend}\next
}

\def\s@pplybdirective#1#2#3\bdend\bdend{%
    #1{#3#2\bdend}%
}

\defc\termname{% a production term :: \termname{\idit|\stringify|\charit}{\idit}
    \let\idit\termnameidit
    \let\stringify\termnamestringify
    \let\optstrextra\optstrextraesc
    \let\termindex\writeidxentry
    \termnam@{#1}{#2}%
    \let\termindex\eatone
}

\def\termnam@#1#2{\toksa{}\toksb{}\toksc{}#1\expandafter\termn@m@\expandafter{\the\toksa}{#2}}

\def\termn@m@#1#2{\toksa{}\toksb{}\toksc{}#2\expandafter\t@rmn@m@\expandafter{\the\toksa}{#1}}

% several versions of term typesetting

\def\t@rmn@m@#1#2{% the termname is #2, the bracketed part is in #1
    \yystringempty{#1}{\toksa{#2}}{\toksa{#2\hbox{$\langle${}#1{}$\rangle$}}}%
    \astappendx{\the\toksa}%
}

\newif\ifbracketedvisible % decide whether to show the bracketed name in productions

\def\t@rmn@m@#1#2{% the termname is #2, the bracketed part is in #1
    \yystringempty{#1}{\toksa{#2}}{%
        \ifbracketedvisible
            \toksa{#2\hbox{$\,\cdot$#1}}%
        \else
            \toksa{#2}%
        \fi
    }%
    \astappendx{\the\toksa}%
}

%

\newtoks\bdirects

\defc\actbraces{% action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
    \readformat{#3}%
    #5% accumulate all the directives
    \yytoksempty\bdirects{}{%
        \edef\next{\bdirects{%
             $\vtop{\halign{\nx\quad####\nx\hfil&####\nx\hfil\cr\the\bdirects}}$}}\next
    }%
    \the\yyformatlocal
    \actbrac@s{#1}{#2}{#4}%
}

\def\actbrac@s#1#2#3{%
    \readstash{#3}%
    \astappendx{\the\bdirects&\toksa{\the\yystashlocal}\cr}%
    \bdirects{}%
}

\def\bpredicate#1#2#3#4#5\bdend{%
    \readformat{#3}%
    #5% accumulate all the directives
    \yytoksempty\bdirects{%
        \bdirects{\quad$\dashv\ \nx\vdash$}%
    }{%
        \edef\next{\bdirects{%
             $\vtop{\halign{\nx\quad####\nx\hfil&####\nx\hfil\cr$\dashv\ $&\omit\nx\hfil$\nx\vdash$\cr
                                                                \the\bdirects}}$}}\next
    }%
    \the\yyformatlocal
    \bpredicat@{#1}{#2}{#4}%
}

\def\bpredicat@#1#2#3{% 
    \readstash{#3}%
    \astappendx{\the\bdirects&\toksa{\the\yystashlocal}\cr}%
    \bdirects{}%
}

\def\bdirective#1#2#3#4{%
    \savecslist{local-namespace}\yyunion
    \restorecslist{table-render:directives}\yyunion
    \toksa{}#2% make a directive list
    \appendr\bdirects{$\nx\langle${\ntt #1} &\the\toksa$\nx\rangle$\nx\cr}%
    \restorecslist{local-namespace}\yyunion
}

\def\mergeop#1#2#3{%
    \bdirective{merge}{#1}{#2}{#3}%
}

\def\dprecop#1#2#3{%
    \bdirective{dprec}{#1}{#2}{#3}%
}

\def\sprecop#1#2#3{%
    \bdirective{prec}{#1}{#2}{#3}%
}

\defc\rules{#1}% a complete right-hand side :: \rules{[\rhs|\rrhssep]}

\defc\oneproduction{#1\setrulecontext{}\stopproduction}% a complete production :: \oneproduction{\prodheader\rules}{fptr}{sptr}

% to make it possible to typeset tokens depending on their context
% the context name is passed to the token typeseting macros; the most
% common context name is the left hand side of a production; special
% context names (such as *left, *right, etc) are defined for terms that are
% not part of a rule; all such context names start with `*' since this
% character cannot be part of a term name

\def\setrulecontext#1{\def\currentrulecontext{#1}}

\let\currentrulecontext\empty

% an empty left hand side is indexed in the index pseudonamespace and not in the main
% parser namespace, since the name parser never encounters this term (as the \emptyterm
% control sequence is inserted by the main parser) while typesetting productions;
% because of this it would be unnatural to define a \prettywordpair sequence for it in the main
% parser namespace; using a special non-\bison\ name should avoid any confusion with
% the regular grammar terms

\def\emptyrhssymbol{$\circ$}

\defc\emptyterm{\astappendx{\nx\emptyrhssymbol\tidxentry{\termexception}{emptyrhs&}}}

\def\termnameidit#1#2#3#4{%
    \readformatlocalonly{#3}%
    \the\yyformatlocal
    \nameproc{#2}\with\parsebin
    \peekstash{#4}% does not clobber \toksa and \toksb
    \expandafter\settermstyle\expandafter{\the\toksb}{#1}%
 }

\def\settermstyle#1#2{% put the \whatsit node (\write) at the end so that the
                      % term macros can remove the whitespace preceeding it
    \edef\next{%
        \toksa{%
            \termmetastyle{%
                \let\nx\idxfont\nx\empty\nx\it\the\toksa
                \gidxentry{\termidstring}{#2}{#1}\nx\/%
            }%
        }%
    }\next
}

\def\termnamestringify#1#2#3#4{%
    \nameproc{#2}\with\parsebin
    \ifyyparsefail
        \toksb{"#1"}\toksc{\termstring}%
    \else
        \toksc{\termidstring}%
    \fi
    \edef\next{\toksa{{%
            \expandafter\gidxentryxv\expandafter{\the\toksc}{#1}{\the\toksb}%
            \ntt\the\toksa
        }}}\next
}

\def\termmetastyle#1{{#1}}

\let\peekstash\eatone

\newif\ifinheader

\def\prodhdridit#1#2#3#4{%
    \setrulecontext{}% TODO: perhaps some special context for the left hand side terms is appropriate
    \nameproc{#2}\with\parsebin
    \setrulecontext{#1}%
    \edef\next{\toksa{{%
            \gidxentryxv{\termidstring}{#1}{\the\toksb}%
            \nx\inheadertrue\let\nx\idxfont\nx\bf\nx\itbold\the\toksa\nx\/%
        }}}\next
}

\defc\charit{% *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
    \expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
        \toksa{#1}\sansfirst\toksa
        \edef\next{\toksa{{\ntt\the\toksa}\gidxentryxb{\termvstring}{\the\toksa}{}}}% remove the possible \\
    \else
        \expandafter\ifx\csname\viskeyref\hostparsernamespace{#1}\endcsname\relax
            \tokse{#1}% we are stripping the quotes so the visible key is adjusted
        \else
            \edef\next{\tokse{\expandafter\expandafter\csname\viskeyref
                \hostparsernamespace{#1}\endcsname\expandafter{\currentrulecontext}}}\next
        \fi
        \edef\next{\expandafter\toksa\expandafter{\expandafter\noexpand
            %\csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termvstring}{'#1'}{\the\tokse}}}%
            \csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termidstring}{'#1'}{\the\tokse}}}%
    \fi
    \next
}

\defc\anint{% INT :: \anint{digits}{fptr}{sptr}
    \astappend{#1}%
}

\defc\hexint{% INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
     \expandafter\astappend\expandafter{\eattwo#1$_{16}$}%
}

\def\optoptsepnone{%
    \concat\opttable\postoks
}

\def\optoptsepsmall{% keep the alignment going but insert a small space
    \concat\opttable\postoks
    \appendrnx\opttable{\noalign{\smallskip}}%
}

\def\prodprodsepsmall{%
    \pushothertables
    \astappend{\noalign{\penalty-150 \smallskip}}%
}

\def\prodprodsep{%
    \pushothertables
    \astappend{\noalign{\medskip}}%
}

\def\prodprodseplarge{%
    \pushothertables
    \astappend{\noalign{\bigskip}}%
}

\def\separatorswitcheq{%
    \oneproduction {%
        \toksd{\prodprodsepsmall}%
    }
    \codepropstype\codeassoc {%
        \toksd{\optoptsepsmall}%
    }
}

\def\separatorswitchdefaulteq{%
    \toksd{\optoptsepnone}%
}

% a switch for inserting position information into the parsed table
% clobbers \toksd

\newtoks\postoks

\def\positionswitch{
    \empty
    {%
        \postoks{}% semicolons should be invisible
    }
    \paramdef\oneparametricoption\optionflag\typedecls\vardef
    {%
        \getfourth{\yy(2)}\to\postoks % the format marker
        \getfifth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
    \prologuecode\ntermdecls\tokendecls
    {%
        \getthird{\yy(2)}\to\postoks % the format marker
        \getfourth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
    \precdecls
    {%
        \getfifth{\yy(2)}\to\postoks % the format marker
        \getsixth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
    \oneproduction
    {%
        \getsecond{\yy(2)}\to\postoks % the \prodheader
        \getfourth{\postoks}\to\toksd % the format marker
        \getfifth{\postoks}\to\postoks % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\toksd}{\the\postoks}}}\next
    }
    \codeassoc\codepropstype
    {%
        \getseventh{\yy(2)}\to\postoks % the format marker
        \geteightth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
}

\def\positionswitchdefault{%
    \errmessage{unknown chunk type in: \the\yy(2)}%
}

\def\posmark#1#2{%
%    \readformat{#1}%
%    no formatting for locations currently
%    \the\yyformatlocal
    \posm@rk{#2}%
}

\def\posm@rk#1{%
    \readstash{#1}%
    \yytoksempty\yystashlocal{\postoks{}}{%
        \toksa\expandafter{\the\yystashlocal}\cleanstash\stripstash
        \setbox\z@=\vbox{\setlazyc\the\toksa}%
        \ifnum\ht\z@=\z@
            \postoks{}%
        \else
            \edef\next{\postoks{\noalign{\indent\toksa{\the\yystashlocal}\nx\makestashbox}}}\next
        \fi
    }%
}

\def\codeoptionlist{%
    \codepropstype\codeassoc\prologuecode
}

\def\simpleoptionsplit{%
    \oneparametricoption\vardef\paramdef\optionflag
}

\def\noncodeoptionlist{%
    \oneparametricoption\vardef\paramdef\optionflag
    \codepropstype\codeassoc\prologuecode
}

\def\alloptionlist{%
    \oneparametricoption\vardef\paramdef\optionflag
    \codepropstype\codeassoc\prologuecode
    \typedecls\tokendecls\precdecls\oneproduction
}

\def\emptyoption{\empty}

\diagprod\codeoptionlist\noncodeoptionlist\in\codenoncodeseplist
\diagprod\simpleoptionsplit\simpleoptionsplit\in\noncodenoncodeseplist
\diagprod\alloptionlist\emptyoption\in\emptyotherseplist

% this is a switch for inserting separators between dissimilar items

\def\separatorswitchneq{% 
    \classexpand\codenoncodeseplist
    {% separate options with code (\codepropstype, \codeassoc) from other options by a small space
        \toksd{\optoptsepsmall}%
    }
    \classexpand\noncodenoncodeseplist
    {%
        \toksd{\optoptsepnone}%
    }
    % a separator after the first rule or before or after a semicolon
    \classexpand\emptyotherseplist
    {%
        \toksd{\optoptsepnone}%
    }
}

\setspecialcharsfrom\separatorswitchneq

\def\separatorswitchdefaultneq{%
    \toksd{\prodprodsep}%
}

\def\pushothertables{%
    %token's
    \yytoksempty\tokdectoks{}{\attachtokentable}%
    %type's
    \yytoksempty\typestable{}{\attachtypestable}%
    %precedence's
    \yytoksempty\prectable{}{\attachprectable}%
    % options
    \yytoksempty\opttable{}{\attachoptionstable}%
    \astappendx{\the\postoks}%
} %

% typesetting token lists, precedence declarations, etc

\def\attachtokentable{% typesetting the token declarations as part of the grammar
    \tempcb3 \tempca\tempcb
    \expandafter\tokdectoks\expandafter{\expandafter}\the\tokdectoks
    \ifnum\tempca=\tempcb
    \else
        \bloop
            \tokdectoks\expandafter{\the\tokdectoks&\omit\hfil}%
        \ifnum\tempca=\z@
        \else
            \advance\tempca\m@ne
        \repeat
        \tokdectoks\expandafter{\the\tokdectoks\cr}%
    \fi
    \toksa{\tt\strut##\hfil\tabskip=0pt plus 1fil}%
    \bloop
        \toksa\expandafter{\the\toksa&\tt\strut##\hfil}%
    \ifnum\tempcb=\tw@
    \else
        \advance\tempcb\m@ne
    \repeat
    \toksa\expandafter{\the\toksa&\tt\strut##\hfil\tabskip\z@\cr}%
    \concat\toksa\tokdectoks
    \tokdectoks\toksa
    \astappendx{\noalign{%
                       \tabskip\parindent
                       %\nx\displaytokenrawtrue % this controls how tokens are displayed
                                                % in the declarations: if true, the macro names will be shown
                       \halign to\hsize{\the\tokdectoks}%
               }%
    }%
    \tokdectoks{}%
}

\def\toksdefline#1#2#3#4#5{%   #1 is the parsed version of the internal bison name
                             % #2 is the index entry
                             % #3 is the type
                             % #4 is the explicit value
                             % #5 is the parsed version of the string value
    \ifnum\tempca=\z@
        \tokdectoks\expandafter{\the\tokdectoks&{#2}#1\hfil\cr}%
        \tempca\tempcb
    \else
        \ifnum\tempcb=\tempca
            \tokdectoks\expandafter{\the\tokdectoks{#2}#1\hfil}%
        \else
            \tokdectoks\expandafter{\the\tokdectoks&{#2}#1\hfil}%
        \fi
        \advance\tempca\m@ne
    \fi
}

\def\attachtypestable{% typesetting the type declarations as part of the grammar
    \toksa{}\the\typestable
    \astappendx{\noalign{{\the\toksa}}}\typestable{}%
}

\long\def\onetype#1#2{%
    \toksb{{%
        \hbadness\@M
        \rightskip=\z@
        \tempda\hsize \advance\tempda-1in 
        \indent{$\langle$\tt union$\rangle.#1$:}\ \hfill\nobreak\hfill\nobreak\null\penalty1\relax\null\nobreak\hfill
        \hbox to\tempda{$\vtop{\noindent\hsize\tempda\raggedright #2\strut}$}\par
    }}%
    \concat\toksa\toksb
}

\def\attachprectable{% typesetting the precedence declarations as part of the grammar
    \toksa{}\the\prectable
    \astappendx{\noalign{{\the\toksa}}}\prectable{}%
}

\long\def\oneprec#1#2#3{%
    \toksc{%
        \hbadness\@M
        \rightskip=\z@
        \tempda\hsize \advance\tempda-1in 
        \indent
    }%
    \toksd{%
        \ \hfill\nobreak\hfill\nobreak\null\penalty1\relax\null\nobreak\hfill
        \hbox to\tempda{$\vtop{\noindent\hsize\tempda\raggedright #3\strut}$}\par
    }
    \def\next{#2}%
    \ifx\next\empty
        \toksb{}%
    \else
        \toksb{${}:{}$#2}%
    \fi
    \edef\next{\toksb{{$\nx\langle$\nx\bf #1\the\toksb$\nx\rangle$}}}\next
    \appendr\toksa{{\the\toksc\the\toksb\the\toksd}}%
}

\def\attachoptionstable{% typesetting the options as part of the grammar
    \astappendx{\noalign{%
                   \tabskip\parindent
                   \halign{####\nx\hfil\tabskip\z@&\nx\qquad\nx\it####\nx\hfil\cr
                       \the\opttable
                   }%
               }%
    }\opttable{}%
}

\let\stopproduction\relax

\defc\tokendecls{% token declarations :: \tokendecls{[\onesymbol]+}{fptr}{sptr}
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:token-declarations}\yyunion
    \let\termindex\writetokenidxentry
    \let\currenttokentype\empty#1%
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

\defc\typedecls{% type declarations :: \typedecls{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:type-declarations}\yyunion
    \let\termindex\writetokenidxentry
    \typesset{}#2\relax#1%
    \appendr\typestable{\nx\onetype{\the\toksa}{\the\typesset}}%
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

\def\precdecls#1#2#3#4#5{% #1 type
                         % #2 tag
                         % #3 list of symbols
                         % #4 format pointer
                         % #5 stash pointer
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:prec-declarations}\yyunion
    \setrulecontext{*#1}%
    \let\termindex\writetokenidxentry
    \typesset{}#3\toksa{}#2%
    \appendr\prectable{\nx\oneprec{#1}{\the\toksa}{\the\typesset}}%
    \restorecslist{local-namespace}{\yyunion\termindex}%
    \setrulecontext{}%
}

% options typesetting

\def\oneparametricoption#1#2#3#4{%
    \restorecs{table-render:token-declarations}{\braceit\stringify\idit\anint\hexint}%
    \toksa{}\toksb{}\toksc{}#2%
    \edef\next{\nx\oneparametricopti@n{#1}{\the\toksa}{\the\toksb}{\the\toksc}}\next
}

\def\oneparametricopti@n#1#2#3#4{%
    \yystringempty{#2}{% there is no identifier
        \yystringempty{#4}{% there is no digit
            \edef\next{\toksa{{\ntt"\the\toksb"}}}\next
        }{%
            \edef\next{\toksa{{\nx\rm\the\toksc}}}\next
        }%
    }{%
        \nameproc{#2}\with\parsebin
    }% now \toksa contains the value of the parameter
    \toksb{#1}%
    \let\termindex\writetokenidxentry
    \appendr\opttable{\gidxentry{\termostring}{#1}{}$\nx\langle${\nx\bf\the\toksb}$\nx\rangle$&\the\toksa\nx\/\nx\cr}%
    \let\termindex\eatone
}

\def\paramdef#1#2#3#4{%
    \restorecs{table-render:token-declarations}\braceit
    \let\termindex\writetokenidxentry
    \appendr\opttable{\gidxentry{\termostring}{#2}{}}%
    \let\termindex\eatone
    \appendrnx\opttable{$\langle${\bf #2}$\rangle$}%
    \toksa{}#1%
    \concat\opttable\toksa
}

\def\optionflag#1#2#3#4{%
    \let\termindex\writetokenidxentry
    \appendr\opttable{\gidxentry{\termostring}{#1}{}}%
    \let\termindex\eatone
    \yystringempty{#2}{%
        \appendrnx\opttable{$\langle${\bf#1}$\rangle$\rlap{$\,\star$}&\cr}%
    }{%
        \appendrnx\opttable{$\langle${\bf#1}$\rangle$\rlap{$\,\star$}& {\rm(set as $\langle${\bf#2}$\rangle$)}\cr}%
    }%
}

\def\codepropstype#1#2#3#4#5#6#7{%
    \readformat{#4}\the\yyformatlocal % this can change \codepropstyp@
    \codepropstyp@{#2}{#3}{#1}{#5}%
}

\def\codepropstyp@#1#2#3#4{%
    \readstash{#4}%
    \restorecs{table-render:type-declarations}{\idit\stringify\hspace}%
    \let\termindex\writetokenidxentry
    \typesset{}#2% collect all the symbols in \typesset
    \appendr\opttable{\gidxentry{\termostring}{#3}{}$\nx\langle${\nx\bf #3}$\nx\rangle$&%
         \toksa{\the\yystashlocal}\nx\makestashbox\nx\hfil\cr
         \noalign{\nx\smallskip}%
         \noalign{\indent$\nx\rlap{\hbox to2em{\nx\hfil$\nx\star$\nx\hfil}}\vcenter{
             \advance\hsize by-\parindent
             \emergencystretch10pt\nx\raggedright\noindent\hangafter\z@\hangindent2em\nx\strut\the\typesset\nx\strut}$}%
    }%
    \let\termindex\eatone
}

\def\codeassoc#1#2#3#4#5#6#7{%
    \readformat{#4}\the\yyformatlocal % this can change \codeass@c
    \codeass@c{#1}{#2}{#3}{#5}%
}

\def\codeass@c#1#2#3#4{%
    \readstash{#4}%
    \let\idit\termnameidit
    \let\termindex\writetokenidxentry
    \toksa{}#2% put the typeset form of the identifier in \toksa
    \appendr\opttable{%
        \yystringempty{#1}{}{\gidxentry{\termostring}{#1}{}}%
        $\nx\langle${\nx\bf #1}$\nx\rangle$\nx\quad{\the\toksa}&%
        \toksa{\the\yystashlocal}\nx\makestashbox\nx\hfil\cr
    }%
    \let\termindex\eatone
}

\def\vardef#1#2#3#4{%
    \readformat{#3}\the\yyformatlocal % this can change \vard@f
    \vard@f{#1}{#2}{#4}%
}

\def\vard@f#1#2#3{%
    \let\idit\termnameidit
    \let\stringify\termnamestringify
    \let\termindex\writetokenidxentry
    #1% put the typeset form of the identifier or string in \toksa
    \toksb\toksa
    \savecslist{local-namespace}\yyunion
    \restorecslist{table-render:variable-definitions}\yyunion
    \toksa{}#2% put the value in toksa
    \toksc\toksa
    \restorecslist{local-namespace}\yyunion
    \let\termindex\eatone
    \appendr\opttable{$\nx\langle${\nx\bf define}$\nx\rangle$\nx\quad{\the\toksb}&\the\toksc\cr}%
}

\defc\prologuecode{% prologue code :: \prologuecode{tex_string}{fptr}{sptr}{fptr}{sptr}
    \readstash{#5}%
    \appendr\opttable{%
        \omit\span\omit\nx\cdotfill\cr
        \omit\span\omit\toksa{\the\yystashlocal}\nx\makestashbox\nx\cr
        \omit\span\omit\nx\cdotfill\cr
    }%
}

\def\yyunion@tablerender{%
    \stashed\rarhss@p\rrhss@p\prodhead@r\hrhss@p\arhss@p\termnam@\termn@m@\t@rmn@m@
    \actbrac@s\hspac@\codepropstyp@\vard@f\termmetastyle\peekstash\stopproduction
}

\savecslist{table-render}\yyunion
\savecslist{table-render}\yyunion@tablerender

\def\setprodtable{%
    \def\aststream{bison.rules}% the list where the AST output is gathered
    \restorecslist{table-render}\yyunion
    \restorecslist{table-render}\yyunion@tablerender
}

% formatting macros

\def\midf#1#2{}

\def\format#1{%
    #1%
}
\let\formatlocal\format
\let\formatbegin\format
\let\formatp\format

\newtoks\yystashlastcontent % contents of the last action
\newif\ifshowlastaction % if true, show the last action of a flattened set of rules

% flatten: omit the actions, just list all the rules in a single line;
% if \showlastactiontrue show just the last action, otherwise display \actionfiller
% instead

\def\flatten{%
    \let\stopproduction\stopproduction@flatten
    \let\actbrac@s\actbrac@s@flatten
    \let\rrhss@p\rrhss@p@flatten
    \let\arhss@p\relax
    \let\rarhss@p\relax
    \def\flattenactiontrace{}%
}

\def\actbrac@s@flatten#1#2#3{% do everything but add the stash contents to the table
    \readstash{#3}%
    \edef\flattenactiontrace{\flattenactiontrace*}%
    \yystashlastcontent\yystashlocal % keep the contents of the braces
                                     % in case we want to show the last action
    \futurelet\actsep\actbrac@s@fl@tten
}

% see the remarks before \emptyterm above for the reasons for choosing the index
% pseudonamespace for this term

\def\inlineactionsymbol{$\diamond$}

\def\actbrac@s@fl@tten{%
    \ifx\actsep\arhssep
        \astappendx{\nx\ \nx\inlineactionsymbol\nx\ \tidxentry{\termexception}{inline_action&}}%
    \fi
}

\def\rrhss@p@flatten{%
    \astappend{$\ \vert\ $}%
}%

\def\stopproduction@flatten{%
    \ifshowlastaction
        {%
            \toksa\yystashlastcontent\cleanstash
            \yytoksempty\toksa{}{% there is a non-empty action
                \def\next{*}%
                \ifx\flattenactiontrace\next % if there were more than one action
                \else                        % preceeding it, prepend an indicator (ellipsis)
                    \appendlnx\toksa{\hbox{$\ldots\ \vert\ $}{}\ignorespaces}%
                \fi
            }%
            \expandafter
        }\expandafter\yystashlastcontent\expandafter{\the\toksa}%
        \yystashlastcontent\expandafter{\expandafter\toksa\expandafter{\the\yystashlastcontent}}%
    \else
        \yystashlastcontent{\actionfiller}%
    \fi
    \def\flattenactiontrace{}%
    \astappendx{&&\the\yystashlastcontent\nx\cr}%
}

% fold: resume normal display

\def\fold{%
    \restorecs{table-render}{\actbrac@s\rrhss@p\arhss@p\rarhss@p\stopproduction}%
    \stopproduction@flatten
}

\def\actionfiller{\omit\quad$\ldots$\hfil}

% inline: put the left hand side of the production on the same line as the rule

\def\inline{%
    \let\prodhead@r\prodhead@r@inline
    \let\hrhss@p\hrhss@p@inline
}

\def\prodhead@r@inline#1#2{%
    \let\idit\prodhdridit
    #1\relax
    \astappendx{\the\toksa}%
    \setbox\z@=\hbox{\strut\the\toksa\/$\,$\rm:\quad}%
    \tempda=\wd\z@\relax
    \advance\tempda-2em
    \hrhss@p
}

\newif\ifsquashterms

\def\termspostformat{\ifsquashterms\aftergroup\noexpand\squashtermstrue\fi}

\def\hrhss@p@inline{%
    \astappendx{&\termspostformat\hbox to \the\tempda{\hss}}%
}

% breakline: break the production line

\def\breakline{%
    \let\hspac@\hspac@@breakline
}

\def\hspac@@breakline{%
    \astappend{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}%
    \restorecs{table-render}\hspac@
}

% breakahead: break the production line after #1 terms
% use: \breakahead{[0-9]+}

\newcount\breakaheadcount

\def\breakahead#1{%
    \let\bahspac@\hspac@
    \let\hspac@\hspac@@breakahead
    \breakaheadcount#1\relax
}

\def\hspac@@breakahead{%
    \ifnum\breakaheadcount=\@ne
        \yybreak{%  
            \astappend{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}%
            \let\hspac@\bahspac@
        }%
    \else
        \advance\breakaheadcount by \m@ne
        \yybreak{\bahspac@}%
    \yycontinue        
}

% skipheader: do not show the left hand side

\def\skipheader{%
    \let\prodhead@r\prodhead@r@skipheader
    \let\hrhss@p\hrhss@p@skipheader
}

\def\prodhead@r@skipheader#1#2{%
    \hrhss@p
}

\def\hrhss@p@skipheader{\astappend{\omit\hbox to 2em{\hfil}&\termspostformat}\restorecs{table-render}{\hrhss@p}}

\def\stashtoterm#1{% in case one needs to insert previous stash as a term
                   % usage: \let\peekstash\stashtoterm
    \readstash{#1}%
    {\edef\next{\toksc{{\toksa{\the\yystashlocal}\nx\makestashbox}}}\expandafter}\next
    \let\termmetastyle\xtoksc
    \restorecs{table-render}\peekstash
}

\def\xtoksc#1{\the\toksc}%

% resetf: resume normal formatting

\def\resetf{%
    \restorecs{table-render}{\prodheader\prodhead@r\hrhss@p\rrhss@p\arhss@p\rhs\termname\actbraces\actbrac@s\rarhss@p\hspac@\rules
        \oneproduction\emptyterm\stringify\idit\charit\prodprodsep\onesymbol\stopproduction}%
    \squashtermsfalse
}

\def\prodstyle#1{% to typeset token names in text;
                 % CAUTION: when many parsers are at play at the same time,
                 % one should use the version of this macro below that takes a
                 % namespace as a parameter, to make sure the term is typeset properly
                 % when another namespace might be in effect (such as when section names are output)  
    {%  
        \let\optstrextra\optstrextraesc
        \def\termidxrank{5}%
        \let\termindex\writeidxentry
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksa{%
                \termmetastyle{%
                    \let\nx\idxfont\nx\empty\nx\it\the\toksa
                    \expandafter\gidxentryxv\expandafter{\romannumeral
                        \ifyyparsefail
                            \yybreak{0 \termidstring}%
                        \else
                            \yybreak{\expandafter\tssextract\the\toksa\end}%
                        \yycontinue}{#1}{\the\toksb}\nx\/%
                }%
            }%
        }\next
        \let\_\uscore
        \the\toksa\nobreak
    }%
}

\def\tssextract#1#2#3\end{%
    \yystringempty{#2}{0 \termidstring}{0 #2}%
}

\def\prodtstyle#1{% to typeset terminal string names in text;
    {%  
        \let\optstrextra\optstrextraesc
        \def\termidxrank{5}%
        \let\termindex\writeidxentry
        \nameproc{#1}\with\parsebin
        \yytoksempty\toksb{\toksb{"#1"}}{}%
        \edef\next{%
            \toksa{%
                \termmetastyle{%
                    \let\nx\idxfont\nx\empty\ntt\the\toksa
                    \gidxentryxv{\termstring}{#1}{\the\toksb}%
                }%
            }%
        }\next
        \let\_\uscore
        \the\toksa\nobreak
    }%
}

\def\prodstylens#1#2{% to typeset token names in text, in a predefined namespace
    {%
        \def\hostparsernamespace{#2}%
        \prodstyle{#1}%
    }%
}

% macros used in typesetting the result of the prologue parsing

%type declarations

\restorecslist{parser-strict}\yyunion


\defc\idit{% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
    \termnameidit{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\defc\stringify{% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
    \termnamestringify{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\defc\hspace{% a space has to take two parameters just like any connector :: \hspace{fptr}{sptr}
    \appendrnx\typesset{ }%
}

\defc\charit{% *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
    \toksa{\.{#2}}%
    \concat\typesset\toksa
}

\defc\tagit{% *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
    \toksa{\\{#2}}%
}

\toyyunion{table-render:type-declarations}

%token declarations

\newif\iftracetokennames

\restorecslist{parser-strict}\yyunion

\defc\hspace{}

\defc\idit{\toksa{#2}}% *ID :: \idit{12string}{tex_string}{fptr}{sptr}

\defc\charit{\toksa{'#1'}}% *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}

\defc\stringify{\toksb{#2}}% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}

\defc\anint{\toksc{#1}}% INT :: \anint{digits}{fptr}{sptr}

\defc\hexint{\toksc\expandafter{\eattwo#1$_{16}$}}% INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}

\defc\tagit{% *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
    \def\currenttokentype{#2}%
}

\def\braceit#1#2#3{% similar to \actbrac@s
    \readstash{#3}%
    \edef\next{\toksa{&\toksa{\the\yystashlocal}\nx\makestashbox\nx\cr}}\next
}

\def\onesymbol#1#2#3{% #1 is the macro name used internally by bison (\idit{}{}{}{})
                     % #2 is the integer value (\anint{}{}{})
                     % #3 is the string value (\stringify{}{}{}{})
    \toksa{}\toksb{}\toksc{}%
    #1#2#3\relax
    \edef\next{\nx\onesymb@l{\the\toksa}{\the\toksc}{\the\toksb}}\next
}

\def\onesymb@l#1#2#3{% #1 is the macro name used internally by bison 
                     % #2 is the integer value 
                     % #3 is the string value 
    \let\optstrextra\optstrextraesc
    \tokse{}\toksf{}\toksg{}% these registers will contain the appropriate visual keys
                            % and index entry processing control sequences
    \yystringempty{#3}{}{%
        \nameproc{#3}\with\parsebin
        \ifyyparsefail % if the parsing failed, create a visible key, process the 
                       % index entry as a \.{\\tt} string
            \tokse{"#3"}\toksf{\termstring}%
        \else % otherwise use the value of the string as the visual key,
              % process the string as a \bison\ identifier
            \tokse\toksb\toksf{\termidstring}%
        \fi
        \tokstoks\toksa
    }%
    \yystringempty{#1}{}{\nameproc{#1}\with\parsebin\toksg\toksb}%
    \iftracetokennames\ferrmessage{token: \the\toksa, string: \the\tokstoks}\fi
    \toksb\expandafter{\currenttokentype}%
    \appendr\tokdectoks{\nx\toksdefline
                            {\the\toksa}%
                            {% 
                                \yystringempty{#1}{}{\gidxentryxv{\termidstring}{#1}{\the\toksg}}%
                                \yystringempty{#3}{}{\expandafter\gidxentryxv\expandafter{\the\toksf}{#3}{\the\tokse}}%
                            }% 
                            {\yytoksempty\toksb{auto}{\the\toksb}}%
                            {#2}%
                            {\the\tokstoks}%
    }%
    \restorecs{nameparser}{\optstrextra}%
}

\toyyunion{table-render:token-declarations}

%precedence declarations

\restorecslist{parser-strict}\yyunion

\defc\idit{% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
    \termnameidit{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\defc\stringify{% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
    \termnamestringify{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\defc\charit{% *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
    \expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
        \toksa{#1}\sansfirst\toksa
        \edef\next{\toksa{{\ntt\the\toksa}\gidxentryxb{\termvstring}{\the\toksa}{}}}% remove the possible \\
    \else
        \expandafter\ifx\csname\viskeyref\hostparsernamespace{#1}\endcsname\relax
            \tokse{#1}%
        \else
            \edef\next{\tokse{\expandafter\expandafter\csname\viskeyref
                \hostparsernamespace{#1}\endcsname\expandafter{\currentrulecontext}}}\next
        \fi
        \edef\next{\expandafter\toksa\expandafter{\expandafter\noexpand
            %\csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termvstring}{'#1'}{\the\tokse}}}%
            \csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termidstring}{'#1'}{\the\tokse}}}%
    \fi
    \next
    \concat\typesset\toksa
}

\defc\anint{% INT :: \anint{digits}{fptr}{sptr}
    \appendrnx\typesset{${}=\hbox{\ntt#1}$}%
}

\defc\hexint{% INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
    \toksa\expandafter{\eattwo#1}%
    \appendr\typesset{${}=\hbox{\ntt\the\toksa}_{16}$}%
}

\defc\hspace{%
    \appendrnx\typesset{ }%
}

\defc\tagit{% *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
    \toksa{\\{#2}}%
}

\def\symbolprec#1#2{%
    #1#2%
}

\toyyunion{table-render:prec-declarations}

% variable definitions

\restorecslist{parser-strict}\yyunion

\defc\idit{\toksa{#2}}% *ID :: \idit{12string}{tex_string}{fptr}{sptr}

\defc\stringify{\toksa{#2}}% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}

\def\bracedvalue#1#2#3{% similar to \actbrac@s
    \readstash{#3}%
    \edef\next{\toksa{\toksa{\the\yystashlocal}\nx\makestashbox}}\next
}

\toyyunion{table-render:variable-definitions}

% directives

\restorecslist{parser-strict}\yyunion

\defc\idit{% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
    \termnameidit{#1}{#2}{#3}{#4}%
}

\defc\stringify{% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
    \termnamestringify{#1}{#2}{#3}{#4}%
}

\defc\tagit{% *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
    \toksa{\\{#2}}%
}

\defc\anint{% INT :: \anint{digits}{fptr}{sptr}
    \toksa{#1}%
}

\defc\hexint{% INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
     \toksa\expandafter{\eattwo#1$_{16}$}%
}

\toyyunion{table-render:directives}

% rule listing macros

\restorecslist{parser-prototypes}\yyunion % we just want to ignore the non-rule related code

\defc\prodheader{% a `left-hand side' of a production :: \prodheader{\idit}{\idit}{fptr}{sptr}
    #1\relax
    \lastlastname\lastname
    \setnumname\numname
    \lastname{}%
    #2\relax
    \edef\next{\toksc{\numname:}\toksd{\noexpand\lhs{\the\lastlastname}{\the\lastname}}}\next
}

% the macros below are defined in `parser-prototypes'
%\defp\rrhssep#1#2{}
%\defp\arhssep#1#2{}
%\defp\rarhssep#1#2{}
%\defp\hspace#1#2{}
%\defp\prodprodsep{}
%\defp\onesymbol#1#2#3{}

\defc\rhs{% a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
    \concat\toksa\toksc
    \concat\toksb\toksd
    #1\relax
    \edef\next{\toksb{\noexpand\onerule{\the\toksb}}}\next
    \appendr\toksa\def
    \possibleimplicitfalse
    \yyreplacestring\toksa\in\newsymswitch\with\toksb
    \toksa{}\toksb{}%
}

\newtoks\lastname
\newtoks\lastlastname

\defc\termname{% a production term :: \termname{\idit|\stringify|\charit}{\idit}
    \flushact
    #1\relax
    \lastlastname\lastname
    \setnumname\numname
    \lastname{}%
    #2\relax
    \appendr\toksa{ \numname}\appendr\toksb{\noexpand\term{\the\lastlastname}{\the\lastname}}%
}

\def\setnumname#1{%
    \expandafter\let\expandafter\next\csname term\parsernamespace\the\lastname \endcsname
    \ifx\next\relax % this is a terminal that was renamed
        \expandafter\ifx\csname token\parsernamespace\the\lastname \endcsname\relax % this is probably a symbol from another grammar
            \def#1{-1}%
            \errmessage{token \the\lastname\space is undefined}%
        \else
            \expandafter\tempca\csname token\parsernamespace\the\lastname \endcsname
            \edef#1{\fgetelemof{yytranslate}\at\tempca}%
        \fi
    \else
        \let#1\next
    \fi
}

\defc\actbraces{% action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
    \flushact
    \lastname{}%
    #2\relax
    \edef\next{\lastactsym{\the\lastname}}\next
    \possibleimplicittrue
}

\defc\rules{#1}% a complete right-hand side :: \rules{[\rhs|\rrhssep]}

\defc\oneproduction{#1}% a complete production :: \oneproduction{\prodheader\rules}{fptr}{sptr}

\defc\emptyterm{%
    \flushact
    \lastname{}%
}

\defc\stringify{% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
    \lastname{"#1"}%
}

\defc\idit{% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
    \lastname{#1}%
}

\defc\charit{% *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
    \lastname{'#1'}%
}

\defc\anint{% INT :: \anint{digits}{fptr}{sptr}
    \lastname{#1}%
}

\defc\hexint{% INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
     \lastname\expandafter{\expandafter\number\eattwo#1$_{16}$}%
}

\newtoks\lastactsym
\newif\ifpossibleimplicit

\def\flushact{%
    \ifpossibleimplicit
        \appendr\toksa{ @implicit@}\appendr\toksb{\noexpand\implicitterm{@implicit@}{\the\lastactsym}}%
        \lastactsym{}%
        \possibleimplicitfalse
    \fi
}

\savecslist{symbols}\yyunion

%% token typesetting macros

\def\prettynamecs#1#2{'parser'#1#2} % naming convention for token typesetting alternatives

\def\setsafe#1{\def\saferword{#1}\setspecialcharsfrom\saferword}
\def\prettyword#1{\setsafe{#1}\expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{{\ntt\lowercase{\saferword}}}}
\def\prettytoken#1{\setsafe{#1}\expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{{\ntt\saferword}}}
\def\prettytoken@#1{\toksa{}\numberstocharsandspaces#1\end\expandafter\edef
    \csname\prettynamecs\parsernamespace{\the\toksa}\endcsname##1{{\ntt\the\toksa}}%
}

\def\prettywordpair@@#1#2{\setsafe{#1}\expandafter\def\csname\prettynamecs\parsernamespace\saferword\endcsname##1{#2}}

\def\prettywordpair#1#2{\prettywordpair@@{#1}{{\ntt@#2}}}

\newif\ifdisplaytokenraw

\def\prettywordpair@#1#2{% a `weak symbol' version of the above macro, so that the new typesetting can be overriden 
    \setsafe{#1}\toksa{}\numberstocharsandspaces#2\end\expandafter\nameproc\expandafter{\the\toksa}\with\parsebin
    \expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{%
        \noexpand\ifdisplaytokenraw
            {\ntt\saferword}% \ntt expands to \noexpand\ntt@
        \noexpand\else
            {\ntt\def\noexpand\_{\char`\noexpand\_}\the\toksa}%
        \noexpand\fi
    }%
}

\def\prettywordpairwvis#1#2#3{%
    \prettywordpair{#1}{#2}%
    \expandafter\def\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{#3}%
}

\def\prettywordpair@wvis#1#2#3{%
    \prettywordpair@{#1}{#2}%
    \expandafter\def\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{#3}%
}

\def\prettywordpair@@wvis#1#2#3{%
    \prettywordpair@@{#1}{#2}%
    \expandafter\def\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{#3}%
}

% a control sequence to reuse the token definition file for typesetting

\newif\iftraceprettytokens % display prettyfying control sequences

\def\tokeneqpretty#1#2{%
    \setsafe{#1}\toksa{}\numberstocharsandspaces#2\end
    \expandafter\nameproc\expandafter{\the\toksa}\with\parsebin
    \ifyyparsefail
        \toksc{{}$_{\rm m}${}}\toksd\toksa
    \else
        \toksc{}\toksd\toksb % quotes should trigger the creation of a visible key
    \fi
    \expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{%
        \noexpand\ifdisplaytokenraw
            {\ntt\saferword}%
        \noexpand\else
            {\ntt\def\noexpand\_{\char`\noexpand\_}\the\toksa\the\toksc}%
        \noexpand\fi
    }%
    \expandafter\edef\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{\the\toksd}%
    \iftraceprettytokens
        \message{token: \expandafter\meaning\csname\prettynamecs\parsernamespace\saferword\endcsname}%
        \message{visual key: \expandafter\meaning\csname\viskeyref\parsernamespace{\saferword}\endcsname}%
    \fi
}

% the bootstrap macros

\def\initbootstrap{%
    \restorecslist{bootstrap}\yyunion
}

\restorecslist{parser-prototypes}\yyunion

\defc\tokendecls{#1}% token declarations :: \tokendecls{[\onesymbol]+}{fptr}{sptr}

\defc\stringify{\toksa{"#1"}}% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}

\defc\idit{\toksb{#1}}% *ID :: \idit{12string}{tex_string}{fptr}{sptr}

%\def\anint#1#2#3{\toksc{#1}}% this definition is not needed as all
                             % the information is extracted from yytname

% note that the definitions below do not handle cases such as 
%token '{' "left brace" since the lexer knows how to return '{'

\defc\onesymbol{%#1#2#3 % a symbol definition :: \onesymbol{\idit}{\anint|\hexint}{\stringify}
    \toksa{}\toksb{}%\toksc{}%
    #1#2#3%
    \yytoksempty\toksa{\expandafter\charstonumbers\the\toksb\end}%
                      {\expandafter\toksa\expandafter{\expandafter}\expandafter\charstonumbers\the\toksa\end}%
    \yytoksempty\toksb{%
        \yytoksempty\toksa{}{%
            \immediate\write\tokendefs{\noexpand\tokeneq{\the\toksb}{\the\toksa}}%
        }%
    }{%
        \immediate\write\tokendefs{\noexpand\tokeneq{\the\toksb}{\the\toksa}}%
    }%
}

% the original, minimal bootstrapping macros were designed to process
% \prodstyle{\%token} declarations only and are enough to establish the
% interface between the \bison\ parser and the \bison\ lexer; to serve 
% the secondary task of providing typesetting information to 
% the \bison\ parser, all forms of token declarations must be processed;
%
% note that the bootstrap parser cannot process these additional declarations
% so these macros are designed to work with a full or prologue parser
% in a bootstrap mode; see \.{\\modebootstrap} in ldman.w and
% its use in \.{ldlex.w} and \.{ldgram.w} for an example of such
% an arrangement.

\def\precdecls#1#2#3#4#5{#3}

\def\symbolprec#1#2{%
    \toksa{}\toksb{}%
    #1%
    \yytoksempty\toksb{}{\immediate\write\tokendefs{\noexpand\tokenpp{\the\toksb}}}%    
}

\savecslist{bootstrap}\yyunion

% correct a few macros for debugging

\restorecslist{parser-debug}\yyunion

\defc\actbraces{% action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
    \noexpand\actbraces{#1}{#2}{#3}{#4}#5\noexpand\bdend
}

\defc\bpredicate{% predicate :: \bpredicate{tex_string}{<nothing>}{fptr}{sptr}directive*\bdend
    \noexpand\bpredicate{#1}{}{#3}{#4}#5\noexpand\bdend
}

\defc\rhs{% a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
    \noexpand\rhs{#1}{#2}{}% omit setting the conditional
}

\toyyunion{parser-debug}

% \romannumeral-expandable macros for character translation

\def\charstonumbersre#1\end{\charstonumbersr@0\end#1\end}

\def\charstonumbersr@#1\end#2\end{%
    \yystringempty{#2}{#1}%
        {\yystartsinspace{#2}{\expandafter\charstonumbers@@swap\expandafter{\number`\ }{#1}#2\end}{\charstonumb@rsr@#1\end#2\end}}%
}

\def\charstonumb@rsr@#1\end#2{%
    \expandafter\charstonumbersr@swap\expandafter{\number`#2}{#1}%
}

\def\charstonumbersr@swap#1#2{\charstonumbersr@#2{#1}\end}

\def\charstonumbers@@swap#1#2 {\charstonumbersr@#2{#1}\end}

% variations: set \toksa to the expansion, \edef-expandable version

\def\charstonumbers#1\end{\toksa\expandafter{\romannumeral\charstonumbersre#1\end}}
\def\charstonumberse#1\end{\romannumeral\charstonumbersre#1\end}

% name parser macros: currently only one suffix is handled, the very last one

\def\yyuniontag{\yynpunion}
\def\parserstrictnamespace{nameparser-strict}
\def\parserprototypesnamespace{nameparser-strict:headers}
\def\parserdebugnamespace{nameparser-debug}

\def\yynpunion{\currentyyunionnamespace}

\defp\idstr#1#2{} % idenifier ::
\defp\bidstr#1#2{} % bison variable ::
\defp\chstr#1#2{} % character ::
\defp\sfxi#1#2{} % literal suffix ::
\defp\sfxn#1#2{} % integer suffix ::
\defp\dotsp{} % suffix separator ::
\defp\optstr#1#2{} % option ::
\defp\qual#1#2{} % qualifier ::
\defp\visflag#1#2{} % visible key change flag ::

\toyyunion{nameparser-prototypes}

\defc\idstr{%
   \appendrnx\toksa{#1}%
   \appendrnx\toksc{#2}%
   \toksb{}%
}

\defc\bidstr{%
   \appendrnx\toksa{{}$\Upsilon${}}%
   \appendrnx\toksc{#2}%
   \toksb{}%
}

\defc\chstr{%
   \appendrnx\toksa{{\ntt@#1}}%
   \appendrnx\toksc{#2}%
   \toksb{}%
}

% the \toksb, set in the next two macros is used by \optstr later

\defc\sfxi{\toksb{\/${}_{\idxfont#1}$}\appendrnx\toksa{\/${}_{\idxfont#1}$}}

\defc\sfxn{\toksb{#1}\appendrnx\toksa{#1}}

\defc\dotsp{\futurelet\next\d@tsp}

\def\d@tsp{%
    \ifx\next\sfxi
    \else
        \ifx\next\qual
        \else
            \appendrnx\toksa{.}%
        \fi
    \fi
}

\defc\optstr{%
   \toksb\expandafter{\eatone#1}%
   \optstrextra
   \concat\toksa\toksb
   \appendrnx\toksc{#2}%
   \toksb{}%
}

\def\optstrextra{%
    \edef\next{\toksb{{$\nx\ulcorner$\ntt\the\toksb$\nx\urcorner$}}}\next
}

\def\optstrextraesc{%
    \edef\next{\toksb{{$\nx\langle$\ntt\the\toksb$\nx\rangle$}}}\next
}

\defc\qual{\appendrnx\toksd{\/${}_{\rm#2}$}\toksb{}}

\let\idxfont\relax

\defc\visflag{\toksf{#1}\appendrnx\tokse{#1}}% flag visible key change
                                   % \toksf contains the typesetting hint sequence

\toyyunion{nameparser}

\newtoks\namechars

\newif\iftracebadnames
\newif\iftracenames

\def\viskeyref#1#2{% getting the possible visual key from the index key
                   % #1 is the namespace of the original key
                   % #2 is the key
      \restorecsname{[nameparser:visual]}{.\prettynamecs{#1}{#2}}%
}

% ignore explicit (\ ) spaces while parsing names
% the macro will be called whenever \tosmallparser is executed (see yyinit.sty);
% this is neccessary due to a shortcut taken by the macros that display
% \bison\ productions in text (see \beginprod macros in yyinit.sty).

\def\ignoreexplicitspace{\yyinput}

% the main name processing routine
% does not have any side effects (other than the ones explicitly produced by the final call of #2);
% the parsed term is returned as #1 of the sequence in #2; #2 of #2 will hold a visual key if
% one exists, otherwise it will be empty; #3 is \yyparsefailtrue or empty
% #4 is the prettyfied version of the term if one exists; #5 is the typesetting hint from \visflag sequences;
% the version below prettifies tokens only if the parsing was successful; if
% needed, prettyfication may be applied after the parsing is complete

\def\nameproc#1\with#2{%
    {% keeping all changes local
        \iftracenames
            \toksa{#1}\ferrmessage{original name: \the\toksa}%
        \fi
        \tosmallparser\basicparserinit\d@parse{#1}%
        \ifyyparsefail
            \iftracebadnames
                \toksa{#1}\ferrmessage{bad name: \the\toksa}%
            \fi
            \toksb{#2{#1}{}{\yyparsefailtrue}{}{}}%
        \else
            \npbuildnames{#2}%   
        \fi
        \expandafter
    }\the\toksb
}

\def\npbuildnames#1{%
    \iftracenames
        \ferrmessage{parsed name (namespace: \parsernamespace): \the\namechars}%
    \fi
    \toksa{}\toksb{}\toksc{}\toksd{}\tokse{}\toksf{}%
    \restorecslist{nameparser}\yynpunion    
    \the\namechars\relax
    \iftracenames
        \ferrmessage{processed name: \the\toksa<identifier>, \the\toksb<suffices>, %
            \the\toksc<hash name>, \the\toksd<options>, \the\tokse<visible key>}%
    \fi
    \expandafter\npsetkeys\expandafter{\the\toksc}%
    \iftracenames
        \ferrmessage{final name: \the\toksa\yytoksempty\toksc{}{, prettyfied name: \the\toksc}%
                                           \yytoksempty\toksf{}{, hint: \the\toksf}%
                                           \yytoksempty\tokse{}{, visual key: \the\tokse}, %
                                           parsing \ifyyparsefail failed\else successful\fi
        }%
    \fi
    \toksb{#1}%
    \edef\next{%
        \toksb{%
            \the\toksb
            {\the\toksa}% the parsed name
            {\the\tokse}% the visual key
            {}% parseflag
            {\the\toksc}% the prettified name
            {\the\toksf}% the typesetting hint
        }%
    }\next
}

\def\npsetkeys#1{%
    \expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
        \toksc{}%
    \else
        \toksc\expandafter{\csname\prettynamecs\hostparsernamespace{#1}\endcsname}%
        \appendr\toksc{{\currentrulecontext}\the\toksb\the\toksd}%
        \toksf{\termidstring}%
    \fi
    \concat\toksa\toksd % changes \toksa and \toksc, no need for \toksb or \toksd after this
    \expandafter\ifx\csname\viskeyref\hostparsernamespace{#1}\endcsname\relax
        % no special visual key for this token
        \yybreak{%
            \yytoksempty\tokse{}{\makeviskeydirect}% if the visual key changed, output the key,
                                                   % attempt to compute it form the parsed output
        }%
    \else
        \yybreak{\makeviskeyfrompretty{#1}}% modify the production of the visible key to use the
                                       % prettified version
    \yycontinue % changes \toksb, \toksd, and \tokse
}

% standard postprocessing function

\def\parsebin#1#2#3#4#5{%
    \yystringempty{#4}{%
        \yystringempty{#3}{\toksa{\eatone{#5}#1}}{\toksa{#1}}%
    }{%
        \yystringempty{#3}{\toksa{\eatone{#5}#4}}{\toksa{#4}}%
    }%
    \toksb{#2}%
    \yystringempty{#3}{\yyparsefailfalse}{\yyparsefailtrue}%
}

% creating a visual key from the stored visual key and the sufixes

\restorecslist{nameparser-prototypes}\yynpunion % make most sequences do nothing

\defc\qual{\toksb{#2}\concat\toksd\toksb}

\toyyunion{nameparser:vispretty}

\def\makeviskeyfrompretty#1{%
    \edef\next{\tokse{\expandafter\expandafter\csname\viskeyref
        \hostparsernamespace{#1}\endcsname\expandafter{\currentrulecontext}}}\next
    \restorecslist{nameparser:vispretty}\yynpunion%
    \toksd{}%
    \the\namechars
    \yytoksempty\toksd{}{\appendr\tokse.\concat\tokse\toksd}%
}

% creating a visual key directly from parsed output

\restorecslist{nameparser-strict}\yynpunion

\defc\idstr{\toksb{#2}\concat\tokse\toksb}

\defc\bidstr{\toksb{#2}\concat\tokse\toksb}

\defc\chstr{\toksb{#2}\concat\tokse\toksb}

\defc\sfxi{\toksb{#2}\concat\tokse\toksb}

\defc\sfxn{\toksb{#2}\concat\tokse\toksb}

\defc\dotsp{\appendr\tokse.}

\defc\optstr{\toksb{#2}\concat\tokse\toksb}

\defc\qual{\toksb{#2}\concat\tokse\toksb}

\defc\visflag{}

\toyyunion{nameparser:visdirect}

\def\makeviskeydirect{%
    \tokse{}%
    \restorecslist{nameparser:visdirect}\yynpunion
    \the\namechars
}

\restorecslist{nameparser-strict}\yynpunion

% \flex\ regex typesetting routine and \flex\ parser value stack types

\def\flrecharadjust{
     `{%
          \yybyte{|}%
          \expandafter\yycp@\expandafter`\the\yybyte\relax
          \mkpurebyte
          \yyreturn
      }%
}

% no need for \setspecialcharsfrom\flrecharadjust

\def\frexproc#1\with#2{%
    {% keeping all changes local
        \iftracenames
            \toksa{#1}\ferrmessage{original regex: \the\toksa}%
        \fi
        \toflexreparser
        \let\acharswitch\flrecharadjust
        \basicparserinit
        \flexreparserinit
        \flexreparserdatainit
        \d@parse{\ #1}% the parsed regex is put in the \table register
        \ifyyparsefail
            \iftracebadnames
                \toksa{#1}\ferrmessage{bad regex: \the\toksa}%
            \fi
            \toksb{#2{#1}{}{\yyparsefailtrue}}%
        \else
            \toksd{#1}%
            \frebuildx{#2}%   
        \fi
        \expandafter
    }\the\toksb
}

\def\frebuildx#1{%
    \toksb{#1}\toksc{}%
    \restorecslist{flexparser-re}\yyflunion
    \the\table % typeset the regex (appears in \toksc)
    \edef\next{%
        \toksb{%
            \the\toksb % the name of the processing routine
            {\the\toksd}% the original regex
            {\the\toksc}% processed regex (or empty)
            {}% parse flag
        }%
    }\next
}

\def\ppregex#1#2#3{%
    \toksc{#2}%
}

\def\setregextable{% activate the appropriate AST control sequences
    \def\aststream{flex.sect2}% the list where the AST output is gathered
    \restorecslist{flexparser-sect2}\yyflunion
}

\def\setregexdeftable{%
    \def\aststream{flex.sect1}% the list where the AST output is gathered
    \restorecslist{flexparser-sect1}\yyflunion
}

% \flex\ parser indexing macros

\def\flexidxdomain{F}
\def\fstatedefidxrank{0}
\def\fstateidxrank{1}
\def\fregexidxrank{2}
\def\ftextrefrank{5}

\def\writeidxfsentry#1{% \flex\ state reference entry
      \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\fstateidxrank}#1}%
}

\def\writeidxfstextentry#1{% \flex\ state reference in text
      \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\ftextrefrank}#1}%
}

\def\writeidxfsdentry#1{% \flex\ state definition entry
      \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\fstatedefidxrank}#1}%
}

\def\writeidxfsrdentry#1{% \flex\ regex name definition entry
      \indxe\gindex{{\secno}{{}{\flexpseudorenamespace}}{\flexidxdomain}{\fstatedefidxrank}#1}%
}

\def\writeidxfsrnentry#1{% \flex\ regex name entry
      \indxe\gindex{{\secno}{{}{\flexpseudorenamespace}}{\flexidxdomain}{\fregexidxrank}#1}%
}

\def\writeidxfscclentry#1{% \flex\ chracter class name entry
      \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\fregexidxrank}#1}%
}

% \flex\ parser typesetting macros and stack types

\def\yyuniontag{\yyflunion}
\def\parserstrictnamespace{flexparser-strict}
\def\parserprototypesnamespace{flexparser-strict:headers}
\def\parserdebugnamespace{flexparser-debug}

\def\yyflunion{\currentyyunionnamespace}

\defp\flnametok#1#2{} % named definition :: \flnametok{matched text}{formatting command}, the text is \{text\}\ *
%
\defp\flcclexpr#1#2#3#4{} % character class expression :: \flcclexpr{matched text}{ccl token}{fptr}{sptr}
\defp\flchar#1#2#3#4{} % character :: \flchar{char}{char12}{fptr}{sptr}
\defp\flstring#1#2#3{} % string :: \flstring{{fptr}{sptr}}{\flchar{...}*}{{fptr}{sptr}}
\defp\flcclrnge#1#2{} % character range :: \flcclrnge{/flchar{...}}{/flchar{...}}
\defp\flbraceccl#1#2#3{} % character class :: \flbraceccl{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
\defp\flbracecclneg#1#2#3{} % negative character class :: \flbracecclneg{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
\defp\flcclunion#1#2{} % union of character classes :: \flcclunion{ccl}{ccl}
\defp\flccldiff#1#2{} % difference of character classes :: \flccldiff{ccl}{ccl}
\defp\flrepeat#1{} % * (asterisk) operation :: \flrepeat{re}
\defp\flrepeatstrict#1{} % + (plus) operation :: \flrepeatstrict{re}
\defp\flrepeatonce#1{} % ? operation :: \flrepeatonce{re}
\defp\flrepeatnm#1#2#3{} % {n, m} operation :: \flrepeatnm{re}{num}{num}
\defp\flrepeatgen#1#2{} % {n,} operation :: \flrepeatgen{re}{num}
\defp\flrepeatn#1#2{} % {n} operation :: \flrepeatn{re}{num}
\defp\flparens#1#2#3#4{} % parenthesized regular expression :: \flparens{{fptr}{sptr}}{re}{{fptr}{sptr}}{formatting command}
\defp\fldot#1#2{} % . (dot) expression :: \fldot{fptr}{sptr}
\defp\flor#1#2{} % `|' expression :: \flor{fptr}{sptr}
\defp\fltrail#1#2{} % regular expression sans trailing suffix :: \fltrail{re}{{fptr}{sptr}}
\defp\flretrail#1#2#3{} % regular expression with a trailing context :: \flretrail{re}{{fptr}{sptr}}{re}
\defp\flreateol#1#2#3{} % regular expression with the end of line as a trailing context :: \flreateol{re}{fptr}{sptr}
\defp\flrule#1#2{} % full regular expression :: \flrule{re}{formatting command}
\defp\flbolrule#1#2{} % full regular expression at the beginning of the line :: \flbolrule{re}{formatting command}
\defp\fleof#1#2{} % end of file rule :: \fleof{fptr}{sptr}
\defp\flsconlist#1#2#3{} % start condition list :: \flsconlist{{fptr}{sptr}}{\flname...}{{fptr}{sptr}}
\defp\flsconuniv#1#2{} % universal start condition :: \flsconuniv{fptr}{sptr}
\defp\flnamesep#1#2{} % separator between names :: \flnamesep{fptr}{sptr}
\defp\flname#1#2#3#4{} % name :: \flname{name}{name12}{fptr}{sptr}
\defp\flopt#1#2#3#4#5{} % \flex option :: \flopt{option}{text}{text12}{fptr}{sptr}
\defp\floptions#1{} % \flex options :: \floptions{\flopt ...}
\defp\flscondecl#1#2#3#4{} % start condition declarations :: \flscondecl{x|s}{fptr}{sptr}{\flname ...}
\defp\flaction#1#2#3#4#5#6#7{} % \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
\defp\flactionc#1#2#3#4#5#6#7{} % \flex continued action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
\defp\flactiongroup#1#2#3#4#5#6{} % a group of \flex actions :: \flactiongroup{\flsconlist...}{fptr}{sptr}{\flaction...}{fptr}{sptr}
\defp\flbareaction#1#2#3#4#5#6{} % bare action :: \flbareaction{fptr}{sptr}
\defp\flptropt#1#2{} % %ponter directive :: \flptropt{fptr}{sptr}
\defp\flarrayopt#1#2{} % %array directive :: \flarrayopt{fptr}{sptr}
\defp\fltopopt#1#2#3#4{} % %top directive :: \fltopopt{fptr}{sptr}{fptr}{sptr}
\defp\flredef#1#2#3#4#5#6#7#8{} % regular expression definition :: \flredef{name}{name12}{fptr}{sptr}{text}{text12}{fptr}{sptr}

\toyyunion{flexparser-prototypes}

\restorecslist{flexparser-strict}\yyflunion

\defc\flrule{%#1#2% full regular expression :: \flrule{re}{formatting command}
    {\toksc{}#1#2\expandafter}\expandafter\toksd\expandafter{\the\toksc}%
    \concat\toksc\toksd
}

\newif\iftextre % check if this is a free standing regular expression or appears in the definitions table

\defc\flbolrule{%#1#2% full regular expression at the beginning of the line :: \flbolrule{re}{formatting command}
    {\toksc{}#1#2\appendlnx\toksc{\raise0.5pt%
        \iftextre\yybreak{\hbox}\else\yybreak{\llap}\yycontinue{\sscmd$\dashv$\ }}\expandafter}\expandafter%
        \toksd\expandafter{\the\toksc}\concat\toksc\toksd
}

\let\astformat@flrule\empty

\def\inscomment#1{% insert a comment before the rule
    \def\astformat@flrule{%
        \appendlnx\toksc{\noalign{#1}}%
    }%
}

\def\insrulealign#1#2{%
    \def\astformat@flrule{%
        \tokse{#1}\toksf{#2}%
        \concatl\tokse\toksc
        \concat\toksc\toksf
    }%
}

\def\rulealigntemplate{% to hide the hash symbols from \\edef operators
    \vbox\bgroup\halign\bgroup##\hfil&##\hfil\cr
}

% regular expression symbols

\defc\flstring{%#1#2#3% string :: \flstring{{fptr}{sptr}}{\flchar{...}*}{{fptr}{sptr}}
%    \toksc\expandafter{\the\toksc\hbox{#2}}%
    #2%
}
  
\defc\flnametok{%#1#2% named definition :: \flnametok{matched text}{formatting command}, the text is \{text\}\ *
    \let\termindex\writeidxfsrnentry
    \fln@metok#1\end{#2}%
    \let\termindex\eatone
}

\def\fln@metok\{#1\}#2\end#3{%
    {%
        \let\hostparsernamespace\flexpseudorenamespace
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \expandafter
    }\the\toksd
    \toksf{}\toksg{}%
    #3%
    \edef\next{%
        \toksc{\the\toksc\the\toksf{}\nx\flexrendisplay{\the\toksd}%
            \gidxentryxv{\termfsrestring}{#1}{\the\tokse}\the\toksg
        }%
    }\next
}

\def\insnamealign#1#2{%
    \def\astformat@flnametok{%
        \toksf{#1}%
        \toksg{#2}%
    }%
}

\let\astformat@flnametok\empty

\defc\flchar{%#1#2#3#4% character :: \flchar{char}{char12}{fptr}{sptr}
    \yystringempty{#2}{#1}{\flch@r#2\end}%
}

\def\flch@r#1#2\end{%
    \ifnum`#1=`\\
        \fl@h@r#2\end
    \else
        \toksc\expandafter{\the\toksc{\ntt#1#2}}%
    \fi
}

\def\fl@h@r#1#2\end{% process the sequence after an escape
    {%
        \let\default\flescdefault
        \tokse{#1}\toksd{#2}%
        \switchon{\the\tokse}\in\flesccharswitch
        \edef\next{\toksc{\toksc{\the\toksc\the\toksd}}}\next
        \expandafter
    }\the\toksc
}

\def\flesccharswitch{% recognizing different types of escape sequences
01234567
    {% octal constant
        \edef\next{\toksd{\nx\.{\the\tokse\the\toksd}$_8${}}}\next
    }
nfrtabv
    {% \Cee\ escape sequence
        \edef\next{\toksd{{\nx\sscmd\nx\sscmd$\nx\langle$\nx\.{\the\tokse\the\toksd}$\nx\rangle$}}}\next
    }
x
    {% hexadecimal constant
        \edef\next{\toksd{\nx\.{\the\toksd}$_{16}${}}}\next
    }
}

\def\flescdefault{%
    \edef\next{\toksd{\nx\.{\the\tokse\the\toksd}}}\next
}

\setspecialcharsfrom\flesccharswitch

\defc\fldot{%#1#2% dot expression :: \fldot{fptr}{sptr}
    \toksc\expandafter{\the\toksc.}%
}

\defc\flparens{%#1#2#3#4% parenthesized regular expression :: \flparens{{fptr}{sptr}}{re}{{fptr}{sptr}}{format command}
    {\toksc{}#2\expandafter}\expandafter\toksd\expandafter
        {\the\toksc{\rm)}}%
    \toksf{}\toksg{}#4%
    \tokse\expandafter{\the\toksf{\rm(}}%
    \concatl\tokse\toksd
    \concat\toksc\toksd
    \concat\toksc\toksg
}

\def\insparensalign#1#2{%
    \def\astformat@flparens{%
        \toksf{#1}%
        \toksg{#2}%
    }%
}

\let\astformat@flparens\empty

\defc\flrepeat{%#1% asterisk operation :: \flrepeat{re}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
        {\the\toksc{{}${}_{*}$}}%
    \flselectiveseparator
    \concat\toksc\toksd%
}

\defc\flrepeatn{%#1#2% {n} operation :: \flrepeatn{re}{num}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
        {\the\toksc{{}${}_{\{\yyfirstoftwo#2\}}$}}%
    \flselectiveseparator
    \concat\toksc\toksd%
}

\defc\flrepeatnm{%#1#2#3% {n, m} operation :: \flrepeatnm{re}{num}{num}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
        {\the\toksc{{}${}_{\{\yyfirstoftwo#2,\yyfirstoftwo#3\}}$}}%
    \flselectiveseparator
    \concat\toksc\toksd%
}

\defc\flrepeatonce{%#1% ? operation :: \flrepeatonce{re}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
        {\the\toksc{{}${}_{?}$}}%
    \flselectiveseparator
    \concat\toksc\toksd%
}

\defc\flrepeatstrict{%#1% plus operation :: \flrepeatstrict{re}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter{\the\toksc{{}${}_{+}$}}%
    \flselectiveseparator
    \concat\toksc\toksd%
}

\def\flselectiveseparator{%
    \yytoksempty\toksc{}{%
        \tokse{{{}$\,$}}%
        \concatl\tokse\toksd
    }%
}

\defc\flcclrnge{%#1#2% character range :: \flcclrnge{\flchar{...}}{\flchar{...}}
%    \toksd{#1}\getfirst\toksd\to\toksd
%    \tokse{#2}\getfirst\tokse\to\tokse
    #1\appendr\toksc{{\nx\rm--}}#2%
%    \edef\next{\toksc{\the\toksc\the\toksd{\nx\rm--}\the\tokse}}\next
}

\defc\flcclexpr{%#1#2#3#4% character class expression :: \flcclexpr{matched text}{ccl token}{fptr}{sptr}
    {%
        \let\hostparsernamespace\flexpseudonamespace
        \nameproc{#2}\with\parsebin
        \edef\next{%
            \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \expandafter
    }\the\toksd
    \let\termindex\writeidxfscclentry
    \edef\next{\toksc{\the\toksc{\the\toksd}%
        \gidxentryxv{\termidstring}{#2}{\the\tokse}%
    }}\next
    \let\termindex\eatone
}

\defc\flbraceccl{%#1#2#3% character class :: \flbraceccl{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
    \appendrnx\toksc{{}$[${}}%
    #2%
    \appendrnx\toksc{{}$]${}}%
}

\def\flbraceccldemo#1#2#3{% character class :: \flbraceccl{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
    % a demo version without the beginning bracket
    \appendrnx\toksc{{}$\ldots${}}%
    #2%
    \appendrnx\toksc{{}$]${}}%
}

\defc\flccldiff{%#1#2% difference of character classes :: \flccldiff{ccl}{ccl}
    #1%
    \appendr\toksc{${}\nx\setminus{}$}%
    #2%
}

\defc\flbracecclneg{%#1#2#3% negative character class :: \flbracecclneg{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
    \appendr\toksc{{}$[${}}%
    #2%
    \appendr\toksc{{}$]^c${}}%
}

\defc\fleof{%#1#2% end of file rule :: \fleof{fptr}{sptr}
    \let\termindex\writeidxfsrnentry
    \edef\next{\toksc{\the\toksc{}\nx\flexrendisplay{EOF}%
        \gidxentry{\termfsrestring}{EOF}{EOF}}}\next
    \let\termindex\eatone
}

\defc\flor{%#1#2% `|' expression :: \flor{fptr}{sptr}
    \appendr\toksc{{}${}\nx\mid{}${}}%
}

\defc\flsconuniv{%#1#2% universal start condition :: \flsconuniv{fptr}{sptr}
    \let\termindex\writeidxfsentry
    \edef\next{\toksb{\the\toksb{}$\langle*\rangle${}%
         \gidxentryxv{\termvstring}{*}{*}}}\next % TODO: create a separate name space for state names
    \let\termindex\eatone
}

\defc\fltrail{%#1#2% regular expression sans trailing suffix :: \fltrail{re}{{fptr}{sptr}}
    \errmessage{\nx\fltrail\space macro has should not be expanded.}
}

\defc\flretrail{%#1#2#3% regular expression with a trailing context :: \flretrail{re}{{fptr}{sptr}}{re}
                % #1 is the regular expression to be matched
                % #2 is the pointer to the context marker (/)
                % #3 is the context
    \errmessage{\nx\flretrail\space macro has not been implemented (see yyunion.sty).}
}

\defc\flreateol{%#1#2#3% regular expression with the end of line as a trailing context :: \flreateol{re}{fptr}{sptr}
    {\toksc{}#1\expandafter}\expandafter\toksd\expandafter{\the\toksc{\raise0.5pt\rlap{\sscmd\ $\vdash$}}}%
    \concat\toksc\toksd%
}

\toyyunion{flexparser-re}

\def\flexsndisplay#1{% state name typesetting style
    {\let\ntt@\relax\idss#1}%
}

\def\flexrendisplay#1{% regular expression name typesetting style
    \penalty-100 \hbox{$\langle$\tt#1$\rangle$}%
}

\def\flexsnstyle#1{% typesetting \flex\ state names in text
    {%
        \let\parsernamespace\flexpseudonamespace
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksd{\the\toksa}\tokse{\the\toksb}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \let\termindex\writeidxfstextentry
        \edef\next{\toksd{\nx\flexsndisplay{\the\toksd}%
            \gidxentryxv{\termfsscstring}{#1}{\the\tokse}}}\next
        \expandafter
    }\the\toksd
}

\def\flexrenstyle#1{% typesetting \flex\ regular expression names in text
    {%
        \let\parsernamespace\flexpseudorenamespace
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksd{\the\toksa}\tokse{\the\toksb}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \let\termindex\writeidxfstextentry
        \edef\next{\toksd{\nx\flexrendisplay{\the\toksd}%
            \gidxentryxv{\termfsrestring}{#1}{\the\tokse}}}\next
        \expandafter
    }\the\toksd
}

\def\flexrestyle#1{% typesetting \flex\ regular expressions in text
    {%
        \frexproc{#1}\with\ppregex
        \edef\next{\toksc{\toksc{\the\toksc}}}\next
        \expandafter
    }\the\toksc
    \hbox{\textretrue\tt\the\toksc}%
}

% \flex\ section~2 typesetting

\defc\flactiongroup{% #1#2#3#4#5#6% a group of \flex actions :: \flactiongroup{\flsconlist...}{fptr}{sptr}{\flaction...}{fptr}{sptr}
    \toksb{}#1% collect all the state name information in \toksb
    % gather the preceeding stash
    \readstash{#3}% read stash late, after \flsconlist (ignore it)
    \edef\next{%
        \toksb{\noalign{\hbox to\nx\hsize{\hskip\the\rgindent
        \nx\flexsndisplay{\the\toksb$^{++}$}\toksa{\relax\the\toksg}\nx\makestashbox\nx\hfil}}}%
    }\next
    \astisempty{}{\astappend{\noalign{\medskip}}}%
    \astappendx{\the\toksb}%
    \advance\rgindent by\parindent
    #4%
    \advance\rgindent by-\parindent
}

\defc\flsconlist{% #1#2#3% start condition list :: \flsconlist{{fptr}{sptr}}{\flname...}{{fptr}{sptr}}
    \readstashwredirect#1\toksg % save the stash preceeding the conditions list to reinsert later
    \savecslist{local-namespace}\yyflunion
    \restorecslist{flexparser-sect2:state-name-display}\yyflunion
    #2% collect the state names (and the stash in between)
    \restorecslist{local-namespace}\yyflunion
    \readstashwredirect#3\toksf
    \appendr\toksb{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
}

\newskip\rgindent
\rgindent=\parindent

\def\appendinteractionskip{%
    \ifnum\prevdepth<\dp\strutbox
        \vskip-\prevdepth\vskip\dp\strutbox
    \fi
    \nointerlineskip
}

\def\flaction@generic#1#2#3#4#5{% #1 is the #1 of \flaction.*
                                % #2 is the #2 of \flaction.*
                                % #3 is the #6 of \flaction.*
                                % #4 is the #7 of \flaction.*
                                % #5 is the \toksa prefix (should be expandable)
    \toksb{}#1% collect all the state name information in \toksb                 
    \toksc{}#2%
    #4% apply formatting
    \readstash{#3}% read stash late, after \flsconlist and \fl...rule
    \yytoksempty\toksb{% no state names were collected
        \astappendx{\hskip\the\rgindent\hbox{\nx\tt\nx\strut\the\toksc}}%
    }{% put state names first
        \astisempty{}{\astappend{\noalign{\medskip}}}%
        \astappendx{%
            \noalign{\nx\appendinteractionskip}%
            \hskip\the\rgindent
            \vbox{%
              %{\nx\sscmd\nx\expandafter}\nx\expandafter\baselineskip\nx\the\baselineskip
              % the line above would allow a smaller vertical skip to be inserted in case
              % the font for state names is reduced in size;
                \halign{&####\hfil\cr
                    \flexstatelisttypeset\cr
                    \hskip\parindent\nx\tt\nx\strut\the\toksc\cr
                }%
            }%
        }%
    }%
    \astappendx{&\toksa{#5\relax\the\yystashlocal}\nx\cr}%
}

\def\flexstatelisttypeset{%
    \hbox to\rgindent{%
        \vbox{\advance\hsize by -\rgindent
            \noindent
            \nx\strut\nx\flexsndisplay{\the\toksb$^+$}
        }\hss
    }%
}

\defc\flaction{%#1#2#3#4#5#6#7% \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
    \flaction@generic{#1}{#2}{#6}{#7}{}%
}

\defc\flactionc{%#1#2#3#4#5#6#7% \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
    \flaction@generic{#1}{#2}{#6}{#7}{{}$\nx\hookleftarrow$}%
}

\def\inscomment#1{%
    \def\astformat@flaction{%
        \toksd{#1}%
        \astappendx{\noalign{\nx\smallskip\noindent\hskip\the\rgindent
            $\nx\triangleright\;$\the\toksd\nx\smallskip}}%
    }%
}

\let\astformat@flaction\empty

\toyyunion{flexparser-sect2}

% flex state names display inside action groups

\restorecslist{flexparser-strict}\yyflunion

\defc\flname{% #1#2#3#4% name :: \flname{name}{name12}{fptr}{sptr}
    {%
        \let\parsernamespace\flexpseudonamespace
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \expandafter
    }\the\toksd
    \let\termindex\writeidxfsentry
    \readstashwredirect{}{#4}\toksf % do not touch \yystashlocal
    \appendr\toksb{{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
        \the\toksd\gidxentryxv{\termfsscstring}{#1}{\the\tokse}}%
    \let\termindex\eatone
}

\defc\flnamesep{%#1#2% separator between names :: \flnamesep{fptr}{sptr}
    \toksb\expandafter{\the\toksb\ }%
}

\toyyunion{flexparser-sect2:state-name-display}

% \flex\ section~1 typesetting

\restorecslist{flexparser-strict}\yyflunion

\defc\flredef{%#1#2#3#4#5#6#7#8% regular expression definition :: \flredef{name}{name12}{fptr}{sptr}{text}{text12}{fptr}{sptr}
    \let\termindex\writeidxfsrdentry
    {%
        \let\hostparsernamespace\flexpseudorenamespace
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \expandafter
    }\the\toksd
    \frexproc{#5}\with\ppregex
    \astappendx{{}$\nx\langle\hbox{\ntt\the\toksd}\nx\rangle${}%
        \gidxentryxv{\termfsrestring}{#1}{\the\tokse}&\the\toksc\cr
    }%
    \let\termindex\eatone
}

% \flex\ options

\defc\floptions{%#1 \flex options :: \floptions{\flopt ...}
    #1%
}

\defc\flopt{%#1#2#3#4#5% \flex option :: \flopt{option}{text:(no||opt_name)}{text12}{fptr}{sptr}
    \expandafter\ifx\csname flex@ption_#1\endcsname\relax
        \yybreak{%
            \errmessage{!option <#1> with value (#3) is not handled.}%
        }%
    \else
        \yybreak{%
            \csname flex@ption_#1\endcsname{#2}{#3}{#4}{#5}%
        }%
    \yycontinue
}

% specific options
\expandafter\def\csname flex@ption_other\endcsname#1#2#3#4{%
    \let\termindex\writeidxfsrnentry
    \storerawidxentry{\tt{\rm(}#1{\rm)}#2}%
    \astappendx{$\nx\langle${\ntt option}$\nx\rangle_{\nx\rm f}$%
        \gidxentryxv{\termfsopstring}{option}{option}%
        \yystringempty{#1}{%
            \gidxentry{\termttstring}{#2}{#2}% index the <option> keyword
        }{%
            \didxentry{\getstackpointer\indexterms}{#2}% index the parameter
        }%
        &\nx\quad{\ntt#1#2}\nx\cr}%
    \let\termindex\empty
}

\expandafter\def\csname flex@ption_file\endcsname#1#2#3#4{%
    \let\termindex\writeidxfsrnentry
    \astappendx{$\nx\langle${\ntt output to}$\nx\rangle_{\nx\rm f}$%
        \gidxentry{\termfsopstring}{output to}{output to}%
        \gidxentryxv{\termttstring}{#2}{\eatone#2}% remove the first quote for sorting
        &\nx\quad{\ntt#1}\nx\cr}%
    \let\termindex\empty
}

\expandafter\def\csname flex@ption_header\endcsname#1#2#3#4{%
    \let\termindex\writeidxfsrnentry
    \astappendx{$\nx\langle${\ntt header file}$\nx\rangle_{\nx\rm f}$%
        \gidxentry{\termfsopstring}{header}{header}%
        \gidxentryxv{\termttstring}{#2}{\eatone#2}% remove the first quote for sorting
        &\nx\quad{\ntt#1}\nx\cr}%
    \let\termindex\empty
}

\expandafter\def\csname flex@ption_prefix\endcsname#1#2#3#4{%
    \let\termindex\writeidxfsrnentry
    \astappendx{$\nx\langle${\ntt prefix}$\nx\rangle_{\nx\rm f}$%
        \gidxentry{\termfsopstring}{prefix}{prefix}%
        \gidxentry{\termttstring}{#2}{#2}%
        &\nx\quad{\ntt#1}\nx\cr}%
    \let\termindex\empty
}

% the option below processes the parameter in C mode
\expandafter\def\csname flex@ption_xtype\endcsname#1#2#3#4{%
    \readstash{#4}%
    \let\termindex\writeidxfsrnentry
    \astappendx{$\nx\langle${\ntt extra type}$\nx\rangle_{\nx\rm f}$%
        \gidxentry{\termfsopstring}{extra type}{extra type}%
        &\nx\quad{\toksa{\the\yystashlocal}\nx\makestashbox}\nx\cr}%
    \let\termindex\empty
}

\expandafter\def\csname flex@ption_deprecated\endcsname#1#2#3#4{%
    \flex@opt@deprecated#2%
}

\def\flex@opt@deprecated#1#2#3^^J{%
    \let\termindex\writeidxfsrnentry
    \astappendx{$\nx\langle${\ntt #2}$\nx\rangle_{\nx\rm f}$\gidxentry{\termfsopstring}{#2}{#2}%
        &\number\flexgetnumber#3.\nx\cr}%
    \let\termindex\empty
}

\def\flexgetnumber#1{%
    \ifnum`#1<"3A
        \yybreak{%
            \ifnum`#1>"2F
                \yybreak{#1\flexgetnumber}%
            \else
                \yybreak{%
                    \ifnum`#1="20
                        \yybreak{\flexgetnumber}%
                    \else
                        \yybreak{}%
                    \yycontinue
                }
            \yycontinue
        }%
    \else
        \yybreak{}%
    \yycontinue
}

\defc\flscondecl{%#1#2#3#4% start condition declarations :: \flscondecl{x|s}{fptr}{sptr}{\flname ...}
    \readstashwredirect{#2}{#3}\toksg % save the stash preceeding the conditions list to reinsert later
    \savecslist{local-namespace}\yyflunion
    \restorecslist{flexparser-sect1:state-name-display}\yyflunion
    \toksb{}#4% collect the state names (and the stash in between) in \toksb
    \restorecslist{local-namespace}\yyflunion
    %\readstashwredirect#3\toksf
    %\appendr\toksb{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
    \let\termindex\writeidxfsentry
    \astappendx{$\langle${\bf state-#1}$\rangle_{\rm f}$%
        \gidxentryxv{\termfsopstring}{state-#1}{state-#1}%
        \nx\quad\nx\flexsndisplay{\the\toksb}\toksa{\relax\the\toksg}\nx\makestashbox\nx\hfil&\nx\cr}%
}

\toyyunion{flexparser-sect1}

% inside state name declarations

\restorecslist{flexparser-strict}\yyflunion

\defc\flname{% #1#2#3#4% name :: \flname{name}{name12}{fptr}{sptr}
    {%
        \let\parsernamespace\flexpseudonamespace
        \nameproc{#1}\with\parsebin
        \edef\next{%
            \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
        }\next % a trick to `reshuffle' the output of \nameproc:
        % the parsed name goes to \toksd and the visual key is put in \tokse
        \expandafter
    }\the\toksd
    \let\termindex\writeidxfsdentry
    \readstashwredirect{}{#4}\toksf % do not touch \yystashlocal
    \appendr\toksb{{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
        \the\toksd\gidxentryxv{\termfsscstring}{#1}{\the\tokse}}%
    \let\termindex\eatone
}

\defc\flnamesep{%#1#2% separator between names :: \flnamesep{fptr}{sptr}
    \toksb\expandafter{\the\toksb\ }%
}

\toyyunion{flexparser-sect1:state-name-display}

% correct a few sequences for debugging purposes

\restorecslist{flexparser-debug}\yyflunion

\defc\flnametok{%#1#2% named definition :: \flnametok{matched text}{formatting command}, the text is \{text\}\ *
    \noexpand\flnametok{#1}{}%
}

\defc\flparens{%#1#2#3#4% parenthesized regular expression :: \flparens{{fptr}{sptr}}{re}{{fptr}{sptr}}{formatting command}
    \noexpand\flparens{#1}{#2}{#3}{}%
}

\defc\flrule{%#1#2% full regular expression :: \flrule{re}{formatting command}
    \noexpand\flrule{#1}{}%
}

\defc\flbolrule{%#1#2% full regular expression at the beginning of the line :: \flbolrule{re}{formatting command}
    \noexpand\flbolrule{#1}{}%
}

\defc\flaction{%#1#2#3#4#5#6#7% \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
    \noexpand\flaction{#1}{#2}{#3}{#4}{#5}{#6}{}%
}

\defc\flactionc{%#1#2#3#4#5#6#7% \flex continued action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
    \noexpand\flactionc{#1}{#2}{#3}{#4}{#5}{#6}{}%
}

\toyyunion{flexparser-debug}

\restorecslist{flexparser-prototypes}\yyflunion

\defc\flscondecl{%#1#2#3#4% start condition declarations :: \flscondecl{x|s}{fptr}{sptr}{\flname ...}
    #4%
}
  
\defc\flname{% #1#2#3#4% name :: \flname{name}{name12}{fptr}{sptr}
    {\immediate\write\stlist{_register_name(#2);}}%
}

\defc\flnamesep{%#1#2% separator between names :: \flnamesep{fptr}{sptr}
}

\toyyunion{bootstrap-flex}

% remembering the rule associated with a specific section reference

\newtoks\currentheader
\newtoks\currentterm
\newtoks\singlerule
\newtoks\ruleset
\newcount\refrulecount

\restorecslist{parser-prototypes}\yyunion % we just want to ignore the non-rule related code
\def\parserprototypesnamespace{parser-strict:headers} % make sure the prototypes come from the correct namespace

\defc\prodheader{#1\currentheader\currentterm}

% the macros below are defined in `parser-prototypes'
%\defp\rrhssep#1#2{}
%\defp\arhssep#1#2{}
%\defp\rarhssep#1#2{}
%\defp\prodprodsep{}
%\defp\onesymbol#1#2#3{}

\defc\rhs{% a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
    \singlerule{}#1%
    \savecs{local-namespace}{\_}%
    \let\_\uscoreletter
    \saveruleforheader
    \restorecs{local-namespace}{\_}%
}

\defc\termname{% a production term :: \termname{\idit|\stringify|\charit}{\idit}
    #1\appendr\singlerule{\the\currentterm}\yystringempty{#2}{}{#2\appendr\singlerule{[\the\currentterm] }}%
}

\defc\actbraces{%
    \readformat{#3}%
    \actbraces@refinit
    \readstash{#4}%
    \savecs{local-namespace}{\format\formatlocal}%
    \let\format\formatreinsert
    \let\formatlocal\formatreinsertlocal
    \the\yyformatlocal
    \appendr\singlerule{ \nx\{\nx\stashed{\the\yystashlocal}\nx\} }%
    \expandafter\lookforX\the\yystashlocal\X:\X\end
    \expandafter\lookforXmark\the\yystashlocal\Xmark:\Xmark\end
    \restorecs{local-namespace}{\format\formatlocal}%
}

\let\actbraces@refinit\empty

% sometimes all format hints have to be erased (an example would be when collecting
% productions marked by \Xmark and omitting an \inline while keeping a \fold)
\newif\iferaselocalformat
\newif\iferaseformat

\def\formatreinsert#1{\iferaseformat\else\appendrnx\singlerule{\format{#1}}\fi}
\def\formatreinsertlocal#1{\iferaselocalformat\else\appendrnx\singlerule{\formatlocal{#1}}\fi}

% macros below (normally) package each production into an individual rule, complete with the header;
% there are two ways to avoid this (i.e. to package the whole production set for a single lhs):
% o using the header term for the label (in this case every production for the given lhs will
%   be collected in the appropriate command sequence, see \saveruleforheader below)
% o using the \Xmark label with the empty name part (e.g. \Xmark label:\Xmark) to mark the
%   productions one wishes to collect for this term; note that the \Xmark mechanism does not
%   verify that the given production is for the same lhs

\def\lookforX#1\X#2:#3\X#4\end{%
    \yystringempty{#2}{}{%
        \expandafter\ifx\csname prodrulefor[#2]\endcsname\relax
            \yybreak{%
                \expandafter\xdef\csname prodrulefor[#2]\endcsname{\the\currentheader: }%
                \addtoprodrulefor{#2}{\the\singlerule}%
            }%
        \else
            \yybreak{%
                \addtoprodrulefor{#2}{\the\currentheader: }%
                \addtoprodrulefor{#2}{\the\singlerule}%
            }%
        \yycontinue
        \lookforX#4\end
    }%
}

\def\lookforXmark#1\Xmark#2:#3\Xmark#4\end{%
    \yystringempty{#2}{}{%
        \expandafter\ifx\csname prodrulefor[#2]\endcsname\relax
            \yybreak{%
                \expandafter\xdef\csname prodrulefor[#2]\endcsname{\the\currentheader: }%
                \addtoprodrulefor{#2}{\the\singlerule}%
            }%
        \else
            \yybreak{%
                \yystringempty{#3}{%
                    \addtoprodrulefor{#2}{|}%
                }{%
                    \addtoprodrulefor{#2}{\the\currentheader: }%
                }%
                \addtoprodrulefor{#2}{\the\singlerule}%
            }%
        \yycontinue
        \lookforXmark#4\end    
    }%
}

\def\saveruleforheader{%
    \expandafter\ifx\csname prodrulefor[\the\currentheader]\endcsname\relax
        \yybreak{%
            \expandafter\xdef\csname prodrulefor[\the\currentheader]\endcsname{\the\currentheader: \the\singlerule}%
        }%
    \else
        \yybreak{%
            \addtoprodrulefor{\the\currentheader}{\expandafter|\the\singlerule}%
        }%
    \yycontinue
}

\def\addtoprodrulefor#1#2{%
    \appendxrt{\gdef}{prodrulefor[#1]}{#2}{}%
}

\def\mergeformatting#1{%
    \readformat{#1}%
    \savecs{local-namespace}{\format\formatlocal}%
    \let\format\formatreinsert
    \let\formatlocal\formatreinsertlocal
    \the\yyformatlocal
    \restorecs{local-namespace}{\format\formatlocal}%
}

\defc\rules{#1}
\defc\oneproduction{#1}
\defc\emptyterm{\appendr\singlerule{ }}
\defc\hspace{\appendr\singlerule{ }}
\defc\stringify{\mergeformatting{#3}\currentterm{"#2"}}
\defc\idit{\mergeformatting{#3}\currentterm{#2}}
\defc\charit{\mergeformatting{#3}\currentterm{'#2'}}
\defc\anint{\mergeformatting{#3}\currentterm{#1}}
\defc\hexint{\mergeformatting{#3}\currentterm{#1}}

\savecslist{reference:delayed}\yyunion

% using the macros above

\def\extractprodrefs{% this is the command sequence that performs the preprocessing of the
                     % parsed table to set the rule references
    \restorecslist{reference:delayed}\yyunion
    \savecs{local-names}{\*}\let\*\empty % in case the name of the section comes from a change file
    \the\table
    \restorecs{local-names}{\*}%
    \resetlistpointer\yystash
    \resetlistpointer\yyformat
}

\def\PB@#1{\PB@@#1\end}
\def\PB@@$#1$\end{#1}
\def\X@#1:#2\X{#1}
\def\Xmark#1\Xmark{}

\def\rulerefrequests{} % here we collect all the reference `requests' for the rules
                       % (parts of) which may be defined later in the file

% referencing previously processed rules:
%   #1 is additional setup code (such as \showlastactiontrue)
%   #2 is formatting commands (typical use is to make it \nx\flatten)
%   #3 is a command sequence that expands to the desired section number
%      (the macros just above make is possible to use |@<Section name@>|
%       for that)

\def\rulereferencex#1#2#3{%
    \begingroup
        \savecs{local-names}{\PB\X\Xmark}%
        \let\PB\PB@ \let\X\X@ \let\Xmark\X@
        \let\textproductionsetup\rulereftextproductionsetup
        {%
            \toksa\expandafter{\rulerefrequests}%
            \edef\rulerefrequests{\the\toksa{#3}}%
            \expandafter
        }\expandafter\gdef\expandafter\rulerefrequests\expandafter{\rulerefrequests}%
        \expandafter\ifx\csname full.prodrulefor[#3]\endcsname\relax
        \else
            \expandafter\expandafter\expandafter
                \singlerule\expandafter\expandafter
                \expandafter{\csname full.prodrulefor[#3]\endcsname}%
            \restorecs{local-names}{\PB\X\Xmark}%
            #1%
            \edef\next{\nx\beginprod#2\the\singlerule\nx\endprod}\next
        \fi
    \endgroup
}

\def\rulereftextproductionsetup{%
    \ninepoint
    \let\returnexplicitspace\splitexplicitspace
    %\let\acharswitch\texcharadjust
    %\let\onecharswitch\texcsadjust% we do not need the adjustments
    %since the names have already gone through the lexer  
    \let\extractprodtableinfo\empty % we do not prepocess the table
    \let\actionfiller\empty
    \fillpstack{b}{%
        \preparsebisongrammar
        \preparsebisonprologue
        {\preparsefallbacktext{**}}%
        \relax % this \relax is necessary so that the braces above 
               % are not stripped by \poppstack
    }%
}

\def\rulereference#1#2{\rulereferencex{\showlastactiontrue}{#1}{#2}}

\def\thisrulereference#1{\rulereference{#1}{\secno}}

% we use lazy evaluation for the references: reference requests are collected 
% and put into the auxilary file at the end of the job

\def\lookupruleref#1{%
    \yystringempty{#1}{}% done
    {%
        \expandafter\expandafter\expandafter
            \toksa
        \expandafter\expandafter\expandafter{\csname prodrulefor[#1]\endcsname}%
        \immediate\write\auxstream{\noexpand\getrulereffor{#1}{\the\toksa}}%
        \lookupruleref
    }%
}

\def\yyuniontag{\auxunionctl}
\defp\gramrefunionctl{}% sequence to activate/deactivate/take other action for
                    % gramref set of macros

\defc\gramrefunionctl{%
    \restorecslist{aux:global:gramrefs}\gramrefunion
}
\savecs{aux:global:activate}\gramrefunionctl

\defc\gramrefunionctl{%
    \restorecslist{aux:global:gramrefs:neutral}\gramrefunion
}
\savecs{aux:global:deactivate}\gramrefunionctl

\defc\gramrefunionctl{% this command should not make any page material contributions
    \message{gramrefs...}%
}
\savecs{aux:global:preend}\gramrefunionctl

\defc\gramrefunionctl{% this command should not make any page material
                      % contributions
    \message{gramrefs.store...}%
    \expandafter\lookupruleref\rulerefrequests{}%
}
\savecs{aux:global:precutoff}\gramrefunionctl

\defc\gramrefunionctl{% writing a headers, etc.
    \message{gramrefs.done...}%
    \toksa\expandafter{\gramrefunion}%
    \immediate\write\auxstream{\harmlesscomment\the\toksa}%
}
\savecs{aux:global:prestart}\gramrefunionctl

\yyuniondeclare\gramrefunion{aux:global:gramrefs}

\defp\getrulereffor#1#2{}

\toyyunion{aux:global:gramrefs:neutral}

\defc\getrulereffor#1#2{%
    \expandafter\def\csname full.prodrulefor[#1]\endcsname{#2}%
}

\toyyunion{aux:global:gramrefs} % do nothing for now
