% \iffalse meta-comment
%
%% File: l3cctab.dtx
%
% Copyright (C) 2018-2024 The LaTeX Project
%
% It may be distributed and/or modified under the conditions of the
% LaTeX Project Public License (LPPL), either version 1.3c of this
% license or (at your option) any later version.  The latest version
% of this license is in the file
%
%    http://www.latex-project.org/lppl.txt
%
% This file is part of the "l3kernel bundle" (The Work in LPPL)
% and all files in that bundle must be distributed together.
%
% -----------------------------------------------------------------------
%
% The development version of the bundle can be found at
%
%    https://github.com/latex3/latex3
%
% for those people who are interested.
%
%<*driver>
\documentclass[full]{l3doc}
\begin{document}
  \DocInput{\jobname.dtx}
\end{document}
%</driver>
% \fi
%
% \title{^^A
%   The \pkg{l3cctab} module\\ Category code tables^^A
% }
%
% \author{^^A
%  The \LaTeX{} Project\thanks
%    {^^A
%      E-mail:
%        \href{mailto:latex-team@latex-project.org}
%          {latex-team@latex-project.org}^^A
%    }^^A
% }
%
% \date{Released 2024-12-09}
%
% \maketitle
%
% \begin{documentation}
%
% A category code table enables rapid switching of all category codes in
% one operation. For \LuaTeX{}, this is possible over the entire Unicode
% range. For other engines, only the $8$-bit range ($0$--$255$) is covered by
% such tables. The implementation of category code tables in \pkg{expl3}
% also saves and restores the \TeX{} \tn{endlinechar} primitive value, meaning
% they could be used for example to implement \cs{ExplSyntaxOn}.
%
% \section{Creating and initialising category code tables}
%
% \begin{function}[updated = 2020-07-02]{\cctab_new:N,\cctab_new:c}
%   \begin{syntax}
%     \cs{cctab_new:N} \meta{category code table}
%   \end{syntax}
%   Creates a new \meta{category code table} variable or raises an error if
%   the name is already taken. The declaration is global.  The
%   \meta{category code table} is initialised with the codes
%   as used by \IniTeX{}.
% \end{function}
%
% \begin{function}[updated = 2020-07-07]{\cctab_const:Nn,\cctab_const:cn}
%   \begin{syntax}
%     \cs{cctab_const:Nn} \meta{category code table} \Arg{category code set up}
%   \end{syntax}
%   Creates a new \meta{category code table}, applies (in a group) the
%   \meta{category code set up} on top of \IniTeX{} settings,
%   then saves them globally as a constant
%   table.  The \meta{category code set up} can include a call to
%   \cs{cctab_select:N}.
% \end{function}
%
% \begin{function}[updated = 2020-07-07]{\cctab_gset:Nn,\cctab_gset:cn}
%   \begin{syntax}
%     \cs{cctab_gset:Nn} \meta{category code table} \Arg{category code set up}
%   \end{syntax}
%   Starting from the \IniTeX{} category codes,
%   applies (in a group) the \meta{category code set up}, then saves them
%   globally in the \meta{category code table}.  The \meta{category code set up}
%   can include a call to \cs{cctab_select:N}.
% \end{function}
%
% \begin{function}[added = 2023-05-26]
%   {\cctab_gsave_current:N,\cctab_gsave_current:c}
%   \begin{syntax}
%     \cs{cctab_gsave_current:N} \meta{category code table}
%   \end{syntax}
%   Saves the current prevailing category codes in the
%   \meta{category code table}.
% \end{function}
%
% \section{Using category code tables}
%
% \begin{function}[updated = 2020-07-02]{\cctab_begin:N,\cctab_begin:c}
%   \begin{syntax}
%     \cs{cctab_begin:N} \meta{category code table}
%   \end{syntax}
%   Switches locally the category codes in force to those stored in the
%   \meta{category code table}.  The prevailing codes before the
%   function is called are added to a stack, for use with
%   \cs{cctab_end:}. This function does not start a \TeX{} group.
% \end{function}
%
% \begin{function}[updated = 2020-07-02]{\cctab_end:}
%   \begin{syntax}
%     \cs{cctab_end:}
%   \end{syntax}
%   Ends the scope of a \meta{category code table} started using
%   \cs{cctab_begin:N}, returning the codes to those in force before the
%   matching \cs{cctab_begin:N} was used.  This must be used within the
%   same \TeX{} group (and at the same \TeX{} group level) as the
%   matching \cs{cctab_begin:N}.
% \end{function}
%
% \begin{function}[added = 2020-05-19, updated = 2020-07-02]{\cctab_select:N, \cctab_select:c}
%   \begin{syntax}
%     \cs{cctab_select:N} \meta{category code table}
%   \end{syntax}
%   Selects the \meta{category code table} for the scope of the current
%   group.  This is in particular useful in the \meta{setup} arguments
%   of \cs{tl_set_rescan:Nnn}, \cs{tl_rescan:nn}, \cs{cctab_const:Nn},
%   and \cs{cctab_gset:Nn}.
% \end{function}
%
% \begin{function}[EXP, added = 2021-05-10]{\cctab_item:Nn, \cctab_item:cn}
%   \begin{syntax}
%     \cs{cctab_item:Nn} \meta{category code table} \Arg{int expr}
%   \end{syntax}
%   Determines the \meta{character} with character code given by the
%   \meta{int expr} and expands to its category code specified
%   by the \meta{category code table}.
% \end{function}
%
% \section{Category code table conditionals}
%
% \begin{function}[pTF]{\cctab_if_exist:N, \cctab_if_exist:c}
%   \begin{syntax}
%     \cs{cctab_if_exist_p:N} \meta{category code table}
%     \cs{cctab_if_exist:NTF} \meta{category code table} \Arg{true code} \Arg{false code}
%   \end{syntax}
%   Tests whether the \meta{category code table} is currently defined.
%   This does not check that the \meta{category code table} really is a
%   category code table.
% \end{function}
%
% \section{Constant and scratch category code tables}
%
% \begin{variable}[updated = 2020-07-10]{\c_code_cctab}
%   Category code table for the \pkg{expl3} code environment; this does
%   \emph{not} include \texttt{@}, which is retained as an \enquote{other}
%   character.  Sets the \tn{endlinechar} value to $32$ (a space).
% \end{variable}
%
% \begin{variable}[updated = 2020-07-08]{\c_document_cctab}
%   Category code table for a standard \LaTeX{} document, as set by the \LaTeX{}
%   kernel. In particular, the upper-half of the $8$-bit range will be set to
%   \enquote{active} with \pdfTeX{} \emph{only}. No \pkg{babel} shorthands
%   will be activated.  Sets the \tn{endlinechar} value to $13$ (normal
%   line ending).
% \end{variable}
%
% \begin{variable}[updated = 2020-07-02]{\c_initex_cctab}
%   Category code table as set up by \IniTeX{}.
% \end{variable}
%
% \begin{variable}[updated = 2020-07-02]{\c_other_cctab}
%   Category code table where all characters have category code $12$
%   (other). Sets the \tn{endlinechar} value to $-1$.
% \end{variable}
%
% \begin{variable}[updated = 2020-07-02]{\c_str_cctab}
%   Category code table where all characters have category code $12$
%   (other) with the exception of spaces, which have category code
%   $10$ (space).  Sets the \tn{endlinechar} value to $-1$.
% \end{variable}
%
% \begin{variable}[added = 2023-05-26]{\g_tmpa_cctab, \g_tmpb_cctab}
%   Scratch category code tables.
% \end{variable}
%
% \end{documentation}
%
% \begin{implementation}
%
% \section{\pkg{l3cctab} implementation}
%
%    \begin{macrocode}
%<*package>
%    \end{macrocode}
%
%    \begin{macrocode}
%<@@=cctab>
%    \end{macrocode}
%
% As \LuaTeX{} offers engine support for category code tables, and this
% is entirely lacking from the other engines, we need two complementary
% approaches. (Some future \XeTeX{} may add support, at which point the
% conditionals below would be different.)
%
% \subsection{Variables}
%
% \begin{variable}{\g_@@_stack_seq, \g_@@_unused_seq}
%   List of catcode tables saved by nested \cs{cctab_begin:N}, to
%   restore catcodes at the matching \cs{cctab_end:}.  When popped from
%   the \cs{g_@@_stack_seq} the table numbers are stored in
%   \cs{g_@@_unused_seq} for later reuse.
%    \begin{macrocode}
\seq_new:N \g_@@_stack_seq
\seq_new:N \g_@@_unused_seq
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\g_@@_group_seq}
%   A stack to store the group level when a catcode table started.
%    \begin{macrocode}
\seq_new:N \g_@@_group_seq
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\g_@@_allocate_int}
%   Integer to keep track of what category code table to allocate.  In
%   \LuaTeX{} it is only used in format mode to implement
%   \cs{cctab_new:N}.  In other engines it is used to make csnames for
%   dynamic tables.
%    \begin{macrocode}
\int_new:N  \g_@@_allocate_int
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\l_@@_internal_a_tl,\l_@@_internal_b_tl}
%   Scratch space.  For instance, when popping
%   \cs{g_@@_stack_seq}/\cs{g_@@_unused_seq}, consists of the
%   catcodetable number (integer denotation) in \LuaTeX{}, or of an
%   intarray variable (as a single token) in other engines.
%    \begin{macrocode}
\tl_new:N \l_@@_internal_a_tl
\tl_new:N \l_@@_internal_b_tl
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\g_@@_endlinechar_prop}
%   In \LuaTeX{} we store the \tn{endlinechar} associated to each
%   \tn{catcodetable} in a property list, unless it is the default
%   value~$13$.
%    \begin{macrocode}
\prop_new:N \g_@@_endlinechar_prop
%    \end{macrocode}
% \end{variable}
%
% \subsection{Allocating category code tables}
%
% \begin{macro}{\cctab_new:N, \cctab_new:c, \@@_new:N, \@@_gstore:Nnn}
%   The \cs{@@_new:N} auxiliary allocates a new catcode table but does
%   not attempt to set its value consistently across engines.  It is
%   used both in \cs{cctab_new:N}, which sets catcodes to \IniTeX{}
%   values, and in \cs{cctab_begin:N}/\cs{cctab_end:} for dynamically
%   allocated tables.
%
%   First, the \LuaTeX{} case.
%   Creating a new category code table is done like other registers.
%   In Con\TeX{}t, \tn{newcatcodetable} does not include the initialisation,
%   so that is added explicitly.
%    \begin{macrocode}
\sys_if_engine_luatex:TF
  {
    \cs_new_protected:Npn \cctab_new:N #1
      {
        \__kernel_chk_if_free_cs:N #1
        \@@_new:N #1
      }
    \cs_new_protected:Npn \@@_new:N #1
      {
        \newcatcodetable #1
        \tex_initcatcodetable:D #1
      }
  }
%    \end{macrocode}
%   Now the case for other engines. Here, each table is an integer
%   array.  Following the \LuaTeX{} pattern, a new table starts with
%   \IniTeX{} codes.  The \cs{debug_suspend:} and \cs{debug_resume:}
%   functions prevent errors and logging from \texttt{debug} commands
%   which are either duplicate or false when \cs{@@_new:N} is used
%   by \cs{cctab_new:N} or \cs{cctab_const:Nn}.
%   The index base is out-by-one, so we have an
%   internal function to handle that.  The \IniTeX{} \tn{endlinechar} is
%   $13$.
%    \begin{macrocode}
  {
    \cs_new_protected:Npn \@@_new:N #1
      {
        \debug_suspend:
        \intarray_new:Nn #1 { 257 }
        \debug_resume:
      }
    \cs_new_protected:Npn \@@_gstore:Nnn #1#2#3
      { \intarray_gset:Nnn #1 { #2 + 1 } {#3} }
    \cs_new_protected:Npn \cctab_new:N #1
      {
        \__kernel_chk_if_free_cs:N #1
        \@@_new:N #1
        \int_step_inline:nn { 256 }
          { \__kernel_intarray_gset:Nnn #1 {##1} { 12 } }
        \__kernel_intarray_gset:Nnn #1 { 257 } { 13 }
        \@@_gstore:Nnn #1 { 0 } { 9 }
        \@@_gstore:Nnn #1 { 13 } { 5 }
        \@@_gstore:Nnn #1 { 32 } { 10 }
        \@@_gstore:Nnn #1 { 37 } { 14 }
        \int_step_inline:nnn { 65 } { 90 }
          { \@@_gstore:Nnn #1 {##1} { 11 } }
        \@@_gstore:Nnn #1 { 92 } { 0 }
        \int_step_inline:nnn { 97 } { 122 }
          { \@@_gstore:Nnn #1 {##1} { 11 } }
        \@@_gstore:Nnn #1 { 127 } { 15 }
      }
  }
\cs_generate_variant:Nn \cctab_new:N { c }
%    \end{macrocode}
% \end{macro}
%
% \subsection{Saving category code tables}
%
% \begin{macro}{\@@_gset:n, \@@_gset_aux:n}
%   In various functions we need to save the current catcodes (globally)
%   in a table.  In \LuaTeX{}, saving the catcodes is a primitives, but
%   the \tn{endlinechar} needs more work: to avoid filling
%   \cs{g_@@_endlinechar_prop} with many entries we special-case the
%   default value $13$.  In other engines we store $256$ current
%   catcodes and the \tn{endlinechar} in an intarray variable.
%    \begin{macrocode}
\sys_if_engine_luatex:TF
  {
    \cs_new_protected:Npn \@@_gset:n #1
      { \exp_args:Nf \@@_gset_aux:n { \int_eval:n {#1} } }
    \cs_new_protected:Npn \@@_gset_aux:n #1
      {
        \tex_savecatcodetable:D #1 \scan_stop:
        \int_compare:nNnTF { \tex_endlinechar:D } = { 13 }
          { \prop_gremove:Nn \g_@@_endlinechar_prop {#1} }
          {
            \prop_gput:NnV \g_@@_endlinechar_prop {#1}
              \tex_endlinechar:D
          }
      }
  }
  {
    \cs_new_protected:Npn \@@_gset:n #1
      {
        \int_step_inline:nn { 256 }
          {
            \__kernel_intarray_gset:Nnn #1 {##1}
              { \char_value_catcode:n { ##1 - 1 } }
          }
        \__kernel_intarray_gset:Nnn #1 { 257 }
          { \tex_endlinechar:D }
      }
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\cctab_gset:Nn, \cctab_gset:cn}
%   Category code tables are always global, so only one version of
%   assignments is needed.  Simply run the setup in a group and save the
%   result in a category code table~|#1|, provided it is valid.  The
%   internal function is defined above depending on the engine.
%    \begin{macrocode}
\cs_new_protected:Npn \cctab_gset:Nn #1#2
  {
    \@@_chk_if_valid:NT #1
      {
        \group_begin:
          \cctab_select:N \c_initex_cctab
          #2 \scan_stop:
          \@@_gset:n {#1}
        \group_end:
      }
  }
\cs_generate_variant:Nn \cctab_gset:Nn { c }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\cctab_gsave_current:N, \cctab_gsave_current:c}
%   Very simple.
%    \begin{macrocode}
\cs_new_protected:Npn \cctab_gsave_current:N #1
  {
    \@@_chk_if_valid:NT #1
      { \@@_gset:n {#1} }
  }
\cs_generate_variant:Nn \cctab_gsave_current:N { c }
%    \end{macrocode}
% \end{macro}
%
% \subsection{Using category code tables}
%
% \begin{variable}{\g_@@_internal_cctab}
% \begin{macro}[EXP]{\@@_internal_cctab_name:}
%   In \LuaTeX{}, we must ensure that the saved tables are read-only.
%   This is done by applying the saved table, then switching immediately
%   to a scratch table.  Any later catcode assignment will affect that
%   scratch table rather than the saved one.  If we simply switched to
%   the saved tables, then \cs{char_set_catcode_other:N} in the example
%   below would change \cs{c_document_cctab} and a later use of that
%   table would give the wrong category code to |_|.
% \begin{verbatim}
% \use:n
%   {
%     \cctab_begin:N \c_document_cctab
%       \char_set_catcode_other:N \_
%     \cctab_end:
%     \cctab_begin:N \c_document_cctab
%       \int_compare:nTF { \char_value_catcode:n { `_ } = 8 }
%         { \TRUE } { \ERROR }
%     \cctab_end:
%   }
% \end{verbatim}
%   We must also make sure that a scratch table is never reused in a
%   nested group: in the following example, the scratch table used by
%   the first \cs{cctab_begin:N} would be changed globally by the second
%   one issuing \tn{savecatcodetable}, and after \cs{group_end:} the
%   wrong category codes (those of \cs{c_str_cctab}) would be imposed.
%   Note that the inner \cs{cctab_end:} restores the correct catcodes
%   only locally, so the problem really comes up because of the
%   different grouping level.  The simplest is to use a scratch table
%   labeled by the \tn{currentgrouplevel}.  We initialize one of them as
%   an example.
% \begin{verbatim}
% \use:n
%   {
%     \cctab_begin:N \c_document_cctab
%       \group_begin:
%         \cctab_begin:N \c_str_cctab
%         \cctab_end:
%       \group_end:
%     \cctab_end:
%   }
% \end{verbatim}
%    \begin{macrocode}
\sys_if_engine_luatex:T
  {
    \@@_new:N \g_@@_internal_cctab
    \cs_new:Npn \@@_internal_cctab_name:
      {
        g_@@_internal
        \tex_romannumeral:D \tex_currentgrouplevel:D
        _cctab
      }
  }
%    \end{macrocode}
% \end{macro}
% \end{variable}
%
% \begin{macro}{\cctab_select:N, \cctab_select:c}
% \begin{macro}{\@@_select:N}
%   The public function simply checks the \meta{cctab~var} exists before
%   using the engine-dependent \cs{@@_select:N}.  Skipping these checks
%   would result in low-level engine-dependent errors.  First, the
%   \LuaTeX{} case.  In other engines, selecting a catcode table is a matter
%   of doing $256$ catcode assignments and setting the \tn{endlinechar}.
%    \begin{macrocode}
\cs_new_protected:Npn \cctab_select:N #1
  { \@@_chk_if_valid:NT #1 { \@@_select:N #1 } }
\cs_generate_variant:Nn \cctab_select:N { c }
\sys_if_engine_luatex:TF
  {
    \cs_new_protected:Npn \@@_select:N #1
      {
        \tex_catcodetable:D #1
        \prop_get:NVNTF \g_@@_endlinechar_prop #1 \l_@@_internal_a_tl
          { \int_set:Nn \tex_endlinechar:D { \l_@@_internal_a_tl } }
          { \int_set:Nn \tex_endlinechar:D { 13 } }
        \cs_if_exist:cF { \@@_internal_cctab_name: }
          { \exp_args:Nc \@@_new:N { \@@_internal_cctab_name: } }
        \exp_args:Nc \tex_savecatcodetable:D { \@@_internal_cctab_name: }
        \exp_args:Nc \tex_catcodetable:D { \@@_internal_cctab_name: }
      }
  }
  {
    \cs_new_protected:Npn \@@_select:N #1
      {
        \int_step_inline:nn { 256 }
          {
            \char_set_catcode:nn { ##1 - 1 }
              { \__kernel_intarray_item:Nn #1 {##1} }
          }
        \int_set:Nn \tex_endlinechar:D
          { \__kernel_intarray_item:Nn #1 { 257 } }
      }
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{variable}{\g_@@_next_cctab}
% \begin{macro}{\@@_begin_aux:}
%   For \cs{cctab_begin:N}/\cs{cctab_end:} we will need to allocate
%   dynamic tables.  This is done here by \cs{@@_begin_aux:}, which puts
%   a table number (in \LuaTeX{}) or name (in other engines) into
%   \cs{l_@@_internal_a_tl}.  In \LuaTeX{} this simply calls \cs{@@_new:N}
%   and uses the resulting catcodetable number; in other engines we need
%   to give a name to the intarray variable and use that.  In \LuaTeX{},
%   to restore catcodes at \cs{cctab_end:} we cannot just set
%   \tn{catcodetable} to its value before \cs{cctab_begin:N}, because
%   that table may have been altered by other code in the mean time.  So
%   we must make sure to save the catcodes in a table we control and
%   restore them at \cs{cctab_end:}.
%    \begin{macrocode}
\sys_if_engine_luatex:TF
  {
    \cs_new_protected:Npn \@@_begin_aux:
      {
        \@@_new:N \g_@@_next_cctab
        \tl_set:NV \l_@@_internal_a_tl \g_@@_next_cctab
        \cs_undefine:N \g_@@_next_cctab
      }
  }
  {
    \cs_new_protected:Npn \@@_begin_aux:
      {
        \int_gincr:N \g_@@_allocate_int
        \exp_args:Nc \@@_new:N
          { g_@@_ \int_use:N \g_@@_allocate_int _cctab }
        \exp_args:NNc \tl_set:Nn \l_@@_internal_a_tl
          { g_@@_ \int_use:N \g_@@_allocate_int _cctab }
      }
  }
%    \end{macrocode}
% \end{macro}
% \end{variable}
%
% \begin{macro}{\cctab_begin:N, \cctab_begin:c}
%   Check the \meta{cctab~var} exists, to avoid low-level errors.  Get
%   in \cs{l_@@_internal_a_tl} the number/name of a dynamic table, either
%   from \cs{g_@@_unused_seq} where we save tables that are not
%   currently in use, or from \cs{@@_begin_aux:} if none are available.
%   Then save the current catcodes into the table (pointed to by)
%   \cs{l_@@_internal_a_tl} and save that table number in a stack before
%   selecting the desired catcodes.
%    \begin{macrocode}
\cs_new_protected:Npn \cctab_begin:N #1
  {
    \@@_chk_if_valid:NT #1
      {
        \seq_gpop:NNF \g_@@_unused_seq \l_@@_internal_a_tl
          { \@@_begin_aux: }
        \@@_chk_group_begin:e
          { \@@_nesting_number:N \l_@@_internal_a_tl }
        \seq_gpush:NV \g_@@_stack_seq \l_@@_internal_a_tl
        \exp_args:NV \@@_gset:n \l_@@_internal_a_tl
        \@@_select:N #1
      }
  }
\cs_generate_variant:Nn \cctab_begin:N { c }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\cctab_end:}
%   Make sure a \cs{cctab_begin:N} was used some time earlier, get in
%   \cs{l_@@_internal_a_tl} the catcode table number/name in which the
%   prevailing catcodes were stored, then restore these catcodes.  The
%   dynamic table is now unused hence stored in \cs{g_@@_unused_seq} for
%   recycling by later \cs{cctab_begin:N}.
%    \begin{macrocode}
\cs_new_protected:Npn \cctab_end:
  {
    \seq_gpop:NNTF \g_@@_stack_seq \l_@@_internal_a_tl
      {
        \seq_gpush:NV \g_@@_unused_seq \l_@@_internal_a_tl
        \exp_args:Ne \@@_chk_group_end:n
          { \@@_nesting_number:N \l_@@_internal_a_tl }
        \@@_select:N \l_@@_internal_a_tl
      }
      { \msg_error:nn { cctab } { extra-end } }
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}
%   {\@@_chk_group_begin:n,\@@_chk_group_begin:e, \@@_chk_group_end:n}
%   Catcode tables are not allowed to be intermixed with groups, so here
%   we check that they are properly nested regarding \TeX{} groups.
%   \cs{@@_chk_group_begin:n} stores the current group level in a stack,
%   and locally defines a dummy control sequence
%   \cs[no-index]{@@_group_\meta{cctab-level}_chk:}.
%
%   \cs{@@_chk_group_end:n} pops the stack, and compares the returned
%   value with \cs{tex_currentgrouplevel:D}.  If they differ,
%   \cs{cctab_end:} is in a different grouping level than the matching
%   \cs{cctab_begin:N}.  If they are the same, both happened at the same
%   level, however a group might have ended and another started between
%   \cs{cctab_begin:N} and \cs{cctab_end:}:
% \begin{verbatim}
%   \group_begin:
%     \cctab_begin:N \c_document_cctab
%   \group_end:
%   \group_begin:
%     \cctab_end:
%   \group_end:
% \end{verbatim}
%   In this case checking \cs{tex_currentgrouplevel:D} is not enough, so
%   we locally define \cs[no-index]{@@_group_\meta{cctab-level}_chk:},
%   and then check if it exist in \cs{cctab_end:}.  If it doesn't,
%   we know there was a group end where it shouldn't.
%
%   The \meta{cctab-level} in the sentinel macro above cannot be
%   replaced by the more convenient \cs{tex_currentgrouplevel:D} because
%   with the latter we might be tricked.  Suppose:
% \begin{verbatim}
%   \group_begin:
%     \cctab_begin:N \c_code_cctab % A
%   \group_end:
%   \group_begin:
%     \cctab_begin:N \c_code_cctab % B
%     \cctab_end: % C
%     \cctab_end: % D
%   \group_end:
% \end{verbatim}
%   The line marked with |A| would start a |cctab| with a sentinel token
%   named \cs[no-index]{@@_group_1_chk:}, which would disappear at the
%   \cs{group_end:} that follows.  But |B| would create the same
%   sentinel token, since both are at the same group level.  Line |C|
%   would end the |cctab| from line |B| correctly, but so would line |D|
%   because line |B| created the same sentinel token.  Using
%   \meta{cctab-level} works correctly because it signals that certain
%   |cctab| level was activated somewhere, but if it doesn't exist when
%   the \cs{cctab_end:} is reached, we had a problem.
%
%   Unfortunately these tests only flag the wrong usage at the
%   \cs{cctab_end:}, which might be far from the \cs{cctab_begin:N}.
%   However it isn't possible to signal the wrong usage at the
%   \cs{group_end:} without using \cs{tex_aftergroup:D}, which is
%   unsafe in certain types of groups.
%
%   The three cases checked here just raise an error, and no recovery is
%   attempted:  usually interleaving groups and catcode tables will work
%   predictably.
%    \begin{macrocode}
\cs_new_protected:Npn \@@_chk_group_begin:n #1
  {
    \seq_gpush:Ne \g_@@_group_seq
      { \int_use:N \tex_currentgrouplevel:D }
    \cs_set_eq:cN { @@_group_ #1 _chk: } \prg_do_nothing:
  }
\cs_generate_variant:Nn \@@_chk_group_begin:n { e }
\cs_new_protected:Npn \@@_chk_group_end:n #1
  {
    \seq_gpop:NN \g_@@_group_seq \l_@@_internal_b_tl
    \bool_lazy_and:nnF
      {
        \int_compare_p:nNn
          { \tex_currentgrouplevel:D } = { \l_@@_internal_b_tl }
      }
      { \cs_if_exist_p:c { @@_group_ #1 _chk: } }
      {
        \msg_error:nne { cctab } { group-mismatch }
          {
            \int_sign:n
              { \tex_currentgrouplevel:D - \l_@@_internal_b_tl }
          }
      }
    \cs_undefine:c { @@_group_ #1 _chk: }
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\@@_nesting_number:N,\@@_nesting_number:w}
%   This macro returns the numeric index of the current catcode table.
%   In \LuaTeX{} this is just the argument, which is a count reference
%   to a \tn{catcodetable} register.  In other engines, the number is
%   extracted from the |cctab| variable.
%    \begin{macrocode}
\sys_if_engine_luatex:TF
  { \cs_new:Npn \@@_nesting_number:N #1 {#1} }
  {
    \cs_new:Npn \@@_nesting_number:N #1
      {
        \exp_after:wN \exp_after:wN \exp_after:wN \@@_nesting_number:w
          \exp_after:wN \token_to_str:N #1
      }
    \use:e
      {
        \cs_new:Npn \exp_not:N \@@_nesting_number:w
          #1 \tl_to_str:n { g_@@_ } #2 \tl_to_str:n { _cctab } {#2}
      }
  }
%    \end{macrocode}
% \end{macro}
%
% Finally, install some code at the end of the \TeX{} run to check that
% all \cs{cctab_begin:N} were ended by some \cs{cctab_end:}.
%    \begin{macrocode}
\cs_if_exist:NT \hook_gput_code:nnn
  {
    \hook_gput_code:nnn { enddocument/end } { cctab }
      {
        \seq_if_empty:NF \g_@@_stack_seq
          { \msg_error:nn { cctab } { missing-end } }
      }
  }
%    \end{macrocode}
%
%
% \begin{macro}{\cctab_item:Nn, \cctab_item:cn}
%   Evaluate the integer argument only once.  In most engines the
%   |cctab| variable only has $256$ entries so we only look up the
%   catcode for these entries, otherwise we use the current catcode.  In
%   particular, for out-of-range values we use whatever fall-back
%   \cs{char_value_catcode:n}.  In \LuaTeX{}, we use the
%   |tex.getcatcode| function.
%    \begin{macrocode}
\cs_new:Npn \cctab_item:Nn #1#2
  { \exp_args:Nf \@@_item:nN { \int_eval:n {#2} } #1 }
\sys_if_engine_luatex:TF
  {
    \cs_new:Npn \@@_item:nN #1#2
      { \lua_now:e { tex.print(-2, tex.getcatcode(\int_use:N #2, #1)) } }
  }
  {
    \cs_new:Npn \@@_item:nN #1#2
      {
        \int_compare:nNnTF {#1} < { 256 }
          { \intarray_item:Nn #2 { #1 + 1 } }
          { \char_value_catcode:n {#1} }
      }
  }
\cs_generate_variant:Nn \cctab_item:Nn { c }
%    \end{macrocode}
% \end{macro}
%
% \subsection{Category code table conditionals}
%
% \begin{macro}[pTF]{\cctab_if_exist:N,\cctab_if_exist:c}
%   Checks whether a \meta{cctab~var} is defined.
%    \begin{macrocode}
\prg_new_eq_conditional:NNn \cctab_if_exist:N \cs_if_exist:N
  { TF , T , F , p }
\prg_new_eq_conditional:NNn \cctab_if_exist:c \cs_if_exist:c
  { TF , T , F , p }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}[TF]{\@@_chk_if_valid:N}
% \begin{macro}{\@@_chk_if_valid_aux:NTF}
%   Checks whether the argument is defined and whether it is a valid
%   \meta{cctab~var}. In \LuaTeX{} the validity of the \meta{cctab~var}
%   is checked by the engine, which complains if the argument is not a
%   \cs{chardef}'ed constant. In other engines, check if the given
%   command is an intarray variable (the underlying definition is a copy
%   of the \texttt{cmr10} font).
%    \begin{macrocode}
\prg_new_protected_conditional:Npnn \@@_chk_if_valid:N #1
  { TF , T , F }
  {
    \cctab_if_exist:NTF #1
      {
        \@@_chk_if_valid_aux:NTF #1
          { \prg_return_true: }
          {
            \msg_error:nne { cctab } { invalid-cctab }
              { \token_to_str:N #1 }
            \prg_return_false:
          }
      }
      {
        \msg_error:nne { kernel } { command-not-defined }
          { \token_to_str:N #1 }
        \prg_return_false:
      }
  }
\sys_if_engine_luatex:TF
  {
    \cs_new_protected:Npn \@@_chk_if_valid_aux:NTF #1
      {
        \int_compare:nNnTF {#1-1} < { \e@alloc@ccodetable@count }
      }
    \cs_if_exist:NT \c_syst_catcodes_n
      {
        \cs_gset_protected:Npn \@@_chk_if_valid_aux:NTF #1
          {
            \int_compare:nTF { #1 <= \c_syst_catcodes_n }
          }
      }
  }
  {
    \cs_new_protected:Npn \@@_chk_if_valid_aux:NTF #1
      {
        \exp_args:Nf \str_if_in:nnTF
          { \cs_meaning:N #1 }
          { select~font~cmr10~at~ }
      }
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \subsection{Constant category code tables}
%
% \begin{macro}{\cctab_const:Nn,\cctab_const:cn}
%  Creates a new \meta{cctab~var} then sets it with the \IniTeX{} and
%  user-supplied codes. To avoid false \texttt{debug} errors, we write
%  out implementation of \cs{cctab_new:N} and \cs{cctab_gset:Nn}
%  instead of directly using them here. The initialization part in
%  \cs{cctab_new:N} in non-\LuaTeX{} is omitted as it's covered by
%  the \IniTeX{} settings.
%    \begin{macrocode}
\cs_new_protected:Npn \cctab_const:Nn #1#2
  {
    \__kernel_chk_if_free_cs:N #1
    \@@_new:N #1
    \group_begin:
      \cctab_select:N \c_initex_cctab
      #2 \scan_stop:
      \@@_gset:n {#1}
    \group_end:
  }
\cs_generate_variant:Nn \cctab_const:Nn { c }
%    \end{macrocode}
% \end{macro}
%
% \begin{variable}
%   {
%     \c_initex_cctab   ,
%     \c_other_cctab    ,
%     \c_str_cctab
%   }
%   Creating category code tables means thinking starting from \IniTeX{}.
%   For all-other and the standard \enquote{string} tables that's easy.
%    \begin{macrocode}
\cctab_new:N \c_initex_cctab
\cctab_const:Nn \c_other_cctab
  {
    \cctab_select:N \c_initex_cctab
    \int_set:Nn \tex_endlinechar:D     { -1 }
    \int_step_inline:nnn { 0 } { 127 }
      { \char_set_catcode_other:n {#1} }
  }
\cctab_const:Nn \c_str_cctab
  {
    \cctab_select:N \c_other_cctab
    \char_set_catcode_space:n { 32 }
  }
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\c_code_cctab, \c_document_cctab}
%  To pick up document-level category codes, we need to delay set up to the
%  end of the format, where that's possible. Also, as there are a \emph{lot}
%  of category codes to set, we avoid using the official interface and store the
%  document codes using internal code. Depending on whether we are in the hook
%  or not, the catcodes may be code or document, so we explicitly set up both
%  correctly.
%    \begin{macrocode}
\cs_if_exist:NTF \@expl@finalise@setup@@@@
  { \tl_gput_right:Nn \@expl@finalise@setup@@@@ }
  { \use:n }
  {
    \@@_new:N \c_code_cctab
    \group_begin:
      \int_set:Nn \tex_endlinechar:D { 32 }
      \char_set_catcode_invalid:n { 0 }
      \sys_if_engine_opentype:TF
        { \int_step_function:nN { 31 } \char_set_catcode_invalid:n }
        { \int_step_function:nN { 31 } \char_set_catcode_active:n }
      \int_step_function:nnN { 33 } { 64 } \char_set_catcode_other:n
      \int_step_function:nnN { 65 } { 90 } \char_set_catcode_letter:n
      \int_step_function:nnN { 91 } { 96 } \char_set_catcode_other:n
      \int_step_function:nnN { 97 } { 122 } \char_set_catcode_letter:n
      \char_set_catcode_ignore:n           { 9 }   % tab
      \char_set_catcode_other:n            { 10 }  % lf
      \char_set_catcode_active:n           { 12 }  % ff
      \char_set_catcode_end_line:n         { 13 }  % cr
      \char_set_catcode_ignore:n           { 32 }  % space
      \char_set_catcode_parameter:n        { 35 }  % hash
      \char_set_catcode_math_toggle:n      { 36 }  % dollar
      \char_set_catcode_comment:n          { 37 }  % percent
      \char_set_catcode_alignment:n        { 38 }  % ampersand
      \char_set_catcode_letter:n           { 58 }  % colon
      \char_set_catcode_escape:n           { 92 }  % backslash
      \char_set_catcode_math_superscript:n { 94 }  % circumflex
      \char_set_catcode_letter:n           { 95 }  % underscore
      \char_set_catcode_group_begin:n      { 123 } % left brace
      \char_set_catcode_other:n            { 124 } % pipe
      \char_set_catcode_group_end:n        { 125 } % right brace
      \char_set_catcode_space:n            { 126 } % tilde
      \char_set_catcode_invalid:n          { 127 } % ^^?
      \sys_if_engine_opentype:F
        { \int_step_function:nnN { 128 } { 255 } \char_set_catcode_active:n }
      \@@_gset:n { \c_code_cctab }
    \group_end:
    \cctab_const:Nn \c_document_cctab
      {
        \cctab_select:N \c_code_cctab
        \int_set:Nn \tex_endlinechar:D { 13 }
        \char_set_catcode_space:n          { 9 }
        \char_set_catcode_space:n          { 32 }
        \char_set_catcode_other:n          { 58 }
        \char_set_catcode_math_subscript:n { 95 }
        \char_set_catcode_active:n         { 126 }
      }
  }
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\g_tmpa_cctab, \g_tmpb_cctab}
%    \begin{macrocode}
\cctab_new:N \g_tmpa_cctab
\cctab_new:N \g_tmpb_cctab
%    \end{macrocode}
% \end{variable}
%
% \subsection{Messages}
%
%    \begin{macrocode}
\msg_new:nnnn { cctab } { stack-full }
  { The~category~code~table~stack~is~exhausted. }
  {
    LaTeX~has~been~asked~to~switch~to~a~new~category~code~table,~
    but~there~is~no~more~space~to~do~this!
  }
\msg_new:nnnn { cctab } { extra-end }
  { Extra~\iow_char:N\\cctab_end:~ignored~\msg_line_context:. }
  {
    LaTeX~came~across~a~\iow_char:N\\cctab_end:~without~a~matching~
    \iow_char:N\\cctab_begin:N.~This~command~will~be~ignored.
  }
\msg_new:nnnn { cctab } { missing-end }
  { Missing~\iow_char:N\\cctab_end:~before~end~of~TeX~run. }
  {
    LaTeX~came~across~more~\iow_char:N\\cctab_begin:N~than~
    \iow_char:N\\cctab_end:.
  }
\msg_new:nnnn { cctab } { invalid-cctab }
  { Invalid~\iow_char:N\\catcode~table. }
  {
    You~can~only~switch~to~a~\iow_char:N\\catcode~table~that~is~
    initialized~using~\iow_char:N\\cctab_new:N~or~
    \iow_char:N\\cctab_const:Nn.
  }
\msg_new:nnnn { cctab } { group-mismatch }
  {
    \iow_char:N\\cctab_end:~occurred~in~a~
    \int_case:nn {#1}
      {
        { 0 } { different~group }
        { 1 } { higher~group~level }
        { -1 } { lower~group~level }
      } ~than~
    the~matching~\iow_char:N\\cctab_begin:N.
  }
  {
    Catcode~tables~and~groups~must~be~properly~nested,~but~
    you~tried~to~interleave~them.~LaTeX~will~try~to~proceed,~
    but~results~may~be~unexpected.
  }
\prop_gput:Nnn \g_msg_module_name_prop { cctab } { LaTeX }
\prop_gput:Nnn \g_msg_module_type_prop { cctab } { }
%    \end{macrocode}
%
%    \begin{macrocode}
%</package>
%    \end{macrocode}
%
% \end{implementation}
%
%\PrintIndex