Source Browser ?

1/*
2    See license.txt in the root of this project.
3*/
4
5# include "luametatex.h"
6
7/*tex
8
9    The nested structure provided by |\char'173| \unknown\ |\char'175| groups in \TEX\ means that
10    |eqtb| entries valid in outer groups should be saved and restored later if they are overridden
11    inside the braces. When a new |eqtb| value is being assigned, the program therefore checks to
12    see if the previous entry belongs to an outer level. In such a case, the old value is placed on
13    the |save_stack| just before the new value enters |eqtb|. At the end of a grouping level, i.e.,
14    when the right brace is sensed, the |save_stack| is used to restore the outer values, and the
15    inner ones are destroyed.
16
17    Entries on the |save_stack| are of type |save_record|. The top item on this stack is
18    |save_stack[p]|, where |p=save_ptr-1|; it contains three fields called |save_type|, |save_level|,
19    and |save_value|, and it is interpreted in one of four ways:
20
21    \startitemize[n]
22
23    \startitem
24        If |save_type(p) = restore_old_value|, then |save_value(p)| is a location in |eqtb| whose
25        current value should be destroyed at the end of the current group and replaced by
26        |save_word(p-1)| (|save_type(p-1) == saved_eqtb|). Furthermore if |save_value(p) >= int_base|,
27        then |save_level(p)| should replace the corresponding entry in |xeq_level| (if |save_value(p)
28        < int_base|, then the level is part of |save_word(p-1)|).
29    \stopitem
30
31    \startitem
32        If |save_type(p) = restore_zero|, then |save_value(p)| is a location in |eqtb| whose current
33        value should be destroyed at the end of the current group, when it should be replaced by the
34        current value of |eqtb[undefined_control_sequence]|.
35    \stopitem
36
37    \startitem
38        If |save_type(p) = insert_token|, then |save_value(p)| is a token that should be inserted
39        into \TeX's input when the current group ends.
40    \stopitem
41
42    \startitem
43        If |save_type(p) = level_boundary|, then |save_level(p)| is a code explaining what kind of
44        group we were previously in, and |save_value(p)| points to the level boundary word at the
45        bottom of the entries for that group. Furthermore, |save_value(p-1)| contains the source
46        line number at which the current level of grouping was entered, this field has itself a
47        type: |save_type(p-1) == saved_line|.
48    \stopitem
49
50    \stopitemize
51
52    Besides this \quote {official} use, various subroutines push temporary variables on the save
53    stack when it is handy to do so. These all have an explicit |save_type|, and they are:
54
55    \starttabulate
56        \NC |saved_adjust|     \NC signifies an adjustment is beging scanned \NC\NR
57        \NC |saved_insert|     \NC an insertion is being scanned \NC\NR
58        \NC |saved_disc|       \NC the |\discretionary| sublist we are working on right now \NC\NR
59        \NC |saved_boxtype|    \NC whether a |\localbox| is |\left| or |\right| \NC\NR
60        \NC |saved_textdir|    \NC a text direction to be restored \NC\NR
61        \NC |saved_eqno|       \NC diffentiates between |\eqno| and |\leqno| \NC\NR
62        \NC |saved_choices|    \NC the |\mathchoices| sublist we are working on right now \NC\NR
63        \NC |saved_above|      \NC used for the \LUAMETATEX\ above variants \NC\NR
64        \NC |saved_math|       \NC and interrupted math list \NC\NR
65        \NC |saved_boxcontext| \NC the box context value \NC\NR
66        \NC |saved_boxspec|    \NC the box |to| or |spread| specification \NC\NR
67        \NC |saved_boxdir|     \NC the box |dir| specification \NC\NR
68        \NC |saved_boxattr|    \NC the box |attr| specification \NC\NR
69        \NC |saved_boxpack|    \NC the box |pack| specification \NC\NR
70        \NC |...|              \NC some more in \LUATEX\ and \LUAMETATEX \NC\NR
71    \stoptabulate
72
73    The global variable |cur_group| keeps track of what sort of group we are currently in. Another
74    global variable, |cur_boundary|, points to the topmost |level_boundary| word. And |cur_level|
75    is the current depth of nesting. The routines are designed to preserve the condition that no
76    entry in the |save_stack| or in |eqtb| ever has a level greater than |cur_level|.
77
78*/
79
80save_state_info lmt_save_state = {
81    .save_stack       = NULL,
82    .save_stack_data  = {
83        .minimum   = min_save_size,
84        .maximum   = max_save_size,
85        .size      = siz_save_size,
86        .step      = stp_save_size,
87        .allocated = 0,
88        .itemsize  = sizeof(save_record),
89        .top       = 0,
90        .ptr       = 0,
91        .initial   = memory_data_unset,
92        .offset    = 0,
93        .extra     = 0, 
94    },
95    .current_level    = 0,
96    .current_group    = 0,
97    .current_boundary = 0,
98 // .padding          = 0,
99};
100
101/*tex
102
103    The comments below are (of course) coming from \LUATEX's ancestor and are still valid! However,
104    in \LUATEX\ we use \UTF\ instead of \ASCII, have attributes, have more primites, etc. But the
105    principles remain the same. We are not 100\% compatible in output and will never be.
106
107*/
108
109static void tex_aux_show_eqtb(halfword n);
110
111static void tex_aux_diagnostic_trace(halfword p, const char *s)
112{
113    tex_begin_diagnostic();
114    tex_print_format("{%s ", s);
115    tex_aux_show_eqtb(p);
116    tex_print_char('}');
117    tex_end_diagnostic();
118}
119
120/*tex
121
122    Now that we have studied the data structures for \TEX's semantic routines (in other modules),
123    we ought to consider the data structures used by its syntactic routines. In other words, our
124    next concern will be the tables that \TEX\ looks at when it is scanning what the user has
125    written.
126
127    The biggest and most important such table is called |eqtb|. It holds the current \quote
128    {equivalents} of things; i.e., it explains what things mean or what their current values are,
129    for all quantities that are subject to the nesting structure provided by \TEX's grouping
130    mechanism. There are six parts to |eqtb|:
131
132    \startitemize[n]
133
134    \startitem
135        |eqtb[null_cs]| holds the current equivalent of the zero-length control sequence.
136    \stopitem
137
138    \startitem
139        |eqtb[hash_base..(glue_base-1)]| holds the current equivalents of single- and multiletter
140        control sequences.
141    \stopitem
142
143    \startitem
144        |eqtb[glue_base..(local_base-1)]| holds the current equivalents of glue parameters like
145        the current baselineskip.
146    \stopitem
147
148    \startitem
149        |eqtb[local_base..(int_base-1)]| holds the current equivalents of local halfword
150        quantities like the current box registers, the current \quote {catcodes}, the current font,
151        and a pointer to the current paragraph shape.
152    \stopitem
153
154    \startitem
155        |eqtb[int_base .. (dimen_base-1)]| holds the current equivalents of fullword integer
156        parameters like the current hyphenation penalty.
157    \stopitem
158
159    \startitem
160        |eqtb[dimen_base .. eqtb_size]| holds the current equivalents of fullword dimension
161        parameters like the current hsize or amount of hanging indentation.
162    \stopitem
163
164    \stopitemize
165
166    Note that, for example, the current amount of baselineskip glue is determined by the setting of
167    a particular location in region~3 of |eqtb|, while the current meaning of the control sequence
168    |\baselineskip| (which might have been changed by |\def| or |\let|) appears in region~2.
169
170    The last two regions of |eqtb| have fullword values instead of the three fields |eq_level|,
171    |eq_type|, and |equiv|. An |eq_type| is unnecessary, but \TEX\ needs to store the |eq_level|
172    information in another array called |xeq_level|.
173
174    The last statement is no longer true. We have plenty of room in the 64 bit memory words now so
175    we no longer need the parallel |x| array. For the moment we keep the commented code.
176
177*/
178
179// equivalents_state_info lmt_equivalents_state = {
180// };
181
182void tex_initialize_levels(void)
183{
184    cur_level = level_one;
185    cur_group = bottom_level_group;
186    cur_boundary = 0;
187    lmt_scanner_state.last_cs_name = null_cs;
188}
189
190void tex_initialize_undefined_cs(void)
191{
192    set_eq_type(undefined_control_sequence, undefined_cs_cmd);
193    set_eq_flag(undefined_control_sequence, 0);
194    set_eq_value(undefined_control_sequence, null);
195    set_eq_level(undefined_control_sequence, level_zero);
196}
197
198/*tex
199    Dump regions 1 to 4 of |eqtb|, the table of equivalents: glue muglue toks boxes. The table of
200    equivalents usually contains repeated information, so we dump it in compressed form: The
201    sequence of $n + 2$ values $(n, x_1, \ldots, x_n, m)$ in the format file represents $n+m$
202    consecutive entries of |eqtb|, with |m| extra copies of $x_n$, namely $(x_1, \ldots, x_n, x_n,
203    \ldots, x_n)$.
204
205    Using a different threshold saves some. In fact we have either undefined or something set
206    in the main area. Packing the sizes in one byte also saves some.
207*/
208
209# define pack_them 1
210
211static void tex_dump_equivalents_mem_hash(dumpstream f)
212{
213    int index = null_cs;
214    int n_of_undefined = 0;
215    int n_of_defined = 0;
216    do {
217        unsigned char undefined = 0;
218        unsigned char defined = 0;
219# if (pack_them)
220        while (index < first_register_base && undefined < 0xE) {
221# else
222        while (index < first_register_base && undefined < 0xFE) {
223# endif
224             if (equal_eqtb_entries(index, undefined_control_sequence)) {
225                ++undefined;
226                ++index;
227            } else {
228                break;
229            }
230        }
231# if (pack_them)
232        while (index < first_register_base && defined < 0xF) {
233# else
234        while (index < first_register_base && defined < 0xFE) {
235# endif
236             if (! equal_eqtb_entries(index, undefined_control_sequence)) {
237                ++defined;
238                ++index;
239            } else {
240                break;
241            }
242        }
243# if (pack_them)
244        dump_via_uchar(f, (undefined << 4) + defined);
245# else
246        dump_uchar(f, undefined);
247        dump_uchar(f, defined);
248# endif
249        if (defined) {
250            dump_things(f, lmt_hash_state.eqtb[index-defined], defined);
251        }
252        n_of_defined += defined;
253        n_of_undefined += undefined;
254    } while (index < first_register_base);
255# if (pack_them)
256    dump_via_uchar(f, 0xFF);
257# else
258    dump_via_uchar(f, 0xFF);
259    dump_via_uchar(f, 0xFF);
260# endif
261    dump_int(f, n_of_undefined);
262    dump_int(f, n_of_defined);
263}
264
265static void tex_dump_equivalents_mem_registers(dumpstream f) /* the old packer */
266{
267    int index = first_register_base;
268    int n_of_different = 0;
269    int n_of_equivalent = 0;
270    do {
271        unsigned char different = 1;
272        unsigned short equivalent = 0;
273        int j = index;
274        while (j < (eqtb_size - 1) && different < 0xFF) {
275            if (equal_eqtb_entries(j, j + 1)) {
276                ++equivalent;
277                goto FOUND1;
278            } else {
279                ++different;
280            }
281            ++j;
282        }
283        goto DONE1;
284      FOUND1:
285        j++;
286        while (j < (eqtb_size - 1) && equivalent < 0xFFFF) {
287            if (equal_eqtb_entries(j, j + 1)) {
288                ++equivalent;
289            } else {
290                goto DONE1;
291            }
292            ++j;
293        }
294      DONE1:
295        dump_uchar(f, different);
296        dump_ushort(f, equivalent);
297        if (different) {
298            dump_things(f, lmt_hash_state.eqtb[index], different);
299        }
300        index = index + different + equivalent;
301        n_of_different += different;
302        n_of_equivalent += equivalent;
303    } while (index <= eqtb_size);
304    dump_int(f, n_of_different);
305    dump_int(f, n_of_equivalent);
306}
307
308static void tex_dump_equivalents_mem_extra(dumpstream f)
309{
310    dump_int(f, lmt_hash_state.hash_data.ptr);
311    if (lmt_hash_state.hash_data.ptr > 0) {
312        /*tex Some 15% of the total saved equivalent section (in \CONTEXT\ about 85K). */
313        dump_things(f, lmt_hash_state.eqtb[eqtb_size + 1], lmt_hash_state.hash_data.ptr);
314    }
315}
316
317static void tex_dump_equivalents_mem_specials(dumpstream f)
318{
319    dump_int(f, lmt_token_state.par_loc);
320 /* dump_int(f, lmt_token_state.line_par_loc); */ /*tex See note in textoken.c|. */
321    dump_int(f, lmt_token_state.empty);
322}
323
324void tex_dump_equivalents_mem(dumpstream f)
325{
326    tex_dump_equivalents_mem_hash(f);
327    tex_dump_equivalents_mem_registers(f);
328    tex_dump_equivalents_mem_extra(f);
329    tex_dump_equivalents_mem_specials(f);
330}
331
332static void tex_undump_equivalents_mem_hash(dumpstream f)
333{
334    int index = null_cs;
335    memoryword undef = lmt_hash_state.eqtb[undefined_control_sequence];
336    int n_of_undefined = 0;
337    int n_of_defined = 0;
338    while (1) {
339        unsigned char undefined = 0;
340        unsigned char defined = 0;
341# if (pack_them)
342        undump_uchar(f, defined);
343        if (defined == 0xFF) {
344# else
345        undump_uchar(f, undefined);
346        undump_uchar(f, defined);
347        if (undefined == 0xFF && defined == 0xFF) {
348# endif
349            break;
350        } else {
351# if (pack_them)
352            undefined = (defined >> 4) & 0xF;
353            defined = defined & 0xF;
354# endif
355            if (undefined) {
356                for (int i = 1; i <= undefined; i++) {
357                    lmt_hash_state.eqtb[index++] = undef;
358                }
359            }
360            if (defined) {
361                undump_things(f, lmt_hash_state.eqtb[index], defined);
362                index += defined;
363            }
364            n_of_defined += defined;
365            n_of_undefined += undefined;
366        }
367    }
368    {
369        int u, d;
370        undump_int(f, u);
371        undump_int(f, d);
372        if (u != n_of_undefined || d != n_of_defined)  {
373            tex_fatal_undump_error("eqtb count mismatch");
374        }
375    }
376}
377
378static void tex_undump_equivalents_mem_registers(dumpstream f) /* the old unpacker */
379{
380    int index = first_register_base;
381    int n_of_different = 0;
382    int n_of_equivalent = 0;
383    do {
384        unsigned char different;
385        unsigned short equivalent;
386        undump_uchar(f, different);
387        undump_ushort(f, equivalent);
388        if (different) {
389            undump_things(f, lmt_hash_state.eqtb[index], different);
390        }
391        if (equivalent > 0) {
392            int last = index + different - 1;
393            for (int i = 1; i <= equivalent; i++) {
394                lmt_hash_state.eqtb[last + i] = lmt_hash_state.eqtb[last];
395            }
396        }
397        index = index + different + equivalent;
398        n_of_different += different;
399        n_of_equivalent += equivalent;
400    } while (index <= eqtb_size);
401    {
402        int d, e;
403        undump_int(f, d);
404        undump_int(f, e);
405        if (d != n_of_different || e != n_of_equivalent)  {
406            tex_fatal_undump_error("eqtb register mismatch");
407        }
408    }
409}
410
411static void tex_undump_equivalents_mem_extra(dumpstream f)
412{
413    undump_int(f, lmt_hash_state.hash_data.ptr);
414    if (lmt_hash_state.hash_data.ptr > 0) {
415        /* we get a warning on possible overrun here */
416        undump_things(f, lmt_hash_state.eqtb[eqtb_size + 1], lmt_hash_state.hash_data.ptr);
417    }
418}
419
420static void tex_undump_equivalents_mem_specials(dumpstream f)
421{
422    undump_int(f, lmt_token_state.par_loc);
423    if (lmt_token_state.par_loc >= hash_base && lmt_token_state.par_loc <= lmt_hash_state.hash_data.top) {
424        lmt_token_state.par_token = cs_token_flag + lmt_token_state.par_loc;
425    } else {
426        tex_fatal_undump_error("parloc");
427    }
428 /* undump_int(f, lmt_token_state.line_par_loc); */
429 /* if (lmt_token_state.line_par_loc >= hash_base && lmt_token_state.line_par_loc <= lmt_hash_state.hash_data.top) { */
430 /*     lmt_token_state.line_par_token = cs_token_flag + lmt_token_state.line_par_loc; */
431 /* } else { */
432 /*     tex_fatal_undump_error("lineparloc"); */
433 /* } */
434    undump_int(f, lmt_token_state.empty);
435}
436
437void tex_undump_equivalents_mem(dumpstream f)
438{
439    tex_undump_equivalents_mem_hash(f);
440    tex_undump_equivalents_mem_registers(f);
441    tex_undump_equivalents_mem_extra(f);
442    tex_undump_equivalents_mem_specials(f);
443}
444
445/*tex
446
447    At this time it might be a good idea for the reader to review the introduction to |eqtb| that
448    was given above just before the long lists of parameter names. Recall that the \quote {outer
449    level} of the program is |level_one|, since undefined control sequences are assumed to be \quote
450    {defined} at |level_zero|.
451
452    The following function is used to test if there is room for up to eight more entries on
453    |save_stack|. By making a conservative test like this, we can get by with testing for overflow
454    in only a few places.
455
456    We now let the save stack dynamically grow. In practice the stack is small but when a large one
457    is needed, the overhead is probably neglectable compared to what the macro need.
458
459*/
460
461# define reserved_save_stack_slots 32 /* We need quite some for boxes so we bump it. */
462
463void tex_initialize_save_stack(void)
464{
465    int size = lmt_save_state.save_stack_data.minimum;
466    lmt_save_state.save_stack = aux_allocate_clear_array(sizeof(save_record), lmt_save_state.save_stack_data.step, reserved_save_stack_slots);
467    if (lmt_save_state.save_stack) {
468        lmt_save_state.save_stack_data.allocated = lmt_save_state.save_stack_data.step;
469    } else {
470        tex_overflow_error("save", size);
471    }
472}
473
474static int tex_room_on_save_stack(void)
475{
476    int top = lmt_save_state.save_stack_data.ptr;
477    if (top > lmt_save_state.save_stack_data.top) {
478        lmt_save_state.save_stack_data.top = top;
479        if (top > lmt_save_state.save_stack_data.allocated) {
480            save_record *tmp = NULL;
481            top = lmt_save_state.save_stack_data.allocated + lmt_save_state.save_stack_data.step;
482            if (top > lmt_save_state.save_stack_data.size) {
483                top = lmt_save_state.save_stack_data.size;
484            }
485            if (top > lmt_save_state.save_stack_data.allocated) {
486                top = lmt_save_state.save_stack_data.allocated + lmt_save_state.save_stack_data.step;
487                tmp = aux_reallocate_array(lmt_save_state.save_stack, sizeof(save_record), top, reserved_save_stack_slots);
488                lmt_save_state.save_stack = tmp;
489            }
490            lmt_run_memory_callback("save", tmp ? 1 : 0);
491            if (! tmp) {
492                tex_overflow_error("save", top);
493                return 0;
494            }
495         // memset((void *) (lmt_save_state.save_stack + lmt_save_state.save_stack_data.allocated + 1), 0, ((size_t) lmt_save_state.save_stack_data.step + reserved_save_stack_slots) * sizeof(save_record));
496            lmt_save_state.save_stack_data.allocated = top;
497        }
498    }
499    return 1;
500}
501
502void tex_save_halfword_on_stack(quarterword t, halfword v) /* todo: also value_2 and value_3 */
503{
504    if (tex_room_on_save_stack()) {
505        saved_type(0) = t;
506        saved_record(0) = 0;
507        saved_value_1(0) = v;
508        ++lmt_save_state.save_stack_data.ptr;
509    }
510}
511
512/*tex
513
514    Procedure |new_save_level| is called when a group begins. The argument is a group identification
515    code like |hbox_group|. After calling this routine, it is safe to put six more entries on
516    |save_stack|.
517
518    In some cases integer-valued items are placed onto the |save_stack| just below a |level_boundary|
519    word, because this is a convenient place to keep information that is supposed to \quote {pop up}
520    just when the group has finished. For example, when |\hbox to 100pt| is being treated, the 100pt
521    dimension is stored on |save_stack| just before |new_save_level| is called.
522
523    The |group_trace| procedure is called when a new level of grouping begins (|e=false|) or ends
524    (|e = true|) with |saved_value (-1)| containing the line number.
525
526*/
527
528static void tex_aux_group_trace(int g)
529{
530    tex_begin_diagnostic();
531    tex_print_format(g ? "{leaving %G}" : "{entering %G}", g);
532    tex_end_diagnostic();
533}
534
535/*tex
536
537    A group entered (or a conditional started) in one file may end in a different file. Such
538    slight anomalies, although perfectly legitimate, may cause errors that are difficult to
539    locate. In order to be able to give a warning message when such anomalies occur, \ETEX\
540    uses the |grp_stack| and |if_stack| arrays to record the initial |cur_boundary| and
541    |condition_ptr| values for each input file.
542
543    When a group ends that was apparently entered in a different input file, the |group_warning|
544    procedure is invoked in order to update the |grp_stack|. If moreover |\tracingnesting| is
545    positive we want to give a warning message. The situation is, however, somewhat complicated
546    by two facts:
547
548    \startitemize[n,packed]
549        \startitem
550            There may be |grp_stack| elements without a corresponding |\input| file or
551            |\scantokens| pseudo file (e.g., error insertions from the terminal); and
552        \stopitem
553        \startitem
554            the relevant information is recorded in the |name_field| of the |input_stack| only
555            loosely synchronized with the |in_open| variable indexing |grp_stack|.
556        \stopitem
557    \stopitemize
558
559    Per end 2023 the decision was made to use the two halfwords in the memory word that is used for
560    saving eq values for other purposes too. It meant for instance that instead if every new group
561    needing 3 stack entries, it now needs just one. Actually there are not that meny mechanism that
562    need the stack. For instance alignments have their own stack, math needs it for multi argument
563    mechanism and the most demanding one is box handling. In addition to the two extra |value_2| and
564    |value_3| fields we also use |options| and |extra| as alias for |level| (these are quarterwords).
565
566*/
567
568static void tex_aux_group_warning(void)
569{
570    /*tex do we need a warning? */
571    bool warning = false;
572    /*tex index into |grp_stack| */
573    int index = lmt_input_state.in_stack_data.ptr;
574    lmt_input_state.base_ptr = lmt_input_state.input_stack_data.ptr;
575    /*tex store current state */
576    lmt_input_state.input_stack[lmt_input_state.base_ptr] = lmt_input_state.cur_input;
577    while ((lmt_input_state.in_stack[index].group == cur_boundary) && (index > 0)) {
578        /*tex
579
580            Set variable |w| to indicate if this case should be reported. This code scans the input
581            stack in order to determine the type of the current input file.
582
583        */
584        if (tracing_nesting_par > 0) {
585            while ((lmt_input_state.input_stack[lmt_input_state.base_ptr].state == token_list_state) || (lmt_input_state.input_stack[lmt_input_state.base_ptr].index > index)) {
586                --lmt_input_state.base_ptr;
587            }
588            if (lmt_input_state.input_stack[lmt_input_state.base_ptr].name > 17) {
589                /*tex |>  max_file_input_code| .. hm */
590                warning = true;
591            }
592        }
593        lmt_input_state.in_stack[index].group = save_value(lmt_save_state.save_stack_data.ptr);
594        --index;
595    }
596    if (warning) {
597        tex_begin_diagnostic();
598        tex_print_format("[warning: end of %G of a different file]", 1);
599        tex_end_diagnostic();
600        if (tracing_nesting_par > 1) {
601            tex_show_context();
602        }
603        if (lmt_error_state.history == spotless) {
604            lmt_error_state.history = warning_issued;
605        }
606    }
607}
608
609/*tex
610    We store the line number and attribute state in the memory word part for the save entry and
611    then we need only one slot instead of three. Each slot is 16 bytes so we're also a bit nicer to
612    caching memory (if it happens at all). Because in practice we don't group that much (400.000
613    times in the luametatex manual and a similar amount in the primitives manual) there is no real
614    significant (positive) impact on performance. It's all about abstraction.
615*/
616
617typedef enum saved_group_entries {
618    saved_group_group_entry           = 0,
619    saved_group_boundary_entry        = 0,
620    saved_group_attribute_state_entry = 0,
621    saved_group_input_line_entry      = 0,
622    saved_group_n_records             = 1,
623} saved_group_entries;
624
625# define saved_group_group           saved_group(saved_group_group_entry)
626# define saved_group_boundary        saved_value_1(saved_group_boundary_entry)
627# define saved_group_attribute_state saved_value_2(saved_group_attribute_state_entry)
628# define saved_group_input_line      saved_value_3(saved_group_input_line_entry)
629
630static inline void saved_group_initialize(void)
631{
632    saved_type(0) = level_boundary_save_type;
633    /* here level is really level */
634}
635
636void tex_new_save_level(quarterword group)
637{
638    /*tex We begin a new level of grouping. This will cost us three entries. */
639    if (tex_room_on_save_stack()) {
640        add_attribute_reference(current_attribute_state);
641        saved_group_initialize();
642        saved_group_group = cur_group;
643        saved_group_boundary = cur_boundary;
644        saved_group_attribute_state = current_attribute_state;
645        saved_group_input_line = lmt_input_state.input_line;
646        if (cur_level == max_quarterword) {
647            /*tex We quit if |(cur_level+1)| is too big to be stored in |eqtb|. */
648            tex_overflow_error("grouping levels", max_quarterword - min_quarterword);
649        }
650        cur_boundary = lmt_save_state.save_stack_data.ptr;
651        cur_group = group;
652        if (tracing_groups_par > 0) {
653            tex_aux_group_trace(0);
654        }
655        ++cur_level;
656        lmt_save_state.save_stack_data.ptr += saved_group_n_records;
657        if (end_of_group_par) {
658            update_tex_end_of_group(null);
659        }
660    }
661}
662
663int tex_saved_line_at_level(void)
664{
665    return lmt_save_state.save_stack_data.ptr > 0 ? (saved_group_input_line > 0 ? saved_group_input_line : 0) : 0;
666}
667
668/*tex
669
670    The |\showgroups| command displays all currently active grouping levels. The modifications of
671    \TEX\ required for the display produced by the |show_save_groups| procedure were first discussed
672    by Donald~E. Knuth in {\em TUGboat} {\bf 11}, 165--170 and 499--511, 1990.
673
674    In order to understand a group type we also have to know its mode. Since unrestricted horizontal
675    modes are not associated with grouping, they are skipped when traversing the semantic nest.
676
677    I have to admit that I never used (or needed) this so we might as well drop it from \LUAMETATEX\
678    and given the already extensive tracing we can decide to drop it.
679
680    The output is not (entirely) downward compatible which is no big deal because we output some more
681    details anyway.
682*/
683
684void tex_show_save_groups(void)
685{
686    int pointer = lmt_nest_state.nest_data.ptr;
687    int saved_pointer = lmt_save_state.save_stack_data.ptr;
688    quarterword saved_level = cur_level;
689    quarterword saved_group = cur_group;
690    halfword saved_tracing = tracing_levels_par;
691    int alignmentstate = 1; /* to keep track of alignments */
692    const char *package = NULL;
693    lmt_save_state.save_stack_data.ptr = cur_boundary;
694    --cur_level;
695    tracing_levels_par |= tracing_levels_group;
696    while (1) {
697        int mode;
698        tex_print_levels();
699        tex_print_group(1);
700        if (cur_group == bottom_level_group) {
701            goto DONE;
702        }
703        do {
704            mode = lmt_nest_state.nest[pointer].mode;
705            if (pointer > 0) {
706                --pointer;
707            } else {
708                mode = vmode;
709            }
710        } while (mode == hmode);
711        tex_print_str(": ");
712        switch (cur_group) {
713            case simple_group:
714                ++pointer;
715                goto FOUND2;
716            case hbox_group:
717            case adjusted_hbox_group:
718                package = "hbox";
719                break;
720            case vbox_group:
721                package = "vbox";
722                break;
723            case vtop_group:
724                package = "vtop";
725                break;
726            case dbox_group:
727                package = "dbox";
728                break;
729            case align_group:
730                if (alignmentstate == 0) {
731                    package = (mode == internal_vmode) ? "halign" : "valign";
732                    alignmentstate = 1;
733                    goto FOUND1;
734                } else {
735                    if (alignmentstate == 1) {
736                        tex_print_str("align entry");
737                    } else {
738                        tex_print_str_esc("cr");
739                    }
740                    if (pointer >= alignmentstate) {
741                        pointer -= alignmentstate;
742                    }
743                    alignmentstate = 0;
744                    goto FOUND2;
745                }
746            case no_align_group:
747                ++pointer;
748                alignmentstate = -1;
749                tex_print_str_esc("noalign");
750                goto FOUND2;
751            case output_group:
752                tex_print_str_esc("output");
753                goto FOUND2;
754         // maybe:
755         //
756         // case math_group:
757         // case math_component_group:
758         // case math_stack_group:
759         //      tex_print_str_esc(lmt_interface.group_code_values[cur_group].name);
760         //      goto FOUND2;
761            case math_group:
762                tex_print_str_esc("mathsubformula");
763                goto FOUND2;
764            case math_component_group:
765                tex_print_str_esc("mathcomponent");
766                goto FOUND2;
767            case math_stack_group:
768                tex_print_str_esc("mathstack");
769                goto FOUND2;
770            case discretionary_group:
771                tex_show_discretionary_group();
772                goto FOUND2;
773            case math_fraction_group:
774                tex_show_math_fraction_group();
775                goto FOUND2;
776            case math_radical_group:
777                tex_show_math_radical_group();
778                goto FOUND2;
779            case math_operator_group:
780                tex_show_math_operator_group();
781                goto FOUND2;
782            case math_choice_group:
783                tex_show_math_choice_group();
784                goto FOUND2;
785            case insert_group:
786                tex_show_insert_group();
787                goto FOUND2;
788            case vadjust_group:
789                tex_show_adjust_group();
790                goto FOUND2;
791            case vcenter_group:
792                package = "vcenter";
793                goto FOUND1;
794            case also_simple_group:
795            case semi_simple_group:
796                ++pointer;
797                tex_print_str_esc("begingroup");
798                goto FOUND2;
799         // case math_simple_group:
800         //     ++pointer;
801         //     tex_print_str_esc("beginmathgroup");
802         //     goto FOUND2;
803            case math_inline_group:
804                tex_print_char('$');
805            case math_display_group:
806                tex_print_char('$');
807                goto FOUND2;
808            case math_number_group:
809                tex_show_math_number_group();
810                goto FOUND2;
811            case math_fence_group:
812                /* kind of ugly ... maybe also save that one */ /* todo: operator */
813                tex_print_str_esc((node_subtype(lmt_nest_state.nest[pointer + 1].delimiter) == left_fence_side) ? "left" : "middle");
814                goto FOUND2;
815            default:
816                tex_confusion("show groups");
817                break;
818        }
819        /*tex
820            Show the box context. In traditional \TEX\ the shift is encoded in the context which is
821            why it had such a large offset for the other context value. That somewhat dirty trick
822            was has stepwise been removed.
823        */
824        switch (tex_get_packaging_context()) {
825            case direct_box_flag:
826                {
827                    scaled shift = tex_get_packaging_shift();
828                    if (shift != null_flag) {
829                        /*tex We passed the safeguard. */
830                        singleword cmd = is_v_mode(lmt_nest_state.nest[pointer].mode) ? hmove_cmd : vmove_cmd;
831                        tex_print_cmd_chr(cmd, (shift > 0) ? move_forward_code : move_backward_code);
832                        tex_print_dimension(abs(shift), pt_unit);
833                    }
834                }
835                break;
836            case global_box_flag:
837                tex_print_str_esc("global");
838            case box_flag:
839                {
840                    tex_print_str_esc("setbox");
841                    tex_print_int(tex_get_packaging_context());
842                    tex_print_char('=');
843                }
844                break;
845            case a_leaders_flag:
846                tex_print_cmd_chr(leader_cmd, a_leaders);
847                break;
848            case c_leaders_flag:
849                tex_print_cmd_chr(leader_cmd, c_leaders);
850                break;
851            case x_leaders_flag:
852                tex_print_cmd_chr(leader_cmd, x_leaders);
853                break;
854            case g_leaders_flag:
855                tex_print_cmd_chr(leader_cmd, g_leaders);
856                break;
857            case u_leaders_flag:
858                tex_print_cmd_chr(leader_cmd, u_leaders);
859                break;
860        }
861      FOUND1:
862        tex_show_packaging_group(package);
863      FOUND2:
864        --cur_level;
865        cur_group = saved_group_group;
866        lmt_save_state.save_stack_data.ptr = saved_group_boundary;
867    }
868  DONE:
869    lmt_save_state.save_stack_data.ptr = saved_pointer;
870    cur_level = saved_level;
871    cur_group = saved_group;
872    tracing_levels_par = saved_tracing;
873}
874
875void tex_show_save_stack(void)
876{
877    halfword savedptr = lmt_save_state.save_stack_data.ptr;
878    tex_print_format("%l[savestack size %i]\n", lmt_save_state.save_stack_data.ptr);
879    while (lmt_save_state.save_stack_data.ptr) {
880        --lmt_save_state.save_stack_data.ptr;
881        tex_print_nlp();
882        tex_print_levels();
883        tex_print_format("[%i: ", lmt_save_state.save_stack_data.ptr);
884        if (save_type(lmt_save_state.save_stack_data.ptr) >= saved_record_0 && save_type(lmt_save_state.save_stack_data.ptr) <= saved_record_9) {
885            tex_print_format("save record %i, ", save_type(lmt_save_state.save_stack_data.ptr) - saved_record_0 + 1, save_record(lmt_save_state.save_stack_data.ptr));
886            /* these have to be provided in the modules */
887            switch (save_record(lmt_save_state.save_stack_data.ptr)) {
888                case unknown_save_type:
889                    break;
890                case box_save_type:
891                    if (tex_show_packaging_record()) {
892                        break;
893                    } else {
894                        goto INVALID_TYPE;
895                    }
896                case local_box_save_type:
897                    if (tex_show_localbox_record()) {
898                        break;
899                    } else {
900                        goto INVALID_TYPE;
901                    }
902                case alignment_save_type:
903                    if (tex_show_alignment_record()) {
904                        break;
905                    } else {
906                        goto INVALID_TYPE;
907                    }
908                case adjust_save_type:
909                    if (tex_show_adjust_record()) {
910                        break;
911                    } else {
912                        goto INVALID_TYPE;
913                    }
914                case math_save_type:
915                    if (tex_show_math_record()) {
916                        break;
917                    } else {
918                        goto INVALID_TYPE;
919                    }
920                case fraction_save_type:
921                    if (tex_show_math_fraction_record()) {
922                        break;
923                    } else {
924                        goto INVALID_TYPE;
925                    }
926                case radical_save_type:
927                    if (tex_show_math_radical_record()) {
928                        break;
929                    } else {
930                        goto INVALID_TYPE;
931                    }
932                case operator_save_type:
933                    if (tex_show_math_operator_record()) {
934                        break;
935                    } else {
936                        goto INVALID_TYPE;
937                    }
938                case math_group_save_type:
939                    if (tex_show_math_group_record()) {
940                        break;
941                    } else {
942                        goto INVALID_TYPE;
943                    }
944                case choice_save_type:
945                    if (tex_show_math_choice_record()) {
946                        break;
947                    } else {
948                        goto INVALID_TYPE;
949                    }
950                case number_save_type:
951                    if (tex_show_math_number_record()) {
952                        break;
953                    } else {
954                        goto INVALID_TYPE;
955                    }
956                case insert_save_type:
957                    if (tex_show_insert_record()) {
958                        break;
959                    } else {
960                        goto INVALID_TYPE;
961                    }
962                case discretionary_save_type:
963                    if (tex_show_discretionary_record()) {
964                        break;
965                    } else {
966                        goto INVALID_TYPE;
967                    }
968                default:
969                    goto INVALID_RECORD;
970            }
971        } else {
972            switch (save_type(lmt_save_state.save_stack_data.ptr)) {
973                case level_boundary_save_type:
974                    tex_print_format("boundary, group '%s', boundary %i, attrlist %i, line %i", lmt_interface.group_code_values[saved_group_group].name, saved_group_boundary, saved_group_attribute_state, tex_saved_line_at_level());
975                    break;
976                case restore_old_value_save_type:
977                    tex_print_format("restore, level %i, cs ", save_level(lmt_save_state.save_stack_data.ptr));
978                    tex_aux_show_eqtb(save_value(lmt_save_state.save_stack_data.ptr));
979                    break;
980                case insert_tokens_save_type:
981                    tex_print_format("insert, pointer %i", save_value(lmt_save_state.save_stack_data.ptr));
982                    break;
983                case restore_lua_save_type:
984                    tex_print_format("restore lua, level %i, function %i", save_level(lmt_save_state.save_stack_data.ptr), save_value(lmt_save_state.save_stack_data.ptr));
985                    break;
986                case restore_zero_save_type:
987                    tex_print_format("restore zero, level %i, cs ", save_level(lmt_save_state.save_stack_data.ptr));
988                    tex_aux_show_eqtb(save_value(lmt_save_state.save_stack_data.ptr));
989                    break;
990                default:
991                    goto INVALID_TYPE;
992            }
993        }
994        goto DONE;
995      INVALID_RECORD:
996        tex_print_format("invalid record %i", save_record(lmt_save_state.save_stack_data.ptr));
997        goto DONE;
998      INVALID_TYPE:
999        tex_print_format("invalid type %i", save_type(lmt_save_state.save_stack_data.ptr));
1000        goto DONE;
1001      DONE:
1002        tex_print_char(']');
1003        tex_print_nlp();
1004    }
1005    tex_print_format("%l[savestack bottom]\n");
1006    lmt_save_state.save_stack_data.ptr = savedptr;
1007}
1008
1009/*tex
1010    This is an experiment. The |handle_overload| function can either go on or quit, depending on
1011    how strong one wants to check for overloads.
1012
1013    \starttabulate[||||||||]
1014        \NC   \NC         \NC immutable \NC permanent \NC primitive \NC frozen \NC instance \NC \NR
1015        \NC 1 \NC warning \NC +         \NC +         \NC +         \NC        \NC          \NC \NR
1016        \NC 2 \NC error   \NC +         \NC +         \NC +         \NC        \NC          \NC \NR
1017        \NC 3 \NC warning \NC +         \NC +         \NC +         \NC +      \NC          \NC \NR
1018        \NC 4 \NC error   \NC +         \NC +         \NC +         \NC +      \NC          \NC \NR
1019        \NC 5 \NC warning \NC +         \NC +         \NC +         \NC +      \NC +        \NC \NR
1020        \NC 6 \NC error   \NC +         \NC +         \NC +         \NC +      \NC +        \NC \NR
1021    \stoptabulate
1022
1023    The overload callback gets passed:
1024        (boolean) error,
1025        (integer) overload,
1026        (string)  csname,
1027        (integer) flags.
1028
1029    See January 2020 files for an alternative implementation.
1030*/
1031
1032static void tex_aux_handle_overload(const char *s, halfword cs, int overload, int error_type)
1033{
1034    int callback_id = lmt_callback_defined(handle_overload_callback);
1035    if (callback_id > 0) {
1036        lmt_run_callback(lmt_lua_state.lua_instance, callback_id, "bdsd->", error_type == normal_error_type, overload, cs_text(cs), eq_flag(cs));
1037    } else {
1038        tex_handle_error(
1039            error_type,
1040            "You can't redefine %s %S.",
1041            s, cs,
1042            NULL
1043        );
1044    }
1045}
1046
1047static int tex_aux_report_overload(halfword cs, int overload)
1048{
1049    int error_type = overload & 1 ? warning_error_type : normal_error_type;
1050    if (has_eq_flag_bits(cs, immutable_flag_bit)) {
1051        tex_aux_handle_overload("immutable", cs, overload, error_type);
1052    } else if (has_eq_flag_bits(cs, primitive_flag_bit)) {
1053        tex_aux_handle_overload("primitive", cs, overload, error_type);
1054    } else if (has_eq_flag_bits(cs, permanent_flag_bit)) {
1055        tex_aux_handle_overload("permanent", cs, overload, error_type);
1056    } else if (has_eq_flag_bits(cs, frozen_flag_bit)) {
1057        tex_aux_handle_overload("frozen", cs, overload, error_type);
1058    } else if (has_eq_flag_bits(cs, instance_flag_bit)) {
1059        tex_aux_handle_overload("instance", cs, overload, warning_error_type);
1060        return 1;
1061    }
1062    return error_type == warning_error_type;
1063}
1064
1065# define overload_error_type(overload) (overload & 1 ? warning_error_type : normal_error_type)
1066
1067int tex_define_permitted(halfword cs, halfword prefixes)
1068{
1069    halfword overload = overload_mode_par;
1070    if (! cs || ! overload || has_eq_flag_bits(cs, mutable_flag_bit)) {
1071        return 1;
1072    } else if (is_overloaded(prefixes)) {
1073        if (overload > 2 && has_eq_flag_bits(cs, immutable_flag_bit | permanent_flag_bit | primitive_flag_bit)) {
1074            return tex_aux_report_overload(cs, overload);
1075        }
1076    } else if (overload > 4) {
1077        if (has_eq_flag_bits(cs, immutable_flag_bit | permanent_flag_bit | primitive_flag_bit | frozen_flag_bit | instance_flag_bit)) {
1078            return tex_aux_report_overload(cs, overload);
1079        }
1080    } else if (overload > 2) {
1081        if (has_eq_flag_bits(cs, immutable_flag_bit | permanent_flag_bit | primitive_flag_bit | frozen_flag_bit)) {
1082            return tex_aux_report_overload(cs, overload);
1083        }
1084    } else if (has_eq_flag_bits(cs, immutable_flag_bit)) {
1085        return tex_aux_report_overload(cs, overload);
1086    }
1087    return 1;
1088}
1089
1090static int tex_aux_mutation_permitted(halfword cs)
1091{
1092    halfword overload = overload_mode_par;
1093    if (cs && overload && has_eq_flag_bits(cs, immutable_flag_bit)) {
1094        return tex_aux_report_overload(cs, overload);
1095    } else {
1096        return 1;
1097    }
1098}
1099
1100/*tex
1101
1102    Just before an entry of |eqtb| is changed, the following procedure should be called to update
1103    the other data structures properly. It is important to keep in mind that reference counts in
1104    |mem| include references from within |save_stack|, so these counts must be handled carefully.
1105
1106*/
1107
1108// static void tex_aux_eq_destroy(memoryword w)
1109// {
1110//     halfword p = eq_value_field(w);
1111//     if (p) {
1112//         switch (eq_type_field(w)) {
1113//             case call_cmd:
1114//             case protected_call_cmd:
1115//             case semi_protected_call_cmd:
1116//             case constant_call_cmd:
1117//             case tolerant_call_cmd:
1118//             case tolerant_protected_call_cmd:
1119//             case tolerant_semi_protected_call_cmd:
1120//             case register_toks_reference_cmd:
1121//             case internal_toks_reference_cmd:
1122//                 tex_delete_token_reference(p);
1123//                 break;
1124//             case internal_glue_reference_cmd:
1125//             case register_glue_reference_cmd:
1126//             case internal_muglue_reference_cmd:
1127//             case register_muglue_reference_cmd:
1128//             case gluespec_cmd:
1129//             case mugluespec_cmd:
1130//             case mathspec_cmd:
1131//             case fontspec_cmd:
1132//             case specification_reference_cmd:
1133//                 tex_flush_node(p);
1134//                 break;
1135//             case internal_box_reference_cmd:
1136//             case register_box_reference_cmd:
1137//                 tex_flush_node_list(p);
1138//                 break;
1139//             default:
1140//                 break;
1141//         }
1142//     }
1143// }
1144
1145// static int tex_aux_eq_destructor(memoryword w)
1146// {
1147//     return eq_value_field(w) ? lmt_hash_state.destructors[eq_type_field(w)] : 0;
1148// }
1149
1150static inline void tex_aux_eq_destroy(memoryword w)
1151{
1152    halfword p = eq_value_field(w);
1153    if (p) {
1154        switch (lmt_hash_state.destructors[eq_type_field(w)]) {
1155            case eq_token_list:
1156                tex_delete_token_reference(p);
1157                break;
1158            case eq_node:
1159                tex_flush_node(p);
1160                break;
1161            case eq_node_list:
1162                tex_flush_node_list(p);
1163                break;
1164        }
1165    }
1166}
1167
1168/*tex
1169
1170    To save a value of |eqtb[p]| that was established at level |l|, we can use the following
1171    subroutine. This code could be simplified after the xeq cleanup so we actually use one slot
1172    less per saved value.
1173
1174*/
1175
1176static void tex_aux_eq_save(halfword p, quarterword l)
1177{
1178    if (tex_room_on_save_stack()) {
1179        if (l == level_zero) {
1180            save_type(lmt_save_state.save_stack_data.ptr) = restore_zero_save_type;
1181        } else {
1182            save_type(lmt_save_state.save_stack_data.ptr) = restore_old_value_save_type;
1183            save_word(lmt_save_state.save_stack_data.ptr) = lmt_hash_state.eqtb[p];
1184        }
1185        save_level(lmt_save_state.save_stack_data.ptr) = l;
1186        save_value(lmt_save_state.save_stack_data.ptr) = p;
1187        ++lmt_save_state.save_stack_data.ptr;
1188    }
1189}
1190
1191/*tex
1192
1193    The procedure |eq_define| defines an |eqtb| entry having specified |eq_type| and |equiv| fields,
1194    and saves the former value if appropriate. This procedure is used only for entries in the first
1195    four regions of |eqtb|, i.e., only for entries that have |eq_type| and |equiv| fields. After
1196    calling this routine, it is safe to put four more entries on |save_stack|, provided that there
1197    was room for four more entries before the call, since |eq_save| makes the necessary test.
1198
1199    The destroy if same branch comes from \ETEX\ but is it really right to destroy here if we
1200    actually want to keep the value? In practice we only come here with zero cases but even then,
1201    it looks like we can destroy the token list or node (list). Not, that might actually work ok in
1202    the case of glue refs that have work by ref count and token lists and node (lists) are always
1203    different so there we do no harm.
1204
1205    There is room for some optimization here.
1206
1207*/
1208
1209static inline int tex_aux_equal_eq(halfword p, singleword cmd, singleword flag, halfword chr)
1210{
1211    /* maybe keep flag test at call end and then only flip flags */
1212    if (eq_flag(p) == flag) {
1213     // printf("eqtest> %03i %03i\n",eq_type(p),cmd);
1214        switch (eq_type(p)) {
1215            case internal_glue_reference_cmd:
1216            case register_glue_reference_cmd:
1217            case internal_muglue_reference_cmd:
1218            case register_muglue_reference_cmd:
1219            case gluespec_cmd:
1220            case mugluespec_cmd:
1221                /*tex We compare the pointer as well as the record. */
1222                if (tex_same_glue(eq_value(p), chr)) {
1223                    if (chr) {
1224                        tex_flush_node(chr);
1225                    }
1226                    return 1;
1227                } else {
1228                    return 0;
1229                }
1230            case mathspec_cmd:
1231                /*tex Idem here. */
1232                if (tex_same_mathspec(eq_value(p), chr)) {
1233                    if (chr) {
1234                        tex_flush_node(chr);
1235                    }
1236                    return 1;
1237                } else {
1238                    return 0;
1239                }
1240            case fontspec_cmd:
1241                /*tex And here. */
1242                if (tex_same_fontspec(eq_value(p), chr)) {
1243                    if (chr) {
1244                        tex_flush_node(chr);
1245                    }
1246                    return 1;
1247                } else {
1248                    return 0;
1249                }
1250            /*
1251            case specificationspec_cmd:
1252                if (eq_type(p) == cmd && eq_value(p) == chr) {
1253                    return 1;
1254                } else {
1255                    return 0;
1256                }
1257            */
1258            case call_cmd:
1259            case protected_call_cmd:
1260            case semi_protected_call_cmd:
1261            case constant_call_cmd:
1262            case tolerant_call_cmd:
1263            case tolerant_protected_call_cmd:
1264            case tolerant_semi_protected_call_cmd:
1265                /*tex The initial token reference will do as it is unique. */
1266             // if (eq_value(p) == chr) {
1267                if (eq_value(p) == chr && eq_level(p) == cur_level) {
1268                    tex_delete_token_reference(eq_value(p));
1269                    return 1;
1270                } else {
1271                    return 0;
1272                }
1273            case specification_reference_cmd:
1274            case unit_reference_cmd:
1275            case internal_box_reference_cmd:
1276            case register_box_reference_cmd:
1277                /*tex These are also references. The ! chr is a bit strange. Todo: test without. */
1278// if (eq_type(p) == cmd && eq_value(p) == chr && chr) {
1279//     printf("SAME\n");
1280//     // destoy ??
1281// }
1282                if (eq_type(p) == cmd && eq_value(p) == chr && ! chr) {
1283             // if (eq_type(p) == cmd && eq_value(p) == chr && ! chr && eq_level(p) == cur_level) {
1284                    return 1;
1285                } else {
1286                    /* play safe */
1287                    return 0;
1288                }
1289            case internal_toks_reference_cmd:
1290            case register_toks_reference_cmd:
1291                /*tex As are these. */
1292                if (p && chr && eq_value(p) == chr) {
1293                    tex_delete_token_reference(eq_value(p));
1294                    return 1;
1295                } else {
1296                    return 0;
1297                }
1298            case internal_toks_cmd:
1299            case register_toks_cmd:
1300                /*tex Again we have references. */
1301                if (eq_value(p) == chr) {
1302             // if (eq_value(p) == chr && eq_level(p) == cur_level) {
1303                    return 1;
1304                } else {
1305                    return 0;
1306                }
1307            case integer_cmd:
1308            case index_cmd:
1309            case dimension_cmd:
1310            case posit_cmd:
1311                if (eq_type(p) == cmd && eq_value(p) == chr) {
1312             // if (eq_type(p) == cmd && eq_value(p) == chr && eq_level(p) == cur_level) {
1313                    return 1;
1314                } else {
1315                    return 0;
1316                }
1317            default:
1318                /*tex
1319                    We can best also check the level because for integer defs etc we run into
1320                    issues otherwise (see testcase tests/luametatex/eqtest.tex based on MS's
1321                    math file).
1322                */
1323                if (eq_type(p) == cmd && eq_value(p) == chr) {
1324             // if (eq_type(p) == cmd && eq_value(p) == chr && eq_level(p) == cur_level) {
1325                    return 1;
1326                } else {
1327                    return 0;
1328                }
1329        }
1330    } else {
1331        return 0;
1332    }
1333}
1334
1335/*tex Used to define a not yet defined cs or box or ... */
1336
1337void tex_eq_define(halfword p, singleword cmd, halfword chr)
1338{
1339    bool trace = tracing_assigns_par > 0;
1340    if (tex_aux_equal_eq(p, cmd, 0, chr)) {
1341        if (trace) {
1342            tex_aux_diagnostic_trace(p, "reassigning");
1343        }
1344    } else {
1345        if (trace) {
1346            tex_aux_diagnostic_trace(p, "changing");
1347        }
1348        if (eq_level(p) == cur_level) {
1349            tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1350        } else if (cur_level > level_one) {
1351            tex_aux_eq_save(p, eq_level(p));
1352        }
1353        set_eq_level(p, cur_level);
1354        set_eq_type(p, cmd);
1355        set_eq_flag(p, 0);
1356        set_eq_value(p, chr);
1357        if (trace) {
1358            tex_aux_diagnostic_trace(p, "into");
1359        }
1360    }
1361}
1362
1363/*tex
1364
1365    The counterpart of |eq_define| for the remaining (fullword) positions in |eqtb| is called
1366    |eq_word_define|. Since |xeq_level[p] >= level_one| for all |p|, a |restore_zero| will never
1367    be used in this case.
1368
1369*/
1370
1371void tex_eq_word_define(halfword p, int w) /* not used */
1372{
1373    bool trace = tracing_assigns_par > 0;
1374    if (eq_value(p) == w) {
1375        if (trace) {
1376            tex_aux_diagnostic_trace(p, "reassigning");
1377        }
1378    } else {
1379        if (trace) {
1380            tex_aux_diagnostic_trace(p, "changing");
1381        }
1382        if (eq_level(p) != cur_level) {
1383            tex_aux_eq_save(p, eq_level(p));
1384            set_eq_level(p, cur_level);
1385        }
1386        eq_value(p) = w;
1387        if (trace) {
1388            tex_aux_diagnostic_trace(p, "into");
1389        }
1390    }
1391}
1392
1393/*tex
1394
1395    The |eq_define| and |eq_word_define| routines take care of local definitions. Global definitions
1396    are done in almost the same way, but there is no need to save old values, and the new value is
1397    associated with |level_one|.
1398
1399*/
1400
1401void tex_geq_define(halfword p, singleword cmd, halfword chr) /* not used */
1402{
1403    bool trace = tracing_assigns_par > 0;
1404    if (trace) {
1405        tex_aux_diagnostic_trace(p, "globally changing");
1406    }
1407    tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1408    set_eq_level(p, level_one);
1409    set_eq_type(p, cmd);
1410    set_eq_flag(p, 0);
1411    set_eq_value(p, chr);
1412    if (trace) {
1413        tex_aux_diagnostic_trace(p, "into");
1414    }
1415}
1416
1417void tex_geq_word_define(halfword p, int w) /* not used */
1418{
1419    bool trace = tracing_assigns_par > 0;
1420    if (trace) {
1421        tex_aux_diagnostic_trace(p, "globally changing");
1422    }
1423    eq_value(p) = w;
1424    set_eq_level(p, level_one);
1425    if (trace) {
1426        tex_aux_diagnostic_trace(p, "into");
1427    }
1428}
1429
1430/*tex
1431    Instead of a macro that distinguishes between global or not we now use a few normal functions.
1432    That way we don't need to define a bogus variable |a| in some cases. This is typically one of
1433    those changes that happened after other bits and pieces got redone. (One can also consider it
1434    a side effect of looking at the code through a visual studio lense.)
1435*/
1436
1437static inline void tex_aux_set_eq_data(halfword p, singleword t, halfword e, singleword f, quarterword l)
1438{
1439    singleword flag = eq_flag(p);
1440    set_eq_level(p, l);
1441    set_eq_type(p, t);
1442    set_eq_value(p, e);
1443    if (is_mutable(f) || is_mutable(flag)) {
1444        set_eq_flag(p, (f | flag) & ~(noaligned_flag_bit | permanent_flag_bit | primitive_flag_bit | immutable_flag_bit));
1445    } else {
1446        set_eq_flag(p, f);
1447    }
1448}
1449
1450void tex_define(int g, halfword p, singleword t, halfword e) /* int g -> singleword g */
1451{
1452    bool trace = tracing_assigns_par > 0;
1453    singleword f = make_eq_flag_bits(g);
1454    if (is_global(g)) {
1455        /* what if already global */
1456        if (trace) {
1457            tex_aux_diagnostic_trace(p, "globally changing");
1458        }
1459     // if (tex_aux_equal_eq(p, t, f, e) && (eq_level(p) == level_one)) {
1460     //     return; /* we can save some stack */
1461     // }
1462        tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1463        tex_aux_set_eq_data(p, t, e, f, level_one);
1464    } else if (! is_constrained(g) && tex_aux_equal_eq(p, t, f, e)) {
1465        /* hm, we tweak the ref ! */
1466        if (trace) {
1467            tex_aux_diagnostic_trace(p, "reassigning");
1468            return;
1469        }
1470    } else {
1471        if (trace) {
1472            tex_aux_diagnostic_trace(p, "changing");
1473        }
1474        if (eq_level(p) == cur_level) {
1475            tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1476        } else if (is_retained(g)) {
1477            /* nothing */
1478        } else if (cur_level > level_one) {
1479            tex_aux_eq_save(p, eq_level(p));
1480        }
1481        tex_aux_set_eq_data(p, t, e, f, cur_level);
1482    }
1483    if (trace) {
1484        tex_aux_diagnostic_trace(p, "into");
1485    }
1486}
1487
1488/*tex
1489    Used in |\dimendef| but also |\dimensiondef| and alike. Before it gets called we already
1490    redefined to |\relax| so we might have saved.
1491*/
1492
1493void tex_define_again(int g, halfword p, singleword t, halfword e) /* int g -> singleword g */
1494{
1495    if (tracing_assigns_par > 0) {
1496        singleword f = make_eq_flag_bits(g);
1497        if (is_global(g)) {
1498            /* what if already global */
1499            tex_aux_diagnostic_trace(p, "globally changing");
1500         // if (tex_aux_equal_eq(p, t, f, e) && (eq_level(p) == level_one)) {
1501         //     return; /* we can save some stack */
1502         // }
1503            tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1504            tex_aux_set_eq_data(p, t, e, f, level_one);
1505        } else if (! is_constrained(g) && tex_aux_equal_eq(p, t, f, e)) {
1506            /* hm, we tweak the ref ! */
1507            tex_aux_diagnostic_trace(p, "reassigning");
1508        } else {
1509            tex_aux_diagnostic_trace(p, is_retained(g) ? "retained changing": "changing");
1510            if (eq_level(p) == cur_level) {
1511                tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1512            } else if (is_retained(g)) {
1513                /* nothing */
1514            } else if (cur_level > level_one) {
1515                tex_aux_eq_save(p, eq_level(p));
1516            }
1517            tex_aux_set_eq_data(p, t, e, f, cur_level);
1518        }
1519        tex_aux_diagnostic_trace(p, "into");
1520    } else {
1521        set_eq_type(p, t);
1522        set_eq_value(p, e);
1523    }
1524}
1525
1526/*tex
1527    The next one is used when we let something.
1528*/
1529
1530void tex_define_inherit(int g, halfword p, singleword f, singleword t, halfword e)
1531{
1532    bool trace = tracing_assigns_par > 0;
1533    if (is_global(g)) {
1534        /* what if already global */
1535        if (trace) {
1536            tex_aux_diagnostic_trace(p, "globally changing");
1537        }
1538     // if (equal_eq(p, t, f, e) && (eq_level(p) == level_one)) {
1539     //    return; /* we can save some stack */
1540     // }
1541        tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1542        tex_aux_set_eq_data(p, t, e, f, level_one);
1543    } else if (! is_constrained(g) && tex_aux_equal_eq(p, t, f, e)) {
1544        if (trace) {
1545            tex_aux_diagnostic_trace(p, "reassigning");
1546            return;
1547        }
1548    } else {
1549        if (trace) {
1550            tex_aux_diagnostic_trace(p, is_retained(g) ? "retained changing" : "changing");
1551        }
1552        if (eq_level(p) == cur_level) {
1553            tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1554        } else if (is_retained(g)) {
1555            /* nothing */
1556        } else if (cur_level > level_one) {
1557            tex_aux_eq_save(p, eq_level(p));
1558        }
1559        tex_aux_set_eq_data(p, t, e, f, cur_level);
1560    }
1561    if (trace) {
1562        tex_aux_diagnostic_trace(p, "into");
1563    }
1564}
1565
1566/*tex
1567    Used in swapping but beware: when we swap a global vsize with a local ... we can get side
1568    effect. No retain here.
1569*/
1570
1571static void tex_aux_just_define(int g, halfword p, halfword e)
1572{
1573    bool trace = tracing_assigns_par > 0;
1574    if (is_global(g)) {
1575        if (trace) {
1576            tex_aux_diagnostic_trace(p, "globally changing");
1577        }
1578        tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1579    } else {
1580        if (trace) {
1581            tex_aux_diagnostic_trace(p, "changing");
1582        }
1583        if (eq_level(p) == cur_level) {
1584            tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1585        } else if (cur_level > level_one) {
1586            tex_aux_eq_save(p, eq_level(p));
1587        }
1588        set_eq_level(p, cur_level);
1589    }
1590    set_eq_value(p, e);
1591    if (trace) {
1592        tex_aux_diagnostic_trace(p, "into");
1593    }
1594}
1595
1596/* We can have a variant that doesn't save/restore so we just have to swap back then. */
1597
1598void tex_define_swapped(int g, halfword p1, halfword p2, int force)
1599{
1600    halfword t1 = eq_type(p1);
1601    halfword t2 = eq_type(p2);
1602    halfword l1 = eq_level(p1);
1603    halfword l2 = eq_level(p2);
1604    singleword f1 = eq_flag(p1);
1605    singleword f2 = eq_flag(p2);
1606    halfword v1 = eq_value(p1);
1607    halfword v2 = eq_value(p2);
1608    if (t1 == t2 && l1 == l2) {
1609        halfword overload = force ? 0 : overload_mode_par;
1610        if (overload) {
1611           if (f1 != f2) {
1612               goto NOTDONE;
1613           } else if (is_immutable(f1)) {
1614               goto NOTDONE;
1615           }
1616        }
1617        if (v1 == v2)  {
1618            return;
1619        } else {
1620            switch (t1) {
1621                case register_posit_cmd:
1622                case register_integer_cmd:
1623                case register_attribute_cmd:
1624                case register_dimension_cmd:
1625                case register_glue_cmd:   /* unchecked */
1626                case register_muglue_cmd: /* unchecked */
1627                case internal_muglue_cmd: /* unchecked */
1628                case integer_cmd:
1629                case index_cmd:
1630                case dimension_cmd:
1631                case posit_cmd:
1632                    tex_aux_just_define(g, p1, v2);
1633                    tex_aux_just_define(g, p2, v1);
1634                    return;
1635                case register_toks_cmd:
1636                case internal_toks_cmd:
1637                    if (v1) tex_add_token_reference(v1);
1638                    if (v2) tex_add_token_reference(v2);
1639                    tex_aux_just_define(g, p1, v2);
1640                    tex_aux_just_define(g, p2, v1);
1641                    if (v1) tex_delete_token_reference(v1);
1642                    if (v2) tex_delete_token_reference(v2);
1643                    return;
1644                case internal_integer_cmd:
1645                    tex_assign_internal_integer_value(g, p1, v2);
1646                    tex_assign_internal_integer_value(g, p2, v1);
1647                    return;
1648                case internal_attribute_cmd:
1649                    tex_assign_internal_attribute_value(g, p1, v2);
1650                    tex_assign_internal_attribute_value(g, p2, v1);
1651                    return;
1652                case internal_posit_cmd:
1653                    tex_assign_internal_posit_value(g, p1, v2);
1654                    tex_assign_internal_posit_value(g, p2, v1);
1655                    return;
1656                case internal_dimension_cmd:
1657                    tex_assign_internal_dimension_value(g, p1, v2);
1658                    tex_assign_internal_dimension_value(g, p2, v1);
1659                    return;
1660                case internal_glue_cmd:
1661                    /* todo */
1662                    tex_assign_internal_skip_value(g, p1, v2);
1663                    tex_assign_internal_skip_value(g, p2, v1);
1664                    return;
1665                default:
1666                    if (overload > 2) {
1667                        if (has_flag_bits(f1, immutable_flag_bit | permanent_flag_bit | primitive_flag_bit)) {
1668                            if (overload > 3) {
1669                                goto NOTDONE;
1670                            }
1671                        }
1672                    }
1673                    if (is_call_cmd(t1)) {
1674                        if (v1) tex_add_token_reference(v1);
1675                        if (v2) tex_add_token_reference(v2);
1676                        tex_aux_just_define(g, p1, v2);
1677                        tex_aux_just_define(g, p2, v1);
1678                        /* no delete here .. hm */
1679                    } else {
1680                        tex_handle_error(
1681                            normal_error_type,
1682                            "\\swapcsvalues not (yet) implemented for commands (%C, %C)",
1683                            t1, v1, t2, v2, NULL
1684                        );
1685
1686                    }
1687                    return;
1688            }
1689        }
1690    }
1691  NOTDONE:
1692    tex_handle_error(
1693        normal_error_type,
1694        "\\swapcsvalues requires equal commands (%C, %C), levels (%i, %i) and flags (%i, %i)",
1695        t1, v1, t2, v2, l1, l2, f1, f2, NULL
1696    );
1697}
1698
1699/*tex
1700    Used in pushing and popping. No retain here.
1701*/
1702
1703void tex_forced_define(int g, halfword p, singleword f, singleword t, halfword e)
1704{
1705    bool trace = tracing_assigns_par > 0;
1706    if (is_global(g)) {
1707        if (trace) {
1708            tex_aux_diagnostic_trace(p, "globally changing");
1709        }
1710        tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1711        set_eq_level(p, level_one);
1712    } else {
1713        if (trace) {
1714            tex_aux_diagnostic_trace(p, "changing");
1715        }
1716        if (eq_level(p) == cur_level) {
1717            tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1718        } else if (cur_level > level_one) {
1719            tex_aux_eq_save(p, eq_level(p));
1720        }
1721        set_eq_level(p, cur_level);
1722    }
1723    set_eq_type(p, t);
1724    set_eq_flag(p, f);
1725    set_eq_value(p, e);
1726    if (trace) {
1727        tex_aux_diagnostic_trace(p, "into");
1728    }
1729}
1730
1731/*tex
1732    Registers and other values that are stored directly without reference.
1733*/
1734
1735void tex_word_define(int g, halfword p, halfword w)
1736{
1737    if (tex_aux_mutation_permitted(p)) {
1738        bool trace = tracing_assigns_par > 0;
1739        if (is_global(g)) {
1740            if (trace) {
1741                tex_aux_diagnostic_trace(p, "globally changing");
1742            }
1743            eq_value(p) = w;
1744            set_eq_level(p, level_one);
1745        } else if (! is_constrained(g) && eq_value(p) == w) {
1746            if (trace) {
1747                tex_aux_diagnostic_trace(p, "reassigning");
1748                return;
1749            }
1750        } else if (is_retained(g)) {
1751            if (trace) {
1752                tex_aux_diagnostic_trace(p, "retained changing");
1753                set_eq_level(p, cur_level);
1754            }
1755            eq_value(p) = w;
1756        } else {
1757            if (trace) {
1758                tex_aux_diagnostic_trace(p, "changing");
1759            }
1760            if (eq_level(p) != cur_level) {
1761                tex_aux_eq_save(p, eq_level(p));
1762                set_eq_level(p, cur_level);
1763            }
1764            eq_value(p) = w;
1765        }
1766        if (trace) {
1767            tex_aux_diagnostic_trace(p, "into");
1768        }
1769        if (is_immutable(g)) {
1770            eq_flag(p) |= immutable_flag_bit;
1771        } else if (is_mutable(g)) {
1772            eq_flag(p) |= mutable_flag_bit;
1773        }
1774    }
1775}
1776
1777/*
1778void tex_forced_word_define(int g, halfword p, singleword f, halfword w)
1779{
1780    if (tex_aux_mutation_permitted(p)) {
1781        bool trace = tracing_assigns_par > 0;
1782        if (is_global(g)) {
1783            if (trace) {
1784                tex_aux_diagnostic_trace(p, "globally changing");
1785            }
1786            eq_value(p) = w;
1787            set_eq_level(p, level_one);
1788        } else if (eq_value(p) == w) {
1789            if (trace) {
1790                tex_aux_diagnostic_trace(p, "reassigning");
1791                return;
1792            }
1793        } else if (is_retained(g)) {
1794            if (trace) {
1795                tex_aux_diagnostic_trace(p, "retained changing");
1796            }
1797            eq_value(p) = w;
1798        } else {
1799            if (trace) {
1800                tex_aux_diagnostic_trace(p, "changing");
1801            }
1802            if (eq_level(p) != cur_level) {
1803                tex_aux_eq_save(p, eq_level(p));
1804                set_eq_level(p, cur_level);
1805            }
1806            eq_value(p) = w;
1807        }
1808        if (trace) {
1809            tex_aux_diagnostic_trace(p, "into");
1810        }
1811        eq_flag(p) = f;
1812    }
1813}
1814*/
1815
1816/*tex
1817
1818    Subroutine |save_for_after_group| puts a token on the stack for save-keeping.
1819
1820*/
1821
1822void tex_save_for_after_group(halfword t)
1823{
1824    if (t && cur_level > level_one && tex_room_on_save_stack()) {
1825        save_type(lmt_save_state.save_stack_data.ptr) = insert_tokens_save_type;
1826        save_level(lmt_save_state.save_stack_data.ptr) = level_zero;
1827        save_value(lmt_save_state.save_stack_data.ptr) = t;
1828        ++lmt_save_state.save_stack_data.ptr;
1829    }
1830}
1831
1832/*tex
1833
1834    The |unsave| routine goes the other way, taking items off of |save_stack|. This routine takes
1835    care of restoration when a level ends. Here, everything belonging to the topmost group is
1836    cleared off of the save stack.
1837
1838    In \TEX\ there are a few |\after...| commands, like |\aftergroup| and |\afterassignment| while
1839    |\futurelet| also has this property of postponed actions. The |\every...| token registers do
1840    the opposite and do stuff up front. In addition to |\aftergrouped| we have a variant that
1841    accepts a token list, as does |\afterassigned|. These items are saved on the stack.
1842
1843    In \LUAMETATEX\ we can also do things just before a group ends as well as just before the
1844    paragraph finishes. In the end it was not that hard to implement in the \LUATEX\ concept,
1845    although it adds a little overhead, but the benefits compensate that. Because we can use some
1846    mechanisms used in other extensions only a few extra lines are needed. All are accumulative
1847    but the paragraph bound one is special in the sense that is is bound to the current paragraph,
1848    so the actual implementation of that one happens elsewhere and differently.
1849
1850    Side note: when |\par| overloading was introduced in \PDFTEX\ and per request also added to
1851    |\LUATEX| it made no sense to add that to \LUAMETATEX\ too. We already have callbacks, and
1852    there is information available about what triggered a |\par|. Another argument against
1853    supporting this is that overloading |\par| is messy and unreliable (macro package and user
1854    demand and actions can badly interfere). The mentioned hooks already give more than enough
1855    opportunities. One doesn't expect users to overload |\relax| either.
1856
1857    Side note: at some point I will look into |\after| hooks in for instance alignments and maybe
1858    something nicer that |\afterassignment| can be used for pushing stuff into boxes (|\everybox|
1859    is not that helpful). But again avoiding extra overhead might is a very good be a reason to
1860    not do that at all.
1861
1862*/
1863
1864void tex_unsave(void)
1865{
1866    if (end_of_group_par) {
1867        /*tex
1868            This is not yet always ok, and looks like we can get weird commands (in some group
1869            ending situations)! But I need a better example of a failure. (low priority)
1870        */
1871        tex_begin_inserted_list(tex_get_available_token(token_val(end_local_cmd, 0)));
1872        tex_begin_token_list(end_of_group_par, end_of_group_text);
1873        if (tracing_nesting_par > 2) {
1874            tex_local_control_message("entering token scanner via endgroup");
1875        }
1876        tex_local_control(1);
1877// tex_cleanup_input_state();
1878    }
1879    delete_attribute_reference(current_attribute_state);
1880    tex_unsave_math_codes(cur_level);
1881    tex_unsave_cat_codes(cat_code_table_par, cur_level);
1882    tex_unsave_text_codes(cur_level);
1883    tex_unsave_math_data(cur_level);
1884    if (cur_level > level_one) {
1885        /*tex
1886            Variable |a| registers if we already have processed an |\aftergroup|. We append when
1887            >= 1.
1888        */
1889        bool append = false;
1890        bool trace = tracing_restores_par > 0;
1891        --cur_level;
1892        /*tex Clear off top level from |save_stack|. */
1893        while (1) {
1894            --lmt_save_state.save_stack_data.ptr;
1895            switch (save_type(lmt_save_state.save_stack_data.ptr)) {
1896                case level_boundary_save_type:
1897                    goto DONE;
1898                case restore_old_value_save_type:
1899                    {
1900                        halfword p = save_value(lmt_save_state.save_stack_data.ptr);
1901                        /*tex
1902                            Store |save_stack[save_ptr]| in |eqtb[p]|, unless |eqtb[p]| holds a global
1903                            value A global definition, which sets the level to |level_one|, will not be
1904                            undone by |unsave|. If at least one global definition of |eqtb[p]| has been
1905                            carried out within the group that just ended, the last such definition will
1906                            therefore survive.
1907                        */
1908                     // if (p < internal_integer_base || p > eqtb_size) {
1909                        if (p < internal_integer_base || p >= internal_specification_base) {
1910                            if (eq_level(p) == level_one) {
1911                                tex_aux_eq_destroy(save_word(lmt_save_state.save_stack_data.ptr));
1912                                if (trace) {
1913                                    tex_aux_diagnostic_trace(p, "retaining");
1914                                }
1915                            } else {
1916                                tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1917                                lmt_hash_state.eqtb[p] = save_word(lmt_save_state.save_stack_data.ptr);
1918                                if (trace) {
1919                                    tex_aux_diagnostic_trace(p, "restoring");
1920                                }
1921                            }
1922                        } else if (eq_level(p) == level_one) {
1923                            if (trace) {
1924                                tex_aux_diagnostic_trace(p, "retaining");
1925                            }
1926                        } else {
1927                            lmt_hash_state.eqtb[p] = save_word(lmt_save_state.save_stack_data.ptr);
1928                            if (trace) {
1929                                tex_aux_diagnostic_trace(p, "restoring");
1930                            }
1931                        }
1932                        break;
1933                    }
1934                case insert_tokens_save_type:
1935                    {
1936                        /*tex A list starts a new input level (for now). */
1937                        halfword p = save_value(lmt_save_state.save_stack_data.ptr);
1938                        if (append) {
1939                            /*tex We stay at the same input level (an \ETEX\ feature). */
1940                            tex_append_input(p);
1941                        } else {
1942                            tex_insert_input(p);
1943                            append = true;
1944                        }
1945                        break;
1946                    }
1947                case restore_lua_save_type:
1948                    {
1949                        /* The same as lua_function_code in |textoken.c|. */
1950                        halfword p = save_value(lmt_save_state.save_stack_data.ptr);
1951                        if (p > 0) {
1952                            strnumber u = tex_save_cur_string();
1953                            lmt_token_state.luacstrings = 0;
1954                            lmt_function_call(p, 0);
1955                            tex_restore_cur_string(u);
1956                            if (lmt_token_state.luacstrings > 0) {
1957                                tex_lua_string_start();
1958                            }
1959                        } else {
1960                            tex_normal_error("lua restore", "invalid number");
1961                        }
1962                        append = true;
1963                        break;
1964                    }
1965                case restore_zero_save_type:
1966                    {
1967                        halfword p = save_value(lmt_save_state.save_stack_data.ptr);
1968                        if (eq_level(p) == level_one) {
1969                            if (trace) {
1970                                tex_aux_diagnostic_trace(p, "retaining");
1971                            }
1972                        } else {
1973                            if (p < internal_integer_base || p > eqtb_size) {
1974                                tex_aux_eq_destroy(lmt_hash_state.eqtb[p]);
1975                            }
1976                            lmt_hash_state.eqtb[p] = lmt_hash_state.eqtb[undefined_control_sequence];
1977                            if (trace) {
1978                                tex_aux_diagnostic_trace(p, "restoring");
1979                            }
1980                        }
1981                        break;
1982                    }
1983                default:
1984                    /* we have a messed up save pointer */
1985                    tex_formatted_error("tex unsave", "bad save type case %d, probably a stack pointer issue", save_type(lmt_save_state.save_stack_data.ptr));
1986                    break;
1987            }
1988        }
1989      DONE:
1990        if (tracing_groups_par > 0) {
1991            tex_aux_group_trace(1);
1992        }
1993        if (lmt_input_state.in_stack[lmt_input_state.in_stack_data.ptr].group == cur_boundary) {
1994            /*tex Groups are possibly not properly nested with files. */
1995            tex_aux_group_warning();
1996        }
1997        cur_group = saved_group_group;
1998        cur_boundary = saved_group_boundary;
1999        set_current_attribute_state(saved_group_attribute_state);
2000    } else {
2001        /*tex |unsave| is not used when |cur_group=bottom_level| */
2002        tex_confusion("current level");
2003    }
2004}
2005
2006/*tex
2007
2008    Most of the parameters kept in |eqtb| can be changed freely, but there's an exception: The
2009    magnification should not be used with two different values during any \TEX\ job, since a
2010    single magnification is applied to an entire run. The global variable |mag_set| is set to the
2011    current magnification whenever it becomes necessary to \quote {freeze} it at a particular value.
2012
2013    The |prepare_mag| subroutine is called whenever \TEX\ wants to use |mag| for magnification. If
2014    nonzero, this magnification should be used henceforth. We might drop magnifaction at some point.
2015
2016    {\em NB: As we delegate the backend to \LUA\ we have no mag.}
2017
2018    Let's pause a moment now and try to look at the Big Picture. The \TEX\ program consists of three
2019    main parts: syntactic routines, semantic routines, and output routines. The chief purpose of the
2020    syntactic routines is to deliver the user's input to the semantic routines, one token at a time.
2021    The semantic routines act as an interpreter responding to these tokens, which may be regarded as
2022    commands. And the output routines are periodically called on to convert box-and-glue lists into a
2023    compact set of instructions that will be sent to a typesetter. We have discussed the basic data
2024    structures and utility routines of \TEX, so we are good and ready to plunge into the real activity
2025    by considering the syntactic routines.
2026
2027    Our current goal is to come to grips with the |get_next| procedure, which is the keystone of
2028    \TEX's input mechanism. Each call of |get_next| sets the value of three variables |cur_cmd|,
2029    |cur_chr|, and |cur_cs|, representing the next input token.
2030
2031    \startitemize
2032        \startitem
2033            |cur_cmd| denotes a command code from the long list of codes given above;
2034        \stopitem
2035        \startitem
2036            |cur_chr| denotes a character code or other modifier of the command code;
2037        \stopitem
2038        \startitem
2039            |cur_cs| is the |eqtb| location of the current control sequence, if the current token
2040            was a control sequence, otherwise it's zero.
2041        \stopitem
2042    \stopitemize
2043
2044    Underlying this external behavior of |get_next| is all the machinery necessary to convert from
2045    character files to tokens. At a given time we may be only partially finished with the reading of
2046    several files (for which |\input| was specified), and partially finished with the expansion of
2047    some user-defined macros and/or some macro parameters, and partially finished with the generation
2048    of some text in a template for |\halign|, and so on. When reading a character file, special
2049    characters must be classified as math delimiters, etc.; comments and extra blank spaces must be
2050    removed, paragraphs must be recognized, and control sequences must be found in the hash table.
2051    Furthermore there are occasions in which the scanning routines have looked ahead for a word like
2052    |plus| but only part of that word was found, hence a few characters must be put back into the input
2053    and scanned again.
2054
2055    To handle these situations, which might all be present simultaneously, \TEX\ uses various stacks
2056    that hold information about the incomplete activities, and there is a finite state control for each
2057    level of the input mechanism. These stacks record the current state of an implicitly recursive
2058    process, but the |get_next| procedure is not recursive. Therefore it will not be difficult to
2059    translate these algorithms into low-level languages that do not support recursion.
2060
2061    In general, |cur_cmd| is the current command as set by |get_next|, while |cur_chr| is the operand
2062    of the current command. The control sequence found here is registsred in |cur_cs| and is zero if
2063    none found. The |cur_tok| variable contains the packed representative of |cur_cmd| and |cur_chr|
2064    and like the other ones is global.
2065
2066    Here is a procedure that displays the current command. The variable |n| holds the level of |\if ...
2067    \fi| nesting and |l| the line where |\if| started.
2068
2069*/
2070
2071void tex_show_cmd_chr(halfword cmd, halfword chr)
2072{
2073    tex_begin_diagnostic();
2074    if (cur_list.mode != lmt_nest_state.shown_mode) {
2075        if (tracing_commands_par >= 4) {
2076            /*tex So, larger than \ETEX's extra info 3 value. We might just always do this. */
2077            tex_print_format("[mode: entering %M]", cur_list.mode);
2078            tex_print_nlp();
2079            tex_print_levels();
2080            tex_print_str("{");
2081        } else {
2082            tex_print_format("{%M: ", cur_list.mode);
2083        }
2084        lmt_nest_state.shown_mode = cur_list.mode;
2085    } else {
2086        tex_print_str("{");
2087    }
2088    tex_print_cmd_chr((singleword) cmd, chr);
2089    if (cmd == if_test_cmd && tracing_ifs_par > 0) {
2090        halfword p;
2091        int n, l;
2092        if (tracing_commands_par >= 4) {
2093            tex_print_str(": ");
2094        } else {
2095            tex_print_char(' ');
2096        }
2097        if (cur_chr >= first_real_if_test_code || cur_chr == or_else_code || cur_chr == or_unless_code) { /* can be other >= test */
2098            n = 1;
2099            l = lmt_input_state.input_line;
2100        } else {
2101            tex_print_cmd_chr(if_test_cmd, lmt_condition_state.cur_if);
2102            tex_print_char(' ');
2103            n = 0;
2104            l = lmt_condition_state.if_line;
2105        }
2106        /*tex
2107            We now also have a proper counter but this is a check for a potential mess up. If
2108            als is right, |lmt_condition_state.if_nesting| often should match |n|.
2109        */
2110        p = lmt_condition_state.cond_ptr;
2111        while (p) {
2112            ++n;
2113            p = node_next(p);
2114        }
2115        if (l) {
2116            if (tracing_commands_par >= 4) {
2117                tex_print_format("(level %i, line %i, nesting %i)", n, l, lmt_condition_state.if_nesting);
2118            } else {
2119             // tex_print_format("(level %i) entered on line %i", n, l);
2120                tex_print_format("(level %i, line %i)", n, l);
2121            }
2122        } else {
2123            tex_print_format("(level %i)", n);
2124        }
2125    }
2126    tex_print_char('}');
2127    tex_end_diagnostic();
2128}
2129
2130/*tex
2131
2132    Here is a procedure that displays the contents of |eqtb[n]| symbolically.
2133
2134    We're now at equivalent |n| in region 4. First we initialize most things to null or undefined
2135    values. An undefined font is represented by the internal code |font_base|.
2136
2137    However, the character code tables are given initial values based on the conventional
2138    interpretation of \ASCII\ code. These initial values should not be changed when \TEX\ is
2139    adapted for use with non-English languages; all changes to the initialization conventions
2140    should be made in format packages, not in \TEX\ itself, so that global interchange of formats
2141    is possible.
2142
2143    The reorganization was done because I wanted a cleaner token interface at the \LUA\ end. So
2144    we also do some more checking. The order differs from traditional \TEX\ but of course the
2145    approach is similar.
2146
2147    The regions in \LUAMETATEX\ are a bit adapted as a side effect of the \ETEX\ extensions as
2148    well as our own. For instance, we tag all regions because we also need a consistent token
2149    interface to \LUA. We also dropped fonts and some more from the table.
2150
2151    A previous, efficient, still range based variant can be found in the my archive but it makes
2152    no sense to keep it commented here (apart from sentimental reasons) so one now only can see
2153    the range agnostic version here.
2154
2155*/
2156
2157void tex_aux_show_eqtb(halfword n)
2158{
2159    if (n < null_cs) {
2160        tex_print_format("bad token %i, case 1", n);
2161    } else if (eqtb_indirect_range(n)) {
2162        tex_print_cs(n);
2163        tex_print_char('=');
2164        tex_print_cmd_chr(eq_type(n), eq_value(n));
2165        if (eq_type(n) >= call_cmd) {
2166            tex_print_char(':');
2167            tex_token_show(eq_value(n));
2168        }
2169    } else {
2170        switch (eq_type(n)) {
2171            case internal_toks_reference_cmd:
2172                tex_print_cmd_chr(internal_toks_cmd, n);
2173                goto TOKS;
2174            case register_toks_reference_cmd:
2175                tex_print_str_esc("toks");
2176                tex_print_int(register_toks_number(n));
2177              TOKS:
2178                tex_print_char('=');
2179                tex_token_show(eq_value(n));
2180                break;
2181            case internal_box_reference_cmd:
2182                tex_print_cmd_chr(eq_type(n), n);
2183                goto BOX;
2184            case register_box_reference_cmd:
2185                tex_print_str_esc("box");
2186                tex_print_int(register_box_number(n));
2187              BOX:
2188                tex_print_char('=');
2189                if (eq_value(n)) {
2190                    tex_show_node_list(eq_value(n), 0, 1);
2191                    tex_print_levels();
2192                } else {
2193                    tex_print_str("void");
2194                }
2195                break;
2196            case internal_glue_reference_cmd:
2197                tex_print_cmd_chr(internal_glue_cmd, n);
2198                goto SKIP;
2199            case register_glue_reference_cmd:
2200                tex_print_str_esc("skip");
2201                tex_print_int(register_glue_number(n));
2202              SKIP:
2203                tex_print_char('=');
2204                if (tracing_nodes_par > 2) {
2205                    tex_print_format("<%i>", eq_value(n));
2206                }
2207                tex_print_spec(eq_value(n), pt_unit);
2208                break;
2209            case internal_muglue_reference_cmd:
2210                tex_print_cmd_chr(internal_muglue_cmd, n);
2211                goto MUSKIP;
2212            case register_muglue_reference_cmd:
2213                tex_print_str_esc("muskip");
2214                tex_print_int(register_muglue_number(n));
2215              MUSKIP:
2216                if (tracing_nodes_par > 2) {
2217                    tex_print_format("<%i>", eq_value(n));
2218                }
2219                tex_print_char('=');
2220                tex_print_spec(eq_value(n), mu_unit);
2221                break;
2222            case internal_integer_reference_cmd:
2223                tex_print_cmd_chr(internal_integer_cmd, n);
2224                goto COUNT;
2225            case register_integer_reference_cmd:
2226                tex_print_str_esc("count");
2227                tex_print_int(register_integer_number(n));
2228              COUNT:
2229                tex_print_char('=');
2230                tex_print_int(eq_value(n));
2231                break;
2232            case internal_attribute_reference_cmd:
2233                tex_print_cmd_chr(internal_attribute_cmd, n);
2234                goto ATTRIBUTE;
2235            case register_attribute_reference_cmd:
2236                tex_print_str_esc("attribute");
2237                tex_print_int(register_attribute_number(n));
2238              ATTRIBUTE:
2239                tex_print_char('=');
2240                tex_print_int(eq_value(n));
2241                break;
2242            case internal_posit_reference_cmd:
2243                tex_print_cmd_chr(internal_posit_cmd, n);
2244                goto POSIT;
2245            case register_posit_reference_cmd:
2246                tex_print_str_esc("posit");
2247                tex_print_int(register_posit_number(n));
2248              POSIT:
2249                tex_print_char('=');
2250                tex_print_posit(eq_value(n));
2251                break;
2252            case internal_dimension_reference_cmd:
2253                tex_print_cmd_chr(internal_dimension_cmd, n);
2254                goto DIMEN;
2255            case register_dimension_reference_cmd:
2256                tex_print_str_esc("dimen");
2257                tex_print_int(register_dimension_number(n));
2258              DIMEN:
2259                tex_print_char('=');
2260                tex_print_dimension(eq_value(n), pt_unit);
2261                break;
2262            case specification_reference_cmd:
2263                tex_print_cmd_chr(specification_cmd, n);
2264                tex_print_char('=');
2265                if (eq_value(n)) {
2266                 // if (tracing_nodes_par > 2) {
2267                 //     tex_print_format("<%i>", eq_value(n));
2268                 // }
2269                    tex_print_int(specification_count(eq_value(n)));
2270                } else {
2271                    tex_print_char('0');
2272                }
2273                break;
2274            case unit_reference_cmd:
2275                tex_print_cmd_chr(association_cmd, n);
2276                tex_print_char('=');
2277                if (eq_value(n)) {
2278                    tex_print_str("todo");
2279                } else {
2280                    tex_print_char('0');
2281                }
2282                break;
2283            default:
2284                tex_print_format("bad token %i, case 2", n);
2285                break;
2286        }
2287    }
2288}
2289
2290/*tex
2291
2292    A couple of (self documenting) convenient helpers. They do what we do in \LUATEX, but we now
2293    have collapsed all the options in one mode parameter that also gets stored in the glyph so
2294    the older functions are gone. Progress.
2295
2296*/
2297
2298halfword tex_automatic_disc_penalty(halfword mode)
2299{
2300    return hyphenation_permitted(mode, automatic_penalty_hyphenation_mode) ? automatic_hyphen_penalty_par : ex_hyphen_penalty_par;
2301}
2302
2303halfword tex_explicit_disc_penalty(halfword mode)
2304{
2305    return hyphenation_permitted(mode, explicit_penalty_hyphenation_mode) ? explicit_hyphen_penalty_par : ex_hyphen_penalty_par;
2306}
2307
2308/*tex
2309
2310    The table of equivalents needs to get (pre)populated by the right commands and references, so
2311    that happens here (called in maincontrol at ini time).
2312
2313    For diagnostic purposes we now have the type set for registers. As a consequence we not have
2314    four |glue_ref| variants, which is a trivial extension.
2315
2316*/
2317
2318static inline void tex_aux_set_eq(halfword base, quarterword level, singleword cmd, halfword value, halfword count)
2319{
2320    if (count > 0) {
2321        set_eq_level(base, level);
2322        set_eq_type(base, cmd);
2323        set_eq_flag(base, 0);
2324        set_eq_value(base, value);
2325        for (int k = base + 1; k <= base + count; k++){
2326            copy_eqtb_entry(k, base);
2327        }
2328    }
2329}
2330
2331void tex_synchronize_equivalents(void)
2332{
2333    tex_aux_set_eq(null_cs, level_zero, undefined_cs_cmd, null, lmt_hash_state.hash_data.top - 1);
2334}
2335
2336void tex_initialize_equivalents(void)
2337{
2338    /*tex Order matters here! */
2339    tex_aux_set_eq(null_cs,                     level_zero, undefined_cs_cmd,                 null,                   lmt_hash_state.hash_data.top - 1);
2340    tex_aux_set_eq(internal_glue_base,          level_one,  internal_glue_reference_cmd,      zero_glue,              number_glue_pars);
2341    tex_aux_set_eq(register_glue_base,          level_one,  register_glue_reference_cmd,      zero_glue,              max_glue_register_index);
2342    tex_aux_set_eq(internal_muglue_base,        level_one,  internal_muglue_reference_cmd,    zero_glue,              number_muglue_pars);
2343    tex_aux_set_eq(register_muglue_base,        level_one,  register_muglue_reference_cmd,    zero_glue,              max_muglue_register_index);
2344    tex_aux_set_eq(internal_toks_base,          level_one,  internal_toks_reference_cmd,      null,                   number_tok_pars);
2345    tex_aux_set_eq(register_toks_base,          level_one,  register_toks_reference_cmd,      null,                   max_toks_register_index);
2346    tex_aux_set_eq(internal_box_base,           level_one,  internal_box_reference_cmd,       null,                   number_box_pars);
2347    tex_aux_set_eq(register_box_base,           level_one,  register_box_reference_cmd,       null,                   max_box_register_index);
2348    tex_aux_set_eq(internal_integer_base,       level_one,  internal_integer_reference_cmd,   0,                      number_integer_pars);
2349    tex_aux_set_eq(register_integer_base,       level_one,  register_integer_reference_cmd,   0,                      max_integer_register_index);
2350    tex_aux_set_eq(internal_attribute_base,     level_one,  internal_attribute_reference_cmd, unused_attribute_value, number_attribute_pars);
2351    tex_aux_set_eq(register_attribute_base,     level_one,  register_attribute_reference_cmd, unused_attribute_value, max_attribute_register_index);
2352    tex_aux_set_eq(internal_posit_base,         level_one,  internal_posit_reference_cmd,     0,                      number_posit_pars);
2353    tex_aux_set_eq(register_posit_base,         level_one,  register_posit_reference_cmd,     0,                      max_posit_register_index);
2354    tex_aux_set_eq(internal_dimension_base,     level_one,  internal_dimension_reference_cmd, 0,                      number_dimension_pars);
2355    tex_aux_set_eq(register_dimension_base,     level_one,  register_dimension_reference_cmd, 0,                      max_dimension_register_index);
2356    tex_aux_set_eq(internal_specification_base, level_one,  specification_reference_cmd,      null,                   number_specification_pars);
2357    tex_aux_set_eq(internal_unit_base,          level_one,  unit_reference_cmd,               unset_unit_class,       max_unit_register_index);
2358    tex_aux_set_eq(undefined_control_sequence,  level_zero, undefined_cs_cmd,                 null,                   0);
2359    /*tex why here? */
2360    cat_code_table_par = 0;
2361}
2362
2363void tex_initialize_destructors(void)
2364{
2365    lmt_hash_state.destructors[call_cmd]                         = eq_token_list;
2366    lmt_hash_state.destructors[protected_call_cmd]               = eq_token_list;
2367    lmt_hash_state.destructors[semi_protected_call_cmd]          = eq_token_list;
2368    lmt_hash_state.destructors[constant_call_cmd]                = eq_token_list;
2369    lmt_hash_state.destructors[tolerant_call_cmd]                = eq_token_list;
2370    lmt_hash_state.destructors[tolerant_protected_call_cmd]      = eq_token_list;
2371    lmt_hash_state.destructors[tolerant_semi_protected_call_cmd] = eq_token_list;
2372    lmt_hash_state.destructors[register_toks_reference_cmd]      = eq_token_list;
2373    lmt_hash_state.destructors[internal_toks_reference_cmd]      = eq_token_list;
2374    lmt_hash_state.destructors[internal_glue_reference_cmd]      = eq_node;
2375    lmt_hash_state.destructors[register_glue_reference_cmd]      = eq_node;
2376    lmt_hash_state.destructors[internal_muglue_reference_cmd]    = eq_node;
2377    lmt_hash_state.destructors[register_muglue_reference_cmd]    = eq_node;
2378    lmt_hash_state.destructors[gluespec_cmd]                     = eq_node;
2379    lmt_hash_state.destructors[mugluespec_cmd]                   = eq_node;
2380    lmt_hash_state.destructors[mathspec_cmd]                     = eq_node;
2381    lmt_hash_state.destructors[fontspec_cmd]                     = eq_node;
2382    lmt_hash_state.destructors[specificationspec_cmd]            = eq_node;
2383    lmt_hash_state.destructors[specification_reference_cmd]      = eq_node;
2384    lmt_hash_state.destructors[internal_box_reference_cmd]       = eq_node_list;
2385    lmt_hash_state.destructors[register_box_reference_cmd]       = eq_node_list;
2386}
2387
2388int tex_located_save_value(int id)
2389{
2390    int i = lmt_save_state.save_stack_data.ptr - 1;
2391    while (save_type(i) != level_boundary_save_type) {
2392        i--;
2393    }
2394    while (i < lmt_save_state.save_stack_data.ptr) {
2395        if (save_type(i) == restore_old_value_save_type && save_value(i) == id) {
2396            /*
2397            if (math_direction_par != save_value(i - 1)) {
2398                return 1;
2399            }
2400            */
2401            return save_value(i - 1);
2402        }
2403        i++;
2404    }
2405    return 0;
2406}
2407
2408int tex_cs_state(halfword p)
2409{
2410    if (p == null_cs) {
2411        return cs_null_error;
2412    } else if (p < hash_base) {
2413        return cs_below_base_error;
2414    } else if (p == undefined_control_sequence) {
2415        return cs_undefined_error;
2416    } else if (eqtb_out_of_range(p)) {
2417        return cs_out_of_range_error;
2418    } else {
2419        return cs_no_error;
2420    }
2421}
2422
2423void tex_save_stack_catch_up(void)
2424{
2425    // save_state_info saved_save_stack_data = lmt_save_state;
2426    halfword saved_stack_ptr = lmt_save_state.save_stack_data.ptr;
2427    quarterword saved_group = cur_group;
2428    quarterword saved_level = cur_level;
2429    lmt_save_state.save_stack_data.ptr = cur_boundary;
2430    while (lmt_input_state.in_stack[lmt_input_state.in_stack_data.ptr].group != lmt_save_state.save_stack_data.ptr) {
2431        --cur_level;
2432        tex_print_nlp();
2433        tex_print_format("Warning: end of file when %G is incomplete", 1);
2434        cur_group = save_level(lmt_save_state.save_stack_data.ptr);
2435        lmt_save_state.save_stack_data.ptr = save_value(lmt_save_state.save_stack_data.ptr);
2436    }
2437    // lmt_save_state = saved_save_stack_data;
2438    lmt_save_state.save_stack_data.ptr = saved_stack_ptr;
2439    cur_level = saved_level;
2440    cur_group = saved_group;
2441}
2442