Source Browser ?

1/*
2    See license.txt in the root of this project.
3*/
4
5# include "luametatex.h"
6
7input_state_info lmt_input_state = {
8    .input_stack      = NULL,
9    .input_stack_data = {
10        .minimum   = min_stack_size,
11        .maximum   = max_stack_size,
12        .size      = siz_stack_size,
13        .step      = stp_stack_size,
14        .allocated = 0,
15        .itemsize  = sizeof(in_state_record),
16        .top       = 0,
17        .ptr       = 0,
18        .initial   = memory_data_unset,
19        .offset    = 0,
20        .extra     = 0, 
21    },
22    .in_stack         = NULL,
23    .in_stack_data    = {
24        .minimum   = min_in_open,
25        .maximum   = max_in_open,
26        .size      = siz_in_open,
27        .step      = stp_in_open,
28        .allocated = 0,
29        .itemsize  = sizeof(input_stack_record),
30        .top       = 0,
31        .ptr       = 0,
32        .initial   = memory_data_unset,
33        .offset    = 0,
34        .extra     = 0, 
35    },
36    .parameter_stack      = NULL,
37    .parameter_stack_data = {
38        .minimum   = min_parameter_size,
39        .maximum   = max_parameter_size,
40        .size      = siz_parameter_size,
41        .step      = stp_parameter_size,
42        .allocated = 0,
43        .itemsize  = sizeof(halfword),
44        .top       = 0,
45        .ptr       = 0,
46        .initial   = memory_data_unset,
47        .offset    = 0,
48        .extra     = 0, 
49    },
50    .cur_input      = { 0 },
51    .input_line     = 0,
52    .scanner_status = 0,
53    .def_ref        = 0,
54    .align_state    = 0,
55    .base_ptr       = 0,
56    .warning_index  = 0,
57    .open_files     = 0,
58    .padding        = 0,
59} ;
60
61input_file_state_info input_file_state = {
62    .forced_file = 0,
63    .forced_line = 0,
64    .mode        = 0,
65    .line        = 0,
66};
67
68/*tex 
69    We play safe and always keep a few batches of parameter slots in reserve so that we 
70    are unlikely to overrun.
71*/
72
73# define reserved_input_stack_slots      2
74# define reserved_in_stack_slots         2
75//define reserved_parameter_stack_slots 32                    
76# define reserved_parameter_stack_slots (2 * max_match_count) 
77
78void tex_initialize_input_state(void)
79{
80    {
81        int size = lmt_input_state.input_stack_data.minimum;
82        lmt_input_state.input_stack = aux_allocate_clear_array(sizeof(in_state_record), size, reserved_input_stack_slots);
83        if (lmt_input_state.input_stack) {
84            lmt_input_state.input_stack_data.allocated = size;
85        } else {
86            tex_overflow_error("input",  size);
87        }
88    }
89    {
90        int size = lmt_input_state.in_stack_data.minimum;
91        lmt_input_state.in_stack = aux_allocate_clear_array(sizeof(input_stack_record), size, reserved_in_stack_slots);
92        if (lmt_input_state.in_stack) {
93            lmt_input_state.in_stack_data.allocated = size;
94        } else {
95            tex_overflow_error("file", size);
96        }
97    }
98    {
99        int size = lmt_input_state.parameter_stack_data.minimum;
100        lmt_input_state.parameter_stack = aux_allocate_clear_array(sizeof(halfword), size, reserved_parameter_stack_slots);
101        if (lmt_input_state.parameter_stack) {
102            lmt_input_state.parameter_stack_data.allocated = size;
103        } else {
104            tex_overflow_error("parameter", size);
105        }
106    }
107}
108
109static bool tex_aux_room_on_input_stack(void) /* quite similar to save_stack checker so maybe share */
110{
111    int top = lmt_input_state.input_stack_data.ptr;
112    if (top > lmt_input_state.input_stack_data.top) {
113        lmt_input_state.input_stack_data.top = top;
114        if (top > lmt_input_state.input_stack_data.allocated) {
115            in_state_record *tmp = NULL;
116            top = lmt_input_state.input_stack_data.allocated + lmt_input_state.input_stack_data.step;
117            if (top > lmt_input_state.input_stack_data.size) {
118                top = lmt_input_state.input_stack_data.size;
119            }
120            if (top > lmt_input_state.input_stack_data.allocated) {
121                lmt_input_state.input_stack_data.allocated = top;
122                tmp = aux_reallocate_array(lmt_input_state.input_stack, sizeof(in_state_record), top, reserved_input_stack_slots);
123                lmt_input_state.input_stack = tmp;
124            }
125            lmt_run_memory_callback("input", tmp ? 1 : 0);
126            if (! tmp) {
127                tex_overflow_error("input", top);
128                return false;
129            }
130        }
131    }
132    return true;
133}
134
135static bool tex_aux_room_on_in_stack(void) /* quite similar to save_stack checker so maybe share */
136{
137    int top = lmt_input_state.in_stack_data.ptr;
138    if (top > lmt_input_state.in_stack_data.top) {
139        lmt_input_state.in_stack_data.top = top;
140        if (top > lmt_input_state.in_stack_data.allocated) {
141            input_stack_record *tmp = NULL;
142            top = lmt_input_state.in_stack_data.allocated + lmt_input_state.in_stack_data.step;
143            if (top > lmt_input_state.in_stack_data.size) {
144                top = lmt_input_state.in_stack_data.size;
145            }
146            if (top > lmt_input_state.in_stack_data.allocated) {
147                lmt_input_state.in_stack_data.allocated = top;
148                tmp = aux_reallocate_array(lmt_input_state.in_stack, sizeof(input_stack_record), top, reserved_in_stack_slots);
149                lmt_input_state.in_stack = tmp;
150            }
151            lmt_run_memory_callback("file", tmp ? 1 : 0);
152            if (! tmp) {
153                tex_overflow_error("file", top);
154                return false;
155            }
156        }
157    }
158    return true;
159}
160
161static bool tex_aux_room_on_parameter_stack(void) /* quite similar to save_stack checker so maybe share */
162{
163    int top = lmt_input_state.parameter_stack_data.ptr;
164    if (top > lmt_input_state.parameter_stack_data.top) {
165        lmt_input_state.parameter_stack_data.top = top;
166        if (top > lmt_input_state.parameter_stack_data.allocated) {
167            halfword *tmp =  NULL;
168            top = lmt_input_state.parameter_stack_data.allocated + lmt_input_state.parameter_stack_data.step;
169            if (top > lmt_input_state.parameter_stack_data.size) {
170                top = lmt_input_state.parameter_stack_data.size;
171            }
172            if (top > lmt_input_state.parameter_stack_data.allocated) {
173                lmt_input_state.parameter_stack_data.allocated = top;
174                tmp = aux_reallocate_array(lmt_input_state.parameter_stack, sizeof(halfword), top, reserved_parameter_stack_slots);
175                lmt_input_state.parameter_stack = tmp;
176            }
177            lmt_run_memory_callback("parameter", tmp ? 1 : 0);
178            if (! tmp) {
179                tex_overflow_error("parameter", top);
180                return false;
181            }
182        }
183    }
184    return true;
185}
186
187void tex_copy_to_parameter_stack(halfword *pstack, int n)
188{
189    if (tex_aux_room_on_parameter_stack()) {
190        if (n == 1) { 
191            lmt_input_state.parameter_stack[lmt_input_state.parameter_stack_data.ptr++] = pstack[0];
192     // } else if (n == 2) { 
193     //        lmt_input_state.parameter_stack[lmt_input_state.parameter_stack_data.ptr++] = pstack[0];
194     //        lmt_input_state.parameter_stack[lmt_input_state.parameter_stack_data.ptr++] = pstack[1];
195        } else { 
196            memcpy(&lmt_input_state.parameter_stack[lmt_input_state.parameter_stack_data.ptr], pstack, n * sizeof(halfword));
197            lmt_input_state.parameter_stack_data.ptr += n;
198        }
199    }
200}
201
202/*tex
203
204    As elsewhere we keep variables that belong together in a structure: |input_stack|, the first
205    unused location of |input_stack| being |input_ptr|, the largest value of |input_ptr| when
206    pushing |max_input_stack|, the the \quote {top} input state|cur_input|, the number of lines in
207    the buffer, less one, |in_open|, the number of open text files |open_files| (in regular \TEX\
208    called |open_parens| because it relates to the way files are reported), the |input_file| and
209    the current line number in the current source file |line|. Furthermore some stacks:
210    |line_stack|. |source_filename_stack| and |full_source_filename_stack|. The |scanner_status|
211    tells if we can a end a subfile now. There is an obscure identifier relevant to non-|normal|
212    scanner status |warning_index|. Then there is the often used reference count pointer of token
213    list being defined: |def_ref|.
214
215    Here is a procedure that uses |scanner_status| to print a warning message when a subfile has
216    ended, and at certain other crucial times. Actually it is only called when we run out of
217    token memory. Because memory errors can be of any kind, we normall will not use the \TEX\
218    error handler (but we do have a callback).
219
220    Similar code is is us in |texerrors.c| for use with the error callback. Maybe some day that
221    will be default.
222
223*/
224
225void tex_show_validity(void)
226{
227    halfword p = null;
228    switch (lmt_input_state.scanner_status) {
229        case scanner_is_defining:
230            p = lmt_input_state.def_ref;
231            break;
232        case scanner_is_matching:
233        case scanner_is_tolerant:
234            p = tex_expand_match_token_head();
235            break;
236        case scanner_is_aligning:
237            p = tex_alignment_hold_token_head();
238            break;
239        case scanner_is_absorbing:
240            p = lmt_input_state.def_ref;
241            break;
242    }
243    if (p) {
244        tex_print_ln();
245        tex_token_show(p);
246        tex_print_ln();
247    }
248}
249
250void tex_show_runaway(void)
251{
252    if (lmt_input_state.scanner_status > scanner_is_skipping) {
253        tex_print_nlp();
254        switch (lmt_input_state.scanner_status) {
255            case scanner_is_defining:
256                tex_print_str("We ran into troubles when scanning a definition.");
257                break;
258            case scanner_is_matching:
259                tex_print_str("We ran into troubles scanning an argument.");
260                break;
261            case scanner_is_tolerant:
262                return;
263            case scanner_is_aligning:
264                tex_print_str("We ran into troubles scanning an alignment preamle.");
265                break;
266            case scanner_is_absorbing:
267                tex_print_str("We ran into troubles absorbing something.");
268                break;
269            default:
270                return;
271        }
272        tex_print_nlp();
273        tex_show_validity();
274    }
275}
276
277/*tex
278
279    The |parameter_stack| is an auxiliary array used to hold pointers to the token lists for parameters
280    at the current level and subsidiary levels of input. This stack is maintained with convention
281    (2), and it grows at a different rate from the others.
282
283    So, the token list pointers for parameters is |parameter_stack|, the first unused entry in
284    |parameter_stack| is |parameter_ptr| which is in the range |0 .. parameter_size + 9|.
285
286    The input routines must also interact with the processing of |\halign| and |\valign|, since the
287    appearance of tab marks and |\cr| in certain places is supposed to trigger the beginning of
288    special |v_j| template text in the scanner. This magic is accomplished by an |align_state|
289    variable that is increased by~1 when a |\char'173| is scanned and decreased by~1 when a |\char
290    '175| is scanned. The |align_state| is nonzero during the $u_j$ template, after which it is set
291    to zero; the |v_j| template begins when a tab mark or |\cr| occurs at a time that |align_state
292    = 0|.
293
294    Thus, the \quote {current input state} can be very complicated indeed; there can be many levels
295    and each level can arise in a variety of ways. The |show_context| procedure, which is used by
296    \TEX's error-reporting routine to print out the current input state on all levels down to the
297    most recent line of characters from an input file, illustrates most of these conventions. The
298    global variable |base_ptr| contains the lowest level that was displayed by this procedure.
299
300    The status at each level is indicated by printing two lines, where the first line indicates
301    what was read so far and the second line shows what remains to be read. The context is cropped,
302    if necessary, so that the first line contains at most |half_error_line| characters, and the
303    second contains at most |error_line|. Non-current input levels whose |token_type| is |backed_up|
304    are shown only if they have not been fully read.
305
306    When applicable, print the location of the current line. This routine should be changed, if
307    necessary, to give the best possible indication of where the current line resides in the input
308    file. For example, on some systems it is best to print both a page and line number.
309
310    Because we also have \LUA\ input en output and because error messages and contexts can go
311    through \LUA, reporting is a bit different in \LUAMETATEX.
312
313*/
314
315static void tex_aux_print_indent(void)
316{
317    for (int q = 1; q <= lmt_error_state.context_indent; q++) {
318        tex_print_char(' ');
319    }
320}
321
322static void tex_aux_print_current_input_state(void)
323{
324    int macro = 0;
325    tex_print_str("<");
326    if (lmt_input_state.cur_input.state == token_list_state) {
327        switch (lmt_input_state.cur_input.token_type) {
328            case parameter_text:
329                tex_print_str("argument");
330                break;
331            case template_pre_text:
332                tex_print_str("templatepre");
333                break;
334            case template_post_text:
335                tex_print_str("templatepost");
336                break;
337            case associated_text:
338                tex_print_str("associated");
339                break;
340            case backed_up_text:
341                tex_print_str(lmt_input_state.cur_input.loc ? "to be read again" : "recently read");
342                break;
343            case inserted_text:
344                tex_print_str("inserted text");
345                break;
346            case macro_text:
347                tex_print_str("macro");
348                macro = lmt_input_state.cur_input.name;
349                break;
350            case output_text:
351                tex_print_str("output");
352                break;
353            case every_par_text:
354                tex_print_str("everypar");
355                break;
356            case every_math_text:
357                tex_print_str("everymath");
358                break;
359            case every_display_text:
360                tex_print_str("everydisplay");
361                break;
362            case every_hbox_text:
363                tex_print_str("everyhbox");
364                break;
365            case every_vbox_text:
366                tex_print_str("everyvbox");
367                break;
368            case every_math_atom_text:
369                tex_print_str("everymathatom");
370                break;
371            case every_job_text:
372                tex_print_str("everyjob");
373                break;
374            case every_cr_text:
375                tex_print_str("everycr");
376                break;
377            case every_tab_text:
378                tex_print_str("everytab");
379                break;
380            case end_of_group_text:
381                tex_print_str("endofgroup");
382                break;
383            case mark_text:
384                tex_print_str("mark");
385                break;
386            case token_text:
387                tex_print_str("token");
388                break;
389            case loop_text:
390                tex_print_str("loop");
391                break;
392            case every_eof_text:
393                tex_print_str("everyeof");
394                break;
395            case every_before_par_text:
396                tex_print_str("everybeforepar");
397                break;
398            case end_paragraph_text:
399                tex_print_str("endpar");
400                break;
401            case end_file_text:
402                tex_print_str("endfile");
403                break;
404            case write_text:
405                tex_print_str("write");
406                break;
407            case local_text:
408                tex_print_str("local");
409                break;
410            case local_loop_text:
411                tex_print_str("localloop");
412                break;
413            default:
414                tex_print_str("unknown");
415                break;
416        }
417    } else {
418        switch (lmt_input_state.cur_input.name) {
419            case io_initial_input_code:
420                tex_print_str("initial input");
421                break;
422            case io_lua_input_code:
423                tex_print_str("lua input");
424                break;
425            case io_token_input_code:
426                tex_print_str("token input");
427                break;
428            case io_token_eof_input_code:
429                tex_print_str("token eof input");
430                break;
431            case io_tex_macro_code:
432            case io_file_input_code:
433            default:
434                {
435                    /* Todo : figure out what the weird line is when we have a premature file end. */
436                    tex_print_str("line ");
437                    tex_print_int(lmt_input_state.cur_input.index);
438                    tex_print_char('.');
439                    tex_print_int(lmt_input_state.cur_input.index == lmt_input_state.in_stack_data.ptr ? lmt_input_state.input_line : lmt_input_state.in_stack[lmt_input_state.cur_input.index + 1].line);
440                }
441                break;
442        }
443    }
444    tex_print_str("> ");
445    if (macro) {
446        tex_print_cs_checked(macro);
447    }
448}
449
450/*tex
451
452    Here it is necessary to explain a little trick. We don't want to store a long string that
453    corresponds to a token list, because that string might take up lots of memory; and we are
454    printing during a time when an error message is being given, so we dare not do anything that
455    might overflow one of \TEX's tables. So \quote {pseudoprinting} is the answer: We enter a mode
456    of printing that stores characters into a buffer of length |error_line|, where character $k +
457    1$ is placed into |trick_buf [k mod error_line]| if |k < trick_count|, otherwise character |k|
458    is dropped. Initially we set |tally := 0| and |trick_count := 1000000|; then when we reach the
459    point where transition from line 1 to line 2 should occur, we set |first_count := tally| and
460    |trick_count := tmax > (error_line, tally + 1 + error_line - half_error_line)|. At the end
461    of the pseudoprinting, the values of |first_count|, |tally|, and |trick_count| give us all the
462    information we need to print the two lines, and all of the necessary text is in |trick_buf|.
463
464    Namely, let |l| be the length of the descriptive information that appears on the first line.
465    The length of the context information gathered for that line is |k = first_count|, and the
466    length of the context information gathered for line~2 is $m=\min(|tally|, |trick_count|) - k$.
467    If |l + k <= h|, where |h = half_error_line|, we print |trick_buf[0 .. k-1]| after the
468    descriptive information on line~1, and set |n := l + k|; here |n| is the length of line~1. If
469    |l + k > h|, some cropping is necessary, so we set |n := h| and print |...| followed by
470    |trick_buf[(l + k - h + 3) .. k - 1]| where subscripts of |trick_buf| are circular modulo
471    |error_line|. The second line consists of |n|~spaces followed by |trick_buf[k .. (k + m - 1)]|,
472    unless |n + m > error_line|; in the latter case, further cropping is done. This is easier to
473    program than to explain.
474
475    The following code sets up the print routines so that they will gather the desired information.
476
477*/
478
479void tex_set_trick_count(void)
480{
481    lmt_print_state.first_count = lmt_print_state.tally;
482    lmt_print_state.trick_count = lmt_print_state.tally + 1 + lmt_error_state.line_limits.size - lmt_error_state.half_line_limits.size;
483    if (lmt_print_state.trick_count < lmt_error_state.line_limits.size) {
484        lmt_print_state.trick_count = lmt_error_state.line_limits.size;
485    }
486}
487
488/*tex
489
490    We don't care too much if we stay a bit too much below the max error_line even if we have more
491    room on the line. If length is really an issue then any length is. After all one can set the
492    length larger. 
493
494    This is not the nicest looking output so at some day I might spend some time on a variant that 
495    is a bit more helpful. 
496
497*/
498
499static void tex_aux_print_valid_utf8(int q)
500{
501    int l = lmt_error_state.line_limits.size;
502    int c = (int) lmt_print_state.trick_buffer[q % l];
503    if (c < 128) {
504        tex_print_char(c);
505    } else if (c < 194) {
506        /* invalid */
507    } else if (c < 224) {
508        tex_print_char(c);
509        tex_print_char(lmt_print_state.trick_buffer[(q + 1) % l]);
510    } else if (c < 240) {
511        tex_print_char(c);
512        tex_print_char(lmt_print_state.trick_buffer[(q + 1) % l]);
513        tex_print_char(lmt_print_state.trick_buffer[(q + 2) % l]);
514    } else if (c < 245) {
515        tex_print_char(c);
516        tex_print_char(lmt_print_state.trick_buffer[(q + 1) % l]);
517        tex_print_char(lmt_print_state.trick_buffer[(q + 2) % l]);
518        tex_print_char(lmt_print_state.trick_buffer[(q + 3) % l]);
519    } else {
520        /*tex Invalid character! */
521    }
522}
523
524void tex_show_context(void)
525{
526    int context_lines = -1; /*tex Number of contexts shown so far, less one: */
527    bool bottom_line = false; /*tex Have we reached the final context to be shown? */
528    lmt_input_state.base_ptr = lmt_input_state.input_stack_data.ptr;
529    lmt_input_state.input_stack[lmt_input_state.base_ptr] = lmt_input_state.cur_input;
530    while (1) {
531        /*tex Enter into the context. */
532        lmt_input_state.cur_input = lmt_input_state.input_stack[lmt_input_state.base_ptr];
533        if ((lmt_input_state.cur_input.state != token_list_state) && (io_file_input(lmt_input_state.cur_input.name) || (lmt_input_state.base_ptr == 0))) {
534            bottom_line = true;
535        }
536        if ((lmt_input_state.base_ptr == lmt_input_state.input_stack_data.ptr) || bottom_line || (context_lines < error_context_lines_par)) {
537            /*tex Display the current context. */
538            if ((lmt_input_state.base_ptr == lmt_input_state.input_stack_data.ptr) || (lmt_input_state.cur_input.state != token_list_state) || (lmt_input_state.cur_input.token_type != backed_up_text) || (lmt_input_state.cur_input.loc)) {
539                /*tex
540                    We omit backed-up token lists that have already been read. Get ready to count
541                    characters. We start pseudo printing.
542
543                    This is complex code. When we display a context, we loop over context lines, but
544                    actually we're talking of two lines: the discriptive line and the token list or
545                    something from the buffer. Then there is that trick buffer stuff. In order to
546                    get a better picture I expanded some variable names. Also, the length of the
547                    input state line never got registered as there was no pseudo printing used.
548
549                    Because in \LUAMETATEX\ the content can come from \LUA\ we display the state
550                    somewhat differently: we also show the input level for line numbers and we tag
551                    for instance a macro, just for consistency. The contexts are separated by
552                    newlines.
553                */
554                bool skip = false;
555                tex_print_nlp();
556                tex_aux_print_current_input_state();
557                /*
558                    The |pseudo_selector_code| selector value is only set in this context. It makes
559                    sure that we end up at the place where the problem happens.
560                */
561                {
562                    int saved_selector = lmt_print_state.selector;
563                    lmt_print_state.tally = 0;
564                    lmt_print_state.selector = pseudo_selector_code;
565                    lmt_print_state.trick_count = 1000000;
566                    if (lmt_input_state.cur_input.state == token_list_state) {
567                        halfword head = lmt_input_state.cur_input.token_type < macro_text ? lmt_input_state.cur_input.start : token_link(lmt_input_state.cur_input.start);
568                        tex_show_token_list_context(head, lmt_input_state.cur_input.loc);
569                    } else if (lmt_input_state.cur_input.name == io_lua_input_code) {
570                        skip = true;
571                    } else {
572                        /*tex Before we pseudo print the line we determine the effective end. */
573                        int j = lmt_input_state.cur_input.limit;
574                        if (lmt_fileio_state.io_buffer[lmt_input_state.cur_input.limit] != end_line_char_par) {
575                            ++j;
576                        }
577                        if (j > 0) {
578                            for (int i = lmt_input_state.cur_input.start; i <= j - 1; i++) {
579                                if (i == lmt_input_state.cur_input.loc) {
580                                    tex_set_trick_count();
581                                }
582                                tex_print_char(lmt_fileio_state.io_buffer[i]);
583                            }
584                        }
585                    }
586                    lmt_print_state.selector = saved_selector;
587                }
588                /*tex Print two lines using the tricky pseudoprinted information. */
589                if (! skip) {
590                    int p; /*tex Starting or ending place in |trick_buf|. */
591                    int m; /*tex Context information gathered for line 2. */
592                    int n; /*tex Length of line 1. */
593                    tex_print_nlp();
594                    tex_aux_print_indent();
595                    if (lmt_print_state.trick_count == 1000000) {
596                        tex_set_trick_count();
597                    }
598                    /*tex The |set_trick_count| must be performed. */
599                    if (lmt_print_state.tally < lmt_print_state.trick_count) {
600                        m = lmt_print_state.tally - lmt_print_state.first_count;
601                    } else {
602                        m = lmt_print_state.trick_count - lmt_print_state.first_count;
603                    }
604                    if (lmt_print_state.first_count <= lmt_error_state.half_line_limits.size) {
605                        p = 0;
606                        n = lmt_print_state.first_count;
607                    } else {
608                        tex_print_str("...");
609                        p = lmt_print_state.first_count - lmt_error_state.half_line_limits.size + 3;
610                        n = lmt_error_state.half_line_limits.size;
611                    }
612                    for (int q = p; q <= lmt_print_state.first_count - 1; q++) {
613                        tex_aux_print_valid_utf8(q);
614                    }
615                    /*tex
616                        Print |n| spaces to begin line 2. Instead of |n| we use a fixed value of
617                        |error_context_indent|.
618                    */
619                    if (m + n > lmt_error_state.line_limits.size) {
620                        p = lmt_print_state.first_count + (lmt_error_state.line_limits.size - n - 3);
621                    } else {
622                        p = lmt_print_state.first_count + m;
623                    }
624                    if (lmt_print_state.first_count <= p - 1) {
625                        tex_print_nlp();
626                        tex_aux_print_indent();
627                        for (int q = lmt_print_state.first_count; q <= p - 1; q++) {
628                            tex_aux_print_valid_utf8(q);
629                        }
630                        if (m + n > lmt_error_state.line_limits.size) {
631                            tex_print_str(" ...");
632                        }
633                    }
634                }
635                ++context_lines;
636            }
637        } else if (context_lines == error_context_lines_par) {
638            tex_print_nlp();
639            tex_print_str(" ...");
640            tex_print_nlp();
641            ++context_lines;
642            /*tex Omitted if |error_context_lines_par < 0|. */
643        }
644        if (bottom_line) {
645            break;
646        } else {
647            --lmt_input_state.base_ptr;
648        }
649    }
650    /*tex Restore the original state. */
651    lmt_input_state.cur_input = lmt_input_state.input_stack[lmt_input_state.input_stack_data.ptr];
652    tex_print_ln();
653    tex_print_nlp();
654}
655
656/*tex
657
658    The following subroutines change the input status in commonly needed ways. First comes
659    |push_input|, which stores the current state and creates a new level (having, initially, the
660    same properties as the old). Enter a new input level, save the old:
661
662*/
663
664static inline void tex_aux_push_input(void)
665{
666    if (tex_aux_room_on_input_stack()) {
667        lmt_input_state.input_stack[lmt_input_state.input_stack_data.ptr] = lmt_input_state.cur_input;
668        ++lmt_input_state.input_stack_data.ptr;
669    } else {
670        tex_overflow_error("input stack size", lmt_input_state.input_stack_data.size);
671    }
672}
673
674static inline void tex_aux_pop_input(void)
675{
676    lmt_input_state.cur_input = lmt_input_state.input_stack[--lmt_input_state.input_stack_data.ptr];
677}
678
679/*tex
680
681    Here is a procedure that starts a new level of token-list input, given a token list |p| and its
682    type |t|. If |t=macro|, the calling routine should set |name| and |loc|.
683
684    I added a few few simple variants because the compiler will then inline the little code involved
685    and these are used often.
686
687*/
688
689void tex_begin_token_list(halfword t, quarterword kind)
690{
691    tex_aux_push_input();
692    lmt_input_state.cur_input.state = token_list_state;
693    lmt_input_state.cur_input.start = t;
694    lmt_input_state.cur_input.token_type = kind;
695    if (kind < macro_text) {
696        lmt_input_state.cur_input.loc = t;
697    } else if (kind == macro_text) {
698        /*tex More frequently when processing a document: */
699        tex_add_token_reference(t);
700        lmt_input_state.cur_input.parameter_start = lmt_input_state.parameter_stack_data.ptr;
701    } else {
702        /*tex More frequently when making a format: */
703        tex_add_token_reference(t);
704        /*tex The token list started with a reference count. */
705        lmt_input_state.cur_input.loc = token_link(t);
706        if (tracing_macros_par > 0) {
707            tex_begin_diagnostic();
708            switch (kind) {
709                case end_of_group_text:
710                    tex_print_str("endgroup");
711                    break;                    
712                case mark_text:
713                    tex_print_str("mark");
714                    break;
715                case token_text:
716                    tex_print_str("token");
717                    break;
718                case loop_text:
719                    tex_print_str("loop");
720                    break;
721                case end_paragraph_text:
722                    tex_print_str("endpar");
723                    break;
724                case end_file_text:
725                    tex_print_str("endfile");
726                    break;
727                case write_text:
728                    tex_print_str("write");
729                    break;
730                case local_text:
731                    tex_print_str("local");
732                    break;
733                case local_loop_text:
734                    tex_print_str("localloop");
735                    break;
736                default:
737                    /* messy offsets */
738                    tex_print_cmd_chr(internal_toks_cmd, kind - output_text + internal_toks_location(output_routine_code));
739                    break;
740            }
741            tex_print_str("->");
742            if (kind == loop_text || kind == local_loop_text) { 
743                tex_print_char('{');
744            }
745            tex_token_show(t);
746            tex_end_diagnostic();
747        }
748    }
749}
750
751/*tex Testing for |t == null| happens before calling the following functions. */
752
753void tex_begin_parameter_list(halfword t)
754{
755    tex_aux_push_input();
756    lmt_input_state.cur_input.state = token_list_state;
757    lmt_input_state.cur_input.start = t;
758    lmt_input_state.cur_input.loc = t;
759    lmt_input_state.cur_input.token_type = parameter_text;
760}
761
762/*
763void tex_begin_backed_up_list_checked(halfword t)
764{
765    if (lmt_input_state.cur_input.state == token_list_state && lmt_input_state.cur_input.start == lmt_input_state.cur_input.loc && lmt_input_state.cur_input.token_type == backed_up_text) {
766        halfword c = t ; 
767        while (token_link(c)) {
768            c = token_link(c);
769        }
770        token_link(c) = lmt_input_state.cur_input.start;
771    } else { 
772        tex_aux_push_input();
773        lmt_input_state.cur_input.state = token_list_state;
774        lmt_input_state.cur_input.token_type = backed_up_text;
775    }
776    lmt_input_state.cur_input.start = t;
777    lmt_input_state.cur_input.loc = t;
778}
779*/
780
781void tex_begin_backed_up_list(halfword t)
782{
783    tex_aux_push_input();
784    lmt_input_state.cur_input.state = token_list_state;
785    lmt_input_state.cur_input.start = t;
786    lmt_input_state.cur_input.loc = t;
787    lmt_input_state.cur_input.token_type = backed_up_text;
788}
789
790void tex_begin_inserted_list(halfword t)
791{
792    tex_aux_push_input();
793    lmt_input_state.cur_input.state = token_list_state;
794    lmt_input_state.cur_input.start = t;
795    lmt_input_state.cur_input.loc = t;
796    lmt_input_state.cur_input.token_type = inserted_text;
797}
798
799void tex_begin_associated_list(halfword t)
800{
801    tex_aux_push_input();
802    lmt_input_state.cur_input.state = token_list_state;
803    lmt_input_state.cur_input.start = t;
804    lmt_input_state.cur_input.loc = t;
805    lmt_input_state.cur_input.token_type = associated_text;
806}
807
808void tex_begin_macro_list(halfword t)
809{
810    tex_aux_push_input();
811    lmt_input_state.cur_input.state = token_list_state;
812    lmt_input_state.cur_input.start = t;
813    tex_add_token_reference(t);
814    lmt_input_state.cur_input.token_type = macro_text;
815    lmt_input_state.cur_input.parameter_start = lmt_input_state.parameter_stack_data.ptr;
816}
817
818/*tex
819
820    When a token list has been fully scanned, the following computations should be done as we leave
821    that level of input. The |token_type| tends to be equal to either |backed_up| or |inserted|
822    about 2/3 of the time.
823
824*/
825
826void tex_end_token_list(void)
827{
828    /*tex Leave a token-list input level: */
829    switch (lmt_input_state.cur_input.token_type) {
830        case parameter_text:
831            break;
832        case template_pre_text:
833            if (lmt_input_state.align_state > interwoven_alignment_threshold) {
834                lmt_input_state.align_state = 0;
835            } else {
836                tex_alignment_interwoven_error(7);
837            }
838            break;
839        case template_post_text:
840        case associated_text:
841            break;
842        case backed_up_text:
843        case inserted_text:
844//        case end_of_group_text:
845     /* case local_text: */
846            tex_flush_token_list(lmt_input_state.cur_input.start);
847            break;
848        case macro_text:
849            {
850                tex_delete_token_reference(lmt_input_state.cur_input.start);
851                if (get_token_preamble(lmt_input_state.cur_input.start)) {
852                    /*tex Parameters must be flushed: */
853                    int ptr = lmt_input_state.parameter_stack_data.ptr;
854                    int start = lmt_input_state.cur_input.parameter_start;
855                    while (ptr > start) {
856                        if (lmt_input_state.parameter_stack[--ptr]) {
857                            tex_flush_token_list(lmt_input_state.parameter_stack[ptr]);
858                         // lmt_input_state.parameter_stack[ptr] = null;
859                        }
860                    }
861                    lmt_input_state.parameter_stack_data.ptr = start;
862                } else { 
863                    /*tex We have no arguments so we save very little runtime here. */
864                }
865                break;
866            }
867        default:
868            /*tex Update the reference count: */
869            tex_delete_token_reference(lmt_input_state.cur_input.start);
870            break;
871    }
872    tex_aux_pop_input();
873 /* check_interrupt(); */
874}
875
876void tex_quit_token_list(void)
877{
878    if (lmt_input_state.cur_input.index > 0) {
879        if (lmt_input_state.cur_input.token_type == backed_up_text) {
880            /* \expandafter \ignorerest */
881            tex_end_token_list();
882        }
883        tex_end_token_list();
884    }
885}
886
887/*tex A special version used in macro expansion. Maybe some day I'll optimize it. */
888
889void tex_cleanup_input_state(void)
890{
891    while (! lmt_input_state.cur_input.loc && lmt_input_state.cur_input.state == token_list_state) {
892        switch (lmt_input_state.cur_input.token_type) {
893            case parameter_text:
894                break;
895            case template_pre_text:
896                if (lmt_input_state.align_state > interwoven_alignment_threshold) {
897                    lmt_input_state.align_state = 0;
898                } else {
899                    tex_alignment_interwoven_error(7);
900                }
901                break;
902            case template_post_text:
903            case associated_text:
904                break;
905            case backed_up_text:
906            case inserted_text:
907         /* case end_of_group_text: */
908         /* case local_text: */
909                tex_flush_token_list(lmt_input_state.cur_input.start);
910                break;
911            case macro_text:
912                {
913                    tex_delete_token_reference(lmt_input_state.cur_input.start);
914                    if (get_token_preamble(lmt_input_state.cur_input.start)) {
915                        /*tex Parameters must be flushed: */
916                        int ptr = lmt_input_state.parameter_stack_data.ptr;
917                        int start = lmt_input_state.cur_input.parameter_start;
918                        while (ptr > start) {
919                            if (lmt_input_state.parameter_stack[--ptr]) {
920                                tex_flush_token_list(lmt_input_state.parameter_stack[ptr]);
921                             // lmt_input_state.parameter_stack[ptr] = null;
922                            }
923                        }
924                        lmt_input_state.parameter_stack_data.ptr = start;
925                    }
926                    break;
927                }
928            default:
929                /*tex Update the reference count: */
930                tex_delete_token_reference(lmt_input_state.cur_input.start);
931                break;
932        }
933        tex_aux_pop_input();
934    }
935}
936
937/*tex
938
939    Sometimes \TEX\ has read too far and wants to \quote {unscan} what it has seen. The |back_input|
940    procedure takes care of this by putting the token just scanned back into the input stream, ready
941    to be read again. This procedure can be used only if |cur_tok| represents the token to be
942    replaced. Some applications of \TEX\ use this procedure a lot, so it has been slightly optimized
943    for speed.
944
945*/
946
947/*tex Undo one token of input: */
948
949void tex_back_input(halfword t)
950{
951    while ((lmt_input_state.cur_input.state == token_list_state) && (! lmt_input_state.cur_input.loc) && (lmt_input_state.cur_input.token_type != template_post_text)) {
952        tex_end_token_list();
953    }
954    {
955        /*tex A token list of length one: */
956        halfword p = tex_get_available_token(t);
957        if (t < right_brace_limit) {
958            if (t < left_brace_limit) {
959                --lmt_input_state.align_state;
960            } else {
961                ++lmt_input_state.align_state;
962            }
963        }
964        if (lmt_input_state.cur_input.state == token_list_state && lmt_input_state.cur_input.start == lmt_input_state.cur_input.loc && lmt_input_state.cur_input.token_type == backed_up_text) {
965            token_link(p) = lmt_input_state.cur_input.start;
966        } else {
967            tex_aux_push_input();
968            lmt_input_state.cur_input.state = token_list_state;
969            lmt_input_state.cur_input.token_type = backed_up_text;
970        }
971        lmt_input_state.cur_input.start = p;
972        lmt_input_state.cur_input.loc = p;
973    }
974}
975
976/*tex Insert token |p| into \TEX's input: */
977
978void tex_reinsert_token(halfword t)
979{
980    halfword p = tex_get_available_token(t);
981    set_token_link(p, lmt_input_state.cur_input.loc);
982    lmt_input_state.cur_input.start = p;
983    lmt_input_state.cur_input.loc = p;
984    if (t < right_brace_limit) {
985        if (t < left_brace_limit) {
986            --lmt_input_state.align_state;
987        } else {
988            ++lmt_input_state.align_state;
989        }
990    }
991}
992
993/*tex Some aftergroup related code: */
994
995void tex_insert_input(halfword h)
996{
997    if (h) {
998        while ((lmt_input_state.cur_input.state == token_list_state) && (! lmt_input_state.cur_input.loc) && (lmt_input_state.cur_input.token_type != template_post_text)) {
999            tex_end_token_list();
1000        }
1001        if (token_info(h) < right_brace_limit) {
1002            if (token_info(h) < left_brace_limit) {
1003                --lmt_input_state.align_state;
1004            } else {
1005                ++lmt_input_state.align_state;
1006            }
1007        }
1008        tex_aux_push_input();
1009        lmt_input_state.cur_input.start = h;
1010        lmt_input_state.cur_input.loc = h;
1011        lmt_input_state.cur_input.state = token_list_state;
1012        lmt_input_state.cur_input.token_type = inserted_text;
1013        /* 
1014            This happens seldom, so we don't save much on pushing / popping the input: 
1015        */ 
1016        /*
1017        if (lmt_input_state.cur_input.state == token_list_state && lmt_input_state.cur_input.start == lmt_input_state.cur_input.loc && lmt_input_state.cur_input.token_type == inserted_text) {
1018            token_link(h) = lmt_input_state.cur_input.start;
1019        } else {
1020            tex_aux_push_input();
1021            lmt_input_state.cur_input.state = token_list_state;
1022            lmt_input_state.cur_input.token_type = inserted_text;
1023        }
1024        lmt_input_state.cur_input.start = h;
1025        lmt_input_state.cur_input.loc = h;
1026        */
1027    }
1028}
1029
1030void tex_append_input(halfword h)
1031{
1032    if (h) {
1033        halfword n = h;
1034        if (n) {
1035            while (token_link(n)) {
1036                n = token_link(n);
1037            }
1038            set_token_link(n, lmt_input_state.cur_input.loc);
1039        } else {
1040            set_token_link(h, lmt_input_state.cur_input.loc);
1041        }
1042        lmt_input_state.cur_input.start = h;
1043        lmt_input_state.cur_input.loc = h;
1044    }
1045}
1046
1047/*tex
1048
1049    The |begin_file_reading| procedure starts a new level of input for lines of characters to be
1050    read from a file, or as an insertion from the terminal. It does not take care of opening the
1051    file, nor does it set |loc| or |limit| or |line|.
1052
1053*/
1054
1055void tex_begin_file_reading(void)
1056{
1057    ++lmt_input_state.in_stack_data.ptr;
1058    if (tex_aux_room_on_in_stack() && tex_room_in_buffer(lmt_fileio_state.io_first)) {
1059        tex_aux_push_input();
1060        lmt_input_state.cur_input.index = (short) lmt_input_state.in_stack_data.ptr;
1061        lmt_input_state.in_stack[lmt_input_state.cur_input.index].full_source_filename = NULL;
1062        lmt_input_state.in_stack[lmt_input_state.cur_input.index].end_of_file_seen = 0;
1063        lmt_input_state.in_stack[lmt_input_state.cur_input.index].at_end_of_file = null;
1064        lmt_input_state.in_stack[lmt_input_state.cur_input.index].group = cur_boundary;
1065        lmt_input_state.in_stack[lmt_input_state.cur_input.index].line = lmt_input_state.input_line;
1066        lmt_input_state.in_stack[lmt_input_state.cur_input.index].if_ptr = lmt_condition_state.cond_ptr;
1067        lmt_input_state.cur_input.start = lmt_fileio_state.io_first;
1068        lmt_input_state.cur_input.state = mid_line_state;
1069        lmt_input_state.cur_input.name = io_initial_input_code;
1070        lmt_input_state.cur_input.cattable = default_catcode_table_preset;
1071        lmt_input_state.cur_input.partial = 0;
1072        /*tex Prepare terminal input \SYNCTEX\ information. */
1073        lmt_input_state.cur_input.state_file = 0;
1074        lmt_input_state.cur_input.state_line = 0;
1075    }
1076}
1077
1078/*tex
1079
1080    Conversely, the variables must be downdated when such a level of input is finished. What needs
1081    to be closed depends on what was opened.
1082
1083*/
1084
1085void tex_end_file_reading(void)
1086{
1087    lmt_fileio_state.io_first = lmt_input_state.cur_input.start;
1088    lmt_input_state.input_line = lmt_input_state.in_stack[lmt_input_state.cur_input.index].line;
1089    switch (lmt_input_state.cur_input.name) {
1090        case io_initial_input_code:
1091            break;
1092        case io_lua_input_code:
1093        case io_token_input_code:
1094        case io_token_eof_input_code:
1095            /*tex happens more frequently than reading from file */
1096            lmt_cstring_close();
1097            break;
1098        case io_tex_macro_code:
1099            break;
1100        default:
1101            /*tex A file opened with |\input|, |\read...| is handled by \LUA.  */
1102            tex_lua_a_close_in();
1103            if (lmt_input_state.in_stack[lmt_input_state.cur_input.index].full_source_filename) {
1104                lmt_memory_free(lmt_input_state.in_stack[lmt_input_state.cur_input.index].full_source_filename);
1105                lmt_input_state.in_stack[lmt_input_state.cur_input.index].full_source_filename = NULL;
1106            }
1107            if (lmt_input_state.in_stack[lmt_input_state.cur_input.index].at_end_of_file) {
1108                tex_flush_token_list(lmt_input_state.in_stack[lmt_input_state.cur_input.index].at_end_of_file);
1109                lmt_input_state.in_stack[lmt_input_state.cur_input.index].at_end_of_file = null;
1110            }
1111            break;
1112    }
1113    tex_aux_pop_input();
1114    --lmt_input_state.in_stack_data.ptr;
1115}
1116
1117/*tex
1118
1119    To get \TEX's whole input mechanism going, we perform the following actions.
1120
1121*/
1122
1123void tex_initialize_inputstack(void)
1124{
1125    lmt_input_state.input_stack_data.ptr = 0;
1126    lmt_input_state.input_stack_data.top = 0;
1127    lmt_input_state.in_stack[0].full_source_filename = NULL;
1128    lmt_input_state.in_stack_data.ptr = 0;
1129    lmt_input_state.open_files = 0;
1130    lmt_fileio_state.io_buffer_data.top = 0;
1131    lmt_input_state.in_stack[0].group = 0;
1132    lmt_input_state.in_stack[0].if_ptr = null;
1133    lmt_input_state.parameter_stack_data.ptr = 0;
1134    lmt_input_state.parameter_stack_data.top = 0;
1135    lmt_input_state.scanner_status = scanner_is_normal;
1136    lmt_input_state.warning_index = null;
1137    lmt_fileio_state.io_first = 1;
1138    lmt_input_state.cur_input.state = new_line_state;
1139    lmt_input_state.cur_input.start = 1;
1140    lmt_input_state.cur_input.index = 0;
1141    lmt_input_state.input_line = 0;
1142    lmt_input_state.cur_input.name = io_initial_input_code;
1143    lmt_token_state.force_eof = 0;
1144    lmt_token_state.luacstrings = 0;
1145    lmt_input_state.cur_input.cattable = default_catcode_table_preset;
1146    lmt_input_state.cur_input.partial = 0;
1147    lmt_input_state.align_state = busy_alignment_state;
1148}
1149
1150/*tex 
1151    Currently |iotype| can be |io_token_input_code| or |io_token_eof_input_code| but the idea 
1152    was to get rid of the eof variant. However, it seems that there are still use cases (not 
1153    in \CONTEXT).
1154*/
1155
1156void tex_tex_string_start(int iotype, int cattable)
1157{
1158 /* (void) iotype; */ 
1159    {
1160        halfword head = tex_scan_general_text(NULL);
1161        int saved_selector = lmt_print_state.selector;
1162        lmt_print_state.selector = new_string_selector_code;
1163        tex_show_token_list(head, 0, 0);
1164        lmt_print_state.selector = saved_selector;
1165        tex_flush_token_list(head);
1166    }
1167    {
1168        int len;
1169        char *str = tex_take_string(&len);
1170        lmt_cstring_store(str, len, tex_valid_catcode_table(cattable) ? cattable : cat_code_table_par);
1171        tex_begin_file_reading();
1172        lmt_input_state.input_line = 0;
1173        lmt_input_state.cur_input.limit = lmt_input_state.cur_input.start;
1174        lmt_input_state.cur_input.loc = lmt_input_state.cur_input.limit + 1;
1175        lmt_input_state.cur_input.name = iotype; /* io_token_input_code; */
1176        lmt_cstring_start();
1177    }
1178}
1179
1180
1181void tex_lua_string_start(void)
1182{
1183    /*tex Set up |cur_file| and a new level of input: */
1184    tex_begin_file_reading();
1185    lmt_input_state.input_line = 0;
1186    lmt_input_state.cur_input.limit = lmt_input_state.cur_input.start;
1187    /*tex Force line read: */
1188    lmt_input_state.cur_input.loc = lmt_input_state.cur_input.limit + 1;
1189    lmt_input_state.cur_input.name = io_lua_input_code;
1190    lmt_cstring_start();
1191}
1192
1193void tex_any_string_start(char* s)
1194{
1195    /* via terminal emulator */
1196    /*
1197        int len = strlen(s);
1198        if (len > 0 && room_in_buffer(len + 1)) {
1199            fileio_state.io_last = fileio_state.io_first;
1200            strcpy((char *) &fileio_state.io_buffer[fileio_state.io_first], s);
1201            fileio_state.io_last += len;
1202            input_state.cur_input.loc = fileio_state.io_first;
1203            input_state.cur_input.limit = fileio_state.io_last;
1204            fileio_state.io_first = fileio_state.io_last + 1;
1205        }
1206    */
1207    /* via token input emulator */
1208    lmt_cstring_store(s, (int) strlen(s), cat_code_table_par);
1209    tex_begin_file_reading();
1210    lmt_input_state.input_line = 0;
1211    lmt_input_state.cur_input.limit = lmt_input_state.cur_input.start;
1212    lmt_input_state.cur_input.loc = lmt_input_state.cur_input.limit + 1;
1213    lmt_input_state.cur_input.name = io_token_input_code;
1214    lmt_cstring_start();
1215}
1216
1217/*tex a list without ref count*/
1218
1219halfword tex_wrapped_token_list(halfword list)
1220{
1221    halfword head = tex_store_new_token(null, left_brace_token + '{');
1222    halfword tail =  head;
1223    token_link(tail) = token_link(list);
1224    while (token_link(tail)) {
1225        tail = token_link(tail);
1226    }
1227    tail = tex_store_new_token(tail, right_brace_token + '}');
1228    return head;
1229}
1230
1231const char *tex_current_input_file_name(void)
1232{
1233    int level = lmt_input_state.in_stack_data.ptr;
1234    while (level > 0) {
1235        const char *s = lmt_input_state.in_stack[level--].full_source_filename;
1236        if (s) {
1237            return s;
1238        }
1239    }
1240    /*tex old method */
1241    level = lmt_input_state.in_stack_data.ptr;
1242    while (level > 0) {
1243        int t = lmt_input_state.input_stack[level--].name;
1244        if (t >= cs_offset_value) {
1245            return (const char *) str_string(t);
1246        }
1247    }
1248    return NULL;
1249}
1250