Source Browser ?

1/*
2    See license.txt in the root of this project.
3*/
4
5# include "luametatex.h"
6
7input_state_info lmt_input_state = {
8    .input_stack      = NULL,
9    .input_stack_data = {
10        .minimum   = min_stack_size,
11        .maximum   = max_stack_size,
12        .size      = siz_stack_size,
13        .step      = stp_stack_size,
14        .allocated = 0,
15        .itemsize  = sizeof(in_state_record),
16        .top       = 0,
17        .ptr       = 0,
18        .initial   = memory_data_unset,
19        .offset    = 0,
20    },
21    .in_stack         = NULL,
22    .in_stack_data    = {
23        .minimum   = min_in_open,
24        .maximum   = max_in_open,
25        .size      = siz_in_open,
26        .step      = stp_in_open,
27        .allocated = 0,
28        .itemsize  = sizeof(input_stack_record),
29        .top       = 0,
30        .ptr       = 0,
31        .initial   = memory_data_unset,
32        .offset    = 0,
33    },
34    .parameter_stack      = NULL,
35    .parameter_stack_data = {
36        .minimum   = min_parameter_size,
37        .maximum   = max_parameter_size,
38        .size      = siz_parameter_size,
39        .step      = stp_parameter_size,
40        .allocated = 0,
41        .itemsize  = sizeof(halfword),
42        .top       = 0,
43        .ptr       = 0,
44        .initial   = memory_data_unset,
45        .offset    = 0,
46    },
47    .cur_input      = { 0 },
48    .input_line     = 0,
49    .scanner_status = 0,
50    .def_ref        = 0,
51    .align_state    = 0,
52    .base_ptr       = 0,
53    .warning_index  = 0,
54    .open_files     = 0,
55    .padding        = 0,
56} ;
57
58input_file_state_info input_file_state = {
59    .forced_file = 0,
60    .forced_line = 0,
61    .mode        = 0,
62    .line        = 0,
63};
64
65/*tex 
66    We play safe and always keep a few batches of parameter slots in reserve so that we 
67    are unlikely to overrun.
68*/
69
70# define reserved_input_stack_slots      2
71# define reserved_in_stack_slots         2
72//define reserved_parameter_stack_slots 32                    
73# define reserved_parameter_stack_slots (2 * max_match_count) 
74
75void tex_initialize_input_state(void)
76{
77    {
78        int size = lmt_input_state.input_stack_data.minimum;
79        lmt_input_state.input_stack = aux_allocate_clear_array(sizeof(in_state_record), size, reserved_input_stack_slots);
80        if (lmt_input_state.input_stack) {
81            lmt_input_state.input_stack_data.allocated = size;
82        } else {
83            tex_overflow_error("input",  size);
84        }
85    }
86    {
87        int size = lmt_input_state.in_stack_data.minimum;
88        lmt_input_state.in_stack = aux_allocate_clear_array(sizeof(input_stack_record), size, reserved_in_stack_slots);
89        if (lmt_input_state.in_stack) {
90            lmt_input_state.in_stack_data.allocated = size;
91        } else {
92            tex_overflow_error("file", size);
93        }
94    }
95    {
96        int size = lmt_input_state.parameter_stack_data.minimum;
97        lmt_input_state.parameter_stack = aux_allocate_clear_array(sizeof(halfword), size, reserved_parameter_stack_slots);
98        if (lmt_input_state.parameter_stack) {
99            lmt_input_state.parameter_stack_data.allocated = size;
100        } else {
101            tex_overflow_error("parameter", size);
102        }
103    }
104}
105
106static bool tex_aux_room_on_input_stack(void) /* quite similar to save_stack checker so maybe share */
107{
108    int top = lmt_input_state.input_stack_data.ptr;
109    if (top > lmt_input_state.input_stack_data.top) {
110        lmt_input_state.input_stack_data.top = top;
111        if (top > lmt_input_state.input_stack_data.allocated) {
112            in_state_record *tmp = NULL;
113            top = lmt_input_state.input_stack_data.allocated + lmt_input_state.input_stack_data.step;
114            if (top > lmt_input_state.input_stack_data.size) {
115                top = lmt_input_state.input_stack_data.size;
116            }
117            if (top > lmt_input_state.input_stack_data.allocated) {
118                lmt_input_state.input_stack_data.allocated = top;
119                tmp = aux_reallocate_array(lmt_input_state.input_stack, sizeof(in_state_record), top, reserved_input_stack_slots);
120                lmt_input_state.input_stack = tmp;
121            }
122            lmt_run_memory_callback("input", tmp ? 1 : 0);
123            if (! tmp) {
124                tex_overflow_error("input", top);
125                return false;
126            }
127        }
128    }
129    return true;
130}
131
132static bool tex_aux_room_on_in_stack(void) /* quite similar to save_stack checker so maybe share */
133{
134    int top = lmt_input_state.in_stack_data.ptr;
135    if (top > lmt_input_state.in_stack_data.top) {
136        lmt_input_state.in_stack_data.top = top;
137        if (top > lmt_input_state.in_stack_data.allocated) {
138            input_stack_record *tmp = NULL;
139            top = lmt_input_state.in_stack_data.allocated + lmt_input_state.in_stack_data.step;
140            if (top > lmt_input_state.in_stack_data.size) {
141                top = lmt_input_state.in_stack_data.size;
142            }
143            if (top > lmt_input_state.in_stack_data.allocated) {
144                lmt_input_state.in_stack_data.allocated = top;
145                tmp = aux_reallocate_array(lmt_input_state.in_stack, sizeof(input_stack_record), top, reserved_in_stack_slots);
146                lmt_input_state.in_stack = tmp;
147            }
148            lmt_run_memory_callback("file", tmp ? 1 : 0);
149            if (! tmp) {
150                tex_overflow_error("file", top);
151                return false;
152            }
153        }
154    }
155    return true;
156}
157
158static bool tex_aux_room_on_parameter_stack(void) /* quite similar to save_stack checker so maybe share */
159{
160    int top = lmt_input_state.parameter_stack_data.ptr;
161    if (top > lmt_input_state.parameter_stack_data.top) {
162        lmt_input_state.parameter_stack_data.top = top;
163        if (top > lmt_input_state.parameter_stack_data.allocated) {
164            halfword *tmp =  NULL;
165            top = lmt_input_state.parameter_stack_data.allocated + lmt_input_state.parameter_stack_data.step;
166            if (top > lmt_input_state.parameter_stack_data.size) {
167                top = lmt_input_state.parameter_stack_data.size;
168            }
169            if (top > lmt_input_state.parameter_stack_data.allocated) {
170                lmt_input_state.parameter_stack_data.allocated = top;
171                tmp = aux_reallocate_array(lmt_input_state.parameter_stack, sizeof(halfword), top, reserved_parameter_stack_slots);
172                lmt_input_state.parameter_stack = tmp;
173            }
174            lmt_run_memory_callback("parameter", tmp ? 1 : 0);
175            if (! tmp) {
176                tex_overflow_error("parameter", top);
177                return false;
178            }
179        }
180    }
181    return true;
182}
183
184void tex_copy_to_parameter_stack(halfword *pstack, int n)
185{
186    if (tex_aux_room_on_parameter_stack()) {
187if (n == 1) { 
188        lmt_input_state.parameter_stack[lmt_input_state.parameter_stack_data.ptr++] = pstack[0];
189//} else if (n == 2) { 
190//        lmt_input_state.parameter_stack[lmt_input_state.parameter_stack_data.ptr++] = pstack[0];
191//        lmt_input_state.parameter_stack[lmt_input_state.parameter_stack_data.ptr++] = pstack[1];
192} else { 
193        memcpy(&lmt_input_state.parameter_stack[lmt_input_state.parameter_stack_data.ptr], pstack, n * sizeof(halfword));
194        lmt_input_state.parameter_stack_data.ptr += n;
195}
196    }
197}
198
199/*tex
200
201    As elsewhere we keep variables that belong together in a structure: |input_stack|, the first
202    unused location of |input_stack| being |input_ptr|, the largest value of |input_ptr| when
203    pushing |max_input_stack|, the the \quote {top} input state|cur_input|, the number of lines in
204    the buffer, less one, |in_open|, the number of open text files |open_files| (in regular \TEX\
205    called |open_parens| because it relates to the way files are reported), the |input_file| and
206    the current line number in the current source file |line|. Furthermore some stacks:
207    |line_stack|. |source_filename_stack| and |full_source_filename_stack|. The |scanner_status|
208    tells if we can a end a subfile now. There is an obscure identifier relevant to non-|normal|
209    scanner status |warning_index|. Then there is the often used reference count pointer of token
210    list being defined: |def_ref|.
211
212    Here is a procedure that uses |scanner_status| to print a warning message when a subfile has
213    ended, and at certain other crucial times. Actually it is only called when we run out of
214    token memory. Because memory errors can be of any kind, we normall will not use the \TEX\
215    error handler (but we do have a callback).
216
217    Similar code is is us in |texerrors.c| for use with the error callback. Maybe some day that
218    will be default.
219
220*/
221
222void tex_show_validity(void)
223{
224    halfword p = null;
225    switch (lmt_input_state.scanner_status) {
226        case scanner_is_defining:
227            p = lmt_input_state.def_ref;
228            break;
229        case scanner_is_matching:
230        case scanner_is_tolerant:
231            p = tex_expand_match_token_head();
232            break;
233        case scanner_is_aligning:
234            p = tex_alignment_hold_token_head();
235            break;
236        case scanner_is_absorbing:
237            p = lmt_input_state.def_ref;
238            break;
239    }
240    if (p) {
241        tex_print_ln();
242        tex_token_show(p);
243        tex_print_ln();
244    }
245}
246
247void tex_show_runaway(void)
248{
249    if (lmt_input_state.scanner_status > scanner_is_skipping) {
250        tex_print_nlp();
251        switch (lmt_input_state.scanner_status) {
252            case scanner_is_defining:
253                tex_print_str("We ran into troubles when scanning a definition.");
254                break;
255            case scanner_is_matching:
256                tex_print_str("We ran into troubles scanning an argument.");
257                break;
258            case scanner_is_tolerant:
259                return;
260            case scanner_is_aligning:
261                tex_print_str("We ran into troubles scanning an alignment preamle.");
262                break;
263            case scanner_is_absorbing:
264                tex_print_str("We ran into troubles absorbing something.");
265                break;
266            default:
267                return;
268        }
269        tex_print_nlp();
270        tex_show_validity();
271    }
272}
273
274/*tex
275
276    The |parameter_stack| is an auxiliary array used to hold pointers to the token lists for parameters
277    at the current level and subsidiary levels of input. This stack is maintained with convention
278    (2), and it grows at a different rate from the others.
279
280    So, the token list pointers for parameters is |parameter_stack|, the first unused entry in
281    |parameter_stack| is |parameter_ptr| which is in the range |0 .. parameter_size + 9|.
282
283    The input routines must also interact with the processing of |\halign| and |\valign|, since the
284    appearance of tab marks and |\cr| in certain places is supposed to trigger the beginning of
285    special |v_j| template text in the scanner. This magic is accomplished by an |align_state|
286    variable that is increased by~1 when a |\char'173| is scanned and decreased by~1 when a |\char
287    '175| is scanned. The |align_state| is nonzero during the $u_j$ template, after which it is set
288    to zero; the |v_j| template begins when a tab mark or |\cr| occurs at a time that |align_state
289    = 0|.
290
291    Thus, the \quote {current input state} can be very complicated indeed; there can be many levels
292    and each level can arise in a variety of ways. The |show_context| procedure, which is used by
293    \TEX's error-reporting routine to print out the current input state on all levels down to the
294    most recent line of characters from an input file, illustrates most of these conventions. The
295    global variable |base_ptr| contains the lowest level that was displayed by this procedure.
296
297    The status at each level is indicated by printing two lines, where the first line indicates
298    what was read so far and the second line shows what remains to be read. The context is cropped,
299    if necessary, so that the first line contains at most |half_error_line| characters, and the
300    second contains at most |error_line|. Non-current input levels whose |token_type| is |backed_up|
301    are shown only if they have not been fully read.
302
303    When applicable, print the location of the current line. This routine should be changed, if
304    necessary, to give the best possible indication of where the current line resides in the input
305    file. For example, on some systems it is best to print both a page and line number.
306
307    Because we also have \LUA\ input en output and because error messages and contexts can go
308    through \LUA, reporting is a bit different in \LUAMETATEX.
309
310*/
311
312static void tex_aux_print_indent(void)
313{
314    for (int q = 1; q <= lmt_error_state.context_indent; q++) {
315        tex_print_char(' ');
316    }
317}
318
319static void tex_aux_print_current_input_state(void)
320{
321    int macro = 0;
322    tex_print_str("<");
323    if (lmt_input_state.cur_input.state == token_list_state) {
324        switch (lmt_input_state.cur_input.token_type) {
325            case parameter_text:
326                tex_print_str("argument");
327                break;
328            case template_pre_text:
329                tex_print_str("templatepre");
330                break;
331            case template_post_text:
332                tex_print_str("templatepost");
333                break;
334            case associated_text:
335                tex_print_str("associated");
336                break;
337            case backed_up_text:
338                tex_print_str(lmt_input_state.cur_input.loc ? "to be read again" : "recently read");
339                break;
340            case inserted_text:
341                tex_print_str("inserted text");
342                break;
343            case macro_text:
344                tex_print_str("macro");
345                macro = lmt_input_state.cur_input.name;
346                break;
347            case output_text:
348                tex_print_str("output");
349                break;
350            case every_par_text:
351                tex_print_str("everypar");
352                break;
353            case every_math_text:
354                tex_print_str("everymath");
355                break;
356            case every_display_text:
357                tex_print_str("everydisplay");
358                break;
359            case every_hbox_text:
360                tex_print_str("everyhbox");
361                break;
362            case every_vbox_text:
363                tex_print_str("everyvbox");
364                break;
365            case every_math_atom_text:
366                tex_print_str("everymathatom");
367                break;
368            case every_job_text:
369                tex_print_str("everyjob");
370                break;
371            case every_cr_text:
372                tex_print_str("everycr");
373                break;
374            case every_tab_text:
375                tex_print_str("everytab");
376                break;
377            case end_of_group_text:
378                tex_print_str("endofgroup");
379                break;
380            case mark_text:
381                tex_print_str("mark");
382                break;
383            case token_text:
384                tex_print_str("token");
385                break;
386            case loop_text:
387                tex_print_str("loop");
388                break;
389            case every_eof_text:
390                tex_print_str("everyeof");
391                break;
392            case every_before_par_text:
393                tex_print_str("everybeforepar");
394                break;
395            case end_paragraph_text:
396                tex_print_str("endpar");
397                break;
398            case end_file_text:
399                tex_print_str("endfile");
400                break;
401            case write_text:
402                tex_print_str("write");
403                break;
404            case local_text:
405                tex_print_str("local");
406                break;
407            case local_loop_text:
408                tex_print_str("localloop");
409                break;
410            default:
411                tex_print_str("unknown");
412                break;
413        }
414    } else {
415        switch (lmt_input_state.cur_input.name) {
416            case io_initial_input_code:
417                tex_print_str("initial input");
418                break;
419            case io_lua_input_code:
420                tex_print_str("lua input");
421                break;
422            case io_token_input_code:
423                tex_print_str("token input");
424                break;
425            case io_token_eof_input_code:
426                tex_print_str("token eof input");
427                break;
428            case io_tex_macro_code:
429            case io_file_input_code:
430            default:
431                {
432                    /* Todo : figure out what the weird line is when we have a premature file end. */
433                    tex_print_str("line ");
434                    tex_print_int(lmt_input_state.cur_input.index);
435                    tex_print_char('.');
436                    tex_print_int(lmt_input_state.cur_input.index == lmt_input_state.in_stack_data.ptr ? lmt_input_state.input_line : lmt_input_state.in_stack[lmt_input_state.cur_input.index + 1].line);
437                }
438                break;
439        }
440    }
441    tex_print_str("> ");
442    if (macro) {
443        tex_print_cs_checked(macro);
444    }
445}
446
447/*tex
448
449    Here it is necessary to explain a little trick. We don't want to store a long string that
450    corresponds to a token list, because that string might take up lots of memory; and we are
451    printing during a time when an error message is being given, so we dare not do anything that
452    might overflow one of \TEX's tables. So \quote {pseudoprinting} is the answer: We enter a mode
453    of printing that stores characters into a buffer of length |error_line|, where character $k +
454    1$ is placed into |trick_buf [k mod error_line]| if |k < trick_count|, otherwise character |k|
455    is dropped. Initially we set |tally := 0| and |trick_count := 1000000|; then when we reach the
456    point where transition from line 1 to line 2 should occur, we set |first_count := tally| and
457    |trick_count := tmax > (error_line, tally + 1 + error_line - half_error_line)|. At the end
458    of the pseudoprinting, the values of |first_count|, |tally|, and |trick_count| give us all the
459    information we need to print the two lines, and all of the necessary text is in |trick_buf|.
460
461    Namely, let |l| be the length of the descriptive information that appears on the first line.
462    The length of the context information gathered for that line is |k = first_count|, and the
463    length of the context information gathered for line~2 is $m=\min(|tally|, |trick_count|) - k$.
464    If |l + k <= h|, where |h = half_error_line|, we print |trick_buf[0 .. k-1]| after the
465    descriptive information on line~1, and set |n := l + k|; here |n| is the length of line~1. If
466    |l + k > h|, some cropping is necessary, so we set |n := h| and print |...| followed by
467    |trick_buf[(l + k - h + 3) .. k - 1]| where subscripts of |trick_buf| are circular modulo
468    |error_line|. The second line consists of |n|~spaces followed by |trick_buf[k .. (k + m - 1)]|,
469    unless |n + m > error_line|; in the latter case, further cropping is done. This is easier to
470    program than to explain.
471
472    The following code sets up the print routines so that they will gather the desired information.
473
474*/
475
476void tex_set_trick_count(void)
477{
478    lmt_print_state.first_count = lmt_print_state.tally;
479    lmt_print_state.trick_count = lmt_print_state.tally + 1 + lmt_error_state.line_limits.size - lmt_error_state.half_line_limits.size;
480    if (lmt_print_state.trick_count < lmt_error_state.line_limits.size) {
481        lmt_print_state.trick_count = lmt_error_state.line_limits.size;
482    }
483}
484
485/*tex
486
487    We don't care too much if we stay a bit too much below the max error_line even if we have more
488    room on the line. If length is really an issue then any length is. After all one can set the
489    length larger.
490
491*/
492
493static void tex_aux_print_valid_utf8(int q)
494{
495    int l = lmt_error_state.line_limits.size;
496    int c = (int) lmt_print_state.trick_buffer[q % l];
497    if (c < 128) {
498        tex_print_char(c);
499    } else if (c < 194) {
500        /* invalid */
501    } else if (c < 224) {
502        tex_print_char(c);
503        tex_print_char(lmt_print_state.trick_buffer[(q + 1) % l]);
504    } else if (c < 240) {
505        tex_print_char(c);
506        tex_print_char(lmt_print_state.trick_buffer[(q + 1) % l]);
507        tex_print_char(lmt_print_state.trick_buffer[(q + 2) % l]);
508    } else if (c < 245) {
509        tex_print_char(c);
510        tex_print_char(lmt_print_state.trick_buffer[(q + 1) % l]);
511        tex_print_char(lmt_print_state.trick_buffer[(q + 2) % l]);
512        tex_print_char(lmt_print_state.trick_buffer[(q + 3) % l]);
513    } else {
514        /*tex Invalid character! */
515    }
516}
517
518void tex_show_context(void)
519{
520    int context_lines = -1; /*tex Number of contexts shown so far, less one: */
521    bool bottom_line = false;    /*tex Have we reached the final context to be shown? */
522    lmt_input_state.base_ptr = lmt_input_state.input_stack_data.ptr;
523    lmt_input_state.input_stack[lmt_input_state.base_ptr] = lmt_input_state.cur_input;
524    while (1) {
525        /*tex Enter into the context. */
526        lmt_input_state.cur_input = lmt_input_state.input_stack[lmt_input_state.base_ptr];
527        if ((lmt_input_state.cur_input.state != token_list_state) && (io_file_input(lmt_input_state.cur_input.name) || (lmt_input_state.base_ptr == 0))) {
528            bottom_line = true;
529        }
530        if ((lmt_input_state.base_ptr == lmt_input_state.input_stack_data.ptr) || bottom_line || (context_lines < error_context_lines_par)) {
531            /*tex Display the current context. */
532            if ((lmt_input_state.base_ptr == lmt_input_state.input_stack_data.ptr) || (lmt_input_state.cur_input.state != token_list_state) || (lmt_input_state.cur_input.token_type != backed_up_text) || (lmt_input_state.cur_input.loc)) {
533                /*tex
534                    We omit backed-up token lists that have already been read. Get ready to count
535                    characters. We start pseudo printing.
536
537                    This is complex code. When we display a context, we loop over context lines, but
538                    actually we're talking of two lines: the discriptive line and the token list or
539                    something from the buffer. Then there is that trick buffer stuff. In order to
540                    get a better picture I expanded some variable names. Also, the length of the
541                    input state line never got registered as there was no pseudo printing used.
542
543                    Because in \LUAMETATEX\ the content can come from \LUA\ we display the state
544                    somewhat differently: we also show the input level for line numbers and we tag
545                    for instance a macro, just for consistency. The contexts are separated by
546                    newlines.
547                */
548                bool skip = false;
549                tex_print_nlp();
550                tex_aux_print_current_input_state();
551                /*
552                    The |pseudo_selector_code| selector value is only set in this context. It makes
553                    sure that we end up at the place where the problem happens.
554                */
555                {
556                    int saved_selector = lmt_print_state.selector;
557                    lmt_print_state.tally = 0;
558                    lmt_print_state.selector = pseudo_selector_code;
559                    lmt_print_state.trick_count = 1000000;
560                    if (lmt_input_state.cur_input.state == token_list_state) {
561                        halfword head = lmt_input_state.cur_input.token_type < macro_text ? lmt_input_state.cur_input.start : token_link(lmt_input_state.cur_input.start);
562                        tex_show_token_list_context(head, lmt_input_state.cur_input.loc);
563                    } else if (lmt_input_state.cur_input.name == io_lua_input_code) {
564                        skip = true;
565                    } else {
566                        /*tex Before we pseudo print the line we determine the effective end. */
567                        int j = lmt_input_state.cur_input.limit;
568                        if (lmt_fileio_state.io_buffer[lmt_input_state.cur_input.limit] != end_line_char_par) {
569                            ++j;
570                        }
571                        if (j > 0) {
572                            for (int i = lmt_input_state.cur_input.start; i <= j - 1; i++) {
573                                if (i == lmt_input_state.cur_input.loc) {
574                                    tex_set_trick_count();
575                                }
576                                tex_print_char(lmt_fileio_state.io_buffer[i]);
577                            }
578                        }
579                    }
580                    lmt_print_state.selector = saved_selector;
581                }
582                /*tex Print two lines using the tricky pseudoprinted information. */
583                if (! skip) {
584                    int p; /*tex Starting or ending place in |trick_buf|. */
585                    int m; /*tex Context information gathered for line 2. */
586                    int n; /*tex Length of line 1. */
587                    tex_print_nlp();
588                    tex_aux_print_indent();
589                    if (lmt_print_state.trick_count == 1000000) {
590                        tex_set_trick_count();
591                    }
592                    /*tex The |set_trick_count| must be performed. */
593                    if (lmt_print_state.tally < lmt_print_state.trick_count) {
594                        m = lmt_print_state.tally - lmt_print_state.first_count;
595                    } else {
596                        m = lmt_print_state.trick_count - lmt_print_state.first_count;
597                    }
598                    if (lmt_print_state.first_count <= lmt_error_state.half_line_limits.size) {
599                        p = 0;
600                        n = lmt_print_state.first_count;
601                    } else {
602                        tex_print_str("...");
603                        p = lmt_print_state.first_count - lmt_error_state.half_line_limits.size + 3;
604                        n = lmt_error_state.half_line_limits.size;
605                    }
606                    for (int q = p; q <= lmt_print_state.first_count - 1; q++) {
607                        tex_aux_print_valid_utf8(q);
608                    }
609                    /*tex
610                        Print |n| spaces to begin line 2. Instead of |n| we use a fixed value of
611                        |error_context_indent|.
612                    */
613                    if (m + n > lmt_error_state.line_limits.size) {
614                        p = lmt_print_state.first_count + (lmt_error_state.line_limits.size - n - 3);
615                    } else {
616                        p = lmt_print_state.first_count + m;
617                    }
618                    if (lmt_print_state.first_count <= p - 1) {
619                        tex_print_nlp();
620                        tex_aux_print_indent();
621                        for (int q = lmt_print_state.first_count; q <= p - 1; q++) {
622                            tex_aux_print_valid_utf8(q);
623                        }
624                        if (m + n > lmt_error_state.line_limits.size) {
625                            tex_print_str(" ...");
626                        }
627                    }
628                }
629                ++context_lines;
630            }
631        } else if (context_lines == error_context_lines_par) {
632            tex_print_nlp();
633            tex_print_str(" ...");
634            tex_print_nlp();
635            ++context_lines;
636            /*tex Omitted if |error_context_lines_par < 0|. */
637        }
638        if (bottom_line) {
639            break;
640        } else {
641            --lmt_input_state.base_ptr;
642        }
643    }
644    /*tex Restore the original state. */
645    lmt_input_state.cur_input = lmt_input_state.input_stack[lmt_input_state.input_stack_data.ptr];
646    tex_print_ln();
647    tex_print_nlp();
648}
649
650/*tex
651
652    The following subroutines change the input status in commonly needed ways. First comes
653    |push_input|, which stores the current state and creates a new level (having, initially, the
654    same properties as the old). Enter a new input level, save the old:
655
656*/
657
658inline static void tex_aux_push_input(void)
659{
660    if (tex_aux_room_on_input_stack()) {
661        lmt_input_state.input_stack[lmt_input_state.input_stack_data.ptr] = lmt_input_state.cur_input;
662        ++lmt_input_state.input_stack_data.ptr;
663    } else {
664        tex_overflow_error("input stack size", lmt_input_state.input_stack_data.size);
665    }
666}
667
668inline static void tex_aux_pop_input(void)
669{
670    lmt_input_state.cur_input = lmt_input_state.input_stack[--lmt_input_state.input_stack_data.ptr];
671}
672
673/*tex
674
675    Here is a procedure that starts a new level of token-list input, given a token list |p| and its
676    type |t|. If |t=macro|, the calling routine should set |name| and |loc|.
677
678    I added a few few simple variants because the compiler will then inline the little code involved
679    and these are used often.
680
681*/
682
683void tex_begin_token_list(halfword t, quarterword kind)
684{
685    tex_aux_push_input();
686    lmt_input_state.cur_input.state = token_list_state;
687    lmt_input_state.cur_input.start = t;
688    lmt_input_state.cur_input.token_type = kind;
689    if (kind < macro_text) {
690        lmt_input_state.cur_input.loc = t;
691    } else if (kind == macro_text) {
692        /*tex More frequently when processing a document: */
693        tex_add_token_reference(t);
694        lmt_input_state.cur_input.parameter_start = lmt_input_state.parameter_stack_data.ptr;
695    } else {
696        /*tex More frequently when making a format: */
697        tex_add_token_reference(t);
698        /*tex The token list started with a reference count. */
699        lmt_input_state.cur_input.loc = token_link(t);
700        if (tracing_macros_par > 0) {
701            tex_begin_diagnostic();
702            switch (kind) {
703                case end_of_group_text:
704                    tex_print_str("endgroup");
705                    break;                    
706                case mark_text:
707                    tex_print_str("mark");
708                    break;
709                case token_text:
710                    tex_print_str("token");
711                    break;
712                case loop_text:
713                    tex_print_str("loop");
714                    break;
715                case end_paragraph_text:
716                    tex_print_str("endpar");
717                    break;
718                case end_file_text:
719                    tex_print_str("endfile");
720                    break;
721                case write_text:
722                    tex_print_str("write");
723                    break;
724                case local_text:
725                    tex_print_str("local");
726                    break;
727                case local_loop_text:
728                    tex_print_str("localloop");
729                    break;
730                default:
731                    /* messy offsets */
732                    tex_print_cmd_chr(internal_toks_cmd, kind - output_text + internal_toks_location(output_routine_code));
733                    break;
734            }
735            tex_print_str("->");
736            if (kind == loop_text || kind == local_loop_text) { 
737                tex_print_char('{');
738            }
739            tex_token_show(t);
740            tex_end_diagnostic();
741        }
742    }
743}
744
745/*tex Testing for |t == null| happens before calling the following functions. */
746
747void tex_begin_parameter_list(halfword t)
748{
749    tex_aux_push_input();
750    lmt_input_state.cur_input.state = token_list_state;
751    lmt_input_state.cur_input.start = t;
752    lmt_input_state.cur_input.loc = t;
753    lmt_input_state.cur_input.token_type = parameter_text;
754}
755
756/*
757void tex_begin_backed_up_list_checked(halfword t)
758{
759    if (lmt_input_state.cur_input.state == token_list_state && lmt_input_state.cur_input.start == lmt_input_state.cur_input.loc && lmt_input_state.cur_input.token_type == backed_up_text) {
760        halfword c = t ; 
761        while (token_link(c)) {
762            c = token_link(c);
763        }
764        token_link(c) = lmt_input_state.cur_input.start;
765    } else { 
766        tex_aux_push_input();
767        lmt_input_state.cur_input.state = token_list_state;
768        lmt_input_state.cur_input.token_type = backed_up_text;
769    }
770    lmt_input_state.cur_input.start = t;
771    lmt_input_state.cur_input.loc = t;
772}
773*/
774
775void tex_begin_backed_up_list(halfword t)
776{
777    tex_aux_push_input();
778    lmt_input_state.cur_input.state = token_list_state;
779    lmt_input_state.cur_input.start = t;
780    lmt_input_state.cur_input.loc = t;
781    lmt_input_state.cur_input.token_type = backed_up_text;
782}
783
784void tex_begin_inserted_list(halfword t)
785{
786    tex_aux_push_input();
787    lmt_input_state.cur_input.state = token_list_state;
788    lmt_input_state.cur_input.start = t;
789    lmt_input_state.cur_input.loc = t;
790    lmt_input_state.cur_input.token_type = inserted_text;
791}
792
793void tex_begin_associated_list(halfword t)
794{
795    tex_aux_push_input();
796    lmt_input_state.cur_input.state = token_list_state;
797    lmt_input_state.cur_input.start = t;
798    lmt_input_state.cur_input.loc = t;
799    lmt_input_state.cur_input.token_type = associated_text;
800}
801
802void tex_begin_macro_list(halfword t)
803{
804    tex_aux_push_input();
805    lmt_input_state.cur_input.state = token_list_state;
806    lmt_input_state.cur_input.start = t;
807    tex_add_token_reference(t);
808    lmt_input_state.cur_input.token_type = macro_text;
809    lmt_input_state.cur_input.parameter_start = lmt_input_state.parameter_stack_data.ptr;
810}
811
812/*tex
813
814    When a token list has been fully scanned, the following computations should be done as we leave
815    that level of input. The |token_type| tends to be equal to either |backed_up| or |inserted|
816    about 2/3 of the time.
817
818*/
819
820void tex_end_token_list(void)
821{
822    /*tex Leave a token-list input level: */
823    switch (lmt_input_state.cur_input.token_type) {
824        case parameter_text:
825            break;
826        case template_pre_text:
827            if (lmt_input_state.align_state > interwoven_alignment_threshold) {
828                lmt_input_state.align_state = 0;
829            } else {
830                tex_alignment_interwoven_error(7);
831            }
832            break;
833        case template_post_text:
834        case associated_text:
835            break;
836        case backed_up_text:
837        case inserted_text:
838//        case end_of_group_text:
839     /* case local_text: */
840            tex_flush_token_list(lmt_input_state.cur_input.start);
841            break;
842        case macro_text:
843            {
844                tex_delete_token_reference(lmt_input_state.cur_input.start);
845                if (get_token_preamble(lmt_input_state.cur_input.start)) {
846                    /*tex Parameters must be flushed: */
847                    int ptr = lmt_input_state.parameter_stack_data.ptr;
848                    int start = lmt_input_state.cur_input.parameter_start;
849                    while (ptr > start) {
850                        --ptr;
851                        if (lmt_input_state.parameter_stack[ptr]) {
852                            tex_flush_token_list(lmt_input_state.parameter_stack[ptr]);
853                        }
854                    }
855                    lmt_input_state.parameter_stack_data.ptr = start;
856                } else { 
857                    /*tex We have no arguments so we save very little runtime here. */
858                }
859                break;
860            }
861        default:
862            /*tex Update the reference count: */
863            tex_delete_token_reference(lmt_input_state.cur_input.start);
864            break;
865    }
866    tex_aux_pop_input();
867 /* check_interrupt(); */
868}
869
870void tex_quit_token_list(void)
871{
872    if (lmt_input_state.cur_input.index > 0) {
873        if (lmt_input_state.cur_input.token_type == backed_up_text) {
874            /* \expandafter \ignorerest */
875            tex_end_token_list();
876        }
877        tex_end_token_list();
878    }
879}
880
881/*tex A special version used in macro expansion. Maybe some day I'll optimize it. */
882
883void tex_cleanup_input_state(void)
884{
885    while (! lmt_input_state.cur_input.loc && lmt_input_state.cur_input.state == token_list_state) {
886        switch (lmt_input_state.cur_input.token_type) {
887            case parameter_text:
888                break;
889            case template_pre_text:
890                if (lmt_input_state.align_state > interwoven_alignment_threshold) {
891                    lmt_input_state.align_state = 0;
892                } else {
893                    tex_alignment_interwoven_error(7);
894                }
895                break;
896            case template_post_text:
897            case associated_text:
898                break;
899            case backed_up_text:
900            case inserted_text:
901         /* case end_of_group_text: */
902         /* case local_text: */
903                tex_flush_token_list(lmt_input_state.cur_input.start);
904                break;
905            case macro_text:
906                {
907                    tex_delete_token_reference(lmt_input_state.cur_input.start);
908                    if (get_token_preamble(lmt_input_state.cur_input.start)) {
909                        /*tex Parameters must be flushed: */
910                        int ptr = lmt_input_state.parameter_stack_data.ptr;
911                        int start = lmt_input_state.cur_input.parameter_start;
912                        while (ptr > start) {
913                            if (lmt_input_state.parameter_stack[--ptr]) {
914                                tex_flush_token_list(lmt_input_state.parameter_stack[ptr]);
915                            }
916                        }
917                        lmt_input_state.parameter_stack_data.ptr = start;
918                    }
919                    break;
920                }
921            default:
922                /*tex Update the reference count: */
923                tex_delete_token_reference(lmt_input_state.cur_input.start);
924                break;
925        }
926        tex_aux_pop_input();
927    }
928}
929
930/*tex
931
932    Sometimes \TEX\ has read too far and wants to \quote {unscan} what it has seen. The |back_input|
933    procedure takes care of this by putting the token just scanned back into the input stream, ready
934    to be read again. This procedure can be used only if |cur_tok| represents the token to be
935    replaced. Some applications of \TEX\ use this procedure a lot, so it has been slightly optimized
936    for speed.
937
938*/
939
940/*tex Undo one token of input: */
941
942void tex_back_input(halfword t)
943{
944    while ((lmt_input_state.cur_input.state == token_list_state) && (! lmt_input_state.cur_input.loc) && (lmt_input_state.cur_input.token_type != template_post_text)) {
945        tex_end_token_list();
946    }
947    {
948        /*tex A token list of length one: */
949        halfword p = tex_get_available_token(t);
950        if (t < right_brace_limit) {
951            if (t < left_brace_limit) {
952                --lmt_input_state.align_state;
953            } else {
954                ++lmt_input_state.align_state;
955            }
956        }
957        if (lmt_input_state.cur_input.state == token_list_state && lmt_input_state.cur_input.start == lmt_input_state.cur_input.loc && lmt_input_state.cur_input.token_type == backed_up_text) {
958            token_link(p) = lmt_input_state.cur_input.start;
959        } else {
960            tex_aux_push_input();
961            lmt_input_state.cur_input.state = token_list_state;
962            lmt_input_state.cur_input.token_type = backed_up_text;
963        }
964        lmt_input_state.cur_input.start = p;
965        lmt_input_state.cur_input.loc = p;
966    }
967}
968
969/*tex Insert token |p| into \TEX's input: */
970
971void tex_reinsert_token(halfword t)
972{
973    halfword p = tex_get_available_token(t);
974    set_token_link(p, lmt_input_state.cur_input.loc);
975    lmt_input_state.cur_input.start = p;
976    lmt_input_state.cur_input.loc = p;
977    if (t < right_brace_limit) {
978        if (t < left_brace_limit) {
979            --lmt_input_state.align_state;
980        } else {
981            ++lmt_input_state.align_state;
982        }
983    }
984}
985
986/*tex Some aftergroup related code: */
987
988void tex_insert_input(halfword h)
989{
990    if (h) {
991        while ((lmt_input_state.cur_input.state == token_list_state) && (! lmt_input_state.cur_input.loc) && (lmt_input_state.cur_input.token_type != template_post_text)) {
992            tex_end_token_list();
993        }
994        if (token_info(h) < right_brace_limit) {
995            if (token_info(h) < left_brace_limit) {
996                --lmt_input_state.align_state;
997            } else {
998                ++lmt_input_state.align_state;
999            }
1000        }
1001        tex_aux_push_input();
1002        lmt_input_state.cur_input.start = h;
1003        lmt_input_state.cur_input.loc = h;
1004        lmt_input_state.cur_input.state = token_list_state;
1005        lmt_input_state.cur_input.token_type = inserted_text;
1006        /* 
1007            This happens seldom, so we don't save much on pushing / popping the input: 
1008        */ 
1009        /*
1010        if (lmt_input_state.cur_input.state == token_list_state && lmt_input_state.cur_input.start == lmt_input_state.cur_input.loc && lmt_input_state.cur_input.token_type == inserted_text) {
1011            token_link(h) = lmt_input_state.cur_input.start;
1012        } else {
1013            tex_aux_push_input();
1014            lmt_input_state.cur_input.state = token_list_state;
1015            lmt_input_state.cur_input.token_type = inserted_text;
1016        }
1017        lmt_input_state.cur_input.start = h;
1018        lmt_input_state.cur_input.loc = h;
1019        */
1020    }
1021}
1022
1023void tex_append_input(halfword h)
1024{
1025    if (h) {
1026        halfword n = h;
1027        if (n) {
1028            while (token_link(n)) {
1029                n = token_link(n);
1030            }
1031            set_token_link(n, lmt_input_state.cur_input.loc);
1032        } else {
1033            set_token_link(h, lmt_input_state.cur_input.loc);
1034        }
1035        lmt_input_state.cur_input.start = h;
1036        lmt_input_state.cur_input.loc = h;
1037    }
1038}
1039
1040/*tex
1041
1042    The |begin_file_reading| procedure starts a new level of input for lines of characters to be
1043    read from a file, or as an insertion from the terminal. It does not take care of opening the
1044    file, nor does it set |loc| or |limit| or |line|.
1045
1046*/
1047
1048void tex_begin_file_reading(void)
1049{
1050    ++lmt_input_state.in_stack_data.ptr;
1051    if (tex_aux_room_on_in_stack() && tex_room_in_buffer(lmt_fileio_state.io_first)) {
1052        tex_aux_push_input();
1053        lmt_input_state.cur_input.index = (short) lmt_input_state.in_stack_data.ptr;
1054        lmt_input_state.in_stack[lmt_input_state.cur_input.index].full_source_filename = NULL;
1055        lmt_input_state.in_stack[lmt_input_state.cur_input.index].end_of_file_seen = 0;
1056        lmt_input_state.in_stack[lmt_input_state.cur_input.index].at_end_of_file = null;
1057        lmt_input_state.in_stack[lmt_input_state.cur_input.index].group = cur_boundary;
1058        lmt_input_state.in_stack[lmt_input_state.cur_input.index].line = lmt_input_state.input_line;
1059        lmt_input_state.in_stack[lmt_input_state.cur_input.index].if_ptr = lmt_condition_state.cond_ptr;
1060        lmt_input_state.cur_input.start = lmt_fileio_state.io_first;
1061        lmt_input_state.cur_input.state = mid_line_state;
1062        lmt_input_state.cur_input.name = io_initial_input_code;
1063        lmt_input_state.cur_input.cattable = default_catcode_table_preset;
1064        lmt_input_state.cur_input.partial = 0;
1065        /*tex Prepare terminal input \SYNCTEX\ information. */
1066        lmt_input_state.cur_input.state_file = 0;
1067        lmt_input_state.cur_input.state_line = 0;
1068    }
1069}
1070
1071/*tex
1072
1073    Conversely, the variables must be downdated when such a level of input is finished. What needs
1074    to be closed depends on what was opened.
1075
1076*/
1077
1078void tex_end_file_reading(void)
1079{
1080    lmt_fileio_state.io_first = lmt_input_state.cur_input.start;
1081    lmt_input_state.input_line = lmt_input_state.in_stack[lmt_input_state.cur_input.index].line;
1082    switch (lmt_input_state.cur_input.name) {
1083        case io_initial_input_code:
1084            break;
1085        case io_lua_input_code:
1086        case io_token_input_code:
1087        case io_token_eof_input_code:
1088            /*tex happens more frequently than reading from file */
1089            lmt_cstring_close();
1090            break;
1091        case io_tex_macro_code:
1092            break;
1093        default:
1094            /*tex A file opened with |\input|, |\read...| is handled by \LUA.  */
1095            tex_lua_a_close_in();
1096            if (lmt_input_state.in_stack[lmt_input_state.cur_input.index].full_source_filename) {
1097                lmt_memory_free(lmt_input_state.in_stack[lmt_input_state.cur_input.index].full_source_filename);
1098                lmt_input_state.in_stack[lmt_input_state.cur_input.index].full_source_filename = NULL;
1099            }
1100            if (lmt_input_state.in_stack[lmt_input_state.cur_input.index].at_end_of_file) {
1101                tex_flush_token_list(lmt_input_state.in_stack[lmt_input_state.cur_input.index].at_end_of_file);
1102                lmt_input_state.in_stack[lmt_input_state.cur_input.index].at_end_of_file = null;
1103            }
1104            break;
1105    }
1106    tex_aux_pop_input();
1107    --lmt_input_state.in_stack_data.ptr;
1108}
1109
1110/*tex
1111
1112    To get \TEX's whole input mechanism going, we perform the following actions.
1113
1114*/
1115
1116void tex_initialize_inputstack(void)
1117{
1118    lmt_input_state.input_stack_data.ptr = 0;
1119    lmt_input_state.input_stack_data.top = 0;
1120    lmt_input_state.in_stack[0].full_source_filename = NULL;
1121    lmt_input_state.in_stack_data.ptr = 0;
1122    lmt_input_state.open_files = 0;
1123    lmt_fileio_state.io_buffer_data.top = 0;
1124    lmt_input_state.in_stack[0].group = 0;
1125    lmt_input_state.in_stack[0].if_ptr = null;
1126    lmt_input_state.parameter_stack_data.ptr = 0;
1127    lmt_input_state.parameter_stack_data.top = 0;
1128    lmt_input_state.scanner_status = scanner_is_normal;
1129    lmt_input_state.warning_index = null;
1130    lmt_fileio_state.io_first = 1;
1131    lmt_input_state.cur_input.state = new_line_state;
1132    lmt_input_state.cur_input.start = 1;
1133    lmt_input_state.cur_input.index = 0;
1134    lmt_input_state.input_line = 0;
1135    lmt_input_state.cur_input.name = io_initial_input_code;
1136    lmt_token_state.force_eof = 0;
1137    lmt_token_state.luacstrings = 0;
1138    lmt_input_state.cur_input.cattable = default_catcode_table_preset;
1139    lmt_input_state.cur_input.partial = 0;
1140    lmt_input_state.align_state = busy_alignment_state;
1141}
1142
1143/*tex 
1144    Currently |iotype| can be |io_token_input_code| or |io_token_eof_input_code| but the idea 
1145    was to get rid of the eof variant. However, it seems that there are still use cases (not 
1146    in \CONTEXT).
1147*/
1148
1149void tex_tex_string_start(int iotype, int cattable)
1150{
1151 /* (void) iotype; */ 
1152    {
1153        halfword head = tex_scan_general_text(NULL);
1154        int saved_selector = lmt_print_state.selector;
1155        lmt_print_state.selector = new_string_selector_code;
1156        tex_show_token_list(head, 0, 0);
1157        lmt_print_state.selector = saved_selector;
1158        tex_flush_token_list(head);
1159    }
1160    {
1161        int len;
1162        char *str = tex_take_string(&len);
1163        lmt_cstring_store(str, len, tex_valid_catcode_table(cattable) ? cattable : cat_code_table_par);
1164        tex_begin_file_reading();
1165        lmt_input_state.input_line = 0;
1166        lmt_input_state.cur_input.limit = lmt_input_state.cur_input.start;
1167        lmt_input_state.cur_input.loc = lmt_input_state.cur_input.limit + 1;
1168        lmt_input_state.cur_input.name = iotype; /* io_token_input_code; */
1169        lmt_cstring_start();
1170    }
1171}
1172
1173
1174void tex_lua_string_start(void)
1175{
1176    /*tex Set up |cur_file| and a new level of input: */
1177    tex_begin_file_reading();
1178    lmt_input_state.input_line = 0;
1179    lmt_input_state.cur_input.limit = lmt_input_state.cur_input.start;
1180    /*tex Force line read: */
1181    lmt_input_state.cur_input.loc = lmt_input_state.cur_input.limit + 1;
1182    lmt_input_state.cur_input.name = io_lua_input_code;
1183    lmt_cstring_start();
1184}
1185
1186void tex_any_string_start(char* s)
1187{
1188    /* via terminal emulator */
1189    /*
1190        int len = strlen(s);
1191        if (len > 0 && room_in_buffer(len + 1)) {
1192            fileio_state.io_last = fileio_state.io_first;
1193            strcpy((char *) &fileio_state.io_buffer[fileio_state.io_first], s);
1194            fileio_state.io_last += len;
1195            input_state.cur_input.loc = fileio_state.io_first;
1196            input_state.cur_input.limit = fileio_state.io_last;
1197            fileio_state.io_first = fileio_state.io_last + 1;
1198        }
1199    */
1200    /* via token input emulator */
1201    lmt_cstring_store(s, (int) strlen(s), cat_code_table_par);
1202    tex_begin_file_reading();
1203    lmt_input_state.input_line = 0;
1204    lmt_input_state.cur_input.limit = lmt_input_state.cur_input.start;
1205    lmt_input_state.cur_input.loc = lmt_input_state.cur_input.limit + 1;
1206    lmt_input_state.cur_input.name = io_token_input_code;
1207    lmt_cstring_start();
1208}
1209
1210/*tex a list without ref count*/
1211
1212halfword tex_wrapped_token_list(halfword list)
1213{
1214    halfword head = tex_store_new_token(null, left_brace_token + '{');
1215    halfword tail =  head;
1216    token_link(tail) = token_link(list);
1217    while (token_link(tail)) {
1218        tail = token_link(tail);
1219    }
1220    tail = tex_store_new_token(tail, right_brace_token + '}');
1221    return head;
1222}
1223
1224const char *tex_current_input_file_name(void)
1225{
1226    int level = lmt_input_state.in_stack_data.ptr;
1227    while (level > 0) {
1228        const char *s = lmt_input_state.in_stack[level--].full_source_filename;
1229        if (s) {
1230            return s;
1231        }
1232    }
1233    /*tex old method */
1234    level = lmt_input_state.in_stack_data.ptr;
1235    while (level > 0) {
1236        int t = lmt_input_state.input_stack[level--].name;
1237        if (t >= cs_offset_value) {
1238            return (const char *) str_string(t);
1239        }
1240    }
1241    return NULL;
1242}
1243