toks-ini.lua /size: 9140 b    last modification: 2021-10-28 13:50
1if not modules then modules = { } end modules ['toks-ini'] = {
2    version   = 1.001,
3    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
4    copyright = "PRAGMA ADE / ConTeXt Development Team",
5    license   = "see context related readme files"
6}
7
8tokens = tokens or { }
9
10local tokens     = tokens
11local token      = token -- the built in one
12local next       = next
13local tonumber   = tonumber
14local tostring   = tostring
15local utfchar    = utf.char
16local char       = string.char
17local printtable = table.print
18local concat     = table.concat
19local format     = string.format
20
21local commands  = token.commands()
22tokens.commands = utilities.storage.allocate(table.swapped(commands,commands))
23tokens.values   = { }
24
25local scantoks        = token.scan_toks
26local scanstring      = token.scan_string
27local scanargument    = token.scan_argument
28local scandelimited   = token.scan_delimited
29local scantokenlist   = token.scan_tokenlist or scanstring
30local scaninteger     = token.scan_integer or token.scan_int
31local scancardinal    = token.scan_cardinal
32local scancode        = token.scan_code
33local scantokencode   = token.scan_token_code
34local scandimen       = token.scan_dimen
35local scanglue        = token.scan_glue
36local scanskip        = token.scan_skip
37local scankeyword     = token.scan_keyword
38local scankeywordcs   = token.scan_keyword_cs or scankeyword
39local scantoken       = token.scan_token
40local scanbox         = token.scan_box
41local scanword        = token.scan_word
42local scanletters     = token.scan_letters or scanword -- lmtx
43local scankey         = token.scan_key
44local scanvalue       = token.scan_value
45local scanchar        = token.scan_char
46local scannumber      = token.scan_number -- not defined
47local scancsname      = token.scan_csname
48local scanreal        = token.scan_real
49local scanfloat       = token.scan_float
50local scanluanumber   = token.scan_luanumber   or scanfloat    -- only lmtx
51local scanluainteger  = token.scan_luainteger  or scaninteger  -- only lmtx
52local scanluacardinal = token.scan_luacardinal or scancardinal -- only lmtx
53
54local setmacro        = token.set_macro
55local setchar         = token.set_char
56local setlua          = token.set_lua
57
58local createtoken     = token.create
59local newtoken        = token.new
60local isdefined       = token.is_defined
61local istoken         = token.is_token
62
63tokens.new            = newtoken
64tokens.create         = createtoken
65tokens.istoken        = istoken
66tokens.isdefined      = isdefined
67tokens.defined        = isdefined
68
69local bits = {
70    escape      = 0x00000001, -- 2^00
71    begingroup  = 0x00000002, -- 2^01
72    endgroup    = 0x00000004, -- 2^02
73    mathshift   = 0x00000008, -- 2^03
74    alignment   = 0x00000010, -- 2^04
75    endofline   = 0x00000020, -- 2^05
76    parameter   = 0x00000040, -- 2^06
77    superscript = 0x00000080, -- 2^07
78    subscript   = 0x00000100, -- 2^08
79    ignore      = 0x00000200, -- 2^09
80    space       = 0x00000400, -- 2^10 -- 1024
81    letter      = 0x00000800, -- 2^11
82    other       = 0x00001000, -- 2^12
83    active      = 0x00002000, -- 2^13
84    comment     = 0x00004000, -- 2^14
85    invalid     = 0x00008000, -- 2^15
86    --
87    character   = 0x00001800, -- 2^11 + 2^12
88    whitespace  = 0x00002400, -- 2^13 + 2^10 --    / needs more checking
89    --
90    open        = 0x00000402, -- 2^10 + 2^01 -- space + begingroup
91    close       = 0x00000404, -- 2^10 + 2^02 -- space + endgroup
92}
93
94-- for k, v in next, bits do bits[v] = k end
95
96tokens.bits = bits
97
98-- words are space or \relax terminated and the trailing space is gobbled; a word
99-- can contain any non-space letter/other (see archive for implementation in lua)
100
101if not scannumber then
102
103    scannumber = function(base)
104        local s = scanword()
105        if not s then
106            return nil
107        elseif base then
108            return tonumber(s,base)
109        else
110            return tonumber(s)
111        end
112    end
113
114end
115
116local function scanboolean()
117    local kw = scanword()
118    if kw == "true" then
119        return true
120    elseif kw == "false" then
121        return false
122    else
123        return nil
124    end
125end
126
127local function scanverbatim()
128    return scanargument(false)
129end
130
131if not scanbox then
132
133    local scanlist = token.scan_list
134    local putnext  = token.put_next
135
136    scanbox = function(s)
137        if s == "hbox" or s == "vbox" or s == "vtop" then
138            putnext(createtoken(s))
139        end
140        return scanlist()
141    end
142
143    token.scanbox = scanbox
144
145end
146
147tokens.scanners = { -- these expand
148    token          = scantoken,
149    toks           = scantoks,
150    tokens         = scantoks,
151    box            = scanbox,
152    hbox           = function() return scanbox("hbox") end,
153    vbox           = function() return scanbox("vbox") end,
154    vtop           = function() return scanbox("vtop") end,
155    dimen          = scandimen,
156    dimension      = scandimen,
157    glue           = scanglue,
158    gluevalues     = function() return scanglue(false,false,true) end,
159    gluespec       = scanskip,
160    integer        = scaninteger,
161    cardinal       = scancardinal,
162    real           = scanreal,
163    float          = scanfloat,
164    luanumber      = scanluanumber,
165    luainteger     = scanluainteger,
166    luacardinal    = scanluacardinal,
167    count          = scaninteger,
168    string         = scanstring,
169    argument       = scanargument,
170    delimited      = scandelimited,
171    tokenlist      = scantokenlist,
172    verbatim       = scanverbatim, -- detokenize
173    code           = scancode,
174    tokencode      = scantokencode,
175    word           = scanword,
176    letters        = scanletters,
177    key            = scankey,
178    value          = scanvalue,
179    char           = scanchar,
180    number         = scannumber,
181    boolean        = scanboolean,
182    keyword        = scankeyword,
183    keywordcs      = scankeywordcs,
184    csname         = scancsname,
185
186    next           = token.scan_next,
187    nextexpanded   = token.scan_next_expanded,
188
189    peek           = token.peek_next,
190    peekexpanded   = token.peek_next_expanded,
191    peekchar       = token.peek_next_char,
192
193    skip           = token.skip_next,
194    skipexpanded   = token.skip_next_expanded,
195
196    cmdchr         = token.scan_cmdchr,
197    cmdchrexpanded = token.scan_cmdchr_expanded,
198
199    ischar         = token.is_next_char,
200}
201
202tokens.getters = { -- these don't expand
203    meaning = token.get_meaning,
204    macro   = token.get_macro,
205    token   = token.scan_next or token.get_next, -- not here, use scanners.next or token
206    cstoken = token.get_cstoken,
207    count   = tex.getcount,
208    dimen   = tex.getdimen,
209    skip    = tex.getglue,
210    glue    = tex.getglue,
211    skip    = tex.getmuglue,
212    glue    = tex.getmuglue,
213    box     = tex.getbox,
214}
215
216tokens.setters = {
217    macro = setmacro,
218    char  = setchar,
219    lua   = setlua,
220    count = tex.setcount,
221    dimen = tex.setdimen,
222    skip  = tex.setglue,
223    glue  = tex.setglue,
224    skip  = tex.setmuglue,
225    glue  = tex.setmuglue,
226    box   = tex.setbox,
227}
228
229tokens.accessors = {
230    command    = token.get_command,
231    cmd        = token.get_command,
232    cmdname    = token.get_cmdname,
233    name       = token.get_cmdname,
234    csname     = token.get_csname,
235    index      = token.get_index,
236    active     = token.get_active,
237    frozen     = token.get_frozen,
238    protected  = token.get_protected,
239    expandable = token.get_protected,
240    user       = token.get_user,
241    cmdchrcs   = token.get_cmdchrcs,
242    active     = token.get_active,
243    range      = token.get_range,
244}
245
246if setinspector then
247
248    local simple = { letter = "letter", other_char = "other" }
249
250    local astable = function(t)
251        if t and istoken(t) then
252            local cmdname = t.cmdname
253            local simple  = simple[cmdname]
254            if simple then
255                return {
256                    id         = t.id,
257                    category   = simple,
258                    character  = utfchar(t.mode) or nil,
259                }
260            else
261                return {
262                    command    = t.command,
263                    id         = t.id,
264                    tok        = t.tok,
265                    csname     = t.csname,
266                    active     = t.active,
267                    expandable = t.expandable,
268                    protected  = t.protected,
269                    frozen     = t.frozen,
270                    mode       = t.mode,
271                    index      = t.index,
272                    user       = t.user,
273                    cmdname    = cmdname,
274                }
275            end
276        end
277    end
278
279    tokens.astable = astable
280
281    setinspector("token",function(v) local t = astable(v) if t then printtable(t,tostring(v)) return true end end)
282
283end
284
285tokens.cache = table.setmetatableindex(function(t,k)
286    if not isdefined(k) then
287        setmacro(k,"","global")
288    end
289    local v = createtoken(k)
290    t[k] = v
291    return v
292end)
293
294if LUATEXVERSION < 114 then
295
296    local d = tokens.defined
297    local c = tokens.create
298
299    function tokens.defined(s,b)
300        if b then
301            return d(s)
302        else
303            return c(s).cmd_name == "undefined_cmd"
304        end
305    end
306
307end
308