s-system-tokens.lmt /size: 8000 b    last modification: 2023-12-21 09:45
1if not modules then modules = { } end modules['s-system-tokens'] = {
2    version   = 1.001,
3    comment   = "companion to s-system-tokens.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9local context = context
10local ctx_NC  = context.NC
11local ctx_BC  = context.BC
12local ctx_NR  = context.NR
13local ctx_FL  = context.FL
14local ctx_ML  = context.ML
15local ctx_LL  = context.LL
16local gsub    = string.gsub
17local find    = string.find
18local concat  = table.concat
19
20moduledata.system        = moduledata.system        or { }
21moduledata.system.tokens = moduledata.system.tokens or { }
22
23local getrange    = token.get_range
24local getcmdchrcs = token.get_cmdchrcs
25local whatever    = { [1] = true, [4] = true, [8] = true }
26
27local collected = nil -- at some point we might have this one preloaded
28
29function moduledata.system.tokens.collect()
30
31    if not collected then
32
33        local allcommands = tokens.commands
34        local primitives  = tex.primitives()
35              collected = { }
36
37        -- 0 and 110 missing
38
39        for k, v in table.sortedhash(allcommands) do
40            if type(k) == "number" then
41                local codes = { }
42                local kind, min, max, fixedvalue = getrange(k)
43                if min and whatever[kind] and max >= 0 and max <= 256 then
44                     for i=min,max do
45                         codes[i] = false
46                     end
47                end
48                collected[k] = codes
49                collected[v] = codes
50            end
51        end
52
53        for i=1,#primitives do
54            local prm = primitives[i]
55            local cmd, chr = getcmdchrcs("normal"..prm)
56            local codes = collected[cmd]
57            if codes and codes[chr] == false then
58                codes[chr] = prm
59                codes[prm] = chr
60            else
61             -- print(cmd,chr)
62            end
63        end
64
65        collected.undefined_cs = nil
66
67     -- table.save("whatever.lua",collected)
68     -- local p = token.getprimitives()
69     -- table.sort(p,function(a,b) if a[1] == b[1] then return a[2] < b[2] else return a[1] < b[1] end end)
70     -- table.save("moreever.lua",p)
71
72     --  local p = token.getprimitives()
73     --  for i=1,#p do
74     --     local t = p[i]
75     --     pi[i] = { t[1], t[2], t[3], getrange(t[1]) }
76     --  end
77
78     -- inspect(collected)
79
80    end
81
82    return collected
83
84end
85
86function moduledata.system.tokens.showlist()
87
88    local l = tokens.commands
89    local t = moduledata.system.tokens.collect()
90
91    context.starttabulate { "|cT|rT|lT|rT|rT|pTA{flushleft,verytolerant,stretch}|" }
92    ctx_FL()
93    ctx_BC() context("")
94    ctx_BC() context("cmd")
95    ctx_BC() context("name")
96    ctx_BC() context("min")
97    ctx_BC() context("max")
98    ctx_BC() context("default or subcommands")
99    ctx_BC() ctx_NR()
100    ctx_ML()
101    for i=0,#l do
102        local kind, min, max, fixedvalue = getrange(i)
103        local what = whatever[kind]
104        ctx_NC() context(kind)
105        ctx_NC() context(i)
106        ctx_NC() context(l[i])
107        ctx_NC() if min then if what or min == 0 then context(min) else context("-0x%X",-min) end end
108        ctx_NC() if max then if what or max == 0 then context(max) else context("0x%X",max) end end
109        ctx_NC()
110        if min and what and max >= 0 and max <= 256 then
111            local l = { }
112            local c = t[i]
113            if c then
114                for j=min, max do
115                    local s = c[j]
116                    if s == " " then
117                        s = "<space>"
118                    elseif not s then
119                        s = "<unavailable>"
120                    end
121                    l[#l+1] = j .. "=" .. s
122                end
123                if (#l > 0) then
124                    context(table.concat(l," "))
125                elseif fixedvalue ~= 0 then
126                    context("0x%X",fixedvalue)
127                end
128            else
129                print("weird")
130            end
131        elseif fixedvalue and fixedvalue ~= 0 then
132            context("0x%X",fixedvalue)
133        end
134        ctx_NC() ctx_NR()
135    end
136    ctx_LL()
137    context.stoptabulate()
138
139end
140
141function moduledata.system.tokens.table(t)
142    local t = t or token.peek_next() -- local t = token.scan_next() token.put_back(t)
143    local n = ""
144    local w = ""
145    local c = t.cmdname
146    if c == "left_brace" then
147        w = "given token list"
148        t = token.scan_toks(false)
149    elseif c == "register_toks" then
150        token.scan_next()
151        w = "token register"
152        n = t.csname or t.index
153        t = tex.gettoks(n,true)
154    elseif c == "internal_toks" then
155        token.scan_next()
156        w = "internal token variable"
157        n = t.csname or t.index
158        t = tex.gettoks(n,true)
159    else
160        n = token.scan_csname()
161        local r = { }
162        local m = token.get_meaning(n,true)
163        if t.frozen    then r[#r+1] = "frozen"    end
164        if t.permanent then r[#r+1] = "permanent" end
165        if t.immutable then r[#r+1] = "immutable" end
166        if t.primitive then r[#r+1] = "primitive" end
167        if t.mutable   then r[#r+1] = "mutable"   end
168        if t.noaligned then r[#r+1] = "noaligned" end
169        if t.instance  then r[#r+1] = "instance"  end
170        if t.tolerant  then r[#r+1] = "tolerant"  end
171        if t.protected then r[#r+1] = "protected" end
172        r[#r+1] = "control sequence"
173        if type(m) == "table" then
174            t = m
175        else
176            t = { t }
177        end
178        w = concat(r, " ")
179    end
180    if type(t) == "table" and #t > 0 then
181        context.starttabulate { "|l|r|r|l|c|l|l|" }
182        ctx_FL()
183        ctx_NC() context.formatted.rlap("\\bold %s: %s",w,n)
184        ctx_NC() ctx_NC() ctx_NC() ctx_NC() ctx_NC() ctx_NC() ctx_NC()
185        ctx_NC() ctx_NR()
186        ctx_ML()
187        for i=1,#t do
188            local ti = t[i]
189            local cs = ti.csname
190            local id = ti.id
191            local ix = ti.index
192            local cd = ti.command
193            local cn = gsub(ti.cmdname,"_"," ")
194            ctx_NC() context(id)
195            ctx_NC() context(cd)
196            ctx_NC() context("%3i",ix)
197            ctx_NC() context(cn)
198            if cs then
199                ctx_NC()
200                ctx_NC()
201                ctx_NC() context(cs)
202                ctx_NC() ctx_NR()
203            elseif cn == "letter" or cn == "other char" then
204                ctx_NC() context.char(ix)
205                ctx_NC() context("%U",ix)
206                ctx_NC()
207                ctx_NC() ctx_NR()
208            elseif cn == "match" then
209                ctx_NC()
210                ctx_NC()
211                    if ix == 32 then context("optional spacer")     -- space
212                elseif ix == 43 then context("keep braces")         -- "+"
213                elseif ix == 45 then context("thrash")              -- "-"
214                elseif ix == 61 then context("mandate braces")      -- "="
215                elseif ix == 94 then context("keep spaces")         -- "^"
216                elseif ix == 95 then context("keep mandate braces") -- "_"
217                else                 context("argument %c",ix)
218                end
219                ctx_NC()
220                ctx_NC() ctx_NR()
221            else
222                ctx_NC()
223                ctx_NC()
224                ctx_NC()
225                ctx_NC() ctx_NR()
226                if cn == "end match" then
227                    context.ML()
228                end
229            end
230        end
231        context.LL()
232        context.stoptabulate()
233    else
234        context.starttabulate { "|l|" }
235        ctx_FL()
236        ctx_NC() context("%s: %s",w,n)
237        ctx_NC() ctx_NR()
238        ctx_ML()
239        ctx_NC() context("<no tokens>")
240        ctx_NC() ctx_NR()
241        ctx_LL()
242        context.stoptabulate()
243    end
244end
245
246interfaces.implement {
247    name      = "luatokentable",
248    public    = true,
249    protected = true,
250    actions   = moduledata.system.tokens.table,
251}
252