toks-ini.lua /size: 10 Kb    last modification: 2020-07-01 14:35
1
if
not
modules
then
modules
=
{
}
end
modules
[
'
toks-ini
'
]
=
{
2
version
=
1
.
001
,
3
author
=
"
Hans Hagen, PRAGMA-ADE, Hasselt NL
"
,
4
copyright
=
"
PRAGMA ADE / ConTeXt Development Team
"
,
5
license
=
"
see context related readme files
"
6
}
7 8
tokens
=
tokens
or
{
}
9 10
local
tokens
=
tokens
11
local
token
=
token
-- the built in one
12
local
next
=
next
13
local
tonumber
=
tonumber
14
local
tostring
=
tostring
15
local
utfchar
=
utf
.
char
16
local
char
=
string
.
char
17
local
printtable
=
table
.
print
18
local
concat
=
table
.
concat
19
local
format
=
string
.
format
20 21
if
token
.
values
then
22 23
local
commands
=
token
.
values
(
"
command
"
)
24
local
values
=
token
.
values
(
"
value
"
)
25
values
.
dimen
=
values
.
dimension
26
values
.
count
=
values
.
integer
27
tokens
.
values
=
utilities
.
storage
.
allocate
(
table
.
swapped
(
values
,
values
)
)
28
tokens
.
commands
=
utilities
.
storage
.
allocate
(
table
.
swapped
(
commands
,
commands
)
)
29 30
elseif
token
.
commands
then
31 32
local
commands
=
token
.
commands
(
)
33
tokens
.
commands
=
utilities
.
storage
.
allocate
(
table
.
swapped
(
commands
,
commands
)
)
34
tokens
.
values
=
{
}
35 36
else
37 38
tokens
.
commands
=
{
}
39
tokens
.
values
=
{
}
40 41
end
42 43
local
scan_toks
=
token
.
scan_toks
44
local
scan_string
=
token
.
scan_string
45
local
scan_argument
=
token
.
scan_argument
46
local
scan_delimited
=
token
.
scan_delimited
47
local
scan_tokenlist
=
token
.
scan_tokenlist
48
local
scan_int
=
token
.
scan_int
49
local
scan_code
=
token
.
scan_code
50
local
scan_token_code
=
token
.
scan_token_code
51
local
scan_dimen
=
token
.
scan_dimen
52
local
scan_glue
=
token
.
scan_glue
53
local
scan_gluespec
=
token
.
scan_skip
54
local
scan_keyword
=
token
.
scan_keyword
55
local
scan_keyword_cs
=
token
.
scan_keyword_cs
or
scan_keyword
56
local
scan_token
=
token
.
scan_token
57
local
scan_box
=
token
.
scan_box
58
local
scan_word
=
token
.
scan_word
59
local
scan_letters
=
token
.
scan_letters
or
scan_word
-- lmtx
60
local
scan_key
=
token
.
scan_key
61
local
scan_value
=
token
.
scan_value
62
local
scan_char
=
token
.
scan_char
63
local
scan_number
=
token
.
scan_number
-- not defined
64
local
scan_csname
=
token
.
scan_csname
65
local
scan_real
=
token
.
scan_real
66
local
scan_float
=
token
.
scan_float
67
local
scan_luanumber
=
token
.
scan_luanumber
or
scan_float
-- only lmtx
68
local
scan_luainteger
=
token
.
scan_luainteger
or
scan_int
-- only lmtx
69
local
scan_luacardinal
=
token
.
scan_luacardinal
or
scan_int
-- only lmtx
70 71
local
get_next
=
token
.
get_next
72
----- get_next_token = token.get_next_token
73
local
skip_next
=
token
.
skip_next
74
local
peek_next_char
=
token
.
peek_next_char
75
local
is_next_char
=
token
.
is_next_char
76 77
local
set_macro
=
token
.
set_macro
78
local
get_macro
=
token
.
get_macro
79
local
get_meaning
=
token
.
get_meaning
80
local
get_cmdname
=
token
.
get_cmdname
81
local
set_char
=
token
.
set_char
82
local
set_lua
=
token
.
set_lua
83 84
local
create_token
=
token
.
create
85
local
new_token
=
token
.
new
86
local
is_defined
=
token
.
is_defined
87
local
is_token
=
token
.
is_token
88 89
tokens
.
new
=
new_token
90
tokens
.
create
=
create_token
91
tokens
.
istoken
=
is_token
92
tokens
.
isdefined
=
is_defined
93
tokens
.
defined
=
is_defined
94 95
tokens
.
getdata
=
token
.
get_data
-- only lmtx
96
tokens
.
setdata
=
token
.
set_data
-- only lmtx
97 98
local
bits
=
{
99
escape
=
0x00000001
,
-- 2^00
100
begingroup
=
0x00000002
,
-- 2^01
101
endgroup
=
0x00000004
,
-- 2^02
102
mathshift
=
0x00000008
,
-- 2^03
103
alignment
=
0x00000010
,
-- 2^04
104
endofline
=
0x00000020
,
-- 2^05
105
parameter
=
0x00000040
,
-- 2^06
106
superscript
=
0x00000080
,
-- 2^07
107
subscript
=
0x00000100
,
-- 2^08
108
ignore
=
0x00000200
,
-- 2^09
109
space
=
0x00000400
,
-- 2^10 -- 1024
110
letter
=
0x00000800
,
-- 2^11
111
other
=
0x00001000
,
-- 2^12
112
active
=
0x00002000
,
-- 2^13
113
comment
=
0x00004000
,
-- 2^14
114
invalid
=
0x00008000
,
-- 2^15
115
--
116
character
=
0x00001800
,
-- 2^11 + 2^12
117
whitespace
=
0x00002400
,
-- 2^13 + 2^10 -- / needs more checking
118
--
119
open
=
0x00000402
,
-- 2^10 + 2^01 -- space + begingroup
120
close
=
0x00000404
,
-- 2^10 + 2^02 -- space + endgroup
121
}
122 123
-- for k, v in next, bits do bits[v] = k end
124 125
tokens
.
bits
=
bits
126 127
-- words are space or \relax terminated and the trailing space is gobbled; a word
128
-- can contain any non-space letter/other (see archive for implementation in lua)
129 130
if
not
scan_number
then
131 132
scan_number
=
function
(
base
)
133
local
s
=
scan_word
(
)
134
if
not
s
then
135
return
nil
136
elseif
base
then
137
return
tonumber
(
s
,
base
)
138
else
139
return
tonumber
(
s
)
140
end
141
end
142 143
end
144 145
local
function
scan_boolean
(
)
146
local
kw
=
scan_word
(
)
147
if
kw
=
=
"
true
"
then
148
return
true
149
elseif
kw
=
=
"
false
"
then
150
return
false
151
else
152
return
nil
153
end
154
end
155 156
local
function
scan_verbatim
(
)
157
return
scan_argument
(
false
)
158
end
159 160
if
not
scan_box
then
161 162
local
scan_list
=
token
.
scan_list
163
local
put_next
=
token
.
put_next
164 165
scan_box
=
function
(
s
)
166
if
s
=
=
"
hbox
"
or
s
=
=
"
vbox
"
or
s
=
=
"
vtop
"
then
167
put_next
(
create_token
(
s
)
)
168
end
169
end
170 171
token
.
scan_box
=
scan_box
172 173
end
174 175
tokens
.
scanners
=
{
-- these expand
176
token
=
scan_token
,
177
toks
=
scan_toks
,
178
tokens
=
scan_toks
,
179
box
=
scan_box
,
180
hbox
=
function
(
)
return
scan_box
(
"
hbox
"
)
end
,
181
vbox
=
function
(
)
return
scan_box
(
"
vbox
"
)
end
,
182
vtop
=
function
(
)
return
scan_box
(
"
vtop
"
)
end
,
183
dimen
=
scan_dimen
,
184
dimension
=
scan_dimen
,
185
glue
=
scan_glue
,
186
gluespec
=
scan_gluespec
,
187
integer
=
scan_int
,
188
real
=
scan_real
,
189
float
=
scan_float
,
190
luanumber
=
scan_luanumber
,
191
luainteger
=
scan_luainteger
,
192
luacardinal
=
scan_luacardinal
,
193
count
=
scan_int
,
194
string
=
scan_string
,
195
argument
=
scan_argument
,
196
delimited
=
scan_delimited
,
197
tokenlist
=
scan_tokenlist
,
198
verbatim
=
scan_verbatim
,
-- detokenize
199
code
=
scan_code
,
200
tokencode
=
scan_token_code
,
201
word
=
scan_word
,
202
letters
=
scan_letters
,
203
key
=
scan_key
,
204
value
=
scan_value
,
205
char
=
scan_char
,
206
number
=
scan_number
,
207
boolean
=
scan_boolean
,
208
keyword
=
scan_keyword
,
209
keywordcs
=
scan_keyword_cs
,
210
csname
=
scan_csname
,
211
peek
=
peek_next_char
,
212
skip
=
skip_next
,
213
ischar
=
is_next_char
,
214
}
215 216
tokens
.
getters
=
{
-- these don't expand
217
meaning
=
get_meaning
,
218
macro
=
get_macro
,
219
token
=
get_next
,
220
count
=
tex
.
getcount
,
221
dimen
=
tex
.
getdimen
,
222
skip
=
tex
.
getglue
,
223
glue
=
tex
.
getglue
,
224
skip
=
tex
.
getmuglue
,
225
glue
=
tex
.
getmuglue
,
226
box
=
tex
.
getbox
,
227
}
228 229
tokens
.
setters
=
{
230
macro
=
set_macro
,
231
char
=
set_char
,
232
lua
=
set_lua
,
233
count
=
tex
.
setcount
,
234
dimen
=
tex
.
setdimen
,
235
skip
=
tex
.
setglue
,
236
glue
=
tex
.
setglue
,
237
skip
=
tex
.
setmuglue
,
238
glue
=
tex
.
setmuglue
,
239
box
=
tex
.
setbox
,
240
}
241 242
-- static int run_scan_token(lua_State * L)
243
-- {
244
-- saved_tex_scanner texstate;
245
-- save_tex_scanner(texstate);
246
-- get_x_token();
247
-- make_new_token(L, cur_cmd, cur_chr, cur_cs);
248
-- unsave_tex_scanner(texstate);
249
-- return 1;
250
-- }
251
--
252
-- static int run_get_future(lua_State * L)
253
-- {
254
-- /* saved_tex_scanner texstate; */
255
-- /* save_tex_scanner(texstate); */
256
-- get_token();
257
-- make_new_token(L, cur_cmd, cur_chr, cur_cs);
258
-- back_input();
259
-- /* unsave_tex_scanner(texstate); */
260
-- return 1;
261
-- }
262 263
if
setinspector
then
264 265
local
simple
=
{
letter
=
"
letter
"
,
other_char
=
"
other
"
}
266 267
local
astable
=
CONTEXTLMTXMODE
=
=
0
and
268 269
function
(
t
)
270
if
t
and
is_token
(
t
)
then
271
local
cmdname
=
t
.
cmdname
272
local
simple
=
simple
[
cmdname
]
273
if
simple
then
274
return
{
275
id
=
t
.
id
,
276
category
=
simple
,
277
character
=
utfchar
(
t
.
mode
)
or
nil
,
278
}
279
else
280
return
{
281
command
=
t
.
command
,
282
id
=
t
.
id
,
283
tok
=
t
.
tok
,
284
csname
=
t
.
csname
,
285
active
=
t
.
active
,
286
expandable
=
t
.
expandable
,
287
protected
=
t
.
protected
,
288
frozen
=
t
.
frozen
,
289
mode
=
t
.
mode
,
290
index
=
t
.
index
,
291
user
=
t
.
user
,
292
cmdname
=
cmdname
,
293
}
294
end
295
end
296
end
297 298
or
299 300
function
(
t
)
301
if
t
and
is_token
(
t
)
then
302
local
cmdname
=
t
.
cmdname
303
local
simple
=
simple
[
cmdname
]
304
if
simple
then
305
return
{
306
id
=
t
.
id
,
307
category
=
simple
,
308
character
=
utfchar
(
t
.
index
)
or
nil
,
309
}
310
else
311
return
{
312
id
=
t
.
id
,
313
command
=
t
.
command
,
314
index
=
t
.
index
,
315
csname
=
t
.
csname
,
316
cmdname
=
cmdname
,
317
active
=
t
.
active
,
318
expandable
=
t
.
expandable
,
319
protected
=
t
.
protected
,
320
frozen
=
t
.
frozen
,
321
user
=
t
.
user
,
322
}
323
end
324
end
325
end
326 327
tokens
.
astable
=
astable
328 329
setinspector
(
"
token
"
,
function
(
v
)
local
t
=
astable
(
v
)
if
t
then
printtable
(
t
,
tostring
(
v
)
)
return
true
end
end
)
330 331
end
332 333
tokens
.
cache
=
table
.
setmetatableindex
(
function
(
t
,
k
)
334
if
not
is_defined
(
k
)
then
335
set_macro
(
k
,
"
"
,
"
global
"
)
336
end
337
local
v
=
create_token
(
k
)
338
t
[
k
]
=
v
339
return
v
340
end
)
341 342
if
LUATEXVERSION
<
114
and
CONTEXTLMTXMODE
=
=
0
then
343 344
local
d
=
tokens
.
defined
345
local
c
=
tokens
.
create
346 347
function
tokens
.
defined
(
s
,
b
)
348
if
b
then
349
return
d
(
s
)
350
else
351
return
c
(
s
)
.
cmd_name
=
=
"
undefined_cmd
"
352
end
353
end
354 355
end
356