1if not modules then modules = { } end modules ['toks-ini'] = {
2 version = 1.001,
3 author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
4 copyright = "PRAGMA ADE / ConTeXt Development Team",
5 license = "see context related readme files"
6}
7
8tokens = tokens or { }
9
10local tokens = tokens
11local token = token
12local next = next
13local tonumber = tonumber
14local tostring = tostring
15local utfchar = utf.char
16local char = string.char
17local printtable = table.print
18local concat = table.concat
19local format = string.format
20
21local commands = token.getcommandvalues()
22local values = token.getfunctionvalues()
23values.dimen = values.dimension
24values.count = values.integer
25tokens.values = utilities.storage.allocate(table.swapped(values, values))
26tokens.commands = utilities.storage.allocate(table.swapped(commands,commands))
27tokens.cache = utilities.storage.allocate()
28
29local scantoks = token.scantoks
30local scanstring = token.scanstring
31local scanargument = token.scanargument
32local scandelimited = token.scandelimited
33local scantokenlist = token.scantokenlist
34local scaninteger = token.scaninteger
35local scancardinal = token.scancardinal
36local scancode = token.scancode
37local scantokencode = token.scantokencode
38local scandimen = token.scandimen
39local scanglue = token.scanglue
40local scanskip = token.scanskip
41local scankeyword = token.scankeyword
42local scankeywordcs = token.scankeywordcs
43local scantoken = token.scantoken
44local scanbox = token.scanbox
45local scanword = token.scanword
46local scanletters = token.scanletters
47local scankey = token.scankey
48local scanvalue = token.scanvalue
49local scanchar = token.scanchar
50local scancsname = token.scancsname
51local scannextchar = token.scannextchar
52local scanposit = token.scanposit
53local scanreal = token.scanreal
54local scanfloat = token.scanfloat
55local scanluanumber = token.scanluanumber
56local scanluainteger = token.scanluainteger
57local scanluacardinal = token.scanluacardinal
58local scanintegerargument = token.scanintegerargument
59local scandimenargument = token.scandimenargument
60local scandetokened = token.scandetokened
61local scantokenstring = token.scantokenstring
62
63local scannumber = token.scannumber
64local scanboolean = token.scanboolean
65
66local setmacro = token.setmacro
67local setchar = token.setchar
68local setlua = token.setlua
69
70local createtoken = token.create
71local newtoken = token.new
72local isdefined = token.isdefined
73local istoken = token.istoken
74
75tokens.new = newtoken
76tokens.create = createtoken
77tokens.istoken = istoken
78tokens.isdefined = isdefined
79tokens.defined = isdefined
80
81tokens.gobble = token.gobble
82
83tokens.getinteger = token.getinteger
84tokens.setinteger = token.setinteger
85
86local bits = {
87 escape = 0x00000001,
88 begingroup = 0x00000002,
89 endgroup = 0x00000004,
90 mathshift = 0x00000008,
91 alignment = 0x00000010,
92 endofline = 0x00000020,
93 parameter = 0x00000040,
94 superscript = 0x00000080,
95 subscript = 0x00000100,
96 ignore = 0x00000200,
97 space = 0x00000400,
98 letter = 0x00000800,
99 other = 0x00001000,
100 active = 0x00002000,
101 comment = 0x00004000,
102 invalid = 0x00008000,
103
104 character = 0x00001800,
105 whitespace = 0x00002400,
106
107 open = 0x00000402,
108 close = 0x00000404,
109}
110
111
112
113tokens.bits = bits
114
115
116
117
118if not scannumber then
119
120 scannumber = function(base)
121 local s = scanword()
122 if not s then
123 return nil
124 elseif base then
125 return tonumber(s,base)
126 else
127 return tonumber(s)
128 end
129 end
130
131end
132
133if not scanboolean then
134
135 scanboolean = function()
136 local kw = scanword()
137 if kw == "true" then
138 return true
139 elseif kw == "false" then
140 return false
141 else
142 return nil
143 end
144 end
145
146end
147
148local function scanverbatim()
149 return scanargument(false)
150end
151
152tokens.scanners = {
153 token = scantoken,
154 toks = scantoks,
155 tokens = scantoks,
156 box = scanbox,
157 hbox = function() return scanbox("hbox") end,
158 vbox = function() return scanbox("vbox") end,
159 vtop = function() return scanbox("vtop") end,
160 dimen = scandimen,
161 dimension = scandimen,
162 glue = scanglue,
163 gluevalues = function() return scanglue(false,false,true) end,
164 gluespec = scanskip,
165 integer = scaninteger,
166 cardinal = scancardinal,
167 posit = scanposit,
168 real = scanreal,
169 float = scanfloat,
170 luanumber = scanluanumber,
171 luainteger = scanluainteger,
172 luacardinal = scanluacardinal,
173 count = scaninteger,
174 string = scanstring,
175 argument = scanargument,
176 delimited = scandelimited,
177 tokenlist = scantokenlist,
178 verbatim = scanverbatim,
179 code = scancode,
180 tokencode = scantokencode,
181 word = scanword,
182 letters = scanletters,
183 key = scankey,
184 value = scanvalue,
185 char = scanchar,
186 number = scannumber,
187 boolean = scanboolean,
188 keyword = scankeyword,
189 keywordcs = scankeywordcs,
190 csname = scancsname,
191 nextchar = scannextchar,
192 detokened = scandetokened,
193 tokenstring = scantokenstring,
194
195 next = token.scannext,
196 nextexpanded = token.scannextexpanded,
197
198 peek = token.peeknext,
199 peekexpanded = token.peeknextexpanded,
200 peekchar = token.peeknextchar,
201
202 skip = token.skipnext,
203 skipexpanded = token.skipnextexpanded,
204
205 cmdchr = token.scancmdchr,
206 cmdchrexpanded = token.scancmdchrexpanded,
207
208 ischar = token.isnextchar,
209
210 integerargument = scanintegerargument,
211 dimenargument = scandimenargument,
212}
213
214tokens.getters = {
215 meaning = token.getmeaning,
216 macro = token.getmacro,
217 token = token.scannext,
218 cstoken = token.getcstoken,
219 count = tex.getcount,
220 dimen = tex.getdimen,
221 skip = tex.getglue,
222 glue = tex.getglue,
223 skip = tex.getmuglue,
224 glue = tex.getmuglue,
225 box = tex.getbox,
226}
227
228tokens.setters = {
229 macro = setmacro,
230 char = setchar,
231 lua = setlua,
232 count = tex.setcount,
233 dimen = tex.setdimen,
234 skip = tex.setglue,
235 glue = tex.setglue,
236 skip = tex.setmuglue,
237 glue = tex.setmuglue,
238 box = tex.setbox,
239}
240
241tokens.accessors = {
242 command = token.getcommand,
243 cmd = token.getcommand,
244 cmdname = token.getcmdname,
245 name = token.getcmdname,
246 csname = token.getcsname,
247 index = token.getindex,
248 active = token.getactive,
249 frozen = token.getfrozen,
250 protected = token.getprotected,
251 expandable = token.getprotected,
252 user = token.getuser,
253 cmdchrcs = token.getcmdchrcs,
254 active = token.getactive,
255 range = token.getrange,
256}
257
258if setinspector then
259
260 local simple = { letter = "letter", other_char = "other" }
261
262 local astable = function(t)
263 if t and istoken(t) then
264 local cmdname = t.cmdname
265 local simple = simple[cmdname]
266 if simple then
267 return {
268 id = t.id,
269 category = simple,
270 character = utfchar(t.index) or nil,
271 }
272 else
273 return {
274 id = t.id,
275 command = t.command,
276 index = t.index,
277 csname = t.csname,
278 cmdname = cmdname,
279 active = t.active,
280 expandable = t.expandable,
281 frozen = t.frozen,
282 tolerant = t.tolerant,
283 protected = t.protected,
284 primitive = t.primitive,
285 permanent = t.permanent,
286 noaligned = t.noaligned,
287 instance = t.instance,
288 immutable = t.imutable,
289 user = t.user,
290 }
291 end
292 end
293 end
294
295 tokens.astable = astable
296
297 setinspector("token",function(v) local t = astable(v) if t then printtable(t,tostring(v)) return true end end)
298
299end
300
301table.setmetatableindex(tokens.cache, function(t,k)
302 if not isdefined(k) then
303 setmacro(k,"","global")
304 end
305 local v = createtoken(k)
306 t[k] = v
307 return v
308end)
309
310
311
312
313token.is_token = token.istoken
314token.is_defined = token.isdefined
315token.scan_next = token.scannext
316token.scan_next_expanded = token.scannextexpanded
317token.scan_next_char = token.scannextchar
318token.skip_next = token.skipnext
319token.skip_next_expanded = token.skipnextexpanded
320token.peek_next = token.peeknext
321token.peek_next_expanded = token.peeknextexpanded
322token.peek_next_char = token.peeknextchar
323token.scan_cmdchr = token.scancmdchr
324token.scan_cmdchr_expanded = token.scancmdchrexpanded
325token.scan_keyword = token.scankeyword
326token.scan_keyword_cs = token.scankeywordcs
327token.scan_integer = token.scaninteger
328token.scan_integer_argument = token.scanintegerargument
329token.scan_dimen_argument = token.scandimenargument
330token.scan_cardinal = token.scancardinal
331token.scan_float = token.scanfloat
332token.scan_real = token.scanreal
333token.scan_luanumber = token.scanluanumber
334token.scan_luainteger = token.scanluainteger
335token.scan_luacardinal = token.scanluacardinal
336token.scan_scale = token.scanscale
337token.scan_dimen = token.scandimen
338token.scan_skip = token.scanskip
339token.scan_glue = token.scanglue
340token.scan_toks = token.scantoks
341token.scan_tokenlist = token.scantokenlist
342token.scan_code = token.scancode
343token.scan_token_code = token.scantokencode
344token.scan_string = token.scanstring
345token.scan_argument = token.scanargument
346token.scan_delimited = token.scandelimited
347token.scan_word = token.scanword
348token.scan_letters = token.scanletters
349token.scan_key = token.scankey
350token.scan_value = token.scanvalue
351token.scan_char = token.scanchar
352token.scan_csname = token.scancsname
353token.scan_token = token.scantoken
354token.scan_box = token.scanbox
355token.is_next_char = token.isnextchar
356token.put_next = token.putnext
357token.put_back = token.putback
358token.get_command = token.getcommand
359token.get_index = token.getindex
360token.get_range = token.getrange
361token.get_cmdname = token.getcmdname
362token.get_csname = token.getcsname
363token.get_id = token.getid
364token.get_tok = token.gettok
365token.get_active = token.getactive
366token.get_expandable = token.getexpandable
367token.get_protected = token.getprotected
368token.get_frozen = token.getfrozen
369token.get_tolerant = token.gettolerant
370token.get_noaligned = token.getnoaligned
371token.get_primitive = token.getprimitive
372token.get_permanent = token.getpermanent
373token.get_immutable = token.getimmutable
374token.get_instance = token.getinstance
375token.get_flags = token.getflags
376token.get_parameters = token.getparameters
377token.get_macro = token.getmacro
378token.get_meaning = token.getmeaning
379token.get_cmdchrcs = token.getcmdchrcs
380token.get_cstoken = token.getcstoken
381token.get_fields = token.getfields
382token.set_macro = token.setmacro
383token.undefine_macro = token.undefinemacro
384token.expand_macro = token.expandmacro
385token.set_char = token.setchar
386token.set_lua = token.setlua
387token.set_integer = token.setinteger
388token.get_integer = token.getinteger
389token.set_dimension = token.setdimension
390token.get_dimension = token.getdimension
391token.gobble_integer = token.gobbleinteger
392token.gobble_dimen = token.gobbledimen
393token.future_expand = token.futureexpand
394token.push_macro = token.pushmacro
395token.pop_macro = token.popmacro
396token.save_lua = token.savelua
397 |