1if not modules then modules = { } end modules ['toks-ini'] = {
2 version = 1.001,
3 author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
4 copyright = "PRAGMA ADE / ConTeXt Development Team",
5 license = "see context related readme files"
6}
7
8tokens = tokens or { }
9
10local tokens = tokens
11local token = token
12local next = next
13local tonumber = tonumber
14local tostring = tostring
15local utfchar = utf.char
16local char = string.char
17local printtable = table.print
18local concat = table.concat
19local format = string.format
20
21local commands = token.getcommandvalues()
22
23local values = token.getfunctionvalues()
24values.dimen = values.dimension
25values.count = values.integer
26
27tokens.values = utilities.storage.allocate(table.swapped(values, values))
28tokens.commands = utilities.storage.allocate(table.swapped(commands,commands))
29tokens.cache = utilities.storage.allocate()
30
31local scantoks = token.scantoks
32local scanstring = token.scanstring
33local scanargument = token.scanargument
34local scandelimited = token.scandelimited
35local scantokenlist = token.scantokenlist
36local scaninteger = token.scaninteger
37local scandimension = token.scandimension
38local scancardinal = token.scancardinal
39local scancode = token.scancode
40local scantokencode = token.scantokencode
41local scanglue = token.scanglue
42local scanskip = token.scanskip
43local scankeyword = token.scankeyword
44local scankeywordcs = token.scankeywordcs
45local scantoken = token.scantoken
46local scanbox = token.scanbox
47local scanword = token.scanword
48local scanletters = token.scanletters
49local scankey = token.scankey
50local scanvalue = token.scanvalue
51local scanchar = token.scanchar
52local scancsname = token.scancsname
53local scannextchar = token.scannextchar
54local scanposit = token.scanposit
55local scanreal = token.scanreal
56local scanfloat = token.scanfloat
57local scanluanumber = token.scanluanumber
58local scanluainteger = token.scanluainteger
59local scanluacardinal = token.scanluacardinal
60local scandetokened = token.scandetokened
61local scantokenstring = token.scantokenstring
62
63local scannumber = token.scannumber
64local scanboolean = token.scanboolean
65
66local setmacro = token.setmacro
67local setlua = token.setlua
68
69local createtoken = token.create
70local newtoken = token.new
71local isdefined = token.isdefined
72local istoken = token.istoken
73
74tokens.new = newtoken
75tokens.create = createtoken
76tokens.istoken = istoken
77tokens.isdefined = isdefined
78tokens.defined = isdefined
79
80tokens.gobble = token.gobble
81
82tokens.getinteger = token.getinteger
83tokens.setinteger = token.setinteger
84
85local bits = {
86 escape = 0x00000001,
87 begingroup = 0x00000002,
88 endgroup = 0x00000004,
89 mathshift = 0x00000008,
90 alignment = 0x00000010,
91 endofline = 0x00000020,
92 parameter = 0x00000040,
93 superscript = 0x00000080,
94 subscript = 0x00000100,
95 ignore = 0x00000200,
96 space = 0x00000400,
97 letter = 0x00000800,
98 other = 0x00001000,
99 active = 0x00002000,
100 comment = 0x00004000,
101 invalid = 0x00008000,
102
103 character = 0x00001800,
104 whitespace = 0x00002400,
105
106 open = 0x00000402,
107 close = 0x00000404,
108}
109
110
111
112tokens.bits = bits
113
114
115
116
117if not scannumber then
118
119 scannumber = function(base)
120 local s = scanword()
121 if not s then
122 return nil
123 elseif base then
124 return tonumber(s,base)
125 else
126 return tonumber(s)
127 end
128 end
129
130end
131
132if not scanboolean then
133
134 scanboolean = function()
135 local kw = scanword()
136 if kw == "true" then
137 return true
138 elseif kw == "false" then
139 return false
140 else
141 return nil
142 end
143 end
144
145end
146
147local function scanverbatim()
148 return scanargument(false)
149end
150
151tokens.scanners = {
152 token = scantoken,
153 toks = scantoks,
154 tokens = scantoks,
155 box = scanbox,
156 hbox = function() return scanbox("hbox") end,
157 vbox = function() return scanbox("vbox") end,
158 vtop = function() return scanbox("vtop") end,
159 dimen = scandimension,
160 dimension = scandimension,
161 glue = scanglue,
162 gluevalues = function() return scanglue(false,false,true) end,
163 gluespec = scanskip,
164 integer = scaninteger,
165 cardinal = scancardinal,
166 posit = scanposit,
167 real = scanreal,
168 float = scanfloat,
169 luanumber = scanluanumber,
170 luainteger = scanluainteger,
171 luacardinal = scanluacardinal,
172 count = scaninteger,
173 string = scanstring,
174 argument = scanargument,
175 delimited = scandelimited,
176 tokenlist = scantokenlist,
177 verbatim = scanverbatim,
178 code = scancode,
179 tokencode = scantokencode,
180 word = scanword,
181 letters = scanletters,
182 key = scankey,
183 value = scanvalue,
184 char = scanchar,
185 number = scannumber,
186 boolean = scanboolean,
187 keyword = scankeyword,
188 keywordcs = scankeywordcs,
189 csname = scancsname,
190 nextchar = scannextchar,
191 detokened = scandetokened,
192 tokenstring = scantokenstring,
193
194 next = token.scannext,
195 nextexpanded = token.scannextexpanded,
196
197 peek = token.peeknext,
198 peekexpanded = token.peeknextexpanded,
199 peekchar = token.peeknextchar,
200
201 skip = token.skipnext,
202 skipexpanded = token.skipnextexpanded,
203
204 cmdchr = token.scancmdchr,
205 cmdchrexpanded = token.scancmdchrexpanded,
206
207 ischar = token.isnextchar,
208
209
210
211 dimen = scandimension,
212 integerargument = token.scanintegerargument or scaninteger,
213 dimensionargument = token.scandimensionargument or scandimension,
214}
215
216tokens.getters = {
217 meaning = token.getmeaning,
218 macro = token.getmacro,
219 token = token.scannext,
220 cstoken = token.getcstoken,
221 count = tex.getcount,
222 dimen = tex.getdimen,
223 skip = tex.getglue,
224 glue = tex.getglue,
225 skip = tex.getmuglue,
226 glue = tex.getmuglue,
227 box = tex.getbox,
228}
229
230tokens.setters = {
231 macro = setmacro,
232 char = tex.chardef,
233 lua = setlua,
234 count = tex.setcount,
235 dimen = tex.setdimen,
236 skip = tex.setglue,
237 glue = tex.setglue,
238 skip = tex.setmuglue,
239 glue = tex.setmuglue,
240 box = tex.setbox,
241}
242
243tokens.accessors = {
244 command = token.getcommand,
245 cmd = token.getcommand,
246 cmdname = token.getcmdname,
247 name = token.getcmdname,
248 csname = token.getcsname,
249 index = token.getindex,
250 active = token.getactive,
251 frozen = token.getfrozen,
252 protected = token.getprotected,
253 expandable = token.getprotected,
254 user = token.getuser,
255 cmdchrcs = token.getcmdchrcs,
256 active = token.getactive,
257 range = token.getrange,
258}
259
260if setinspector then
261
262 local simple = { letter = "letter", other_char = "other" }
263
264 local astable = function(t)
265 if t and istoken(t) then
266 local cmdname = t.cmdname
267 local simple = simple[cmdname]
268 if simple then
269 return {
270 id = t.id,
271 category = simple,
272 character = utfchar(t.index) or nil,
273 }
274 else
275 return {
276 id = t.id,
277 command = t.command,
278 index = t.index,
279 csname = t.csname,
280 cmdname = cmdname,
281 active = t.active,
282 expandable = t.expandable,
283 frozen = t.frozen,
284 tolerant = t.tolerant,
285 protected = t.protected,
286 primitive = t.primitive,
287 permanent = t.permanent,
288 noaligned = t.noaligned,
289 instance = t.instance,
290 mutable = t.mutable,
291 immutable = t.immutable,
292 user = t.user,
293 }
294 end
295 end
296 end
297
298 tokens.astable = astable
299
300 setinspector("token",function(v) local t = astable(v) if t then printtable(t,tostring(v)) return true end end)
301
302end
303
304table.setmetatableindex(tokens.cache, function(t,k)
305 if not isdefined(k) then
306 setmacro(k,"","global")
307 end
308 local v = createtoken(k)
309 t[k] = v
310 return v
311end)
312
313
314
315
316token.is_token = token.istoken
317token.is_defined = token.isdefined
318token.scan_next = token.scannext
319token.scan_next_expanded = token.scannextexpanded
320token.scan_next_char = token.scannextchar
321token.skip_next = token.skipnext
322token.skip_next_expanded = token.skipnextexpanded
323token.peek_next = token.peeknext
324token.peek_next_expanded = token.peeknextexpanded
325token.peek_next_char = token.peeknextchar
326token.scan_cmdchr = token.scancmdchr
327token.scan_cmdchr_expanded = token.scancmdchrexpanded
328token.scan_keyword = token.scankeyword
329token.scan_keyword_cs = token.scankeywordcs
330token.scan_integer = token.scaninteger
331
332
333token.scan_cardinal = token.scancardinal
334token.scan_float = token.scanfloat
335token.scan_real = token.scanreal
336token.scan_luanumber = token.scanluanumber
337token.scan_luainteger = token.scanluainteger
338token.scan_luacardinal = token.scanluacardinal
339token.scan_scale = token.scanscale
340token.scan_dimen = token.scandimen
341token.scan_skip = token.scanskip
342token.scan_glue = token.scanglue
343token.scan_toks = token.scantoks
344token.scan_tokenlist = token.scantokenlist
345token.scan_code = token.scancode
346token.scan_token_code = token.scantokencode
347token.scan_string = token.scanstring
348token.scan_argument = token.scanargument
349token.scan_delimited = token.scandelimited
350token.scan_word = token.scanword
351token.scan_letters = token.scanletters
352token.scan_key = token.scankey
353token.scan_value = token.scanvalue
354token.scan_char = token.scanchar
355token.scan_csname = token.scancsname
356token.scan_token = token.scantoken
357token.scan_box = token.scanbox
358token.is_next_char = token.isnextchar
359token.put_next = token.putnext
360token.put_back = token.putback
361token.get_command = token.getcommand
362token.get_index = token.getindex
363token.get_range = token.getrange
364token.get_cmdname = token.getcmdname
365token.get_csname = token.getcsname
366token.get_id = token.getid
367token.get_tok = token.gettok
368token.get_active = token.getactive
369token.get_expandable = token.getexpandable
370token.get_protected = token.getprotected
371token.get_frozen = token.getfrozen
372token.get_tolerant = token.gettolerant
373token.get_noaligned = token.getnoaligned
374token.get_primitive = token.getprimitive
375token.get_permanent = token.getpermanent
376token.get_immutable = token.getimmutable
377token.get_instance = token.getinstance
378token.get_flags = token.getflags
379token.get_parameters = token.getparameters
380token.get_macro = token.getmacro
381token.get_meaning = token.getmeaning
382token.get_cmdchrcs = token.getcmdchrcs
383token.get_cstoken = token.getcstoken
384token.get_fields = token.getfields
385token.set_macro = token.setmacro
386token.undefine_macro = token.undefinemacro
387token.expand_macro = token.expandmacro
388token.set_lua = token.setlua
389token.set_integer = token.setinteger
390token.get_integer = token.getinteger
391token.set_dimension = token.setdimension
392token.get_dimension = token.getdimension
393token.gobble_integer = token.gobbleinteger
394token.gobble_dimension = token.gobbledimension
395token.future_expand = token.futureexpand
396token.push_macro = token.pushmacro
397token.pop_macro = token.popmacro
398token.save_lua = token.savelua
399 |
