1if not modules then modules = { } end modules ['toks-ini'] = {
2 version = 1.001,
3 author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
4 copyright = "PRAGMA ADE / ConTeXt Development Team",
5 license = "see context related readme files"
6}
7
8tokens = tokens or { }
9
10local tokens = tokens
11local token = token
12local next = next
13local tonumber = tonumber
14local tostring = tostring
15local utfchar = utf.char
16local char = string.char
17local printtable = table.print
18local concat = table.concat
19local format = string.format
20
21local commands = token.commands()
22tokens.commands = utilities.storage.allocate(table.swapped(commands,commands))
23tokens.values = { }
24
25local scantoks = token.scan_toks
26local scanstring = token.scan_string
27local scanargument = token.scan_argument
28local scandelimited = token.scan_delimited
29local scantokenlist = token.scan_tokenlist or scanstring
30local scaninteger = token.scan_integer or token.scan_int
31local scancardinal = token.scan_cardinal
32local scancode = token.scan_code
33local scantokencode = token.scan_token_code
34local scandimen = token.scan_dimen
35local scanglue = token.scan_glue
36local scanskip = token.scan_skip
37local scankeyword = token.scan_keyword
38local scankeywordcs = token.scan_keyword_cs or scankeyword
39local scantoken = token.scan_token
40local scanbox = token.scan_box
41local scanword = token.scan_word
42local scanletters = token.scan_letters or scanword
43local scankey = token.scan_key
44local scanvalue = token.scan_value
45local scanchar = token.scan_char
46local scannumber = token.scan_number
47local scancsname = token.scan_csname
48local scanreal = token.scan_real
49local scanfloat = token.scan_float
50local scanluanumber = token.scan_luanumber or scanfloat
51local scanluainteger = token.scan_luainteger or scaninteger
52local scanluacardinal = token.scan_luacardinal or scancardinal
53
54local setmacro = token.set_macro
55local setchar = token.set_char
56local setlua = token.set_lua
57
58local createtoken = token.create
59local newtoken = token.new
60local isdefined = token.is_defined
61local istoken = token.is_token
62
63tokens.new = newtoken
64tokens.create = createtoken
65tokens.istoken = istoken
66tokens.isdefined = isdefined
67tokens.defined = isdefined
68
69local bits = {
70 escape = 0x00000001,
71 begingroup = 0x00000002,
72 endgroup = 0x00000004,
73 mathshift = 0x00000008,
74 alignment = 0x00000010,
75 endofline = 0x00000020,
76 parameter = 0x00000040,
77 superscript = 0x00000080,
78 subscript = 0x00000100,
79 ignore = 0x00000200,
80 space = 0x00000400,
81 letter = 0x00000800,
82 other = 0x00001000,
83 active = 0x00002000,
84 comment = 0x00004000,
85 invalid = 0x00008000,
86
87 character = 0x00001800,
88 whitespace = 0x00002400,
89
90 open = 0x00000402,
91 close = 0x00000404,
92}
93
94
95
96tokens.bits = bits
97
98
99
100
101if not scannumber then
102
103 scannumber = function(base)
104 local s = scanword()
105 if not s then
106 return nil
107 elseif base then
108 return tonumber(s,base)
109 else
110 return tonumber(s)
111 end
112 end
113
114end
115
116local function scanboolean()
117 local kw = scanword()
118 if kw == "true" then
119 return true
120 elseif kw == "false" then
121 return false
122 else
123 return nil
124 end
125end
126
127local function scanverbatim()
128 return scanargument(false)
129end
130
131if not scanbox then
132
133 local scanlist = token.scan_list
134 local putnext = token.put_next
135
136 scanbox = function(s)
137 if s == "hbox" or s == "vbox" or s == "vtop" then
138 putnext(createtoken(s))
139 end
140 return scanlist()
141 end
142
143 token.scanbox = scanbox
144
145end
146
147tokens.scanners = {
148 token = scantoken,
149 toks = scantoks,
150 tokens = scantoks,
151 box = scanbox,
152 hbox = function() return scanbox("hbox") end,
153 vbox = function() return scanbox("vbox") end,
154 vtop = function() return scanbox("vtop") end,
155 dimen = scandimen,
156 dimension = scandimen,
157 glue = scanglue,
158 gluevalues = function() return scanglue(false,false,true) end,
159 gluespec = scanskip,
160 integer = scaninteger,
161 cardinal = scancardinal,
162 real = scanreal,
163 float = scanfloat,
164 luanumber = scanluanumber,
165 luainteger = scanluainteger,
166 luacardinal = scanluacardinal,
167 count = scaninteger,
168 string = scanstring,
169 argument = scanargument,
170 delimited = scandelimited,
171 tokenlist = scantokenlist,
172 verbatim = scanverbatim,
173 code = scancode,
174 tokencode = scantokencode,
175 word = scanword,
176 letters = scanletters,
177 key = scankey,
178 value = scanvalue,
179 char = scanchar,
180 number = scannumber,
181 boolean = scanboolean,
182 keyword = scankeyword,
183 keywordcs = scankeywordcs,
184 csname = scancsname,
185
186 next = token.scan_next,
187 nextexpanded = token.scan_next_expanded,
188
189 peek = token.peek_next,
190 peekexpanded = token.peek_next_expanded,
191 peekchar = token.peek_next_char,
192
193 skip = token.skip_next,
194 skipexpanded = token.skip_next_expanded,
195
196 cmdchr = token.scan_cmdchr,
197 cmdchrexpanded = token.scan_cmdchr_expanded,
198
199 ischar = token.is_next_char,
200}
201
202tokens.getters = {
203 meaning = token.get_meaning,
204 macro = token.get_macro,
205 token = token.scan_next or token.get_next,
206 cstoken = token.get_cstoken,
207 count = tex.getcount,
208 dimen = tex.getdimen,
209 skip = tex.getglue,
210 glue = tex.getglue,
211 skip = tex.getmuglue,
212 glue = tex.getmuglue,
213 box = tex.getbox,
214}
215
216tokens.setters = {
217 macro = setmacro,
218 char = setchar,
219 lua = setlua,
220 count = tex.setcount,
221 dimen = tex.setdimen,
222 skip = tex.setglue,
223 glue = tex.setglue,
224 skip = tex.setmuglue,
225 glue = tex.setmuglue,
226 box = tex.setbox,
227}
228
229tokens.accessors = {
230 command = token.get_command,
231 cmd = token.get_command,
232 cmdname = token.get_cmdname,
233 name = token.get_cmdname,
234 csname = token.get_csname,
235 index = token.get_index,
236 active = token.get_active,
237 frozen = token.get_frozen,
238 protected = token.get_protected,
239 expandable = token.get_protected,
240 user = token.get_user,
241 cmdchrcs = token.get_cmdchrcs,
242 active = token.get_active,
243 range = token.get_range,
244}
245
246if setinspector then
247
248 local simple = { letter = "letter", other_char = "other" }
249
250 local astable = function(t)
251 if t and istoken(t) then
252 local cmdname = t.cmdname
253 local simple = simple[cmdname]
254 if simple then
255 return {
256 id = t.id,
257 category = simple,
258 character = utfchar(t.mode) or nil,
259 }
260 else
261 return {
262 command = t.command,
263 id = t.id,
264 tok = t.tok,
265 csname = t.csname,
266 active = t.active,
267 expandable = t.expandable,
268 protected = t.protected,
269 frozen = t.frozen,
270 mode = t.mode,
271 index = t.index,
272 user = t.user,
273 cmdname = cmdname,
274 }
275 end
276 end
277 end
278
279 tokens.astable = astable
280
281 setinspector("token",function(v) local t = astable(v) if t then printtable(t,tostring(v)) return true end end)
282
283end
284
285tokens.cache = table.setmetatableindex(function(t,k)
286 if not isdefined(k) then
287 setmacro(k,"","global")
288 end
289 local v = createtoken(k)
290 t[k] = v
291 return v
292end)
293
294if LUATEXVERSION < 114 then
295
296 local d = tokens.defined
297 local c = tokens.create
298
299 function tokens.defined(s,b)
300 if b then
301 return d(s)
302 else
303 return c(s).cmd_name == "undefined_cmd"
304 end
305 end
306
307end
308 |