1if not modules then modules = { } end modules ['toks-scn'] = {
2 version = 1.001,
3 author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
4 copyright = "PRAGMA ADE / ConTeXt Development Team",
5 license = "see context related readme files"
6}
7
8
9
10
11
12
13
14
15local type, next, tostring, tonumber = type, next, tostring, tonumber
16
17local formatters = string.formatters
18local concat = table.concat
19
20local scanners = tokens.scanners
21local tokenbits = tokens.bits
22
23local scanstring = scanners.string
24local scanargument = scanners.argument
25local scandelimited = scanners.delimited
26local scanverbatim = scanners.verbatim
27local scantokenlist = scanners.tokenlist
28local scantoks = scanners.toks
29local scaninteger = scanners.integer
30local scancardinal = scanners.cardinal
31local scannumber = scanners.number
32local scankeyword = scanners.keyword
33local scankeywordcs = scanners.keywordcs
34local scanword = scanners.word
35local scanletters = scanners.letters
36local scankey = scanners.key
37local scancode = scanners.code
38local scanboolean = scanners.boolean
39local scandimen = scanners.dimen
40local scanglue = scanners.glue
41local scangluevalues = scanners.gluevalues
42local scangluespec = scanners.gluespec
43local scancsname = scanners.csname
44
45local todimen = number.todimen
46local toboolean = toboolean
47
48local lpegmatch = lpeg.match
49local p_unquoted = lpeg.Cs(lpeg.patterns.unquoted)
50
51local trace_compile = false trackers.register("tokens.compile", function(v) trace_compile = v end)
52local report_compile = logs.reporter("tokens","compile")
53local report_scan = logs.reporter("tokens","scan")
54
55local open = tokenbits.open
56local close = tokenbits.close
57
58local function scanopen()
59 while true do
60 local c = scancode(open)
61 if c == 123 then
62 return true
63
64 elseif not c then
65 return
66 end
67 end
68end
69
70local function scanclose()
71 while true do
72 local c = scancode(close)
73 if c == 125 then
74 return true
75
76 elseif not c then
77 return
78 end
79 end
80end
81
82scanners.scanopen = scanopen
83scanners.scanclose = scanclose
84
85local function scanlist()
86 local wrapped = scanopen()
87 local list = { }
88 local size = 0
89 while true do
90 local entry = scanstring()
91 if entry then
92 size = size + 1
93 list[size] = entry
94 else
95 break
96 end
97 end
98 if wrapped then
99 scanclose()
100 end
101 return list
102end
103
104local function scanconditional()
105 local kw = scanword()
106 if kw == "true" then
107 return true
108 end
109 if kw == "false" then
110 return false
111 end
112 local c = scaninteger()
113 if c then
114 return c == 0
115 end
116 return nil
117end
118
119local function scantable(t,data)
120 if not data then
121 data = { }
122 end
123 if t then
124 local wrapped = scanopen()
125 while true do
126 local key = scanword(true)
127 if key then
128 local get = t[key]
129 if get then
130 data[key] = get()
131 else
132
133 end
134 else
135 break
136 end
137 end
138 if wrapped then
139 scanclose()
140 end
141 end
142 return data
143end
144
145function tokens.constant(s)
146 if type(s) == "string" then
147 return "'" .. s .. "'"
148 else
149 return s
150 end
151end
152
153scanners.list = scanlist
154scanners.table = scantable
155scanners.conditional = scanconditional
156
157function scanners.whd()
158 local width, height, depth
159 while true do
160 if scankeyword("width") then
161 width = scandimen()
162 elseif scankeyword("height") then
163 height = scandimen()
164 elseif scankeyword("depth") then
165 depth = scandimen()
166 else
167 break
168 end
169 end
170 if width or height or depth then
171 return width or 0, height or 0, depth or 0
172 else
173
174 end
175end
176
177
178
179local l = utf.byte("[")
180local r = utf.byte("]")
181
182local function scanbracketed()
183 local s = scandelimited(l, r)
184 if s then
185 return s
186 else
187 local readstate = status.getreadstate()
188 report_scan("missing argument in line %i of %a", readstate.linenumber, readstate.filename)
189 return ""
190 end
191end
192
193local function scanoptional()
194 return scandelimited(l, r) or ""
195end
196
197local function scanbracketedasis()
198 return scandelimited(l, r, false)
199end
200
201local function scanargumentasis()
202 return scanargument(false)
203end
204
205scanners.bracketed = scanbracketed
206scanners.optional = scanoptional
207scanners.bracketedasis = scanbracketedasis
208scanners.argumentasis = scanargumentasis
209
210
211
212local shortcuts = {
213 tokens = tokens,
214 bits = tokenbits,
215 open = open,
216 close = close,
217 scanners = scanners,
218 scanstring = scanstring,
219 scanargument = scanargument,
220 scanverbatim = scanverbatim,
221 scantokenlist = scantokenlist,
222 scantoks = scantoks,
223 scaninteger = scaninteger,
224 scancardinal = scancardinal,
225 scannumber = scannumber,
226 scantable = scantable,
227 scankeyword = scankeyword,
228 scankeywordcs = scankeywordcs,
229 scanword = scanword,
230 scanletters = scanletters,
231
232 scancode = scancode,
233 scanboolean = scanboolean,
234 scanglue = scanglue,
235 scangluespec = scangluespec,
236 scangluevalues = scangluevalues,
237 scandimen = scandimen,
238 scandimension = scandimen,
239 scanbox = scanners.box,
240 scanhbox = scanners.hbox,
241 scanvbox = scanners.vbox,
242 scanvtop = scanners.vtop,
243 scanconditional = scanconditional,
244 scanopen = scanopen,
245 scanclose = scanclose,
246 scanlist = scanlist,
247 scancsname = scancsname,
248 todimen = todimen,
249 tonumber = tonumber,
250 tostring = tostring,
251 toboolean = toboolean,
252 inspect = inspect,
253 report = report_scan,
254
255 scandelimited = scandelimited,
256 scanbracketed = scanbracketed,
257 scanoptional = scanoptional,
258 scanbracketedasis = scanbracketedasis,
259 scanargumentasis = scanargumentasis,
260}
261
262tokens.shortcuts = shortcuts
263
264local load = load
265local dump = string.dump
266
267local function loadstripped(code)
268 return load(code,nil,nil,shortcuts)
269
270end
271
272tokens.converters = {
273 tonumber = "tonumber",
274 tostring = "tostring",
275 toboolean = "toboolean",
276 todimen = "todimen",
277 toglue = "todimen",
278}
279
280
281
282
283
284
285
286
287
288
289
290
291local f_if = formatters[ " if scankeywordcs('%s') then data['%s'] = scan%s()"]
292local f_elseif = formatters[" elseif scankeywordcs('%s') then data['%s'] = scan%s()"]
293
294
295
296
297local f_local = formatters["local scan%s = scanners.%s"]
298local f_scan = formatters["scan%s()"]
299local f_shortcut = formatters["local %s = scanners.converters.%s"]
300
301local f_if_c = formatters[ " if scankeywordcs('%s') then data['%s'] = %s(scan%s())"]
302local f_elseif_c = formatters[" elseif scankeywordcs('%s') then data['%s'] = %s(scan%s())"]
303local f_scan_c = formatters["%s(scan%s())"]
304
305
306
307local f_any = formatters[" else local key = scanword(true) if key then data[key] = scan%s() else break end end"]
308local f_any_c = formatters[" else local key = scanword(true) if key then data[key] = %s(scan%s()) else break end end"]
309local s_done = " else break end"
310
311local f_any_all = formatters[" local key = scanword(true) if key then data[key] = scan%s() else break end"]
312local f_any_all_c= formatters[" local key = scanword(true) if key then data[key] = %s(scan%s()) else break end"]
313
314local f_table = formatters["%\nt\nreturn function()\n local data = { }\n%s\n return %s\nend\n"]
315local f_sequence = formatters["%\nt\n%\nt\n%\nt\nreturn function()\n return %s\nend\n"]
316local f_singular = formatters["%\nt\n%\nt\n\nreturn function(%s)\n return %s\nend\n"]
317local f_simple = formatters["%\nt\nreturn function()\n return %s\nend\n"]
318local f_string = formatters["%q"]
319local f_action_f = formatters["action%s(%s)"]
320local f_action_s = formatters["local action%s = tokens._action[%s]"]
321local f_nested = formatters["local function scan%s()\n local data = { }\n%s\n return data\nend\n"]
322
323local f_check = formatters[ [[
324 local wrapped = scanopen()
325 while true do
326 ]] .. "%\nt\n" .. [[
327 %s
328 end
329 if wrapped then
330 scanclose()
331 end
332]] ]
333
334
335
336local presets = {
337 ["1 string" ] = { "string" },
338 ["2 strings"] = { "string", "string" },
339 ["3 strings"] = { "string", "string", "string" },
340 ["4 strings"] = { "string", "string", "string", "string" },
341 ["5 strings"] = { "string", "string", "string", "string", "string" },
342 ["6 strings"] = { "string", "string", "string", "string", "string", "string" },
343 ["7 strings"] = { "string", "string", "string", "string", "string", "string", "string" },
344 ["8 strings"] = { "string", "string", "string", "string", "string", "string", "string", "string" },
345
346 ["1 argument" ] = { "argument" },
347 ["2 arguments"] = { "argument", "argument" },
348 ["3 arguments"] = { "argument", "argument", "argument" },
349 ["4 arguments"] = { "argument", "argument", "argument", "argument" },
350}
351
352tokens.presets = presets
353
354function tokens.compile(specification)
355 local f = { }
356 local n = 0
357 local c = { }
358 local t = specification.arguments or specification
359 local a = specification.actions or nil
360 if type(a) == "function" then
361 a = { a }
362 end
363 local code
364 local args
365 local function compile(t,nested)
366 local done = s_done
367 local r = { }
368 local m = 0
369 for i=1,#t do
370 local ti = t[i]
371 if ti == "*" and i == 1 then
372 done = f_any_all("string")
373 else
374 local t1 = ti[1]
375 local t2 = ti[2] or "string"
376 if type(t2) == "table" then
377 n = n + 1
378 f[n] = compile(t2,n)
379 t2 = n
380 end
381 local t3 = ti[3]
382 if type(t3) == "function" then
383
384 elseif t3 then
385 c[t3] = f_shortcut(t3,t3)
386 if t1 == "*" then
387 if i == 1 then
388 done = f_any_all_c(t3,t2)
389 break
390 else
391 done = f_any_c(t3,t2)
392 end
393 else
394 m = m + 1
395 r[m] = (m > 1 and f_elseif_c or f_if_c)(t1,t1,t3,t2)
396 end
397 else
398 if t1 == "*" then
399 if i == 1 then
400 done = f_any_all(t2)
401 break
402 else
403 done = f_any(t2)
404 end
405 else
406 m = m + 1
407 r[m] = (m > 1 and f_elseif or f_if )(t1,t1,t2)
408
409 end
410 end
411 end
412 end
413 local c = f_check(r,done)
414 if nested then
415 return f_nested(nested,c)
416 else
417 return c
418 end
419 end
420 local p = t and presets[t]
421 if p then
422 t = p
423 end
424 local tt = type(t)
425 if tt == "string" then
426 if a then
427 local s = lpegmatch(p_unquoted,t)
428 if s and t ~= s then
429 code = t
430 else
431 code = f_scan(t)
432 end
433 tokens._action = a
434 for i=1,#a do
435 code = f_action_f(i,code)
436 n = n + 1
437 f[n] = f_action_s(i,i)
438 end
439 code = f_simple(f,code)
440 else
441 return scanners[t]
442 end
443 elseif tt ~= "table" then
444 return
445 elseif #t == 1 then
446 local ti = t[1]
447 if type(ti) == "table" then
448 ti = compile(ti)
449 code = "data"
450 if a then
451 tokens._action = a
452 for i=1,#a do
453 code = f_action_f(i,code)
454 n = n + 1
455 f[n] = f_action_s(i,i)
456 end
457 end
458 code = f_table(f,ti,code)
459 elseif a then
460 code = f_scan(ti)
461 tokens._action = a
462 for i=1,#a do
463 code = f_action_f(i,code)
464 n = n + 1
465 f[n] = f_action_s(i,i)
466 end
467 code = f_simple(f,code)
468 else
469 return scanners[ti]
470 end
471 elseif #t == 0 then
472 if specification.usage == "value" then
473 code = "b"
474 args = "_,b"
475 else
476 code = ""
477 args = ""
478 end
479 if a then
480 tokens._action = a
481 for i=1,#a do
482 code = f_action_f(i,code)
483 n = n + 1
484 f[n] = f_action_s(i,i)
485 end
486 end
487 code = f_singular(c,f,args,code)
488 else
489 local r = { }
490 local p = { }
491 local m = 0
492 for i=1,#t do
493 local ti = t[i]
494 local tt = type(ti)
495 if tt == "table" then
496 if ti[1] == "_constant_" then
497 local v = ti[2]
498 if type(v) == "string" then
499 r[i] = f_string(v)
500 else
501 r[i] = tostring(v)
502 end
503 else
504 m = m + 1
505 p[m] = compile(ti,100+m)
506 r[i] = f_scan(100+m)
507 end
508 elseif tt == "number" then
509 r[i] = tostring(ti)
510 elseif tt == "boolean" then
511 r[i] = tostring(ti)
512 else
513 local s = lpegmatch(p_unquoted,ti)
514 if s and ti ~= s then
515 r[i] = ti
516 elseif scanners[ti] then
517 r[i] = f_scan(ti)
518 else
519 report_compile("unknown scanner %a",ti)
520 r[i] = ti
521 end
522 end
523 end
524 code = concat(r,",")
525 if a then
526 tokens._action = a
527 for i=1,#a do
528 code = f_action_f(i,code)
529 n = n + 1
530 f[n] = f_action_s(i,i)
531 end
532 end
533 code = f_sequence(c,f,p,code)
534 end
535 if not code then
536 return
537 end
538 if trace_compile then
539 report_compile("code: %s",code)
540 end
541 local code, message = loadstripped(code)
542 if code then
543 code = code()
544 else
545 report_compile("error in code: %s",code)
546 report_compile("error message: %s",message)
547 end
548 if a then
549 tokens._action = nil
550 end
551 if code then
552 return code
553 end
554end
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578 |