1if not modules then modules = { } end modules ['toks-scn'] = {
2 version = 1.001,
3 author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
4 copyright = "PRAGMA ADE / ConTeXt Development Team",
5 license = "see context related readme files"
6}
7
8
9
10
11
12
13
14
15local type, next, tostring, tonumber = type, next, tostring, tonumber
16
17local formatters = string.formatters
18local concat = table.concat
19
20local scanners = tokens.scanners
21local tokenbits = tokens.bits
22
23local scanstring = scanners.string
24local scanargument = scanners.argument
25local scandelimited = scanners.delimited
26local scanverbatim = scanners.verbatim
27local scantokenlist = scanners.tokenlist
28local scantoks = scanners.toks
29local scaninteger = scanners.integer
30local scancardinal = scanners.cardinal
31local scannumber = scanners.number
32local scanfloat = scanners.float
33local scankeyword = scanners.keyword
34local scankeywordcs = scanners.keywordcs
35local scanword = scanners.word
36local scanletters = scanners.letters
37local scankey = scanners.key
38local scancode = scanners.code
39local scanboolean = scanners.boolean
40local scandimen = scanners.dimen
41local scanglue = scanners.glue
42local scangluevalues = scanners.gluevalues
43local scangluespec = scanners.gluespec
44local scancsname = scanners.csname
45local scanintegerargument = scanners.integerargument
46local scandimenargument = scanners.dimenargument
47local scandetokened = scanners.detokened
48
49local todimen = number.todimen
50local toboolean = toboolean
51
52local lpegmatch = lpeg.match
53local p_unquoted = lpeg.Cs(lpeg.patterns.unquoted)
54
55local trace_compile = false trackers.register("tokens.compile", function(v) trace_compile = v end)
56local report_compile = logs.reporter("tokens","compile")
57local report_scan = logs.reporter("tokens","scan")
58
59local open = tokenbits.open
60local close = tokenbits.close
61
62local function scanopen()
63 while true do
64 local c = scancode(open)
65 if c == 123 then
66 return true
67
68 elseif not c then
69 return
70 end
71 end
72end
73
74local function scanclose()
75 while true do
76 local c = scancode(close)
77 if c == 125 then
78 return true
79
80 elseif not c then
81 return
82 end
83 end
84end
85
86scanners.scanopen = scanopen
87scanners.scanclose = scanclose
88
89local function scanlist()
90 local wrapped = scanopen()
91 local list = { }
92 local size = 0
93 while true do
94 local entry = scanstring()
95 if entry then
96 size = size + 1
97 list[size] = entry
98 else
99 break
100 end
101 end
102 if wrapped then
103 scanclose()
104 end
105 return list
106end
107
108local function scanconditional()
109 local kw = scanword()
110 if kw == "true" then
111 return true
112 end
113 if kw == "false" then
114 return false
115 end
116 local c = scaninteger()
117 if c then
118 return c == 0
119 end
120 return nil
121end
122
123local function scantable(t,data)
124 if not data then
125 data = { }
126 end
127 if t then
128 local wrapped = scanopen()
129 while true do
130 local key = scanword(true)
131 if key then
132 local get = t[key]
133 if get then
134 data[key] = get()
135 else
136
137 end
138 else
139 break
140 end
141 end
142 if wrapped then
143 scanclose()
144 end
145 end
146 return data
147end
148
149function tokens.constant(s)
150 if type(s) == "string" then
151 return "'" .. s .. "'"
152 else
153 return s
154 end
155end
156
157scanners.list = scanlist
158scanners.table = scantable
159scanners.conditional = scanconditional
160
161function scanners.whd()
162 local width, height, depth
163 while true do
164 if scankeyword("width") then
165 width = scandimen()
166 elseif scankeyword("height") then
167 height = scandimen()
168 elseif scankeyword("depth") then
169 depth = scandimen()
170 else
171 break
172 end
173 end
174 if width or height or depth then
175 return width or 0, height or 0, depth or 0
176 else
177
178 end
179end
180
181local l = utf.byte("[")
182local r = utf.byte("]")
183
184local function scanbracketed()
185 local s = scandelimited(l, r)
186 if s then
187 return s
188 else
189 local readstate = status.getreadstate()
190 report_scan("missing argument in line %i of %a", readstate.linenumber, readstate.filename)
191 return ""
192 end
193end
194
195local function scanoptional()
196 return scandelimited(l, r) or ""
197end
198
199local function scanbracketedasis()
200 return scandelimited(l, r, false)
201end
202
203local function scanargumentasis()
204 return scanargument(false)
205end
206
207local function scancsnameunchecked()
208 return scancsname(true)
209end
210
211scanners.bracketed = scanbracketed
212scanners.optional = scanoptional
213scanners.bracketedasis = scanbracketedasis
214scanners.argumentasis = scanargumentasis
215scanners.csnameunchecked = scancsnameunchecked
216
217
218
219local shortcuts = {
220 tokens = tokens,
221 bits = tokenbits,
222 open = open,
223 close = close,
224 scanners = scanners,
225 scanstring = scanstring,
226 scanargument = scanargument,
227 scantokenstring = scanners.tokenstring,
228
229 scanverbatim = scanverbatim,
230 scantokenlist = scantokenlist,
231 scantoks = scantoks,
232 scaninteger = scaninteger,
233 scancardinal = scancardinal,
234 scannumber = scannumber,
235 scanfloat = scanfloat,
236 scantable = scantable,
237 scankeyword = scankeyword,
238 scankeywordcs = scankeywordcs,
239 scanword = scanword,
240 scanletters = scanletters,
241
242 scancode = scancode,
243 scanboolean = scanboolean,
244 scanglue = scanglue,
245 scangluespec = scangluespec,
246 scangluevalues = scangluevalues,
247 scandimen = scandimen,
248 scandimension = scandimen,
249 scanbox = scanners.box,
250 scanhbox = scanners.hbox,
251 scanvbox = scanners.vbox,
252 scanvtop = scanners.vtop,
253 scanconditional = scanconditional,
254 scanopen = scanopen,
255 scanclose = scanclose,
256 scanlist = scanlist,
257 scancsname = scancsname,
258 scancsnameunchecked = scancsnameunchecked,
259 scandelimited = scandelimited,
260 scanbracketed = scanbracketed,
261 scanoptional = scanoptional,
262 scanbracketedasis = scanbracketedasis,
263 scanargumentasis = scanargumentasis,
264 scanintegerargument = scanintegerargument,
265 scandimenargument = scandimenargument,
266 scandetokened = scandetokened,
267 todimen = todimen,
268 tonumber = tonumber,
269 tostring = tostring,
270 toboolean = toboolean,
271 inspect = inspect,
272 report = report_scan,
273 posit = posit,
274}
275
276tokens.shortcuts = shortcuts
277
278local load = load
279local dump = string.dump
280
281local function loadstripped(code)
282 return load(code,nil,nil,shortcuts)
283
284end
285
286tokens.converters = {
287 tonumber = "tonumber",
288 tostring = "tostring",
289 toboolean = "toboolean",
290 todimen = "todimen",
291 toglue = "todimen",
292}
293
294
295
296
297
298
299
300
301
302
303
304
305local f_if = formatters[ " if scankeywordcs('%s') then data['%s'] = scan%s()"]
306local f_elseif = formatters[" elseif scankeywordcs('%s') then data['%s'] = scan%s()"]
307
308
309
310
311
312
313
314local f_local = formatters["local scan%s = scanners.%s"]
315local f_scan = formatters["scan%s()"]
316local f_shortcut = formatters["local %s = scanners.converters.%s"]
317
318local f_if_c = formatters[ " if scankeywordcs('%s') then data['%s'] = %s(scan%s())"]
319local f_elseif_c = formatters[" elseif scankeywordcs('%s') then data['%s'] = %s(scan%s())"]
320local f_scan_c = formatters["%s(scan%s())"]
321
322
323
324
325
326
327local f_any = formatters[" else local key = scanword(true) if key then data[key] = scan%s() else break end end"]
328local f_any_c = formatters[" else local key = scanword(true) if key then data[key] = %s(scan%s()) else break end end"]
329local s_done <const> = " else break end"
330
331local f_any_all = formatters[" local key = scanword(true) if key then data[key] = scan%s() else break end"]
332local f_any_all_c = formatters[" local key = scanword(true) if key then data[key] = %s(scan%s()) else break end"]
333
334local f_table = formatters["%\nt\nreturn function()\n local data = { }\n%s\n return %s\nend\n"]
335local f_sequence = formatters["%\nt\n%\nt\n%\nt\nreturn function()\n return %s\nend\n"]
336local f_singular = formatters["%\nt\n%\nt\n\nreturn function(%s)\n return %s\nend\n"]
337local f_simple = formatters["%\nt\nreturn function()\n return %s\nend\n"]
338local f_string = formatters["%q"]
339local f_action_f = formatters["action%s(%s)"]
340local f_action_s = formatters["local action%s = tokens._action[%s]"]
341local f_nested = formatters["local function scan%s()\n local data = { }\n%s\n return data\nend\n"]
342
343local f_check = formatters[ [[
344 local wrapped = scanopen()
345 while true do
346 ]] .. "%\nt\n" .. [[
347 %s
348 end
349 if wrapped then
350 scanclose()
351 end
352]] ]
353
354
355
356local presets = {
357 ["1 string" ] = { "string" },
358 ["2 strings"] = { "string", "string" },
359 ["3 strings"] = { "string", "string", "string" },
360 ["4 strings"] = { "string", "string", "string", "string" },
361 ["5 strings"] = { "string", "string", "string", "string", "string" },
362 ["6 strings"] = { "string", "string", "string", "string", "string", "string" },
363 ["7 strings"] = { "string", "string", "string", "string", "string", "string", "string" },
364 ["8 strings"] = { "string", "string", "string", "string", "string", "string", "string", "string" },
365
366 ["1 argument" ] = { "argument" },
367 ["2 arguments"] = { "argument", "argument" },
368 ["3 arguments"] = { "argument", "argument", "argument" },
369 ["4 arguments"] = { "argument", "argument", "argument", "argument" },
370
371 ["1 integer"] = { "integer" },
372 ["2 integers"] = { "integer", "integer" },
373 ["3 integers"] = { "integer", "integer", "integer" },
374 ["4 integers"] = { "integer", "integer", "integer", "integer" },
375
376 ["1 optional"] = { "optional" },
377 ["2 optionals"] = { "optional", "optional" },
378 ["3 optionals"] = { "optional", "optional", "optional" },
379 ["4 optionals"] = { "optional", "optional", "optional", "optional" },
380}
381
382tokens.presets = presets
383
384function tokens.compile(specification)
385 local f = { }
386 local n = 0
387 local c = { }
388 local t = specification.arguments or specification
389 local a = specification.actions or nil
390 if type(a) == "function" then
391 a = { a }
392 end
393 local code
394 local args
395 local function compile(t,nested)
396 local done = s_done
397 local r = { }
398 local m = 0
399 for i=1,#t do
400 local ti = t[i]
401 if ti == "*" and i == 1 then
402 done = f_any_all("string")
403 else
404 local t1 = ti[1]
405 local t2 = ti[2] or "string"
406 if type(t2) == "table" then
407 n = n + 1
408 f[n] = compile(t2,n)
409 t2 = n
410 end
411 local t3 = ti[3]
412 if type(t3) == "function" then
413
414 elseif t3 then
415 c[t3] = f_shortcut(t3,t3)
416 if t1 == "*" then
417 if i == 1 then
418 done = f_any_all_c(t3,t2)
419 break
420 else
421 done = f_any_c(t3,t2)
422 end
423 else
424 m = m + 1
425 r[m] = (m > 1 and f_elseif_c or f_if_c)(t1,t1,t3,t2)
426 end
427 else
428 if t1 == "*" then
429 if i == 1 then
430 done = f_any_all(t2)
431 break
432 else
433 done = f_any(t2)
434 end
435 else
436 m = m + 1
437 r[m] = (m > 1 and f_elseif or f_if )(t1,t1,t2)
438
439 end
440 end
441 end
442 end
443 local c = f_check(r,done)
444 if nested then
445 return f_nested(nested,c)
446 else
447 return c
448 end
449 end
450 local p = t and presets[t]
451 if p then
452 t = p
453 end
454 local tt = type(t)
455 if tt == "string" then
456 if a then
457 local s = lpegmatch(p_unquoted,t)
458 if s and t ~= s then
459 code = t
460 else
461 code = f_scan(t)
462 end
463 tokens._action = a
464 for i=1,#a do
465 code = f_action_f(i,code)
466 n = n + 1
467 f[n] = f_action_s(i,i)
468 end
469 code = f_simple(f,code)
470 else
471 return scanners[t]
472 end
473 elseif tt ~= "table" then
474 return
475 elseif #t == 1 then
476 local ti = t[1]
477 if type(ti) == "table" then
478 ti = compile(ti)
479 code = "data"
480 if a then
481 tokens._action = a
482 for i=1,#a do
483 code = f_action_f(i,code)
484 n = n + 1
485 f[n] = f_action_s(i,i)
486 end
487 end
488 code = f_table(f,ti,code)
489 elseif a then
490 code = f_scan(ti)
491 tokens._action = a
492 for i=1,#a do
493 code = f_action_f(i,code)
494 n = n + 1
495 f[n] = f_action_s(i,i)
496 end
497 code = f_simple(f,code)
498 else
499 return scanners[ti]
500 end
501 elseif #t == 0 then
502 if specification.usage == "value" then
503 code = "b"
504 args = "_,b"
505 else
506 code = ""
507 args = ""
508 end
509 if a then
510 tokens._action = a
511 for i=1,#a do
512 code = f_action_f(i,code)
513 n = n + 1
514 f[n] = f_action_s(i,i)
515 end
516 end
517 code = f_singular(c,f,args,code)
518 else
519 local r = { }
520 local p = { }
521 local m = 0
522 for i=1,#t do
523 local ti = t[i]
524 local tt = type(ti)
525 if tt == "table" then
526 if ti[1] == "_constant_" then
527 local v = ti[2]
528 if type(v) == "string" then
529 r[i] = f_string(v)
530 else
531 r[i] = tostring(v)
532 end
533 else
534 m = m + 1
535 p[m] = compile(ti,100+m)
536 r[i] = f_scan(100+m)
537 end
538 elseif tt == "number" then
539 r[i] = tostring(ti)
540 elseif tt == "boolean" then
541 r[i] = tostring(ti)
542 else
543 local s = lpegmatch(p_unquoted,ti)
544 if s and ti ~= s then
545 r[i] = ti
546 elseif scanners[ti] then
547 r[i] = f_scan(ti)
548 else
549 report_compile("unknown scanner %a",ti)
550 r[i] = ti
551 end
552 end
553 end
554 code = concat(r,",")
555 if a then
556 tokens._action = a
557 for i=1,#a do
558 code = f_action_f(i,code)
559 n = n + 1
560 f[n] = f_action_s(i,i)
561 end
562 end
563 code = f_sequence(c,f,p,code)
564 end
565 if not code then
566 return
567 end
568 if trace_compile then
569 report_compile("code: %s",code)
570 end
571 local code, message = loadstripped(code)
572 if code then
573 code = code()
574 else
575 report_compile("error in code: %s",code)
576 report_compile("error message: %s",message)
577 end
578 if a then
579 tokens._action = nil
580 end
581 if code then
582 return code
583 end
584end
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608 |