cldf-ini.lua /size: 51 Kb    last modification: 2020-07-01 14:35
1
if
not
modules
then
modules
=
{
}
end
modules
[
'
cldf-ini
'
]
=
{
2
version
=
1
.
001
,
3
comment
=
"
companion to cldf-ini.mkiv
"
,
4
author
=
"
Hans Hagen, PRAGMA-ADE, Hasselt NL
"
,
5
copyright
=
"
PRAGMA ADE / ConTeXt Development Team
"
,
6
license
=
"
see context related readme files
"
7
}
8 9
-- todo: {token} -> 3 tokens
10 11
-- This started as an experiment: generating context code at the lua end. After all
12
-- it is surprisingly simple to implement due to metatables. I was wondering if
13
-- there was a more natural way to deal with commands at the lua end. Of course it's
14
-- a bit slower but often more readable when mixed with lua code. It can also be handy
15
-- when generating documents from databases or when constructing large tables or so.
16
--
17
-- In cldf-tod.lua I have some code that might end up here. In cldf-old.lua the
18
-- code that precedes more modern solutions (made possible by features in the engine),
19
-- most noticeably function handling, which worked well for a decade, but by now the
20
-- more efficient approach is stable enough to move the original code to the obsolete
21
-- module.
22
--
23
-- to be considered:
24
--
25
-- 0.528 local foo = tex.ctxcatcodes
26
-- 0.651 local foo = getcount("ctxcatcodes")
27
-- 0.408 local foo = getcount(ctxcatcodes) -- local ctxcatcodes = tex.iscount("ctxcatcodes")
28 29
-- maybe:  (escape) or 0x2061 (apply function) or 0x2394 (software function ⎔) (old)
30
-- note : tex.print == line with endlinechar appended
31
-- todo : context("%bold{total: }%s",total)
32
-- todo : context.documentvariable("title")
33
--
34
-- During the crited project we ran into the situation that luajittex was 10-20 times
35
-- slower that luatex ... after 3 days of testing and probing we finally figured out that
36
-- the the differences between the lua and luajit hashers can lead to quite a slowdown
37
-- in some cases.
38
--
39
-- context(lpeg.match(lpeg.patterns.texescape,"${}"))
40
-- context(string.formatters["%!tex!"]("${}"))
41
-- context("%!tex!","${}")
42
--
43
-- We try not to polute the context namespace too much. For that reason the commands are
44
-- registered in the context.core namespace. You can call all context commands using
45
-- context.foo etc. and pipe to context with context("foo"). Defining a local command
46
-- foo being context.foo is okay, as is context.core.foo. We will have some definitions
47
-- that are plugged into the core namespace (mostly for speedup) but otherwise specialized
48
-- writers are in the context namespace only. In your documents you can best use the
49
-- context.foo(...) and context(...) variant but in some core modules we use the faster
50
-- ones in critical places (no one will notice of course). The standard syntax highlighter
51
-- that I use knows how to visualize ctx related code.
52 53
-- I cleaned this code up a bit when updating the cld manual but got distracted a bit by
54
-- watching Trio Hiromi Uehara, Anthony Jackson & Simon Phillips (discovered via the
55
-- later on YT). Hopefully the video breaks made the following better in the end.
56 57
-- This module is also a test bed for experimental features so the content changes over
58
-- time (for the better or worse). There have been no fundamental changes for many years
59
-- and performance has not changed much either.
60 61
-- The code here has evolved over time, and was already present in the early versions of
62
-- mkiv. However, it got adapted when feature were added to luatex, like the ability to
63
-- "print" nodes and later tokens. We use to have control over the method used (if only
64
-- for testing) but the older code has been removed now. The history is in the archives.
65
-- These were the controls:
66 67
-- local tokenflushmode = true
68
-- local nodeflushmode = true
69
-- local scannerdefmode = true
70
-- local maxflushnodeindex = 0x10FFFF - 1
71 72
-- In earlier experiments a function tables was referred to as lua.calls and the primitive
73
-- \luafunctions was \luacall and we used our own implementation of a function table (more
74
-- indirectness).
75 76
local
format
,
stripstring
=
string
.
format
,
string
.
strip
77
local
next
,
type
,
tostring
,
tonumber
,
unpack
,
select
,
rawset
=
next
,
type
,
tostring
,
tonumber
,
unpack
,
select
,
rawset
78
local
insert
,
remove
,
concat
=
table
.
insert
,
table
.
remove
,
table
.
concat
79
local
lpegmatch
,
lpegC
,
lpegS
,
lpegP
,
lpegV
,
lpegCc
,
lpegCs
,
patterns
=
lpeg
.
match
,
lpeg
.
C
,
lpeg
.
S
,
lpeg
.
P
,
lpeg
.
V
,
lpeg
.
Cc
,
lpeg
.
Cs
,
lpeg
.
patterns
80 81
local
formatters
=
string
.
formatters
-- using formatters is slower in this case
82
local
setmetatableindex
=
table
.
setmetatableindex
83
local
setmetatablecall
=
table
.
setmetatablecall
84
local
setmetatablenewindex
=
table
.
setmetatablenewindex
85 86
context
=
context
or
{
}
87
commands
=
commands
or
{
}
88
interfaces
=
interfaces
or
{
}
89 90
local
context
=
context
91
local
commands
=
commands
92
local
interfaces
=
interfaces
93 94
local
loaddata
=
io
.
loaddata
95 96
local
tex
=
tex
97
local
texsprint
=
tex
.
sprint
-- just appended (no space,eol treatment)
98
local
texprint
=
tex
.
print
-- each arg a separate line (not last in directlua)
99
----- texwrite = tex.write -- all 'space' and 'character'
100
local
texgetcount
=
tex
.
getcount
101 102
local
isnode
=
node
.
is_node
103
local
writenode
=
node
.
write
104
local
copynodelist
=
node
.
copy_list
105
local
tonut
=
node
.
direct
.
todirect
106
local
tonode
=
node
.
direct
.
tonode
107 108
local
istoken
=
token
.
is_token
109
local
newtoken
=
token
.
new
110
local
createtoken
=
token
.
create
111
local
setluatoken
=
token
.
set_lua
112 113
local
catcodenumbers
=
catcodes
.
numbers
114 115
local
ctxcatcodes
=
catcodenumbers
.
ctxcatcodes
116
local
prtcatcodes
=
catcodenumbers
.
prtcatcodes
117
local
texcatcodes
=
catcodenumbers
.
texcatcodes
118
local
txtcatcodes
=
catcodenumbers
.
txtcatcodes
119
local
vrbcatcodes
=
catcodenumbers
.
vrbcatcodes
120
local
xmlcatcodes
=
catcodenumbers
.
xmlcatcodes
121 122
local
flush
=
texsprint
-- snippets
123
local
flushdirect
=
texprint
-- lines
124
----- flushraw = texwrite
125 126
local
report_context
=
logs
.
reporter
(
"
cld
"
,
"
tex
"
)
127
local
report_cld
=
logs
.
reporter
(
"
cld
"
,
"
stack
"
)
128
----- report_template = logs.reporter("cld","template")
129 130
local
processlines
=
true
-- experiments.register("context.processlines", function(v) processlines = v end)
131 132
local
trialtypesettingstate
=
createtoken
(
"
trialtypesettingstate
"
)
.
index
133 134
function
context
.
trialtypesetting
(
)
135
return
texgetcount
(
trialtypesettingstate
)
~
=
0
136
end
137 138
local
knownfunctions
=
(
lua
.
getfunctionstable
or
lua
.
get_functions_table
)
(
true
)
139
local
showstackusage
=
false
140 141
trackers
.
register
(
"
context.stack
"
,
function
(
v
)
showstackusage
=
v
end
)
142 143
local
freed
=
{
}
144
local
nofused
=
0
145
local
noffreed
=
0
146 147
local
usedstack
=
function
(
)
148
return
nofused
,
noffreed
149
end
150 151
local
flushfunction
=
function
(
slot
,
arg
)
152
if
arg
(
)
then
153
-- keep
154
elseif
texgetcount
(
trialtypesettingstate
)
=
=
0
then
155
noffreed
=
noffreed
+
1
156
freed
[
noffreed
]
=
slot
157
knownfunctions
[
slot
]
=
false
158
else
159
-- keep
160
end
161
end
162 163
local
storefunction
=
function
(
arg
)
164
local
f
=
function
(
slot
)
flushfunction
(
slot
,
arg
)
end
165
if
noffreed
>
0
then
166
local
n
=
freed
[
noffreed
]
167
freed
[
noffreed
]
=
nil
168
noffreed
=
noffreed
-
1
169
knownfunctions
[
n
]
=
f
170
return
n
171
else
172
nofused
=
nofused
+
1
173
knownfunctions
[
nofused
]
=
f
174
return
nofused
175
end
176
end
177 178
local
flushnode
=
function
(
slot
,
arg
)
179
if
texgetcount
(
trialtypesettingstate
)
=
=
0
then
180
writenode
(
arg
)
181
noffreed
=
noffreed
+
1
182
freed
[
noffreed
]
=
slot
183
knownfunctions
[
slot
]
=
false
184
else
185
writenode
(
copynodelist
(
arg
)
)
186
end
187
end
188 189
local
storenode
=
function
(
arg
)
190
local
f
=
function
(
slot
)
flushnode
(
slot
,
arg
)
end
191
if
noffreed
>
0
then
192
local
n
=
freed
[
noffreed
]
193
freed
[
noffreed
]
=
nil
194
noffreed
=
noffreed
-
1
195
knownfunctions
[
n
]
=
f
196
return
n
197
else
198
nofused
=
nofused
+
1
199
knownfunctions
[
nofused
]
=
f
200
return
nofused
201
end
202
end
203 204
storage
.
storedfunctions
=
storage
.
storedfunctions
or
{
}
205
local
storedfunctions
=
storage
.
storedfunctions
206
local
initex
=
environment
.
initex
207 208
storage
.
register
(
"
storage/storedfunctions
"
,
storedfunctions
,
"
storage.storedfunctions
"
)
209 210
local
f_resolve
=
nil
211
local
p_resolve
=
(
(
1
-
lpegP
(
"
.
"
)
)
^
1
/
function
(
s
)
f_resolve
=
f_resolve
[
s
]
end
*
lpegP
(
"
.
"
)
^
0
)
^
1
212 213
local
function
resolvestoredfunction
(
str
)
214
if
type
(
str
)
=
=
"
string
"
then
215
f_resolve
=
global
-- namespace
216
lpegmatch
(
p_resolve
,
str
)
217
return
f_resolve
218
else
219
return
str
220
end
221
end
222 223
local
function
expose
(
slot
,
f
,
...
)
-- so we can register yet undefined functions
224
local
func
=
resolvestoredfunction
(
f
)
225
if
not
func
then
226
func
=
function
(
)
report_cld
(
"
beware: unknown function %i called: %s
"
,
slot
,
f
)
end
227
end
228
knownfunctions
[
slot
]
=
func
229
return
func
(
...
)
230
end
231 232
if
initex
then
233
-- todo: log stored functions
234
else
235
local
slots
=
table
.
sortedkeys
(
storedfunctions
)
236
local
last
=
#
slots
237
if
last
>
0
then
238
-- we restore the references
239
for
i
=
1
,
last
do
240
local
slot
=
slots
[
i
]
241
local
data
=
storedfunctions
[
slot
]
242
knownfunctions
[
slot
]
=
function
(
...
)
243
-- print(data) -- could be trace
244
return
expose
(
slot
,
data
,
...
)
245
end
246
end
247
-- we now know how many are defined
248
nofused
=
slots
[
last
]
249
-- normally there are no holes in the list yet
250
for
i
=
1
,
nofused
do
251
if
not
knownfunctions
[
i
]
then
252
noffreed
=
noffreed
+
1
253
freed
[
noffreed
]
=
i
254
end
255
end
256
-- report_cld("%s registered functions, %s freed slots",last,noffreed)
257
end
258
end
259 260
local
registerfunction
=
function
(
f
,
direct
,
slot
)
-- either f=code or f=namespace,direct=name
261
local
func
262
if
slot
then
263
-- already used
264
elseif
noffreed
>
0
then
265
slot
=
freed
[
noffreed
]
266
freed
[
noffreed
]
=
nil
267
noffreed
=
noffreed
-
1
268
else
269
nofused
=
nofused
+
1
270
slot
=
nofused
271
end
272
if
direct
then
273
if
initex
then
274
func
=
function
(
...
)
expose
(
slot
,
f
,
...
)
end
275
storedfunctions
[
slot
]
=
f
276
else
277
func
=
resolvestoredfunction
(
f
)
278
end
279
if
type
(
func
)
~
=
"
function
"
then
280
func
=
function
(
)
report_cld
(
"
invalid resolve %A, case %s
"
,
f
,
1
)
end
281
end
282
elseif
type
(
f
)
=
=
"
string
"
then
283
func
=
loadstring
(
f
)
284
if
type
(
func
)
~
=
"
function
"
then
285
func
=
function
(
)
report_cld
(
"
invalid code %A, case %s
"
,
f
,
2
)
end
286
end
287
elseif
type
(
f
)
=
=
"
function
"
then
288
func
=
f
289
else
290
func
=
function
(
)
report_cld
(
"
invalid function %A, case %s
"
,
f
,
3
)
end
291
end
292
knownfunctions
[
slot
]
=
func
293
return
slot
294
end
295 296
local
unregisterfunction
=
function
(
slot
)
297
if
knownfunctions
[
slot
]
then
298
noffreed
=
noffreed
+
1
299
freed
[
noffreed
]
=
slot
300
knownfunctions
[
slot
]
=
false
301
else
302
report_cld
(
"
invalid function slot %A
"
,
slot
)
303
end
304
end
305 306
local
reservefunction
=
function
(
)
307
if
noffreed
>
0
then
308
local
n
=
freed
[
noffreed
]
309
freed
[
noffreed
]
=
nil
310
noffreed
=
noffreed
-
1
311
return
n
312
else
313
nofused
=
nofused
+
1
314
return
nofused
315
end
316
end
317 318
local
callfunctiononce
=
function
(
slot
)
319
knownfunctions
[
slot
]
(
slot
)
320
noffreed
=
noffreed
+
1
321
freed
[
noffreed
]
=
slot
322
knownfunctions
[
slot
]
=
false
323
end
324 325
setmetatablecall
(
knownfunctions
,
function
(
t
,
n
)
return
knownfunctions
[
n
]
(
n
)
end
)
326 327
-- some protection
328 329
do
330 331
local
stub
=
{
}
332
local
done
=
false
333
local
message
=
function
(
)
334
-- one time message
335
if
not
done
then
336
report_cld
(
"
"
)
337
report_cld
(
"
use : slot = context.functions.register(f)
"
)
338
report_cld
(
"
and : context.functions.unregister(slot)
"
)
339
report_cld
(
"
"
)
340
done
=
true
341
end
342
end
343 344
setmetatable
(
stub
,
{
345
__index
=
message
,
346
__newindex
=
message
,
347
}
)
348 349
function
lua
.
getfunctionstable
(
)
350
message
(
)
351
return
stub
352
end
353 354
lua
.
get_functions_table
=
lua
.
getfunctionstable
355 356
end
357 358
-- The next hack is a convenient way to define scanners at the Lua end and get them
359
-- available at the TeX end. There is some dirty magic needed to prevent overload
360
-- during format loading. Nowadays we prefer to use the slightly less efficient way
361
-- of defining interfaces using the implementer. There is a little more overhead in
362
-- defining as well as runtime overhead, but we accept that.
363 364
-- interfaces.scanners.foo = function() context("[%s]",tokens.scanners.string()) end : \scan_foo
365 366
local
storedscanners
=
interfaces
.
storedscanners
or
{
}
367
local
interfacescanners
=
{
}
368
local
privatenamespace
=
"
clf_
"
369 370
interfaces
.
storedscanners
=
storedscanners
371 372
storage
.
register
(
"
interfaces/storedscanners
"
,
storedscanners
,
"
interfaces.storedscanners
"
)
373 374
local
function
registerscanner
(
name
,
action
,
protected
,
public
,
value
)
375
rawset
(
interfacescanners
,
name
,
action
)
376
local
n
=
storedscanners
[
name
]
377
n
=
registerfunction
(
"
interfaces.scanners.
"
.
.
name
,
true
,
n
)
378
storedscanners
[
name
]
=
n
379
name
=
public
and
name
or
(
privatenamespace
.
.
name
)
380
setluatoken
(
name
,
n
,
"
global
"
,
protected
and
"
protected
"
or
"
"
,
value
and
"
value
"
or
"
"
)
381
end
382 383
interfaces
.
registerscanner
=
registerscanner
384 385
function
interfaces
.
knownscanner
(
name
)
386
return
interfacescanners
[
name
]
387
end
388 389
setmetatablenewindex
(
interfacescanners
,
function
(
t
,
k
,
v
)
390
report_cld
(
"
don't register scanner %a directly
"
,
k
)
391
-- registerscanner(k,v)
392
end
)
393 394
interfaces
.
scanners
=
storage
.
mark
(
interfacescanners
)
395 396
context
.
functions
=
{
397
register
=
function
(
qualifiedname
)
return
registerfunction
(
qualifiedname
)
end
,
-- only one argument
398
unregister
=
unregisterfunction
,
399
reserve
=
reservefunction
,
400
known
=
knownfunctions
,
401
callonce
=
callfunctiononce
,
402
}
403 404
function
commands
.
ctxfunction
(
code
,
namespace
)
405
context
(
registerfunction
(
code
,
namespace
)
)
406
end
407 408
function
commands
.
ctxscanner
(
name
,
code
,
namespace
)
409
local
n
=
registerfunction
(
code
,
namespace
)
410
if
storedscanners
[
name
]
then
411
storedscanners
[
name
]
=
n
412
end
413
context
(
n
)
414
end
415 416
local
function
dummy
(
)
end
417 418
function
commands
.
ctxresetter
(
name
)
-- to be checked
419
return
function
(
)
420
if
storedscanners
[
name
]
then
421
rawset
(
interfacescanners
,
name
,
dummy
)
422
context
.
resetctxscanner
(
privatenamespace
.
.
name
)
423
end
424
end
425
end
426 427
-- Should we keep the catcodes with the function?
428 429
local
catcodestack
=
{
}
430
local
currentcatcodes
=
ctxcatcodes
431
local
contentcatcodes
=
ctxcatcodes
432 433
local
catcodes
=
{
434
ctx
=
ctxcatcodes
,
ctxcatcodes
=
ctxcatcodes
,
context
=
ctxcatcodes
,
435
prt
=
prtcatcodes
,
prtcatcodes
=
prtcatcodes
,
protect
=
prtcatcodes
,
436
tex
=
texcatcodes
,
texcatcodes
=
texcatcodes
,
plain
=
texcatcodes
,
437
txt
=
txtcatcodes
,
txtcatcodes
=
txtcatcodes
,
text
=
txtcatcodes
,
438
vrb
=
vrbcatcodes
,
vrbcatcodes
=
vrbcatcodes
,
verbatim
=
vrbcatcodes
,
439
xml
=
xmlcatcodes
,
xmlcatcodes
=
xmlcatcodes
,
440
}
441 442
-- maybe just increment / decrement
443 444
-- local function pushcatcodes(c)
445
-- insert(catcodestack,currentcatcodes)
446
-- currentcatcodes = (c and catcodes[c] or tonumber(c)) or currentcatcodes
447
-- contentcatcodes = currentcatcodes
448
-- end
449
--
450
-- local function popcatcodes()
451
-- currentcatcodes = remove(catcodestack) or currentcatcodes
452
-- contentcatcodes = currentcatcodes
453
-- end
454 455
local
catcodelevel
=
0
456 457
local
function
pushcatcodes
(
c
)
458
catcodelevel
=
catcodelevel
+
1
459
catcodestack
[
catcodelevel
]
=
currentcatcodes
460
currentcatcodes
=
(
c
and
catcodes
[
c
]
or
tonumber
(
c
)
)
or
currentcatcodes
461
contentcatcodes
=
currentcatcodes
462
end
463 464
local
function
popcatcodes
(
)
465
if
catcodelevel
>
0
then
466
currentcatcodes
=
catcodestack
[
catcodelevel
]
or
currentcatcodes
467
catcodelevel
=
catcodelevel
-
1
468
end
469
contentcatcodes
=
currentcatcodes
470
end
471 472
function
context
.
unprotect
(
)
473
-- at the lua end
474
catcodelevel
=
catcodelevel
+
1
475
catcodestack
[
catcodelevel
]
=
currentcatcodes
476
currentcatcodes
=
prtcatcodes
477
contentcatcodes
=
prtcatcodes
478
-- at the tex end
479
flush
(
"
\\unprotect
"
)
480
end
481 482
function
context
.
protect
(
)
483
-- at the tex end
484
flush
(
"
\\protect
"
)
485
-- at the lua end
486
if
catcodelevel
>
0
then
487
currentcatcodes
=
catcodestack
[
catcodelevel
]
or
currentcatcodes
488
catcodelevel
=
catcodelevel
-
1
489
end
490
contentcatcodes
=
currentcatcodes
491
end
492 493
context
.
catcodes
=
catcodes
494
context
.
pushcatcodes
=
pushcatcodes
495
context
.
popcatcodes
=
popcatcodes
496 497
-- -- --
498 499
local
newline
=
patterns
.
newline
500
local
space
=
patterns
.
spacer
501
local
spacing
=
newline
*
space
^
0
502
local
content
=
lpegC
(
(
1
-
spacing
)
^
1
)
-- texsprint
503
local
emptyline
=
space
^
0
*
newline
^
2
-- texprint("")
504
local
endofline
=
space
^
0
*
newline
*
space
^
0
-- texsprint(" ")
505
local
simpleline
=
endofline
*
lpegP
(
-1
)
--
506 507
local
verbose
=
lpegC
(
(
1
-
space
-
newline
)
^
1
)
508
local
beginstripper
=
(
lpegS
(
"
\t
"
)
^
1
*
newline
^
1
)
/
"
"
509
local
endstripper
=
beginstripper
*
lpegP
(
-1
)
510 511
local
justaspace
=
space
*
lpegCc
(
"
"
)
512
local
justanewline
=
newline
*
lpegCc
(
"
"
)
513 514
local
function
n_content
(
s
)
flush
(
contentcatcodes
,
s
)
end
515
local
function
n_verbose
(
s
)
flush
(
vrbcatcodes
,
s
)
end
516
local
function
n_endofline
(
)
flush
(
currentcatcodes
,
"
\r
"
)
end
517
local
function
n_emptyline
(
)
flushdirect
(
currentcatcodes
,
"
\r
"
)
end
518
local
function
n_simpleline
(
)
flush
(
currentcatcodes
,
"
\r
"
)
end
519 520
local
n_exception
=
"
"
521 522
-- better a table specification
523 524
function
context
.
newtexthandler
(
specification
)
525
specification
=
specification
or
{
}
526
--
527
local
s_catcodes
=
specification
.
catcodes
528
--
529
local
f_before
=
specification
.
before
530
local
f_after
=
specification
.
after
531
--
532
local
f_endofline
=
specification
.
endofline
or
n_endofline
533
local
f_emptyline
=
specification
.
emptyline
or
n_emptyline
534
local
f_simpleline
=
specification
.
simpleline
or
n_simpleline
535
local
f_content
=
specification
.
content
or
n_content
536
local
f_space
=
specification
.
space
537
--
538
local
p_exception
=
specification
.
exception
539
--
540
if
s_catcodes
then
541
f_content
=
function
(
s
)
542
flush
(
s_catcodes
,
s
)
543
end
544
end
545
--
546
local
pattern
547
if
f_space
then
548
if
p_exception
then
549
local
content
=
lpegC
(
(
1
-
spacing
-
p_exception
)
^
1
)
550
pattern
=
551
(
552
justaspace
/
f_space
553
+
justanewline
/
f_endofline
554
+
p_exception
555
+
content
/
f_content
556
)
^
0
557
else
558
local
content
=
lpegC
(
(
1
-
space
-
endofline
)
^
1
)
559
pattern
=
560
(
561
justaspace
/
f_space
562
+
justanewline
/
f_endofline
563
+
content
/
f_content
564
)
^
0
565
end
566
else
567
if
p_exception
then
568
local
content
=
lpegC
(
(
1
-
spacing
-
p_exception
)
^
1
)
569
pattern
=
570
simpleline
/
f_simpleline
571
+
572
(
573
emptyline
/
f_emptyline
574
+
endofline
/
f_endofline
575
+
p_exception
576
+
content
/
f_content
577
)
^
0
578
else
579
local
content
=
lpegC
(
(
1
-
spacing
)
^
1
)
580
pattern
=
581
simpleline
/
f_simpleline
582
+
583
(
584
emptyline
/
f_emptyline
585
+
endofline
/
f_endofline
586
+
content
/
f_content
587
)
^
0
588
end
589
end
590
--
591
if
f_before
then
592
pattern
=
(
P
(
true
)
/
f_before
)
*
pattern
593
end
594
--
595
if
f_after
then
596
pattern
=
pattern
*
(
P
(
true
)
/
f_after
)
597
end
598
--
599
return
function
(
str
)
return
lpegmatch
(
pattern
,
str
)
end
,
pattern
600
end
601 602
function
context
.
newverbosehandler
(
specification
)
-- a special variant for e.g. cdata in lxml-tex
603
specification
=
specification
or
{
}
604
--
605
local
f_line
=
specification
.
line
or
function
(
)
flushdirect
(
"
\r
"
)
end
606
local
f_space
=
specification
.
space
or
function
(
)
flush
(
"
"
)
end
607
local
f_content
=
specification
.
content
or
n_verbose
608
local
f_before
=
specification
.
before
609
local
f_after
=
specification
.
after
610
--
611
local
pattern
=
612
justanewline
/
f_line
-- so we get call{}
613
+
verbose
/
f_content
614
+
justaspace
/
f_space
-- so we get call{}
615
--
616
if
specification
.
strip
then
617
pattern
=
beginstripper
^
0
*
(
endstripper
+
pattern
)
^
0
618
else
619
pattern
=
pattern
^
0
620
end
621
--
622
if
f_before
then
623
pattern
=
(
lpegP
(
true
)
/
f_before
)
*
pattern
624
end
625
--
626
if
f_after
then
627
pattern
=
pattern
*
(
lpegP
(
true
)
/
f_after
)
628
end
629
--
630
return
function
(
str
)
return
lpegmatch
(
pattern
,
str
)
end
,
pattern
631
end
632 633
local
flushlines
=
context
.
newtexthandler
{
634
content
=
n_content
,
635
endofline
=
n_endofline
,
636
emptyline
=
n_emptyline
,
637
simpleline
=
n_simpleline
,
638
}
639 640
-- The next variant is only used in rare cases (buffer to mp):
641 642
local
printlines_ctx
=
(
643
(
newline
)
/
function
(
)
texprint
(
"
"
)
end
+
644
(
1
-
newline
)
^
1
/
function
(
s
)
texprint
(
ctxcatcodes
,
s
)
end
*
newline
^
-1
645
)
^
0
646 647
local
printlines_raw
=
(
648
(
newline
)
/
function
(
)
texprint
(
"
"
)
end
+
649
(
1
-
newline
)
^
1
/
function
(
s
)
texprint
(
s
)
end
*
newline
^
-1
650
)
^
0
651 652
function
context
.
printlines
(
str
,
raw
)
-- todo: see if via file is useable
653
if
raw
then
654
lpegmatch
(
printlines_raw
,
str
)
655
else
656
lpegmatch
(
printlines_ctx
,
str
)
657
end
658
end
659 660
function
context
.
printtable
(
t
,
separator
)
-- todo: see if via file is useable
661
if
separator
=
=
nil
or
separator
=
=
true
then
662
separator
=
"
\r
"
663
elseif
separator
=
=
"
"
or
separator
=
=
false
then
664
separator
=
"
"
665
end
666
local
s
=
concat
(
t
,
separator
)
667
if
s
~
=
"
"
then
668
context
(
s
)
669
end
670
end
671 672
-- -- -- "{" .. ti .. "}" is somewhat slower in a cld-mkiv run than "{",ti,"}"
673 674
local
containseol
=
patterns
.
containseol
675 676
local
t_cldl_luafunction
=
newtoken
(
"
luafunctioncall
"
,
0
)
677
local
lua_expandable_call_token_code
=
token
.
command_id
and
token
.
command_id
(
"
lua_expandable_call
"
)
678 679
local
sortedhashindeed
=
false
680 681
directives
.
register
(
"
context.sorthash
"
,
function
(
v
)
682
sortedhashindeed
=
v
and
table
.
sortedhash
or
nil
683
end
)
684 685
local
function
writer
(
parent
,
command
,
...
)
-- already optimized before call
686
if
type
(
command
)
=
=
"
string
"
then
-- for now
687
flush
(
currentcatcodes
,
command
)
-- todo: ctx|prt|texcatcodes
688
else
689
flush
(
command
)
-- todo: ctx|prt|texcatcodes
690
end
691
local
direct
=
false
692
for
i
=
1
,
select
(
"
#
"
,
...
)
do
693
local
ti
=
select
(
i
,
...
)
694
if
direct
then
695
local
typ
=
type
(
ti
)
696
if
typ
=
=
"
string
"
or
typ
=
=
"
number
"
then
697
flush
(
currentcatcodes
,
ti
)
698
else
-- node.write
699
report_context
(
"
error: invalid use of direct in %a, only strings and numbers can be flushed directly, not %a
"
,
command
,
typ
)
700
end
701
direct
=
false
702
elseif
ti
=
=
nil
then
703
-- nothing
704
elseif
ti
=
=
"
"
then
705
flush
(
currentcatcodes
,
"
{}
"
)
706
-- elseif ti == 1 then
707
-- flush(currentcatcodes,"{1}")
708
else
709
local
typ
=
type
(
ti
)
710
if
typ
=
=
"
string
"
then
711
-- is processlines seen ?
712
if
processlines
and
lpegmatch
(
containseol
,
ti
)
then
713
flush
(
currentcatcodes
,
"
{
"
)
714
flushlines
(
ti
)
715
flush
(
currentcatcodes
,
"
}
"
)
716
elseif
currentcatcodes
=
=
contentcatcodes
then
717
flush
(
currentcatcodes
,
"
{
"
,
ti
,
"
}
"
)
718
else
719
flush
(
currentcatcodes
,
"
{
"
)
720
flush
(
contentcatcodes
,
ti
)
721
flush
(
currentcatcodes
,
"
}
"
)
722
end
723
elseif
typ
=
=
"
number
"
then
724
-- numbers never have funny catcodesz
725
flush
(
currentcatcodes
,
"
{
"
,
ti
,
"
}
"
)
726
elseif
typ
=
=
"
table
"
then
727
local
tn
=
#
ti
728
if
tn
=
=
0
then
729
local
done
=
false
730
if
sortedhashindeed
then
731
for
k
,
v
in
sortedhashindeed
(
ti
)
do
732
if
done
then
733
if
v
=
=
"
"
then
734
flush
(
currentcatcodes
,
"
,
"
,
k
,
'
=
'
)
735
else
736
flush
(
currentcatcodes
,
"
,
"
,
k
,
"
={
"
,
v
,
"
}
"
)
737
end
738
else
739
if
v
=
=
"
"
then
740
flush
(
currentcatcodes
,
"
[
"
,
k
,
"
=
"
)
741
else
742
flush
(
currentcatcodes
,
"
[
"
,
k
,
"
={
"
,
v
,
"
}
"
)
743
end
744
done
=
true
745
end
746
end
747
else
748
for
k
,
v
in
next
,
ti
do
749
if
done
then
750
if
v
=
=
"
"
then
751
flush
(
currentcatcodes
,
"
,
"
,
k
,
'
=
'
)
752
else
753
flush
(
currentcatcodes
,
"
,
"
,
k
,
"
={
"
,
v
,
"
}
"
)
754
end
755
else
756
if
v
=
=
"
"
then
757
flush
(
currentcatcodes
,
"
[
"
,
k
,
"
=
"
)
758
else
759
flush
(
currentcatcodes
,
"
[
"
,
k
,
"
={
"
,
v
,
"
}
"
)
760
end
761
done
=
true
762
end
763
end
764
end
765
if
done
then
766
flush
(
currentcatcodes
,
"
]
"
)
767
else
768
flush
(
currentcatcodes
,
"
[]
"
)
769
end
770
elseif
tn
=
=
1
then
-- some 20% faster than the next loop
771
local
tj
=
ti
[
1
]
772
if
type
(
tj
)
=
=
"
function
"
then
773
tj
=
storefunction
(
tj
)
774
flush
(
currentcatcodes
,
"
[
"
,
newtoken
(
tj
,
lua_expandable_call_token_code
)
,
"
]
"
)
775
else
776
flush
(
currentcatcodes
,
"
[
"
,
tj
,
"
]
"
)
777
end
778
else
-- is concat really faster than flushes here? probably needed anyway (print artifacts)
779
flush
(
currentcatcodes
,
"
[
"
)
780
for
j
=
1
,
tn
do
781
local
tj
=
ti
[
j
]
782
if
type
(
tj
)
=
=
"
function
"
then
783
tj
=
storefunction
(
tj
)
784
flush
(
currentcatcodes
,
"
[
"
,
newtoken
(
tj
,
lua_expandable_call_token_code
)
,
j
=
=
tn
and
"
]
"
or
"
,
"
)
785
else
786
if
j
=
=
tn
then
787
flush
(
currentcatcodes
,
tj
,
"
]
"
)
788
else
789
flush
(
currentcatcodes
,
tj
,
"
,
"
)
790
end
791
end
792
end
793
end
794
elseif
typ
=
=
"
function
"
then
795
-- todo: ctx|prt|texcatcodes
796
ti
=
storefunction
(
ti
)
797
flush
(
currentcatcodes
,
"
{
"
,
newtoken
(
ti
,
lua_expandable_call_token_code
)
,
"
}
"
)
798
elseif
typ
=
=
"
boolean
"
then
799
if
ti
then
800
flushdirect
(
currentcatcodes
,
"
\r
"
)
801
else
802
direct
=
true
803
end
804
elseif
typ
=
=
"
thread
"
then
805
report_context
(
"
coroutines not supported as we cannot yield across boundaries
"
)
806
elseif
isnode
(
ti
)
then
-- slow | why {} here ?
807
flush
(
currentcatcodes
,
"
{
"
,
ti
,
"
}
"
)
808
else
809
local
s
=
tostring
(
ti
)
810
if
s
then
811
flushdirect
(
currentcatcodes
,
s
)
812
else
813
report_context
(
"
error: %a gets a weird argument %a
"
,
command
,
ti
)
814
end
815
end
816
-- else
817
-- local n = isnode(ti)
818
-- if n then
819
-- flush(ti)
820
-- else
821
-- report_context("error: %a gets a weird argument %a",command,ti)
822
-- end
823
end
824
end
825
end
826 827
local
toks
=
tokens
.
cache
828
context
.
tokenizedcs
=
toks
829 830
local
core
=
setmetatableindex
(
function
(
parent
,
k
)
831
local
t
832
local
f
=
function
(
first
,
...
)
833
if
not
t
then
834
t
=
toks
[
k
]
835
end
836
if
first
=
=
nil
then
837
flush
(
t
)
838
else
839
return
writer
(
context
,
t
,
first
,
...
)
840
end
841
end
842
parent
[
k
]
=
f
843
return
f
844
end
)
845 846
core
.
cs
=
setmetatableindex
(
function
(
parent
,
k
)
847
local
t
848
local
f
=
function
(
)
849
if
not
t
then
850
t
=
toks
[
k
]
851
end
852
flush
(
t
)
853
end
854
parent
[
k
]
=
f
855
return
f
856
end
)
857 858
local
indexer
=
function
(
parent
,
k
)
859
if
type
(
k
)
=
=
"
string
"
then
860
return
core
[
k
]
861
else
862
return
context
-- catch
863
end
864
end
865 866
context
.
core
=
core
867 868
-- only for internal usage:
869 870
-- local prtindexer = nil
871
--
872
-- do
873
--
874
-- -- the only variant is not much faster than the full but it's more
875
-- -- memory efficient
876
--
877
-- local protected = { }
878
-- local protectedcs = { }
879
-- context.protected = protected
880
-- context.protectedcs = protectedcs
881
--
882
-- local function fullindexer(t,k)
883
-- local c = "\\" .. k -- tostring(k)
884
-- local v = function(first,...)
885
-- if first == nil then
886
-- flush(prtcatcodes,c)
887
-- else
888
-- return prtwriter(c,first,...)
889
-- end
890
-- end
891
-- rawset(t,k,v) -- protected namespace
892
-- return v
893
-- end
894
--
895
-- local function onlyindexer(t,k)
896
-- local c = "\\" .. k -- tostring(k)
897
-- local v = function()
898
-- flush(prtcatcodes,c)
899
-- end
900
-- rawset(protected,k,v)
901
-- rawset(t,k,v)
902
-- return v
903
-- end
904
--
905
-- protected.cs = setmetatableindex(function(parent,k)
906
-- local c = "\\" .. k -- tostring(k)
907
-- local v = function()
908
-- flush(prtcatcodes,c)
909
-- end
910
-- parent[k] = v
911
-- return v
912
-- end
913
--
914
-- setmetatableindex(protected,fullindexer)
915
-- setmetatablecall (protected,prtwriter)
916
--
917
-- setmetatableindex(protectedcs,onlyindexer)
918
-- setmetatablecall (protectedcs,prtwriter)
919
--
920
-- end
921 922
-- local splitformatters = utilities.strings.formatters.new(true) -- not faster (yet)
923 924
local
caller
=
function
(
parent
,
f
,
a
,
...
)
925
if
not
parent
then
926
-- so we don't need to test in the calling (slower but often no issue)
927
elseif
f
~
=
nil
then
928
local
typ
=
type
(
f
)
929
if
typ
=
=
"
string
"
then
930
if
f
=
=
"
"
then
931
-- new, can save a bit sometimes
932
-- if trace_context then
933
-- report_context("empty argument to context()")
934
-- end
935
elseif
a
then
936
flush
(
contentcatcodes
,
formatters
[
f
]
(
a
,
...
)
)
-- was currentcatcodes
937
-- flush(contentcatcodes,splitformatters[f](a,...)) -- was currentcatcodes
938
elseif
processlines
and
lpegmatch
(
containseol
,
f
)
then
939
flushlines
(
f
)
940
else
941
flush
(
contentcatcodes
,
f
)
942
end
943
elseif
typ
=
=
"
number
"
then
944
if
a
then
945
flush
(
currentcatcodes
,
f
,
a
,
...
)
946
else
947
flush
(
currentcatcodes
,
f
)
948
end
949
elseif
typ
=
=
"
function
"
then
950
-- ignored: a ...
951
f
=
storefunction
(
f
)
952
flush
(
currentcatcodes
,
"
{
"
,
newtoken
(
f
,
lua_expandable_call_token_code
)
,
"
}
"
)
953
elseif
typ
=
=
"
boolean
"
then
954
if
f
then
955
if
a
~
=
nil
then
956
flushlines
(
a
)
957
else
958
flushdirect
(
currentcatcodes
,
"
\n
"
)
-- no \r, else issues with \startlines ... use context.par() otherwise
959
end
960
else
961
if
a
~
=
nil
then
962
-- no command, same as context(a,...)
963
writer
(
parent
,
"
"
,
a
,
...
)
964
else
965
-- ignored
966
end
967
end
968
elseif
typ
=
=
"
thread
"
then
969
report_context
(
"
coroutines not supported as we cannot yield across boundaries
"
)
970
elseif
isnode
(
f
)
then
-- slow
971
flush
(
f
)
972
else
973
local
s
=
tostring
(
f
)
974
if
s
then
975
flushdirect
(
currentcatcodes
,
s
)
976
else
977
report_context
(
"
error: %a gets a weird argument %a
"
,
"
context
"
,
f
)
978
end
979
end
980
-- else
981
-- local n = isnode(f)
982
-- if n then
983
-- flush(f)
984
-- else
985
-- report_context("error: %a gets a weird argument %a","context",f)
986
-- end
987
end
988
end
989 990
context
.
nodes
=
{
-- todo
991
store
=
storenode
,
992
flush
=
function
(
n
)
993
flush
(
n
)
994
end
,
995
}
996 997
context
.
nuts
=
{
-- todo
998
store
=
function
(
n
)
999
return
storenode
(
tonut
(
n
)
)
1000
end
,
1001
flush
=
function
(
n
,
d
)
1002
flush
(
tonode
(
n
)
)
1003
end
,
1004
}
1005 1006
local
defaultcaller
=
caller
1007 1008
setmetatableindex
(
context
,
indexer
)
1009
setmetatablecall
(
context
,
caller
)
1010 1011
function
context
.
sprint
(
...
)
-- takes catcodes as first argument
1012
flush
(
...
)
1013
end
1014 1015
function
context
.
fprint
(
first
,
second
,
third
,
...
)
1016
if
type
(
first
)
=
=
"
number
"
then
1017
if
third
then
1018
flush
(
first
,
formatters
[
second
]
(
third
,
...
)
)
1019
else
1020
flush
(
first
,
second
)
1021
end
1022
else
1023
if
second
then
1024
flush
(
formatters
[
first
]
(
second
,
third
,
...
)
)
1025
else
1026
flush
(
first
)
1027
end
1028
end
1029
end
1030 1031
tex
.
fprint
=
context
.
fprint
1032 1033
-- logging
1034 1035
local
trace_stack
=
{
report_context
}
1036 1037
local
normalflush
=
flush
1038
local
normalflushdirect
=
flushdirect
1039
----- normalflushraw = flushraw
1040
local
normalwriter
=
writer
1041
local
currenttrace
=
report_context
1042
local
nofwriters
=
0
1043
local
nofflushes
=
0
1044 1045
local
tracingpermitted
=
true
1046 1047
local
visualizer
=
lpeg
.
replacer
{
1048
{
"
\n
"
,
"
<<newline>>
"
}
,
1049
{
"
\r
"
,
"
<<par>>
"
}
,
1050
}
1051 1052
statistics
.
register
(
"
traced context
"
,
function
(
)
1053
local
used
,
freed
=
usedstack
(
)
1054
local
unreachable
=
used
-
freed
1055
if
nofwriters
>
0
or
nofflushes
>
0
then
1056
return
format
(
"
writers: %s, flushes: %s, maxstack: %s
"
,
nofwriters
,
nofflushes
,
used
,
freed
,
unreachable
)
1057
elseif
showstackusage
or
unreachable
>
0
then
1058
return
format
(
"
maxstack: %s, freed: %s, unreachable: %s
"
,
used
,
freed
,
unreachable
)
1059
end
1060
end
)
1061 1062
local
function
userdata
(
argument
)
1063
if
isnode
(
argument
)
then
1064
return
formatters
[
"
<< %s node %i>>
"
]
(
nodes
.
nodecodes
[
argument
.
id
]
,
tonut
(
argument
)
)
1065
end
1066
if
istoken
(
argument
)
then
1067
local
csname
=
argument
.
csname
1068
if
csname
then
1069
-- return formatters["<<\\%s>>"](csname)
1070
return
formatters
[
"
\\%s
"
]
(
csname
)
1071
end
1072
local
cmdname
=
argument
.
cmdname
1073
if
cmdname
=
=
"
lua_expandable_call
"
or
cmdname
=
=
"
lua_call
"
then
1074
return
"
<<function>>
"
-- argument.mode
1075
end
1076
return
"
<<token>>
"
1077
end
1078
return
"
<<userdata>>
"
1079
end
1080 1081 1082
local
tracedwriter
=
function
(
parent
,
...
)
-- also catcodes ?
1083
nofwriters
=
nofwriters
+
1
1084
local
savedflush
=
flush
1085
local
savedflushdirect
=
flushdirect
-- unlikely to be used here
1086
local
t
=
{
"
w : - :
"
}
1087
local
n
=
1
1088
local
traced
=
function
(
catcodes
,
...
)
-- todo: check for catcodes
1089
local
s
=
type
(
catcodes
)
=
=
"
number
"
and
{
...
}
or
{
catcodes
,
...
}
1090
for
i
=
1
,
#
s
do
1091
local
argument
=
s
[
i
]
1092
local
argtype
=
type
(
argument
)
1093
if
argtype
=
=
"
string
"
then
1094
s
[
i
]
=
lpegmatch
(
visualizer
,
argument
)
1095
elseif
argtype
=
=
"
number
"
then
1096
s
[
i
]
=
argument
1097
elseif
argtype
=
=
"
userdata
"
then
1098
s
[
i
]
=
userdata
(
argument
)
1099
else
1100
s
[
i
]
=
formatters
[
"
<<%S>>
"
]
(
argument
)
1101
end
1102
end
1103
s
=
concat
(
s
)
1104
s
=
lpegmatch
(
visualizer
,
s
)
1105
n
=
n
+
1
1106
t
[
n
]
=
s
1107
end
1108
flush
=
function
(
...
)
1109
normalflush
(
...
)
1110
if
tracingpermitted
then
1111
traced
(
...
)
1112
end
1113
end
1114
flushdirect
=
function
(
...
)
1115
normalflushdirect
(
...
)
1116
if
tracingpermitted
then
1117
traced
(
...
)
1118
end
1119
end
1120
normalwriter
(
parent
,
...
)
1121
flush
=
savedflush
1122
flushdirect
=
savedflushdirect
1123
currenttrace
(
concat
(
t
)
)
1124
end
1125 1126
-- we could reuse collapsed
1127 1128
local
traced
=
function
(
one
,
two
,
...
)
1129
if
two
~
=
nil
then
1130
-- only catcodes if 'one' is number
1131
local
catcodes
=
type
(
one
)
=
=
"
number
"
and
one
1132
local
arguments
=
catcodes
and
{
two
,
...
}
or
{
one
,
two
,
...
}
1133
local
collapsed
=
{
formatters
[
"
f : %s :
"
]
(
catcodes
or
'
-
'
)
}
1134
local
c
=
1
1135
for
i
=
1
,
#
arguments
do
1136
local
argument
=
arguments
[
i
]
1137
local
argtype
=
type
(
argument
)
1138
c
=
c
+
1
1139
if
argtype
=
=
"
string
"
then
1140
collapsed
[
c
]
=
lpegmatch
(
visualizer
,
argument
)
1141
elseif
argtype
=
=
"
number
"
then
1142
collapsed
[
c
]
=
argument
1143
elseif
argtype
=
=
"
userdata
"
then
1144
collapsed
[
c
]
=
userdata
(
argument
)
1145
else
1146
collapsed
[
c
]
=
formatters
[
"
<<%S>>
"
]
(
argument
)
1147
end
1148
end
1149
currenttrace
(
concat
(
collapsed
)
)
1150
elseif
one
~
=
nil
then
1151
-- no catcodes
1152
local
argtype
=
type
(
one
)
1153
if
argtype
=
=
"
string
"
then
1154
currenttrace
(
formatters
[
"
f : - : %s
"
]
(
lpegmatch
(
visualizer
,
one
)
)
)
1155
elseif
argtype
=
=
"
number
"
then
1156
currenttrace
(
formatters
[
"
f : - : %s
"
]
(
one
)
)
1157
elseif
argtype
=
=
"
userdata
"
then
1158
currenttrace
(
formatters
[
"
F : - : %s
"
]
(
userdata
(
one
)
)
)
1159
else
1160
currenttrace
(
formatters
[
"
f : - : <<%S>>
"
]
(
one
)
)
1161
end
1162
end
1163
end
1164 1165
local
tracedflush
=
function
(
one
,
two
,
...
)
1166
nofflushes
=
nofflushes
+
1
1167
if
two
~
=
nil
then
1168
normalflush
(
one
,
two
,
...
)
1169
else
1170
normalflush
(
one
)
1171
end
1172
if
tracingpermitted
then
1173
traced
(
one
,
two
,
...
)
1174
end
1175
end
1176 1177
local
tracedflushdirect
=
function
(
one
,
two
,
...
)
1178
nofflushes
=
nofflushes
+
1
1179
if
two
~
=
nil
then
1180
normalflushdirect
(
one
,
two
,
...
)
1181
else
1182
normalflushdirect
(
one
)
1183
end
1184
if
tracingpermitted
then
1185
traced
(
one
,
two
,
...
)
1186
end
1187
end
1188 1189
function
context
.
pushlogger
(
trace
)
1190
trace
=
trace
or
report_context
1191
insert
(
trace_stack
,
currenttrace
)
1192
currenttrace
=
trace
1193
end
1194 1195
function
context
.
poplogger
(
)
1196
if
#
trace_stack
>
1
then
1197
currenttrace
=
remove
(
trace_stack
)
or
report_context
1198
else
1199
currenttrace
=
report_context
1200
end
1201
end
1202 1203
function
context
.
settracing
(
v
)
1204
if
v
then
1205
flush
=
tracedflush
1206
flushdirect
=
tracedflushdirect
1207
writer
=
tracedwriter
1208
else
1209
flush
=
normalflush
1210
flushdirect
=
normalflushdirect
1211
writer
=
normalwriter
1212
end
1213
return
flush
,
writer
,
flushdirect
1214
end
1215 1216
function
context
.
getlogger
(
)
1217
return
flush
,
writer
,
flushdirect
1218
end
1219 1220
trackers
.
register
(
"
context.trace
"
,
context
.
settracing
)
1221 1222
local
trace_cld
=
false
trackers
.
register
(
"
context.files
"
,
function
(
v
)
trace_cld
=
v
end
)
1223 1224
do
1225 1226
-- This is the most reliable way to deal with nested buffers and other
1227
-- catcode sensitive data.
1228 1229
local
resolve
=
resolvers
.
savers
.
byscheme
1230
local
validstring
=
string
.
valid
1231
local
input
=
context
.
input
1232 1233
local
function
viafile
(
data
,
tag
)
1234
if
data
and
data
~
=
"
"
then
1235
local
filename
=
resolve
(
"
virtual
"
,
validstring
(
tag
,
"
viafile
"
)
,
data
)
1236
-- context.startregime { "utf" }
1237
input
(
filename
)
1238
-- context.stopregime()
1239
end
1240
end
1241 1242
context
.
viafile
=
viafile
1243 1244
-- experiment for xtables, don't use it elsewhere yet
1245 1246
local
collected
=
nil
1247
local
nofcollected
=
0
1248
local
sentinel
=
string
.
char
(
26
)
-- ASCII SUB character : endoffileasciicode : ignorecatcode
1249
local
level
=
0
1250 1251
local
function
collect
(
c
,
a
,
...
)
-- can be optimized
1252
if
type
(
c
)
=
=
"
userdata
"
then
1253
nofcollected
=
nofcollected
+
1
1254
-- collected[nofcollected] = userdata(c)
1255
collected
[
nofcollected
]
=
"
\\
"
.
.
c
.
csname
1256
end
1257
if
a
then
1258
for
i
=
1
,
select
(
"
#
"
,
a
,
...
)
do
1259
local
c
=
select
(
i
,
a
,
...
)
1260
nofcollected
=
nofcollected
+
1
1261
collected
[
nofcollected
]
=
type
(
c
)
=
=
"
userdata
"
and
userdata
(
c
)
or
c
1262
end
1263
end
1264
end
1265 1266
local
collectdirect
=
collect
1267
local
permitted
=
true
1268 1269
-- doesn't work well with tracing do we need to avoid that when
1270
-- collecting stuff
1271 1272
function
context
.
startcollecting
(
)
1273
if
level
=
=
0
then
1274
collected
=
{
}
1275
nofcollected
=
0
1276
flush
=
collect
1277
flushdirect
=
collectdirect
1278
permitted
=
tracingpermitted
1279
end
1280
level
=
level
+
1
1281
end
1282 1283
function
context
.
stopcollecting
(
)
1284
level
=
level
-
1
1285
if
level
<
1
then
1286
local
result
=
concat
(
collected
,
sentinel
)
1287
flush
=
normalflush
1288
flushdirect
=
normalflushdirect
1289
tracingpermitted
=
permitted
1290
collected
=
nil
1291
nofcollected
=
0
1292
level
=
0
1293
viafile
(
result
)
1294
end
1295
end
1296 1297
local
findtexfile
=
resolvers
.
findtexfile
1298
local
findfile
=
resolvers
.
findfile
1299 1300
function
context
.
runfile
(
filename
)
1301
local
foundname
=
findtexfile
(
file
.
addsuffix
(
filename
,
"
cld
"
)
)
or
"
"
1302
if
foundname
~
=
"
"
then
1303
local
ok
=
dofile
(
foundname
)
1304
if
type
(
ok
)
=
=
"
function
"
then
1305
if
trace_cld
then
1306
report_context
(
"
begin of file %a (function call)
"
,
foundname
)
1307
end
1308
ok
(
)
1309
if
trace_cld
then
1310
report_context
(
"
end of file %a (function call)
"
,
foundname
)
1311
end
1312
elseif
ok
then
1313
report_context
(
"
file %a is processed and returns true
"
,
foundname
)
1314
else
1315
report_context
(
"
file %a is processed and returns nothing
"
,
foundname
)
1316
end
1317
else
1318
report_context
(
"
unknown file %a
"
,
filename
)
1319
end
1320
end
1321 1322
function
context
.
loadfile
(
filename
)
1323
context
(
stripstring
(
loaddata
(
findfile
(
filename
)
)
)
)
1324
end
1325 1326
function
context
.
loadviafile
(
filename
)
1327
viafile
(
stripstring
(
loaddata
(
findfile
(
filename
)
)
)
)
1328
end
1329 1330
end
1331 1332
-- some functions
1333 1334
function
context
.
direct
(
first
,
...
)
1335
if
first
~
=
nil
then
1336
return
writer
(
context
,
"
"
,
first
,
...
)
1337
end
1338
end
1339 1340
-- context.delayed (todo: lines)
1341 1342
do
1343 1344
local
delayed
=
{
}
1345 1346
local
function
indexer
(
parent
,
k
)
1347
local
f
=
function
(
...
)
1348
local
a
=
{
...
}
-- this also freezes ...
1349
return
function
(
)
1350
-- return context[k](unpack(a))
1351
return
core
[
k
]
(
unpack
(
a
)
)
1352
end
1353
end
1354
parent
[
k
]
=
f
1355
return
f
1356
end
1357 1358
local
function
caller
(
parent
,
...
)
-- todo: nodes
1359
local
a
=
{
...
}
1360
return
function
(
)
1361
-- return context(unpack(a))
1362
return
defaultcaller
(
context
,
unpack
(
a
)
)
1363
end
1364
end
1365 1366
setmetatableindex
(
delayed
,
indexer
)
1367
setmetatablecall
(
delayed
,
caller
)
1368 1369
context
.
delayed
=
delayed
1370 1371
end
1372 1373
-- do
1374
--
1375
-- -- context.nested (todo: lines), creates strings
1376
--
1377
-- local nested = { }
1378
--
1379
-- local function indexer(parent,k) -- not ok when traced
1380
-- local f = function(...)
1381
-- local t, savedflush, n = { }, flush, 0
1382
-- flush = function(c,f,s,...) -- catcodes are ignored
1383
-- n = n + 1
1384
-- t[n] = s and concat{f,s,...} or f -- optimized for #args == 1
1385
-- end
1386
-- -- context[k](...)
1387
-- core[k](...)
1388
-- flush = savedflush
1389
-- return concat(t)
1390
-- end
1391
-- parent[k] = f
1392
-- return f
1393
-- end
1394
--
1395
-- local function caller(parent,...)
1396
-- local t, savedflush, n = { }, flush, 0
1397
-- flush = function(c,f,s,...) -- catcodes are ignored
1398
-- n = n + 1
1399
-- t[n] = s and concat{f,s,...} or f -- optimized for #args == 1
1400
-- end
1401
-- -- context(...)
1402
-- defaultcaller(context,...)
1403
-- flush = savedflush
1404
-- return concat(t)
1405
-- end
1406
--
1407
-- setmetatableindex(nested,indexer)
1408
-- setmetatablecall (nested,caller)
1409
--
1410
-- context.nested = nested
1411
--
1412
-- end
1413 1414
context
.
nested
=
context
.
delayed
1415 1416
-- verbatim
1417 1418
function
context
.
newindexer
(
catcodes
,
cmdcodes
)
1419
local
handler
=
{
}
1420 1421
local
function
indexer
(
parent
,
k
)
1422
local
command
=
core
[
k
]
1423
local
f
=
function
(
...
)
1424
local
savedcatcodes
=
contentcatcodes
1425
contentcatcodes
=
catcodes
1426
command
(
...
)
1427
contentcatcodes
=
savedcatcodes
1428
end
1429
parent
[
k
]
=
f
1430
return
f
1431
end
1432 1433
local
function
caller
(
parent
,
...
)
1434
local
savedcatcodes
=
contentcatcodes
1435
contentcatcodes
=
catcodes
1436
defaultcaller
(
parent
,
...
)
1437
contentcatcodes
=
savedcatcodes
1438
end
1439 1440
handler
.
cs
=
core
.
cs
1441 1442
setmetatableindex
(
handler
,
indexer
)
1443
setmetatablecall
(
handler
,
caller
)
1444 1445
return
handler
1446
end
1447 1448
context
.
verbatim
=
context
.
newindexer
(
vrbcatcodes
,
ctxcatcodes
)
1449
context
.
puretext
=
context
.
newindexer
(
txtcatcodes
,
ctxcatcodes
)
1450
context
.
protected
=
context
.
newindexer
(
prtcatcodes
,
prtcatcodes
)
1451 1452
-- formatted
1453 1454
do
1455 1456
local
formatted
=
{
}
1457 1458
-- formatted.command([catcodes,]format[,...])
1459 1460
local
function
formattedflush
(
parent
,
c
,
catcodes
,
fmt
,
...
)
1461
if
not
catcodes
then
1462
return
writer
(
parent
,
c
)
1463
elseif
not
fmt
then
1464
return
writer
(
parent
,
c
,
catcodes
)
1465
elseif
type
(
catcodes
)
=
=
"
number
"
then
1466
local
result
1467
pushcatcodes
(
catcodes
)
1468
result
=
writer
(
parent
,
c
,
formatters
[
fmt
]
(
...
)
)
1469
popcatcodes
(
)
1470
return
result
1471
else
1472
return
writer
(
parent
,
c
,
formatters
[
catcodes
]
(
fmt
,
...
)
)
1473
end
1474
end
1475 1476
local
toks
=
tokens
.
cache
1477 1478
local
indexer
=
function
(
parent
,
k
)
1479
if
type
(
k
)
=
=
"
string
"
then
1480
local
t
1481
local
f
=
function
(
first
,
...
)
1482
if
not
t
then
1483
t
=
toks
[
k
]
1484
end
1485
if
first
=
=
nil
then
1486
flush
(
t
)
1487
else
1488
return
formattedflush
(
parent
,
t
,
first
,
...
)
1489
end
1490
end
1491
parent
[
k
]
=
f
1492
return
f
1493
else
1494
return
context
-- catch
1495
end
1496
end
1497 1498
-- formatted([catcodes,]format[,...])
1499 1500
local
function
caller
(
parent
,
catcodes
,
fmt
,
...
)
1501
if
not
catcodes
then
1502
-- nothing
1503
elseif
not
fmt
then
1504
flush
(
catcodes
)
1505
elseif
type
(
catcodes
)
=
=
"
number
"
then
1506
flush
(
catcodes
,
formatters
[
fmt
]
(
...
)
)
1507
else
1508
flush
(
formatters
[
catcodes
]
(
fmt
,
...
)
)
1509
end
1510
end
1511 1512
setmetatableindex
(
formatted
,
indexer
)
1513
setmetatablecall
(
formatted
,
caller
)
1514 1515
context
.
formatted
=
formatted
1516 1517
end
1518 1519
do
1520 1521
-- metafun (this will move to another file)
1522 1523
local
metafun
=
{
}
1524 1525
function
metafun
.
start
(
)
1526
context
.
startMPcode
(
)
1527
end
1528 1529
function
metafun
.
stop
(
)
1530
context
.
stopMPcode
(
)
1531
end
1532 1533
setmetatablecall
(
metafun
,
defaultcaller
)
1534 1535
function
metafun
.
color
(
name
)
-- obsolete
1536
return
name
-- formatters[ [[\MPcolor{%s}]] ](name)
1537
end
1538 1539
-- metafun.delayed
1540 1541
local
delayed
=
{
}
1542 1543
local
function
indexer
(
parent
,
k
)
1544
local
f
=
function
(
...
)
1545
local
a
=
{
...
}
1546
return
function
(
)
1547
return
metafun
[
k
]
(
unpack
(
a
)
)
1548
end
1549
end
1550
parent
[
k
]
=
f
1551
return
f
1552
end
1553 1554 1555
local
function
caller
(
parent
,
...
)
1556
local
a
=
{
...
}
1557
return
function
(
)
1558
return
metafun
(
unpack
(
a
)
)
1559
end
1560
end
1561 1562
setmetatableindex
(
delayed
,
indexer
)
1563
setmetatablecall
(
delayed
,
caller
)
1564 1565
context
.
metafun
=
metafun
1566
metafun
.
delayed
=
delayed
1567 1568
end
1569 1570
-- helpers:
1571 1572
do
1573 1574
function
context
.
concat
(
...
)
1575
context
(
concat
(
...
)
)
1576
end
1577 1578
local
p_texescape
=
patterns
.
texescape
1579 1580
function
context
.
escaped
(
s
)
1581
if
s
then
1582
context
(
lpegmatch
(
p_texescape
,
s
)
or
s
)
1583
else
1584
-- context("")
1585
end
1586
end
1587 1588
function
context
.
escape
(
s
)
1589
if
s
then
1590
return
lpegmatch
(
p_texescape
,
s
)
or
s
1591
else
1592
return
"
"
1593
end
1594
end
1595 1596
end
1597 1598
-- templates
1599 1600
do
1601 1602
local
single
=
lpegP
(
"
%
"
)
1603
local
double
=
lpegP
(
"
%%
"
)
1604
local
lquoted
=
lpegP
(
"
%[
"
)
1605
local
rquoted
=
lpegP
(
"
]%
"
)
1606
local
space
=
lpegP
(
"
"
)
1607 1608
local
start
=
[[
1609 local texescape = lpeg.patterns.texescape 1610 local lpegmatch = lpeg.match 1611 return function(variables) return 1612
]]
1613 1614
local
stop
=
[[
1615 end 1616
]]
1617 1618
local
replacer
=
lpegP
{
"
parser
"
,
1619
parser
=
lpegCs
(
lpegCc
(
start
)
*
lpegV
(
"
step
"
)
*
(
lpegCc
(
"
..
"
)
*
lpegV
(
"
step
"
)
)
^
0
*
lpegCc
(
stop
)
)
,
1620
unquoted
=
(
lquoted
*
space
/
'
'
)
1621
*
(
(
lpegC
(
(
1
-
space
*
rquoted
)
^
1
)
)
/
"
lpegmatch(texescape,variables%0 or '')
"
)
1622
*
(
space
*
rquoted
/
'
'
)
1623
+
(
lquoted
/
'
'
)
1624
*
(
(
lpegC
(
(
1
-
rquoted
)
^
1
)
)
/
"
lpegmatch(texescape,variables['%0'] or '')
"
)
1625
*
(
rquoted
/
'
'
)
,
1626
key
=
(
single
*
space
/
'
'
)
1627
*
(
(
lpegC
(
(
1
-
space
*
single
)
^
1
)
)
/
"
(variables%0 or '')
"
)
1628
*
(
space
*
single
/
'
'
)
1629
+
(
single
/
'
'
)
1630
*
(
(
lpegC
(
(
1
-
single
)
^
1
)
)
/
"
(variables['%0'] or '')
"
)
1631
*
(
single
/
'
'
)
,
1632
escape
=
double
/
'
%%
'
,
1633
step
=
lpegV
(
"
unquoted
"
)
1634
+
lpegV
(
"
escape
"
)
1635
+
lpegV
(
"
key
"
)
1636
+
lpegCc
(
"
\n[===[
"
)
*
(
1
-
lpegV
(
"
unquoted
"
)
-
lpegV
(
"
escape
"
)
-
lpegV
(
"
key
"
)
)
^
1
*
lpegCc
(
"
]===]\n
"
)
,
1637
}
1638 1639
local
templates
=
{
}
1640 1641
local
function
indexer
(
parent
,
k
)
1642
local
v
=
lpegmatch
(
replacer
,
k
)
1643
if
not
v
then
1644
-- report_template("invalid template:\n%s",k)
1645
v
=
"
error: no valid template (1)
"
1646
else
1647
local
f
=
loadstring
(
v
)
1648
if
type
(
f
)
~
=
"
function
"
then
1649
-- report_template("invalid template:\n%s\n=>\n%s",k,v)
1650
v
=
"
error: no valid template (2)
"
1651
else
1652
f
=
f
(
)
1653
if
not
f
then
1654
-- report_template("invalid template:\n%s\n=>\n%s",k,v)
1655
v
=
"
error: no valid template (3)
"
1656
else
1657
v
=
f
1658
end
1659
end
1660
end
1661
if
type
(
v
)
=
=
"
function
"
then
1662
local
f
=
function
(
first
,
second
)
1663
if
second
then
1664
pushcatcodes
(
first
)
1665
flushlines
(
v
(
second
)
)
1666
popcatcodes
(
)
1667
else
1668
flushlines
(
v
(
first
)
)
1669
end
1670
end
1671
parent
[
k
]
=
f
1672
return
f
1673
else
1674
return
function
(
)
1675
flush
(
v
)
1676
end
1677
end
1678 1679
end
1680 1681
local
function
caller
(
parent
,
k
,
...
)
1682
return
parent
[
k
]
(
...
)
1683
end
1684 1685
setmetatableindex
(
templates
,
indexer
)
1686
setmetatablecall
(
templates
,
caller
)
1687 1688
context
.
templates
=
templates
1689 1690
end
1691 1692
-- The above is a bit over the top as we could also stick to a simple context.replace
1693
-- which is fast enough anyway, but the above fits in nicer, also with the catcodes.
1694
--
1695
-- local replace = utilities.templates.replace
1696
--
1697
-- function context.template(template,variables)
1698
-- context(replace(template,variables))
1699
-- end
1700 1701
do
1702 1703
-- not the best namespace -- some day maybe texs a la nodes and tokens .. also,
1704
-- we already have tex.modes so we need a different name
1705 1706
local
modelevels
=
tex
.
getmodevalues
(
)
1707
local
t
=
table
.
keys
(
modelevels
)
1708
tex
.
modelevels
=
table
.
swapped
(
modelevels
,
modelevels
)
1709 1710
for
i
=
1
,
#
t
do
local
k
=
t
[
i
]
modelevels
[
-
k
]
=
modelevels
[
k
]
end
1711 1712
end
1713