core-uti.lua /size: 15 Kb    last modification: 2020-07-01 14:35
1
if
not
modules
then
modules
=
{
}
end
modules
[
'
core-uti
'
]
=
{
2
version
=
1
.
001
,
3
comment
=
"
companion to core-uti.mkiv
"
,
4
author
=
"
Hans Hagen, PRAGMA-ADE, Hasselt NL
"
,
5
copyright
=
"
PRAGMA ADE / ConTeXt Development Team
"
,
6
license
=
"
see context related readme files
"
7
}
8 9
-- todo: keep track of changes here (hm, track access, and only true when
10
-- accessed and changed)
11 12
--[[ldx-- 13<p>A utility file has always been part of <l n='context'/> and with 14the move to <l n='luatex'/> we also moved a lot of multi-pass info 15to a <l n='lua'/> table. Instead of loading a <l n='tex'/> based 16utility file under different setups, we now load a table once. This 17saves much runtime but at the cost of more memory usage.</p> 18--ldx]]
--
19 20
local
math
=
math
21
local
format
,
match
=
string
.
format
,
string
.
match
22
local
next
,
type
,
tostring
,
tonumber
=
next
,
type
,
tostring
,
tonumber
23
local
concat
=
table
.
concat
24 25
local
definetable
=
utilities
.
tables
.
definetable
26
local
accesstable
=
utilities
.
tables
.
accesstable
27
local
migratetable
=
utilities
.
tables
.
migratetable
28
local
serialize
=
table
.
serialize
29
local
packers
=
utilities
.
packers
30
local
allocate
=
utilities
.
storage
.
allocate
31
local
mark
=
utilities
.
storage
.
mark
32 33
local
getrandom
=
utilities
.
randomizer
.
get
34
local
setrandomseedi
=
utilities
.
randomizer
.
setseedi
35
local
getrandomseed
=
utilities
.
randomizer
.
getseed
36 37
local
implement
=
interfaces
.
implement
38 39
local
texgetcount
=
tex
.
getcount
40 41
local
report_passes
=
logs
.
reporter
(
"
job
"
,
"
passes
"
)
42 43
job
=
job
or
{
}
44
local
job
=
job
45 46
job
.
version
=
1
.
31
47
job
.
packversion
=
1
.
02
48 49
-- some day we will implement loading of other jobs and then we need
50
-- job.jobs
51 52
--[[ldx-- 53<p>Variables are saved using in the previously defined table and passed 54onto <l n='tex'/> using the following method. Of course one can also 55directly access the variable using a <l n='lua'/> call.</p> 56--ldx]]
--
57 58
local
savelist
,
comment
=
{
}
,
{
}
59 60
function
job
.
comment
(
key
,
value
)
61
if
type
(
key
)
=
=
"
table
"
then
62
for
k
,
v
in
next
,
key
do
63
comment
[
k
]
=
v
64
end
65
else
66
comment
[
key
]
=
value
67
end
68
end
69 70
job
.
comment
(
"
version
"
,
job
.
version
)
71 72
local
enabled
=
true
73
local
initialized
=
false
74 75
directives
.
register
(
"
job.save
"
,
function
(
v
)
enabled
=
v
end
)
76
----------.register("job.keep",function(v) kept = v end)
77 78
function
job
.
disablesave
(
)
-- can be command
79
enabled
=
false
80
end
81 82
function
job
.
initialize
(
loadname
,
savename
)
83
if
not
initialized
then
84
if
not
loadname
or
loadname
=
=
"
"
then
85
loadname
=
tex
.
jobname
.
.
"
.tuc
"
86
end
87
if
not
savename
or
savename
=
=
"
"
then
88
savename
=
tex
.
jobname
.
.
"
.tua
"
89
end
90
job
.
load
(
loadname
)
-- has to come after structure is defined !
91
luatex
.
registerstopactions
(
function
(
)
92
if
enabled
and
not
status
.
lasterrorstring
or
status
.
lasterrorstring
=
=
"
"
then
93
-- if kept then
94
-- job.keep(loadname) -- could move to mtx-context instead
95
-- end
96
job
.
save
(
savename
)
97
end
98
end
)
99
initialized
=
true
100
end
101
end
102 103
function
job
.
register
(
collected
,
tobesaved
,
initializer
,
finalizer
,
serializer
)
104
savelist
[
#
savelist
+
1
]
=
{
collected
,
tobesaved
,
initializer
,
finalizer
,
serializer
}
105
end
106 107
-- as an example we implement variables
108 109
local
tobesaved
,
collected
,
checksums
=
allocate
(
)
,
allocate
(
)
,
allocate
(
)
110 111
local
jobvariables
=
{
112
collected
=
collected
,
113
tobesaved
=
tobesaved
,
114
checksums
=
checksums
,
115
}
116 117
-- if not checksums.old then checksums.old = md5.HEX("old") end -- used in experiment
118
-- if not checksums.new then checksums.new = md5.HEX("new") end -- used in experiment
119 120
job
.
variables
=
jobvariables
121 122
local
function
initializer
(
)
123
checksums
=
jobvariables
.
checksums
124
end
125 126
job
.
register
(
'
job.variables.checksums
'
,
'
job.variables.checksums
'
,
initializer
)
127 128
local
rmethod
,
rvalue
129
local
collectedmacros
,
tobesavedmacros
130 131
local
ctx_setxvalue
=
context
.
setxvalue
132 133
local
function
initializer
(
)
134
tobesaved
=
jobvariables
.
tobesaved
135
collected
=
jobvariables
.
collected
136
--
137
rvalue
=
collected
.
randomseed
138
if
not
rvalue
then
139
rvalue
=
getrandom
(
"
initialize
"
)
140
setrandomseedi
(
rvalue
)
141
rmethod
=
"
initialized
"
142
else
143
setrandomseedi
(
rvalue
)
144
rmethod
=
"
resumed
"
145
end
146
tobesaved
.
randomseed
=
rvalue
147
--
148
collectedmacros
=
collected
.
macros
149
tobesavedmacros
=
tobesaved
.
macros
150
if
not
collectedmacros
then
151
collectedmacros
=
{
}
152
collected
.
macros
=
collectedmacros
153
end
154
if
not
tobesavedmacros
then
155
tobesavedmacros
=
{
}
156
tobesaved
.
macros
=
tobesavedmacros
157
end
158
-- will become collected.macros
159
for
cs
,
value
in
next
,
collectedmacros
do
160
if
type
(
value
)
=
=
"
string
"
then
-- safeguard
161
ctx_setxvalue
(
cs
,
value
)
162
end
163
end
164
end
165 166
job
.
register
(
'
job.variables.collected
'
,
tobesaved
,
initializer
)
167 168
function
jobvariables
.
save
(
cs
,
value
)
169
tobesavedmacros
[
cs
]
=
value
170
end
171 172
function
jobvariables
.
restore
(
cs
)
173
return
collectedmacros
[
cs
]
or
tobesavedmacros
[
cs
]
174
end
175 176
function
job
.
getrandomseed
(
)
177
return
tobesaved
.
randomseed
or
getrandomseed
(
)
178
end
179 180
-- checksums
181 182
function
jobvariables
.
getchecksum
(
tag
)
183
return
checksums
[
tag
]
-- no default
184
end
185 186
function
jobvariables
.
makechecksum
(
data
)
187
return
data
and
md5
.
HEX
(
data
)
-- no default
188
end
189 190
function
jobvariables
.
setchecksum
(
tag
,
checksum
)
191
checksums
[
tag
]
=
checksum
192
end
193 194
--
195 196
local
packlist
=
{
197
"
numbers
"
,
198
"
ownnumbers
"
,
199
"
metadata
"
,
200
"
sectiondata
"
,
201
"
prefixdata
"
,
202
"
numberdata
"
,
203
"
pagedata
"
,
204
"
directives
"
,
205
"
specification
"
,
206
"
processors
"
,
-- might become key under directives or metadata
207
-- "references", -- we need to rename of them as only one packs (not structures.lists.references)
208
}
209 210
local
skiplist
=
{
211
"
datasets
"
,
212
"
userdata
"
,
213
"
positions
"
,
214
}
215 216
-- not ok as we can have arbitrary keys in userdata and dataset so some day we
217
-- might need a bit more granularity, like skippers
218 219
local
jobpacker
=
packers
.
new
(
packlist
,
job
.
packversion
,
skiplist
)
-- jump number when changs in hash
220 221
job
.
pack
=
true
222
-- job.pack = false
223 224
directives
.
register
(
"
job.pack
"
,
function
(
v
)
job
.
pack
=
v
end
)
225 226
local
_save_
,
_load_
,
_others_
=
{
}
,
{
}
,
{
}
-- registers timing
227 228
function
job
.
save
(
filename
)
-- we could return a table but it can get pretty large
229
statistics
.
starttiming
(
_save_
)
230
local
f
=
io
.
open
(
filename
,
'
w
'
)
231
if
f
then
232
f
:
write
(
"
local utilitydata = { }\n\n
"
)
233
f
:
write
(
serialize
(
comment
,
"
utilitydata.comment
"
,
true
)
,
"
\n\n
"
)
234
for
l
=
1
,
#
savelist
do
235
-- f:write("do\n\n") -- no solution for the jit limitatione either
236
local
list
=
savelist
[
l
]
237
local
target
=
format
(
"
utilitydata.%s
"
,
list
[
1
]
)
238
local
data
=
list
[
2
]
239
local
finalizer
=
list
[
4
]
240
local
serializer
=
list
[
5
]
241
if
type
(
data
)
=
=
"
string
"
then
242
data
=
utilities
.
tables
.
accesstable
(
data
)
243
end
244
if
type
(
finalizer
)
=
=
"
function
"
then
245
finalizer
(
)
246
end
247
if
job
.
pack
then
248
packers
.
pack
(
data
,
jobpacker
,
true
)
249
end
250
local
definer
,
name
=
definetable
(
target
,
true
,
true
)
-- no first and no last
251
if
serializer
then
252
f
:
write
(
definer
,
"
\n\n
"
,
serializer
(
data
,
name
,
true
)
,
"
\n\n
"
)
253
else
254
f
:
write
(
definer
,
"
\n\n
"
,
serialize
(
data
,
name
,
true
)
,
"
\n\n
"
)
255
end
256
-- f:write("end\n\n")
257
end
258
if
job
.
pack
then
259
packers
.
strip
(
jobpacker
)
260
-- f:write("do\n\n")
261
f
:
write
(
serialize
(
jobpacker
,
"
utilitydata.job.packed
"
,
true
)
,
"
\n\n
"
)
262
-- f:write("end\n\n")
263
end
264
f
:
write
(
"
return utilitydata
"
)
265
f
:
close
(
)
266
end
267
statistics
.
stoptiming
(
_save_
)
268
end
269 270
local
function
load
(
filename
)
271
if
lfs
.
isfile
(
filename
)
then
272 273
local
function
dofile
(
filename
)
274
local
result
=
loadstring
(
io
.
loaddata
(
filename
)
)
275
if
result
then
276
return
result
(
)
277
else
278
return
nil
279
end
280
end
281 282
local
okay
,
data
=
pcall
(
dofile
,
filename
)
283
if
okay
and
type
(
data
)
=
=
"
table
"
then
284
local
jobversion
=
job
.
version
285
local
datacomment
=
data
.
comment
286
local
dataversion
=
datacomment
and
datacomment
.
version
or
"
?
"
287
if
dataversion
~
=
jobversion
then
288
report_passes
(
"
version mismatch: %s <> %s
"
,
dataversion
,
jobversion
)
289
else
290
return
data
291
end
292
else
293
os
.
remove
(
filename
)
-- probably a bad file (or luajit overflow as it cannot handle large tables well)
294
report_passes
(
"
removing stale job data file %a, restart job, message: %s%s
"
,
filename
,
tostring
(
data
)
,
295
jit
and
"
(try luatex instead of luajittex)
"
or
"
"
)
296
os
.
exit
(
true
)
-- trigger second run
297
end
298
end
299
end
300 301
function
job
.
load
(
filename
)
302
statistics
.
starttiming
(
_load_
)
303
local
utilitydata
=
load
(
filename
)
304
if
utilitydata
then
305
local
jobpacker
=
utilitydata
.
job
.
packed
306
local
handlers
=
{
}
307
for
i
=
1
,
#
savelist
do
308
local
list
=
savelist
[
i
]
309
local
target
=
list
[
1
]
310
local
initializer
=
list
[
3
]
311
local
result
=
accesstable
(
target
,
utilitydata
)
312
if
result
then
313
local
done
=
packers
.
unpack
(
result
,
jobpacker
,
true
)
314
if
done
then
315
migratetable
(
target
,
mark
(
result
)
)
316
if
type
(
initializer
)
=
=
"
function
"
then
317
handlers
[
#
handlers
+
1
]
=
{
initializer
,
result
}
318
end
319
else
320
report_passes
(
"
pack version mismatch
"
)
321
end
322
end
323
end
324
-- so we have all tables available (unpacked)
325
for
i
=
1
,
#
handlers
do
326
local
handler
=
handlers
[
i
]
327
handler
[
1
]
(
handler
[
2
]
)
328
end
329
end
330
statistics
.
stoptiming
(
_load_
)
331
end
332 333
function
job
.
loadother
(
filename
)
334
statistics
.
starttiming
(
_load_
)
335
_others_
[
#
_others_
+
1
]
=
file
.
nameonly
(
filename
)
336
local
utilitydata
=
load
(
filename
)
337
if
utilitydata
then
338
local
jobpacker
=
utilitydata
.
job
.
packed
339
local
unpacked
=
{
}
340
for
l
=
1
,
#
savelist
do
341
local
list
=
savelist
[
l
]
342
local
target
=
list
[
1
]
343
local
result
=
accesstable
(
target
,
utilitydata
)
344
local
done
=
packers
.
unpack
(
result
,
jobpacker
,
true
)
345
if
done
then
346
migratetable
(
target
,
result
,
unpacked
)
347
end
348
end
349
unpacked
.
job
.
packed
=
nil
-- nicer in inspecting
350
return
unpacked
351
end
352
statistics
.
stoptiming
(
_load_
)
353
end
354 355
-- function job.keep(filename)
356
-- local suffix = file.suffix(filename)
357
-- local base = file.removesuffix(filename)
358
-- if suffix == "" then
359
-- suffix = "tuc"
360
-- end
361
-- for i=1,10 do
362
-- local tmpname = format("%s-%s-%02d.tmp",base,suffix,i)
363
-- if lfs.isfile(tmpname) then
364
-- os.remove(tmpname)
365
-- report_passes("removing %a",tmpname)
366
-- end
367
-- end
368
-- if lfs.isfile(filename) then
369
-- local tmpname = format("%s-%s-%02d.tmp",base,suffix,environment.currentrun or 1)
370
-- report_passes("copying %a into %a",filename,tmpname)
371
-- file.copy(filename,tmpname)
372
-- else
373
-- report_passes("no file %a, nothing kept",filename)
374
-- end
375
-- end
376 377
-- eventually this will end up in strc-ini
378 379
statistics
.
register
(
"
startup time
"
,
function
(
)
380
return
statistics
.
elapsedseconds
(
statistics
,
"
including runtime option file processing
"
)
381
end
)
382 383
statistics
.
register
(
"
jobdata time
"
,
function
(
)
384
if
enabled
then
385
if
#
_others_
>
0
then
386
return
format
(
"
%s seconds saving, %s seconds loading, other files: %s
"
,
statistics
.
elapsedtime
(
_save_
)
,
statistics
.
elapsedtime
(
_load_
)
,
concat
(
_others_
,
"
"
)
)
387
else
388
return
format
(
"
%s seconds saving, %s seconds loading
"
,
statistics
.
elapsedtime
(
_save_
)
,
statistics
.
elapsedtime
(
_load_
)
)
389
end
390
else
391
if
#
_others_
>
0
then
392
return
format
(
"
nothing saved, %s seconds loading, other files: %s
"
,
statistics
.
elapsedtime
(
_load_
)
,
concat
(
_others_
,
"
"
)
)
393
else
394
return
format
(
"
nothing saved, %s seconds loading
"
,
statistics
.
elapsedtime
(
_load_
)
)
395
end
396
end
397
end
)
398 399
-- statistics.register("callbacks", function()
400
-- local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0
401
-- local pages = texgetcount('realpageno') - 1
402
-- if pages > 1 then
403
-- return format("direct: %s, indirect: %s, total: %s (%i per page)", total-indirect, indirect, total, total/pages)
404
-- else
405
-- return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total)
406
-- end
407
-- end)
408 409
function
statistics
.
callbacks
(
)
410
local
c_internal
=
status
.
callbacks
or
0
411
local
c_file
=
status
.
indirect_callbacks
or
0
412
local
c_direct
=
status
.
direct_callbacks
or
0
413
local
c_late
=
backends
.
noflatelua
(
)
or
0
414
local
c_function
=
status
.
function_callbacks
or
0
415
local
c_total
=
c_internal
+
c_file
+
c_direct
+
c_late
+
c_function
416
local
n_pages
=
texgetcount
(
'
realpageno
'
)
-
1
417
local
c_average
=
n_pages
>
0
and
math
.
round
(
c_total
/
n_pages
)
or
0
418
local
s_result
=
format
(
419
c_average
>
0
and
"
internal: %s, file: %s, direct: %s, late: %s, function %s, total: %s (%s per page)
"
420
or
"
internal: %s, file: %s, direct: %s, late: %s, function %s, total: %s
"
,
421
c_internal
,
c_file
,
c_direct
,
c_late
,
c_function
,
c_total
,
c_average
422
)
423
statistics
.
callbacks
=
function
(
)
424
return
s_result
425
end
426
return
s_result
427
end
428 429
statistics
.
register
(
"
callbacks
"
,
statistics
.
callbacks
)
430 431
statistics
.
register
(
"
randomizer
"
,
function
(
)
432
if
rmethod
and
rvalue
then
433
return
format
(
"
%s with value %s
"
,
rmethod
,
rvalue
)
434
end
435
end
)
436 437
-- a sort of joke (for ctx meeting)
438 439
-- local kg_per_watt_per_second = 1 / 15000000
440
-- local watts_per_core = 50
441
-- local speedup_by_other_engine = 1.2
442
-- local used_wood_factor = watts_per_core * kg_per_watt_per_second / speedup_by_other_engine
443
-- local used_wood_factor = (50 / 15000000) / 1.2
444 445 446
function
statistics
.
formatruntime
(
runtime
)
447
if
not
environment
.
initex
then
-- else error when testing as not counters yet
448
-- stoptiming(statistics) -- to be sure
449
local
shipped
=
texgetcount
(
'
nofshipouts
'
)
450
local
pages
=
texgetcount
(
'
realpageno
'
)
451
if
pages
>
shipped
then
452
pages
=
shipped
453
end
454
runtime
=
tonumber
(
runtime
)
455
if
shipped
>
0
or
pages
>
0
then
456
local
persecond
=
(
runtime
>
0
)
and
(
shipped
/
runtime
)
or
pages
457
if
pages
=
=
0
then
458
pages
=
shipped
459
end
460
return
format
(
"
%0.3f seconds, %i processed pages, %i shipped pages, %.3f pages/second
"
,
runtime
,
pages
,
shipped
,
persecond
)
461
else
462
return
format
(
"
%0.3f seconds
"
,
runtime
)
463
end
464
end
465
end
466 467
implement
{
468
name
=
"
savevariable
"
,
469
actions
=
job
.
variables
.
save
,
470
arguments
=
"
2 strings
"
,
471
}
472 473
implement
{
474
name
=
"
setjobcomment
"
,
475
actions
=
job
.
comment
,
476
arguments
=
{
{
"
*
"
}
}
477
}
478 479
implement
{
480
name
=
"
initializejob
"
,
481
actions
=
job
.
initialize
482
}
483 484
implement
{
485
name
=
"
disablejobsave
"
,
486
actions
=
job
.
disablesave
487
}
488