data-tmp.lua /size: 15 Kb    last modification: 2020-07-01 14:35
1
if
not
modules
then
modules
=
{
}
end
modules
[
'
data-tmp
'
]
=
{
2
version
=
1
.
100
,
3
comment
=
"
companion to luat-lib.mkiv
"
,
4
author
=
"
Hans Hagen, PRAGMA-ADE, Hasselt NL
"
,
5
copyright
=
"
PRAGMA ADE / ConTeXt Development Team
"
,
6
license
=
"
see context related readme files
"
7
}
8 9
--[[ldx-- 10<p>This module deals with caching data. It sets up the paths and implements 11loaders and savers for tables. Best is to set the following variable. When not 12set, the usual paths will be checked. Personally I prefer the (users) temporary 13path.</p> 14 15</code> 16TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. 17</code> 18 19<p>Currently we do no locking when we write files. This is no real problem 20because most caching involves fonts and the chance of them being written at the 21same time is small. We also need to extend luatools with a recache feature.</p> 22--ldx]]
--
23 24
local
next
,
type
=
next
,
type
25
local
pcall
,
loadfile
,
collectgarbage
=
pcall
,
loadfile
,
collectgarbage
26
local
format
,
lower
,
gsub
=
string
.
format
,
string
.
lower
,
string
.
gsub
27
local
concat
,
serialize
,
fastserialize
,
serializetofile
=
table
.
concat
,
table
.
serialize
,
table
.
fastserialize
,
table
.
tofile
28
local
mkdirs
,
expanddirname
,
isdir
,
isfile
=
dir
.
mkdirs
,
dir
.
expandname
,
lfs
.
isdir
,
lfs
.
isfile
29
local
is_writable
,
is_readable
=
file
.
is_writable
,
file
.
is_readable
30
local
collapsepath
,
joinfile
,
addsuffix
,
dirname
=
file
.
collapsepath
,
file
.
join
,
file
.
addsuffix
,
file
.
dirname
31
local
savedata
=
file
.
savedata
32
local
formatters
=
string
.
formatters
33
local
osexit
,
osdate
,
osuuid
=
os
.
exit
,
os
.
date
,
os
.
uuid
34
local
removefile
=
os
.
remove
35
local
md5hex
=
md5
.
hex
36 37
local
trace_locating
=
false
trackers
.
register
(
"
resolvers.locating
"
,
function
(
v
)
trace_locating
=
v
end
)
38
local
trace_cache
=
false
trackers
.
register
(
"
resolvers.cache
"
,
function
(
v
)
trace_cache
=
v
end
)
39 40
local
report_caches
=
logs
.
reporter
(
"
resolvers
"
,
"
caches
"
)
41
local
report_resolvers
=
logs
.
reporter
(
"
resolvers
"
,
"
caching
"
)
42 43
local
resolvers
=
resolvers
44
local
cleanpath
=
resolvers
.
cleanpath
45
local
resolvepath
=
resolvers
.
resolve
46 47
local
luautilities
=
utilities
.
lua
48 49
-- intermezzo
50 51
do
52 53
local
directive_cleanup
=
false
directives
.
register
(
"
system.compile.cleanup
"
,
function
(
v
)
directive_cleanup
=
v
end
)
54
local
directive_strip
=
false
directives
.
register
(
"
system.compile.strip
"
,
function
(
v
)
directive_strip
=
v
end
)
55 56
local
compilelua
=
luautilities
.
compile
57 58
function
luautilities
.
compile
(
luafile
,
lucfile
,
cleanup
,
strip
)
59
if
cleanup
=
=
nil
then
cleanup
=
directive_cleanup
end
60
if
strip
=
=
nil
then
strip
=
directive_strip
end
61
return
compilelua
(
luafile
,
lucfile
,
cleanup
,
strip
)
62
end
63 64
end
65 66
-- end of intermezzo
67 68
caches
=
caches
or
{
}
69
local
caches
=
caches
70
local
writable
=
nil
71
local
readables
=
{
}
72
local
usedreadables
=
{
}
73 74
local
compilelua
=
luautilities
.
compile
75
local
luasuffixes
=
luautilities
.
suffixes
76 77
caches
.
base
=
caches
.
base
or
"
luatex-cache
"
-- can be local
78
caches
.
more
=
caches
.
more
or
"
context
"
-- can be local
79
caches
.
defaults
=
{
"
TMPDIR
"
,
"
TEMPDIR
"
,
"
TMP
"
,
"
TEMP
"
,
"
HOME
"
,
"
HOMEPATH
"
}
80 81
local
direct_cache
=
false
-- true is faster but may need huge amounts of memory
82
local
fast_cache
=
false
83
local
cache_tree
=
false
84 85
directives
.
register
(
"
system.caches.direct
"
,
function
(
v
)
direct_cache
=
true
end
)
86
directives
.
register
(
"
system.caches.fast
"
,
function
(
v
)
fast_cache
=
true
end
)
87 88
-- we could use a metatable for writable and readable but not yet
89 90
local
function
configfiles
(
)
91
return
concat
(
resolvers
.
configurationfiles
(
)
,
"
;
"
)
92
end
93 94
local
function
hashed
(
tree
)
95
tree
=
gsub
(
tree
,
"
[\\/]+$
"
,
"
"
)
96
tree
=
lower
(
tree
)
97
local
hash
=
md5hex
(
tree
)
98
if
trace_cache
or
trace_locating
then
99
report_caches
(
"
hashing tree %a, hash %a
"
,
tree
,
hash
)
100
end
101
return
hash
102
end
103 104
local
function
treehash
(
)
105
local
tree
=
configfiles
(
)
106
if
not
tree
or
tree
=
=
"
"
then
107
return
false
108
else
109
return
hashed
(
tree
)
110
end
111
end
112 113
caches
.
hashed
=
hashed
114
caches
.
treehash
=
treehash
115
caches
.
configfiles
=
configfiles
116 117
local
function
identify
(
)
118
-- Combining the loops makes it messy. First we check the format cache path
119
-- and when the last component is not present we try to create it.
120
local
texmfcaches
=
resolvers
.
cleanpathlist
(
"
TEXMFCACHE
"
)
-- forward ref
121
if
texmfcaches
then
122
for
k
=
1
,
#
texmfcaches
do
123
local
cachepath
=
texmfcaches
[
k
]
124
if
cachepath
~
=
"
"
then
125
cachepath
=
resolvepath
(
cachepath
)
126
cachepath
=
cleanpath
(
cachepath
)
127
cachepath
=
collapsepath
(
cachepath
)
128
local
valid
=
isdir
(
cachepath
)
129
if
valid
then
130
if
is_readable
(
cachepath
)
then
131
readables
[
#
readables
+
1
]
=
cachepath
132
if
not
writable
and
is_writable
(
cachepath
)
then
133
writable
=
cachepath
134
end
135
end
136
elseif
not
writable
then
137
local
cacheparent
=
dirname
(
cachepath
)
138
if
is_writable
(
cacheparent
)
then
-- we go on anyway (needed for mojca's kind of paths)
139
mkdirs
(
cachepath
)
140
if
isdir
(
cachepath
)
and
is_writable
(
cachepath
)
then
141
report_caches
(
"
path %a created
"
,
cachepath
)
142
writable
=
cachepath
143
readables
[
#
readables
+
1
]
=
cachepath
144
end
145
end
146
end
147
end
148
end
149
end
150
-- As a last resort we check some temporary paths but this time we don't
151
-- create them.
152
local
texmfcaches
=
caches
.
defaults
153
if
texmfcaches
then
154
for
k
=
1
,
#
texmfcaches
do
155
local
cachepath
=
texmfcaches
[
k
]
156
cachepath
=
resolvers
.
expansion
(
cachepath
)
-- was getenv
157
if
cachepath
~
=
"
"
then
158
cachepath
=
resolvepath
(
cachepath
)
159
cachepath
=
cleanpath
(
cachepath
)
160
local
valid
=
isdir
(
cachepath
)
161
if
valid
and
is_readable
(
cachepath
)
then
162
if
not
writable
and
is_writable
(
cachepath
)
then
163
readables
[
#
readables
+
1
]
=
cachepath
164
writable
=
cachepath
165
break
166
end
167
end
168
end
169
end
170
end
171
-- Some extra checking. If we have no writable or readable path then we simply
172
-- quit.
173
if
not
writable
then
174
report_caches
(
"
fatal error: there is no valid writable cache path defined
"
)
175
osexit
(
)
176
elseif
#
readables
=
=
0
then
177
report_caches
(
"
fatal error: there is no valid readable cache path defined
"
)
178
osexit
(
)
179
end
180
-- why here
181
writable
=
expanddirname
(
cleanpath
(
writable
)
)
-- just in case
182
-- moved here ( we have only one writable tree)
183
local
base
=
caches
.
base
184
local
more
=
caches
.
more
185
local
tree
=
cache_tree
or
treehash
(
)
-- we have only one writable tree
186
if
tree
then
187
cache_tree
=
tree
188
writable
=
mkdirs
(
writable
,
base
,
more
,
tree
)
189
for
i
=
1
,
#
readables
do
190
readables
[
i
]
=
joinfile
(
readables
[
i
]
,
base
,
more
,
tree
)
191
end
192
else
193
writable
=
mkdirs
(
writable
,
base
,
more
)
194
for
i
=
1
,
#
readables
do
195
readables
[
i
]
=
joinfile
(
readables
[
i
]
,
base
,
more
)
196
end
197
end
198
-- end
199
if
trace_cache
then
200
for
i
=
1
,
#
readables
do
201
report_caches
(
"
using readable path %a (order %s)
"
,
readables
[
i
]
,
i
)
202
end
203
report_caches
(
"
using writable path %a
"
,
writable
)
204
end
205
identify
=
function
(
)
206
return
writable
,
readables
207
end
208
return
writable
,
readables
209
end
210 211
function
caches
.
usedpaths
(
separator
)
212
local
writable
,
readables
=
identify
(
)
213
if
#
readables
>
1
then
214
local
result
=
{
}
215
local
done
=
{
}
216
for
i
=
1
,
#
readables
do
217
local
readable
=
readables
[
i
]
218
if
readable
=
=
writable
then
219
done
[
readable
]
=
true
220
result
[
#
result
+
1
]
=
formatters
[
"
readable+writable: %a
"
]
(
readable
)
221
elseif
usedreadables
[
i
]
then
222
done
[
readable
]
=
true
223
result
[
#
result
+
1
]
=
formatters
[
"
readable: %a
"
]
(
readable
)
224
end
225
end
226
if
not
done
[
writable
]
then
227
result
[
#
result
+
1
]
=
formatters
[
"
writable: %a
"
]
(
writable
)
228
end
229
return
concat
(
result
,
separator
or
"
|
"
)
230
else
231
return
writable
or
"
?
"
232
end
233
end
234 235
local
r_cache
=
{
}
236
local
w_cache
=
{
}
237 238
local
function
getreadablepaths
(
...
)
239
local
tags
=
{
...
}
240
local
hash
=
concat
(
tags
,
"
/
"
)
241
local
done
=
r_cache
[
hash
]
242
if
not
done
then
243
local
writable
,
readables
=
identify
(
)
-- exit if not found
244
if
#
tags
>
0
then
245
done
=
{
}
246
for
i
=
1
,
#
readables
do
247
done
[
i
]
=
joinfile
(
readables
[
i
]
,
...
)
248
end
249
else
250
done
=
readables
251
end
252
r_cache
[
hash
]
=
done
253
end
254
return
done
255
end
256 257
local
function
getwritablepath
(
...
)
258
local
tags
=
{
...
}
259
local
hash
=
concat
(
tags
,
"
/
"
)
260
local
done
=
w_cache
[
hash
]
261
if
not
done
then
262
local
writable
,
readables
=
identify
(
)
-- exit if not found
263
if
#
tags
>
0
then
264
done
=
mkdirs
(
writable
,
...
)
265
else
266
done
=
writable
267
end
268
w_cache
[
hash
]
=
done
269
end
270
return
done
271
end
272 273
local
function
setfirstwritablefile
(
filename
,
...
)
274
local
wr
=
getwritablepath
(
...
)
275
local
fullname
=
joinfile
(
wr
,
filename
)
276
return
fullname
,
wr
277
end
278 279
local
function
setluanames
(
path
,
name
)
280
return
281
format
(
"
%s/%s.%s
"
,
path
,
name
,
luasuffixes
.
tma
)
,
282
format
(
"
%s/%s.%s
"
,
path
,
name
,
luasuffixes
.
tmc
)
283
end
284 285
local
function
getfirstreadablefile
(
filename
,
...
)
286
-- check if we have already written once
287
local
fullname
,
path
=
setfirstwritablefile
(
filename
,
...
)
288
if
is_readable
(
fullname
)
then
289
return
fullname
,
path
-- , true
290
end
291
-- otherwise search for pregenerated
292
local
rd
=
getreadablepaths
(
...
)
293
for
i
=
1
,
#
rd
do
294
local
path
=
rd
[
i
]
295
local
fullname
=
joinfile
(
path
,
filename
)
296
if
is_readable
(
fullname
)
then
297
usedreadables
[
i
]
=
true
298
return
fullname
,
path
-- , false
299
end
300
end
301
-- else assume new written
302
return
fullname
,
path
-- , true
303
end
304 305
caches
.
getreadablepaths
=
getreadablepaths
306
caches
.
getwritablepath
=
getwritablepath
307
caches
.
setfirstwritablefile
=
setfirstwritablefile
308
caches
.
getfirstreadablefile
=
getfirstreadablefile
309
caches
.
setluanames
=
setluanames
310 311
-- -- not used:
312
--
313
-- function caches.define(category,subcategory)
314
-- return function()
315
-- return getwritablepath(category,subcategory)
316
-- end
317
-- end
318 319
-- This works best if the first writable is the first readable too. In practice
320
-- we can have these situations for file databases:
321
--
322
-- tma in readable
323
-- tma + tmb/c in readable
324
--
325
-- runtime files like fonts are written to the writable cache anyway
326 327
function
caches
.
loaddata
(
readables
,
name
,
writable
)
328
if
type
(
readables
)
=
=
"
string
"
then
329
readables
=
{
readables
}
330
end
331
for
i
=
1
,
#
readables
do
332
local
path
=
readables
[
i
]
333
local
loader
=
false
334
local
state
=
false
335
local
tmaname
,
tmcname
=
setluanames
(
path
,
name
)
336
if
isfile
(
tmcname
)
then
337
state
,
loader
=
pcall
(
loadfile
,
tmcname
)
338
end
339
if
not
loader
and
isfile
(
tmaname
)
then
340
-- can be different paths when we read a file database from disk
341
local
tmacrap
,
tmcname
=
setluanames
(
writable
,
name
)
342
if
isfile
(
tmcname
)
then
343
state
,
loader
=
pcall
(
loadfile
,
tmcname
)
344
end
345
compilelua
(
tmaname
,
tmcname
)
346
if
isfile
(
tmcname
)
then
347
state
,
loader
=
pcall
(
loadfile
,
tmcname
)
348
end
349
if
not
loader
then
350
state
,
loader
=
pcall
(
loadfile
,
tmaname
)
351
end
352
end
353
if
loader
then
354
loader
=
loader
(
)
355
collectgarbage
(
"
step
"
)
356
return
loader
357
end
358
end
359
return
false
360
end
361 362
function
caches
.
is_writable
(
filepath
,
filename
)
363
local
tmaname
,
tmcname
=
setluanames
(
filepath
,
filename
)
364
return
is_writable
(
tmaname
)
365
end
366 367
local
saveoptions
=
{
compact
=
true
,
accurate
=
not
JITSUPPORTED
}
368 369
function
caches
.
savedata
(
filepath
,
filename
,
data
,
fast
)
370
local
tmaname
,
tmcname
=
setluanames
(
filepath
,
filename
)
371
data
.
cache_uuid
=
osuuid
(
)
372
if
fast
or
fast_cache
then
373
savedata
(
tmaname
,
fastserialize
(
data
,
true
)
)
374
elseif
direct_cache
then
375
savedata
(
tmaname
,
serialize
(
data
,
true
,
saveoptions
)
)
376
else
377
serializetofile
(
tmaname
,
data
,
true
,
saveoptions
)
378
end
379
compilelua
(
tmaname
,
tmcname
)
380
end
381 382
-- moved from data-res:
383 384
local
content_state
=
{
}
385 386
function
caches
.
contentstate
(
)
387
return
content_state
or
{
}
388
end
389 390
function
caches
.
loadcontent
(
cachename
,
dataname
,
filename
)
391
if
not
filename
then
392
local
name
=
hashed
(
cachename
)
393
local
full
,
path
=
getfirstreadablefile
(
addsuffix
(
name
,
luasuffixes
.
lua
)
,
"
trees
"
)
394
filename
=
joinfile
(
path
,
name
)
395
end
396
local
state
,
blob
=
pcall
(
loadfile
,
addsuffix
(
filename
,
luasuffixes
.
luc
)
)
397
if
not
blob
then
398
state
,
blob
=
pcall
(
loadfile
,
addsuffix
(
filename
,
luasuffixes
.
lua
)
)
399
end
400
if
blob
then
401
local
data
=
blob
(
)
402
if
data
and
data
.
content
then
403
if
data
.
type
=
=
dataname
then
404
if
data
.
version
=
=
resolvers
.
cacheversion
then
405
content_state
[
#
content_state
+
1
]
=
data
.
uuid
406
if
trace_locating
then
407
report_resolvers
(
"
loading %a for %a from %a
"
,
dataname
,
cachename
,
filename
)
408
end
409
return
data
.
content
410
else
411
report_resolvers
(
"
skipping %a for %a from %a (version mismatch)
"
,
dataname
,
cachename
,
filename
)
412
end
413
else
414
report_resolvers
(
"
skipping %a for %a from %a (datatype mismatch)
"
,
dataname
,
cachename
,
filename
)
415
end
416
elseif
trace_locating
then
417
report_resolvers
(
"
skipping %a for %a from %a (no content)
"
,
dataname
,
cachename
,
filename
)
418
end
419
elseif
trace_locating
then
420
report_resolvers
(
"
skipping %a for %a from %a (invalid file)
"
,
dataname
,
cachename
,
filename
)
421
end
422
end
423 424
function
caches
.
collapsecontent
(
content
)
425
for
k
,
v
in
next
,
content
do
426
if
type
(
v
)
=
=
"
table
"
and
#
v
=
=
1
then
427
content
[
k
]
=
v
[
1
]
428
end
429
end
430
end
431 432
function
caches
.
savecontent
(
cachename
,
dataname
,
content
,
filename
)
433
if
not
filename
then
434
local
name
=
hashed
(
cachename
)
435
local
full
,
path
=
setfirstwritablefile
(
addsuffix
(
name
,
luasuffixes
.
lua
)
,
"
trees
"
)
436
filename
=
joinfile
(
path
,
name
)
-- is full
437
end
438
local
luaname
=
addsuffix
(
filename
,
luasuffixes
.
lua
)
439
local
lucname
=
addsuffix
(
filename
,
luasuffixes
.
luc
)
440
if
trace_locating
then
441
report_resolvers
(
"
preparing %a for %a
"
,
dataname
,
cachename
)
442
end
443
local
data
=
{
444
type
=
dataname
,
445
root
=
cachename
,
446
version
=
resolvers
.
cacheversion
,
447
date
=
osdate
(
"
%Y-%m-%d
"
)
,
448
time
=
osdate
(
"
%H:%M:%S
"
)
,
449
content
=
content
,
450
uuid
=
osuuid
(
)
,
451
}
452
local
ok
=
savedata
(
luaname
,
serialize
(
data
,
true
)
)
453
if
ok
then
454
if
trace_locating
then
455
report_resolvers
(
"
category %a, cachename %a saved in %a
"
,
dataname
,
cachename
,
luaname
)
456
end
457
if
compilelua
(
luaname
,
lucname
)
then
458
if
trace_locating
then
459
report_resolvers
(
"
%a compiled to %a
"
,
dataname
,
lucname
)
460
end
461
return
true
462
else
463
if
trace_locating
then
464
report_resolvers
(
"
compiling failed for %a, deleting file %a
"
,
dataname
,
lucname
)
465
end
466
removefile
(
lucname
)
467
end
468
elseif
trace_locating
then
469
report_resolvers
(
"
unable to save %a in %a (access error)
"
,
dataname
,
luaname
)
470
end
471
end
472