data-sch.lua /size: 6945 b    last modification: 2020-07-01 14:35
1
if
not
modules
then
modules
=
{
}
end
modules
[
'
data-sch
'
]
=
{
2
version
=
1
.
001
,
3
comment
=
"
companion to luat-lib.mkiv
"
,
4
author
=
"
Hans Hagen, PRAGMA-ADE, Hasselt NL
"
,
5
copyright
=
"
PRAGMA ADE / ConTeXt Development Team
"
,
6
license
=
"
see context related readme files
"
7
}
8 9
-- this one will be redone / localized
10 11
local
load
,
tonumber
=
load
,
tonumber
12
local
gsub
,
format
=
string
.
gsub
,
string
.
format
13
local
sortedhash
,
concat
=
table
.
sortedhash
,
table
.
concat
14
local
finders
,
openers
,
loaders
=
resolvers
.
finders
,
resolvers
.
openers
,
resolvers
.
loaders
15
local
addsuffix
,
suffix
,
splitbase
=
file
.
addsuffix
,
file
.
suffix
,
file
.
splitbase
16
local
md5hex
=
md5
.
hex
17 18
local
trace_schemes
=
false
trackers
.
register
(
"
resolvers.schemes
"
,
function
(
v
)
trace_schemes
=
v
end
)
19
local
report_schemes
=
logs
.
reporter
(
"
resolvers
"
,
"
schemes
"
)
20 21
local
http
=
require
(
"
socket.http
"
)
22
local
ltn12
=
require
(
"
ltn12
"
)
23 24
if
mbox
then
mbox
=
nil
end
-- useless and even bugged (helper overwrites lib)
25 26
local
resolvers
=
resolvers
27
local
schemes
=
resolvers
.
schemes
or
{
}
28
resolvers
.
schemes
=
schemes
29 30
local
cleaners
=
{
}
31
schemes
.
cleaners
=
cleaners
32 33
local
threshold
=
24
*
60
*
60
34 35
directives
.
register
(
"
schemes.threshold
"
,
function
(
v
)
threshold
=
tonumber
(
v
)
or
threshold
end
)
36 37
function
cleaners
.
none
(
specification
)
38
return
specification
.
original
39
end
40 41
-- function cleaners.strip(specification)
42
-- -- todo: only keep suffix periods, so after the last
43
-- return (gsub(specification.original,"[^%a%d%.]+","-")) -- so we keep periods
44
-- end
45 46
function
cleaners
.
strip
(
specification
)
-- keep suffixes
47
local
path
,
name
=
splitbase
(
specification
.
original
)
48
if
path
=
=
"
"
then
49
return
(
gsub
(
name
,
"
[^%a%d%.]+
"
,
"
-
"
)
)
50
else
51
return
(
gsub
(
(
gsub
(
path
,
"
%.
"
,
"
-
"
)
.
.
"
-
"
.
.
name
)
,
"
[^%a%d%.]+
"
,
"
-
"
)
)
52
end
53
end
54 55
function
cleaners
.
md5
(
specification
)
56
return
addsuffix
(
md5hex
(
specification
.
original
)
,
suffix
(
specification
.
path
)
)
57
end
58 59
local
cleaner
=
cleaners
.
strip
60 61
directives
.
register
(
"
schemes.cleanmethod
"
,
function
(
v
)
cleaner
=
cleaners
[
v
]
or
cleaners
.
strip
end
)
62 63
function
resolvers
.
schemes
.
cleanname
(
specification
)
64
local
hash
=
cleaner
(
specification
)
65
if
trace_schemes
then
66
report_schemes
(
"
hashing %a to %a
"
,
specification
.
original
,
hash
)
67
end
68
return
hash
69
end
70 71
local
cached
=
{
}
72
local
loaded
=
{
}
73
local
reused
=
{
}
74
local
thresholds
=
{
}
75
local
handlers
=
{
}
76
local
runner
=
sandbox
.
registerrunner
{
77
name
=
"
curl resolver
"
,
78
method
=
"
execute
"
,
79
program
=
"
curl
"
,
80
template
=
'
--silent --insecure --create-dirs --output "%cachename%" "%original%"
'
,
81
checkers
=
{
82
cachename
=
"
cache
"
,
83
original
=
"
url
"
,
84
}
85
}
86 87
local
function
fetch
(
specification
)
88
local
original
=
specification
.
original
89
local
scheme
=
specification
.
scheme
90
local
cleanname
=
schemes
.
cleanname
(
specification
)
91
local
cachename
=
caches
.
setfirstwritablefile
(
cleanname
,
"
schemes
"
)
92
if
not
cached
[
original
]
then
93
statistics
.
starttiming
(
schemes
)
94
if
not
io
.
exists
(
cachename
)
or
(
os
.
difftime
(
os
.
time
(
)
,
lfs
.
attributes
(
cachename
)
.
modification
)
>
(
thresholds
[
protocol
]
or
threshold
)
)
then
95
cached
[
original
]
=
cachename
96
local
handler
=
handlers
[
scheme
]
97
if
handler
then
98
if
trace_schemes
then
99
report_schemes
(
"
fetching %a, protocol %a, method %a
"
,
original
,
scheme
,
"
built-in
"
)
100
end
101
logs
.
flush
(
)
102
handler
(
specification
,
cachename
)
103
else
104
if
trace_schemes
then
105
report_schemes
(
"
fetching %a, protocol %a, method %a
"
,
original
,
scheme
,
"
curl
"
)
106
end
107
logs
.
flush
(
)
108
runner
{
109
original
=
original
,
110
cachename
=
cachename
,
111
}
112
end
113
end
114
if
io
.
exists
(
cachename
)
then
115
cached
[
original
]
=
cachename
116
if
trace_schemes
then
117
report_schemes
(
"
using cached %a, protocol %a, cachename %a
"
,
original
,
scheme
,
cachename
)
118
end
119
else
120
cached
[
original
]
=
"
"
121
if
trace_schemes
then
122
report_schemes
(
"
using missing %a, protocol %a
"
,
original
,
scheme
)
123
end
124
end
125
loaded
[
scheme
]
=
loaded
[
scheme
]
+
1
126
statistics
.
stoptiming
(
schemes
)
127
else
128
if
trace_schemes
then
129
report_schemes
(
"
reusing %a, protocol %a
"
,
original
,
scheme
)
130
end
131
reused
[
scheme
]
=
reused
[
scheme
]
+
1
132
end
133
return
cached
[
original
]
134
end
135 136
local
function
finder
(
specification
,
filetype
)
137
return
resolvers
.
methodhandler
(
"
finders
"
,
fetch
(
specification
)
,
filetype
)
138
end
139 140
local
opener
=
openers
.
file
141
local
loader
=
loaders
.
file
142 143
local
function
install
(
scheme
,
handler
,
newthreshold
)
144
handlers
[
scheme
]
=
handler
145
loaded
[
scheme
]
=
0
146
reused
[
scheme
]
=
0
147
finders
[
scheme
]
=
finder
148
openers
[
scheme
]
=
opener
149
loaders
[
scheme
]
=
loader
150
thresholds
[
scheme
]
=
newthreshold
or
threshold
151
end
152 153
schemes
.
install
=
install
154 155
local
function
http_handler
(
specification
,
cachename
)
156
local
tempname
=
cachename
.
.
"
.tmp
"
157
local
handle
=
io
.
open
(
tempname
,
"
wb
"
)
158
local
status
,
message
=
http
.
request
{
159
url
=
specification
.
original
,
160
sink
=
ltn12
.
sink
.
file
(
handle
)
161
}
162
if
not
status
then
163
os
.
remove
(
tempname
)
164
else
165
os
.
remove
(
cachename
)
166
os
.
rename
(
tempname
,
cachename
)
167
end
168
return
cachename
169
end
170 171
install
(
'
http
'
,
http_handler
)
172
install
(
'
https
'
)
-- see pod
173
install
(
'
ftp
'
)
174 175
statistics
.
register
(
"
scheme handling time
"
,
function
(
)
176
local
l
,
r
,
nl
,
nr
=
{
}
,
{
}
,
0
,
0
177
for
k
,
v
in
sortedhash
(
loaded
)
do
178
if
v
>
0
then
179
nl
=
nl
+
1
180
l
[
nl
]
=
k
.
.
"
:
"
.
.
v
181
end
182
end
183
for
k
,
v
in
sortedhash
(
reused
)
do
184
if
v
>
0
then
185
nr
=
nr
+
1
186
r
[
nr
]
=
k
.
.
"
:
"
.
.
v
187
end
188
end
189
local
n
=
nl
+
nr
190
if
n
>
0
then
191
if
nl
=
=
0
then
l
=
{
"
none
"
}
end
192
if
nr
=
=
0
then
r
=
{
"
none
"
}
end
193
return
format
(
"
%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s
"
,
194
statistics
.
elapsedtime
(
schemes
)
,
n
,
threshold
,
concat
(
l
,
"
"
)
,
concat
(
l
,
"
"
)
)
195
else
196
return
nil
197
end
198
end
)
199 200
-- We provide a few more helpers:
201 202
----- http = require("socket.http")
203
local
httprequest
=
http
.
request
204
local
toquery
=
url
.
toquery
205 206
local
function
fetchstring
(
url
,
data
)
207
local
q
=
data
and
toquery
(
data
)
208
if
q
then
209
url
=
url
.
.
"
?
"
.
.
q
210
end
211
local
reply
=
httprequest
(
url
)
212
return
reply
-- just one argument
213
end
214 215
schemes
.
fetchstring
=
fetchstring
216 217
function
schemes
.
fetchtable
(
url
,
data
)
218
local
reply
=
fetchstring
(
url
,
data
)
219
if
reply
then
220
local
s
=
load
(
"
return
"
.
.
reply
)
221
if
s
then
222
return
s
(
)
223
end
224
end
225
end
226