1if not modules then modules = { } end modules ['data-sch'] = {
2 version = 1.001,
3 comment = "companion to luat-lib.mkiv",
4 author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5 copyright = "PRAGMA ADE / ConTeXt Development Team",
6 license = "see context related readme files"
7}
8
9
10
11local load, tonumber = load, tonumber
12local gsub, format = string.gsub, string.format
13local sortedhash, concat = table.sortedhash, table.concat
14local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders
15local addsuffix, suffix, splitbase = file.addsuffix, file.suffix, file.splitbase
16local md5hex = md5.hex
17
18local trace_schemes = false trackers.register("resolvers.schemes",function(v) trace_schemes = v end)
19local report_schemes = logs.reporter("resolvers","schemes")
20
21local http = require("socket.http")
22local ltn12 = require("ltn12")
23
24if mbox then mbox = nil end
25
26local resolvers = resolvers
27local schemes = resolvers.schemes or { }
28resolvers.schemes = schemes
29
30local cleaners = { }
31schemes.cleaners = cleaners
32
33local threshold = 24 * 60 * 60
34
35directives.register("schemes.threshold", function(v) threshold = tonumber(v) or threshold end)
36
37function cleaners.none(specification)
38 return specification.original
39end
40
41
42
43
44
45
46function cleaners.strip(specification)
47 local path, name = splitbase(specification.original)
48 if path == "" then
49 return (gsub(name,"[^%a%d%.]+","-"))
50 else
51 return (gsub((gsub(path,"%.","-") .. "-" .. name),"[^%a%d%.]+","-"))
52 end
53end
54
55function cleaners.md5(specification)
56 return addsuffix(md5hex(specification.original),suffix(specification.path))
57end
58
59local cleaner = cleaners.strip
60
61directives.register("schemes.cleanmethod", function(v) cleaner = cleaners[v] or cleaners.strip end)
62
63function resolvers.schemes.cleanname(specification)
64 local hash = cleaner(specification)
65 if trace_schemes then
66 report_schemes("hashing %a to %a",specification.original,hash)
67 end
68 return hash
69end
70
71local cached = { }
72local loaded = { }
73local reused = { }
74local thresholds = { }
75local handlers = { }
76local runner = sandbox.registerrunner {
77 name = "curl resolver",
78 method = "execute",
79 program = "curl",
80 template = '--silent --insecure --create-dirs --output "%cachename%" "%original%"',
81 checkers = {
82 cachename = "cache",
83 original = "url",
84 }
85}
86
87local function fetch(specification)
88 local original = specification.original
89 local scheme = specification.scheme
90 local cleanname = schemes.cleanname(specification)
91 local cachename = caches.setfirstwritablefile(cleanname,"schemes")
92 if not cached[original] then
93 statistics.starttiming(schemes)
94 if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > (thresholds[protocol] or threshold)) then
95 cached[original] = cachename
96 local handler = handlers[scheme]
97 if handler then
98 if trace_schemes then
99 report_schemes("fetching %a, protocol %a, method %a",original,scheme,"built-in")
100 end
101 logs.flush()
102 handler(specification,cachename)
103 else
104 if trace_schemes then
105 report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl")
106 end
107 logs.flush()
108 runner {
109 original = original,
110 cachename = cachename,
111 }
112 end
113 end
114 if io.exists(cachename) then
115 cached[original] = cachename
116 if trace_schemes then
117 report_schemes("using cached %a, protocol %a, cachename %a",original,scheme,cachename)
118 end
119 else
120 cached[original] = ""
121 if trace_schemes then
122 report_schemes("using missing %a, protocol %a",original,scheme)
123 end
124 end
125 loaded[scheme] = loaded[scheme] + 1
126 statistics.stoptiming(schemes)
127 else
128 if trace_schemes then
129 report_schemes("reusing %a, protocol %a",original,scheme)
130 end
131 reused[scheme] = reused[scheme] + 1
132 end
133 return cached[original]
134end
135
136local function finder(specification,filetype)
137 return resolvers.methodhandler("finders",fetch(specification),filetype)
138end
139
140local opener = openers.file
141local loader = loaders.file
142
143local function install(scheme,handler,newthreshold)
144 handlers [scheme] = handler
145 loaded [scheme] = 0
146 reused [scheme] = 0
147 finders [scheme] = finder
148 openers [scheme] = opener
149 loaders [scheme] = loader
150 thresholds[scheme] = newthreshold or threshold
151end
152
153schemes.install = install
154
155local function http_handler(specification,cachename)
156 local tempname = cachename .. ".tmp"
157 local handle = io.open(tempname,"wb")
158 local status, message = http.request {
159 url = specification.original,
160 sink = ltn12.sink.file(handle)
161 }
162 if not status then
163 os.remove(tempname)
164 else
165 os.remove(cachename)
166 os.rename(tempname,cachename)
167 end
168 return cachename
169end
170
171install('http',http_handler)
172install('https')
173install('ftp')
174
175statistics.register("scheme handling time", function()
176 local l, r, nl, nr = { }, { }, 0, 0
177 for k, v in sortedhash(loaded) do
178 if v > 0 then
179 nl = nl + 1
180 l[nl] = k .. ":" .. v
181 end
182 end
183 for k, v in sortedhash(reused) do
184 if v > 0 then
185 nr = nr + 1
186 r[nr] = k .. ":" .. v
187 end
188 end
189 local n = nl + nr
190 if n > 0 then
191 if nl == 0 then l = { "none" } end
192 if nr == 0 then r = { "none" } end
193 return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s",
194 statistics.elapsedtime(schemes), n, threshold, concat(l," "), concat(l," "))
195 else
196 return nil
197 end
198end)
199
200
201
202
203local httprequest = http.request
204local toquery = url.toquery
205
206local function fetchstring(url,data)
207 local q = data and toquery(data)
208 if q then
209 url = url .. "?" .. q
210 end
211 local reply = httprequest(url)
212 return reply
213end
214
215schemes.fetchstring = fetchstring
216
217function schemes.fetchtable(url,data)
218 local reply = fetchstring(url,data)
219 if reply then
220 local s = load("return " .. reply)
221 if s then
222 return s()
223 end
224 end
225end
226 |
