data-sch.lua /size: 6945 b    last modification: 2020-07-01 14:35
1if not modules then modules = { } end modules ['data-sch'] = {
2    version   = 1.001,
3    comment   = "companion to luat-lib.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- this one will be redone / localized
10
11local load, tonumber = load, tonumber
12local gsub, format = string.gsub, string.format
13local sortedhash, concat = table.sortedhash, table.concat
14local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders
15local addsuffix, suffix, splitbase = file.addsuffix, file.suffix, file.splitbase
16local md5hex = md5.hex
17
18local trace_schemes  = false  trackers.register("resolvers.schemes",function(v) trace_schemes = v end)
19local report_schemes = logs.reporter("resolvers","schemes")
20
21local http           = require("socket.http")
22local ltn12          = require("ltn12")
23
24if mbox then mbox = nil end -- useless and even bugged (helper overwrites lib)
25
26local resolvers      = resolvers
27local schemes        = resolvers.schemes or { }
28resolvers.schemes    = schemes
29
30local cleaners       = { }
31schemes.cleaners     = cleaners
32
33local threshold      = 24 * 60 * 60
34
35directives.register("schemes.threshold", function(v) threshold = tonumber(v) or threshold end)
36
37function cleaners.none(specification)
38    return specification.original
39end
40
41-- function cleaners.strip(specification)
42--     -- todo: only keep suffix periods, so after the last
43--     return (gsub(specification.original,"[^%a%d%.]+","-")) -- so we keep periods
44-- end
45
46function cleaners.strip(specification) -- keep suffixes
47    local path, name = splitbase(specification.original)
48    if path == "" then
49        return (gsub(name,"[^%a%d%.]+","-"))
50    else
51        return (gsub((gsub(path,"%.","-") .. "-" .. name),"[^%a%d%.]+","-"))
52    end
53end
54
55function cleaners.md5(specification)
56    return addsuffix(md5hex(specification.original),suffix(specification.path))
57end
58
59local cleaner = cleaners.strip
60
61directives.register("schemes.cleanmethod", function(v) cleaner = cleaners[v] or cleaners.strip end)
62
63function resolvers.schemes.cleanname(specification)
64    local hash = cleaner(specification)
65    if trace_schemes then
66        report_schemes("hashing %a to %a",specification.original,hash)
67    end
68    return hash
69end
70
71local cached     = { }
72local loaded     = { }
73local reused     = { }
74local thresholds = { }
75local handlers   = { }
76local runner     = sandbox.registerrunner {
77    name     = "curl resolver",
78    method   = "execute",
79    program  = "curl",
80    template = '--silent --insecure --create-dirs --output "%cachename%" "%original%"',
81    checkers = {
82        cachename = "cache",
83        original  = "url",
84    }
85}
86
87local function fetch(specification)
88    local original  = specification.original
89    local scheme    = specification.scheme
90    local cleanname = schemes.cleanname(specification)
91    local cachename = caches.setfirstwritablefile(cleanname,"schemes")
92    if not cached[original] then
93        statistics.starttiming(schemes)
94        if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > (thresholds[protocol] or threshold)) then
95            cached[original] = cachename
96            local handler = handlers[scheme]
97            if handler then
98                if trace_schemes then
99                    report_schemes("fetching %a, protocol %a, method %a",original,scheme,"built-in")
100                end
101                logs.flush()
102                handler(specification,cachename)
103            else
104                if trace_schemes then
105                    report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl")
106                end
107                logs.flush()
108                runner {
109                    original  = original,
110                    cachename = cachename,
111                }
112            end
113        end
114        if io.exists(cachename) then
115            cached[original] = cachename
116            if trace_schemes then
117                report_schemes("using cached %a, protocol %a, cachename %a",original,scheme,cachename)
118            end
119        else
120            cached[original] = ""
121            if trace_schemes then
122                report_schemes("using missing %a, protocol %a",original,scheme)
123            end
124        end
125        loaded[scheme] = loaded[scheme] + 1
126        statistics.stoptiming(schemes)
127    else
128        if trace_schemes then
129            report_schemes("reusing %a, protocol %a",original,scheme)
130        end
131        reused[scheme] = reused[scheme] + 1
132    end
133    return cached[original]
134end
135
136local function finder(specification,filetype)
137    return resolvers.methodhandler("finders",fetch(specification),filetype)
138end
139
140local opener = openers.file
141local loader = loaders.file
142
143local function install(scheme,handler,newthreshold)
144    handlers  [scheme] = handler
145    loaded    [scheme] = 0
146    reused    [scheme] = 0
147    finders   [scheme] = finder
148    openers   [scheme] = opener
149    loaders   [scheme] = loader
150    thresholds[scheme] = newthreshold or threshold
151end
152
153schemes.install = install
154
155local function http_handler(specification,cachename)
156    local tempname = cachename .. ".tmp"
157    local handle   = io.open(tempname,"wb")
158    local status, message = http.request {
159        url  = specification.original,
160        sink = ltn12.sink.file(handle)
161    }
162    if not status then
163        os.remove(tempname)
164    else
165        os.remove(cachename)
166        os.rename(tempname,cachename)
167    end
168    return cachename
169end
170
171install('http',http_handler)
172install('https') -- see pod
173install('ftp')
174
175statistics.register("scheme handling time", function()
176    local l, r, nl, nr = { }, { }, 0, 0
177    for k, v in sortedhash(loaded) do
178        if v > 0 then
179            nl = nl + 1
180            l[nl] = k .. ":" .. v
181        end
182    end
183    for k, v in sortedhash(reused) do
184        if v > 0 then
185            nr = nr + 1
186            r[nr] = k .. ":" .. v
187        end
188    end
189    local n = nl + nr
190    if n > 0 then
191        if nl == 0 then l = { "none" } end
192        if nr == 0 then r = { "none" } end
193        return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s",
194            statistics.elapsedtime(schemes), n, threshold, concat(l," "), concat(l," "))
195    else
196        return nil
197    end
198end)
199
200-- We provide a few more helpers:
201
202----- http        = require("socket.http")
203local httprequest = http.request
204local toquery     = url.toquery
205
206local function fetchstring(url,data)
207    local q = data and toquery(data)
208    if q then
209        url = url .. "?" .. q
210    end
211    local reply = httprequest(url)
212    return reply -- just one argument
213end
214
215schemes.fetchstring = fetchstring
216
217function schemes.fetchtable(url,data)
218    local reply = fetchstring(url,data)
219    if reply then
220        local s = load("return " .. reply)
221        if s then
222            return s()
223        end
224    end
225end
226