data-tre.lua /size: 10 Kb    last modification: 2020-07-01 14:35
1if not modules then modules = { } end modules ['data-tre'] = {
2    version   = 1.001,
3    comment   = "companion to luat-lib.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- A tree search is rather dumb ... there is some basic caching of searched trees
10-- but nothing is cached over runs ... it's also a wildcard one so we cannot use
11-- the normal scanner.
12
13-- tree://e:/temporary/mb-mp/**/drawing.jpg
14-- tree://e:/temporary/mb-mp/**/Drawing.jpg
15-- tree://t:./**/tufte.tex
16-- tree://t:/./**/tufte.tex
17-- tree://t:/**/tufte.tex
18-- dirlist://e:/temporary/mb-mp/**/drawing.jpg
19-- dirlist://e:/temporary/mb-mp/**/Drawing.jpg
20-- dirlist://e:/temporary/mb-mp/**/just/some/place/drawing.jpg
21-- dirlist://e:/temporary/mb-mp/**/images/drawing.jpg
22-- dirlist://e:/temporary/mb-mp/**/images/drawing.jpg?option=fileonly
23-- dirlist://///storage-2/resources/mb-mp/**/drawing.jpg
24-- dirlist://e:/**/drawing.jpg
25
26local type = type
27local find, gsub, lower = string.find, string.gsub, string.lower
28local basename, dirname, joinname = file.basename, file.dirname, file.join
29local globdir, isdir, isfile = dir.glob, lfs.isdir, lfs.isfile
30local P, lpegmatch = lpeg.P, lpeg.match
31
32local trace_locating = false  trackers.register("resolvers.locating", function(v) trace_locating = v end)
33
34local report_trees   = logs.reporter("resolvers","trees")
35
36local resolvers  = resolvers
37local finders    = resolvers.finders
38local openers    = resolvers.openers
39local loaders    = resolvers.loaders
40local locators   = resolvers.locators
41local hashers    = resolvers.hashers
42local generators = resolvers.generators
43
44do
45
46    local collectors = { }
47    local found      = { }
48    local notfound   = finders.notfound
49
50    function finders.tree(specification) -- to be adapted to new formats
51        local spec = specification.filename
52        local okay = found[spec]
53        if okay == nil then
54            if spec ~= "" then
55                local path = dirname(spec)
56                local name = basename(spec)
57                if path == "" then
58                    path = "."
59                end
60                local names = collectors[path]
61                if not names then
62                    local pattern = find(path,"/%*+$") and path or (path .. "/*")
63                    names = globdir(pattern)
64                    collectors[path] = names
65                end
66                local pattern = "/" .. gsub(name,"([%.%-%+])", "%%%1") .. "$"
67                for i=1,#names do
68                    local fullname = names[i]
69                    if find(fullname,pattern) then
70                        found[spec] = fullname
71                        return fullname
72                    end
73                end
74                -- let's be nice:
75                local pattern = lower(pattern)
76                for i=1,#names do
77                    local fullname = lower(names[i])
78                    if find(fullname,pattern) then
79                        if isfile(fullname) then
80                            found[spec] = fullname
81                            return fullname
82                        else
83                            -- no os name mapping
84                            break
85                        end
86                    end
87                end
88            end
89            okay = notfound() -- false
90            found[spec] = okay
91        end
92        return okay
93    end
94
95end
96
97do
98
99    local resolveprefix = resolvers.resolve
100    local appendhash    = resolvers.appendhash
101
102    local function dolocate(specification)
103        local name     = specification.filename
104        local realname = resolveprefix(name) -- no shortcut
105        if realname and realname ~= '' and isdir(realname) then
106            if trace_locating then
107                report_trees("locator %a found",realname)
108            end
109            appendhash('tree',name,false) -- don't cache
110        elseif trace_locating then
111            report_trees("locator %a not found",name)
112        end
113    end
114
115    locators.tree    = dolocate
116    locators.dirlist = dolocate
117    locators.dirfile = dolocate
118
119end
120
121
122do
123
124    local filegenerator = generators.file
125
126    generators.dirlist = filegenerator
127    generators.dirfile = filegenerator
128
129end
130
131do
132
133    local filegenerator = generators.file
134    local methodhandler = resolvers.methodhandler
135
136    local function dohash(specification)
137        local name = specification.filename
138        if trace_locating then
139            report_trees("analyzing %a",name)
140        end
141        methodhandler("hashers",name)
142        filegenerator(specification)
143    end
144
145    hashers.tree    = dohash
146    hashers.dirlist = dohash
147    hashers.dirfile = dohash
148
149end
150
151-- This is a variation on tree lookups but this time we do cache in the given
152-- root. We use a similar hasher as the resolvers because we have to deal with
153-- for instance trees with 50K xml files plus a similar amount of resources to
154-- deal and we don't want too much overhead.
155
156local resolve  do
157
158    local collectors  = { }
159    local splitter    = lpeg.splitat("/**/")
160    local stripper    = lpeg.replacer { [P("/") * P("*")^1 * P(-1)] = "" }
161
162    local loadcontent = caches.loadcontent
163    local savecontent = caches.savecontent
164
165    local notfound    = finders.notfound
166
167    local scanfiles   = resolvers.scanfiles
168    local lookup      = resolvers.get_from_content
169
170    table.setmetatableindex(collectors, function(t,k)
171        local rootname = lpegmatch(stripper,k)
172        local dataname = joinname(rootname,"dirlist")
173        local content  = loadcontent(dataname,"files",dataname)
174        if not content then
175            -- path branch usecache onlyonce tolerant
176            content = scanfiles(rootname,nil,nil,false,true) -- so we accept crap
177            savecontent(dataname,"files",content,dataname)
178        end
179        t[k] = content
180        return content
181    end)
182
183    local function checked(root,p,n)
184        if p then
185            if type(p) == "table" then
186                for i=1,#p do
187                    local fullname = joinname(root,p[i],n)
188                    if isfile(fullname) then -- safeguard
189                        return fullname
190                    end
191                end
192            else
193                local fullname = joinname(root,p,n)
194                if isfile(fullname) then -- safeguard
195                    return fullname
196                end
197            end
198        end
199        return notfound()
200    end
201
202    -- no funny characters in path but in filename permitted .. sigh
203
204    resolve = function(specification) -- can be called directly too
205        local filename = specification.filename
206     -- inspect(specification)
207        if filename ~= "" then
208            local root, rest = lpegmatch(splitter,filename)
209            if root and rest then
210                local path, name = dirname(rest), basename(rest)
211                if name ~= rest then
212                    local content = collectors[root]
213                    local p, n = lookup(content,name)
214                    if not p then
215                        return notfound()
216                    end
217                    local pattern = ".*/" .. path .. "$"
218                    local istable = type(p) == "table"
219                    if istable then
220                        for i=1,#p do
221                            local pi = p[i]
222                            if pi == path or find(pi,pattern) then
223                                local fullname = joinname(root,pi,n)
224                                if isfile(fullname) then -- safeguard
225                                    return fullname
226                                end
227                            end
228                        end
229                    elseif p == path or find(p,pattern) then
230                        local fullname = joinname(root,p,n)
231                        if isfile(fullname) then -- safeguard
232                            return fullname
233                        end
234                    end
235                    local queries = specification.queries
236                    if queries and queries.option == "fileonly" then
237                        return checked(root,p,n)
238                    else
239                        return notfound()
240                    end
241                end
242            end
243            local path    = dirname(filename)
244            local name    = basename(filename)
245            local root    = lpegmatch(stripper,path)
246            local content = collectors[path]
247            local p, n = lookup(content,name)
248            if p then
249                return checked(root,p,n)
250            end
251        end
252        return notfound()
253    end
254
255    finders.dirlist = resolve
256
257    function finders.dirfile(specification)
258        local queries = specification.queries
259        if queries then
260            queries.option = "fileonly"
261        else
262            specification.queries = { option = "fileonly" }
263        end
264        return resolve(specification)
265    end
266
267end
268
269do
270
271    local fileopener = openers.file
272    local fileloader = loaders.file
273
274    openers.dirlist = fileopener
275    loaders.dirlist = fileloader
276
277    openers.dirfile = fileopener
278    loaders.dirfile = fileloader
279
280end
281
282-- print(resolvers.findtexfile("tree://e:/temporary/mb-mp/**/VB_wmf_03_vw_01d_ant.jpg"))
283-- print(resolvers.findtexfile("tree://t:/**/tufte.tex"))
284-- print(resolvers.findtexfile("dirlist://e:/temporary/mb-mp/**/VB_wmf_03_vw_01d_ant.jpg"))
285
286
287do
288
289    local hashfile = "dirhash.lua"
290    local kind     = "HASH256"
291    local version  = 1.0
292
293    local loadtable = table.load
294    local savetable = table.save
295    local loaddata  = io.loaddata
296
297    function resolvers.dirstatus(patterns)
298        local t = type(patterns)
299        if t == "string" then
300            patterns = { patterns }
301        elseif t ~= "table" then
302            return false
303        end
304        local status = loadtable(hashfile)
305        if not status or status.version ~= version or status.kind ~= kind then
306            status = {
307                version = 1.0,
308                kind    = kind,
309                hashes  = { },
310            }
311        end
312        local hashes  = status.hashes
313        local changed = { }
314        local action  = sha2[kind]
315        local update  = { }
316        for i=1,#patterns do
317            local pattern = patterns[i]
318            local files   = globdir(pattern)
319            for i=1,#files do
320                local name = files[i]
321                local hash = action(loaddata(name))
322                if hashes[name] ~= hash then
323                    changed[#changed+1] = name
324                end
325                update[name] = hash
326            end
327        end
328        status.hashes = update
329        savetable(hashfile,status)
330        return #changed > 0 and changed or false
331    end
332
333end
334