data-res.lua /size: 67 Kb    last modification: 2020-07-01 14:35
1if not modules then modules = { } end modules ['data-res'] = {
2    version   = 1.001,
3    comment   = "companion to luat-lib.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files",
7}
8
9-- todo: cache:/// home:/// selfautoparent:/// (sometime end 2012)
10
11local gsub, find, lower, upper, match, gmatch = string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
12local concat, insert, remove = table.concat, table.insert, table.remove
13local next, type, rawget, loadfile = next, type, rawget, loadfile
14local mergedtable = table.merged
15
16local os = os
17
18local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg
19local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns
20
21local formatters        = string.formatters
22local filedirname       = file.dirname
23local filebasename      = file.basename
24local suffixonly        = file.suffixonly
25local addsuffix         = file.addsuffix
26local removesuffix      = file.removesuffix
27local filejoin          = file.join
28local collapsepath      = file.collapsepath
29local joinpath          = file.joinpath
30local is_qualified_path = file.is_qualified_path
31
32local allocate          = utilities.storage.allocate
33local settings_to_array = utilities.parsers.settings_to_array
34
35local urlhasscheme      = url.hasscheme
36
37local getcurrentdir     = lfs.currentdir
38local isfile            = lfs.isfile
39local isdir             = lfs.isdir
40
41local setmetatableindex = table.setmetatableindex
42local luasuffixes       = utilities.lua.suffixes
43
44local trace_locating    = false  trackers  .register("resolvers.locating",   function(v) trace_locating    = v end)
45local trace_details     = false  trackers  .register("resolvers.details",    function(v) trace_details     = v end)
46local trace_expansions  = false  trackers  .register("resolvers.expansions", function(v) trace_expansions  = v end)
47local trace_paths       = false  trackers  .register("resolvers.paths",      function(v) trace_paths       = v end)
48local resolve_otherwise = true   directives.register("resolvers.otherwise",  function(v) resolve_otherwise = v end)
49
50local report_resolving = logs.reporter("resolvers","resolving")
51
52local resolvers              = resolvers
53
54local expandedpathfromlist   = resolvers.expandedpathfromlist
55local checkedvariable        = resolvers.checkedvariable
56local splitconfigurationpath = resolvers.splitconfigurationpath
57local methodhandler          = resolvers.methodhandler
58local filtered               = resolvers.filtered_from_content
59local lookup                 = resolvers.get_from_content
60local cleanpath              = resolvers.cleanpath
61local resolveprefix          = resolvers.resolve
62
63local initializesetter       = utilities.setters.initialize
64
65local ostype, osname, osenv, ossetenv, osgetenv = os.type, os.name, os.env, os.setenv, os.getenv
66
67resolvers.cacheversion   = "1.100"
68resolvers.configbanner   = ""
69resolvers.homedir        = environment.homedir
70resolvers.luacnfname     = "texmfcnf.lua"
71resolvers.luacnffallback = "contextcnf.lua"
72resolvers.luacnfstate    = "unknown"
73
74local criticalvars = {
75    "SELFAUTOLOC",
76    "SELFAUTODIR",
77    "SELFAUTOPARENT",
78    "TEXMFCNF",
79    "TEXMF",
80    "TEXOS",
81}
82
83-- The web2c tex binaries as well as kpse have built in paths for the configuration
84-- files and there can be a depressing truckload of them. This is actually the weak
85-- spot of a distribution. So we don't want:
86--
87-- resolvers.luacnfspec = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}'
88--
89-- but instead (for instance) use:
90--
91-- resolvers.luacnfspec = 'selfautoparent:{/texmf{-local,}{,/web2c}}'
92--
93-- which does not make texlive happy as there is a texmf-local tree one level up
94-- (sigh), so we need this. We can assume web2c as mkiv does not run on older
95-- texlives anyway.
96--
97-- texlive:
98--
99-- selfautoloc:
100-- selfautoloc:/share/texmf-local/web2c
101-- selfautoloc:/share/texmf-dist/web2c
102-- selfautoloc:/share/texmf/web2c
103-- selfautoloc:/texmf-local/web2c
104-- selfautoloc:/texmf-dist/web2c
105-- selfautoloc:/texmf/web2c
106-- selfautodir:
107-- selfautodir:/share/texmf-local/web2c
108-- selfautodir:/share/texmf-dist/web2c
109-- selfautodir:/share/texmf/web2c
110-- selfautodir:/texmf-local/web2c
111-- selfautodir:/texmf-dist/web2c
112-- selfautodir:/texmf/web2c
113-- selfautoparent:/../texmf-local/web2c
114-- selfautoparent:
115-- selfautoparent:/share/texmf-local/web2c
116-- selfautoparent:/share/texmf-dist/web2c
117-- selfautoparent:/share/texmf/web2c
118-- selfautoparent:/texmf-local/web2c
119-- selfautoparent:/texmf-dist/web2c
120-- selfautoparent:/texmf/web2c
121--
122-- minimals:
123--
124-- home:texmf/web2c
125-- selfautoparent:texmf-local/web2c
126-- selfautoparent:texmf-context/web2c
127-- selfautoparent:texmf/web2c
128
129-- This is a real mess: you don't want to know what creepy paths end up in the default
130-- configuration spec, for instance nested texmf- paths. I'd rather get away from it and
131-- specify a proper search sequence but alas ... it is not permitted in texlive and there
132-- is no way to check if we run a minimals as texmf-context is not in that spec. It's a
133-- compiled-in permutation of historics with the selfautoloc, selfautodir, selfautoparent
134-- resulting in weird combinations. So, when we eventually check the 30 something paths
135-- we also report weird ones, with weird being: (1) duplicate /texmf or (2) no /web2c in
136-- the names.
137
138if environment.default_texmfcnf then
139    resolvers.luacnfspec = "home:texmf/web2c;" .. environment.default_texmfcnf -- texlive + home: for taco etc
140else
141    resolvers.luacnfspec = concat ( {
142        "home:texmf/web2c",
143        "selfautoparent:/texmf-local/web2c",
144        "selfautoparent:/texmf-context/web2c",
145        "selfautoparent:/texmf-dist/web2c",
146        "selfautoparent:/texmf/web2c",
147    }, ";")
148end
149
150local unset_variable = "unset"
151
152local formats   = resolvers.formats
153local suffixes  = resolvers.suffixes
154local usertypes = resolvers.usertypes
155local dangerous = resolvers.dangerous
156local suffixmap = resolvers.suffixmap
157
158resolvers.defaultsuffixes = { "tex" } --  "mkiv", "cld" -- too tricky
159
160local instance  = nil -- the current one (fast access)
161
162-- forward declarations
163
164local variable
165local expansion
166local setenv
167local getenv
168
169-- done
170
171local formatofsuffix = resolvers.formatofsuffix
172local splitpath      = resolvers.splitpath
173local splitmethod    = resolvers.splitmethod
174
175-- An instance has an environment (coming from the outside, kept raw), variables
176-- (coming from the configuration file), and expansions (variables with nested
177-- variables replaced). One can push something into the outer environment and
178-- its internal copy, but only the later one will be the raw unprefixed variant.
179
180setenv = function(key,value,raw)
181    if instance then
182        -- this one will be consulted first when we stay inside
183        -- the current environment (prefixes are not resolved here)
184        instance.environment[key] = value
185        -- we feed back into the environment, and as this is used
186        -- by other applications (via os.execute) we need to make
187        -- sure that prefixes are resolve
188        ossetenv(key,raw and value or resolveprefix(value))
189    end
190end
191
192-- Beware we don't want empty here as this one can be called early on
193-- and therefore we use rawget.
194
195getenv = function(key)
196    local value = rawget(instance.environment,key)
197    if value and value ~= "" then
198        return value
199    else
200        local e = osgetenv(key)
201        return e ~= nil and e ~= "" and checkedvariable(e) or ""
202    end
203end
204
205resolvers.getenv = getenv
206resolvers.setenv = setenv
207
208-- We are going to use some metatable trickery where we backtrack from
209-- expansion to variable to environment.
210
211local dollarstripper   = lpeg.stripper("$")
212local inhibitstripper  = P("!")^0 * Cs(P(1)^0)
213
214local expandedvariable, resolvedvariable  do
215
216    local function resolveinstancevariable(k)
217        return instance.expansions[k]
218    end
219
220    local p_variable       = P("$") / ""
221    local p_key            = C(R("az","AZ","09","__","--")^1)
222    local p_whatever       = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "")
223                           + P(";") * (P(";") / "")
224                           + P(1)
225    local variableexpander = Cs( (p_variable * (p_key/resolveinstancevariable) + p_whatever)^1 )
226
227    local p_cleaner        = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";"
228    local variablecleaner  = Cs((p_cleaner  + P(1))^0)
229
230    local p_variable       = R("az","AZ","09","__","--")^1 / resolveinstancevariable
231    local p_variable       = (P("$")/"") * (p_variable + (P("{")/"") * p_variable * (P("}")/""))
232    local variableresolver = Cs((p_variable + P(1))^0)
233
234    expandedvariable = function(var)
235        return lpegmatch(variableexpander,var) or var
236    end
237
238    function resolvers.reset()
239
240        -- normally we only need one instance but for special cases we can (re)load one so
241        -- we stick to this model.
242
243        if trace_locating then
244            report_resolving("creating instance")
245        end
246
247        local environment = { }
248        local variables   = { }
249        local expansions  = { }
250        local order       = { }
251
252        instance = {
253            environment    = environment,
254            variables      = variables,
255            expansions     = expansions,
256            order          = order,
257            files          = { },
258            setups         = { },
259            found          = { },
260            foundintrees   = { },
261            hashes         = { },
262            hashed         = { },
263            pathlists      = false,-- delayed
264            specification  = { },
265            lists          = { },
266            data           = { }, -- only for loading
267            fakepaths      = { },
268            remember       = true,
269            diskcache      = true,
270            renewcache     = false,
271            renewtree      = false,
272            loaderror      = false,
273            savelists      = true,
274            pattern        = nil, -- lists
275            force_suffixes = true,
276            pathstack      = { },
277        }
278
279        setmetatableindex(variables,function(t,k)
280            local v
281            for i=1,#order do
282                v = order[i][k]
283                if v ~= nil then
284                    t[k] = v
285                    return v
286                end
287            end
288            if v == nil then
289                v = ""
290            end
291            t[k] = v
292            return v
293        end)
294
295        local repath = resolvers.repath
296
297        setmetatableindex(environment, function(t,k)
298            local v = osgetenv(k)
299            if v == nil then
300                v = variables[k]
301            end
302            if v ~= nil then
303                v = checkedvariable(v) or ""
304            end
305            v = repath(v) -- for taco who has a : separated osfontdir
306            t[k] = v
307            return v
308        end)
309
310        setmetatableindex(expansions, function(t,k)
311            local v = environment[k]
312            if type(v) == "string" then
313                v = lpegmatch(variableresolver,v)
314                v = lpegmatch(variablecleaner,v)
315            end
316            t[k] = v
317            return v
318        end)
319
320    end
321
322end
323
324function resolvers.initialized()
325    return instance ~= nil
326end
327
328local function reset_hashes()
329    instance.lists     = { }
330    instance.pathlists = false
331    instance.found     = { }
332end
333
334local function reset_caches()
335    instance.lists     = { }
336    instance.pathlists = false
337end
338
339local makepathexpression  do
340
341    local slash = P("/")
342
343    local pathexpressionpattern = Cs ( -- create lpeg instead (2013/2014)
344        Cc("^") * (
345            Cc("%") * S(".-")
346          + slash^2 * P(-1) / "/.*"
347       -- + slash^2 / "/.-/"
348       -- + slash^2 / "/[^/]*/*"   -- too general
349          + slash^2 / "/"
350          + (1-slash) * P(-1) * Cc("/")
351          + P(1)
352        )^1 * Cc("$") -- yes or no $
353    )
354
355    local cache = { }
356
357    makepathexpression = function(str)
358        if str == "." then
359            return "^%./$"
360        else
361            local c = cache[str]
362            if not c then
363                c = lpegmatch(pathexpressionpattern,str)
364                cache[str] = c
365            end
366            return c
367        end
368    end
369
370end
371
372local function reportcriticalvariables(cnfspec)
373    if trace_locating then
374        for i=1,#criticalvars do
375            local k = criticalvars[i]
376            local v = getenv(k) or "unknown" -- this one will not resolve !
377            report_resolving("variable %a set to %a",k,v)
378        end
379        report_resolving()
380        if cnfspec then
381            report_resolving("using configuration specification %a",type(cnfspec) == "table" and concat(cnfspec,",") or cnfspec)
382        end
383        report_resolving()
384    end
385    reportcriticalvariables = function() end
386end
387
388local function identify_configuration_files()
389    local specification = instance.specification
390    if #specification == 0 then
391        local cnfspec = getenv("TEXMFCNF")
392        if cnfspec == "" then
393            cnfspec = resolvers.luacnfspec
394            resolvers.luacnfstate = "default"
395        else
396            resolvers.luacnfstate = "environment"
397        end
398        reportcriticalvariables(cnfspec)
399        local cnfpaths = expandedpathfromlist(splitpath(cnfspec))
400
401        local function locatecnf(luacnfname,kind)
402            for i=1,#cnfpaths do
403                local filepath = cnfpaths[i]
404                local filename = collapsepath(filejoin(filepath,luacnfname))
405                local realname = resolveprefix(filename) -- can still have "//" ... needs checking
406                -- todo: environment.skipweirdcnfpaths directive
407                if trace_locating then
408                    local fullpath  = gsub(resolveprefix(collapsepath(filepath)),"//","/")
409                    local weirdpath = find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true)
410                    report_resolving("looking for %s %a on %s path %a from specification %a",
411                        kind,luacnfname,weirdpath and "weird" or "given",fullpath,filepath)
412                end
413                if isfile(realname) then
414                    specification[#specification+1] = filename -- unresolved as we use it in matching, relocatable
415                    if trace_locating then
416                        report_resolving("found %s configuration file %a",kind,realname)
417                    end
418                end
419            end
420        end
421
422        locatecnf(resolvers.luacnfname,"regular")
423        if #specification == 0 then
424            locatecnf(resolvers.luacnffallback,"fallback")
425        end
426        if trace_locating then
427            report_resolving()
428        end
429    elseif trace_locating then
430        report_resolving("configuration files already identified")
431    end
432end
433
434local function load_configuration_files()
435    local specification = instance.specification
436    local setups        = instance.setups
437    local order         = instance.order
438    if #specification > 0 then
439        local luacnfname = resolvers.luacnfname
440        for i=1,#specification do
441            local filename = specification[i]
442            local pathname = filedirname(filename)
443            local filename = filejoin(pathname,luacnfname)
444            local realname = resolveprefix(filename) -- no shortcut
445            local blob = loadfile(realname)
446            if blob then
447                local data = blob()
448                local parent = data and data.parent
449                if parent then
450                    local filename = filejoin(pathname,parent)
451                    local realname = resolveprefix(filename) -- no shortcut
452                    local blob = loadfile(realname)
453                    if blob then
454                        local parentdata = blob()
455                        if parentdata then
456                            report_resolving("loading configuration file %a",filename)
457                            data = mergedtable(parentdata,data)
458                        end
459                    end
460                end
461                data = data and data.content
462                if data then
463                    if trace_locating then
464                        report_resolving("loading configuration file %a",filename)
465                        report_resolving()
466                    end
467                    local variables = data.variables or { }
468                    local warning = false
469                    for k, v in next, data do
470                        local variant = type(v)
471                        if variant == "table" then
472                            initializesetter(filename,k,v)
473                        elseif variables[k] == nil then
474                            if trace_locating and not warning then
475                                report_resolving("variables like %a in configuration file %a should move to the 'variables' subtable",
476                                    k,resolveprefix(filename))
477                                warning = true
478                            end
479                            variables[k] = v
480                        end
481                    end
482                    setups[pathname] = variables
483                    if resolvers.luacnfstate == "default" then
484                        -- the following code is not tested
485                        local cnfspec = variables["TEXMFCNF"]
486                        if cnfspec then
487                            if trace_locating then
488                                report_resolving("reloading configuration due to TEXMF redefinition")
489                            end
490                            -- we push the value into the main environment (osenv) so
491                            -- that it takes precedence over the default one and therefore
492                            -- also over following definitions
493                            setenv("TEXMFCNF",cnfspec) -- resolves prefixes
494                            -- we now identify and load the specified configuration files
495                            instance.specification = { }
496                            identify_configuration_files()
497                            load_configuration_files()
498                            -- we prevent further overload of the configuration variable
499                            resolvers.luacnfstate = "configuration"
500                            -- we quit the outer loop
501                            break
502                        end
503                    end
504
505                else
506                    if trace_locating then
507                        report_resolving("skipping configuration file %a (no content)",filename)
508                    end
509                    setups[pathname] = { }
510                    instance.loaderror = true
511                end
512            elseif trace_locating then
513                report_resolving("skipping configuration file %a (no valid format)",filename)
514            end
515            order[#order+1] = setups[pathname]
516            if instance.loaderror then
517                break
518            end
519        end
520    elseif trace_locating then
521        report_resolving("warning: no lua configuration files found")
522    end
523end
524
525-- forward declarations:
526
527local expandedpathlist
528local unexpandedpathlist
529
530-- done
531
532function resolvers.configurationfiles()
533    return instance.specification or { }
534end
535
536-- scheme magic ... database loading
537
538local function load_file_databases()
539    instance.loaderror = false
540    instance.files     = { }
541    if not instance.renewcache then
542        local hashes = instance.hashes
543        for k=1,#hashes do
544            local hash = hashes[k]
545            resolvers.hashers.byscheme(hash.type,hash.name)
546            if instance.loaderror then break end
547        end
548    end
549end
550
551local function locate_file_databases()
552    -- todo: cache:// and tree:// (runtime)
553    local texmfpaths = expandedpathlist("TEXMF")
554    if #texmfpaths > 0 then
555        for i=1,#texmfpaths do
556            local path = collapsepath(texmfpaths[i])
557            path = gsub(path,"/+$","") -- in case $HOME expands to something with a trailing /
558            local stripped = lpegmatch(inhibitstripper,path) -- the !! thing
559            if stripped ~= "" then
560                local runtime = stripped == path
561                path = cleanpath(path)
562                local spec = splitmethod(stripped)
563                if runtime and (spec.noscheme or spec.scheme == "file") then
564                    stripped = "tree:///" .. stripped
565                elseif spec.scheme == "cache" or spec.scheme == "file" then
566                    stripped = spec.path
567                end
568                if trace_locating then
569                    if runtime then
570                        report_resolving("locating list of %a (runtime) (%s)",path,stripped)
571                    else
572                        report_resolving("locating list of %a (cached)",path)
573                    end
574                end
575                methodhandler('locators',stripped)
576            end
577        end
578        if trace_locating then
579            report_resolving()
580        end
581    elseif trace_locating then
582        report_resolving("no texmf paths are defined (using TEXMF)")
583    end
584end
585
586local function generate_file_databases()
587    local hashes = instance.hashes
588    for k=1,#hashes do
589        local hash = hashes[k]
590        methodhandler('generators',hash.name)
591    end
592    if trace_locating then
593        report_resolving()
594    end
595end
596
597local function save_file_databases() -- will become cachers
598    local hashes = instance.hashes
599    local files  = instance.files
600    for i=1,#hashes do
601        local hash      = hashes[i]
602        local cachename = hash.name
603        if hash.cache then
604            local content = files[cachename]
605            caches.collapsecontent(content)
606            if trace_locating then
607                report_resolving("saving tree %a",cachename)
608            end
609            caches.savecontent(cachename,"files",content)
610        elseif trace_locating then
611            report_resolving("not saving runtime tree %a",cachename)
612        end
613    end
614end
615
616function resolvers.renew(hashname)
617    local files = instance.files
618    if hashname and hashname ~= "" then
619        local expanded = expansion(hashname) or ""
620        if expanded ~= "" then
621            if trace_locating then
622                report_resolving("identifying tree %a from %a",expanded,hashname)
623            end
624            hashname = expanded
625        else
626            if trace_locating then
627                report_resolving("identifying tree %a",hashname)
628            end
629        end
630        local realpath = resolveprefix(hashname)
631        if isdir(realpath) then
632            if trace_locating then
633                report_resolving("using path %a",realpath)
634            end
635            methodhandler('generators',hashname)
636            -- could be shared
637            local content = files[hashname]
638            caches.collapsecontent(content)
639            if trace_locating then
640                report_resolving("saving tree %a",hashname)
641            end
642            caches.savecontent(hashname,"files",content)
643            -- till here
644        else
645            report_resolving("invalid path %a",realpath)
646        end
647    end
648end
649
650local function load_databases()
651    locate_file_databases()
652    if instance.diskcache and not instance.renewcache then
653        load_file_databases()
654        if instance.loaderror then
655            generate_file_databases()
656            save_file_databases()
657        end
658    else
659        generate_file_databases()
660        if instance.renewcache then
661            save_file_databases()
662        end
663    end
664end
665
666function resolvers.appendhash(type,name,cache)
667    local hashed = instance.hashed
668    local hashes = instance.hashes
669    if hashed[name] then
670        -- safeguard ... tricky as it's actually a bug when seen twice
671    else
672        if trace_locating then
673            report_resolving("hash %a appended",name)
674        end
675        insert(hashes, { type = type, name = name, cache = cache } )
676        hashed[name] = cache
677    end
678end
679
680function resolvers.prependhash(type,name,cache)
681    local hashed = instance.hashed
682    local hashes = instance.hashes
683    if hashed[name] then
684        -- safeguard ... tricky as it's actually a bug when seen twice
685    else
686        if trace_locating then
687            report_resolving("hash %a prepended",name)
688        end
689        insert(hashes, 1, { type = type, name = name, cache = cache } )
690        hashed[name] = cache
691    end
692end
693
694function resolvers.extendtexmfvariable(specification) -- crap, we could better prepend the hash
695    local environment = instance.environment
696    local variables   = instance.variables
697    local texmftrees  = splitpath(getenv("TEXMF")) -- okay?
698    insert(texmftrees,1,specification)
699    texmftrees = concat(texmftrees,",") -- not ;
700    if environment["TEXMF"] then
701        environment["TEXMF"] = texmftrees
702    elseif variables["TEXMF"] then
703        variables["TEXMF"] = texmftrees
704    else
705        -- weird
706    end
707    reset_hashes()
708end
709
710function resolvers.splitexpansions()
711    local expansions = instance.expansions
712    for k, v in next, expansions do
713        local t, tn, h, p = { }, 0, { }, splitconfigurationpath(v)
714        for kk=1,#p do
715            local vv = p[kk]
716            if vv ~= "" and not h[vv] then
717                tn = tn + 1
718                t[tn] = vv
719                h[vv] = true
720            end
721        end
722        if tn > 1 then
723            expansions[k] = t
724        else
725            expansions[k] = t[1]
726        end
727    end
728end
729
730-- end of split/join code
731
732-- we used to have 'files' and 'configurations' so therefore the following
733-- shared function
734
735function resolvers.datastate()
736    return caches.contentstate()
737end
738
739variable = function(name)
740    local variables = instance.variables
741    local name   = name and lpegmatch(dollarstripper,name)
742    local result = name and variables[name]
743    return result ~= nil and result or ""
744end
745
746expansion = function(name)
747    local expansions = instance.expansions
748    local name   = name and lpegmatch(dollarstripper,name)
749    local result = name and expansions[name]
750    return result ~= nil and result or ""
751end
752
753resolvers.variable  = variable
754resolvers.expansion = expansion
755
756unexpandedpathlist = function(str)
757    local pth = variable(str)
758    local lst = splitpath(pth)
759    return expandedpathfromlist(lst)
760end
761
762function resolvers.unexpandedpath(str)
763    return joinpath(unexpandedpathlist(str))
764end
765
766function resolvers.pushpath(name)
767    local pathstack = instance.pathstack
768    local lastpath  = pathstack[#pathstack]
769    local pluspath  = filedirname(name)
770    if lastpath then
771        lastpath = collapsepath(filejoin(lastpath,pluspath))
772    else
773        lastpath = collapsepath(pluspath)
774    end
775    insert(pathstack,lastpath)
776    if trace_paths then
777        report_resolving("pushing path %a",lastpath)
778    end
779end
780
781function resolvers.poppath()
782    local pathstack = instance.pathstack
783    if trace_paths and #pathstack > 0 then
784        report_resolving("popping path %a",pathstack[#pathstack])
785    end
786    remove(pathstack)
787end
788
789function resolvers.stackpath()
790    local pathstack   = instance.pathstack
791    local currentpath = pathstack[#pathstack]
792    return currentpath ~= "" and currentpath or nil
793end
794
795local done = { }
796
797function resolvers.resetextrapaths()
798    local extra_paths = instance.extra_paths
799    if not extra_paths then
800        done                 = { }
801        instance.extra_paths = { }
802    elseif #ep > 0 then
803        done = { }
804        reset_caches()
805    end
806end
807
808function resolvers.getextrapaths()
809    return instance.extra_paths or { }
810end
811
812function resolvers.registerextrapath(paths,subpaths)
813    if not subpaths or subpaths == "" then
814        if not paths or path == "" then
815            return -- invalid spec
816        elseif done[paths] then
817            return -- already done
818        end
819    end
820    local paths       = settings_to_array(paths)
821    local subpaths    = settings_to_array(subpaths)
822    local extra_paths = instance.extra_paths or { }
823    local oldn        = #extra_paths
824    local newn        = oldn
825    local nofpaths    = #paths
826    local nofsubpaths = #subpaths
827    if nofpaths > 0 then
828        if nofsubpaths > 0 then
829            for i=1,nofpaths do
830                local p = paths[i]
831                for j=1,nofsubpaths do
832                    local s = subpaths[j]
833                    local ps = p .. "/" .. s
834                    if not done[ps] then
835                        newn = newn + 1
836                        extra_paths[newn] = cleanpath(ps)
837                        done[ps] = true
838                    end
839                end
840            end
841        else
842            for i=1,nofpaths do
843                local p = paths[i]
844                if not done[p] then
845                    newn = newn + 1
846                    extra_paths[newn] = cleanpath(p)
847                    done[p] = true
848                end
849            end
850        end
851    elseif nofsubpaths > 0 then
852        for i=1,oldn do
853            for j=1,nofsubpaths do
854                local s = subpaths[j]
855                local ps = extra_paths[i] .. "/" .. s
856                if not done[ps] then
857                    newn = newn + 1
858                    extra_paths[newn] = cleanpath(ps)
859                    done[ps] = true
860                end
861            end
862        end
863    end
864    if newn > 0 then
865        instance.extra_paths = extra_paths -- register paths
866    end
867    if newn ~= oldn then
868        reset_caches()
869    end
870end
871
872function resolvers.pushextrapath(path)
873    local paths = settings_to_array(path)
874    local extra_stack = instance.extra_stack
875    if extra_stack then
876        insert(extra_stack,1,paths)
877    else
878        instance.extra_stack = { paths }
879    end
880    reset_caches()
881end
882
883function resolvers.popextrapath()
884    local extra_stack = instance.extra_stack
885    if extra_stack then
886        reset_caches()
887        return remove(extra_stack,1)
888    end
889end
890
891local function made_list(instance,list,extra_too)
892    local done = { }
893    local new  = { }
894    local newn = 0
895    -- a helper
896    local function add(p)
897        for k=1,#p do
898            local v = p[k]
899            if not done[v] then
900                done[v] = true
901                newn = newn + 1
902                new[newn] = v
903            end
904        end
905    end
906    -- honour . .. ../.. but only when at the start
907    for k=1,#list do
908        local v = list[k]
909        if done[v] then
910            -- skip
911        elseif find(v,"^[%.%/]$") then
912            done[v] = true
913            newn = newn + 1
914            new[newn] = v
915        else
916            break
917        end
918    end
919    if extra_too then
920        local extra_stack = instance.extra_stack
921        local extra_paths = instance.extra_paths
922        -- first the stacked paths
923        if extra_stack and #extra_stack > 0 then
924            for k=1,#extra_stack do
925                add(extra_stack[k])
926            end
927        end
928        -- then the extra paths
929        if extra_paths and #extra_paths > 0 then
930            add(extra_paths)
931        end
932    end
933    -- last the formal paths
934    add(list)
935    return new
936end
937
938expandedpathlist = function(str,extra_too)
939    if not str then
940        return { }
941    elseif instance.savelists then -- hm, what if two cases, with and without extra_too
942        str = lpegmatch(dollarstripper,str)
943        local lists = instance.lists
944        local lst = lists[str]
945        if not lst then
946            local l = made_list(instance,splitpath(expansion(str)),extra_too)
947            lst = expandedpathfromlist(l)
948            lists[str] = lst
949        end
950        return lst
951    else
952        local lst = splitpath(expansion(str))
953        return made_list(instance,expandedpathfromlist(lst),extra_too)
954    end
955end
956
957resolvers.expandedpathlist   = expandedpathlist
958resolvers.unexpandedpathlist = unexpandedpathlist
959
960function resolvers.cleanpathlist(str)
961    local t = expandedpathlist(str)
962    if t then
963        for i=1,#t do
964            t[i] = collapsepath(cleanpath(t[i]))
965        end
966    end
967    return t
968end
969
970function resolvers.expandpath(str)
971    return joinpath(expandedpathlist(str))
972end
973
974local function expandedpathlistfromvariable(str) -- brrr / could also have cleaner ^!! /$ //
975    str = lpegmatch(dollarstripper,str)
976    local tmp = resolvers.variableofformatorsuffix(str)
977    return expandedpathlist(tmp ~= "" and tmp or str)
978end
979
980function resolvers.expandpathfromvariable(str)
981    return joinpath(expandedpathlistfromvariable(str))
982end
983
984resolvers.expandedpathlistfromvariable = expandedpathlistfromvariable
985
986function resolvers.cleanedpathlist(v) -- can be cached if needed
987    local t = expandedpathlist(v)
988    for i=1,#t do
989        t[i] = resolveprefix(cleanpath(t[i]))
990    end
991    return t
992end
993
994function resolvers.expandbraces(str) -- output variable and brace expansion of STRING
995    local pth = expandedpathfromlist(splitpath(str))
996    return joinpath(pth)
997end
998
999function resolvers.registerfilehash(name,content,someerror)
1000    local files = instance.files
1001    if content then
1002        files[name] = content
1003    else
1004        files[name] = { }
1005        if somerror == true then -- can be unset
1006            instance.loaderror = someerror
1007        end
1008    end
1009end
1010
1011function resolvers.getfilehashes()
1012    return instance and instance.files or { }
1013end
1014
1015function resolvers.gethashes()
1016    return instance and instance.hashes or { }
1017end
1018
1019function resolvers.renewcache()
1020    if instance then
1021        instance.renewcache = true
1022    end
1023end
1024
1025local function isreadable(name)
1026    local readable = isfile(name) -- not file.is_readable(name) as it can be a dir
1027    if trace_details then
1028        if readable then
1029            report_resolving("file %a is readable",name)
1030        else
1031            report_resolving("file %a is not readable", name)
1032        end
1033    end
1034    return readable
1035end
1036
1037-- name | name/name
1038
1039local function collect_files(names) -- potential files .. sort of too much when asking for just one file
1040    local filelist = { }            -- but we need it for pattern matching later on
1041    local noffiles = 0
1042    local function check(hash,root,pathname,path,basename,name)
1043        if not pathname or find(path,pathname) then
1044            local variant = hash.type
1045            local search  = filejoin(root,path,name) -- funny no concatinator
1046            local result  = methodhandler('concatinators',variant,root,path,name)
1047            if trace_details then
1048                report_resolving("match: variant %a, search %a, result %a",variant,search,result)
1049            end
1050            noffiles = noffiles + 1
1051            filelist[noffiles] = { variant, search, result }
1052        end
1053    end
1054    for k=1,#names do
1055        local filename = names[k]
1056        if trace_details then
1057            report_resolving("checking name %a",filename)
1058        end
1059        local basename = filebasename(filename)
1060        local pathname = filedirname(filename)
1061        if pathname == "" or find(pathname,"^%.") then
1062            pathname = false
1063        else
1064            pathname = gsub(pathname,"%*",".*")
1065            pathname = "/" .. pathname .. "$"
1066        end
1067        local hashes = instance.hashes
1068        local files  = instance.files
1069        for h=1,#hashes do
1070            local hash     = hashes[h]
1071            local hashname = hash.name
1072            local content  = hashname and files[hashname]
1073            if content then
1074                if trace_details then
1075                    report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname)
1076                end
1077                local path, name = lookup(content,basename)
1078                if path then
1079                    local metadata = content.metadata
1080                    local realroot = metadata and metadata.path or hashname
1081                    if type(path) == "string" then
1082                        check(hash,realroot,pathname,path,basename,name)
1083                    else
1084                        for i=1,#path do
1085                            check(hash,realroot,pathname,path[i],basename,name)
1086                        end
1087                    end
1088                end
1089            elseif trace_locating then
1090                report_resolving("no match in %a (%s)",hashname,basename)
1091            end
1092        end
1093    end
1094    return noffiles > 0 and filelist or nil
1095end
1096
1097local fit = { }
1098
1099function resolvers.registerintrees(filename,format,filetype,usedmethod,foundname)
1100    local foundintrees = instance.foundintrees
1101    if usedmethod == "direct" and filename == foundname and fit[foundname] then
1102        -- just an extra lookup after a test on presence
1103    else
1104        local collapsed = collapsepath(foundname,true)
1105        local t = {
1106            filename   = filename,
1107            format     = format    ~= "" and format   or nil,
1108            filetype   = filetype  ~= "" and filetype or nil,
1109            usedmethod = usedmethod,
1110            foundname  = foundname,
1111            fullname   = collapsed,
1112        }
1113        fit[foundname] = t
1114        foundintrees[#foundintrees+1] = t
1115    end
1116end
1117
1118function resolvers.foundintrees()
1119    return instance.foundintrees or { }
1120end
1121
1122function resolvers.foundintree(fullname)
1123    local f = fit[fullname]
1124    return f and f.usedmethod == "database"
1125end
1126
1127-- split the next one up for readability (but this module needs a cleanup anyway)
1128
1129local function can_be_dir(name) -- can become local
1130    local fakepaths = instance.fakepaths
1131    if not fakepaths[name] then
1132        if isdir(name) then
1133            fakepaths[name] = 1 -- directory
1134        else
1135            fakepaths[name] = 2 -- no directory
1136        end
1137    end
1138    return fakepaths[name] == 1
1139end
1140
1141local preparetreepattern = Cs((P(".")/"%%." + P("-")/"%%-" + P(1))^0 * Cc("$"))
1142
1143-- -- -- begin of main file search routing -- -- -- needs checking as previous has been patched
1144
1145local collect_instance_files
1146
1147local function find_analyze(filename,askedformat,allresults)
1148    local filetype    = ''
1149    local filesuffix  = suffixonly(filename)
1150    local wantedfiles = { }
1151    -- too tricky as filename can be bla.1.2.3:
1152    --
1153    -- if not suffixmap[filesuffix] then
1154    --     wantedfiles[#wantedfiles+1] = filename
1155    -- end
1156    wantedfiles[#wantedfiles+1] = filename
1157    if askedformat == "" then
1158        if filesuffix == "" or not suffixmap[filesuffix] then
1159            local defaultsuffixes = resolvers.defaultsuffixes
1160            for i=1,#defaultsuffixes do
1161                local forcedname = filename .. '.' .. defaultsuffixes[i]
1162                wantedfiles[#wantedfiles+1] = forcedname
1163                filetype = formatofsuffix(forcedname)
1164                if trace_locating then
1165                    report_resolving("forcing filetype %a",filetype)
1166                end
1167            end
1168        else
1169            filetype = formatofsuffix(filename)
1170            if trace_locating then
1171                report_resolving("using suffix based filetype %a",filetype)
1172            end
1173        end
1174    else
1175        if filesuffix == "" or not suffixmap[filesuffix] then
1176            local format_suffixes = suffixes[askedformat]
1177            if format_suffixes then
1178                for i=1,#format_suffixes do
1179                    wantedfiles[#wantedfiles+1] = filename .. "." .. format_suffixes[i]
1180                end
1181            end
1182        end
1183        filetype = askedformat
1184        if trace_locating then
1185            report_resolving("using given filetype %a",filetype)
1186        end
1187    end
1188    return filetype, wantedfiles
1189end
1190
1191local function find_direct(filename,allresults)
1192    if not dangerous[askedformat] and isreadable(filename) then
1193        if trace_details then
1194            report_resolving("file %a found directly",filename)
1195        end
1196        return "direct", { filename }
1197    end
1198end
1199
1200local function find_wildcard(filename,allresults)
1201    if find(filename,'*',1,true) then
1202        if trace_locating then
1203            report_resolving("checking wildcard %a", filename)
1204        end
1205        local result = resolvers.findwildcardfiles(filename)
1206        if result then
1207            return "wildcard", result
1208        end
1209    end
1210end
1211
1212local function find_qualified(filename,allresults,askedformat,alsostripped) -- this one will be split too
1213    if not is_qualified_path(filename) then
1214        return
1215    end
1216    if trace_locating then
1217        report_resolving("checking qualified name %a", filename)
1218    end
1219    if isreadable(filename) then
1220        if trace_details then
1221            report_resolving("qualified file %a found", filename)
1222        end
1223        return "qualified", { filename }
1224    end
1225    if trace_details then
1226        report_resolving("locating qualified file %a", filename)
1227    end
1228    local forcedname, suffix = "", suffixonly(filename)
1229    if suffix == "" then -- why
1230        local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat]
1231        if format_suffixes then
1232            for i=1,#format_suffixes do
1233                local suffix = format_suffixes[i]
1234                forcedname   = filename .. "." .. suffix
1235                if isreadable(forcedname) then
1236                    if trace_locating then
1237                        report_resolving("no suffix, forcing format filetype %a", suffix)
1238                    end
1239                    return "qualified", { forcedname }
1240                end
1241            end
1242        end
1243    end
1244    if alsostripped and suffix and suffix ~= "" then
1245        -- try to find in tree (no suffix manipulation), here we search for the
1246        -- matching last part of the name
1247        local basename    = filebasename(filename)
1248        local pattern     = lpegmatch(preparetreepattern,filename)
1249        local savedformat = askedformat
1250        local format      = savedformat or ""
1251        if format == "" then
1252            askedformat = formatofsuffix(suffix)
1253        end
1254        if not format then
1255            askedformat = "othertextfiles" -- kind of everything, maybe all
1256        end
1257        --
1258        -- is this really what we want? basename if we have an explicit path?
1259        --
1260        if basename ~= filename then
1261            local resolved = collect_instance_files(basename,askedformat,allresults)
1262            if #resolved == 0 then
1263                local lowered = lower(basename)
1264                if filename ~= lowered then
1265                    resolved = collect_instance_files(lowered,askedformat,allresults)
1266                end
1267            end
1268            resolvers.format = savedformat
1269            --
1270            if #resolved > 0 then
1271                local result = { }
1272                for r=1,#resolved do
1273                    local rr = resolved[r]
1274                    if find(rr,pattern) then
1275                        result[#result+1] = rr
1276                    end
1277                end
1278                if #result > 0 then
1279                    return "qualified", result
1280                end
1281            end
1282        end
1283        -- a real wildcard:
1284        --
1285        -- local filelist = collect_files({basename})
1286        -- result = { }
1287        -- for f=1,#filelist do
1288        --     local ff = filelist[f][3] or ""
1289        --     if find(ff,pattern) then
1290        --         result[#result+1], ok = ff, true
1291        --     end
1292        -- end
1293        -- if #result > 0 then
1294        --     return "qualified", result
1295        -- end
1296    end
1297end
1298
1299local function check_subpath(fname)
1300    if isreadable(fname) then
1301        if trace_details then
1302            report_resolving("found %a by deep scanning",fname)
1303        end
1304        return fname
1305    end
1306end
1307
1308-- this caching is not really needed (seldom accessed) but more readable
1309-- we could probably move some to a higher level but then we need to adapt
1310-- more code ... maybe some day
1311
1312local function makepathlist(list,filetype)
1313    local typespec = resolvers.variableofformat(filetype)
1314    local pathlist = expandedpathlist(typespec,filetype and usertypes[filetype]) -- only extra path with user files
1315    local entry    = { }
1316    if pathlist and #pathlist > 0 then
1317        for k=1,#pathlist do
1318            local path       = pathlist[k]
1319            local prescanned = find(path,'^!!')
1320            local resursive  = find(path,'//$')
1321            local pathname   = lpegmatch(inhibitstripper,path)
1322            local expression = makepathexpression(pathname)
1323            local barename   = gsub(pathname,"/+$","")
1324            barename         = resolveprefix(barename)
1325            local scheme     = urlhasscheme(barename)
1326            local schemename = gsub(barename,"%.%*$",'') -- after scheme
1327         -- local prescanned = path ~= pathname -- ^!!
1328         -- local resursive  = find(pathname,'//$')
1329            entry[k] = {
1330                path       = path,
1331                pathname   = pathname,
1332                prescanned = prescanned,
1333                recursive  = recursive,
1334                expression = expression,
1335                barename   = barename,
1336                scheme     = scheme,
1337                schemename = schemename,
1338            }
1339        end
1340        entry.typespec = typespec
1341        list[filetype] = entry
1342    else
1343        list[filetype] = false
1344    end
1345    return entry
1346end
1347
1348-- pathlist : resolved
1349-- dirlist  : unresolved or resolved
1350-- filelist : unresolved
1351
1352local function find_intree(filename,filetype,wantedfiles,allresults)
1353    local pathlists = instance.pathlists
1354    if not pathlists then
1355        pathlists = setmetatableindex({ },makepathlist)
1356        instance.pathlists = pathlists
1357    end
1358    local pathlist = pathlists[filetype]
1359    if pathlist then
1360        -- list search
1361        local method   = "intree"
1362        local filelist = collect_files(wantedfiles) -- okay, a bit over the top when we just look relative to the current path
1363        local dirlist  = { }
1364        local result   = { }
1365        if filelist then
1366            for i=1,#filelist do
1367                dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble
1368            end
1369        end
1370        if trace_details then
1371            report_resolving("checking filename %a in tree",filename)
1372        end
1373        for k=1,#pathlist do
1374            local entry    = pathlist[k]
1375            local path     = entry.path
1376            local pathname = entry.pathname
1377            local done     = false
1378            -- using file list
1379            if filelist then -- database
1380                -- compare list entries with permitted pattern -- /xx /xx//
1381                local expression = entry.expression
1382                if trace_details then
1383                    report_resolving("using pattern %a for path %a",expression,pathname)
1384                end
1385                for k=1,#filelist do
1386                    local fl = filelist[k]
1387                    local f  = fl[2]
1388                    local d  = dirlist[k]
1389                    -- resolve is new:
1390                    if find(d,expression) or find(resolveprefix(d),expression) then
1391                        -- todo, test for readable
1392                        result[#result+1] = resolveprefix(fl[3]) -- no shortcut
1393                        done = true
1394                        if allresults then
1395                            if trace_details then
1396                                report_resolving("match to %a in hash for file %a and path %a, continue scanning",expression,f,d)
1397                            end
1398                        else
1399                            if trace_details then
1400                                report_resolving("match to %a in hash for file %a and path %a, quit scanning",expression,f,d)
1401                            end
1402                            break
1403                        end
1404                    elseif trace_details then
1405                        report_resolving("no match to %a in hash for file %a and path %a",expression,f,d)
1406                    end
1407                end
1408            end
1409            if done then
1410                method = "database"
1411            else
1412                -- beware: we don't honor allresults here in a next attempt (done false)
1413                -- but that is kind of special anyway
1414                method       = "filesystem" -- bonus, even when !! is specified
1415                local scheme = entry.scheme
1416                if not scheme or scheme == "file" then
1417                    local pname = entry.schemename
1418                    if not find(pname,"*",1,true) then
1419                        if can_be_dir(pname) then
1420                            -- hm, rather useless as we don't go deeper and if we would we could also
1421                            -- auto generate the file database .. however, we need this for extra paths
1422                            -- that are not hashed (like sources on my machine) .. so, this is slightly
1423                            -- out of order but at least fast (and we seldom end up here, only when a file
1424                            -- is not already found
1425                            if not done and not entry.prescanned then
1426                                if trace_details then
1427                                    report_resolving("quick root scan for %a",pname)
1428                                end
1429                                for k=1,#wantedfiles do
1430                                    local w = wantedfiles[k]
1431                                    local fname = check_subpath(filejoin(pname,w))
1432                                    if fname then
1433                                        result[#result+1] = fname
1434                                        done = true
1435                                        if not allresults then
1436                                            break
1437                                        end
1438                                    end
1439                                end
1440                                if not done and entry.recursive then -- maybe also when allresults
1441                                    -- collect files in path (and cache the result)
1442                                    if trace_details then
1443                                        report_resolving("scanning filesystem for %a",pname)
1444                                    end
1445                                    local files = resolvers.simplescanfiles(pname,false,true)
1446                                    for k=1,#wantedfiles do
1447                                        local w = wantedfiles[k]
1448                                        local subpath = files[w]
1449                                        if not subpath or subpath == "" then
1450                                            -- rootscan already done
1451                                        elseif type(subpath) == "string" then
1452                                            local fname = check_subpath(filejoin(pname,subpath,w))
1453                                            if fname then
1454                                                result[#result+1] = fname
1455                                                done = true
1456                                                if not allresults then
1457                                                    break
1458                                                end
1459                                            end
1460                                        else
1461                                            for i=1,#subpath do
1462                                                local sp = subpath[i]
1463                                                if sp == "" then
1464                                                    -- roottest already done
1465                                                else
1466                                                    local fname = check_subpath(filejoin(pname,sp,w))
1467                                                    if fname then
1468                                                        result[#result+1] = fname
1469                                                        done = true
1470                                                        if not allresults then
1471                                                            break
1472                                                        end
1473                                                    end
1474                                                end
1475                                            end
1476                                            if done and not allresults then
1477                                                break
1478                                            end
1479                                        end
1480                                    end
1481                                end
1482                            end
1483                        end
1484                    else
1485                        -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
1486                    end
1487                else
1488                    -- we can have extra_paths that are urls
1489                    for k=1,#wantedfiles do
1490                        -- independent url scanner
1491                        local pname = entry.barename
1492                        local fname = methodhandler('finders',pname .. "/" .. wantedfiles[k])
1493                        if fname then
1494                            result[#result+1] = fname
1495                            done = true
1496                            if not allresults then
1497                                break
1498                            end
1499                        end
1500                    end
1501                end
1502            end
1503            -- todo recursive scanning
1504            if done and not allresults then
1505                break
1506            end
1507        end
1508        if #result > 0 then
1509            return method, result
1510        end
1511    end
1512end
1513
1514local function find_onpath(filename,filetype,wantedfiles,allresults)
1515    if trace_details then
1516        report_resolving("checking filename %a, filetype %a, wanted files %a",filename,filetype,concat(wantedfiles," | "))
1517    end
1518    local result = { }
1519    for k=1,#wantedfiles do
1520        local fname = wantedfiles[k]
1521        if fname and isreadable(fname) then
1522            filename = fname
1523            result[#result+1] = filejoin('.',fname)
1524            if not allresults then
1525                break
1526            end
1527        end
1528    end
1529    if #result > 0 then
1530        return "onpath", result
1531    end
1532end
1533
1534local function find_otherwise(filename,filetype,wantedfiles,allresults) -- other text files | any | whatever
1535    local filelist = collect_files(wantedfiles)
1536    local fl = filelist and filelist[1]
1537    if fl then
1538        return "otherwise", { resolveprefix(fl[3]) } -- filename
1539    end
1540end
1541
1542-- we could have a loop over the 6 functions but then we'd have to
1543-- always analyze .. todo: use url split
1544
1545collect_instance_files = function(filename,askedformat,allresults) -- uses nested
1546    if not filename or filename == "" then
1547        return { }
1548    end
1549    askedformat = askedformat or ""
1550    filename    = collapsepath(filename,".")
1551    filename    = gsub(filename,"^%./",getcurrentdir().."/") -- we will merge dir.expandname and collapse some day
1552    if allresults then
1553        -- no need for caching, only used for tracing
1554        local filetype, wantedfiles = find_analyze(filename,askedformat)
1555        local results = {
1556            { find_direct   (filename,true) },
1557            { find_wildcard (filename,true) },
1558            { find_qualified(filename,true,askedformat) }, -- we can add ,true if we want to find dups
1559            { find_intree   (filename,filetype,wantedfiles,true) },
1560            { find_onpath   (filename,filetype,wantedfiles,true) },
1561            { find_otherwise(filename,filetype,wantedfiles,true) },
1562        }
1563        local result = { }
1564        local status = { }
1565        local done   = { }
1566        for k, r in next, results do
1567            local method, list = r[1], r[2]
1568            if method and list then
1569                for i=1,#list do
1570                    local c = collapsepath(list[i])
1571                    if not done[c] then
1572                        result[#result+1] = c
1573                        done[c] = true
1574                    end
1575                    status[#status+1] = formatters["%-10s: %s"](method,c)
1576                end
1577            end
1578        end
1579        if trace_details then
1580            report_resolving("lookup status: %s",table.serialize(status,filename))
1581        end
1582        return result, status
1583    else
1584        local method, result, stamp, filetype, wantedfiles
1585        if instance.remember then
1586            if askedformat == "" then
1587                stamp = formatters["%s::%s"](suffixonly(filename),filename)
1588            else
1589                stamp = formatters["%s::%s"](askedformat,filename)
1590            end
1591            result = stamp and instance.found[stamp]
1592            if result then
1593                if trace_locating then
1594                    report_resolving("remembered file %a",filename)
1595                end
1596                return result
1597            end
1598        end
1599        method, result = find_direct(filename)
1600        if not result then
1601            method, result = find_wildcard(filename)
1602            if not result then
1603                method, result = find_qualified(filename,false,askedformat)
1604                if not result then
1605                    filetype, wantedfiles = find_analyze(filename,askedformat)
1606                    method, result = find_intree(filename,filetype,wantedfiles)
1607                    if not result then
1608                        method, result = find_onpath(filename,filetype,wantedfiles)
1609                        if resolve_otherwise and not result then
1610                            -- this will search everywhere in the tree
1611                            method, result = find_otherwise(filename,filetype,wantedfiles)
1612                        end
1613                    end
1614                end
1615            end
1616        end
1617        if result and #result > 0 then
1618            local foundname = collapsepath(result[1])
1619            resolvers.registerintrees(filename,askedformat,filetype,method,foundname)
1620            result = { foundname }
1621        else
1622            result = { } -- maybe false
1623        end
1624        if stamp then
1625            if trace_locating then
1626                report_resolving("remembering file %a using hash %a",filename,stamp)
1627            end
1628            instance.found[stamp] = result
1629        end
1630        return result
1631    end
1632end
1633
1634-- -- -- end of main file search routing -- -- --
1635
1636local function findfiles(filename,filetype,allresults)
1637    if not filename or filename == "" then
1638        return { }
1639    end
1640    if allresults == nil then
1641        allresults = true
1642    end
1643    local result, status = collect_instance_files(filename,filetype or "",allresults)
1644    if not result or #result == 0 then
1645        local lowered = lower(filename)
1646        if filename ~= lowered then
1647            result, status = collect_instance_files(lowered,filetype or "",allresults)
1648        end
1649    end
1650    return result or { }, status
1651end
1652
1653local function findfile(filename,filetype)
1654    if not filename or filename == "" then
1655        return ""
1656    else
1657        return findfiles(filename,filetype,false)[1] or ""
1658    end
1659end
1660
1661resolvers.findfiles  = findfiles
1662resolvers.findfile   = findfile
1663
1664resolvers.find_file  = findfile  -- obsolete
1665resolvers.find_files = findfiles -- obsolete
1666
1667function resolvers.findpath(filename,filetype)
1668    return filedirname(findfiles(filename,filetype,false)[1] or "")
1669end
1670
1671local function findgivenfiles(filename,allresults)
1672    local hashes = instance.hashes
1673    local files  = instance.files
1674    local base   = filebasename(filename)
1675    local result = { }
1676    --
1677    local function okay(hash,path,name)
1678        local found = methodhandler('concatinators',hash.type,hash.name,path,name)
1679        if found and found ~= "" then
1680            result[#result+1] = resolveprefix(found)
1681            return not allresults
1682        end
1683    end
1684    --
1685    for k=1,#hashes do
1686        local hash    = hashes[k]
1687        local content = files[hash.name]
1688        if content then
1689            local path, name = lookup(content,base)
1690            if not path then
1691                -- no match
1692            elseif type(path) == "string" then
1693                if okay(hash,path,name) then
1694                    return result
1695                end
1696            else
1697                for i=1,#path do
1698                    if okay(hash,path[i],name) then
1699                        return result
1700                    end
1701                end
1702            end
1703        end
1704    end
1705    --
1706    return result
1707end
1708
1709function resolvers.findgivenfiles(filename)
1710    return findgivenfiles(filename,true)
1711end
1712
1713function resolvers.findgivenfile(filename)
1714    return findgivenfiles(filename,false)[1] or ""
1715end
1716
1717local makewildcard = Cs(
1718    (P("^")^0 * P("/") * P(-1) + P(-1)) /".*"
1719  + (P("^")^0 * P("/") / "")^0 * (P("*")/".*" + P("-")/"%%-" + P(".")/"%%." + P("?")/"."+ P("\\")/"/" + P(1))^0
1720)
1721
1722function resolvers.wildcardpattern(pattern)
1723    return lpegmatch(makewildcard,pattern) or pattern
1724end
1725
1726-- we use more function calls than before but we also have smaller trees so
1727-- why bother
1728
1729local function findwildcardfiles(filename,allresults,result)
1730    local files  = instance.files
1731    local hashes = instance.hashes
1732    --
1733    local result = result or { }
1734    local base   = filebasename(filename)
1735    local dirn   = filedirname(filename)
1736    local path   = lower(lpegmatch(makewildcard,dirn) or dirn)
1737    local name   = lower(lpegmatch(makewildcard,base) or base)
1738    --
1739    if find(name,"*",1,true) then
1740        local function okay(found,path,base,hashname,hashtype)
1741            if find(found,path) then
1742                local full = methodhandler('concatinators',hashtype,hashname,found,base)
1743                if full and full ~= "" then
1744                    result[#result+1] = resolveprefix(full)
1745                    return not allresults
1746                end
1747            end
1748        end
1749        for k=1,#hashes do
1750            local hash     = hashes[k]
1751            local hashname = hash.name
1752            local hashtype = hash.type
1753            if hashname and hashtype then
1754                for found, base in filtered(files[hashname],name) do
1755                    if type(found) == 'string' then
1756                        if okay(found,path,base,hashname,hashtype) then
1757                            break
1758                        end
1759                    else
1760                        for i=1,#found do
1761                            if okay(found[i],path,base,hashname,hashtype) then
1762                                break
1763                            end
1764                        end
1765                    end
1766                end
1767            end
1768        end
1769    else
1770        local function okayokay(found,path,base,hashname,hashtype)
1771            if find(found,path) then
1772                local full = methodhandler('concatinators',hashtype,hashname,found,base)
1773                if full and full ~= "" then
1774                    result[#result+1] = resolveprefix(full)
1775                    return not allresults
1776                end
1777            end
1778        end
1779        --
1780        for k=1,#hashes do
1781            local hash     = hashes[k]
1782            local hashname = hash.name
1783            local hashtype = hash.type
1784            if hashname and hashtype then
1785                local found, base = lookup(content,base)
1786                if not found then
1787                    -- nothing
1788                elseif type(found) == 'string' then
1789                    if okay(found,path,base,hashname,hashtype) then
1790                        break
1791                    end
1792                else
1793                    for i=1,#found do
1794                        if okay(found[i],path,base,hashname,hashtype) then
1795                            break
1796                        end
1797                    end
1798                end
1799            end
1800        end
1801    end
1802    -- we can consider also searching the paths not in the database, but then
1803    -- we end up with a messy search (all // in all path specs)
1804    return result
1805end
1806
1807function resolvers.findwildcardfiles(filename,result)
1808    return findwildcardfiles(filename,true,result)
1809end
1810
1811function resolvers.findwildcardfile(filename)
1812    return findwildcardfiles(filename,false)[1] or ""
1813end
1814
1815do
1816
1817    local starttiming = statistics.starttiming
1818    local stoptiming  = statistics.stoptiming
1819    local elapsedtime = statistics.elapsedtime
1820
1821    function resolvers.starttiming()
1822        starttiming(instance)
1823    end
1824
1825    function resolvers.stoptiming()
1826        stoptiming(instance)
1827    end
1828
1829    function resolvers.loadtime()
1830        return elapsedtime(instance)
1831    end
1832
1833end
1834
1835-- main user functions
1836
1837function resolvers.automount()
1838    -- implemented later (maybe a one-time setter like textopener)
1839end
1840
1841function resolvers.load(option)
1842    resolvers.starttiming()
1843    identify_configuration_files()
1844    load_configuration_files()
1845    if option ~= "nofiles" then
1846        load_databases()
1847        resolvers.automount()
1848    end
1849    resolvers.stoptiming()
1850    local files = instance.files
1851    return files and next(files) and true
1852end
1853
1854local function report(str)
1855    if trace_locating then
1856        report_resolving(str) -- has already verbose
1857    else
1858        print(str)
1859    end
1860end
1861
1862function resolvers.dowithfilesandreport(command, files, ...) -- will move
1863    if files and #files > 0 then
1864        if trace_locating then
1865            report('') -- ?
1866        end
1867        if type(files) == "string" then
1868            files = { files }
1869        end
1870        for f=1,#files do
1871            local file = files[f]
1872            local result = command(file,...)
1873            if type(result) == 'string' then
1874                report(result)
1875            else
1876                for i=1,#result do
1877                    report(result[i]) -- could be unpack
1878                end
1879            end
1880        end
1881    end
1882end
1883
1884-- resolvers.varvalue  = resolvers.variable   -- output the value of variable $STRING.
1885-- resolvers.expandvar = expansion  -- output variable expansion of STRING.
1886
1887function resolvers.showpath(str)     -- output search path for file type NAME
1888    return joinpath(expandedpathlist(resolvers.formatofvariable(str)))
1889end
1890
1891function resolvers.registerfile(files, name, path)
1892    if files[name] then
1893        if type(files[name]) == 'string' then
1894            files[name] = { files[name], path }
1895        else
1896            files[name] = path
1897        end
1898    else
1899        files[name] = path
1900    end
1901end
1902
1903function resolvers.dowithpath(name,func)
1904    local pathlist = expandedpathlist(name)
1905    for i=1,#pathlist do
1906        func("^"..cleanpath(pathlist[i]))
1907    end
1908end
1909
1910function resolvers.dowithvariable(name,func)
1911    func(expandedvariable(name))
1912end
1913
1914function resolvers.locateformat(name)
1915    local engine   = environment.ownmain or "luatex"
1916    local barename = removesuffix(file.basename(name))
1917    local fullname = addsuffix(barename,"fmt")
1918    local fmtname  = caches.getfirstreadablefile(fullname,"formats",engine) or ""
1919    if fmtname == "" then
1920        fmtname = findfile(fullname)
1921        fmtname = cleanpath(fmtname)
1922    end
1923    if fmtname ~= "" then
1924        local barename = removesuffix(fmtname)
1925        local luaname = addsuffix(barename,luasuffixes.lua)
1926        local lucname = addsuffix(barename,luasuffixes.luc)
1927        local luiname = addsuffix(barename,luasuffixes.lui)
1928        if isfile(luiname) then
1929            return fmtname, luiname
1930        elseif isfile(lucname) then
1931            return fmtname, lucname
1932        elseif isfile(luaname) then
1933            return fmtname, luaname
1934        end
1935    end
1936    return nil, nil
1937end
1938
1939function resolvers.booleanvariable(str,default)
1940    local b = expansion(str)
1941    if b == "" then
1942        return default
1943    else
1944        b = toboolean(b)
1945        return (b == nil and default) or b
1946    end
1947end
1948
1949function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, can be a nice iterator instead
1950    local hashes = instance.hashes
1951    local files  = instance.files
1952    for i=1,#hashes do
1953        local hash     = hashes[i]
1954        local blobtype = hash.type
1955        local blobpath = hash.name
1956        if blobtype and blobpath then
1957            local total   = 0
1958            local checked = 0
1959            local done    = 0
1960            if before then
1961                before(blobtype,blobpath,pattern)
1962            end
1963            for path, name in filtered(files[blobpath],pattern) do
1964                if type(path) == "string" then
1965                    checked = checked + 1
1966                    if handle(blobtype,blobpath,path,name) then
1967                        done = done + 1
1968                    end
1969                else
1970                    checked = checked + #path
1971                    for i=1,#path do
1972                        if handle(blobtype,blobpath,path[i],name) then
1973                            done = done + 1
1974                        end
1975                    end
1976                end
1977            end
1978            if after then
1979                after(blobtype,blobpath,pattern,checked,done)
1980            end
1981        end
1982    end
1983end
1984
1985-- moved here
1986
1987function resolvers.knownvariables(pattern)
1988    if instance then
1989        local environment = instance.environment
1990        local variables   = instance.variables
1991        local expansions  = instance.expansions
1992        local order       = instance.order
1993        local pattern     = upper(pattern or "")
1994        local result      = { }
1995        for i=1,#order do
1996            for key in next, order[i] do
1997                if result[key] == nil and key ~= "" and (pattern == "" or find(upper(key),pattern)) then
1998                    result[key] = {
1999                        environment = rawget(environment,key),
2000                        variable    = key,
2001                        expansion   = expansions[key],
2002                        resolved    = resolveprefix(expansions[key]),
2003                    }
2004                end
2005            end
2006        end
2007        return result
2008    else
2009        return { }
2010    end
2011end
2012