data-res.lua /size: 69 Kb    last modification: 2023-12-21 09:44
1if not modules then modules = { } end modules ['data-res'] = {
2    version   = 1.001,
3    comment   = "companion to luat-lib.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files",
7}
8
9-- todo: cache:/// home:/// selfautoparent:/// (sometime end 2012)
10
11local gsub, find, lower, upper, match, gmatch = string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
12local concat, insert, remove = table.concat, table.insert, table.remove
13local next, type, rawget, loadfile = next, type, rawget, loadfile
14local mergedtable = table.merged
15
16local os = os
17
18local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg
19local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns
20
21local formatters        = string.formatters
22local filedirname       = file.dirname
23local filebasename      = file.basename
24local suffixonly        = file.suffixonly
25local addsuffix         = file.addsuffix
26local removesuffix      = file.removesuffix
27local filejoin          = file.join
28local collapsepath      = file.collapsepath
29local joinpath          = file.joinpath
30local is_qualified_path = file.is_qualified_path
31
32local allocate          = utilities.storage.allocate
33local settings_to_array = utilities.parsers.settings_to_array
34
35local urlhasscheme      = url.hasscheme
36
37local getcurrentdir     = lfs.currentdir
38local isfile            = lfs.isfile
39local isdir             = lfs.isdir
40
41local setmetatableindex = table.setmetatableindex
42local luasuffixes       = utilities.lua.suffixes
43
44local trace_locating    = false  trackers  .register("resolvers.locating",   function(v) trace_locating    = v end)
45local trace_details     = false  trackers  .register("resolvers.details",    function(v) trace_details     = v end)
46local trace_expansions  = false  trackers  .register("resolvers.expansions", function(v) trace_expansions  = v end)
47local trace_paths       = false  trackers  .register("resolvers.paths",      function(v) trace_paths       = v end)
48local resolve_otherwise = true   directives.register("resolvers.otherwise",  function(v) resolve_otherwise = v end)
49
50local report_resolving = logs.reporter("resolvers","resolving")
51
52local resolvers              = resolvers
53
54local expandedpathfromlist   = resolvers.expandedpathfromlist
55local checkedvariable        = resolvers.checkedvariable
56local splitconfigurationpath = resolvers.splitconfigurationpath
57local methodhandler          = resolvers.methodhandler
58local filtered               = resolvers.filtered_from_content
59local lookup                 = resolvers.get_from_content
60local cleanpath              = resolvers.cleanpath
61local resolveprefix          = resolvers.resolve
62
63local initializesetter       = utilities.setters.initialize
64
65local ostype, osname, osenv, ossetenv, osgetenv = os.type, os.name, os.env, os.setenv, os.getenv
66
67resolvers.cacheversion   = "1.100"
68resolvers.configbanner   = ""
69resolvers.homedir        = environment.homedir
70resolvers.luacnfname     = "texmfcnf.lua"
71resolvers.luacnffallback = "contextcnf.lua"
72resolvers.luacnfstate    = "unknown"
73
74local criticalvars = {
75    "SELFAUTOLOC",
76    "SELFAUTODIR",
77    "SELFAUTOPARENT",
78    "TEXMFCNF",
79    "TEXMF",
80    "TEXOS",
81}
82
83-- The web2c tex binaries as well as kpse have built in paths for the configuration
84-- files and there can be a depressing truckload of them. This is actually the weak
85-- spot of a distribution. So we don't want:
86--
87-- resolvers.luacnfspec = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}'
88--
89-- but instead (for instance) use:
90--
91-- resolvers.luacnfspec = 'selfautoparent:{/texmf{-local,}{,/web2c}}'
92--
93-- which does not make texlive happy as there is a texmf-local tree one level up
94-- (sigh), so we need this. We can assume web2c as mkiv does not run on older
95-- texlives anyway.
96--
97-- texlive:
98--
99-- selfautoloc:
100-- selfautoloc:/share/texmf-local/web2c
101-- selfautoloc:/share/texmf-dist/web2c
102-- selfautoloc:/share/texmf/web2c
103-- selfautoloc:/texmf-local/web2c
104-- selfautoloc:/texmf-dist/web2c
105-- selfautoloc:/texmf/web2c
106-- selfautodir:
107-- selfautodir:/share/texmf-local/web2c
108-- selfautodir:/share/texmf-dist/web2c
109-- selfautodir:/share/texmf/web2c
110-- selfautodir:/texmf-local/web2c
111-- selfautodir:/texmf-dist/web2c
112-- selfautodir:/texmf/web2c
113-- selfautoparent:/../texmf-local/web2c
114-- selfautoparent:
115-- selfautoparent:/share/texmf-local/web2c
116-- selfautoparent:/share/texmf-dist/web2c
117-- selfautoparent:/share/texmf/web2c
118-- selfautoparent:/texmf-local/web2c
119-- selfautoparent:/texmf-dist/web2c
120-- selfautoparent:/texmf/web2c
121--
122-- minimals:
123--
124-- home:texmf/web2c
125-- selfautoparent:texmf-local/web2c
126-- selfautoparent:texmf-context/web2c
127-- selfautoparent:texmf/web2c
128
129-- This is a real mess: you don't want to know what creepy paths end up in the default
130-- configuration spec, for instance nested texmf- paths. I'd rather get away from it and
131-- specify a proper search sequence but alas ... it is not permitted in texlive and there
132-- is no way to check if we run a minimals as texmf-context is not in that spec. It's a
133-- compiled-in permutation of historics with the selfautoloc, selfautodir, selfautoparent
134-- resulting in weird combinations. So, when we eventually check the 30 something paths
135-- we also report weird ones, with weird being: (1) duplicate /texmf or (2) no /web2c in
136-- the names.
137
138-- if environment.default_texmfcnf then
139--     resolvers.luacnfspec = "home:texmf/web2c;" .. environment.default_texmfcnf -- texlive + home: for taco etc
140-- else
141--     resolvers.luacnfspec = concat ( {
142--         "home:texmf/web2c",
143--         "selfautoparent:/texmf-local/web2c",
144--         "selfautoparent:/texmf-context/web2c",
145--         "selfautoparent:/texmf-dist/web2c",
146--         "selfautoparent:/texmf/web2c",
147--     }, ";")
148-- end
149
150if environment.default_texmfcnf then
151    -- this will go away (but then also no more checking in mtxrun.lua itself)
152    resolvers.luacnfspec = "home:texmf/web2c;" .. environment.default_texmfcnf -- texlive + home: for taco etc
153else
154    local texroot = environment.texroot
155    resolvers.luacnfspec = "home:texmf/web2c;selfautoparent:/texmf-local/web2c;selfautoparent:/texmf-context/web2c;selfautoparent:/texmf/web2c"
156    if texroot and isdir(texroot .. "/texmf-context") then
157        -- we're okay and run the lean and mean installation
158    elseif texroot and isdir(texroot .. "/texmf-dist") then
159        -- we're in texlive where texmf-dist is leading
160        resolvers.luacnfspec = "home:texmf/web2c;selfautoparent:/texmf-local/web2c;selfautoparent:/texmf-dist/web2c;selfautoparent:/texmf/web2c"
161    elseif ostype ~= "windows" and isdir("/etc/texmf/web2c") then
162        -- we have some linux distribution that does it its own way
163        resolvers.luacnfspec = "home:texmf/web2c;/etc/texmf/web2c;selfautodir:/share/texmf/web2c"
164    else
165        -- we stick to the reference specification
166    end
167end
168
169local unset_variable = "unset"
170
171local formats   = resolvers.formats
172local suffixes  = resolvers.suffixes
173local usertypes = resolvers.usertypes
174local dangerous = resolvers.dangerous
175local suffixmap = resolvers.suffixmap
176
177resolvers.defaultsuffixes = { "tex" } --  "mkiv", "cld" -- too tricky
178
179local instance  = nil -- the current one (fast access)
180
181-- forward declarations
182
183local variable
184local expansion
185local setenv
186local getenv
187
188-- done
189
190local formatofsuffix = resolvers.formatofsuffix
191local splitpath      = resolvers.splitpath
192local splitmethod    = resolvers.splitmethod
193
194-- An instance has an environment (coming from the outside, kept raw), variables
195-- (coming from the configuration file), and expansions (variables with nested
196-- variables replaced). One can push something into the outer environment and
197-- its internal copy, but only the later one will be the raw unprefixed variant.
198
199setenv = function(key,value,raw)
200    if instance then
201        -- this one will be consulted first when we stay inside
202        -- the current environment (prefixes are not resolved here)
203        instance.environment[key] = value
204        -- we feed back into the environment, and as this is used
205        -- by other applications (via os.execute) we need to make
206        -- sure that prefixes are resolve
207        ossetenv(key,raw and value or resolveprefix(value))
208    end
209end
210
211-- Beware we don't want empty here as this one can be called early on
212-- and therefore we use rawget.
213
214getenv = function(key)
215    local value = rawget(instance.environment,key)
216    if value and value ~= "" then
217        return value
218    else
219        local e = osgetenv(key)
220        return e ~= nil and e ~= "" and checkedvariable(e) or ""
221    end
222end
223
224resolvers.getenv = getenv
225resolvers.setenv = setenv
226
227-- We are going to use some metatable trickery where we backtrack from
228-- expansion to variable to environment.
229
230local dollarstripper   = lpeg.stripper("$")
231local inhibitstripper  = P("!")^0 * Cs(P(1)^0)
232
233local expandedvariable, resolvedvariable  do
234
235    local function resolveinstancevariable(k)
236        return instance.expansions[k]
237    end
238
239    local p_variable       = P("$") / ""
240    local p_key            = C(R("az","AZ","09","__","--")^1)
241    local p_whatever       = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "")
242                           + P(";") * (P(";") / "")
243                           + P(1)
244    local variableexpander = Cs( (p_variable * (p_key/resolveinstancevariable) + p_whatever)^1 )
245
246    local p_cleaner        = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";"
247    local variablecleaner  = Cs((p_cleaner  + P(1))^0)
248
249    local p_variable       = R("az","AZ","09","__","--")^1 / resolveinstancevariable
250    local p_variable       = (P("$")/"") * (p_variable + (P("{")/"") * p_variable * (P("}")/""))
251    local variableresolver = Cs((p_variable + P(1))^0)
252
253    expandedvariable = function(var)
254        return lpegmatch(variableexpander,var) or var
255    end
256
257    function resolvers.reset()
258
259        -- normally we only need one instance but for special cases we can (re)load one so
260        -- we stick to this model.
261
262        if trace_locating then
263            report_resolving("creating instance")
264        end
265
266        local environment = { }
267        local variables   = { }
268        local expansions  = { }
269        local order       = { }
270
271        instance = {
272            environment    = environment,
273            variables      = variables,
274            expansions     = expansions,
275            order          = order,
276            files          = { },
277            setups         = { },
278            found          = { },
279            foundintrees   = { },
280            hashes         = { },
281            hashed         = { },
282            pathlists      = false,-- delayed
283            specification  = { },
284            lists          = { },
285            data           = { }, -- only for loading
286            fakepaths      = { },
287            remember       = true,
288            diskcache      = true,
289            renewcache     = false,
290            renewtree      = false,
291            loaderror      = false,
292            savelists      = true,
293            pattern        = nil, -- lists
294            force_suffixes = true,
295            pathstack      = { },
296        }
297
298        setmetatableindex(variables,function(t,k)
299            local v
300            for i=1,#order do
301                v = order[i][k]
302                if v ~= nil then
303                    t[k] = v
304                    return v
305                end
306            end
307            if v == nil then
308                v = ""
309            end
310            t[k] = v
311            return v
312        end)
313
314        local repath = resolvers.repath
315
316        setmetatableindex(environment, function(t,k)
317            local v = osgetenv(k)
318            if v == nil then
319                v = variables[k]
320            end
321            if v ~= nil then
322                v = checkedvariable(v) or ""
323            end
324            v = repath(v) -- for taco who has a : separated osfontdir
325            t[k] = v
326            return v
327        end)
328
329        setmetatableindex(expansions, function(t,k)
330            local v = environment[k]
331            if type(v) == "string" then
332                v = lpegmatch(variableresolver,v)
333                v = lpegmatch(variablecleaner,v)
334            end
335            t[k] = v
336            return v
337        end)
338
339    end
340
341end
342
343function resolvers.initialized()
344    return instance ~= nil
345end
346
347local function reset_hashes()
348    instance.lists     = { }
349    instance.pathlists = false
350    instance.found     = { }
351end
352
353local function reset_caches()
354    instance.lists     = { }
355    instance.pathlists = false
356end
357
358local makepathexpression  do
359
360    local slash = P("/")
361
362    local pathexpressionpattern = Cs ( -- create lpeg instead (2013/2014)
363        Cc("^") * (
364            Cc("%") * S(".-")
365          + slash^2 * P(-1) / "/.*"
366       -- + slash^2 / "/.-/"
367       -- + slash^2 / "/[^/]*/*"   -- too general
368          + slash^2 / "/"
369          + (1-slash) * P(-1) * Cc("/")
370          + P(1)
371        )^1 * Cc("$") -- yes or no $
372    )
373
374    local cache = { }
375
376    makepathexpression = function(str)
377        if str == "." then
378            return "^%./$"
379        else
380            local c = cache[str]
381            if not c then
382                c = lpegmatch(pathexpressionpattern,str)
383                cache[str] = c
384            end
385            return c
386        end
387    end
388
389end
390
391local function reportcriticalvariables(cnfspec)
392    if trace_locating then
393        for i=1,#criticalvars do
394            local k = criticalvars[i]
395            local v = getenv(k) or "unknown" -- this one will not resolve !
396            report_resolving("variable %a set to %a",k,v)
397        end
398        report_resolving()
399        if cnfspec then
400            report_resolving("using configuration specification %a",type(cnfspec) == "table" and concat(cnfspec,",") or cnfspec)
401        end
402        report_resolving()
403    end
404    reportcriticalvariables = function() end
405end
406
407local function identify_configuration_files()
408    local specification = instance.specification
409    if #specification == 0 then
410        local cnfspec = getenv("TEXMFCNF")
411        if cnfspec == "" then
412            cnfspec = resolvers.luacnfspec
413            resolvers.luacnfstate = "default"
414        else
415            resolvers.luacnfstate = "environment"
416        end
417        reportcriticalvariables(cnfspec)
418        local cnfpaths = expandedpathfromlist(splitpath(cnfspec))
419
420        local function locatecnf(luacnfname,kind)
421            for i=1,#cnfpaths do
422                local filepath = cnfpaths[i]
423                local filename = collapsepath(filejoin(filepath,luacnfname))
424                local realname = resolveprefix(filename) -- can still have "//" ... needs checking
425                -- todo: environment.skipweirdcnfpaths directive
426                if trace_locating then
427                    local fullpath  = gsub(resolveprefix(collapsepath(filepath)),"//","/")
428                    local weirdpath = find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true)
429                    report_resolving("looking for %s %a on %s path %a from specification %a",
430                        kind,luacnfname,weirdpath and "weird" or "given",fullpath,filepath)
431                end
432                if isfile(realname) then
433                    specification[#specification+1] = filename -- unresolved as we use it in matching, relocatable
434                    if trace_locating then
435                        report_resolving("found %s configuration file %a",kind,realname)
436                    end
437                end
438            end
439        end
440
441        locatecnf(resolvers.luacnfname,"regular")
442        if #specification == 0 then
443            locatecnf(resolvers.luacnffallback,"fallback")
444        end
445        if trace_locating then
446            report_resolving()
447        end
448    elseif trace_locating then
449        report_resolving("configuration files already identified")
450    end
451end
452
453local function load_configuration_files()
454    local specification = instance.specification
455    local setups        = instance.setups
456    local order         = instance.order
457    if #specification > 0 then
458        local luacnfname = resolvers.luacnfname
459        for i=1,#specification do
460            local filename = specification[i]
461            local pathname = filedirname(filename)
462            local filename = filejoin(pathname,luacnfname)
463            local realname = resolveprefix(filename) -- no shortcut
464            local blob = loadfile(realname)
465            if blob then
466                local data = blob()
467                local parent = data and data.parent
468                if parent then
469                    local filename = filejoin(pathname,parent)
470                    local realname = resolveprefix(filename) -- no shortcut
471                    local blob = loadfile(realname)
472                    if blob then
473                        local parentdata = blob()
474                        if parentdata then
475                            report_resolving("loading configuration file %a",filename)
476                            data = mergedtable(parentdata,data)
477                        end
478                    end
479                end
480                data = data and data.content
481                if data then
482                    if trace_locating then
483                        report_resolving("loading configuration file %a",filename)
484                        report_resolving()
485                    end
486                    local variables = data.variables or { }
487                    local warning = false
488                    for k, v in next, data do
489                        local variant = type(v)
490                        if variant == "table" then
491                            initializesetter(filename,k,v)
492                        elseif variables[k] == nil then
493                            if trace_locating and not warning then
494                                report_resolving("variables like %a in configuration file %a should move to the 'variables' subtable",
495                                    k,resolveprefix(filename))
496                                warning = true
497                            end
498                            variables[k] = v
499                        end
500                    end
501                    setups[pathname] = variables
502                    if resolvers.luacnfstate == "default" then
503                        -- the following code is not tested
504                        local cnfspec = variables["TEXMFCNF"]
505                        if cnfspec then
506                            if trace_locating then
507                                report_resolving("reloading configuration due to TEXMF redefinition")
508                            end
509                            -- we push the value into the main environment (osenv) so
510                            -- that it takes precedence over the default one and therefore
511                            -- also over following definitions
512                            setenv("TEXMFCNF",cnfspec) -- resolves prefixes
513                            -- we now identify and load the specified configuration files
514                            instance.specification = { }
515                            identify_configuration_files()
516                            load_configuration_files()
517                            -- we prevent further overload of the configuration variable
518                            resolvers.luacnfstate = "configuration"
519                            -- we quit the outer loop
520                            break
521                        end
522                    end
523
524                else
525                    if trace_locating then
526                        report_resolving("skipping configuration file %a (no content)",filename)
527                    end
528                    setups[pathname] = { }
529                    instance.loaderror = true
530                end
531            elseif trace_locating then
532                report_resolving("skipping configuration file %a (no valid format)",filename)
533            end
534            order[#order+1] = setups[pathname]
535            if instance.loaderror then
536                break
537            end
538        end
539    elseif trace_locating then
540        report_resolving("warning: no lua configuration files found")
541    end
542end
543
544-- forward declarations:
545
546local expandedpathlist
547local unexpandedpathlist
548
549-- done
550
551function resolvers.configurationfiles()
552    return instance.specification or { }
553end
554
555-- scheme magic ... database loading
556
557local function load_file_databases()
558    instance.loaderror = false
559    instance.files     = { }
560    if not instance.renewcache then
561        local hashes = instance.hashes
562        for k=1,#hashes do
563            local hash = hashes[k]
564            resolvers.hashers.byscheme(hash.type,hash.name)
565            if instance.loaderror then break end
566        end
567    end
568end
569
570local function locate_file_databases()
571    -- todo: cache:// and tree:// (runtime)
572    local texmfpaths = expandedpathlist("TEXMF")
573    if #texmfpaths > 0 then
574        for i=1,#texmfpaths do
575            local path = collapsepath(texmfpaths[i])
576            path = gsub(path,"/+$","") -- in case $HOME expands to something with a trailing /
577            local stripped = lpegmatch(inhibitstripper,path) -- the !! thing
578            if stripped ~= "" then
579                local runtime = stripped == path
580                path = cleanpath(path)
581                local spec = splitmethod(stripped)
582                if runtime and (spec.noscheme or spec.scheme == "file") then
583                    stripped = "tree:///" .. stripped
584                elseif spec.scheme == "cache" or spec.scheme == "file" then
585                    stripped = spec.path
586                end
587                if trace_locating then
588                    if runtime then
589                        report_resolving("locating list of %a (runtime) (%s)",path,stripped)
590                    else
591                        report_resolving("locating list of %a (cached)",path)
592                    end
593                end
594                methodhandler('locators',stripped)
595            end
596        end
597        if trace_locating then
598            report_resolving()
599        end
600    elseif trace_locating then
601        report_resolving("no texmf paths are defined (using TEXMF)")
602    end
603end
604
605local function generate_file_databases()
606    local hashes = instance.hashes
607    for k=1,#hashes do
608        local hash = hashes[k]
609        methodhandler('generators',hash.name)
610    end
611    if trace_locating then
612        report_resolving()
613    end
614end
615
616local function save_file_databases() -- will become cachers
617    local hashes = instance.hashes
618    local files  = instance.files
619    for i=1,#hashes do
620        local hash      = hashes[i]
621        local cachename = hash.name
622        if hash.cache then
623            local content = files[cachename]
624            caches.collapsecontent(content)
625            if trace_locating then
626                report_resolving("saving tree %a",cachename)
627            end
628            caches.savecontent(cachename,"files",content)
629        elseif trace_locating then
630            report_resolving("not saving runtime tree %a",cachename)
631        end
632    end
633end
634
635function resolvers.renew(hashname)
636    local files = instance.files
637    if hashname and hashname ~= "" then
638        local expanded = expansion(hashname) or ""
639        if expanded ~= "" then
640            if trace_locating then
641                report_resolving("identifying tree %a from %a",expanded,hashname)
642            end
643            hashname = expanded
644        else
645            if trace_locating then
646                report_resolving("identifying tree %a",hashname)
647            end
648        end
649        local realpath = resolveprefix(hashname)
650        if isdir(realpath) then
651            if trace_locating then
652                report_resolving("using path %a",realpath)
653            end
654            methodhandler('generators',hashname)
655            -- could be shared
656            local content = files[hashname]
657            caches.collapsecontent(content)
658            if trace_locating then
659                report_resolving("saving tree %a",hashname)
660            end
661            caches.savecontent(hashname,"files",content)
662            -- till here
663        else
664            report_resolving("invalid path %a",realpath)
665        end
666    end
667end
668
669local function load_databases()
670    locate_file_databases()
671    if instance.diskcache and not instance.renewcache then
672        load_file_databases()
673        if instance.loaderror then
674            generate_file_databases()
675            save_file_databases()
676        end
677    else
678        generate_file_databases()
679        if instance.renewcache then
680            save_file_databases()
681        end
682    end
683end
684
685function resolvers.appendhash(type,name,cache)
686    local hashed = instance.hashed
687    local hashes = instance.hashes
688    if hashed[name] then
689        -- safeguard ... tricky as it's actually a bug when seen twice
690    else
691        if trace_locating then
692            report_resolving("hash %a appended",name)
693        end
694        insert(hashes, { type = type, name = name, cache = cache } )
695        hashed[name] = cache
696    end
697end
698
699function resolvers.prependhash(type,name,cache)
700    local hashed = instance.hashed
701    local hashes = instance.hashes
702    if hashed[name] then
703        -- safeguard ... tricky as it's actually a bug when seen twice
704    else
705        if trace_locating then
706            report_resolving("hash %a prepended",name)
707        end
708        insert(hashes, 1, { type = type, name = name, cache = cache } )
709        hashed[name] = cache
710    end
711end
712
713function resolvers.extendtexmfvariable(specification) -- crap, we could better prepend the hash
714    local environment = instance.environment
715    local variables   = instance.variables
716    local texmftrees  = splitpath(getenv("TEXMF")) -- okay?
717    insert(texmftrees,1,specification)
718    texmftrees = concat(texmftrees,",") -- not ;
719    if environment["TEXMF"] then
720        environment["TEXMF"] = texmftrees
721    elseif variables["TEXMF"] then
722        variables["TEXMF"] = texmftrees
723    else
724        -- weird
725    end
726    reset_hashes()
727end
728
729function resolvers.splitexpansions()
730    local expansions = instance.expansions
731    for k, v in next, expansions do
732        local t, tn, h, p = { }, 0, { }, splitconfigurationpath(v)
733        for kk=1,#p do
734            local vv = p[kk]
735            if vv ~= "" and not h[vv] then
736                tn = tn + 1
737                t[tn] = vv
738                h[vv] = true
739            end
740        end
741        if tn > 1 then
742            expansions[k] = t
743        else
744            expansions[k] = t[1]
745        end
746    end
747end
748
749-- end of split/join code
750
751-- we used to have 'files' and 'configurations' so therefore the following
752-- shared function
753
754function resolvers.datastate()
755    return caches.contentstate()
756end
757
758variable = function(name)
759    local variables = instance.variables
760    local name   = name and lpegmatch(dollarstripper,name)
761    local result = name and variables[name]
762    return result ~= nil and result or ""
763end
764
765expansion = function(name)
766    local expansions = instance.expansions
767    local name   = name and lpegmatch(dollarstripper,name)
768    local result = name and expansions[name]
769    return result ~= nil and result or ""
770end
771
772resolvers.variable  = variable
773resolvers.expansion = expansion
774
775unexpandedpathlist = function(str)
776    local pth = variable(str)
777    local lst = splitpath(pth)
778    return expandedpathfromlist(lst)
779end
780
781function resolvers.unexpandedpath(str)
782    return joinpath(unexpandedpathlist(str))
783end
784
785function resolvers.pushpath(name)
786    local pathstack = instance.pathstack
787    local lastpath  = pathstack[#pathstack]
788    local pluspath  = filedirname(name)
789    if lastpath then
790        lastpath = collapsepath(filejoin(lastpath,pluspath))
791    else
792        lastpath = collapsepath(pluspath)
793    end
794    insert(pathstack,lastpath)
795    if trace_paths then
796        report_resolving("pushing path %a",lastpath)
797    end
798end
799
800function resolvers.poppath()
801    local pathstack = instance.pathstack
802    if trace_paths and #pathstack > 0 then
803        report_resolving("popping path %a",pathstack[#pathstack])
804    end
805    remove(pathstack)
806end
807
808function resolvers.stackpath()
809    local pathstack   = instance.pathstack
810    local currentpath = pathstack[#pathstack]
811    return currentpath ~= "" and currentpath or nil
812end
813
814local done = { }
815
816function resolvers.resetextrapaths()
817    local extra_paths = instance.extra_paths
818    if not extra_paths then
819        done                 = { }
820        instance.extra_paths = { }
821    elseif #ep > 0 then
822        done = { }
823        reset_caches()
824    end
825end
826
827function resolvers.getextrapaths()
828    return instance.extra_paths or { }
829end
830
831function resolvers.registerextrapath(paths,subpaths)
832    if not subpaths or subpaths == "" then
833        if not paths or path == "" then
834            return -- invalid spec
835        elseif done[paths] then
836            return -- already done
837        end
838    end
839    local paths       = settings_to_array(paths)
840    local subpaths    = settings_to_array(subpaths)
841    local extra_paths = instance.extra_paths or { }
842    local oldn        = #extra_paths
843    local newn        = oldn
844    local nofpaths    = #paths
845    local nofsubpaths = #subpaths
846    if nofpaths > 0 then
847        if nofsubpaths > 0 then
848            for i=1,nofpaths do
849                local p = paths[i]
850                for j=1,nofsubpaths do
851                    local s = subpaths[j]
852                    local ps = p .. "/" .. s
853                    if not done[ps] then
854                        newn = newn + 1
855                        extra_paths[newn] = cleanpath(ps)
856                        done[ps] = true
857                    end
858                end
859            end
860        else
861            for i=1,nofpaths do
862                local p = paths[i]
863                if not done[p] then
864                    newn = newn + 1
865                    extra_paths[newn] = cleanpath(p)
866                    done[p] = true
867                end
868            end
869        end
870    elseif nofsubpaths > 0 then
871        for i=1,oldn do
872            for j=1,nofsubpaths do
873                local s = subpaths[j]
874                local ps = extra_paths[i] .. "/" .. s
875                if not done[ps] then
876                    newn = newn + 1
877                    extra_paths[newn] = cleanpath(ps)
878                    done[ps] = true
879                end
880            end
881        end
882    end
883    if newn > 0 then
884        instance.extra_paths = extra_paths -- register paths
885    end
886    if newn ~= oldn then
887        reset_caches()
888    end
889end
890
891function resolvers.pushextrapath(path)
892    local paths = settings_to_array(path)
893    local extra_stack = instance.extra_stack
894    if extra_stack then
895        insert(extra_stack,1,paths)
896    else
897        instance.extra_stack = { paths }
898    end
899    reset_caches()
900end
901
902function resolvers.popextrapath()
903    local extra_stack = instance.extra_stack
904    if extra_stack then
905        reset_caches()
906        return remove(extra_stack,1)
907    end
908end
909
910local function made_list(instance,list,extra_too)
911    local done = { }
912    local new  = { }
913    local newn = 0
914    -- a helper
915    local function add(p)
916        for k=1,#p do
917            local v = p[k]
918            if not done[v] then
919                done[v] = true
920                newn = newn + 1
921                new[newn] = v
922            end
923        end
924    end
925    -- honour . .. ../.. but only when at the start
926    for k=1,#list do
927        local v = list[k]
928        if done[v] then
929            -- skip
930        elseif find(v,"^[%.%/]$") then
931            done[v] = true
932            newn = newn + 1
933            new[newn] = v
934        else
935            break
936        end
937    end
938    if extra_too then
939        local extra_stack = instance.extra_stack
940        local extra_paths = instance.extra_paths
941        -- first the stacked paths
942        if extra_stack and #extra_stack > 0 then
943            for k=1,#extra_stack do
944                add(extra_stack[k])
945            end
946        end
947        -- then the extra paths
948        if extra_paths and #extra_paths > 0 then
949            add(extra_paths)
950        end
951    end
952    -- last the formal paths
953    add(list)
954    return new
955end
956
957expandedpathlist = function(str,extra_too)
958    if not str then
959        return { }
960    elseif instance.savelists then -- hm, what if two cases, with and without extra_too
961        str = lpegmatch(dollarstripper,str)
962        local lists = instance.lists
963        local lst = lists[str]
964        if not lst then
965            local l = made_list(instance,splitpath(expansion(str)),extra_too)
966            lst = expandedpathfromlist(l)
967            lists[str] = lst
968        end
969        return lst
970    else
971        local lst = splitpath(expansion(str))
972        return made_list(instance,expandedpathfromlist(lst),extra_too)
973    end
974end
975
976resolvers.expandedpathlist   = expandedpathlist
977resolvers.unexpandedpathlist = unexpandedpathlist
978
979function resolvers.cleanpathlist(str)
980    local t = expandedpathlist(str)
981    if t then
982        for i=1,#t do
983            t[i] = collapsepath(cleanpath(t[i]))
984        end
985    end
986    return t
987end
988
989function resolvers.expandpath(str)
990    return joinpath(expandedpathlist(str))
991end
992
993local function expandedpathlistfromvariable(str) -- brrr / could also have cleaner ^!! /$ //
994    str = lpegmatch(dollarstripper,str)
995    local tmp = resolvers.variableofformatorsuffix(str)
996    return expandedpathlist(tmp ~= "" and tmp or str)
997end
998
999function resolvers.expandpathfromvariable(str)
1000    return joinpath(expandedpathlistfromvariable(str))
1001end
1002
1003resolvers.expandedpathlistfromvariable = expandedpathlistfromvariable
1004
1005function resolvers.cleanedpathlist(v) -- can be cached if needed
1006    local t = expandedpathlist(v)
1007    for i=1,#t do
1008        t[i] = resolveprefix(cleanpath(t[i]))
1009    end
1010    return t
1011end
1012
1013function resolvers.expandbraces(str) -- output variable and brace expansion of STRING
1014    local pth = expandedpathfromlist(splitpath(str))
1015    return joinpath(pth)
1016end
1017
1018function resolvers.registerfilehash(name,content,someerror)
1019    local files = instance.files
1020    if content then
1021        files[name] = content
1022    else
1023        files[name] = { }
1024        if somerror == true then -- can be unset
1025            instance.loaderror = someerror
1026        end
1027    end
1028end
1029
1030function resolvers.getfilehashes()
1031    return instance and instance.files or { }
1032end
1033
1034function resolvers.gethashes()
1035    return instance and instance.hashes or { }
1036end
1037
1038function resolvers.renewcache()
1039    if instance then
1040        instance.renewcache = true
1041    end
1042end
1043
1044local function isreadable(name)
1045    local readable = isfile(name) -- not file.is_readable(name) as it can be a dir
1046    if trace_details then
1047        if readable then
1048            report_resolving("file %a is readable",name)
1049        else
1050            report_resolving("file %a is not readable", name)
1051        end
1052    end
1053    return readable
1054end
1055
1056-- name | name/name
1057
1058local function collect_files(names) -- potential files .. sort of too much when asking for just one file
1059    local filelist = { }            -- but we need it for pattern matching later on
1060    local noffiles = 0
1061    local function check(hash,root,pathname,path,basename,name)
1062        if not pathname or find(path,pathname) then
1063            local variant = hash.type
1064            local search  = filejoin(root,path,name) -- funny no concatinator
1065            local result  = methodhandler('concatinators',variant,root,path,name)
1066            if trace_details then
1067                report_resolving("match: variant %a, search %a, result %a",variant,search,result)
1068            end
1069            noffiles = noffiles + 1
1070            filelist[noffiles] = { variant, search, result }
1071        end
1072    end
1073    for k=1,#names do
1074        local filename = names[k]
1075        if trace_details then
1076            report_resolving("checking name %a",filename)
1077        end
1078        local basename = filebasename(filename)
1079        local pathname = filedirname(filename)
1080        if pathname == "" or find(pathname,"^%.") then
1081            pathname = false
1082        else
1083            pathname = gsub(pathname,"%*",".*")
1084            pathname = "/" .. pathname .. "$"
1085        end
1086        local hashes = instance.hashes
1087        local files  = instance.files
1088        for h=1,#hashes do
1089            local hash     = hashes[h]
1090            local hashname = hash.name
1091            local content  = hashname and files[hashname]
1092            if content then
1093                if trace_details then
1094                    report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname)
1095                end
1096                local path, name = lookup(content,basename)
1097                if path then
1098                    local metadata = content.metadata
1099                    local realroot = metadata and metadata.path or hashname
1100                    if type(path) == "string" then
1101                        check(hash,realroot,pathname,path,basename,name)
1102                    else
1103                        for i=1,#path do
1104                            check(hash,realroot,pathname,path[i],basename,name)
1105                        end
1106                    end
1107                end
1108            elseif trace_locating then
1109                report_resolving("no match in %a (%s)",hashname,basename)
1110            end
1111        end
1112    end
1113    return noffiles > 0 and filelist or nil
1114end
1115
1116local fit = { }
1117
1118function resolvers.registerintrees(filename,format,filetype,usedmethod,foundname)
1119    local foundintrees = instance.foundintrees
1120    if usedmethod == "direct" and filename == foundname and fit[foundname] then
1121        -- just an extra lookup after a test on presence
1122    else
1123        local collapsed = collapsepath(foundname,true)
1124        local t = {
1125            filename   = filename,
1126            format     = format    ~= "" and format   or nil,
1127            filetype   = filetype  ~= "" and filetype or nil,
1128            usedmethod = usedmethod,
1129            foundname  = foundname,
1130            fullname   = collapsed,
1131        }
1132        fit[foundname] = t
1133        foundintrees[#foundintrees+1] = t
1134    end
1135end
1136
1137function resolvers.foundintrees()
1138    return instance.foundintrees or { }
1139end
1140
1141function resolvers.foundintree(fullname)
1142    local f = fit[fullname]
1143    return f and f.usedmethod == "database"
1144end
1145
1146-- split the next one up for readability (but this module needs a cleanup anyway)
1147
1148local function can_be_dir(name) -- can become local
1149    local fakepaths = instance.fakepaths
1150    if not fakepaths[name] then
1151        if isdir(name) then
1152            fakepaths[name] = 1 -- directory
1153        else
1154            fakepaths[name] = 2 -- no directory
1155        end
1156    end
1157    return fakepaths[name] == 1
1158end
1159
1160local preparetreepattern = Cs((P(".")/"%%." + P("-")/"%%-" + P(1))^0 * Cc("$"))
1161
1162-- -- -- begin of main file search routing -- -- -- needs checking as previous has been patched
1163
1164local collect_instance_files
1165
1166local function find_analyze(filename,askedformat,allresults)
1167    local filetype    = ""
1168    local filesuffix  = suffixonly(filename)
1169    local wantedfiles = { }
1170    -- too tricky as filename can be bla.1.2.3:
1171    --
1172    -- if not suffixmap[filesuffix] then
1173    --     wantedfiles[#wantedfiles+1] = filename
1174    -- end
1175    wantedfiles[#wantedfiles+1] = filename
1176    if askedformat == "" then
1177        if filesuffix == "" or not suffixmap[filesuffix] then
1178            local defaultsuffixes = resolvers.defaultsuffixes
1179            for i=1,#defaultsuffixes do
1180                local forcedname = filename .. "." .. defaultsuffixes[i]
1181                wantedfiles[#wantedfiles+1] = forcedname
1182                filetype = formatofsuffix(forcedname)
1183                if trace_locating then
1184                    report_resolving("forcing filetype %a",filetype)
1185                end
1186            end
1187        else
1188            filetype = formatofsuffix(filename)
1189            if trace_locating then
1190                report_resolving("using suffix based filetype %a",filetype)
1191            end
1192        end
1193    else
1194        if filesuffix == "" or not suffixmap[filesuffix] then
1195            local format_suffixes = suffixes[askedformat]
1196            if format_suffixes then
1197                for i=1,#format_suffixes do
1198                    wantedfiles[#wantedfiles+1] = filename .. "." .. format_suffixes[i]
1199                end
1200            end
1201        end
1202        filetype = askedformat
1203        if trace_locating then
1204            report_resolving("using given filetype %a",filetype)
1205        end
1206    end
1207    return filetype, wantedfiles
1208end
1209
1210local function find_direct(filename,allresults)
1211    if not dangerous[askedformat] and isreadable(filename) then
1212        if trace_details then
1213            report_resolving("file %a found directly",filename)
1214        end
1215        return "direct", { filename }
1216    end
1217end
1218
1219local function find_wildcard(filename,allresults)
1220    if find(filename,'*',1,true) then
1221        if trace_locating then
1222            report_resolving("checking wildcard %a", filename)
1223        end
1224        local result = resolvers.findwildcardfiles(filename)
1225        if result then
1226            return "wildcard", result
1227        end
1228    end
1229end
1230
1231local function find_qualified(filename,allresults,askedformat,alsostripped) -- this one will be split too
1232    if not is_qualified_path(filename) then
1233        return
1234    end
1235    if trace_locating then
1236        report_resolving("checking qualified name %a", filename)
1237    end
1238    if isreadable(filename) then
1239        if trace_details then
1240            report_resolving("qualified file %a found", filename)
1241        end
1242        return "qualified", { filename }
1243    end
1244    if trace_details then
1245        report_resolving("locating qualified file %a", filename)
1246    end
1247    local forcedname, suffix = "", suffixonly(filename)
1248    if suffix == "" then -- why
1249        local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat]
1250        if format_suffixes then
1251            for i=1,#format_suffixes do
1252                local suffix = format_suffixes[i]
1253                forcedname   = filename .. "." .. suffix
1254                if isreadable(forcedname) then
1255                    if trace_locating then
1256                        report_resolving("no suffix, forcing format filetype %a", suffix)
1257                    end
1258                    return "qualified", { forcedname }
1259                end
1260            end
1261        end
1262    end
1263    if alsostripped and suffix and suffix ~= "" then
1264        -- try to find in tree (no suffix manipulation), here we search for the
1265        -- matching last part of the name
1266        local basename    = filebasename(filename)
1267        local pattern     = lpegmatch(preparetreepattern,filename)
1268        local savedformat = askedformat
1269        local format      = savedformat or ""
1270        if format == "" then
1271            askedformat = formatofsuffix(suffix)
1272        end
1273        if not format then
1274            askedformat = "othertextfiles" -- kind of everything, maybe all
1275        end
1276        --
1277        -- is this really what we want? basename if we have an explicit path?
1278        --
1279        if basename ~= filename then
1280            local resolved = collect_instance_files(basename,askedformat,allresults)
1281            if #resolved == 0 then
1282                local lowered = lower(basename)
1283                if filename ~= lowered then
1284                    resolved = collect_instance_files(lowered,askedformat,allresults)
1285                end
1286            end
1287            resolvers.format = savedformat
1288            --
1289            if #resolved > 0 then
1290                local result = { }
1291                for r=1,#resolved do
1292                    local rr = resolved[r]
1293                    if find(rr,pattern) then
1294                        result[#result+1] = rr
1295                    end
1296                end
1297                if #result > 0 then
1298                    return "qualified", result
1299                end
1300            end
1301        end
1302        -- a real wildcard:
1303        --
1304        -- local filelist = collect_files({basename})
1305        -- result = { }
1306        -- for f=1,#filelist do
1307        --     local ff = filelist[f][3] or ""
1308        --     if find(ff,pattern) then
1309        --         result[#result+1], ok = ff, true
1310        --     end
1311        -- end
1312        -- if #result > 0 then
1313        --     return "qualified", result
1314        -- end
1315    end
1316end
1317
1318local function check_subpath(fname)
1319    if isreadable(fname) then
1320        if trace_details then
1321            report_resolving("found %a by deep scanning",fname)
1322        end
1323        return fname
1324    end
1325end
1326
1327-- this caching is not really needed (seldom accessed) but more readable
1328-- we could probably move some to a higher level but then we need to adapt
1329-- more code ... maybe some day
1330
1331local function makepathlist(list,filetype)
1332    local typespec = resolvers.variableofformat(filetype)
1333    local pathlist = expandedpathlist(typespec,filetype and usertypes[filetype]) -- only extra path with user files
1334    local entry    = { }
1335    if pathlist and #pathlist > 0 then
1336        for k=1,#pathlist do
1337            local path       = pathlist[k]
1338            local prescanned = find(path,'^!!')
1339            local resursive  = find(path,'//$')
1340            local pathname   = lpegmatch(inhibitstripper,path)
1341            local expression = makepathexpression(pathname)
1342            local barename   = gsub(pathname,"/+$","")
1343            barename         = resolveprefix(barename)
1344            local scheme     = urlhasscheme(barename)
1345            local schemename = gsub(barename,"%.%*$",'') -- after scheme
1346         -- local prescanned = path ~= pathname -- ^!!
1347         -- local resursive  = find(pathname,'//$')
1348            entry[k] = {
1349                path       = path,
1350                pathname   = pathname,
1351                prescanned = prescanned,
1352                recursive  = recursive,
1353                expression = expression,
1354                barename   = barename,
1355                scheme     = scheme,
1356                schemename = schemename,
1357            }
1358        end
1359        entry.typespec = typespec
1360        list[filetype] = entry
1361    else
1362        list[filetype] = false
1363    end
1364    return entry
1365end
1366
1367-- pathlist : resolved
1368-- dirlist  : unresolved or resolved
1369-- filelist : unresolved
1370
1371local function find_intree(filename,filetype,wantedfiles,allresults)
1372    local pathlists = instance.pathlists
1373    if not pathlists then
1374        pathlists = setmetatableindex({ },makepathlist)
1375        instance.pathlists = pathlists
1376    end
1377    local pathlist = pathlists[filetype]
1378    if pathlist then
1379        -- list search
1380        local method   = "intree"
1381        local filelist = collect_files(wantedfiles) -- okay, a bit over the top when we just look relative to the current path
1382        local dirlist  = { }
1383        local result   = { }
1384        if filelist then
1385            for i=1,#filelist do
1386                dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble
1387            end
1388        end
1389        if trace_details then
1390            report_resolving("checking filename %a in tree",filename)
1391        end
1392        for k=1,#pathlist do
1393            local entry    = pathlist[k]
1394            local path     = entry.path
1395            local pathname = entry.pathname
1396            local done     = false
1397            -- using file list
1398            if filelist then -- database
1399                -- compare list entries with permitted pattern -- /xx /xx//
1400                local expression = entry.expression
1401                if trace_details then
1402                    report_resolving("using pattern %a for path %a",expression,pathname)
1403                end
1404                for k=1,#filelist do
1405                    local fl = filelist[k]
1406                    local f  = fl[2]
1407                    local d  = dirlist[k]
1408                    -- resolve is new:
1409                    if find(d,expression) or find(resolveprefix(d),expression) then
1410                        -- todo, test for readable
1411                        result[#result+1] = resolveprefix(fl[3]) -- no shortcut
1412                        done = true
1413                        if allresults then
1414                            if trace_details then
1415                                report_resolving("match to %a in hash for file %a and path %a, continue scanning",expression,f,d)
1416                            end
1417                        else
1418                            if trace_details then
1419                                report_resolving("match to %a in hash for file %a and path %a, quit scanning",expression,f,d)
1420                            end
1421                            break
1422                        end
1423                    elseif trace_details then
1424                        report_resolving("no match to %a in hash for file %a and path %a",expression,f,d)
1425                    end
1426                end
1427            end
1428            if done then
1429                method = "database"
1430            else
1431                -- beware: we don't honor allresults here in a next attempt (done false)
1432                -- but that is kind of special anyway
1433                method       = "filesystem" -- bonus, even when !! is specified
1434                local scheme = entry.scheme
1435                if not scheme or scheme == "file" then
1436                    local pname = entry.schemename
1437                    if not find(pname,"*",1,true) then
1438                        if can_be_dir(pname) then
1439                            -- hm, rather useless as we don't go deeper and if we would we could also
1440                            -- auto generate the file database .. however, we need this for extra paths
1441                            -- that are not hashed (like sources on my machine) .. so, this is slightly
1442                            -- out of order but at least fast (and we seldom end up here, only when a file
1443                            -- is not already found
1444                            if not done and not entry.prescanned then
1445                                if trace_details then
1446                                    report_resolving("quick root scan for %a",pname)
1447                                end
1448                                for k=1,#wantedfiles do
1449                                    local w = wantedfiles[k]
1450                                    local fname = check_subpath(filejoin(pname,w))
1451                                    if fname then
1452                                        result[#result+1] = fname
1453                                        done = true
1454                                        if not allresults then
1455                                            break
1456                                        end
1457                                    end
1458                                end
1459                                if not done and entry.recursive then -- maybe also when allresults
1460                                    -- collect files in path (and cache the result)
1461                                    if trace_details then
1462                                        report_resolving("scanning filesystem for %a",pname)
1463                                    end
1464                                    local files = resolvers.simplescanfiles(pname,false,true)
1465                                    for k=1,#wantedfiles do
1466                                        local w = wantedfiles[k]
1467                                        local subpath = files[w]
1468                                        if not subpath or subpath == "" then
1469                                            -- rootscan already done
1470                                        elseif type(subpath) == "string" then
1471                                            local fname = check_subpath(filejoin(pname,subpath,w))
1472                                            if fname then
1473                                                result[#result+1] = fname
1474                                                done = true
1475                                                if not allresults then
1476                                                    break
1477                                                end
1478                                            end
1479                                        else
1480                                            for i=1,#subpath do
1481                                                local sp = subpath[i]
1482                                                if sp == "" then
1483                                                    -- roottest already done
1484                                                else
1485                                                    local fname = check_subpath(filejoin(pname,sp,w))
1486                                                    if fname then
1487                                                        result[#result+1] = fname
1488                                                        done = true
1489                                                        if not allresults then
1490                                                            break
1491                                                        end
1492                                                    end
1493                                                end
1494                                            end
1495                                            if done and not allresults then
1496                                                break
1497                                            end
1498                                        end
1499                                    end
1500                                end
1501                            end
1502                        end
1503                    else
1504                        -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
1505                    end
1506                else
1507                    -- we can have extra_paths that are urls
1508                    for k=1,#wantedfiles do
1509                        -- independent url scanner
1510                        local pname = entry.barename
1511                        local fname = methodhandler('finders',pname .. "/" .. wantedfiles[k])
1512                        if fname then
1513                            result[#result+1] = fname
1514                            done = true
1515                            if not allresults then
1516                                break
1517                            end
1518                        end
1519                    end
1520                end
1521            end
1522            -- todo recursive scanning
1523            if done and not allresults then
1524                break
1525            end
1526        end
1527        if #result > 0 then
1528            return method, result
1529        end
1530    end
1531end
1532
1533local function find_onpath(filename,filetype,wantedfiles,allresults)
1534    if trace_details then
1535        report_resolving("checking filename %a, filetype %a, wanted files %a",filename,filetype,concat(wantedfiles," | "))
1536    end
1537    local result = { }
1538    for k=1,#wantedfiles do
1539        local fname = wantedfiles[k]
1540        if fname and isreadable(fname) then
1541            filename = fname
1542            result[#result+1] = filejoin('.',fname)
1543            if not allresults then
1544                break
1545            end
1546        end
1547    end
1548    if #result > 0 then
1549        return "onpath", result
1550    end
1551end
1552
1553local function find_otherwise(filename,filetype,wantedfiles,allresults) -- other text files | any | whatever
1554    local filelist = collect_files(wantedfiles)
1555    local fl = filelist and filelist[1]
1556    if fl then
1557        return "otherwise", { resolveprefix(fl[3]) } -- filename
1558    end
1559end
1560
1561-- we could have a loop over the 6 functions but then we'd have to
1562-- always analyze .. todo: use url split
1563
1564collect_instance_files = function(filename,askedformat,allresults) -- uses nested
1565    if not filename or filename == "" then
1566        return { }
1567    end
1568    askedformat = askedformat or ""
1569    filename    = collapsepath(filename,".")
1570    filename    = gsub(filename,"^%./",getcurrentdir().."/") -- we will merge dir.expandname and collapse some day
1571    if allresults then
1572        -- no need for caching, only used for tracing
1573        local filetype, wantedfiles = find_analyze(filename,askedformat)
1574        local results = {
1575            { find_direct   (filename,true) },
1576            { find_wildcard (filename,true) },
1577            { find_qualified(filename,true,askedformat) }, -- we can add ,true if we want to find dups
1578            { find_intree   (filename,filetype,wantedfiles,true) },
1579            { find_onpath   (filename,filetype,wantedfiles,true) },
1580            { find_otherwise(filename,filetype,wantedfiles,true) },
1581        }
1582        local result = { }
1583        local status = { }
1584        local done   = { }
1585--         for k, r in next, results do
1586        for k=1,#results do
1587            local r = results[k]
1588            local method, list = r[1], r[2]
1589            if method and list then
1590                for i=1,#list do
1591                    local c = collapsepath(list[i])
1592                    if not done[c] then
1593                        result[#result+1] = c
1594                        done[c] = true
1595                    end
1596                    status[#status+1] = formatters["%-10s: %s"](method,c)
1597                end
1598            end
1599        end
1600        if trace_details then
1601            report_resolving("lookup status: %s",table.serialize(status,filename))
1602        end
1603        return result, status
1604    else
1605        local method, result, stamp, filetype, wantedfiles
1606        if instance.remember then
1607            if askedformat == "" then
1608                stamp = formatters["%s::%s"](suffixonly(filename),filename)
1609            else
1610                stamp = formatters["%s::%s"](askedformat,filename)
1611            end
1612            result = stamp and instance.found[stamp]
1613            if result then
1614                if trace_locating then
1615                    report_resolving("remembered file %a",filename)
1616                end
1617                return result
1618            end
1619        end
1620        method, result = find_direct(filename)
1621        if not result then
1622            method, result = find_wildcard(filename)
1623            if not result then
1624                method, result = find_qualified(filename,false,askedformat)
1625                if not result then
1626                    filetype, wantedfiles = find_analyze(filename,askedformat)
1627                    method, result = find_intree(filename,filetype,wantedfiles)
1628                    if not result then
1629                        method, result = find_onpath(filename,filetype,wantedfiles)
1630                        if resolve_otherwise and not result then
1631                            -- this will search everywhere in the tree
1632                            method, result = find_otherwise(filename,filetype,wantedfiles)
1633                        end
1634                    end
1635                end
1636            end
1637        end
1638        if result and #result > 0 then
1639            local foundname = collapsepath(result[1])
1640            resolvers.registerintrees(filename,askedformat,filetype,method,foundname)
1641            result = { foundname }
1642        else
1643            result = { } -- maybe false
1644        end
1645        if stamp then
1646            if trace_locating then
1647                report_resolving("remembering file %a using hash %a",filename,stamp)
1648            end
1649            instance.found[stamp] = result
1650        end
1651        return result
1652    end
1653end
1654
1655-- -- -- end of main file search routing -- -- --
1656
1657local function findfiles(filename,filetype,allresults)
1658    if not filename or filename == "" then
1659        return { }
1660    end
1661    if allresults == nil then
1662        allresults = true
1663    end
1664    local result, status = collect_instance_files(filename,filetype or "",allresults)
1665    if not result or #result == 0 then
1666        local lowered = lower(filename)
1667        if filename ~= lowered then
1668            result, status = collect_instance_files(lowered,filetype or "",allresults)
1669        end
1670    end
1671    return result or { }, status
1672end
1673
1674local function findfile(filename,filetype)
1675    if not filename or filename == "" then
1676        return ""
1677    else
1678        return findfiles(filename,filetype,false)[1] or ""
1679    end
1680end
1681
1682resolvers.findfiles  = findfiles
1683resolvers.findfile   = findfile
1684
1685resolvers.find_file  = findfile  -- obsolete
1686resolvers.find_files = findfiles -- obsolete
1687
1688function resolvers.findpath(filename,filetype)
1689    return filedirname(findfiles(filename,filetype,false)[1] or "")
1690end
1691
1692local function findgivenfiles(filename,allresults)
1693    local hashes = instance.hashes
1694    local files  = instance.files
1695    local base   = filebasename(filename)
1696    local result = { }
1697    --
1698    local function okay(hash,path,name)
1699        local found = methodhandler('concatinators',hash.type,hash.name,path,name)
1700        if found and found ~= "" then
1701            result[#result+1] = resolveprefix(found)
1702            return not allresults
1703        end
1704    end
1705    --
1706    for k=1,#hashes do
1707        local hash    = hashes[k]
1708        local content = files[hash.name]
1709        if content then
1710            local path, name = lookup(content,base)
1711            if not path then
1712                -- no match
1713            elseif type(path) == "string" then
1714                if okay(hash,path,name) then
1715                    return result
1716                end
1717            else
1718                for i=1,#path do
1719                    if okay(hash,path[i],name) then
1720                        return result
1721                    end
1722                end
1723            end
1724        end
1725    end
1726    --
1727    return result
1728end
1729
1730function resolvers.findgivenfiles(filename)
1731    return findgivenfiles(filename,true)
1732end
1733
1734function resolvers.findgivenfile(filename)
1735    return findgivenfiles(filename,false)[1] or ""
1736end
1737
1738local makewildcard = Cs(
1739    (P("^")^0 * P("/") * P(-1) + P(-1)) /".*"
1740  + (P("^")^0 * P("/") / "")^0 * (P("*")/".*" + P("-")/"%%-" + P(".")/"%%." + P("?")/"."+ P("\\")/"/" + P(1))^0
1741)
1742
1743function resolvers.wildcardpattern(pattern)
1744    return lpegmatch(makewildcard,pattern) or pattern
1745end
1746
1747-- we use more function calls than before but we also have smaller trees so
1748-- why bother
1749
1750local function findwildcardfiles(filename,allresults,result)
1751    local files  = instance.files
1752    local hashes = instance.hashes
1753    --
1754    local result = result or { }
1755    local base   = filebasename(filename)
1756    local dirn   = filedirname(filename)
1757    local path   = lower(lpegmatch(makewildcard,dirn) or dirn)
1758    local name   = lower(lpegmatch(makewildcard,base) or base)
1759    --
1760    if find(name,"*",1,true) then
1761        local function okay(found,path,base,hashname,hashtype)
1762            if find(found,path) then
1763                local full = methodhandler('concatinators',hashtype,hashname,found,base)
1764                if full and full ~= "" then
1765                    result[#result+1] = resolveprefix(full)
1766                    return not allresults
1767                end
1768            end
1769        end
1770        for k=1,#hashes do
1771            local hash     = hashes[k]
1772            local hashname = hash.name
1773            local hashtype = hash.type
1774            if hashname and hashtype then
1775                for found, base in filtered(files[hashname],name) do
1776                    if type(found) == 'string' then
1777                        if okay(found,path,base,hashname,hashtype) then
1778                            break
1779                        end
1780                    else
1781                        for i=1,#found do
1782                            if okay(found[i],path,base,hashname,hashtype) then
1783                                break
1784                            end
1785                        end
1786                    end
1787                end
1788            end
1789        end
1790    else
1791        local function okayokay(found,path,base,hashname,hashtype)
1792            if find(found,path) then
1793                local full = methodhandler('concatinators',hashtype,hashname,found,base)
1794                if full and full ~= "" then
1795                    result[#result+1] = resolveprefix(full)
1796                    return not allresults
1797                end
1798            end
1799        end
1800        --
1801        for k=1,#hashes do
1802            local hash     = hashes[k]
1803            local hashname = hash.name
1804            local hashtype = hash.type
1805            if hashname and hashtype then
1806                local found, base = lookup(content,base)
1807                if not found then
1808                    -- nothing
1809                elseif type(found) == 'string' then
1810                    if okay(found,path,base,hashname,hashtype) then
1811                        break
1812                    end
1813                else
1814                    for i=1,#found do
1815                        if okay(found[i],path,base,hashname,hashtype) then
1816                            break
1817                        end
1818                    end
1819                end
1820            end
1821        end
1822    end
1823    -- we can consider also searching the paths not in the database, but then
1824    -- we end up with a messy search (all // in all path specs)
1825    return result
1826end
1827
1828function resolvers.findwildcardfiles(filename,result)
1829    return findwildcardfiles(filename,true,result)
1830end
1831
1832function resolvers.findwildcardfile(filename)
1833    return findwildcardfiles(filename,false)[1] or ""
1834end
1835
1836do
1837
1838    local starttiming = statistics.starttiming
1839    local stoptiming  = statistics.stoptiming
1840    local elapsedtime = statistics.elapsedtime
1841
1842    function resolvers.starttiming()
1843        starttiming(instance)
1844    end
1845
1846    function resolvers.stoptiming()
1847        stoptiming(instance)
1848    end
1849
1850    function resolvers.loadtime()
1851        return elapsedtime(instance)
1852    end
1853
1854end
1855
1856-- main user functions
1857
1858function resolvers.automount()
1859    -- implemented later (maybe a one-time setter like textopener)
1860end
1861
1862function resolvers.load(option)
1863    resolvers.starttiming()
1864    identify_configuration_files()
1865    load_configuration_files()
1866    if option ~= "nofiles" then
1867        load_databases()
1868        resolvers.automount()
1869    end
1870    resolvers.stoptiming()
1871    local files = instance.files
1872    return files and next(files) and true
1873end
1874
1875local function report(str)
1876    if trace_locating then
1877        report_resolving(str) -- has already verbose
1878    else
1879        print(str)
1880    end
1881end
1882
1883function resolvers.dowithfilesandreport(command, files, ...) -- will move
1884    if files and #files > 0 then
1885        if trace_locating then
1886            report('') -- ?
1887        end
1888        if type(files) == "string" then
1889            files = { files }
1890        end
1891        for f=1,#files do
1892            local file = files[f]
1893            local result = command(file,...)
1894            if type(result) == 'string' then
1895                report(result)
1896            else
1897                for i=1,#result do
1898                    report(result[i]) -- could be unpack
1899                end
1900            end
1901        end
1902    end
1903end
1904
1905-- resolvers.varvalue  = resolvers.variable   -- output the value of variable $STRING.
1906-- resolvers.expandvar = expansion  -- output variable expansion of STRING.
1907
1908function resolvers.showpath(str)     -- output search path for file type NAME
1909    return joinpath(expandedpathlist(resolvers.formatofvariable(str)))
1910end
1911
1912function resolvers.registerfile(files, name, path)
1913    if files[name] then
1914        if type(files[name]) == 'string' then
1915            files[name] = { files[name], path }
1916        else
1917            files[name] = path
1918        end
1919    else
1920        files[name] = path
1921    end
1922end
1923
1924function resolvers.dowithpath(name,func)
1925    local pathlist = expandedpathlist(name)
1926    for i=1,#pathlist do
1927        func("^"..cleanpath(pathlist[i]))
1928    end
1929end
1930
1931function resolvers.dowithvariable(name,func)
1932    func(expandedvariable(name))
1933end
1934
1935function resolvers.locateformat(name)
1936    local engine   = environment.ownmain or "luatex"
1937    local barename = removesuffix(file.basename(name))
1938    local fullname = addsuffix(barename,"fmt")
1939    local fmtname  = caches.getfirstreadablefile(fullname,"formats",engine) or ""
1940    if fmtname == "" then
1941        fmtname = findfile(fullname)
1942        fmtname = cleanpath(fmtname)
1943    end
1944    if fmtname ~= "" then
1945        local barename = removesuffix(fmtname)
1946        local luaname = addsuffix(barename,luasuffixes.lua)
1947        local lucname = addsuffix(barename,luasuffixes.luc)
1948        local luiname = addsuffix(barename,luasuffixes.lui)
1949        if isfile(luiname) then
1950            return fmtname, luiname
1951        elseif isfile(lucname) then
1952            return fmtname, lucname
1953        elseif isfile(luaname) then
1954            return fmtname, luaname
1955        end
1956    end
1957    return nil, nil
1958end
1959
1960function resolvers.booleanvariable(str,default)
1961    local b = expansion(str)
1962    if b == "" then
1963        return default
1964    else
1965        b = toboolean(b)
1966        return (b == nil and default) or b
1967    end
1968end
1969
1970function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, can be a nice iterator instead
1971    local hashes = instance.hashes
1972    local files  = instance.files
1973    for i=1,#hashes do
1974        local hash     = hashes[i]
1975        local blobtype = hash.type
1976        local blobpath = hash.name
1977        if blobtype and blobpath then
1978            local total   = 0
1979            local checked = 0
1980            local done    = 0
1981            if before then
1982                before(blobtype,blobpath,pattern)
1983            end
1984            for path, name in filtered(files[blobpath],pattern) do
1985                if type(path) == "string" then
1986                    checked = checked + 1
1987                    if handle(blobtype,blobpath,path,name) then
1988                        done = done + 1
1989                    end
1990                else
1991                    checked = checked + #path
1992                    for i=1,#path do
1993                        if handle(blobtype,blobpath,path[i],name) then
1994                            done = done + 1
1995                        end
1996                    end
1997                end
1998            end
1999            if after then
2000                after(blobtype,blobpath,pattern,checked,done)
2001            end
2002        end
2003    end
2004end
2005
2006-- moved here
2007
2008function resolvers.knownvariables(pattern)
2009    if instance then
2010        local environment = instance.environment
2011        local variables   = instance.variables
2012        local expansions  = instance.expansions
2013        local order       = instance.order
2014        local pattern     = upper(pattern or "")
2015        local result      = { }
2016        for i=1,#order do
2017            for key in next, order[i] do
2018                if result[key] == nil and key ~= "" and (pattern == "" or find(upper(key),pattern)) then
2019                    result[key] = {
2020                        environment = rawget(environment,key),
2021                        variable    = key,
2022                        expansion   = expansions[key],
2023                        resolved    = resolveprefix(expansions[key]),
2024                    }
2025                end
2026            end
2027        end
2028        return result
2029    else
2030        return { }
2031    end
2032end
2033