data-res.lua /size: 69 Kb    last modification: 2025-02-21 11:03
1if not modules then modules = { } end modules ['data-res'] = {
2    version   = 1.001,
3    comment   = "companion to luat-lib.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files",
7}
8
9-- todo: cache:/// home:/// selfautoparent:/// (sometime end 2012)
10
11local gsub, find, lower, upper, match, gmatch = string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
12local concat, insert, remove = table.concat, table.insert, table.remove
13local next, type, rawget, loadfile = next, type, rawget, loadfile
14local mergedtable = table.merged
15
16local os = os
17
18local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg
19local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns
20
21local formatters        = string.formatters
22local filedirname       = file.dirname
23local filebasename      = file.basename
24local suffixonly        = file.suffixonly
25local addsuffix         = file.addsuffix
26local removesuffix      = file.removesuffix
27local filejoin          = file.join
28local collapsepath      = file.collapsepath
29local joinpath          = file.joinpath
30local is_qualified_path = file.is_qualified_path
31
32local allocate          = utilities.storage.allocate
33local settings_to_array = utilities.parsers.settings_to_array
34
35local urlhasscheme      = url.hasscheme
36
37local getcurrentdir     = lfs.currentdir
38local isfile            = lfs.isfile
39local isdir             = lfs.isdir
40
41local setmetatableindex = table.setmetatableindex
42local luasuffixes       = utilities.lua.suffixes
43
44local trace_locating    = false  trackers  .register("resolvers.locating",   function(v) trace_locating    = v end)
45local trace_details     = false  trackers  .register("resolvers.details",    function(v) trace_details     = v end)
46local trace_expansions  = false  trackers  .register("resolvers.expansions", function(v) trace_expansions  = v end)
47local trace_paths       = false  trackers  .register("resolvers.paths",      function(v) trace_paths       = v end)
48local resolve_otherwise = true   directives.register("resolvers.otherwise",  function(v) resolve_otherwise = v end)
49
50local report_resolving = logs.reporter("resolvers","resolving")
51
52local resolvers              = resolvers
53
54local expandedpathfromlist   = resolvers.expandedpathfromlist
55local checkedvariable        = resolvers.checkedvariable
56local splitconfigurationpath = resolvers.splitconfigurationpath
57local methodhandler          = resolvers.methodhandler
58local filtered               = resolvers.filtered_from_content
59local lookup                 = resolvers.get_from_content
60local cleanpath              = resolvers.cleanpath
61local resolveprefix          = resolvers.resolve
62
63local initializesetter       = utilities.setters.initialize
64
65local ostype, osname, osenv, ossetenv, osgetenv = os.type, os.name, os.env, os.setenv, os.getenv
66
67resolvers.cacheversion   = "1.100"
68resolvers.configbanner   = ""
69resolvers.homedir        = environment.homedir
70resolvers.luacnfname     = "texmfcnf.lua"
71resolvers.luacnffallback = "contextcnf.lua"
72resolvers.luacnfstate    = "unknown"
73
74local criticalvars = {
75    "SELFAUTOLOC",
76    "SELFAUTODIR",
77    "SELFAUTOPARENT",
78    "TEXMFCNF",
79    "TEXMF",
80    "TEXOS",
81}
82
83-- The web2c tex binaries as well as kpse have built in paths for the configuration
84-- files and there can be a depressing truckload of them. This is actually the weak
85-- spot of a distribution. So we don't want:
86--
87-- resolvers.luacnfspec = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}'
88--
89-- but instead (for instance) use:
90--
91-- resolvers.luacnfspec = 'selfautoparent:{/texmf{-local,}{,/web2c}}'
92--
93-- which does not make texlive happy as there is a texmf-local tree one level up
94-- (sigh), so we need this. We can assume web2c as mkiv does not run on older
95-- texlives anyway.
96--
97-- texlive:
98--
99-- selfautoloc:
100-- selfautoloc:/share/texmf-local/web2c
101-- selfautoloc:/share/texmf-dist/web2c
102-- selfautoloc:/share/texmf/web2c
103-- selfautoloc:/texmf-local/web2c
104-- selfautoloc:/texmf-dist/web2c
105-- selfautoloc:/texmf/web2c
106-- selfautodir:
107-- selfautodir:/share/texmf-local/web2c
108-- selfautodir:/share/texmf-dist/web2c
109-- selfautodir:/share/texmf/web2c
110-- selfautodir:/texmf-local/web2c
111-- selfautodir:/texmf-dist/web2c
112-- selfautodir:/texmf/web2c
113-- selfautoparent:/../texmf-local/web2c
114-- selfautoparent:
115-- selfautoparent:/share/texmf-local/web2c
116-- selfautoparent:/share/texmf-dist/web2c
117-- selfautoparent:/share/texmf/web2c
118-- selfautoparent:/texmf-local/web2c
119-- selfautoparent:/texmf-dist/web2c
120-- selfautoparent:/texmf/web2c
121--
122-- minimals:
123--
124-- home:texmf/web2c
125-- selfautoparent:texmf-local/web2c
126-- selfautoparent:texmf-context/web2c
127-- selfautoparent:texmf/web2c
128
129-- This is a real mess: you don't want to know what creepy paths end up in the default
130-- configuration spec, for instance nested texmf- paths. I'd rather get away from it and
131-- specify a proper search sequence but alas ... it is not permitted in texlive and there
132-- is no way to check if we run a minimals as texmf-context is not in that spec. It's a
133-- compiled-in permutation of historics with the selfautoloc, selfautodir, selfautoparent
134-- resulting in weird combinations. So, when we eventually check the 30 something paths
135-- we also report weird ones, with weird being: (1) duplicate /texmf or (2) no /web2c in
136-- the names.
137
138do
139
140    local texroot = environment.texroot
141
142    resolvers.luacnfspec = {
143        "home:texmf/web2c",
144        "selfautoparent:/texmf-local/web2c",
145        "selfautoparent:/texmf-context/web2c",
146        "selfautoparent:/texmf/web2c",
147    }
148
149    if environment.default_texmfcnf then
150        -- this will go away (but then also no more checking in mtxrun.lua itself)
151        resolvers.luacnfspec = {
152            "home:texmf/web2c",
153            environment.default_texmfcnf, -- texlive + home: for taco etc
154        }
155    elseif texroot and isdir(texroot .. "/texmf-context") then
156        -- we're okay and run the lean and mean reference installation
157    elseif texroot and isdir(texroot .. "/texmf-dist") then
158        -- we're in texlive where texmf-dist is leading
159        resolvers.luacnfspec = {
160            "home:texmf/web2c",
161            "selfautoparent:/texmf-local/web2c",
162            "selfautoparent:", -- new per 2024 as it's needed for osx
163            "selfautoparent:/texmf-dist/web2c",
164            "selfautoparent:/texmf/web2c",
165        }
166    elseif ostype ~= "windows" and isdir("/etc/texmf/web2c") then
167        -- we have some linux distribution that does it its own way
168        resolvers.luacnfspec = {
169            "home:texmf/web2c",
170            "/etc/texmf/web2c",
171            "selfautodir:/share/texmf/web2c",
172        }
173    else
174        -- we stick to the reference specification
175    end
176
177    resolvers.luacnfspec = concat(resolvers.luacnfspec,";")
178
179end
180
181local unset_variable = "unset"
182
183local formats   = resolvers.formats
184local suffixes  = resolvers.suffixes
185local usertypes = resolvers.usertypes
186local dangerous = resolvers.dangerous
187local suffixmap = resolvers.suffixmap
188
189resolvers.defaultsuffixes = { "tex" } --  "mkiv", "cld" -- too tricky
190
191local instance  = nil -- the current one (fast access)
192
193-- forward declarations
194
195local variable
196local expansion
197local setenv
198local getenv
199
200-- done
201
202local formatofsuffix = resolvers.formatofsuffix
203local splitpath      = resolvers.splitpath
204local splitmethod    = resolvers.splitmethod
205
206-- An instance has an environment (coming from the outside, kept raw), variables
207-- (coming from the configuration file), and expansions (variables with nested
208-- variables replaced). One can push something into the outer environment and
209-- its internal copy, but only the later one will be the raw unprefixed variant.
210
211setenv = function(key,value,raw)
212    if instance then
213        -- this one will be consulted first when we stay inside
214        -- the current environment (prefixes are not resolved here)
215        instance.environment[key] = value
216        -- we feed back into the environment, and as this is used
217        -- by other applications (via os.execute) we need to make
218        -- sure that prefixes are resolve
219        ossetenv(key,raw and value or resolveprefix(value))
220    end
221end
222
223-- Beware we don't want empty here as this one can be called early on
224-- and therefore we use rawget.
225
226getenv = function(key)
227    local value = rawget(instance.environment,key)
228    if value and value ~= "" then
229        return value
230    else
231        local e = osgetenv(key)
232        return e ~= nil and e ~= "" and checkedvariable(e) or ""
233    end
234end
235
236resolvers.getenv = getenv
237resolvers.setenv = setenv
238
239-- We are going to use some metatable trickery where we backtrack from
240-- expansion to variable to environment.
241
242local dollarstripper   = lpeg.stripper("$")
243local inhibitstripper  = P("!")^0 * Cs(P(1)^0)
244
245local expandedvariable, resolvedvariable  do
246
247    local function resolveinstancevariable(k)
248        return instance.expansions[k]
249    end
250
251    local p_variable       = P("$") / ""
252    local p_key            = C(R("az","AZ","09","__","--")^1)
253    local p_whatever       = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "")
254                           + P(";") * (P(";") / "")
255                           + P(1)
256    local variableexpander = Cs( (p_variable * (p_key/resolveinstancevariable) + p_whatever)^1 )
257
258    local p_cleaner        = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";"
259    local variablecleaner  = Cs((p_cleaner  + P(1))^0)
260
261    local p_variable       = R("az","AZ","09","__","--")^1 / resolveinstancevariable
262    local p_variable       = (P("$")/"") * (p_variable + (P("{")/"") * p_variable * (P("}")/""))
263    local variableresolver = Cs((p_variable + P(1))^0)
264
265    expandedvariable = function(var)
266        return lpegmatch(variableexpander,var) or var
267    end
268
269    function resolvers.reset()
270
271        -- normally we only need one instance but for special cases we can (re)load one so
272        -- we stick to this model.
273
274        if trace_locating then
275            report_resolving("creating instance")
276        end
277
278        local environment = { }
279        local variables   = { }
280        local expansions  = { }
281        local order       = { }
282
283        instance = {
284            environment    = environment,
285            variables      = variables,
286            expansions     = expansions,
287            order          = order,
288            files          = { },
289            setups         = { },
290            found          = { },
291            foundintrees   = { },
292            hashes         = { },
293            hashed         = { },
294            pathlists      = false,-- delayed
295            specification  = { },
296            lists          = { },
297            data           = { }, -- only for loading
298            fakepaths      = { },
299            remember       = true,
300            diskcache      = true,
301            renewcache     = false,
302            renewtree      = false,
303            loaderror      = false,
304            savelists      = true,
305            pattern        = nil, -- lists
306            force_suffixes = true,
307            pathstack      = { },
308        }
309
310        setmetatableindex(variables,function(t,k)
311            local v
312            for i=1,#order do
313                v = order[i][k]
314                if v ~= nil then
315                    t[k] = v
316                    return v
317                end
318            end
319            if v == nil then
320                v = ""
321            end
322            t[k] = v
323            return v
324        end)
325
326        local repath = resolvers.repath
327
328        setmetatableindex(environment, function(t,k)
329            local v = osgetenv(k)
330            if v == nil then
331                v = variables[k]
332            end
333            if v ~= nil then
334                v = checkedvariable(v) or ""
335            end
336            v = repath(v) -- for taco who has a : separated osfontdir
337            t[k] = v
338            return v
339        end)
340
341        setmetatableindex(expansions, function(t,k)
342            local v = environment[k]
343            if type(v) == "string" then
344                v = lpegmatch(variableresolver,v)
345                v = lpegmatch(variablecleaner,v)
346            end
347            t[k] = v
348            return v
349        end)
350
351    end
352
353end
354
355function resolvers.initialized()
356    return instance ~= nil
357end
358
359local function reset_hashes()
360    instance.lists     = { }
361    instance.pathlists = false
362    instance.found     = { }
363end
364
365local function reset_caches()
366    instance.lists     = { }
367    instance.pathlists = false
368end
369
370local makepathexpression  do
371
372    local slash = P("/")
373
374    local pathexpressionpattern = Cs ( -- create lpeg instead (2013/2014)
375        Cc("^") * (
376            Cc("%") * S(".-")
377          + slash^2 * P(-1) / "/.*"
378       -- + slash^2 / "/.-/"
379       -- + slash^2 / "/[^/]*/*"   -- too general
380          + slash^2 / "/"
381          + (1-slash) * P(-1) * Cc("/")
382          + P(1)
383        )^1 * Cc("$") -- yes or no $
384    )
385
386    local cache = { }
387
388    makepathexpression = function(str)
389        if str == "." then
390            return "^%./$"
391        else
392            local c = cache[str]
393            if not c then
394                c = lpegmatch(pathexpressionpattern,str)
395                cache[str] = c
396            end
397            return c
398        end
399    end
400
401end
402
403local function reportcriticalvariables(cnfspec)
404    if trace_locating then
405        for i=1,#criticalvars do
406            local k = criticalvars[i]
407            local v = getenv(k) or "unknown" -- this one will not resolve !
408            report_resolving("variable %a set to %a",k,v)
409        end
410        report_resolving()
411        if cnfspec then
412            report_resolving("using configuration specification %a",type(cnfspec) == "table" and concat(cnfspec,",") or cnfspec)
413        end
414        report_resolving()
415    end
416    reportcriticalvariables = function() end
417end
418
419local function identify_configuration_files()
420    local specification = instance.specification
421    if #specification == 0 then
422        local cnfspec = getenv("TEXMFCNF")
423        if cnfspec == "" then
424            cnfspec = resolvers.luacnfspec
425            resolvers.luacnfstate = "default"
426        else
427            resolvers.luacnfstate = "environment"
428        end
429        reportcriticalvariables(cnfspec)
430        local cnfpaths = expandedpathfromlist(splitpath(cnfspec))
431
432        local function locatecnf(luacnfname,kind)
433            for i=1,#cnfpaths do
434                local filepath = cnfpaths[i]
435                local filename = collapsepath(filejoin(filepath,luacnfname))
436                local realname = resolveprefix(filename) -- can still have "//" ... needs checking
437                -- todo: environment.skipweirdcnfpaths directive
438                if trace_locating then
439                    local fullpath  = gsub(resolveprefix(collapsepath(filepath)),"//","/")
440                    local weirdpath = find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true)
441                    report_resolving("looking for %s %a on %s path %a from specification %a",
442                        kind,luacnfname,weirdpath and "weird" or "given",fullpath,filepath)
443                end
444                if isfile(realname) then
445                    specification[#specification+1] = filename -- unresolved as we use it in matching, relocatable
446                    if trace_locating then
447                        report_resolving("found %s configuration file %a",kind,realname)
448                    end
449                end
450            end
451        end
452
453        locatecnf(resolvers.luacnfname,"regular")
454        if #specification == 0 then
455            locatecnf(resolvers.luacnffallback,"fallback")
456        end
457        if trace_locating then
458            report_resolving()
459        end
460    elseif trace_locating then
461        report_resolving("configuration files already identified")
462    end
463end
464
465local function load_configuration_files()
466    local specification = instance.specification
467    local setups        = instance.setups
468    local order         = instance.order
469    if #specification > 0 then
470        local luacnfname = resolvers.luacnfname
471        for i=1,#specification do
472            local filename = specification[i]
473            local pathname = filedirname(filename)
474            local filename = filejoin(pathname,luacnfname)
475            local realname = resolveprefix(filename) -- no shortcut
476            local blob = loadfile(realname)
477            if blob then
478                local data = blob()
479                local parent = data and data.parent
480                if parent then
481                    local filename = filejoin(pathname,parent)
482                    local realname = resolveprefix(filename) -- no shortcut
483                    local blob = loadfile(realname)
484                    if blob then
485                        local parentdata = blob()
486                        if parentdata then
487                            report_resolving("loading configuration file %a",filename)
488                            data = mergedtable(parentdata,data)
489                        end
490                    end
491                end
492                data = data and data.content
493                if data then
494                    if trace_locating then
495                        report_resolving("loading configuration file %a",filename)
496                        report_resolving()
497                    end
498                    local variables = data.variables or { }
499                    local warning = false
500                    for k, v in next, data do
501                        local variant = type(v)
502                        if variant == "table" then
503                            initializesetter(filename,k,v)
504                        elseif variables[k] == nil then
505                            if trace_locating and not warning then
506                                report_resolving("variables like %a in configuration file %a should move to the 'variables' subtable",
507                                    k,resolveprefix(filename))
508                                warning = true
509                            end
510                            variables[k] = v
511                        end
512                    end
513                    setups[pathname] = variables
514                    if resolvers.luacnfstate == "default" then
515                        -- the following code is not tested
516                        local cnfspec = variables["TEXMFCNF"]
517                        if cnfspec then
518                            if trace_locating then
519                                report_resolving("reloading configuration due to TEXMF redefinition")
520                            end
521                            -- we push the value into the main environment (osenv) so
522                            -- that it takes precedence over the default one and therefore
523                            -- also over following definitions
524                            setenv("TEXMFCNF",cnfspec) -- resolves prefixes
525                            -- we now identify and load the specified configuration files
526                            instance.specification = { }
527                            identify_configuration_files()
528                            load_configuration_files()
529                            -- we prevent further overload of the configuration variable
530                            resolvers.luacnfstate = "configuration"
531                            -- we quit the outer loop
532                            break
533                        end
534                    end
535
536                else
537                    if trace_locating then
538                        report_resolving("skipping configuration file %a (no content)",filename)
539                    end
540                    setups[pathname] = { }
541                    instance.loaderror = true
542                end
543            elseif trace_locating then
544                report_resolving("skipping configuration file %a (no valid format)",filename)
545            end
546            order[#order+1] = setups[pathname]
547            if instance.loaderror then
548                break
549            end
550        end
551    elseif trace_locating then
552        report_resolving("warning: no lua configuration files found")
553    end
554end
555
556-- forward declarations:
557
558local expandedpathlist
559local unexpandedpathlist
560
561-- done
562
563function resolvers.configurationfiles()
564    return instance.specification or { }
565end
566
567-- scheme magic ... database loading
568
569local function load_file_databases()
570    instance.loaderror = false
571    instance.files     = { }
572    if not instance.renewcache then
573        local hashes = instance.hashes
574        for k=1,#hashes do
575            local hash = hashes[k]
576            resolvers.hashers.byscheme(hash.type,hash.name)
577            if instance.loaderror then break end
578        end
579    end
580end
581
582local function locate_file_databases()
583    -- todo: cache:// and tree:// (runtime)
584    local texmfpaths = expandedpathlist("TEXMF")
585    if #texmfpaths > 0 then
586        for i=1,#texmfpaths do
587            local path = collapsepath(texmfpaths[i])
588            path = gsub(path,"/+$","") -- in case $HOME expands to something with a trailing /
589            local stripped = lpegmatch(inhibitstripper,path) -- the !! thing
590            if stripped ~= "" then
591                local runtime = stripped == path
592                path = cleanpath(path)
593                local spec = splitmethod(stripped)
594                if runtime and (spec.noscheme or spec.scheme == "file") then
595                    stripped = "tree:///" .. stripped
596                elseif spec.scheme == "cache" or spec.scheme == "file" then
597                    stripped = spec.path
598                end
599                if trace_locating then
600                    if runtime then
601                        report_resolving("locating list of %a (runtime) (%s)",path,stripped)
602                    else
603                        report_resolving("locating list of %a (cached)",path)
604                    end
605                end
606                methodhandler('locators',stripped)
607            end
608        end
609        if trace_locating then
610            report_resolving()
611        end
612    elseif trace_locating then
613        report_resolving("no texmf paths are defined (using TEXMF)")
614    end
615end
616
617local function generate_file_databases()
618    local hashes = instance.hashes
619    for k=1,#hashes do
620        local hash = hashes[k]
621        methodhandler('generators',hash.name)
622    end
623    if trace_locating then
624        report_resolving()
625    end
626end
627
628local function save_file_databases() -- will become cachers
629    local hashes = instance.hashes
630    local files  = instance.files
631    for i=1,#hashes do
632        local hash      = hashes[i]
633        local cachename = hash.name
634        if hash.cache then
635            local content = files[cachename]
636            caches.collapsecontent(content)
637            if trace_locating then
638                report_resolving("saving tree %a",cachename)
639            end
640            caches.savecontent(cachename,"files",content)
641        elseif trace_locating then
642            report_resolving("not saving runtime tree %a",cachename)
643        end
644    end
645end
646
647function resolvers.renew(hashname)
648    local files = instance.files
649    if hashname and hashname ~= "" then
650        local expanded = expansion(hashname) or ""
651        if expanded ~= "" then
652            if trace_locating then
653                report_resolving("identifying tree %a from %a",expanded,hashname)
654            end
655            hashname = expanded
656        else
657            if trace_locating then
658                report_resolving("identifying tree %a",hashname)
659            end
660        end
661        local realpath = resolveprefix(hashname)
662        if isdir(realpath) then
663            if trace_locating then
664                report_resolving("using path %a",realpath)
665            end
666            methodhandler('generators',hashname)
667            -- could be shared
668            local content = files[hashname]
669            caches.collapsecontent(content)
670            if trace_locating then
671                report_resolving("saving tree %a",hashname)
672            end
673            caches.savecontent(hashname,"files",content)
674            -- till here
675        else
676            report_resolving("invalid path %a",realpath)
677        end
678    end
679end
680
681local function load_databases()
682    locate_file_databases()
683    if instance.diskcache and not instance.renewcache then
684        load_file_databases()
685        if instance.loaderror then
686            generate_file_databases()
687            save_file_databases()
688        end
689    else
690        generate_file_databases()
691        if instance.renewcache then
692            save_file_databases()
693        end
694    end
695end
696
697function resolvers.appendhash(type,name,cache)
698    local hashed = instance.hashed
699    local hashes = instance.hashes
700    if hashed[name] then
701        -- safeguard ... tricky as it's actually a bug when seen twice
702    else
703        if trace_locating then
704            report_resolving("hash %a appended",name)
705        end
706        insert(hashes, { type = type, name = name, cache = cache } )
707        hashed[name] = cache
708    end
709end
710
711function resolvers.prependhash(type,name,cache)
712    local hashed = instance.hashed
713    local hashes = instance.hashes
714    if hashed[name] then
715        -- safeguard ... tricky as it's actually a bug when seen twice
716    else
717        if trace_locating then
718            report_resolving("hash %a prepended",name)
719        end
720        insert(hashes, 1, { type = type, name = name, cache = cache } )
721        hashed[name] = cache
722    end
723end
724
725function resolvers.extendtexmfvariable(specification) -- crap, we could better prepend the hash
726    local environment = instance.environment
727    local variables   = instance.variables
728    local texmftrees  = splitpath(getenv("TEXMF")) -- okay?
729    insert(texmftrees,1,specification)
730    texmftrees = concat(texmftrees,",") -- not ;
731    if environment["TEXMF"] then
732        environment["TEXMF"] = texmftrees
733    elseif variables["TEXMF"] then
734        variables["TEXMF"] = texmftrees
735    else
736        -- weird
737    end
738    reset_hashes()
739end
740
741function resolvers.splitexpansions()
742    local expansions = instance.expansions
743    for k, v in next, expansions do
744        local t, tn, h, p = { }, 0, { }, splitconfigurationpath(v)
745        for kk=1,#p do
746            local vv = p[kk]
747            if vv ~= "" and not h[vv] then
748                tn = tn + 1
749                t[tn] = vv
750                h[vv] = true
751            end
752        end
753        if tn > 1 then
754            expansions[k] = t
755        else
756            expansions[k] = t[1]
757        end
758    end
759end
760
761-- end of split/join code
762
763-- we used to have 'files' and 'configurations' so therefore the following
764-- shared function
765
766function resolvers.datastate()
767    return caches.contentstate()
768end
769
770variable = function(name)
771    local variables = instance.variables
772    local name   = name and lpegmatch(dollarstripper,name)
773    local result = name and variables[name]
774    return result ~= nil and result or ""
775end
776
777expansion = function(name)
778    local expansions = instance.expansions
779    local name   = name and lpegmatch(dollarstripper,name)
780    local result = name and expansions[name]
781    return result ~= nil and result or ""
782end
783
784resolvers.variable  = variable
785resolvers.expansion = expansion
786
787unexpandedpathlist = function(str)
788    local pth = variable(str)
789    local lst = splitpath(pth)
790    return expandedpathfromlist(lst)
791end
792
793function resolvers.unexpandedpath(str)
794    return joinpath(unexpandedpathlist(str))
795end
796
797function resolvers.pushpath(name)
798    local pathstack = instance.pathstack
799    local lastpath  = pathstack[#pathstack]
800    local pluspath  = filedirname(name)
801    if lastpath then
802        lastpath = collapsepath(filejoin(lastpath,pluspath))
803    else
804        lastpath = collapsepath(pluspath)
805    end
806    insert(pathstack,lastpath)
807    if trace_paths then
808        report_resolving("pushing path %a",lastpath)
809    end
810end
811
812function resolvers.poppath()
813    local pathstack = instance.pathstack
814    if trace_paths and #pathstack > 0 then
815        report_resolving("popping path %a",pathstack[#pathstack])
816    end
817    remove(pathstack)
818end
819
820function resolvers.stackpath()
821    local pathstack   = instance.pathstack
822    local currentpath = pathstack[#pathstack]
823    return currentpath ~= "" and currentpath or nil
824end
825
826local done = { }
827
828function resolvers.resetextrapaths()
829    local extra_paths = instance.extra_paths
830    if not extra_paths then
831        done                 = { }
832        instance.extra_paths = { }
833    elseif #ep > 0 then
834        done = { }
835        reset_caches()
836    end
837end
838
839function resolvers.getextrapaths()
840    return instance.extra_paths or { }
841end
842
843function resolvers.registerextrapath(paths,subpaths)
844    if not subpaths or subpaths == "" then
845        if not paths or path == "" then
846            return -- invalid spec
847        elseif done[paths] then
848            return -- already done
849        end
850    end
851    local paths       = settings_to_array(paths)
852    local subpaths    = settings_to_array(subpaths)
853    local extra_paths = instance.extra_paths or { }
854    local oldn        = #extra_paths
855    local newn        = oldn
856    local nofpaths    = #paths
857    local nofsubpaths = #subpaths
858    if nofpaths > 0 then
859        if nofsubpaths > 0 then
860            for i=1,nofpaths do
861                local p = paths[i]
862                for j=1,nofsubpaths do
863                    local s = subpaths[j]
864                    local ps = p .. "/" .. s
865                    if not done[ps] then
866                        newn = newn + 1
867                        extra_paths[newn] = cleanpath(ps)
868                        done[ps] = true
869                    end
870                end
871            end
872        else
873            for i=1,nofpaths do
874                local p = paths[i]
875                if not done[p] then
876                    newn = newn + 1
877                    extra_paths[newn] = cleanpath(p)
878                    done[p] = true
879                end
880            end
881        end
882    elseif nofsubpaths > 0 then
883        for i=1,oldn do
884            for j=1,nofsubpaths do
885                local s = subpaths[j]
886                local ps = extra_paths[i] .. "/" .. s
887                if not done[ps] then
888                    newn = newn + 1
889                    extra_paths[newn] = cleanpath(ps)
890                    done[ps] = true
891                end
892            end
893        end
894    end
895    if newn > 0 then
896        instance.extra_paths = extra_paths -- register paths
897    end
898    if newn ~= oldn then
899        reset_caches()
900    end
901end
902
903function resolvers.pushextrapath(path)
904    local paths = settings_to_array(path)
905    local extra_stack = instance.extra_stack
906    if extra_stack then
907        insert(extra_stack,1,paths)
908    else
909        instance.extra_stack = { paths }
910    end
911    reset_caches()
912end
913
914function resolvers.popextrapath()
915    local extra_stack = instance.extra_stack
916    if extra_stack then
917        reset_caches()
918        return remove(extra_stack,1)
919    end
920end
921
922local function made_list(instance,list,extra_too)
923    local done = { }
924    local new  = { }
925    local newn = 0
926    -- a helper
927    local function add(p)
928        for k=1,#p do
929            local v = p[k]
930            if not done[v] then
931                done[v] = true
932                newn = newn + 1
933                new[newn] = v
934            end
935        end
936    end
937    -- honour . .. ../.. but only when at the start
938    for k=1,#list do
939        local v = list[k]
940        if done[v] then
941            -- skip
942        elseif find(v,"^[%.%/]$") then
943            done[v] = true
944            newn = newn + 1
945            new[newn] = v
946        else
947            break
948        end
949    end
950    if extra_too then
951        local extra_stack = instance.extra_stack
952        local extra_paths = instance.extra_paths
953        -- first the stacked paths
954        if extra_stack and #extra_stack > 0 then
955            for k=1,#extra_stack do
956                add(extra_stack[k])
957            end
958        end
959        -- then the extra paths
960        if extra_paths and #extra_paths > 0 then
961            add(extra_paths)
962        end
963    end
964    -- last the formal paths
965    add(list)
966    return new
967end
968
969expandedpathlist = function(str,extra_too)
970    if not str then
971        return { }
972    elseif instance.savelists then -- hm, what if two cases, with and without extra_too
973        str = lpegmatch(dollarstripper,str)
974        local lists = instance.lists
975        local lst = lists[str]
976        if not lst then
977            local l = made_list(instance,splitpath(expansion(str)),extra_too)
978            lst = expandedpathfromlist(l)
979            lists[str] = lst
980        end
981        return lst
982    else
983        local lst = splitpath(expansion(str))
984        return made_list(instance,expandedpathfromlist(lst),extra_too)
985    end
986end
987
988resolvers.expandedpathlist   = expandedpathlist
989resolvers.unexpandedpathlist = unexpandedpathlist
990
991function resolvers.cleanpathlist(str)
992    local t = expandedpathlist(str)
993    if t then
994        for i=1,#t do
995            t[i] = collapsepath(cleanpath(t[i]))
996        end
997    end
998    return t
999end
1000
1001function resolvers.expandpath(str)
1002    return joinpath(expandedpathlist(str))
1003end
1004
1005local function expandedpathlistfromvariable(str) -- brrr / could also have cleaner ^!! /$ //
1006    str = lpegmatch(dollarstripper,str)
1007    local tmp = resolvers.variableofformatorsuffix(str)
1008    return expandedpathlist(tmp ~= "" and tmp or str)
1009end
1010
1011function resolvers.expandpathfromvariable(str)
1012    return joinpath(expandedpathlistfromvariable(str))
1013end
1014
1015resolvers.expandedpathlistfromvariable = expandedpathlistfromvariable
1016
1017function resolvers.cleanedpathlist(v) -- can be cached if needed
1018    local t = expandedpathlist(v)
1019    for i=1,#t do
1020        t[i] = resolveprefix(cleanpath(t[i]))
1021    end
1022    return t
1023end
1024
1025function resolvers.expandbraces(str) -- output variable and brace expansion of STRING
1026    local pth = expandedpathfromlist(splitpath(str))
1027    return joinpath(pth)
1028end
1029
1030function resolvers.registerfilehash(name,content,someerror)
1031    local files = instance.files
1032    if content then
1033        files[name] = content
1034    else
1035        files[name] = { }
1036        if somerror == true then -- can be unset
1037            instance.loaderror = someerror
1038        end
1039    end
1040end
1041
1042function resolvers.getfilehashes()
1043    return instance and instance.files or { }
1044end
1045
1046function resolvers.gethashes()
1047    return instance and instance.hashes or { }
1048end
1049
1050function resolvers.renewcache()
1051    if instance then
1052        instance.renewcache = true
1053    end
1054end
1055
1056local function isreadable(name)
1057    local readable = isfile(name) -- not file.is_readable(name) as it can be a dir
1058    if trace_details then
1059        if readable then
1060            report_resolving("file %a is readable",name)
1061        else
1062            report_resolving("file %a is not readable", name)
1063        end
1064    end
1065    return readable
1066end
1067
1068-- name | name/name
1069
1070local function collect_files(names) -- potential files .. sort of too much when asking for just one file
1071    local filelist = { }            -- but we need it for pattern matching later on
1072    local noffiles = 0
1073    local function check(hash,root,pathname,path,basename,name)
1074        if not pathname or find(path,pathname) then
1075            local variant = hash.type
1076            local search  = filejoin(root,path,name) -- funny no concatinator
1077            local result  = methodhandler('concatinators',variant,root,path,name)
1078            if trace_details then
1079                report_resolving("match: variant %a, search %a, result %a",variant,search,result)
1080            end
1081            noffiles = noffiles + 1
1082            filelist[noffiles] = { variant, search, result }
1083        end
1084    end
1085    for k=1,#names do
1086        local filename = names[k]
1087        if trace_details then
1088            report_resolving("checking name %a",filename)
1089        end
1090        local basename = filebasename(filename)
1091        local pathname = filedirname(filename)
1092        if pathname == "" or find(pathname,"^%.") then
1093            pathname = false
1094        else
1095            pathname = gsub(pathname,"%*",".*")
1096            pathname = "/" .. pathname .. "$"
1097        end
1098        local hashes = instance.hashes
1099        local files  = instance.files
1100        for h=1,#hashes do
1101            local hash     = hashes[h]
1102            local hashname = hash.name
1103            local content  = hashname and files[hashname]
1104            if content then
1105                if trace_details then
1106                    report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname)
1107                end
1108                local path, name = lookup(content,basename)
1109                if path then
1110                    local metadata = content.metadata
1111                    local realroot = metadata and metadata.path or hashname
1112                    if type(path) == "string" then
1113                        check(hash,realroot,pathname,path,basename,name)
1114                    else
1115                        for i=1,#path do
1116                            check(hash,realroot,pathname,path[i],basename,name)
1117                        end
1118                    end
1119                end
1120            elseif trace_locating then
1121                report_resolving("no match in %a (%s)",hashname,basename)
1122            end
1123        end
1124    end
1125    return noffiles > 0 and filelist or nil
1126end
1127
1128local fit = { }
1129
1130function resolvers.registerintrees(filename,format,filetype,usedmethod,foundname)
1131    local foundintrees = instance.foundintrees
1132    if usedmethod == "direct" and filename == foundname and fit[foundname] then
1133        -- just an extra lookup after a test on presence
1134    else
1135        local collapsed = collapsepath(foundname,true)
1136        local t = {
1137            filename   = filename,
1138            format     = format    ~= "" and format   or nil,
1139            filetype   = filetype  ~= "" and filetype or nil,
1140            usedmethod = usedmethod,
1141            foundname  = foundname,
1142            fullname   = collapsed,
1143        }
1144        fit[foundname] = t
1145        foundintrees[#foundintrees+1] = t
1146    end
1147end
1148
1149function resolvers.foundintrees()
1150    return instance.foundintrees or { }
1151end
1152
1153function resolvers.foundintree(fullname)
1154    local f = fit[fullname]
1155    return f and f.usedmethod == "database"
1156end
1157
1158-- split the next one up for readability (but this module needs a cleanup anyway)
1159
1160local function can_be_dir(name) -- can become local
1161    local fakepaths = instance.fakepaths
1162    if not fakepaths[name] then
1163        if isdir(name) then
1164            fakepaths[name] = 1 -- directory
1165        else
1166            fakepaths[name] = 2 -- no directory
1167        end
1168    end
1169    return fakepaths[name] == 1
1170end
1171
1172local preparetreepattern = Cs((P(".")/"%%." + P("-")/"%%-" + P(1))^0 * Cc("$"))
1173
1174-- -- -- begin of main file search routing -- -- -- needs checking as previous has been patched
1175
1176local collect_instance_files
1177
1178local function find_analyze(filename,askedformat,allresults)
1179    local filetype    = ""
1180    local filesuffix  = suffixonly(filename)
1181    local wantedfiles = { }
1182    -- too tricky as filename can be bla.1.2.3:
1183    --
1184    -- if not suffixmap[filesuffix] then
1185    --     wantedfiles[#wantedfiles+1] = filename
1186    -- end
1187    wantedfiles[#wantedfiles+1] = filename
1188    if askedformat == "" then
1189        if filesuffix == "" or not suffixmap[filesuffix] then
1190            local defaultsuffixes = resolvers.defaultsuffixes
1191            for i=1,#defaultsuffixes do
1192                local forcedname = filename .. "." .. defaultsuffixes[i]
1193                wantedfiles[#wantedfiles+1] = forcedname
1194                filetype = formatofsuffix(forcedname)
1195                if trace_locating then
1196                    report_resolving("forcing filetype %a",filetype)
1197                end
1198            end
1199        else
1200            filetype = formatofsuffix(filename)
1201            if trace_locating then
1202                report_resolving("using suffix based filetype %a",filetype)
1203            end
1204        end
1205    else
1206        if filesuffix == "" or not suffixmap[filesuffix] then
1207            local format_suffixes = suffixes[askedformat]
1208            if format_suffixes then
1209                for i=1,#format_suffixes do
1210                    wantedfiles[#wantedfiles+1] = filename .. "." .. format_suffixes[i]
1211                end
1212            end
1213        end
1214        filetype = askedformat
1215        if trace_locating then
1216            report_resolving("using given filetype %a",filetype)
1217        end
1218    end
1219    return filetype, wantedfiles
1220end
1221
1222local function find_direct(filename,allresults)
1223    if not dangerous[askedformat] and isreadable(filename) then
1224        if trace_details then
1225            report_resolving("file %a found directly",filename)
1226        end
1227        return "direct", { filename }
1228    end
1229end
1230
1231local function find_wildcard(filename,allresults)
1232    if find(filename,'*',1,true) then
1233        if trace_locating then
1234            report_resolving("checking wildcard %a", filename)
1235        end
1236        local result = resolvers.findwildcardfiles(filename)
1237        if result then
1238            return "wildcard", result
1239        end
1240    end
1241end
1242
1243local function find_qualified(filename,allresults,askedformat,alsostripped) -- this one will be split too
1244    if not is_qualified_path(filename) then
1245        return
1246    end
1247    if trace_locating then
1248        report_resolving("checking qualified name %a", filename)
1249    end
1250    if isreadable(filename) then
1251        if trace_details then
1252            report_resolving("qualified file %a found", filename)
1253        end
1254        return "qualified", { filename }
1255    end
1256    if trace_details then
1257        report_resolving("locating qualified file %a", filename)
1258    end
1259    local forcedname, suffix = "", suffixonly(filename)
1260    if suffix == "" then -- why
1261        local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat]
1262        if format_suffixes then
1263            for i=1,#format_suffixes do
1264                local suffix = format_suffixes[i]
1265                forcedname   = filename .. "." .. suffix
1266                if isreadable(forcedname) then
1267                    if trace_locating then
1268                        report_resolving("no suffix, forcing format filetype %a", suffix)
1269                    end
1270                    return "qualified", { forcedname }
1271                end
1272            end
1273        end
1274    end
1275    if alsostripped and suffix and suffix ~= "" then
1276        -- try to find in tree (no suffix manipulation), here we search for the
1277        -- matching last part of the name
1278        local basename    = filebasename(filename)
1279        local pattern     = lpegmatch(preparetreepattern,filename)
1280        local savedformat = askedformat
1281        local format      = savedformat or ""
1282        if format == "" then
1283            askedformat = formatofsuffix(suffix)
1284        end
1285        if not format then
1286            askedformat = "othertextfiles" -- kind of everything, maybe all
1287        end
1288        --
1289        -- is this really what we want? basename if we have an explicit path?
1290        --
1291        if basename ~= filename then
1292            local resolved = collect_instance_files(basename,askedformat,allresults)
1293            if #resolved == 0 then
1294                local lowered = lower(basename)
1295                if filename ~= lowered then
1296                    resolved = collect_instance_files(lowered,askedformat,allresults)
1297                end
1298            end
1299            resolvers.format = savedformat
1300            --
1301            if #resolved > 0 then
1302                local result = { }
1303                for r=1,#resolved do
1304                    local rr = resolved[r]
1305                    if find(rr,pattern) then
1306                        result[#result+1] = rr
1307                    end
1308                end
1309                if #result > 0 then
1310                    return "qualified", result
1311                end
1312            end
1313        end
1314        -- a real wildcard:
1315        --
1316        -- local filelist = collect_files({basename})
1317        -- result = { }
1318        -- for f=1,#filelist do
1319        --     local ff = filelist[f][3] or ""
1320        --     if find(ff,pattern) then
1321        --         result[#result+1], ok = ff, true
1322        --     end
1323        -- end
1324        -- if #result > 0 then
1325        --     return "qualified", result
1326        -- end
1327    end
1328end
1329
1330local function check_subpath(fname)
1331    if isreadable(fname) then
1332        if trace_details then
1333            report_resolving("found %a by deep scanning",fname)
1334        end
1335        return fname
1336    end
1337end
1338
1339-- this caching is not really needed (seldom accessed) but more readable
1340-- we could probably move some to a higher level but then we need to adapt
1341-- more code ... maybe some day
1342
1343local function makepathlist(list,filetype)
1344    local typespec = resolvers.variableofformat(filetype)
1345    local pathlist = expandedpathlist(typespec,filetype and usertypes[filetype]) -- only extra path with user files
1346    local entry    = { }
1347    if pathlist and #pathlist > 0 then
1348        for k=1,#pathlist do
1349            local path       = pathlist[k]
1350            local prescanned = find(path,'^!!')
1351            local resursive  = find(path,'//$')
1352            local pathname   = lpegmatch(inhibitstripper,path)
1353            local expression = makepathexpression(pathname)
1354            local barename   = gsub(pathname,"/+$","")
1355            barename         = resolveprefix(barename)
1356            local scheme     = urlhasscheme(barename)
1357            local schemename = gsub(barename,"%.%*$",'') -- after scheme
1358         -- local prescanned = path ~= pathname -- ^!!
1359         -- local resursive  = find(pathname,'//$')
1360            entry[k] = {
1361                path       = path,
1362                pathname   = pathname,
1363                prescanned = prescanned,
1364                recursive  = recursive,
1365                expression = expression,
1366                barename   = barename,
1367                scheme     = scheme,
1368                schemename = schemename,
1369            }
1370        end
1371        entry.typespec = typespec
1372        list[filetype] = entry
1373    else
1374        list[filetype] = false
1375    end
1376    return entry
1377end
1378
1379-- pathlist : resolved
1380-- dirlist  : unresolved or resolved
1381-- filelist : unresolved
1382
1383local function find_intree(filename,filetype,wantedfiles,allresults)
1384    local pathlists = instance.pathlists
1385    if not pathlists then
1386        pathlists = setmetatableindex({ },makepathlist)
1387        instance.pathlists = pathlists
1388    end
1389    local pathlist = pathlists[filetype]
1390    if pathlist then
1391        -- list search
1392        local method   = "intree"
1393        local filelist = collect_files(wantedfiles) -- okay, a bit over the top when we just look relative to the current path
1394        local dirlist  = { }
1395        local result   = { }
1396        if filelist then
1397            for i=1,#filelist do
1398                dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble
1399            end
1400        end
1401        if trace_details then
1402            report_resolving("checking filename %a in tree",filename)
1403        end
1404        for k=1,#pathlist do
1405            local entry    = pathlist[k]
1406            local path     = entry.path
1407            local pathname = entry.pathname
1408            local done     = false
1409            -- using file list
1410            if filelist then -- database
1411                -- compare list entries with permitted pattern -- /xx /xx//
1412                local expression = entry.expression
1413                if trace_details then
1414                    report_resolving("using pattern %a for path %a",expression,pathname)
1415                end
1416                for k=1,#filelist do
1417                    local fl = filelist[k]
1418                    local f  = fl[2]
1419                    local d  = dirlist[k]
1420                    -- resolve is new:
1421                    if find(d,expression) or find(resolveprefix(d),expression) then
1422                        -- todo, test for readable
1423                        result[#result+1] = resolveprefix(fl[3]) -- no shortcut
1424                        done = true
1425                        if allresults then
1426                            if trace_details then
1427                                report_resolving("match to %a in hash for file %a and path %a, continue scanning",expression,f,d)
1428                            end
1429                        else
1430                            if trace_details then
1431                                report_resolving("match to %a in hash for file %a and path %a, quit scanning",expression,f,d)
1432                            end
1433                            break
1434                        end
1435                    elseif trace_details then
1436                        report_resolving("no match to %a in hash for file %a and path %a",expression,f,d)
1437                    end
1438                end
1439            end
1440            if done then
1441                method = "database"
1442            else
1443                -- beware: we don't honor allresults here in a next attempt (done false)
1444                -- but that is kind of special anyway
1445                method       = "filesystem" -- bonus, even when !! is specified
1446                local scheme = entry.scheme
1447                if not scheme or scheme == "file" then
1448                    local pname = entry.schemename
1449                    if not find(pname,"*",1,true) then
1450                        if can_be_dir(pname) then
1451                            -- hm, rather useless as we don't go deeper and if we would we could also
1452                            -- auto generate the file database .. however, we need this for extra paths
1453                            -- that are not hashed (like sources on my machine) .. so, this is slightly
1454                            -- out of order but at least fast (and we seldom end up here, only when a file
1455                            -- is not already found
1456                            if not done and not entry.prescanned then
1457                                if trace_details then
1458                                    report_resolving("quick root scan for %a",pname)
1459                                end
1460                                for k=1,#wantedfiles do
1461                                    local w = wantedfiles[k]
1462                                    local fname = check_subpath(filejoin(pname,w))
1463                                    if fname then
1464                                        result[#result+1] = fname
1465                                        done = true
1466                                        if not allresults then
1467                                            break
1468                                        end
1469                                    end
1470                                end
1471                                if not done and entry.recursive then -- maybe also when allresults
1472                                    -- collect files in path (and cache the result)
1473                                    if trace_details then
1474                                        report_resolving("scanning filesystem for %a",pname)
1475                                    end
1476                                    local files = resolvers.simplescanfiles(pname,false,true)
1477                                    for k=1,#wantedfiles do
1478                                        local w = wantedfiles[k]
1479                                        local subpath = files[w]
1480                                        if not subpath or subpath == "" then
1481                                            -- rootscan already done
1482                                        elseif type(subpath) == "string" then
1483                                            local fname = check_subpath(filejoin(pname,subpath,w))
1484                                            if fname then
1485                                                result[#result+1] = fname
1486                                                done = true
1487                                                if not allresults then
1488                                                    break
1489                                                end
1490                                            end
1491                                        else
1492                                            for i=1,#subpath do
1493                                                local sp = subpath[i]
1494                                                if sp == "" then
1495                                                    -- roottest already done
1496                                                else
1497                                                    local fname = check_subpath(filejoin(pname,sp,w))
1498                                                    if fname then
1499                                                        result[#result+1] = fname
1500                                                        done = true
1501                                                        if not allresults then
1502                                                            break
1503                                                        end
1504                                                    end
1505                                                end
1506                                            end
1507                                            if done and not allresults then
1508                                                break
1509                                            end
1510                                        end
1511                                    end
1512                                end
1513                            end
1514                        end
1515                    else
1516                        -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
1517                    end
1518                else
1519                    -- we can have extra_paths that are urls
1520                    for k=1,#wantedfiles do
1521                        -- independent url scanner
1522                        local pname = entry.barename
1523                        local fname = methodhandler('finders',pname .. "/" .. wantedfiles[k])
1524                        if fname then
1525                            result[#result+1] = fname
1526                            done = true
1527                            if not allresults then
1528                                break
1529                            end
1530                        end
1531                    end
1532                end
1533            end
1534            -- todo recursive scanning
1535            if done and not allresults then
1536                break
1537            end
1538        end
1539        if #result > 0 then
1540            return method, result
1541        end
1542    end
1543end
1544
1545local function find_onpath(filename,filetype,wantedfiles,allresults)
1546    if trace_details then
1547        report_resolving("checking filename %a, filetype %a, wanted files %a",filename,filetype,concat(wantedfiles," | "))
1548    end
1549    local result = { }
1550    for k=1,#wantedfiles do
1551        local fname = wantedfiles[k]
1552        if fname and isreadable(fname) then
1553            filename = fname
1554            result[#result+1] = filejoin('.',fname)
1555            if not allresults then
1556                break
1557            end
1558        end
1559    end
1560    if #result > 0 then
1561        return "onpath", result
1562    end
1563end
1564
1565local function find_otherwise(filename,filetype,wantedfiles,allresults) -- other text files | any | whatever
1566    local filelist = collect_files(wantedfiles)
1567    local fl = filelist and filelist[1]
1568    if fl then
1569        return "otherwise", { resolveprefix(fl[3]) } -- filename
1570    end
1571end
1572
1573-- we could have a loop over the 6 functions but then we'd have to
1574-- always analyze .. todo: use url split
1575
1576collect_instance_files = function(filename,askedformat,allresults) -- uses nested
1577    if not filename or filename == "" then
1578        return { }
1579    end
1580    askedformat = askedformat or ""
1581    filename    = collapsepath(filename,".")
1582    filename    = gsub(filename,"^%./",getcurrentdir().."/") -- we will merge dir.expandname and collapse some day
1583    if allresults then
1584        -- no need for caching, only used for tracing
1585        local filetype, wantedfiles = find_analyze(filename,askedformat)
1586        local results = {
1587            { find_direct   (filename,true) },
1588            { find_wildcard (filename,true) },
1589            { find_qualified(filename,true,askedformat) }, -- we can add ,true if we want to find dups
1590            { find_intree   (filename,filetype,wantedfiles,true) },
1591            { find_onpath   (filename,filetype,wantedfiles,true) },
1592            { find_otherwise(filename,filetype,wantedfiles,true) },
1593        }
1594        local result = { }
1595        local status = { }
1596        local done   = { }
1597--         for k, r in next, results do
1598        for k=1,#results do
1599            local r = results[k]
1600            local method, list = r[1], r[2]
1601            if method and list then
1602                for i=1,#list do
1603                    local c = collapsepath(list[i])
1604                    if not done[c] then
1605                        result[#result+1] = c
1606                        done[c] = true
1607                    end
1608                    status[#status+1] = formatters["%-10s: %s"](method,c)
1609                end
1610            end
1611        end
1612        if trace_details then
1613            report_resolving("lookup status: %s",table.serialize(status,filename))
1614        end
1615        return result, status
1616    else
1617        local method, result, stamp, filetype, wantedfiles
1618        if instance.remember then
1619            if askedformat == "" then
1620                stamp = formatters["%s::%s"](suffixonly(filename),filename)
1621            else
1622                stamp = formatters["%s::%s"](askedformat,filename)
1623            end
1624            result = stamp and instance.found[stamp]
1625            if result then
1626                if trace_locating then
1627                    report_resolving("remembered file %a",filename)
1628                end
1629                return result
1630            end
1631        end
1632        method, result = find_direct(filename)
1633        if not result then
1634            method, result = find_wildcard(filename)
1635            if not result then
1636                method, result = find_qualified(filename,false,askedformat)
1637                if not result then
1638                    filetype, wantedfiles = find_analyze(filename,askedformat)
1639                    method, result = find_intree(filename,filetype,wantedfiles)
1640                    if not result then
1641                        method, result = find_onpath(filename,filetype,wantedfiles)
1642                        if resolve_otherwise and not result then
1643                            -- this will search everywhere in the tree
1644                            method, result = find_otherwise(filename,filetype,wantedfiles)
1645                        end
1646                    end
1647                end
1648            end
1649        end
1650        if result and #result > 0 then
1651            local foundname = collapsepath(result[1])
1652            resolvers.registerintrees(filename,askedformat,filetype,method,foundname)
1653            result = { foundname }
1654        else
1655            result = { } -- maybe false
1656        end
1657        if stamp then
1658            if trace_locating then
1659                report_resolving("remembering file %a using hash %a",filename,stamp)
1660            end
1661            instance.found[stamp] = result
1662        end
1663        return result
1664    end
1665end
1666
1667-- -- -- end of main file search routing -- -- --
1668
1669local function findfiles(filename,filetype,allresults)
1670    if not filename or filename == "" then
1671        return { }
1672    end
1673    if allresults == nil then
1674        allresults = true
1675    end
1676    local result, status = collect_instance_files(filename,filetype or "",allresults)
1677    if not result or #result == 0 then
1678        local lowered = lower(filename)
1679        if filename ~= lowered then
1680            result, status = collect_instance_files(lowered,filetype or "",allresults)
1681        end
1682    end
1683    return result or { }, status
1684end
1685
1686local function findfile(filename,filetype)
1687    if not filename or filename == "" then
1688        return ""
1689    else
1690        return findfiles(filename,filetype,false)[1] or ""
1691    end
1692end
1693
1694resolvers.findfiles  = findfiles
1695resolvers.findfile   = findfile
1696
1697resolvers.find_file  = findfile  -- obsolete
1698resolvers.find_files = findfiles -- obsolete
1699
1700function resolvers.findpath(filename,filetype)
1701    return filedirname(findfiles(filename,filetype,false)[1] or "")
1702end
1703
1704local function findgivenfiles(filename,allresults)
1705    local hashes = instance.hashes
1706    local files  = instance.files
1707    local base   = filebasename(filename)
1708    local result = { }
1709    --
1710    local function okay(hash,path,name)
1711        local found = methodhandler('concatinators',hash.type,hash.name,path,name)
1712        if found and found ~= "" then
1713            result[#result+1] = resolveprefix(found)
1714            return not allresults
1715        end
1716    end
1717    --
1718    for k=1,#hashes do
1719        local hash    = hashes[k]
1720        local content = files[hash.name]
1721        if content then
1722            local path, name = lookup(content,base)
1723            if not path then
1724                -- no match
1725            elseif type(path) == "string" then
1726                if okay(hash,path,name) then
1727                    return result
1728                end
1729            else
1730                for i=1,#path do
1731                    if okay(hash,path[i],name) then
1732                        return result
1733                    end
1734                end
1735            end
1736        end
1737    end
1738    --
1739    return result
1740end
1741
1742function resolvers.findgivenfiles(filename)
1743    return findgivenfiles(filename,true)
1744end
1745
1746function resolvers.findgivenfile(filename)
1747    return findgivenfiles(filename,false)[1] or ""
1748end
1749
1750local makewildcard = Cs(
1751    (P("^")^0 * P("/") * P(-1) + P(-1)) /".*"
1752  + (P("^")^0 * P("/") / "")^0 * (P("*")/".*" + P("-")/"%%-" + P(".")/"%%." + P("?")/"."+ P("\\")/"/" + P(1))^0
1753)
1754
1755function resolvers.wildcardpattern(pattern)
1756    return lpegmatch(makewildcard,pattern) or pattern
1757end
1758
1759-- we use more function calls than before but we also have smaller trees so
1760-- why bother
1761
1762local function findwildcardfiles(filename,allresults,result)
1763    local files  = instance.files
1764    local hashes = instance.hashes
1765    --
1766    local result = result or { }
1767    local base   = filebasename(filename)
1768    local dirn   = filedirname(filename)
1769    local path   = lower(lpegmatch(makewildcard,dirn) or dirn)
1770    local name   = lower(lpegmatch(makewildcard,base) or base)
1771    --
1772    if find(name,"*",1,true) then
1773        local function okay(found,path,base,hashname,hashtype)
1774            if find(found,path) then
1775                local full = methodhandler('concatinators',hashtype,hashname,found,base)
1776                if full and full ~= "" then
1777                    result[#result+1] = resolveprefix(full)
1778                    return not allresults
1779                end
1780            end
1781        end
1782        for k=1,#hashes do
1783            local hash     = hashes[k]
1784            local hashname = hash.name
1785            local hashtype = hash.type
1786            if hashname and hashtype then
1787                for found, base in filtered(files[hashname],name) do
1788                    if type(found) == 'string' then
1789                        if okay(found,path,base,hashname,hashtype) then
1790                            break
1791                        end
1792                    else
1793                        for i=1,#found do
1794                            if okay(found[i],path,base,hashname,hashtype) then
1795                                break
1796                            end
1797                        end
1798                    end
1799                end
1800            end
1801        end
1802    else
1803        local function okayokay(found,path,base,hashname,hashtype)
1804            if find(found,path) then
1805                local full = methodhandler('concatinators',hashtype,hashname,found,base)
1806                if full and full ~= "" then
1807                    result[#result+1] = resolveprefix(full)
1808                    return not allresults
1809                end
1810            end
1811        end
1812        --
1813        for k=1,#hashes do
1814            local hash     = hashes[k]
1815            local hashname = hash.name
1816            local hashtype = hash.type
1817            if hashname and hashtype then
1818                local found, base = lookup(content,base)
1819                if not found then
1820                    -- nothing
1821                elseif type(found) == 'string' then
1822                    if okay(found,path,base,hashname,hashtype) then
1823                        break
1824                    end
1825                else
1826                    for i=1,#found do
1827                        if okay(found[i],path,base,hashname,hashtype) then
1828                            break
1829                        end
1830                    end
1831                end
1832            end
1833        end
1834    end
1835    -- we can consider also searching the paths not in the database, but then
1836    -- we end up with a messy search (all // in all path specs)
1837    return result
1838end
1839
1840function resolvers.findwildcardfiles(filename,result)
1841    return findwildcardfiles(filename,true,result)
1842end
1843
1844function resolvers.findwildcardfile(filename)
1845    return findwildcardfiles(filename,false)[1] or ""
1846end
1847
1848do
1849
1850    local starttiming = statistics.starttiming
1851    local stoptiming  = statistics.stoptiming
1852    local elapsedtime = statistics.elapsedtime
1853
1854    function resolvers.starttiming()
1855        starttiming(instance)
1856    end
1857
1858    function resolvers.stoptiming()
1859        stoptiming(instance)
1860    end
1861
1862    function resolvers.loadtime()
1863        return elapsedtime(instance)
1864    end
1865
1866end
1867
1868-- main user functions
1869
1870function resolvers.automount()
1871    -- implemented later (maybe a one-time setter like textopener)
1872end
1873
1874function resolvers.load(option)
1875    resolvers.starttiming()
1876    identify_configuration_files()
1877    load_configuration_files()
1878    if option ~= "nofiles" then
1879        load_databases()
1880        resolvers.automount()
1881    end
1882    resolvers.stoptiming()
1883    local files = instance.files
1884    return files and next(files) and true
1885end
1886
1887local function report(str)
1888    if trace_locating then
1889        report_resolving(str) -- has already verbose
1890    else
1891        print(str)
1892    end
1893end
1894
1895function resolvers.dowithfilesandreport(command, files, ...) -- will move
1896    if files and #files > 0 then
1897        if trace_locating then
1898            report('') -- ?
1899        end
1900        if type(files) == "string" then
1901            files = { files }
1902        end
1903        for f=1,#files do
1904            local file = files[f]
1905            local result = command(file,...)
1906            if type(result) == 'string' then
1907                report(result)
1908            else
1909                for i=1,#result do
1910                    report(result[i]) -- could be unpack
1911                end
1912            end
1913        end
1914    end
1915end
1916
1917-- resolvers.varvalue  = resolvers.variable   -- output the value of variable $STRING.
1918-- resolvers.expandvar = expansion  -- output variable expansion of STRING.
1919
1920function resolvers.showpath(str)     -- output search path for file type NAME
1921    return joinpath(expandedpathlist(resolvers.formatofvariable(str)))
1922end
1923
1924function resolvers.registerfile(files, name, path)
1925    if files[name] then
1926        if type(files[name]) == 'string' then
1927            files[name] = { files[name], path }
1928        else
1929            files[name] = path
1930        end
1931    else
1932        files[name] = path
1933    end
1934end
1935
1936function resolvers.dowithpath(name,func)
1937    local pathlist = expandedpathlist(name)
1938    for i=1,#pathlist do
1939        func("^"..cleanpath(pathlist[i]))
1940    end
1941end
1942
1943function resolvers.dowithvariable(name,func)
1944    func(expandedvariable(name))
1945end
1946
1947function resolvers.locateformat(name)
1948    local engine   = environment.ownmain or "luatex"
1949    local barename = removesuffix(file.basename(name))
1950    local fullname = addsuffix(barename,"fmt")
1951    local fmtname  = caches.getfirstreadablefile(fullname,"formats",engine) or ""
1952    if fmtname == "" then
1953        fmtname = findfile(fullname)
1954        fmtname = cleanpath(fmtname)
1955    end
1956    if fmtname ~= "" then
1957        local barename = removesuffix(fmtname)
1958        local luaname = addsuffix(barename,luasuffixes.lua)
1959        local lucname = addsuffix(barename,luasuffixes.luc)
1960        local luiname = addsuffix(barename,luasuffixes.lui)
1961        if isfile(luiname) then
1962            return fmtname, luiname
1963        elseif isfile(lucname) then
1964            return fmtname, lucname
1965        elseif isfile(luaname) then
1966            return fmtname, luaname
1967        end
1968    end
1969    return nil, nil
1970end
1971
1972function resolvers.booleanvariable(str,default)
1973    local b = expansion(str)
1974    if b == "" then
1975        return default
1976    else
1977        b = toboolean(b)
1978        return (b == nil and default) or b
1979    end
1980end
1981
1982function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, can be a nice iterator instead
1983    local hashes = instance.hashes
1984    local files  = instance.files
1985    for i=1,#hashes do
1986        local hash     = hashes[i]
1987        local blobtype = hash.type
1988        local blobpath = hash.name
1989        if blobtype and blobpath then
1990            local total   = 0
1991            local checked = 0
1992            local done    = 0
1993            if before then
1994                before(blobtype,blobpath,pattern)
1995            end
1996            for path, name in filtered(files[blobpath],pattern) do
1997                if type(path) == "string" then
1998                    checked = checked + 1
1999                    if handle(blobtype,blobpath,path,name) then
2000                        done = done + 1
2001                    end
2002                else
2003                    checked = checked + #path
2004                    for i=1,#path do
2005                        if handle(blobtype,blobpath,path[i],name) then
2006                            done = done + 1
2007                        end
2008                    end
2009                end
2010            end
2011            if after then
2012                after(blobtype,blobpath,pattern,checked,done)
2013            end
2014        end
2015    end
2016end
2017
2018-- moved here
2019
2020function resolvers.knownvariables(pattern)
2021    if instance then
2022        local environment = instance.environment
2023        local variables   = instance.variables
2024        local expansions  = instance.expansions
2025        local order       = instance.order
2026        local pattern     = upper(pattern or "")
2027        local result      = { }
2028        for i=1,#order do
2029            for key in next, order[i] do
2030                if result[key] == nil and key ~= "" and (pattern == "" or find(upper(key),pattern)) then
2031                    result[key] = {
2032                        environment = rawget(environment,key),
2033                        variable    = key,
2034                        expansion   = expansions[key],
2035                        resolved    = resolveprefix(expansions[key]),
2036                    }
2037                end
2038            end
2039        end
2040        return result
2041    else
2042        return { }
2043    end
2044end
2045