SourceBrowser

strc-ref.lmt /size: 102 Kb last modification: 2024-01-16 10:22
1if not modules then modules = { } end modules ['strc-ref'] = {
2    version   = 1.001,
3    comment   = "companion to strc-ref.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- beware, this is a first step in the rewrite (just getting rid of
10-- the tuo file); later all access and parsing will also move to lua
11
12-- the useddata and pagedata names might change
13-- todo: pack exported data
14
15-- todo: autoload components when :::
16
17local format, gmatch, match, strip = string.format, string.gmatch, string.match, string.strip
18local floor = math.floor
19local rawget, tonumber, type, next = rawget, tonumber, type, next
20local lpegmatch = lpeg.match
21local insert, remove, copytable, sortedhash = table.insert, table.remove, table.copy, table.sortedhash
22local formatters = string.formatters
23local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match
24
25local allocate           = utilities.storage.allocate
26local mark               = utilities.storage.mark
27local setmetatableindex  = table.setmetatableindex
28
29local trace_referencing  = false  trackers.register("structures.referencing",             function(v) trace_referencing = v end)
30local trace_analyzing    = false  trackers.register("structures.referencing.analyzing",   function(v) trace_analyzing   = v end)
31local trace_identifying  = false  trackers.register("structures.referencing.identifying", function(v) trace_identifying = v end)
32local trace_importing    = false  trackers.register("structures.referencing.importing",   function(v) trace_importing   = v end)
33local trace_external     = false  trackers.register("structures.referencing.external",    function(v) trace_external    = v end)
34local trace_empty        = false  trackers.register("structures.referencing.empty",       function(v) trace_empty       = v end)
35
36local check_duplicates   = true
37
38directives.register("structures.referencing.checkduplicates", function(v) check_duplicates = v end)
39
40local report_references  = logs.reporter("references")
41local report_identifying = logs.reporter("references","identifying")
42local report_importing   = logs.reporter("references","importing")
43local report_external    = logs.reporter("references","external")
44local report_empty       = logs.reporter("references","empty")
45local report             = report_references
46
47local variables          = interfaces.variables
48local v_page             = variables.page
49local v_auto             = variables.auto
50local v_yes              = variables.yes
51local v_name             = variables.name
52
53local context            = context
54local commands           = commands
55local implement          = interfaces.implement
56
57local ctx_latelua        = context.latelua
58local ctx_doifelse       = commands.doifelse
59
60local texiscount         = tex.iscount
61local texgetcount        = tex.getcount
62local texsetcount        = tex.setcount
63local texconditionals    = tex.conditionals
64
65local getexpansion       = token.getexpansion
66
67local productcomponent   = resolvers.jobs.productcomponent
68local justacomponent     = resolvers.jobs.justacomponent
69
70local settings_to_array  = utilities.parsers.settings_to_array
71local settings_to_table  = utilities.parsers.settings_to_array_obey_fences
72local process_settings   = utilities.parsers.process_stripped_settings
73local unsetvalue         = attributes.unsetvalue
74
75local structures         = structures
76local helpers            = structures.helpers
77local sections           = structures.sections
78local references         = structures.references
79local lists              = structures.lists
80local counters           = structures.counters
81
82local jobpositions       = job.positions
83local getpos             = jobpositions.getpos
84
85-- some might become local
86
87references.defined       = references.defined or allocate()
88
89local defined            = references.defined
90local derived            = allocate()
91local specials           = allocate()
92local functions          = allocate()
93local runners            = allocate()
94local internals          = allocate()
95local filters            = allocate()
96local executers          = allocate()
97local handlers           = allocate()
98local tobesaved          = allocate()
99local collected          = allocate()
100local tobereferred       = allocate()
101local referred           = allocate()
102local usedinternals      = allocate()
103local flaginternals      = allocate()
104local usedviews          = allocate()
105
106references.derived       = derived
107references.specials      = specials
108references.functions     = functions
109references.runners       = runners
110references.internals     = internals
111references.filters       = filters
112references.executers     = executers
113references.handlers      = handlers
114references.tobesaved     = tobesaved
115references.collected     = collected
116references.tobereferred  = tobereferred
117references.referred      = referred
118references.usedinternals = usedinternals
119references.flaginternals = flaginternals
120references.usedviews     = usedviews
121
122local splitreference     = references.splitreference
123local splitprefix        = references.splitcomponent -- replaces: references.splitprefix
124local prefixsplitter     = references.prefixsplitter
125local componentsplitter  = references.componentsplitter
126
127local currentreference   = nil
128
129local txtcatcodes        = catcodes.numbers.txtcatcodes -- or just use "txtcatcodes"
130
131local c_realpageno               = texiscount("realpageno")
132local c_locationcount            = texiscount("locationcount")
133local c_locationorder            = texiscount("locationorder")
134local c_lastdestinationattribute = texiscount("lastdestinationattribute")
135
136local p_splitter = lpeg.splitat(":")
137local p_lower    = lpeg.patterns.utf8lower
138
139local context                      = context
140
141local ctx_pushcatcodes             = context.pushcatcodes
142local ctx_popcatcodes              = context.popcatcodes
143local ctx_dofinishreference        = context.dofinishreference
144local ctx_dofromurldescription     = context.dofromurldescription
145local ctx_dofromurlliteral         = context.dofromurlliteral
146local ctx_dofromfiledescription    = context.dofromfiledescription
147local ctx_dofromfileliteral        = context.dofromfileliteral
148local ctx_expandreferenceoperation = context.expandreferenceoperation
149local ctx_expandreferencearguments = context.expandreferencearguments
150local ctx_convertnumber            = context.convertnumber
151local ctx_emptyreference           = context.emptyreference
152
153storage.register("structures/references/defined", references.defined, "structures.references.defined")
154
155local initializers = { }
156local finalizers   = { }
157local somefound    = false -- so we don't report missing when we have a fresh start
158
159function references.registerinitializer(func) -- we could use a token register instead
160    initializers[#initializers+1] = func
161end
162
163function references.registerfinalizer(func) -- we could use a token register instead
164    finalizers[#finalizers+1] = func
165end
166
167local function initializer() -- can we use a tobesaved as metatable for collected?
168    tobesaved = references.tobesaved
169    collected = references.collected
170    for i=1,#initializers do
171        initializers[i](tobesaved,collected)
172    end
173    for prefix, list in next, collected do
174        for tag, data in next, list do
175            local r = data.references
176            local i = r.internal
177            if i then
178                internals[i]     = data
179                usedinternals[i] = r.used
180                local structure = r.structure
181                if structure then
182                    structure = resolvers.jobs.namelist(structure,namestack)
183                    r.namestack = structure
184                end
185            end
186        end
187    end
188    somefound = next(collected)
189end
190
191local function finalizer()
192    for i=1,#finalizers do
193        finalizers[i](tobesaved)
194    end
195    for prefix, list in next, tobesaved do
196        for tag, data in next, list do
197            local r = data.references
198            local i = r.internal
199            local f = flaginternals[i]
200            if f then
201                r.used = usedviews[i] or true
202            end
203        end
204    end
205end
206
207job.register('structures.references.collected', tobesaved, initializer, finalizer)
208
209local maxreferred = 1
210local nofreferred = 0
211
212local function initializer() -- can we use a tobesaved as metatable for collected?
213    tobereferred = references.tobereferred
214    referred     = references.referred
215    nofreferred  = #referred
216end
217
218-- no longer done this way
219
220-- references.resolvers = references.resolvers or { }
221-- local resolvers = references.resolvers
222--
223-- function resolvers.section(var)
224--     local vi = lists.collected[var.i[2]]
225--     if vi then
226--         var.i = vi
227--         var.r = (vi.references and vi.references.realpage) or (vi.pagedata and vi.pagedata.realpage) or 1
228--     else
229--         var.i = nil
230--         var.r = 1
231--     end
232-- end
233--
234-- resolvers.float       = resolvers.section
235-- resolvers.description = resolvers.section
236-- resolvers.formula     = resolvers.section
237-- resolvers.note        = resolvers.section
238--
239-- function resolvers.reference(var)
240--     local vi = var.i[2]
241--     if vi then
242--         var.i = vi
243--         var.r = (vi.references and vi.references.realpage) or (vi.pagedata and vi.pagedata.realpage) or 1
244--     else
245--         var.i = nil
246--         var.r = 1
247--     end
248-- end
249
250-- We make the array sparse (maybe a finalizer should optionally return a table) because
251-- there can be quite some page links involved. We only store one action number per page
252-- which is normally good enough for what we want (e.g. see above/below) and we do
253-- a combination of a binary search and traverse backwards. A previous implementation
254-- always did a traverse and was pretty slow on a large number of links (given that this
255-- methods was used). It took me about a day to locate this as a bottleneck in processing
256-- a 2500 page interactive document with 60 links per page. In that case, traversing
257-- thousands of slots per link then brings processing to a grinding halt (especially when
258-- there are no slots at all, which is the case in a first run).
259
260local sparsetobereferred = { }
261
262local function finalizer()
263    local lastr, lasti
264    local n = 0
265    for i=1,maxreferred do
266        local r = tobereferred[i]
267        if not lastr then
268            lastr = r
269            lasti = i
270        elseif r ~= lastr then
271            n = n + 1
272            sparsetobereferred[n] = { lastr, lasti }
273            lastr = r
274            lasti = i
275        end
276    end
277    if lastr then
278        n = n + 1
279        sparsetobereferred[n] = { lastr, lasti }
280    end
281end
282
283job.register('structures.references.referred', sparsetobereferred, initializer, finalizer)
284
285local function referredpage(n)
286    local max = nofreferred
287    if max > 0 then
288        -- find match
289        local min = 1
290        while true do
291            local mid = floor((min+max)/2)
292            local r = referred[mid]
293            local m = r[2]
294            if n == m then
295                return r[1]
296            elseif n > m then
297                min = mid + 1
298            else
299                max = mid - 1
300            end
301            if min > max then
302                break
303            end
304        end
305        -- find first previous
306        for i=min,1,-1 do
307            local r = referred[i]
308            if r and r[2] < n then
309                return r[1]
310            end
311        end
312    end
313    -- fallback
314    return texgetcount(c_realpageno)
315end
316
317references.referredpage = referredpage
318
319function references.registerpage(n) -- called in the backend code
320    if not tobereferred[n] then
321        if n > maxreferred then
322            maxreferred = n
323        end
324        tobereferred[n] = texgetcount(c_realpageno)
325    end
326end
327
328-- todo: delay split till later as in destinations we split anyway
329
330local orders, lastorder = { }, 0
331
332local function setnextorder(kind,name)
333    lastorder = 0
334    if kind and name then
335        local ok = orders[kind]
336        if not ok then
337            ok = { }
338            orders[kind] = ok
339        end
340        lastorder = (ok[name] or 0) + 1
341        ok[name] = lastorder
342    end
343    texsetcount("global",c_locationorder,lastorder)
344end
345
346
347local function setnextinternal(kind,name)
348    setnextorder(kind,name) -- always incremented with internal
349    local n = texgetcount(c_locationcount) + 1
350    texsetcount("global",c_locationcount,n)
351    return n
352end
353
354local function currentorder(kind,name)
355    return orders[kind] and orders[kind][name] or lastorder
356end
357
358-- local function setstructure(data)
359--     local structure = resolvers.jobs.currentnamehash()
360--     if structure then
361--         local references = data and data.references
362--         if references then
363--             references.structure = structure
364--         end
365--     end
366-- end
367
368-- local function setcomponent(data)
369--     -- we might consider doing this at the tex end, just like prefix
370--     local component = productcomponent() -- todo: maybe a list
371--  -- local component = resolvers.jobs.currentcomponent()
372--     if component then
373--         local references = data and data.references
374--         if references then
375--             references.component = component
376--             if references.prefix == component then
377--                 references.prefix = nil
378--             end
379--         end
380--         return component
381--     end
382-- end
383
384references.setnextorder    = setnextorder
385references.setnextinternal = setnextinternal
386references.currentorder    = currentorder
387references.setstructure    = resolvers.jobs.currentstructure -- yes or no here
388----------.setcomponent    = setcomponent
389
390implement {
391    name      = "setnextreferenceorder", -- used in strc-enu
392    actions   = setnextorder,
393    arguments = "2 strings",
394}
395
396-- implement {
397--     name      = "currentreferenceorder",
398--     actions = { currentorder, context },
399--     arguments = "2 strings",
400-- }
401
402implement {
403    name      = "setnextinternalreferences",
404    public    = true,
405    protected = true,
406    actions   = setnextinternal,
407    arguments = "2 arguments",
408}
409
410implement {
411    name      = "getinternalorderreference",
412    public    = true,
413    protected = true,
414    actions   = { currentorder, context },
415    arguments = "2 arguments",
416}
417
418local reported = setmetatableindex("table")
419
420function references.set(data)
421    local references = data.references
422    local reference  = references.reference
423    if not reference or reference == "" then
424     -- report_references("invalid reference") -- harmless
425        return 0
426    end
427    --
428    references.structure = resolvers.jobs.currentstructure()
429    --
430    local prefix = references.prefix or ""
431    local pd     = tobesaved[prefix] -- nicer is a metatable
432    if not pd then
433        pd = { }
434        tobesaved[prefix] = pd
435    end
436    local n = 0
437    local function action(ref)
438        if ref == "" then
439            -- skip
440        elseif check_duplicates and pd[ref] then
441            if not prefix then
442                prefix = ""
443            end
444            if not reported[prefix][ref] then
445                if prefix ~= "" then
446                    report_references("redundant reference %a in namespace %a",ref,prefix)
447                else
448                    report_references("redundant reference %a",ref)
449                end
450                reported[prefix][ref] = true
451            end
452        else
453            n = n + 1
454            pd[ref] = data
455            local r = data.references
456            ctx_dofinishreference(prefix or "",ref or "",r and r.internal or 0)
457         -- ctx_latelua(function() structures.references.enhance(prefix or ref,ref or "") end)
458        end
459    end
460    process_settings(reference,action)
461    return n > 0
462end
463
464-- function references.enhance(prefix,tag)
465--     local l = tobesaved[prefix][tag]
466--     if l then
467--         l.references.realpage = texgetcount(c_realpageno)
468--     end
469-- end
470
471local function synchronizepage(reference) -- non public helper
472    reference.realpage = texgetcount(c_realpageno)
473    if jobpositions.used() then
474        reference.x, reference.y = getpos()
475    end
476end
477
478references.synchronizepage = synchronizepage
479
480local function enhancereference(specification)
481    local prefix = specification.prefix
482    if prefix then
483        local entry = tobesaved[prefix]
484        if entry then
485            entry = entry[specification.tag]
486            if entry then
487                synchronizepage(entry.references)
488            else
489                -- normally a bug
490            end
491        else
492            -- normally a bug
493        end
494    else
495        -- normally a bug
496    end
497end
498
499references.enhance = enhancereference
500
501-- implement {
502--     name      = "enhancereference",
503--     arguments = "2 strings",
504--     actions   = function(prefix,tag)
505--        enhancereference { prefix = prefix, tag = tag }
506--     end,
507-- }
508
509implement {
510    name      = "deferredenhancereference",
511    arguments = "2 strings",
512    protected = true,
513    actions   = function(prefix,tag)
514        ctx_latelua { action = enhancereference, prefix = prefix, tag = tag }
515    end,
516}
517
518-- -- -- related to strc-ini.lua -- -- --
519
520-- no metatable here .. better be sparse
521
522local function register_from_list(collected,derived,external,namestack)
523    local derived_g = derived[""] -- global
524    local derived_p = nil
525    local derived_c = nil
526    local prefix    = nil
527--     local component = nil
528    local entry     = nil
529    if not derived_g then
530        derived_g = { }
531        derived[""] = derived_g
532    end
533    local action = trace_external and
534        function(s)
535            if structure then
536                for i=1,#structure do
537                    local si = structure[i]
538                 -- if si == component then
539                 --     -- skipped
540                 -- elseif si == prefix then
541                 --     -- skipped
542                 -- else
543                        local ds = derived[si]
544                        if not ds then
545                            derived[si] = { [s] = entry }
546                            report_external("reference %a, %s %a",s,"structure",si)
547                        elseif not ds[s] then
548                            ds[s] = entry
549                            report_external("reference %a, %s %a",s,"structure",si)
550                        end
551                 -- end
552                end
553            end
554            if derived_p and not derived_p[s] then
555                report_external("reference %a, %s %a",s,"prefix",prefix)
556                derived_p[s] = entry
557
558            end
559--             if derived_c and not derived_c[s] then
560--                 report_external("reference %a, %s %a",s,"component",component)
561--                 derived_c[s] = entry
562--             end
563            if not derived_g[s] then
564                report_external("reference %a, %s %a",s,"global","")
565                derived_g[s] = entry -- first wins
566            end
567        end
568    or
569        function(s)
570            if derived_p and not derived_p[s] then
571                derived_p[s] = entry
572            end
573--             if derived_c and not derived_c[s] then
574--                 derived_c[s] = entry
575--             end
576            if not derived_g[s] then
577                derived_g[s] = entry -- first wins
578            end
579            if structure then
580                for i=1,#structure do
581                    local si = structure[i]
582                    if si == component then
583                        -- skipped
584                    elseif si == prefix then
585                        -- skipped
586                    else
587                        local ds = derived[si]
588                        if not ds then
589                            derived[si] = { [s] = entry }
590                        elseif not ds[s] then
591                            ds[s] = entry
592                        end
593                    end
594                end
595            end
596        end
597    --
598    for i=1,#collected do
599        entry = collected[i]
600        local metadata = entry.metadata
601        if metadata then
602            local kind = metadata.kind -- why this check
603            if kind then
604                local references = entry.references
605                if references then
606                    local reference = references.reference
607                    if reference and reference ~= "" then
608                        local realpage = references.realpage
609                        if realpage then
610                            prefix    = references.prefix
611--                             component = references.component
612                            if prefix and prefix ~= "" then
613                                derived_p = derived[prefix]
614                                if not derived_p then
615                                    derived_p = { }
616                                    derived[prefix] = derived_p
617                                end
618                            else
619                                derived_p = nil
620                            end
621--                             if component and component ~= "" and component ~= prefix then
622--                                 derived_c = derived[component]
623--                                 if not derived_c then
624--                                     derived_c = { }
625--                                     derived[component] = derived_c
626--                                 end
627--                             else
628--                                 derived_c = nil
629--                             end
630                            structure = references.structure
631                            if structure then
632                                structure = resolvers.jobs.namelist(structure,namestack)
633                                references.namestack = structure
634                            end
635                            if trace_referencing or trace_external then
636                                report_references("list entry %a provides %a reference %a on realpage %a",i,kind,reference,realpage)
637                            end
638                            process_settings(reference,action)
639                        end
640                    end
641                end
642            end
643        end
644    end
645end
646
647function references.integrate(utilitydata)
648    local filename = utilitydata.comment.file
649    if filename and filename ~= environment.jobname then
650        -- lists are already internalized
651        local structures = utilitydata.structures
652        if structures then
653            local lists = structures.lists.collected
654            if lists then
655                register_from_list(lists,derived,filename,utilitydata)
656            end
657        end
658    end
659end
660
661references.registerinitializer(function()
662    -- the main document
663    register_from_list(lists.collected,derived) -- false, false
664end)
665
666-- tracing
667
668local function collectbypage(tracedpages)
669    -- lists
670    do
671        local collected = structures.lists.collected
672        local data      = nil
673        local function action(reference)
674            local prefix    = data.prefix
675            local component = data.component
676            local realpage  = data.realpage
677            if realpage then
678                local pagelist  = rawget(tracedpages,realpage)
679                local internal  = data.internal or 0
680                local prefix    = (prefix ~= "" and prefix) or (component ~= "" and component) or ""
681                local pagedata  = { prefix, reference, internal }
682                if pagelist then
683                    pagelist[#pagelist+1] = pagedata
684                else
685                    tracedpages[realpage] = { pagedata }
686                end
687                if internal > 0 then
688                    data.usedprefix = prefix
689                end
690            end
691        end
692        for i=1,#collected do
693            local entry = collected[i]
694            local metadata = entry.metadata
695            if metadata and metadata.kind then
696                data = entry.references
697                if data then
698                    local reference = data.reference
699                    if reference and reference ~= "" then
700                        process_settings(reference,action)
701                    end
702                end
703            end
704        end
705    end
706    -- references
707    do
708        for prefix, list in next, collected do
709            for reference, entry in next, list do
710                local data = entry.references
711                if data then
712                    local realpage = data.realpage
713                    local internal = data.internal or 0
714                    local pagelist = rawget(tracedpages,realpage)
715                    local pagedata = { prefix, reference, internal }
716                    if pagelist then
717                        pagelist[#pagelist+1] = pagedata
718                    else
719                        tracedpages[realpage] = { pagedata }
720                    end
721                    if internal > 0 then
722                        data.usedprefix = prefix
723                    end
724                end
725            end
726        end
727    end
728end
729
730references.tracedpages = table.setmetatableindex(allocate(),function(t,k)
731    if collectbypage then
732        collectbypage(t)
733        collectbypage = nil
734    end
735    return rawget(t,k)
736end)
737
738-- urls
739
740local urls      = references.urls or { }
741references.urls = urls
742local urldata   = urls.data or { }
743urls.data       = urldata
744
745local p_untexurl = Cs ( (
746    P("\\")/"" * (P("%")/"%%" + P(1))
747  + P(" ")/"%%20"
748  + P(1)
749)^1 )
750
751function urls.untex(url)
752    return lpegmatch(p_untexurl,url) or url
753end
754
755function urls.define(name,url,file,description)
756    if name and name ~= "" then
757     -- url = lpegmatch(replacer,url)
758        urldata[name] = { url or "", file or "", description or url or file or ""}
759    end
760end
761
762function urls.get(name)
763    local u = urldata[name]
764    if u then
765        local url, file = u[1], u[2]
766        if file and file ~= "" then
767            return formatters["%s/%s"](url,file)
768        else
769            return url
770        end
771    end
772end
773
774function urls.found(name)
775    return urldata[name]
776end
777
778local function geturl(name)
779    local url = urls.get(name)
780    if url and url ~= "" then
781        ctx_pushcatcodes(txtcatcodes)
782        context(url)
783        ctx_popcatcodes()
784    end
785end
786
787implement {
788    name      = "doifelseurldefined",
789    actions   = { urls.found, ctx_doifelse },
790    arguments = "string"
791}
792
793implement {
794    name      = "useurl",
795    actions   = urls.define,
796    arguments = "4 strings",
797}
798
799implement {
800    name      = "geturl",
801    actions   = geturl,
802    arguments = "string",
803}
804
805-- files
806
807local files      = references.files or { }
808references.files = files
809local filedata   = files.data or { }
810files.data       = filedata
811
812function files.define(name,file,description)
813    if name and name ~= "" then
814        filedata[name] = { file or "", description or file or "" }
815    end
816end
817
818function files.get(name,method,space) -- method: none, before, after, both, space: yes/no
819    local f = filedata[name]
820    if f then
821        context(f[1])
822    end
823end
824
825function files.found(name)
826    return filedata[name]
827end
828
829local function getfile(name)
830    local fil = files.get(name)
831    if fil and fil ~= "" then
832        ctx_pushcatcodes(txtcatcodes)
833        context(fil)
834        ctx_popcatcodes()
835    end
836end
837
838implement {
839    name      = "doifelsefiledefined",
840    actions   = { files.found, ctx_doifelse },
841    arguments = "string"
842}
843
844implement {
845    name      = "usefile",
846    actions   = files.define,
847    arguments = "3 strings"
848}
849
850implement {
851    name      = "getfile",
852    actions   = getfile,
853    arguments = "string"
854}
855
856-- helpers
857
858function references.checkedfile(whatever) -- return whatever if not resolved
859    if whatever then
860        local w = filedata[whatever]
861        if w then
862            return w[1]
863        else
864            return whatever
865        end
866    end
867end
868
869function references.checkedurl(whatever) -- return whatever if not resolved
870    if whatever then
871        local w = urldata[whatever]
872        if w then
873            local u, f = w[1], w[2]
874            if f and f ~= "" then
875                return u .. "/" .. f
876            else
877                return u
878            end
879        else
880            return whatever
881        end
882    end
883end
884
885function references.checkedfileorurl(whatever,default) -- return nil, nil if not resolved
886    if whatever then
887        local w = filedata[whatever]
888        if w then
889            return w[1], nil
890        else
891            local w = urldata[whatever]
892            if w then
893                local u, f = w[1], w[2]
894                if f and f ~= "" then
895                    return nil, u .. "/" .. f
896                else
897                    return nil, u
898                end
899            end
900        end
901    end
902    return default
903end
904
905-- programs
906
907local programs      = references.programs or { }
908references.programs = programs
909local programdata   = programs.data or { }
910programs.data       = programdata
911
912function programs.define(name,file,description)
913    if name and name ~= "" then
914        programdata[name] = { file or "", description or file or ""}
915    end
916end
917
918function programs.get(name)
919    local f = programdata[name]
920    return f and f[1]
921end
922
923function references.checkedprogram(whatever) -- return whatever if not resolved
924    if whatever then
925        local w = programdata[whatever]
926        if w then
927            return w[1]
928        else
929            return whatever
930        end
931    end
932end
933
934implement {
935    name      = "defineprogram",
936    actions   = programs.define,
937    arguments = "3 strings",
938}
939
940local function getprogram(name)
941    local p = programdata[name]
942    if p then
943        context(p[1])
944    end
945end
946
947implement {
948    name      = "getprogram",
949    actions   = getprogram,
950    arguments = "string"
951}
952
953-- shared by urls and files
954
955function references.from(name)
956    local u = urldata[name]
957    if u then
958        local url, file, description = u[1], u[2], u[3]
959        if description ~= "" then
960            return description
961            -- ok
962        elseif file and file ~= "" then
963            return url .. "/" .. file
964        else
965            return url
966        end
967    else
968        local f = filedata[name]
969        if f then
970            local file, description = f[1], f[2]
971            if description ~= "" then
972                return description
973            else
974                return file
975            end
976        end
977    end
978end
979
980local function from(name)
981    local u = urldata[name]
982    if u then
983        local url, file, description = u[1], u[2], u[3]
984        if description ~= "" then
985            ctx_dofromurldescription(description)
986            -- ok
987        elseif file and file ~= "" then
988            ctx_dofromurlliteral(url .. "/" .. file)
989        else
990            ctx_dofromurlliteral(url)
991        end
992    else
993        local f = filedata[name]
994        if f then
995            local file, description = f[1], f[2]
996            if description ~= "" then
997                ctx_dofromfiledescription(description)
998            else
999                ctx_dofromfileliteral(file)
1000            end
1001        end
1002    end
1003end
1004
1005implement {
1006    name      = "from",
1007    actions   = from,
1008    arguments = "string"
1009}
1010
1011function references.define(prefix,reference,list)
1012    local d = defined[prefix] if not d then d = { } defined[prefix] = d end
1013    d[reference] = list
1014end
1015
1016function references.reset(prefix,reference)
1017    local d = defined[prefix]
1018    if d then
1019        d[reference] = nil
1020    end
1021end
1022
1023implement {
1024    name      = "definereference",
1025    actions   = references.define,
1026    arguments = "3 strings",
1027}
1028
1029implement {
1030    name      = "resetreference",
1031    actions   = references.reset,
1032    arguments = "2 strings",
1033}
1034
1035setmetatableindex(defined,"table")
1036
1037local function resolve(prefix,reference,args,set) -- we start with prefix,reference
1038    if reference and reference ~= "" then
1039        if not set then
1040            set = { prefix = prefix, reference = reference }
1041        else
1042            if not set.reference then set.reference = reference end
1043            if not set.prefix    then set.prefix    = prefix    end
1044        end
1045     -- local r = settings_to_array(reference)
1046        local r = settings_to_table(reference) -- maybe option to honor () []
1047        for i=1,#r do
1048            local ri = r[i]
1049            local d = defined[prefix][ri] or defined[""][ri]
1050            if d then
1051                d = getexpansion(d)
1052                resolve(prefix,d,nil,set)
1053            else
1054                local var = splitreference(ri)
1055                if var then
1056                    var.reference = ri
1057                    local vo = var.outer
1058                    local vi = var.inner
1059                    -- we catch this here .. it's a way to pass references with commas
1060                    if vi == "name" then
1061                        local arguments = var.arguments
1062                        if arguments then
1063                            vi            = arguments
1064                            var.inner     = arguments
1065                            var.reference = arguments
1066                            var.arguments = nil
1067                        end
1068                    elseif var.special == "name" then
1069                        local operation = var.operation
1070                        if operation then
1071                            vi            = operation
1072                            var.inner     = operation
1073                            var.reference = operation
1074                            var.operation = nil
1075                            var.special   = nil
1076                        end
1077                    end
1078                    -- end of catch
1079                    if not vo and vi then
1080                        -- to be checked
1081                        d = defined[prefix][vi] or defined[""][vi]
1082                        --
1083                        if d then
1084                            d = getexpansion(d)
1085                            resolve(prefix,d,var.arguments,set) -- args can be nil
1086                        else
1087                            if args then var.arguments = args end
1088                            set[#set+1] = var
1089                        end
1090                    else
1091                        if args then var.arguments = args end
1092                        set[#set+1] = var
1093                    end
1094                else
1095                --  report_references("funny pattern %a",ri)
1096                end
1097            end
1098        end
1099        return set
1100    else
1101        return { }
1102    end
1103end
1104
1105-- prefix == "" is valid prefix which saves multistep lookup
1106
1107references.currentset = nil
1108
1109local externals = { }
1110
1111-- we have prefixes but also components:
1112--
1113-- :    prefix
1114-- ::   always external
1115-- :::  internal (for products) or external (for components)
1116
1117local function loadexternalreferences(name,utilitydata)
1118    local struc = utilitydata.structures
1119    if struc then
1120        local external = struc.references.collected -- direct references
1121        local lists    = struc.lists.collected      -- indirect references (derived)
1122        local pages    = struc.pages.collected      -- pagenumber data
1123        local sections = struc.sections.collected
1124        -- a bit weird one, as we don't have the externals in the collected
1125        for prefix, set in next, external do
1126            if prefix == "" then
1127                prefix = name -- this can clash!
1128            end
1129            for reference, data in next, set do
1130                if trace_importing then
1131                    report_importing("registering %a reference, kind %a, name %a, prefix %a, reference %a",
1132                        "external","regular",name,prefix,reference)
1133                end
1134                -- weird code
1135             -- local section  = reference.section
1136             -- local realpage = reference.realpage
1137             -- if section then
1138             --     reference.sectiondata = lists[section]
1139             -- end
1140             -- if realpage then
1141             --     reference.pagedata = pages[realpage]
1142             -- end
1143                -- better?
1144                local references = data.references -- hm. plural
1145                if references then
1146                    local section  = references.section
1147                    local realpage = references.realpage
1148                    if not references.sectiondata then
1149                        references.sectiondata = sections[references.section or false]
1150                    end
1151                    if not references.pagedata then
1152                        references.pagedata = pages[references.realpage or false]
1153                    end
1154                end
1155            end
1156        end
1157        -- maybe store utilitydata in metatable so that we can access all
1158        for i=1,#lists do
1159            local entry      = lists[i]
1160            local metadata   = entry.metadata
1161            local references = entry.references
1162            if metadata and references then
1163                local reference = references.reference
1164                if reference and reference ~= "" then
1165                    local kind     = metadata.kind
1166                    local realpage = references.realpage
1167                    if kind and realpage then
1168                        -- quite likely already done so this might go
1169                        if pages and not references.pagedata then
1170                            references.pagedata = pages[references.realpage or false]
1171                        end
1172                        if sections and not references.sectiondata then
1173                            references.sectiondata = sections[references.section or false]
1174                        end
1175                        --
1176                        local prefix = references.prefix or ""
1177                        if prefix == "" then
1178                            prefix = name -- this can clash!
1179                        end
1180                        local target = external[prefix]
1181                        if not target then
1182                            target = { }
1183                            external[prefix] = target
1184                        end
1185                        local function action(s)
1186                            if trace_importing then
1187                                report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1188                                    "external",kind,name,prefix,s)
1189                            end
1190                            target[s] = target[s] or entry
1191                        end
1192                        process_settings(reference,action)
1193                    end
1194                end
1195            end
1196        end
1197        externals[name] = external
1198        return external
1199    end
1200end
1201
1202local externalfiles = { }
1203
1204setmetatableindex(externalfiles, function(t,k)
1205    local v = filedata[k]
1206    if not v then
1207        v = { k, k }
1208    end
1209    externalfiles[k] = v
1210    return v
1211end)
1212
1213setmetatableindex(externals, function(t,k) -- either or not automatically
1214    local filename = externalfiles[k][1] -- filename
1215    local fullname = file.replacesuffix(filename,"tuc")
1216    if lfs.isfile(fullname) then -- todo: use other locator
1217        local utilitydata = job.loadother(fullname)
1218        if utilitydata then
1219            local external = loadexternalreferences(k,utilitydata)
1220            t[k] = external or false
1221            return external
1222        end
1223    end
1224    t[k] = false
1225    return false
1226end)
1227
1228local productdata = allocate { -- will go
1229    productreferences   = { },
1230    componentreferences = { },
1231    components          = { },
1232}
1233
1234references.productdata = productdata
1235
1236local function loadproductreferences(productname,componentname,utilitydata)
1237    local struc = utilitydata.structures
1238    if struc then
1239        local productreferences = struc.references.collected -- direct references
1240        local lists    = struc.lists.collected      -- indirect references (derived)
1241        local pages    = struc.pages.collected      -- pagenumber data
1242        local sections = struc.sections.collected   -- section number data
1243        -- we use indirect tables to save room but as they are eventually
1244        -- just references we resolve them to data here (the mechanisms
1245        -- that use this data check for indirectness)
1246        for prefix, set in next, productreferences do
1247            for reference, data in next, set do
1248                if trace_importing then
1249                    report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1250                        "product","regular",productname,prefix,reference)
1251                end
1252                local references = data.references
1253                if references then
1254                    if not references.sectiondata then
1255                        references.sectiondata = sections[references.section or false]
1256                    end
1257                    if not references.pagedata then
1258                        references.pagedata = pages[references.realpage or false]
1259                    end
1260                end
1261            end
1262        end
1263        --
1264        local componentreferences = { }
1265        for i=1,#lists do
1266            local entry      = lists[i]
1267            local metadata   = entry.metadata
1268            local references = entry.references
1269            if metadata and references then
1270                local reference = references.reference
1271                if reference and reference ~= "" then
1272                    local kind     = metadata.kind
1273                    local realpage = references.realpage
1274                    if kind and realpage then
1275                        local prefix    = references.prefix or ""
1276                        local component = references.component
1277
1278                    -- local structure = references.structure
1279                    -- print("!!!!!!!!",structure)
1280                    -- if structure then
1281                    --     inspect(resolvers.job.namelist(structure))
1282                    -- end
1283
1284                        local ctarget, ptarget
1285                        if not component or component == componentname then
1286                            -- skip
1287                        else
1288                            -- one level up
1289                            local external = componentreferences[component]
1290                            if not external then
1291                                external = { }
1292                                componentreferences[component] = external
1293                            end
1294                            if component == prefix then
1295                                prefix = ""
1296                            end
1297                            ctarget = external[prefix]
1298                            if not ctarget then
1299                                ctarget = { }
1300                                external[prefix] = ctarget
1301                            end
1302                        end
1303                        ptarget = productreferences[prefix]
1304                        if not ptarget then
1305                            ptarget = { }
1306                            productreferences[prefix] = ptarget
1307                        end
1308                        local function action(s)
1309                            if ptarget then
1310                                if trace_importing then
1311                                    report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1312                                        "product",kind,productname,prefix,s)
1313                                end
1314                                ptarget[s] = ptarget[s] or entry
1315                            end
1316                            if ctarget then
1317                                if trace_importing then
1318                                    report_importing("registering %s reference, kind %a, name %a, prefix %a, referenc %a",
1319                                        "component",kind,productname,prefix,s)
1320                                end
1321                                ctarget[s] = ctarget[s] or entry
1322                            end
1323                        end
1324                        process_settings(reference,action)
1325                    end
1326                end
1327            end
1328        end
1329        productdata.productreferences   = productreferences -- not yet used
1330        productdata.componentreferences = componentreferences
1331    end
1332end
1333
1334local function loadproductvariables(product,component,utilitydata)
1335    local struc = utilitydata.structures
1336    if struc then
1337        local lists = struc.lists and struc.lists.collected
1338        if lists then
1339            local pages = struc.pages and struc.pages.collected
1340            for i=1,#lists do
1341                local li = lists[i]
1342                if li.metadata.kind == "section" and li.references.component == component then
1343                    local firstsection = li
1344                    if firstsection.numberdata then
1345                        local numbers = firstsection.numberdata.numbers
1346                        if numbers then
1347                            if trace_importing then
1348                                report_importing("initializing section number to %:t",numbers)
1349                            end
1350                            productdata.firstsection = firstsection
1351                            structures.documents.preset(numbers)
1352                        end
1353                    end
1354                    if pages and firstsection.references then
1355                        local firstpage = pages[firstsection.references.realpage]
1356                        local number = firstpage and firstpage.number
1357                        if number then
1358                            if trace_importing then
1359                                report_importing("initializing page number to %a",number)
1360                            end
1361                            productdata.firstpage = firstpage
1362                            counters.set("userpage",1,number)
1363                        end
1364                    end
1365                    break
1366                end
1367            end
1368        end
1369    end
1370end
1371
1372local function componentlist(tree,target)
1373    local branches = tree and tree.branches
1374    if branches then
1375        for i=1,#branches do
1376            local branch = branches[i]
1377            local type = branch.type
1378            if type == "component" then
1379                if target then
1380                    target[#target+1] = branch.name
1381                else
1382                    target = { branch.name }
1383                end
1384            elseif type == "product" or type == "component" then
1385                target = componentlist(branch,target)
1386            end
1387        end
1388    end
1389    return target
1390end
1391
1392local function loadproductcomponents(product,component,utilitydata)
1393    local job = utilitydata.job
1394    productdata.components = componentlist(job and job.structure and job.structure.collected) or { }
1395end
1396
1397references.registerinitializer(function(tobesaved,collected)
1398    -- not that much related to tobesaved or collected
1399    productdata.components = componentlist(job.structure.collected) or { }
1400end)
1401
1402function references.loadpresets(product,component) -- we can consider a special components hash
1403    if product and component and product~= "" and component ~= "" and not productdata.product then -- maybe: productdata.filename ~= filename
1404        productdata.product   = product
1405        productdata.component = component
1406        -- todo: use other locator
1407        local utilitydata = job.loadother(product)
1408        if utilitydata then
1409            if trace_importing then
1410                report_importing("loading references for component %a of product %a",component,product)
1411            end
1412            loadproductvariables (product,component,utilitydata)
1413            loadproductcomponents(product,component,utilitydata)
1414            loadproductreferences(product,component,utilitydata)
1415        --             loadproductcomponents(product,component,utilitydata)
1416        end
1417    end
1418end
1419
1420references.productdata = productdata
1421
1422implement {
1423    name      = "usereferences",
1424    public    = true,
1425    protected = true,
1426    arguments = "optional",
1427    actions   = function(product)
1428        local utilitydata = job.loadother(product)
1429        if utilitydata then
1430            loadexternalreferences(product,utilitydata)
1431        end
1432    end
1433}
1434
1435local ctx_useproductuseproduct = commands.useproduct
1436
1437if ctx_useproduct then
1438
1439    local function newuseproduct(product)
1440        ctx_useproduct(product)
1441        if texconditionals.autocrossfilereferences then
1442            local component = justacomponent()
1443            if component then
1444                if trace_referencing or trace_importing then
1445                    report_references("loading presets for component %a of product %a",component,product)
1446                end
1447                references.loadpresets(product,component)
1448            end
1449        end
1450    end
1451
1452    implement {
1453        name      = "useproduct",
1454        actions   = newuseproduct,
1455        arguments = "optional",
1456        overload  = true,
1457    }
1458
1459end
1460
1461-- productdata.firstsection.numberdata.numbers
1462-- productdata.firstpage.number
1463
1464local function report_identify_special(set,var,i,type)
1465    local reference = set.reference
1466    local prefix    = set.prefix or ""
1467    local special   = var.special
1468    local error     = var.error
1469    local kind      = var.kind
1470    if error then
1471        report_identifying("type %a, reference %a, index %a, prefix %a, special %a, error %a",type,reference,i,prefix,special,error)
1472    else
1473        report_identifying("type %a, reference %a, index %a, prefix %a, special %a, kind %a",type,reference,i,prefix,special,kind)
1474    end
1475end
1476
1477local function report_identify_arguments(set,var,i,type)
1478    local reference = set.reference
1479    local prefix    = set.prefix or ""
1480    local arguments = var.arguments
1481    local error     = var.error
1482    local kind      = var.kind
1483    if error then
1484        report_identifying("type %a, reference %a, index %a, prefix %a, arguments %a, error %a",type,reference,i,prefix,arguments,error)
1485    else
1486        report_identifying("type %a, reference %a, index %a, prefix %a, arguments %a, kind %a",type,reference,i,prefix,arguments,kind)
1487    end
1488end
1489
1490local function report_identify_outer(set,var,i,type)
1491    local reference = set.reference
1492    local prefix    = set.prefix or ""
1493    local outer     = var.outer
1494    local error     = var.error
1495    local kind      = var.kind
1496    if outer then
1497        if error then
1498            report_identifying("type %a, reference %a, index %a, prefix %a, outer %a, error %a",type,reference,i,prefix,outer,error)
1499        else
1500            report_identifying("type %a, reference %a, index %a, prefix %a, outer %a, kind %a",type,reference,i,prefix,outer,kind)
1501        end
1502    else
1503        if error then
1504            report_identifying("type %a, reference %a, index %a, prefix %a, error %a",type,reference,i,prefix,error)
1505        else
1506            report_identifying("type %a, reference %a, index %a, prefix %a, kind %a",type,reference,i,prefix,kind)
1507        end
1508    end
1509end
1510
1511local function report_identify_auto(set,var,i,type)
1512    report_identifying("type %a, reference %a, using auto outer %a",type,reference,var.outer)
1513end
1514
1515local function identify_special(set,var,i)
1516    local special = var.special
1517    local s = specials[special]
1518    if s then
1519        local outer     = var.outer
1520        local operation = var.operation
1521        local arguments = var.arguments
1522        if outer then
1523            if operation then
1524                -- special(outer::operation)
1525                var.kind = "special outer with operation"
1526            else
1527                -- special()
1528                var.kind = "special outer"
1529            end
1530            var.f = outer
1531        elseif operation then
1532            if arguments then
1533                -- special(operation{argument,argument})
1534                var.kind = "special operation with arguments"
1535            else
1536                -- special(operation)
1537                var.kind = "special operation"
1538            end
1539        else
1540            -- special()
1541            var.kind = "special"
1542        end
1543        if trace_identifying then
1544            report_identify_special(set,var,i,"1a")
1545        end
1546    else
1547        var.error = "unknown special"
1548    end
1549    return var
1550end
1551
1552local function identify_arguments(set,var,i)
1553    local s = specials[var.inner]
1554    if s then
1555        -- inner{argument}
1556        var.kind = "special operation with arguments"
1557    else
1558        var.error = "unknown inner or special"
1559    end
1560    if trace_identifying then
1561        report_identify_arguments(set,var,i,"3a")
1562    end
1563    return var
1564end
1565
1566-- needs checking: if we don't do too much (redundant) checking now
1567-- inner ... we could move the prefix logic into the parser so that we have 'm for each entry
1568-- foo:bar -> foo == prefix (first we try the global one)
1569-- -:bar   -> ignore prefix
1570
1571local function finish_inner(set,var,p,i)
1572    local external = i.references.external
1573    local realpage = (i.references and i.references.realpage) or (i.pagedata and i.pagedata.realpage) or 1
1574    if external then
1575        var.kind     = "outer with inner"
1576        var.external = true
1577        var.outer    = external
1578        set.external = true -- probably not needed
1579    else
1580        var.kind = "inner"
1581        var.p    = p
1582    end
1583    var.r = realpage
1584    var.i = i
1585    return var
1586end
1587
1588local function identify_inner(set,var,prefix,collected,derived)
1589    local inner = var.inner
1590    -- the next test is a safeguard when references are auto loaded from outer
1591    if not inner or inner == "" then
1592        return false
1593    end
1594    local splitprefix, splitinner = lpegmatch(prefixsplitter,inner)
1595    if splitprefix and splitinner then
1596        -- we check for a prefix:reference instance in the regular set of collected
1597        -- references; a special case is -: which forces a lookup in the global list
1598        if splitprefix == "-" then
1599            local i = collected[""]
1600            if i then
1601                i = i[splitinner]
1602                if i then
1603                    return finish_inner(set,var,"",i)
1604                end
1605            end
1606        end
1607        local i = collected[splitprefix]
1608        if i then
1609            i = i[splitinner]
1610            if i then
1611                return finish_inner(set,var,splitprefix,i)
1612            end
1613        end
1614        if derived then
1615            -- next we look for a reference in the regular set of collected references
1616            -- using the prefix that is active at this moment (so we overload the given
1617            -- these are taken from other data structures (like lists)
1618            if splitprefix == "-" then
1619-- loop over the current structure list reversed
1620-- and end with "" .. needs example
1621                local i = derived[""]
1622                if i then
1623                    i = i[splitinner]
1624                    if i then
1625                        return finish_inner(set,var,"",i)
1626                    end
1627                end
1628            end
1629            local i = derived[splitprefix]
1630            if i then
1631                i = i[splitinner]
1632                if i then
1633                    return finish_inner(set,var,splitprefix,i)
1634                end
1635            end
1636        end
1637    end
1638    -- we now ignore the split prefix and treat the whole inner as a potential
1639    -- reference into the global list
1640    local i = collected[prefix]
1641    if i then
1642        i = i[inner]
1643        if i then
1644            return finish_inner(set,var,prefix,i)
1645        end
1646    end
1647    if not i and derived then
1648        -- and if not found we look in the derived references
1649        local i = derived[prefix]
1650        if i then
1651            i = i[inner]
1652            if i then
1653                return finish_inner(set,var,prefix,i)
1654            end
1655        end
1656        --
1657        local n, list = resolvers.jobs.currentstructure()
1658-- print(prefix)
1659-- inspect(set)
1660-- inspect(list)
1661        if list then
1662            for i=#list,1,-1 do
1663                local l = list[i]
1664                local i = derived[l]
1665                if i then
1666                    i = i[inner]
1667                    if i then
1668local p = i.references.prefix or l -- which is what we use in the destination
1669--                         return finish_inner(set,var,l,i)
1670                        return finish_inner(set,var,p,i)
1671                    end
1672                end
1673            end
1674        end
1675    end
1676    return false
1677end
1678
1679local function unprefixed_inner(set,var,prefix,collected,derived,tobesaved)
1680    local inner = var.inner
1681    local s = specials[inner]
1682    if s then
1683        var.kind = "special"
1684    else
1685        local i = (collected and collected[""] and collected[""][inner]) or
1686                  (derived   and derived  [""] and derived  [""][inner]) or
1687                  (tobesaved and tobesaved[""] and tobesaved[""][inner])
1688        if i then
1689            finish_inner(set,var,"",i)
1690        else
1691            var.error = "unknown inner or special"
1692        end
1693    end
1694    return var
1695end
1696
1697local function identify_auto(set,var,i)
1698    local inner  = var.inner
1699    local prefix = set.prefix or ""
1700    if prefix == "" then
1701        prefix, inner = lpegmatch(p_splitter,inner)
1702        if not inner then
1703            inner  = var.inner
1704            prefix = false
1705        end
1706    end
1707    local function okay(o,where)
1708        var.outer = o
1709        if trace_identifying then
1710            report_identify_auto(set,var,i,where)
1711        end
1712        return true
1713    end
1714    for o, odata in sortedhash(externals) do
1715        if o == outer or not odata then
1716            -- skip this one
1717        elseif prefix and prefix ~= "" then
1718            local pdata = odata[prefix]
1719            if pdata then
1720                local r = pdata[inner]
1721                if r then
1722                    return okay(o,"2h")
1723                end
1724            end
1725            -- escape from prefix
1726        --  local pdata = odata[""]
1727        --  if pdata then
1728        --      local r = pdata[inner]
1729        --      if r then
1730        --          return okay(o,"2h")
1731        --      end
1732        --  end
1733        else
1734            for p, pdata in sortedhash(odata) do
1735                local r = pdata[inner]
1736                if r then
1737                    return okay(o,"2j")
1738                end
1739            end
1740            local pdata = odata[""]
1741            if pdata then
1742                r = pdata[inner]
1743                if r then
1744                    return okay(o,"2k")
1745                end
1746            end
1747        end
1748    end
1749    return false
1750end
1751
1752local function identify_outer(set,var,i)
1753    local outer    = var.outer
1754    local inner    = var.inner
1755    local external = externals[outer]
1756    if external then
1757        local v = identify_inner(set,var,"",external)
1758        if v then
1759            v.kind = "outer with inner"
1760            set.external = true
1761            if trace_identifying then
1762                report_identify_outer(set,v,i,"2a")
1763            end
1764            return v
1765        end
1766-- weird too (we really need to check how this table is build
1767        local v = identify_inner(set,var,var.outer,external)
1768        if v then
1769            v.kind = "outer with inner"
1770            set.external = true
1771            if trace_identifying then
1772                report_identify_outer(set,v,i,"2c")
1773            end
1774            return v
1775        end
1776--
1777        -- somewhat rubish: we use outer as first step in the externals table so it makes no
1778        -- sense to have it as prefix so the next could be an option
1779        local external = external[""]
1780        if external then
1781            local v = identify_inner(set,var,var.outer,external)
1782            if v then
1783                v.kind = "outer with inner"
1784                set.external = true
1785                if trace_identifying then
1786                    report_identify_outer(set,v,i,"2b")
1787                end
1788                return v
1789            end
1790        end
1791    end
1792    local external = productdata.productreferences[outer]
1793    if external then
1794        local vi = external[inner]
1795        if vi then
1796            var.kind = "outer with inner"
1797            var.i = vi
1798            set.external = true
1799            if trace_identifying then
1800                report_identify_outer(set,var,i,"2d")
1801            end
1802            return var
1803        end
1804    end
1805    local external = productdata.componentreferences[outer]
1806    if external then
1807        local v = identify_inner(set,var,"",external)
1808        if v then
1809            v.kind = "outer with inner"
1810            set.external = true
1811            if trace_identifying then
1812                report_identify_outer(set,v,i,"2c")
1813            end
1814            return v
1815        end
1816    end
1817    -- the rest
1818    local special   = var.special
1819    local arguments = var.arguments
1820    local operation = var.operation
1821    if inner then
1822        -- tricky: in this case we can only use views when we're sure that all inners
1823        -- are flushed in the outer document so that should become an option
1824        if arguments then
1825            -- outer::inner{argument}
1826            var.kind = "outer with inner with arguments"
1827        else
1828            -- outer::inner
1829            var.kind = "outer with inner"
1830        end
1831        var.i = inner
1832        var.f = outer
1833        if type(inner) == "table" then
1834            -- can this really happen?
1835            var.r = (inner.references and inner.references.realpage) or (inner.pagedata and inner.pagedata.realpage) or 1
1836        else
1837            var.r = 1
1838        end
1839        if trace_identifying then
1840            report_identify_outer(set,var,i,"2e")
1841        end
1842    elseif special then
1843        local s = specials[special]
1844        if s then
1845            if operation then
1846                if arguments then
1847                    -- outer::special(operation{argument,argument})
1848                    var.kind = "outer with special and operation and arguments"
1849                else
1850                    -- outer::special(operation)
1851                    var.kind = "outer with special and operation"
1852                end
1853            else
1854                -- outer::special()
1855                var.kind = "outer with special"
1856            end
1857            var.f = outer
1858        else
1859            var.error = "unknown outer with special"
1860        end
1861        if trace_identifying then
1862            report_identify_outer(set,var,i,"2f")
1863        end
1864    else
1865        -- outer::
1866        var.kind = "outer"
1867        var.f = outer
1868        if trace_identifying then
1869            report_identify_outer(set,var,i,"2g")
1870        end
1871    end
1872    return var
1873end
1874
1875-- todo: avoid copy
1876
1877local function identify_inner_or_outer(set,var,i)
1878    -- here we fall back on product data
1879    local inner = var.inner
1880    if inner and inner ~= "" then
1881
1882        -- first we look up in collected and derived using the current prefix
1883
1884        local prefix = set.prefix
1885
1886        local v = identify_inner(set,var,set.prefix,collected,derived)
1887        if v then
1888            if trace_identifying then
1889                report_identify_outer(set,v,i,"4a")
1890            end
1891            return v
1892        end
1893
1894--         if var.inner == var.reference and identify_auto(set,var,i) and var.outer and var.outer ~= "" then
1895        if var.inner == var.reference and var.outer and var.outer ~= "" then
1896            return identify_outer(set,var,i)
1897        end
1898
1899        -- nest we look at each component (but we can omit the already consulted one
1900
1901        local jobstructure = job.structure
1902        local components   = jobstructure and jobstructure.components
1903        if components then
1904            for c=1,#components do
1905                local component = components[c]
1906                if component ~= prefix then
1907                    local v = identify_inner(set,var,component,collected,derived)
1908                    if v then
1909                        if trace_identifying then
1910                            report_identify_outer(set,var,i,"4b")
1911                        end
1912                        return v
1913                    end
1914                end
1915            end
1916        end
1917
1918        -- as a last resort we will consult the global lists
1919        local v = unprefixed_inner(set,var,"",collected,derived,tobesaved)
1920        if v then
1921            if trace_identifying then
1922                report_identify_outer(set,v,i,"4c")
1923            end
1924            return v
1925        end
1926
1927        -- not it gets bad ... we need to look in external files ... keep in mind that
1928        -- we can best use explicit references for this ... we might issue a warning
1929
1930        local componentreferences = productdata.componentreferences
1931        local productreferences = productdata.productreferences
1932        local components = productdata.components
1933        if components and componentreferences then
1934            for c=1,#components do
1935                local component = components[c]
1936                local data = componentreferences[component]
1937                if data then
1938                    local d = data[""]
1939                    local vi = d and d[inner]
1940                    if vi then
1941                        var.outer = component
1942                        var.i = vi
1943                        var.kind = "outer with inner"
1944                        set.external = true
1945                        if trace_identifying then
1946                            report_identify_outer(set,var,i,"4d")
1947                        end
1948                        return var
1949                    end
1950                end
1951            end
1952        end
1953        local component, inner = lpegmatch(componentsplitter,inner)
1954        if component then
1955            local data = componentreferences and componentreferences[component]
1956            if data then
1957                local d = data[""]
1958                local vi = d and d[inner]
1959                if vi then
1960                    var.inner = inner
1961                    var.outer = component
1962                    var.i = vi
1963                    var.kind = "outer with inner"
1964                    set.external = true
1965                    if trace_identifying then
1966                        report_identify_outer(set,var,i,"4e")
1967                    end
1968                    return var
1969                end
1970            end
1971            local data = productreferences and productreferences[component]
1972            if data then
1973                local vi = data[inner]
1974                if vi then
1975                    var.inner = inner
1976                    var.outer = component
1977                    var.i = vi
1978                    var.kind = "outer with inner"
1979                    set.external = true
1980                    if trace_identifying then
1981                        report_identify_outer(set,var,i,"4f")
1982                    end
1983                    return var
1984                end
1985            end
1986        end
1987
1988        var.error = "unknown inner"
1989    else
1990        var.error = "no inner"
1991    end
1992    if trace_identifying then
1993        report_identify_outer(set,var,i,"4g")
1994    end
1995    return var
1996end
1997
1998local function identify_inner_component(set,var,i)
1999    -- we're in a product (maybe ignore when same as component)
2000    local component = var.component
2001    local v = identify_inner(set,var,component,collected,derived)
2002    if not v then
2003        var.error = "unknown inner in component"
2004    end
2005    if trace_identifying then
2006        report_identify_outer(set,var,i,"5a")
2007    end
2008    return var
2009end
2010
2011local function identify_outer_component_step(set,var,vi,i,component,where)
2012    if vi then
2013        var.outer    = component
2014        var.i        = vi
2015        var.kind     = "outer with inner"
2016        set.external = true
2017        if trace_identifying then
2018            report_identify_outer(set,var,i,where)
2019        end
2020        return true
2021    end
2022end
2023
2024local function identify_outer_component(set,var,i)
2025    local component = var.component
2026    local inner     = var.inner
2027    --
2028    local data = productdata.componentreferences[component]
2029    if data then
2030        local d = data[""]
2031        local d = d and d[inner]
2032        if d and identify_outer_component_step(set,var,d,i,component,"6a") then
2033            return var
2034        end
2035    end
2036    --
2037    local data = productdata.productreferences[component]
2038    if data then
2039        local d = data[inner]
2040        if d and identify_outer_component_step(set,var,d,i,component,"6b") then
2041            return var
2042        end
2043    end
2044    --
2045if texconditionals.autocrossfilereferences then
2046    references.loadpresets(component,component)
2047    local data = productdata.productreferences[component]
2048    if data then
2049        local d = data[inner]
2050        if d and identify_outer_component_step(set,var,d,i,component,"6c") then
2051            return var
2052        end
2053    end
2054end
2055    --
2056    var.error = "unknown component"
2057    if trace_identifying then
2058        report_identify_outer(set,var,i,"6c")
2059    end
2060    return var
2061end
2062
2063local nofidentified = 0
2064
2065local function identify(prefix,reference)
2066    if not reference then
2067        prefix, reference = "", prefix
2068    end
2069    local set = resolve(prefix,reference)
2070    local bug = false
2071    nofidentified = nofidentified + 1
2072    set.n = nofidentified
2073    for i=1,#set do
2074        local var = set[i]
2075        local spe = var.special
2076        local fnc = functions[spe]
2077        if fnc then
2078            var = fnc(var) or { error = "invalid special function" }
2079        elseif spe then
2080            var = identify_special(set,var,i)
2081        elseif var.outer then
2082            if var.outer == "auto" then
2083                identify_auto(set,var,i)
2084            end
2085            var = identify_outer(set,var,i)
2086        elseif var.arguments then
2087            var = identify_arguments(set,var,i)
2088-- else
2089--     var = identify_inner_or_outer(set,var,i)
2090        elseif not var.component then
2091            var = identify_inner_or_outer(set,var,i)
2092        elseif productcomponent() then
2093            var = identify_inner_component(set,var,i)
2094        else
2095            var = identify_outer_component(set,var,i)
2096        end
2097        set[i] = var
2098        bug = bug or var.error
2099    end
2100    references.currentset = mark(set) -- mark, else in api doc
2101    if trace_analyzing then
2102        report_references(table.serialize(set,reference))
2103    end
2104    return set, bug
2105end
2106
2107references.identify = identify
2108
2109local unknowns, nofunknowns, f_valid = { }, 0, formatters["[%s][%s]"]
2110
2111function references.valid(prefix,reference,specification,silent)
2112    local set, bug = identify(prefix,reference)
2113    local unknown = bug or #set == 0
2114    if unknown then
2115        currentreference = nil -- will go away
2116        local str = f_valid(prefix,reference)
2117        local u = unknowns[str]
2118        if not u then
2119            if somefound and not silent then
2120                interfaces.showmessage("references",1,str) -- 1 = unknown, 4 = illegal
2121            end
2122            unknowns[str] = 1
2123            nofunknowns = nofunknowns + 1
2124        else
2125            unknowns[str] = u + 1
2126        end
2127    else
2128        set.highlight    = specification.highlight
2129        set.newwindow    = specification.newwindow
2130        set.layer        = specification.layer
2131        currentreference = set[1]
2132    end
2133    -- we can do the expansion here which saves a call
2134    return not unknown
2135end
2136
2137function references.checked(prefix,reference,specification)
2138    return references.valid(prefix,reference,specification,true)
2139end
2140
2141implement {
2142    name      = "doifelsereference",
2143    actions   = { references.valid, ctx_doifelse },
2144    arguments = {
2145        "string",
2146        "string",
2147        {
2148            { "highlight", "boolean" },
2149            { "newwindow", "boolean" },
2150            { "layer" },
2151        }
2152    }
2153}
2154
2155implement {
2156    name      = "doifelsereferencechecked",
2157    actions   = { references.checked, ctx_doifelse },
2158    arguments = {
2159        "string",
2160        "string",
2161        {
2162            { "highlight", "boolean" },
2163            { "newwindow", "boolean" },
2164            { "layer" },
2165        }
2166    }
2167}
2168
2169logs.registerfinalactions(function()
2170    if nofunknowns > 0 then
2171        statistics.register("cross referencing", function()
2172            return format("%s identified, %s unknown",nofidentified,nofunknowns)
2173        end)
2174        local sortedhash = table.sortedhash
2175        logs.startfilelogging(report,"missing references")
2176        for k, v in table.sortedhash(unknowns) do
2177            report("%4i  %s",v,k)
2178        end
2179        logs.stopfilelogging()
2180        if logs.loggingerrors() then
2181            logs.starterrorlogging(report,"missing references")
2182            for k, v in table.sortedhash(unknowns) do
2183                report("%4i  %s",v,k)
2184            end
2185            logs.stoperrorlogging()
2186        end
2187    end
2188end)
2189
2190-- The auto method will try to avoid named internals in a clever way which
2191-- can make files smaller without sacrificing external references. Some of
2192-- the housekeeping happens the backend side.
2193
2194local innermethod        = v_auto       -- only page|auto now
2195local outermethod        = v_auto       -- only page|auto now
2196local defaultinnermethod = defaultinnermethod
2197local defaultoutermethod = defaultoutermethod
2198references.innermethod   = innermethod  -- don't mess with this one directly
2199references.outermethod   = outermethod  -- don't mess with this one directly
2200
2201function references.setlinkmethod(inner,outer)
2202    if not outer and type(inner) == "string" then
2203        local m = settings_to_array(inner)
2204        inner = m[1]
2205        outer = m[2] or v_auto
2206    end
2207    if toboolean(inner) or inner == v_page or inner == v_yes then
2208        innermethod = v_page
2209    elseif inner == v_name then
2210        innermethod = v_name
2211    else
2212        innermethod = v_auto
2213    end
2214    if toboolean(outer) or outer == v_page or outer == v_yes then
2215        outermethod = v_page
2216    elseif outer == v_name then
2217        outermethod = v_name
2218    else
2219        outermethod = v_auto
2220    end
2221    references.innermethod = innermethod
2222    references.outermethod = outermethod
2223    function references.setlinkmethod()
2224        report_references("link method is already set and frozen: inner %a, outer %a",innermethod,outermethod)
2225    end
2226end
2227
2228implement {
2229    name      = "setreferencelinkmethod",
2230    actions   = references.setlinkmethod,
2231    arguments = "string",
2232 -- onlyonce  = true
2233}
2234
2235function references.getinnermethod()
2236    return innermethod or defaultinnermethod
2237end
2238
2239function references.getoutermethod()
2240    return outermethod or defaultoutermethod
2241end
2242
2243directives.register("references.linkmethod", function(v) -- page auto
2244    references.setlinkmethod(v)
2245end)
2246
2247-- we can call setinternalreference with an already known internal or with
2248-- a reference/prefix specification
2249
2250local destinationattributes = { }
2251
2252local function setinternalreference(specification)
2253    local internal    = specification.internal
2254    local destination = unsetvalue
2255    if innermethod == v_auto or innermethod == v_name then
2256        local t         = { } -- maybe add to current (now only used for tracing)
2257        local tn        = 0
2258        local reference = specification.reference
2259        local view      = specification.view
2260        if reference then
2261            local prefix = specification.prefix
2262            if prefix and prefix ~= "" then
2263                local prefix = prefix .. ":" -- watch out, : here
2264                local function action(ref)
2265                    tn = tn + 1
2266                    t[tn] = prefix .. ref
2267                end
2268                process_settings(reference,action)
2269            else
2270                local function action(ref)
2271                    tn = tn + 1
2272                    t[tn] = ref
2273                end
2274                process_settings(reference,action)
2275            end
2276        end
2277        -- ugly .. later we decide to ignore it when we have a real one
2278        -- but for testing we might want to see them all
2279        if internal then
2280            if innermethod ~= v_name then -- innermethod == v_auto
2281             -- we don't want too many #1 #2 #3 etc
2282                tn = tn + 1
2283                t[tn] = internal -- when number it's internal
2284            end
2285            if not view then
2286                local i = references.internals[internal]
2287                if i then
2288                    view = i.references.view
2289                end
2290            end
2291        end
2292        destination = references.mark(t,nil,nil,view) -- returns an attribute
2293    end
2294    if internal then -- new
2295        destinationattributes[internal] = destination
2296    end
2297    texsetcount(c_lastdestinationattribute,destination)
2298    return destination
2299end
2300
2301local function getinternalreference(internal)
2302    return destinationattributes[internal] or 0
2303end
2304
2305references.setinternalreference = setinternalreference
2306references.getinternalreference = getinternalreference
2307
2308implement {
2309    name      = "setinternalreference",
2310    actions   = setinternalreference,
2311    arguments = {
2312        {
2313            { "prefix" },
2314            { "reference" },
2315            { "internal", "integer" },
2316            { "view" }
2317        }
2318    }
2319}
2320
2321-- implement {
2322--     name      = "getinternalreference",
2323--     actions   = { getinternalreference, context },
2324--     arguments = "integer",
2325-- }
2326
2327function references.setandgetattribute(data) -- maybe do internal automatically here
2328    local attr = unsetvalue
2329    local mdat = data.metadata
2330    local rdat = data.references
2331    if mdat and rdat then
2332        if not rdat.section then
2333            rdat.section = structures.sections.currentid()
2334        end
2335        local ndat = data.numberdata
2336        if ndat then
2337            local numbers = ndat.numbers
2338            if type(numbers) == "string" then
2339                counters.compact(ndat,numbers)
2340            end
2341            data.numberdata = helpers.simplify(ndat)
2342        end
2343        local pdat = data.prefixdata
2344        if pdat then
2345            data.prefixdata = helpers.simplify(pdat)
2346        end
2347        local udat = data.userdata
2348        if type(udat) == "string"  then
2349            data.userdata = helpers.touserdata(udat)
2350        end
2351        if not rdat.block then
2352            rdat.block = structures.sections.currentblock()
2353        end
2354        local done = references.set(data) -- we had kind i.e .item -> full
2355        if done then
2356            attr = setinternalreference {
2357                prefix    = rdat.prefix,
2358                reference = rdat.reference,
2359                internal  = rdat.internal,
2360                view      = rdat.view
2361            } or unsetvalue
2362        end
2363    end
2364    texsetcount(c_lastdestinationattribute,attr)
2365    return attr
2366end
2367
2368implement {
2369    name      = "setdestinationattribute",
2370    actions   = references.setandgetattribute,
2371    arguments = {
2372        {
2373            {
2374                "references", {
2375                    { "internal", "integer" },
2376                    { "block" },
2377                    { "view" },
2378                    { "prefix" },
2379                    { "reference" },
2380                },
2381            },
2382            {
2383                "metadata", {
2384                    { "kind" },
2385                    { "xmlroot" },
2386                    { "catcodes", "integer" },
2387                },
2388            },
2389            {
2390                "prefixdata", { "*" }
2391            },
2392            {
2393                "numberdata", { "*" }
2394            },
2395            {
2396                "entries", { "*" }
2397            },
2398            {
2399                "userdata"
2400            }
2401        }
2402    }
2403}
2404
2405function references.getinternallistreference(n) -- n points into list (todo: registers)
2406    local l = lists.collected[n]
2407    local i = l and l.references.internal
2408    return i and destinationattributes[i] or 0
2409end
2410
2411function references.getinternalcachedlistreference(n) -- n points into list (todo: registers)
2412    local l = lists.cached[n]
2413    local i = l and l.references.internal
2414    return i and destinationattributes[i] or 0
2415end
2416
2417implement {
2418    name      = "getinternallistreference",
2419    actions   = { references.getinternallistreference, context },
2420    arguments = "integer"
2421}
2422
2423implement {
2424    name      = "getinternalcachedlistreference",
2425    actions   = { references.getinternalcachedlistreference, context },
2426    arguments = "integer"
2427}
2428
2429
2430--
2431
2432function references.getcurrentmetadata(tag)
2433    local data = currentreference and currentreference.i
2434    return data and data.metadata and data.metadata[tag]
2435end
2436
2437implement {
2438    name      = "getcurrentreferencemetadata",
2439    actions   = { references.getcurrentmetadata, context },
2440    arguments = "string",
2441}
2442
2443local function currentmetadata(tag)
2444    local data = currentreference and currentreference.i
2445    return data and data.metadata and data.metadata[tag]
2446end
2447
2448references.currentmetadata = currentmetadata
2449
2450local function getcurrentprefixspec(default)
2451    local data     = currentreference and currentreference.i
2452    local metadata = data and data.metadata
2453    return
2454        metadata and metadata.kind or "?",
2455        metadata and metadata.name or "?",
2456        default                    or "?"
2457end
2458
2459references.getcurrentprefixspec = getcurrentprefixspec
2460
2461-- implement {
2462--     name      = "getcurrentprefixspec",
2463--     actions   = { getcurrentprefixspec, context }, -- returns 3 arguments
2464--     arguments = "string",
2465-- }
2466
2467implement {
2468    name      = "getcurrentprefixspec",
2469    actions   = function(tag)
2470        context("{%s}{%s}{%s}",getcurrentprefixspec(tag))
2471    end,
2472    arguments = "string",
2473}
2474
2475local genericfilters = { }
2476local userfilters    = { }
2477local textfilters    = { }
2478local fullfilters    = { }
2479local sectionfilters = { }
2480
2481filters.generic = genericfilters
2482filters.user    = userfilters
2483filters.text    = textfilters
2484filters.full    = fullfilters
2485filters.section = sectionfilters
2486
2487local function filterreference(name,prefixspec,numberspec) -- number page title ...
2488    local data = currentreference and currentreference.i -- maybe we should take realpage from here
2489    if data then
2490        if name == "realpage" then
2491            local cs = references.analyze() -- normally already analyzed but also sets state
2492            -- or just get from references.realpage when present
2493            context(tonumber(cs.realpage) or 0)
2494        else -- assumes data is table
2495            local kind = false
2496            if type(data) == "string" then
2497                -- todo
2498            end
2499            if type(data) == "table" then
2500                kind = data.metadata and data.metadata.kind
2501            end
2502            if kind then
2503                local filter = filters[kind] or genericfilters
2504                filter = filter and (filter[name] or filter.unknown or genericfilters[name] or genericfilters.unknown)
2505                if filter then
2506                    if trace_referencing then
2507                        report_references("name %a, kind %a, using dedicated filter",name,kind)
2508                    end
2509                    filter(data,name,prefixspec,numberspec)
2510                elseif trace_referencing then
2511                    report_references("name %a, kind %a, using generic filter",name,kind)
2512                end
2513            elseif trace_referencing then
2514                report_references("name %a, unknown kind",name)
2515            end
2516        end
2517    elseif name == "realpage" then
2518        context(0)
2519    elseif trace_referencing then
2520        report_references("name %a, no reference",name)
2521    end
2522end
2523
2524local function filterreferencedefault()
2525    return filterreference("default",getcurrentprefixspec("default"))
2526end
2527
2528local function getreferencesectionnumber(reference)
2529    if references.valid("",reference,{},false) then
2530        local data = currentreference and currentreference.i
2531        if type(data) == "table" then
2532            local metadata = data.metadata
2533            if metadata and metadata.kind == "section" then
2534                local numberdata = data.numberdata
2535                if numberdata then
2536                    numberdata = numberdata.numbers
2537                    context(numberdata[#numberdata] or 0)
2538                    return
2539                end
2540            end
2541        elseif trace_referencing then
2542            report_references("name %a, no reference",reference)
2543        end
2544    end
2545    context(0)
2546end
2547
2548references.filter        = filterreference
2549references.filterdefault = filterreferencedefault
2550
2551implement {
2552    name      = "filterreference",
2553    actions   = filterreference,
2554    arguments = "string",
2555}
2556
2557implement {
2558    name      = "filterdefaultreference",
2559    actions   = filterreference,
2560    arguments = {
2561        "string",    -- 'default'
2562        { { "*" } }, -- prefixspec
2563        { { "*" } }, -- numberspec
2564    }
2565}
2566
2567implement {
2568    name      = "getreferencesectionnumber",
2569    actions   = getreferencesectionnumber,
2570    arguments = "string",
2571    public    = true,
2572}
2573
2574function genericfilters.title(data)
2575    if data then
2576        local titledata = data.titledata or data.useddata
2577        if titledata then
2578            helpers.title(titledata.reference or titledata.title or "?",data.metadata)
2579        end
2580    end
2581end
2582
2583function genericfilters.text(data)
2584    if data then
2585        local entries = data.entries or data.useddata
2586        if entries then
2587            helpers.title(entries.text or "?",data.metadata)
2588        end
2589    end
2590end
2591
2592function genericfilters.number(data,what,prefixspec,numberspec)
2593    if data then
2594        numberdata = lists.reordered(data) -- data.numberdata
2595        if numberdata then
2596            helpers.prefix(data,prefixspec)
2597            sections.typesetnumber(numberdata,"number",numberspec,numberdata)
2598        else
2599            local useddata = data.useddata
2600            if useddata and useddata.number then
2601                context(useddata.number)
2602            end
2603        end
2604    end
2605end
2606
2607genericfilters.default = genericfilters.text
2608
2609function genericfilters.page(data,prefixspec,pagespec)
2610    local pagedata = data.pagedata
2611    if pagedata then
2612        local number     = pagedata.number
2613        local conversion = pagedata.conversion
2614        if not number then
2615            -- error
2616        elseif conversion then
2617            ctx_convertnumber(conversion,number)
2618        else
2619            context(number)
2620        end
2621    else
2622        helpers.prefixpage(data,prefixspec,pagespec)
2623    end
2624end
2625
2626function userfilters.unknown(data,name)
2627    if data then
2628        local userdata = data.userdata
2629        local userkind = userdata and userdata.kind
2630        if userkind then
2631            local filter = filters[userkind] or genericfilters
2632            filter = filter and (filter[name] or filter.unknown)
2633            if filter then
2634                filter(data,name)
2635                return
2636            end
2637        end
2638        local namedata = userdata and userdata[name]
2639        if namedata then
2640            context(namedata)
2641        end
2642    end
2643end
2644
2645function textfilters.title(data)
2646    helpers.title(data.entries.text or "?",data.metadata)
2647end
2648
2649-- no longer considered useful:
2650--
2651-- function filters.text.number(data)
2652--     helpers.title(data.entries.text or "?",data.metadata)
2653-- end
2654
2655function textfilters.page(data,prefixspec,pagespec)
2656    helpers.prefixpage(data,prefixspec,pagespec)
2657end
2658
2659fullfilters.title = textfilters.title
2660fullfilters.page  = textfilters.page
2661
2662function sectionfilters.number(data,what,prefixspec)
2663    if data then
2664        local numberdata = data.numberdata
2665        if not numberdata then
2666            local useddata = data.useddata
2667            if useddata and useddata.number then
2668                context(useddata.number)
2669            end
2670        elseif numberdata.hidenumber then
2671            local references = data.references
2672            if trace_empty then
2673                report_empty("reference %a has a hidden number",references.reference)
2674                ctx_emptyreference() -- maybe an option
2675            end
2676        else
2677            sections.typesetnumber(numberdata,"number",prefixspec,numberdata)
2678        end
2679    end
2680end
2681
2682sectionfilters.title   = genericfilters.title
2683sectionfilters.page    = genericfilters.page
2684sectionfilters.default = sectionfilters.number
2685
2686-- filters.note        = { default = genericfilters.number }
2687-- filters.formula     = { default = genericfilters.number }
2688-- filters.float       = { default = genericfilters.number }
2689-- filters.description = { default = genericfilters.number }
2690-- filters.item        = { default = genericfilters.number }
2691
2692setmetatableindex(filters, function(t,k) -- beware, test with rawget
2693    local v = { default = genericfilters.number } -- not copy as it might be extended differently
2694    t[k] = v
2695    return v
2696end)
2697
2698-- function references.sectiontitle(n)
2699--     helpers.sectiontitle(lists.collected[tonumber(n) or 0])
2700-- end
2701
2702-- function references.sectionnumber(n)
2703--     helpers.sectionnumber(lists.collected[tonumber(n) or 0])
2704-- end
2705
2706-- function references.sectionpage(n,prefixspec,pagespec)
2707--     helpers.prefixedpage(lists.collected[tonumber(n) or 0],prefixspec,pagespec)
2708-- end
2709
2710-- analyze
2711
2712references.testrunners  = references.testrunners  or { }
2713references.testspecials = references.testspecials or { }
2714
2715local runners  = references.testrunners
2716local specials = references.testspecials
2717
2718-- We need to prevent ending up in the 'relative location' analyzer as it is
2719-- pretty slow (progressively). In the pagebody one can best check the reference
2720-- real page to determine if we need contrastlocation as that is more lightweight.
2721
2722local function checkedpagestate(n,page,actions,position,spread)
2723    local p = tonumber(page)
2724    if not p then
2725        return 0
2726    end
2727    if position and #actions > 0 then
2728        local i = actions[1].i -- brrr
2729        if i then
2730            local a = i.references
2731            if a then
2732                local x = a.x
2733                local y = a.y
2734                if x and y then
2735                    local jp = jobpositions.collected[position]
2736                    if jp then
2737                        local px = jp.x
2738                        local py = jp.y
2739                        local pp = jp.p
2740                        if p == pp then
2741                            -- same page
2742                            if py > y then
2743                                return 5 -- above
2744                            elseif py < y then
2745                                return 4 -- below
2746                            elseif px > x then
2747                                return 4 -- below
2748                            elseif px < x then
2749                                return 5 -- above
2750                            else
2751                                return 1 -- same
2752                            end
2753                        elseif spread then
2754                            if pp % 2 == 0 then
2755                                -- left page
2756                                if pp > p then
2757                                    return 2 -- before
2758                                elseif pp + 1 == p then
2759--                                     return 4 -- below (on right page)
2760                                    return 5 -- above (on left page)
2761                                else
2762                                    return 3 -- after
2763                                end
2764                            else
2765                                -- right page
2766                                if pp < p then
2767                                    return 3 -- after
2768                                elseif pp - 1 == p then
2769--                                     return 5 -- above (on left page)
2770                                    return 4 -- below (on right page)
2771                                else
2772                                    return 2 -- before
2773                                end
2774                            end
2775                        elseif pp > p then
2776                            return 2 -- before
2777                        else
2778                            return 3 -- after
2779                        end
2780                    end
2781                end
2782            end
2783        end
2784    end
2785    local r = referredpage(n) -- sort of obsolete
2786    if p > r then
2787        return 3 -- after
2788    elseif p < r then
2789        return 2 -- before
2790    else
2791        return 1 -- same
2792    end
2793end
2794
2795local function setreferencerealpage(actions)
2796    if not actions then
2797        actions = references.currentset
2798    end
2799    if type(actions) == "table" then
2800        local realpage = actions.realpage
2801        if realpage then
2802            return realpage
2803        end
2804        local nofactions = #actions
2805        if nofactions > 0 then
2806            for i=1,nofactions do
2807                local a = actions[i]
2808                local what = runners[a.kind]
2809                if what then
2810                    what = what(a,actions) -- needs documentation
2811                end
2812            end
2813            realpage = actions.realpage
2814            if realpage then
2815                return realpage
2816            end
2817        end
2818        actions.realpage = 0
2819    end
2820    return 0
2821end
2822
2823references.setreferencerealpage = setreferencerealpage
2824
2825-- we store some analysis data alongside the indexed array
2826-- at this moment only the real reference page is analyzed
2827-- normally such an analysis happens in the backend code
2828
2829function references.analyze(actions,position,spread)
2830    if not actions then
2831        actions = references.currentset
2832    end
2833    if not actions then
2834        actions = { realpage = 0, pagestate = 0 }
2835    elseif actions.pagestate then
2836        -- already done
2837    else
2838        local realpage = actions.realpage or setreferencerealpage(actions)
2839        if realpage == 0 then
2840            actions.pagestate = 0
2841        elseif actions.external then
2842            actions.pagestate = 0
2843        else
2844            actions.pagestate = checkedpagestate(actions.n,realpage,actions,position,spread)
2845        end
2846    end
2847-- inspect(actions)
2848    return actions
2849end
2850
2851local function referencepagestate(position,detail,spread)
2852    local actions = references.currentset
2853    if not actions then
2854        return 0
2855    elseif actions.external then
2856        return 0
2857    else
2858        local pagestate = actions.pagestate
2859        for i=1,#actions do
2860            local a = actions[i]
2861            if a.outer then
2862                pagestate = 0
2863                actions.pagestate = pagestate
2864                break
2865            end
2866        end
2867        if not pagestate then
2868            references.analyze(actions,position,spread) -- delayed unless explicitly asked for
2869            pagestate = actions.pagestate
2870        end
2871        if detail then
2872            return pagestate
2873        elseif pagestate == 4 then
2874            return 2 -- compatible
2875        elseif pagestate == 5 then
2876            return 3 -- compatible
2877        else
2878            return pagestate
2879        end
2880    end
2881end
2882
2883implement {
2884    name      = "referencepagestate",
2885    actions   = { referencepagestate, context },
2886    arguments = "string"
2887}
2888
2889implement {
2890    name      = "referencepagedetail",
2891    actions   = { referencepagestate, context },
2892    arguments = { "string", "boolean", "boolean" }
2893}
2894
2895-- local function referencerealpage()
2896--     local actions = references.currentset
2897--     return not actions and 0 or actions.realpage or setreferencerealpage(actions)
2898-- end
2899--
2900-- implement {
2901--     name      = "referencerealpage",
2902--     actions   = { referencerealpage, context },
2903--  -- arguments = "string" -- hm, weird
2904-- }
2905
2906implement {
2907    name      = "askedreference",
2908    public    = true,
2909    protected = true,
2910    actions   = function()
2911        local actions = references.currentset
2912        if actions then
2913            context("[p=%s,r=%s]",actions.prefix or "",actions.reference)
2914        end
2915    end
2916}
2917
2918implement {
2919    name    = "referencerealpage",
2920    actions = function()
2921        local actions = references.currentset
2922        context(not actions and 0 or actions.realpage or setreferencerealpage(actions))
2923    end
2924}
2925
2926local function referencepos(key)
2927    local actions = references.currentset
2928    local i = actions[1].i -- brrr
2929    local v = 0
2930    if i then
2931        local a = i.references
2932        if a then
2933            v = a[key] or 0
2934        end
2935    end
2936    return v
2937end
2938
2939implement { name = "referenceposx", actions = function() context("%p",referencepos("x")) end }
2940implement { name = "referenceposy", actions = function() context("%p",referencepos("y")) end }
2941
2942implement {
2943    name    = "referencecolumn",
2944    actions = function()
2945        local actions = references.currentset
2946        local column  = 1
2947        if actions then
2948            column = jobpositions.columnofpos(actions.realpage or setreferencerealpage(actions),referencepos("x"))
2949        end
2950        context(column or 1)
2951    end
2952}
2953
2954local plist, nofrealpages
2955
2956local function realpageofpage(p) -- the last one counts !
2957    if not plist then
2958        local pages = structures.pages.collected
2959        nofrealpages = #pages
2960        plist = { }
2961        for rp=1,nofrealpages do
2962            local page = pages[rp]
2963            if page then
2964                plist[page.number] = rp
2965            end
2966        end
2967        references.nofrealpages = nofrealpages
2968    end
2969    return plist[p]
2970end
2971
2972references.realpageofpage = realpageofpage
2973
2974function references.checkedrealpage(r)
2975    if not plist then
2976        realpageofpage(r) -- just initialize
2977    end
2978    if not r then
2979        return texgetcount(c_realpageno)
2980    elseif r < 1 then
2981        return 1
2982    elseif r > nofrealpages then
2983        return nofrealpages
2984    else
2985        return r
2986    end
2987end
2988
2989-- use local ?
2990
2991local pages = allocate {
2992    [variables.firstpage]       = function() return counters.record("realpage")["first"]    end,
2993    [variables.previouspage]    = function() return counters.record("realpage")["previous"] end,
2994    [variables.nextpage]        = function() return counters.record("realpage")["next"]     end,
2995    [variables.lastpage]        = function() return counters.record("realpage")["last"]     end,
2996
2997    [variables.firstsubpage]    = function() return counters.record("subpage" )["first"]    end,
2998    [variables.previoussubpage] = function() return counters.record("subpage" )["previous"] end,
2999    [variables.nextsubpage]     = function() return counters.record("subpage" )["next"]     end,
3000    [variables.lastsubpage]     = function() return counters.record("subpage" )["last"]     end,
3001
3002    [variables.forward]         = function() return counters.record("realpage")["forward"]  end,
3003    [variables.backward]        = function() return counters.record("realpage")["backward"] end,
3004}
3005
3006references.pages = pages
3007
3008-- maybe some day i will merge this in the backend code with a testmode (so each
3009-- runner then implements a branch)
3010
3011runners["inner"] = function(var,actions)
3012    local r = var.r
3013    if r then
3014        actions.realpage = r
3015    end
3016end
3017
3018runners["special"] = function(var,actions)
3019    local handler = specials[var.special]
3020    return handler and handler(var,actions)
3021end
3022
3023runners["special operation"]                = runners["special"]
3024runners["special operation with arguments"] = runners["special"]
3025
3026function specials.internal(var,actions)
3027    local v = internals[tonumber(var.operation)]
3028    local r = v and v.references
3029    if r then
3030        local p = r.realpage
3031        if p then
3032-- setmetatableindex(actions,r)
3033            actions.realpage = p
3034            actions.view     = r.view
3035        end
3036    end
3037end
3038
3039specials.i = specials.internal
3040
3041function specials.page(var,actions)
3042    local o = var.operation
3043    local p = pages[o]
3044    if type(p) == "function" then
3045        p = p()
3046    else
3047        p = tonumber(realpageofpage(tonumber(o)))
3048    end
3049    if p then
3050        var.r = p
3051        actions.realpage = actions.realpage or p -- first wins
3052    end
3053end
3054
3055function specials.realpage(var,actions)
3056    local p = tonumber(var.operation)
3057    if p then
3058        var.r = p
3059        actions.realpage = actions.realpage or p -- first wins
3060    end
3061end
3062
3063function specials.userpage(var,actions)
3064    local p = tonumber(realpageofpage(var.operation))
3065    if p then
3066        var.r = p
3067        actions.realpage = actions.realpage or p -- first wins
3068    end
3069end
3070
3071function specials.deltapage(var,actions)
3072    local p = tonumber(var.operation)
3073    if p then
3074        p = references.checkedrealpage(p + texgetcount(c_realpageno))
3075        var.r = p
3076        actions.realpage = actions.realpage or p -- first wins
3077    end
3078end
3079
3080function specials.section(var,actions)
3081    local sectionname = var.arguments
3082    local destination = var.operation
3083    local internal    = structures.sections.internalreference(sectionname,destination)
3084    if internal then
3085        var.special   = "internal"
3086        var.operation = internal
3087        var.arguments = nil
3088        specials.internal(var,actions)
3089    end
3090end
3091
3092-- experimental:
3093
3094-- We can cache lowercased titles which saves a lot of time, but then
3095-- we can better have a global cache with weak keys.
3096
3097-- local lowercache = table.setmetatableindex(function(t,k)
3098--     local v = lpegmatch(p_lower,k)
3099--     t[k] = v
3100--     return v
3101-- end)
3102
3103local lowercache = false
3104
3105local function locate(list,askedkind,askedname,pattern)
3106    local kinds = lists.kinds
3107    local names = lists.names
3108    if askedkind and not kinds[askedkind] then
3109        return false
3110    end
3111    if askedname and not names[askedname] then
3112        return false
3113    end
3114    for i=1,#list do
3115        local entry    = list[i]
3116        local metadata = entry.metadata
3117        if metadata then
3118            local found = false
3119            if askedname then
3120                local name = metadata.name
3121                if name then
3122                    found = name == askedname
3123                end
3124            elseif askedkind then
3125                local kind = metadata.kind
3126                if kind then
3127                    found = kind == askedkind
3128                end
3129            end
3130            if found then
3131                local titledata = entry.titledata
3132                if titledata then
3133                    local title = titledata.title
3134                    if title then
3135                        if lowercache then
3136                            found = lpegmatch(pattern,lowercache[title])
3137                        else
3138                            found = lpegmatch(pattern,lpegmatch(p_lower,title))
3139                        end
3140                        if found then
3141                            return {
3142                                inner     = pattern,
3143                                kind      = "inner",
3144                                reference = pattern,
3145                                i         = entry,
3146                                p         = "",
3147                                r         = entry.references.realpage,
3148                            }
3149                        end
3150                    end
3151                end
3152            end
3153        end
3154    end
3155end
3156
3157function functions.match(var,actions)
3158    if not var.outer then
3159        local operation   = var.operation
3160        if operation and operation ~= "" then
3161            local operation   = lpegmatch(p_lower,operation)
3162            local list        = lists.collected
3163            local names       = false
3164            local kinds       = false
3165            local where, what = lpegmatch(p_splitter,operation)
3166            if where and what then
3167                local pattern = lpeg.finder(what)
3168                return
3169                    locate(list,false,where,pattern)
3170                 or locate(list,where,false,pattern)
3171                 or { error = "no match" }
3172            else
3173                local pattern = lpeg.finder(operation)
3174                -- todo: don't look at section and float in last pass
3175                return
3176                    locate(list,"section",false,pattern)
3177                 or locate(list,"float",false,pattern)
3178                 or locate(list,false,false,pattern)
3179                 or { error = "no match" }
3180            end
3181        end
3182    end
3183end
3184
3185-- needs a better split ^^^
3186
3187-- done differently now:
3188
3189function references.export(usedname) end
3190function references.import(usedname) end
3191function references.load  (usedname) end
3192
3193implement { name = "exportreferences", actions =references.export }
3194
3195-- better done here .... we don't insert/remove, just use a pointer
3196
3197local prefixstack = { "" }
3198local prefixlevel = 1
3199
3200local function pushreferenceprefix(prefix)
3201    prefixlevel = prefixlevel + 1
3202    prefixstack[prefixlevel] = prefix
3203    return prefix
3204end
3205
3206local function popreferenceprefix()
3207    prefixlevel = prefixlevel - 1
3208    if prefixlevel > 0 then
3209        return prefixstack[prefixlevel]
3210    else
3211        report_references("unable to pop referenceprefix")
3212        return ""
3213    end
3214end
3215
3216implement {
3217    name      = "pushreferenceprefix",
3218    actions   = { pushreferenceprefix, context }, -- we can use setmacro
3219    arguments = "string",
3220}
3221
3222implement {
3223    name      = "popreferenceprefix",
3224    actions   = { popreferenceprefix, context }, -- we can use setmacro
3225}
3226
3227-- a bit weird one, used for checkinh in the backend
3228
3229function references.pagefromhash(hash,destination,page,actions)
3230    local r = actions[1]
3231    if type(r) == "table" then
3232        r = r.i
3233        if r then
3234            r = r.references
3235            if r then
3236                local structure = r.structure
3237                if structure then
3238                    local namestack = r.namestack
3239                    if namestack then
3240                        for i=#namestack,1,-1 do
3241                            local d = namestack[i] .. ":" .. destination
3242                            page = hash[d]
3243                            if page then
3244                                destination = d
3245                                break
3246                            end
3247                        end
3248                    end
3249                end
3250            end
3251        end
3252    end
3253    return page, destination
3254end
3255
Source Browser ?
