Source Browser ?

1if not modules then modules = { } end modules ['strc-ref'] = {
2    version   = 1.001,
3    comment   = "companion to strc-ref.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- beware, this is a first step in the rewrite (just getting rid of
10-- the tuo file); later all access and parsing will also move to lua
11
12-- the useddata and pagedata names might change
13-- todo: pack exported data
14
15-- todo: autoload components when :::
16
17local format, gmatch, match, strip = string.format, string.gmatch, string.match, string.strip
18local floor = math.floor
19local rawget, tonumber, type, next = rawget, tonumber, type, next
20local lpegmatch = lpeg.match
21local insert, remove, copytable, sortedhash = table.insert, table.remove, table.copy, table.sortedhash
22local formatters = string.formatters
23local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match
24
25local allocate           = utilities.storage.allocate
26local mark               = utilities.storage.mark
27local setmetatableindex  = table.setmetatableindex
28
29local trace_referencing  = false  trackers.register("structures.referencing",             function(v) trace_referencing = v end)
30local trace_analyzing    = false  trackers.register("structures.referencing.analyzing",   function(v) trace_analyzing   = v end)
31local trace_identifying  = false  trackers.register("structures.referencing.identifying", function(v) trace_identifying = v end)
32local trace_importing    = false  trackers.register("structures.referencing.importing",   function(v) trace_importing   = v end)
33local trace_external     = false  trackers.register("structures.referencing.external",    function(v) trace_external    = v end)
34local trace_empty        = false  trackers.register("structures.referencing.empty",       function(v) trace_empty       = v end)
35
36local check_duplicates   = true
37
38directives.register("structures.referencing.checkduplicates", function(v) check_duplicates = v end)
39
40local report_references  = logs.reporter("references")
41local report_identifying = logs.reporter("references","identifying")
42local report_importing   = logs.reporter("references","importing")
43local report_external    = logs.reporter("references","external")
44local report_empty       = logs.reporter("references","empty")
45local report             = report_references
46
47local variables          = interfaces.variables
48local v_page             <const> = variables.page
49local v_auto             <const> = variables.auto
50local v_yes              <const> = variables.yes
51local v_name             <const> = variables.name
52
53local context            = context
54local commands           = commands
55local implement          = interfaces.implement
56
57local ctx_latelua        = context.latelua
58local ctx_doifelse       = commands.doifelse
59
60local texiscount         = tex.iscount
61local texgetcount        = tex.getcount
62local texsetcount        = tex.setcount
63local texconditionals    = tex.conditionals
64
65local getexpansion       = token.getexpansion
66
67local productcomponent   = resolvers.jobs.productcomponent
68local justacomponent     = resolvers.jobs.justacomponent
69
70local settings_to_array  = utilities.parsers.settings_to_array
71local settings_to_table  = utilities.parsers.settings_to_array_obey_fences
72local process_settings   = utilities.parsers.process_stripped_settings
73
74local unsetvalue         <const> = attributes.unsetvalue
75
76local structures         = structures
77local helpers            = structures.helpers
78local sections           = structures.sections
79local references         = structures.references
80local lists              = structures.lists
81local counters           = structures.counters
82
83local jobpositions       = job.positions
84local getpos             = jobpositions.getpos
85
86-- some might become local
87
88references.defined       = references.defined or allocate()
89
90local defined            = references.defined
91local derived            = allocate()
92local specials           = allocate()
93local functions          = allocate()
94local runners            = allocate()
95local internals          = allocate()
96local filters            = allocate()
97local executers          = allocate()
98local handlers           = allocate()
99local tobesaved          = allocate()
100local collected          = allocate()
101local tobereferred       = allocate()
102local referred           = allocate()
103local usedinternals      = allocate()
104local flaginternals      = allocate()
105local usedviews          = allocate()
106
107references.derived       = derived
108references.specials      = specials
109references.functions     = functions
110references.runners       = runners
111references.internals     = internals
112references.filters       = filters
113references.executers     = executers
114references.handlers      = handlers
115references.tobesaved     = tobesaved
116references.collected     = collected
117references.tobereferred  = tobereferred
118references.referred      = referred
119references.usedinternals = usedinternals
120references.flaginternals = flaginternals
121references.usedviews     = usedviews
122
123local splitreference     = references.splitreference
124local splitprefix        = references.splitcomponent -- replaces: references.splitprefix
125local prefixsplitter     = references.prefixsplitter
126local componentsplitter  = references.componentsplitter
127
128local currentreference   = nil
129
130local txtcatcodes        <const> = catcodes.numbers.txtcatcodes -- or just use "txtcatcodes"
131
132local c_realpageno               <const> = texiscount("realpageno")
133local c_locationcount            <const> = texiscount("locationcount")
134local c_locationorder            <const> = texiscount("locationorder")
135local c_lastdestinationattribute <const> = texiscount("lastdestinationattribute")
136
137local p_splitter = lpeg.splitat(":")
138local p_lower    = lpeg.patterns.utf8lower
139
140local context                      = context
141
142local ctx_pushcatcodes             = context.pushcatcodes
143local ctx_popcatcodes              = context.popcatcodes
144local ctx_dofinishreference        = context.dofinishreference
145local ctx_dofromurldescription     = context.dofromurldescription
146local ctx_dofromurlliteral         = context.dofromurlliteral
147local ctx_dofromfiledescription    = context.dofromfiledescription
148local ctx_dofromfileliteral        = context.dofromfileliteral
149local ctx_expandreferenceoperation = context.expandreferenceoperation
150local ctx_expandreferencearguments = context.expandreferencearguments
151local ctx_convertnumber            = context.convertnumber
152local ctx_emptyreference           = context.emptyreference
153
154storage.register("structures/references/defined", references.defined, "structures.references.defined")
155
156local initializers = { }
157local finalizers   = { }
158local somefound    = false -- so we don't report missing when we have a fresh start
159
160function references.registerinitializer(func) -- we could use a token register instead
161    initializers[#initializers+1] = func
162end
163
164function references.registerfinalizer(func) -- we could use a token register instead
165    finalizers[#finalizers+1] = func
166end
167
168local function initializer() -- can we use a tobesaved as metatable for collected?
169    tobesaved = references.tobesaved
170    collected = references.collected
171    for i=1,#initializers do
172        initializers[i](tobesaved,collected)
173    end
174    for prefix, list in next, collected do
175        for tag, data in next, list do
176            local r = data.references
177            local i = r.internal
178            if i then
179                internals[i]     = data
180                usedinternals[i] = r.used
181                local structure = r.structure
182                if structure then
183                    structure = resolvers.jobs.namelist(structure,namestack)
184                    r.namestack = structure
185                end
186            end
187        end
188    end
189    somefound = next(collected)
190end
191
192local function finalizer()
193    for i=1,#finalizers do
194        finalizers[i](tobesaved)
195    end
196    for prefix, list in next, tobesaved do
197        for tag, data in next, list do
198            local r = data.references
199            local i = r.internal
200            local f = flaginternals[i]
201            if f then
202                r.used = usedviews[i] or true
203            end
204        end
205    end
206end
207
208job.register('structures.references.collected', tobesaved, initializer, finalizer)
209
210local maxreferred = 1
211local nofreferred = 0
212
213local function initializer() -- can we use a tobesaved as metatable for collected?
214    tobereferred = references.tobereferred
215    referred     = references.referred
216    nofreferred  = #referred
217end
218
219-- no longer done this way
220
221-- references.resolvers = references.resolvers or { }
222-- local resolvers = references.resolvers
223--
224-- function resolvers.section(var)
225--     local vi = lists.collected[var.i[2]]
226--     if vi then
227--         var.i = vi
228--         var.r = (vi.references and vi.references.realpage) or (vi.pagedata and vi.pagedata.realpage) or 1
229--     else
230--         var.i = nil
231--         var.r = 1
232--     end
233-- end
234--
235-- resolvers.float       = resolvers.section
236-- resolvers.description = resolvers.section
237-- resolvers.formula     = resolvers.section
238-- resolvers.note        = resolvers.section
239--
240-- function resolvers.reference(var)
241--     local vi = var.i[2]
242--     if vi then
243--         var.i = vi
244--         var.r = (vi.references and vi.references.realpage) or (vi.pagedata and vi.pagedata.realpage) or 1
245--     else
246--         var.i = nil
247--         var.r = 1
248--     end
249-- end
250
251-- We make the array sparse (maybe a finalizer should optionally return a table) because
252-- there can be quite some page links involved. We only store one action number per page
253-- which is normally good enough for what we want (e.g. see above/below) and we do
254-- a combination of a binary search and traverse backwards. A previous implementation
255-- always did a traverse and was pretty slow on a large number of links (given that this
256-- methods was used). It took me about a day to locate this as a bottleneck in processing
257-- a 2500 page interactive document with 60 links per page. In that case, traversing
258-- thousands of slots per link then brings processing to a grinding halt (especially when
259-- there are no slots at all, which is the case in a first run).
260
261local sparsetobereferred = { }
262
263local function finalizer()
264    local lastr, lasti
265    local n = 0
266    for i=1,maxreferred do
267        local r = tobereferred[i]
268        if not lastr then
269            lastr = r
270            lasti = i
271        elseif r ~= lastr then
272            n = n + 1
273            sparsetobereferred[n] = { lastr, lasti }
274            lastr = r
275            lasti = i
276        end
277    end
278    if lastr then
279        n = n + 1
280        sparsetobereferred[n] = { lastr, lasti }
281    end
282end
283
284job.register('structures.references.referred', sparsetobereferred, initializer, finalizer)
285
286local function referredpage(n)
287    local max = nofreferred
288    if max > 0 then
289        -- find match
290        local min = 1
291        while true do
292            local mid = floor((min+max)/2)
293            local r = referred[mid]
294            local m = r[2]
295            if n == m then
296                return r[1]
297            elseif n > m then
298                min = mid + 1
299            else
300                max = mid - 1
301            end
302            if min > max then
303                break
304            end
305        end
306        -- find first previous
307        for i=min,1,-1 do
308            local r = referred[i]
309            if r and r[2] < n then
310                return r[1]
311            end
312        end
313    end
314    -- fallback
315    return texgetcount(c_realpageno)
316end
317
318references.referredpage = referredpage
319
320function references.registerpage(n) -- called in the backend code
321    if not tobereferred[n] then
322        if n > maxreferred then
323            maxreferred = n
324        end
325        tobereferred[n] = texgetcount(c_realpageno)
326    end
327end
328
329-- todo: delay split till later as in destinations we split anyway
330
331local orders, lastorder = { }, 0
332
333local function setnextorder(kind,name)
334    lastorder = 0
335    if kind and name then
336        local ok = orders[kind]
337        if not ok then
338            ok = { }
339            orders[kind] = ok
340        end
341        lastorder = (ok[name] or 0) + 1
342        ok[name] = lastorder
343    end
344    texsetcount("global",c_locationorder,lastorder)
345end
346
347
348local function setnextinternal(kind,name)
349    setnextorder(kind,name) -- always incremented with internal
350    local n = texgetcount(c_locationcount) + 1
351    texsetcount("global",c_locationcount,n)
352    return n
353end
354
355local function currentorder(kind,name)
356    return orders[kind] and orders[kind][name] or lastorder
357end
358
359-- local function setstructure(data)
360--     local structure = resolvers.jobs.currentnamehash()
361--     if structure then
362--         local references = data and data.references
363--         if references then
364--             references.structure = structure
365--         end
366--     end
367-- end
368
369-- local function setcomponent(data)
370--     -- we might consider doing this at the tex end, just like prefix
371--     local component = productcomponent() -- todo: maybe a list
372--  -- local component = resolvers.jobs.currentcomponent()
373--     if component then
374--         local references = data and data.references
375--         if references then
376--             references.component = component
377--             if references.prefix == component then
378--                 references.prefix = nil
379--             end
380--         end
381--         return component
382--     end
383-- end
384
385references.setnextorder    = setnextorder
386references.setnextinternal = setnextinternal
387references.currentorder    = currentorder
388references.setstructure    = resolvers.jobs.currentstructure -- yes or no here
389----------.setcomponent    = setcomponent
390
391implement {
392    name      = "setnextreferenceorder", -- used in strc-enu
393    actions   = setnextorder,
394    arguments = "2 strings",
395}
396
397-- implement {
398--     name      = "currentreferenceorder",
399--     actions = { currentorder, context },
400--     arguments = "2 strings",
401-- }
402
403implement {
404    name      = "setnextinternalreferences",
405    public    = true,
406    protected = true,
407    actions   = setnextinternal,
408    arguments = "2 arguments",
409}
410
411implement {
412    name      = "getinternalorderreference",
413    public    = true,
414    protected = true,
415    actions   = { currentorder, context },
416    arguments = "2 arguments",
417}
418
419local reported = setmetatableindex("table")
420
421function references.set(data)
422    local references = data.references
423    local reference  = references.reference
424    if not reference or reference == "" then
425     -- report_references("invalid reference") -- harmless
426        return 0
427    end
428    --
429    references.structure = resolvers.jobs.currentstructure()
430    --
431    local prefix = references.prefix or ""
432    local pd     = tobesaved[prefix] -- nicer is a metatable
433    if not pd then
434        pd = { }
435        tobesaved[prefix] = pd
436    end
437    local n = 0
438    local function action(ref)
439        if ref == "" then
440            -- skip
441        elseif check_duplicates and pd[ref] then
442            if not prefix then
443                prefix = ""
444            end
445            if not reported[prefix][ref] then
446                if prefix ~= "" then
447                    report_references("redundant reference %a in namespace %a",ref,prefix)
448                else
449                    report_references("redundant reference %a",ref)
450                end
451                reported[prefix][ref] = true
452            end
453        else
454            n = n + 1
455            pd[ref] = data
456            local r = data.references
457            ctx_dofinishreference(prefix or "",ref or "",r and r.internal or 0)
458         -- ctx_latelua(function() structures.references.enhance(prefix or ref,ref or "") end)
459        end
460    end
461    process_settings(reference,action)
462    return n > 0
463end
464
465-- function references.enhance(prefix,tag)
466--     local l = tobesaved[prefix][tag]
467--     if l then
468--         l.references.realpage = texgetcount(c_realpageno)
469--     end
470-- end
471
472local function synchronizepage(reference) -- non public helper
473    reference.realpage = texgetcount(c_realpageno)
474    if jobpositions.used() then
475        reference.x, reference.y = getpos()
476    end
477end
478
479references.synchronizepage = synchronizepage
480
481local function enhancereference(specification)
482    local prefix = specification.prefix
483    if prefix then
484        local entry = tobesaved[prefix]
485        if entry then
486            entry = entry[specification.tag]
487            if entry then
488                synchronizepage(entry.references)
489            else
490                -- normally a bug
491            end
492        else
493            -- normally a bug
494        end
495    else
496        -- normally a bug
497    end
498end
499
500references.enhance = enhancereference
501
502-- implement {
503--     name      = "enhancereference",
504--     arguments = "2 strings",
505--     actions   = function(prefix,tag)
506--        enhancereference { prefix = prefix, tag = tag }
507--     end,
508-- }
509
510implement {
511    name      = "deferredenhancereference",
512    arguments = "2 strings",
513    protected = true,
514    actions   = function(prefix,tag)
515        ctx_latelua { action = enhancereference, prefix = prefix, tag = tag }
516    end,
517}
518
519-- -- -- related to strc-ini.lua -- -- --
520
521-- no metatable here .. better be sparse
522
523local function register_from_list(collected,derived,external,namestack)
524    local derived_g = derived[""] -- global
525    local derived_p = nil
526    local derived_c = nil
527    local prefix    = nil
528--     local component = nil
529    local entry     = nil
530    if not derived_g then
531        derived_g = { }
532        derived[""] = derived_g
533    end
534    local action = trace_external and
535        function(s)
536            if structure then
537                for i=1,#structure do
538                    local si = structure[i]
539                 -- if si == component then
540                 --     -- skipped
541                 -- elseif si == prefix then
542                 --     -- skipped
543                 -- else
544                        local ds = derived[si]
545                        if not ds then
546                            derived[si] = { [s] = entry }
547                            report_external("reference %a, %s %a",s,"structure",si)
548                        elseif not ds[s] then
549                            ds[s] = entry
550                            report_external("reference %a, %s %a",s,"structure",si)
551                        end
552                 -- end
553                end
554            end
555            if derived_p and not derived_p[s] then
556                report_external("reference %a, %s %a",s,"prefix",prefix)
557                derived_p[s] = entry
558
559            end
560--             if derived_c and not derived_c[s] then
561--                 report_external("reference %a, %s %a",s,"component",component)
562--                 derived_c[s] = entry
563--             end
564            if not derived_g[s] then
565                report_external("reference %a, %s %a",s,"global","")
566                derived_g[s] = entry -- first wins
567            end
568        end
569    or
570        function(s)
571            if derived_p and not derived_p[s] then
572                derived_p[s] = entry
573            end
574--             if derived_c and not derived_c[s] then
575--                 derived_c[s] = entry
576--             end
577            if not derived_g[s] then
578                derived_g[s] = entry -- first wins
579            end
580            if structure then
581                for i=1,#structure do
582                    local si = structure[i]
583                    if si == component then
584                        -- skipped
585                    elseif si == prefix then
586                        -- skipped
587                    else
588                        local ds = derived[si]
589                        if not ds then
590                            derived[si] = { [s] = entry }
591                        elseif not ds[s] then
592                            ds[s] = entry
593                        end
594                    end
595                end
596            end
597        end
598    --
599    for i=1,#collected do
600        entry = collected[i]
601        local metadata = entry.metadata
602        if metadata then
603            local kind = metadata.kind -- why this check
604            if kind then
605                local references = entry.references
606                if references then
607                    local reference = references.reference
608                    if reference and reference ~= "" then
609                        local realpage = references.realpage
610                        if realpage then
611                            prefix    = references.prefix
612--                             component = references.component
613                            if prefix and prefix ~= "" then
614                                derived_p = derived[prefix]
615                                if not derived_p then
616                                    derived_p = { }
617                                    derived[prefix] = derived_p
618                                end
619                            else
620                                derived_p = nil
621                            end
622--                             if component and component ~= "" and component ~= prefix then
623--                                 derived_c = derived[component]
624--                                 if not derived_c then
625--                                     derived_c = { }
626--                                     derived[component] = derived_c
627--                                 end
628--                             else
629--                                 derived_c = nil
630--                             end
631                            structure = references.structure
632                            if structure then
633                                structure = resolvers.jobs.namelist(structure,namestack)
634                                references.namestack = structure
635                            end
636                            if trace_referencing or trace_external then
637                                report_references("list entry %a provides %a reference %a on realpage %a",i,kind,reference,realpage)
638                            end
639                            process_settings(reference,action)
640                        end
641                    end
642                end
643            end
644        end
645    end
646end
647
648function references.integrate(utilitydata)
649    local filename = utilitydata.comment.file
650    if filename and filename ~= environment.jobname then
651        -- lists are already internalized
652        local structures = utilitydata.structures
653        if structures then
654            local lists = structures.lists.collected
655            if lists then
656                register_from_list(lists,derived,filename,utilitydata)
657            end
658        end
659    end
660end
661
662references.registerinitializer(function()
663    -- the main document
664    register_from_list(lists.collected,derived) -- false, false
665end)
666
667-- tracing
668
669local function collectbypage(tracedpages)
670    -- lists
671    do
672        local collected = structures.lists.collected
673        local data      = nil
674        local function action(reference)
675            local prefix    = data.prefix
676            local component = data.component
677            local realpage  = data.realpage
678            if realpage then
679                local pagelist  = rawget(tracedpages,realpage)
680                local internal  = data.internal or 0
681                local prefix    = (prefix ~= "" and prefix) or (component ~= "" and component) or ""
682                local pagedata  = { prefix, reference, internal }
683                if pagelist then
684                    pagelist[#pagelist+1] = pagedata
685                else
686                    tracedpages[realpage] = { pagedata }
687                end
688                if internal > 0 then
689                    data.usedprefix = prefix
690                end
691            end
692        end
693        for i=1,#collected do
694            local entry = collected[i]
695            local metadata = entry.metadata
696            if metadata and metadata.kind then
697                data = entry.references
698                if data then
699                    local reference = data.reference
700                    if reference and reference ~= "" then
701                        process_settings(reference,action)
702                    end
703                end
704            end
705        end
706    end
707    -- references
708    do
709        for prefix, list in next, collected do
710            for reference, entry in next, list do
711                local data = entry.references
712                if data then
713                    local realpage = data.realpage
714                    local internal = data.internal or 0
715                    local pagelist = rawget(tracedpages,realpage)
716                    local pagedata = { prefix, reference, internal }
717                    if pagelist then
718                        pagelist[#pagelist+1] = pagedata
719                    else
720                        tracedpages[realpage] = { pagedata }
721                    end
722                    if internal > 0 then
723                        data.usedprefix = prefix
724                    end
725                end
726            end
727        end
728    end
729end
730
731references.tracedpages = table.setmetatableindex(allocate(),function(t,k)
732    if collectbypage then
733        collectbypage(t)
734        collectbypage = nil
735    end
736    return rawget(t,k)
737end)
738
739-- urls
740
741local urls      = references.urls or { }
742references.urls = urls
743local urldata   = urls.data or { }
744urls.data       = urldata
745
746local p_untexurl = Cs ( (
747    P("\\")/"" * (P("%")/"%%" + P(1))
748  + P(" ")/"%%20"
749  + P(1)
750)^1 )
751
752function urls.untex(url)
753    return lpegmatch(p_untexurl,url) or url
754end
755
756function urls.define(name,url,file,description)
757    if name and name ~= "" then
758     -- url = lpegmatch(replacer,url)
759        urldata[name] = { url or "", file or "", description or url or file or ""}
760    end
761end
762
763function urls.get(name)
764    local u = urldata[name]
765    if u then
766        local url, file = u[1], u[2]
767        if file and file ~= "" then
768            return formatters["%s/%s"](url,file)
769        else
770            return url
771        end
772    end
773end
774
775function urls.found(name)
776    return urldata[name]
777end
778
779local function geturl(name)
780    local url = urls.get(name)
781    if url and url ~= "" then
782        ctx_pushcatcodes(txtcatcodes)
783        context(url)
784        ctx_popcatcodes()
785    end
786end
787
788implement {
789    name      = "doifelseurldefined",
790    actions   = { urls.found, ctx_doifelse },
791    arguments = "string"
792}
793
794implement {
795    name      = "useurl",
796    actions   = urls.define,
797    arguments = "4 strings",
798}
799
800implement {
801    name      = "geturl",
802    actions   = geturl,
803    arguments = "string",
804}
805
806-- files
807
808local files      = references.files or { }
809references.files = files
810local filedata   = files.data or { }
811files.data       = filedata
812
813function files.define(name,file,description)
814    if name and name ~= "" then
815        filedata[name] = { file or "", description or file or "" }
816    end
817end
818
819function files.get(name,method,space) -- method: none, before, after, both, space: yes/no
820    local f = filedata[name]
821    if f then
822        context(f[1])
823    end
824end
825
826function files.found(name)
827    return filedata[name]
828end
829
830local function getfile(name)
831    local fil = files.get(name)
832    if fil and fil ~= "" then
833        ctx_pushcatcodes(txtcatcodes)
834        context(fil)
835        ctx_popcatcodes()
836    end
837end
838
839implement {
840    name      = "doifelsefiledefined",
841    actions   = { files.found, ctx_doifelse },
842    arguments = "string"
843}
844
845implement {
846    name      = "usefile",
847    actions   = files.define,
848    arguments = "3 strings"
849}
850
851implement {
852    name      = "getfile",
853    actions   = getfile,
854    arguments = "string"
855}
856
857-- helpers
858
859function references.checkedfile(whatever) -- return whatever if not resolved
860    if whatever then
861        local w = filedata[whatever]
862        if w then
863            return w[1]
864        else
865            return whatever
866        end
867    end
868end
869
870function references.checkedurl(whatever) -- return whatever if not resolved
871    if whatever then
872        local w = urldata[whatever]
873        if w then
874            local u, f = w[1], w[2]
875            if f and f ~= "" then
876                return u .. "/" .. f
877            else
878                return u
879            end
880        else
881            return whatever
882        end
883    end
884end
885
886function references.checkedfileorurl(whatever,default) -- return nil, nil if not resolved
887    if whatever then
888        local w = filedata[whatever]
889        if w then
890            return w[1], nil
891        else
892            local w = urldata[whatever]
893            if w then
894                local u, f = w[1], w[2]
895                if f and f ~= "" then
896                    return nil, u .. "/" .. f
897                else
898                    return nil, u
899                end
900            end
901        end
902    end
903    return default
904end
905
906-- programs
907
908local programs      = references.programs or { }
909references.programs = programs
910local programdata   = programs.data or { }
911programs.data       = programdata
912
913function programs.define(name,file,description)
914    if name and name ~= "" then
915        programdata[name] = { file or "", description or file or ""}
916    end
917end
918
919function programs.get(name)
920    local f = programdata[name]
921    return f and f[1]
922end
923
924function references.checkedprogram(whatever) -- return whatever if not resolved
925    if whatever then
926        local w = programdata[whatever]
927        if w then
928            return w[1]
929        else
930            return whatever
931        end
932    end
933end
934
935implement {
936    name      = "defineprogram",
937    actions   = programs.define,
938    arguments = "3 strings",
939}
940
941local function getprogram(name)
942    local p = programdata[name]
943    if p then
944        context(p[1])
945    end
946end
947
948implement {
949    name      = "getprogram",
950    actions   = getprogram,
951    arguments = "string"
952}
953
954-- shared by urls and files
955
956function references.from(name)
957    local u = urldata[name]
958    if u then
959        local url, file, description = u[1], u[2], u[3]
960        if description ~= "" then
961            return description
962            -- ok
963        elseif file and file ~= "" then
964            return url .. "/" .. file
965        else
966            return url
967        end
968    else
969        local f = filedata[name]
970        if f then
971            local file, description = f[1], f[2]
972            if description ~= "" then
973                return description
974            else
975                return file
976            end
977        end
978    end
979end
980
981local function from(name)
982    local u = urldata[name]
983    if u then
984        local url, file, description = u[1], u[2], u[3]
985        if description ~= "" then
986            ctx_dofromurldescription(description)
987            -- ok
988        elseif file and file ~= "" then
989            ctx_dofromurlliteral(url .. "/" .. file)
990        else
991            ctx_dofromurlliteral(url)
992        end
993    else
994        local f = filedata[name]
995        if f then
996            local file, description = f[1], f[2]
997            if description ~= "" then
998                ctx_dofromfiledescription(description)
999            else
1000                ctx_dofromfileliteral(file)
1001            end
1002        end
1003    end
1004end
1005
1006implement {
1007    name      = "from",
1008    actions   = from,
1009    arguments = "string"
1010}
1011
1012function references.define(prefix,reference,list)
1013    local d = defined[prefix] if not d then d = { } defined[prefix] = d end
1014    d[reference] = list
1015end
1016
1017function references.reset(prefix,reference)
1018    local d = defined[prefix]
1019    if d then
1020        d[reference] = nil
1021    end
1022end
1023
1024implement {
1025    name      = "definereference",
1026    actions   = references.define,
1027    arguments = "3 strings",
1028}
1029
1030implement {
1031    name      = "resetreference",
1032    actions   = references.reset,
1033    arguments = "2 strings",
1034}
1035
1036setmetatableindex(defined,"table")
1037
1038local function resolve(prefix,reference,args,set) -- we start with prefix,reference
1039    if reference and reference ~= "" then
1040        if not set then
1041            set = { prefix = prefix, reference = reference }
1042        else
1043            if not set.reference then set.reference = reference end
1044            if not set.prefix    then set.prefix    = prefix    end
1045        end
1046     -- local r = settings_to_array(reference)
1047        local r = settings_to_table(reference) -- maybe option to honor () []
1048        for i=1,#r do
1049            local ri = r[i]
1050            local d = defined[prefix][ri] or defined[""][ri]
1051            if d then
1052                d = getexpansion(d)
1053                resolve(prefix,d,nil,set)
1054            else
1055                local var = splitreference(ri)
1056                if var then
1057                    var.reference = ri
1058                    local vo = var.outer
1059                    local vi = var.inner
1060                    -- we catch this here .. it's a way to pass references with commas
1061                    if vi == "name" then
1062                        local arguments = var.arguments
1063                        if arguments then
1064                            vi            = arguments
1065                            var.inner     = arguments
1066                            var.reference = arguments
1067                            var.arguments = nil
1068                        end
1069                    elseif var.special == "name" then
1070                        local operation = var.operation
1071                        if operation then
1072                            vi            = operation
1073                            var.inner     = operation
1074                            var.reference = operation
1075                            var.operation = nil
1076                            var.special   = nil
1077                        end
1078                    end
1079                    -- end of catch
1080                    if not vo and vi then
1081                        -- to be checked
1082                        d = defined[prefix][vi] or defined[""][vi]
1083                        --
1084                        if d then
1085                            d = getexpansion(d)
1086                            resolve(prefix,d,var.arguments,set) -- args can be nil
1087                        else
1088                            if args then var.arguments = args end
1089                            set[#set+1] = var
1090                        end
1091                    else
1092                        if args then var.arguments = args end
1093                        set[#set+1] = var
1094                    end
1095                else
1096                --  report_references("funny pattern %a",ri)
1097                end
1098            end
1099        end
1100        return set
1101    else
1102        return { }
1103    end
1104end
1105
1106-- prefix == "" is valid prefix which saves multistep lookup
1107
1108references.currentset = nil
1109
1110local externals = { }
1111
1112-- we have prefixes but also components:
1113--
1114-- :    prefix
1115-- ::   always external
1116-- :::  internal (for products) or external (for components)
1117
1118local function loadexternalreferences(name,utilitydata)
1119    local struc = utilitydata.structures
1120    if struc then
1121        local external = struc.references.collected -- direct references
1122        local lists    = struc.lists.collected      -- indirect references (derived)
1123        local pages    = struc.pages.collected      -- pagenumber data
1124        local sections = struc.sections.collected
1125        -- a bit weird one, as we don't have the externals in the collected
1126        for prefix, set in next, external do
1127            -- prefix is currently not used, only in reporting
1128            local usedprefix = prefix == "" and name or prefix -- this can clash!
1129            for reference, data in next, set do
1130                if trace_importing then
1131                    report_importing("registering %a reference, kind %a, name %a, prefix %a, reference %a",
1132                        "external","regular",name,usedprefix,reference)
1133                end
1134                -- weird code
1135             -- local section  = reference.section
1136             -- local realpage = reference.realpage
1137             -- if section then
1138             --     reference.sectiondata = lists[section]
1139             -- end
1140             -- if realpage then
1141             --     reference.pagedata = pages[realpage]
1142             -- end
1143                -- better?
1144                local references = data.references -- hm. plural
1145                if references then
1146                    local section  = references.section
1147                    local realpage = references.realpage
1148                    if not references.sectiondata then
1149                        references.sectiondata = sections[references.section or false]
1150                    end
1151                    if not references.pagedata then
1152                        references.pagedata = pages[references.realpage or false]
1153                    end
1154                end
1155            end
1156        end
1157        -- maybe store utilitydata in metatable so that we can access all
1158        for i=1,#lists do
1159            local entry      = lists[i]
1160            local metadata   = entry.metadata
1161            local references = entry.references
1162            if metadata and references then
1163                local reference = references.reference
1164                if reference and reference ~= "" then
1165                    local kind     = metadata.kind
1166                    local realpage = references.realpage
1167                    if kind and realpage then
1168                        -- quite likely already done so this might go
1169                        if pages and not references.pagedata then
1170                            references.pagedata = pages[references.realpage or false]
1171                        end
1172                        if sections and not references.sectiondata then
1173                            references.sectiondata = sections[references.section or false]
1174                        end
1175                        --
1176                        local prefix = references.prefix or ""
1177                        if prefix == "" then
1178                            prefix = name -- this can clash!
1179                        end
1180                        local target = external[prefix]
1181                        if not target then
1182                            target = { }
1183                            external[prefix] = target
1184                        end
1185                        local function action(s)
1186                            if trace_importing then
1187                                report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1188                                    "external",kind,name,prefix,s)
1189                            end
1190                            target[s] = target[s] or entry
1191                        end
1192                        process_settings(reference,action)
1193                    end
1194                end
1195            end
1196        end
1197        externals[name] = external
1198        return external
1199    end
1200end
1201
1202local externalfiles = { }
1203
1204setmetatableindex(externalfiles, function(t,k)
1205    local v = filedata[k]
1206    if not v then
1207        v = { k, k }
1208    end
1209    externalfiles[k] = v
1210    return v
1211end)
1212
1213setmetatableindex(externals, function(t,k) -- either or not automatically
1214    local filename = externalfiles[k][1] -- filename
1215    local fullname = file.replacesuffix(filename,"tuc")
1216    if lfs.isfile(fullname) then -- todo: use other locator
1217        local utilitydata = job.loadother(fullname)
1218        if utilitydata then
1219            local external = loadexternalreferences(k,utilitydata)
1220            t[k] = external or false
1221            return external
1222        end
1223    end
1224    t[k] = false
1225    return false
1226end)
1227
1228local productdata = allocate { -- will go
1229    productreferences   = { },
1230    componentreferences = { },
1231    components          = { },
1232}
1233
1234references.productdata = productdata
1235
1236local function loadproductreferences(productname,componentname,utilitydata)
1237    local struc = utilitydata.structures
1238    if struc then
1239        local productreferences = struc.references.collected -- direct references
1240        local lists    = struc.lists.collected      -- indirect references (derived)
1241        local pages    = struc.pages.collected      -- pagenumber data
1242        local sections = struc.sections.collected   -- section number data
1243        -- we use indirect tables to save room but as they are eventually
1244        -- just references we resolve them to data here (the mechanisms
1245        -- that use this data check for indirectness)
1246        for prefix, set in next, productreferences do
1247            for reference, data in next, set do
1248                if trace_importing then
1249                    report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1250                        "product","regular",productname,prefix,reference)
1251                end
1252                local references = data.references
1253                if references then
1254                    if not references.sectiondata then
1255                        references.sectiondata = sections[references.section or false]
1256                    end
1257                    if not references.pagedata then
1258                        references.pagedata = pages[references.realpage or false]
1259                    end
1260                end
1261            end
1262        end
1263        --
1264        local componentreferences = { }
1265        for i=1,#lists do
1266            local entry      = lists[i]
1267            local metadata   = entry.metadata
1268            local references = entry.references
1269            if metadata and references then
1270                local reference = references.reference
1271                if reference and reference ~= "" then
1272                    local kind     = metadata.kind
1273                    local realpage = references.realpage
1274                    if kind and realpage then
1275                        local prefix    = references.prefix or ""
1276                        local component = references.component
1277
1278                    -- local structure = references.structure
1279                    -- print("!!!!!!!!",structure)
1280                    -- if structure then
1281                    --     inspect(resolvers.job.namelist(structure))
1282                    -- end
1283
1284                        local ctarget, ptarget
1285                        if not component or component == componentname then
1286                            -- skip
1287                        else
1288                            -- one level up
1289                            local external = componentreferences[component]
1290                            if not external then
1291                                external = { }
1292                                componentreferences[component] = external
1293                            end
1294                            if component == prefix then
1295                                prefix = ""
1296                            end
1297                            ctarget = external[prefix]
1298                            if not ctarget then
1299                                ctarget = { }
1300                                external[prefix] = ctarget
1301                            end
1302                        end
1303                        ptarget = productreferences[prefix]
1304                        if not ptarget then
1305                            ptarget = { }
1306                            productreferences[prefix] = ptarget
1307                        end
1308                        local function action(s)
1309                            if ptarget then
1310                                if trace_importing then
1311                                    report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1312                                        "product",kind,productname,prefix,s)
1313                                end
1314                                ptarget[s] = ptarget[s] or entry
1315                            end
1316                            if ctarget then
1317                                if trace_importing then
1318                                    report_importing("registering %s reference, kind %a, name %a, prefix %a, referenc %a",
1319                                        "component",kind,productname,prefix,s)
1320                                end
1321                                ctarget[s] = ctarget[s] or entry
1322                            end
1323                        end
1324                        process_settings(reference,action)
1325                    end
1326                end
1327            end
1328        end
1329        productdata.productreferences   = productreferences -- not yet used
1330        productdata.componentreferences = componentreferences
1331    end
1332end
1333
1334local function loadproductvariables(product,component,utilitydata)
1335    local struc = utilitydata.structures
1336    if struc then
1337        local lists = struc.lists and struc.lists.collected
1338        if lists then
1339            local pages = struc.pages and struc.pages.collected
1340            for i=1,#lists do
1341                local li = lists[i]
1342                if li.metadata.kind == "section" and li.references.component == component then
1343                    local firstsection = li
1344                    if firstsection.numberdata then
1345                        local numbers = firstsection.numberdata.numbers
1346                        if numbers then
1347                            if trace_importing then
1348                                report_importing("initializing section number to %:t",numbers)
1349                            end
1350                            productdata.firstsection = firstsection
1351                            structures.documents.preset(numbers)
1352                        end
1353                    end
1354                    if pages and firstsection.references then
1355                        local firstpage = pages[firstsection.references.realpage]
1356                        local number = firstpage and firstpage.number
1357                        if number then
1358                            if trace_importing then
1359                                report_importing("initializing page number to %a",number)
1360                            end
1361                            productdata.firstpage = firstpage
1362                            counters.set("userpage",1,number)
1363                        end
1364                    end
1365                    break
1366                end
1367            end
1368        end
1369    end
1370end
1371
1372local function componentlist(tree,target)
1373    local branches = tree and tree.branches
1374    if branches then
1375        for i=1,#branches do
1376            local branch = branches[i]
1377            local type = branch.type
1378            if type == "component" then
1379                if target then
1380                    target[#target+1] = branch.name
1381                else
1382                    target = { branch.name }
1383                end
1384            elseif type == "product" or type == "component" then
1385                target = componentlist(branch,target)
1386            end
1387        end
1388    end
1389    return target
1390end
1391
1392local function loadproductcomponents(product,component,utilitydata)
1393    local job = utilitydata.job
1394    productdata.components = componentlist(job and job.structure and job.structure.collected) or { }
1395end
1396
1397references.registerinitializer(function(tobesaved,collected)
1398    -- not that much related to tobesaved or collected
1399    productdata.components = componentlist(job.structure.collected) or { }
1400end)
1401
1402function references.loadpresets(product,component) -- we can consider a special components hash
1403    if product and component and product~= "" and component ~= "" and not productdata.product then -- maybe: productdata.filename ~= filename
1404        productdata.product   = product
1405        productdata.component = component
1406        -- todo: use other locator
1407        local utilitydata = job.loadother(product)
1408        if utilitydata then
1409            if trace_importing then
1410                report_importing("loading references for component %a of product %a",component,product)
1411            end
1412            loadproductvariables (product,component,utilitydata)
1413            loadproductcomponents(product,component,utilitydata)
1414            loadproductreferences(product,component,utilitydata)
1415        --             loadproductcomponents(product,component,utilitydata)
1416        end
1417    end
1418end
1419
1420references.productdata = productdata
1421
1422implement {
1423    name      = "usereferences",
1424    public    = true,
1425    protected = true,
1426    arguments = "optional",
1427    actions   = function(product)
1428        local utilitydata = job.loadother(product)
1429        if utilitydata then
1430            loadexternalreferences(product,utilitydata)
1431        end
1432    end
1433}
1434
1435local ctx_useproductuseproduct = commands.useproduct
1436
1437if ctx_useproduct then
1438
1439    local function newuseproduct(product)
1440        ctx_useproduct(product)
1441        if texconditionals.autocrossfilereferences then
1442            local component = justacomponent()
1443            if component then
1444                if trace_referencing or trace_importing then
1445                    report_references("loading presets for component %a of product %a",component,product)
1446                end
1447                references.loadpresets(product,component)
1448            end
1449        end
1450    end
1451
1452    implement {
1453        name      = "useproduct",
1454        actions   = newuseproduct,
1455        arguments = "optional",
1456        overload  = true,
1457    }
1458
1459end
1460
1461-- productdata.firstsection.numberdata.numbers
1462-- productdata.firstpage.number
1463
1464local function report_identify_special(set,var,i,type)
1465    local reference = set.reference
1466    local prefix    = set.prefix or ""
1467    local special   = var.special
1468    local error     = var.error
1469    local kind      = var.kind
1470    if error then
1471        report_identifying("type %a, reference %a, index %a, prefix %a, special %a, error %a",type,reference,i,prefix,special,error)
1472    else
1473        report_identifying("type %a, reference %a, index %a, prefix %a, special %a, kind %a",type,reference,i,prefix,special,kind)
1474    end
1475end
1476
1477local function report_identify_arguments(set,var,i,type)
1478    local reference = set.reference
1479    local prefix    = set.prefix or ""
1480    local arguments = var.arguments
1481    local error     = var.error
1482    local kind      = var.kind
1483    if error then
1484        report_identifying("type %a, reference %a, index %a, prefix %a, arguments %a, error %a",type,reference,i,prefix,arguments,error)
1485    else
1486        report_identifying("type %a, reference %a, index %a, prefix %a, arguments %a, kind %a",type,reference,i,prefix,arguments,kind)
1487    end
1488end
1489
1490local function report_identify_outer(set,var,i,type)
1491    local reference = set.reference
1492    local prefix    = set.prefix or ""
1493    local outer     = var.outer
1494    local error     = var.error
1495    local kind      = var.kind
1496    if outer then
1497        if error then
1498            report_identifying("type %a, reference %a, index %a, prefix %a, outer %a, error %a",type,reference,i,prefix,outer,error)
1499        else
1500            report_identifying("type %a, reference %a, index %a, prefix %a, outer %a, kind %a",type,reference,i,prefix,outer,kind)
1501        end
1502    else
1503        if error then
1504            report_identifying("type %a, reference %a, index %a, prefix %a, error %a",type,reference,i,prefix,error)
1505        else
1506            report_identifying("type %a, reference %a, index %a, prefix %a, kind %a",type,reference,i,prefix,kind)
1507        end
1508    end
1509end
1510
1511local function report_identify_auto(set,var,i,type)
1512    report_identifying("type %a, reference %a, using auto outer %a",type,reference,var.outer)
1513end
1514
1515local function identify_special(set,var,i)
1516    local special = var.special
1517    local s = specials[special]
1518    if s then
1519        local outer     = var.outer
1520        local operation = var.operation
1521        local arguments = var.arguments
1522        if outer then
1523            if operation then
1524                -- special(outer::operation)
1525                var.kind = "special outer with operation"
1526            else
1527                -- special()
1528                var.kind = "special outer"
1529            end
1530            var.f = outer
1531        elseif operation then
1532            if arguments then
1533                -- special(operation{argument,argument})
1534                var.kind = "special operation with arguments"
1535            else
1536                -- special(operation)
1537                var.kind = "special operation"
1538            end
1539        else
1540            -- special()
1541            var.kind = "special"
1542        end
1543        if trace_identifying then
1544            report_identify_special(set,var,i,"1a")
1545        end
1546    else
1547        var.error = "unknown special"
1548    end
1549    return var
1550end
1551
1552local function identify_arguments(set,var,i)
1553    local s = specials[var.inner]
1554    if s then
1555        -- inner{argument}
1556        var.kind = "special operation with arguments"
1557    else
1558        var.error = "unknown inner or special"
1559    end
1560    if trace_identifying then
1561        report_identify_arguments(set,var,i,"3a")
1562    end
1563    return var
1564end
1565
1566-- needs checking: if we don't do too much (redundant) checking now
1567-- inner ... we could move the prefix logic into the parser so that we have 'm for each entry
1568-- foo:bar -> foo == prefix (first we try the global one)
1569-- -:bar   -> ignore prefix
1570
1571local function finish_inner(set,var,p,i)
1572    local external = i.references.external
1573    local realpage = (i.references and i.references.realpage) or (i.pagedata and i.pagedata.realpage) or 1
1574    if external then
1575        var.kind     = "outer with inner"
1576        var.external = true
1577        var.outer    = external
1578        set.external = true -- probably not needed
1579    else
1580        var.kind = "inner"
1581        var.p    = p
1582    end
1583    var.r = realpage
1584    var.i = i
1585    return var
1586end
1587
1588local function identify_inner(set,var,prefix,collected,derived)
1589    local inner = var.inner
1590    -- the next test is a safeguard when references are auto loaded from outer
1591    if not inner or inner == "" then
1592        return false
1593    end
1594    local splitprefix, splitinner = lpegmatch(prefixsplitter,inner)
1595    if splitprefix and splitinner then
1596        -- we check for a prefix:reference instance in the regular set of collected
1597        -- references; a special case is -: which forces a lookup in the global list
1598        if splitprefix == "-" then
1599            local i = collected[""]
1600            if i then
1601                i = i[splitinner]
1602                if i then
1603                    return finish_inner(set,var,"",i)
1604                end
1605            end
1606        end
1607        local i = collected[splitprefix]
1608        if i then
1609            i = i[splitinner]
1610            if i then
1611                return finish_inner(set,var,splitprefix,i)
1612            end
1613        end
1614        if derived then
1615            -- next we look for a reference in the regular set of collected references
1616            -- using the prefix that is active at this moment (so we overload the given
1617            -- these are taken from other data structures (like lists)
1618            if splitprefix == "-" then
1619-- loop over the current structure list reversed
1620-- and end with "" .. needs example
1621                local i = derived[""]
1622                if i then
1623                    i = i[splitinner]
1624                    if i then
1625                        return finish_inner(set,var,"",i)
1626                    end
1627                end
1628            end
1629            local i = derived[splitprefix]
1630            if i then
1631                i = i[splitinner]
1632                if i then
1633                    return finish_inner(set,var,splitprefix,i)
1634                end
1635            end
1636        end
1637    end
1638    -- we now ignore the split prefix and treat the whole inner as a potential
1639    -- reference into the global list
1640    local i = collected[prefix]
1641    if i then
1642        i = i[inner]
1643        if i then
1644            return finish_inner(set,var,prefix,i)
1645        end
1646    end
1647    if not i and derived then
1648        -- and if not found we look in the derived references
1649        local i = derived[prefix]
1650        if i then
1651            i = i[inner]
1652            if i then
1653                return finish_inner(set,var,prefix,i)
1654            end
1655        end
1656        --
1657        local n, list = resolvers.jobs.currentstructure()
1658        if list then
1659            for i=#list,1,-1 do
1660                local l = list[i]
1661                local i = derived[l]
1662                if i then
1663                    i = i[inner]
1664                    if i then
1665                     -- return finish_inner(set,var,l,i)
1666                        local p = i.references.prefix or l -- which is what we use in the destination
1667                        return finish_inner(set,var,p,i)
1668                    end
1669                end
1670            end
1671        end
1672    end
1673    return false
1674end
1675
1676local function unprefixed_inner(set,var,prefix,collected,derived,tobesaved)
1677    local inner = var.inner
1678    local s = specials[inner]
1679    if s then
1680        var.kind = "special"
1681    else
1682        local i = (collected and collected[""] and collected[""][inner]) or
1683                  (derived   and derived  [""] and derived  [""][inner]) or
1684                  (tobesaved and tobesaved[""] and tobesaved[""][inner])
1685        if i then
1686            finish_inner(set,var,"",i)
1687        else
1688            var.error = "unknown inner or special"
1689        end
1690    end
1691    return var
1692end
1693
1694local function identify_auto(set,var,i)
1695    local inner  = var.inner
1696    local prefix = set.prefix or ""
1697    if prefix == "" then
1698        prefix, inner = lpegmatch(p_splitter,inner)
1699        if not inner then
1700            inner  = var.inner
1701            prefix = false
1702        end
1703    end
1704    local function okay(o,where)
1705        var.outer = o
1706        if trace_identifying then
1707            report_identify_auto(set,var,i,where)
1708        end
1709        return true
1710    end
1711    for o, odata in sortedhash(externals) do
1712        if o == outer or not odata then
1713            -- skip this one
1714        elseif prefix and prefix ~= "" then
1715            local pdata = odata[prefix]
1716            if pdata then
1717                local r = pdata[inner]
1718                if r then
1719                    return okay(o,"2h")
1720                end
1721            end
1722            -- escape from prefix
1723        --  local pdata = odata[""]
1724        --  if pdata then
1725        --      local r = pdata[inner]
1726        --      if r then
1727        --          return okay(o,"2h")
1728        --      end
1729        --  end
1730        else
1731            for p, pdata in sortedhash(odata) do
1732                local r = pdata[inner]
1733                if r then
1734                    return okay(o,"2j")
1735                end
1736            end
1737            local pdata = odata[""]
1738            if pdata then
1739                r = pdata[inner]
1740                if r then
1741                    return okay(o,"2k")
1742                end
1743            end
1744        end
1745    end
1746    return false
1747end
1748
1749local function identify_outer(set,var,i)
1750    local outer    = var.outer
1751    local inner    = var.inner
1752    local external = externals[outer]
1753    if external then
1754        local v = identify_inner(set,var,"",external)
1755        if v then
1756            v.kind = "outer with inner"
1757            set.external = true
1758            if trace_identifying then
1759                report_identify_outer(set,v,i,"2a")
1760            end
1761            return v
1762        end
1763-- weird too (we really need to check how this table is build
1764        local v = identify_inner(set,var,var.outer,external)
1765        if v then
1766            v.kind = "outer with inner"
1767            set.external = true
1768            if trace_identifying then
1769                report_identify_outer(set,v,i,"2c")
1770            end
1771            return v
1772        end
1773--
1774        -- somewhat rubish: we use outer as first step in the externals table so it makes no
1775        -- sense to have it as prefix so the next could be an option
1776        local external = external[""]
1777        if external then
1778            local v = identify_inner(set,var,var.outer,external)
1779            if v then
1780                v.kind = "outer with inner"
1781                set.external = true
1782                if trace_identifying then
1783                    report_identify_outer(set,v,i,"2b")
1784                end
1785                return v
1786            end
1787        end
1788    end
1789    local external = productdata.productreferences[outer]
1790    if external then
1791        local vi = external[inner]
1792        if vi then
1793            var.kind = "outer with inner"
1794            var.i = vi
1795            set.external = true
1796            if trace_identifying then
1797                report_identify_outer(set,var,i,"2d")
1798            end
1799            return var
1800        end
1801    end
1802    local external = productdata.componentreferences[outer]
1803    if external then
1804        local v = identify_inner(set,var,"",external)
1805        if v then
1806            v.kind = "outer with inner"
1807            set.external = true
1808            if trace_identifying then
1809                report_identify_outer(set,v,i,"2c")
1810            end
1811            return v
1812        end
1813    end
1814    -- the rest
1815    local special   = var.special
1816    local arguments = var.arguments
1817    local operation = var.operation
1818    if inner then
1819        -- tricky: in this case we can only use views when we're sure that all inners
1820        -- are flushed in the outer document so that should become an option
1821        if arguments then
1822            -- outer::inner{argument}
1823            var.kind = "outer with inner with arguments"
1824        else
1825            -- outer::inner
1826            var.kind = "outer with inner"
1827        end
1828        var.i = inner
1829        var.f = outer
1830        if type(inner) == "table" then
1831            -- can this really happen?
1832            var.r = (inner.references and inner.references.realpage) or (inner.pagedata and inner.pagedata.realpage) or 1
1833        else
1834            var.r = 1
1835        end
1836        if trace_identifying then
1837            report_identify_outer(set,var,i,"2e")
1838        end
1839    elseif special then
1840        local s = specials[special]
1841        if s then
1842            if operation then
1843                if arguments then
1844                    -- outer::special(operation{argument,argument})
1845                    var.kind = "outer with special and operation and arguments"
1846                else
1847                    -- outer::special(operation)
1848                    var.kind = "outer with special and operation"
1849                end
1850            else
1851                -- outer::special()
1852                var.kind = "outer with special"
1853            end
1854            var.f = outer
1855        else
1856            var.error = "unknown outer with special"
1857        end
1858        if trace_identifying then
1859            report_identify_outer(set,var,i,"2f")
1860        end
1861    else
1862        -- outer::
1863        var.kind = "outer"
1864        var.f = outer
1865        if trace_identifying then
1866            report_identify_outer(set,var,i,"2g")
1867        end
1868    end
1869    return var
1870end
1871
1872-- todo: avoid copy
1873
1874local function identify_inner_or_outer(set,var,i)
1875    -- here we fall back on product data
1876    local inner = var.inner
1877    if inner and inner ~= "" then
1878
1879        -- first we look up in collected and derived using the current prefix
1880
1881        local prefix = set.prefix
1882
1883        local v = identify_inner(set,var,set.prefix,collected,derived)
1884        if v then
1885            if trace_identifying then
1886                report_identify_outer(set,v,i,"4a")
1887            end
1888            return v
1889        end
1890
1891--         if var.inner == var.reference and identify_auto(set,var,i) and var.outer and var.outer ~= "" then
1892        if var.inner == var.reference and var.outer and var.outer ~= "" then
1893            return identify_outer(set,var,i)
1894        end
1895
1896        -- nest we look at each component (but we can omit the already consulted one
1897
1898        local jobstructure = job.structure
1899        local components   = jobstructure and jobstructure.components
1900        if components then
1901            for c=1,#components do
1902                local component = components[c]
1903                if component ~= prefix then
1904                    local v = identify_inner(set,var,component,collected,derived)
1905                    if v then
1906                        if trace_identifying then
1907                            report_identify_outer(set,var,i,"4b")
1908                        end
1909                        return v
1910                    end
1911                end
1912            end
1913        end
1914
1915        -- as a last resort we will consult the global lists
1916        local v = unprefixed_inner(set,var,"",collected,derived,tobesaved)
1917        if v then
1918            if trace_identifying then
1919                report_identify_outer(set,v,i,"4c")
1920            end
1921            return v
1922        end
1923
1924        -- not it gets bad ... we need to look in external files ... keep in mind that
1925        -- we can best use explicit references for this ... we might issue a warning
1926
1927        local componentreferences = productdata.componentreferences
1928        local productreferences = productdata.productreferences
1929        local components = productdata.components
1930        if components and componentreferences then
1931            for c=1,#components do
1932                local component = components[c]
1933                local data = componentreferences[component]
1934                if data then
1935                    local d = data[""]
1936                    local vi = d and d[inner]
1937                    if vi then
1938                        var.outer = component
1939                        var.i = vi
1940                        var.kind = "outer with inner"
1941                        set.external = true
1942                        if trace_identifying then
1943                            report_identify_outer(set,var,i,"4d")
1944                        end
1945                        return var
1946                    end
1947                end
1948            end
1949        end
1950        local component, inner = lpegmatch(componentsplitter,inner)
1951        if component then
1952            local data = componentreferences and componentreferences[component]
1953            if data then
1954                local d = data[""]
1955                local vi = d and d[inner]
1956                if vi then
1957                    var.inner = inner
1958                    var.outer = component
1959                    var.i = vi
1960                    var.kind = "outer with inner"
1961                    set.external = true
1962                    if trace_identifying then
1963                        report_identify_outer(set,var,i,"4e")
1964                    end
1965                    return var
1966                end
1967            end
1968            local data = productreferences and productreferences[component]
1969            if data then
1970                local vi = data[inner]
1971                if vi then
1972                    var.inner = inner
1973                    var.outer = component
1974                    var.i = vi
1975                    var.kind = "outer with inner"
1976                    set.external = true
1977                    if trace_identifying then
1978                        report_identify_outer(set,var,i,"4f")
1979                    end
1980                    return var
1981                end
1982            end
1983        end
1984
1985        var.error = "unknown inner"
1986    else
1987        var.error = "no inner"
1988    end
1989    if trace_identifying then
1990        report_identify_outer(set,var,i,"4g")
1991    end
1992    return var
1993end
1994
1995local function identify_inner_component(set,var,i)
1996    -- we're in a product (maybe ignore when same as component)
1997    local component = var.component
1998    local v = identify_inner(set,var,component,collected,derived)
1999    if not v then
2000        var.error = "unknown inner in component"
2001    end
2002    if trace_identifying then
2003        report_identify_outer(set,var,i,"5a")
2004    end
2005    return var
2006end
2007
2008local function identify_outer_component_step(set,var,vi,i,component,where)
2009    if vi then
2010        var.outer    = component
2011        var.i        = vi
2012        var.kind     = "outer with inner"
2013        set.external = true
2014        if trace_identifying then
2015            report_identify_outer(set,var,i,where)
2016        end
2017        return true
2018    end
2019end
2020
2021local function identify_outer_component(set,var,i)
2022    local component = var.component
2023    local inner     = var.inner
2024    --
2025    local data = productdata.componentreferences[component]
2026    if data then
2027        local d = data[""]
2028        local d = d and d[inner]
2029        if d and identify_outer_component_step(set,var,d,i,component,"6a") then
2030            return var
2031        end
2032    end
2033    --
2034    local data = productdata.productreferences[component]
2035    if data then
2036        local d = data[inner]
2037        if d and identify_outer_component_step(set,var,d,i,component,"6b") then
2038            return var
2039        end
2040    end
2041    --
2042if texconditionals.autocrossfilereferences then
2043    references.loadpresets(component,component)
2044    local data = productdata.productreferences[component]
2045    if data then
2046        local d = data[inner]
2047        if d and identify_outer_component_step(set,var,d,i,component,"6c") then
2048            return var
2049        end
2050    end
2051end
2052    --
2053    var.error = "unknown component"
2054    if trace_identifying then
2055        report_identify_outer(set,var,i,"6c")
2056    end
2057    return var
2058end
2059
2060local nofidentified = 0
2061
2062local function identify(prefix,reference)
2063    if not reference then
2064        prefix, reference = "", prefix
2065    end
2066    local set = resolve(prefix,reference)
2067    local bug = false
2068    nofidentified = nofidentified + 1
2069    set.n = nofidentified
2070    for i=1,#set do
2071        local var = set[i]
2072        local spe = var.special
2073        local fnc = functions[spe]
2074        if fnc then
2075            var = fnc(var) or { error = "invalid special function" }
2076        elseif spe then
2077            var = identify_special(set,var,i)
2078        elseif var.outer then
2079            if var.outer == "auto" then
2080                identify_auto(set,var,i)
2081            end
2082            var = identify_outer(set,var,i)
2083        elseif var.arguments then
2084            var = identify_arguments(set,var,i)
2085-- else
2086--     var = identify_inner_or_outer(set,var,i)
2087        elseif not var.component then
2088            var = identify_inner_or_outer(set,var,i)
2089        elseif productcomponent() then
2090            var = identify_inner_component(set,var,i)
2091        else
2092            var = identify_outer_component(set,var,i)
2093        end
2094        set[i] = var
2095        bug = bug or var.error
2096    end
2097    references.currentset = mark(set) -- mark, else in api doc
2098    if trace_analyzing then
2099        report_references(table.serialize(set,reference))
2100    end
2101    return set, bug
2102end
2103
2104references.identify = identify
2105
2106local unknowns, nofunknowns, f_valid = { }, 0, formatters["[%s][%s]"]
2107
2108function references.valid(prefix,reference,specification,silent)
2109    local set, bug = identify(prefix,reference)
2110    local unknown = bug or #set == 0
2111    if unknown then
2112        currentreference = nil -- will go away
2113        local str = f_valid(prefix,reference)
2114        local u = unknowns[str]
2115        if not u then
2116            if somefound and not silent then
2117                interfaces.showmessage("references",1,str) -- 1 = unknown, 4 = illegal
2118            end
2119            unknowns[str] = 1
2120            nofunknowns = nofunknowns + 1
2121        else
2122            unknowns[str] = u + 1
2123        end
2124    else
2125        set.highlight    = specification.highlight
2126        set.newwindow    = specification.newwindow
2127        set.layer        = specification.layer
2128        currentreference = set[1]
2129    end
2130    -- we can do the expansion here which saves a call
2131    return not unknown
2132end
2133
2134function references.checked(prefix,reference,specification)
2135    return references.valid(prefix,reference,specification,true)
2136end
2137
2138implement {
2139    name      = "doifelsereference",
2140    actions   = { references.valid, ctx_doifelse },
2141    arguments = {
2142        "string",
2143        "string",
2144        {
2145            { "highlight", "boolean" },
2146            { "newwindow", "boolean" },
2147            { "layer" },
2148        }
2149    }
2150}
2151
2152implement {
2153    name      = "doifelsereferencechecked",
2154    actions   = { references.checked, ctx_doifelse },
2155    arguments = {
2156        "string",
2157        "string",
2158        {
2159            { "highlight", "boolean" },
2160            { "newwindow", "boolean" },
2161            { "layer" },
2162        }
2163    }
2164}
2165
2166logs.registerfinalactions(function()
2167    if nofunknowns > 0 then
2168        statistics.register("cross referencing", function()
2169            return format("%s identified, %s unknown",nofidentified,nofunknowns)
2170        end)
2171        local sortedhash = table.sortedhash
2172        logs.startfilelogging(report,"missing references")
2173        for k, v in table.sortedhash(unknowns) do
2174            report("%4i  %s",v,k)
2175        end
2176        logs.stopfilelogging()
2177        if logs.loggingerrors() then
2178            logs.starterrorlogging(report,"missing references")
2179            for k, v in table.sortedhash(unknowns) do
2180                report("%4i  %s",v,k)
2181            end
2182            logs.stoperrorlogging()
2183        end
2184    end
2185end)
2186
2187-- The auto method will try to avoid named internals in a clever way which
2188-- can make files smaller without sacrificing external references. Some of
2189-- the housekeeping happens the backend side.
2190
2191local innermethod        = v_auto       -- only page|auto now
2192local outermethod        = v_auto       -- only page|auto now
2193local defaultinnermethod = defaultinnermethod
2194local defaultoutermethod = defaultoutermethod
2195references.innermethod   = innermethod  -- don't mess with this one directly
2196references.outermethod   = outermethod  -- don't mess with this one directly
2197
2198function references.setlinkmethod(inner,outer)
2199    if not outer and type(inner) == "string" then
2200        local m = settings_to_array(inner)
2201        inner = m[1]
2202        outer = m[2] or v_auto
2203    end
2204    if toboolean(inner) or inner == v_page or inner == v_yes then
2205        innermethod = v_page
2206    elseif inner == v_name then
2207        innermethod = v_name
2208    else
2209        innermethod = v_auto
2210    end
2211    if toboolean(outer) or outer == v_page or outer == v_yes then
2212        outermethod = v_page
2213    elseif outer == v_name then
2214        outermethod = v_name
2215    else
2216        outermethod = v_auto
2217    end
2218    references.innermethod = innermethod
2219    references.outermethod = outermethod
2220    function references.setlinkmethod()
2221        report_references("link method is already set and frozen: inner %a, outer %a",innermethod,outermethod)
2222    end
2223end
2224
2225implement {
2226    name      = "setreferencelinkmethod",
2227    actions   = references.setlinkmethod,
2228    arguments = "string",
2229 -- onlyonce  = true
2230}
2231
2232function references.getinnermethod()
2233    return innermethod or defaultinnermethod
2234end
2235
2236function references.getoutermethod()
2237    return outermethod or defaultoutermethod
2238end
2239
2240directives.register("references.linkmethod", function(v) -- page auto
2241    references.setlinkmethod(v)
2242end)
2243
2244-- we can call setinternalreference with an already known internal or with
2245-- a reference/prefix specification
2246
2247local destinationattributes = { }
2248local forcedinnermode       = false
2249
2250function references.forceinnermode()
2251    forcedinnermode = true
2252end
2253
2254local function setinternalreference(specification)
2255    local internal    = specification.internal
2256    local destination = unsetvalue
2257    if forcedinnermode or innermethod == v_auto or innermethod == v_name then
2258        local t         = { } -- maybe add to current (now only used for tracing)
2259        local tn        = 0
2260        local reference = specification.reference
2261        local view      = specification.view
2262        if reference then
2263            local prefix = specification.prefix
2264            if prefix and prefix ~= "" then
2265                local prefix = prefix .. ":" -- watch out, : here
2266                local function action(ref)
2267                    tn = tn + 1
2268                    t[tn] = prefix .. ref
2269                end
2270                process_settings(reference,action)
2271            else
2272                local function action(ref)
2273                    tn = tn + 1
2274                    t[tn] = ref
2275                end
2276                process_settings(reference,action)
2277            end
2278        end
2279        -- ugly .. later we decide to ignore it when we have a real one
2280        -- but for testing we might want to see them all
2281        if internal then
2282            if innermethod ~= v_name then -- innermethod == v_auto
2283             -- we don't want too many #1 #2 #3 etc
2284                tn = tn + 1
2285                t[tn] = internal -- when number it's internal
2286            end
2287            if not view then
2288                local i = references.internals[internal]
2289                if i then
2290                    view = i.references.view
2291                end
2292            end
2293        end
2294        destination = references.mark(t,nil,nil,view,internal) -- returns an attribute
2295    end
2296    if internal then -- new
2297        destinationattributes[internal] = destination
2298    end
2299    texsetcount(c_lastdestinationattribute,destination)
2300    return destination
2301end
2302
2303local function getinternalreference(internal)
2304    return destinationattributes[internal] or 0
2305end
2306
2307references.setinternalreference = setinternalreference
2308references.getinternalreference = getinternalreference
2309
2310implement {
2311    name      = "setinternalreference",
2312    actions   = setinternalreference,
2313    arguments = {
2314        {
2315            { "prefix" },
2316            { "reference" },
2317            { "internal", "integer" },
2318            { "view" }
2319        }
2320    }
2321}
2322
2323-- implement {
2324--     name      = "getinternalreference",
2325--     actions   = { getinternalreference, context },
2326--     arguments = "integer",
2327-- }
2328
2329function references.setandgetattribute(data) -- maybe do internal automatically here
2330    local attr = unsetvalue
2331    local mdat = data.metadata
2332    local rdat = data.references
2333    if mdat and rdat then
2334        if not rdat.section then
2335            rdat.section = structures.sections.currentid()
2336        end
2337        local ndat = data.numberdata
2338        if ndat then
2339            local numbers = ndat.numbers
2340            if type(numbers) == "string" then
2341                counters.compact(ndat,numbers)
2342            end
2343            data.numberdata = helpers.simplify(ndat)
2344        end
2345        local pdat = data.prefixdata
2346        if pdat then
2347            data.prefixdata = helpers.simplify(pdat)
2348        end
2349        local udat = data.userdata
2350        if type(udat) == "string"  then
2351            data.userdata = helpers.touserdata(udat)
2352        end
2353        if not rdat.block then
2354            rdat.block = structures.sections.currentblock()
2355        end
2356        local done = references.set(data) -- we had kind i.e .item -> full
2357        if done then
2358            attr = setinternalreference {
2359                prefix    = rdat.prefix,
2360                reference = rdat.reference,
2361                internal  = rdat.internal,
2362                view      = rdat.view
2363            } or unsetvalue
2364        end
2365    end
2366    texsetcount(c_lastdestinationattribute,attr)
2367    return attr
2368end
2369
2370implement {
2371    name      = "setdestinationattribute",
2372    actions   = references.setandgetattribute,
2373    arguments = {
2374        {
2375            {
2376                "references", {
2377                    { "internal", "integer" },
2378                    { "block" },
2379                    { "view" },
2380                    { "prefix" },
2381                    { "reference" },
2382                },
2383            },
2384            {
2385                "metadata", {
2386                    { "kind" },
2387                    { "xmlroot" },
2388                    { "catcodes", "integer" },
2389                },
2390            },
2391            {
2392                "prefixdata", { "*" }
2393            },
2394            {
2395                "numberdata", { "*" }
2396            },
2397            {
2398                "entries", { "*" }
2399            },
2400            {
2401                "userdata"
2402            }
2403        }
2404    }
2405}
2406
2407function references.getinternallistreference(n) -- n points into list (todo: registers)
2408    local l = lists.collected[n]
2409    local i = l and l.references.internal
2410    return i and destinationattributes[i] or 0
2411end
2412
2413function references.getinternalcachedlistreference(n) -- n points into list (todo: registers)
2414    local l = lists.cached[n]
2415    local i = l and l.references.internal
2416    return i and destinationattributes[i] or 0
2417end
2418
2419implement {
2420    name      = "getinternallistreference",
2421    actions   = { references.getinternallistreference, context },
2422    arguments = "integer"
2423}
2424
2425implement {
2426    name      = "getinternalcachedlistreference",
2427    actions   = { references.getinternalcachedlistreference, context },
2428    arguments = "integer"
2429}
2430
2431
2432--
2433
2434function references.getcurrentmetadata(tag)
2435    local data = currentreference and currentreference.i
2436    if data then
2437        local entry = data.metadata
2438        local value = entry[tag]
2439        if value then
2440            return value
2441        end
2442    end
2443end
2444
2445implement {
2446    name      = "getcurrentreferencemetadata",
2447    actions   = { references.getcurrentmetadata, context },
2448    arguments = "string",
2449}
2450
2451function references.getcurrentlabel()
2452    local data = currentreference and currentreference.i
2453    if data then
2454        local entry = data.titledata
2455        if entry then
2456            local value = entry.label
2457            if value then
2458                return value
2459            end
2460        end
2461     -- if entry then
2462     --     local entry = data.metadata
2463     --     local value = entry[tag]
2464     --     if value then
2465     --         return value
2466     --     end
2467     -- end
2468    end
2469end
2470
2471implement {
2472    name    = "getcurrentreferencelabel",
2473    actions = { references.getcurrentlabel, context },
2474}
2475
2476local function currentmetadata(tag)
2477    local data = currentreference and currentreference.i
2478    return data and data.metadata and data.metadata[tag]
2479end
2480
2481references.currentmetadata = currentmetadata
2482
2483local function getcurrentprefixspec(default)
2484    local data     = currentreference and currentreference.i
2485    local metadata = data and data.metadata
2486    return
2487        metadata and metadata.kind or "?",
2488        metadata and metadata.name or "?",
2489        default                    or "?"
2490end
2491
2492references.getcurrentprefixspec = getcurrentprefixspec
2493
2494-- implement {
2495--     name      = "getcurrentprefixspec",
2496--     actions   = { getcurrentprefixspec, context }, -- returns 3 arguments
2497--     arguments = "string",
2498-- }
2499
2500implement {
2501    name      = "getcurrentprefixspec",
2502    actions   = function(tag)
2503        context("{%s}{%s}{%s}",getcurrentprefixspec(tag))
2504    end,
2505    arguments = "string",
2506}
2507
2508local genericfilters = { }
2509local userfilters    = { }
2510local textfilters    = { }
2511local fullfilters    = { }
2512local sectionfilters = { }
2513
2514filters.generic = genericfilters
2515filters.user    = userfilters
2516filters.text    = textfilters
2517filters.full    = fullfilters
2518filters.section = sectionfilters
2519
2520local function filterreference(name,prefixspec,numberspec) -- number page title ...
2521    local data = currentreference and currentreference.i -- maybe we should take realpage from here
2522    if data then
2523        if name == "realpage" then
2524            local cs = references.analyze() -- normally already analyzed but also sets state
2525            -- or just get from references.realpage when present
2526            context(tonumber(cs.realpage) or 0)
2527        else -- assumes data is table
2528            local kind = false
2529            if type(data) == "string" then
2530                -- todo
2531            end
2532            if type(data) == "table" then
2533                kind = data.metadata and data.metadata.kind
2534            end
2535            if kind then
2536                local filter = filters[kind] or genericfilters
2537                filter = filter and (filter[name] or filter.unknown or genericfilters[name] or genericfilters.unknown)
2538                if filter then
2539                    if trace_referencing then
2540                        report_references("name %a, kind %a, using dedicated filter",name,kind)
2541                    end
2542                    filter(data,name,prefixspec,numberspec)
2543                elseif trace_referencing then
2544                    report_references("name %a, kind %a, using generic filter",name,kind)
2545                end
2546            elseif trace_referencing then
2547                report_references("name %a, unknown kind",name)
2548            end
2549        end
2550    elseif name == "realpage" then
2551        context(0)
2552    elseif trace_referencing then
2553        report_references("name %a, no reference",name)
2554    end
2555end
2556
2557local function filterreferencedefault()
2558    return filterreference("default",getcurrentprefixspec("default"))
2559end
2560
2561local function getreferencesectionnumber(reference)
2562    if references.valid("",reference,{},false) then
2563        local data = currentreference and currentreference.i
2564        if type(data) == "table" then
2565            local metadata = data.metadata
2566            if metadata and metadata.kind == "section" then
2567                local numberdata = data.numberdata
2568                if numberdata then
2569                    numberdata = numberdata.numbers
2570                    context(numberdata[#numberdata] or 0)
2571                    return
2572                end
2573            end
2574        elseif trace_referencing then
2575            report_references("name %a, no reference",reference)
2576        end
2577    end
2578    context(0)
2579end
2580
2581references.filter        = filterreference
2582references.filterdefault = filterreferencedefault
2583
2584implement {
2585    name      = "filterreference",
2586    actions   = filterreference,
2587    arguments = "string",
2588}
2589
2590implement {
2591    name      = "filterdefaultreference",
2592    actions   = filterreference,
2593    arguments = {
2594        "string",    -- 'default'
2595        { { "*" } }, -- prefixspec
2596        { { "*" } }, -- numberspec
2597    }
2598}
2599
2600implement {
2601    name      = "getreferencesectionnumber",
2602    actions   = getreferencesectionnumber,
2603    arguments = "string",
2604    public    = true,
2605}
2606
2607function genericfilters.title(data)
2608    if data then
2609        local titledata = data.titledata or data.useddata
2610        if titledata then
2611            helpers.title(titledata.reference or titledata.title or "?",data.metadata)
2612        end
2613    end
2614end
2615
2616function genericfilters.text(data)
2617    if data then
2618        local entries = data.entries or data.useddata
2619        if entries then
2620            helpers.title(entries.text or "?",data.metadata)
2621        end
2622    end
2623end
2624
2625function genericfilters.number(data,what,prefixspec,numberspec)
2626    if data then
2627        numberdata = lists.reordered(data) -- data.numberdata
2628        if numberdata then
2629            helpers.prefix(data,prefixspec)
2630            sections.typesetnumber(numberdata,"number",numberspec,numberdata)
2631        else
2632            local useddata = data.useddata
2633            if useddata and useddata.number then
2634                context(useddata.number)
2635            end
2636        end
2637    end
2638end
2639
2640genericfilters.default = genericfilters.text
2641
2642function genericfilters.page(data,prefixspec,pagespec)
2643    local pagedata = data.pagedata
2644    if pagedata then
2645        local number     = pagedata.number
2646        local conversion = pagedata.conversion
2647        if not number then
2648            -- error
2649        elseif conversion then
2650            ctx_convertnumber(conversion,number)
2651        else
2652            context(number)
2653        end
2654    else
2655        helpers.prefixpage(data,prefixspec,pagespec)
2656    end
2657end
2658
2659function userfilters.unknown(data,name)
2660    if data then
2661        local userdata = data.userdata
2662        local userkind = userdata and userdata.kind
2663        if userkind then
2664            local filter = filters[userkind] or genericfilters
2665            filter = filter and (filter[name] or filter.unknown)
2666            if filter then
2667                filter(data,name)
2668                return
2669            end
2670        end
2671        local namedata = userdata and userdata[name]
2672        if namedata then
2673            context(namedata)
2674        end
2675    end
2676end
2677
2678function textfilters.title(data)
2679    helpers.title(data.entries.text or "?",data.metadata)
2680end
2681
2682-- no longer considered useful:
2683--
2684-- function filters.text.number(data)
2685--     helpers.title(data.entries.text or "?",data.metadata)
2686-- end
2687
2688function textfilters.page(data,prefixspec,pagespec)
2689    helpers.prefixpage(data,prefixspec,pagespec)
2690end
2691
2692fullfilters.title = textfilters.title
2693fullfilters.page  = textfilters.page
2694
2695function sectionfilters.number(data,what,prefixspec)
2696    if data then
2697        local numberdata = data.numberdata
2698        if not numberdata then
2699            local useddata = data.useddata
2700            if useddata and useddata.number then
2701                context(useddata.number)
2702            end
2703        elseif numberdata.hidenumber then
2704            local references = data.references
2705            if trace_empty then
2706                report_empty("reference %a has a hidden number",references.reference)
2707                ctx_emptyreference() -- maybe an option
2708            end
2709        else
2710            sections.typesetnumber(numberdata,"number",prefixspec,numberdata)
2711        end
2712    end
2713end
2714
2715sectionfilters.title   = genericfilters.title
2716sectionfilters.page    = genericfilters.page
2717sectionfilters.default = sectionfilters.number
2718
2719-- filters.note        = { default = genericfilters.number }
2720-- filters.formula     = { default = genericfilters.number }
2721-- filters.float       = { default = genericfilters.number }
2722-- filters.description = { default = genericfilters.number }
2723-- filters.item        = { default = genericfilters.number }
2724
2725setmetatableindex(filters, function(t,k) -- beware, test with rawget
2726    local v = { default = genericfilters.number } -- not copy as it might be extended differently
2727    t[k] = v
2728    return v
2729end)
2730
2731-- function references.sectiontitle(n)
2732--     helpers.sectiontitle(lists.collected[tonumber(n) or 0])
2733-- end
2734
2735-- function references.sectionnumber(n)
2736--     helpers.sectionnumber(lists.collected[tonumber(n) or 0])
2737-- end
2738
2739-- function references.sectionpage(n,prefixspec,pagespec)
2740--     helpers.prefixedpage(lists.collected[tonumber(n) or 0],prefixspec,pagespec)
2741-- end
2742
2743-- analyze
2744
2745references.testrunners  = references.testrunners  or { }
2746references.testspecials = references.testspecials or { }
2747
2748local runners  = references.testrunners
2749local specials = references.testspecials
2750
2751-- We need to prevent ending up in the 'relative location' analyzer as it is
2752-- pretty slow (progressively). In the pagebody one can best check the reference
2753-- real page to determine if we need contrastlocation as that is more lightweight.
2754
2755local function checkedpagestate(n,page,actions,position,spread)
2756    local p = tonumber(page)
2757    if not p then
2758        return 0
2759    end
2760    if position and #actions > 0 then
2761        local i = actions[1].i -- brrr
2762        if i then
2763            local a = i.references
2764            if a then
2765                local x = a.x
2766                local y = a.y
2767                if x and y then
2768                    local jp = jobpositions.collected[position]
2769                    if jp then
2770                        local px = jp.x
2771                        local py = jp.y
2772                        local pp = jp.p
2773                        if p == pp then
2774                            -- same page
2775                            if py > y then
2776                                return 5 -- above
2777                            elseif py < y then
2778                                return 4 -- below
2779                            elseif px > x then
2780                                return 4 -- below
2781                            elseif px < x then
2782                                return 5 -- above
2783                            else
2784                                return 1 -- same
2785                            end
2786                        elseif spread then
2787                            if pp % 2 == 0 then
2788                                -- left page
2789                                if pp > p then
2790                                    return 2 -- before
2791                                elseif pp + 1 == p then
2792--                                     return 4 -- below (on right page)
2793                                    return 5 -- above (on left page)
2794                                else
2795                                    return 3 -- after
2796                                end
2797                            else
2798                                -- right page
2799                                if pp < p then
2800                                    return 3 -- after
2801                                elseif pp - 1 == p then
2802--                                     return 5 -- above (on left page)
2803                                    return 4 -- below (on right page)
2804                                else
2805                                    return 2 -- before
2806                                end
2807                            end
2808                        elseif pp > p then
2809                            return 2 -- before
2810                        else
2811                            return 3 -- after
2812                        end
2813                    end
2814                end
2815            end
2816        end
2817    end
2818    local r = referredpage(n) -- sort of obsolete
2819    if p > r then
2820        return 3 -- after
2821    elseif p < r then
2822        return 2 -- before
2823    else
2824        return 1 -- same
2825    end
2826end
2827
2828local function setreferencerealpage(actions)
2829    if not actions then
2830        actions = references.currentset
2831    end
2832    if type(actions) == "table" then
2833        local realpage = actions.realpage
2834        if realpage then
2835            return realpage
2836        end
2837        local nofactions = #actions
2838        if nofactions > 0 then
2839            for i=1,nofactions do
2840                local a = actions[i]
2841                local what = runners[a.kind]
2842                if what then
2843                    what = what(a,actions) -- needs documentation
2844                end
2845            end
2846            realpage = actions.realpage
2847            if realpage then
2848                return realpage
2849            end
2850        end
2851        actions.realpage = 0
2852    end
2853    return 0
2854end
2855
2856references.setreferencerealpage = setreferencerealpage
2857
2858-- we store some analysis data alongside the indexed array
2859-- at this moment only the real reference page is analyzed
2860-- normally such an analysis happens in the backend code
2861
2862function references.analyze(actions,position,spread)
2863    if not actions then
2864        actions = references.currentset
2865    end
2866    if not actions then
2867        actions = { realpage = 0, pagestate = 0 }
2868    elseif actions.pagestate then
2869        -- already done
2870    else
2871        local realpage = actions.realpage or setreferencerealpage(actions)
2872        if realpage == 0 then
2873            actions.pagestate = 0
2874        elseif actions.external then
2875            actions.pagestate = 0
2876        else
2877            actions.pagestate = checkedpagestate(actions.n,realpage,actions,position,spread)
2878        end
2879    end
2880-- inspect(actions)
2881    return actions
2882end
2883
2884local function referencepagestate(position,detail,spread)
2885    local actions = references.currentset
2886    if not actions then
2887        return 0
2888    elseif actions.external then
2889        return 0
2890    else
2891        local pagestate = actions.pagestate
2892        for i=1,#actions do
2893            local a = actions[i]
2894            if a.outer then
2895                pagestate = 0
2896                actions.pagestate = pagestate
2897                break
2898            end
2899        end
2900        if not pagestate then
2901            references.analyze(actions,position,spread) -- delayed unless explicitly asked for
2902            pagestate = actions.pagestate
2903        end
2904        if detail then
2905            return pagestate
2906        elseif pagestate == 4 then
2907            return 2 -- compatible
2908        elseif pagestate == 5 then
2909            return 3 -- compatible
2910        else
2911            return pagestate
2912        end
2913    end
2914end
2915
2916implement {
2917    name      = "referencepagestate",
2918    actions   = { referencepagestate, context },
2919    arguments = "string"
2920}
2921
2922implement {
2923    name      = "referencepagedetail",
2924    actions   = { referencepagestate, context },
2925    arguments = { "string", "boolean", "boolean" }
2926}
2927
2928-- local function referencerealpage()
2929--     local actions = references.currentset
2930--     return not actions and 0 or actions.realpage or setreferencerealpage(actions)
2931-- end
2932--
2933-- implement {
2934--     name      = "referencerealpage",
2935--     actions   = { referencerealpage, context },
2936--  -- arguments = "string" -- hm, weird
2937-- }
2938
2939implement {
2940    name      = "askedreference",
2941    public    = true,
2942    protected = true,
2943    actions   = function()
2944        local actions = references.currentset
2945        if actions then
2946            context("[p=%s,r=%s]",actions.prefix or "",actions.reference)
2947        end
2948    end
2949}
2950
2951implement {
2952    name    = "referencerealpage",
2953    actions = function()
2954        local actions = references.currentset
2955        context(not actions and 0 or actions.realpage or setreferencerealpage(actions))
2956    end
2957}
2958
2959local function referencepos(key)
2960    local actions = references.currentset
2961    local i = actions[1].i -- brrr
2962    local v = 0
2963    if i then
2964        local a = i.references
2965        if a then
2966            v = a[key] or 0
2967        end
2968    end
2969    return v
2970end
2971
2972implement { name = "referenceposx", actions = function() context("%p",referencepos("x")) end }
2973implement { name = "referenceposy", actions = function() context("%p",referencepos("y")) end }
2974
2975implement {
2976    name    = "referencecolumn",
2977    actions = function()
2978        local actions = references.currentset
2979        local column  = 1
2980        if actions then
2981            column = jobpositions.columnofpos(actions.realpage or setreferencerealpage(actions),referencepos("x"))
2982        end
2983        context(column or 1)
2984    end
2985}
2986
2987local plist, nofrealpages
2988
2989local function realpageofpage(p) -- the last one counts !
2990    if not plist then
2991        local pages = structures.pages.collected
2992        nofrealpages = #pages
2993        plist = { }
2994        for rp=1,nofrealpages do
2995            local page = pages[rp]
2996            if page then
2997                plist[page.number] = rp
2998            end
2999        end
3000        references.nofrealpages = nofrealpages
3001    end
3002    return plist[p]
3003end
3004
3005references.realpageofpage = realpageofpage
3006
3007function references.checkedrealpage(r)
3008    if not plist then
3009        realpageofpage(r) -- just initialize
3010    end
3011    if not r then
3012        return texgetcount(c_realpageno)
3013    elseif r < 1 then
3014        return 1
3015    elseif r > nofrealpages then
3016        return nofrealpages
3017    else
3018        return r
3019    end
3020end
3021
3022-- use local ?
3023
3024local pages = allocate {
3025    [variables.firstpage]       = function() return counters.record("realpage")["first"]    end,
3026    [variables.previouspage]    = function() return counters.record("realpage")["previous"] end,
3027    [variables.nextpage]        = function() return counters.record("realpage")["next"]     end,
3028    [variables.lastpage]        = function() return counters.record("realpage")["last"]     end,
3029
3030    [variables.firstsubpage]    = function() return counters.record("subpage" )["first"]    end,
3031    [variables.previoussubpage] = function() return counters.record("subpage" )["previous"] end,
3032    [variables.nextsubpage]     = function() return counters.record("subpage" )["next"]     end,
3033    [variables.lastsubpage]     = function() return counters.record("subpage" )["last"]     end,
3034
3035    [variables.forward]         = function() return counters.record("realpage")["forward"]  end,
3036    [variables.backward]        = function() return counters.record("realpage")["backward"] end,
3037}
3038
3039references.pages = pages
3040
3041-- maybe some day i will merge this in the backend code with a testmode (so each
3042-- runner then implements a branch)
3043
3044runners["inner"] = function(var,actions)
3045    local r = var.r
3046    if r then
3047        actions.realpage = r
3048    end
3049end
3050
3051runners["special"] = function(var,actions)
3052    local handler = specials[var.special]
3053    return handler and handler(var,actions)
3054end
3055
3056runners["special operation"]                = runners["special"]
3057runners["special operation with arguments"] = runners["special"]
3058
3059function specials.internal(var,actions)
3060    local v = internals[tonumber(var.operation)]
3061    local r = v and v.references
3062    if r then
3063        local p = r.realpage
3064        if p then
3065-- setmetatableindex(actions,r)
3066            actions.realpage = p
3067            actions.view     = r.view
3068        end
3069    end
3070end
3071
3072specials.i = specials.internal
3073
3074function specials.page(var,actions)
3075    local o = var.operation
3076    local p = pages[o]
3077    if type(p) == "function" then
3078        p = p()
3079    else
3080        p = tonumber(realpageofpage(tonumber(o)))
3081    end
3082    if p then
3083        var.r = p
3084        actions.realpage = actions.realpage or p -- first wins
3085    end
3086end
3087
3088function specials.realpage(var,actions)
3089    local p = tonumber(var.operation)
3090    if p then
3091        var.r = p
3092        actions.realpage = actions.realpage or p -- first wins
3093    end
3094end
3095
3096function specials.userpage(var,actions)
3097    local p = tonumber(realpageofpage(var.operation))
3098    if p then
3099        var.r = p
3100        actions.realpage = actions.realpage or p -- first wins
3101    end
3102end
3103
3104function specials.deltapage(var,actions)
3105    local p = tonumber(var.operation)
3106    if p then
3107        p = references.checkedrealpage(p + texgetcount(c_realpageno))
3108        var.r = p
3109        actions.realpage = actions.realpage or p -- first wins
3110    end
3111end
3112
3113function specials.section(var,actions)
3114    local sectionname = var.arguments
3115    local destination = var.operation
3116    local internal    = structures.sections.internalreference(sectionname,destination)
3117    if internal then
3118        var.special   = "internal"
3119        var.operation = internal
3120        var.arguments = nil
3121        specials.internal(var,actions)
3122    end
3123end
3124
3125-- experimental:
3126
3127-- We can cache lowercased titles which saves a lot of time, but then
3128-- we can better have a global cache with weak keys.
3129
3130-- local lowercache = table.setmetatableindex(function(t,k)
3131--     local v = lpegmatch(p_lower,k)
3132--     t[k] = v
3133--     return v
3134-- end)
3135
3136local lowercache = false
3137
3138local function locate(list,askedkind,askedname,pattern)
3139    local kinds = lists.kinds
3140    local names = lists.names
3141    if askedkind and not kinds[askedkind] then
3142        return false
3143    end
3144    if askedname and not names[askedname] then
3145        return false
3146    end
3147    for i=1,#list do
3148        local entry    = list[i]
3149        local metadata = entry.metadata
3150        if metadata then
3151            local found = false
3152            if askedname then
3153                local name = metadata.name
3154                if name then
3155                    found = name == askedname
3156                end
3157            elseif askedkind then
3158                local kind = metadata.kind
3159                if kind then
3160                    found = kind == askedkind
3161                end
3162            end
3163            if found then
3164                local titledata = entry.titledata
3165                if titledata then
3166                    local title = titledata.title
3167                    if title then
3168                        if lowercache then
3169                            found = lpegmatch(pattern,lowercache[title])
3170                        else
3171                            found = lpegmatch(pattern,lpegmatch(p_lower,title))
3172                        end
3173                        if found then
3174                            return {
3175                                inner     = pattern,
3176                                kind      = "inner",
3177                                reference = pattern,
3178                                i         = entry,
3179                                p         = "",
3180                                r         = entry.references.realpage,
3181                            }
3182                        end
3183                    end
3184                end
3185            end
3186        end
3187    end
3188end
3189
3190function functions.match(var,actions)
3191    if not var.outer then
3192        local operation   = var.operation
3193        if operation and operation ~= "" then
3194            local operation   = lpegmatch(p_lower,operation)
3195            local list        = lists.collected
3196            local names       = false
3197            local kinds       = false
3198            local where, what = lpegmatch(p_splitter,operation)
3199            if where and what then
3200                local pattern = lpeg.finder(what)
3201                return
3202                    locate(list,false,where,pattern)
3203                 or locate(list,where,false,pattern)
3204                 or { error = "no match" }
3205            else
3206                local pattern = lpeg.finder(operation)
3207                -- todo: don't look at section and float in last pass
3208                return
3209                    locate(list,"section",false,pattern)
3210                 or locate(list,"float",false,pattern)
3211                 or locate(list,false,false,pattern)
3212                 or { error = "no match" }
3213            end
3214        end
3215    end
3216end
3217
3218-- needs a better split ^^^
3219
3220-- done differently now:
3221
3222function references.export(usedname) end
3223function references.import(usedname) end
3224function references.load  (usedname) end
3225
3226implement { name = "exportreferences", actions =references.export }
3227
3228-- better done here .... we don't insert/remove, just use a pointer
3229
3230local prefixstack = { "" }
3231local prefixlevel = 1
3232
3233local function pushreferenceprefix(prefix)
3234    prefixlevel = prefixlevel + 1
3235    prefixstack[prefixlevel] = prefix
3236    return prefix
3237end
3238
3239local function popreferenceprefix()
3240    prefixlevel = prefixlevel - 1
3241    if prefixlevel > 0 then
3242        return prefixstack[prefixlevel]
3243    else
3244        report_references("unable to pop referenceprefix")
3245        return ""
3246    end
3247end
3248
3249implement {
3250    name      = "pushreferenceprefix",
3251    actions   = { pushreferenceprefix, context }, -- we can use setmacro
3252    arguments = "string",
3253}
3254
3255implement {
3256    name      = "popreferenceprefix",
3257    actions   = { popreferenceprefix, context }, -- we can use setmacro
3258}
3259
3260-- a bit weird one, used for checkinh in the backend
3261
3262function references.pagefromhash(hash,destination,page,actions)
3263    local r = actions[1]
3264    if type(r) == "table" then
3265        r = r.i
3266        if r then
3267            r = r.references
3268            if r then
3269                local structure = r.structure
3270                if structure then
3271                    local namestack = r.namestack
3272                    if namestack then
3273                        for i=#namestack,1,-1 do
3274                            local d = namestack[i] .. ":" .. destination
3275                            page = hash[d]
3276                            if page then
3277                                destination = d
3278                                break
3279                            end
3280                        end
3281                    end
3282                end
3283            end
3284        end
3285    end
3286    return page, destination
3287end
3288