Source Browser ?



Optional search indexes:
    Comments Docs MkII Data
Whole words only
Lua string pattern matches
strc-ref.lua /size: 93 Kb    last modification: 2025-02-21 11:03
1if not modules then modules = { } end modules ['strc-ref'] = {
2    version   = 1.001,
3    comment   = "companion to strc-ref.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- beware, this is a first step in the rewrite (just getting rid of
10-- the tuo file); later all access and parsing will also move to lua
11
12-- the useddata and pagedata names might change
13-- todo: pack exported data
14
15-- todo: autoload components when :::
16
17local format, find, gmatch, match, strip = string.format, string.find, string.gmatch, string.match, string.strip
18local floor = math.floor
19local rawget, tonumber, type, next = rawget, tonumber, type, next
20local lpegmatch = lpeg.match
21local insert, remove, copytable = table.insert, table.remove, table.copy
22local formatters = string.formatters
23local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match
24
25local allocate           = utilities.storage.allocate
26local mark               = utilities.storage.mark
27local setmetatableindex  = table.setmetatableindex
28
29local trace_referencing  = false  trackers.register("structures.referencing",             function(v) trace_referencing = v end)
30local trace_analyzing    = false  trackers.register("structures.referencing.analyzing",   function(v) trace_analyzing   = v end)
31local trace_identifying  = false  trackers.register("structures.referencing.identifying", function(v) trace_identifying = v end)
32local trace_importing    = false  trackers.register("structures.referencing.importing",   function(v) trace_importing   = v end)
33local trace_empty        = false  trackers.register("structures.referencing.empty",       function(v) trace_empty       = v end)
34
35local check_duplicates   = true
36
37directives.register("structures.referencing.checkduplicates", function(v) check_duplicates = v end)
38
39local report_references  = logs.reporter("references")
40local report_identifying = logs.reporter("references","identifying")
41local report_importing   = logs.reporter("references","importing")
42local report_empty       = logs.reporter("references","empty")
43local report             = report_references
44
45local variables          = interfaces.variables
46local v_page             = variables.page
47local v_auto             = variables.auto
48local v_yes              = variables.yes
49local v_name             = variables.name
50
51local context            = context
52local commands           = commands
53local implement          = interfaces.implement
54
55local ctx_latelua        = context.latelua
56
57local texgetcount        = tex.getcount
58local texsetcount        = tex.setcount
59local texconditionals    = tex.conditionals
60
61local productcomponent   = resolvers.jobs.productcomponent
62local justacomponent     = resolvers.jobs.justacomponent
63
64local settings_to_array  = utilities.parsers.settings_to_array
65local settings_to_table  = utilities.parsers.settings_to_array_obey_fences
66local process_settings   = utilities.parsers.process_stripped_settings
67local unsetvalue         = attributes.unsetvalue
68
69local structures         = structures
70local helpers            = structures.helpers
71local sections           = structures.sections
72local references         = structures.references
73local lists              = structures.lists
74local counters           = structures.counters
75
76local jobpositions       = job.positions
77local getpos             = jobpositions.getpos
78
79-- some might become local
80
81references.defined       = references.defined or allocate()
82
83local defined            = references.defined
84local derived            = allocate()
85local specials           = allocate()
86local functions          = allocate()
87local runners            = allocate()
88local internals          = allocate()
89local filters            = allocate()
90local executers          = allocate()
91local handlers           = allocate()
92local tobesaved          = allocate()
93local collected          = allocate()
94local tobereferred       = allocate()
95local referred           = allocate()
96local usedinternals      = allocate()
97local flaginternals      = allocate()
98local usedviews          = allocate()
99
100references.derived       = derived
101references.specials      = specials
102references.functions     = functions
103references.runners       = runners
104references.internals     = internals
105references.filters       = filters
106references.executers     = executers
107references.handlers      = handlers
108references.tobesaved     = tobesaved
109references.collected     = collected
110references.tobereferred  = tobereferred
111references.referred      = referred
112references.usedinternals = usedinternals
113references.flaginternals = flaginternals
114references.usedviews     = usedviews
115
116local splitreference     = references.splitreference
117local splitprefix        = references.splitcomponent -- replaces: references.splitprefix
118local prefixsplitter     = references.prefixsplitter
119local componentsplitter  = references.componentsplitter
120
121local currentreference   = nil
122
123local txtcatcodes        = catcodes.numbers.txtcatcodes -- or just use "txtcatcodes"
124
125local context                      = context
126
127local ctx_pushcatcodes             = context.pushcatcodes
128local ctx_popcatcodes              = context.popcatcodes
129local ctx_dofinishreference        = context.dofinishreference
130local ctx_dofromurldescription     = context.dofromurldescription
131local ctx_dofromurlliteral         = context.dofromurlliteral
132local ctx_dofromfiledescription    = context.dofromfiledescription
133local ctx_dofromfileliteral        = context.dofromfileliteral
134local ctx_expandreferenceoperation = context.expandreferenceoperation
135local ctx_expandreferencearguments = context.expandreferencearguments
136local ctx_convertnumber            = context.convertnumber
137local ctx_emptyreference           = context.emptyreference
138
139storage.register("structures/references/defined", references.defined, "structures.references.defined")
140
141local initializers = { }
142local finalizers   = { }
143local somefound    = false -- so we don't report missing when we have a fresh start
144
145function references.registerinitializer(func) -- we could use a token register instead
146    initializers[#initializers+1] = func
147end
148
149function references.registerfinalizer(func) -- we could use a token register instead
150    finalizers[#finalizers+1] = func
151end
152
153local function initializer() -- can we use a tobesaved as metatable for collected?
154    tobesaved = references.tobesaved
155    collected = references.collected
156    for i=1,#initializers do
157        initializers[i](tobesaved,collected)
158    end
159    for prefix, list in next, collected do
160        for tag, data in next, list do
161            local r = data.references
162            local i = r.internal
163            if i then
164                internals[i]     = data
165                usedinternals[i] = r.used
166            end
167        end
168    end
169    somefound = next(collected)
170end
171
172local function finalizer()
173    for i=1,#finalizers do
174        finalizers[i](tobesaved)
175    end
176    for prefix, list in next, tobesaved do
177        for tag, data in next, list do
178            local r = data.references
179            local i = r.internal
180            local f = flaginternals[i]
181            if f then
182                r.used = usedviews[i] or true
183            end
184        end
185    end
186end
187
188job.register('structures.references.collected', tobesaved, initializer, finalizer)
189
190local maxreferred = 1
191local nofreferred = 0
192
193local function initializer() -- can we use a tobesaved as metatable for collected?
194    tobereferred = references.tobereferred
195    referred     = references.referred
196    nofreferred = #referred
197end
198
199-- no longer done this way
200
201-- references.resolvers = references.resolvers or { }
202-- local resolvers = references.resolvers
203--
204-- function resolvers.section(var)
205--     local vi = lists.collected[var.i[2]]
206--     if vi then
207--         var.i = vi
208--         var.r = (vi.references and vi.references.realpage) or (vi.pagedata and vi.pagedata.realpage) or 1
209--     else
210--         var.i = nil
211--         var.r = 1
212--     end
213-- end
214--
215-- resolvers.float       = resolvers.section
216-- resolvers.description = resolvers.section
217-- resolvers.formula     = resolvers.section
218-- resolvers.note        = resolvers.section
219--
220-- function resolvers.reference(var)
221--     local vi = var.i[2]
222--     if vi then
223--         var.i = vi
224--         var.r = (vi.references and vi.references.realpage) or (vi.pagedata and vi.pagedata.realpage) or 1
225--     else
226--         var.i = nil
227--         var.r = 1
228--     end
229-- end
230
231-- We make the array sparse (maybe a finalizer should optionally return a table) because
232-- there can be quite some page links involved. We only store one action number per page
233-- which is normally good enough for what we want (e.g. see above/below) and we do
234-- a combination of a binary search and traverse backwards. A previous implementation
235-- always did a traverse and was pretty slow on a large number of links (given that this
236-- methods was used). It took me about a day to locate this as a bottleneck in processing
237-- a 2500 page interactive document with 60 links per page. In that case, traversing
238-- thousands of slots per link then brings processing to a grinding halt (especially when
239-- there are no slots at all, which is the case in a first run).
240
241local sparsetobereferred = { }
242
243local function finalizer()
244    local lastr, lasti
245    local n = 0
246    for i=1,maxreferred do
247        local r = tobereferred[i]
248        if not lastr then
249            lastr = r
250            lasti = i
251        elseif r ~= lastr then
252            n = n + 1
253            sparsetobereferred[n] = { lastr, lasti }
254            lastr = r
255            lasti = i
256        end
257    end
258    if lastr then
259        n = n + 1
260        sparsetobereferred[n] = { lastr, lasti }
261    end
262end
263
264job.register('structures.references.referred', sparsetobereferred, initializer, finalizer)
265
266local function referredpage(n)
267    local max = nofreferred
268    if max > 0 then
269        -- find match
270        local min = 1
271        while true do
272            local mid = floor((min+max)/2)
273            local r = referred[mid]
274            local m = r[2]
275            if n == m then
276                return r[1]
277            elseif n > m then
278                min = mid + 1
279            else
280                max = mid - 1
281            end
282            if min > max then
283                break
284            end
285        end
286        -- find first previous
287        for i=min,1,-1 do
288            local r = referred[i]
289            if r and r[2] < n then
290                return r[1]
291            end
292        end
293    end
294    -- fallback
295    return texgetcount("realpageno")
296end
297
298references.referredpage = referredpage
299
300function references.registerpage(n) -- called in the backend code
301    if not tobereferred[n] then
302        if n > maxreferred then
303            maxreferred = n
304        end
305        tobereferred[n] = texgetcount("realpageno")
306    end
307end
308
309-- todo: delay split till later as in destinations we split anyway
310
311local orders, lastorder = { }, 0
312
313local function setnextorder(kind,name)
314    lastorder = 0
315    if kind and name then
316        local ok = orders[kind]
317        if not ok then
318            ok = { }
319            orders[kind] = ok
320        end
321        lastorder = (ok[name] or 0) + 1
322        ok[name] = lastorder
323    end
324    texsetcount("global","locationorder",lastorder)
325end
326
327
328local function setnextinternal(kind,name)
329    setnextorder(kind,name) -- always incremented with internal
330    local n = texgetcount("locationcount") + 1
331    texsetcount("global","locationcount",n)
332    return n
333end
334
335local function currentorder(kind,name)
336    return orders[kind] and orders[kind][name] or lastorder
337end
338
339local function setcomponent(data)
340    -- we might consider doing this at the tex end, just like prefix
341    local component = productcomponent()
342    if component then
343        local references = data and data.references
344        if references then
345            references.component = component
346            if references.prefix == component then
347                references.prefix = nil
348            end
349        end
350        return component
351    end
352    -- but for the moment we do it here (experiment)
353end
354
355references.setnextorder    = setnextorder
356references.setnextinternal = setnextinternal
357references.currentorder    = currentorder
358references.setcomponent    = setcomponent
359
360implement {
361    name      = "setnextreferenceorder",
362    actions   = setnextorder,
363    arguments = "2 strings",
364}
365
366implement {
367    name      = "setnextinternalreference",
368    actions   = setnextinternal,
369    arguments = "2 strings",
370}
371
372implement {
373    name      = "currentreferenceorder",
374    actions = { currentorder, context },
375    arguments = "2 strings",
376}
377
378local reported = setmetatableindex("table")
379
380function references.set(data)
381    local references = data.references
382    local reference  = references.reference
383    if not reference or reference == "" then
384     -- report_references("invalid reference") -- harmless
385        return 0
386    end
387    local prefix = references.prefix or ""
388    local pd     = tobesaved[prefix] -- nicer is a metatable
389    if not pd then
390        pd = { }
391        tobesaved[prefix] = pd
392    end
393    local n = 0
394    local function action(ref)
395        if ref == "" then
396            -- skip
397        elseif check_duplicates and pd[ref] then
398            if not prefix then
399                prefix = ""
400            end
401            if not reported[prefix][ref] then
402                if prefix ~= "" then
403                    report_references("redundant reference %a in namespace %a",ref,prefix)
404                else
405                    report_references("redundant reference %a",ref)
406                end
407                reported[prefix][ref] = true
408            end
409        else
410            n = n + 1
411            pd[ref] = data
412            local r = data.references
413            ctx_dofinishreference(prefix or "",ref or "",r and r.internal or 0)
414         -- ctx_latelua(function() structures.references.enhance(prefix or ref,ref or "") end)
415        end
416    end
417    process_settings(reference,action)
418    return n > 0
419end
420
421-- function references.enhance(prefix,tag)
422--     local l = tobesaved[prefix][tag]
423--     if l then
424--         l.references.realpage = texgetcount("realpageno")
425--     end
426-- end
427
428local function synchronizepage(reference) -- non public helper
429    reference.realpage = texgetcount("realpageno")
430    if jobpositions.used() then
431        reference.x, reference.y = getpos()
432    end
433end
434
435references.synchronizepage = synchronizepage
436
437local function enhancereference(specification)
438    local prefix = specification.prefix
439    if prefix then
440        local entry = tobesaved[prefix]
441        if entry then
442            entry = entry[specification.tag]
443            if entry then
444                synchronizepage(entry.references)
445            else
446                -- normally a bug
447            end
448        else
449            -- normally a bug
450        end
451    else
452        -- normally a bug
453    end
454end
455
456references.enhance = enhancereference
457
458-- implement {
459--     name      = "enhancereference",
460--     arguments = "2 strings",
461--     actions   = function(prefix,tag)
462--        enhancereference { prefix = prefix, tag = tag }
463--     end,
464-- }
465
466implement {
467    name      = "deferredenhancereference",
468    arguments = "2 strings",
469    protected = true,
470    actions   = function(prefix,tag)
471        ctx_latelua { action = enhancereference, prefix = prefix, tag = tag }
472    end,
473}
474
475-- -- -- related to strc-ini.lua -- -- --
476
477-- no metatable here .. better be sparse
478
479local function register_from_lists(collected,derived,pages,sections)
480    local derived_g = derived[""] -- global
481    local derived_p = nil
482    local derived_c = nil
483    local prefix    = nil
484    local component = nil
485    local entry     = nil
486    if not derived_g then
487        derived_g = { }
488        derived[""] = derived_g
489    end
490    local function action(s)
491        if trace_referencing then
492            report_references("list entry %a provides %a reference %a on realpage %a",i,kind,s,realpage)
493        end
494        if derived_p and not derived_p[s] then
495            derived_p[s] = entry
496        end
497        if derived_c and not derived_c[s] then
498            derived_c[s] = entry
499        end
500        if not derived_g[s] then
501            derived_g[s] = entry -- first wins
502        end
503    end
504    for i=1,#collected do
505        entry = collected[i]
506        local metadata = entry.metadata
507        if metadata then
508            local kind = metadata.kind -- why this check
509            if kind then
510                local references = entry.references
511                if references then
512                    local reference = references.reference
513                    if reference and reference ~= "" then
514                        local realpage = references.realpage
515                        if realpage then
516                            prefix    = references.prefix
517                            component = references.component
518                            if prefix and prefix ~= "" then
519                                derived_p = derived[prefix]
520                                if not derived_p then
521                                    derived_p = { }
522                                    derived[prefix] = derived_p
523                                end
524                            end
525                            if component and component ~= "" and component ~= prefix then
526                                derived_c = derived[component]
527                                if not derived_c then
528                                    derived_c = { }
529                                    derived[component] = derived_c
530                                end
531                            end
532                            process_settings(reference,action)
533                        end
534                    end
535                end
536            end
537        end
538    end
539end
540
541references.registerinitializer(function() register_from_lists(lists.collected,derived) end)
542
543-- tracing
544
545local function collectbypage(tracedpages)
546    -- lists
547    do
548        local collected = structures.lists.collected
549        local data      = nil
550        local function action(reference)
551            local prefix    = data.prefix
552            local component = data.component
553            local realpage  = data.realpage
554            if realpage then
555                local pagelist  = rawget(tracedpages,realpage)
556                local internal  = data.internal or 0
557                local prefix    = (prefix ~= "" and prefix) or (component ~= "" and component) or ""
558                local pagedata  = { prefix, reference, internal }
559                if pagelist then
560                    pagelist[#pagelist+1] = pagedata
561                else
562                    tracedpages[realpage] = { pagedata }
563                end
564                if internal > 0 then
565                    data.usedprefix = prefix
566                end
567            end
568        end
569        for i=1,#collected do
570            local entry = collected[i]
571            local metadata = entry.metadata
572            if metadata and metadata.kind then
573                data = entry.references
574                if data then
575                    local reference = data.reference
576                    if reference and reference ~= "" then
577                        process_settings(reference,action)
578                    end
579                end
580            end
581        end
582    end
583    -- references
584    do
585        for prefix, list in next, collected do
586            for reference, entry in next, list do
587                local data = entry.references
588                if data then
589                    local realpage = data.realpage
590                    local internal = data.internal or 0
591                    local pagelist = rawget(tracedpages,realpage)
592                    local pagedata = { prefix, reference, internal }
593                    if pagelist then
594                        pagelist[#pagelist+1] = pagedata
595                    else
596                        tracedpages[realpage] = { pagedata }
597                    end
598                    if internal > 0 then
599                        data.usedprefix = prefix
600                    end
601                end
602            end
603        end
604    end
605end
606
607references.tracedpages = table.setmetatableindex(allocate(),function(t,k)
608    if collectbypage then
609        collectbypage(t)
610        collectbypage = nil
611    end
612    return rawget(t,k)
613end)
614
615-- urls
616
617local urls       = references.urls or { }
618references.urls  = urls
619local urldata    = urls.data or { }
620urls.data        = urldata
621
622local p_untexurl = Cs ( (
623    P("\\")/"" * (P("%")/"%%" + P(1))
624  + P(" ")/"%%20"
625  + P(1)
626)^1 )
627
628function urls.untex(url)
629    return lpegmatch(p_untexurl,url) or url
630end
631
632function urls.define(name,url,file,description)
633    if name and name ~= "" then
634     -- url = lpegmatch(replacer,url)
635        urldata[name] = { url or "", file or "", description or url or file or ""}
636    end
637end
638
639function urls.get(name)
640    local u = urldata[name]
641    if u then
642        local url, file = u[1], u[2]
643        if file and file ~= "" then
644            return formatters["%s/%s"](url,file)
645        else
646            return url
647        end
648    end
649end
650
651function urls.found(name)
652    return urldata[name]
653end
654
655local function geturl(name)
656    local url = urls.get(name)
657    if url and url ~= "" then
658        ctx_pushcatcodes(txtcatcodes)
659        context(url)
660        ctx_popcatcodes()
661    end
662end
663
664implement {
665    name      = "doifelseurldefined",
666    actions   = { urls.found, commands.doifelse },
667    arguments = "string"
668}
669
670implement {
671    name      = "useurl",
672    actions   = urls.define,
673    arguments = "4 strings",
674}
675
676implement {
677    name      = "geturl",
678    actions   = geturl,
679    arguments = "string",
680}
681
682-- files
683
684local files      = references.files or { }
685references.files = files
686local filedata   = files.data or { }
687files.data       = filedata
688
689function files.define(name,file,description)
690    if name and name ~= "" then
691        filedata[name] = { file or "", description or file or "" }
692    end
693end
694
695function files.get(name,method,space) -- method: none, before, after, both, space: yes/no
696    local f = filedata[name]
697    if f then
698        context(f[1])
699    end
700end
701
702function files.found(name)
703    return filedata[name]
704end
705
706local function getfile(name)
707    local fil = files.get(name)
708    if fil and fil ~= "" then
709        ctx_pushcatcodes(txtcatcodes)
710        context(fil)
711        ctx_popcatcodes()
712    end
713end
714
715implement {
716    name      = "doifelsefiledefined",
717    actions   = { files.found, commands.doifelse },
718    arguments = "string"
719}
720
721implement {
722    name      = "usefile",
723    actions   = files.define,
724    arguments = "3 strings"
725}
726
727implement {
728    name      = "getfile",
729    actions   = getfile,
730    arguments = "string"
731}
732
733-- helpers
734
735function references.checkedfile(whatever) -- return whatever if not resolved
736    if whatever then
737        local w = filedata[whatever]
738        if w then
739            return w[1]
740        else
741            return whatever
742        end
743    end
744end
745
746function references.checkedurl(whatever) -- return whatever if not resolved
747    if whatever then
748        local w = urldata[whatever]
749        if w then
750            local u, f = w[1], w[2]
751            if f and f ~= "" then
752                return u .. "/" .. f
753            else
754                return u
755            end
756        else
757            return whatever
758        end
759    end
760end
761
762function references.checkedfileorurl(whatever,default) -- return nil, nil if not resolved
763    if whatever then
764        local w = filedata[whatever]
765        if w then
766            return w[1], nil
767        else
768            local w = urldata[whatever]
769            if w then
770                local u, f = w[1], w[2]
771                if f and f ~= "" then
772                    return nil, u .. "/" .. f
773                else
774                    return nil, u
775                end
776            end
777        end
778    end
779    return default
780end
781
782-- programs
783
784local programs      = references.programs or { }
785references.programs = programs
786local programdata   = programs.data or { }
787programs.data       = programdata
788
789function programs.define(name,file,description)
790    if name and name ~= "" then
791        programdata[name] = { file or "", description or file or ""}
792    end
793end
794
795function programs.get(name)
796    local f = programdata[name]
797    return f and f[1]
798end
799
800function references.checkedprogram(whatever) -- return whatever if not resolved
801    if whatever then
802        local w = programdata[whatever]
803        if w then
804            return w[1]
805        else
806            return whatever
807        end
808    end
809end
810
811implement {
812    name      = "defineprogram",
813    actions   = programs.define,
814    arguments = "3 strings",
815}
816
817local function getprogram(name)
818    local p = programdata[name]
819    if p then
820        context(p[1])
821    end
822end
823
824implement {
825    name      = "getprogram",
826    actions   = getprogram,
827    arguments = "string"
828}
829
830-- shared by urls and files
831
832function references.from(name)
833    local u = urldata[name]
834    if u then
835        local url, file, description = u[1], u[2], u[3]
836        if description ~= "" then
837            return description
838            -- ok
839        elseif file and file ~= "" then
840            return url .. "/" .. file
841        else
842            return url
843        end
844    else
845        local f = filedata[name]
846        if f then
847            local file, description = f[1], f[2]
848            if description ~= "" then
849                return description
850            else
851                return file
852            end
853        end
854    end
855end
856
857local function from(name)
858    local u = urldata[name]
859    if u then
860        local url, file, description = u[1], u[2], u[3]
861        if description ~= "" then
862            ctx_dofromurldescription(description)
863            -- ok
864        elseif file and file ~= "" then
865            ctx_dofromurlliteral(url .. "/" .. file)
866        else
867            ctx_dofromurlliteral(url)
868        end
869    else
870        local f = filedata[name]
871        if f then
872            local file, description = f[1], f[2]
873            if description ~= "" then
874                ctx_dofromfiledescription(description)
875            else
876                ctx_dofromfileliteral(file)
877            end
878        end
879    end
880end
881
882implement {
883    name      = "from",
884    actions   = from,
885    arguments = "string"
886}
887
888function references.define(prefix,reference,list)
889    local d = defined[prefix] if not d then d = { } defined[prefix] = d end
890    d[reference] = list
891end
892
893function references.reset(prefix,reference)
894    local d = defined[prefix]
895    if d then
896        d[reference] = nil
897    end
898end
899
900implement {
901    name      = "definereference",
902    actions   = references.define,
903    arguments = "3 strings",
904}
905
906implement {
907    name      = "resetreference",
908    actions   = references.reset,
909    arguments = "2 strings",
910}
911
912setmetatableindex(defined,"table")
913
914local function resolve(prefix,reference,args,set) -- we start with prefix,reference
915    if reference and reference ~= "" then
916        if not set then
917            set = { prefix = prefix, reference = reference }
918        else
919            if not set.reference then set.reference = reference end
920            if not set.prefix    then set.prefix    = prefix    end
921        end
922     -- local r = settings_to_array(reference)
923        local r = settings_to_table(reference) -- maybe option to honor () []
924        for i=1,#r do
925            local ri = r[i]
926            local d = defined[prefix][ri] or defined[""][ri]
927            if d then
928                resolve(prefix,d,nil,set)
929            else
930                local var = splitreference(ri)
931                if var then
932                    var.reference = ri
933                    local vo, vi = var.outer, var.inner
934                    -- we catch this here .. it's a way to pass references with commas
935                    if vi == "name" then
936                        local arguments = var.arguments
937                        if arguments then
938                            vi            = arguments
939                            var.inner     = arguments
940                            var.reference = arguments
941                            var.arguments = nil
942                        end
943                    elseif var.special == "name" then
944                        local operation = var.operation
945                        if operation then
946                            vi            = operation
947                            var.inner     = operation
948                            var.reference = operation
949                            var.operation = nil
950                            var.special   = nil
951                        end
952                    end
953                    -- end of catch
954                    if not vo and vi then
955                        -- to be checked
956                        d = defined[prefix][vi] or defined[""][vi]
957                        --
958                        if d then
959                            resolve(prefix,d,var.arguments,set) -- args can be nil
960                        else
961                            if args then var.arguments = args end
962                            set[#set+1] = var
963                        end
964                    else
965                        if args then var.arguments = args end
966                        set[#set+1] = var
967                    end
968                    if var.has_tex then
969                        set.has_tex = true
970                    end
971                else
972                --  report_references("funny pattern %a",ri)
973                end
974            end
975        end
976        return set
977    else
978        return { }
979    end
980end
981
982-- prefix == "" is valid prefix which saves multistep lookup
983
984references.currentset = nil
985
986local function setreferenceoperation(k,v)
987    references.currentset[k].operation = v
988end
989
990local function setreferencearguments(k,v)
991    references.currentset[k].arguments = v
992end
993
994function references.expandcurrent() -- todo: two booleans: o_has_tex& a_has_tex
995    local currentset = references.currentset
996    if currentset and currentset.has_tex then
997        for i=1,#currentset do
998            local ci = currentset[i]
999            local operation = ci.operation
1000            if operation and find(operation,"\\",1,true) then -- if o_has_tex then
1001                ctx_expandreferenceoperation(i,operation)
1002            end
1003            local arguments = ci.arguments
1004            if arguments and find(arguments,"\\",1,true) then -- if a_has_tex then
1005                ctx_expandreferencearguments(i,arguments)
1006            end
1007        end
1008    end
1009end
1010
1011implement {
1012    name      = "expandcurrentreference",
1013    actions   = references.expandcurrent
1014}
1015
1016implement {
1017    name      = "setreferenceoperation",
1018    actions   = setreferenceoperation,
1019    arguments = { "integer", "string" }
1020}
1021
1022implement {
1023    name      = "setreferencearguments",
1024    actions   = setreferencearguments,
1025    arguments = { "integer", "string" }
1026}
1027
1028local externals = { }
1029
1030-- we have prefixes but also components:
1031--
1032-- :    prefix
1033-- ::   always external
1034-- :::  internal (for products) or external (for components)
1035
1036local function loadexternalreferences(name,utilitydata)
1037    local struc = utilitydata.structures
1038    if struc then
1039        local external = struc.references.collected -- direct references
1040        local lists    = struc.lists.collected      -- indirect references (derived)
1041        local pages    = struc.pages.collected      -- pagenumber data
1042        local sections = struc.sections.collected
1043        -- a bit weird one, as we don't have the externals in the collected
1044        for prefix, set in next, external do
1045            if prefix == "" then
1046                prefix = name -- this can clash!
1047            end
1048            for reference, data in next, set do
1049                if trace_importing then
1050                    report_importing("registering %a reference, kind %a, name %a, prefix %a, reference %a",
1051                        "external","regular",name,prefix,reference)
1052                end
1053                local section  = reference.section
1054                local realpage = reference.realpage
1055                if section then
1056                    reference.sectiondata = lists[section]
1057                end
1058                if realpage then
1059                    reference.pagedata = pages[realpage]
1060                end
1061            end
1062        end
1063        for i=1,#lists do
1064            local entry      = lists[i]
1065            local metadata   = entry.metadata
1066            local references = entry.references
1067            if metadata and references then
1068                local reference = references.reference
1069                if reference and reference ~= "" then
1070                    local kind     = metadata.kind
1071                    local realpage = references.realpage
1072                    if kind and realpage then
1073                        references.pagedata = pages[realpage]
1074                        local prefix = references.prefix or ""
1075                        if prefix == "" then
1076                            prefix = name -- this can clash!
1077                        end
1078                        local section = references.section
1079                        if section then
1080                            -- we have to make sure that the right section is used, see helpers.prefix
1081                            if sections then
1082                                references.sectiondata = sections[section]
1083                            else
1084                                -- warning
1085                            end
1086                        end
1087                        local target = external[prefix]
1088                        if not target then
1089                            target = { }
1090                            external[prefix] = target
1091                        end
1092                     -- for s in gmatch(reference,"%s*([^,]+)") do
1093                     --     if trace_importing then
1094                     --         report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1095                     --             "external",kind,name,prefix,s)
1096                     --     end
1097                     --     target[s] = target[s] or entry
1098                     -- end
1099                        local function action(s)
1100                            if trace_importing then
1101                                report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1102                                    "external",kind,name,prefix,s)
1103                            end
1104                            target[s] = target[s] or entry
1105                        end
1106                        process_settings(reference,action)
1107                    end
1108                end
1109            end
1110        end
1111        externals[name] = external
1112        return external
1113    end
1114end
1115
1116local externalfiles = { }
1117
1118setmetatableindex(externalfiles, function(t,k)
1119    local v = filedata[k]
1120    if not v then
1121        v = { k, k }
1122    end
1123    externalfiles[k] = v
1124    return v
1125end)
1126
1127setmetatableindex(externals, function(t,k) -- either or not automatically
1128    local filename = externalfiles[k][1] -- filename
1129    local fullname = file.replacesuffix(filename,"tuc")
1130    if lfs.isfile(fullname) then -- todo: use other locator
1131        local utilitydata = job.loadother(fullname)
1132        if utilitydata then
1133            local external = loadexternalreferences(k,utilitydata)
1134            t[k] = external or false
1135            return external
1136        end
1137    end
1138    t[k] = false
1139    return false
1140end)
1141
1142local productdata = allocate {
1143    productreferences   = { },
1144    componentreferences = { },
1145    components          = { },
1146}
1147
1148references.productdata = productdata
1149
1150local function loadproductreferences(productname,componentname,utilitydata)
1151    local struc = utilitydata.structures
1152    if struc then
1153        local productreferences = struc.references.collected -- direct references
1154        local lists = struc.lists.collected      -- indirect references (derived)
1155        local pages = struc.pages.collected      -- pagenumber data
1156        -- we use indirect tables to save room but as they are eventually
1157        -- just references we resolve them to data here (the mechanisms
1158        -- that use this data check for indirectness)
1159        for prefix, set in next, productreferences do
1160            for reference, data in next, set do
1161                if trace_importing then
1162                    report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1163                        "product","regular",productname,prefix,reference)
1164                end
1165                local section  = reference.section
1166                local realpage = reference.realpage
1167                if section then
1168                    reference.sectiondata = lists[section]
1169                end
1170                if realpage then
1171                    reference.pagedata = pages[realpage]
1172                end
1173            end
1174        end
1175        --
1176        local componentreferences = { }
1177        for i=1,#lists do
1178            local entry      = lists[i]
1179            local metadata   = entry.metadata
1180            local references = entry.references
1181            if metadata and references then
1182                local reference = references.reference
1183                if reference and reference ~= "" then
1184                    local kind     = metadata.kind
1185                    local realpage = references.realpage
1186                    if kind and realpage then
1187                        references.pagedata = pages[realpage]
1188                        local prefix    = references.prefix or ""
1189                        local component = references.component
1190                        local ctarget, ptarget
1191                        if not component or component == componentname then
1192                            -- skip
1193                        else
1194                            -- one level up
1195                            local external = componentreferences[component]
1196                            if not external then
1197                                external = { }
1198                                componentreferences[component] = external
1199                            end
1200                            if component == prefix then
1201                                prefix = ""
1202                            end
1203                            ctarget = external[prefix]
1204                            if not ctarget then
1205                                ctarget = { }
1206                                external[prefix] = ctarget
1207                            end
1208                        end
1209                        ptarget = productreferences[prefix]
1210                        if not ptarget then
1211                            ptarget = { }
1212                            productreferences[prefix] = ptarget
1213                        end
1214                        local function action(s)
1215                            if ptarget then
1216                                if trace_importing then
1217                                    report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1218                                        "product",kind,productname,prefix,s)
1219                                end
1220                                ptarget[s] = ptarget[s] or entry
1221                            end
1222                            if ctarget then
1223                                if trace_importing then
1224                                    report_importing("registering %s reference, kind %a, name %a, prefix %a, referenc %a",
1225                                        "component",kind,productname,prefix,s)
1226                                end
1227                                ctarget[s] = ctarget[s] or entry
1228                            end
1229                        end
1230                        process_settings(reference,action)
1231                    end
1232                end
1233            end
1234        end
1235        productdata.productreferences   = productreferences -- not yet used
1236        productdata.componentreferences = componentreferences
1237    end
1238end
1239
1240local function loadproductvariables(product,component,utilitydata)
1241    local struc = utilitydata.structures
1242    if struc then
1243        local lists = struc.lists and struc.lists.collected
1244        if lists then
1245            local pages = struc.pages and struc.pages.collected
1246            for i=1,#lists do
1247                local li = lists[i]
1248                if li.metadata.kind == "section" and li.references.component == component then
1249                    local firstsection = li
1250                    if firstsection.numberdata then
1251                        local numbers = firstsection.numberdata.numbers
1252                        if numbers then
1253                            if trace_importing then
1254                                report_importing("initializing section number to %:t",numbers)
1255                            end
1256                            productdata.firstsection = firstsection
1257                            structures.documents.preset(numbers)
1258                        end
1259                    end
1260                    if pages and firstsection.references then
1261                        local firstpage = pages[firstsection.references.realpage]
1262                        local number = firstpage and firstpage.number
1263                        if number then
1264                            if trace_importing then
1265                                report_importing("initializing page number to %a",number)
1266                            end
1267                            productdata.firstpage = firstpage
1268                            counters.set("userpage",1,number)
1269                        end
1270                    end
1271                    break
1272                end
1273            end
1274        end
1275    end
1276end
1277
1278local function componentlist(tree,target)
1279    local branches = tree and tree.branches
1280    if branches then
1281        for i=1,#branches do
1282            local branch = branches[i]
1283            local type = branch.type
1284            if type == "component" then
1285                if target then
1286                    target[#target+1] = branch.name
1287                else
1288                    target = { branch.name }
1289                end
1290            elseif type == "product" or type == "component" then
1291                target = componentlist(branch,target)
1292            end
1293        end
1294    end
1295    return target
1296end
1297
1298local function loadproductcomponents(product,component,utilitydata)
1299    local job = utilitydata.job
1300    productdata.components = componentlist(job and job.structure and job.structure.collected) or { }
1301end
1302
1303references.registerinitializer(function(tobesaved,collected)
1304    -- not that much related to tobesaved or collected
1305    productdata.components = componentlist(job.structure.collected) or { }
1306end)
1307
1308function references.loadpresets(product,component) -- we can consider a special components hash
1309    if product and component and product~= "" and component ~= "" and not productdata.product then -- maybe: productdata.filename ~= filename
1310        productdata.product = product
1311        productdata.component = component
1312        local fullname = file.replacesuffix(product,"tuc")
1313        if lfs.isfile(fullname) then -- todo: use other locator
1314            local utilitydata = job.loadother(fullname)
1315            if utilitydata then
1316                if trace_importing then
1317                    report_importing("loading references for component %a of product %a from %a",component,product,fullname)
1318                end
1319                loadproductvariables (product,component,utilitydata)
1320                loadproductreferences(product,component,utilitydata)
1321                loadproductcomponents(product,component,utilitydata)
1322            end
1323        end
1324    end
1325end
1326
1327references.productdata = productdata
1328
1329local useproduct = commands.useproduct
1330
1331if useproduct then
1332
1333    local function newuseproduct(product)
1334        useproduct(product)
1335        if texconditionals.autocrossfilereferences then
1336            local component = justacomponent()
1337            if component then
1338                if trace_referencing or trace_importing then
1339                    report_references("loading presets for component %a of product %a",component,product)
1340                end
1341                references.loadpresets(product,component)
1342            end
1343        end
1344    end
1345
1346    implement {
1347        name      = "useproduct",
1348        actions   = newuseproduct,
1349        arguments = "string",
1350        overload  = true,
1351    }
1352
1353end
1354
1355-- productdata.firstsection.numberdata.numbers
1356-- productdata.firstpage.number
1357
1358local function report_identify_special(set,var,i,type)
1359    local reference = set.reference
1360    local prefix    = set.prefix or ""
1361    local special   = var.special
1362    local error     = var.error
1363    local kind      = var.kind
1364    if error then
1365        report_identifying("type %a, reference %a, index %a, prefix %a, special %a, error %a",type,reference,i,prefix,special,error)
1366    else
1367        report_identifying("type %a, reference %a, index %a, prefix %a, special %a, kind %a",type,reference,i,prefix,special,kind)
1368    end
1369end
1370
1371local function report_identify_arguments(set,var,i,type)
1372    local reference = set.reference
1373    local prefix    = set.prefix or ""
1374    local arguments = var.arguments
1375    local error     = var.error
1376    local kind      = var.kind
1377    if error then
1378        report_identifying("type %a, reference %a, index %a, prefix %a, arguments %a, error %a",type,reference,i,prefix,arguments,error)
1379    else
1380        report_identifying("type %a, reference %a, index %a, prefix %a, arguments %a, kind %a",type,reference,i,prefix,arguments,kind)
1381    end
1382end
1383
1384local function report_identify_outer(set,var,i,type)
1385    local reference = set.reference
1386    local prefix    = set.prefix or ""
1387    local outer     = var.outer
1388    local error     = var.error
1389    local kind      = var.kind
1390    if outer then
1391        if error then
1392            report_identifying("type %a, reference %a, index %a, prefix %a, outer %a, error %a",type,reference,i,prefix,outer,error)
1393        else
1394            report_identifying("type %a, reference %a, index %a, prefix %a, outer %a, kind %a",type,reference,i,prefix,outer,kind)
1395        end
1396    else
1397        if error then
1398            report_identifying("type %a, reference %a, index %a, prefix %a, error %a",type,reference,i,prefix,error)
1399        else
1400            report_identifying("type %a, reference %a, index %a, prefix %a, kind %a",type,reference,i,prefix,kind)
1401        end
1402    end
1403end
1404
1405local function identify_special(set,var,i)
1406    local special = var.special
1407    local s = specials[special]
1408    if s then
1409        local outer     = var.outer
1410        local operation = var.operation
1411        local arguments = var.arguments
1412        if outer then
1413            if operation then
1414                -- special(outer::operation)
1415                var.kind = "special outer with operation"
1416            else
1417                -- special()
1418                var.kind = "special outer"
1419            end
1420            var.f = outer
1421        elseif operation then
1422            if arguments then
1423                -- special(operation{argument,argument})
1424                var.kind = "special operation with arguments"
1425            else
1426                -- special(operation)
1427                var.kind = "special operation"
1428            end
1429        else
1430            -- special()
1431            var.kind = "special"
1432        end
1433        if trace_identifying then
1434            report_identify_special(set,var,i,"1a")
1435        end
1436    else
1437        var.error = "unknown special"
1438    end
1439    return var
1440end
1441
1442local function identify_arguments(set,var,i)
1443    local s = specials[var.inner]
1444    if s then
1445        -- inner{argument}
1446        var.kind = "special operation with arguments"
1447    else
1448        var.error = "unknown inner or special"
1449    end
1450    if trace_identifying then
1451        report_identify_arguments(set,var,i,"3a")
1452    end
1453    return var
1454end
1455
1456-- needs checking: if we don't do too much (redundant) checking now
1457-- inner ... we could move the prefix logic into the parser so that we have 'm for each entry
1458-- foo:bar -> foo == prefix (first we try the global one)
1459-- -:bar   -> ignore prefix
1460
1461local function finish_inner(var,p,i)
1462    var.kind = "inner"
1463    var.i = i
1464    var.p = p
1465    var.r = (i.references and i.references.realpage) or (i.pagedata and i.pagedata.realpage) or 1
1466    return var
1467end
1468
1469local function identify_inner(set,var,prefix,collected,derived)
1470    local inner = var.inner
1471    -- the next test is a safeguard when references are auto loaded from outer
1472    if not inner or inner == "" then
1473        return false
1474    end
1475    local splitprefix, splitinner = lpegmatch(prefixsplitter,inner)
1476    if splitprefix and splitinner then
1477        -- we check for a prefix:reference instance in the regular set of collected
1478        -- references; a special case is -: which forces a lookup in the global list
1479        if splitprefix == "-" then
1480            local i = collected[""]
1481            if i then
1482                i = i[splitinner]
1483                if i then
1484                    return finish_inner(var,"",i)
1485                end
1486            end
1487        end
1488        local i = collected[splitprefix]
1489        if i then
1490            i = i[splitinner]
1491            if i then
1492                return finish_inner(var,splitprefix,i)
1493            end
1494        end
1495        if derived then
1496            -- next we look for a reference in the regular set of collected references
1497            -- using the prefix that is active at this moment (so we overload the given
1498            -- these are taken from other data structures (like lists)
1499            if splitprefix == "-" then
1500                local i = derived[""]
1501                if i then
1502                    i = i[splitinner]
1503                    if i then
1504                        return finish_inner(var,"",i)
1505                    end
1506                end
1507            end
1508            local i = derived[splitprefix]
1509            if i then
1510                i = i[splitinner]
1511                if i then
1512                    return finish_inner(var,splitprefix,i)
1513                end
1514            end
1515        end
1516    end
1517    -- we now ignore the split prefix and treat the whole inner as a potential
1518    -- reference into the global list
1519    local i = collected[prefix]
1520    if i then
1521        i = i[inner]
1522        if i then
1523            return finish_inner(var,prefix,i)
1524        end
1525    end
1526    if not i and derived then
1527        -- and if not found we look in the derived references
1528        local i = derived[prefix]
1529        if i then
1530            i = i[inner]
1531            if i then
1532                return finish_inner(var,prefix,i)
1533            end
1534        end
1535    end
1536    return false
1537end
1538
1539local function unprefixed_inner(set,var,prefix,collected,derived,tobesaved)
1540    local inner = var.inner
1541    local s = specials[inner]
1542    if s then
1543        var.kind = "special"
1544    else
1545        local i = (collected and collected[""] and collected[""][inner]) or
1546                  (derived   and derived  [""] and derived  [""][inner]) or
1547                  (tobesaved and tobesaved[""] and tobesaved[""][inner])
1548        if i then
1549            var.kind = "inner"
1550            var.p    = ""
1551            var.i    = i
1552            var.r    = (i.references and i.references.realpage) or (i.pagedata and i.pagedata.realpage) or 1
1553        else
1554            var.error = "unknown inner or special"
1555        end
1556    end
1557    return var
1558end
1559
1560local function identify_outer(set,var,i)
1561    local outer    = var.outer
1562    local inner    = var.inner
1563    local external = externals[outer]
1564    if external then
1565        local v = identify_inner(set,var,"",external)
1566        if v then
1567            v.kind = "outer with inner"
1568            set.external = true
1569            if trace_identifying then
1570                report_identify_outer(set,v,i,"2a")
1571            end
1572            return v
1573        end
1574-- weird too (we really need to check how this table is build
1575        local v = identify_inner(set,var,var.outer,external)
1576        if v then
1577            v.kind = "outer with inner"
1578            set.external = true
1579            if trace_identifying then
1580                report_identify_outer(set,v,i,"2c")
1581            end
1582            return v
1583        end
1584--
1585        -- somewhat rubish: we use outer as first step in the externals table so it makes no
1586        -- sense to have it as prefix so the next could be an option
1587        local external = external[""]
1588        if external then
1589            local v = identify_inner(set,var,var.outer,external)
1590            if v then
1591                v.kind = "outer with inner"
1592                set.external = true
1593                if trace_identifying then
1594                    report_identify_outer(set,v,i,"2b")
1595                end
1596                return v
1597            end
1598        end
1599    end
1600    local external = productdata.componentreferences[outer]
1601    if external then
1602        local v = identify_inner(set,var,"",external)
1603        if v then
1604            v.kind = "outer with inner"
1605            set.external = true
1606            if trace_identifying then
1607                report_identify_outer(set,v,i,"2c")
1608            end
1609            return v
1610        end
1611    end
1612    local external = productdata.productreferences[outer]
1613    if external then
1614        local vi = external[inner]
1615        if vi then
1616            var.kind = "outer with inner"
1617            var.i = vi
1618            set.external = true
1619            if trace_identifying then
1620                report_identify_outer(set,var,i,"2d")
1621            end
1622            return var
1623        end
1624    end
1625    -- the rest
1626    local special   = var.special
1627    local arguments = var.arguments
1628    local operation = var.operation
1629    if inner then
1630        -- tricky: in this case we can only use views when we're sure that all inners
1631        -- are flushed in the outer document so that should become an option
1632        if arguments then
1633            -- outer::inner{argument}
1634            var.kind = "outer with inner with arguments"
1635        else
1636            -- outer::inner
1637            var.kind = "outer with inner"
1638        end
1639        var.i = inner
1640        var.f = outer
1641        if type(inner) == "table" then
1642            -- can this really happen?
1643            var.r = (inner.references and inner.references.realpage) or (inner.pagedata and inner.pagedata.realpage) or 1
1644        else
1645            var.r = 1
1646        end
1647        if trace_identifying then
1648            report_identify_outer(set,var,i,"2e")
1649        end
1650    elseif special then
1651        local s = specials[special]
1652        if s then
1653            if operation then
1654                if arguments then
1655                    -- outer::special(operation{argument,argument})
1656                    var.kind = "outer with special and operation and arguments"
1657                else
1658                    -- outer::special(operation)
1659                    var.kind = "outer with special and operation"
1660                end
1661            else
1662                -- outer::special()
1663                var.kind = "outer with special"
1664            end
1665            var.f = outer
1666        else
1667            var.error = "unknown outer with special"
1668        end
1669        if trace_identifying then
1670            report_identify_outer(set,var,i,"2f")
1671        end
1672    else
1673        -- outer::
1674        var.kind = "outer"
1675        var.f = outer
1676        if trace_identifying then
1677            report_identify_outer(set,var,i,"2g")
1678        end
1679    end
1680    return var
1681end
1682
1683-- todo: avoid copy
1684
1685local function identify_inner_or_outer(set,var,i)
1686    -- here we fall back on product data
1687    local inner = var.inner
1688    if inner and inner ~= "" then
1689
1690        -- first we look up in collected and derived using the current prefix
1691
1692        local prefix = set.prefix
1693
1694        local v = identify_inner(set,var,set.prefix,collected,derived)
1695        if v then
1696            if trace_identifying then
1697                report_identify_outer(set,v,i,"4a")
1698            end
1699            return v
1700        end
1701
1702        -- nest we look at each component (but we can omit the already consulted one
1703
1704        local jobstructure = job.structure
1705        local components   = jobstructure and jobstructure.components
1706        if components then
1707            for c=1,#components do
1708                local component = components[c]
1709                if component ~= prefix then
1710                    local v = identify_inner(set,var,component,collected,derived)
1711                    if v then
1712                        if trace_identifying then
1713                            report_identify_outer(set,var,i,"4b")
1714                        end
1715                        return v
1716                    end
1717                end
1718            end
1719        end
1720
1721        -- as a last resort we will consult the global lists
1722
1723        local v = unprefixed_inner(set,var,"",collected,derived,tobesaved)
1724        if v then
1725            if trace_identifying then
1726                report_identify_outer(set,v,i,"4c")
1727            end
1728            return v
1729        end
1730
1731        -- not it gets bad ... we need to look in external files ... keep in mind that
1732        -- we can best use explicit references for this ... we might issue a warning
1733
1734        local componentreferences = productdata.componentreferences
1735        local productreferences = productdata.productreferences
1736        local components = productdata.components
1737        if components and componentreferences then
1738            for c=1,#components do
1739                local component = components[c]
1740                local data = componentreferences[component]
1741                if data then
1742                    local d = data[""]
1743                    local vi = d and d[inner]
1744                    if vi then
1745                        var.outer = component
1746                        var.i = vi
1747                        var.kind = "outer with inner"
1748                        set.external = true
1749                        if trace_identifying then
1750                            report_identify_outer(set,var,i,"4d")
1751                        end
1752                        return var
1753                    end
1754                end
1755            end
1756        end
1757        local component, inner = lpegmatch(componentsplitter,inner)
1758        if component then
1759            local data = componentreferences and componentreferences[component]
1760            if data then
1761                local d = data[""]
1762                local vi = d and d[inner]
1763                if vi then
1764                    var.inner = inner
1765                    var.outer = component
1766                    var.i = vi
1767                    var.kind = "outer with inner"
1768                    set.external = true
1769                    if trace_identifying then
1770                        report_identify_outer(set,var,i,"4e")
1771                    end
1772                    return var
1773                end
1774            end
1775            local data = productreferences and productreferences[component]
1776            if data then
1777                local vi = data[inner]
1778                if vi then
1779                    var.inner = inner
1780                    var.outer = component
1781                    var.i = vi
1782                    var.kind = "outer with inner"
1783                    set.external = true
1784                    if trace_identifying then
1785                        report_identify_outer(set,var,i,"4f")
1786                    end
1787                    return var
1788                end
1789            end
1790        end
1791        var.error = "unknown inner"
1792    else
1793        var.error = "no inner"
1794    end
1795    if trace_identifying then
1796        report_identify_outer(set,var,i,"4g")
1797    end
1798    return var
1799end
1800
1801local function identify_inner_component(set,var,i)
1802    -- we're in a product (maybe ignore when same as component)
1803    local component = var.component
1804    local v = identify_inner(set,var,component,collected,derived)
1805    if not v then
1806        var.error = "unknown inner in component"
1807    end
1808    if trace_identifying then
1809        report_identify_outer(set,var,i,"5a")
1810    end
1811    return var
1812end
1813
1814local function identify_outer_component(set,var,i)
1815    local component = var.component
1816    local inner = var.inner
1817    local data = productdata.componentreferences[component]
1818    if data then
1819        local d = data[""]
1820        local vi = d and d[inner]
1821        if vi then
1822            var.inner = inner
1823            var.outer = component
1824            var.i = vi
1825            var.kind = "outer with inner"
1826            set.external = true
1827            if trace_identifying then
1828                report_identify_outer(set,var,i,"6a")
1829            end
1830            return var
1831        end
1832    end
1833    local data = productdata.productreferences[component]
1834    if data then
1835        local vi = data[inner]
1836        if vi then
1837            var.inner = inner
1838            var.outer = component
1839            var.i = vi
1840            var.kind = "outer with inner"
1841            set.external = true
1842            if trace_identifying then
1843                report_identify_outer(set,var,i,"6b")
1844            end
1845            return var
1846        end
1847    end
1848    var.error = "unknown component"
1849    if trace_identifying then
1850        report_identify_outer(set,var,i,"6c")
1851    end
1852    return var
1853end
1854
1855local nofidentified = 0
1856
1857local function identify(prefix,reference)
1858    if not reference then
1859        prefix, reference = "", prefix
1860    end
1861    local set = resolve(prefix,reference)
1862    local bug = false
1863    texsetcount("referencehastexstate",set.has_tex and 1 or 0)
1864    nofidentified = nofidentified + 1
1865    set.n = nofidentified
1866    for i=1,#set do
1867        local var = set[i]
1868        local spe = var.special
1869        local fnc = functions[spe]
1870        if fnc then
1871            var = fnc(var) or { error = "invalid special function" }
1872        elseif spe then
1873            var = identify_special(set,var,i)
1874        elseif var.outer then
1875            var = identify_outer(set,var,i)
1876        elseif var.arguments then
1877            var = identify_arguments(set,var,i)
1878        elseif not var.component then
1879            var = identify_inner_or_outer(set,var,i)
1880        elseif productcomponent() then
1881            var = identify_inner_component(set,var,i)
1882        else
1883            var = identify_outer_component(set,var,i)
1884        end
1885        set[i] = var
1886        bug = bug or var.error
1887    end
1888    references.currentset = mark(set) -- mark, else in api doc
1889    if trace_analyzing then
1890        report_references(table.serialize(set,reference))
1891    end
1892    return set, bug
1893end
1894
1895references.identify = identify
1896
1897local unknowns, nofunknowns, f_valid = { }, 0, formatters["[%s][%s]"]
1898
1899function references.valid(prefix,reference,specification)
1900    local set, bug = identify(prefix,reference)
1901    local unknown = bug or #set == 0
1902    if unknown then
1903        currentreference = nil -- will go away
1904        local str = f_valid(prefix,reference)
1905        local u = unknowns[str]
1906        if not u then
1907            if somefound then
1908                interfaces.showmessage("references",1,str) -- 1 = unknown, 4 = illegal
1909            end
1910            unknowns[str] = 1
1911            nofunknowns = nofunknowns + 1
1912        else
1913            unknowns[str] = u + 1
1914        end
1915    else
1916        set.highlight    = specification.highlight
1917        set.newwindow    = specification.newwindow
1918        set.layer        = specification.layer
1919        currentreference = set[1]
1920    end
1921    -- we can do the expansion here which saves a call
1922    return not unknown
1923end
1924
1925implement {
1926    name      = "doifelsereference",
1927    actions   = { references.valid, commands.doifelse },
1928    arguments = {
1929        "string",
1930        "string",
1931        {
1932            { "highlight", "boolean" },
1933            { "newwindow", "boolean" },
1934            { "layer" },
1935        }
1936    }
1937}
1938
1939logs.registerfinalactions(function()
1940    if nofunknowns > 0 then
1941        statistics.register("cross referencing", function()
1942            return format("%s identified, %s unknown",nofidentified,nofunknowns)
1943        end)
1944        local sortedhash = table.sortedhash
1945        logs.startfilelogging(report,"missing references")
1946        for k, v in table.sortedhash(unknowns) do
1947            report("%4i  %s",v,k)
1948        end
1949        logs.stopfilelogging()
1950        if logs.loggingerrors() then
1951            logs.starterrorlogging(report,"missing references")
1952            for k, v in table.sortedhash(unknowns) do
1953                report("%4i  %s",v,k)
1954            end
1955            logs.stoperrorlogging()
1956        end
1957    end
1958end)
1959
1960-- The auto method will try to avoid named internals in a clever way which
1961-- can make files smaller without sacrificing external references. Some of
1962-- the housekeeping happens the backend side.
1963
1964local innermethod        = v_auto       -- only page|auto now
1965local outermethod        = v_auto       -- only page|auto now
1966local defaultinnermethod = defaultinnermethod
1967local defaultoutermethod = defaultoutermethod
1968references.innermethod   = innermethod  -- don't mess with this one directly
1969references.outermethod   = outermethod  -- don't mess with this one directly
1970
1971function references.setlinkmethod(inner,outer)
1972    if not outer and type(inner) == "string" then
1973        local m = settings_to_array(inner)
1974        inner = m[1]
1975        outer = m[2] or v_auto
1976    end
1977    if toboolean(inner) or inner == v_page or inner == v_yes then
1978        innermethod = v_page
1979    elseif inner == v_name then
1980        innermethod = v_name
1981    else
1982        innermethod = v_auto
1983    end
1984    if toboolean(outer) or outer == v_page or outer == v_yes then
1985        outermethod = v_page
1986    elseif inner == v_name then
1987        outermethod = v_name
1988    else
1989        outermethod = v_auto
1990    end
1991    references.innermethod = innermethod
1992    references.outermethod = outermethod
1993    function references.setlinkmethod()
1994        report_references("link method is already set and frozen: inner %a, outer %a",innermethod,outermethod)
1995    end
1996end
1997
1998implement {
1999    name      = "setreferencelinkmethod",
2000    actions   = references.setlinkmethod,
2001    arguments = "string",
2002 -- onlyonce  = true
2003}
2004
2005function references.getinnermethod()
2006    return innermethod or defaultinnermethod
2007end
2008
2009function references.getoutermethod()
2010    return outermethod or defaultoutermethod
2011end
2012
2013directives.register("references.linkmethod", function(v) -- page auto
2014    references.setlinkmethod(v)
2015end)
2016
2017-- we can call setinternalreference with an already known internal or with
2018-- a reference/prefix specification
2019
2020local destinationattributes = { }
2021
2022local function setinternalreference(specification)
2023    local internal    = specification.internal
2024    local destination = unsetvalue
2025    if innermethod == v_auto or innermethod == v_name then
2026        local t         = { } -- maybe add to current (now only used for tracing)
2027        local tn        = 0
2028        local reference = specification.reference
2029        local view      = specification.view
2030        if reference then
2031            local prefix = specification.prefix
2032            if prefix and prefix ~= "" then
2033                local prefix = prefix .. ":" -- watch out, : here
2034                local function action(ref)
2035                    tn = tn + 1
2036                    t[tn] = prefix .. ref
2037                end
2038                process_settings(reference,action)
2039            else
2040                local function action(ref)
2041                    tn = tn + 1
2042                    t[tn] = ref
2043                end
2044                process_settings(reference,action)
2045            end
2046        end
2047        -- ugly .. later we decide to ignore it when we have a real one
2048        -- but for testing we might want to see them all
2049        if internal then
2050            if innermethod ~= v_name then -- innermethod == v_auto
2051             -- we don't want too many #1 #2 #3 etc
2052                tn = tn + 1
2053                t[tn] = internal -- when number it's internal
2054            end
2055            if not view then
2056                local i = references.internals[internal]
2057                if i then
2058                    view = i.references.view
2059                end
2060            end
2061        end
2062        destination = references.mark(t,nil,nil,view) -- returns an attribute
2063    end
2064    if internal then -- new
2065        destinationattributes[internal] = destination
2066    end
2067    texsetcount("lastdestinationattribute",destination)
2068    return destination
2069end
2070
2071local function getinternalreference(internal)
2072    return destinationattributes[internal] or 0
2073end
2074
2075references.setinternalreference = setinternalreference
2076references.getinternalreference = getinternalreference
2077
2078implement {
2079    name      = "setinternalreference",
2080    actions   = setinternalreference,
2081    arguments = {
2082        {
2083            { "prefix" },
2084            { "reference" },
2085            { "internal", "integer" },
2086            { "view" }
2087        }
2088    }
2089}
2090
2091-- implement {
2092--     name      = "getinternalreference",
2093--     actions   = { getinternalreference, context },
2094--     arguments = "integer",
2095-- }
2096
2097function references.setandgetattribute(data) -- maybe do internal automatically here
2098    local attr = unsetvalue
2099    local mdat = data.metadata
2100    local rdat = data.references
2101    if mdat and rdat then
2102        if not rdat.section then
2103            rdat.section = structures.sections.currentid()
2104        end
2105        local ndat = data.numberdata
2106        if ndat then
2107            local numbers = ndat.numbers
2108            if type(numbers) == "string" then
2109                counters.compact(ndat,numbers)
2110            end
2111            data.numberdata = helpers.simplify(ndat)
2112        end
2113        local pdat = data.prefixdata
2114        if pdat then
2115            data.prefixdata = helpers.simplify(pdat)
2116        end
2117        local udat = data.userdata
2118        if type(udat) == "string"  then
2119            data.userdata = helpers.touserdata(udat)
2120        end
2121        if not rdat.block then
2122            rdat.block = structures.sections.currentblock()
2123        end
2124        local done = references.set(data) -- we had kind i.e .item -> full
2125        if done then
2126            attr = setinternalreference {
2127                prefix    = rdat.prefix,
2128                reference = rdat.reference,
2129                internal  = rdat.internal,
2130                view      = rdat.view
2131            } or unsetvalue
2132        end
2133    end
2134    texsetcount("lastdestinationattribute",attr)
2135    return attr
2136end
2137
2138implement {
2139    name      = "setdestinationattribute",
2140    actions   = references.setandgetattribute,
2141    arguments = {
2142        {
2143            {
2144                "references", {
2145                    { "internal", "integer" },
2146                    { "block" },
2147                    { "view" },
2148                    { "prefix" },
2149                    { "reference" },
2150                },
2151            },
2152            {
2153                "metadata", {
2154                    { "kind" },
2155                    { "xmlroot" },
2156                    { "catcodes", "integer" },
2157                },
2158            },
2159            {
2160                "prefixdata", { "*" }
2161            },
2162            {
2163                "numberdata", { "*" }
2164            },
2165            {
2166                "entries", { "*" }
2167            },
2168            {
2169                "userdata"
2170            }
2171        }
2172    }
2173}
2174
2175function references.getinternallistreference(n) -- n points into list (todo: registers)
2176    local l = lists.collected[n]
2177    local i = l and l.references.internal
2178    return i and destinationattributes[i] or 0
2179end
2180
2181function references.getinternalcachedlistreference(n) -- n points into list (todo: registers)
2182    local l = lists.cached[n]
2183    local i = l and l.references.internal
2184    return i and destinationattributes[i] or 0
2185end
2186
2187implement {
2188    name      = "getinternallistreference",
2189    actions   = { references.getinternallistreference, context },
2190    arguments = "integer"
2191}
2192
2193implement {
2194    name      = "getinternalcachedlistreference",
2195    actions   = { references.getinternalcachedlistreference, context },
2196    arguments = "integer"
2197}
2198
2199
2200--
2201
2202function references.getcurrentmetadata(tag)
2203    local data = currentreference and currentreference.i
2204    if data then
2205        local entry = data.metadata
2206        local value = entry[tag]
2207        if value then
2208            return value
2209        end
2210    end
2211end
2212
2213implement {
2214    name      = "getcurrentreferencemetadata",
2215    actions   = { references.getcurrentmetadata, context },
2216    arguments = "string",
2217}
2218
2219function references.getcurrentlabel()
2220    local data = currentreference and currentreference.i
2221    if data then
2222        local entry = data.titledata
2223        local value = entry.label
2224        if value then
2225            return value
2226        end
2227        local entry = data.metadata
2228        local value = entry[tag]
2229        if value then
2230            return value
2231        end
2232    end
2233end
2234
2235implement {
2236    name    = "getcurrentreferencelabel",
2237    actions = { references.getcurrentlabel, context },
2238}
2239
2240local function currentmetadata(tag)
2241    local data = currentreference and currentreference.i
2242    return data and data.metadata and data.metadata[tag]
2243end
2244
2245references.currentmetadata = currentmetadata
2246
2247local function getcurrentprefixspec(default)
2248    local data     = currentreference and currentreference.i
2249    local metadata = data and data.metadata
2250    return
2251        metadata and metadata.kind or "?",
2252        metadata and metadata.name or "?",
2253        default                    or "?"
2254end
2255
2256references.getcurrentprefixspec = getcurrentprefixspec
2257
2258-- implement {
2259--     name      = "getcurrentprefixspec",
2260--     actions   = { getcurrentprefixspec, context }, -- returns 3 arguments
2261--     arguments = "string",
2262-- }
2263
2264implement {
2265    name      = "getcurrentprefixspec",
2266    actions   = function(tag)
2267        context("{%s}{%s}{%s}",getcurrentprefixspec(tag))
2268    end,
2269    arguments = "string",
2270}
2271
2272local genericfilters = { }
2273local userfilters    = { }
2274local textfilters    = { }
2275local fullfilters    = { }
2276local sectionfilters = { }
2277
2278filters.generic = genericfilters
2279filters.user    = userfilters
2280filters.text    = textfilters
2281filters.full    = fullfilters
2282filters.section = sectionfilters
2283
2284local function filterreference(name,prefixspec,numberspec) -- number page title ...
2285    local data = currentreference and currentreference.i -- maybe we should take realpage from here
2286    if data then
2287        if name == "realpage" then
2288            local cs = references.analyze() -- normally already analyzed but also sets state
2289            context(tonumber(cs.realpage) or 0)
2290        else -- assumes data is table
2291            local kind = type(data) == "table" and data.metadata and data.metadata.kind
2292            if kind then
2293                local filter = filters[kind] or genericfilters
2294                filter = filter and (filter[name] or filter.unknown or genericfilters[name] or genericfilters.unknown)
2295                if filter then
2296                    if trace_referencing then
2297                        report_references("name %a, kind %a, using dedicated filter",name,kind)
2298                    end
2299                    filter(data,name,prefixspec,numberspec)
2300                elseif trace_referencing then
2301                    report_references("name %a, kind %a, using generic filter",name,kind)
2302                end
2303            elseif trace_referencing then
2304                report_references("name %a, unknown kind",name)
2305            end
2306        end
2307    elseif name == "realpage" then
2308        context(0)
2309    elseif trace_referencing then
2310        report_references("name %a, no reference",name)
2311    end
2312end
2313
2314local function filterreferencedefault()
2315    return filterreference("default",getcurrentprefixspec("default"))
2316end
2317
2318references.filter        = filterreference
2319references.filterdefault = filterreferencedefault
2320
2321implement {
2322    name      = "filterreference",
2323    actions   = filterreference,
2324    arguments = "string",
2325}
2326
2327implement {
2328    name      = "filterdefaultreference",
2329    actions   = filterreference,
2330    arguments = {
2331        "string",    -- 'default'
2332        { { "*" } }, -- prefixspec
2333        { { "*" } }, -- numberspec
2334    }
2335}
2336
2337function genericfilters.title(data)
2338    if data then
2339        local titledata = data.titledata or data.useddata
2340        if titledata then
2341            helpers.title(titledata.reference or titledata.title or "?",data.metadata)
2342        end
2343    end
2344end
2345
2346function genericfilters.text(data)
2347    if data then
2348        local entries = data.entries or data.useddata
2349        if entries then
2350            helpers.title(entries.text or "?",data.metadata)
2351        end
2352    end
2353end
2354
2355function genericfilters.number(data,what,prefixspec,numberspec)
2356    if data then
2357        numberdata = lists.reordered(data) -- data.numberdata
2358        if numberdata then
2359            helpers.prefix(data,prefixspec)
2360            sections.typesetnumber(numberdata,"number",numberspec,numberdata)
2361        else
2362            local useddata = data.useddata
2363            if useddata and useddata.number then
2364                context(useddata.number)
2365            end
2366        end
2367    end
2368end
2369
2370genericfilters.default = genericfilters.text
2371
2372function genericfilters.page(data,prefixspec,pagespec)
2373    local pagedata = data.pagedata
2374    if pagedata then
2375        local number     = pagedata.number
2376        local conversion = pagedata.conversion
2377        if not number then
2378            -- error
2379        elseif conversion then
2380            ctx_convertnumber(conversion,number)
2381        else
2382            context(number)
2383        end
2384    else
2385        helpers.prefixpage(data,prefixspec,pagespec)
2386    end
2387end
2388
2389function userfilters.unknown(data,name)
2390    if data then
2391        local userdata = data.userdata
2392        local userkind = userdata and userdata.kind
2393        if userkind then
2394            local filter = filters[userkind] or genericfilters
2395            filter = filter and (filter[name] or filter.unknown)
2396            if filter then
2397                filter(data,name)
2398                return
2399            end
2400        end
2401        local namedata = userdata and userdata[name]
2402        if namedata then
2403            context(namedata)
2404        end
2405    end
2406end
2407
2408function textfilters.title(data)
2409    helpers.title(data.entries.text or "?",data.metadata)
2410end
2411
2412-- no longer considered useful:
2413--
2414-- function filters.text.number(data)
2415--     helpers.title(data.entries.text or "?",data.metadata)
2416-- end
2417
2418function textfilters.page(data,prefixspec,pagespec)
2419    helpers.prefixpage(data,prefixspec,pagespec)
2420end
2421
2422fullfilters.title = textfilters.title
2423fullfilters.page  = textfilters.page
2424
2425function sectionfilters.number(data,what,prefixspec)
2426    if data then
2427        local numberdata = data.numberdata
2428        if not numberdata then
2429            local useddata = data.useddata
2430            if useddata and useddata.number then
2431                context(useddata.number)
2432            end
2433        elseif numberdata.hidenumber then
2434            local references = data.references
2435            if trace_empty then
2436                report_empty("reference %a has a hidden number",references.reference)
2437                ctx_emptyreference() -- maybe an option
2438            end
2439        else
2440            sections.typesetnumber(numberdata,"number",prefixspec,numberdata)
2441        end
2442    end
2443end
2444
2445sectionfilters.title   = genericfilters.title
2446sectionfilters.page    = genericfilters.page
2447sectionfilters.default = sectionfilters.number
2448
2449-- filters.note        = { default = genericfilters.number }
2450-- filters.formula     = { default = genericfilters.number }
2451-- filters.float       = { default = genericfilters.number }
2452-- filters.description = { default = genericfilters.number }
2453-- filters.item        = { default = genericfilters.number }
2454
2455setmetatableindex(filters, function(t,k) -- beware, test with rawget
2456    local v = { default = genericfilters.number } -- not copy as it might be extended differently
2457    t[k] = v
2458    return v
2459end)
2460
2461-- function references.sectiontitle(n)
2462--     helpers.sectiontitle(lists.collected[tonumber(n) or 0])
2463-- end
2464
2465-- function references.sectionnumber(n)
2466--     helpers.sectionnumber(lists.collected[tonumber(n) or 0])
2467-- end
2468
2469-- function references.sectionpage(n,prefixspec,pagespec)
2470--     helpers.prefixedpage(lists.collected[tonumber(n) or 0],prefixspec,pagespec)
2471-- end
2472
2473-- analyze
2474
2475references.testrunners  = references.testrunners  or { }
2476references.testspecials = references.testspecials or { }
2477
2478local runners  = references.testrunners
2479local specials = references.testspecials
2480
2481-- We need to prevent ending up in the 'relative location' analyzer as it is
2482-- pretty slow (progressively). In the pagebody one can best check the reference
2483-- real page to determine if we need contrastlocation as that is more lightweight.
2484
2485local function checkedpagestate(n,page,actions,position,spread)
2486    local p = tonumber(page)
2487    if not p then
2488        return 0
2489    end
2490    if position and #actions > 0 then
2491        local i = actions[1].i -- brrr
2492        if i then
2493            local a = i.references
2494            if a then
2495                local x = a.x
2496                local y = a.y
2497                if x and y then
2498                    local jp = jobpositions.collected[position]
2499                    if jp then
2500                        local px = jp.x
2501                        local py = jp.y
2502                        local pp = jp.p
2503                        if p == pp then
2504                            -- same page
2505                            if py > y then
2506                                return 5 -- above
2507                            elseif py < y then
2508                                return 4 -- below
2509                            elseif px > x then
2510                                return 4 -- below
2511                            elseif px < x then
2512                                return 5 -- above
2513                            else
2514                                return 1 -- same
2515                            end
2516                        elseif spread then
2517                            if pp % 2 == 0 then
2518                                -- left page
2519                                if pp > p then
2520                                    return 2 -- before
2521                                elseif pp + 1 == p then
2522--                                     return 4 -- below (on right page)
2523                                    return 5 -- above (on left page)
2524                                else
2525                                    return 3 -- after
2526                                end
2527                            else
2528                                -- right page
2529                                if pp < p then
2530                                    return 3 -- after
2531                                elseif pp - 1 == p then
2532--                                     return 5 -- above (on left page)
2533                                    return 4 -- below (on right page)
2534                                else
2535                                    return 2 -- before
2536                                end
2537                            end
2538                        elseif pp > p then
2539                            return 2 -- before
2540                        else
2541                            return 3 -- after
2542                        end
2543                    end
2544                end
2545            end
2546        end
2547    end
2548    local r = referredpage(n) -- sort of obsolete
2549    if p > r then
2550        return 3 -- after
2551    elseif p < r then
2552        return 2 -- before
2553    else
2554        return 1 -- same
2555    end
2556end
2557
2558local function setreferencerealpage(actions)
2559    if not actions then
2560        actions = references.currentset
2561    end
2562    if type(actions) == "table" then
2563        local realpage = actions.realpage
2564        if realpage then
2565            return realpage
2566        end
2567        local nofactions = #actions
2568        if nofactions > 0 then
2569            for i=1,nofactions do
2570                local a = actions[i]
2571                local what = runners[a.kind]
2572                if what then
2573                    what = what(a,actions) -- needs documentation
2574                end
2575            end
2576            realpage = actions.realpage
2577            if realpage then
2578                return realpage
2579            end
2580        end
2581        actions.realpage = 0
2582    end
2583    return 0
2584end
2585
2586references.setreferencerealpage = setreferencerealpage
2587
2588-- we store some analysis data alongside the indexed array
2589-- at this moment only the real reference page is analyzed
2590-- normally such an analysis happens in the backend code
2591
2592function references.analyze(actions,position,spread)
2593    if not actions then
2594        actions = references.currentset
2595    end
2596    if not actions then
2597        actions = { realpage = 0, pagestate = 0 }
2598    elseif actions.pagestate then
2599        -- already done
2600    else
2601        local realpage = actions.realpage or setreferencerealpage(actions)
2602        if realpage == 0 then
2603            actions.pagestate = 0
2604        elseif actions.external then
2605            actions.pagestate = 0
2606        else
2607            actions.pagestate = checkedpagestate(actions.n,realpage,actions,position,spread)
2608        end
2609    end
2610    return actions
2611end
2612
2613local function referencepagestate(position,detail,spread)
2614    local actions = references.currentset
2615    if not actions then
2616        return 0
2617    else
2618        local pagestate = actions.pagestate
2619        for i=1,#actions do
2620            local a = actions[i]
2621            if a.outer then
2622                pagestate = 0
2623                actions.pagestate = pagestate
2624                break
2625            end
2626        end
2627        if not pagestate then
2628            references.analyze(actions,position,spread) -- delayed unless explicitly asked for
2629            pagestate = actions.pagestate
2630        end
2631        if detail then
2632            return pagestate
2633        elseif pagestate == 4 then
2634            return 2 -- compatible
2635        elseif pagestate == 5 then
2636            return 3 -- compatible
2637        else
2638            return pagestate
2639        end
2640    end
2641end
2642
2643implement {
2644    name      = "referencepagestate",
2645    actions   = { referencepagestate, context },
2646    arguments = "string"
2647}
2648
2649implement {
2650    name      = "referencepagedetail",
2651    actions   = { referencepagestate, context },
2652    arguments = { "string", "boolean", "boolean" }
2653}
2654
2655-- local function referencerealpage()
2656--     local actions = references.currentset
2657--     return not actions and 0 or actions.realpage or setreferencerealpage(actions)
2658-- end
2659--
2660-- implement {
2661--     name      = "referencerealpage",
2662--     actions   = { referencerealpage, context },
2663--  -- arguments = "string" -- hm, weird
2664-- }
2665
2666implement {
2667    name      = "askedreference",
2668    public    = true,
2669    protected = true,
2670    actions   = function()
2671        local actions = references.currentset
2672        if actions then
2673            context("[p=%s,r=%s]",actions.prefix or "",actions.reference)
2674        end
2675    end
2676}
2677
2678implement {
2679    name    = "referencerealpage",
2680    actions = function()
2681        local actions = references.currentset
2682        context(not actions and 0 or actions.realpage or setreferencerealpage(actions))
2683    end
2684}
2685
2686local function referencepos(key)
2687    local actions = references.currentset
2688    local i = actions[1].i -- brrr
2689    local v = 0
2690    if i then
2691        local a = i.references
2692        if a then
2693            v = a[key] or 0
2694        end
2695    end
2696    return v
2697end
2698
2699implement { name = "referenceposx", actions = function() context("%p",referencepos("x")) end }
2700implement { name = "referenceposy", actions = function() context("%p",referencepos("y")) end }
2701
2702implement {
2703    name    = "referencecolumn",
2704    actions = function()
2705        local actions = references.currentset
2706        local column  = 1
2707        if actions then
2708            column = jobpositions.columnofpos(actions.realpage or setreferencerealpage(actions),referencepos("x"))
2709        end
2710        context(column or 1)
2711    end
2712}
2713
2714local plist, nofrealpages
2715
2716local function realpageofpage(p) -- the last one counts !
2717    if not plist then
2718        local pages = structures.pages.collected
2719        nofrealpages = #pages
2720        plist = { }
2721        for rp=1,nofrealpages do
2722            local page = pages[rp]
2723            if page then
2724                plist[page.number] = rp
2725            end
2726        end
2727        references.nofrealpages = nofrealpages
2728    end
2729    return plist[p]
2730end
2731
2732references.realpageofpage = realpageofpage
2733
2734function references.checkedrealpage(r)
2735    if not plist then
2736        realpageofpage(r) -- just initialize
2737    end
2738    if not r then
2739        return texgetcount("realpageno")
2740    elseif r < 1 then
2741        return 1
2742    elseif r > nofrealpages then
2743        return nofrealpages
2744    else
2745        return r
2746    end
2747end
2748
2749-- use local ?
2750
2751local pages = allocate {
2752    [variables.firstpage]       = function() return counters.record("realpage")["first"]    end,
2753    [variables.previouspage]    = function() return counters.record("realpage")["previous"] end,
2754    [variables.nextpage]        = function() return counters.record("realpage")["next"]     end,
2755    [variables.lastpage]        = function() return counters.record("realpage")["last"]     end,
2756
2757    [variables.firstsubpage]    = function() return counters.record("subpage" )["first"]    end,
2758    [variables.previoussubpage] = function() return counters.record("subpage" )["previous"] end,
2759    [variables.nextsubpage]     = function() return counters.record("subpage" )["next"]     end,
2760    [variables.lastsubpage]     = function() return counters.record("subpage" )["last"]     end,
2761
2762    [variables.forward]         = function() return counters.record("realpage")["forward"]  end,
2763    [variables.backward]        = function() return counters.record("realpage")["backward"] end,
2764}
2765
2766references.pages = pages
2767
2768-- maybe some day i will merge this in the backend code with a testmode (so each
2769-- runner then implements a branch)
2770
2771runners["inner"] = function(var,actions)
2772    local r = var.r
2773    if r then
2774        actions.realpage = r
2775    end
2776end
2777
2778runners["special"] = function(var,actions)
2779    local handler = specials[var.special]
2780    return handler and handler(var,actions)
2781end
2782
2783runners["special operation"]                = runners["special"]
2784runners["special operation with arguments"] = runners["special"]
2785
2786function specials.internal(var,actions)
2787    local v = internals[tonumber(var.operation)]
2788    local r = v and v.references
2789    if r then
2790        local p = r.realpage
2791        if p then
2792-- setmetatableindex(actions,r)
2793            actions.realpage = p
2794            actions.view     = r.view
2795        end
2796    end
2797end
2798
2799specials.i = specials.internal
2800
2801function specials.page(var,actions)
2802    local o = var.operation
2803    local p = pages[o]
2804    if type(p) == "function" then
2805        p = p()
2806    else
2807        p = tonumber(realpageofpage(tonumber(o)))
2808    end
2809    if p then
2810        var.r = p
2811        actions.realpage = actions.realpage or p -- first wins
2812    end
2813end
2814
2815function specials.realpage(var,actions)
2816    local p = tonumber(var.operation)
2817    if p then
2818        var.r = p
2819        actions.realpage = actions.realpage or p -- first wins
2820    end
2821end
2822
2823function specials.userpage(var,actions)
2824    local p = tonumber(realpageofpage(var.operation))
2825    if p then
2826        var.r = p
2827        actions.realpage = actions.realpage or p -- first wins
2828    end
2829end
2830
2831function specials.deltapage(var,actions)
2832    local p = tonumber(var.operation)
2833    if p then
2834        p = references.checkedrealpage(p + texgetcount("realpageno"))
2835        var.r = p
2836        actions.realpage = actions.realpage or p -- first wins
2837    end
2838end
2839
2840function specials.section(var,actions)
2841    local sectionname = var.arguments
2842    local destination = var.operation
2843    local internal    = structures.sections.internalreference(sectionname,destination)
2844    if internal then
2845        var.special   = "internal"
2846        var.operation = internal
2847        var.arguments = nil
2848        specials.internal(var,actions)
2849    end
2850end
2851
2852-- experimental:
2853
2854local p_splitter = lpeg.splitat(":")
2855local p_lower    = lpeg.patterns.utf8lower
2856
2857-- We can cache lowercased titles which saves a lot of time, but then
2858-- we can better have a global cache with weak keys.
2859
2860-- local lowercache = table.setmetatableindex(function(t,k)
2861--     local v = lpegmatch(p_lower,k)
2862--     t[k] = v
2863--     return v
2864-- end)
2865
2866local lowercache = false
2867
2868local function locate(list,askedkind,askedname,pattern)
2869    local kinds = lists.kinds
2870    local names = lists.names
2871    if askedkind and not kinds[askedkind] then
2872        return false
2873    end
2874    if askedname and not names[askedname] then
2875        return false
2876    end
2877    for i=1,#list do
2878        local entry    = list[i]
2879        local metadata = entry.metadata
2880        if metadata then
2881            local found = false
2882            if askedname then
2883                local name = metadata.name
2884                if name then
2885                    found = name == askedname
2886                end
2887            elseif askedkind then
2888                local kind = metadata.kind
2889                if kind then
2890                    found = kind == askedkind
2891                end
2892            end
2893            if found then
2894                local titledata = entry.titledata
2895                if titledata then
2896                    local title = titledata.title
2897                    if title then
2898                        if lowercache then
2899                            found = lpegmatch(pattern,lowercache[title])
2900                        else
2901                            found = lpegmatch(pattern,lpegmatch(p_lower,title))
2902                        end
2903                        if found then
2904                            return {
2905                                inner     = pattern,
2906                                kind      = "inner",
2907                                reference = pattern,
2908                                i         = entry,
2909                                p         = "",
2910                                r         = entry.references.realpage,
2911                            }
2912                        end
2913                    end
2914                end
2915            end
2916        end
2917    end
2918end
2919
2920function functions.match(var,actions)
2921    if not var.outer then
2922        local operation   = var.operation
2923        if operation and operation ~= "" then
2924            local operation   = lpegmatch(p_lower,operation)
2925            local list        = lists.collected
2926            local names       = false
2927            local kinds       = false
2928            local where, what = lpegmatch(p_splitter,operation)
2929            if where and what then
2930                local pattern = lpeg.finder(what)
2931                return
2932                    locate(list,false,where,pattern)
2933                 or locate(list,where,false,pattern)
2934                 or { error = "no match" }
2935            else
2936                local pattern = lpeg.finder(operation)
2937                -- todo: don't look at section and float in last pass
2938                return
2939                    locate(list,"section",false,pattern)
2940                 or locate(list,"float",false,pattern)
2941                 or locate(list,false,false,pattern)
2942                 or { error = "no match" }
2943            end
2944        end
2945    end
2946end
2947
2948-- needs a better split ^^^
2949
2950-- done differently now:
2951
2952function references.export(usedname) end
2953function references.import(usedname) end
2954function references.load  (usedname) end
2955
2956implement { name = "exportreferences", actions =references.export }
2957
2958-- better done here .... we don't insert/remove, just use a pointer
2959
2960local prefixstack = { "" }
2961local prefixlevel = 1
2962
2963local function pushreferenceprefix(prefix)
2964    prefixlevel = prefixlevel + 1
2965    prefixstack[prefixlevel] = prefix
2966    return prefix
2967end
2968
2969local function popreferenceprefix()
2970    prefixlevel = prefixlevel - 1
2971    if prefixlevel > 0 then
2972        return prefixstack[prefixlevel]
2973    else
2974        report_references("unable to pop referenceprefix")
2975        return ""
2976    end
2977end
2978
2979implement {
2980    name      = "pushreferenceprefix",
2981    actions   = { pushreferenceprefix, context }, -- we can use setmacro
2982    arguments = "string",
2983}
2984
2985implement {
2986    name      = "popreferenceprefix",
2987    actions   = { popreferenceprefix, context }, -- we can use setmacro
2988}
2989