Source Browser ?

1if not modules then modules = { } end modules ['strc-ref'] = {
2    version   = 1.001,
3    comment   = "companion to strc-ref.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- beware, this is a first step in the rewrite (just getting rid of
10-- the tuo file); later all access and parsing will also move to lua
11
12-- the useddata and pagedata names might change
13-- todo: pack exported data
14
15-- todo: autoload components when :::
16
17local format, find, gmatch, match, strip = string.format, string.find, string.gmatch, string.match, string.strip
18local floor = math.floor
19local rawget, tonumber, type, next = rawget, tonumber, type, next
20local lpegmatch = lpeg.match
21local insert, remove, copytable = table.insert, table.remove, table.copy
22local formatters = string.formatters
23local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match
24
25local allocate           = utilities.storage.allocate
26local mark               = utilities.storage.mark
27local setmetatableindex  = table.setmetatableindex
28
29local trace_referencing  = false  trackers.register("structures.referencing",             function(v) trace_referencing = v end)
30local trace_analyzing    = false  trackers.register("structures.referencing.analyzing",   function(v) trace_analyzing   = v end)
31local trace_identifying  = false  trackers.register("structures.referencing.identifying", function(v) trace_identifying = v end)
32local trace_importing    = false  trackers.register("structures.referencing.importing",   function(v) trace_importing   = v end)
33local trace_empty        = false  trackers.register("structures.referencing.empty",       function(v) trace_empty       = v end)
34
35local check_duplicates   = true
36
37directives.register("structures.referencing.checkduplicates", function(v) check_duplicates = v end)
38
39local report_references  = logs.reporter("references")
40local report_identifying = logs.reporter("references","identifying")
41local report_importing   = logs.reporter("references","importing")
42local report_empty       = logs.reporter("references","empty")
43local report             = report_references
44
45local variables          = interfaces.variables
46local v_page             = variables.page
47local v_auto             = variables.auto
48local v_yes              = variables.yes
49local v_name             = variables.name
50
51local context            = context
52local commands           = commands
53local implement          = interfaces.implement
54
55local ctx_latelua        = context.latelua
56
57local texgetcount        = tex.getcount
58local texsetcount        = tex.setcount
59local texconditionals    = tex.conditionals
60
61local productcomponent   = resolvers.jobs.productcomponent
62local justacomponent     = resolvers.jobs.justacomponent
63
64local settings_to_array  = utilities.parsers.settings_to_array
65local settings_to_table  = utilities.parsers.settings_to_array_obey_fences
66local process_settings   = utilities.parsers.process_stripped_settings
67local unsetvalue         = attributes.unsetvalue
68
69local structures         = structures
70local helpers            = structures.helpers
71local sections           = structures.sections
72local references         = structures.references
73local lists              = structures.lists
74local counters           = structures.counters
75
76local jobpositions       = job.positions
77local getpos             = jobpositions.getpos
78
79-- some might become local
80
81references.defined       = references.defined or allocate()
82
83local defined            = references.defined
84local derived            = allocate()
85local specials           = allocate()
86local functions          = allocate()
87local runners            = allocate()
88local internals          = allocate()
89local filters            = allocate()
90local executers          = allocate()
91local handlers           = allocate()
92local tobesaved          = allocate()
93local collected          = allocate()
94local tobereferred       = allocate()
95local referred           = allocate()
96local usedinternals      = allocate()
97local flaginternals      = allocate()
98local usedviews          = allocate()
99
100references.derived       = derived
101references.specials      = specials
102references.functions     = functions
103references.runners       = runners
104references.internals     = internals
105references.filters       = filters
106references.executers     = executers
107references.handlers      = handlers
108references.tobesaved     = tobesaved
109references.collected     = collected
110references.tobereferred  = tobereferred
111references.referred      = referred
112references.usedinternals = usedinternals
113references.flaginternals = flaginternals
114references.usedviews     = usedviews
115
116local splitreference     = references.splitreference
117local splitprefix        = references.splitcomponent -- replaces: references.splitprefix
118local prefixsplitter     = references.prefixsplitter
119local componentsplitter  = references.componentsplitter
120
121local currentreference   = nil
122
123local txtcatcodes        = catcodes.numbers.txtcatcodes -- or just use "txtcatcodes"
124
125local context                      = context
126
127local ctx_pushcatcodes             = context.pushcatcodes
128local ctx_popcatcodes              = context.popcatcodes
129local ctx_dofinishreference        = context.dofinishreference
130local ctx_dofromurldescription     = context.dofromurldescription
131local ctx_dofromurlliteral         = context.dofromurlliteral
132local ctx_dofromfiledescription    = context.dofromfiledescription
133local ctx_dofromfileliteral        = context.dofromfileliteral
134local ctx_expandreferenceoperation = context.expandreferenceoperation
135local ctx_expandreferencearguments = context.expandreferencearguments
136local ctx_convertnumber            = context.convertnumber
137local ctx_emptyreference           = context.emptyreference
138
139storage.register("structures/references/defined", references.defined, "structures.references.defined")
140
141local initializers = { }
142local finalizers   = { }
143local somefound    = false -- so we don't report missing when we have a fresh start
144
145function references.registerinitializer(func) -- we could use a token register instead
146    initializers[#initializers+1] = func
147end
148
149function references.registerfinalizer(func) -- we could use a token register instead
150    finalizers[#finalizers+1] = func
151end
152
153local function initializer() -- can we use a tobesaved as metatable for collected?
154    tobesaved = references.tobesaved
155    collected = references.collected
156    for i=1,#initializers do
157        initializers[i](tobesaved,collected)
158    end
159    for prefix, list in next, collected do
160        for tag, data in next, list do
161            local r = data.references
162            local i = r.internal
163            if i then
164                internals[i]     = data
165                usedinternals[i] = r.used
166            end
167        end
168    end
169    somefound = next(collected)
170end
171
172local function finalizer()
173    for i=1,#finalizers do
174        finalizers[i](tobesaved)
175    end
176    for prefix, list in next, tobesaved do
177        for tag, data in next, list do
178            local r = data.references
179            local i = r.internal
180            local f = flaginternals[i]
181            if f then
182                r.used = usedviews[i] or true
183            end
184        end
185    end
186end
187
188job.register('structures.references.collected', tobesaved, initializer, finalizer)
189
190local maxreferred = 1
191local nofreferred = 0
192
193local function initializer() -- can we use a tobesaved as metatable for collected?
194    tobereferred = references.tobereferred
195    referred     = references.referred
196    nofreferred = #referred
197end
198
199-- no longer done this way
200
201-- references.resolvers = references.resolvers or { }
202-- local resolvers = references.resolvers
203--
204-- function resolvers.section(var)
205--     local vi = lists.collected[var.i[2]]
206--     if vi then
207--         var.i = vi
208--         var.r = (vi.references and vi.references.realpage) or (vi.pagedata and vi.pagedata.realpage) or 1
209--     else
210--         var.i = nil
211--         var.r = 1
212--     end
213-- end
214--
215-- resolvers.float       = resolvers.section
216-- resolvers.description = resolvers.section
217-- resolvers.formula     = resolvers.section
218-- resolvers.note        = resolvers.section
219--
220-- function resolvers.reference(var)
221--     local vi = var.i[2]
222--     if vi then
223--         var.i = vi
224--         var.r = (vi.references and vi.references.realpage) or (vi.pagedata and vi.pagedata.realpage) or 1
225--     else
226--         var.i = nil
227--         var.r = 1
228--     end
229-- end
230
231-- We make the array sparse (maybe a finalizer should optionally return a table) because
232-- there can be quite some page links involved. We only store one action number per page
233-- which is normally good enough for what we want (e.g. see above/below) and we do
234-- a combination of a binary search and traverse backwards. A previous implementation
235-- always did a traverse and was pretty slow on a large number of links (given that this
236-- methods was used). It took me about a day to locate this as a bottleneck in processing
237-- a 2500 page interactive document with 60 links per page. In that case, traversing
238-- thousands of slots per link then brings processing to a grinding halt (especially when
239-- there are no slots at all, which is the case in a first run).
240
241local sparsetobereferred = { }
242
243local function finalizer()
244    local lastr, lasti
245    local n = 0
246    for i=1,maxreferred do
247        local r = tobereferred[i]
248        if not lastr then
249            lastr = r
250            lasti = i
251        elseif r ~= lastr then
252            n = n + 1
253            sparsetobereferred[n] = { lastr, lasti }
254            lastr = r
255            lasti = i
256        end
257    end
258    if lastr then
259        n = n + 1
260        sparsetobereferred[n] = { lastr, lasti }
261    end
262end
263
264job.register('structures.references.referred', sparsetobereferred, initializer, finalizer)
265
266local function referredpage(n)
267    local max = nofreferred
268    if max > 0 then
269        -- find match
270        local min = 1
271        while true do
272            local mid = floor((min+max)/2)
273            local r = referred[mid]
274            local m = r[2]
275            if n == m then
276                return r[1]
277            elseif n > m then
278                min = mid + 1
279            else
280                max = mid - 1
281            end
282            if min > max then
283                break
284            end
285        end
286        -- find first previous
287        for i=min,1,-1 do
288            local r = referred[i]
289            if r and r[2] < n then
290                return r[1]
291            end
292        end
293    end
294    -- fallback
295    return texgetcount("realpageno")
296end
297
298references.referredpage = referredpage
299
300function references.registerpage(n) -- called in the backend code
301    if not tobereferred[n] then
302        if n > maxreferred then
303            maxreferred = n
304        end
305        tobereferred[n] = texgetcount("realpageno")
306    end
307end
308
309-- todo: delay split till later as in destinations we split anyway
310
311local orders, lastorder = { }, 0
312
313local function setnextorder(kind,name)
314    lastorder = 0
315    if kind and name then
316        local ok = orders[kind]
317        if not ok then
318            ok = { }
319            orders[kind] = ok
320        end
321        lastorder = (ok[name] or 0) + 1
322        ok[name] = lastorder
323    end
324    texsetcount("global","locationorder",lastorder)
325end
326
327
328local function setnextinternal(kind,name)
329    setnextorder(kind,name) -- always incremented with internal
330    local n = texgetcount("locationcount") + 1
331    texsetcount("global","locationcount",n)
332    return n
333end
334
335local function currentorder(kind,name)
336    return orders[kind] and orders[kind][name] or lastorder
337end
338
339local function setcomponent(data)
340    -- we might consider doing this at the tex end, just like prefix
341    local component = productcomponent()
342    if component then
343        local references = data and data.references
344        if references then
345            references.component = component
346            if references.prefix == component then
347                references.prefix = nil
348            end
349        end
350        return component
351    end
352    -- but for the moment we do it here (experiment)
353end
354
355references.setnextorder    = setnextorder
356references.setnextinternal = setnextinternal
357references.currentorder    = currentorder
358references.setcomponent    = setcomponent
359
360implement {
361    name      = "setnextreferenceorder",
362    actions   = setnextorder,
363    arguments = "2 strings",
364}
365
366implement {
367    name      = "setnextinternalreference",
368    actions   = setnextinternal,
369    arguments = "2 strings",
370}
371
372implement {
373    name      = "currentreferenceorder",
374    actions = { currentorder, context },
375    arguments = "2 strings",
376}
377
378local reported = setmetatableindex("table")
379
380function references.set(data)
381    local references = data.references
382    local reference  = references.reference
383    if not reference or reference == "" then
384     -- report_references("invalid reference") -- harmless
385        return 0
386    end
387    local prefix = references.prefix or ""
388    local pd     = tobesaved[prefix] -- nicer is a metatable
389    if not pd then
390        pd = { }
391        tobesaved[prefix] = pd
392    end
393    local n = 0
394    local function action(ref)
395        if ref == "" then
396            -- skip
397        elseif check_duplicates and pd[ref] then
398            if not prefix then
399                prefix = ""
400            end
401            if not reported[prefix][ref] then
402                if prefix ~= "" then
403                    report_references("redundant reference %a in namespace %a",ref,prefix)
404                else
405                    report_references("redundant reference %a",ref)
406                end
407                reported[prefix][ref] = true
408            end
409        else
410            n = n + 1
411            pd[ref] = data
412            local r = data.references
413            ctx_dofinishreference(prefix or "",ref or "",r and r.internal or 0)
414         -- ctx_latelua(function() structures.references.enhance(prefix or ref,ref or "") end)
415        end
416    end
417    process_settings(reference,action)
418    return n > 0
419end
420
421-- function references.enhance(prefix,tag)
422--     local l = tobesaved[prefix][tag]
423--     if l then
424--         l.references.realpage = texgetcount("realpageno")
425--     end
426-- end
427
428local function synchronizepage(reference) -- non public helper
429    reference.realpage = texgetcount("realpageno")
430    if jobpositions.used() then
431        reference.x, reference.y = getpos()
432    end
433end
434
435references.synchronizepage = synchronizepage
436
437local function enhancereference(specification)
438    local prefix = specification.prefix
439    if prefix then
440        local entry = tobesaved[prefix]
441        if entry then
442            entry = entry[specification.tag]
443            if entry then
444                synchronizepage(entry.references)
445            else
446                -- normally a bug
447            end
448        else
449            -- normally a bug
450        end
451    else
452        -- normally a bug
453    end
454end
455
456references.enhance = enhancereference
457
458-- implement {
459--     name      = "enhancereference",
460--     arguments = "2 strings",
461--     actions   = function(prefix,tag)
462--        enhancereference { prefix = prefix, tag = tag }
463--     end,
464-- }
465
466implement {
467    name      = "deferredenhancereference",
468    arguments = "2 strings",
469    protected = true,
470    actions   = function(prefix,tag)
471        ctx_latelua { action = enhancereference, prefix = prefix, tag = tag }
472    end,
473}
474
475-- -- -- related to strc-ini.lua -- -- --
476
477-- no metatable here .. better be sparse
478
479local function register_from_lists(collected,derived,pages,sections)
480    local derived_g = derived[""] -- global
481    local derived_p = nil
482    local derived_c = nil
483    local prefix    = nil
484    local component = nil
485    local entry     = nil
486    if not derived_g then
487        derived_g = { }
488        derived[""] = derived_g
489    end
490    local function action(s)
491        if trace_referencing then
492            report_references("list entry %a provides %a reference %a on realpage %a",i,kind,s,realpage)
493        end
494        if derived_p and not derived_p[s] then
495            derived_p[s] = entry
496        end
497        if derived_c and not derived_c[s] then
498            derived_c[s] = entry
499        end
500        if not derived_g[s] then
501            derived_g[s] = entry -- first wins
502        end
503    end
504    for i=1,#collected do
505        entry = collected[i]
506        local metadata = entry.metadata
507        if metadata then
508            local kind = metadata.kind -- why this check
509            if kind then
510                local references = entry.references
511                if references then
512                    local reference = references.reference
513                    if reference and reference ~= "" then
514                        local realpage = references.realpage
515                        if realpage then
516                            prefix    = references.prefix
517                            component = references.component
518                            if prefix and prefix ~= "" then
519                                derived_p = derived[prefix]
520                                if not derived_p then
521                                    derived_p = { }
522                                    derived[prefix] = derived_p
523                                end
524                            end
525                            if component and component ~= "" and component ~= prefix then
526                                derived_c = derived[component]
527                                if not derived_c then
528                                    derived_c = { }
529                                    derived[component] = derived_c
530                                end
531                            end
532                            process_settings(reference,action)
533                        end
534                    end
535                end
536            end
537        end
538    end
539end
540
541references.registerinitializer(function() register_from_lists(lists.collected,derived) end)
542
543-- tracing
544
545local function collectbypage(tracedpages)
546    -- lists
547    do
548        local collected = structures.lists.collected
549        local data      = nil
550        local function action(reference)
551            local prefix    = data.prefix
552            local component = data.component
553            local realpage  = data.realpage
554            if realpage then
555                local pagelist  = rawget(tracedpages,realpage)
556                local internal  = data.internal or 0
557                local prefix    = (prefix ~= "" and prefix) or (component ~= "" and component) or ""
558                local pagedata  = { prefix, reference, internal }
559                if pagelist then
560                    pagelist[#pagelist+1] = pagedata
561                else
562                    tracedpages[realpage] = { pagedata }
563                end
564                if internal > 0 then
565                    data.usedprefix = prefix
566                end
567            end
568        end
569        for i=1,#collected do
570            local entry = collected[i]
571            local metadata = entry.metadata
572            if metadata and metadata.kind then
573                data = entry.references
574                if data then
575                    local reference = data.reference
576                    if reference and reference ~= "" then
577                        process_settings(reference,action)
578                    end
579                end
580            end
581        end
582    end
583    -- references
584    do
585        for prefix, list in next, collected do
586            for reference, entry in next, list do
587                local data = entry.references
588                if data then
589                    local realpage = data.realpage
590                    local internal = data.internal or 0
591                    local pagelist = rawget(tracedpages,realpage)
592                    local pagedata = { prefix, reference, internal }
593                    if pagelist then
594                        pagelist[#pagelist+1] = pagedata
595                    else
596                        tracedpages[realpage] = { pagedata }
597                    end
598                    if internal > 0 then
599                        data.usedprefix = prefix
600                    end
601                end
602            end
603        end
604    end
605end
606
607references.tracedpages = table.setmetatableindex(allocate(),function(t,k)
608    if collectbypage then
609        collectbypage(t)
610        collectbypage = nil
611    end
612    return rawget(t,k)
613end)
614
615-- urls
616
617local urls       = references.urls or { }
618references.urls  = urls
619local urldata    = urls.data or { }
620urls.data        = urldata
621
622local p_untexurl = Cs ( (
623    P("\\")/"" * (P("%")/"%%" + P(1))
624  + P(" ")/"%%20"
625  + P(1)
626)^1 )
627
628function urls.untex(url)
629    return lpegmatch(p_untexurl,url) or url
630end
631
632function urls.define(name,url,file,description)
633    if name and name ~= "" then
634     -- url = lpegmatch(replacer,url)
635        urldata[name] = { url or "", file or "", description or url or file or ""}
636    end
637end
638
639function urls.get(name)
640    local u = urldata[name]
641    if u then
642        local url, file = u[1], u[2]
643        if file and file ~= "" then
644            return formatters["%s/%s"](url,file)
645        else
646            return url
647        end
648    end
649end
650
651function urls.found(name)
652    return urldata[name]
653end
654
655local function geturl(name)
656    local url = urls.get(name)
657    if url and url ~= "" then
658        ctx_pushcatcodes(txtcatcodes)
659        context(url)
660        ctx_popcatcodes()
661    end
662end
663
664implement {
665    name      = "doifelseurldefined",
666    actions   = { urls.found, commands.doifelse },
667    arguments = "string"
668}
669
670implement {
671    name      = "useurl",
672    actions   = urls.define,
673    arguments = "4 strings",
674}
675
676implement {
677    name      = "geturl",
678    actions   = geturl,
679    arguments = "string",
680}
681
682-- files
683
684local files      = references.files or { }
685references.files = files
686local filedata   = files.data or { }
687files.data       = filedata
688
689function files.define(name,file,description)
690    if name and name ~= "" then
691        filedata[name] = { file or "", description or file or "" }
692    end
693end
694
695function files.get(name,method,space) -- method: none, before, after, both, space: yes/no
696    local f = filedata[name]
697    if f then
698        context(f[1])
699    end
700end
701
702function files.found(name)
703    return filedata[name]
704end
705
706local function getfile(name)
707    local fil = files.get(name)
708    if fil and fil ~= "" then
709        ctx_pushcatcodes(txtcatcodes)
710        context(fil)
711        ctx_popcatcodes()
712    end
713end
714
715implement {
716    name      = "doifelsefiledefined",
717    actions   = { files.found, commands.doifelse },
718    arguments = "string"
719}
720
721implement {
722    name      = "usefile",
723    actions   = files.define,
724    arguments = "3 strings"
725}
726
727implement {
728    name      = "getfile",
729    actions   = getfile,
730    arguments = "string"
731}
732
733-- helpers
734
735function references.checkedfile(whatever) -- return whatever if not resolved
736    if whatever then
737        local w = filedata[whatever]
738        if w then
739            return w[1]
740        else
741            return whatever
742        end
743    end
744end
745
746function references.checkedurl(whatever) -- return whatever if not resolved
747    if whatever then
748        local w = urldata[whatever]
749        if w then
750            local u, f = w[1], w[2]
751            if f and f ~= "" then
752                return u .. "/" .. f
753            else
754                return u
755            end
756        else
757            return whatever
758        end
759    end
760end
761
762function references.checkedfileorurl(whatever,default) -- return nil, nil if not resolved
763    if whatever then
764        local w = filedata[whatever]
765        if w then
766            return w[1], nil
767        else
768            local w = urldata[whatever]
769            if w then
770                local u, f = w[1], w[2]
771                if f and f ~= "" then
772                    return nil, u .. "/" .. f
773                else
774                    return nil, u
775                end
776            end
777        end
778    end
779    return default
780end
781
782-- programs
783
784local programs      = references.programs or { }
785references.programs = programs
786local programdata   = programs.data or { }
787programs.data       = programdata
788
789function programs.define(name,file,description)
790    if name and name ~= "" then
791        programdata[name] = { file or "", description or file or ""}
792    end
793end
794
795function programs.get(name)
796    local f = programdata[name]
797    return f and f[1]
798end
799
800function references.checkedprogram(whatever) -- return whatever if not resolved
801    if whatever then
802        local w = programdata[whatever]
803        if w then
804            return w[1]
805        else
806            return whatever
807        end
808    end
809end
810
811implement {
812    name      = "defineprogram",
813    actions   = programs.define,
814    arguments = "3 strings",
815}
816
817local function getprogram(name)
818    local p = programdata[name]
819    if p then
820        context(p[1])
821    end
822end
823
824implement {
825    name      = "getprogram",
826    actions   = getprogram,
827    arguments = "string"
828}
829
830-- shared by urls and files
831
832function references.from(name)
833    local u = urldata[name]
834    if u then
835        local url, file, description = u[1], u[2], u[3]
836        if description ~= "" then
837            return description
838            -- ok
839        elseif file and file ~= "" then
840            return url .. "/" .. file
841        else
842            return url
843        end
844    else
845        local f = filedata[name]
846        if f then
847            local file, description = f[1], f[2]
848            if description ~= "" then
849                return description
850            else
851                return file
852            end
853        end
854    end
855end
856
857local function from(name)
858    local u = urldata[name]
859    if u then
860        local url, file, description = u[1], u[2], u[3]
861        if description ~= "" then
862            ctx_dofromurldescription(description)
863            -- ok
864        elseif file and file ~= "" then
865            ctx_dofromurlliteral(url .. "/" .. file)
866        else
867            ctx_dofromurlliteral(url)
868        end
869    else
870        local f = filedata[name]
871        if f then
872            local file, description = f[1], f[2]
873            if description ~= "" then
874                ctx_dofromfiledescription(description)
875            else
876                ctx_dofromfileliteral(file)
877            end
878        end
879    end
880end
881
882implement {
883    name      = "from",
884    actions   = from,
885    arguments = "string"
886}
887
888function references.define(prefix,reference,list)
889    local d = defined[prefix] if not d then d = { } defined[prefix] = d end
890    d[reference] = list
891end
892
893function references.reset(prefix,reference)
894    local d = defined[prefix]
895    if d then
896        d[reference] = nil
897    end
898end
899
900implement {
901    name      = "definereference",
902    actions   = references.define,
903    arguments = "3 strings",
904}
905
906implement {
907    name      = "resetreference",
908    actions   = references.reset,
909    arguments = "2 strings",
910}
911
912setmetatableindex(defined,"table")
913
914local function resolve(prefix,reference,args,set) -- we start with prefix,reference
915    if reference and reference ~= "" then
916        if not set then
917            set = { prefix = prefix, reference = reference }
918        else
919            if not set.reference then set.reference = reference end
920            if not set.prefix    then set.prefix    = prefix    end
921        end
922     -- local r = settings_to_array(reference)
923        local r = settings_to_table(reference) -- maybe option to honor () []
924        for i=1,#r do
925            local ri = r[i]
926            local d = defined[prefix][ri] or defined[""][ri]
927            if d then
928                resolve(prefix,d,nil,set)
929            else
930                local var = splitreference(ri)
931                if var then
932                    var.reference = ri
933                    local vo, vi = var.outer, var.inner
934                    -- we catch this here .. it's a way to pass references with commas
935                    if vi == "name" then
936                        local arguments = var.arguments
937                        if arguments then
938                            vi            = arguments
939                            var.inner     = arguments
940                            var.reference = arguments
941                            var.arguments = nil
942                        end
943                    elseif var.special == "name" then
944                        local operation = var.operation
945                        if operation then
946                            vi            = operation
947                            var.inner     = operation
948                            var.reference = operation
949                            var.operation = nil
950                            var.special   = nil
951                        end
952                    end
953                    -- end of catch
954                    if not vo and vi then
955                        -- to be checked
956                        d = defined[prefix][vi] or defined[""][vi]
957                        --
958                        if d then
959                            resolve(prefix,d,var.arguments,set) -- args can be nil
960                        else
961                            if args then var.arguments = args end
962                            set[#set+1] = var
963                        end
964                    else
965                        if args then var.arguments = args end
966                        set[#set+1] = var
967                    end
968                    if var.has_tex then
969                        set.has_tex = true
970                    end
971                else
972                --  report_references("funny pattern %a",ri)
973                end
974            end
975        end
976        return set
977    else
978        return { }
979    end
980end
981
982-- prefix == "" is valid prefix which saves multistep lookup
983
984references.currentset = nil
985
986local function setreferenceoperation(k,v)
987    references.currentset[k].operation = v
988end
989
990local function setreferencearguments(k,v)
991    references.currentset[k].arguments = v
992end
993
994function references.expandcurrent() -- todo: two booleans: o_has_tex& a_has_tex
995    local currentset = references.currentset
996    if currentset and currentset.has_tex then
997        for i=1,#currentset do
998            local ci = currentset[i]
999            local operation = ci.operation
1000            if operation and find(operation,"\\",1,true) then -- if o_has_tex then
1001                ctx_expandreferenceoperation(i,operation)
1002            end
1003            local arguments = ci.arguments
1004            if arguments and find(arguments,"\\",1,true) then -- if a_has_tex then
1005                ctx_expandreferencearguments(i,arguments)
1006            end
1007        end
1008    end
1009end
1010
1011implement {
1012    name      = "expandcurrentreference",
1013    actions   = references.expandcurrent
1014}
1015
1016implement {
1017    name      = "setreferenceoperation",
1018    actions   = setreferenceoperation,
1019    arguments = { "integer", "string" }
1020}
1021
1022implement {
1023    name      = "setreferencearguments",
1024    actions   = setreferencearguments,
1025    arguments = { "integer", "string" }
1026}
1027
1028local externals = { }
1029
1030-- we have prefixes but also components:
1031--
1032-- :    prefix
1033-- ::   always external
1034-- :::  internal (for products) or external (for components)
1035
1036local function loadexternalreferences(name,utilitydata)
1037    local struc = utilitydata.structures
1038    if struc then
1039        local external = struc.references.collected -- direct references
1040        local lists    = struc.lists.collected      -- indirect references (derived)
1041        local pages    = struc.pages.collected      -- pagenumber data
1042        -- a bit weird one, as we don't have the externals in the collected
1043        for prefix, set in next, external do
1044            if prefix == "" then
1045                prefix = name -- this can clash!
1046            end
1047            for reference, data in next, set do
1048                if trace_importing then
1049                    report_importing("registering %a reference, kind %a, name %a, prefix %a, reference %a",
1050                        "external","regular",name,prefix,reference)
1051                end
1052                local section  = reference.section
1053                local realpage = reference.realpage
1054                if section then
1055                    reference.sectiondata = lists[section]
1056                end
1057                if realpage then
1058                    reference.pagedata = pages[realpage]
1059                end
1060            end
1061        end
1062        for i=1,#lists do
1063            local entry      = lists[i]
1064            local metadata   = entry.metadata
1065            local references = entry.references
1066            if metadata and references then
1067                local reference = references.reference
1068                if reference and reference ~= "" then
1069                    local kind     = metadata.kind
1070                    local realpage = references.realpage
1071                    if kind and realpage then
1072                        references.pagedata = pages[realpage]
1073                        local prefix = references.prefix or ""
1074                        if prefix == "" then
1075                            prefix = name -- this can clash!
1076                        end
1077                        local target = external[prefix]
1078                        if not target then
1079                            target = { }
1080                            external[prefix] = target
1081                        end
1082                     -- for s in gmatch(reference,"%s*([^,]+)") do
1083                     --     if trace_importing then
1084                     --         report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1085                     --             "external",kind,name,prefix,s)
1086                     --     end
1087                     --     target[s] = target[s] or entry
1088                     -- end
1089                        local function action(s)
1090                            if trace_importing then
1091                                report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1092                                    "external",kind,name,prefix,s)
1093                            end
1094                            target[s] = target[s] or entry
1095                        end
1096                        process_settings(reference,action)
1097                    end
1098                end
1099            end
1100        end
1101        externals[name] = external
1102        return external
1103    end
1104end
1105
1106local externalfiles = { }
1107
1108setmetatableindex(externalfiles, function(t,k)
1109    local v = filedata[k]
1110    if not v then
1111        v = { k, k }
1112    end
1113    externalfiles[k] = v
1114    return v
1115end)
1116
1117setmetatableindex(externals, function(t,k) -- either or not automatically
1118    local filename = externalfiles[k][1] -- filename
1119    local fullname = file.replacesuffix(filename,"tuc")
1120    if lfs.isfile(fullname) then -- todo: use other locator
1121        local utilitydata = job.loadother(fullname)
1122        if utilitydata then
1123            local external = loadexternalreferences(k,utilitydata)
1124            t[k] = external or false
1125            return external
1126        end
1127    end
1128    t[k] = false
1129    return false
1130end)
1131
1132local productdata = allocate {
1133    productreferences   = { },
1134    componentreferences = { },
1135    components          = { },
1136}
1137
1138references.productdata = productdata
1139
1140local function loadproductreferences(productname,componentname,utilitydata)
1141    local struc = utilitydata.structures
1142    if struc then
1143        local productreferences = struc.references.collected -- direct references
1144        local lists = struc.lists.collected      -- indirect references (derived)
1145        local pages = struc.pages.collected      -- pagenumber data
1146        -- we use indirect tables to save room but as they are eventually
1147        -- just references we resolve them to data here (the mechanisms
1148        -- that use this data check for indirectness)
1149        for prefix, set in next, productreferences do
1150            for reference, data in next, set do
1151                if trace_importing then
1152                    report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1153                        "product","regular",productname,prefix,reference)
1154                end
1155                local section  = reference.section
1156                local realpage = reference.realpage
1157                if section then
1158                    reference.sectiondata = lists[section]
1159                end
1160                if realpage then
1161                    reference.pagedata = pages[realpage]
1162                end
1163            end
1164        end
1165        --
1166        local componentreferences = { }
1167        for i=1,#lists do
1168            local entry      = lists[i]
1169            local metadata   = entry.metadata
1170            local references = entry.references
1171            if metadata and references then
1172                local reference = references.reference
1173                if reference and reference ~= "" then
1174                    local kind     = metadata.kind
1175                    local realpage = references.realpage
1176                    if kind and realpage then
1177                        references.pagedata = pages[realpage]
1178                        local prefix    = references.prefix or ""
1179                        local component = references.component
1180                        local ctarget, ptarget
1181                        if not component or component == componentname then
1182                            -- skip
1183                        else
1184                            -- one level up
1185                            local external = componentreferences[component]
1186                            if not external then
1187                                external = { }
1188                                componentreferences[component] = external
1189                            end
1190                            if component == prefix then
1191                                prefix = ""
1192                            end
1193                            ctarget = external[prefix]
1194                            if not ctarget then
1195                                ctarget = { }
1196                                external[prefix] = ctarget
1197                            end
1198                        end
1199                        ptarget = productreferences[prefix]
1200                        if not ptarget then
1201                            ptarget = { }
1202                            productreferences[prefix] = ptarget
1203                        end
1204                        local function action(s)
1205                            if ptarget then
1206                                if trace_importing then
1207                                    report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1208                                        "product",kind,productname,prefix,s)
1209                                end
1210                                ptarget[s] = ptarget[s] or entry
1211                            end
1212                            if ctarget then
1213                                if trace_importing then
1214                                    report_importing("registering %s reference, kind %a, name %a, prefix %a, referenc %a",
1215                                        "component",kind,productname,prefix,s)
1216                                end
1217                                ctarget[s] = ctarget[s] or entry
1218                            end
1219                        end
1220                        process_settings(reference,action)
1221                    end
1222                end
1223            end
1224        end
1225        productdata.productreferences   = productreferences -- not yet used
1226        productdata.componentreferences = componentreferences
1227    end
1228end
1229
1230local function loadproductvariables(product,component,utilitydata)
1231    local struc = utilitydata.structures
1232    if struc then
1233        local lists = struc.lists and struc.lists.collected
1234        if lists then
1235            local pages = struc.pages and struc.pages.collected
1236            for i=1,#lists do
1237                local li = lists[i]
1238                if li.metadata.kind == "section" and li.references.component == component then
1239                    local firstsection = li
1240                    if firstsection.numberdata then
1241                        local numbers = firstsection.numberdata.numbers
1242                        if numbers then
1243                            if trace_importing then
1244                                report_importing("initializing section number to %:t",numbers)
1245                            end
1246                            productdata.firstsection = firstsection
1247                            structures.documents.preset(numbers)
1248                        end
1249                    end
1250                    if pages and firstsection.references then
1251                        local firstpage = pages[firstsection.references.realpage]
1252                        local number = firstpage and firstpage.number
1253                        if number then
1254                            if trace_importing then
1255                                report_importing("initializing page number to %a",number)
1256                            end
1257                            productdata.firstpage = firstpage
1258                            counters.set("userpage",1,number)
1259                        end
1260                    end
1261                    break
1262                end
1263            end
1264        end
1265    end
1266end
1267
1268local function componentlist(tree,target)
1269    local branches = tree and tree.branches
1270    if branches then
1271        for i=1,#branches do
1272            local branch = branches[i]
1273            local type = branch.type
1274            if type == "component" then
1275                if target then
1276                    target[#target+1] = branch.name
1277                else
1278                    target = { branch.name }
1279                end
1280            elseif type == "product" or type == "component" then
1281                target = componentlist(branch,target)
1282            end
1283        end
1284    end
1285    return target
1286end
1287
1288local function loadproductcomponents(product,component,utilitydata)
1289    local job = utilitydata.job
1290    productdata.components = componentlist(job and job.structure and job.structure.collected) or { }
1291end
1292
1293references.registerinitializer(function(tobesaved,collected)
1294    -- not that much related to tobesaved or collected
1295    productdata.components = componentlist(job.structure.collected) or { }
1296end)
1297
1298function references.loadpresets(product,component) -- we can consider a special components hash
1299    if product and component and product~= "" and component ~= "" and not productdata.product then -- maybe: productdata.filename ~= filename
1300        productdata.product = product
1301        productdata.component = component
1302        local fullname = file.replacesuffix(product,"tuc")
1303        if lfs.isfile(fullname) then -- todo: use other locator
1304            local utilitydata = job.loadother(fullname)
1305            if utilitydata then
1306                if trace_importing then
1307                    report_importing("loading references for component %a of product %a from %a",component,product,fullname)
1308                end
1309                loadproductvariables (product,component,utilitydata)
1310                loadproductreferences(product,component,utilitydata)
1311                loadproductcomponents(product,component,utilitydata)
1312            end
1313        end
1314    end
1315end
1316
1317references.productdata = productdata
1318
1319local useproduct = commands.useproduct
1320
1321if useproduct then
1322
1323    local function newuseproduct(product)
1324        useproduct(product)
1325        if texconditionals.autocrossfilereferences then
1326            local component = justacomponent()
1327            if component then
1328                if trace_referencing or trace_importing then
1329                    report_references("loading presets for component %a of product %a",component,product)
1330                end
1331                references.loadpresets(product,component)
1332            end
1333        end
1334    end
1335
1336    implement {
1337        name      = "useproduct",
1338        actions   = newuseproduct,
1339        arguments = "string",
1340        overload  = true,
1341    }
1342
1343end
1344
1345-- productdata.firstsection.numberdata.numbers
1346-- productdata.firstpage.number
1347
1348local function report_identify_special(set,var,i,type)
1349    local reference = set.reference
1350    local prefix    = set.prefix or ""
1351    local special   = var.special
1352    local error     = var.error
1353    local kind      = var.kind
1354    if error then
1355        report_identifying("type %a, reference %a, index %a, prefix %a, special %a, error %a",type,reference,i,prefix,special,error)
1356    else
1357        report_identifying("type %a, reference %a, index %a, prefix %a, special %a, kind %a",type,reference,i,prefix,special,kind)
1358    end
1359end
1360
1361local function report_identify_arguments(set,var,i,type)
1362    local reference = set.reference
1363    local prefix    = set.prefix or ""
1364    local arguments = var.arguments
1365    local error     = var.error
1366    local kind      = var.kind
1367    if error then
1368        report_identifying("type %a, reference %a, index %a, prefix %a, arguments %a, error %a",type,reference,i,prefix,arguments,error)
1369    else
1370        report_identifying("type %a, reference %a, index %a, prefix %a, arguments %a, kind %a",type,reference,i,prefix,arguments,kind)
1371    end
1372end
1373
1374local function report_identify_outer(set,var,i,type)
1375    local reference = set.reference
1376    local prefix    = set.prefix or ""
1377    local outer     = var.outer
1378    local error     = var.error
1379    local kind      = var.kind
1380    if outer then
1381        if error then
1382            report_identifying("type %a, reference %a, index %a, prefix %a, outer %a, error %a",type,reference,i,prefix,outer,error)
1383        else
1384            report_identifying("type %a, reference %a, index %a, prefix %a, outer %a, kind %a",type,reference,i,prefix,outer,kind)
1385        end
1386    else
1387        if error then
1388            report_identifying("type %a, reference %a, index %a, prefix %a, error %a",type,reference,i,prefix,error)
1389        else
1390            report_identifying("type %a, reference %a, index %a, prefix %a, kind %a",type,reference,i,prefix,kind)
1391        end
1392    end
1393end
1394
1395local function identify_special(set,var,i)
1396    local special = var.special
1397    local s = specials[special]
1398    if s then
1399        local outer     = var.outer
1400        local operation = var.operation
1401        local arguments = var.arguments
1402        if outer then
1403            if operation then
1404                -- special(outer::operation)
1405                var.kind = "special outer with operation"
1406            else
1407                -- special()
1408                var.kind = "special outer"
1409            end
1410            var.f = outer
1411        elseif operation then
1412            if arguments then
1413                -- special(operation{argument,argument})
1414                var.kind = "special operation with arguments"
1415            else
1416                -- special(operation)
1417                var.kind = "special operation"
1418            end
1419        else
1420            -- special()
1421            var.kind = "special"
1422        end
1423        if trace_identifying then
1424            report_identify_special(set,var,i,"1a")
1425        end
1426    else
1427        var.error = "unknown special"
1428    end
1429    return var
1430end
1431
1432local function identify_arguments(set,var,i)
1433    local s = specials[var.inner]
1434    if s then
1435        -- inner{argument}
1436        var.kind = "special operation with arguments"
1437    else
1438        var.error = "unknown inner or special"
1439    end
1440    if trace_identifying then
1441        report_identify_arguments(set,var,i,"3a")
1442    end
1443    return var
1444end
1445
1446-- needs checking: if we don't do too much (redundant) checking now
1447-- inner ... we could move the prefix logic into the parser so that we have 'm for each entry
1448-- foo:bar -> foo == prefix (first we try the global one)
1449-- -:bar   -> ignore prefix
1450
1451local function finish_inner(var,p,i)
1452    var.kind = "inner"
1453    var.i = i
1454    var.p = p
1455    var.r = (i.references and i.references.realpage) or (i.pagedata and i.pagedata.realpage) or 1
1456    return var
1457end
1458
1459local function identify_inner(set,var,prefix,collected,derived)
1460    local inner = var.inner
1461    -- the next test is a safeguard when references are auto loaded from outer
1462    if not inner or inner == "" then
1463        return false
1464    end
1465    local splitprefix, splitinner = lpegmatch(prefixsplitter,inner)
1466    if splitprefix and splitinner then
1467        -- we check for a prefix:reference instance in the regular set of collected
1468        -- references; a special case is -: which forces a lookup in the global list
1469        if splitprefix == "-" then
1470            local i = collected[""]
1471            if i then
1472                i = i[splitinner]
1473                if i then
1474                    return finish_inner(var,"",i)
1475                end
1476            end
1477        end
1478        local i = collected[splitprefix]
1479        if i then
1480            i = i[splitinner]
1481            if i then
1482                return finish_inner(var,splitprefix,i)
1483            end
1484        end
1485        if derived then
1486            -- next we look for a reference in the regular set of collected references
1487            -- using the prefix that is active at this moment (so we overload the given
1488            -- these are taken from other data structures (like lists)
1489            if splitprefix == "-" then
1490                local i = derived[""]
1491                if i then
1492                    i = i[splitinner]
1493                    if i then
1494                        return finish_inner(var,"",i)
1495                    end
1496                end
1497            end
1498            local i = derived[splitprefix]
1499            if i then
1500                i = i[splitinner]
1501                if i then
1502                    return finish_inner(var,splitprefix,i)
1503                end
1504            end
1505        end
1506    end
1507    -- we now ignore the split prefix and treat the whole inner as a potential
1508    -- reference into the global list
1509    local i = collected[prefix]
1510    if i then
1511        i = i[inner]
1512        if i then
1513            return finish_inner(var,prefix,i)
1514        end
1515    end
1516    if not i and derived then
1517        -- and if not found we look in the derived references
1518        local i = derived[prefix]
1519        if i then
1520            i = i[inner]
1521            if i then
1522                return finish_inner(var,prefix,i)
1523            end
1524        end
1525    end
1526    return false
1527end
1528
1529local function unprefixed_inner(set,var,prefix,collected,derived,tobesaved)
1530    local inner = var.inner
1531    local s = specials[inner]
1532    if s then
1533        var.kind = "special"
1534    else
1535        local i = (collected and collected[""] and collected[""][inner]) or
1536                  (derived   and derived  [""] and derived  [""][inner]) or
1537                  (tobesaved and tobesaved[""] and tobesaved[""][inner])
1538        if i then
1539            var.kind = "inner"
1540            var.p    = ""
1541            var.i    = i
1542            var.r    = (i.references and i.references.realpage) or (i.pagedata and i.pagedata.realpage) or 1
1543        else
1544            var.error = "unknown inner or special"
1545        end
1546    end
1547    return var
1548end
1549
1550local function identify_outer(set,var,i)
1551    local outer    = var.outer
1552    local inner    = var.inner
1553    local external = externals[outer]
1554    if external then
1555        local v = identify_inner(set,var,"",external)
1556        if v then
1557            v.kind = "outer with inner"
1558            set.external = true
1559            if trace_identifying then
1560                report_identify_outer(set,v,i,"2a")
1561            end
1562            return v
1563        end
1564-- weird too (we really need to check how this table is build
1565        local v = identify_inner(set,var,var.outer,external)
1566        if v then
1567            v.kind = "outer with inner"
1568            set.external = true
1569            if trace_identifying then
1570                report_identify_outer(set,v,i,"2c")
1571            end
1572            return v
1573        end
1574--
1575        -- somewhat rubish: we use outer as first step in the externals table so it makes no
1576        -- sense to have it as prefix so the next could be an option
1577        local external = external[""]
1578        if external then
1579            local v = identify_inner(set,var,var.outer,external)
1580            if v then
1581                v.kind = "outer with inner"
1582                set.external = true
1583                if trace_identifying then
1584                    report_identify_outer(set,v,i,"2b")
1585                end
1586                return v
1587            end
1588        end
1589    end
1590    local external = productdata.componentreferences[outer]
1591    if external then
1592        local v = identify_inner(set,var,"",external)
1593        if v then
1594            v.kind = "outer with inner"
1595            set.external = true
1596            if trace_identifying then
1597                report_identify_outer(set,v,i,"2c")
1598            end
1599            return v
1600        end
1601    end
1602    local external = productdata.productreferences[outer]
1603    if external then
1604        local vi = external[inner]
1605        if vi then
1606            var.kind = "outer with inner"
1607            var.i = vi
1608            set.external = true
1609            if trace_identifying then
1610                report_identify_outer(set,var,i,"2d")
1611            end
1612            return var
1613        end
1614    end
1615    -- the rest
1616    local special   = var.special
1617    local arguments = var.arguments
1618    local operation = var.operation
1619    if inner then
1620        -- tricky: in this case we can only use views when we're sure that all inners
1621        -- are flushed in the outer document so that should become an option
1622        if arguments then
1623            -- outer::inner{argument}
1624            var.kind = "outer with inner with arguments"
1625        else
1626            -- outer::inner
1627            var.kind = "outer with inner"
1628        end
1629        var.i = inner
1630        var.f = outer
1631        if type(inner) == "table" then
1632            -- can this really happen?
1633            var.r = (inner.references and inner.references.realpage) or (inner.pagedata and inner.pagedata.realpage) or 1
1634        else
1635            var.r = 1
1636        end
1637        if trace_identifying then
1638            report_identify_outer(set,var,i,"2e")
1639        end
1640    elseif special then
1641        local s = specials[special]
1642        if s then
1643            if operation then
1644                if arguments then
1645                    -- outer::special(operation{argument,argument})
1646                    var.kind = "outer with special and operation and arguments"
1647                else
1648                    -- outer::special(operation)
1649                    var.kind = "outer with special and operation"
1650                end
1651            else
1652                -- outer::special()
1653                var.kind = "outer with special"
1654            end
1655            var.f = outer
1656        else
1657            var.error = "unknown outer with special"
1658        end
1659        if trace_identifying then
1660            report_identify_outer(set,var,i,"2f")
1661        end
1662    else
1663        -- outer::
1664        var.kind = "outer"
1665        var.f = outer
1666        if trace_identifying then
1667            report_identify_outer(set,var,i,"2g")
1668        end
1669    end
1670    return var
1671end
1672
1673-- todo: avoid copy
1674
1675local function identify_inner_or_outer(set,var,i)
1676    -- here we fall back on product data
1677    local inner = var.inner
1678    if inner and inner ~= "" then
1679
1680        -- first we look up in collected and derived using the current prefix
1681
1682        local prefix = set.prefix
1683
1684        local v = identify_inner(set,var,set.prefix,collected,derived)
1685        if v then
1686            if trace_identifying then
1687                report_identify_outer(set,v,i,"4a")
1688            end
1689            return v
1690        end
1691
1692        -- nest we look at each component (but we can omit the already consulted one
1693
1694        local jobstructure = job.structure
1695        local components   = jobstructure and jobstructure.components
1696        if components then
1697            for c=1,#components do
1698                local component = components[c]
1699                if component ~= prefix then
1700                    local v = identify_inner(set,var,component,collected,derived)
1701                    if v then
1702                        if trace_identifying then
1703                            report_identify_outer(set,var,i,"4b")
1704                        end
1705                        return v
1706                    end
1707                end
1708            end
1709        end
1710
1711        -- as a last resort we will consult the global lists
1712
1713        local v = unprefixed_inner(set,var,"",collected,derived,tobesaved)
1714        if v then
1715            if trace_identifying then
1716                report_identify_outer(set,v,i,"4c")
1717            end
1718            return v
1719        end
1720
1721        -- not it gets bad ... we need to look in external files ... keep in mind that
1722        -- we can best use explicit references for this ... we might issue a warning
1723
1724        local componentreferences = productdata.componentreferences
1725        local productreferences = productdata.productreferences
1726        local components = productdata.components
1727        if components and componentreferences then
1728            for c=1,#components do
1729                local component = components[c]
1730                local data = componentreferences[component]
1731                if data then
1732                    local d = data[""]
1733                    local vi = d and d[inner]
1734                    if vi then
1735                        var.outer = component
1736                        var.i = vi
1737                        var.kind = "outer with inner"
1738                        set.external = true
1739                        if trace_identifying then
1740                            report_identify_outer(set,var,i,"4d")
1741                        end
1742                        return var
1743                    end
1744                end
1745            end
1746        end
1747        local component, inner = lpegmatch(componentsplitter,inner)
1748        if component then
1749            local data = componentreferences and componentreferences[component]
1750            if data then
1751                local d = data[""]
1752                local vi = d and d[inner]
1753                if vi then
1754                    var.inner = inner
1755                    var.outer = component
1756                    var.i = vi
1757                    var.kind = "outer with inner"
1758                    set.external = true
1759                    if trace_identifying then
1760                        report_identify_outer(set,var,i,"4e")
1761                    end
1762                    return var
1763                end
1764            end
1765            local data = productreferences and productreferences[component]
1766            if data then
1767                local vi = data[inner]
1768                if vi then
1769                    var.inner = inner
1770                    var.outer = component
1771                    var.i = vi
1772                    var.kind = "outer with inner"
1773                    set.external = true
1774                    if trace_identifying then
1775                        report_identify_outer(set,var,i,"4f")
1776                    end
1777                    return var
1778                end
1779            end
1780        end
1781        var.error = "unknown inner"
1782    else
1783        var.error = "no inner"
1784    end
1785    if trace_identifying then
1786        report_identify_outer(set,var,i,"4g")
1787    end
1788    return var
1789end
1790
1791local function identify_inner_component(set,var,i)
1792    -- we're in a product (maybe ignore when same as component)
1793    local component = var.component
1794    local v = identify_inner(set,var,component,collected,derived)
1795    if not v then
1796        var.error = "unknown inner in component"
1797    end
1798    if trace_identifying then
1799        report_identify_outer(set,var,i,"5a")
1800    end
1801    return var
1802end
1803
1804local function identify_outer_component(set,var,i)
1805    local component = var.component
1806    local inner = var.inner
1807    local data = productdata.componentreferences[component]
1808    if data then
1809        local d = data[""]
1810        local vi = d and d[inner]
1811        if vi then
1812            var.inner = inner
1813            var.outer = component
1814            var.i = vi
1815            var.kind = "outer with inner"
1816            set.external = true
1817            if trace_identifying then
1818                report_identify_outer(set,var,i,"6a")
1819            end
1820            return var
1821        end
1822    end
1823    local data = productdata.productreferences[component]
1824    if data then
1825        local vi = data[inner]
1826        if vi then
1827            var.inner = inner
1828            var.outer = component
1829            var.i = vi
1830            var.kind = "outer with inner"
1831            set.external = true
1832            if trace_identifying then
1833                report_identify_outer(set,var,i,"6b")
1834            end
1835            return var
1836        end
1837    end
1838    var.error = "unknown component"
1839    if trace_identifying then
1840        report_identify_outer(set,var,i,"6c")
1841    end
1842    return var
1843end
1844
1845local nofidentified = 0
1846
1847local function identify(prefix,reference)
1848    if not reference then
1849        prefix, reference = "", prefix
1850    end
1851    local set = resolve(prefix,reference)
1852    local bug = false
1853    texsetcount("referencehastexstate",set.has_tex and 1 or 0)
1854    nofidentified = nofidentified + 1
1855    set.n = nofidentified
1856    for i=1,#set do
1857        local var = set[i]
1858        local spe = var.special
1859        local fnc = functions[spe]
1860        if fnc then
1861            var = fnc(var) or { error = "invalid special function" }
1862        elseif spe then
1863            var = identify_special(set,var,i)
1864        elseif var.outer then
1865            var = identify_outer(set,var,i)
1866        elseif var.arguments then
1867            var = identify_arguments(set,var,i)
1868        elseif not var.component then
1869            var = identify_inner_or_outer(set,var,i)
1870        elseif productcomponent() then
1871            var = identify_inner_component(set,var,i)
1872        else
1873            var = identify_outer_component(set,var,i)
1874        end
1875        set[i] = var
1876        bug = bug or var.error
1877    end
1878    references.currentset = mark(set) -- mark, else in api doc
1879    if trace_analyzing then
1880        report_references(table.serialize(set,reference))
1881    end
1882    return set, bug
1883end
1884
1885references.identify = identify
1886
1887local unknowns, nofunknowns, f_valid = { }, 0, formatters["[%s][%s]"]
1888
1889function references.valid(prefix,reference,specification)
1890    local set, bug = identify(prefix,reference)
1891    local unknown = bug or #set == 0
1892    if unknown then
1893        currentreference = nil -- will go away
1894        local str = f_valid(prefix,reference)
1895        local u = unknowns[str]
1896        if not u then
1897            if somefound then
1898                interfaces.showmessage("references",1,str) -- 1 = unknown, 4 = illegal
1899            end
1900            unknowns[str] = 1
1901            nofunknowns = nofunknowns + 1
1902        else
1903            unknowns[str] = u + 1
1904        end
1905    else
1906        set.highlight    = specification.highlight
1907        set.newwindow    = specification.newwindow
1908        set.layer        = specification.layer
1909        currentreference = set[1]
1910    end
1911    -- we can do the expansion here which saves a call
1912    return not unknown
1913end
1914
1915implement {
1916    name      = "doifelsereference",
1917    actions   = { references.valid, commands.doifelse },
1918    arguments = {
1919        "string",
1920        "string",
1921        {
1922            { "highlight", "boolean" },
1923            { "newwindow", "boolean" },
1924            { "layer" },
1925        }
1926    }
1927}
1928
1929logs.registerfinalactions(function()
1930    if nofunknowns > 0 then
1931        statistics.register("cross referencing", function()
1932            return format("%s identified, %s unknown",nofidentified,nofunknowns)
1933        end)
1934        local sortedhash = table.sortedhash
1935        logs.startfilelogging(report,"missing references")
1936        for k, v in table.sortedhash(unknowns) do
1937            report("%4i  %s",v,k)
1938        end
1939        logs.stopfilelogging()
1940        if logs.loggingerrors() then
1941            logs.starterrorlogging(report,"missing references")
1942            for k, v in table.sortedhash(unknowns) do
1943                report("%4i  %s",v,k)
1944            end
1945            logs.stoperrorlogging()
1946        end
1947    end
1948end)
1949
1950-- The auto method will try to avoid named internals in a clever way which
1951-- can make files smaller without sacrificing external references. Some of
1952-- the housekeeping happens the backend side.
1953
1954local innermethod        = v_auto       -- only page|auto now
1955local outermethod        = v_auto       -- only page|auto now
1956local defaultinnermethod = defaultinnermethod
1957local defaultoutermethod = defaultoutermethod
1958references.innermethod   = innermethod  -- don't mess with this one directly
1959references.outermethod   = outermethod  -- don't mess with this one directly
1960
1961function references.setlinkmethod(inner,outer)
1962    if not outer and type(inner) == "string" then
1963        local m = settings_to_array(inner)
1964        inner = m[1]
1965        outer = m[2] or v_auto
1966    end
1967    if toboolean(inner) or inner == v_page or inner == v_yes then
1968        innermethod = v_page
1969    elseif inner == v_name then
1970        innermethod = v_name
1971    else
1972        innermethod = v_auto
1973    end
1974    if toboolean(outer) or outer == v_page or outer == v_yes then
1975        outermethod = v_page
1976    elseif inner == v_name then
1977        outermethod = v_name
1978    else
1979        outermethod = v_auto
1980    end
1981    references.innermethod = innermethod
1982    references.outermethod = outermethod
1983    function references.setlinkmethod()
1984        report_references("link method is already set and frozen: inner %a, outer %a",innermethod,outermethod)
1985    end
1986end
1987
1988implement {
1989    name      = "setreferencelinkmethod",
1990    actions   = references.setlinkmethod,
1991    arguments = "string",
1992 -- onlyonce  = true
1993}
1994
1995function references.getinnermethod()
1996    return innermethod or defaultinnermethod
1997end
1998
1999function references.getoutermethod()
2000    return outermethod or defaultoutermethod
2001end
2002
2003directives.register("references.linkmethod", function(v) -- page auto
2004    references.setlinkmethod(v)
2005end)
2006
2007-- we can call setinternalreference with an already known internal or with
2008-- a reference/prefix specification
2009
2010local destinationattributes = { }
2011
2012local function setinternalreference(specification)
2013    local internal    = specification.internal
2014    local destination = unsetvalue
2015    if innermethod == v_auto or innermethod == v_name then
2016        local t         = { } -- maybe add to current (now only used for tracing)
2017        local tn        = 0
2018        local reference = specification.reference
2019        local view      = specification.view
2020        if reference then
2021            local prefix = specification.prefix
2022            if prefix and prefix ~= "" then
2023                local prefix = prefix .. ":" -- watch out, : here
2024                local function action(ref)
2025                    tn = tn + 1
2026                    t[tn] = prefix .. ref
2027                end
2028                process_settings(reference,action)
2029            else
2030                local function action(ref)
2031                    tn = tn + 1
2032                    t[tn] = ref
2033                end
2034                process_settings(reference,action)
2035            end
2036        end
2037        -- ugly .. later we decide to ignore it when we have a real one
2038        -- but for testing we might want to see them all
2039        if internal then
2040            if innermethod ~= v_name then -- innermethod == v_auto
2041             -- we don't want too many #1 #2 #3 etc
2042                tn = tn + 1
2043                t[tn] = internal -- when number it's internal
2044            end
2045            if not view then
2046                local i = references.internals[internal]
2047                if i then
2048                    view = i.references.view
2049                end
2050            end
2051        end
2052        destination = references.mark(t,nil,nil,view) -- returns an attribute
2053    end
2054    if internal then -- new
2055        destinationattributes[internal] = destination
2056    end
2057    texsetcount("lastdestinationattribute",destination)
2058    return destination
2059end
2060
2061local function getinternalreference(internal)
2062    return destinationattributes[internal] or 0
2063end
2064
2065references.setinternalreference = setinternalreference
2066references.getinternalreference = getinternalreference
2067
2068implement {
2069    name      = "setinternalreference",
2070    actions   = setinternalreference,
2071    arguments = {
2072        {
2073            { "prefix" },
2074            { "reference" },
2075            { "internal", "integer" },
2076            { "view" }
2077        }
2078    }
2079}
2080
2081-- implement {
2082--     name      = "getinternalreference",
2083--     actions   = { getinternalreference, context },
2084--     arguments = "integer",
2085-- }
2086
2087function references.setandgetattribute(data) -- maybe do internal automatically here
2088    local attr = unsetvalue
2089    local mdat = data.metadata
2090    local rdat = data.references
2091    if mdat and rdat then
2092        if not rdat.section then
2093            rdat.section = structures.sections.currentid()
2094        end
2095        local ndat = data.numberdata
2096        if ndat then
2097            local numbers = ndat.numbers
2098            if type(numbers) == "string" then
2099                ndat.numbers = counters.compact(numbers,nil,true)
2100            end
2101            data.numberdata = helpers.simplify(ndat)
2102        end
2103        local pdat = data.prefixdata
2104        if pdat then
2105            data.prefixdata = helpers.simplify(pdat)
2106        end
2107        local udat = data.userdata
2108        if type(udat) == "string"  then
2109            data.userdata = helpers.touserdata(udat)
2110        end
2111        if not rdat.block then
2112            rdat.block = structures.sections.currentblock()
2113        end
2114        local done = references.set(data) -- we had kind i.e .item -> full
2115        if done then
2116            attr = setinternalreference {
2117                prefix    = rdat.prefix,
2118                reference = rdat.reference,
2119                internal  = rdat.internal,
2120                view      = rdat.view
2121            } or unsetvalue
2122        end
2123    end
2124    texsetcount("lastdestinationattribute",attr)
2125    return attr
2126end
2127
2128implement {
2129    name      = "setdestinationattribute",
2130    actions   = references.setandgetattribute,
2131    arguments = {
2132        {
2133            {
2134                "references", {
2135                    { "internal", "integer" },
2136                    { "block" },
2137                    { "view" },
2138                    { "prefix" },
2139                    { "reference" },
2140                },
2141            },
2142            {
2143                "metadata", {
2144                    { "kind" },
2145                    { "xmlroot" },
2146                    { "catcodes", "integer" },
2147                },
2148            },
2149            {
2150                "prefixdata", { "*" }
2151            },
2152            {
2153                "numberdata", { "*" }
2154            },
2155            {
2156                "entries", { "*" }
2157            },
2158            {
2159                "userdata"
2160            }
2161        }
2162    }
2163}
2164
2165function references.getinternallistreference(n) -- n points into list (todo: registers)
2166    local l = lists.collected[n]
2167    local i = l and l.references.internal
2168    return i and destinationattributes[i] or 0
2169end
2170
2171function references.getinternalcachedlistreference(n) -- n points into list (todo: registers)
2172    local l = lists.cached[n]
2173    local i = l and l.references.internal
2174    return i and destinationattributes[i] or 0
2175end
2176
2177implement {
2178    name      = "getinternallistreference",
2179    actions   = { references.getinternallistreference, context },
2180    arguments = "integer"
2181}
2182
2183implement {
2184    name      = "getinternalcachedlistreference",
2185    actions   = { references.getinternalcachedlistreference, context },
2186    arguments = "integer"
2187}
2188
2189
2190--
2191
2192function references.getcurrentmetadata(tag)
2193    local data = currentreference and currentreference.i
2194    return data and data.metadata and data.metadata[tag]
2195end
2196
2197implement {
2198    name      = "getcurrentreferencemetadata",
2199    actions   = { references.getcurrentmetadata, context },
2200    arguments = "string",
2201}
2202
2203local function currentmetadata(tag)
2204    local data = currentreference and currentreference.i
2205    return data and data.metadata and data.metadata[tag]
2206end
2207
2208references.currentmetadata = currentmetadata
2209
2210local function getcurrentprefixspec(default)
2211    local data     = currentreference and currentreference.i
2212    local metadata = data and data.metadata
2213    return
2214        metadata and metadata.kind or "?",
2215        metadata and metadata.name or "?",
2216        default                    or "?"
2217end
2218
2219references.getcurrentprefixspec = getcurrentprefixspec
2220
2221-- implement {
2222--     name      = "getcurrentprefixspec",
2223--     actions   = { getcurrentprefixspec, context }, -- returns 3 arguments
2224--     arguments = "string",
2225-- }
2226
2227implement {
2228    name      = "getcurrentprefixspec",
2229    actions   = function(tag)
2230        context("{%s}{%s}{%s}",getcurrentprefixspec(tag))
2231    end,
2232    arguments = "string",
2233}
2234
2235local genericfilters = { }
2236local userfilters    = { }
2237local textfilters    = { }
2238local fullfilters    = { }
2239local sectionfilters = { }
2240
2241filters.generic = genericfilters
2242filters.user    = userfilters
2243filters.text    = textfilters
2244filters.full    = fullfilters
2245filters.section = sectionfilters
2246
2247local function filterreference(name,prefixspec,numberspec) -- number page title ...
2248    local data = currentreference and currentreference.i -- maybe we should take realpage from here
2249    if data then
2250        if name == "realpage" then
2251            local cs = references.analyze() -- normally already analyzed but also sets state
2252            context(tonumber(cs.realpage) or 0)
2253        else -- assumes data is table
2254            local kind = type(data) == "table" and data.metadata and data.metadata.kind
2255            if kind then
2256                local filter = filters[kind] or genericfilters
2257                filter = filter and (filter[name] or filter.unknown or genericfilters[name] or genericfilters.unknown)
2258                if filter then
2259                    if trace_referencing then
2260                        report_references("name %a, kind %a, using dedicated filter",name,kind)
2261                    end
2262                    filter(data,name,prefixspec,numberspec)
2263                elseif trace_referencing then
2264                    report_references("name %a, kind %a, using generic filter",name,kind)
2265                end
2266            elseif trace_referencing then
2267                report_references("name %a, unknown kind",name)
2268            end
2269        end
2270    elseif name == "realpage" then
2271        context(0)
2272    elseif trace_referencing then
2273        report_references("name %a, no reference",name)
2274    end
2275end
2276
2277local function filterreferencedefault()
2278    return filterreference("default",getcurrentprefixspec("default"))
2279end
2280
2281references.filter        = filterreference
2282references.filterdefault = filterreferencedefault
2283
2284implement {
2285    name      = "filterreference",
2286    actions   = filterreference,
2287    arguments = "string",
2288}
2289
2290implement {
2291    name      = "filterdefaultreference",
2292    actions   = filterreference,
2293    arguments = {
2294        "string",    -- 'default'
2295        { { "*" } }, -- prefixspec
2296        { { "*" } }, -- numberspec
2297    }
2298}
2299
2300function genericfilters.title(data)
2301    if data then
2302        local titledata = data.titledata or data.useddata
2303        if titledata then
2304            helpers.title(titledata.reference or titledata.title or "?",data.metadata)
2305        end
2306    end
2307end
2308
2309function genericfilters.text(data)
2310    if data then
2311        local entries = data.entries or data.useddata
2312        if entries then
2313            helpers.title(entries.text or "?",data.metadata)
2314        end
2315    end
2316end
2317
2318function genericfilters.number(data,what,prefixspec,numberspec)
2319    if data then
2320        numberdata = lists.reordered(data) -- data.numberdata
2321        if numberdata then
2322            helpers.prefix(data,prefixspec)
2323            sections.typesetnumber(numberdata,"number",numberspec,numberdata)
2324        else
2325            local useddata = data.useddata
2326            if useddata and useddata.number then
2327                context(useddata.number)
2328            end
2329        end
2330    end
2331end
2332
2333genericfilters.default = genericfilters.text
2334
2335function genericfilters.page(data,prefixspec,pagespec)
2336    local pagedata = data.pagedata
2337    if pagedata then
2338        local number     = pagedata.number
2339        local conversion = pagedata.conversion
2340        if not number then
2341            -- error
2342        elseif conversion then
2343            ctx_convertnumber(conversion,number)
2344        else
2345            context(number)
2346        end
2347    else
2348        helpers.prefixpage(data,prefixspec,pagespec)
2349    end
2350end
2351
2352function userfilters.unknown(data,name)
2353    if data then
2354        local userdata = data.userdata
2355        local userkind = userdata and userdata.kind
2356        if userkind then
2357            local filter = filters[userkind] or genericfilters
2358            filter = filter and (filter[name] or filter.unknown)
2359            if filter then
2360                filter(data,name)
2361                return
2362            end
2363        end
2364        local namedata = userdata and userdata[name]
2365        if namedata then
2366            context(namedata)
2367        end
2368    end
2369end
2370
2371function textfilters.title(data)
2372    helpers.title(data.entries.text or "?",data.metadata)
2373end
2374
2375-- no longer considered useful:
2376--
2377-- function filters.text.number(data)
2378--     helpers.title(data.entries.text or "?",data.metadata)
2379-- end
2380
2381function textfilters.page(data,prefixspec,pagespec)
2382    helpers.prefixpage(data,prefixspec,pagespec)
2383end
2384
2385fullfilters.title = textfilters.title
2386fullfilters.page  = textfilters.page
2387
2388function sectionfilters.number(data,what,prefixspec)
2389    if data then
2390        local numberdata = data.numberdata
2391        if not numberdata then
2392            local useddata = data.useddata
2393            if useddata and useddata.number then
2394                context(useddata.number)
2395            end
2396        elseif numberdata.hidenumber then
2397            local references = data.references
2398            if trace_empty then
2399                report_empty("reference %a has a hidden number",references.reference)
2400                ctx_emptyreference() -- maybe an option
2401            end
2402        else
2403            sections.typesetnumber(numberdata,"number",prefixspec,numberdata)
2404        end
2405    end
2406end
2407
2408sectionfilters.title   = genericfilters.title
2409sectionfilters.page    = genericfilters.page
2410sectionfilters.default = sectionfilters.number
2411
2412-- filters.note        = { default = genericfilters.number }
2413-- filters.formula     = { default = genericfilters.number }
2414-- filters.float       = { default = genericfilters.number }
2415-- filters.description = { default = genericfilters.number }
2416-- filters.item        = { default = genericfilters.number }
2417
2418setmetatableindex(filters, function(t,k) -- beware, test with rawget
2419    local v = { default = genericfilters.number } -- not copy as it might be extended differently
2420    t[k] = v
2421    return v
2422end)
2423
2424-- function references.sectiontitle(n)
2425--     helpers.sectiontitle(lists.collected[tonumber(n) or 0])
2426-- end
2427
2428-- function references.sectionnumber(n)
2429--     helpers.sectionnumber(lists.collected[tonumber(n) or 0])
2430-- end
2431
2432-- function references.sectionpage(n,prefixspec,pagespec)
2433--     helpers.prefixedpage(lists.collected[tonumber(n) or 0],prefixspec,pagespec)
2434-- end
2435
2436-- analyze
2437
2438references.testrunners  = references.testrunners  or { }
2439references.testspecials = references.testspecials or { }
2440
2441local runners  = references.testrunners
2442local specials = references.testspecials
2443
2444-- We need to prevent ending up in the 'relative location' analyzer as it is
2445-- pretty slow (progressively). In the pagebody one can best check the reference
2446-- real page to determine if we need contrastlocation as that is more lightweight.
2447
2448local function checkedpagestate(n,page,actions,position,spread)
2449    local p = tonumber(page)
2450    if not p then
2451        return 0
2452    end
2453    if position and #actions > 0 then
2454        local i = actions[1].i -- brrr
2455        if i then
2456            local a = i.references
2457            if a then
2458                local x = a.x
2459                local y = a.y
2460                if x and y then
2461                    local jp = jobpositions.collected[position]
2462                    if jp then
2463                        local px = jp.x
2464                        local py = jp.y
2465                        local pp = jp.p
2466                        if p == pp then
2467                            -- same page
2468                            if py > y then
2469                                return 5 -- above
2470                            elseif py < y then
2471                                return 4 -- below
2472                            elseif px > x then
2473                                return 4 -- below
2474                            elseif px < x then
2475                                return 5 -- above
2476                            else
2477                                return 1 -- same
2478                            end
2479                        elseif spread then
2480                            if pp % 2 == 0 then
2481                                -- left page
2482                                if pp > p then
2483                                    return 2 -- before
2484                                elseif pp + 1 == p then
2485--                                     return 4 -- below (on right page)
2486                                    return 5 -- above (on left page)
2487                                else
2488                                    return 3 -- after
2489                                end
2490                            else
2491                                -- right page
2492                                if pp < p then
2493                                    return 3 -- after
2494                                elseif pp - 1 == p then
2495--                                     return 5 -- above (on left page)
2496                                    return 4 -- below (on right page)
2497                                else
2498                                    return 2 -- before
2499                                end
2500                            end
2501                        elseif pp > p then
2502                            return 2 -- before
2503                        else
2504                            return 3 -- after
2505                        end
2506                    end
2507                end
2508            end
2509        end
2510    end
2511    local r = referredpage(n) -- sort of obsolete
2512    if p > r then
2513        return 3 -- after
2514    elseif p < r then
2515        return 2 -- before
2516    else
2517        return 1 -- same
2518    end
2519end
2520
2521local function setreferencerealpage(actions)
2522    if not actions then
2523        actions = references.currentset
2524    end
2525    if type(actions) == "table" then
2526        local realpage = actions.realpage
2527        if realpage then
2528            return realpage
2529        end
2530        local nofactions = #actions
2531        if nofactions > 0 then
2532            for i=1,nofactions do
2533                local a = actions[i]
2534                local what = runners[a.kind]
2535                if what then
2536                    what = what(a,actions) -- needs documentation
2537                end
2538            end
2539            realpage = actions.realpage
2540            if realpage then
2541                return realpage
2542            end
2543        end
2544        actions.realpage = 0
2545    end
2546    return 0
2547end
2548
2549references.setreferencerealpage = setreferencerealpage
2550
2551-- we store some analysis data alongside the indexed array
2552-- at this moment only the real reference page is analyzed
2553-- normally such an analysis happens in the backend code
2554
2555function references.analyze(actions,position,spread)
2556    if not actions then
2557        actions = references.currentset
2558    end
2559    if not actions then
2560        actions = { realpage = 0, pagestate = 0 }
2561    elseif actions.pagestate then
2562        -- already done
2563    else
2564        local realpage = actions.realpage or setreferencerealpage(actions)
2565        if realpage == 0 then
2566            actions.pagestate = 0
2567        elseif actions.external then
2568            actions.pagestate = 0
2569        else
2570            actions.pagestate = checkedpagestate(actions.n,realpage,actions,position,spread)
2571        end
2572    end
2573    return actions
2574end
2575
2576local function referencepagestate(position,detail,spread)
2577    local actions = references.currentset
2578    if not actions then
2579        return 0
2580    else
2581        local pagestate = actions.pagestate
2582        for i=1,#actions do
2583            local a = actions[i]
2584            if a.outer then
2585                pagestate = 0
2586                actions.pagestate = pagestate
2587                break
2588            end
2589        end
2590        if not pagestate then
2591            references.analyze(actions,position,spread) -- delayed unless explicitly asked for
2592            pagestate = actions.pagestate
2593        end
2594        if detail then
2595            return pagestate
2596        elseif pagestate == 4 then
2597            return 2 -- compatible
2598        elseif pagestate == 5 then
2599            return 3 -- compatible
2600        else
2601            return pagestate
2602        end
2603    end
2604end
2605
2606implement {
2607    name      = "referencepagestate",
2608    actions   = { referencepagestate, context },
2609    arguments = "string"
2610}
2611
2612implement {
2613    name      = "referencepagedetail",
2614    actions   = { referencepagestate, context },
2615    arguments = { "string", "boolean", "boolean" }
2616}
2617
2618-- local function referencerealpage()
2619--     local actions = references.currentset
2620--     return not actions and 0 or actions.realpage or setreferencerealpage(actions)
2621-- end
2622--
2623-- implement {
2624--     name      = "referencerealpage",
2625--     actions   = { referencerealpage, context },
2626--  -- arguments = "string" -- hm, weird
2627-- }
2628
2629implement {
2630    name      = "askedreference",
2631    public    = true,
2632    protected = true,
2633    actions   = function()
2634        local actions = references.currentset
2635        if actions then
2636            context("[p=%s,r=%s]",actions.prefix or "",actions.reference)
2637        end
2638    end
2639}
2640
2641implement {
2642    name    = "referencerealpage",
2643    actions = function()
2644        local actions = references.currentset
2645        context(not actions and 0 or actions.realpage or setreferencerealpage(actions))
2646    end
2647}
2648
2649local function referencepos(key)
2650    local actions = references.currentset
2651    local i = actions[1].i -- brrr
2652    local v = 0
2653    if i then
2654        local a = i.references
2655        if a then
2656            v = a[key] or 0
2657        end
2658    end
2659    return v
2660end
2661
2662implement { name = "referenceposx", actions = function() context("%p",referencepos("x")) end }
2663implement { name = "referenceposy", actions = function() context("%p",referencepos("y")) end }
2664
2665implement {
2666    name    = "referencecolumn",
2667    actions = function()
2668        local actions = references.currentset
2669        local column  = 1
2670        if actions then
2671            column = jobpositions.columnofpos(actions.realpage or setreferencerealpage(actions),referencepos("x"))
2672        end
2673        context(column or 1)
2674    end
2675}
2676
2677local plist, nofrealpages
2678
2679local function realpageofpage(p) -- the last one counts !
2680    if not plist then
2681        local pages = structures.pages.collected
2682        nofrealpages = #pages
2683        plist = { }
2684        for rp=1,nofrealpages do
2685            local page = pages[rp]
2686            if page then
2687                plist[page.number] = rp
2688            end
2689        end
2690        references.nofrealpages = nofrealpages
2691    end
2692    return plist[p]
2693end
2694
2695references.realpageofpage = realpageofpage
2696
2697function references.checkedrealpage(r)
2698    if not plist then
2699        realpageofpage(r) -- just initialize
2700    end
2701    if not r then
2702        return texgetcount("realpageno")
2703    elseif r < 1 then
2704        return 1
2705    elseif r > nofrealpages then
2706        return nofrealpages
2707    else
2708        return r
2709    end
2710end
2711
2712-- use local ?
2713
2714local pages = allocate {
2715    [variables.firstpage]       = function() return counters.record("realpage")["first"]    end,
2716    [variables.previouspage]    = function() return counters.record("realpage")["previous"] end,
2717    [variables.nextpage]        = function() return counters.record("realpage")["next"]     end,
2718    [variables.lastpage]        = function() return counters.record("realpage")["last"]     end,
2719
2720    [variables.firstsubpage]    = function() return counters.record("subpage" )["first"]    end,
2721    [variables.previoussubpage] = function() return counters.record("subpage" )["previous"] end,
2722    [variables.nextsubpage]     = function() return counters.record("subpage" )["next"]     end,
2723    [variables.lastsubpage]     = function() return counters.record("subpage" )["last"]     end,
2724
2725    [variables.forward]         = function() return counters.record("realpage")["forward"]  end,
2726    [variables.backward]        = function() return counters.record("realpage")["backward"] end,
2727}
2728
2729references.pages = pages
2730
2731-- maybe some day i will merge this in the backend code with a testmode (so each
2732-- runner then implements a branch)
2733
2734runners["inner"] = function(var,actions)
2735    local r = var.r
2736    if r then
2737        actions.realpage = r
2738    end
2739end
2740
2741runners["special"] = function(var,actions)
2742    local handler = specials[var.special]
2743    return handler and handler(var,actions)
2744end
2745
2746runners["special operation"]                = runners["special"]
2747runners["special operation with arguments"] = runners["special"]
2748
2749function specials.internal(var,actions)
2750    local v = internals[tonumber(var.operation)]
2751    local r = v and v.references
2752    if r then
2753        local p = r.realpage
2754        if p then
2755-- setmetatableindex(actions,r)
2756            actions.realpage = p
2757            actions.view     = r.view
2758        end
2759    end
2760end
2761
2762specials.i = specials.internal
2763
2764function specials.page(var,actions)
2765    local o = var.operation
2766    local p = pages[o]
2767    if type(p) == "function" then
2768        p = p()
2769    else
2770        p = tonumber(realpageofpage(tonumber(o)))
2771    end
2772    if p then
2773        var.r = p
2774        actions.realpage = actions.realpage or p -- first wins
2775    end
2776end
2777
2778function specials.realpage(var,actions)
2779    local p = tonumber(var.operation)
2780    if p then
2781        var.r = p
2782        actions.realpage = actions.realpage or p -- first wins
2783    end
2784end
2785
2786function specials.userpage(var,actions)
2787    local p = tonumber(realpageofpage(var.operation))
2788    if p then
2789        var.r = p
2790        actions.realpage = actions.realpage or p -- first wins
2791    end
2792end
2793
2794function specials.deltapage(var,actions)
2795    local p = tonumber(var.operation)
2796    if p then
2797        p = references.checkedrealpage(p + texgetcount("realpageno"))
2798        var.r = p
2799        actions.realpage = actions.realpage or p -- first wins
2800    end
2801end
2802
2803function specials.section(var,actions)
2804    local sectionname = var.arguments
2805    local destination = var.operation
2806    local internal    = structures.sections.internalreference(sectionname,destination)
2807    if internal then
2808        var.special   = "internal"
2809        var.operation = internal
2810        var.arguments = nil
2811        specials.internal(var,actions)
2812    end
2813end
2814
2815-- experimental:
2816
2817local p_splitter = lpeg.splitat(":")
2818local p_lower    = lpeg.patterns.utf8lower
2819
2820-- We can cache lowercased titles which saves a lot of time, but then
2821-- we can better have a global cache with weak keys.
2822
2823-- local lowercache = table.setmetatableindex(function(t,k)
2824--     local v = lpegmatch(p_lower,k)
2825--     t[k] = v
2826--     return v
2827-- end)
2828
2829local lowercache = false
2830
2831local function locate(list,askedkind,askedname,pattern)
2832    local kinds = lists.kinds
2833    local names = lists.names
2834    if askedkind and not kinds[askedkind] then
2835        return false
2836    end
2837    if askedname and not names[askedname] then
2838        return false
2839    end
2840    for i=1,#list do
2841        local entry    = list[i]
2842        local metadata = entry.metadata
2843        if metadata then
2844            local found = false
2845            if askedname then
2846                local name = metadata.name
2847                if name then
2848                    found = name == askedname
2849                end
2850            elseif askedkind then
2851                local kind = metadata.kind
2852                if kind then
2853                    found = kind == askedkind
2854                end
2855            end
2856            if found then
2857                local titledata = entry.titledata
2858                if titledata then
2859                    local title = titledata.title
2860                    if title then
2861                        if lowercache then
2862                            found = lpegmatch(pattern,lowercache[title])
2863                        else
2864                            found = lpegmatch(pattern,lpegmatch(p_lower,title))
2865                        end
2866                        if found then
2867                            return {
2868                                inner     = pattern,
2869                                kind      = "inner",
2870                                reference = pattern,
2871                                i         = entry,
2872                                p         = "",
2873                                r         = entry.references.realpage,
2874                            }
2875                        end
2876                    end
2877                end
2878            end
2879        end
2880    end
2881end
2882
2883function functions.match(var,actions)
2884    if not var.outer then
2885        local operation   = var.operation
2886        if operation and operation ~= "" then
2887            local operation   = lpegmatch(p_lower,operation)
2888            local list        = lists.collected
2889            local names       = false
2890            local kinds       = false
2891            local where, what = lpegmatch(p_splitter,operation)
2892            if where and what then
2893                local pattern = lpeg.finder(what)
2894                return
2895                    locate(list,false,where,pattern)
2896                 or locate(list,where,false,pattern)
2897                 or { error = "no match" }
2898            else
2899                local pattern = lpeg.finder(operation)
2900                -- todo: don't look at section and float in last pass
2901                return
2902                    locate(list,"section",false,pattern)
2903                 or locate(list,"float",false,pattern)
2904                 or locate(list,false,false,pattern)
2905                 or { error = "no match" }
2906            end
2907        end
2908    end
2909end
2910
2911-- needs a better split ^^^
2912
2913-- done differently now:
2914
2915function references.export(usedname) end
2916function references.import(usedname) end
2917function references.load  (usedname) end
2918
2919implement { name = "exportreferences", actions =references.export }
2920
2921-- better done here .... we don't insert/remove, just use a pointer
2922
2923local prefixstack = { "" }
2924local prefixlevel = 1
2925
2926local function pushreferenceprefix(prefix)
2927    prefixlevel = prefixlevel + 1
2928    prefixstack[prefixlevel] = prefix
2929    return prefix
2930end
2931
2932local function popreferenceprefix()
2933    prefixlevel = prefixlevel - 1
2934    if prefixlevel > 0 then
2935        return prefixstack[prefixlevel]
2936    else
2937        report_references("unable to pop referenceprefix")
2938        return ""
2939    end
2940end
2941
2942implement {
2943    name      = "pushreferenceprefix",
2944    actions   = { pushreferenceprefix, context }, -- we can use setmacro
2945    arguments = "string",
2946}
2947
2948implement {
2949    name      = "popreferenceprefix",
2950    actions   = { popreferenceprefix, context }, -- we can use setmacro
2951}
2952