SourceBrowser

strc-ref.lua /size: 92 Kb last modification: 2023-12-21 09:44
1if not modules then modules = { } end modules ['strc-ref'] = {
2    version   = 1.001,
3    comment   = "companion to strc-ref.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- beware, this is a first step in the rewrite (just getting rid of
10-- the tuo file); later all access and parsing will also move to lua
11
12-- the useddata and pagedata names might change
13-- todo: pack exported data
14
15-- todo: autoload components when :::
16
17local format, find, gmatch, match, strip = string.format, string.find, string.gmatch, string.match, string.strip
18local floor = math.floor
19local rawget, tonumber, type, next = rawget, tonumber, type, next
20local lpegmatch = lpeg.match
21local insert, remove, copytable = table.insert, table.remove, table.copy
22local formatters = string.formatters
23local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match
24
25local allocate           = utilities.storage.allocate
26local mark               = utilities.storage.mark
27local setmetatableindex  = table.setmetatableindex
28
29local trace_referencing  = false  trackers.register("structures.referencing",             function(v) trace_referencing = v end)
30local trace_analyzing    = false  trackers.register("structures.referencing.analyzing",   function(v) trace_analyzing   = v end)
31local trace_identifying  = false  trackers.register("structures.referencing.identifying", function(v) trace_identifying = v end)
32local trace_importing    = false  trackers.register("structures.referencing.importing",   function(v) trace_importing   = v end)
33local trace_empty        = false  trackers.register("structures.referencing.empty",       function(v) trace_empty       = v end)
34
35local check_duplicates   = true
36
37directives.register("structures.referencing.checkduplicates", function(v) check_duplicates = v end)
38
39local report_references  = logs.reporter("references")
40local report_identifying = logs.reporter("references","identifying")
41local report_importing   = logs.reporter("references","importing")
42local report_empty       = logs.reporter("references","empty")
43local report             = report_references
44
45local variables          = interfaces.variables
46local v_page             = variables.page
47local v_auto             = variables.auto
48local v_yes              = variables.yes
49local v_name             = variables.name
50
51local context            = context
52local commands           = commands
53local implement          = interfaces.implement
54
55local ctx_latelua        = context.latelua
56
57local texgetcount        = tex.getcount
58local texsetcount        = tex.setcount
59local texconditionals    = tex.conditionals
60
61local productcomponent   = resolvers.jobs.productcomponent
62local justacomponent     = resolvers.jobs.justacomponent
63
64local settings_to_array  = utilities.parsers.settings_to_array
65local settings_to_table  = utilities.parsers.settings_to_array_obey_fences
66local process_settings   = utilities.parsers.process_stripped_settings
67local unsetvalue         = attributes.unsetvalue
68
69local structures         = structures
70local helpers            = structures.helpers
71local sections           = structures.sections
72local references         = structures.references
73local lists              = structures.lists
74local counters           = structures.counters
75
76local jobpositions       = job.positions
77local getpos             = jobpositions.getpos
78
79-- some might become local
80
81references.defined       = references.defined or allocate()
82
83local defined            = references.defined
84local derived            = allocate()
85local specials           = allocate()
86local functions          = allocate()
87local runners            = allocate()
88local internals          = allocate()
89local filters            = allocate()
90local executers          = allocate()
91local handlers           = allocate()
92local tobesaved          = allocate()
93local collected          = allocate()
94local tobereferred       = allocate()
95local referred           = allocate()
96local usedinternals      = allocate()
97local flaginternals      = allocate()
98local usedviews          = allocate()
99
100references.derived       = derived
101references.specials      = specials
102references.functions     = functions
103references.runners       = runners
104references.internals     = internals
105references.filters       = filters
106references.executers     = executers
107references.handlers      = handlers
108references.tobesaved     = tobesaved
109references.collected     = collected
110references.tobereferred  = tobereferred
111references.referred      = referred
112references.usedinternals = usedinternals
113references.flaginternals = flaginternals
114references.usedviews     = usedviews
115
116local splitreference     = references.splitreference
117local splitprefix        = references.splitcomponent -- replaces: references.splitprefix
118local prefixsplitter     = references.prefixsplitter
119local componentsplitter  = references.componentsplitter
120
121local currentreference   = nil
122
123local txtcatcodes        = catcodes.numbers.txtcatcodes -- or just use "txtcatcodes"
124
125local context                      = context
126
127local ctx_pushcatcodes             = context.pushcatcodes
128local ctx_popcatcodes              = context.popcatcodes
129local ctx_dofinishreference        = context.dofinishreference
130local ctx_dofromurldescription     = context.dofromurldescription
131local ctx_dofromurlliteral         = context.dofromurlliteral
132local ctx_dofromfiledescription    = context.dofromfiledescription
133local ctx_dofromfileliteral        = context.dofromfileliteral
134local ctx_expandreferenceoperation = context.expandreferenceoperation
135local ctx_expandreferencearguments = context.expandreferencearguments
136local ctx_convertnumber            = context.convertnumber
137local ctx_emptyreference           = context.emptyreference
138
139storage.register("structures/references/defined", references.defined, "structures.references.defined")
140
141local initializers = { }
142local finalizers   = { }
143local somefound    = false -- so we don't report missing when we have a fresh start
144
145function references.registerinitializer(func) -- we could use a token register instead
146    initializers[#initializers+1] = func
147end
148
149function references.registerfinalizer(func) -- we could use a token register instead
150    finalizers[#finalizers+1] = func
151end
152
153local function initializer() -- can we use a tobesaved as metatable for collected?
154    tobesaved = references.tobesaved
155    collected = references.collected
156    for i=1,#initializers do
157        initializers[i](tobesaved,collected)
158    end
159    for prefix, list in next, collected do
160        for tag, data in next, list do
161            local r = data.references
162            local i = r.internal
163            if i then
164                internals[i]     = data
165                usedinternals[i] = r.used
166            end
167        end
168    end
169    somefound = next(collected)
170end
171
172local function finalizer()
173    for i=1,#finalizers do
174        finalizers[i](tobesaved)
175    end
176    for prefix, list in next, tobesaved do
177        for tag, data in next, list do
178            local r = data.references
179            local i = r.internal
180            local f = flaginternals[i]
181            if f then
182                r.used = usedviews[i] or true
183            end
184        end
185    end
186end
187
188job.register('structures.references.collected', tobesaved, initializer, finalizer)
189
190local maxreferred = 1
191local nofreferred = 0
192
193local function initializer() -- can we use a tobesaved as metatable for collected?
194    tobereferred = references.tobereferred
195    referred     = references.referred
196    nofreferred = #referred
197end
198
199-- no longer done this way
200
201-- references.resolvers = references.resolvers or { }
202-- local resolvers = references.resolvers
203--
204-- function resolvers.section(var)
205--     local vi = lists.collected[var.i[2]]
206--     if vi then
207--         var.i = vi
208--         var.r = (vi.references and vi.references.realpage) or (vi.pagedata and vi.pagedata.realpage) or 1
209--     else
210--         var.i = nil
211--         var.r = 1
212--     end
213-- end
214--
215-- resolvers.float       = resolvers.section
216-- resolvers.description = resolvers.section
217-- resolvers.formula     = resolvers.section
218-- resolvers.note        = resolvers.section
219--
220-- function resolvers.reference(var)
221--     local vi = var.i[2]
222--     if vi then
223--         var.i = vi
224--         var.r = (vi.references and vi.references.realpage) or (vi.pagedata and vi.pagedata.realpage) or 1
225--     else
226--         var.i = nil
227--         var.r = 1
228--     end
229-- end
230
231-- We make the array sparse (maybe a finalizer should optionally return a table) because
232-- there can be quite some page links involved. We only store one action number per page
233-- which is normally good enough for what we want (e.g. see above/below) and we do
234-- a combination of a binary search and traverse backwards. A previous implementation
235-- always did a traverse and was pretty slow on a large number of links (given that this
236-- methods was used). It took me about a day to locate this as a bottleneck in processing
237-- a 2500 page interactive document with 60 links per page. In that case, traversing
238-- thousands of slots per link then brings processing to a grinding halt (especially when
239-- there are no slots at all, which is the case in a first run).
240
241local sparsetobereferred = { }
242
243local function finalizer()
244    local lastr, lasti
245    local n = 0
246    for i=1,maxreferred do
247        local r = tobereferred[i]
248        if not lastr then
249            lastr = r
250            lasti = i
251        elseif r ~= lastr then
252            n = n + 1
253            sparsetobereferred[n] = { lastr, lasti }
254            lastr = r
255            lasti = i
256        end
257    end
258    if lastr then
259        n = n + 1
260        sparsetobereferred[n] = { lastr, lasti }
261    end
262end
263
264job.register('structures.references.referred', sparsetobereferred, initializer, finalizer)
265
266local function referredpage(n)
267    local max = nofreferred
268    if max > 0 then
269        -- find match
270        local min = 1
271        while true do
272            local mid = floor((min+max)/2)
273            local r = referred[mid]
274            local m = r[2]
275            if n == m then
276                return r[1]
277            elseif n > m then
278                min = mid + 1
279            else
280                max = mid - 1
281            end
282            if min > max then
283                break
284            end
285        end
286        -- find first previous
287        for i=min,1,-1 do
288            local r = referred[i]
289            if r and r[2] < n then
290                return r[1]
291            end
292        end
293    end
294    -- fallback
295    return texgetcount("realpageno")
296end
297
298references.referredpage = referredpage
299
300function references.registerpage(n) -- called in the backend code
301    if not tobereferred[n] then
302        if n > maxreferred then
303            maxreferred = n
304        end
305        tobereferred[n] = texgetcount("realpageno")
306    end
307end
308
309-- todo: delay split till later as in destinations we split anyway
310
311local orders, lastorder = { }, 0
312
313local function setnextorder(kind,name)
314    lastorder = 0
315    if kind and name then
316        local ok = orders[kind]
317        if not ok then
318            ok = { }
319            orders[kind] = ok
320        end
321        lastorder = (ok[name] or 0) + 1
322        ok[name] = lastorder
323    end
324    texsetcount("global","locationorder",lastorder)
325end
326
327
328local function setnextinternal(kind,name)
329    setnextorder(kind,name) -- always incremented with internal
330    local n = texgetcount("locationcount") + 1
331    texsetcount("global","locationcount",n)
332    return n
333end
334
335local function currentorder(kind,name)
336    return orders[kind] and orders[kind][name] or lastorder
337end
338
339local function setcomponent(data)
340    -- we might consider doing this at the tex end, just like prefix
341    local component = productcomponent()
342    if component then
343        local references = data and data.references
344        if references then
345            references.component = component
346            if references.prefix == component then
347                references.prefix = nil
348            end
349        end
350        return component
351    end
352    -- but for the moment we do it here (experiment)
353end
354
355references.setnextorder    = setnextorder
356references.setnextinternal = setnextinternal
357references.currentorder    = currentorder
358references.setcomponent    = setcomponent
359
360implement {
361    name      = "setnextreferenceorder",
362    actions   = setnextorder,
363    arguments = "2 strings",
364}
365
366implement {
367    name      = "setnextinternalreference",
368    actions   = setnextinternal,
369    arguments = "2 strings",
370}
371
372implement {
373    name      = "currentreferenceorder",
374    actions = { currentorder, context },
375    arguments = "2 strings",
376}
377
378local reported = setmetatableindex("table")
379
380function references.set(data)
381    local references = data.references
382    local reference  = references.reference
383    if not reference or reference == "" then
384     -- report_references("invalid reference") -- harmless
385        return 0
386    end
387    local prefix = references.prefix or ""
388    local pd     = tobesaved[prefix] -- nicer is a metatable
389    if not pd then
390        pd = { }
391        tobesaved[prefix] = pd
392    end
393    local n = 0
394    local function action(ref)
395        if ref == "" then
396            -- skip
397        elseif check_duplicates and pd[ref] then
398            if not prefix then
399                prefix = ""
400            end
401            if not reported[prefix][ref] then
402                if prefix ~= "" then
403                    report_references("redundant reference %a in namespace %a",ref,prefix)
404                else
405                    report_references("redundant reference %a",ref)
406                end
407                reported[prefix][ref] = true
408            end
409        else
410            n = n + 1
411            pd[ref] = data
412            local r = data.references
413            ctx_dofinishreference(prefix or "",ref or "",r and r.internal or 0)
414         -- ctx_latelua(function() structures.references.enhance(prefix or ref,ref or "") end)
415        end
416    end
417    process_settings(reference,action)
418    return n > 0
419end
420
421-- function references.enhance(prefix,tag)
422--     local l = tobesaved[prefix][tag]
423--     if l then
424--         l.references.realpage = texgetcount("realpageno")
425--     end
426-- end
427
428local function synchronizepage(reference) -- non public helper
429    reference.realpage = texgetcount("realpageno")
430    if jobpositions.used() then
431        reference.x, reference.y = getpos()
432    end
433end
434
435references.synchronizepage = synchronizepage
436
437local function enhancereference(specification)
438    local prefix = specification.prefix
439    if prefix then
440        local entry = tobesaved[prefix]
441        if entry then
442            entry = entry[specification.tag]
443            if entry then
444                synchronizepage(entry.references)
445            else
446                -- normally a bug
447            end
448        else
449            -- normally a bug
450        end
451    else
452        -- normally a bug
453    end
454end
455
456references.enhance = enhancereference
457
458-- implement {
459--     name      = "enhancereference",
460--     arguments = "2 strings",
461--     actions   = function(prefix,tag)
462--        enhancereference { prefix = prefix, tag = tag }
463--     end,
464-- }
465
466implement {
467    name      = "deferredenhancereference",
468    arguments = "2 strings",
469    protected = true,
470    actions   = function(prefix,tag)
471        ctx_latelua { action = enhancereference, prefix = prefix, tag = tag }
472    end,
473}
474
475-- -- -- related to strc-ini.lua -- -- --
476
477-- no metatable here .. better be sparse
478
479local function register_from_lists(collected,derived,pages,sections)
480    local derived_g = derived[""] -- global
481    local derived_p = nil
482    local derived_c = nil
483    local prefix    = nil
484    local component = nil
485    local entry     = nil
486    if not derived_g then
487        derived_g = { }
488        derived[""] = derived_g
489    end
490    local function action(s)
491        if trace_referencing then
492            report_references("list entry %a provides %a reference %a on realpage %a",i,kind,s,realpage)
493        end
494        if derived_p and not derived_p[s] then
495            derived_p[s] = entry
496        end
497        if derived_c and not derived_c[s] then
498            derived_c[s] = entry
499        end
500        if not derived_g[s] then
501            derived_g[s] = entry -- first wins
502        end
503    end
504    for i=1,#collected do
505        entry = collected[i]
506        local metadata = entry.metadata
507        if metadata then
508            local kind = metadata.kind -- why this check
509            if kind then
510                local references = entry.references
511                if references then
512                    local reference = references.reference
513                    if reference and reference ~= "" then
514                        local realpage = references.realpage
515                        if realpage then
516                            prefix    = references.prefix
517                            component = references.component
518                            if prefix and prefix ~= "" then
519                                derived_p = derived[prefix]
520                                if not derived_p then
521                                    derived_p = { }
522                                    derived[prefix] = derived_p
523                                end
524                            end
525                            if component and component ~= "" and component ~= prefix then
526                                derived_c = derived[component]
527                                if not derived_c then
528                                    derived_c = { }
529                                    derived[component] = derived_c
530                                end
531                            end
532                            process_settings(reference,action)
533                        end
534                    end
535                end
536            end
537        end
538    end
539end
540
541references.registerinitializer(function() register_from_lists(lists.collected,derived) end)
542
543-- tracing
544
545local function collectbypage(tracedpages)
546    -- lists
547    do
548        local collected = structures.lists.collected
549        local data      = nil
550        local function action(reference)
551            local prefix    = data.prefix
552            local component = data.component
553            local realpage  = data.realpage
554            if realpage then
555                local pagelist  = rawget(tracedpages,realpage)
556                local internal  = data.internal or 0
557                local prefix    = (prefix ~= "" and prefix) or (component ~= "" and component) or ""
558                local pagedata  = { prefix, reference, internal }
559                if pagelist then
560                    pagelist[#pagelist+1] = pagedata
561                else
562                    tracedpages[realpage] = { pagedata }
563                end
564                if internal > 0 then
565                    data.usedprefix = prefix
566                end
567            end
568        end
569        for i=1,#collected do
570            local entry = collected[i]
571            local metadata = entry.metadata
572            if metadata and metadata.kind then
573                data = entry.references
574                if data then
575                    local reference = data.reference
576                    if reference and reference ~= "" then
577                        process_settings(reference,action)
578                    end
579                end
580            end
581        end
582    end
583    -- references
584    do
585        for prefix, list in next, collected do
586            for reference, entry in next, list do
587                local data = entry.references
588                if data then
589                    local realpage = data.realpage
590                    local internal = data.internal or 0
591                    local pagelist = rawget(tracedpages,realpage)
592                    local pagedata = { prefix, reference, internal }
593                    if pagelist then
594                        pagelist[#pagelist+1] = pagedata
595                    else
596                        tracedpages[realpage] = { pagedata }
597                    end
598                    if internal > 0 then
599                        data.usedprefix = prefix
600                    end
601                end
602            end
603        end
604    end
605end
606
607references.tracedpages = table.setmetatableindex(allocate(),function(t,k)
608    if collectbypage then
609        collectbypage(t)
610        collectbypage = nil
611    end
612    return rawget(t,k)
613end)
614
615-- urls
616
617local urls       = references.urls or { }
618references.urls  = urls
619local urldata    = urls.data or { }
620urls.data        = urldata
621
622local p_untexurl = Cs ( (
623    P("\\")/"" * (P("%")/"%%" + P(1))
624  + P(" ")/"%%20"
625  + P(1)
626)^1 )
627
628function urls.untex(url)
629    return lpegmatch(p_untexurl,url) or url
630end
631
632function urls.define(name,url,file,description)
633    if name and name ~= "" then
634     -- url = lpegmatch(replacer,url)
635        urldata[name] = { url or "", file or "", description or url or file or ""}
636    end
637end
638
639function urls.get(name)
640    local u = urldata[name]
641    if u then
642        local url, file = u[1], u[2]
643        if file and file ~= "" then
644            return formatters["%s/%s"](url,file)
645        else
646            return url
647        end
648    end
649end
650
651function urls.found(name)
652    return urldata[name]
653end
654
655local function geturl(name)
656    local url = urls.get(name)
657    if url and url ~= "" then
658        ctx_pushcatcodes(txtcatcodes)
659        context(url)
660        ctx_popcatcodes()
661    end
662end
663
664implement {
665    name      = "doifelseurldefined",
666    actions   = { urls.found, commands.doifelse },
667    arguments = "string"
668}
669
670implement {
671    name      = "useurl",
672    actions   = urls.define,
673    arguments = "4 strings",
674}
675
676implement {
677    name      = "geturl",
678    actions   = geturl,
679    arguments = "string",
680}
681
682-- files
683
684local files      = references.files or { }
685references.files = files
686local filedata   = files.data or { }
687files.data       = filedata
688
689function files.define(name,file,description)
690    if name and name ~= "" then
691        filedata[name] = { file or "", description or file or "" }
692    end
693end
694
695function files.get(name,method,space) -- method: none, before, after, both, space: yes/no
696    local f = filedata[name]
697    if f then
698        context(f[1])
699    end
700end
701
702function files.found(name)
703    return filedata[name]
704end
705
706local function getfile(name)
707    local fil = files.get(name)
708    if fil and fil ~= "" then
709        ctx_pushcatcodes(txtcatcodes)
710        context(fil)
711        ctx_popcatcodes()
712    end
713end
714
715implement {
716    name      = "doifelsefiledefined",
717    actions   = { files.found, commands.doifelse },
718    arguments = "string"
719}
720
721implement {
722    name      = "usefile",
723    actions   = files.define,
724    arguments = "3 strings"
725}
726
727implement {
728    name      = "getfile",
729    actions   = getfile,
730    arguments = "string"
731}
732
733-- helpers
734
735function references.checkedfile(whatever) -- return whatever if not resolved
736    if whatever then
737        local w = filedata[whatever]
738        if w then
739            return w[1]
740        else
741            return whatever
742        end
743    end
744end
745
746function references.checkedurl(whatever) -- return whatever if not resolved
747    if whatever then
748        local w = urldata[whatever]
749        if w then
750            local u, f = w[1], w[2]
751            if f and f ~= "" then
752                return u .. "/" .. f
753            else
754                return u
755            end
756        else
757            return whatever
758        end
759    end
760end
761
762function references.checkedfileorurl(whatever,default) -- return nil, nil if not resolved
763    if whatever then
764        local w = filedata[whatever]
765        if w then
766            return w[1], nil
767        else
768            local w = urldata[whatever]
769            if w then
770                local u, f = w[1], w[2]
771                if f and f ~= "" then
772                    return nil, u .. "/" .. f
773                else
774                    return nil, u
775                end
776            end
777        end
778    end
779    return default
780end
781
782-- programs
783
784local programs      = references.programs or { }
785references.programs = programs
786local programdata   = programs.data or { }
787programs.data       = programdata
788
789function programs.define(name,file,description)
790    if name and name ~= "" then
791        programdata[name] = { file or "", description or file or ""}
792    end
793end
794
795function programs.get(name)
796    local f = programdata[name]
797    return f and f[1]
798end
799
800function references.checkedprogram(whatever) -- return whatever if not resolved
801    if whatever then
802        local w = programdata[whatever]
803        if w then
804            return w[1]
805        else
806            return whatever
807        end
808    end
809end
810
811implement {
812    name      = "defineprogram",
813    actions   = programs.define,
814    arguments = "3 strings",
815}
816
817local function getprogram(name)
818    local p = programdata[name]
819    if p then
820        context(p[1])
821    end
822end
823
824implement {
825    name      = "getprogram",
826    actions   = getprogram,
827    arguments = "string"
828}
829
830-- shared by urls and files
831
832function references.from(name)
833    local u = urldata[name]
834    if u then
835        local url, file, description = u[1], u[2], u[3]
836        if description ~= "" then
837            return description
838            -- ok
839        elseif file and file ~= "" then
840            return url .. "/" .. file
841        else
842            return url
843        end
844    else
845        local f = filedata[name]
846        if f then
847            local file, description = f[1], f[2]
848            if description ~= "" then
849                return description
850            else
851                return file
852            end
853        end
854    end
855end
856
857local function from(name)
858    local u = urldata[name]
859    if u then
860        local url, file, description = u[1], u[2], u[3]
861        if description ~= "" then
862            ctx_dofromurldescription(description)
863            -- ok
864        elseif file and file ~= "" then
865            ctx_dofromurlliteral(url .. "/" .. file)
866        else
867            ctx_dofromurlliteral(url)
868        end
869    else
870        local f = filedata[name]
871        if f then
872            local file, description = f[1], f[2]
873            if description ~= "" then
874                ctx_dofromfiledescription(description)
875            else
876                ctx_dofromfileliteral(file)
877            end
878        end
879    end
880end
881
882implement {
883    name      = "from",
884    actions   = from,
885    arguments = "string"
886}
887
888function references.define(prefix,reference,list)
889    local d = defined[prefix] if not d then d = { } defined[prefix] = d end
890    d[reference] = list
891end
892
893function references.reset(prefix,reference)
894    local d = defined[prefix]
895    if d then
896        d[reference] = nil
897    end
898end
899
900implement {
901    name      = "definereference",
902    actions   = references.define,
903    arguments = "3 strings",
904}
905
906implement {
907    name      = "resetreference",
908    actions   = references.reset,
909    arguments = "2 strings",
910}
911
912setmetatableindex(defined,"table")
913
914local function resolve(prefix,reference,args,set) -- we start with prefix,reference
915    if reference and reference ~= "" then
916        if not set then
917            set = { prefix = prefix, reference = reference }
918        else
919            if not set.reference then set.reference = reference end
920            if not set.prefix    then set.prefix    = prefix    end
921        end
922     -- local r = settings_to_array(reference)
923        local r = settings_to_table(reference) -- maybe option to honor () []
924        for i=1,#r do
925            local ri = r[i]
926            local d = defined[prefix][ri] or defined[""][ri]
927            if d then
928                resolve(prefix,d,nil,set)
929            else
930                local var = splitreference(ri)
931                if var then
932                    var.reference = ri
933                    local vo, vi = var.outer, var.inner
934                    -- we catch this here .. it's a way to pass references with commas
935                    if vi == "name" then
936                        local arguments = var.arguments
937                        if arguments then
938                            vi            = arguments
939                            var.inner     = arguments
940                            var.reference = arguments
941                            var.arguments = nil
942                        end
943                    elseif var.special == "name" then
944                        local operation = var.operation
945                        if operation then
946                            vi            = operation
947                            var.inner     = operation
948                            var.reference = operation
949                            var.operation = nil
950                            var.special   = nil
951                        end
952                    end
953                    -- end of catch
954                    if not vo and vi then
955                        -- to be checked
956                        d = defined[prefix][vi] or defined[""][vi]
957                        --
958                        if d then
959                            resolve(prefix,d,var.arguments,set) -- args can be nil
960                        else
961                            if args then var.arguments = args end
962                            set[#set+1] = var
963                        end
964                    else
965                        if args then var.arguments = args end
966                        set[#set+1] = var
967                    end
968                    if var.has_tex then
969                        set.has_tex = true
970                    end
971                else
972                --  report_references("funny pattern %a",ri)
973                end
974            end
975        end
976        return set
977    else
978        return { }
979    end
980end
981
982-- prefix == "" is valid prefix which saves multistep lookup
983
984references.currentset = nil
985
986local function setreferenceoperation(k,v)
987    references.currentset[k].operation = v
988end
989
990local function setreferencearguments(k,v)
991    references.currentset[k].arguments = v
992end
993
994function references.expandcurrent() -- todo: two booleans: o_has_tex& a_has_tex
995    local currentset = references.currentset
996    if currentset and currentset.has_tex then
997        for i=1,#currentset do
998            local ci = currentset[i]
999            local operation = ci.operation
1000            if operation and find(operation,"\\",1,true) then -- if o_has_tex then
1001                ctx_expandreferenceoperation(i,operation)
1002            end
1003            local arguments = ci.arguments
1004            if arguments and find(arguments,"\\",1,true) then -- if a_has_tex then
1005                ctx_expandreferencearguments(i,arguments)
1006            end
1007        end
1008    end
1009end
1010
1011implement {
1012    name      = "expandcurrentreference",
1013    actions   = references.expandcurrent
1014}
1015
1016implement {
1017    name      = "setreferenceoperation",
1018    actions   = setreferenceoperation,
1019    arguments = { "integer", "string" }
1020}
1021
1022implement {
1023    name      = "setreferencearguments",
1024    actions   = setreferencearguments,
1025    arguments = { "integer", "string" }
1026}
1027
1028local externals = { }
1029
1030-- we have prefixes but also components:
1031--
1032-- :    prefix
1033-- ::   always external
1034-- :::  internal (for products) or external (for components)
1035
1036local function loadexternalreferences(name,utilitydata)
1037    local struc = utilitydata.structures
1038    if struc then
1039        local external = struc.references.collected -- direct references
1040        local lists    = struc.lists.collected      -- indirect references (derived)
1041        local pages    = struc.pages.collected      -- pagenumber data
1042        local sections = struc.sections.collected
1043        -- a bit weird one, as we don't have the externals in the collected
1044        for prefix, set in next, external do
1045            if prefix == "" then
1046                prefix = name -- this can clash!
1047            end
1048            for reference, data in next, set do
1049                if trace_importing then
1050                    report_importing("registering %a reference, kind %a, name %a, prefix %a, reference %a",
1051                        "external","regular",name,prefix,reference)
1052                end
1053                local section  = reference.section
1054                local realpage = reference.realpage
1055                if section then
1056                    reference.sectiondata = lists[section]
1057                end
1058                if realpage then
1059                    reference.pagedata = pages[realpage]
1060                end
1061            end
1062        end
1063        for i=1,#lists do
1064            local entry      = lists[i]
1065            local metadata   = entry.metadata
1066            local references = entry.references
1067            if metadata and references then
1068                local reference = references.reference
1069                if reference and reference ~= "" then
1070                    local kind     = metadata.kind
1071                    local realpage = references.realpage
1072                    if kind and realpage then
1073                        references.pagedata = pages[realpage]
1074                        local prefix = references.prefix or ""
1075                        if prefix == "" then
1076                            prefix = name -- this can clash!
1077                        end
1078                        local section = references.section
1079                        if section then
1080                            -- we have to make sure that the right section is used, see helpers.prefix
1081                            if sections then
1082                                references.sectiondata = sections[section]
1083                            else
1084                                -- warning
1085                            end
1086                        end
1087                        local target = external[prefix]
1088                        if not target then
1089                            target = { }
1090                            external[prefix] = target
1091                        end
1092                     -- for s in gmatch(reference,"%s*([^,]+)") do
1093                     --     if trace_importing then
1094                     --         report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1095                     --             "external",kind,name,prefix,s)
1096                     --     end
1097                     --     target[s] = target[s] or entry
1098                     -- end
1099                        local function action(s)
1100                            if trace_importing then
1101                                report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1102                                    "external",kind,name,prefix,s)
1103                            end
1104                            target[s] = target[s] or entry
1105                        end
1106                        process_settings(reference,action)
1107                    end
1108                end
1109            end
1110        end
1111        externals[name] = external
1112        return external
1113    end
1114end
1115
1116local externalfiles = { }
1117
1118setmetatableindex(externalfiles, function(t,k)
1119    local v = filedata[k]
1120    if not v then
1121        v = { k, k }
1122    end
1123    externalfiles[k] = v
1124    return v
1125end)
1126
1127setmetatableindex(externals, function(t,k) -- either or not automatically
1128    local filename = externalfiles[k][1] -- filename
1129    local fullname = file.replacesuffix(filename,"tuc")
1130    if lfs.isfile(fullname) then -- todo: use other locator
1131        local utilitydata = job.loadother(fullname)
1132        if utilitydata then
1133            local external = loadexternalreferences(k,utilitydata)
1134            t[k] = external or false
1135            return external
1136        end
1137    end
1138    t[k] = false
1139    return false
1140end)
1141
1142local productdata = allocate {
1143    productreferences   = { },
1144    componentreferences = { },
1145    components          = { },
1146}
1147
1148references.productdata = productdata
1149
1150local function loadproductreferences(productname,componentname,utilitydata)
1151    local struc = utilitydata.structures
1152    if struc then
1153        local productreferences = struc.references.collected -- direct references
1154        local lists = struc.lists.collected      -- indirect references (derived)
1155        local pages = struc.pages.collected      -- pagenumber data
1156        -- we use indirect tables to save room but as they are eventually
1157        -- just references we resolve them to data here (the mechanisms
1158        -- that use this data check for indirectness)
1159        for prefix, set in next, productreferences do
1160            for reference, data in next, set do
1161                if trace_importing then
1162                    report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1163                        "product","regular",productname,prefix,reference)
1164                end
1165                local section  = reference.section
1166                local realpage = reference.realpage
1167                if section then
1168                    reference.sectiondata = lists[section]
1169                end
1170                if realpage then
1171                    reference.pagedata = pages[realpage]
1172                end
1173            end
1174        end
1175        --
1176        local componentreferences = { }
1177        for i=1,#lists do
1178            local entry      = lists[i]
1179            local metadata   = entry.metadata
1180            local references = entry.references
1181            if metadata and references then
1182                local reference = references.reference
1183                if reference and reference ~= "" then
1184                    local kind     = metadata.kind
1185                    local realpage = references.realpage
1186                    if kind and realpage then
1187                        references.pagedata = pages[realpage]
1188                        local prefix    = references.prefix or ""
1189                        local component = references.component
1190                        local ctarget, ptarget
1191                        if not component or component == componentname then
1192                            -- skip
1193                        else
1194                            -- one level up
1195                            local external = componentreferences[component]
1196                            if not external then
1197                                external = { }
1198                                componentreferences[component] = external
1199                            end
1200                            if component == prefix then
1201                                prefix = ""
1202                            end
1203                            ctarget = external[prefix]
1204                            if not ctarget then
1205                                ctarget = { }
1206                                external[prefix] = ctarget
1207                            end
1208                        end
1209                        ptarget = productreferences[prefix]
1210                        if not ptarget then
1211                            ptarget = { }
1212                            productreferences[prefix] = ptarget
1213                        end
1214                        local function action(s)
1215                            if ptarget then
1216                                if trace_importing then
1217                                    report_importing("registering %s reference, kind %a, name %a, prefix %a, reference %a",
1218                                        "product",kind,productname,prefix,s)
1219                                end
1220                                ptarget[s] = ptarget[s] or entry
1221                            end
1222                            if ctarget then
1223                                if trace_importing then
1224                                    report_importing("registering %s reference, kind %a, name %a, prefix %a, referenc %a",
1225                                        "component",kind,productname,prefix,s)
1226                                end
1227                                ctarget[s] = ctarget[s] or entry
1228                            end
1229                        end
1230                        process_settings(reference,action)
1231                    end
1232                end
1233            end
1234        end
1235        productdata.productreferences   = productreferences -- not yet used
1236        productdata.componentreferences = componentreferences
1237    end
1238end
1239
1240local function loadproductvariables(product,component,utilitydata)
1241    local struc = utilitydata.structures
1242    if struc then
1243        local lists = struc.lists and struc.lists.collected
1244        if lists then
1245            local pages = struc.pages and struc.pages.collected
1246            for i=1,#lists do
1247                local li = lists[i]
1248                if li.metadata.kind == "section" and li.references.component == component then
1249                    local firstsection = li
1250                    if firstsection.numberdata then
1251                        local numbers = firstsection.numberdata.numbers
1252                        if numbers then
1253                            if trace_importing then
1254                                report_importing("initializing section number to %:t",numbers)
1255                            end
1256                            productdata.firstsection = firstsection
1257                            structures.documents.preset(numbers)
1258                        end
1259                    end
1260                    if pages and firstsection.references then
1261                        local firstpage = pages[firstsection.references.realpage]
1262                        local number = firstpage and firstpage.number
1263                        if number then
1264                            if trace_importing then
1265                                report_importing("initializing page number to %a",number)
1266                            end
1267                            productdata.firstpage = firstpage
1268                            counters.set("userpage",1,number)
1269                        end
1270                    end
1271                    break
1272                end
1273            end
1274        end
1275    end
1276end
1277
1278local function componentlist(tree,target)
1279    local branches = tree and tree.branches
1280    if branches then
1281        for i=1,#branches do
1282            local branch = branches[i]
1283            local type = branch.type
1284            if type == "component" then
1285                if target then
1286                    target[#target+1] = branch.name
1287                else
1288                    target = { branch.name }
1289                end
1290            elseif type == "product" or type == "component" then
1291                target = componentlist(branch,target)
1292            end
1293        end
1294    end
1295    return target
1296end
1297
1298local function loadproductcomponents(product,component,utilitydata)
1299    local job = utilitydata.job
1300    productdata.components = componentlist(job and job.structure and job.structure.collected) or { }
1301end
1302
1303references.registerinitializer(function(tobesaved,collected)
1304    -- not that much related to tobesaved or collected
1305    productdata.components = componentlist(job.structure.collected) or { }
1306end)
1307
1308function references.loadpresets(product,component) -- we can consider a special components hash
1309    if product and component and product~= "" and component ~= "" and not productdata.product then -- maybe: productdata.filename ~= filename
1310        productdata.product = product
1311        productdata.component = component
1312        local fullname = file.replacesuffix(product,"tuc")
1313        if lfs.isfile(fullname) then -- todo: use other locator
1314            local utilitydata = job.loadother(fullname)
1315            if utilitydata then
1316                if trace_importing then
1317                    report_importing("loading references for component %a of product %a from %a",component,product,fullname)
1318                end
1319                loadproductvariables (product,component,utilitydata)
1320                loadproductreferences(product,component,utilitydata)
1321                loadproductcomponents(product,component,utilitydata)
1322            end
1323        end
1324    end
1325end
1326
1327references.productdata = productdata
1328
1329local useproduct = commands.useproduct
1330
1331if useproduct then
1332
1333    local function newuseproduct(product)
1334        useproduct(product)
1335        if texconditionals.autocrossfilereferences then
1336            local component = justacomponent()
1337            if component then
1338                if trace_referencing or trace_importing then
1339                    report_references("loading presets for component %a of product %a",component,product)
1340                end
1341                references.loadpresets(product,component)
1342            end
1343        end
1344    end
1345
1346    implement {
1347        name      = "useproduct",
1348        actions   = newuseproduct,
1349        arguments = "string",
1350        overload  = true,
1351    }
1352
1353end
1354
1355-- productdata.firstsection.numberdata.numbers
1356-- productdata.firstpage.number
1357
1358local function report_identify_special(set,var,i,type)
1359    local reference = set.reference
1360    local prefix    = set.prefix or ""
1361    local special   = var.special
1362    local error     = var.error
1363    local kind      = var.kind
1364    if error then
1365        report_identifying("type %a, reference %a, index %a, prefix %a, special %a, error %a",type,reference,i,prefix,special,error)
1366    else
1367        report_identifying("type %a, reference %a, index %a, prefix %a, special %a, kind %a",type,reference,i,prefix,special,kind)
1368    end
1369end
1370
1371local function report_identify_arguments(set,var,i,type)
1372    local reference = set.reference
1373    local prefix    = set.prefix or ""
1374    local arguments = var.arguments
1375    local error     = var.error
1376    local kind      = var.kind
1377    if error then
1378        report_identifying("type %a, reference %a, index %a, prefix %a, arguments %a, error %a",type,reference,i,prefix,arguments,error)
1379    else
1380        report_identifying("type %a, reference %a, index %a, prefix %a, arguments %a, kind %a",type,reference,i,prefix,arguments,kind)
1381    end
1382end
1383
1384local function report_identify_outer(set,var,i,type)
1385    local reference = set.reference
1386    local prefix    = set.prefix or ""
1387    local outer     = var.outer
1388    local error     = var.error
1389    local kind      = var.kind
1390    if outer then
1391        if error then
1392            report_identifying("type %a, reference %a, index %a, prefix %a, outer %a, error %a",type,reference,i,prefix,outer,error)
1393        else
1394            report_identifying("type %a, reference %a, index %a, prefix %a, outer %a, kind %a",type,reference,i,prefix,outer,kind)
1395        end
1396    else
1397        if error then
1398            report_identifying("type %a, reference %a, index %a, prefix %a, error %a",type,reference,i,prefix,error)
1399        else
1400            report_identifying("type %a, reference %a, index %a, prefix %a, kind %a",type,reference,i,prefix,kind)
1401        end
1402    end
1403end
1404
1405local function identify_special(set,var,i)
1406    local special = var.special
1407    local s = specials[special]
1408    if s then
1409        local outer     = var.outer
1410        local operation = var.operation
1411        local arguments = var.arguments
1412        if outer then
1413            if operation then
1414                -- special(outer::operation)
1415                var.kind = "special outer with operation"
1416            else
1417                -- special()
1418                var.kind = "special outer"
1419            end
1420            var.f = outer
1421        elseif operation then
1422            if arguments then
1423                -- special(operation{argument,argument})
1424                var.kind = "special operation with arguments"
1425            else
1426                -- special(operation)
1427                var.kind = "special operation"
1428            end
1429        else
1430            -- special()
1431            var.kind = "special"
1432        end
1433        if trace_identifying then
1434            report_identify_special(set,var,i,"1a")
1435        end
1436    else
1437        var.error = "unknown special"
1438    end
1439    return var
1440end
1441
1442local function identify_arguments(set,var,i)
1443    local s = specials[var.inner]
1444    if s then
1445        -- inner{argument}
1446        var.kind = "special operation with arguments"
1447    else
1448        var.error = "unknown inner or special"
1449    end
1450    if trace_identifying then
1451        report_identify_arguments(set,var,i,"3a")
1452    end
1453    return var
1454end
1455
1456-- needs checking: if we don't do too much (redundant) checking now
1457-- inner ... we could move the prefix logic into the parser so that we have 'm for each entry
1458-- foo:bar -> foo == prefix (first we try the global one)
1459-- -:bar   -> ignore prefix
1460
1461local function finish_inner(var,p,i)
1462    var.kind = "inner"
1463    var.i = i
1464    var.p = p
1465    var.r = (i.references and i.references.realpage) or (i.pagedata and i.pagedata.realpage) or 1
1466    return var
1467end
1468
1469local function identify_inner(set,var,prefix,collected,derived)
1470    local inner = var.inner
1471    -- the next test is a safeguard when references are auto loaded from outer
1472    if not inner or inner == "" then
1473        return false
1474    end
1475    local splitprefix, splitinner = lpegmatch(prefixsplitter,inner)
1476    if splitprefix and splitinner then
1477        -- we check for a prefix:reference instance in the regular set of collected
1478        -- references; a special case is -: which forces a lookup in the global list
1479        if splitprefix == "-" then
1480            local i = collected[""]
1481            if i then
1482                i = i[splitinner]
1483                if i then
1484                    return finish_inner(var,"",i)
1485                end
1486            end
1487        end
1488        local i = collected[splitprefix]
1489        if i then
1490            i = i[splitinner]
1491            if i then
1492                return finish_inner(var,splitprefix,i)
1493            end
1494        end
1495        if derived then
1496            -- next we look for a reference in the regular set of collected references
1497            -- using the prefix that is active at this moment (so we overload the given
1498            -- these are taken from other data structures (like lists)
1499            if splitprefix == "-" then
1500                local i = derived[""]
1501                if i then
1502                    i = i[splitinner]
1503                    if i then
1504                        return finish_inner(var,"",i)
1505                    end
1506                end
1507            end
1508            local i = derived[splitprefix]
1509            if i then
1510                i = i[splitinner]
1511                if i then
1512                    return finish_inner(var,splitprefix,i)
1513                end
1514            end
1515        end
1516    end
1517    -- we now ignore the split prefix and treat the whole inner as a potential
1518    -- reference into the global list
1519    local i = collected[prefix]
1520    if i then
1521        i = i[inner]
1522        if i then
1523            return finish_inner(var,prefix,i)
1524        end
1525    end
1526    if not i and derived then
1527        -- and if not found we look in the derived references
1528        local i = derived[prefix]
1529        if i then
1530            i = i[inner]
1531            if i then
1532                return finish_inner(var,prefix,i)
1533            end
1534        end
1535    end
1536    return false
1537end
1538
1539local function unprefixed_inner(set,var,prefix,collected,derived,tobesaved)
1540    local inner = var.inner
1541    local s = specials[inner]
1542    if s then
1543        var.kind = "special"
1544    else
1545        local i = (collected and collected[""] and collected[""][inner]) or
1546                  (derived   and derived  [""] and derived  [""][inner]) or
1547                  (tobesaved and tobesaved[""] and tobesaved[""][inner])
1548        if i then
1549            var.kind = "inner"
1550            var.p    = ""
1551            var.i    = i
1552            var.r    = (i.references and i.references.realpage) or (i.pagedata and i.pagedata.realpage) or 1
1553        else
1554            var.error = "unknown inner or special"
1555        end
1556    end
1557    return var
1558end
1559
1560local function identify_outer(set,var,i)
1561    local outer    = var.outer
1562    local inner    = var.inner
1563    local external = externals[outer]
1564    if external then
1565        local v = identify_inner(set,var,"",external)
1566        if v then
1567            v.kind = "outer with inner"
1568            set.external = true
1569            if trace_identifying then
1570                report_identify_outer(set,v,i,"2a")
1571            end
1572            return v
1573        end
1574-- weird too (we really need to check how this table is build
1575        local v = identify_inner(set,var,var.outer,external)
1576        if v then
1577            v.kind = "outer with inner"
1578            set.external = true
1579            if trace_identifying then
1580                report_identify_outer(set,v,i,"2c")
1581            end
1582            return v
1583        end
1584--
1585        -- somewhat rubish: we use outer as first step in the externals table so it makes no
1586        -- sense to have it as prefix so the next could be an option
1587        local external = external[""]
1588        if external then
1589            local v = identify_inner(set,var,var.outer,external)
1590            if v then
1591                v.kind = "outer with inner"
1592                set.external = true
1593                if trace_identifying then
1594                    report_identify_outer(set,v,i,"2b")
1595                end
1596                return v
1597            end
1598        end
1599    end
1600    local external = productdata.componentreferences[outer]
1601    if external then
1602        local v = identify_inner(set,var,"",external)
1603        if v then
1604            v.kind = "outer with inner"
1605            set.external = true
1606            if trace_identifying then
1607                report_identify_outer(set,v,i,"2c")
1608            end
1609            return v
1610        end
1611    end
1612    local external = productdata.productreferences[outer]
1613    if external then
1614        local vi = external[inner]
1615        if vi then
1616            var.kind = "outer with inner"
1617            var.i = vi
1618            set.external = true
1619            if trace_identifying then
1620                report_identify_outer(set,var,i,"2d")
1621            end
1622            return var
1623        end
1624    end
1625    -- the rest
1626    local special   = var.special
1627    local arguments = var.arguments
1628    local operation = var.operation
1629    if inner then
1630        -- tricky: in this case we can only use views when we're sure that all inners
1631        -- are flushed in the outer document so that should become an option
1632        if arguments then
1633            -- outer::inner{argument}
1634            var.kind = "outer with inner with arguments"
1635        else
1636            -- outer::inner
1637            var.kind = "outer with inner"
1638        end
1639        var.i = inner
1640        var.f = outer
1641        if type(inner) == "table" then
1642            -- can this really happen?
1643            var.r = (inner.references and inner.references.realpage) or (inner.pagedata and inner.pagedata.realpage) or 1
1644        else
1645            var.r = 1
1646        end
1647        if trace_identifying then
1648            report_identify_outer(set,var,i,"2e")
1649        end
1650    elseif special then
1651        local s = specials[special]
1652        if s then
1653            if operation then
1654                if arguments then
1655                    -- outer::special(operation{argument,argument})
1656                    var.kind = "outer with special and operation and arguments"
1657                else
1658                    -- outer::special(operation)
1659                    var.kind = "outer with special and operation"
1660                end
1661            else
1662                -- outer::special()
1663                var.kind = "outer with special"
1664            end
1665            var.f = outer
1666        else
1667            var.error = "unknown outer with special"
1668        end
1669        if trace_identifying then
1670            report_identify_outer(set,var,i,"2f")
1671        end
1672    else
1673        -- outer::
1674        var.kind = "outer"
1675        var.f = outer
1676        if trace_identifying then
1677            report_identify_outer(set,var,i,"2g")
1678        end
1679    end
1680    return var
1681end
1682
1683-- todo: avoid copy
1684
1685local function identify_inner_or_outer(set,var,i)
1686    -- here we fall back on product data
1687    local inner = var.inner
1688    if inner and inner ~= "" then
1689
1690        -- first we look up in collected and derived using the current prefix
1691
1692        local prefix = set.prefix
1693
1694        local v = identify_inner(set,var,set.prefix,collected,derived)
1695        if v then
1696            if trace_identifying then
1697                report_identify_outer(set,v,i,"4a")
1698            end
1699            return v
1700        end
1701
1702        -- nest we look at each component (but we can omit the already consulted one
1703
1704        local jobstructure = job.structure
1705        local components   = jobstructure and jobstructure.components
1706        if components then
1707            for c=1,#components do
1708                local component = components[c]
1709                if component ~= prefix then
1710                    local v = identify_inner(set,var,component,collected,derived)
1711                    if v then
1712                        if trace_identifying then
1713                            report_identify_outer(set,var,i,"4b")
1714                        end
1715                        return v
1716                    end
1717                end
1718            end
1719        end
1720
1721        -- as a last resort we will consult the global lists
1722
1723        local v = unprefixed_inner(set,var,"",collected,derived,tobesaved)
1724        if v then
1725            if trace_identifying then
1726                report_identify_outer(set,v,i,"4c")
1727            end
1728            return v
1729        end
1730
1731        -- not it gets bad ... we need to look in external files ... keep in mind that
1732        -- we can best use explicit references for this ... we might issue a warning
1733
1734        local componentreferences = productdata.componentreferences
1735        local productreferences = productdata.productreferences
1736        local components = productdata.components
1737        if components and componentreferences then
1738            for c=1,#components do
1739                local component = components[c]
1740                local data = componentreferences[component]
1741                if data then
1742                    local d = data[""]
1743                    local vi = d and d[inner]
1744                    if vi then
1745                        var.outer = component
1746                        var.i = vi
1747                        var.kind = "outer with inner"
1748                        set.external = true
1749                        if trace_identifying then
1750                            report_identify_outer(set,var,i,"4d")
1751                        end
1752                        return var
1753                    end
1754                end
1755            end
1756        end
1757        local component, inner = lpegmatch(componentsplitter,inner)
1758        if component then
1759            local data = componentreferences and componentreferences[component]
1760            if data then
1761                local d = data[""]
1762                local vi = d and d[inner]
1763                if vi then
1764                    var.inner = inner
1765                    var.outer = component
1766                    var.i = vi
1767                    var.kind = "outer with inner"
1768                    set.external = true
1769                    if trace_identifying then
1770                        report_identify_outer(set,var,i,"4e")
1771                    end
1772                    return var
1773                end
1774            end
1775            local data = productreferences and productreferences[component]
1776            if data then
1777                local vi = data[inner]
1778                if vi then
1779                    var.inner = inner
1780                    var.outer = component
1781                    var.i = vi
1782                    var.kind = "outer with inner"
1783                    set.external = true
1784                    if trace_identifying then
1785                        report_identify_outer(set,var,i,"4f")
1786                    end
1787                    return var
1788                end
1789            end
1790        end
1791        var.error = "unknown inner"
1792    else
1793        var.error = "no inner"
1794    end
1795    if trace_identifying then
1796        report_identify_outer(set,var,i,"4g")
1797    end
1798    return var
1799end
1800
1801local function identify_inner_component(set,var,i)
1802    -- we're in a product (maybe ignore when same as component)
1803    local component = var.component
1804    local v = identify_inner(set,var,component,collected,derived)
1805    if not v then
1806        var.error = "unknown inner in component"
1807    end
1808    if trace_identifying then
1809        report_identify_outer(set,var,i,"5a")
1810    end
1811    return var
1812end
1813
1814local function identify_outer_component(set,var,i)
1815    local component = var.component
1816    local inner = var.inner
1817    local data = productdata.componentreferences[component]
1818    if data then
1819        local d = data[""]
1820        local vi = d and d[inner]
1821        if vi then
1822            var.inner = inner
1823            var.outer = component
1824            var.i = vi
1825            var.kind = "outer with inner"
1826            set.external = true
1827            if trace_identifying then
1828                report_identify_outer(set,var,i,"6a")
1829            end
1830            return var
1831        end
1832    end
1833    local data = productdata.productreferences[component]
1834    if data then
1835        local vi = data[inner]
1836        if vi then
1837            var.inner = inner
1838            var.outer = component
1839            var.i = vi
1840            var.kind = "outer with inner"
1841            set.external = true
1842            if trace_identifying then
1843                report_identify_outer(set,var,i,"6b")
1844            end
1845            return var
1846        end
1847    end
1848    var.error = "unknown component"
1849    if trace_identifying then
1850        report_identify_outer(set,var,i,"6c")
1851    end
1852    return var
1853end
1854
1855local nofidentified = 0
1856
1857local function identify(prefix,reference)
1858    if not reference then
1859        prefix, reference = "", prefix
1860    end
1861    local set = resolve(prefix,reference)
1862    local bug = false
1863    texsetcount("referencehastexstate",set.has_tex and 1 or 0)
1864    nofidentified = nofidentified + 1
1865    set.n = nofidentified
1866    for i=1,#set do
1867        local var = set[i]
1868        local spe = var.special
1869        local fnc = functions[spe]
1870        if fnc then
1871            var = fnc(var) or { error = "invalid special function" }
1872        elseif spe then
1873            var = identify_special(set,var,i)
1874        elseif var.outer then
1875            var = identify_outer(set,var,i)
1876        elseif var.arguments then
1877            var = identify_arguments(set,var,i)
1878        elseif not var.component then
1879            var = identify_inner_or_outer(set,var,i)
1880        elseif productcomponent() then
1881            var = identify_inner_component(set,var,i)
1882        else
1883            var = identify_outer_component(set,var,i)
1884        end
1885        set[i] = var
1886        bug = bug or var.error
1887    end
1888    references.currentset = mark(set) -- mark, else in api doc
1889    if trace_analyzing then
1890        report_references(table.serialize(set,reference))
1891    end
1892    return set, bug
1893end
1894
1895references.identify = identify
1896
1897local unknowns, nofunknowns, f_valid = { }, 0, formatters["[%s][%s]"]
1898
1899function references.valid(prefix,reference,specification)
1900    local set, bug = identify(prefix,reference)
1901    local unknown = bug or #set == 0
1902    if unknown then
1903        currentreference = nil -- will go away
1904        local str = f_valid(prefix,reference)
1905        local u = unknowns[str]
1906        if not u then
1907            if somefound then
1908                interfaces.showmessage("references",1,str) -- 1 = unknown, 4 = illegal
1909            end
1910            unknowns[str] = 1
1911            nofunknowns = nofunknowns + 1
1912        else
1913            unknowns[str] = u + 1
1914        end
1915    else
1916        set.highlight    = specification.highlight
1917        set.newwindow    = specification.newwindow
1918        set.layer        = specification.layer
1919        currentreference = set[1]
1920    end
1921    -- we can do the expansion here which saves a call
1922    return not unknown
1923end
1924
1925implement {
1926    name      = "doifelsereference",
1927    actions   = { references.valid, commands.doifelse },
1928    arguments = {
1929        "string",
1930        "string",
1931        {
1932            { "highlight", "boolean" },
1933            { "newwindow", "boolean" },
1934            { "layer" },
1935        }
1936    }
1937}
1938
1939logs.registerfinalactions(function()
1940    if nofunknowns > 0 then
1941        statistics.register("cross referencing", function()
1942            return format("%s identified, %s unknown",nofidentified,nofunknowns)
1943        end)
1944        local sortedhash = table.sortedhash
1945        logs.startfilelogging(report,"missing references")
1946        for k, v in table.sortedhash(unknowns) do
1947            report("%4i  %s",v,k)
1948        end
1949        logs.stopfilelogging()
1950        if logs.loggingerrors() then
1951            logs.starterrorlogging(report,"missing references")
1952            for k, v in table.sortedhash(unknowns) do
1953                report("%4i  %s",v,k)
1954            end
1955            logs.stoperrorlogging()
1956        end
1957    end
1958end)
1959
1960-- The auto method will try to avoid named internals in a clever way which
1961-- can make files smaller without sacrificing external references. Some of
1962-- the housekeeping happens the backend side.
1963
1964local innermethod        = v_auto       -- only page|auto now
1965local outermethod        = v_auto       -- only page|auto now
1966local defaultinnermethod = defaultinnermethod
1967local defaultoutermethod = defaultoutermethod
1968references.innermethod   = innermethod  -- don't mess with this one directly
1969references.outermethod   = outermethod  -- don't mess with this one directly
1970
1971function references.setlinkmethod(inner,outer)
1972    if not outer and type(inner) == "string" then
1973        local m = settings_to_array(inner)
1974        inner = m[1]
1975        outer = m[2] or v_auto
1976    end
1977    if toboolean(inner) or inner == v_page or inner == v_yes then
1978        innermethod = v_page
1979    elseif inner == v_name then
1980        innermethod = v_name
1981    else
1982        innermethod = v_auto
1983    end
1984    if toboolean(outer) or outer == v_page or outer == v_yes then
1985        outermethod = v_page
1986    elseif inner == v_name then
1987        outermethod = v_name
1988    else
1989        outermethod = v_auto
1990    end
1991    references.innermethod = innermethod
1992    references.outermethod = outermethod
1993    function references.setlinkmethod()
1994        report_references("link method is already set and frozen: inner %a, outer %a",innermethod,outermethod)
1995    end
1996end
1997
1998implement {
1999    name      = "setreferencelinkmethod",
2000    actions   = references.setlinkmethod,
2001    arguments = "string",
2002 -- onlyonce  = true
2003}
2004
2005function references.getinnermethod()
2006    return innermethod or defaultinnermethod
2007end
2008
2009function references.getoutermethod()
2010    return outermethod or defaultoutermethod
2011end
2012
2013directives.register("references.linkmethod", function(v) -- page auto
2014    references.setlinkmethod(v)
2015end)
2016
2017-- we can call setinternalreference with an already known internal or with
2018-- a reference/prefix specification
2019
2020local destinationattributes = { }
2021
2022local function setinternalreference(specification)
2023    local internal    = specification.internal
2024    local destination = unsetvalue
2025    if innermethod == v_auto or innermethod == v_name then
2026        local t         = { } -- maybe add to current (now only used for tracing)
2027        local tn        = 0
2028        local reference = specification.reference
2029        local view      = specification.view
2030        if reference then
2031            local prefix = specification.prefix
2032            if prefix and prefix ~= "" then
2033                local prefix = prefix .. ":" -- watch out, : here
2034                local function action(ref)
2035                    tn = tn + 1
2036                    t[tn] = prefix .. ref
2037                end
2038                process_settings(reference,action)
2039            else
2040                local function action(ref)
2041                    tn = tn + 1
2042                    t[tn] = ref
2043                end
2044                process_settings(reference,action)
2045            end
2046        end
2047        -- ugly .. later we decide to ignore it when we have a real one
2048        -- but for testing we might want to see them all
2049        if internal then
2050            if innermethod ~= v_name then -- innermethod == v_auto
2051             -- we don't want too many #1 #2 #3 etc
2052                tn = tn + 1
2053                t[tn] = internal -- when number it's internal
2054            end
2055            if not view then
2056                local i = references.internals[internal]
2057                if i then
2058                    view = i.references.view
2059                end
2060            end
2061        end
2062        destination = references.mark(t,nil,nil,view) -- returns an attribute
2063    end
2064    if internal then -- new
2065        destinationattributes[internal] = destination
2066    end
2067    texsetcount("lastdestinationattribute",destination)
2068    return destination
2069end
2070
2071local function getinternalreference(internal)
2072    return destinationattributes[internal] or 0
2073end
2074
2075references.setinternalreference = setinternalreference
2076references.getinternalreference = getinternalreference
2077
2078implement {
2079    name      = "setinternalreference",
2080    actions   = setinternalreference,
2081    arguments = {
2082        {
2083            { "prefix" },
2084            { "reference" },
2085            { "internal", "integer" },
2086            { "view" }
2087        }
2088    }
2089}
2090
2091-- implement {
2092--     name      = "getinternalreference",
2093--     actions   = { getinternalreference, context },
2094--     arguments = "integer",
2095-- }
2096
2097function references.setandgetattribute(data) -- maybe do internal automatically here
2098    local attr = unsetvalue
2099    local mdat = data.metadata
2100    local rdat = data.references
2101    if mdat and rdat then
2102        if not rdat.section then
2103            rdat.section = structures.sections.currentid()
2104        end
2105        local ndat = data.numberdata
2106        if ndat then
2107            local numbers = ndat.numbers
2108            if type(numbers) == "string" then
2109                counters.compact(ndat,numbers)
2110            end
2111            data.numberdata = helpers.simplify(ndat)
2112        end
2113        local pdat = data.prefixdata
2114        if pdat then
2115            data.prefixdata = helpers.simplify(pdat)
2116        end
2117        local udat = data.userdata
2118        if type(udat) == "string"  then
2119            data.userdata = helpers.touserdata(udat)
2120        end
2121        if not rdat.block then
2122            rdat.block = structures.sections.currentblock()
2123        end
2124        local done = references.set(data) -- we had kind i.e .item -> full
2125        if done then
2126            attr = setinternalreference {
2127                prefix    = rdat.prefix,
2128                reference = rdat.reference,
2129                internal  = rdat.internal,
2130                view      = rdat.view
2131            } or unsetvalue
2132        end
2133    end
2134    texsetcount("lastdestinationattribute",attr)
2135    return attr
2136end
2137
2138implement {
2139    name      = "setdestinationattribute",
2140    actions   = references.setandgetattribute,
2141    arguments = {
2142        {
2143            {
2144                "references", {
2145                    { "internal", "integer" },
2146                    { "block" },
2147                    { "view" },
2148                    { "prefix" },
2149                    { "reference" },
2150                },
2151            },
2152            {
2153                "metadata", {
2154                    { "kind" },
2155                    { "xmlroot" },
2156                    { "catcodes", "integer" },
2157                },
2158            },
2159            {
2160                "prefixdata", { "*" }
2161            },
2162            {
2163                "numberdata", { "*" }
2164            },
2165            {
2166                "entries", { "*" }
2167            },
2168            {
2169                "userdata"
2170            }
2171        }
2172    }
2173}
2174
2175function references.getinternallistreference(n) -- n points into list (todo: registers)
2176    local l = lists.collected[n]
2177    local i = l and l.references.internal
2178    return i and destinationattributes[i] or 0
2179end
2180
2181function references.getinternalcachedlistreference(n) -- n points into list (todo: registers)
2182    local l = lists.cached[n]
2183    local i = l and l.references.internal
2184    return i and destinationattributes[i] or 0
2185end
2186
2187implement {
2188    name      = "getinternallistreference",
2189    actions   = { references.getinternallistreference, context },
2190    arguments = "integer"
2191}
2192
2193implement {
2194    name      = "getinternalcachedlistreference",
2195    actions   = { references.getinternalcachedlistreference, context },
2196    arguments = "integer"
2197}
2198
2199
2200--
2201
2202function references.getcurrentmetadata(tag)
2203    local data = currentreference and currentreference.i
2204    return data and data.metadata and data.metadata[tag]
2205end
2206
2207implement {
2208    name      = "getcurrentreferencemetadata",
2209    actions   = { references.getcurrentmetadata, context },
2210    arguments = "string",
2211}
2212
2213local function currentmetadata(tag)
2214    local data = currentreference and currentreference.i
2215    return data and data.metadata and data.metadata[tag]
2216end
2217
2218references.currentmetadata = currentmetadata
2219
2220local function getcurrentprefixspec(default)
2221    local data     = currentreference and currentreference.i
2222    local metadata = data and data.metadata
2223    return
2224        metadata and metadata.kind or "?",
2225        metadata and metadata.name or "?",
2226        default                    or "?"
2227end
2228
2229references.getcurrentprefixspec = getcurrentprefixspec
2230
2231-- implement {
2232--     name      = "getcurrentprefixspec",
2233--     actions   = { getcurrentprefixspec, context }, -- returns 3 arguments
2234--     arguments = "string",
2235-- }
2236
2237implement {
2238    name      = "getcurrentprefixspec",
2239    actions   = function(tag)
2240        context("{%s}{%s}{%s}",getcurrentprefixspec(tag))
2241    end,
2242    arguments = "string",
2243}
2244
2245local genericfilters = { }
2246local userfilters    = { }
2247local textfilters    = { }
2248local fullfilters    = { }
2249local sectionfilters = { }
2250
2251filters.generic = genericfilters
2252filters.user    = userfilters
2253filters.text    = textfilters
2254filters.full    = fullfilters
2255filters.section = sectionfilters
2256
2257local function filterreference(name,prefixspec,numberspec) -- number page title ...
2258    local data = currentreference and currentreference.i -- maybe we should take realpage from here
2259    if data then
2260        if name == "realpage" then
2261            local cs = references.analyze() -- normally already analyzed but also sets state
2262            context(tonumber(cs.realpage) or 0)
2263        else -- assumes data is table
2264            local kind = type(data) == "table" and data.metadata and data.metadata.kind
2265            if kind then
2266                local filter = filters[kind] or genericfilters
2267                filter = filter and (filter[name] or filter.unknown or genericfilters[name] or genericfilters.unknown)
2268                if filter then
2269                    if trace_referencing then
2270                        report_references("name %a, kind %a, using dedicated filter",name,kind)
2271                    end
2272                    filter(data,name,prefixspec,numberspec)
2273                elseif trace_referencing then
2274                    report_references("name %a, kind %a, using generic filter",name,kind)
2275                end
2276            elseif trace_referencing then
2277                report_references("name %a, unknown kind",name)
2278            end
2279        end
2280    elseif name == "realpage" then
2281        context(0)
2282    elseif trace_referencing then
2283        report_references("name %a, no reference",name)
2284    end
2285end
2286
2287local function filterreferencedefault()
2288    return filterreference("default",getcurrentprefixspec("default"))
2289end
2290
2291references.filter        = filterreference
2292references.filterdefault = filterreferencedefault
2293
2294implement {
2295    name      = "filterreference",
2296    actions   = filterreference,
2297    arguments = "string",
2298}
2299
2300implement {
2301    name      = "filterdefaultreference",
2302    actions   = filterreference,
2303    arguments = {
2304        "string",    -- 'default'
2305        { { "*" } }, -- prefixspec
2306        { { "*" } }, -- numberspec
2307    }
2308}
2309
2310function genericfilters.title(data)
2311    if data then
2312        local titledata = data.titledata or data.useddata
2313        if titledata then
2314            helpers.title(titledata.reference or titledata.title or "?",data.metadata)
2315        end
2316    end
2317end
2318
2319function genericfilters.text(data)
2320    if data then
2321        local entries = data.entries or data.useddata
2322        if entries then
2323            helpers.title(entries.text or "?",data.metadata)
2324        end
2325    end
2326end
2327
2328function genericfilters.number(data,what,prefixspec,numberspec)
2329    if data then
2330        numberdata = lists.reordered(data) -- data.numberdata
2331        if numberdata then
2332            helpers.prefix(data,prefixspec)
2333            sections.typesetnumber(numberdata,"number",numberspec,numberdata)
2334        else
2335            local useddata = data.useddata
2336            if useddata and useddata.number then
2337                context(useddata.number)
2338            end
2339        end
2340    end
2341end
2342
2343genericfilters.default = genericfilters.text
2344
2345function genericfilters.page(data,prefixspec,pagespec)
2346    local pagedata = data.pagedata
2347    if pagedata then
2348        local number     = pagedata.number
2349        local conversion = pagedata.conversion
2350        if not number then
2351            -- error
2352        elseif conversion then
2353            ctx_convertnumber(conversion,number)
2354        else
2355            context(number)
2356        end
2357    else
2358        helpers.prefixpage(data,prefixspec,pagespec)
2359    end
2360end
2361
2362function userfilters.unknown(data,name)
2363    if data then
2364        local userdata = data.userdata
2365        local userkind = userdata and userdata.kind
2366        if userkind then
2367            local filter = filters[userkind] or genericfilters
2368            filter = filter and (filter[name] or filter.unknown)
2369            if filter then
2370                filter(data,name)
2371                return
2372            end
2373        end
2374        local namedata = userdata and userdata[name]
2375        if namedata then
2376            context(namedata)
2377        end
2378    end
2379end
2380
2381function textfilters.title(data)
2382    helpers.title(data.entries.text or "?",data.metadata)
2383end
2384
2385-- no longer considered useful:
2386--
2387-- function filters.text.number(data)
2388--     helpers.title(data.entries.text or "?",data.metadata)
2389-- end
2390
2391function textfilters.page(data,prefixspec,pagespec)
2392    helpers.prefixpage(data,prefixspec,pagespec)
2393end
2394
2395fullfilters.title = textfilters.title
2396fullfilters.page  = textfilters.page
2397
2398function sectionfilters.number(data,what,prefixspec)
2399    if data then
2400        local numberdata = data.numberdata
2401        if not numberdata then
2402            local useddata = data.useddata
2403            if useddata and useddata.number then
2404                context(useddata.number)
2405            end
2406        elseif numberdata.hidenumber then
2407            local references = data.references
2408            if trace_empty then
2409                report_empty("reference %a has a hidden number",references.reference)
2410                ctx_emptyreference() -- maybe an option
2411            end
2412        else
2413            sections.typesetnumber(numberdata,"number",prefixspec,numberdata)
2414        end
2415    end
2416end
2417
2418sectionfilters.title   = genericfilters.title
2419sectionfilters.page    = genericfilters.page
2420sectionfilters.default = sectionfilters.number
2421
2422-- filters.note        = { default = genericfilters.number }
2423-- filters.formula     = { default = genericfilters.number }
2424-- filters.float       = { default = genericfilters.number }
2425-- filters.description = { default = genericfilters.number }
2426-- filters.item        = { default = genericfilters.number }
2427
2428setmetatableindex(filters, function(t,k) -- beware, test with rawget
2429    local v = { default = genericfilters.number } -- not copy as it might be extended differently
2430    t[k] = v
2431    return v
2432end)
2433
2434-- function references.sectiontitle(n)
2435--     helpers.sectiontitle(lists.collected[tonumber(n) or 0])
2436-- end
2437
2438-- function references.sectionnumber(n)
2439--     helpers.sectionnumber(lists.collected[tonumber(n) or 0])
2440-- end
2441
2442-- function references.sectionpage(n,prefixspec,pagespec)
2443--     helpers.prefixedpage(lists.collected[tonumber(n) or 0],prefixspec,pagespec)
2444-- end
2445
2446-- analyze
2447
2448references.testrunners  = references.testrunners  or { }
2449references.testspecials = references.testspecials or { }
2450
2451local runners  = references.testrunners
2452local specials = references.testspecials
2453
2454-- We need to prevent ending up in the 'relative location' analyzer as it is
2455-- pretty slow (progressively). In the pagebody one can best check the reference
2456-- real page to determine if we need contrastlocation as that is more lightweight.
2457
2458local function checkedpagestate(n,page,actions,position,spread)
2459    local p = tonumber(page)
2460    if not p then
2461        return 0
2462    end
2463    if position and #actions > 0 then
2464        local i = actions[1].i -- brrr
2465        if i then
2466            local a = i.references
2467            if a then
2468                local x = a.x
2469                local y = a.y
2470                if x and y then
2471                    local jp = jobpositions.collected[position]
2472                    if jp then
2473                        local px = jp.x
2474                        local py = jp.y
2475                        local pp = jp.p
2476                        if p == pp then
2477                            -- same page
2478                            if py > y then
2479                                return 5 -- above
2480                            elseif py < y then
2481                                return 4 -- below
2482                            elseif px > x then
2483                                return 4 -- below
2484                            elseif px < x then
2485                                return 5 -- above
2486                            else
2487                                return 1 -- same
2488                            end
2489                        elseif spread then
2490                            if pp % 2 == 0 then
2491                                -- left page
2492                                if pp > p then
2493                                    return 2 -- before
2494                                elseif pp + 1 == p then
2495--                                     return 4 -- below (on right page)
2496                                    return 5 -- above (on left page)
2497                                else
2498                                    return 3 -- after
2499                                end
2500                            else
2501                                -- right page
2502                                if pp < p then
2503                                    return 3 -- after
2504                                elseif pp - 1 == p then
2505--                                     return 5 -- above (on left page)
2506                                    return 4 -- below (on right page)
2507                                else
2508                                    return 2 -- before
2509                                end
2510                            end
2511                        elseif pp > p then
2512                            return 2 -- before
2513                        else
2514                            return 3 -- after
2515                        end
2516                    end
2517                end
2518            end
2519        end
2520    end
2521    local r = referredpage(n) -- sort of obsolete
2522    if p > r then
2523        return 3 -- after
2524    elseif p < r then
2525        return 2 -- before
2526    else
2527        return 1 -- same
2528    end
2529end
2530
2531local function setreferencerealpage(actions)
2532    if not actions then
2533        actions = references.currentset
2534    end
2535    if type(actions) == "table" then
2536        local realpage = actions.realpage
2537        if realpage then
2538            return realpage
2539        end
2540        local nofactions = #actions
2541        if nofactions > 0 then
2542            for i=1,nofactions do
2543                local a = actions[i]
2544                local what = runners[a.kind]
2545                if what then
2546                    what = what(a,actions) -- needs documentation
2547                end
2548            end
2549            realpage = actions.realpage
2550            if realpage then
2551                return realpage
2552            end
2553        end
2554        actions.realpage = 0
2555    end
2556    return 0
2557end
2558
2559references.setreferencerealpage = setreferencerealpage
2560
2561-- we store some analysis data alongside the indexed array
2562-- at this moment only the real reference page is analyzed
2563-- normally such an analysis happens in the backend code
2564
2565function references.analyze(actions,position,spread)
2566    if not actions then
2567        actions = references.currentset
2568    end
2569    if not actions then
2570        actions = { realpage = 0, pagestate = 0 }
2571    elseif actions.pagestate then
2572        -- already done
2573    else
2574        local realpage = actions.realpage or setreferencerealpage(actions)
2575        if realpage == 0 then
2576            actions.pagestate = 0
2577        elseif actions.external then
2578            actions.pagestate = 0
2579        else
2580            actions.pagestate = checkedpagestate(actions.n,realpage,actions,position,spread)
2581        end
2582    end
2583    return actions
2584end
2585
2586local function referencepagestate(position,detail,spread)
2587    local actions = references.currentset
2588    if not actions then
2589        return 0
2590    else
2591        local pagestate = actions.pagestate
2592        for i=1,#actions do
2593            local a = actions[i]
2594            if a.outer then
2595                pagestate = 0
2596                actions.pagestate = pagestate
2597                break
2598            end
2599        end
2600        if not pagestate then
2601            references.analyze(actions,position,spread) -- delayed unless explicitly asked for
2602            pagestate = actions.pagestate
2603        end
2604        if detail then
2605            return pagestate
2606        elseif pagestate == 4 then
2607            return 2 -- compatible
2608        elseif pagestate == 5 then
2609            return 3 -- compatible
2610        else
2611            return pagestate
2612        end
2613    end
2614end
2615
2616implement {
2617    name      = "referencepagestate",
2618    actions   = { referencepagestate, context },
2619    arguments = "string"
2620}
2621
2622implement {
2623    name      = "referencepagedetail",
2624    actions   = { referencepagestate, context },
2625    arguments = { "string", "boolean", "boolean" }
2626}
2627
2628-- local function referencerealpage()
2629--     local actions = references.currentset
2630--     return not actions and 0 or actions.realpage or setreferencerealpage(actions)
2631-- end
2632--
2633-- implement {
2634--     name      = "referencerealpage",
2635--     actions   = { referencerealpage, context },
2636--  -- arguments = "string" -- hm, weird
2637-- }
2638
2639implement {
2640    name      = "askedreference",
2641    public    = true,
2642    protected = true,
2643    actions   = function()
2644        local actions = references.currentset
2645        if actions then
2646            context("[p=%s,r=%s]",actions.prefix or "",actions.reference)
2647        end
2648    end
2649}
2650
2651implement {
2652    name    = "referencerealpage",
2653    actions = function()
2654        local actions = references.currentset
2655        context(not actions and 0 or actions.realpage or setreferencerealpage(actions))
2656    end
2657}
2658
2659local function referencepos(key)
2660    local actions = references.currentset
2661    local i = actions[1].i -- brrr
2662    local v = 0
2663    if i then
2664        local a = i.references
2665        if a then
2666            v = a[key] or 0
2667        end
2668    end
2669    return v
2670end
2671
2672implement { name = "referenceposx", actions = function() context("%p",referencepos("x")) end }
2673implement { name = "referenceposy", actions = function() context("%p",referencepos("y")) end }
2674
2675implement {
2676    name    = "referencecolumn",
2677    actions = function()
2678        local actions = references.currentset
2679        local column  = 1
2680        if actions then
2681            column = jobpositions.columnofpos(actions.realpage or setreferencerealpage(actions),referencepos("x"))
2682        end
2683        context(column or 1)
2684    end
2685}
2686
2687local plist, nofrealpages
2688
2689local function realpageofpage(p) -- the last one counts !
2690    if not plist then
2691        local pages = structures.pages.collected
2692        nofrealpages = #pages
2693        plist = { }
2694        for rp=1,nofrealpages do
2695            local page = pages[rp]
2696            if page then
2697                plist[page.number] = rp
2698            end
2699        end
2700        references.nofrealpages = nofrealpages
2701    end
2702    return plist[p]
2703end
2704
2705references.realpageofpage = realpageofpage
2706
2707function references.checkedrealpage(r)
2708    if not plist then
2709        realpageofpage(r) -- just initialize
2710    end
2711    if not r then
2712        return texgetcount("realpageno")
2713    elseif r < 1 then
2714        return 1
2715    elseif r > nofrealpages then
2716        return nofrealpages
2717    else
2718        return r
2719    end
2720end
2721
2722-- use local ?
2723
2724local pages = allocate {
2725    [variables.firstpage]       = function() return counters.record("realpage")["first"]    end,
2726    [variables.previouspage]    = function() return counters.record("realpage")["previous"] end,
2727    [variables.nextpage]        = function() return counters.record("realpage")["next"]     end,
2728    [variables.lastpage]        = function() return counters.record("realpage")["last"]     end,
2729
2730    [variables.firstsubpage]    = function() return counters.record("subpage" )["first"]    end,
2731    [variables.previoussubpage] = function() return counters.record("subpage" )["previous"] end,
2732    [variables.nextsubpage]     = function() return counters.record("subpage" )["next"]     end,
2733    [variables.lastsubpage]     = function() return counters.record("subpage" )["last"]     end,
2734
2735    [variables.forward]         = function() return counters.record("realpage")["forward"]  end,
2736    [variables.backward]        = function() return counters.record("realpage")["backward"] end,
2737}
2738
2739references.pages = pages
2740
2741-- maybe some day i will merge this in the backend code with a testmode (so each
2742-- runner then implements a branch)
2743
2744runners["inner"] = function(var,actions)
2745    local r = var.r
2746    if r then
2747        actions.realpage = r
2748    end
2749end
2750
2751runners["special"] = function(var,actions)
2752    local handler = specials[var.special]
2753    return handler and handler(var,actions)
2754end
2755
2756runners["special operation"]                = runners["special"]
2757runners["special operation with arguments"] = runners["special"]
2758
2759function specials.internal(var,actions)
2760    local v = internals[tonumber(var.operation)]
2761    local r = v and v.references
2762    if r then
2763        local p = r.realpage
2764        if p then
2765-- setmetatableindex(actions,r)
2766            actions.realpage = p
2767            actions.view     = r.view
2768        end
2769    end
2770end
2771
2772specials.i = specials.internal
2773
2774function specials.page(var,actions)
2775    local o = var.operation
2776    local p = pages[o]
2777    if type(p) == "function" then
2778        p = p()
2779    else
2780        p = tonumber(realpageofpage(tonumber(o)))
2781    end
2782    if p then
2783        var.r = p
2784        actions.realpage = actions.realpage or p -- first wins
2785    end
2786end
2787
2788function specials.realpage(var,actions)
2789    local p = tonumber(var.operation)
2790    if p then
2791        var.r = p
2792        actions.realpage = actions.realpage or p -- first wins
2793    end
2794end
2795
2796function specials.userpage(var,actions)
2797    local p = tonumber(realpageofpage(var.operation))
2798    if p then
2799        var.r = p
2800        actions.realpage = actions.realpage or p -- first wins
2801    end
2802end
2803
2804function specials.deltapage(var,actions)
2805    local p = tonumber(var.operation)
2806    if p then
2807        p = references.checkedrealpage(p + texgetcount("realpageno"))
2808        var.r = p
2809        actions.realpage = actions.realpage or p -- first wins
2810    end
2811end
2812
2813function specials.section(var,actions)
2814    local sectionname = var.arguments
2815    local destination = var.operation
2816    local internal    = structures.sections.internalreference(sectionname,destination)
2817    if internal then
2818        var.special   = "internal"
2819        var.operation = internal
2820        var.arguments = nil
2821        specials.internal(var,actions)
2822    end
2823end
2824
2825-- experimental:
2826
2827local p_splitter = lpeg.splitat(":")
2828local p_lower    = lpeg.patterns.utf8lower
2829
2830-- We can cache lowercased titles which saves a lot of time, but then
2831-- we can better have a global cache with weak keys.
2832
2833-- local lowercache = table.setmetatableindex(function(t,k)
2834--     local v = lpegmatch(p_lower,k)
2835--     t[k] = v
2836--     return v
2837-- end)
2838
2839local lowercache = false
2840
2841local function locate(list,askedkind,askedname,pattern)
2842    local kinds = lists.kinds
2843    local names = lists.names
2844    if askedkind and not kinds[askedkind] then
2845        return false
2846    end
2847    if askedname and not names[askedname] then
2848        return false
2849    end
2850    for i=1,#list do
2851        local entry    = list[i]
2852        local metadata = entry.metadata
2853        if metadata then
2854            local found = false
2855            if askedname then
2856                local name = metadata.name
2857                if name then
2858                    found = name == askedname
2859                end
2860            elseif askedkind then
2861                local kind = metadata.kind
2862                if kind then
2863                    found = kind == askedkind
2864                end
2865            end
2866            if found then
2867                local titledata = entry.titledata
2868                if titledata then
2869                    local title = titledata.title
2870                    if title then
2871                        if lowercache then
2872                            found = lpegmatch(pattern,lowercache[title])
2873                        else
2874                            found = lpegmatch(pattern,lpegmatch(p_lower,title))
2875                        end
2876                        if found then
2877                            return {
2878                                inner     = pattern,
2879                                kind      = "inner",
2880                                reference = pattern,
2881                                i         = entry,
2882                                p         = "",
2883                                r         = entry.references.realpage,
2884                            }
2885                        end
2886                    end
2887                end
2888            end
2889        end
2890    end
2891end
2892
2893function functions.match(var,actions)
2894    if not var.outer then
2895        local operation   = var.operation
2896        if operation and operation ~= "" then
2897            local operation   = lpegmatch(p_lower,operation)
2898            local list        = lists.collected
2899            local names       = false
2900            local kinds       = false
2901            local where, what = lpegmatch(p_splitter,operation)
2902            if where and what then
2903                local pattern = lpeg.finder(what)
2904                return
2905                    locate(list,false,where,pattern)
2906                 or locate(list,where,false,pattern)
2907                 or { error = "no match" }
2908            else
2909                local pattern = lpeg.finder(operation)
2910                -- todo: don't look at section and float in last pass
2911                return
2912                    locate(list,"section",false,pattern)
2913                 or locate(list,"float",false,pattern)
2914                 or locate(list,false,false,pattern)
2915                 or { error = "no match" }
2916            end
2917        end
2918    end
2919end
2920
2921-- needs a better split ^^^
2922
2923-- done differently now:
2924
2925function references.export(usedname) end
2926function references.import(usedname) end
2927function references.load  (usedname) end
2928
2929implement { name = "exportreferences", actions =references.export }
2930
2931-- better done here .... we don't insert/remove, just use a pointer
2932
2933local prefixstack = { "" }
2934local prefixlevel = 1
2935
2936local function pushreferenceprefix(prefix)
2937    prefixlevel = prefixlevel + 1
2938    prefixstack[prefixlevel] = prefix
2939    return prefix
2940end
2941
2942local function popreferenceprefix()
2943    prefixlevel = prefixlevel - 1
2944    if prefixlevel > 0 then
2945        return prefixstack[prefixlevel]
2946    else
2947        report_references("unable to pop referenceprefix")
2948        return ""
2949    end
2950end
2951
2952implement {
2953    name      = "pushreferenceprefix",
2954    actions   = { pushreferenceprefix, context }, -- we can use setmacro
2955    arguments = "string",
2956}
2957
2958implement {
2959    name      = "popreferenceprefix",
2960    actions   = { popreferenceprefix, context }, -- we can use setmacro
2961}
2962
Source Browser ?
