strc-lst.lmt /size: 50 Kb    last modification: 2024-01-16 10:22
1if not modules then modules = { } end modules ['strc-lst'] = {
2    version   = 1.001,
3    comment   = "companion to strc-lst.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- when all datastructures are stable a packer will be added which will
10-- bring down memory consumption a bit; we can use for instance a pagenumber,
11-- section, metadata cache (internal then has to move up one level) or a
12-- shared cache [we can use a fast and stupid serializer]
13
14-- todo: tag entry in list is crap
15--
16-- move more to commands
17
18local tonumber, type, next = tonumber, type, next
19local concat, insert, remove, sort = table.concat, table.insert, table.remove, table.sort
20local lpegmatch = lpeg.match
21
22local setmetatableindex = table.setmetatableindex
23local sortedkeys        = table.sortedkeys
24
25local settings_to_set   = utilities.parsers.settings_to_set
26local allocate          = utilities.storage.allocate
27local checked           = utilities.storage.checked
28
29local trace_lists       = false  trackers.register("structures.lists", function(v) trace_lists = v end)
30
31local report_lists      = logs.reporter("structure","lists")
32
33local context           = context
34local commands          = commands
35local implement         = interfaces.implement
36local conditionals      = tex.conditionals
37
38local ctx_latelua       = context.latelua
39
40----- cheat             = true  -- fragile when we have a multi-component document with cross references
41local cheat             = false -- so this will become a directive or maybe even just an experiment
42
43local structures        = structures
44local lists             = structures.lists
45local sections          = structures.sections
46local helpers           = structures.helpers
47local documents         = structures.documents
48local tags              = structures.tags
49local counters          = structures.counters
50local references        = structures.references
51
52local collected         = allocate()
53local tobesaved         = allocate()
54local cached            = allocate()
55local pushed            = allocate()
56local kinds             = allocate()
57local names             = allocate()
58
59lists.collected         = collected
60lists.tobesaved         = tobesaved
61
62lists.enhancers         = lists.enhancers or { }
63-----.internals         = allocate(lists.internals or { }) -- to be checked
64lists.ordered           = allocate(lists.ordered   or { }) -- to be checked
65lists.cached            = cached
66lists.pushed            = pushed
67lists.kinds             = kinds
68lists.names             = names
69
70local sorters           = sorters
71local sortstripper      = sorters.strip
72local sortsplitter      = sorters.splitters.utf
73local sortcomparer      = sorters.comparers.basic
74
75local sectionblocks     = allocate()
76lists.sectionblocks     = sectionblocks
77
78references.specials     = references.specials or { }
79
80local matchingtilldepth = sections.matchingtilldepth
81local numberatdepth     = sections.numberatdepth
82local getsectionlevel   = sections.getlevel
83local typesetnumber     = sections.typesetnumber
84local autosectiondepth  = sections.autodepth
85
86local variables         = interfaces.variables
87
88local v_all             = variables.all
89local v_reference       = variables.reference
90local v_title           = variables.title
91local v_command         = variables.command
92local v_text            = variables.text
93local v_current         = variables.current
94local v_previous        = variables.previous
95local v_intro           = variables.intro
96local v_here            = variables.here
97local v_component       = variables.component
98local v_product         = variables.product
99local v_file            = variables.file
100local v_local           = variables["local"]
101local v_default         = variables.default
102
103local cheats = {
104    [variables.fit]   = true,
105    [variables.tight] = true,
106}
107
108local function zerostrippedconcat(t,separator)
109    local f = 1
110    local l = #t
111    for i=f,l do
112        if t[i] == 0 then
113            f = f + 1
114        end
115    end
116    for i=l,f,-1 do
117        if t[i] == 0 then
118            l = l - 1
119        end
120    end
121    return concat(t,separator,f,l)
122end
123
124-- -- -- -- -- --
125
126local function initializer()
127    -- create a cross reference between internal references
128    -- and list entries
129    local collected     = lists.collected
130    local internals     = checked(references.internals)
131    local ordered       = lists.ordered
132    local usedinternals = references.usedinternals
133    local blockdone     = { }
134    local lastblock     = nil
135    for i=1,#collected do
136        local c = collected[i]
137        local m = c.metadata
138        local r = c.references
139        if m then
140            -- access by internal reference
141            if r then
142                local internal = r.internal
143                if internal then
144                    internals[internal] = c
145                    usedinternals[internal] = r.used
146                end
147                local block = r.block
148                if not block then
149                    -- shouldn't happen
150                elseif lastblock == block then
151                    -- we're okay
152                elseif lastblock then
153                    if blockdone[block] then
154                        if trace_lists then
155                            report_lists("out of order sectionsblocks, maybe use \\setsectionblock")
156                        end
157                    else
158                        blockdone[block] = true
159                        sectionblocks[#sectionblocks+1] = block
160                    end
161                    lastblock = block
162                elseif not blockdone[block] then
163                    blockdone[block] = true
164                    sectionblocks[#sectionblocks+1] = block
165                    lastblock = block
166                end
167            end
168            -- access by order in list
169            local kind = m.kind
170            local name = m.name
171            if kind and name then
172                local ok = ordered[kind]
173                if ok then
174                    local on = ok[name]
175                    if on then
176                        on[#on+1] = c
177                    else
178                        ok[name] = { c }
179                    end
180                else
181                    ordered[kind] = { [name] = { c } }
182                end
183                kinds[kind] = true
184                names[name] = true
185            elseif kind then
186                kinds[kind] = true
187            elseif name then
188                names[name] = true
189            end
190        end
191        if r then
192            r.listindex = i -- handy to have
193        end
194    end
195end
196
197local function finalizer()
198    local flaginternals = references.flaginternals
199    local usedviews     = references.usedviews
200    for i=1,#tobesaved do
201        local r = tobesaved[i].references
202        if r then
203            local i = r.internal
204            local f = flaginternals[i]
205            local v = usedviews[i]
206            if cheat and v and cheats[v] then -- cheats check added, to be tested by RKB
207                -- this permits runs=2 with interactivity
208                r.view = v
209            end
210            if f then
211                r.used = v or true
212            end
213        end
214    end
215end
216
217job.register('structures.lists.collected', tobesaved, initializer, finalizer)
218
219local groupindices = setmetatableindex("table")
220
221function lists.groupindex(name,group)
222    local groupindex = groupindices[name]
223    return groupindex and groupindex[group] or 0
224end
225
226-- we could use t (as hash key) in order to check for dup entries
227
228function lists.addto(t) -- maybe more more here (saves parsing at the tex end)
229    local metadata   = t.metadata
230    local userdata   = t.userdata
231    local numberdata = t.numberdata
232    if userdata and type(userdata) == "string" then
233        t.userdata = helpers.touserdata(userdata)
234    end
235    if not metadata.level then
236        metadata.level = structures.sections.currentlevel() -- this is not used so it will go away
237    end
238    --
239 -- if not conditionals.inlinelefttoright then
240 --     metadata.idir = "r2l"
241 -- end
242 -- if not conditionals.displaylefttoright then
243 --     metadata.ddir = "r2l"
244 -- end
245    --
246    if numberdata then
247        local numbers = numberdata.numbers
248        if type(numbers) == "string" then
249            counters.compact(numberdata,numbers,numberdata.level)
250        end
251    end
252    local group = numberdata and numberdata.group
253    local name  = metadata.name
254    local kind  = metadata.kind
255    if not group then
256        -- forget about it
257    elseif group == "" then
258        group, numberdata.group = nil, nil
259    else
260        local groupindex = groupindices[name][group]
261        if groupindex then
262            numberdata.numbers = cached[groupindex].numberdata.numbers
263        end
264    end
265    --
266--     local setcomponent = references.setcomponent
267--     if setcomponent then
268--         setcomponent(t) -- can be inlined, will be replaced by setstructure below
269--     end
270    --
271    local r = t.references
272    local i = r and r.internal or 0 -- brrr
273    if r then
274        if not r.section then
275            r.section = structures.sections.currentid()
276        end
277        if not r.structure then
278            r.structure = resolvers.jobs.currentstructure()
279        end
280        local b = r and t.block
281        if not b then
282            local s = r.section
283            if s then
284                s = structures.sections.tobesaved[s]
285                r.block = s and s.block or nil
286            end
287        end
288        if kind and name then
289            local tag = tags.getid(kind,name)
290            if tag and tag ~= "?" then
291                r.tag = tag -- todo: use internal ... is unique enough
292            end
293        end
294    end
295    local p = pushed[i]
296    if not p then
297        p = #cached + 1
298        cached[p] = helpers.simplify(t)
299        pushed[i] = p
300        if r then
301            r.listindex = p
302        end
303    end
304    if group then
305        groupindices[name][group] = p
306    end
307    if trace_lists then
308        report_lists("added %a, internal %a",name,p)
309    end
310    return p
311end
312
313function lists.discard(n)
314    n = tonumber(n)
315    if not n then
316        -- maybe an error message
317    elseif n == #cached then
318        cached[n] = nil
319        n = n - 1
320        while n > 0 and cached[n] == false do
321            cached[n] = nil -- collect garbage
322            n = n - 1
323        end
324    else
325        cached[n] = false
326    end
327end
328
329function lists.iscached(n)
330    return cached[tonumber(n)]
331end
332
333-- this is the main pagenumber enhancer
334
335local enhanced = { }
336
337local synchronizepage = function(r)  -- bah ... will move
338    synchronizepage = references.synchronizepage
339    return synchronizepage(r)
340end
341
342local function enhancelist(specification)
343    local n = specification.n
344    local l = cached[n]
345    if not l then
346        report_lists("enhancing %a, unknown internal",n)
347    elseif enhanced[n] then
348        if trace_lists then
349            report_lists("enhancing %a, name %a, duplicate ignored",n,name)
350        end
351    else
352        local metadata   = l.metadata
353        local references = l.references
354        --
355        l.directives = nil -- might change
356        -- save in the right order (happens at shipout)
357        lists.tobesaved[#lists.tobesaved+1] = l
358        -- default enhancer (cross referencing)
359        synchronizepage(references)
360        -- tags
361        local kind = metadata.kind
362        local name = metadata.name
363        if trace_lists then
364            report_lists("enhancing %a, name %a, page %a",n,name,references.realpage or 0)
365        end
366--         if references then
367--             -- is this used ?
368--             local tag = tags.getid(kind,name)
369--             if tag and tag ~= "?" then
370--                 references.tag = tag
371--             end
372--         end
373        -- specific enhancer (kind of obsolete)
374        local enhancer = kind and lists.enhancers[kind]
375        if enhancer then
376            enhancer(l)
377        end
378        --
379        enhanced[n] = true
380        return l
381    end
382end
383
384lists.enhance = enhancelist
385
386-- we can use level instead but we can also decide to remove level from the metadata
387
388local nesting = { }
389
390function lists.pushnesting(i)
391    local parent     = lists.result[i]
392    local name       = parent.metadata.name
393    local numberdata = parent and parent.numberdata
394    local numbers    = numberdata and numberdata.numbers
395    local number     = numbers and numbers[getsectionlevel(name)] or 0
396    insert(nesting, {
397        number = number,
398        name   = name,
399        result = lists.result,
400        parent = parent
401    })
402end
403
404function lists.popnesting()
405    local old = remove(nesting)
406    if old then
407        lists.result = old.result
408    else
409        report_lists("nesting error")
410    end
411end
412
413-- Historically we had blocks but in the mkiv approach that could as well be a level
414-- which would simplify things a bit.
415
416local splitter = lpeg.splitat(":") -- maybe also :: or have a block parameter
417
418local listsorters = {
419    [v_command] = function(a,b)
420        if a.metadata.kind == "command" or b.metadata.kind == "command" then
421            return a.references.internal < b.references.internal
422        else
423            return a.references.order < b.references.order
424        end
425    end,
426    [v_all] = function(a,b)
427        return a.references.internal < b.references.internal
428    end,
429    [v_title] = function(a,b)
430        local da = a.titledata
431        local db = b.titledata
432        if da and db then
433            local ta = da.title
434            local tb = db.title
435            if ta and tb then
436                local sa = da.split
437                if not sa then
438                    sa = sortsplitter(sortstripper(ta))
439                    da.split = sa
440                end
441                local sb = db.split
442                if not sb then
443                    sb = sortsplitter(sortstripper(tb))
444                    db.split = sb
445                end
446                return sortcomparer(da,db) == -1
447            end
448        end
449        return a.references.internal < b.references.internal
450    end
451}
452
453-- was: names, criterium, number, collected, forced, nested, sortorder
454
455local filters = setmetatableindex(function(t,k) return t[v_default] end)
456
457
458local used = { }
459
460function lists.use(tag,filename,class)
461    used[tag] = {
462        names    = names, -- optional
463        filename = filename,
464        data     = job.loadother(filename),
465    }
466end
467
468implement {
469    name      = "uselist",
470    arguments = "3 strings",
471    actions   = lists.use,
472}
473
474filters.external = function(specification)
475    local collected = specification.collected or { }
476    local result    = { }
477    local nofresult = 0
478    local all       = specification.all
479    local names     = specification.names
480    setmetatableindex(result,{ external = specification.reference }) -- brr, we also have the name in
481    for i=1,#collected do
482        local v = collected[i]
483        local m = v.metadata
484        if m and names[m.name] or all then
485            nofresult = nofresult + 1
486            result[nofresult] = v
487        end
488    end
489    return result
490end
491
492
493local function filtercollected(specification)
494    --
495    local names     = specification.names     or { }
496    local criterium = specification.criterium or v_default
497    local number    = 0 -- specification.number
498    local reference = specification.reference or ""
499    local collected = specification.collected or lists.collected
500    local forced    = specification.forced    or { }
501    local nested    = specification.nested    or false
502    local sortorder = specification.sortorder or specification.order
503    local numbers   = documents.data.numbers
504    local depth     = documents.data.depth
505    local block     = false -- all
506    --
507    local wantedblock, wantedcriterium = lpegmatch(splitter,criterium) -- block:criterium
508    if wantedblock == "" or wantedblock == v_all or wantedblock == v_text then
509        criterium = wantedcriterium ~= "" and wantedcriterium or criterium
510    elseif not wantedcriterium then
511        block = documents.data.block
512    else
513        block     = wantedblock
514        criterium = wantedcriterium
515    end
516    if block == "" then
517        block = false
518    end
519    if type(names) == "string" then
520        names = settings_to_set(names)
521    end
522    local all = not next(names) or names[v_all] or false
523    --    --
524    specification.names     = names
525    specification.criterium = criterium
526    specification.number    = 0 -- obsolete
527    specification.reference = reference -- new
528    specification.collected = collected
529    specification.forced    = forced -- todo: also on other branched, for the moment only needed for bookmarks
530    specification.nested    = nested
531    specification.sortorder = sortorder
532    specification.numbers   = numbers
533    specification.depth     = depth
534    specification.block     = block
535    specification.all       = all
536    --
537    if specification.atmost then
538        criterium = v_text
539    end
540    --
541    if trace_lists then
542        report_lists("filtering names %,t, criterium %a, block %a",sortedkeys(names), criterium, block or "*")
543    end
544    local result = filters[criterium](specification)
545    if trace_lists then
546        report_lists("criterium %a, block %a, found %a",specification.criterium, specification.block or "*", #result)
547    end
548    --
549    local levels = tonumber(specification.levels)
550    if levels then
551        local minlevel  = 1000
552        local found     = result
553        local nofresult = #result
554        for i=1,nofresult do
555            local v = found[i]
556            local l = v.metadata.level or 1
557            if l < minlevel then
558                minlevel = l
559            end
560        end
561        local maxlevel = minlevel + levels - 1
562        result    = { }
563        nofresult = 0
564        for i=1,#found do
565            local v = found[i]
566            local l = v.metadata.level or 1
567            if l >= minlevel and l <= maxlevel then
568                nofresult = nofresult + 1
569                result[nofresult] = v
570            end
571        end
572    end
573    --
574    if sortorder then -- experiment
575        local sorter = listsorters[sortorder]
576        if sorter then
577            if trace_lists then
578                report_lists("sorting list using method %a",sortorder)
579            end
580            for i=1,#result do
581                result[i].references.order = i
582            end
583            sort(result,sorter)
584        end
585    end
586    --
587    return result
588end
589
590filters[v_intro] = function(specification)
591    local collected = specification.collected
592    local result    = { }
593    local nofresult = 0
594    local all       = specification.all
595    local names     = specification.names
596    for i=1,#collected do
597        local v = collected[i]
598        local metadata = v.metadata
599        if metadata and (all or names[metadata.name or false]) then
600            local r = v.references
601            if r and r.section == 0 then
602                nofresult = nofresult + 1
603                result[nofresult] = v
604            end
605        end
606    end
607    return result
608end
609
610filters[v_reference] = function(specification) -- this will go away
611    local collected = specification.collected
612    local result    = { }
613    local nofresult = 0
614    local names     = specification.names
615    local sections  = sections.collected
616    local reference = specification.reference
617    if reference ~= "" then
618        local prefix, rest = lpegmatch(references.prefixsplitter,reference) -- p::r
619        local r = prefix and rest and references.derived[prefix][rest] or references.derived[""][reference]
620        local s = r and r.numberdata -- table ref !
621        if s then
622            local depth   = getsectionlevel(r.metadata.name)
623            local numbers = s.numbers
624            for i=1,#collected do
625                local v = collected[i]
626                local r = v.references
627                if r and (not block or not r.block or block == r.block) then
628                    local metadata = v.metadata
629                    if metadata and names[metadata.name or false] then
630                        local sectionnumber = (r.section == 0) or sections[r.section]
631                        if sectionnumber then
632                            if matchingtilldepth(depth,numbers,sectionnumber.numbers) then
633                                nofresult = nofresult + 1
634                                result[nofresult] = v
635                            end
636                        end
637                    end
638                end
639            end
640        else
641            report_lists("unknown reference %a specified",reference)
642        end
643    else
644        report_lists("no reference specified")
645    end
646    return result
647end
648
649-- maybe:
650--
651-- filters[v_reference] = function(specification)
652--     local collected = specification.collected
653--     local result    = { }
654--     local nofresult = 0
655--     local names     = specification.names
656--     local sections  = sections.collected
657--     local reference = specification.reference
658--     if reference ~= "" then
659--         local split = references.splitreference(reference)
660--         if split then
661--             local prefix = split and split.outer
662--             local rest   = split and split.inner
663--             if prefix and rest then
664--                 local d = references.derived
665--                 local r = (d[prefix] and d[prefix][rest]) or (d[""] and d[""][reference])
666--                 local s = r and r.numberdata -- table ref !
667--                 if s then
668--                     local depth   = getsectionlevel(r.metadata.name)
669--                     local numbers = s.numbers
670--                     for i=1,#collected do
671--                         local v = collected[i]
672--                         local r = v.references
673--                         if r and (not block or not r.block or block == r.block) then
674--                             local metadata = v.metadata
675--                             if metadata and names[metadata.name or false] then
676--                                 local sectionnumber = (r.section == 0) or sections[r.section]
677--                                 if sectionnumber then
678--                                     if matchingtilldepth(depth,numbers,sectionnumber.numbers) then
679--                                         nofresult = nofresult + 1
680--                                         result[nofresult] = v
681--                                     end
682--                                 end
683--                             end
684--                         end
685--                     end
686--                 else
687--                     report_lists("unknown reference %a specified",reference)
688--                 end
689--             end
690--         end
691--     else
692--         report_lists("no reference specified")
693--     end
694--     return result
695-- end
696
697filters[v_all] = function(specification)
698    local collected = specification.collected
699    local result    = { }
700    local nofresult = 0
701    local block     = specification.block
702    local all       = specification.all
703    local forced    = specification.forced
704    local names     = specification.names
705    local sections  = sections.collected
706    for i=1,#collected do
707        local v = collected[i]
708        local r = v.references
709        if r and (not block or not r.block or block == r.block) then
710            local metadata = v.metadata
711            if metadata then
712                local name  = metadata.name  or false
713                local sectionnumber = (r.section == 0) or sections[r.section]
714                if forced[name] or (sectionnumber and not metadata.nolist and (all or names[name])) then -- and not sectionnumber.hidenumber then
715                    nofresult = nofresult + 1
716                    result[nofresult] = v
717                end
718            end
719        end
720    end
721    return result
722end
723
724filters[v_text] = filters[v_all]
725
726filters[v_current] = function(specification)
727    if specification.depth == 0 then
728        specification.nested    = false
729        specification.criterium = v_intro
730        return filters[v_intro](specification)
731    end
732    local collected = specification.collected
733    local result    = { }
734    local nofresult = 0
735    local depth     = specification.depth
736    local block     = specification.block
737    local all       = specification.all
738    local names     = specification.names
739    local numbers   = specification.numbers
740    local sections  = sections.collected
741    for i=1,#collected do
742        local v = collected[i]
743        local r = v.references
744        if r and (not block or not r.block or block == r.block) then
745            local sectionnumber = sections[r.section]
746            if sectionnumber then -- and not sectionnumber.hidenumber then
747                local cnumbers = sectionnumber.numbers
748                local metadata = v.metadata
749                if cnumbers then
750                    if metadata and not metadata.nolist and (all or names[metadata.name or false]) and #cnumbers > depth then
751                        local ok = true
752                        for d=1,depth do
753                            local cnd = cnumbers[d]
754                            if not (cnd == 0 or cnd == numbers[d]) then
755                                ok = false
756                                break
757                            end
758                        end
759                        if ok then
760                            nofresult = nofresult + 1
761                            result[nofresult] = v
762                        end
763                    end
764                end
765            end
766        end
767    end
768    return result
769end
770
771filters[v_here] = function(specification)
772    -- this is quite dirty ... as cnumbers is not sparse we can misuse #cnumbers
773    if specification.depth == 0 then
774        specification.nested    = false
775        specification.criterium = v_intro
776        return filters[v_intro](specification)
777    end
778    local collected = specification.collected
779    local result    = { }
780    local nofresult = 0
781    local depth     = specification.depth
782    local block     = specification.block
783    local all       = specification.all
784    local names     = specification.names
785    local numbers   = specification.numbers
786    local sections  = sections.collected
787    for i=1,#collected do
788        local v = collected[i]
789        local r = v.references
790        if r then -- and (not block or not r.block or block == r.block) then
791            local sectionnumber = sections[r.section]
792            if sectionnumber then -- and not sectionnumber.hidenumber then
793                local cnumbers = sectionnumber.numbers
794                local metadata = v.metadata
795                if cnumbers then
796                    if metadata and not metadata.nolist and (all or names[metadata.name or false]) and #cnumbers >= depth then
797                        local ok = true
798                        for d=1,depth do
799                            local cnd = cnumbers[d]
800                            if not (cnd == 0 or cnd == numbers[d]) then
801                                ok = false
802                                break
803                            end
804                        end
805                        if ok then
806                            nofresult = nofresult + 1
807                            result[nofresult] = v
808                        end
809                    end
810                end
811            end
812        end
813    end
814    return result
815end
816
817filters[v_previous] = function(specification)
818    if specification.depth == 0 then
819        specification.nested    = false
820        specification.criterium = v_intro
821        return filters[v_intro](specification)
822    end
823    local collected = specification.collected
824    local result    = { }
825    local nofresult = 0
826    local block     = specification.block
827    local all       = specification.all
828    local names     = specification.names
829    local numbers   = specification.numbers
830    local sections  = sections.collected
831    local depth     = specification.depth
832    for i=1,#collected do
833        local v = collected[i]
834        local r = v.references
835        if r and (not block or not r.block or block == r.block) then
836            local sectionnumber = sections[r.section]
837            if sectionnumber then -- and not sectionnumber.hidenumber then
838                local cnumbers = sectionnumber.numbers
839                local metadata = v.metadata
840                if cnumbers then
841                    if metadata and not metadata.nolist and (all or names[metadata.name or false]) and #cnumbers >= depth then
842                        local ok = true
843                        for d=1,depth-1 do
844                            local cnd = cnumbers[d]
845                            if not (cnd == 0 or cnd == numbers[d]) then
846                                ok = false
847                                break
848                            end
849                        end
850                        if ok then
851                            nofresult = nofresult + 1
852                            result[nofresult] = v
853                        end
854                    end
855                end
856            end
857        end
858    end
859    return result
860end
861
862filters[v_local] = function(specification)
863    local numbers = specification.numbers
864    local nested  = nesting[#nesting]
865    if nested then
866        return filtercollected {
867            names     = specification.names,
868            criterium = nested.name,
869            collected = specification.collected,
870            forced    = specification.forced,
871            nested    = nested,
872            sortorder = specification.sortorder,
873        }
874    else
875        specification.criterium = autosectiondepth(numbers) == 0 and v_all or v_current
876        specification.nested    = false
877        return filtercollected(specification) -- rechecks, so better (for determining all)
878    end
879end
880
881filters[v_component] = function(specification)
882    -- special case, no structure yet
883    local collected = specification.collected
884    local result    = { }
885    local nofresult = 0
886    local all       = specification.all
887    local names     = specification.names
888    local component = resolvers.jobs.currentcomponent() or ""
889    if component ~= "" then
890        for i=1,#collected do
891            local v = collected[i]
892            local r = v.references
893            local m = v.metadata
894            if r and r.component == component and (m and names[m.name] or all) then
895                nofresult = nofresult + 1
896                result[nofresult] = v
897            end
898        end
899    end
900    return result
901end
902
903filters[v_product] = function(specification)
904    local reference = specification.reference
905    if reference and reference ~= "" then
906        local utilitydata = job.loadother(reference)
907        if utilitydata then
908            local collected = utilitydata.structures.lists.collected or { }
909            local result    = { }
910            local nofresult = 0
911            local all       = specification.all
912            local names     = specification.names
913setmetatableindex(result,{ external = reference }) -- brr
914            for i=1,#collected do
915                local v = collected[i]
916                local m = v.metadata
917--                 if m and names[m.name] or all then
918                if m and not m.nolist and (names[m.name] or all) then
919                    nofresult = nofresult + 1
920                    result[nofresult] = v
921                end
922            end
923            return result
924        end
925    end
926    return { }
927end
928
929-- local number = tonumber(number) or numberatdepth(depth) or 0
930-- if number > 0 then
931--     ...
932-- end
933
934filters[v_default] = function(specification) -- is named
935    local collected = specification.collected
936    local result    = { }
937    local nofresult = 0
938    ----- depth     = specification.depth
939    local block     = specification.block
940    local criterium = specification.criterium
941    local all       = specification.all
942    local names     = specification.names
943    local numbers   = specification.numbers
944    local sections  = sections.collected
945    local reference = specification.reference
946    local nested    = specification.nested
947    --
948    if reference then
949        reference = tonumber(reference)
950    end
951    --
952    local depth     = getsectionlevel(criterium)
953    local pnumbers  = nil
954    local pblock    = block
955    local parent    = nested and nested.parent
956    --
957    if parent then
958        pnumbers = parent.numberdata.numbers or pnumbers -- so local as well as nested
959        pblock   = parent.references.block or pblock
960        if trace_lists then
961            report_lists("filtering by block %a and section %a",pblock,criterium)
962        end
963    end
964    --
965    for i=1,#collected do
966        local v = collected[i]
967        local r = v.references
968        if r and (not block or not r.block or pblock == r.block) then
969            local sectionnumber = sections[r.section]
970            if sectionnumber then
971                local metadata = v.metadata
972                local cnumbers = sectionnumber.numbers
973                if cnumbers then
974                    if all or names[metadata.name or false] then
975                        if reference then
976                            -- filter by number
977                            if reference == cnumbers[depth] then
978                                nofresult = nofresult + 1
979                                result[nofresult] = v
980                            end
981                        else
982                            if #cnumbers >= depth and matchingtilldepth(depth,cnumbers,pnumbers) then
983                                nofresult = nofresult + 1
984                                result[nofresult] = v
985                            end
986                        end
987                    end
988                end
989            end
990        end
991    end
992    return result
993end
994
995-- names, criterium, number, collected, forced, nested, sortorder) -- names is hash or string
996
997lists.filter = filtercollected
998
999lists.result = { }
1000
1001function lists.getresult(r)
1002    return lists.result[r]
1003end
1004
1005function lists.process(specification)
1006    local names    = specification.names
1007    local external = used[names or ""]
1008    if external then
1009        specification.reference = names
1010        specification.names     = external.names
1011        specification.collected = external.data.structures.lists.collected
1012        specification.criterium = "external"
1013    end
1014    local result = filtercollected(specification)
1015    local total  = #result
1016    lists.result = result
1017    if total > 0 then
1018        local usedinternals = references.usedinternals
1019        local usedviews     = references.usedviews
1020        local specials = settings_to_set(specification.extras or "")
1021              specials = next(specials) and specials or nil
1022        for i=1,total do
1023            local listentry  = result[i]
1024            local metadata   = listentry.metadata
1025            local numberdata = listentry.numberdata
1026            local references = listentry.references
1027            local special    = specials and numberdata and specials[zerostrippedconcat(numberdata.numbers,".")] or ""
1028            local view       = usedviews[i]
1029--             if cheat and references and cheats[view] then
1030            if cheat and references and view and cheats[view] then
1031             -- this permits runs=2 with interactivity
1032                local internal = references.internal
1033                usedinternals[internal] = true
1034                usedviews    [internal] = references.view
1035            end
1036            context.strclistsentryprocess(metadata.name,metadata.kind,i,special)
1037        end
1038    end
1039end
1040
1041function lists.analyze(specification)
1042    lists.result = filtercollected(specification)
1043end
1044
1045function lists.userdata(name,r,tag) -- to tex (todo: xml)
1046    local result = lists.result[r]
1047    if result then
1048        local userdata = result.userdata
1049        local str = userdata and userdata[tag]
1050        if str then
1051            return str, result.metadata
1052        end
1053    end
1054end
1055
1056function lists.uservalue(name,r,tag,default) -- to lua
1057    local str = lists.result[r]
1058    if str then
1059        str = str.userdata
1060    end
1061    if str then
1062        str = str[tag]
1063    end
1064    return str or default
1065end
1066
1067function lists.size()
1068    return #lists.result
1069end
1070
1071function lists.external(n)
1072    return lists.result.external or ""
1073end
1074
1075function lists.location(n)
1076    local l = lists.result[n]
1077    return l and l.references.internal or n
1078end
1079
1080function lists.label(n,default)
1081    local l = lists.result[n]
1082    local t = l.titledata
1083    return t and t.label or default or ""
1084end
1085
1086function lists.sectionnumber(name,n,spec)
1087    local data = lists.result[n]
1088    local sectiondata = sections.collected[data.references.section]
1089    -- hm, prefixnumber?
1090    typesetnumber(sectiondata,"prefix",spec,sectiondata) -- data happens to contain the spec too
1091end
1092
1093-- some basics (todo: helpers for pages)
1094
1095function lists.title(name,n,tag) -- tag becomes obsolete
1096    local data = lists.result[n]
1097    if data then
1098        local titledata = data.titledata
1099        if titledata then
1100            helpers.title(titledata[tag] or titledata.list or titledata.title or "",data.metadata)
1101        end
1102    end
1103end
1104
1105function lists.hastitledata(name,n,tag)
1106    local data = cached[tonumber(n)]
1107    if data then
1108        local titledata = data.titledata
1109        if titledata then
1110            return (titledata[tag] or titledata.title or "") ~= ""
1111        end
1112    end
1113    return false
1114end
1115
1116function lists.haspagedata(name,n)
1117    local data = lists.result[n]
1118    if data then
1119        local references = data.references
1120        if references and references.realpage then -- or references.pagedata
1121            return true
1122        end
1123    end
1124    return false
1125end
1126
1127function lists.hasnumberdata(name,n)
1128    local data = lists.result[n]
1129    if data then
1130        local numberdata = data.numberdata
1131        if numberdata and not numberdata.hidenumber then -- the hide number is true
1132            return true
1133        end
1134    end
1135    return false
1136end
1137
1138function lists.rawnumber(n,name)
1139    local data = lists.result[n]
1140    if data then
1141        local numberdata = data.numberdata
1142        if numberdata then
1143            numberdata = numberdata.numbers
1144            return numberdata and numberdata[getsectionlevel(name)] or numberdata[name] or 0
1145        end
1146    end
1147    return 0
1148end
1149
1150function lists.prefix(name,n,spec)
1151    helpers.prefix(lists.result[n],spec)
1152end
1153
1154function lists.page(name,n,pagespec)
1155    helpers.page(lists.result[n],pagespec)
1156end
1157
1158function lists.prefixedpage(name,n,prefixspec,pagespec)
1159    helpers.prefixpage(lists.result[n],prefixspec,pagespec)
1160end
1161
1162function lists.realpage(name,n)
1163    local data = lists.result[n]
1164    if data then
1165        local references = data.references
1166        return references and references.realpage or 0
1167    else
1168        return 0
1169    end
1170end
1171
1172-- numbers stored in entry.numberdata + entry.numberprefix
1173
1174function lists.number(name,n,spec)
1175    local data = lists.result[n]
1176    if data then
1177        local numberdata = data.numberdata
1178        if numberdata then
1179            typesetnumber(numberdata,"number",spec or false,numberdata or false)
1180        end
1181    end
1182end
1183
1184function lists.prefixednumber(name,n,prefixspec,numberspec,forceddata)
1185    local data = lists.result[n]
1186    if data then
1187        helpers.prefix(data,prefixspec)
1188        local numberdata = data.numberdata or forceddata
1189        if numberdata then
1190            typesetnumber(numberdata,"number",numberspec or false,numberdata or false)
1191        end
1192    end
1193end
1194
1195-- todo, do this in references namespace ordered instead (this is an experiment)
1196--
1197-- also see lpdf-ano (maybe move this there)
1198
1199local splitter = lpeg.splitat(":")
1200
1201function references.specials.order(var,actions) -- references.specials !
1202    local operation = var.operation
1203    if operation then
1204        local kind, name, n = lpegmatch(splitter,operation)
1205        local order = lists.ordered[kind]
1206        order = order and order[name]
1207        local v = order[tonumber(n)]
1208        local r = v and v.references.realpage
1209        if r then
1210            actions.realpage = r
1211            var.operation = r -- brrr, but test anyway
1212            return references.specials.page(var,actions)
1213        end
1214    end
1215end
1216
1217-- interface (maybe strclistpush etc)
1218
1219if not lists.reordered then
1220    function lists.reordered(data)
1221        return data.numberdata
1222    end
1223end
1224
1225implement { name = "pushlist", actions = lists.pushnesting, arguments = "integer" }
1226implement { name = "poplist",  actions = lists.popnesting  }
1227
1228implement {
1229    name      = "addtolist",
1230    actions   = { lists.addto, context },
1231    arguments = {
1232        {
1233            { "references", {
1234                    { "internal", "integer" },
1235                    { "block" },
1236                    { "section", "integer" },
1237                    { "location" },
1238                    { "prefix" },
1239                    { "reference" },
1240                    { "view" },
1241                    { "order", "integer" },
1242                }
1243            },
1244            { "metadata", {
1245                    { "kind" },
1246                    { "name" },
1247                    { "level", "integer" },
1248                    { "catcodes", "integer" },
1249                    { "coding" },
1250                    { "xmlroot" },
1251                    { "setup" },
1252                }
1253            },
1254            { "userdata" },
1255            { "titledata", {
1256                    { "label" },
1257                    { "title" },
1258                    { "bookmark" },
1259                    { "marking" },
1260                    { "list" },
1261                    { "reference" },
1262                }
1263            },
1264            { "prefixdata", {
1265                    { "prefix" },
1266                    { "separatorset" },
1267                    { "conversionset" },
1268                    { "conversion" },
1269                    { "set" },
1270                    { "segments" },
1271                    { "connector" },
1272                }
1273            },
1274            { "numberdata", {
1275                    { "level", "integer" },
1276                    { "numbers" },
1277                    { "groupsuffix" },
1278                    { "group" },
1279                    { "counter" },
1280                    { "separatorset" },
1281                    { "conversionset" },
1282                    { "conversion" },
1283                    { "starter" },
1284                    { "stopper" },
1285                    { "segments" },
1286                }
1287            }
1288        }
1289    }
1290}
1291
1292implement {
1293    name      = "enhancelist",
1294    arguments = "integer",
1295    actions   = function(n)
1296        enhancelist { n = n }
1297    end
1298}
1299
1300implement {
1301    name      = "deferredenhancelist",
1302    arguments = "integer",
1303    protected = true, -- for now, pre 1.09
1304    actions   = function(n)
1305        ctx_latelua { action = enhancelist, n = n }
1306    end,
1307}
1308
1309implement {
1310    name      = "processlist",
1311    actions   = lists.process,
1312    arguments = {
1313        {
1314            { "names" },
1315            { "criterium" },
1316            { "reference" },
1317            { "extras" },
1318            { "order" },
1319            { "levels" },
1320        }
1321    }
1322}
1323
1324implement {
1325    name      = "analyzelist",
1326    actions   = lists.analyze,
1327    arguments = {
1328        {
1329            { "names" },
1330            { "criterium" },
1331            { "reference" },
1332        }
1333    }
1334}
1335
1336implement {
1337    name      = "listtitle",
1338    actions   = lists.title,
1339    arguments = { "string", "integer" }
1340}
1341
1342implement {
1343    name      = "listprefixednumber",
1344    actions   = lists.prefixednumber,
1345    arguments = {
1346        "string",
1347        "integer",
1348        {
1349            { "prefix" },
1350            { "separatorset" },
1351            { "conversionset" },
1352            { "starter" },
1353            { "stopper" },
1354            { "set" },
1355            { "segments" },
1356            { "connector" },
1357        },
1358        {
1359            { "separatorset" },
1360            { "conversionset" },
1361            { "starter" },
1362            { "stopper" },
1363            { "segments" },
1364        }
1365    }
1366}
1367
1368implement {
1369    name      = "listprefixedpage",
1370    actions   = lists.prefixedpage,
1371    arguments = {
1372        "string",
1373        "integer",
1374        {
1375            { "separatorset" },
1376            { "conversionset" },
1377            { "set" },
1378            { "segments" },
1379            { "connector" },
1380        },
1381        {
1382            { "prefix" },
1383            { "conversionset" },
1384            { "starter" },
1385            { "stopper" },
1386        }
1387    }
1388}
1389
1390implement { name = "listsize",       actions = { lists.size,       context } }
1391implement { name = "listexternal",   actions = { lists.external,   context }, arguments = "integer" }
1392implement { name = "listlocation",   actions = { lists.location,   context }, arguments = "integer" }
1393implement { name = "listlabel",      actions = { lists.label,      context }, arguments = { "integer", "string" } }
1394implement { name = "listrealpage",   actions = { lists.realpage,   context }, arguments = { "string", "integer" } }
1395implement { name = "listgroupindex", actions = { lists.groupindex, context }, arguments = "2 strings", }
1396
1397implement {
1398    name    = "currentsectiontolist",
1399    actions = { sections.current, lists.addto, context }
1400}
1401
1402local function userdata(name,r,tag)
1403    local str, metadata = lists.userdata(name,r,tag)
1404    if str then
1405     -- local catcodes = metadata and metadata.catcodes
1406     -- if catcodes then
1407     --     context.sprint(catcodes,str)
1408     -- else
1409     --     context(str)
1410     -- end
1411        helpers.title(str,metadata)
1412    end
1413end
1414
1415implement {
1416    name      = "listuserdata",
1417    actions   = userdata,
1418    arguments = { "string", "integer", "string" }
1419}
1420
1421-- we could also set variables .. names will change (when this module is done)
1422-- maybe strc_lists_savedtitle etc
1423
1424implement { name = "doifelselisthastitle",  actions = { lists.hastitledata,  commands.doifelse }, arguments = { "string", "integer" } }
1425implement { name = "doifelselisthaspage",   actions = { lists.haspagedata,   commands.doifelse }, arguments = { "string", "integer" } }
1426implement { name = "doifelselisthasnumber", actions = { lists.hasnumberdata, commands.doifelse }, arguments = { "string", "integer" } }
1427implement { name = "doifelselisthasentry",  actions = { lists.iscached,      commands.doifelse }, arguments = "integer" }
1428
1429local function savedlisttitle(name,n,tag)
1430    local data = cached[tonumber(n)]
1431    if data then
1432        local titledata = data.titledata
1433        if titledata then
1434            helpers.title(titledata[tag] or titledata.title or "",data.metadata)
1435        end
1436    end
1437end
1438
1439local function savedlistnumber(name,n)
1440    local data = cached[tonumber(n)]
1441    if data then
1442        local numberdata = data.numberdata
1443        if numberdata then
1444            typesetnumber(numberdata,"number",numberdata or false)
1445        end
1446    end
1447end
1448
1449local function savedlistprefixednumber(name,n)
1450    local data = cached[tonumber(n)]
1451    if data then
1452        local numberdata = lists.reordered(data)
1453        if numberdata then
1454            helpers.prefix(data,data.prefixdata)
1455            typesetnumber(numberdata,"number",numberdata or false)
1456        end
1457    end
1458end
1459
1460lists.savedlisttitle          = savedlisttitle
1461lists.savedlistnumber         = savedlistnumber
1462lists.savedlistprefixednumber = savedlistprefixednumber
1463
1464implement {
1465    name      = "savedlistnumber",
1466    actions   = savedlistnumber,
1467    arguments = { "string", "integer" }
1468}
1469
1470implement {
1471    name      = "savedlisttitle",
1472    actions   = savedlisttitle,
1473    arguments = { "string", "integer" }
1474}
1475
1476implement {
1477    name      = "savedlistprefixednumber",
1478    actions   = savedlistprefixednumber,
1479    arguments = { "string", "integer" }
1480}
1481
1482implement {
1483    name      = "discardfromlist",
1484    actions   = lists.discard,
1485    arguments = "integer"
1486}
1487
1488implement {
1489    name      = "rawlistnumber",
1490    actions   = { lists.rawnumber, context },
1491    arguments = { "integer", "string" },
1492}
1493
1494-- new and experimental and therefore off by default
1495
1496lists.autoreorder = false -- true
1497
1498local function addlevel(t,k)
1499    local v = { }
1500    setmetatableindex(v,function(t,k)
1501        local v = { }
1502        t[k] = v
1503        return v
1504    end)
1505    t[k] = v
1506    return v
1507end
1508
1509local internals = setmetatableindex({ }, function(t,k)
1510
1511    local sublists = setmetatableindex({ },addlevel)
1512
1513    local collected = lists.collected or { }
1514
1515    for i=1,#collected do
1516        local entry = collected[i]
1517        local numberdata = entry.numberdata
1518        if numberdata then
1519            local metadata = entry.metadata
1520            if metadata then
1521                local references = entry.references
1522                if references then
1523                    local kind = metadata.kind
1524                    local name = numberdata.counter or metadata.name
1525                    local internal = references.internal
1526                    if kind and name and internal then
1527                        local sublist = sublists[kind][name]
1528                        sublist[#sublist + 1] = { internal, numberdata }
1529                    end
1530                end
1531            end
1532        end
1533    end
1534
1535    for k, v in next, sublists do
1536        for k, v in next, v do
1537            local tmp = { }
1538            for i=1,#v do
1539                tmp[i] = v[i]
1540            end
1541            sort(v,function(a,b) return a[1] < b[1] end)
1542            for i=1,#v do
1543                t[v[i][1]] = tmp[i][2]
1544            end
1545        end
1546    end
1547
1548    setmetatableindex(t,nil)
1549
1550    return t[k]
1551
1552end)
1553
1554function lists.reordered(entry)
1555    local numberdata = entry.numberdata
1556    if lists.autoreorder then
1557        if numberdata then
1558            local metadata = entry.metadata
1559            if metadata then
1560                local references = entry.references
1561                if references then
1562                    local kind = metadata.kind
1563                    local name = numberdata.counter or metadata.name
1564                    local internal = references.internal
1565                    if kind and name and internal then
1566                        return internals[internal] or numberdata
1567                    end
1568                end
1569            end
1570        end
1571    else
1572        function lists.reordered(entry)
1573            return entry.numberdata
1574        end
1575    end
1576    return numberdata
1577end
1578
1579-- external data
1580
1581function lists.integrate(utilitydata)
1582    local filename = utilitydata.comment.file
1583    if filename and filename ~= environment.jobname then
1584        local structures = utilitydata.structures
1585        if structures then
1586            local lists = structures.lists.collected or { }
1587            if lists then
1588                local sections = structures.sections.collected or { }
1589                local pages    = structures.pages.collected or { }
1590                for i=1,#lists do
1591                    local entry      = lists[i]
1592                    local references = entry.references
1593                    if references then
1594                        local section  = references.section
1595                        local realpage = references.realpage
1596                        -- maybe entry.* instead:
1597                        references.sectiondata = section  and sections[section]
1598                        references.pagedata    = realpage and pages[realpage]
1599                        -- get rid of these, maybe even crash as these are bogus
1600                     -- references.internal = nil
1601                     -- references.realpage = nil
1602                     -- references.section  = nil
1603                        if references.view then
1604                            references.view = nil
1605                        end
1606                        if references.used then
1607                            references.used = nil
1608                        end
1609                        if references.x then
1610                            references.x = nil
1611                        end
1612                        if references.y then
1613                            references.y = nil
1614                        end
1615                        references.external = filename
1616                    end
1617                end
1618            end
1619        end
1620    end
1621end
1622