strc-reg.lua /size: 53 Kb    last modification: 2023-12-21 09:44
1if not modules then modules = { } end modules ['strc-reg'] = {
2    version   = 1.001,
3    comment   = "companion to strc-reg.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9local next, type, tonumber, rawget = next, type, tonumber, rawget
10local char, format, gmatch = string.char, string.format, string.gmatch
11local equal, concat, remove = table.are_equal, table.concat, table.remove
12local lpegmatch, P, C, Ct = lpeg.match, lpeg.P, lpeg.C, lpeg.Ct
13local allocate = utilities.storage.allocate
14
15local trace_registers    = false  trackers.register("structures.registers", function(v) trace_registers = v end)
16
17local report_registers     = logs.reporter("structure","registers")
18
19local structures           = structures
20local registers            = structures.registers
21local helpers              = structures.helpers
22local sections             = structures.sections
23local documents            = structures.documents
24local pages                = structures.pages
25local references           = structures.references
26
27local usedinternals        = references.usedinternals
28
29local mappings             = sorters.mappings
30local entries              = sorters.entries
31local replacements         = sorters.replacements
32
33local processors           = typesetters.processors
34local splitprocessor       = processors.split
35
36local texgetcount          = tex.getcount
37
38local variables            = interfaces.variables
39local v_forward            = variables.forward
40local v_all                = variables.all
41local v_no                 = variables.no
42local v_yes                = variables.yes
43local v_packed             = variables.packed
44local v_current            = variables.current
45local v_previous           = variables.previous
46local v_first              = variables.first
47local v_last               = variables.last
48local v_text               = variables.text
49local v_section            = variables.section
50
51local context              = context
52local ctx_latelua          = context.latelua
53
54local implement            = interfaces.implement
55
56local matchingtilldepth    = sections.matchingtilldepth
57local numberatdepth        = sections.numberatdepth
58local currentlevel         = sections.currentlevel
59local currentid            = sections.currentid
60
61local touserdata           = helpers.touserdata
62
63local internalreferences   = references.internals
64local setinternalreference = references.setinternalreference
65
66local setmetatableindex    = table.setmetatableindex
67
68local absmaxlevel          = 5 -- \c_strc_registers_maxlevel
69
70local h_prefixpage              = helpers.prefixpage
71local h_prefixlastpage          = helpers.prefixlastpage
72local h_title                   = helpers.title
73local h_prefix                  = helpers.prefix
74
75local ctx_startregisteroutput   = context.startregisteroutput
76local ctx_stopregisteroutput    = context.stopregisteroutput
77local ctx_startregistersection  = context.startregistersection
78local ctx_stopregistersection   = context.stopregistersection
79local ctx_startregisterentries  = context.startregisterentries
80local ctx_stopregisterentries   = context.stopregisterentries
81local ctx_startregisterentry    = context.startregisterentry
82local ctx_stopregisterentry     = context.stopregisterentry
83local ctx_startregisterpages    = context.startregisterpages
84local ctx_stopregisterpages     = context.stopregisterpages
85local ctx_startregisterseewords = context.startregisterseewords
86local ctx_stopregisterseewords  = context.stopregisterseewords
87
88local ctx_registerentry         = context.registerentry
89local ctx_registerseeword       = context.registerseeword
90local ctx_registerpagerange     = context.registerpagerange
91local ctx_registeronepage       = context.registeronepage
92local ctx_registersection       = context.registersection
93local ctx_registerpacked        = context.registerpacked
94
95-- possible export, but ugly code (overloads)
96--
97-- local output, section, entries, nofentries, pages, words, rawtext
98--
99-- h_title = function(a,b) rawtext = a end
100--
101-- local function ctx_startregisteroutput()
102--     output     = { }
103--     section    = nil
104--     entries    = nil
105--     nofentries = nil
106--     pages      = nil
107--     words      = nil
108--     rawtext    = nil
109-- end
110-- local function ctx_stopregisteroutput()
111--     inspect(output)
112--     output     = nil
113--     section    = nil
114--     entries    = nil
115--     nofentries = nil
116--     pages      = nil
117--     words      = nil
118--     rawtext    = nil
119-- end
120-- local function ctx_startregistersection(tag)
121--     section = { }
122--     output[#output+1] = {
123--         section = section,
124--         tag     = tag,
125--     }
126-- end
127-- local function ctx_stopregistersection()
128-- end
129-- local function ctx_startregisterentries(n)
130--     entries = { }
131--     nofentries = 0
132--     section[#section+1] = entries
133-- end
134-- local function ctx_stopregisterentries()
135-- end
136-- local function ctx_startregisterentry(n) -- or subentries (nested?)
137--     nofentries = nofentries + 1
138--     entry = { }
139--     entries[nofentries] = entry
140-- end
141-- local function ctx_stopregisterentry()
142--     nofentries = nofentries - 1
143--     entry = entries[nofentries]
144-- end
145-- local function ctx_startregisterpages()
146--     pages = { }
147--     entry.pages = pages
148-- end
149-- local function ctx_stopregisterpages()
150-- end
151-- local function ctx_startregisterseewords()
152--     words = { }
153--     entry.words = words
154-- end
155-- local function ctx_stopregisterseewords()
156-- end
157-- local function ctx_registerentry(processor,internal,seeparent,text)
158--     text()
159--     entry.text = {
160--         processor = processor,
161--         internal  = internal,
162--         seeparent = seeparent,
163--         text      = rawtext,
164--     }
165-- end
166-- local function ctx_registerseeword(i,n,processor,internal,seeindex,seetext)
167--     seetext()
168--     entry.words[i] = {
169--         processor = processor,
170--         internal  = internal,
171--         seeparent = seeparent,
172--         seetext   = rawtext,
173--     }
174-- end
175-- local function ctx_registerpagerange(fprocessor,finternal,frealpage,lprocessor,linternal,lrealpage)
176--     pages[#pages+1] = {
177--         first = {
178--             processor = fprocessor,
179--             internal  = finternal,
180--             realpage  = frealpage,
181--         },
182--         last = {
183--             processor = lprocessor,
184--             internal  = linternal,
185--             realpage  = lrealpage,
186--         },
187--     }
188-- end
189-- local function ctx_registeronepage(processor,internal,realpage)
190--     pages[#pages+1] = {
191--         processor = processor,
192--         internal  = internal,
193--         realpage  = realpage,
194--     }
195-- end
196
197-- some day we will share registers and lists (although there are some conceptual
198-- differences in the application of keywords)
199
200local function filtercollected(names,criterium,number,collected,prevmode)
201    if not criterium or criterium == "" then
202        criterium = v_all
203    end
204    local data      = documents.data
205    local numbers   = data.numbers
206    local depth     = data.depth
207    local hash      = { }
208    local result    = { }
209    local nofresult = 0
210    local all       = not names or names == "" or names == v_all
211    local detail    = nil
212    if not all then
213        for s in gmatch(names,"[^, ]+") do
214            hash[s] = true
215        end
216    end
217    if criterium == v_all or criterium == v_text then
218        for i=1,#collected do
219            local v = collected[i]
220            if all then
221                nofresult = nofresult + 1
222                result[nofresult] = v
223            else
224                local vmn = v.metadata and v.metadata.name
225                if hash[vmn] then
226                    nofresult = nofresult + 1
227                    result[nofresult] = v
228                end
229            end
230        end
231    elseif criterium == v_current then
232        local collectedsections = sections.collected
233        for i=1,#collected do
234            local v = collected[i]
235            local sectionnumber = collectedsections[v.references.section]
236            if sectionnumber then
237                local cnumbers = sectionnumber.numbers
238                if prevmode then
239                    if (all or hash[v.metadata.name]) and #cnumbers >= depth then -- is the = ok for lists as well?
240                        local ok = true
241                        for d=1,depth do
242                            if not (cnumbers[d] == numbers[d]) then -- no zero test
243                                ok = false
244                                break
245                            end
246                        end
247                        if ok then
248                            nofresult = nofresult + 1
249                            result[nofresult] = v
250                        end
251                    end
252                else
253                    if (all or hash[v.metadata.name]) and #cnumbers > depth then
254                        local ok = true
255                        for d=1,depth do
256                            local cnd = cnumbers[d]
257                            if not (cnd == 0 or cnd == numbers[d]) then
258                                ok = false
259                                break
260                            end
261                        end
262                        if ok then
263                            nofresult = nofresult + 1
264                            result[nofresult] = v
265                        end
266                    end
267                end
268            end
269        end
270    elseif criterium == v_previous then
271        local collectedsections = sections.collected
272        for i=1,#collected do
273            local v = collected[i]
274            local sectionnumber = collectedsections[v.references.section]
275            if sectionnumber then
276                local cnumbers = sectionnumber.numbers
277                if (all or hash[v.metadata.name]) and #cnumbers >= depth then
278                    local ok = true
279                    if prevmode then
280                        for d=1,depth do
281                            if not (cnumbers[d] == numbers[d]) then
282                                ok = false
283                                break
284                            end
285                        end
286                    else
287                        for d=1,depth do
288                            local cnd = cnumbers[d]
289                            if not (cnd == 0 or cnd == numbers[d]) then
290                                ok = false
291                                break
292                            end
293                        end
294                    end
295                    if ok then
296                        nofresult = nofresult + 1
297                        result[nofresult] = v
298                    end
299                end
300            end
301        end
302    elseif criterium == variables["local"] then
303        if sections.autodepth(data.numbers) == 0 then
304            return filtercollected(names,v_all,number,collected,prevmode)
305        else
306            return filtercollected(names,v_current,number,collected,prevmode)
307        end
308    else -- sectionname, number
309        -- beware, this works ok for registers
310        -- to be redone with reference instead
311        local depth = sections.getlevel(criterium)
312        local number = tonumber(number) or numberatdepth(depth) or 0
313        if trace_registers then
314            detail = format("depth: %s, number: %s, numbers: %s, startset: %s",depth,number,concat(sections.numbers(),".",1,depth),#collected)
315        end
316        if number > 0 then
317            for i=1,#collected do
318                local v = collected[i]
319                local r = v.references
320                if r then
321                    local sectionnumber = sections.collected[r.section]
322                    if sectionnumber then
323                        local metadata = v.metadata
324                        local cnumbers = sectionnumber.numbers
325                        if cnumbers then
326                            if (all or hash[metadata.name or false]) and #cnumbers >= depth and matchingtilldepth(depth,cnumbers) then
327                                nofresult = nofresult + 1
328                                result[nofresult] = v
329                            end
330                        end
331                    end
332                end
333            end
334        end
335    end
336    if trace_registers then
337        if detail then
338            report_registers("criterium %a, detail %a, found %a",criterium,detail,#result)
339        else
340            report_registers("criterium %a, detail %a, found %a",criterium,nil,#result)
341        end
342    end
343    return result
344end
345
346local tobesaved           = allocate()
347local collected           = allocate()
348
349registers.collected       = collected
350registers.tobesaved       = tobesaved
351registers.filtercollected = filtercollected
352
353-- we follow a different strategy than by lists, where we have a global
354-- result table; we might do that here as well but since sorting code is
355-- older we delay that decision
356
357-- maybe store the specification in the format (although we predefine only
358-- saved registers)
359
360local function checker(t,k)
361    local v = {
362        metadata = {
363            language = 'en',
364            sorted   = false,
365            class    = class,
366        },
367        entries  = { },
368    }
369    t[k] = v
370    return v
371end
372
373local function initializer()
374    tobesaved = registers.tobesaved
375    collected = registers.collected
376    setmetatableindex(tobesaved,checker)
377    setmetatableindex(collected,checker)
378    local usedinternals = references.usedinternals
379    for name, list in next, collected do
380        local entries = list.entries
381        if not list.metadata.notsaved then
382            for e=1,#entries do
383                local entry = entries[e]
384                local r = entry.references
385                if r then
386                    local internal = r and r.internal
387                    if internal then
388                        internalreferences[internal] = entry
389                        usedinternals[internal] = r.used
390                    end
391                end
392            end
393        end
394    end
395 -- references.sortedinternals = sortedkeys(internalreferences) -- todo: when we need it more than once
396end
397
398local function finalizer()
399    local flaginternals = references.flaginternals
400    local usedviews     = references.usedviews
401    for k, v in next, tobesaved do
402        local entries = v.entries
403        if entries then
404            for i=1,#entries do
405                local r = entries[i].references
406                if r then
407                    local i = r.internal
408                    local f = flaginternals[i]
409                    if f then
410                        r.used = usedviews[i] or true
411                    end
412                end
413            end
414        end
415    end
416end
417
418job.register('structures.registers.collected', tobesaved, initializer, finalizer)
419
420setmetatableindex(tobesaved,checker)
421setmetatableindex(collected,checker)
422
423local function defineregister(class,method)
424    local d = tobesaved[class]
425    if method == v_forward then
426        d.metadata.notsaved = true
427    end
428end
429
430registers.define           = defineregister -- 4 times is somewhat over the top but we want consistency
431registers.setmethod        = defineregister -- and we might have a difference some day
432
433implement {
434    name      = "defineregister",
435    actions   = defineregister,
436    arguments = "2 strings",
437}
438
439implement {
440    name      = "setregistermethod",
441    actions   = defineregister, -- duplicate use
442    arguments = "2 strings",
443}
444
445
446local p_s = P("+")
447local p_e = P("&") * (1-P(";"))^0 * P(";")
448local p_r = C((p_e + (1-p_s))^0)
449
450local entrysplitter_xml = Ct(p_r * (p_s * p_r)^0) -- bah
451local entrysplitter_tex = lpeg.tsplitat('+')      -- & obsolete in mkiv
452
453local tagged = { }
454
455-- this whole splitting is an inheritance of mkii
456
457local function preprocessentries(rawdata)
458    local entries = rawdata.entries
459    if entries then
460        local processors = rawdata.processors
461        local et         = entries.entries
462        local kt         = entries.keys
463        local pt         = entries.processors
464        local entryproc  = processors and processors.entry
465        local pageproc   = processors and processors.page
466        local coding     = rawdata.metadata.coding
467        if entryproc == "" then
468            entryproc = nil
469        end
470        if pageproc == "" then
471            pageproc = nil
472        end
473        if not et then
474            local p, e = splitprocessor(entries.entry or "")
475            if p then
476                entryproc = p
477            end
478            et = lpegmatch(coding == "xml" and entrysplitter_xml or entrysplitter_tex,e)
479        end
480        if not kt then
481            local p, k = splitprocessor(entries.key or "")
482            if p then
483                pageproc = p
484            end
485            kt = lpegmatch(coding == "xml" and entrysplitter_xml or entrysplitter_tex,k)
486        end
487        if not pt then
488            pt = { }
489        end
490        --
491        entries = { }
492        local ok = false
493        for k=#et,1,-1 do
494            local etk = et[k]
495            local ktk = kt[k]
496            local ptk = pt[k]
497            if not ok and etk == "" then
498                entries[k] = nil
499            else
500                entries[k] = { etk or "", ktk ~= "" and ktk or false, ptk ~= "" and ptk or false }
501                ok = true
502            end
503        end
504        rawdata.list = entries
505        if pageproc or entryproc then
506            rawdata.processors = { entryproc, pageproc } -- old way: indexed .. will be keys
507        end
508        rawdata.entries = nil
509    end
510    local seeword = rawdata.seeword
511    if seeword then
512        seeword.processor, seeword.text = splitprocessor(seeword.text or "")
513    end
514end
515
516local function storeregister(rawdata) -- metadata, references, entries
517    local references = rawdata.references
518    local metadata   = rawdata.metadata
519    -- checking
520    if not metadata then
521        metadata = { }
522        rawdata.metadata = metadata
523    end
524    --
525    if not metadata.kind then
526        metadata.kind = "entry"
527    end
528    --
529    --
530    if not metadata.catcodes then
531        metadata.catcodes = tex.catcodetable -- get
532    end
533    --
534    local name     = metadata.name
535    local notsaved = tobesaved[name].metadata.notsaved
536    --
537    if not references then
538        references         = { }
539        rawdata.references = references
540    end
541    --
542    local internal = references.internal
543    if not internal then
544        internal = texgetcount("locationcount") -- we assume that it has been set
545        references.internal = internal
546    end
547    --
548    if notsaved then
549        usedinternals[internal] = references.used -- todo view (we assume that forward references index entries are used)
550    end
551    --
552    if not references.realpage then
553        references.realpage = 0 -- just to be sure as it can be refered to
554    end
555    --
556    local userdata = rawdata.userdata
557    if userdata then
558        rawdata.userdata = touserdata(userdata)
559    end
560    --
561    references.section = currentid()
562    metadata.level     = currentlevel()
563    --
564    local data     = notsaved and collected[name] or tobesaved[name]
565    local entries  = data.entries
566    internalreferences[internal] = rawdata
567    preprocessentries(rawdata)
568    entries[#entries+1] = rawdata
569    local label = references.label
570    if label and label ~= "" then
571        tagged[label] = #entries
572    else
573        references.label = nil
574    end
575    return #entries
576end
577
578local function enhanceregister(specification)
579    local name  = specification.name
580    local n     = specification.n
581    local saved = tobesaved[name]
582    local data  = saved.metadata.notsaved and collected[name] or saved
583    local entry = data.entries[n]
584    if entry then
585        entry.references.realpage = texgetcount("realpageno")
586    end
587end
588
589-- This can become extendregister(specification)!
590
591local function extendregister(name,tag,rawdata) -- maybe do lastsection internally
592    if type(tag) == "string" then
593        tag = tagged[tag]
594    end
595    if tag then
596        local data = tobesaved[name].metadata.notsaved and collected[name] or tobesaved[name]
597        local entry = data.entries[tag]
598        if entry then
599            local references = entry.references
600            references.lastrealpage = texgetcount("realpageno")
601            references.lastsection = currentid()
602            if rawdata then
603                local userdata = rawdata.userdata
604                if userdata then
605                    rawdata.userdata = touserdata(userdata)
606                end
607                if rawdata.entries then
608                    preprocessentries(rawdata)
609                end
610                local metadata = rawdata.metadata
611                if metadata and not metadata.catcodes then
612                    metadata.catcodes = tex.catcodetable -- get
613                end
614                for k, v in next, rawdata do
615                    local rk = references[k]
616                    if not rk then
617                        references[k] = v
618                    else
619                        for kk, vv in next, v do
620                            if type(vv) == "table" then
621                                if next(vv) then
622                                    rk[kk] = vv
623                                end
624                            elseif vv ~= "" then
625                                rk[kk] = vv
626                            end
627                        end
628                    end
629                end
630            end
631        end
632    end
633end
634
635registers.store   = storeregister
636registers.enhance = enhanceregister
637registers.extend  = extendregister
638
639function registers.get(tag,n)
640    local list = tobesaved[tag]
641    return list and list.entries[n]
642end
643
644implement {
645    name      = "enhanceregister",
646    arguments = { "string", "integer" },
647    actions   = function(name,n)
648        enhanceregister { name = name, n = n } -- todo: move to scanner
649    end,
650}
651
652implement {
653    name      = "deferredenhanceregister",
654    arguments = { "string", "integer" },
655    protected = true,
656    actions   = function(name,n)
657        ctx_latelua { action = enhanceregister, name = name, n = n }
658    end,
659}
660
661implement {
662    name      = "extendregister",
663    actions   = extendregister,
664    arguments = "2 strings",
665}
666
667implement {
668    name      = "storeregister",
669 -- actions   = function(rawdata)
670 --     local nofentries = storeregister(rawdata)
671 --     setinternalreference { internal = rawdata.references.internal }
672 --     context(nofentries)
673 -- end,
674    actions   = { storeregister, context },
675    arguments = {
676        {
677            { "metadata", {
678                    { "kind" },
679                    { "name" },
680                    { "coding" },
681                    { "level", "integer" },
682                    { "catcodes", "integer" },
683                    { "own" },
684                    { "xmlroot" },
685                    { "xmlsetup" },
686                }
687            },
688            { "entries", {
689                    { "entries", "list" },
690                    { "keys", "list" },
691                    { "processors", "list" },
692                    { "entry" },
693                    { "key" },
694                    { "processor" },
695                }
696            },
697            { "references", {
698                    { "internal", "integer" },
699                    { "section", "integer" },
700                    { "view" },
701                    { "label" },
702                }
703            },
704            { "seeword", {
705                    { "text" },
706                }
707            },
708            { "processors", {
709                    { "entry" },
710                    { "key" },
711                    { "page" },
712                }
713            },
714            { "userdata" },
715        }
716    }
717}
718
719-- sorting and rendering
720
721local compare = sorters.comparers.basic
722
723function registers.compare(a,b)
724    local result = compare(a,b)
725    if result ~= 0 then
726        return result
727    else
728        local ka = a.metadata.kind
729        local kb = b.metadata.kind
730        if ka == kb then
731            local ra = a.references
732            local rb = b.references
733            local pa = ra.realpage
734            local pb = rb.realpage
735            if not pa or not pb then
736                return 0
737            elseif pa < pb then
738                return -1
739            elseif pa > pb then
740                return  1
741            else
742                -- new, we need to pick the right one of multiple and
743                -- we want to prevent oscillation in the tuc file so:
744                local ia = ra.internal
745                local ib = rb.internal
746                if not ia or not ib then
747                    return 0
748                elseif ia < ib then
749                    return -1
750                elseif ia > ib then
751                    return  1
752                else
753                    return 0
754                end
755            end
756        elseif ka == "see" then
757            return 1
758        elseif kb == "see" then
759            return -1
760        end
761    end
762    return 0
763end
764
765function registers.filter(data,options)
766    data.result = registers.filtercollected(nil,options.criterium,options.number,data.entries,true)
767end
768
769local seeindex = 0
770
771-- meerdere loops, seewords, dan words, anders seewords
772
773-- todo: split seeword
774
775local function crosslinkseewords(result,check) -- all words
776    -- collect all seewords
777    local seewords = { }
778    for i=1,#result do
779        local data = result[i]
780        local seeword = data.seeword
781        if seeword then
782            local seetext = seeword.text
783            if seetext and not seewords[seetext] then
784                seeindex = seeindex + 1
785                seewords[seetext] = seeindex
786                if trace_registers then
787                    report_registers("see word %03i: %s",seeindex,seetext)
788                end
789            end
790        end
791    end
792
793    -- mark seeparents
794
795 -- local seeparents = { }
796 -- for i=1,#result do
797 --     local data = result[i]
798 --     local word = data.list[1]
799 --     local okay = word and word[1]
800 --     if okay then
801 --         local seeindex = seewords[okay]
802 --         if seeindex then
803 --             seeparents[okay] = data
804 --             data.references.seeparent = seeindex
805 --             if trace_registers then
806 --                 report_registers("see parent %03i: %s",seeindex,okay)
807 --             end
808 --         end
809 --     end
810 -- end
811
812    local entries    = { }
813    local keywords   = { }
814    local seeparents = { }
815    for i=1,#result do
816        local data = result[i]
817        local word = data.list
818        local size = #word
819        if data.seeword then
820            -- beware: a seeword has an extra entry for sorting purposes
821            size = size - 1
822        end
823        for i=1,size do
824            local w = word[i]
825            local e = w[1]
826            local k = w[2] or e
827            entries [i] = e
828            keywords[i] = k
829        end
830        -- first try the keys
831        local okay, seeindex
832        for n=size,1,-1 do
833            okay = concat(keywords,"+",1,n)
834            seeindex = seewords[okay]
835            -- first try the entries
836            if seeindex then
837                break
838            end
839            okay = concat(entries,"+",1,n)
840            seeindex = seewords[okay]
841            if seeindex then
842                break
843            end
844        end
845        if seeindex then
846            seeparents[okay] = data
847            data.references.seeparent = seeindex
848            if trace_registers then
849                report_registers("see parent %03i: %s",seeindex,okay)
850            end
851        end
852    end
853
854    -- mark seewords and extend sort list
855    for i=1,#result do
856        local data = result[i]
857        local seeword = data.seeword
858        if seeword then
859            local text = seeword.text
860            if text then
861                local seeparent = seeparents[text]
862                if seeparent then
863                    local seeindex = seewords[text]
864                    data.references.seeindex = seeindex
865                    if trace_registers then
866                        report_registers("see crosslink %03i: %s",seeindex,text)
867                    end
868                    seeword.valid = true
869                elseif check then
870                    report_registers("invalid crosslink : %s, %s",text,"ignored")
871                    seeword.valid = false
872                else
873                    report_registers("invalid crosslink : %s, %s",text,"passed")
874                    seeword.valid = true
875                end
876            end
877        end
878    end
879end
880
881local function removeemptyentries(result)
882    local i, n, m = 1, #result, 0
883    while i <= n do
884        local entry = result[i]
885        if #entry.list == 0 or #entry.split == 0 then
886            remove(result,i)
887            n = n - 1
888            m = m + 1
889        else
890            i = i + 1
891        end
892    end
893    if m > 0 then
894        report_registers("%s empty entries removed in register",m)
895    end
896end
897
898function registers.prepare(data,options)
899    -- data has 'list' table
900    local strip    = sorters.strip
901    local splitter = sorters.splitters.utf
902    local result   = data.result
903    if result then
904        local seeprefix = char(0)
905        for i=1, #result do
906            local entry = result[i]
907            local split = { }
908            local list  = entry.list
909            if list then
910                if entry.seeword then
911                    -- we can have multiple seewords, only two levels supported
912                    list[#list+1] = { seeprefix .. strip(entry.seeword.text) }
913                end
914                for l=1,#list do
915                    local ll   = list[l]
916                    local word = ll[1]
917                    local key  = ll[2]
918                    if not key or key == "" then
919                        key = word
920                    end
921                    split[l] = splitter(strip(key))
922                end
923            end
924            entry.split = split
925        end
926        removeemptyentries(result)
927        crosslinkseewords(result,options.check ~= v_no)
928    end
929end
930
931function registers.sort(data,options)
932 -- if options.pagenumber == false then
933 --     sorters.sort(data.result,compare)
934 -- else
935        sorters.sort(data.result,registers.compare)
936 -- end
937end
938
939function registers.unique(data,options)
940    local result     = { }
941    local nofresult  = 0
942    local prev       = nil
943    local dataresult = data.result
944    local bysection  = options.pagemethod == v_section -- normally page
945    for k=1,#dataresult do
946        local v = dataresult[k]
947        if prev then
948            local vr = v.references
949            local pr = prev.references
950            if not equal(prev.list,v.list) then
951                -- ok
952            elseif bysection and vr.section == pr.section then
953                v = nil
954                -- ok
955            elseif pr.realpage ~= vr.realpage then
956                -- ok
957            else
958                local pl = pr.lastrealpage
959                local vl = vr.lastrealpage
960                if pl or vl then
961                    if not vl then
962                        -- ok
963                    elseif not pl then
964                        -- ok
965                    elseif pl ~= vl then
966                        -- ok
967                    else
968                        v = nil
969                    end
970                else
971                    v = nil
972                end
973            end
974        end
975        if v then
976            nofresult = nofresult + 1
977            result[nofresult] = v
978            prev = v
979        end
980    end
981    data.result = result
982end
983
984function registers.finalize(data,options) -- maps character to index (order)
985    local result = data.result
986    data.metadata.nofsorted = #result
987    local split    = { }
988    local nofsplit = 0
989    local lasttag  = nil
990    local done     = nil
991    local nofdone  = 0
992    local firstofsplit = sorters.firstofsplit
993    for k=1,#result do
994        local v = result[k]
995        local entry, tag = firstofsplit(v)
996        if tag ~= lasttag then
997            if trace_registers then
998                report_registers("splitting at %a",tag)
999            end
1000            done     = { }
1001            nofdone  = 0
1002            nofsplit = nofsplit + 1
1003            lasttag  = tag
1004            split[nofsplit] = { tag = tag, data = done }
1005        end
1006        nofdone = nofdone + 1
1007        done[nofdone] = v
1008    end
1009    data.result = split
1010end
1011
1012-- local function analyzeregister(class,options)
1013--     local data = collected[class]
1014--     if data and data.entries then
1015--         options = options or { }
1016--         sorters.setlanguage(options.language,options.method,options.numberorder)
1017--         registers.filter(data,options)   -- filter entries into results (criteria)
1018--         registers.prepare(data,options)  -- adds split table parallel to list table
1019--         registers.sort(data,options)     -- sorts results
1020--         registers.unique(data,options)   -- get rid of duplicates
1021--         registers.finalize(data,options) -- split result in ranges
1022--         data.metadata.sorted = true
1023--         return data.metadata.nofsorted or 0
1024--     else
1025--         return 0
1026--     end
1027-- end
1028
1029local function analyzeregister(class,options)
1030    local data = rawget(collected,class)
1031    if not data then
1032        local list     = utilities.parsers.settings_to_array(class)
1033        local entries  = { }
1034        local metadata = false
1035        for i=1,#list do
1036            local l = list[i]
1037            local d = collected[l]
1038            local e = d.entries
1039            for i=1,#e do
1040                entries[#entries+1] = e[i]
1041            end
1042            if not metadata then
1043                metadata = d.metadata
1044            end
1045        end
1046        data = {
1047            metadata = metadata or { },
1048            entries  = entries,
1049        }
1050        collected[class] = data
1051    end
1052    if data and data.entries then
1053        options = options or { }
1054        sorters.setlanguage(options.language,options.method,options.numberorder)
1055        registers.filter(data,options)   -- filter entries into results (criteria)
1056        registers.prepare(data,options)  -- adds split table parallel to list table
1057        registers.sort(data,options)     -- sorts results
1058        registers.unique(data,options)   -- get rid of duplicates
1059        registers.finalize(data,options) -- split result in ranges
1060        data.metadata.sorted = true
1061        return data.metadata.nofsorted or 0
1062    else
1063        return 0
1064    end
1065end
1066
1067registers.analyze = analyzeregister
1068
1069implement {
1070    name      = "analyzeregister",
1071    actions   = { analyzeregister, context },
1072    arguments = {
1073        "string",
1074        {
1075            { "language" },
1076            { "method" },
1077            { "numberorder" },
1078            { "compress" },
1079            { "criterium" },
1080            { "pagenumber", "boolean" },
1081        }
1082    }
1083}
1084
1085-- todo take conversion from index
1086
1087function registers.userdata(index,name)
1088    local data = references.internals[tonumber(index)]
1089    return data and data.userdata and data.userdata[name] or nil
1090end
1091
1092implement {
1093    name      = "registeruserdata",
1094    actions   = { registers.userdata, context },
1095    arguments = { "integer", "string" }
1096}
1097
1098-- todo: ownnumber
1099
1100local function pagerange(f_entry,t_entry,is_last,prefixspec,pagespec)
1101    local fer = f_entry.references
1102    local ter = t_entry.references
1103    ctx_registerpagerange(
1104        f_entry.metadata.name or "",
1105        f_entry.processors and f_entry.processors[2] or "",
1106        fer.internal or 0,
1107        fer.realpage or 0,
1108        function()
1109            h_prefixpage(f_entry,prefixspec,pagespec)
1110        end,
1111        ter.internal or 0,
1112        ter.lastrealpage or ter.realpage or 0,
1113        function()
1114            if is_last then
1115                h_prefixlastpage(t_entry,prefixspec,pagespec) -- swaps page and realpage keys
1116            else
1117                h_prefixpage    (t_entry,prefixspec,pagespec)
1118            end
1119        end
1120    )
1121end
1122
1123local function pagenumber(entry,prefixspec,pagespec)
1124    local er = entry.references
1125    ctx_registeronepage(
1126        entry.metadata.name or "",
1127        entry.processors and entry.processors[2] or "",
1128        er.internal or 0,
1129        er.realpage or 0,
1130        function() h_prefixpage(entry,prefixspec,pagespec) end
1131    )
1132end
1133
1134local function packed(f_entry,t_entry)
1135    local fer = f_entry.references
1136    local ter = t_entry.references
1137    ctx_registerpacked(
1138        fer.internal or 0,
1139        ter.internal or 0
1140    )
1141end
1142
1143local function collapsedpage(pages)
1144    for i=2,#pages do
1145        local first             = pages[i-1]
1146        local second            = pages[i]
1147        local first_first       = first[1]
1148        local first_last        = first[2]
1149        local second_first      = second[1]
1150        local second_last       = second[2]
1151        local first_last_pn     = first_last  .references.realpage
1152        local second_first_pn   = second_first.references.realpage
1153        local second_last_pn    = second_last .references.realpage
1154        local first_last_last   = first_last  .references.lastrealpage
1155        local second_first_last = second_first.references.lastrealpage
1156        if first_last_last then
1157            first_last_pn = first_last_last
1158            if first_last_pn == second_first_pn then
1159                -- 2-3, 3-9 -> 2-9
1160                pages[i-1] = { first_first, second_last }
1161                remove(pages,i)
1162                return true
1163            elseif second_first == second_last and second_first_pn <= first_last_pn then
1164                -- 2=8, 5 -> 12=8
1165                remove(pages,i)
1166                return true
1167            elseif second_first == second_last and second_first_pn > first_last_pn then
1168                -- 2=8, 9 -> 2-9
1169                pages[i-1] = { first_first, second_last }
1170                remove(pages,i)
1171                return true
1172            elseif second_last_pn < first_last_pn then
1173                -- 2=8, 3-4 -> 2=8
1174                remove(pages,i)
1175                return true
1176            elseif first_last_pn < second_last_pn then
1177                -- 2=8, 3-9 -> 2-9
1178                pages[i-1] = { first_first, second_last }
1179                remove(pages,i)
1180                return true
1181            elseif first_last_pn + 1 == second_first_pn and second_last_pn > first_last_pn then
1182                -- 2=8, 9-11 -> 2-11
1183                pages[i-1] = { first_first, second_last }
1184                remove(pages,i)
1185                return true
1186            elseif second_first.references.lastrealpage then
1187                -- 2=8, 9=11 -> 2-11
1188                pages[i-1] = { first_first, second_last }
1189                remove(pages,i)
1190                return true
1191            end
1192        elseif second_first_last then
1193            second_first_pn = second_first_last
1194            if first_last_pn == second_first_pn then
1195                -- 2-4, 5=9 -> 2-9
1196                pages[i-1] = { first_first, second_last }
1197                remove(pages,i)
1198                return true
1199            end
1200        elseif first_last_pn == second_first_pn then
1201            -- 2-3, 3-4 -> 2-4
1202            pages[i-1] = { first_last, second_last }
1203            remove(pages,i)
1204            return true
1205        end
1206    end
1207    return false
1208end
1209
1210local function collapsepages(pages)
1211    while collapsedpage(pages) do end
1212    return #pages
1213end
1214
1215-- todo: create an intermediate structure and flush that
1216
1217function registers.flush(data,options,prefixspec,pagespec)
1218    local compress         = options.compress
1219    local collapse_singles = compress == v_yes
1220    local collapse_ranges  = compress == v_all
1221    local collapse_packed  = compress == v_packed
1222    local show_page_number = options.pagenumber ~= false -- true or false
1223    local bysection        = options.pagemethod == v_section
1224    local result = data.result
1225    local maxlevel = 0
1226    --
1227    for i=1,#result do
1228        local data = result[i].data
1229        for d=1,#data do
1230            local m = #data[d].list
1231            if m > maxlevel then
1232                maxlevel = m
1233            end
1234        end
1235    end
1236    if maxlevel > absmaxlevel then
1237        maxlevel = absmaxlevel
1238        report_registers("limiting level to %a",maxlevel)
1239    end
1240    --
1241    ctx_startregisteroutput()
1242    local done    = { }
1243    local started = false
1244    for i=1,#result do
1245     -- ranges need checking !
1246        local sublist = result[i]
1247     -- local done = { false, false, false, false }
1248        for i=1,maxlevel do
1249            done[i] = false
1250        end
1251        local data = sublist.data
1252        local d    = 0
1253        local n    = 0
1254        ctx_startregistersection(sublist.tag)
1255        for d=1,#data do
1256            local entry = data[d]
1257            if entry.metadata.kind == "see" then
1258                local list = entry.list
1259                if #list > 1 then
1260                    list[#list] = nil
1261                else
1262                 -- we have an \seeindex{Foo}{Bar} without Foo being defined anywhere .. somehow this message is wrong
1263                 -- report_registers("invalid see entry in register %a, reference %a",entry.metadata.name,list[1][1])
1264                end
1265            end
1266        end
1267        -- ok, this is tricky: we use e[i] delayed so we need it to be local
1268        -- but we don't want to allocate too many entries so there we go
1269
1270        while d < #data do
1271            d = d + 1
1272            local entry    = data[d]
1273-- inspect(entry)
1274            local metadata = entry.metadata
1275            local kind     = metadata.kind
1276            local list     = entry.list
1277            local e = { false, false, false }
1278            for i=3,maxlevel do
1279                e[i] = false
1280            end
1281            for i=1,maxlevel do
1282                local li = list[i]
1283                if list[i] then
1284                    e[i] = li[1]
1285                end
1286                if e[i] == done[i] then
1287                    -- skip
1288                elseif not e[i] then
1289                    -- see ends up here
1290                    -- can't happen any more
1291                    done[i] = false
1292                    for j=i+1,maxlevel do
1293                        done[j] = false
1294                    end
1295                elseif e[i] == "" then
1296                    done[i] = false
1297                    for j=i+1,maxlevel do
1298                        done[j] = false
1299                    end
1300                else
1301                    done[i] = e[i]
1302                    for j=i+1,maxlevel do
1303                        done[j] = false
1304                    end
1305                    if started then
1306                        ctx_stopregisterentry()
1307                        started = false
1308                    end
1309                    if n == i then
1310                     -- ctx_stopregisterentries()
1311                     -- ctx_startregisterentries(n)
1312                    else
1313                        while n > i do
1314                            n = n - 1
1315                            ctx_stopregisterentries()
1316                        end
1317                        while n < i do
1318                            n = n + 1
1319                            ctx_startregisterentries(n)
1320                        end
1321                    end
1322                    local references = entry.references
1323                    local processors = entry.processors
1324                    local internal   = references.internal or 0
1325                    local seeparent  = references.seeparent or ""
1326                    local processor  = (li and li[3]) or (processors and processors[1]) or ""
1327                    -- so, we need to keep e as is (local), or we need local title = e[i] ... which might be
1328                    -- more of a problem
1329                    ctx_startregisterentry(0) -- will become a counter
1330                    started = true
1331                    if metadata then
1332                        ctx_registerentry(metadata.name or "",processor,internal,seeparent,function() h_title(e[i],metadata) end)
1333                    else
1334                        -- can this happen?
1335                        ctx_registerentry("",processor,internal,seeindex,e[i])
1336                    end
1337                end
1338            end
1339
1340            local function case_1()
1341                -- we collapse ranges and keep existing ranges as they are
1342                -- so we get prebuilt as well as built ranges
1343                local first, last, prev, pages, dd, nofpages = entry, nil, entry, { }, d, 0
1344                while dd < #data do
1345                    dd = dd + 1
1346                    local next = data[dd]
1347                    if next and next.metadata.kind == "see" then
1348                        dd = dd - 1
1349                        break
1350                    else
1351                        local el, nl = entry.list, next.list
1352                        if not equal(el,nl) then
1353                            dd = dd - 1
1354                        --~ first = nil
1355                            break
1356                        elseif next.references.lastrealpage then
1357                            nofpages = nofpages + 1
1358                            pages[nofpages] = first and { first, last or first } or { entry, entry }
1359                            nofpages = nofpages + 1
1360                            pages[nofpages] = { next, next }
1361                            first, last, prev = nil, nil, nil
1362                        elseif not first then
1363                            first, prev = next, next
1364                        elseif next.references.realpage - prev.references.realpage == 1 then -- 1 ?
1365                            last, prev = next, next
1366                        else
1367                            nofpages = nofpages + 1
1368                            pages[nofpages] = { first, last or first }
1369                            first, last, prev = next, nil, next
1370                        end
1371                    end
1372                end
1373                if first then
1374                    nofpages = nofpages + 1
1375                    pages[nofpages] = { first, last or first }
1376                end
1377                if collapse_ranges and nofpages > 1 then
1378                    nofpages = collapsepages(pages)
1379                end
1380                if nofpages > 0 then -- or 0
1381                    d = dd
1382                    for p=1,nofpages do
1383                        local first, last = pages[p][1], pages[p][2]
1384                        if first == last then
1385                            if first.references.lastrealpage then
1386                                pagerange(first,first,true,prefixspec,pagespec)
1387                            else
1388                                pagenumber(first,prefixspec,pagespec)
1389                            end
1390                        elseif last.references.lastrealpage then
1391                            pagerange(first,last,true,prefixspec,pagespec)
1392                        else
1393                            pagerange(first,last,false,prefixspec,pagespec)
1394                        end
1395                    end
1396                elseif entry.references.lastrealpage then
1397                    pagerange(entry,entry,true,prefixspec,pagespec)
1398                else
1399                    pagenumber(entry,prefixspec,pagespec)
1400                end
1401            end
1402
1403            local function case_2()
1404                local first = nil
1405                local last  = nil
1406                while true do
1407                    if not first then
1408                        first = entry
1409                    end
1410                    last  = entry
1411                    if d == #data then
1412                        break
1413                    else
1414                        d = d + 1
1415                        local next = data[d]
1416                        if next.metadata.kind == "see" or not equal(entry.list,next.list) then
1417                            d = d - 1
1418                            break
1419                        else
1420                            entry = next
1421                        end
1422                    end
1423                end
1424                packed(first,last) -- returns internals
1425            end
1426
1427            local function case_3()
1428                while true do
1429                    if entry.references.lastrealpage then
1430                        pagerange(entry,entry,true,prefixspec,pagespec)
1431                    else
1432                        pagenumber(entry,prefixspec,pagespec)
1433                    end
1434                    if d == #data then
1435                        break
1436                    else
1437                        d = d + 1
1438                        local next = data[d]
1439                        if next.metadata.kind == "see" or not equal(entry.list,next.list) then
1440                            d = d - 1
1441                            break
1442                        else
1443                            entry = next
1444                        end
1445                    end
1446                end
1447            end
1448
1449            local function case_4()
1450                local t  = { }
1451                local nt = 0
1452                while true do
1453                    if entry.seeword and entry.seeword.valid then
1454                        nt = nt + 1
1455                        t[nt] = entry
1456                    end
1457                    if d == #data then
1458                        break
1459                    else
1460                        d = d + 1
1461                        local next = data[d]
1462                        if next.metadata.kind ~= "see" or not equal(entry.list,next.list) then
1463                            d = d - 1
1464                            break
1465                        else
1466                            entry = next
1467                        end
1468                    end
1469                end
1470                for i=1,nt do
1471                    local entry     = t[i]
1472                    local seeword   = entry.seeword
1473                    local seetext   = seeword.text or ""
1474                    local processor = seeword.processor or (entry.processors and entry.processors[1]) or ""
1475                    local seeindex  = entry.references.seeindex or ""
1476                    ctx_registerseeword(
1477                        metadata.name or "",
1478                        i,
1479                        nt,
1480                        processor,
1481                        0,
1482                        seeindex,
1483                        function() h_title(seetext,metadata) end
1484                    )
1485                end
1486            end
1487
1488            local function case_5()
1489                local first = d
1490                while true do
1491                    if d == #data then
1492                        break
1493                    else
1494                        d = d + 1
1495                        local next = data[d]
1496                        if next.metadata.kind == "see" or not equal(entry.list,next.list) then
1497                            d = d - 1
1498                            break
1499                        else
1500                            entry = next
1501                        end
1502                    end
1503                end
1504                local last      = d
1505                local n         = last - first + 1
1506                local i         = 0
1507                local name      = metadata.name or ""
1508                local processor = entry.processors and entry.processors[1] or ""
1509                for e=first,last do
1510                    local d = data[e]
1511                    local sectionindex = d.references.internal or 0
1512                    i = i + 1
1513                    ctx_registersection(
1514                        name,
1515                        i,
1516                        n,
1517                        processor,
1518                        0,
1519                        sectionindex,
1520                        function() h_prefix(d,prefixspec,true) end
1521                    )
1522                end
1523            end
1524
1525            if kind == "entry" then
1526                if show_page_number then
1527                    ctx_startregisterpages()
1528                    if bysection then
1529                        case_5()
1530                    elseif collapse_singles or collapse_ranges then
1531                        case_1()
1532                    elseif collapse_packed then
1533                        case_2()
1534                    else
1535                        case_3()
1536                    end
1537                    ctx_stopregisterpages()
1538                end
1539            elseif kind == "see" then
1540                ctx_startregisterseewords()
1541                case_4()
1542                ctx_stopregisterseewords()
1543            end
1544
1545        end
1546
1547        if started then
1548            ctx_stopregisterentry()
1549            started = false
1550        end
1551        while n > 0 do
1552            ctx_stopregisterentries()
1553            n = n - 1
1554        end
1555        ctx_stopregistersection()
1556    end
1557    ctx_stopregisteroutput()
1558    -- for now, maybe at some point we will do a multipass or so
1559    data.result = nil
1560    data.metadata.sorted = false
1561    -- temp hack for luajittex :
1562    local entries = data.entries
1563    for i=1,#entries do
1564        entries[i].split = nil
1565    end
1566 -- collectgarbage("collect")
1567end
1568
1569function registers.process(class,...)
1570    if analyzeregister(class,...) > 0 then
1571        local data = collected[class]
1572        registers.flush(data,...)
1573    end
1574end
1575
1576implement {
1577    name      = "processregister",
1578    actions   = registers.process,
1579    arguments = {
1580        "string",
1581        {
1582            { "language" },
1583            { "method" },
1584            { "numberorder" },
1585            { "compress" },
1586            { "criterium" },
1587            { "check" },
1588            { "pagemethod" },
1589            { "pagenumber", "boolean" },
1590        },
1591        {
1592            { "separatorset" },
1593            { "conversionset" },
1594            { "starter" },
1595            { "stopper" },
1596            { "set" },
1597            { "segments" },
1598            { "connector" },
1599        },
1600        {
1601            { "prefix" },
1602            { "separatorset" },
1603            { "conversionset" },
1604            { "starter" },
1605            { "stopper" },
1606            { "segments" },
1607        }
1608    }
1609}
1610
1611-- linked registers
1612
1613function registers.findinternal(tag,where,n)
1614 -- local collected = registers.collected
1615    local current   = collected[tag]
1616    if not current then
1617        return 0
1618    end
1619    local entries = current.entries
1620    if not entries then
1621        return 0
1622    end
1623    local entry = entries[n]
1624    if not entry then
1625        return 0
1626    end
1627    local list = entry.list
1628    local size = #list
1629    --
1630    local start, stop, step
1631    if where == v_previous then
1632        start = n - 1
1633        stop  = 1
1634        step  = -1
1635    elseif where == v_first then
1636        start = 1
1637        stop  = #entries
1638        step  = 1
1639    elseif where == v_last then
1640        start = #entries
1641        stop  = 1
1642        step  = -1
1643    else
1644        start = n + 1
1645        stop  = #entries
1646        step  = 1
1647    end
1648    --
1649    for i=start,stop,step do
1650        local r = entries[i]
1651        local l = r.list
1652        local s = #l
1653        if s == size then
1654            local ok = true
1655            for i=1,size do
1656                if list[i][1] ~= l[i][1] then
1657                    ok = false
1658                    break
1659                end
1660            end
1661            if ok then
1662                return r.references.internal or 0
1663            end
1664        end
1665    end
1666    return 0
1667end
1668
1669interfaces.implement {
1670    name      = "findregisterinternal",
1671    arguments = { "string", "string", "integer" },
1672    actions   = { registers.findinternal, context },
1673}
1674