back-exp.lua /size: 150 Kb    last modification: 2025-02-21 11:03
1if not modules then modules = { } end modules ['back-exp'] = {
2    version   = 1.001,
3    comment   = "companion to back-exp.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- Todo: share properties more with tagged pdf (or thge reverse)
10
11-- Because we run into the 200 local limit we quite some do .. end wrappers .. not always
12-- that nice but it has to be.
13
14-- Experiments demonstrated that mapping to <div> and classes is messy because we have to
15-- package attributes (some 30) into one set of (space seperatated but prefixed classes)
16-- which only makes things worse .. so if you want something else, use xslt to get there.
17
18-- language       -> only mainlanguage, local languages should happen through start/stoplanguage
19-- tocs/registers -> maybe add a stripper (i.e. just don't flush entries in final tree)
20-- footnotes      -> css 3
21-- bodyfont       -> in styles.css
22
23-- Because we need to look ahead we now always build a tree (this was optional in
24-- the beginning). The extra overhead in the frontend is neglectable.
25--
26-- We can optimize the code ... currently the overhead is some 10% for xml + html so
27-- there is no hurry.
28
29-- todo: move critital formatters out of functions
30-- todo: delay loading (apart from basic tag stuff)
31
32-- problem : too many local variables
33
34-- check setting __i__
35
36local next, type, tonumber = next, type, tonumber
37local sub, gsub, match = string.sub, string.gsub, string.match
38local validstring = string.valid
39local lpegmatch = lpeg.match
40local utfchar, utfvalues, utflen = utf.char, utf.values, utf.len
41local concat, insert, remove, merge, sort = table.concat, table.insert, table.remove, table.merge, table.sort
42local sortedhash, sortedkeys = table.sortedhash, table.sortedkeys
43local formatters = string.formatters
44local todimen = number.todimen
45local replacetemplate = utilities.templates.replace
46
47local addsuffix, joinfile, nameonly, basename, filesuffix = file.addsuffix, file.join, file.nameonly, file.basename, file.suffix
48
49local trace_export  = false  trackers.register  ("export.trace",         function(v) trace_export  = v end)
50local trace_spacing = false  trackers.register  ("export.trace.spacing", function(v) trace_spacing = v end)
51local trace_details = false  trackers.register  ("export.trace.details", function(v) trace_details = v end)
52
53local less_state    = false  directives.register("export.lessstate",     function(v) less_state    = v end)
54local show_comment  = true   directives.register("export.comment",       function(v) show_comment  = v end)
55
56show_comment = false -- figure out why break comment
57
58-- maybe we will also support these:
59--
60-- local css_hyphens       = false  directives.register("export.css.hyphens",      function(v) css_hyphens      = v end)
61-- local css_textalign     = false  directives.register("export.css.textalign",    function(v) css_textalign    = v end)
62-- local css_bodyfontsize  = false  directives.register("export.css.bodyfontsize", function(v) css_bodyfontsize = v end)
63-- local css_textwidth     = false  directives.register("export.css.textwidth",    function(v) css_textwidth    = v end)
64
65local report_export     = logs.reporter("backend","export")
66
67local nodes             = nodes
68local attributes        = attributes
69
70local variables         = interfaces.variables
71local v_yes             = variables.yes
72local v_no              = variables.no
73local v_xml             = variables.xml
74local v_hidden          = variables.hidden
75
76local implement         = interfaces.implement
77
78local included          = backends.included
79
80local settings_to_array = utilities.parsers.settings_to_array
81local settings_to_hash  = utilities.parsers.settings_to_hash
82
83local setmetatableindex = table.setmetatableindex
84local tasks             = nodes.tasks
85local fontchar          = fonts.hashes.characters
86local fontquads         = fonts.hashes.quads
87local languagenames     = languages.numbers
88
89local texgetcount       = tex.getcount
90
91local references        = structures.references
92local structurestags    = structures.tags
93local taglist           = structurestags.taglist
94local specifications    = structurestags.specifications
95local properties        = structurestags.properties
96local locatedtag        = structurestags.locatedtag
97
98structurestags.usewithcare = { }
99
100local starttiming       = statistics.starttiming
101local stoptiming        = statistics.stoptiming
102
103local characterdata     = characters.data
104local overloads         = fonts.mappings.overloads
105
106-- todo: more locals (and optimize)
107
108local exportversion     = "0.35"
109local mathmlns          = "http://www.w3.org/1998/Math/MathML"
110local contextns         = "http://www.contextgarden.net/context/export" -- whatever suits
111local cssnamespaceurl   = "@namespace context url('%namespace%') ;"
112local cssnamespace      = "context|"
113----- cssnamespacenop   = "/* no namespace */"
114
115local usecssnamespace   = false
116
117local nofcurrentcontent = 0 -- so we don't free (less garbage collection)
118local currentcontent    = { }
119local currentnesting    = nil
120local currentattribute  = nil
121local last              = nil
122local currentparagraph  = nil
123
124local noftextblocks     = 0
125
126----- hyphencode        = 0xAD
127local hyphen            = utfchar(0xAD) -- todo: also emdash etc
128local tagsplitter       = structurestags.patterns.splitter
129----- colonsplitter     = lpeg.splitat(":")
130----- dashsplitter      = lpeg.splitat("-")
131local threshold         = 65536
132local indexing          = false
133local keephyphens       = false
134local exportproperties  = false
135
136local finetuning        = { }
137
138local treestack         = { }
139local nesting           = { }
140local currentdepth      = 0
141
142local wrapups           = { }
143
144local tree              = { data = { }, fulltag == "root" } -- root
145local treeroot          = tree
146local treehash          = { }
147local extras            = { }
148local checks            = { }
149local fixes             = { }
150local finalizers        = { }
151local nofbreaks         = 0
152local used              = { }
153local exporting         = false
154local restart           = false
155local specialspaces     = { [0x20] = " "  }               -- for conversion
156local somespace         = { [0x20] = true, [" "] = true } -- for testing
157local entities          = { ["&"] = "&amp;", [">"] = "&gt;", ["<"] = "&lt;" }
158local attribentities    = { ["&"] = "&amp;", [">"] = "&gt;", ["<"] = "&lt;", ['"'] = "quot;" }
159
160local p_entity          = lpeg.replacer(entities) -- was: entityremapper = utf.remapper(entities)
161local p_attribute       = lpeg.replacer(attribentities)
162local p_stripper        = lpeg.patterns.stripper
163local p_escaped         = lpeg.patterns.xml.escaped
164
165local f_tagid           = formatters["%s-%04i"]
166
167-- local alignmapping = {
168--     flushright = "right",
169--     middle     = "center",
170--     flushleft  = "left",
171-- }
172
173local defaultnature = "mixed" -- "inline"
174
175setmetatableindex(used, function(t,k)
176    if k then
177        local v = { }
178        t[k] = v
179        return v
180    end
181end)
182
183local f_entity    = formatters["&#x%X;"]
184local f_attribute = formatters[" %s=%q"]
185local f_property  = formatters[" %s%s=%q"]
186
187setmetatableindex(specialspaces, function(t,k)
188    local v = utfchar(k)
189    t[k] = v
190    entities[v] = f_entity(k)
191    somespace[k] = true
192    somespace[v] = true
193    return v
194end)
195
196
197local namespaced = {
198    -- filled on
199}
200
201local namespaces = {
202    msubsup      = "m",
203    msub         = "m",
204    msup         = "m",
205    mn           = "m",
206    mi           = "m",
207    ms           = "m",
208    mo           = "m",
209    mtext        = "m",
210    mrow         = "m",
211    mfrac        = "m",
212    mroot        = "m",
213    msqrt        = "m",
214    munderover   = "m",
215    munder       = "m",
216    mover        = "m",
217    merror       = "m",
218    math         = "m",
219    mrow         = "m",
220    mtable       = "m",
221    mtr          = "m",
222    mtd          = "m",
223    mfenced      = "m",
224    maction      = "m",
225    mspace       = "m",
226    -- only when testing
227    mstacker     = "m",
228    mstackertop  = "m",
229    mstackermid  = "m",
230    mstackerbot  = "m",
231}
232
233setmetatableindex(namespaced, function(t,k)
234    if k then
235        local namespace = namespaces[k]
236        local v = namespace and namespace .. ":" .. k or k
237        t[k] = v
238        return v
239    end
240end)
241
242local function attribute(key,value)
243    if value and value ~= "" then
244        return f_attribute(key,lpegmatch(p_attribute,value))
245    else
246        return ""
247    end
248end
249
250local function setattribute(di,key,value,escaped)
251    if value and value ~= "" then
252        local a = di.attributes
253        if escaped then
254            value = lpegmatch(p_escaped,value)
255        end
256        if not a then
257            di.attributes = { [key] = value }
258        else
259            a[key] = value
260        end
261    end
262end
263
264local listdata = { } -- this has to be done otherwise: each element can just point back to ...
265
266function wrapups.hashlistdata()
267    local c = structures.lists.collected
268    for i=1,#c do
269        local ci = c[i]
270        local tag = ci.references.tag
271        if tag then
272            local m = ci.metadata
273            local t = m.kind .. ">" .. tag -- todo: use internal (see strc-lst.lua where it's set)
274            listdata[t] = ci
275        end
276    end
277end
278
279function structurestags.setattributehash(attr,key,value) -- public hash
280    local specification = taglist[attr]
281    if specification then
282        specification[key] = value
283    else
284        -- some kind of error
285    end
286end
287
288local usedstyles = { }
289
290local namespacetemplate = [[
291/* %what% for file %filename% */
292
293%cssnamespaceurl%
294]]
295
296do
297
298    -- experiment: styles and images
299    --
300    -- officially we should convert to bp but we round anyway
301
302    -- /* padding      : ; */
303    -- /* text-justify : inter-word ; */
304    -- /* text-align : justify ; */
305
306local documenttemplate = [[
307document,
308%namespace%div.document {
309    font-size  : %size% !important ;
310    max-width  : %width% !important ;
311    text-align : %align% !important ;
312    hyphens    : %hyphens% !important ;
313}]]
314
315local styletemplate = [[
316%element%[detail="%detail%"],
317%namespace%div.%element%.%detail% {
318    display      : inline ;
319    font-style   : %style% ;
320    font-variant : %variant% ;
321    font-weight  : %weight% ;
322    font-family  : %family% ;
323    color        : %color% ;
324}]]
325
326    local numbertoallign = {
327        [0] = "justify", ["0"] = "justify", [variables.normal    ] = "justify",
328              "right",   ["1"] = "right",   [variables.flushright] = "right",
329              "center",  ["2"] = "center",  [variables.middle    ] = "center",
330              "left",    ["3"] = "left",    [variables.flushleft ] = "left",
331    }
332
333    function wrapups.allusedstyles(filename)
334        local result = { replacetemplate(namespacetemplate, {
335            what            = "styles",
336            filename        = filename,
337            namespace       = contextns,
338         -- cssnamespaceurl = usecssnamespace and cssnamespaceurl or cssnamespacenop,
339            cssnamespaceurl = cssnamespaceurl,
340        },false,true) }
341        --
342        local bodyfont = finetuning.bodyfont
343        local width    = finetuning.width
344        local hyphen   = finetuning.hyphen
345        local align    = finetuning.align
346        --
347        if type(bodyfont) == "number" then
348            bodyfont = todimen(bodyfont)
349        else
350            bodyfont = "12pt"
351        end
352        if type(width) == "number" then
353            width = todimen(width) or "50em"
354        else
355            width = "50em"
356        end
357        if hyphen == v_yes then
358            hyphen = "manual"
359        else
360            hyphen = "inherited"
361        end
362        if align then
363            align = numbertoallign[align]
364        end
365        if not align then
366            align = hyphen and "justify" or "inherited"
367        end
368        --
369        result[#result+1] = replacetemplate(documenttemplate,{
370            size    = bodyfont,
371            width   = width,
372            align   = align,
373            hyphens = hyphen
374        })
375        --
376        local colorspecification = xml.css.colorspecification
377        local fontspecification  = xml.css.fontspecification
378        for element, details in sortedhash(usedstyles) do
379            for detail, data in sortedhash(details) do
380                local s = fontspecification(data.style)
381                local c = colorspecification(data.color)
382                detail = gsub(detail,"[^A-Za-z0-9]+","-")
383                result[#result+1] = replacetemplate(styletemplate,{
384                    namespace = usecssnamespace and cssnamespace or "",
385                    element   = element,
386                    detail    = detail,
387                    style     = s.style   or "inherit",
388                    variant   = s.variant or "inherit",
389                    weight    = s.weight  or "inherit",
390                    family    = s.family  or "inherit",
391                    color     = c         or "inherit",
392                    display   = s.display and "block" or nil,
393                })
394            end
395        end
396        return concat(result,"\n\n")
397    end
398
399end
400
401local usedimages = { }
402
403do
404
405local imagetemplate = [[
406%element%[id="%id%"], %namespace%div.%element%[id="%id%"] {
407    display           : block ;
408    background-image  : url('%url%') ;
409    background-size   : 100%% auto ;
410    background-repeat : no-repeat ;
411    width             : %width% ;
412    height            : %height% ;
413}]]
414
415    local f_svgname = formatters["%s.svg"]
416    local f_svgpage = formatters["%s-page-%s.svg"]
417    local collected = { }
418
419    local function usedname(name,page)
420        if filesuffix(name) == "pdf" then
421            -- temp hack .. we will have a remapper
422            if page and page > 1 then
423                name = f_svgpage(nameonly(name),page)
424            else
425                name = f_svgname(nameonly(name))
426            end
427        end
428        local scheme = url.hasscheme(name)
429        if not scheme or scheme == "file" then
430            -- or can we just use the name ?
431            return joinfile("../images",basename(url.filename(name)))
432        else
433            return name
434        end
435    end
436
437    function wrapups.allusedimages(filename)
438        local result = { replacetemplate(namespacetemplate, {
439            what            = "images",
440            filename        = filename,
441            namespace       = contextns,
442         -- cssnamespaceurl = usecssnamespace and cssnamespaceurl or "",
443            cssnamespaceurl = cssnamespaceurl,
444        },false,true) }
445        for element, details in sortedhash(usedimages) do
446            for detail, data in sortedhash(details) do
447                local name = data.name
448                local page = tonumber(data.page) or 1
449                local spec = {
450                    element   = element,
451                    id        = data.id,
452                    name      = name,
453                    page      = page,
454                    url       = usedname(name,page),
455                    width     = data.width,
456                    height    = data.height,
457                    used      = data.used,
458                    namespace = usecssnamespace and cssnamespace or "",
459                }
460                result[#result+1] = replacetemplate(imagetemplate,spec)
461                collected[detail] = spec
462            end
463        end
464        return concat(result,"\n\n")
465    end
466
467    function wrapups.uniqueusedimages() -- todo: combine these two
468        return collected
469    end
470
471end
472
473--
474
475properties.vspace = { export = "break",     nature = "display" }
476----------------- = { export = "pagebreak", nature = "display" }
477
478local function makebreaklist(list)
479    nofbreaks = nofbreaks + 1
480    local t = { }
481    local l = list and list.taglist
482    if l then
483        for i=1,#list do
484            t[i] = l[i]
485        end
486    end
487    t[#t+1] = "break>" .. nofbreaks -- maybe no number or 0
488    return { taglist = t }
489end
490
491local breakattributes = {
492    type = "collapse"
493}
494
495local function makebreaknode(attributes) -- maybe no fulltag
496    nofbreaks = nofbreaks + 1
497    return {
498        tg         = "break",
499        fulltag    = "break>" .. nofbreaks,
500        n          = nofbreaks,
501        element    = "break",
502        nature     = "display",
503        attributes = attributes or nil,
504     -- data       = { }, -- not needed
505     -- attribute  = 0, -- not needed
506     -- parnumber  = 0,
507    }
508end
509
510do
511
512    local fields = { "title", "subtitle", "author", "keywords", "url", "version" }
513
514    local ignoredelements = false
515
516    local function checkdocument(root)
517        local data = root.data
518        if data then
519            for i=1,#data do
520                local di = data[i]
521                local tg = di.tg
522                if tg == "noexport" then
523                    local s = specifications[di.fulltag]
524                    local u = s and s.userdata
525                    if u then
526                        local comment = u.comment
527                        if comment then
528                            di.element = "comment"
529                            di.data = { { content = comment } }
530                            u.comment = nil
531                        else
532                            data[i] = false
533                        end
534                    else
535                        data[i] = false
536                    end
537                elseif di.content then
538                    -- okay
539                elseif tg == "ignore" then
540                    di.element = ""
541                    checkdocument(di)
542                elseif ignoredelements and ignoredelements[tg] then
543                    di.element = ""
544                    checkdocument(di)
545                else
546                    checkdocument(di) -- new, else no noexport handling
547                end
548            end
549        end
550    end
551
552    function extras.document(di,element,n,fulltag)
553        setattribute(di,"language",languagenames[texgetcount("mainlanguagenumber")])
554        if not less_state then
555            setattribute(di,"file",tex.jobname)
556            if included.date then
557                setattribute(di,"date",os.fulltime())
558            end
559            setattribute(di,"context",environment.version)
560            setattribute(di,"version",exportversion)
561            setattribute(di,"xmlns:m",mathmlns)
562            local identity = interactions.general.getidentity()
563            for i=1,#fields do
564                local key   = fields[i]
565                local value = identity[key]
566                if value and value ~= "" then
567                    setattribute(di,key,value)
568                end
569            end
570        end
571        checkdocument(di)
572    end
573
574    implement {
575        name      = "ignoretagsinexport",
576        arguments = "string",
577        actions   = function(list)
578            for tag in string.gmatch(list,"[a-z]+") do
579                if ignoredelements then
580                    ignoredelements[tag] = true
581                else
582                    ignoredelements = { [tag] = true }
583                end
584            end
585        end,
586    }
587
588end
589
590do
591
592    local marginanchors = { }
593    local margincontent = { }
594
595    implement {
596        name      = "settagmargintext",
597        arguments = "integer",
598        actions   = function(n)
599            marginanchors[locatedtag("margintext")] = n
600        end
601    }
602
603    implement {
604        name      = "settagmarginanchor",
605        arguments = "integer",
606        actions   = function(n)
607            marginanchors[locatedtag("marginanchor")] = n
608        end
609    }
610
611    function checks.margintext(di)
612        local i = marginanchors[di.fulltag]
613        margincontent[i] = di
614    end
615
616    function checks.marginanchor(di)
617        local i = marginanchors[di.fulltag]
618        local d = margincontent[i]
619        --
620        di.attribute = d.attribute
621        di.data      = d.data
622        di.detail    = d.detail
623        di.element   = d.element
624        di.fulltag   = d.fulltag
625        di.nature    = d.nature
626        di.samepar   = true
627        di.tg        = d.tg
628        --
629        d.skip       = "ignore"
630  end
631
632end
633
634do
635
636    local symbols = { }
637
638    function structurestags.settagdelimitedsymbol(symbol)
639        symbols[locatedtag("delimitedsymbol")] = {
640            symbol = symbol,
641        }
642    end
643
644    function extras.delimitedsymbol(di,element,n,fulltag)
645        local hash = symbols[fulltag]
646        if hash then
647            setattribute(di,"symbol",hash.symbol or nil)
648        end
649    end
650
651end
652
653do
654
655    local symbols = { }
656
657    function structurestags.settagsubsentencesymbol(symbol)
658        symbols[locatedtag("subsentencesymbol")] = {
659            symbol = symbol,
660        }
661    end
662
663    function extras.subsentencesymbol(di,element,n,fulltag)
664        local hash = symbols[fulltag]
665        if hash then
666            setattribute(di,"symbol",hash.symbol or nil)
667        end
668    end
669
670end
671
672do
673
674    local itemgroups = { }
675
676    function structurestags.setitemgroup(packed,level,symbol)
677        itemgroups[locatedtag("itemgroup")] = {
678            packed = packed,
679            symbol = symbol,
680            level  = level,
681        }
682    end
683
684    function structurestags.setitem(kind)
685        itemgroups[locatedtag("item")] = {
686            kind = kind,
687        }
688    end
689
690    function extras.itemgroup(di,element,n,fulltag)
691        local hash = itemgroups[fulltag]
692        if hash then
693            setattribute(di,"packed",hash.packed and "yes" or nil)
694            setattribute(di,"symbol",hash.symbol)
695            setattribute(di,"level",hash.level)
696        end
697    end
698
699    function extras.item(di,element,n,fulltag)
700        local hash = itemgroups[fulltag]
701        if hash then
702            local kind = hash.kind
703            if kind and kind ~= "" then
704                setattribute(di,"kind",kind)
705            end
706        end
707    end
708
709end
710
711do
712
713    function fixes.linenumber(di,data,i)
714        local ni = data[i+1]
715        if ni then
716            if ni.data then
717                while true do
718                    local d = ni.data[1]
719                    if d then
720                        local e = d.element
721                        if e then
722                            if e == "line" or e == "verbatimline" then
723                                insert(d.data,1,di)
724                                data[i] = false
725                                return
726                            else
727                                ni = d
728                            end
729                        else
730                            return
731                        end
732                    else
733                        return
734                    end
735                end
736            end
737        end
738    end
739
740end
741
742do
743
744    local synonyms = { }
745    local sortings = { }
746
747    function structurestags.setsynonym(tag)
748        synonyms[locatedtag("synonym")] = tag
749    end
750
751    function extras.synonym(di,element,n,fulltag)
752        local tag = synonyms[fulltag]
753        if tag then
754            setattribute(di,"tag",tag)
755        end
756    end
757
758    function structurestags.setsorting(tag)
759        sortings[locatedtag("sorting")] = tag
760    end
761
762    function extras.sorting(di,element,n,fulltag)
763        local tag = sortings[fulltag]
764        if tag then
765            setattribute(di,"tag",tag)
766        end
767    end
768
769end
770
771do
772
773    local strippedtag    = structurestags.strip -- we assume global styles
774
775    local highlight      = { }
776    local construct      = { }
777
778    usedstyles.highlight = highlight
779    usedstyles.construct = construct
780
781    function structurestags.sethighlight(name,style,color,mode)
782        if not highlight[name] then
783            highlight[name] = {
784                style = style,
785                color = color,
786                mode  = mode == 1 and "display" or nil,
787            }
788        end
789    end
790
791    function structurestags.setconstruct(name,style,color,mode)
792        if not construct[name] then
793            construct[name] = {
794                style = style,
795                color = color,
796                mode  = mode == 1 and "display" or nil,
797            }
798        end
799    end
800
801end
802
803do
804
805    local descriptions = { }
806    local symbols      = { }
807    local linked       = { }
808
809    -- we could move the notation itself to the first reference (can be an option)
810
811    function structurestags.setnotation(tag,n) -- needs checking (is tag needed)
812        -- we can also use the internals hash or list
813        local nd = structures.notes.get(tag,n)
814        if nd then
815            local references = nd.references
816            descriptions[references and references.internal] = locatedtag("description")
817        end
818    end
819
820    function structurestags.setnotationsymbol(tag,n) -- needs checking (is tag needed)
821        local nd = structures.notes.get(tag,n) -- todo: use listdata instead
822        if nd then
823            local references = nd.references
824            symbols[references and references.internal] = locatedtag("descriptionsymbol")
825        end
826    end
827
828    function finalizers.descriptions(tree)
829        local n = 0
830        for id, tag in sortedhash(descriptions) do
831            local sym = symbols[id]
832            if sym then
833                n = n + 1
834                linked[tag] = n
835                linked[sym] = n
836            end
837        end
838    end
839
840    function extras.description(di,element,n,fulltag)
841        local id = linked[fulltag]
842        if id then
843            setattribute(di,"insert",id)
844        end
845    end
846
847    function extras.descriptionsymbol(di,element,n,fulltag)
848        local id = linked[fulltag]
849        if id then
850            setattribute(di,"insert",id)
851        end
852    end
853
854end
855
856-- -- todo: ignore breaks
857--
858-- function extras.verbatimline(di,element,n,fulltag)
859--     inspect(di)
860-- end
861
862do
863
864    local f_id       = formatters["%s-%s"]
865    local image      = { }
866    usedimages.image = image
867
868    structurestags.usewithcare.images = image
869
870    function structurestags.setfigure(name,used,page,width,height,label)
871        local fulltag = locatedtag("image")
872        local spec    = specifications[fulltag]
873        if spec then
874            local page = tonumber(page)
875            image[fulltag] = {
876                id     = f_id(spec.tagname,spec.tagindex),
877                name   = name,
878                used   = used,
879                page   = page and page > 1 and page or nil,
880                width  = todimen(width, "cm","%0.3F%s"),
881                height = todimen(height,"cm","%0.3F%s"),
882                label  = label,
883            }
884        else
885            -- we ignore images in layers in the background / pagebody
886        end
887    end
888
889    function extras.image(di,element,n,fulltag)
890        local data = image[fulltag]
891        if data then
892            setattribute(di,"name",data.name)
893            setattribute(di,"page",data.page)
894            setattribute(di,"id",data.id)
895            setattribute(di,"width",data.width)
896            setattribute(di,"height",data.height)
897            setattribute(di,"label",data.label)
898        end
899    end
900
901end
902
903do
904
905    local combinations = { }
906
907    function structurestags.setcombination(nx,ny)
908        combinations[locatedtag("combination")] = {
909            nx = nx,
910            ny = ny,
911        }
912    end
913
914    function extras.combination(di,element,n,fulltag)
915        local data = combinations[fulltag]
916        if data then
917            setattribute(di,"nx",data.nx)
918            setattribute(di,"ny",data.ny)
919        end
920    end
921
922end
923
924-- quite some code deals with exporting references  --
925
926-- links:
927--
928-- url      :
929-- file     :
930-- internal : automatic location
931-- location : named reference
932
933-- references:
934--
935-- implicit : automatic reference
936-- explicit : named reference
937
938local evaluators = { }
939local specials   = { }
940local explicits  = { }
941
942evaluators.inner = function(di,var)
943    local inner = var.inner
944    if inner then
945        setattribute(di,"location",inner,true)
946    end
947end
948
949evaluators.outer = function(di,var)
950    local file, url = references.checkedfileorurl(var.outer,var.outer)
951    if url then
952        setattribute(di,"url",url,true)
953    elseif file then
954        setattribute(di,"file",file,true)
955    end
956end
957
958evaluators["outer with inner"] = function(di,var)
959    local file = references.checkedfile(var.f)
960    if file then
961        setattribute(di,"file",file,true)
962    end
963    local inner = var.inner
964    if inner then
965        setattribute(di,"inner",inner,true)
966    end
967end
968
969evaluators.special = function(di,var)
970    local handler = specials[var.special]
971    if handler then
972        handler(di,var)
973    end
974end
975
976local referencehash   = { }
977local destinationhash = { }
978
979do
980
981    evaluators["special outer with operation"]     = evaluators.special
982    evaluators["special operation"]                = evaluators.special
983    evaluators["special operation with arguments"] = evaluators.special
984
985    function specials.url(di,var)
986        local url = references.checkedurl(var.operation)
987        if url and url ~= "" then
988            setattribute(di,"url",url,true)
989        end
990    end
991
992    function specials.file(di,var)
993        local file = references.checkedfile(var.operation)
994        if file and file ~= "" then
995            setattribute(di,"file",file,true)
996        end
997    end
998
999    function specials.fileorurl(di,var)
1000        local file, url = references.checkedfileorurl(var.operation,var.operation)
1001        if url and url ~= "" then
1002            setattribute(di,"url",url,true)
1003        elseif file and file ~= "" then
1004            setattribute(di,"file",file,true)
1005        end
1006    end
1007
1008    function specials.internal(di,var)
1009        local internal = references.checkedurl(var.operation)
1010        if internal then
1011            setattribute(di,"location",internal)
1012        end
1013    end
1014
1015    local function adddestination(di,references) -- todo: specials -> exporters and then concat
1016        if references then
1017            local reference = references.reference
1018            if reference and reference ~= "" then
1019                local prefix = references.prefix
1020                if prefix and prefix ~= "" then
1021                    setattribute(di,"prefix",prefix,true)
1022                end
1023                setattribute(di,"destination",reference,true)
1024                for i=1,#references do
1025                    local r = references[i]
1026                    local e = evaluators[r.kind]
1027                    if e then
1028                        e(di,r)
1029                    end
1030                end
1031            end
1032        end
1033    end
1034
1035    function extras.addimplicit(di,references)
1036        if references then
1037            local internal = references.internal
1038            if internal then
1039                setattribute(di,"implicit",internal)
1040            end
1041        end
1042    end
1043
1044    function extras.addinternal(di,references)
1045        if references then
1046            local internal = references.internal
1047            if internal then
1048                setattribute(di,"internal",internal)
1049            end
1050        end
1051    end
1052
1053    local p_firstpart = lpeg.Cs((1-lpeg.P(","))^0)
1054
1055    local function addreference(di,references)
1056        if references then
1057            local reference = references.reference
1058            if reference and reference ~= "" then
1059                local prefix = references.prefix
1060                if prefix and prefix ~= "" then
1061                    setattribute(di,"prefix",prefix)
1062                end
1063                setattribute(di,"reference",reference,true)
1064                setattribute(di,"explicit",lpegmatch(p_firstpart,reference),true)
1065            end
1066            local internal = references.internal
1067            if internal and internal ~= "" then
1068                setattribute(di,"implicit",internal)
1069            end
1070        end
1071    end
1072
1073    local function link(di,element,n,fulltag)
1074        -- for instance in lists a link has nested elements and no own text
1075        local reference = referencehash[fulltag]
1076        if reference then
1077            adddestination(di,structures.references.get(reference))
1078            return true
1079        else
1080            local data = di.data
1081            if data then
1082                for i=1,#data do
1083                    local di = data[i]
1084                    if di then
1085                        local fulltag = di.fulltag
1086                        if fulltag and link(di,element,n,fulltag) then
1087                            return true
1088                        end
1089                    end
1090                end
1091            end
1092        end
1093    end
1094
1095    local function reference(di,element,n,fulltag)
1096        local destination = destinationhash[fulltag]
1097        if destination then
1098            local d = structures.references.internals[destination]
1099            if d then
1100                addreference(di,d.references)
1101                return true
1102            else
1103                return false
1104            end
1105        else
1106            local data = di.data
1107            if data then
1108                for i=1,#data do
1109                    local di = data[i]
1110                    if di then
1111                        local fulltag = di.fulltag
1112                        if fulltag and reference(di,element,n,fulltag) then
1113                            return true
1114                        end
1115                    end
1116                end
1117            end
1118        end
1119    end
1120
1121    extras.adddestination = adddestination
1122    extras.addreference   = addreference
1123
1124    extras.link           = link
1125    extras.reference      = reference
1126
1127end
1128
1129-- no settings, as these are obscure ones
1130
1131do
1132
1133    local automathrows   = true  directives.register("export.math.autorows",   function(v) automathrows   = v end)
1134    local automathapply  = true  directives.register("export.math.autoapply",  function(v) automathapply  = v end)
1135    local automathnumber = true  directives.register("export.math.autonumber", function(v) automathnumber = v end)
1136    local automathstrip  = true  directives.register("export.math.autostrip",  function(v) automathstrip  = v end)
1137
1138    local functions      = mathematics.categories.functions
1139
1140    local function collapse(di,i,data,ndata,detail,element)
1141        local collapsing = di.data
1142        if data then
1143            di.element = element
1144            di.detail = nil
1145            i = i + 1
1146            while i <= ndata do
1147                local dn = data[i]
1148                if dn.detail == detail then
1149                    collapsing[#collapsing+1] = dn.data[1]
1150                    dn.skip = "ignore"
1151                    i = i + 1
1152                else
1153                    break
1154                end
1155            end
1156        end
1157        return i
1158    end
1159
1160    local function collapse_mn(di,i,data,ndata)
1161        -- this is tricky ... we need to make sure that we wrap in mrows if we want
1162        -- to bypass this one
1163        local collapsing = di.data
1164        if data then
1165            i = i + 1
1166            while i <= ndata do
1167                local dn = data[i]
1168                local tg = dn.tg
1169                if tg == "mn" then
1170                    collapsing[#collapsing+1] = dn.data[1]
1171                    dn.skip = "ignore"
1172                    i = i + 1
1173                elseif tg == "mo" then
1174                    local d = dn.data[1]
1175                    if d == "." then
1176                        collapsing[#collapsing+1] = d
1177                        dn.skip = "ignore"
1178                        i = i + 1
1179                    else
1180                        break
1181                    end
1182                else
1183                    break
1184                end
1185            end
1186        end
1187        return i
1188    end
1189
1190    -- maybe delay __i__ till we need it
1191
1192    local apply_function = {
1193        {
1194            element = "mo",
1195         -- comment = "apply function",
1196         -- data    = { utfchar(0x2061) },
1197            data    = { "&#x2061;" },
1198            nature  = "mixed",
1199        }
1200    }
1201
1202    local functioncontent = { }
1203
1204    setmetatableindex(functioncontent,function(t,k)
1205        local v = { { content = k } }
1206        t[k] = v
1207        return v
1208    end)
1209
1210    local dummy_nucleus = {
1211        element   = "mtext",
1212        data      = { content = "" },
1213        nature    = "inline",
1214        comment   = "dummy nucleus",
1215        fulltag   = "mtext>0"
1216    }
1217
1218    local function accentchar(d)
1219        for i=1,3 do
1220            d = d.data
1221            if not d then
1222                return
1223            end
1224            d = d[1]
1225            if not d then
1226                return
1227            end
1228            local tg = d.tg
1229            if tg == "mover" then
1230                local s = specifications[d.fulltag]
1231                local t = s.top
1232                if t then
1233                    d = d.data[1]
1234                    local d1 = d.data[1]
1235                    d1.content = utfchar(t)
1236                    d.data = { d1 }
1237                    return d
1238                end
1239            elseif tg == "munder" then
1240                local s = specifications[d.fulltag]
1241                local b = s.bottom
1242                if b then
1243                    d = d.data[1]
1244                    local d1 = d.data[1]
1245                    d1.content = utfchar(b)
1246                    d.data = { d1 }
1247                    return d
1248                end
1249            end
1250        end
1251    end
1252
1253    local no_mrow = {
1254        mrow     = true,
1255        mfenced  = true,
1256        mfrac    = true,
1257        mroot    = true,
1258        msqrt    = true,
1259        mtable   = true,
1260        mi       = true,
1261        mo       = true,
1262        mn       = true,
1263    }
1264
1265    local function checkmath(root) -- we can provide utf.toentities as an option
1266        local data = root.data
1267        if data then
1268            local ndata = #data
1269            local roottg = root.tg
1270            if roottg == "msubsup" then
1271                -- kind of tricky: we have a diufferent order in display mode
1272                local nucleus, superscript, subscript
1273                if ndata > 3 then
1274                    -- error
1275                else
1276                    for i=1,ndata do
1277                        local di = data[i]
1278                        if not di then
1279                            -- weird
1280                        elseif di.content then
1281                            -- text
1282                        else
1283                            local s = specifications[di.fulltag]
1284                            if s.subscript then
1285                                subscript = i
1286                            elseif s.superscript then
1287                                superscript = i
1288                            else
1289                                nucleus = i
1290                            end
1291                        end
1292                    end
1293                    if superscript or subscript then
1294                        -- we probably always have 3 anyway ... needs checking
1295                        local nuc = nucleus     and data[nucleus]
1296                        local sub = subscript   and data[subscript]
1297                        local sup = superscript and data[superscript]
1298                        local n = 0 -- play safe
1299                        if nuc then n = n + 1 ; data[n] = nuc end
1300                        if sub then n = n + 1 ; data[n] = sub end
1301                        if sup then n = n + 1 ; data[n] = sup end
1302                    end
1303                end
1304         -- elseif roottg == "msup" or roottg == "msub" then
1305         --     -- m$^2$
1306         --     if ndata == 1 then
1307         --         local d = data[1]
1308         --         data[2] = d
1309         --         d.__i__ = 2
1310         --         data[1] = dummy_nucleus
1311         --     end
1312            elseif roottg == "mfenced" then
1313                local s = specifications[root.fulltag]
1314                local l, m, r = s.left, s.middle, s.right
1315                if l then
1316                    l = utfchar(l)
1317                end
1318                if m then
1319                    local t = { }
1320                    for i=1,#m do
1321                        t[i] = utfchar(m[i])
1322                    end
1323                    m = concat(t)
1324                end
1325                if r then
1326                    r = utfchar(r)
1327                end
1328                root.attributes = {
1329                    open       = l,
1330                    separators = m,
1331                    close      = r,
1332                }
1333            end
1334            if ndata == 0 then
1335                root.skip = "comment" -- get rid of weird artefacts
1336                root.nota = "weird"
1337                return
1338            elseif ndata == 1 then
1339                local d = data[1]
1340                if not d or d == "" then
1341                    root.skip = "comment"
1342                    return
1343                elseif d.content then
1344                    return
1345                else -- if ndata == 1 then
1346                    local tg = d.tg
1347                    if automathrows and (roottg == "mrow" or roottg == "mtext") then
1348                        -- maybe just always ! check spec first
1349                        -- or we can have chesks.* for each as we then can flatten
1350                        if no_mrow[tg] then
1351                            root.skip = "comment"
1352                        end
1353                    elseif roottg == "mo" then
1354                        if tg == "mo" then
1355                            root.skip = "comment"
1356                        end
1357                    end
1358                end
1359            end
1360            local i = 1
1361            while i <= ndata do                   -- -- -- TOO MUCH NESTED CHECKING -- -- --
1362                local di = data[i]
1363                if di and not di.content then
1364                    local tg = di.tg
1365                    if tg == "math" then
1366                     -- di.element = "mrow" -- when properties
1367                        di.skip = "comment"
1368                        checkmath(di)
1369                        i = i + 1
1370                    elseif tg == "mover" then
1371                        local s = specifications[di.fulltag]
1372                        if s.accent then
1373                            local t = s.top
1374                            local d = di.data
1375                            -- todo: accent = "false" (for scripts like limits)
1376                            di.attributes = {
1377                                accent = "true",
1378                            }
1379                            -- todo: p.topfixed
1380                            if t then
1381                                -- mover
1382                                d[1].data[1].content = utfchar(t)
1383                                di.data = { d[2], d[1] }
1384                            end
1385                        else
1386                            -- can't happen
1387                        end
1388                        checkmath(di)
1389                        i = i + 1
1390                    elseif tg == "munder" then
1391                        local s = specifications[di.fulltag]
1392                        if s.accent then
1393                            local b = s.bottom
1394                            local d = di.data
1395                            -- todo: accent = "false" (for scripts like limits)
1396                            di.attributes = {
1397                                accent = "true",
1398                            }
1399                         -- todo: p.bottomfixed
1400                            if b then
1401                                -- munder
1402                                d[2].data[1].content = utfchar(b)
1403                            end
1404                        else
1405                            -- can't happen
1406                        end
1407                        checkmath(di)
1408                        i = i + 1
1409                    elseif tg == "munderover" then
1410                        local s = specifications[di.fulltag]
1411                        if s.accent then
1412                            local t = s.top
1413                            local b = s.bottom
1414                            local d = di.data
1415                            -- todo: accent      = "false" (for scripts like limits)
1416                            -- todo: accentunder = "false" (for scripts like limits)
1417                            di.attributes = {
1418                                accent      = "true",
1419                                accentunder = "true",
1420                            }
1421                         -- todo: p.topfixed
1422                         -- todo: p.bottomfixed
1423                            if t and b then
1424                                -- munderover
1425                                d[1].data[1].content = utfchar(t)
1426                                d[3].data[1].content = utfchar(b)
1427                                di.data = { d[2], d[3], d[1] }
1428                            else
1429                                -- can't happen
1430                            end
1431                        else
1432                            -- can't happen
1433                        end
1434                        checkmath(di)
1435                        i = i + 1
1436                    elseif tg == "mstacker" then
1437                        local d = di.data
1438                        local d1 = d[1]
1439                        local d2 = d[2]
1440                        local d3 = d[3]
1441                        local t1 = d1 and d1.tg
1442                        local t2 = d2 and d2.tg
1443                        local t3 = d3 and d3.tg
1444                        local m  = nil -- d1.data[1]
1445                        local t  = nil
1446                        local b  = nil
1447                        -- only accent when top / bot have stretch
1448                        -- normally we flush [base under over] which is better for tagged pdf
1449                        if t1 == "mstackermid" then
1450                            m = accentchar(d1) -- or m
1451                            if t2 == "mstackertop" then
1452                                if t3 == "mstackerbot" then
1453                                    t = accentchar(d2)
1454                                    b = accentchar(d3)
1455                                    di.element = "munderover"
1456                                    di.data    = { m or d1.data[1], b or d3.data[1], t or d2.data[1] }
1457                                else
1458                                    t = accentchar(d2)
1459                                    di.element = "mover"
1460                                    di.data    = { m or d1.data[1], t or d2.data[1] }
1461                                end
1462                            elseif t2 == "mstackerbot" then
1463                                if t3 == "mstackertop" then
1464                                    b = accentchar(d2)
1465                                    t = accentchar(d3)
1466                                    di.element = "munderover"
1467                                    di.data    = { m or d1.data[1], t or d3.data[1], m, b or d2.data[1] }
1468                                else
1469                                    b = accentchar(d2)
1470                                    di.element = "munder"
1471                                    di.data    = { m or d1.data[1], b or d2.data[1] }
1472                                end
1473                            else
1474                                -- can't happen
1475                            end
1476                        else
1477                            -- can't happen
1478                        end
1479                        if t or b then
1480                            di.attributes = {
1481                                accent      = t and "true" or nil,
1482                                accentunder = b and "true" or nil,
1483                            }
1484                            di.detail = nil
1485                        end
1486                        checkmath(di)
1487                        i = i + 1
1488                    elseif tg == "mroot" then
1489                        local data = di.data
1490                        local size = #data
1491                        if size == 1 then
1492                            -- else firefox complains ... code in math-tag (for pdf tagging)
1493                            di.element = "msqrt"
1494                        elseif size == 2 then
1495                            data[1], data[2] = data[2], data[1]
1496                        end
1497                        checkmath(di)
1498                        i = i + 1
1499                    elseif tg == "break" then
1500                        di.skip = "comment"
1501                        i = i + 1
1502                    elseif tg == "mtext" then
1503                        -- this is only needed for unboxed mtexts ... all kind of special
1504                        -- tex border cases and optimizations ... trial and error
1505                        local data = di.data
1506                        if #data > 1 then
1507                            for i=1,#data do
1508                                local di = data[i]
1509                                local content = di.content
1510                                if content then
1511                                    data[i] = {
1512                                        element = "mtext",
1513                                        nature  = "inline",
1514                                        data    = { di },
1515                                        n       = 0,
1516                                    }
1517                                elseif di.tg == "math" then
1518                                    local di = di.data[1]
1519                                    if di then
1520                                        data[i] = di
1521                                        checkmath(di)
1522                                    end
1523                                end
1524                            end
1525                            di.element = "mrow"
1526                         -- di.tg = "mrow"
1527                         -- di.nature  = "inline"
1528                        end
1529                        checkmath(di)
1530                        i = i + 1
1531                    elseif tg == "mrow" and di.detail then -- hm, falls through
1532                        checkmath(di)
1533                        di = {
1534                            element    = "maction",
1535                            nature     = "display",
1536                            attributes = { actiontype = di.detail },
1537                            data       = { di },
1538                            n          = 0,
1539                        }
1540                        di.detail = nil
1541                        data[i] = di
1542                        i = i + 1
1543                    else
1544                        local category = di.mathcategory
1545                        if category then
1546                         -- no checkmath(di) here
1547                            if category == 1 then -- mo
1548                                i = collapse(di,i,data,ndata,di.detail,"mo")
1549                            elseif category == 2 then -- mi
1550                                i = collapse(di,i,data,ndata,di.detail,"mi")
1551                            elseif category == 3 then -- mn
1552                                i = collapse(di,i,data,ndata,di.detail,"mn")
1553                            elseif category == 4 then -- ms
1554                                i = collapse(di,i,data,ndata,di.detail,"ms")
1555                            elseif category >= 1000 then
1556                                -- Can this still happen .. maybe it's broken.
1557                                local apply = category >= 2000
1558                                if apply then
1559                                    category = category - 1000
1560                                end
1561                                if tg == "mi" then -- function
1562                                    if roottg == "mrow" then
1563                                        root.skip = "comment"
1564                                        root.element = "function"
1565                                    end
1566                                    i = collapse(di,i,data,ndata,di.detail,"mi")
1567                                    local tag = functions[category]
1568                                    if tag then
1569                                        di.data = functioncontent[tag]
1570                                    end
1571                                    if apply then
1572                                        di.after = apply_function
1573                                    elseif automathapply then -- make function
1574                                        local following
1575                                        if i <= ndata then
1576                                            -- normally not the case
1577                                            following = data[i]
1578                                        else
1579                                            local parent = di.__p__ -- == root
1580                                            if parent.tg == "mrow" then
1581                                                parent = parent.__p__
1582                                            end
1583                                            local index = parent.__i__
1584                                            following = parent.data[index+1]
1585                                        end
1586                                        if following then
1587                                            local tg = following.tg
1588                                            if tg == "mrow" or tg == "mfenced" then -- we need to figure out the right condition
1589                                                di.after = apply_function
1590                                            end
1591                                        end
1592                                    end
1593                                else -- some problem
1594                                    checkmath(di)
1595                                    i = i + 1
1596                                end
1597                            else
1598                                checkmath(di)
1599                                i = i + 1
1600                            end
1601                        elseif automathnumber and tg == "mn" then
1602                            checkmath(di)
1603                            i = collapse_mn(di,i,data,ndata)
1604                        else
1605                            checkmath(di)
1606                            i = i + 1
1607                        end
1608                    end
1609                else -- can be string or boolean
1610                    if parenttg ~= "mtext" and di == " " then
1611                        data[i] = false
1612                    end
1613                    i = i + 1
1614                end
1615            end
1616        end
1617    end
1618
1619    local function stripmath(di)
1620        if not di then
1621            --
1622        elseif di.content then
1623            return di
1624        else
1625            local tg = di.tg
1626            if tg == "mtext" or tg == "ms" then
1627                return di
1628            else
1629                local data = di.data
1630                local ndata = #data
1631                local n = 0
1632                for i=1,ndata do
1633                    local d = data[i]
1634                    if d and not d.content then
1635                        d = stripmath(d)
1636                    end
1637                    if d then
1638                        local content = d.content
1639                        if not content then
1640                            n = n + 1
1641                            d.__i__ = n
1642                            data[n] = d
1643                        elseif content == " " or content == "" then
1644                            if d.tg == "mspace" then
1645                                -- we append or prepend a space to a preceding or following mtext
1646                                local parent = di.__p__
1647                                local index  = di.__i__ -- == i
1648                                local data   = parent.data
1649                                if index > 1 then
1650                                    local d = data[index-1]
1651                                    if d.tg == "mtext" then
1652                                        local dd = d.data
1653                                        local dn = dd[#dd]
1654                                        local dc = dn.content
1655                                        if dc then
1656                                            dn.content = dc .. content
1657                                        end
1658                                    end
1659                                elseif index < ndata then
1660                                    local d = data[index+1]
1661                                    if d.tg == "mtext" then
1662                                        local dd = d.data
1663                                        local dn = dd[1]
1664                                        local dc = dn.content
1665                                        if dc then
1666                                            dn.content = content .. dc
1667                                        end
1668                                    end
1669                                end
1670                            end
1671                        else
1672                            n = n + 1
1673                            data[n] = d
1674                        end
1675                    end
1676                end
1677                for i=ndata,n+1,-1 do
1678                    data[i] = nil
1679                end
1680                if #data > 0 then
1681                    return di
1682                end
1683            end
1684        end
1685    end
1686
1687    function checks.math(di)
1688        if di.skip == "comment" then
1689            -- already done, kind of weird, happens in mathmatrix, maybe some collapse
1690            -- issue that i need to look into
1691        else
1692            local specification = specifications[di.fulltag]
1693            local mode = specification and specification.mode == "display" and "block" or "inline"
1694            di.attributes = {
1695                ["display"] = mode,
1696                ["xmlns:m"] = mathmlns,
1697            }
1698            -- can be option if needed:
1699            if mode == "inline" then
1700             -- di.nature = "mixed"  -- else spacing problem (maybe inline)
1701                di.nature = "inline" -- we need to catch x$X$x and x $X$ x
1702            else
1703                di.nature = "display"
1704            end
1705            if automathstrip then
1706                stripmath(di)
1707            end
1708            checkmath(di)
1709        end
1710    end
1711
1712    -- this one can replace some of the previous code .. todo (test on mathmatrix)
1713
1714    -- ignore with no data can be removed
1715
1716    local function checked(d)
1717        local n = #d
1718        if n == 1 then
1719            local di = d[1]
1720            local tg = di.tg
1721            if tg == "ignore" then
1722                -- todo: we can move ignore's data one level up
1723                return 1
1724            elseif di.content then
1725                return 1
1726            else
1727                local dd = di.data
1728                if #dd > 0 and checked(dd) > 0 then
1729                    return 1
1730                else
1731                    return 0
1732                end
1733            end
1734        else
1735            local m = 0
1736            for i=1,n do
1737                local di = d[i]
1738                local tg = di.tg
1739                if tg == "ignore" then
1740                    -- skip
1741                elseif di.content then
1742                    m = m + 1
1743                    d[m] = di
1744                else
1745                    local dd = di.data
1746                    if #dd > 0 and checked(dd) > 0 then
1747                        m = m + 1
1748                        d[m] = di
1749                    end
1750                end
1751            end
1752            if m < n then
1753                for i=n,m+1,-1 do
1754                    d[i] = nil
1755                end
1756            end
1757            return m
1758        end
1759    end
1760
1761    function checks.mrow(di)
1762     -- local d = di.data
1763     -- if d then
1764     --     checked(d)
1765     -- end
1766    end
1767
1768    -- we can move more checks here
1769
1770    local function flatten(di)
1771        local r = di.__p__
1772        while r do
1773            local d = r.data
1774            local n = #d
1775            if d and n > 1 then
1776                n = checked(d)
1777            end
1778            local tg = r.tg
1779            if n == 1 and (tg == "mtext" or tg == "mrow") then
1780                r.skip = "comment" -- weird error
1781                r = r.__p__
1782            else
1783                break
1784            end
1785        end
1786    end
1787
1788    function checks.mtable(di)
1789        flatten(di)
1790        local d = di.data
1791        for i=1,#d do
1792            local d = d[i]
1793            if d.tg == "mtr" then
1794                local d = d.data
1795                for i=1,#d do
1796                    local d = d[i]
1797                    if d.tg == "mtd" then
1798                        -- okay
1799                    elseif d.content then
1800                        d.content = ""
1801                    else
1802                        d.skip = "comment" -- weird error
1803                    end
1804                end
1805            elseif d.content then
1806                d.content = ""
1807            else
1808                d.skip = "comment" -- weird error
1809            end
1810        end
1811    end
1812
1813    do
1814
1815        local a, z, A, Z = 0x61, 0x7A, 0x41, 0x5A
1816
1817        function extras.mi(di,element,n,fulltag) -- check with content
1818            local str = di.data[1].content
1819            if str and sub(str,1,1) ~= "&" then -- hack but good enough (maybe gsub op eerste)
1820                for v in utfvalues(str) do
1821                    if (v >= a and v <= z) or (v >= A and v <= Z) then
1822                        local a = di.attributes
1823                        if a then
1824                            a.mathvariant = "normal"
1825                        else
1826                            di.attributes = { mathvariant = "normal" }
1827                        end
1828                    end
1829                end
1830            end
1831        end
1832
1833    end
1834
1835    function extras.msub(di,element,n,fulltag)
1836        -- m$^2$
1837        local data = di.data
1838        if #data == 1 then
1839            local d = data[1]
1840            data[2] = d
1841            d.__i__ = 2
1842            data[1] = dummy_nucleus
1843        end
1844    end
1845
1846    extras.msup = extras.msub
1847
1848end
1849
1850do
1851
1852    local registered = { }
1853
1854    function structurestags.setformulacontent(n)
1855        registered[locatedtag("formulacontent")] = {
1856            n = n,
1857        }
1858    end
1859
1860    function extras.formulacontent(di,element,n,fulltag)
1861        local r = registered[fulltag]
1862        if r then
1863            setattribute(di,"n",r.n)
1864        end
1865    end
1866
1867end
1868
1869do
1870
1871    local registered = structures.sections.registered
1872
1873    local function resolve(di,element,n,fulltag)
1874        local data = listdata[fulltag]
1875        if data then
1876            extras.addreference(di,data.references)
1877            return true
1878        else
1879            local data = di.data
1880            if data then
1881                for i=1,#data do
1882                    local di = data[i]
1883                    if di then
1884                        local ft = di.fulltag
1885                        if ft and resolve(di,element,n,ft) then
1886                            return true
1887                        end
1888                    end
1889                end
1890            end
1891        end
1892    end
1893
1894    function extras.section(di,element,n,fulltag)
1895        local r = registered[specifications[fulltag].detail]
1896        if r then
1897            setattribute(di,"level",r.level)
1898        end
1899        resolve(di,element,n,fulltag)
1900    end
1901
1902    local floats = { }
1903
1904    function structurestags.setfloat(options,method)
1905        floats[locatedtag("float")] = {
1906            options = options,
1907            method  = method,
1908        }
1909    end
1910
1911    function extras.float(di,element,n,fulltag)
1912        local hash = floats[fulltag]
1913        if hash then
1914            local method  = hash.method
1915            if not method or method == "" then
1916                method = "here"
1917            end
1918            setattribute(di,"method",method)
1919            local options = hash.options
1920            if options and options ~= "" then
1921                options = settings_to_hash(options)
1922                options[method] = nil
1923                options = concat(sortedkeys(options),",")
1924                if #options > 0 then
1925                    setattribute(di,"options",options)
1926                end
1927            end
1928        end
1929        resolve(di,element,n,fulltag)
1930    end
1931
1932    -- todo: internal is already hashed
1933
1934    function structurestags.setlist(n)
1935        local data = structures.lists.getresult(n)
1936        if data then
1937            referencehash[locatedtag("listitem")] = data
1938        end
1939    end
1940
1941    function extras.listitem(di,element,n,fulltag)
1942        local data = referencehash[fulltag]
1943        if data then
1944            extras.addinternal(di,data.references)
1945            return true
1946        end
1947    end
1948
1949end
1950
1951do
1952
1953    -- todo: internal is already hashed
1954
1955    function structurestags.setregister(tag,n) -- check if tag is needed
1956        local data = structures.registers.get(tag,n)
1957        if data then
1958            referencehash[locatedtag("registerlocation")] = data
1959        end
1960    end
1961
1962    function extras.registerlocation(di,element,n,fulltag)
1963        local data = referencehash[fulltag]
1964        if type(data) == "table" then
1965            extras.addinternal(di,data.references)
1966            return true
1967        else
1968            -- needs checking, probably bookmarks
1969        end
1970    end
1971
1972    function extras.registerpages(di,element,n,fulltag) -- ignorebreaks
1973        local data = di.data
1974        for i=1,#data do
1975            local d = data[i]
1976            if d.content == " " then
1977                d.content = ""
1978            end
1979        end
1980    end
1981
1982    function extras.registerseparator(di,element,n,fulltag) -- ignorespaces
1983        local data = di.data
1984        for i=1,#data do
1985            local d = data[i]
1986            local c = d.content
1987            if type(c) == "string" then
1988                d.content = lpegmatch(p_stripper,c)
1989            end
1990        end
1991    end
1992
1993end
1994
1995do
1996
1997    local tabledata = { }
1998
1999    local function hascontent(data)
2000        for i=1,#data do
2001            local di = data[i]
2002            if not di or di.tg == "ignore" then
2003                --
2004            else
2005                local content = di.content
2006                if content == " " then
2007                    --
2008                elseif content then
2009                    return true
2010                else
2011                    local d = di.data
2012                    if d and #d > 0 and hascontent(d) then
2013                        return true
2014                    end
2015                end
2016            end
2017        end
2018    end
2019
2020    function structurestags.settablecell(rows,columns,align)
2021        if align > 0 or rows > 1 or columns > 1 then -- or kind > 0
2022            tabledata[locatedtag("tablecell")] = {
2023                rows    = rows,
2024                columns = columns,
2025                align   = align,
2026            }
2027        end
2028    end
2029
2030    function structurestags.gettablecell(fulltag)
2031        return tabledata[fulltag]
2032    end
2033
2034    function extras.tablecell(di,element,n,fulltag)
2035        local hash = tabledata[fulltag]
2036        if hash then
2037            local columns = hash.columns
2038            if columns and columns > 1 then
2039                setattribute(di,"columns",columns)
2040            end
2041            local rows = hash.rows
2042            if rows and rows > 1 then
2043                setattribute(di,"rows",rows)
2044            end
2045            local align = hash.align
2046            if not align or align == 0 then
2047                -- normal
2048            elseif align == 1 then -- use numbertoalign here
2049                setattribute(di,"align","flushright")
2050            elseif align == 2 then
2051                setattribute(di,"align","middle")
2052            elseif align == 3 then
2053                setattribute(di,"align","flushleft")
2054            end
2055        end
2056    end
2057
2058    local tabulatedata = { }
2059
2060    function structurestags.settabulatecell(align,kind)
2061        if align > 0 or kind > 0 then
2062            tabulatedata[locatedtag("tabulatecell")] = {
2063                align = align,
2064                kind  = kind, -- 1 = bold head
2065            }
2066        end
2067    end
2068
2069    function structurestags.gettabulatecell(fulltag)
2070        return tabulatedata[fulltag]
2071    end
2072
2073    function extras.tabulate(di,element,n,fulltag)
2074        local data = di.data
2075        for i=1,#data do
2076            local di = data[i]
2077            if di.tg == "tabulaterow" and not hascontent(di.data) then
2078                di.element = "" -- or simply remove
2079            end
2080        end
2081    end
2082
2083    function extras.tabulatecell(di,element,n,fulltag)
2084        local hash = tabulatedata[fulltag]
2085        if hash then
2086            local align = hash.align
2087            if not align or align == 0 then
2088                -- normal
2089            elseif align == 1 then
2090                setattribute(di,"align","flushleft")
2091            elseif align == 2 then
2092                setattribute(di,"align","flushright")
2093            elseif align == 3 then
2094                setattribute(di,"align","middle")
2095            end
2096            local kind = hash.kind
2097            if kind == 1 then
2098                setattribute(di,"kind","strong")
2099            elseif kind == 2 then
2100                setattribute(di,"kind","equals")
2101            end
2102        end
2103    end
2104
2105end
2106
2107do
2108
2109    local usedpublications = { }
2110    local tagsindatasets   = setmetatableindex("table")
2111    local serialize        = false
2112
2113    function structurestags.setpublication(dataset,tag,rendering)
2114        usedpublications[locatedtag("publication")] = {
2115            dataset   = dataset,
2116            tag       = tag,
2117            rendering = rendering
2118        }
2119        tagsindatasets[dataset][tag] = true
2120        if not serialize then
2121            structures.tags.registerextradata("btx",function()
2122                local t = { "<btxdata>"}
2123                for dataset, used in sortedhash(tagsindatasets) do
2124                    t[#t+1] = publications.converttoxml(dataset,true,false,true,false,true,true)
2125                end
2126                t[#t+1] = "</btxdata>"
2127                return concat(t,"\n")
2128            end)
2129        end
2130    end
2131
2132    function extras.publication(di,element,n,fulltag)
2133        local hash = usedpublications[fulltag]
2134        if hash then
2135            setattribute(di,"dataset",hash.dataset)
2136            setattribute(di,"tag",hash.tag)
2137        end
2138    end
2139
2140end
2141
2142do
2143
2144    local usedparagraphs = { }
2145
2146    function structurestags.setparagraph(align)
2147        if align ~= "" then
2148            usedparagraphs[locatedtag("paragraph")] = {
2149                align = align,
2150            }
2151        end
2152    end
2153
2154    function extras.paragraph(di,element,n,fulltag)
2155        local hash = usedparagraphs[fulltag]
2156        if hash then
2157            setattribute(di,"align",hash.align)
2158        end
2159    end
2160
2161end
2162
2163-- flusher
2164
2165do
2166
2167    local f_detail                     = formatters[' detail="%s"']
2168    local f_chain                      = formatters[' chain="%s"']
2169    local f_index                      = formatters[' n="%s"']
2170    local f_spacing                    = formatters['<c p="%s">%s</c>']
2171
2172    local f_empty_inline               = formatters["<%s/>"]
2173    local f_empty_mixed                = formatters["%w<%s/>\n"]
2174    local f_empty_display              = formatters["\n%w<%s/>\n"]
2175    local f_empty_inline_attr          = formatters["<%s%s/>"]
2176    local f_empty_mixed_attr           = formatters["%w<%s%s/>"]
2177    local f_empty_display_attr         = formatters["\n%w<%s%s/>\n"]
2178
2179    local f_begin_inline               = formatters["<%s>"]
2180    local f_begin_mixed                = formatters["%w<%s>"]
2181    local f_begin_display              = formatters["\n%w<%s>\n"]
2182    local f_begin_inline_attr          = formatters["<%s%s>"]
2183    local f_begin_mixed_attr           = formatters["%w<%s%s>"]
2184    local f_begin_display_attr         = formatters["\n%w<%s%s>\n"]
2185
2186    local f_end_inline                 = formatters["</%s>"]
2187    local f_end_mixed                  = formatters["</%s>\n"]
2188    local f_end_display                = formatters["%w</%s>\n"]
2189
2190    local f_begin_inline_comment       = formatters["<!-- %s --><%s>"]
2191    local f_begin_mixed_comment        = formatters["%w<!-- %s --><%s>"]
2192    local f_begin_display_comment      = formatters["\n%w<!-- %s -->\n%w<%s>\n"]
2193    local f_begin_inline_attr_comment  = formatters["<!-- %s --><%s%s>"]
2194    local f_begin_mixed_attr_comment   = formatters["%w<!-- %s --><%s%s>"]
2195    local f_begin_display_attr_comment = formatters["\n%w<!-- %s -->\n%w<%s%s>\n"]
2196
2197    local f_comment_begin_inline       = formatters["<!-- begin %s -->"]
2198    local f_comment_begin_mixed        = formatters["%w<!-- begin %s -->"]
2199    local f_comment_begin_display      = formatters["\n%w<!-- begin %s -->\n"]
2200
2201    local f_comment_end_inline         = formatters["<!-- end %s -->"]
2202    local f_comment_end_mixed          = formatters["<!-- end %s -->\n"]
2203    local f_comment_end_display        = formatters["%w<!-- end %s -->\n"]
2204
2205    local f_metadata_begin             = formatters["\n%w<metadata>\n"]
2206    local f_metadata                   = formatters["%w<metavariable name=%q>%s</metavariable>\n"]
2207    local f_metadata_end               = formatters["%w</metadata>\n"]
2208
2209    local function attributes(a)
2210        local r = { }
2211        local n = 0
2212        for k, v in next, a do
2213            n = n + 1
2214            r[n] = f_attribute(k,tostring(v)) -- tostring because of %q
2215        end
2216        sort(r)
2217        return concat(r,"")
2218    end
2219
2220    local function properties(a)
2221        local r = { }
2222        local n = 0
2223        for k, v in next, a do
2224            n = n + 1
2225            r[n] = f_property(exportproperties,k,tostring(v)) -- tostring because of %q
2226        end
2227        sort(r)
2228        return concat(r,"")
2229    end
2230
2231    local depth  = 0
2232    local inline = 0
2233
2234    local function emptytag(result,element,nature,di) -- currently only break but at some point
2235        local a = di.attributes                       -- we might add detail etc
2236        if a then -- happens seldom
2237            if nature == "display" then
2238                result[#result+1] = f_empty_display_attr(depth,namespaced[element],attributes(a))
2239            elseif nature == "mixed" then
2240                result[#result+1] = f_empty_mixed_attr(depth,namespaced[element],attributes(a))
2241            else
2242                result[#result+1] = f_empty_inline_attr(namespaced[element],attributes(a))
2243            end
2244        else
2245            if nature == "display" then
2246                result[#result+1] = f_empty_display(depth,namespaced[element])
2247            elseif nature == "mixed" then
2248                result[#result+1] = f_empty_mixed(depth,namespaced[element])
2249            else
2250                result[#result+1] = f_empty_inline(namespaced[element])
2251            end
2252        end
2253    end
2254
2255 -- local function stripspaces(di)
2256 --     local d = di.data
2257 --     local n = #d
2258 --     local m = 0
2259 --     for i=1,n do
2260 --         local di = d[i]
2261 --         if di.tg then
2262 --             m = m + 1
2263 --             d[m] = di
2264 --         end
2265 --     end
2266 --     for i=n,m+1,-1 do
2267 --         d[i] = nil
2268 --     end
2269 -- end
2270 --
2271 -- -- simpler:
2272
2273    local function stripspaces(di)
2274        local d = di.data
2275        for i=1,#d do
2276            local di = d[i]
2277            if not di.tg then
2278                di.content = ""
2279            end
2280        end
2281    end
2282
2283    local function begintag(result,element,nature,di,skip)
2284        local index         = di.n
2285        local fulltag       = di.fulltag
2286        local specification = specifications[fulltag] or { } -- we can have a dummy
2287        local comment       = di.comment
2288        local detail        = specification.detail
2289        if skip == "comment" then
2290            if show_comment then
2291                if nature == "inline" or inline > 0 then
2292                    result[#result+1] = f_comment_begin_inline(namespaced[element])
2293                    inline = inline + 1
2294                elseif nature == "mixed" then
2295                    result[#result+1] = f_comment_begin_mixed(depth,namespaced[element])
2296                    depth = depth + 1
2297                    inline = 1
2298                else
2299                    result[#result+1] = f_comment_begin_display(depth,namespaced[element])
2300                    depth = depth + 1
2301                end
2302            end
2303        elseif skip then
2304            -- ignore
2305        else
2306
2307            local n = 0
2308            local r = { } -- delay this
2309            if detail then
2310                detail = gsub(detail,"[^A-Za-z0-9]+","-")
2311                specification.detail = detail -- we use it later in for the div
2312                n = n + 1
2313                r[n] = f_detail(detail)
2314            end
2315            local parents = specification.parents
2316            if parents then
2317                parents = gsub(parents,"[^A-Za-z0-9 ]+","-")
2318                specification.parents = parents -- we use it later in for the div
2319                n = n + 1
2320                r[n] = f_chain(parents)
2321            end
2322            if indexing and index then
2323                n = n + 1
2324                r[n] = f_index(index)
2325            end
2326            --
2327            local extra = extras[element]
2328            if extra then
2329                extra(di,element,index,fulltag)
2330            end
2331            --
2332            if di.record then
2333                stripspaces(di)
2334            end
2335            --
2336            if exportproperties then
2337                local p = specification.userdata
2338                if not p then
2339                    -- skip
2340                elseif exportproperties == v_yes then
2341                    n = n + 1
2342                    r[n] = attributes(p)
2343                else
2344                    n = n + 1
2345                    r[n] = properties(p)
2346                end
2347            end
2348            local a = di.attributes
2349            if a then
2350                if trace_spacing then
2351                    a.p = di.parnumber or 0
2352                end
2353                n = n + 1
2354                r[n] = attributes(a)
2355            elseif trace_spacing then
2356                n = n + 1
2357                r[n] = attributes { p = di.parnumber or 0 }
2358            end
2359            if n == 0 then
2360                if nature == "inline" or inline > 0 then
2361                    if show_comment and comment then
2362                        result[#result+1] = f_begin_inline_comment(comment,namespaced[element])
2363                    else
2364                        result[#result+1] = f_begin_inline(namespaced[element])
2365                    end
2366                    inline = inline + 1
2367                elseif nature == "mixed" then
2368                    if show_comment and comment then
2369                        result[#result+1] = f_begin_mixed_comment(depth,comment,namespaced[element])
2370                    else
2371                        result[#result+1] = f_begin_mixed(depth,namespaced[element])
2372                    end
2373                    depth = depth + 1
2374                    inline = 1
2375                else
2376                    if show_comment and comment then
2377                        result[#result+1] = f_begin_display_comment(depth,comment,depth,namespaced[element])
2378                    else
2379                        result[#result+1] = f_begin_display(depth,namespaced[element])
2380                    end
2381                    depth = depth + 1
2382                end
2383            else
2384                r = concat(r,"",1,n)
2385                if nature == "inline" or inline > 0 then
2386                    if show_comment and comment then
2387                        result[#result+1] = f_begin_inline_attr_comment(comment,namespaced[element],r)
2388                    else
2389                        result[#result+1] = f_begin_inline_attr(namespaced[element],r)
2390                    end
2391                    inline = inline + 1
2392                elseif nature == "mixed" then
2393                    if show_comment and comment then
2394                        result[#result+1] = f_begin_mixed_attr_comment(depth,comment,namespaced[element],r)
2395                    else
2396                        result[#result+1] = f_begin_mixed_attr(depth,namespaced[element],r)
2397                    end
2398                    depth = depth + 1
2399                    inline = 1
2400                else
2401                    if show_comment and comment then
2402                        result[#result+1] = f_begin_display_attr_comment(depth,comment,depth,namespaced[element],r)
2403                    else
2404                        result[#result+1] = f_begin_display_attr(depth,namespaced[element],r)
2405                    end
2406                    depth = depth + 1
2407                end
2408            end
2409        end
2410        used[element][detail or ""] = { nature, specification.parents }  -- for template css
2411        -- also in last else ?
2412        local metadata = specification.metadata
2413        if metadata then
2414            result[#result+1] = f_metadata_begin(depth)
2415            for k, v in table.sortedpairs(metadata) do
2416                if v ~= "" then
2417                    result[#result+1] = f_metadata(depth+1,k,lpegmatch(p_entity,v))
2418                end
2419            end
2420            result[#result+1] = f_metadata_end(depth)
2421        end
2422    end
2423
2424    local function endtag(result,element,nature,di,skip)
2425        if skip == "comment" then
2426            if show_comment then
2427                if nature == "display" and (inline == 0 or inline == 1) then
2428                    depth = depth - 1
2429                    result[#result+1] = f_comment_end_display(depth,namespaced[element])
2430                    inline = 0
2431                elseif nature == "mixed" and (inline == 0 or inline == 1) then
2432                    depth = depth - 1
2433                    result[#result+1] = f_comment_end_mixed(namespaced[element])
2434                    inline = 0
2435                else
2436                    inline = inline - 1
2437                    result[#result+1] = f_comment_end_inline(namespaced[element])
2438                end
2439            end
2440        elseif skip then
2441            -- ignore
2442        else
2443            if nature == "display" and (inline == 0 or inline == 1) then
2444                depth = depth - 1
2445                result[#result+1] = f_end_display(depth,namespaced[element])
2446                inline = 0
2447            elseif nature == "mixed" and (inline == 0 or inline == 1) then
2448                depth = depth - 1
2449                result[#result+1] = f_end_mixed(namespaced[element])
2450                inline = 0
2451            else
2452                inline = inline - 1
2453                result[#result+1] = f_end_inline(namespaced[element])
2454            end
2455        end
2456    end
2457
2458    local function flushtree(result,data,nature)
2459        local nofdata = #data
2460        for i=1,nofdata do
2461            local di = data[i]
2462            if not di then -- hm, di can be string
2463                -- whatever
2464            else
2465                local content = di.content
2466             -- also optimize for content == "" : trace that first
2467                if content then
2468                    -- already has breaks
2469                    local content = lpegmatch(p_entity,content)
2470                    if i == nofdata and sub(content,-1) == "\n" then -- move check
2471                        -- can be an end of line in par but can also be the last line
2472                        if trace_spacing then
2473                            result[#result+1] = f_spacing(di.parnumber or 0,sub(content,1,-2))
2474                        else
2475                            result[#result+1] = sub(content,1,-2)
2476                        end
2477                        result[#result+1] = " "
2478                    else
2479                        if trace_spacing then
2480                            result[#result+1] = f_spacing(di.parnumber or 0,content)
2481                        else
2482                            result[#result+1] = content
2483                        end
2484                    end
2485                elseif not di.collapsed then -- ignore collapsed data (is appended, reconstructed par)
2486                    local element = di.element
2487                    if not element then
2488                        -- skip
2489                    elseif element == "break" then -- or element == "pagebreak"
2490                        emptytag(result,element,nature,di)
2491                    elseif element == "" or di.skip == "ignore" then
2492                        -- skip
2493                    else
2494                        if di.before then
2495                            flushtree(result,di.before,nature)
2496                        end
2497                        local natu = di.nature
2498                        local skip = di.skip
2499                        if di.breaknode then
2500                            emptytag(result,"break","display",di)
2501                        end
2502                        begintag(result,element,natu,di,skip)
2503                        flushtree(result,di.data,natu)
2504                        endtag(result,element,natu,di,skip)
2505                        if di.after then
2506                            flushtree(result,di.after,nature)
2507                        end
2508                    end
2509                end
2510            end
2511        end
2512    end
2513
2514    local function breaktree(tree,parent,parentelement) -- also removes double breaks
2515        local data = tree.data
2516        if data then
2517            local nofdata = #data
2518            local prevelement
2519            local prevnature
2520            local prevparnumber
2521            local newdata = { }
2522            local nofnewdata = 0
2523            for i=1,nofdata do
2524                local di = data[i]
2525                if not di then
2526                    -- skip
2527                elseif di.skip == "ignore" then
2528                    -- skip (new)
2529elseif di.tg == "ignore" then
2530    -- skip (new)
2531                elseif di.content then
2532                    if di.samepar then
2533                        prevparnumber = false
2534                    else
2535                        local parnumber = di.parnumber
2536                        if prevnature == "inline" and prevparnumber and prevparnumber ~= parnumber then
2537                            nofnewdata = nofnewdata + 1
2538                            if trace_spacing then
2539                                newdata[nofnewdata] = makebreaknode { type = "a", p = prevparnumber, n = parnumber }
2540                            else
2541                                newdata[nofnewdata] = makebreaknode()
2542                            end
2543                        end
2544                        prevelement = nil
2545                        prevparnumber = parnumber
2546                    end
2547                    prevnature = "inline"
2548                    nofnewdata = nofnewdata + 1
2549                    newdata[nofnewdata] = di
2550                elseif not di.collapsed then
2551                    local element = di.element
2552                    if element == "break" then -- or element == "pagebreak"
2553                        if prevelement == "break" then
2554                            di.element = ""
2555                        end
2556                        prevelement = element
2557                        prevnature = "display"
2558                        nofnewdata = nofnewdata + 1
2559                        newdata[nofnewdata] = di
2560                    elseif element == "" or di.skip == "ignore" then
2561                        -- skip
2562                    else
2563                        if di.samepar then
2564                            prevnature    = "inline"
2565                            prevparnumber = false
2566                        else
2567                            local nature = di.nature
2568                            local parnumber = di.parnumber
2569                            if prevnature == "inline" and nature == "inline" and prevparnumber and prevparnumber ~= parnumber then
2570                                nofnewdata = nofnewdata + 1
2571                                if trace_spacing then
2572                                    newdata[nofnewdata] = makebreaknode { type = "b", p = prevparnumber, n = parnumber }
2573                                else
2574                                    newdata[nofnewdata] = makebreaknode()
2575                                end
2576                            end
2577                            prevnature = nature
2578                            prevparnumber = parnumber
2579                        end
2580                        prevelement = element
2581                        breaktree(di,tree,element)
2582                        nofnewdata = nofnewdata + 1
2583                        newdata[nofnewdata] = di
2584                    end
2585                else
2586                    if di.samepar then
2587                        prevnature    = "inline"
2588                        prevparnumber = false
2589                    else
2590                        local nature = di.nature
2591                        local parnumber = di.parnumber
2592                        if prevnature == "inline" and nature == "inline" and prevparnumber and prevparnumber ~= parnumber then
2593                            nofnewdata = nofnewdata + 1
2594                            if trace_spacing then
2595                                newdata[nofnewdata] = makebreaknode { type = "c", p = prevparnumber, n = parnumber }
2596                            else
2597                                newdata[nofnewdata] = makebreaknode()
2598                            end
2599                        end
2600                        prevnature = nature
2601                        prevparnumber = parnumber
2602                    end
2603                    nofnewdata = nofnewdata + 1
2604                    newdata[nofnewdata] = di
2605                end
2606            end
2607            tree.data = newdata
2608        end
2609    end
2610
2611    -- also tabulaterow reconstruction .. maybe better as a checker
2612    -- i.e cell attribute
2613
2614    local function collapsetree(tree)
2615--         for tag, trees in sortedhash(treehash) do
2616        for tag, trees in next, treehash do
2617            local d = trees[1].data
2618-- print("!!!!!!!!",tag)
2619-- inspect(trees)
2620            if d then
2621                local nd = #d
2622                if nd > 0 then
2623                    for i=2,#trees do
2624                        local currenttree = trees[i]
2625                        local currentdata = currenttree.data
2626                        local currentpar  = currenttree.parnumber
2627                        local previouspar = trees[i-1].parnumber
2628                        currenttree.collapsed = true
2629                        -- is the next ok?
2630                        if previouspar == 0 or not (di and di.content) then
2631                            previouspar = nil -- no need anyway so no further testing needed
2632                        end
2633                        for j=1,#currentdata do
2634                            local cd = currentdata[j]
2635                            if not cd or cd == "" then
2636                                -- skip
2637                            elseif cd.skip == "ignore" then
2638                                -- skip
2639                            elseif cd.content then
2640                                if not currentpar then
2641                                    -- add space ?
2642                                elseif not previouspar then
2643                                    -- add space ?
2644                                elseif currentpar ~= previouspar then
2645                                    nd = nd + 1
2646                                    if trace_spacing then
2647                                        d[nd] = makebreaknode { type = "d", p = previouspar, n = currentpar }
2648                                    else
2649                                        d[nd] = makebreaknode()
2650                                    end
2651                                end
2652                                previouspar = currentpar
2653                                nd = nd + 1
2654                                d[nd] = cd
2655                            else
2656                                nd = nd + 1
2657                                d[nd] = cd
2658                            end
2659                            currentdata[j] = false
2660                        end
2661                    end
2662                end
2663            end
2664        end
2665    end
2666
2667    local function finalizetree(tree)
2668        for _, finalizer in next, finalizers do
2669            finalizer(tree)
2670        end
2671    end
2672
2673 -- local function showtree(data,when,where)
2674 --     if data then
2675 --         for i=1,#data do
2676 --             local d = data[i]
2677 --             if type(d) == "table" and d.element then
2678 --                 print(when,where,i,d.element,d.parnumber or 0)
2679 --             end
2680 --         end
2681 --     end
2682 -- end
2683
2684    local function indextree(tree)
2685        local data = tree.data
2686        if data then
2687            local n, new = 0, { }
2688         -- showtree(data,"before","index")
2689            for i=1,#data do
2690                local d = data[i]
2691                if not d then
2692                    -- skip
2693                elseif d.content then
2694                    n = n + 1
2695                    new[n] = d
2696                elseif not d.collapsed then
2697                    n = n + 1
2698                    d.__i__ = n
2699                    d.__p__ = tree
2700                    indextree(d)
2701                    new[n] = d
2702                end
2703            end
2704            tree.data = new
2705         -- showtree(new,"after","index")
2706        end
2707    end
2708
2709    local function checktree(tree)
2710        local data = tree.data
2711        if data then
2712         -- showtree(data,"before","check")
2713            for i=1,#data do
2714                local d = data[i]
2715                if type(d) == "table" then
2716                    local check = checks[d.tg]
2717                    if check then
2718                        check(d,data,i)
2719                    end
2720                    checktree(d) -- so parts can pass twice
2721                end
2722            end
2723         -- showtree(data,"after","check")
2724        end
2725    end
2726
2727    local function fixtree(tree)
2728        local data = tree.data
2729        if data then
2730         -- showtree(data,"before","fix")
2731            for i=1,#data do
2732                local d = data[i]
2733                if type(d) == "table" then
2734                    local fix = fixes[d.tg]
2735                    if fix then
2736                        fix(d,data,i)
2737                    end
2738                    fixtree(d) -- so parts can pass twice
2739                end
2740            end
2741         -- showtree(data,"after","fix")
2742        end
2743    end
2744
2745    wrapups.flushtree    = flushtree
2746    wrapups.breaktree    = breaktree
2747    wrapups.collapsetree = collapsetree
2748    wrapups.finalizetree = finalizetree
2749    wrapups.indextree    = indextree
2750    wrapups.checktree    = checktree
2751    wrapups.fixtree      = fixtree
2752
2753end
2754
2755-- collector code
2756
2757local function push(fulltag,depth)
2758    local tg, n, detail, element, nature, record
2759    local specification = specifications[fulltag]
2760    if specification then
2761        tg     = specification.tagname
2762        n      = specification.tagindex
2763        detail = specification.detail
2764    else
2765        -- a break (more efficient if we don't store those in specifications)
2766        tg, n = lpegmatch(tagsplitter,fulltag)
2767        n = tonumber(n) -- to tonumber in tagsplitter
2768    end
2769    local p = properties[tg]
2770    if p then
2771        element = p.export or tg
2772        nature  = p.nature or "inline" -- defaultnature
2773        record  = p.record
2774    end
2775    local treedata = tree.data
2776    local t = { -- maybe we can use the tag table
2777        tg        = tg,
2778        fulltag   = fulltag,
2779        detail    = detail,
2780        n         = n, -- already a number
2781        element   = element,
2782        nature    = nature,
2783        data      = { },
2784        attribute = currentattribute,
2785        parnumber = currentparagraph,
2786        record    = record, -- we can consider storing properties
2787    }
2788    treedata[#treedata+1] = t
2789    currentdepth = currentdepth + 1
2790    nesting[currentdepth] = fulltag
2791    treestack[currentdepth] = tree
2792    if trace_export then
2793        if detail and detail ~= "" then
2794            report_export("%w<%s trigger=%q n=%q paragraph=%q index=%q detail=%q>",currentdepth-1,tg,n,currentattribute or 0,currentparagraph or 0,#treedata,detail)
2795        else
2796            report_export("%w<%s trigger=%q n=%q paragraph=%q index=%q>",currentdepth-1,tg,n,currentattribute or 0,currentparagraph or 0,#treedata)
2797        end
2798    end
2799    tree = t
2800    if tg == "break" then
2801        -- no need for this
2802    else
2803        local h = treehash[fulltag]
2804        if h then
2805            h[#h+1] = t
2806        else
2807            treehash[fulltag] = { t }
2808        end
2809    end
2810end
2811
2812local function pop()
2813    if currentdepth > 0 then
2814        local top = nesting[currentdepth]
2815        tree = treestack[currentdepth]
2816        currentdepth = currentdepth - 1
2817        if trace_export then
2818            if top then
2819                report_export("%w</%s>",currentdepth,match(top,"[^>]+"))
2820            else
2821                report_export("</BAD>")
2822            end
2823        end
2824    else
2825        report_export("%w<!-- too many pops -->",currentdepth)
2826    end
2827end
2828
2829local function continueexport()
2830    if nofcurrentcontent > 0 then
2831        if trace_export then
2832            report_export("%w<!-- injecting pagebreak space -->",currentdepth)
2833        end
2834        nofcurrentcontent = nofcurrentcontent + 1
2835        currentcontent[nofcurrentcontent] = " " -- pagebreak
2836    end
2837end
2838
2839local function pushentry(current)
2840    if not current then
2841        -- bad news
2842        return
2843    end
2844    current = current.taglist
2845    if not current then
2846        -- even worse news
2847        return
2848    end
2849    if restart then
2850        continueexport()
2851        restart = false
2852    end
2853    local newdepth = #current
2854    local olddepth = currentdepth
2855    if trace_export then
2856        report_export("%w<!-- moving from depth %s to %s (%s) -->",currentdepth,olddepth,newdepth,current[newdepth])
2857    end
2858    if olddepth <= 0 then
2859        for i=1,newdepth do
2860            push(current[i],i)
2861        end
2862    else
2863        local difference
2864        if olddepth < newdepth then
2865            for i=1,olddepth do
2866                if current[i] ~= nesting[i] then
2867                    difference = i
2868                    break
2869                end
2870            end
2871        else
2872            for i=1,newdepth do
2873                if current[i] ~= nesting[i] then
2874                    difference = i
2875                    break
2876                end
2877            end
2878        end
2879        if difference then
2880            for i=olddepth,difference,-1 do
2881                pop()
2882            end
2883            for i=difference,newdepth do
2884                push(current[i],i)
2885            end
2886        elseif newdepth > olddepth then
2887            for i=olddepth+1,newdepth do
2888                push(current[i],i)
2889            end
2890        elseif newdepth < olddepth then
2891            for i=olddepth,newdepth,-1 do
2892                pop()
2893            end
2894        elseif trace_export then
2895            report_export("%w<!-- staying at depth %s (%s) -->",currentdepth,newdepth,nesting[newdepth] or "?")
2896        end
2897    end
2898    return olddepth, newdepth
2899end
2900
2901local function pushcontent(oldparagraph,newparagraph)
2902    if nofcurrentcontent > 0 then
2903        if oldparagraph then
2904            if currentcontent[nofcurrentcontent] == "\n" then
2905                if trace_export then
2906                    report_export("%w<!-- removing newline -->",currentdepth)
2907                end
2908                nofcurrentcontent = nofcurrentcontent - 1
2909            end
2910        end
2911        local content = concat(currentcontent,"",1,nofcurrentcontent)
2912        if content == "" then
2913            -- omit; when oldparagraph we could push, remove spaces, pop
2914        elseif somespace[content] and oldparagraph then
2915            -- omit; when oldparagraph we could push, remove spaces, pop
2916        else
2917            local olddepth, newdepth
2918            local list = taglist[currentattribute]
2919            if list then
2920                olddepth, newdepth = pushentry(list)
2921            end
2922            if tree then
2923                local td = tree.data
2924                local nd = #td
2925                td[nd+1] = { parnumber = oldparagraph or currentparagraph, content = content }
2926                if trace_export then
2927                    report_export("%w<!-- start content with length %s -->",currentdepth,utflen(content))
2928                    report_export("%w%s",currentdepth,(gsub(content,"\n","\\n")))
2929                    report_export("%w<!-- stop content -->",currentdepth)
2930                end
2931                if olddepth then
2932                    for i=newdepth-1,olddepth,-1 do
2933                        pop()
2934                    end
2935                end
2936            end
2937        end
2938        nofcurrentcontent = 0
2939    end
2940    if oldparagraph then
2941        pushentry(makebreaklist(currentnesting))
2942        if trace_export then
2943            report_export("%w<!-- break added between paragraph %a and %a -->",currentdepth,oldparagraph,newparagraph)
2944        end
2945    end
2946end
2947
2948local function finishexport()
2949    if trace_export then
2950        report_export("%w<!-- start finalizing -->",currentdepth)
2951    end
2952    if nofcurrentcontent > 0 then
2953        if somespace[currentcontent[nofcurrentcontent]] then
2954            if trace_export then
2955                report_export("%w<!-- removing space -->",currentdepth)
2956            end
2957            nofcurrentcontent = nofcurrentcontent - 1
2958        end
2959        pushcontent()
2960    end
2961    for i=currentdepth,1,-1 do
2962        pop()
2963    end
2964    currentcontent = { } -- we're nice and do a cleanup
2965    if trace_export then
2966        report_export("%w<!-- stop finalizing -->",currentdepth)
2967    end
2968end
2969
2970-- inserts ?
2971
2972local collectresults  do -- too many locals otherwise
2973
2974    local nodecodes        = nodes.nodecodes
2975    local gluecodes        = nodes.gluecodes
2976    local listcodes        = nodes.listcodes
2977    local whatsitcodes     = nodes.whatsitcodes
2978
2979    local subtypes         = nodes.subtypes
2980
2981    local hlist_code       = nodecodes.hlist
2982    local vlist_code       = nodecodes.vlist
2983    local glyph_code       = nodecodes.glyph
2984    local glue_code        = nodecodes.glue
2985    local kern_code        = nodecodes.kern
2986    local disc_code        = nodecodes.disc
2987    local whatsit_code     = nodecodes.whatsit
2988    local par_code         = nodecodes.par
2989
2990    local userskip_code    = gluecodes.userskip
2991    local rightskip_code   = gluecodes.rightskip
2992    local parfillskip_code = gluecodes.parfillskip
2993    local spaceskip_code   = gluecodes.spaceskip
2994    local xspaceskip_code  = gluecodes.xspaceskip
2995
2996    local linelist_code    = listcodes.line
2997
2998    local userdefinedwhatsit_code  = whatsitcodes.userdefined
2999
3000    local privateattribute = attributes.private
3001    local a_image          = privateattribute('image')
3002    local a_reference      = privateattribute('reference')
3003    local a_destination    = privateattribute('destination')
3004    local a_characters     = privateattribute('characters')
3005    local a_exportstatus   = privateattribute('exportstatus')
3006    local a_tagged         = privateattribute('tagged')
3007    local a_taggedpar      = privateattribute("taggedpar")
3008    local a_textblock      = privateattribute("textblock")
3009
3010    local inline_mark      = nodes.pool.userids["margins.inline"]
3011
3012    local nuts             = nodes.nuts
3013
3014    local getnext          = nuts.getnext
3015    local getdisc          = nuts.getdisc
3016    local getlist          = nuts.getlist
3017    local getid            = nuts.getid
3018    local getattr          = nuts.getattr
3019    local setattr          = nuts.setattr -- maybe use properties
3020    local isglyph          = nuts.isglyph
3021    local getkern          = nuts.getkern
3022    local getwidth         = nuts.getwidth
3023
3024    local startofpar       = nuts.startofpar
3025
3026    local nexthlist        = nuts.traversers.hlist
3027    local nextnode         = nuts.traversers.node
3028
3029    local function addtomaybe(maybewrong,c,case)
3030        if trace_export then
3031            report_export("%w<!-- possible paragraph mixup at %C case %i -->",currentdepth,c,case)
3032        else
3033            local s = formatters["%C"](c)
3034            if maybewrong then
3035                maybewrong[#maybewrong+1] = s
3036            else
3037                maybewrong = { s }
3038            end
3039            return maybewrong
3040        end
3041    end
3042
3043    local function showmaybe(maybewrong)
3044        if not trace_export then
3045            report_export("fuzzy paragraph: % t",maybewrong)
3046        end
3047    end
3048
3049    local function showdetail(n,id,subtype)
3050        local a = getattr(n,a_tagged)
3051        local t = taglist[a]
3052        local c = nodecodes[id]
3053        local s = subtypes[id][subtype]
3054        if a and t then
3055            report_export("node %a, subtype %a, tag %a, element %a, tree '% t'",c,s,a,t.tagname,t.taglist)
3056        else
3057            report_export("node %a, subtype %a, untagged",c,s)
3058        end
3059    end
3060
3061    local function collectresults(head,list,pat,pap) -- is last used (we also have currentattribute)
3062        local p
3063        local paragraph
3064        local maybewrong
3065        local pid
3066        for n, id, subtype in nextnode, head do
3067            if trace_details then
3068                showdetail(n,id,subtype)
3069            end
3070            if id == glyph_code then
3071                local c, f = isglyph(n)
3072                local at   = getattr(n,a_tagged) or pat
3073                if not at then
3074                 -- we need to tag the pagebody stuff as being valid skippable
3075                 --
3076                 -- report_export("skipping character: %C (no attribute)",n.char)
3077                else
3078                    if last ~= at then
3079                        local tl = taglist[at]
3080                        local ap = getattr(n,a_taggedpar) or pap
3081                        if paragraph and (not ap or ap < paragraph) then
3082                            maybewrong = addtomaybe(maybewrong,c,1)
3083                        end
3084                        pushcontent()
3085                        currentnesting   = tl
3086                        currentparagraph = ap
3087                        currentattribute = at
3088                        last = at
3089                        pushentry(currentnesting)
3090                        if trace_export then
3091                            report_export("%w<!-- processing glyph %C tagged %a -->",currentdepth,c,at)
3092                        end
3093                        -- We need to intercept this here; maybe I will also move this
3094                        -- to a regular setter at the tex end.
3095                        local r = getattr(n,a_reference)
3096                        if r then
3097                            local t = tl.taglist
3098                            referencehash[t[#t]] = r -- fulltag
3099                        end
3100                        local d = getattr(n,a_destination)
3101                        if d then
3102                            local t = tl.taglist
3103                            destinationhash[t[#t]] = d -- fulltag
3104                        end
3105                        --
3106                    elseif last then
3107                        -- we can consider tagging the pars (lines) in the parbuilder but then we loose some
3108                        -- information unless we inject a special node (but even then we can run into nesting
3109                        -- issues)
3110                        local ap = getattr(n,a_taggedpar) or pap
3111                        if ap ~= currentparagraph then
3112                            pushcontent(currentparagraph,ap)
3113                            pushentry(currentnesting)
3114                            currentattribute = last
3115                            currentparagraph = ap
3116                        end
3117                        if paragraph and (not ap or ap < paragraph) then
3118                            maybewrong = addtomaybe(maybewrong,c,2)
3119                        end
3120                        if trace_export then
3121                            report_export("%w<!-- processing glyph %C tagged %a -->",currentdepth,c,last)
3122                        end
3123                    else
3124                        if trace_export then
3125                            report_export("%w<!-- processing glyph %C tagged %a -->",currentdepth,c,at)
3126                        end
3127                    end
3128                    local s = getattr(n,a_exportstatus)
3129                    if s then
3130                        c = s
3131                    end
3132                    if c == 0 then
3133                        if trace_export then
3134                            report_export("%w<!-- skipping last glyph -->",currentdepth)
3135                        end
3136                    elseif c == 0x20 then
3137                        local a = getattr(n,a_characters)
3138                        nofcurrentcontent = nofcurrentcontent + 1
3139                        if a then
3140                            if trace_export then
3141                                report_export("%w<!-- turning last space into special space %U -->",currentdepth,a)
3142                            end
3143                            currentcontent[nofcurrentcontent] = specialspaces[a] -- special space
3144                        else
3145                            currentcontent[nofcurrentcontent] = " "
3146                        end
3147                    else
3148                        local fc = fontchar[f]
3149                        if fc then
3150                            fc = fc and fc[c]
3151                            if fc then
3152                                local u = fc.unicode
3153                                if not u then
3154                                    nofcurrentcontent = nofcurrentcontent + 1
3155                                    currentcontent[nofcurrentcontent] = utfchar(c)
3156                                elseif type(u) == "table" then
3157                                    for i=1,#u do
3158                                        nofcurrentcontent = nofcurrentcontent + 1
3159                                        currentcontent[nofcurrentcontent] = utfchar(u[i])
3160                                    end
3161                                else
3162                                    nofcurrentcontent = nofcurrentcontent + 1
3163                                    currentcontent[nofcurrentcontent] = utfchar(u)
3164                                end
3165                            elseif c > 0 then
3166                                nofcurrentcontent = nofcurrentcontent + 1
3167                                currentcontent[nofcurrentcontent] = utfchar(c)
3168                            else
3169                                -- we can have -1 as side effect of an explicit hyphen (unless we expand)
3170                            end
3171                        elseif c > 0 then
3172                            nofcurrentcontent = nofcurrentcontent + 1
3173                            currentcontent[nofcurrentcontent] = utfchar(c)
3174                        else
3175                            -- we can have -1 as side effect of an explicit hyphen (unless we expand)
3176                        end
3177                    end
3178                end
3179            elseif id == glue_code then
3180                -- we need to distinguish between hskips and vskips
3181                local ca = getattr(n,a_characters)
3182                if ca == 0 then
3183                    -- skip this one ... already converted special character (node-acc)
3184                elseif ca then
3185                    local a = getattr(n,a_tagged) or pat
3186                    if a then
3187                        local c = specialspaces[ca]
3188                        if last ~= a then
3189                            local tl = taglist[a]
3190                            if trace_export then
3191                                report_export("%w<!-- processing space glyph %U tagged %a case 1 -->",currentdepth,ca,a)
3192                            end
3193                            pushcontent()
3194                            currentnesting = tl
3195                            currentparagraph = getattr(n,a_taggedpar) or pap
3196                            currentattribute = a
3197                            last = a
3198                            pushentry(currentnesting)
3199                            -- no reference check (see above)
3200                        elseif last then
3201                            local ap = getattr(n,a_taggedpar) or pap
3202                            if ap ~= currentparagraph then
3203                                pushcontent(currentparagraph,ap)
3204                                pushentry(currentnesting)
3205                                currentattribute = last
3206                                currentparagraph = ap
3207                            end
3208                            if trace_export then
3209                                report_export("%w<!-- processing space glyph %U tagged %a case 2 -->",currentdepth,ca,last)
3210                            end
3211                        end
3212                        -- if somespace[currentcontent[nofcurrentcontent]] then
3213                        --     if trace_export then
3214                        --         report_export("%w<!-- removing space -->",currentdepth)
3215                        --     end
3216                        --     nofcurrentcontent = nofcurrentcontent - 1
3217                        -- end
3218                        nofcurrentcontent = nofcurrentcontent + 1
3219                        currentcontent[nofcurrentcontent] = c
3220                    end
3221                elseif subtype == userskip_code then
3222                    if getwidth(n) > threshold then
3223                        if last and not somespace[currentcontent[nofcurrentcontent]] then
3224                            local a = getattr(n,a_tagged) or pat
3225                            if a == last then
3226                                if trace_export then
3227                                    report_export("%w<!-- injecting spacing 5a -->",currentdepth)
3228                                end
3229                                nofcurrentcontent = nofcurrentcontent + 1
3230                                currentcontent[nofcurrentcontent] = " "
3231                            elseif a then
3232                                -- e.g LOGO<space>LOGO
3233                                if trace_export then
3234                                    report_export("%w<!-- processing glue > threshold tagged %s becomes %s -->",currentdepth,last,a)
3235                                end
3236                                pushcontent()
3237                                if trace_export then
3238                                    report_export("%w<!-- injecting spacing 5b -->",currentdepth)
3239                                end
3240                                last = a
3241                                nofcurrentcontent = nofcurrentcontent + 1
3242                                currentcontent[nofcurrentcontent] = " "
3243                                currentnesting = taglist[last]
3244                                pushentry(currentnesting)
3245                                currentattribute = last
3246                            end
3247                        end
3248                    end
3249                elseif subtype == spaceskip_code or subtype == xspaceskip_code then
3250                    if not somespace[currentcontent[nofcurrentcontent]] then
3251                        local a = getattr(n,a_tagged) or pat
3252                        if a == last then
3253                            if trace_export then
3254                                report_export("%w<!-- injecting spacing 7 (stay in element) -->",currentdepth)
3255                            end
3256                            nofcurrentcontent = nofcurrentcontent + 1
3257                            currentcontent[nofcurrentcontent] = " "
3258                        else
3259                            if trace_export then
3260                                report_export("%w<!-- injecting spacing 7 (end of element) -->",currentdepth)
3261                            end
3262                            last = a
3263                            pushcontent()
3264                            nofcurrentcontent = nofcurrentcontent + 1
3265                            currentcontent[nofcurrentcontent] = " "
3266                            currentnesting = taglist[last]
3267                            pushentry(currentnesting)
3268                            currentattribute = last
3269                        end
3270                    end
3271                elseif subtype == rightskip_code then
3272                    -- a line
3273                    if nofcurrentcontent > 0 then
3274                        local r = currentcontent[nofcurrentcontent]
3275                        if r == hyphen then
3276                            if not keephyphens then
3277                                nofcurrentcontent = nofcurrentcontent - 1
3278                            end
3279                        elseif pid == disc_code then
3280                            -- go on .. tricky: we should mark the glyhs as coming from a disc
3281                        elseif not somespace[r] then
3282                            local a = getattr(n,a_tagged) or pat
3283                            if a == last then
3284                                if trace_export then
3285                                    report_export("%w<!-- injecting spacing 1 (end of line, stay in element) -->",currentdepth)
3286                                end
3287                                nofcurrentcontent = nofcurrentcontent + 1
3288                                currentcontent[nofcurrentcontent] = " "
3289                            else
3290                                if trace_export then
3291                                    report_export("%w<!-- injecting spacing 1 (end of line, end of element) -->",currentdepth)
3292                                end
3293                                last = a
3294                                pushcontent()
3295                                nofcurrentcontent = nofcurrentcontent + 1
3296                                currentcontent[nofcurrentcontent] = " "
3297                                currentnesting = taglist[last]
3298                                pushentry(currentnesting)
3299                                currentattribute = last
3300                            end
3301                        end
3302                    end
3303                elseif subtype == parfillskip_code then
3304                    -- deal with paragraph endings (crossings) elsewhere and we quit here
3305                    -- as we don't want the rightskip space addition
3306                    if maybewrong then
3307                        showmaybe(maybewrong)
3308                    end
3309                    return
3310                end
3311            elseif id == hlist_code or id == vlist_code then
3312                local ai = getattr(n,a_image)
3313                if ai then
3314                    local at = getattr(n,a_tagged) or pat
3315                    if nofcurrentcontent > 0 then
3316                        pushcontent()
3317                        pushentry(currentnesting) -- ??
3318                    end
3319                    pushentry(taglist[at]) -- has an index, todo: flag empty element
3320                    if trace_export then
3321                        report_export("%w<!-- processing image tagged %a",currentdepth,last)
3322                    end
3323                    last = nil
3324                    currentparagraph = nil
3325                else
3326                    -- we need to determine an end-of-line
3327                    local list = getlist(n)
3328                    if list then
3329                        -- todo: no par checking needed in math
3330                        local at = getattr(n,a_tagged) or pat
3331                        collectresults(list,n,at)
3332                    end
3333                end
3334            elseif id == kern_code then
3335                local kern = getkern(n)
3336                if kern > 0 then
3337local a = getattr(n,a_tagged) or pat
3338local t = taglist[a]
3339if not t or t.tagname ~= "ignore" then -- maybe earlier on top)
3340                    local limit = threshold
3341                    if p then
3342                        local c, f = isglyph(p)
3343                        if c then
3344                            limit = fontquads[f] / 4
3345                        end
3346                    end
3347                    if kern > limit then
3348                        if last and not somespace[currentcontent[nofcurrentcontent]] then
3349--                             local a = getattr(n,a_tagged) or pat
3350                            if a == last then
3351                                if not somespace[currentcontent[nofcurrentcontent]] then
3352                                    if trace_export then
3353                                        report_export("%w<!-- injecting spacing 8 (kern %p) -->",currentdepth,kern)
3354                                    end
3355                                    nofcurrentcontent = nofcurrentcontent + 1
3356                                    currentcontent[nofcurrentcontent] = " "
3357                                end
3358                            elseif a then
3359                                -- e.g LOGO<space>LOGO
3360                                if trace_export then
3361                                    report_export("%w<!-- processing kern, threshold %p, tag %s => %s -->",currentdepth,limit,last,a)
3362                                end
3363                                last = a
3364                                pushcontent()
3365                                if trace_export then
3366                                    report_export("%w<!-- injecting spacing 9 (kern %p) -->",currentdepth,kern)
3367                                end
3368                                nofcurrentcontent = nofcurrentcontent + 1
3369                                currentcontent[nofcurrentcontent] = " "
3370--                                 currentnesting = taglist[last]
3371currentnesting = t
3372                                pushentry(currentnesting)
3373                                currentattribute = last
3374                            end
3375                        end
3376                    end
3377end
3378                end
3379            elseif id == whatsit_code then
3380                if subtype == userdefinedwhatsit_code then
3381                    -- similar to images, see above
3382                    local at = getattr(n,a_tagged)
3383                    if nofcurrentcontent > 0 then
3384                        pushcontent()
3385                        pushentry(currentnesting) -- ??
3386                    end
3387                    pushentry(taglist[at])
3388                    if trace_export then
3389                        report_export("%w<!-- processing anchor tagged %a",currentdepth,last)
3390                    end
3391                    last = nil
3392                    currentparagraph = nil
3393                end
3394            elseif not paragraph and id == par_code and startofpar(n) then
3395                paragraph = getattr(n,a_taggedpar)
3396            elseif id == disc_code then
3397                -- very unlikely because we stripped them
3398                local pre, post, replace = getdisc(n)
3399                if keephyphens then
3400                    if pre and not getnext(pre) and isglyph(pre) == 0xAD then -- hyphencode then
3401                        nofcurrentcontent = nofcurrentcontent + 1
3402                        currentcontent[nofcurrentcontent] = hyphen
3403                    end
3404                end
3405                if replace then
3406                    collectresults(replace,nil)
3407                end
3408            end
3409            p   = n
3410            pid = id
3411        end
3412        if maybewrong then
3413            showmaybe(maybewrong)
3414        end
3415    end
3416
3417    function nodes.handlers.export(head) -- hooks into the page builder
3418        starttiming(treehash)
3419        if trace_export then
3420            report_export("%w<!-- start flushing page -->",currentdepth)
3421        end
3422     -- continueexport()
3423        restart = true
3424        collectresults(head)
3425        if trace_export then
3426            report_export("%w<!-- stop flushing page -->",currentdepth)
3427        end
3428        stoptiming(treehash)
3429        return head
3430    end
3431
3432    function nodes.handlers.checkparcounter(p)
3433        setattr(p,a_taggedpar,texgetcount("tagparcounter") + 1)
3434        return p
3435    end
3436
3437    function builders.paragraphs.tag(head)
3438        noftextblocks = noftextblocks + 1
3439        for n, subtype in nexthlist, head do
3440            if subtype == linelist_code then
3441                setattr(n,a_textblock,noftextblocks)
3442--             elseif subtype == glue_code or subtype == kern_code then -- weird, no list
3443--                 setattr(n,a_textblock,0)
3444            end
3445        end
3446        return false
3447    end
3448
3449end
3450
3451do
3452
3453    local xmlcollected  = xml.collected
3454    local xmlsetcomment = xml.setcomment
3455
3456local xmlpreamble = [[
3457<?xml version="1.0" encoding="UTF-8" standalone="%standalone%" ?>
3458
3459<!--
3460
3461    input filename   : %filename%
3462    processing date  : %date%
3463    context version  : %contextversion%
3464    exporter version : %exportversion%
3465
3466-->
3467
3468]]
3469
3470    local flushtree = wrapups.flushtree
3471
3472    local function wholepreamble(standalone)
3473        return replacetemplate(xmlpreamble, {
3474            standalone     = standalone and "yes" or "no",
3475            filename       = tex.jobname,
3476            date           = included.date and os.fulltime(),
3477            contextversion = environment.version,
3478            exportversion  = exportversion,
3479        })
3480    end
3481
3482
3483local csspreamble = [[
3484<?xml-stylesheet type="text/css" href="%filename%" ?>
3485]]
3486
3487local cssheadlink = [[
3488<link type="text/css" rel="stylesheet" href="%filename%" />
3489]]
3490
3491    local function allusedstylesheets(cssfiles,files,path)
3492        local done   = { }
3493        local result = { }
3494        local extras = { }
3495        for i=1,#cssfiles do
3496            local cssfile = cssfiles[i]
3497            if type(cssfile) ~= "string" then
3498                -- error
3499            elseif cssfile == "export-example.css" then
3500                -- ignore
3501            elseif not done[cssfile] then
3502                cssfile = joinfile(path,basename(cssfile))
3503                report_export("adding css reference '%s'",cssfile)
3504                files[#files+1]   = cssfile
3505                result[#result+1] = replacetemplate(csspreamble, { filename = cssfile })
3506                extras[#extras+1] = replacetemplate(cssheadlink, { filename = cssfile })
3507                done[cssfile]     = true
3508            end
3509        end
3510        return concat(result), concat(extras)
3511    end
3512
3513local elementtemplate = [[
3514/* element="%element%" detail="%detail%" chain="%chain%" */
3515
3516%element%,
3517%namespace%div.%element% {
3518    display: %display% ;
3519}]]
3520
3521local detailtemplate = [[
3522/* element="%element%" detail="%detail%" chain="%chain%" */
3523
3524%element%[detail=%detail%],
3525%namespace%div.%element%.%detail% {
3526    display: %display% ;
3527}]]
3528
3529-- <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1 plus MathML 2.0 plus SVG 1.1//EN" "http://www.w3.org/2002/04/xhtml-math-svg/xhtml-math-svg.dtd" >
3530
3531local htmltemplate = [[
3532%preamble%
3533
3534<html xmlns="http://www.w3.org/1999/xhtml" xmlns:math="http://www.w3.org/1998/Math/MathML">
3535
3536    <head>
3537
3538        <meta charset="utf-8"/>
3539
3540        <title>%title%</title>
3541
3542%style%
3543
3544    </head>
3545    <body>
3546        <div class="document" xmlns="http://www.pragma-ade.com/context/export">
3547
3548<div class="warning">Rendering can be suboptimal because there is no default/fallback css loaded.</div>
3549
3550%body%
3551
3552        </div>
3553    </body>
3554</html>
3555]]
3556
3557    local displaymapping = {
3558        inline  = "inline",
3559        display = "block",
3560        mixed   = "inline",
3561    }
3562
3563    local function allusedelements(filename)
3564        local result = { replacetemplate(namespacetemplate, {
3565            what            = "template",
3566            filename        = filename,
3567            namespace       = contextns,
3568         -- cssnamespaceurl = usecssnamespace and cssnamespaceurl or "",
3569            cssnamespaceurl = cssnamespaceurl,
3570        },false,true) }
3571        for element, details in sortedhash(used) do
3572            if namespaces[element] then
3573                -- skip math
3574            else
3575                for detail, what in sortedhash(details) do
3576                    local nature  = what[1] or "display"
3577                    local chain   = what[2]
3578                    local display = displaymapping[nature] or "block"
3579                    if detail == "" then
3580                        result[#result+1] = replacetemplate(elementtemplate, {
3581                            element   = element,
3582                            display   = display,
3583                            chain     = chain,
3584                            namespace = usecssnamespace and namespace or "",
3585                        })
3586                    else
3587                        result[#result+1] = replacetemplate(detailtemplate, {
3588                            element   = element,
3589                            display   = display,
3590                            detail    = detail,
3591                            chain     = chain,
3592                            namespace = usecssnamespace and cssnamespace or "",
3593                        })
3594                    end
3595                end
3596            end
3597        end
3598        return concat(result,"\n\n")
3599    end
3600
3601    local function allcontent(tree,embed)
3602        local result   = { }
3603        flushtree(result,tree.data,"display") -- we need to collect images
3604        result = concat(result)
3605        -- no need to lpeg .. fast enough
3606        result = gsub(result,"\n *\n","\n")
3607        result = gsub(result,"\n +([^< ])","\n%1")
3608        return result
3609    end
3610
3611    -- local xhtmlpreamble = [[
3612    --     <!DOCTYPE html PUBLIC
3613    --         "-//W3C//DTD XHTML 1.1 plus MathML 2.0 plus SVG 1.1//EN"
3614    --         "http://www.w3.org/2002/04/xhtml-math-svg/xhtml-math-svg.dtd"
3615    --     >
3616    -- ]]
3617
3618    local function cleanxhtmltree(xmltree)
3619        if xmltree then
3620            local implicits = { }
3621            local explicits = { }
3622            local overloads = { }
3623            for e in xmlcollected(xmltree,"*") do
3624                local at = e.at
3625                if at then
3626                    local explicit = at.explicit
3627                    local implicit = at.implicit
3628                    if explicit then
3629                        if not explicits[explicit] then
3630                            explicits[explicit] = true
3631                            at.id = explicit
3632                            if implicit then
3633                                overloads[implicit] = explicit
3634                            end
3635                        end
3636                    else
3637                        if implicit and not implicits[implicit] then
3638                            implicits[implicit] = true
3639                            at.id = "aut:" .. implicit
3640                        end
3641                    end
3642                end
3643            end
3644            for e in xmlcollected(xmltree,"*") do
3645                local at = e.at
3646                if at then
3647                    local internal = at.internal
3648                    local location = at.location
3649                    if internal then
3650                        if location then
3651                            local explicit = overloads[location]
3652                            if explicit then
3653                                at.href = "#" .. explicit
3654                            else
3655                                at.href = "#aut:" .. internal
3656                            end
3657                        else
3658                            at.href = "#aut:" .. internal
3659                        end
3660                    else
3661                        if location then
3662                            at.href = "#" .. location
3663                        else
3664                            local url = at.url
3665                            if url then
3666                                at.href = url
3667                            else
3668                                local file = at.file
3669                                if file then
3670                                    at.href = file
3671                                end
3672                            end
3673                        end
3674                    end
3675                end
3676            end
3677            return xmltree
3678        else
3679            return xml.convert('<?xml version="1.0"?>\n<error>invalid xhtml tree</error>')
3680        end
3681    end
3682
3683    -- maybe the reverse: be explicit about what is permitted
3684
3685    local private = {
3686        destination = true,
3687        prefix      = true,
3688        reference   = true,
3689        --
3690        id          = true,
3691        href        = true,
3692        --
3693        implicit    = true,
3694        explicit    = true,
3695        --
3696        url         = true,
3697        file        = true,
3698        internal    = true,
3699        location    = true,
3700        --
3701        name        = true, -- image name
3702        used        = true, -- image name
3703        page        = true, -- image name
3704        width       = true,
3705        height      = true,
3706        --
3707    }
3708
3709    local addclicks   = true
3710    local f_onclick   = formatters[ [[location.href='%s']] ]
3711    local f_onclick   = formatters[ [[location.href='%s']] ]
3712
3713    local p_cleanid   = lpeg.replacer { [":"] = "-" }
3714    local p_cleanhref = lpeg.Cs(lpeg.P("#") * p_cleanid)
3715
3716    local p_splitter  = lpeg.Ct ( (
3717        lpeg.Carg(1) * lpeg.C((1-lpeg.P(" "))^1) / function(d,s) if not d[s] then d[s] = true return s end end
3718      * lpeg.P(" ")^0 )^1 )
3719
3720
3721    local classes = table.setmetatableindex(function(t,k)
3722        local v = concat(lpegmatch(p_splitter,k,1,{})," ")
3723        t[k] = v
3724        return v
3725    end)
3726
3727    local function makeclass(tg,at)
3728        local detail     = at.detail
3729        local chain      = at.chain
3730        local extra      = nil
3731        local classes    = { }
3732        local nofclasses = 0
3733        at.detail        = nil
3734        at.chain         = nil
3735        for k, v in next, at do
3736            if not private[k] then
3737                nofclasses = nofclasses + 1
3738                classes[nofclasses] = k .. "-" .. v
3739            end
3740        end
3741        if detail and detail ~= "" then
3742            if chain and chain ~= "" then
3743                if chain ~= detail then
3744                    extra = classes[tg .. " " .. chain .. " " .. detail]
3745                elseif tg ~= detail then
3746                    extra = detail
3747                end
3748            elseif tg ~= detail then
3749                extra = detail
3750            end
3751        elseif chain and chain ~= "" then
3752            if tg ~= chain then
3753                extra = chain
3754            end
3755        end
3756        -- in this order
3757        if nofclasses > 0 then
3758            sort(classes)
3759            classes = concat(classes," ")
3760            if extra then
3761                return tg .. " " .. extra .. " " .. classes
3762            else
3763                return tg .. " " .. classes
3764            end
3765        else
3766            if extra then
3767                return tg .. " " .. extra
3768            else
3769                return tg
3770            end
3771        end
3772    end
3773
3774    -- Some elements are not supported (well) in css so we need to retain them. For
3775    -- instance, tablecells have no colspan so basically that renders css table div
3776    -- elements quite useless. A side effect is that we nwo can have conflicts when
3777    -- we mix in with other html (as there is no reset). Of course, when it eventually
3778    -- gets added, there is a change then that those not using the div abstraction
3779    -- will be rediculed.
3780    --
3781    -- a table tr td th thead tbody tfoot
3782    --
3783
3784    local crappycss = {
3785        table     = "table", tabulate      = "table",
3786        tablehead = "thead", tabulatehead  = "thead",
3787        tablebody = "tbody", tabulatebody  = "tbody",
3788        tablefoot = "tfoot", tabulatefoot  = "tfoot",
3789        tablerow  = "tr",    tabulaterow   = "tr",
3790        tablecell = "td",    tabulatecell  = "td",
3791    }
3792
3793    local cssmapping = false
3794
3795    directives.register("export.nativetags", function(v)
3796        cssmapping = v and crappycss or false
3797    end)
3798
3799    local function remap(specification,source,target)
3800        local comment = nil -- share comments
3801        for c in xmlcollected(source,"*") do
3802            if not c.special then
3803                local tg = c.tg
3804                local ns = c.ns
3805                if ns == "m" then
3806                    if false then -- yes or no
3807                        c.ns = ""
3808                        c.at["xmlns:m"] = nil
3809                    end
3810             -- elseif tg == "a" then
3811             --     c.ns = ""
3812                else
3813                    local dt = c.dt
3814                    local nt = #dt
3815                    if nt == 0 or (nt == 1 and dt[1] == "") then
3816                        if comment then
3817                            c.dt = comment
3818                        else
3819                            xmlsetcomment(c,"empty")
3820                            comment = c.dt
3821                        end
3822                    end
3823                    local at    = c.at
3824                    local class = nil
3825                    local label = nil
3826                    if tg == "document" then
3827                        at.href   = nil
3828                        at.detail = nil
3829                        at.chain  = nil
3830                    elseif tg == "metavariable" then
3831                        label = at.name
3832                        at.detail = "metaname-" .. label
3833                        class = makeclass(tg,at)
3834                    else
3835                        class = makeclass(tg,at)
3836                    end
3837                    local id   = at.id
3838                    local href = at.href
3839                    local attr = nil
3840                    if id then
3841                        id = lpegmatch(p_cleanid, id) or id
3842                        if href then
3843                            href = lpegmatch(p_cleanhref,href) or href
3844                            attr = {
3845                                class   = class,
3846                                id      = id,
3847                                href    = href,
3848                                onclick = addclicks and f_onclick(href) or nil,
3849                            }
3850                        else
3851                            attr = {
3852                                class = class,
3853                                id    = id,
3854                            }
3855                        end
3856                    else
3857                        if href then
3858                            href = lpegmatch(p_cleanhref,href) or href
3859                            attr = {
3860                                class   = class,
3861                                href    = href,
3862                                onclick = addclicks and f_onclick(href) or nil,
3863                            }
3864                        else
3865                            attr = {
3866                                class = class,
3867                            }
3868                        end
3869                    end
3870                    c.at = attr
3871                    if label then
3872                        attr.label = label
3873                    end
3874                    c.tg = cssmapping and cssmapping[tg] or "div"
3875                end
3876            end
3877        end
3878    end
3879
3880 -- local cssfile = nil  directives.register("backend.export.css", function(v) cssfile = v end)
3881
3882    local embedfile = false  directives.register("export.embed",function(v) embedfile = v end)
3883
3884    function structurestags.finishexport()
3885
3886        if exporting then
3887            exporting = false
3888        else
3889            return
3890        end
3891
3892        local onlyxml = finetuning.export == v_xml
3893
3894        starttiming(treehash)
3895        --
3896        finishexport()
3897        --
3898        report_export("")
3899        if onlyxml then
3900            report_export("exporting xml, no other files")
3901        else
3902            report_export("exporting xml, xhtml, html and css files")
3903        end
3904        report_export("")
3905        --
3906        wrapups.fixtree(tree)
3907        wrapups.collapsetree(tree)
3908        wrapups.indextree(tree)
3909        wrapups.checktree(tree)
3910        wrapups.breaktree(tree)
3911        wrapups.finalizetree(tree)
3912        --
3913        wrapups.hashlistdata()
3914        --
3915        local askedname = finetuning.file
3916        --
3917        -- we use a dedicated subpath:
3918        --
3919        -- ./jobname-export
3920        -- ./jobname-export/images
3921        -- ./jobname-export/styles
3922        -- ./jobname-export/styles
3923        -- ./jobname-export/jobname-export.xml
3924        -- ./jobname-export/jobname-export.xhtml
3925        -- ./jobname-export/jobname-export.html
3926        -- ./jobname-export/jobname-specification.lua
3927        -- ./jobname-export/styles/jobname-defaults.css
3928        -- ./jobname-export/styles/jobname-styles.css
3929        -- ./jobname-export/styles/jobname-images.css
3930        -- ./jobname-export/styles/jobname-templates.css
3931
3932        if type(askedname) ~= "string" or askedname == "" then
3933            askedname = tex.jobname
3934        end
3935
3936        local usedname  = nameonly(askedname)
3937        local basepath  = usedname .. "-export"
3938        local imagepath = joinfile(basepath,"images")
3939        local stylepath = joinfile(basepath,"styles")
3940
3941        local function validpath(what,pathname)
3942            if lfs.isdir(pathname) then
3943                report_export("using existing %s path %a",what,pathname)
3944                return pathname
3945            end
3946            lfs.mkdir(pathname)
3947            if lfs.isdir(pathname) then
3948                report_export("using cretated %s path %a",what,basepath)
3949                return pathname
3950            else
3951                report_export("unable to create %s path %a",what,basepath)
3952                return false
3953            end
3954        end
3955
3956        if not (validpath("export",basepath) and validpath("images",imagepath) and validpath("styles",stylepath)) then
3957            return
3958        end
3959
3960        -- we're now on the dedicated export subpath so we can't clash names
3961        --
3962        -- a xhtml suffix no longer seems to be work well with browsers
3963
3964        local xmlfilebase           = addsuffix(usedname .. "-raw","xml"  )
3965        local xhtmlfilebase         = addsuffix(usedname .. "-tag","xhtml")
3966        local htmlfilebase          = addsuffix(usedname .. "-div","html")
3967        local specificationfilebase = addsuffix(usedname .. "-pub","lua"  )
3968
3969        local xmlfilename           = joinfile(basepath, xmlfilebase          )
3970        local xhtmlfilename         = joinfile(basepath, xhtmlfilebase        )
3971        local htmlfilename          = joinfile(basepath, htmlfilebase         )
3972        local specificationfilename = joinfile(basepath, specificationfilebase)
3973        --
3974        local defaultfilebase       = addsuffix(usedname .. "-defaults", "css")
3975        local imagefilebase         = addsuffix(usedname .. "-images",   "css")
3976        local stylefilebase         = addsuffix(usedname .. "-styles",   "css")
3977        local templatefilebase      = addsuffix(usedname .. "-templates","css")
3978        --
3979        local defaultfilename       = joinfile(stylepath,defaultfilebase )
3980        local imagefilename         = joinfile(stylepath,imagefilebase   )
3981        local stylefilename         = joinfile(stylepath,stylefilebase   )
3982        local templatefilename      = joinfile(stylepath,templatefilebase)
3983
3984        local cssfile               = finetuning.cssfile
3985
3986        -- we keep track of all used files
3987
3988        local files = {
3989        }
3990
3991        -- we always load the defaults and optionally extra css files; we also copy the example
3992        -- css file so that we always have the latest version
3993
3994        local cssfiles = {
3995            defaultfilebase,
3996            imagefilebase,
3997            stylefilebase,
3998        }
3999
4000        local cssextra = cssfile and table.unique(settings_to_array(cssfile)) or { }
4001
4002        -- at this point we're ready for the content; the collector also does some
4003        -- housekeeping and data collecting; at this point we still have an xml
4004        -- representation that uses verbose element names and carries information in
4005        -- attributes
4006
4007        local data = tree.data
4008        for i=1,#data do
4009            if data[i].tg ~= "document" then
4010                data[i] = { }
4011            end
4012        end
4013
4014        local result = allcontent(tree,embedmath) -- embedfile is for testing
4015
4016        -- ugly but so be it:
4017
4018        local extradata = structures.tags.getextradata()
4019        if extradata then
4020            local t = { "" }
4021            t[#t+1] = "<extradata>"
4022            for name, action in sortedhash(extradata) do
4023                t[#t+1] = action()
4024            end
4025            t[#t+1] = "</extradata>"
4026            t[#t+1] = "</document>"
4027            -- we use a function because otherwise we can have a bad capture index
4028            result = gsub(result,"</document>",function()
4029                return concat(t,"\n")
4030            end)
4031        end
4032
4033        -- done with ugly
4034
4035        if onlyxml then
4036
4037            os.remove(defaultfilename)
4038            os.remove(imagefilename)
4039            os.remove(stylefilename)
4040            os.remove(templatefilename)
4041
4042            for i=1,#cssextra do
4043                os.remove(joinfile(stylepath,basename(source)))
4044            end
4045
4046         -- os.remove(xmlfilename)
4047
4048            os.remove(imagefilename)
4049            os.remove(stylefilename)
4050            os.remove(templatefilename)
4051            os.remove(xhtmlfilename)
4052            os.remove(specificationfilename)
4053            os.remove(htmlfilename)
4054
4055            result = concat {
4056                wholepreamble(true),
4057                "<!-- This export file is used for filtering runtime only! -->\n",
4058                result,
4059            }
4060
4061            report_export("saving xml data in %a",xmlfilename)
4062            io.savedata(xmlfilename,result)
4063
4064            return
4065
4066        end
4067
4068        local examplefilename = resolvers.findfile("export-example.css")
4069        if examplefilename then
4070            local data = io.loaddata(examplefilename)
4071            if not data or data == "" then
4072                data = "/* missing css file */"
4073            elseif not usecssnamespace then
4074                data = gsub(data,cssnamespace,"")
4075            end
4076            io.savedata(defaultfilename,data)
4077        end
4078
4079        if cssfile then
4080            for i=1,#cssextra do
4081                local source = addsuffix(cssextra[i],"css")
4082                local target = joinfile(stylepath,basename(source))
4083                cssfiles[#cssfiles+1] = source
4084                if not lfs.isfile(source) then
4085                    source = joinfile("../",source)
4086                end
4087                if lfs.isfile(source) then
4088                    report_export("copying %s",source)
4089                    file.copy(source,target)
4090                end
4091            end
4092        end
4093
4094        local x_styles, h_styles = allusedstylesheets(cssfiles,files,"styles")
4095
4096        local attach = backends.nodeinjections.attachfile
4097
4098        if embedfile and attach then
4099            -- only for testing
4100            attach {
4101                data       = concat{ wholepreamble(true), result },
4102                name       = basename(xmlfilename),
4103                registered = "export",
4104                title      = "raw xml export",
4105                method     = v_hidden,
4106                mimetype   = "application/mathml+xml",
4107            }
4108        end
4109
4110        result = concat {
4111            wholepreamble(true),
4112            x_styles, -- adds to files
4113            result,
4114        }
4115
4116        cssfiles = table.unique(cssfiles)
4117
4118        -- we're now ready for saving the result in the xml file
4119
4120        report_export("saving xml data in %a",xmlfilename)
4121        io.savedata(xmlfilename,result)
4122
4123        report_export("saving css image definitions in %a",imagefilename)
4124        io.savedata(imagefilename,wrapups.allusedimages(usedname))
4125
4126        report_export("saving css style definitions in %a",stylefilename)
4127        io.savedata(stylefilename,wrapups.allusedstyles(usedname))
4128
4129        report_export("saving css template in %a",templatefilename)
4130        io.savedata(templatefilename,allusedelements(usedname))
4131
4132        -- additionally we save an xhtml file; for that we load the file as xml tree
4133
4134        report_export("saving xhtml variant in %a",xhtmlfilename)
4135
4136        local xmltree = cleanxhtmltree(xml.convert(result))
4137
4138     -- local xmltree = xml.convert(result)
4139     -- for c in xml.collected(xmltree,"m:mtext[lastindex()=1]/m:mrow") do
4140     --     print(c)
4141     -- end
4142     -- for c in xml.collected(xmltree,"mtext/mrow") do
4143     --     print(c)
4144     -- end
4145     -- local xmltree = cleanxhtmltree(xmltree)
4146
4147        xml.save(xmltree,xhtmlfilename)
4148
4149        -- now we save a specification file that can b eused for generating an epub file
4150
4151        -- looking at identity is somewhat redundant as we also inherit from interaction
4152        -- at the tex end
4153
4154        local identity  = interactions.general.getidentity()
4155        local metadata  = structures.tags.getmetadata()
4156
4157        local specification = {
4158            name       = usedname,
4159            identifier = os.uuid(),
4160            images     = wrapups.uniqueusedimages(),
4161            imagefile  = joinfile("styles",imagefilebase),
4162            imagepath  = "images",
4163            stylepath  = "styles",
4164            xmlfiles   = { xmlfilebase },
4165            xhtmlfiles = { xhtmlfilebase },
4166            htmlfiles  = { htmlfilebase },
4167            styles     = cssfiles,
4168            htmlroot   = htmlfilebase,
4169            language   = languagenames[texgetcount("mainlanguagenumber")],
4170            title      = validstring(finetuning.title) or validstring(identity.title),
4171            subtitle   = validstring(finetuning.subtitle) or validstring(identity.subtitle),
4172            author     = validstring(finetuning.author) or validstring(identity.author),
4173            firstpage  = validstring(finetuning.firstpage),
4174            lastpage   = validstring(finetuning.lastpage),
4175            metadata   = metadata,
4176        }
4177
4178        report_export("saving specification in %a",specificationfilename,specificationfilename)
4179
4180        xml.wipe(xmltree,"metadata") -- maybe optional
4181
4182        io.savedata(specificationfilename,table.serialize(specification,true))
4183
4184        -- the html export for epub is different in the sense that it uses div's instead of
4185        -- specific tags
4186
4187        report_export("saving div based alternative in %a",htmlfilename)
4188
4189        remap(specification,xmltree)
4190
4191        -- believe it or not, but a <title/> can prevent viewing in browsers
4192
4193        local title = specification.title
4194
4195        if not title or title == "" then
4196            title = metadata.title
4197            if not title or title == "" then
4198                title = usedname -- was: "no title"
4199            end
4200        end
4201
4202        local variables = {
4203            style    = h_styles,
4204            body     = xml.tostring(xml.first(xmltree,"/div")),
4205            preamble = wholepreamble(false),
4206            title    = title,
4207        }
4208
4209        io.savedata(htmlfilename,replacetemplate(htmltemplate,variables,"xml"))
4210
4211        -- finally we report how an epub file can be made (using the specification)
4212
4213        report_export("")
4214        report_export('create epub with: mtxrun --script epub --make "%s" [--purge --rename --svgmath]',usedname)
4215        report_export("")
4216
4217        stoptiming(treehash)
4218    end
4219
4220    local enableaction = nodes.tasks.enableaction
4221
4222    function structurestags.initializeexport()
4223        if not exporting then
4224            report_export("enabling export to xml")
4225            enableaction("shipouts","nodes.handlers.export")
4226            enableaction("shipouts","nodes.handlers.accessibility")
4227            enableaction("math",    "noads.handlers.tags")
4228            enableaction("everypar","nodes.handlers.checkparcounter")
4229            luatex.registerstopactions(structurestags.finishexport)
4230            exporting = true
4231        end
4232    end
4233
4234    function structurestags.setupexport(t)
4235        merge(finetuning,t)
4236        keephyphens      = finetuning.hyphen == v_yes
4237        exportproperties = finetuning.properties
4238        if exportproperties == v_no then
4239            exportproperties = false
4240        end
4241    end
4242
4243    statistics.register("xml exporting time", function()
4244        if exporting then
4245            return string.format("%s seconds, version %s", statistics.elapsedtime(treehash),exportversion)
4246        end
4247    end)
4248
4249end
4250
4251-- These are called at the tex end:
4252
4253implement {
4254    name      = "setupexport",
4255    actions   = structurestags.setupexport,
4256    arguments = {
4257        {
4258            { "align" },
4259            { "bodyfont", "dimen" },
4260            { "width", "dimen" },
4261            { "properties" },
4262            { "hyphen" },
4263            { "title" },
4264            { "subtitle" },
4265            { "author" },
4266            { "firstpage" },
4267            { "lastpage" },
4268            { "svgstyle" },
4269            { "cssfile" },
4270            { "file" },
4271            { "export" },
4272        }
4273    }
4274}
4275
4276implement {
4277    name      = "finishexport",
4278    actions   = structurestags.finishexport,
4279}
4280
4281implement {
4282    name      = "initializeexport",
4283    actions   = structurestags.initializeexport,
4284}
4285
4286implement {
4287    name      = "settagitemgroup",
4288    actions   = structurestags.setitemgroup,
4289    arguments = { "boolean", "integer", "string" }
4290}
4291
4292implement {
4293    name      = "settagitem",
4294    actions   = structurestags.setitem,
4295    arguments = "string"
4296}
4297
4298implement {
4299    name      = "settagfloat",
4300    actions   = structurestags.setfloat,
4301    arguments = "2 strings",
4302}
4303
4304implement {
4305    name      = "settagformulacontent",
4306    actions   = structurestags.setformulacontent,
4307    arguments = "integer",
4308}
4309
4310implement {
4311    name      = "settagdelimitedsymbol",
4312    actions   = structurestags.settagdelimitedsymbol,
4313    arguments = "string"
4314}
4315
4316implement {
4317    name      = "settagsubsentencesymbol",
4318    actions   = structurestags.settagsubsentencesymbol,
4319    arguments = "string"
4320}
4321
4322implement {
4323    name      = "settagsynonym",
4324    actions   = structurestags.setsynonym,
4325    arguments = "string"
4326}
4327
4328implement {
4329    name      = "settagsorting",
4330    actions   = structurestags.setsorting,
4331    arguments = "string"
4332}
4333
4334implement {
4335    name      = "settagnotation",
4336    actions   = structurestags.setnotation,
4337    arguments = { "string", "integer" }
4338}
4339
4340implement {
4341    name      = "settagnotationsymbol",
4342    actions   = structurestags.setnotationsymbol,
4343    arguments = { "string", "integer" }
4344}
4345
4346implement {
4347    name      = "settaghighlight",
4348    actions   = structurestags.sethighlight,
4349    arguments = { "string", "string", "integer", "integer" }
4350}
4351
4352implement {
4353    name      = "settagconstruct",
4354    actions   = structurestags.setconstruct,
4355    arguments = { "string", "string", "integer", "integer" }
4356}
4357
4358implement {
4359    name      = "settagfigure",
4360    actions    = structurestags.setfigure,
4361    arguments = { "string", "string", "string", "dimen", "dimen", "string" }
4362}
4363
4364implement {
4365    name      = "settagcombination",
4366    actions   = structurestags.setcombination,
4367    arguments = { "integer", "integer" }
4368}
4369
4370implement {
4371    name      = "settagtablecell",
4372    actions   = structurestags.settablecell,
4373    arguments = { "integer", "integer", "integer" }
4374}
4375
4376implement {
4377    name      = "settagtabulatecell",
4378    actions   = structurestags.settabulatecell,
4379    arguments = { "integer", "integer" },
4380}
4381
4382implement {
4383    name      = "settagregister",
4384    actions   = structurestags.setregister,
4385    arguments = { "string", "integer" }
4386}
4387
4388implement {
4389    name      = "settaglist",
4390    actions   = structurestags.setlist,
4391    arguments = "integer"
4392}
4393
4394implement {
4395    name      = "settagpublication",
4396    actions   = structurestags.setpublication,
4397    arguments = "2 strings"
4398}
4399
4400implement {
4401    name      = "settagparagraph",
4402    actions   = structurestags.setparagraph,
4403    arguments = "string"
4404}
4405