math-noa.lmt /size: 103 Kb    last modification: 2024-01-16 09:02
1if not modules then modules = { } end modules ['math-noa'] = {
2    version   = 1.001,
3    optimize  = true,
4    comment   = "companion to math-ini.mkiv",
5    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
6    copyright = "PRAGMA ADE / ConTeXt Development Team",
7    license   = "see context related readme files"
8}
9
10-- TODO: SET CLASSES !
11
12-- if specials and (specials[1] == "char" or specials[1] == "font") then -- can we avoid this
13-- ... better create a reverse mapping from the already present vectors
14
15-- beware: this is experimental code and there will be a more generic (attribute value
16-- driven) interface too but for the moment this is ok (sometime in 2015-2016 i will
17-- start cleaning up as by then the bigger picture is clear and code has been used for
18-- years; the main handlers will get some extensions)
19--
20-- we will also make dedicated processors (faster)
21--
22-- beware: names will change as we wil make noads.xxx.handler i.e. xxx
23-- subnamespaces
24
25-- 20D6 -> 2190
26-- 20D7 -> 2192
27
28-- todo: most is mathchar_code so we can have simple dedicated loops
29
30-- nota bene: uunderdelimiter uoverdelimiter etc are radicals (we have 5 types)
31
32local next, tonumber = next, tonumber
33local formatters, gmatch, match = string.formatters, string.gmatch, string.match
34local sortedhash = table.sortedhash
35local insert, remove = table.insert, table.remove
36local div, round = math.div, math.round
37
38local fonts              = fonts
39local nodes              = nodes
40local node               = node
41local mathematics        = mathematics
42
43local privateattribute   = attributes.private
44local registertracker    = trackers.register
45local registerdirective  = directives.register
46local logreporter        = logs.reporter
47local setmetatableindex  = table.setmetatableindex
48
49local texgetmode         = tex.getmode
50local mathmode_code      = tex.modelevels.math
51
52local colortracers       = nodes.tracers.colors
53
54-- most trace/report will move into the closures
55
56local trace_remapping     = false  registertracker("math.remapping",    function(v) trace_remapping    = v end)
57local trace_processing    = false  registertracker("math.processing",   function(v) trace_processing   = v end)
58local trace_analyzing     = false  registertracker("math.analyzing",    function(v) trace_analyzing    = v end)
59local trace_normalizing   = false  registertracker("math.normalizing",  function(v) trace_normalizing  = v end)
60local trace_collapsing    = false  registertracker("math.collapsing",   function(v) trace_collapsing   = v end)
61local trace_goodies       = false  registertracker("math.goodies",      function(v) trace_goodies      = v end)
62
63local check_coverage      = true   registerdirective("math.checkcoverage", function(v) check_coverage   = v     end)
64local use_math_goodies    = true   registerdirective("math.nogoodies",     function(v) use_math_goodies = not v end)
65
66local report_processing   = logreporter("mathematics","processing")
67local report_remapping    = logreporter("mathematics","remapping")
68local report_normalizing  = logreporter("mathematics","normalizing")
69local report_collapsing   = logreporter("mathematics","collapsing")
70local report_goodies      = logreporter("mathematics","goodies")
71
72local a_mathrendering    = privateattribute("mathrendering")
73local a_exportstatus     = privateattribute("exportstatus")
74
75local nuts               = nodes.nuts
76local nodepool           = nuts.pool
77local tonut              = nuts.tonut
78local nutstring          = nuts.tostring
79
80local setfield           = nuts.setfield
81local setlink            = nuts.setlink
82local setlist            = nuts.setlist
83local setnext            = nuts.setnext
84local setprev            = nuts.setprev
85local setchar            = nuts.setchar
86local setfam             = nuts.setfam
87local setsubtype         = nuts.setsubtype
88local setattr            = nuts.setattr
89local setattrlist        = nuts.setattrlist
90local setwidth           = nuts.setwidth
91local setheight          = nuts.setheight
92local setdepth           = nuts.setdepth
93local setdelimiter       = nuts.setdelimiter
94local setclass           = nuts.setclass
95
96local getfield           = nuts.getfield
97local getnext            = nuts.getnext
98local getprev            = nuts.getprev
99local getboth            = nuts.getboth
100local isnext             = nuts.isnext
101local isprev             = nuts.isprev
102local isboth             = nuts.isboth
103local getid              = nuts.getid
104local getsubtype         = nuts.getsubtype
105local getchar            = nuts.getchar
106local getfont            = nuts.getfont
107local getfam             = nuts.getfam
108local getcharspec        = nuts.getcharspec
109local getattr            = nuts.getattr
110local getattrs           = nuts.getattrs
111local getlist            = nuts.getlist
112local getwidth           = nuts.getwidth
113local getheight          = nuts.getheight
114local getdepth           = nuts.getdepth
115local getwhd             = nuts.getwhd
116local getdelimiter       = nuts.getdelimiter
117local getleftdelimiter   = nuts.getleftdelimiter
118local getrightdelimiter  = nuts.getrightdelimiter
119local gettopdelimiter    = nuts.gettopdelimiter
120local getbottomdelimiter = nuts.getbottomdelimiter
121local getnumerator       = nuts.getnumerator
122local getdenominator     = nuts.getdenominator
123local getdegree          = nuts.getdegree
124local gettop             = nuts.gettop
125local getmiddle          = nuts.getmiddle
126local getbottom          = nuts.getbottom
127local getchoice          = nuts.getchoice
128
129local getnucleus         = nuts.getnucleus
130local getsub             = nuts.getsub
131local getsup             = nuts.getsup
132local getsubpre          = nuts.getsubpre
133local getsuppre          = nuts.getsuppre
134local getprime           = nuts.getprime
135
136local setnucleus         = nuts.setnucleus
137local setsub             = nuts.setsub
138local setsup             = nuts.setsup
139local setsubpre          = nuts.setsubpre
140local setsuppre          = nuts.setsuppre
141local setprime           = nuts.setprime
142
143local getoffsets         = nuts.getoffsets
144local setoffsets         = nuts.setoffsets
145
146local getoptions         = nuts.getoptions
147local setoptions         = nuts.setoptions
148
149local flushnode          = nuts.flush
150local copy_node          = nuts.copy
151local slide_nodes        = nuts.slide
152local set_visual         = nuts.setvisual
153
154local mlisttohlist       = nuts.mlisttohlist
155
156local new_kern           = nodepool.kern
157local new_submlist       = nodepool.submlist
158local new_noad           = nodepool.noad
159local new_delimiter      = nodepool.delimiter
160local new_fence          = nodepool.fence
161
162local fonthashes         = fonts.hashes
163local fontdata           = fonthashes.identifiers
164local fontcharacters     = fonthashes.characters
165local fontitalics        = fonthashes.italics
166local fontparameters     = fonthashes.parameters
167
168local variables          = interfaces.variables
169local texsetattribute    = tex.setattribute
170local texgetattribute    = tex.getattribute
171local getfontoffamily    = tex.getfontoffamily
172local unsetvalue         = attributes.unsetvalue
173local implement          = interfaces.implement
174
175local v_reset            = variables.reset
176local v_small            = variables.small
177local v_medium           = variables.medium
178local v_big              = variables.big
179local v_line             = variables.line
180
181local chardata           = characters.data
182
183noads                    = noads or { }  -- todo: only here
184local noads              = noads
185
186noads.processors         = noads.processors or { }
187local processors         = noads.processors
188
189noads.handlers           = noads.handlers   or { }
190local handlers           = noads.handlers
191
192local tasks              = nodes.tasks
193local enableaction       = tasks.enableaction
194local setaction          = tasks.setaction
195
196local nodecodes          = nodes.nodecodes
197----- noadcodes          = nodes.noadcodes
198local fencecodes         = nodes.fencecodes
199local classes            = mathematics.classes -- or nodes.noadcodes
200
201local ordinary_class     = classes.ordinary
202local operator_class     = classes.operator
203local binary_class       = classes.binary
204local relation_class     = classes.relation
205local open_class         = classes.open
206local close_class        = classes.close
207local middle_class       = classes.middle
208local punctuation_class  = classes.punctuation
209local fenced_class       = classes.fenced
210local fraction_class     = classes.fraction
211local radical_class      = classes.radical
212local accent_class       = classes.accent
213local numbergroup_class  = classes.numbergroup
214local digit_class        = classes.digit
215
216local noad_code          = nodecodes.noad
217local accent_code        = nodecodes.accent
218local radical_code       = nodecodes.radical
219local fraction_code      = nodecodes.fraction
220local subbox_code        = nodecodes.subbox
221local submlist_code      = nodecodes.submlist
222local mathchar_code      = nodecodes.mathchar
223local mathtextchar_code  = nodecodes.mathtextchar
224local delimiter_code     = nodecodes.delimiter
225----- style_code         = nodecodes.style
226----- parameter_code     = nodecodes.parameter
227local math_choice        = nodecodes.choice
228local fence_code         = nodecodes.fence
229
230local leftfence_code     = fencecodes.left
231local middlefence_code   = fencecodes.middle
232local rightfence_code    = fencecodes.right
233
234-- this initial stuff is tricky as we can have removed and new nodes with the same address
235-- the only way out is a free-per-page list of nodes (not bad anyway)
236
237-- local gf = getfield local gt = setmetatableindex("number") getfield = function(n,f)   gt[f] = gt[f] + 1 return gf(n,f)   end mathematics.GETFIELD = gt
238-- local sf = setfield local st = setmetatableindex("number") setfield = function(n,f,v) st[f] = st[f] + 1        sf(n,f,v) end mathematics.SETFIELD = st
239
240-- TODO : get rid of done
241
242local function process(start,what,n,parent)
243
244    if n then
245        n = n + 1
246    else
247        n = 0
248    end
249    --
250    local initial = start
251    --
252--     slide_nodes(start) -- we still miss a prev in noads -- fences test code
253    --
254    while start do
255        local noad = nil
256        local id   = getid(start)
257        if trace_processing then
258            if id == noad_code then
259                report_processing("%w%S, class %a",n*2,nutstring(start),classes[getsubtype(start)])
260            elseif id == mathchar_code then
261                local char, font, fam = getcharspec(start)
262                report_processing("%w%S, family %a, font %a, char %a, shape %c",n*2,nutstring(start),fam,font,char,char)
263            else
264                report_processing("%w%S",n*2,nutstring(start))
265            end
266        end
267        local proc = what[id]
268        if proc then
269         -- report_processing("start processing")
270            local done, newstart, newinitial = proc(start,what,n,parent) -- prev is bugged:  or getprev(start)
271            if newinitial then
272                initial = newinitial -- temp hack .. we will make all return head
273                if newstart then
274                    start = newstart
275                 -- report_processing("stop processing (new start)")
276                else
277                 -- report_processing("quit processing (done)")
278                    break
279                end
280            else
281                if newstart then
282                    start = newstart
283                 -- report_processing("stop processing (new start)")
284                else
285                 -- report_processing("stop processing")
286                end
287            end
288        elseif id == noad_code then
289            -- single characters are like this
290            noad = getnucleus(start)        if noad then process(noad,what,n,start) end -- list
291            noad = getsup(start)            if noad then process(noad,what,n,start) end -- list
292            noad = getsub(start)            if noad then process(noad,what,n,start) end -- list
293            noad = getsuppre(start)         if noad then process(noad,what,n,start) end -- list
294            noad = getsubpre(start)         if noad then process(noad,what,n,start) end -- list
295            noad = getprime(start)          if noad then process(noad,what,n,start) end -- list
296        elseif id == mathchar_code or id == mathtextchar_code or id == delimiter_code then
297            break
298        elseif id == subbox_code or id == submlist_code then
299            noad = getlist(start)            if noad then process(noad,what,n,start) end -- list (not getlist !)
300        elseif id == fraction_code then
301            noad = getnumerator(start)       if noad then process(noad,what,n,start) end -- list
302            noad = getdenominator(start)     if noad then process(noad,what,n,start) end -- list
303            noad = getleftdelimiter(start)   if noad then process(noad,what,n,start) end -- delimiter
304            noad = getdelimiter(start)       if noad then process(noad,what,n,start) end -- delimiter
305            noad = getrightdelimiter(start)  if noad then process(noad,what,n,start) end -- delimiter
306        elseif id == fence_code then
307            noad = getdelimiter(start)       if noad then process(noad,what,n,start) end -- delimiter
308            noad = gettop(start)             if noad then process(noad,what,n,start) end -- list
309            noad = getbottom(start)          if noad then process(noad,what,n,start) end -- list
310        elseif id == radical_code then
311            noad = getnucleus(start)         if noad then process(noad,what,n,start) end -- list
312            noad = getsup(start)             if noad then process(noad,what,n,start) end -- list
313            noad = getsub(start)             if noad then process(noad,what,n,start) end -- list
314            noad = getsuppre(start)          if noad then process(noad,what,n,start) end -- list
315            noad = getsubpre(start)          if noad then process(noad,what,n,start) end -- list
316            noad = getprime(start)           if noad then process(noad,what,n,start) end -- delimiter
317            noad = getleftdelimiter(start)   if noad then process(noad,what,n,start) end -- delimiter
318            noad = getrightdelimiter(start)  if noad then process(noad,what,n,start) end -- delimiter
319            noad = gettopdelimiter(start)    if noad then process(noad,what,n,start) end -- delimiter
320            noad = getbottomdelimiter(start) if noad then process(noad,what,n,start) end -- delimiter
321            noad = getdegree(start)          if noad then process(noad,what,n,start) end -- list
322        elseif id == accent_code then
323            noad = getnucleus(start)         if noad then process(noad,what,n,start) end -- list
324            noad = getsup(start)             if noad then process(noad,what,n,start) end -- list
325            noad = getsub(start)             if noad then process(noad,what,n,start) end -- list
326            noad = getsuppre(start)          if noad then process(noad,what,n,start) end -- list
327            noad = getsubpre(start)          if noad then process(noad,what,n,start) end -- list
328            noad = getprime(start)           if noad then process(noad,what,n,start) end -- list
329            noad = gettop(start)             if noad then process(noad,what,n,start) end -- list
330            noad = getmiddle(start)          if noad then process(noad,what,n,start) end -- list
331            noad = getbottom(start)          if noad then process(noad,what,n,start) end -- list
332        elseif id == math_choice then
333            noad = getchoice(start,1)        if noad then process(noad,what,n,start) end -- list
334            noad = getchoice(start,2)        if noad then process(noad,what,n,start) end -- list
335            noad = getchoice(start,3)        if noad then process(noad,what,n,start) end -- list
336            noad = getchoice(start,4)        if noad then process(noad,what,n,start) end -- list
337     -- elseif id == style_code then
338     --     -- has a next
339     -- elseif id == parameter_code then
340     --     -- has a next
341     -- else
342     --     -- glue, penalty, etc
343        end
344        start = getnext(start)
345    end
346    if not parent then
347        return initial -- only first level -- for now
348    end
349end
350
351local function processnested(current,what,n)
352    local noad = nil
353    local id   = getid(current)
354    if id == noad_code then
355        noad = getnucleus(current)         if noad then process(noad,what,n,current) end -- list
356        noad = getsup(current)             if noad then process(noad,what,n,current) end -- list
357        noad = getsub(current)             if noad then process(noad,what,n,current) end -- list
358        noad = getsuppre(current)          if noad then process(noad,what,n,current) end -- list
359        noad = getsubpre(current)          if noad then process(noad,what,n,current) end -- list
360        noad = getprime(current)           if noad then process(noad,what,n,current) end -- list
361    elseif id == subbox_code or id == submlist_code then
362        noad = getlist(current)            if noad then process(noad,what,n,current) end -- list (not getlist !)
363    elseif id == fraction_code then
364        noad = getnumerator(current)       if noad then process(noad,what,n,current) end -- list
365        noad = getdenominator(current)     if noad then process(noad,what,n,current) end -- list
366        noad = getleftdelimiter(current)   if noad then process(noad,what,n,current) end -- delimiter
367        noad = getdelimiter(current)       if noad then process(noad,what,n,current) end -- delimiter
368        noad = getrightdelimiter(current)  if noad then process(noad,what,n,current) end -- delimiter
369    elseif id == fence_code then
370        noad = getdelimiter(current)       if noad then process(noad,what,n,current) end -- delimiter
371        noad = gettop(current)             if noad then process(noad,what,n,current) end -- list
372        noad = getbottom(current)          if noad then process(noad,what,n,current) end -- list
373    elseif id == radical_code then
374        noad = getnucleus(current)         if noad then process(noad,what,n,current) end -- list
375        noad = getsup(current)             if noad then process(noad,what,n,current) end -- list
376        noad = getsub(current)             if noad then process(noad,what,n,current) end -- list
377        noad = getsuppre(current)          if noad then process(noad,what,n,current) end -- list
378        noad = getsubpre(current)          if noad then process(noad,what,n,current) end -- list
379        noad = getprime(current)           if noad then process(noad,what,n,current) end -- list
380        noad = getleftdelimiter(current)   if noad then process(noad,what,n,current) end -- delimiter
381        noad = getrightdelimiter(current)  if noad then process(noad,what,n,current) end -- delimiter
382        noad = gettopdelimiter(current)    if noad then process(noad,what,n,current) end -- delimiter
383        noad = getbottomdelimiter(current) if noad then process(noad,what,n,current) end -- delimiter
384        noad = getdegree(current)          if noad then process(noad,what,n,current) end -- list
385    elseif id == accent_code then
386        noad = getnucleus(current)         if noad then process(noad,what,n,current) end -- list
387        noad = getsup(current)             if noad then process(noad,what,n,current) end -- list
388        noad = getsub(current)             if noad then process(noad,what,n,current) end -- list
389        noad = getsuppre(current)          if noad then process(noad,what,n,current) end -- list
390        noad = getsubpre(current)          if noad then process(noad,what,n,current) end -- list
391        noad = getprime(current)           if noad then process(noad,what,n,current) end -- list
392        noad = gettop(current)             if noad then process(noad,what,n,current) end -- list
393        noad = getmiddle(current)          if noad then process(noad,what,n,current) end -- list
394        noad = getbottom(current)          if noad then process(noad,what,n,current) end -- list
395    elseif id == math_choice then
396        noad = getchoice(start,1)          if noad then process(noad,what,n,current) end -- list
397        noad = getchoice(start,2)          if noad then process(noad,what,n,current) end -- list
398        noad = getchoice(start,3)          if noad then process(noad,what,n,current) end -- list
399        noad = getchoice(start,4)          if noad then process(noad,what,n,current) end -- list
400    end
401end
402
403local function processstep(current,process,n,id)
404    local noad = nil
405    local id   = id or getid(current)
406    if id == noad_code then
407        noad = getnucleus(current)         if noad then process(noad,n,current) end -- list
408        noad = getsup(current)             if noad then process(noad,n,current) end -- list
409        noad = getsub(current)             if noad then process(noad,n,current) end -- list
410        noad = getsuppre(current)          if noad then process(noad,n,current) end -- list
411        noad = getsubpre(current)          if noad then process(noad,n,current) end -- list
412        noad = getprime(current)           if noad then process(noad,n,current) end -- list
413    elseif id == subbox_code or id == submlist_code then
414        noad = getlist(current)            if noad then process(noad,n,current) end -- list (not getlist !)
415    elseif id == fraction_code then
416        noad = getnumerator(current)       if noad then process(noad,n,current) end -- list
417        noad = getdenominator(current)     if noad then process(noad,n,current) end -- list
418        noad = getleftdelimiter(current)   if noad then process(noad,n,current) end -- delimiter
419        noad = getdelimiter(current)       if noad then process(noad,n,current) end -- delimiter
420        noad = getrightdelimiter(current)  if noad then process(noad,n,current) end -- delimiter
421    elseif id == fence_code then
422        noad = getdelimiter(current)       if noad then process(noad,n,current) end -- delimiter
423        noad = gettop(current)             if noad then process(noad,n,current) end -- list
424        noad = getbottom(current)          if noad then process(noad,n,current) end -- list
425    elseif id == radical_code then
426        noad = getnucleus(current)         if noad then process(noad,n,current) end -- list
427        noad = getsup(current)             if noad then process(noad,n,current) end -- list
428        noad = getsub(current)             if noad then process(noad,n,current) end -- list
429        noad = getsuppre(current)          if noad then process(noad,n,current) end -- list
430        noad = getsubpre(current)          if noad then process(noad,n,current) end -- list
431        noad = getprime(current)           if noad then process(noad,n,current) end -- delimiter
432        noad = getleftdelimiter(current)   if noad then process(noad,n,current) end -- delimiter
433        noad = getrightdelimiter(current)  if noad then process(noad,n,current) end -- delimiter
434        noad = gettopdelimiter(current)    if noad then process(noad,n,current) end -- delimiter
435        noad = getbottomdelimiter(current) if noad then process(noad,n,current) end -- delimiter
436        noad = getdegree(current)          if noad then process(noad,n,current) end -- list
437    elseif id == accent_code then
438        noad = getnucleus(current)         if noad then process(noad,n,current) end -- list
439        noad = getsup(current)             if noad then process(noad,n,current) end -- list
440        noad = getsub(current)             if noad then process(noad,n,current) end -- list
441        noad = getsuppre(current)          if noad then process(noad,n,current) end -- list
442        noad = getsubpre(current)          if noad then process(noad,n,current) end -- list
443        noad = getprime(current)           if noad then process(noad,n,current) end -- list
444        noad = gettop(current)             if noad then process(noad,n,current) end -- list
445        noad = getmiddle(current)          if noad then process(noad,n,current) end -- list
446        noad = getbottom(current)          if noad then process(noad,n,current) end -- list
447    elseif id == math_choice then
448        noad = getchoice(start,1)          if noad then process(noad,n,current) end -- list
449        noad = getchoice(start,2)          if noad then process(noad,n,current) end -- list
450        noad = getchoice(start,3)          if noad then process(noad,n,current) end -- list
451        noad = getchoice(start,4)          if noad then process(noad,n,current) end -- list
452    end
453end
454
455local function processnoads(head,actions,banner)
456    if trace_processing then
457        report_processing("start %a",banner)
458        head = process(head,actions)
459        report_processing("stop %a",banner)
460    else
461        head = process(head,actions)
462    end
463    return head
464end
465
466noads.process       = processnoads
467noads.processnested = processnested
468noads.processouter  = process
469
470-- experiment (when not present fall back to fam 0) -- needs documentation
471
472local unknowns = { }
473local checked  = { } -- simple case
474local tracked  = false  trackers.register("fonts.missing", function(v) tracked = v end)
475local cached   = setmetatableindex("table") -- complex case
476
477local variantselectors = {
478    [0xFE00] = true,
479    [0xFE01] = true
480}
481
482local function errorchar(font,char)
483    if variantselectors[char] then
484        return char
485    end
486    local done = unknowns[char]
487    if done then
488        unknowns[char] = done  + 1
489    else
490        unknowns[char] = 1
491    end
492    if tracked then
493        -- slower as we check each font too and we always replace as math has
494        -- more demands than text
495        local fake = cached[font][char]
496        if fake then
497            return fake
498        else
499            local kind, fake = fonts.checkers.placeholder(font,char)
500            if not fake or kind ~= "char" then -- Also check for "with" here?
501                fake = 0x3F
502            end
503            cached[font][char] = fake
504            return fake
505        end
506    else
507        -- only simple checking, report at the end so one should take
508        -- action anyway ... we can miss a few checks but that is ok
509        -- as there is at least one reported
510        if not checked[char] then
511            if trace_normalizing then
512                report_normalizing("character %C is not available",char)
513            end
514            checked[char] = true
515        end
516        return 0x3F
517    end
518end
519
520-- 0-2 regular
521-- 3-5 bold
522-- 6-8 pseudobold
523
524-- this could best be integrated in the remapper, and if we run into problems, we
525-- might as well do this
526
527do
528
529    local families     = { }
530    local a_mathfamily = privateattribute("mathfamily")
531    local boldmap      = mathematics.boldmap
532
533    local trace_families  = false  registertracker("math.families", function(v) trace_families = v end)
534    local report_families = logreporter("mathematics","families")
535
536    local familymap = { [0] =
537        "regular",
538        "regular",
539        "regular",
540        "bold",
541        "bold",
542        "bold",
543        "pseudobold",
544        "pseudobold",
545        "pseudobold",
546    }
547
548    families[noad_code] = function(pointer,what,n,parent)
549        local a = getattr(pointer,a_mathfamily)
550        if a and a >= 0 then
551            if a > 0 then
552                setattr(pointer,a_mathfamily,0)
553                if a > 5 then
554                    a = a - 3
555                end
556            end
557            setfam(pointer,a)
558        end
559        processnested(pointer,families,n+1)
560    end
561
562    families[fraction_code] = families[noad_code]
563    families[accent_code]   = families[noad_code] -- added 2023-07
564    families[radical_code]  = families[noad_code] -- added 2023-07
565    families[fence_code]    = families[noad_code] -- added 2023-07
566
567    families[mathchar_code] = function(pointer)
568        if getfam(pointer) == 0 then
569            local a = getattr(pointer,a_mathfamily)
570            if a and a > 0 then
571                setattr(pointer,a_mathfamily,0)
572                if a > 5 then
573                    local char = getchar(pointer)
574                    local bold = boldmap[char]
575                    local newa = a - 3
576                    if not bold then
577                        if trace_families then
578                            report_families("no bold replacement for %C, family %s with remap %s becomes %s with remap %s",char,a,familymap[a],newa,familymap[newa])
579                        end
580                        setfam(pointer,newa)
581                    elseif not fontcharacters[getfontoffamily(newa)][bold] then
582                        if trace_families then
583                            report_families("no bold character for %C, family %s with remap %s becomes %s with remap %s",char,a,familymap[a],newa,familymap[newa])
584                        end
585                        if newa > 3 then
586                            setfam(pointer,newa-3)
587                        end
588                    else
589                        setattr(pointer,a_exportstatus,char)
590                        setchar(pointer,bold)
591                        if trace_families then
592                            report_families("replacing %C by bold %C, family %s with remap %s becomes %s with remap %s",char,bold,a,familymap[a],newa,familymap[newa])
593                        end
594                        setfam(pointer,newa)
595                    end
596                else
597                    local char = getchar(pointer)
598                    if not fontcharacters[getfontoffamily(a)][char] then
599                        if trace_families then
600                            report_families("no bold replacement for %C",char)
601                        end
602                    else
603                        if trace_families then
604                            report_families("family of %C becomes %s with remap %s",char,a,familymap[a])
605                        end
606                        setfam(pointer,a)
607                    end
608                end
609            end
610        end
611    end
612
613    -- has become:
614
615    families[delimiter_code] = function(pointer)
616        if getfam(pointer) == 0 then
617            local a = getattr(pointer,a_mathfamily)
618            if a and a > 0 then
619                setattr(pointer,a_mathfamily,0)
620                if a > 5 then
621                    -- no bold delimiters in unicode
622                    a = a - 3
623                end
624                local char = getchar(pointer)
625                local okay = fontcharacters[getfontoffamily(a)][char]
626                if okay then
627                    setfam(pointer,a)
628                elseif a > 2 then
629                    setfam(pointer,a-3)
630                end
631            else
632                setfam(pointer,0)
633            end
634        end
635    end
636
637    families[mathtextchar_code] = families[mathchar_code]
638
639    function handlers.families(head,style,penalties)
640        processnoads(head,families,"families")
641    end
642
643end
644
645-- character remapping
646
647do
648
649    local a_mathalphabet       = privateattribute("mathalphabet")
650    local a_mathgreek          = privateattribute("mathgreek")
651
652    local relocate             = { }
653
654    local remapalphabets       = mathematics.remapalphabets
655    local fallbackstyleattr    = mathematics.fallbackstyleattr
656    local fallbackalphabetattr = mathematics.fallbackalphabetattr
657
658    local function report_remap(tag,id,old,new,extra)
659        if new then
660            report_remapping("remapping %s in font (%s,%s) from %C to %C%s",
661                tag,id,fontdata[id].properties.fontname or "",old,new,extra)
662        else
663            -- error !
664            report_remapping("remapping %s in font (%s,%s) from %C to ?",
665                tag,id,fontdata[id].properties.fontname or "",old)
666        end
667    end
668
669    local function checked(pointer)
670        local char, font = getcharspec(pointer)
671        local data = fontcharacters[font]
672        if not data[char] then
673            local specials = characters.data[char].specials
674            if specials and (specials[1] == "char" or specials[1] == "font") then
675                local newchar = specials[#specials]
676                if trace_remapping then
677                    report_remap("fallback",font,char,newchar)
678                end
679                if trace_analyzing then
680                    setnodecolor(pointer,"font:isol")
681                end
682                setattr(pointer,a_exportstatus,char) -- testcase: exponentiale
683                setchar(pointer,newchar)
684                return true
685            end
686        end
687    end
688
689    -- We can optimize this if we really think that math is a bottleneck which it never
690    -- really is. Beware: the font is the text font in the family, so we only check the
691    -- text font here.
692
693    relocate[mathchar_code] = function(pointer)
694        local g          = getattr(pointer,a_mathgreek) or 0
695        local a          = getattr(pointer,a_mathalphabet) or 0
696     -- local g, a       = getattrs(pointer,a_mathgreek,a_mathalphabet)
697     -- if not a then a = 0 end
698     -- if not g then g = 0 end
699        local char, font, fam = getcharspec(pointer)
700        local characters = fontcharacters[font]
701        if a > 0 or g > 0 then
702            if a > 0 then
703                -- not really critital but we could use properties
704                setattr(pointer,a_mathgreek,0)
705            end
706            if g > 0 then
707                -- not really critital but we could use properties
708                setattr(pointer,a_mathalphabet,0)
709            end
710            local newchar = remapalphabets(char,a,g)
711            if newchar then
712                local newchardata = characters[newchar]
713                if newchardata then
714                    if trace_remapping then
715                        report_remap("char",font,char,newchar,newchardata.commands and " (virtual)" or "")
716                    end
717                    if trace_analyzing then
718                        setnodecolor(pointer,"font:isol")
719                    end
720                    setchar(pointer,newchar)
721                    return true
722                else
723                    local fallback = fallbackstyleattr(a)
724                    if fallback then
725                        local newchar = remapalphabets(char,fallback,g)
726                        if newchar then
727                            if characters[newchar] then
728                                if trace_remapping then
729                                    report_remap("char",font,char,newchar," (fallback remapping used)")
730                                end
731                                if trace_analyzing then
732                                    setnodecolor(pointer,"font:isol")
733                                end
734                                setchar(pointer,newchar)
735                                return true
736                            elseif trace_remapping then
737                                report_remap("char",font,char,newchar," fails (no fallback character)")
738                            end
739                        elseif trace_remapping then
740                            report_remap("char",font,char,newchar," fails (no fallback remap character)")
741                        end
742                    elseif trace_remapping then
743                        report_remap("char",font,char,newchar," fails (no fallback style)")
744                    end
745                end
746            elseif trace_remapping then
747                local chardata = characters[char]
748                if chardata and chardata.commands then
749                    report_remap("char",font,char,char," (virtual)")
750                end
751            end
752        end
753        if not characters[char] then
754            local fnt = getfontoffamily(fam,1)
755            setchar(pointer,errorchar(font,char))
756            if font ~= fnt then
757                errorchar(fnt,char)
758                errorchar(getfontoffamily(fam,2),char)
759            end
760        end
761        if trace_analyzing then
762            setnodecolor(pointer,"font:medi")
763        end
764        if check_coverage then
765            return checked(pointer)
766        end
767    end
768
769    relocate[mathtextchar_code] = function(pointer)
770        if trace_analyzing then
771            setnodecolor(pointer,"font:init")
772        end
773    end
774
775    relocate[delimiter_code] = function(pointer)
776        if trace_analyzing then
777            setnodecolor(pointer,"font:fina")
778        end
779    end
780
781    function handlers.relocate(head,style,penalties)
782        processnoads(head,relocate,"relocate")
783    end
784
785end
786
787-- rendering (beware, not exported)
788
789do
790
791    local render     = { }
792
793    local rendersets = mathematics.renderings.numbers or { } -- store
794
795    render[mathchar_code] = function(pointer)
796        local attr = getattr(pointer,a_mathrendering)
797        if attr and attr > 0 then
798            local char, font = getcharspec(pointer)
799            local renderset = rendersets[attr]
800            if renderset then
801                local newchar = renderset[char]
802                if newchar then
803                    local characters = fontcharacters[font]
804                    if characters and characters[newchar] then
805                        setchar(pointer,newchar)
806                        setattr(pointer,a_exportstatus,char) -- yes or no
807                    end
808                end
809            end
810        end
811    end
812
813    function handlers.render(head,style,penalties)
814        processnoads(head,render,"render")
815    end
816
817end
818
819-- some resize options (this works ok because the content is
820-- empty and no larger next will be forced)
821--
822-- beware: we don't use \delcode but \Udelcode and as such have
823-- no largefamily; also, we need to check for subtype and/or
824-- smallfamily not being 0 because \. sits in 0,0 by default
825--
826-- todo: just replace the character by an ord noad
827-- and remove the right delimiter as well
828
829do
830
831    -- todo: use registerattribute
832
833    local a_mathsize  = privateattribute("mathsize") -- this might move into other fence code
834    local resize      = { }
835
836    local trace_fences  = false  registertracker("math.fences", function(v) trace_fences = v end)
837    local report_fences = logreporter("mathematics","fences")
838
839    resize[fence_code] = function(pointer)
840        local subtype = getsubtype(pointer)
841     -- if subtype == leftfence_code or subtype == rightfence_code then
842            local a = getattr(pointer,a_mathsize)
843            if a and a > 0 then
844                local method = div(a,100)
845                local size   = a % 100
846                setattr(pointer,a_mathsize,0)
847                if size ~= 0 then
848                    local delimiter = getdelimiter(pointer)
849                    if delimiter then
850                        local oldchar, font, fam = getcharspec(delimiter)
851                        if oldchar > 0 and font > 0 then
852                            local ht         = getheight(pointer)
853                            local dp         = getdepth(pointer)
854                            local data       = fontdata[font]
855                            local characters = data.characters
856                            local olddata    = characters[oldchar]
857                            if olddata then
858                                local template  = olddata.varianttemplate
859                                local newchar   = mathematics.big(data,template or oldchar,size,method)
860                                local newdata   = characters[newchar]
861                                local newheight = newdata.height or 0
862                                local newdepth  = newdata.depth or 0
863                                if template then
864                                    setheight(pointer,newheight)
865                                    setdepth(pointer,newdepth)
866                                    if not olddata.extensible then
867                                        -- check this on bonum and antykwa
868                                        setoptions(pointer,0)
869                                    end
870                                    setoptions(pointer,getoptions(pointer)| tex.noadoptioncodes.scale                        )
871                                    if trace_fences then
872                                        report_fences("replacing %C using method %a, size %a and template %C",newchar,method,size,template)
873                                    end
874                                else
875                                    -- 1 scaled point is a signal, for now
876                                    if ht == 1 then
877                                        setheight(pointer,newheight)
878                                    end
879                                    if dp == 1 then
880                                        setdepth(pointer,newdepth)
881                                    end
882                                    setchar(delimiter,newchar)
883                                    if trace_fences then
884                                        report_fences("replacing %C by %C using method %a and size %a",oldchar,char,method,size)
885                                    end
886                                end
887                            elseif trace_fences then
888                                report_fences("not replacing %C using method %a and size %a",oldchar,method,size)
889                            end
890                        end
891                    end
892                end
893            end
894     -- end
895    end
896
897    function handlers.resize(head,style,penalties)
898        processnoads(head,resize,"resize")
899    end
900
901end
902
903-- still not perfect:
904
905do
906
907    local a_autofence     = privateattribute("mathautofence")
908    local autofences      = { }
909    local dummyfencechar  = 0x2E
910
911    local function makefence(what,char,template)
912        local d = new_delimiter()
913        local f = new_fence()
914        if char then
915            local sym = getnucleus(char)
916            local chr, fnt, fam = getcharspec(sym)
917            if chr == dummyfencechar then
918                chr = 0
919            end
920            setchar(d,chr)
921            setfam(d,fam)
922            flushnode(sym)
923        end
924        setattrlist(d,template)
925        setattrlist(f,template)
926        setsubtype(f,what)
927        setdelimiter(f,d)
928        setclass(f,-1) -- tex itself does this, so not fenceclasses[what]
929        return f
930    end
931
932    local function show(where,pointer)
933        print("")
934        local i = 0
935        for n in nuts.traverse(pointer) do
936            i = i + 1
937            print(i,where,nuts.tonode(n))
938        end
939        print("")
940    end
941
942    local function makelist(middle,noad,f_o,o_next,c_prev,f_c)
943-- report_fences(
944--     "middle %s, noad %s, open %s, opennext %s, closeprev %s, close %s",
945--     middle or "?",
946--     noad   or "?",
947--     f_o    or "?",
948--     o_next or "?",
949--     c_prev or "?",
950--     f_c    or "?"
951-- )
952        local list = new_submlist()
953        setsubtype(noad,fenced_class)
954        setnucleus(noad,list)
955        setattrlist(list,noad)
956        setlist(list,f_o)
957        setlink(f_o,o_next) -- prev of list is nil
958        setlink(c_prev,f_c) -- next of list is nil
959-- show("list",f_o)
960        if middle and next(middle) then
961            local prev    = f_o
962            local current = o_next
963            while current ~= f_c do
964                local midl = middle[current]
965                local next = getnext(current)
966                if midl then
967                    local fence = makefence(middlefence_code,current,current)
968                    setnucleus(current)
969                    flushnode(current)
970                    middle[current] = nil
971                    -- replace_node
972                    setlink(prev,fence,next)
973                    prev = fence
974                else
975                    prev = current
976                end
977                current = next
978            end
979        end
980        return noad
981    end
982
983    -- relinking is now somewhat overdone
984
985    local function convert_both(open,close,middle)
986        local o_next = getnext(open)
987        if o_next == close then
988            return close
989        else
990            local c_prev, c_next = getboth(close)
991            local f_o = makefence(leftfence_code,open,open)
992            local f_c = makefence(rightfence_code,close,close)
993            makelist(middle,open,f_o,o_next,c_prev,f_c)
994            setnucleus(close)
995            flushnode(close)
996            -- open is now a list
997            setlink(open,c_next)
998            return open
999        end
1000    end
1001
1002    local function convert_open(open,last,middle) -- last is really last (final case)
1003        local f_o = makefence(leftfence_code,open,open)
1004        local f_c = makefence(rightfence_code,nil,open)
1005        local o_next = getnext(open)
1006        makelist(middle,open,f_o,o_next,last,nil)
1007        -- open is now a list
1008        setlink(open,l_next)
1009        return open
1010    end
1011
1012    local function convert_close(first,close,middle)
1013        local f_o = makefence(leftfence_code,nil,close)
1014        local f_c = makefence(rightfence_code,close)
1015        local c_prev = getprev(close)
1016        local f_next = getnext(first)
1017        makelist(middle,close,f_o,f_next,c_prev,f_c)
1018        -- close is now a list : inner but should be fenced
1019        if c_prev ~= first then
1020            setlink(first,close)
1021        end
1022        return close
1023    end
1024
1025    local stacks = setmetatableindex("table")
1026
1027    -- 1=open 2=close 3=middle 4=both : todo check both
1028
1029    local function processfences(pointer,n,parent)
1030        local current = pointer
1031        local last    = pointer
1032        local start   = pointer
1033        local done    = false
1034        local initial = pointer
1035        local stack   = nil
1036        local middle  = nil -- todo: use properties
1037        while current do
1038-- show("before",pointer)
1039            local id = getid(current)
1040            if id == noad_code then
1041                local a = getattr(current,a_autofence)
1042                if a and a > 0 then
1043                    local stack = stacks[n]
1044                    setattr(current,a_autofence,0) -- hm, better use a property
1045                    local level = #stack
1046                    if a == 1 then
1047                        if trace_fences then
1048                            report_fences("%2i: level %i, handling %s, action %s",n,level,"open","open")
1049                        end
1050                        insert(stack,current)
1051                    elseif a == 2 then
1052                        local open = remove(stack)
1053                        if open then
1054                            if trace_fences then
1055                                report_fences("%2i: level %i, handling %s, action %s",n,level,"close","both")
1056                            end
1057                            current = convert_both(open,current,middle)
1058                        elseif current == start then
1059                            if trace_fences then
1060                                report_fences("%2i: level %i, handling %s, action %s",n,level,"close","skip")
1061                            end
1062                        else
1063                            if trace_fences then
1064                                report_fences("%2i: level %i, handling %s, action %s",n,level,"close","close")
1065                            end
1066                            current = convert_close(initial,current,middle)
1067                            if not parent then
1068                                initial = current
1069                            end
1070                        end
1071                    elseif a == 3 then
1072                        if trace_fences then
1073                            report_fences("%2i: level %i, handling %s, action %s",n,level,"middle","middle")
1074                        end
1075                        if middle then
1076                            middle[current] = last
1077                        else
1078                            middle = { [current] = last }
1079                        end
1080                    elseif a == 4 then
1081                        if not stack or #stack == 0 then
1082                            if trace_fences then
1083                                report_fences("%2i: level %i, handling %s, action %s",n,level,"both","open")
1084                            end
1085                            insert(stack,current)
1086                        else
1087                            local open = remove(stack)
1088                            if open then
1089                                if trace_fences then
1090                                    report_fences("%2i: level %i, handling %s, action %s",n,level,"both","both")
1091                                end
1092                                current = convert_both(open,current,middle)
1093                            elseif current == start then
1094                                if trace_fences then
1095                                    report_fences("%2i: level %i, handling %s, action %s",n,level,"both","skip")
1096                                end
1097                            else
1098                                if trace_fences then
1099                                    report_fences("%2i: level %i, handling %s, action %s",n,level,"both","close")
1100                                end
1101                                current = convert_close(initial,current,middle)
1102                                if not parent then
1103                                    initial = current
1104                                end
1105                            end
1106                        end
1107                    end
1108                    done = true
1109                else
1110                    processstep(current,processfences,n+1,id)
1111                end
1112            else
1113                -- next at current level
1114                processstep(current,processfences,n,id)
1115            end
1116-- show("after",pointer)
1117            last    = current
1118            current = getnext(current)
1119        end
1120        if done then
1121            local stack = stacks[n]
1122            local s = #stack
1123            if s > 0 then
1124                for i=1,s do
1125                    local open = remove(stack)
1126                    if trace_fences then
1127                        report_fences("%2i: level %i, handling %s, action %s",n,#stack,"flush","open")
1128                    end
1129                    last = convert_open(open,last,middle)
1130                end
1131-- show("done",pointer)
1132            end
1133        end
1134    end
1135
1136    -- we can have a first changed node .. an option is to have a leading dummy node in math
1137    -- lists like the par node as it can save a  lot of mess
1138
1139    local enabled = false
1140
1141    implement {
1142        name     = "enableautofences",
1143        onlyonce = true,
1144        actions  = function()
1145            enableaction("math","noads.handlers.autofences")
1146            enabled = true
1147        end
1148    }
1149
1150    function handlers.autofences(head,style,penalties)
1151        if enabled then -- tex.modes.c_math_fences_auto
1152         -- inspect(nodes.totree(head))
1153            processfences(head,1)
1154         -- inspect(nodes.totree(head))
1155        end
1156    end
1157
1158end
1159
1160-- normalize scripts
1161
1162do
1163
1164    local unscript     = { }  noads.processors.unscript = unscript
1165    local superscripts = characters.superscripts
1166    local subscripts   = characters.subscripts
1167    local fractions    = characters.fractions
1168    local replaced     = { }
1169
1170    local function replace(pointer,what,n,parent)
1171        pointer = parent -- we're following the parent list (chars trigger this)
1172        local next = getnext(pointer)
1173        local start_super, stop_super, start_sub, stop_sub
1174        local mode = "unset"
1175        while next and getid(next) == noad_code do
1176            local nextnucleus = getnucleus(next)
1177            if nextnucleus and getid(nextnucleus) == mathchar_code and not getsub(next) and not getsup(next) then
1178                local char = getchar(nextnucleus)
1179                local s = superscripts[char]
1180                if s then
1181                    if not start_super then
1182                        start_super = next
1183                        mode = "super"
1184                    elseif mode == "sub" then
1185                        break
1186                    end
1187                    stop_super = next
1188                    next = getnext(next)
1189                    setchar(nextnucleus,s)
1190                    replaced[char] = (replaced[char] or 0) + 1
1191                    if trace_normalizing then
1192                        report_normalizing("superscript %C becomes %C",char,s)
1193                    end
1194                else
1195                    local s = subscripts[char]
1196                    if s then
1197                        if not start_sub then
1198                            start_sub = next
1199                            mode = "sub"
1200                        elseif mode == "super" then
1201                            break
1202                        end
1203                        stop_sub = next
1204                        next = getnext(next)
1205                        setchar(nextnucleus,s)
1206                        replaced[char] = (replaced[char] or 0) + 1
1207                        if trace_normalizing then
1208                            report_normalizing("subscript %C becomes %C",char,s)
1209                        end
1210                    else
1211                        break
1212                    end
1213                end
1214            else
1215                break
1216            end
1217        end
1218        if start_super then
1219            if start_super == stop_super then
1220                setsup(pointer,getnucleus(start_super))
1221            else
1222                local list = new_submlist()
1223                setattrlist(list,pointer)
1224                setlist(list,start_super)
1225                setsup(pointer,list)
1226            end
1227            if mode == "super" then
1228                setnext(pointer,getnext(stop_super))
1229            end
1230            setnext(stop_super)
1231        end
1232        if start_sub then
1233            if start_sub == stop_sub then
1234                setsub(pointer,getnucleus(start_sub))
1235            else
1236                local list = new_submlist()
1237                setattrlist(list,pointer)
1238                setlist(list,start_sub)
1239                setsub(pointer,list)
1240            end
1241            if mode == "sub" then
1242                setnext(pointer,getnext(stop_sub))
1243            end
1244            setnext(stop_sub)
1245        end
1246        -- we could return stop
1247    end
1248
1249    unscript[mathchar_code] = replace -- not noads as we need to recurse
1250
1251    function handlers.unscript(head,style,penalties)
1252        processnoads(head,unscript,"unscript")
1253    end
1254
1255end
1256
1257do
1258
1259    local unstack   = { }    noads.processors.unstack = unstack
1260    local enabled   = false
1261    local a_unstack = privateattribute("mathunstack")
1262
1263    local trace_unstacking  = false  registertracker("math.unstack", function(v) trace_unstacking  = v end)
1264    local report_unstacking = logreporter("mathematics","unstack")
1265
1266    unstack[noad_code] = function(pointer)
1267        local a = getattr(pointer,a_unstack)
1268        if a then
1269            local sup = getsup(pointer)
1270            local sub = getsub(pointer)
1271            if sup and sub then
1272             -- if trace_unstacking then
1273             --     report_unstacking() -- todo ... what to show ...
1274             -- end
1275                if a == 1 then
1276                    a = tex.noadoptioncodes.shiftedsubscript
1277                elseif a == 2 then
1278                    a = tex.noadoptioncodes.shiftedsuperscript
1279                else
1280                    a = 0
1281                end
1282                setoptions(pointer,getoptions(pointer) | a)
1283            end
1284        end
1285    end
1286
1287    function handlers.unstack(head,style,penalties)
1288        if enabled then
1289            processnoads(head,unstack,"unstack")
1290
1291        end
1292    end
1293
1294    implement {
1295        name     = "enablescriptunstacking",
1296        onlyonce = true,
1297        actions  = function()
1298            enableaction("math","noads.handlers.unstack")
1299            enabled = true
1300        end
1301    }
1302
1303end
1304
1305do
1306
1307    local function collected(list)
1308        if list and next(list) then
1309            local n, t = 0, { }
1310            for k, v in sortedhash(list) do
1311                n = n + 1
1312                t[n] = formatters["%C"](k)
1313            end
1314            return formatters["% t (n=%s)"](t,n)
1315        end
1316    end
1317
1318    statistics.register("math script replacements", function()
1319        return collected(replaced)
1320    end)
1321
1322    statistics.register("unknown math characters", function()
1323        return collected(unknowns)
1324    end)
1325
1326end
1327
1328-- math alternates: (in xits     lgf: $ABC$ $\cal ABC$ $\mathalternate{cal}\cal ABC$)
1329-- math alternates: (in lucidaot lgf: $ABC \mathalternate{italic} ABC$)
1330
1331-- todo: set alternate for specific symbols
1332-- todo: no need to do this when already loaded
1333-- todo: use a fonts.hashes.mathalternates
1334
1335do
1336
1337    local trace_alternates  = false  registertracker("math.alternates", function(v) trace_alternates = v end)
1338    local report_alternates = logreporter("mathematics","alternates")
1339
1340    local last = 0
1341
1342    local known = setmetatableindex(function(t,k)
1343        local v = 0 | 2^last
1344        t[k] = v
1345        last = last + 1
1346        return v
1347    end)
1348
1349    local defaults = {
1350        dotless = { feature = 'dtls', value = 1, comment = "Mathematical Dotless Forms" },
1351     -- zero    = { feature = 'zero', value = 1, comment = "Slashed or Dotted Zero" }, -- in no math font (yet)
1352    }
1353
1354    local function initializemathalternates(tfmdata)
1355        if use_math_goodies then
1356
1357            local goodies  = tfmdata.goodies
1358            local autolist = defaults -- table.copy(defaults)
1359
1360            local function setthem(newalternates)
1361                local resources      = tfmdata.resources -- was tfmdata.shared
1362                local mathalternates = resources.mathalternates
1363                local alternates, attributes, registered, presets
1364                if mathalternates then
1365                    alternates = mathalternates.alternates
1366                    attributes = mathalternates.attributes
1367                    registered = mathalternates.registered
1368                else
1369                    alternates, attributes, registered = { }, { }, { }
1370                    mathalternates = {
1371                        attributes = attributes,
1372                        alternates = alternates,
1373                        registered = registered,
1374                        presets    = { },
1375                        resets     = { },
1376                        hashes     = setmetatableindex("table")
1377                    }
1378                    resources.mathalternates = mathalternates
1379                end
1380                --
1381                for name, data in sortedhash(newalternates) do
1382                    if alternates[name] then
1383                        -- ignore
1384                    else
1385                        local attr = known[name]
1386                        attributes[attr] = data
1387                        alternates[name] = attr
1388                        registered[#registered+1] = attr
1389                    end
1390                end
1391            end
1392
1393            if goodies then
1394                local done = { }
1395                for i=1,#goodies do
1396                    -- first one counts
1397                    -- we can consider sharing the attributes ... todo (only once scan)
1398                    local mathgoodies = goodies[i].mathematics
1399                    local alternates  = mathgoodies and mathgoodies.alternates
1400                    if alternates then
1401                        if trace_goodies then
1402                            report_goodies("loading alternates for font %a",tfmdata.properties.name)
1403                        end
1404                        for k, v in next, autolist do
1405                            if not alternates[k] then
1406                                alternates[k] = v
1407                            end
1408                        end
1409                        setthem(alternates)
1410                        return
1411                    end
1412                end
1413            end
1414
1415            if trace_goodies then
1416                report_goodies("loading default alternates for font %a",tfmdata.properties.name)
1417            end
1418            setthem(autolist)
1419        end
1420
1421    end
1422
1423    local otf                = fonts.handlers.otf
1424    local registerotffeature = fonts.constructors.features.otf.register
1425    ----- getalternate       = otf.getalternate (runtime new method so ...)
1426
1427    registerotffeature {
1428        name        = "mathalternates",
1429        description = "additional math alternative shapes",
1430        initializers = {
1431            base = initializemathalternates,
1432            node = initializemathalternates,
1433        }
1434    }
1435
1436    -- todo: not shared but copies ... one never knows
1437
1438    local a_mathalternate = privateattribute("mathalternate")
1439    local alternate       = { } -- processors.alternate = alternate
1440    local fontdata        = fonts.hashes.identifiers
1441    local fontresources   = fonts.hashes.resources
1442
1443    local function getalternate(fam,tag,current)
1444        -- fam is always zero, so we assume a very consistent setup
1445        local resources = fontresources[getfontoffamily(fam)]
1446        local attribute = unsetvalue
1447        if resources then
1448            local mathalternates = resources.mathalternates
1449            if mathalternates then
1450                local presets = mathalternates.presets
1451                if presets then
1452                    local resets = mathalternates.resets
1453                    attribute = presets[tag]
1454                    if not attribute then
1455                        attribute = 0
1456                        local alternates = mathalternates.alternates
1457                        for s in gmatch(tag,"[^, ]+") do
1458                            local n, s = match(s,"^([+-]*)(.*)$")
1459                            if s == v_reset then
1460                                resets[tag] = true
1461                                current = unsetvalue
1462                            else
1463                                local a = alternates[s] -- or known[s]
1464                                if a then
1465                                    if n == "-" then
1466                                        attribute = attribute & ~a
1467                                    else
1468                                        attribute = attribute | a
1469                                    end
1470                                end
1471                            end
1472                        end
1473                        if attribute == 0 then
1474                            attribute = unsetvalue
1475                        end
1476                        presets[tag] = attribute
1477                    elseif resets[tag] then
1478                        current = unsetvalue
1479                    end
1480                end
1481            end
1482        end
1483        if attribute > 0 and current and current > 0 then
1484            return current | attribute
1485        else
1486            return attribute
1487        end
1488    end
1489
1490    implement {
1491        name      = "presetmathfontalternate",
1492        arguments = "argument",
1493        public    = true,
1494        protected = true,
1495        actions   = function(tag)
1496            if texgetmode() == mathmode_code then
1497                texsetattribute(a_mathalternate,getalternate(0,tag))
1498            end
1499        end,
1500    }
1501
1502    implement {
1503        name      = "setmathfontalternate",
1504        arguments = "argument",
1505        public    = true,
1506        protected = true,
1507        actions   = function(tag)
1508            if texgetmode() == mathmode_code then
1509                texsetattribute(a_mathalternate,getalternate(0,tag,texgetattribute(a_mathalternate)))
1510            end
1511        end,
1512    }
1513
1514    alternate[mathchar_code] = function(pointer) -- slow
1515        local a = getattr(pointer,a_mathalternate)
1516        if a and a > 0 then
1517            setattr(pointer,a_mathalternate,0)
1518            local fontid    = getfont(pointer)
1519            local resources = fontresources[fontid]
1520            if resources then
1521                local mathalternates = resources.mathalternates
1522                if mathalternates then
1523                    local attributes = mathalternates.attributes
1524                    local registered = mathalternates.registered
1525                    local hashes     = mathalternates.hashes
1526                    local newchar    = nil
1527                    for i=1,#registered do
1528                        local r = registered[i]
1529                        if (a & r) ~= 0 then
1530                            local char = newchar or getchar(pointer)
1531                            local alt  = hashes[i][char]
1532                            if alt == nil then
1533                                local what = attributes[r]
1534                                local list = what.list
1535                                if not list or list[char] then
1536                                alt = otf.getalternate(fontdata[fontid],char,what.feature,what.value) or false
1537                                    if alt == char then
1538                                        alt = false
1539                                    end
1540                                    hashes[i][char] = alt
1541                                end
1542                            end
1543                            if alt then
1544                                if trace_alternates then
1545                                    local what = attributes[r]
1546                                    report_alternates("alternate %a, value %a, replacing glyph %U by glyph %U",
1547                                        tostring(what.feature),tostring(what.value),char,alt)
1548                                end
1549                             -- setchar(pointer,alt)
1550                             -- break
1551                                newchar = alt
1552                            end
1553                        end
1554                    end
1555                    if newchar then
1556                        setchar(pointer,newchar)
1557                    end
1558                end
1559            end
1560        end
1561    end
1562
1563    alternate[delimiter_code] = alternate[mathchar_code]
1564
1565    function handlers.alternates(head,style,penalties)
1566        processnoads(head,alternate,"alternate")
1567    end
1568
1569end
1570
1571-- Italic correction control has been removed here. You can find it in the files
1572-- for \MKIV. The left-over code deals with text-math boundaries.
1573
1574do
1575
1576    local enable = function()
1577        if trace_italics then
1578            report_italics("enabling math italics")
1579        end
1580        -- we enable math (unless already enabled elsewhere)
1581        typesetters.italics.enablemath()
1582        enable = false
1583    end
1584
1585    implement {
1586        name      = "initializemathitalics",
1587        actions   = enable,
1588        onlyonce  = true,
1589    }
1590
1591end
1592
1593do
1594
1595    -- math kerns (experiment) in goodies:
1596    --
1597    -- mathematics = {
1598    --     kernpairs = {
1599    --         [0x1D44E] = {
1600    --             [0x1D44F] = 400, -- 𝑎𝑏
1601    --         }
1602    --     },
1603    -- }
1604
1605    local a_kernpairs = privateattribute("mathkernpairs")
1606    local kernpairs   = { }
1607
1608    local trace_kernpairs  = false  registertracker("math.kernpairs", function(v) trace_kernpairs = v end)
1609    local report_kernpairs = logreporter("mathematics","kernpairs")
1610
1611    local function enable()
1612        enableaction("math", "noads.handlers.kernpairs")
1613        if trace_kernpairs then
1614            report_kernpairs("enabling math kern pairs")
1615        end
1616        enable = false
1617    end
1618
1619    implement {
1620        name      = "initializemathkernpairs",
1621        actions   = enable,
1622        onlyonce  = true,
1623    }
1624
1625    local hash = setmetatableindex(function(t,font)
1626        local g = fontdata[font].goodies
1627        local m = g and g[1] and g[1].mathematics
1628        local k = m and m.kernpairs
1629        t[font] = k
1630        return k
1631    end)
1632
1633    -- no correction after prime because that moved to a superscript
1634
1635    kernpairs[mathchar_code] = function(pointer,what,n,parent)
1636        if getattr(pointer,a_kernpairs) == 1 then
1637            local first, firstfont = getcharspec(pointer)
1638            local list = hash[firstfont]
1639            if list then
1640                local found = list[first]
1641                if found then
1642                    local next = isnext(parent,noad_code)
1643                    if next then
1644--                         pointer = getnucleus(next)
1645--                         if pointer then
1646                            local second, secondfont = getcharspec(pointer)
1647                            if secondfont == firstfont then
1648                                local kern = found[second]
1649                                if kern then
1650                                    kern = kern * fontparameters[firstfont].hfactor
1651                                    if trace_kernpairs then
1652                                        report_kernpairs("adding %p kerning between %C and %C",kern,first,second)
1653                                    end
1654                                    kern = new_kern(kern)
1655                                    setlink(parent,kern,getnext(parent))
1656                                    setattrlist(kern,pointer)
1657                                end
1658                            end
1659--                         end
1660                    end
1661                end
1662            end
1663        end
1664    end
1665
1666    function handlers.kernpairs(head,style,penalties)
1667        if use_math_goodies then
1668            processnoads(head,kernpairs,"kernpairs")
1669        end
1670    end
1671
1672end
1673
1674do
1675
1676    local a_numbers = privateattribute("mathnumbers")
1677    local a_spacing = privateattribute("mathspacing")
1678    local a_fencing = privateattribute("mathfencing")
1679
1680    local numbers = { }
1681    local spacing = { }
1682    local fencing = { }
1683
1684    local separators = {
1685        [0x2E] = { 0x2E, 0x2C, 0x002E, 0x002C, 0x2008, 0x2008 }, -- . -- punctuationspace
1686        [0x2C] = { 0x2C, 0x2E, 0x2008, 0x2008, 0x002E, 0x002C }, -- ,
1687    }
1688
1689    local digits = {
1690        [0x30] = true, [0x31] = true,
1691        [0x32] = true, [0x33] = true,
1692        [0x34] = true, [0x35] = true,
1693        [0x36] = true, [0x37] = true,
1694        [0x38] = true, [0x39] = true,
1695    }
1696
1697    local snoloc = {
1698        [punctuation_class] = 0x003A,
1699        [relation_class]    = 0x2236,
1700    }
1701
1702    local colons = {
1703        [0x003A] = snoloc,
1704        [0x2236] = snoloc,
1705    }
1706
1707    local middles = {
1708        [0x007C] = true,
1709        [0x2016] = true,
1710        [0x2980] = true,
1711    }
1712
1713    local singles = {
1714        0x007C,
1715        0x2016,
1716        0x2980,
1717    }
1718
1719    local followedbyspace_code = tex.noadoptioncodes.followedbyspace
1720
1721    local function followedbyspace(n)
1722        return n and (getoptions(n) & followedbyspace_code == followedbyspace_code)
1723    end
1724
1725    local function followbyspace(n)
1726        setoptions(n,getoptions(n) | followedbyspace_code)
1727    end
1728
1729    numbers[mathchar_code] = function(pointer,what,n,parent)
1730        local alternative = getattr(pointer,a_numbers)
1731        if alternative then
1732            local oldchar = getcharspec(pointer)
1733            local found   = separators[oldchar]
1734            if found then
1735                local prev, next = isboth(parent,noad_code)
1736                if prev and next then
1737                 -- local lc = getcharspec(getnucleus(prev))
1738                    local lc = getcharspec(prev)
1739                    if digits[lc] then
1740                     -- local rc = getcharspec(getnucleus(next))
1741                        local rc = getcharspec(next)
1742                        if digits[rc] then
1743                            local newchar = found[alternative]
1744                            local class   = followedbyspace(parent) and punctuation_class or ordinary_class
1745                            setsubtype(parent,class)
1746                            if newchar ~= oldchar then
1747                                setchar(pointer,newchar)
1748                            end
1749                         -- if trace_numbers then
1750                         --     report_numbers("digit separator digit")
1751                         -- end
1752                        end
1753                    end
1754                end
1755                return
1756            end
1757            found = digits[oldchar]
1758            if found then
1759                if followedbyspace(parent) then
1760                    local next = isnext(parent,noad_code)
1761                    if next then
1762                 --     local rc = getcharspec(getnucleus(next))
1763                        local rc = getcharspec(next)
1764                        if rc and digits[rc] then
1765                            local n = new_noad(numbergroup_class)
1766                            local s = new_submlist()
1767                            setnucleus(n,s)
1768                            setattrlist(n,pointer)
1769                            setattrlist(s,pointer)
1770                            setlink(parent,n,next)
1771                         -- if trace_numbers then
1772                         --     report_numbers("digit spacer digit")
1773                         -- end
1774                        end
1775                    end
1776                end
1777                return
1778            end
1779        end
1780    end
1781
1782    spacing[mathchar_code] = function(pointer,what,n,parent)
1783        if getattr(pointer,a_spacing) then
1784            local oldchar = getcharspec(pointer)
1785            local found   = colons[oldchar]
1786            if found then
1787                local prev = isprev(parent,noad_code)
1788                if prev then
1789                    local class   = followedbyspace(prev) and relation_class or punctuation_class
1790                    local newchar = found[class]
1791                    setsubtype(parent,class)
1792                    if newchar ~= oldchar then
1793                        setchar(pointer,newchar)
1794                    end
1795                 -- if trace_spacing then
1796                 --     report_spacing("spacer colon")
1797                 -- end
1798                end
1799                return
1800            end
1801        end
1802    end
1803
1804    -- we can share code, see earlier
1805
1806    local function makefence(chr,fam,subtype,class,template)
1807        local f = new_fence()
1808        local d = new_delimiter()
1809        setchar(d,chr)
1810        setfam(d,fam)
1811        setattrlist(d,template)
1812        setattrlist(f,template)
1813        setsubtype(f,subtype)
1814        setdelimiter(f,d)
1815        setclass(f,class) -- tex itself does this, so not fenceclasses[what]
1816        return f
1817    end
1818
1819    -- we loose scripts so maybe also copy these
1820
1821    fencing[mathchar_code] = function(pointer,what,n,parent)
1822        if getattr(pointer,a_fencing) and pointer == getnucleus(parent) then
1823            local oldchar = getcharspec(pointer)
1824            local found   = middles[oldchar]
1825            if found then
1826                local prev, next = getboth(parent)
1827                if getcharspec(next) == oldchar and not followedbyspace(parent) then
1828                    local nextnext = getnext(next)
1829                    -- we need to preserve the followed property
1830                    if getcharspec(nextnext) == oldchar and not followedbyspace(next) then
1831                        oldchar = singles[3]
1832                        prev, parent = nuts.remove(prev,parent,true)
1833                        prev, parent = nuts.remove(prev,parent,true)
1834                    else
1835                        oldchar = singles[2]
1836                        prev, parent = nuts.remove(prev,parent,true)
1837                    end
1838                    next = getnext(parent)
1839                    pointer = getnucleus(parent)
1840                    setchar(pointer,oldchar)
1841                end
1842                if followedbyspace(prev) and followedbyspace(parent) then
1843                    local chr, fnt, fam = getcharspec(pointer)
1844                    local f1 = makefence(0,0,0,0,pointer)
1845                    local f2 = makefence(chr,fam,middlefence_code,middle_class,pointer)
1846                    setlink(prev,f1,f2,next)
1847                    flushnode(parent)
1848followbyspace(f1)
1849followbyspace(f2)
1850                    return true, f2
1851                else
1852                    return true, parent
1853                end
1854            end
1855        end
1856    end
1857
1858    -- numbers
1859
1860    function handlers.numbers(head,style,penalties)
1861        processnoads(head,numbers,"numbers")
1862    end
1863
1864    local enable = function()
1865        enableaction("math", "noads.handlers.numbers")
1866     -- if trace_numbers then
1867     --     report_numbers("enabling math numbers")
1868     -- end
1869        enable = false
1870    end
1871
1872    implement {
1873        name     = "initializemathnumbers",
1874        actions  = enable,
1875        onlyonce = true,
1876    }
1877
1878    -- spacing
1879
1880    function handlers.spacing(head,style,penalties)
1881        processnoads(head,spacing,"spacing")
1882    end
1883
1884    local enable = function()
1885        enableaction("math", "noads.handlers.spacing")
1886     -- if trace_spacing then
1887     --     report_spacing("enabling math spacing")
1888     -- end
1889        enable = false
1890    end
1891
1892    implement {
1893        name     = "initializemathspacing",
1894        actions  = enable,
1895        onlyonce = true,
1896    }
1897
1898    -- fences
1899
1900    function handlers.fencing(head,style,penalties)
1901        processnoads(head,fencing,"fencing")
1902    end
1903
1904    local enable = function()
1905        enableaction("math", "noads.handlers.fencing")
1906     -- if trace_fencing then
1907     --     report_fencing("enabling math fencing")
1908     -- end
1909        enable = false
1910    end
1911
1912    implement {
1913        name     = "initializemathfencing",
1914        actions  = enable,
1915        onlyonce = true,
1916    }
1917
1918end
1919
1920-- primes and such
1921
1922do
1923
1924    -- is validpair stil needed? why not always now?
1925
1926    local a_mathcollapsing = privateattribute("mathcollapsing")
1927    local collapse         = { }
1928    local mathlists        = characters.mathlists
1929    local validpair        = {
1930        [ordinary_class]    = true,
1931        [operator_class]    = true,
1932        [binary_class]      = true, -- new
1933        [relation_class]    = true,
1934        [open_class]        = true, -- new
1935        [middle_class]      = true, -- new
1936        [close_class]       = true, -- new
1937        [punctuation_class] = true, -- new
1938        [fraction_class]    = true,
1939        [accent_class]      = true,
1940    }
1941
1942    local reported = setmetatableindex("table")
1943
1944    mathlists[39] = { [39] = { [39] = { enforced = 0x2034, [39] = { enforced = 0x2057 } }, enforced = 0x2033 }, enforced = 0x2032 }
1945    mathlists[96] = { [96] = { [96] = { enforced = 0x2037                               }, enforced = 0x2036 }, enforced = 0x2035 }
1946
1947    collapse[mathchar_code] = function(pointer,what,n,parent)
1948        if parent and mathlists[getchar(pointer)] then
1949            local found, last, lucleus, lsup, lsub, category
1950            local tree    = mathlists
1951            local current = parent
1952            while current and validpair[getsubtype(current)] do
1953                local nucleus, prime, sup, sub = getnucleus(current,true)
1954                local char = getchar(nucleus)
1955                if char then
1956                    local match = tree[char]
1957                    if match then
1958                        local method = getattr(current,a_mathcollapsing)
1959                        if method and method > 0 and method <= 3 then
1960                            local enforced = match.enforced
1961                            local specials = match.specials
1962                            local mathlist = match.mathlist
1963                            local ligature
1964                            if method == 0 then
1965                                ligature = enforced
1966                            elseif method == 1 then
1967                                ligature = enforced or specials
1968                            elseif method == 2 then
1969                                ligature = enforced or specials or mathlist
1970                            else -- 3
1971                                ligature = enforced or mathlist or specials
1972                            end
1973                            if ligature then
1974                                category = mathlist and "mathlist" or "specials"
1975                                found    = ligature
1976                                last     = current
1977                                lucleus  = nucleus
1978                                lsup     = sup
1979                                lsub     = sub
1980                            end
1981                            tree = match
1982                            if sub or sup then
1983                                break
1984                            else
1985                                current = getnext(current)
1986                            end
1987                        else
1988                            break
1989                        end
1990                    else
1991                        break
1992                    end
1993                else
1994                    break
1995                end
1996            end
1997            if found and last and lucleus then
1998                local id         = getfont(lucleus)
1999                local characters = fontcharacters[id]
2000                local replace    = characters and characters[found]
2001                if not replace then
2002                    if not reported[id][found] then
2003                        reported[id][found] = true
2004                        report_collapsing("%s ligature %C from %s","ignoring",found,category)
2005                    end
2006                elseif trace_collapsing then
2007                    report_collapsing("%s ligature %C from %s","creating",found,category)
2008                end
2009                setchar(pointer,found)
2010                local l = getnext(last)
2011                local c = getnext(parent)
2012                if lsub then
2013                    setsub(parent,lsub)
2014                    setsub(last)
2015                end
2016                if lsup then
2017                    setsup(parent,lsup)
2018                    setsup(last)
2019                end
2020                while c ~= l do
2021                    local n = getnext(c)
2022                    flushnode(c)
2023                    c = n
2024                end
2025                setlink(parent,l)
2026            end
2027        end
2028    end
2029
2030    function noads.handlers.collapse(head,style,penalties)
2031        processnoads(head,collapse,"collapse")
2032    end
2033
2034    local enable = function()
2035        enableaction("math", "noads.handlers.collapse")
2036        if trace_collapsing then
2037            report_collapsing("enabling math collapsing")
2038        end
2039        enable = false
2040    end
2041
2042    implement {
2043        name     = "initializemathcollapsing",
2044        actions  = enable,
2045        onlyonce = true,
2046    }
2047
2048end
2049
2050do
2051
2052    local fixscripts = { }
2053
2054    local primes = {
2055        -- primes
2056        [0x2032] = true,
2057        [0x2033] = true,
2058        [0x2034] = true,
2059        [0x2057] = true,
2060        -- reverse primes
2061        [0x2035] = true,
2062        [0x2036] = true,
2063        [0x2037] = true,
2064    }
2065
2066    local fixable = {
2067        [noad_code]     = true,
2068        [accent_code]   = true,
2069        [radical_code]  = true,
2070        [fraction_code] = true,
2071    }
2072
2073    -- [prime|sub|sup]first
2074
2075    fixscripts[mathchar_code] = function(pointer,what,n,parent,nested) -- todo: switch to turn it on and off
2076        if parent then
2077            local char = getchar(pointer)
2078            if char and primes[char] then
2079                local nuc = getnucleus(parent)
2080                if pointer == nuc then
2081                    local prev = getprev(parent)
2082                    if prev and fixable[getid(prev)] then
2083                        local prevsup  = getsup(prev)
2084                        local prevsub  = getsub(prev)
2085                        local primesup = getsup(parent)
2086                        local primesub = getsub(parent)
2087                        setfield(prev,"scriptorder",prevsub and 2 or 1) -- sub first, then prime
2088                        if primesup and not prevsup then
2089                            setsup(prev,primesup)
2090                            primesup = nil
2091                        end
2092                        if primesub and not prevsub then
2093                            setsub(prev,primesub)
2094                            primesub = nil
2095                        end
2096                        setprime(prev,nuc)
2097                        setnucleus(parent)
2098                        if not primesup then
2099                            setsup(parent)
2100                        end
2101                        if not primesub then
2102                            setsub(parent)
2103                        end
2104                        if not (primesup or primesub) then
2105                            setlink(prev,getnext(parent))
2106                            flushnode(parent)
2107                            return true, prev, prev
2108                        end
2109                    end
2110                end
2111            end
2112        end
2113    end
2114
2115    function noads.handlers.fixscripts(head,style,penalties)
2116        processnoads(head,fixscripts,"fixscripts")
2117    end
2118
2119end
2120
2121-- variants (upgraded for script too)
2122
2123do
2124
2125    local variants  = { }
2126    local chardata  = characters.data
2127    local a_variant = privateattribute("mathvariant")
2128
2129    local trace_variants  = false  registertracker("math.variants", function(v) trace_variants = v end)
2130    local report_variants = logreporter("mathematics","variants")
2131
2132    local function setvariant(pointer,selector,char)
2133        local tfmdata      = fontdata[getfont(pointer)]
2134        local mathvariants = tfmdata.resources.variants -- and variantdata / can be a hash
2135        if mathvariants then
2136            mathvariants = mathvariants[selector]
2137            if mathvariants then
2138                local variant = mathvariants[char]
2139                if variant then
2140                    setchar(pointer,variant)
2141                    setattr(pointer,a_exportstatus,char) -- we don't export the variant as it's visual markup
2142                    if trace_variants then
2143                        report_variants("variant (%U,%U) replaced by %U",char,selector,variant)
2144                    end
2145                else
2146                    if trace_variants then
2147                        report_variants("no variant (%U,%U)",char,selector)
2148                    end
2149                end
2150            end
2151        end
2152    end
2153
2154    variants[mathchar_code] = function(pointer,what,n,parent) -- also set export value
2155        local char = getchar(pointer)
2156        local data = chardata[char]
2157        if data then
2158            local variants = data.variants
2159            if variants then
2160                local next = isnext(parent,noad_code)
2161                if next then
2162                    local nucleus = getnucleus(next)
2163                    if nucleus and getid(nucleus) == mathchar_code then
2164                        local selector = getchar(nucleus)
2165                        if variants[selector] then
2166                            setvariant(pointer,selector,char)
2167                            setprev(next,pointer)
2168                            setnext(parent,getnext(next))
2169                            flushnode(next)
2170                        end
2171                    end
2172                end
2173                local selector = getattr(pointer,a_variant)
2174                if selector and variants[selector] then
2175                    setvariant(pointer,selector,char)
2176                end
2177            end
2178        end
2179    end
2180
2181    function mathematics.addvariant(tfmdata,char,variant,selector)
2182        if char and variant and selector then
2183            local data = chardata[char]
2184            if data then
2185                local variants = data.variants
2186                if variants and variants[selector] then
2187                    local resources = tfmdata.resources
2188                    local variants  = resources.variants -- and variantdata
2189                    if not variants then
2190                        variants           = { }
2191                        resources.variants = variants
2192                    end
2193                    local selectors = variants[selector]
2194                    if not selectors then
2195                        selectors          = { }
2196                        variants[selector] = selectors
2197                    end
2198                    selectors[char] = variant
2199                    return true
2200                end
2201            end
2202        end
2203        return false
2204    end
2205
2206    function handlers.variants(head,style,penalties)
2207        processnoads(head,variants,"unicode variant")
2208    end
2209
2210    local valid = {
2211        calligraphic = 0xFE00,
2212        calligraphy  = 0xFE00,
2213        script       = 0xFE01,
2214        handwriting  = 0xFE01,
2215    }
2216
2217    function mathematics.setvariant(s)
2218        texsetattribute(a_variant,valid[s] or unsetvalue)
2219    end
2220
2221    implement {
2222        name      = "setmathvariant",
2223        public    = true,
2224        protected = true,
2225        arguments = "argument",
2226        actions   = mathematics.setvariant,
2227    }
2228
2229end
2230
2231-- for manuals
2232
2233do
2234
2235    -- Given the amount of classes this no longer makes much sense or we need to
2236    -- extend it.
2237
2238    local classes = { }
2239    local colors  = {
2240        [relation_class]    = "trace:dr",
2241        [ordinary_class]    = "trace:db",
2242        [binary_class]      = "trace:dg",
2243        [open_class]        = "trace:dm",
2244        [middle_class]      = "trace:dm",
2245        [close_class]       = "trace:dm",
2246        [punctuation_class] = "trace:dc",
2247    }
2248
2249    local setcolor   = colortracers.set
2250    local resetcolor = colortracers.reset
2251
2252    classes[mathchar_code] = function(pointer,what,n,parent)
2253        local color = colors[getsubtype(parent)]
2254        if color then
2255            setcolor(pointer,color)
2256        else
2257            resetcolor(pointer)
2258        end
2259    end
2260
2261    function handlers.classes(head,style,penalties)
2262        processnoads(head,classes,"classes")
2263    end
2264
2265    registertracker("math.classes",function(v)
2266        setaction("math","noads.handlers.classes",v)
2267    end)
2268
2269end
2270
2271
2272do
2273
2274    local traversehlist = nuts.traversers.hlist
2275
2276    local getshift      = nuts.getshift
2277    local setwhd        = nuts.setwhd
2278    local setshift      = nuts.setshift
2279
2280    -- normalizer: can become engine feature (native tex loves shifts)
2281
2282    local function normalize(h)
2283        for n, s in traversehlist, h do
2284            if s > 0 then
2285                local sh = getshift(n)
2286                local ox, oy = getoffsets(n)
2287                if sh ~= 0 then
2288                    local w, h, d = getwhd(n)
2289                    h = h - sh
2290                    d = d + sh
2291                    setshift(n)
2292                    setwhd(n,w,h > 0 and h or 0,d > 0 and d or 0)
2293                    setoffsets(n,ox,oy - sh)
2294                end
2295            end
2296            local l = getlist(l)
2297            if l then
2298                normalize(l)
2299            end
2300        end
2301    end
2302
2303    function handlers.normalize(h)
2304        return normalize(h)
2305    end
2306
2307end
2308
2309do
2310
2311    local traversehlist = nuts.traversers.hlist
2312
2313    local texgetdimen   = tex.getdimen
2314    local texgetcount   = tex.getcount
2315
2316    local newrule       = nuts.pool.outlinerule
2317    local newkern       = nuts.pool.kern
2318    local setcolor      = nodes.tracers.colors.set
2319
2320    local a_mathsnap    = attributes.private("mathsnap")
2321
2322    local d_mathstrutht = tex.isdimen("mathstrutht")
2323    local d_mathstrutdp = tex.isdimen("mathstrutdp")
2324    local c_mathnesting = tex.iscount("mathnestinglevel")
2325
2326    local trace_snapping  = false  registertracker("math.snapping", function(v) trace_snapping = v end)
2327    local report_snapping = logreporter("mathematics","snapping")
2328
2329    function handlers.snap(h,_,_,_,_,level)
2330     -- if not level or level == 0 then
2331        if texgetcount(c_mathnesting) == 1 then
2332            local trace_color
2333            if trace_snapping == "frame" then
2334                trace_color = "darkgray"
2335            elseif type(trace_snapping) == "string" then
2336                trace_color = trace_snapping
2337            else
2338                trace_color = false
2339            end
2340            local ht, dp, dd, hs, ds, hd
2341            for n, s in traversehlist, h do
2342                local step = getattr(n,a_mathsnap)
2343                if step then
2344                    local done = false
2345                    if not dd then
2346                        ht = texgetdimen(d_mathstrutht)
2347                        dp = texgetdimen(d_mathstrutdp)
2348                        hd = ht + dp
2349                        -- lineskip can be large in alignments
2350                     -- dd = hd / 12
2351                        dd = hd / 6
2352                        if step == 0xFFFF then
2353                            hs = dd
2354                            ds = dd
2355                        else
2356                            hs = ht/step
2357                            ds = dp/step
2358                        end
2359                    end
2360                    local w, h, d = getwhd(n)
2361                    -- snap to line
2362                  ::height::
2363                    if h-dd < ht then
2364                        if trace_snapping == true then
2365                            report_snapping("adapting ht: old %p, new %p, lineskip %p",h,ht,dd)
2366                        end
2367                        done = true
2368                        setheight(n,ht)
2369                        goto depth
2370                    end
2371                    if h > ht then
2372--                         while ht < (h-dd) do
2373                        while ht < h do
2374                            ht = round(ht + hs)
2375                        end
2376                        if h ~= ht then
2377                            setheight(n,ht)
2378                            if trace_snapping == true then
2379                                report_snapping("enlarging ht: old %p, new %p, step %p",h,ht,hs)
2380                            end
2381                            done = true
2382                        end
2383                    end
2384                  ::depth::
2385                    if d-dd < dp then
2386                        if trace_snapping == true then
2387                            report_snapping("adapting dp: old %p, new %p, lineskip %p",d,dp,dd)
2388                        end
2389                        setdepth(n,dp)
2390                        done = true
2391                        goto done
2392                    end
2393                    if d > dp then
2394--                         while dp < (d-dd) do
2395                        while dp < d do
2396                            dp = round(dp + ds)
2397                        end
2398                        if d ~= dp then
2399                            setdepth(n,dp)
2400                            if trace_snapping == true then
2401                                report_snapping("enlarging dp: old %p, new %p, step %p",d,dp,ds)
2402                            end
2403                            done = true
2404                        end
2405                    end
2406                 ::done::
2407                    if done and trace_color then
2408                     -- w, h, d = getwhd(n)
2409                     -- local r = newrule(w,h,d,65536)
2410                     -- setcolor(r,trace_color)
2411                     -- setlink(r,newkern(-w),getlist(n))
2412                     -- setlist(n,r)
2413
2414                        local old = newrule(w,h,d,65536)
2415                        setcolor(old,"middlegray")
2416                        w, h, d = getwhd(n)
2417                        local new = newrule(w,h,d,65536/4)
2418                        setcolor(new,trace_color)
2419                        setlink(old,newkern(-w),new,newkern(-w),getlist(n))
2420                        local ox, oy = getoffsets(n)
2421                        setoffsets(old,-ox,-oy)
2422                        setoffsets(new,-ox,-oy)
2423                        setlist(n,old)
2424                    end
2425                end
2426            end
2427        end
2428    end
2429
2430    local valid = {
2431        [v_reset]  = unsetvalue,
2432        [v_line]   = 0xFFFF,
2433        [v_small]  = 8,
2434        [v_medium] = 4,
2435        [v_big]    = 2,
2436    }
2437
2438    function mathematics.setsnapping(s)
2439        if not enabled then
2440            enableaction("math", "noads.handlers.snap")
2441            enabled = true
2442        end
2443        texsetattribute(a_mathsnap,valid[s] or unsetvalue)
2444    end
2445
2446    implement {
2447        name      = "setmathsnapping",
2448        public    = true,
2449        protected = true,
2450        arguments = "argument",
2451        actions   = mathematics.setsnapping,
2452    }
2453
2454end
2455
2456-- experimental : replaced by dictionaries but for now we keep the code
2457--
2458-- do
2459--
2460--  -- mathematics.registerdomain {
2461--  --     name       = "foo",
2462--  --     parents    = { "bar" },
2463--  --     characters = {
2464--  --         [0x123] = { char = 0x234, class = binary },
2465--  --     },
2466--  -- }
2467--
2468--     local trace_domains  = false  registertracker("math.domains", function(v) trace_domains = v end)
2469--     local report_domains = logreporter("mathematics","domains")
2470--
2471--     local domains       = { }
2472--     local categories    = { }
2473--     local numbers       = { }
2474--     local a_mathdomain  = privateattribute("mathdomain")
2475--     mathematics.domains = categories
2476--     local permitted     = {
2477--         ordinary    = ordinary_class,
2478--         binary      = binary_class,
2479--         relation    = relation_class,
2480--         punctuation = punctuation_class,
2481--         inner       = innernoad_code,
2482--         fenced      = fenced_class,
2483--      -- fraction    = fraction_class,
2484--      -- radical     = radical_class,
2485--     }
2486--
2487--     function mathematics.registerdomain(data)
2488--         local name = data.name
2489--         if not name then
2490--             return
2491--         end
2492--         local attr       = #numbers + 1
2493--         categories[name] = data
2494--         numbers[attr]    = data
2495--         data.attribute   = attr
2496--         -- we delay hashing
2497--         return attr
2498--     end
2499--
2500--     local enable
2501--
2502--     enable = function()
2503--         enableaction("math", "noads.handlers.domains")
2504--         if trace_domains then
2505--             report_domains("enabling math domains")
2506--         end
2507--         enable = false
2508--     end
2509--
2510--     function mathematics.setdomain(name)
2511--         if enable then
2512--             enable()
2513--         end
2514--         local data = name and name ~= v_reset and categories[name]
2515--         texsetattribute(a_mathdomain,data and data.attribute or unsetvalue)
2516--     end
2517--
2518--     function mathematics.getdomain(name)
2519--         if enable then
2520--             enable()
2521--         end
2522--         local data = name and name ~= v_reset and categories[name]
2523--         context(data and data.attribute or unsetvalue)
2524--     end
2525--
2526--     implement {
2527--         name      = "initializemathdomain",
2528--         actions   = enable,
2529--         onlyonce  = true,
2530--     }
2531--
2532--     implement {
2533--         name      = "setmathdomain",
2534--         arguments = "string",
2535--         actions   = mathematics.setdomain,
2536--     }
2537--
2538--     implement {
2539--         name      = "getmathdomain",
2540--         arguments = "string",
2541--         actions   = mathematics.getdomain,
2542--     }
2543--
2544--     local function makehash(data)
2545--         local hash    = { }
2546--         local parents = data.parents
2547--         if parents then
2548--             local function merge(name)
2549--                 if name then
2550--                     local c = categories[name]
2551--                     if c then
2552--                         local hash = c.hash
2553--                         if not hash then
2554--                             hash = makehash(c)
2555--                         end
2556--                         for k, v in next, hash do
2557--                             hash[k] = v
2558--                         end
2559--                     end
2560--                 end
2561--             end
2562--             if type(parents) == "string" then
2563--                 merge(parents)
2564--             elseif type(parents) == "table" then
2565--                 for i=1,#parents do
2566--                     merge(parents[i])
2567--                 end
2568--             end
2569--         end
2570--         local characters = data.characters
2571--         if characters then
2572--             for k, v in next, characters do
2573--              -- local chr = n.char
2574--                 local cls = v.class
2575--                 if cls then
2576--                     v.code = permitted[cls]
2577--                 else
2578--                     -- invalid class
2579--                 end
2580--                 hash[k] = v
2581--             end
2582--         end
2583--         data.hash = hash
2584--         return hash
2585--     end
2586--
2587--     domains[mathchar_code] = function(pointer,what,n,parent)
2588--         local attr = getattr(pointer,a_mathdomain)
2589--         if attr then
2590--             local domain = numbers[attr]
2591--             if domain then
2592--                 local hash = domain.hash
2593--                 if not hash then
2594--                     hash = makehash(domain)
2595--                 end
2596--                 local char = getchar(pointer)
2597--                 local okay = hash[char]
2598--                 if okay then
2599--                     local chr = okay.char
2600--                     local cls = okay.code
2601--                     if chr and chr ~= char then
2602--                         setchar(pointer,chr)
2603--                     end
2604--                     if cls and cls ~= getsubtype(parent) then
2605--                         setsubtype(parent,cls)
2606--                     end
2607--                 end
2608--             end
2609--         end
2610--     end
2611--
2612--     function handlers.domains(head,style,penalties)
2613--         processnoads(head,domains,"domains")
2614--     end
2615--
2616-- end
2617
2618-- just for me
2619
2620function handlers.showtree(head,style,penalties)
2621    inspect(nodes.totree(tonut(head)))
2622end
2623
2624registertracker("math.showtree",function(v)
2625    setaction("math","noads.handlers.showtree",v)
2626end)
2627
2628-- also for me
2629
2630do
2631
2632    local applyvisuals = nuts.applyvisuals
2633    local visual       = false
2634
2635    function handlers.makeup(head)
2636        applyvisuals(head,visual)
2637    end
2638
2639    registertracker("math.makeup",function(v)
2640        visual = v
2641        setaction("math","noads.handlers.makeup",v)
2642    end)
2643
2644end
2645
2646-- Musical timestamp: August 2022 with "Meditation by Cory Wong (Live @ Brooklyn
2647-- Steel FEB 2022). Seen live earlier that year and its gets better and better!
2648--
2649-- As we also try to do here:
2650
2651do
2652
2653    local trace_dictionaries  = false  registertracker("math.dictionaries",         function(v) trace_dictionaries = v end)
2654    local trace_details       = false  registertracker("math.dictionaries.details", function(v) trace_details      = v end)
2655
2656    local report_dictionaries = logreporter("mathematics","dictionaries")
2657
2658    local setnodecolor = colortracers.set
2659    local getchardict  = nuts.getchardict
2660    local setchardict  = nuts.setchardict
2661
2662    local dictionaries = { }  noads.processors.dictionaries = dictionaries
2663    local groups       = mathematics.dictionaries.groups
2664    local sets         = mathematics.dictionaries.sets
2665    local variants     = mathematics.dictionaries.variants
2666    local defaults     = mathematics.dictionaries.defaults
2667
2668    local function check(pointer,group,index)
2669        local v = variants[index]
2670        if v then
2671            local c = v[group]
2672            if c then
2673                return group, c
2674            end
2675        end
2676        return 1
2677    end
2678
2679    dictionaries[mathchar_code] = function(pointer,what,n,parent)
2680        local properties, oldgroup, index, font, char = getchardict(pointer)
2681        local newgroup = 1
2682        local newclass = false
2683--         local oldclass = getsubtype(pointer)
2684        local oldclass = getsubtype(parent)
2685        if (properties & 0x1) == 0x1 then
2686            newclass = oldclass
2687            newgroup = oldgroup
2688        else
2689            local set = sets[oldgroup]
2690            if set then
2691                local groups    = set.groups
2692                local nofgroups = groups and #groups
2693                if nofgroups > 0 then
2694                    for i=1,nofgroups do
2695                        local group = groups[i]
2696                        local real, class = check(pointer,group,index)
2697                        if real ~= 1 then
2698                            newclass = class
2699                            newgroup = group
2700                            goto done
2701                        end
2702                    end
2703                end
2704            else
2705                newgroup, newclass = check(pointer,oldgroup,index)
2706            end
2707          ::done::
2708            if newgroup == 1 then
2709                newgroup = defaults[index] or 1
2710            end
2711            setchardict(pointer,properties,newgroup,index)
2712            if type(newclass) == "number" then
2713-- print(newgroup,newclass,oldclass)
2714                setsubtype(parent,newclass)
2715--                 setsubtype(pointer,newclass)
2716            else
2717                newclass = oldclass
2718            end
2719        end
2720        if trace_dictionaries or trace_details then
2721            if newgroup > 1 then
2722                local groupname = groups[newgroup]
2723                if groupname then
2724                    setnodecolor(pointer,"dictionary:"..groupname)
2725                end
2726            end
2727            if trace_details then
2728                report_dictionaries("properties 0x%02X, group 0x%02X -> 0x%02X, class 0x%02X -> 0x%02X, index %05X, %U %c",properties,oldgroup,newgroup,oldclass,newclass,index,char,char)
2729            end
2730        end
2731    end
2732
2733    function handlers.dictionaries(head,style,penalties)
2734        processnoads(head,dictionaries,"dictionaries")
2735    end
2736
2737end
2738
2739do
2740
2741    local trace_suspicious  = false  registertracker("math.suspicious", function(v) trace_suspicious = v end)
2742    local report_suspicious = logreporter("mathematics","suspicious")
2743
2744    local suspicious = { }  noads.processors.suspicious = suspicious
2745
2746    local candidates = {
2747        [classes.maybeordinary] = "maybeordinary",
2748        [classes.mayberelation] = "mayberelation",
2749        [classes.maybebinary  ] = "maybebinary",
2750    }
2751    local registered = setmetatableindex("table")
2752
2753    suspicious[mathchar_code] = function(pointer,what,n,parent)
2754        local class = getsubtype(pointer)
2755        local found = candidates[class]
2756        if found then
2757            local char = getchar(pointer)
2758            if not registered[class][char] then
2759                report_suspicious("class %a, %U %c",found,char,char)
2760                registered[class][char] = true
2761            end
2762        end
2763    end
2764
2765    function handlers.suspicious(head,style,penalties)
2766        if trace_suspicious then
2767            processnoads(head,suspicious,"suspicious")
2768        end
2769    end
2770
2771end
2772
2773-- the normal builder
2774
2775-- do
2776--
2777--     local force_penalties = false
2778--
2779--     function builders.kernel.mlisttohlist(head,style,penalties)
2780--         return mlisttohlist(head,style,force_penalties or penalties)
2781--     end
2782--
2783--     implement {
2784--         name      = "setmathpenalties",
2785--         arguments = "integer",
2786--         actions   = function(p)
2787--             force_penalties = p > 0
2788--         end,
2789--     }
2790--
2791-- end
2792
2793builders.kernel.mlisttohlist = mlisttohlist
2794
2795local actions = tasks.actions("math") -- head, style, penalties
2796
2797local starttiming, stoptiming = statistics.starttiming, statistics.stoptiming
2798
2799function processors.mlisttohlist(head,style,penalties,beginclass,endclass,level)
2800    starttiming(noads)
2801    head = actions(head,style,penalties,beginclass,endclass,level)
2802    stoptiming(noads)
2803    return head
2804end
2805
2806callbacks.register("mlist_to_hlist",processors.mlisttohlist,"convert a noad list into a node list")
2807
2808-- tracing
2809
2810statistics.register("math processing time", function()
2811    return statistics.elapsedseconds(noads)
2812end)
2813