font-oup.lua /size: 120 Kb    last modification: 2021-10-28 13:50
1if not modules then modules = { } end modules ['font-oup'] = {
2    version   = 1.001,
3    comment   = "companion to font-ini.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9local next, type = next, type
10local P, R, S = lpeg.P, lpeg.R, lpeg.S
11local lpegmatch = lpeg.match
12local insert, remove, copy, unpack = table.insert, table.remove, table.copy, table.unpack
13
14local formatters           = string.formatters
15local sortedkeys           = table.sortedkeys
16local sortedhash           = table.sortedhash
17local tohash               = table.tohash
18local setmetatableindex    = table.setmetatableindex
19
20local report_error         = logs.reporter("otf reader","error")
21local report_markwidth     = logs.reporter("otf reader","markwidth")
22local report_cleanup       = logs.reporter("otf reader","cleanup")
23local report_optimizations = logs.reporter("otf reader","merges")
24local report_unicodes      = logs.reporter("otf reader","unicodes")
25
26local trace_markwidth      = false  trackers.register("otf.markwidth",     function(v) trace_markwidth     = v end)
27local trace_cleanup        = false  trackers.register("otf.cleanups",      function(v) trace_cleanups      = v end)
28local trace_optimizations  = false  trackers.register("otf.optimizations", function(v) trace_optimizations = v end)
29local trace_unicodes       = false  trackers.register("otf.unicodes",      function(v) trace_unicodes      = v end)
30
31local readers              = fonts.handlers.otf.readers
32local privateoffset        = fonts.constructors and fonts.constructors.privateoffset or 0xF0000 -- 0x10FFFF
33
34local f_private            = formatters["P%05X"]
35local f_unicode            = formatters["U%05X"]
36local f_index              = formatters["I%05X"]
37local f_character_y        = formatters["%C"]
38local f_character_n        = formatters["[ %C ]"]
39
40local check_duplicates     = true    -- can become an option (pseudo feature) / always needed anyway
41local check_soft_hyphen    = context -- only in context
42
43directives.register("otf.checksofthyphen",function(v)
44    check_soft_hyphen = v -- only for testing
45end)
46
47-- After (!) the unicodes have been resolved we compact ligature tables so before that happens
48-- we don't need to check for numbers.
49
50local function replaced(list,index,replacement)
51    if type(list) == "number" then
52        return replacement
53    elseif type(replacement) == "table" then
54        local t = { }
55        local n = index-1
56        for i=1,n do
57            t[i] = list[i]
58        end
59        for i=1,#replacement do
60            n = n + 1
61            t[n] = replacement[i]
62        end
63        for i=index+1,#list do
64            n = n + 1
65            t[n] = list[i]
66        end
67    else
68        list[index] = replacement
69        return list
70    end
71end
72
73local function unifyresources(fontdata,indices)
74    local descriptions = fontdata.descriptions
75    local resources    = fontdata.resources
76    if not descriptions or not resources then
77        return
78    end
79    --
80    local nofindices = #indices
81    --
82    local variants = fontdata.resources.variants
83    if variants then
84        for selector, unicodes in next, variants do
85            for unicode, index in next, unicodes do
86                unicodes[unicode] = indices[index]
87            end
88        end
89    end
90    --
91    local function remark(marks)
92        if marks then
93            local newmarks = { }
94            for k, v in next, marks do
95                local u = indices[k]
96                if u then
97                    newmarks[u] = v
98                elseif trace_optimizations then
99                    report_optimizations("discarding mark %i",k)
100                end
101            end
102            return newmarks
103        end
104    end
105    --
106    local marks = resources.marks
107    if marks then
108        resources.marks  = remark(marks)
109    end
110    --
111    local markclasses = resources.markclasses
112    if markclasses then
113        for class, marks in next, markclasses do
114            markclasses[class] = remark(marks)
115        end
116    end
117    --
118    local marksets = resources.marksets
119    if marksets then
120        for class, marks in next, marksets do
121            marksets[class] = remark(marks)
122        end
123    end
124    --
125    local done = { } -- we need to deal with shared !
126    --
127    local duplicates = check_duplicates and resources.duplicates
128    if duplicates and not next(duplicates) then
129        duplicates = false
130    end
131    --
132    local function recover(cover) -- can be packed
133        for i=1,#cover do
134            local c = cover[i]
135            if not done[c] then
136                local t = { }
137                for k, v in next, c do
138                    local ug = indices[k]
139                    if ug then
140                        t[ug] = v
141                    else
142                        report_error("case %i, bad index in unifying %s: %s of %s",1,"coverage",k,nofindices)
143                    end
144                end
145                cover[i] = t
146                done[c]  = d
147            end
148        end
149    end
150    --
151    local function recursed(c,kind) -- ligs are not packed
152        local t = { }
153        for g, d in next, c do
154            if type(d) == "table" then
155                local ug = indices[g]
156                if ug then
157                    t[ug] = recursed(d,kind)
158                else
159                    report_error("case %i, bad index in unifying %s: %s of %s",1,kind,g,nofindices)
160                end
161            else
162                t[g] = indices[d] -- ligature
163            end
164        end
165        return t
166    end
167    --
168    -- the duplicates need checking (probably only in cjk fonts): currently we only check
169    -- gsub_single, gsub_alternate, gsub_multiple, gpos_single and gpos_cursive
170    --
171    local function unifythem(sequences)
172        if not sequences then
173            return
174        end
175        for i=1,#sequences do
176            local sequence  = sequences[i]
177            local kind      = sequence.type
178            local steps     = sequence.steps
179            local features  = sequence.features
180            if steps then
181                for i=1,#steps do
182                    local step = steps[i]
183                    if kind == "gsub_single" then
184                        local c = step.coverage
185                        if c then
186                            local t1 = done[c]
187                            if not t1 then
188                                t1 = { }
189                                if duplicates then
190                                    for g1, d1 in next, c do
191                                        local ug1 = indices[g1]
192                                        if ug1 then
193                                            local ud1 = indices[d1]
194                                            if ud1 then
195                                                t1[ug1] = ud1
196                                                local dg1 = duplicates[ug1]
197                                                if dg1 then
198                                                    for u in next, dg1 do
199                                                        t1[u] = ud1
200                                                    end
201                                                end
202                                            else
203                                                report_error("case %i, bad index in unifying %s: %s of %s",3,kind,d1,nofindices)
204                                            end
205                                        else
206                                            report_error("case %i, bad index in unifying %s: %s of %s",1,kind,g1,nofindices)
207                                        end
208                                    end
209                                else
210                                    for g1, d1 in next, c do
211                                        local ug1 = indices[g1]
212                                        if ug1 then
213                                            t1[ug1] = indices[d1]
214                                        else
215                                            report_error("fuzzy case %i in unifying %s: %i",2,kind,g1)
216                                        end
217                                    end
218                                end
219                                done[c] = t1
220                            end
221                            step.coverage = t1
222                        end
223                    elseif kind == "gpos_pair" then
224                        local c = step.coverage
225                        if c then
226                            local t1 = done[c]
227                            if not t1 then
228                                t1 = { }
229                                for g1, d1 in next, c do
230                                    local ug1 = indices[g1]
231                                    if ug1 then
232                                        local t2 = done[d1]
233                                        if not t2 then
234                                            t2 = { }
235                                            for g2, d2 in next, d1 do
236                                                local ug2 = indices[g2]
237                                                if ug2 then
238                                                    t2[ug2] = d2
239                                                else
240                                                    report_error("case %i, bad index in unifying %s: %s of %s",1,kind,g2,nofindices,nofindices)
241                                                end
242                                            end
243                                            done[d1] = t2
244                                        end
245                                        t1[ug1] = t2
246                                    else
247                                        report_error("case %i, bad index in unifying %s: %s of %s",2,kind,g1,nofindices)
248                                    end
249                                end
250                                done[c] = t1
251                            end
252                            step.coverage = t1
253                        end
254                    elseif kind == "gsub_ligature" then
255                        local c = step.coverage
256                        if c then
257                            step.coverage = recursed(c,kind)
258                        end
259                    elseif kind == "gsub_alternate" or kind == "gsub_multiple" then
260                        local c = step.coverage
261                        if c then
262                            local t1 = done[c]
263                            if not t1 then
264                                t1 = { }
265                                if duplicates then
266                                    for g1, d1 in next, c do
267                                        for i=1,#d1 do
268                                            local d1i = d1[i]
269                                            local d1u = indices[d1i]
270                                            if d1u then
271                                                d1[i] = d1u
272                                            else
273                                                report_error("case %i, bad index in unifying %s: %s of %s",1,kind,i,d1i,nofindices)
274                                            end
275                                        end
276                                        local ug1 = indices[g1]
277                                        if ug1 then
278                                            t1[ug1] = d1
279                                            local dg1 = duplicates[ug1]
280                                            if dg1 then
281                                                for u in next, dg1 do
282                                                    t1[u] = copy(d1)
283                                                end
284                                            end
285                                        else
286                                            report_error("case %i, bad index in unifying %s: %s of %s",2,kind,g1,nofindices)
287                                        end
288                                    end
289                                else
290                                    for g1, d1 in next, c do
291                                        for i=1,#d1 do
292                                            local d1i = d1[i]
293                                            local d1u = indices[d1i]
294                                            if d1u then
295                                                d1[i] = d1u
296                                            else
297                                                report_error("case %i, bad index in unifying %s: %s of %s",2,kind,d1i,nofindices)
298                                            end
299                                        end
300                                        t1[indices[g1]] = d1
301                                    end
302                                end
303                                done[c] = t1
304                            end
305                            step.coverage = t1
306                        end
307                    elseif kind == "gpos_single" then
308                        local c = step.coverage
309                        if c then
310                            local t1 = done[c]
311                            if not t1 then
312                                t1 = { }
313                                if duplicates then
314                                    for g1, d1 in next, c do
315                                        local ug1 = indices[g1]
316                                        if ug1 then
317                                            t1[ug1] = d1
318                                            local dg1 = duplicates[ug1]
319                                            if dg1 then
320                                                for u in next, dg1 do
321                                                    t1[u] = d1
322                                                end
323                                            end
324                                        else
325                                            report_error("case %i, bad index in unifying %s: %s of %s",1,kind,g1,nofindices)
326                                        end
327                                    end
328                                else
329                                    for g1, d1 in next, c do
330                                        local ug1 = indices[g1]
331                                        if ug1 then
332                                            t1[ug1] = d1
333                                        else
334                                            report_error("case %i, bad index in unifying %s: %s of %s",2,kind,g1,nofindices)
335                                        end
336                                    end
337                                end
338                                done[c] = t1
339                            end
340                            step.coverage = t1
341                        end
342                    elseif kind == "gpos_mark2base" or kind == "gpos_mark2mark" or kind == "gpos_mark2ligature" then
343                        local c = step.coverage
344                        if c then
345                            local t1 = done[c]
346                            if not t1 then
347                                t1 = { }
348                                for g1, d1 in next, c do
349                                    local ug1 = indices[g1]
350                                    if ug1 then
351                                        t1[ug1] = d1
352                                    else
353                                        report_error("case %i, bad index in unifying %s: %s of %s",1,kind,g1,nofindices)
354                                    end
355                                end
356                                done[c] = t1
357                            end
358                            step.coverage = t1
359                        end
360                        local c = step.baseclasses
361                        if c then
362                            local t1 = done[c]
363                            if not t1 then
364                                for g1, d1 in next, c do
365                                    local t2 = done[d1]
366                                    if not t2 then
367                                        t2 = { }
368                                        for g2, d2 in next, d1 do
369                                            local ug2 = indices[g2]
370                                            if ug2 then
371                                                t2[ug2] = d2
372                                            else
373                                                report_error("case %i, bad index in unifying %s: %s of %s",2,kind,g2,nofindices)
374                                            end
375                                        end
376                                        done[d1] = t2
377                                    end
378                                    c[g1] = t2
379                                end
380                                done[c] = c
381                            end
382                        end
383                    elseif kind == "gpos_cursive" then
384                        local c = step.coverage
385                        if c then
386                            local t1 = done[c]
387                            if not t1 then
388                                t1 = { }
389                                if duplicates then
390                                    for g1, d1 in next, c do
391                                        local ug1 = indices[g1]
392                                        if ug1 then
393                                            t1[ug1] = d1
394                                            --
395                                            local dg1 = duplicates[ug1]
396                                            if dg1 then
397                                                -- probably needs a bit more
398                                                for u in next, dg1 do
399                                                    t1[u] = copy(d1)
400                                                end
401                                            end
402                                        else
403                                            report_error("case %i, bad index in unifying %s: %s of %s",1,kind,g1,nofindices)
404                                        end
405                                    end
406                                else
407                                    for g1, d1 in next, c do
408                                        local ug1 = indices[g1]
409                                        if ug1 then
410                                            t1[ug1] = d1
411                                        else
412                                            report_error("case %i, bad index in unifying %s: %s of %s",2,kind,g1,nofindices)
413                                        end
414                                    end
415                                end
416                                done[c] = t1
417                            end
418                            step.coverage = t1
419                        end
420                    end
421                    --
422                    local rules = step.rules
423                    if rules then
424                        for i=1,#rules do
425                            local rule = rules[i]
426                            --
427                            local before   = rule.before   if before  then recover(before)  end
428                            local after    = rule.after    if after   then recover(after)   end
429                            local current  = rule.current  if current then recover(current) end
430                            --
431                            local replacements = rule.replacements
432                            if replacements then
433                                if not done[replacements] then
434                                    local r = { }
435                                    for k, v in next, replacements do
436                                        r[indices[k]] = indices[v]
437                                    end
438                                    rule.replacements = r
439                                    done[replacements] = r
440                                end
441                            end
442                        end
443                    end
444                end
445            end
446       end
447    end
448    --
449    unifythem(resources.sequences)
450    unifythem(resources.sublookups)
451end
452
453local function copyduplicates(fontdata)
454    if check_duplicates then
455        local descriptions = fontdata.descriptions
456        local resources    = fontdata.resources
457        local duplicates   = resources.duplicates
458        if check_soft_hyphen then
459         -- ebgaramond has a zero width empty soft hyphen
460         -- antykwatorunska lacks a soft hyphen
461         -- lucidaot has a halfwidth soft hyphen
462
463         -- local dh = descriptions[0x2D]
464         -- if dh then
465         --     descriptions[0xAD] = nil
466         --     local d = duplicates[0x2D]
467         --     if d then
468         --         d[#d+1] = { [0xAD] = true }
469         --     else
470         --         duplicates[0x2D] = { [0xAD] = true }
471         --     end
472         -- end
473
474            local dh = descriptions[0x2D]
475            if dh then
476                local ds = descriptions[0xAD]
477                if not ds or ds.width ~= dh.width then
478                    descriptions[0xAD] = nil
479                    if ds then
480                        if trace_unicodes then
481                            report_unicodes("patching soft hyphen")
482                        end
483                    else
484                        if trace_unicodes then
485                            report_unicodes("adding soft hyphen")
486                        end
487                    end
488                    if not duplicates then
489                        duplicates = { }
490                        resources.duplicates = duplicates
491                    end
492                    local d = duplicates[0x2D]
493                    if d then
494                        d[0xAD] = true
495                    else
496                        duplicates[0x2D] = { [0xAD] = true }
497                    end
498                end
499            end
500
501        end
502        if duplicates then
503           for u, d in next, duplicates do
504                local du = descriptions[u]
505                if du then
506                    local t = { f_character_y(u), "@", f_index(du.index), "->" }
507                    local n = 0
508                    local m = 25
509                    for u in next, d do
510                        if descriptions[u] then
511                            if n < m then
512                                t[n+4] = f_character_n(u)
513                            end
514                        else
515                            local c = copy(du)
516                            c.unicode = u -- better this way
517                            descriptions[u] = c
518                            if n < m then
519                                t[n+4] = f_character_y(u)
520                            end
521                        end
522                        n = n + 1
523                    end
524                    if trace_unicodes then
525                        if n <= m then
526                            report_unicodes("%i : % t",n,t)
527                        else
528                            report_unicodes("%i : % t ...",n,t)
529                        end
530                    end
531                else
532                    -- what a mess
533                end
534            end
535        end
536    end
537end
538
539local ignore = { -- should we fix them?
540    ["notdef"]            = true,
541    [".notdef"]           = true,
542    ["null"]              = true,
543    [".null"]             = true,
544    ["nonmarkingreturn"]  = true,
545}
546
547
548local function checklookups(fontdata,missing,nofmissing)
549    local descriptions = fontdata.descriptions
550    local resources    = fontdata.resources
551    if missing and nofmissing and nofmissing <= 0 then
552        return
553    end
554    --
555    local singles    = { }
556    local alternates = { }
557    local ligatures  = { }
558
559    if not missing then
560        missing    = { }
561        nofmissing = 0
562        for u, d in next, descriptions do
563            if not d.unicode then
564                nofmissing = nofmissing + 1
565                missing[u] = true
566            end
567        end
568    end
569    local function collectthem(sequences)
570        if not sequences then
571            return
572        end
573        for i=1,#sequences do
574            local sequence = sequences[i]
575            local kind     = sequence.type
576            local steps    = sequence.steps
577            if steps then
578                for i=1,#steps do
579                    local step = steps[i]
580                    if kind == "gsub_single" then
581                        local c = step.coverage
582                        if c then
583                            singles[#singles+1] = c
584                        end
585                    elseif kind == "gsub_alternate" then
586                        local c = step.coverage
587                        if c then
588                            alternates[#alternates+1] = c
589                        end
590                    elseif kind == "gsub_ligature" then
591                        local c = step.coverage
592                        if c then
593                            ligatures[#ligatures+1] = c
594                        end
595                    end
596                end
597            end
598        end
599    end
600
601    collectthem(resources.sequences)
602    collectthem(resources.sublookups)
603
604    local loops = 0
605    while true do
606        loops = loops + 1
607        local old = nofmissing
608        for i=1,#singles do
609            local c = singles[i]
610            for g1, g2 in next, c do
611                if missing[g1] then
612                    local u2 = descriptions[g2].unicode
613                    if u2 then
614                        missing[g1] = false
615                        descriptions[g1].unicode = u2
616                        nofmissing = nofmissing - 1
617                    end
618                end
619                if missing[g2] then
620                    local u1 = descriptions[g1].unicode
621                    if u1 then
622                        missing[g2] = false
623                        descriptions[g2].unicode = u1
624                        nofmissing = nofmissing - 1
625                    end
626                end
627            end
628        end
629        for i=1,#alternates do
630            local c = alternates[i]
631            -- maybe first a g1 loop and then a g2
632            for g1, d1 in next, c do
633                if missing[g1] then
634                    for i=1,#d1 do
635                        local g2 = d1[i]
636                        local u2 = descriptions[g2].unicode
637                        if u2 then
638                            missing[g1] = false
639                            descriptions[g1].unicode = u2
640                            nofmissing = nofmissing - 1
641                        end
642                    end
643                end
644                if not missing[g1] then
645                    for i=1,#d1 do
646                        local g2 = d1[i]
647                        if missing[g2] then
648                            local u1 = descriptions[g1].unicode
649                            if u1 then
650                                missing[g2] = false
651                                descriptions[g2].unicode = u1
652                                nofmissing = nofmissing - 1
653                            end
654                        end
655                    end
656                end
657            end
658        end
659        if nofmissing <= 0 then
660            if trace_unicodes then
661                report_unicodes("all missings done in %s loops",loops)
662            end
663            return
664        elseif old == nofmissing then
665            break
666        end
667    end
668
669    local t, n -- no need to insert/remove and allocate many times
670
671    local function recursed(c)
672        for g, d in next, c do
673            if g ~= "ligature" then
674                local u = descriptions[g].unicode
675                if u then
676                    n = n + 1
677                    t[n] = u
678                    recursed(d)
679                    n = n - 1
680                end
681            elseif missing[d] then
682                local l = { }
683                local m = 0
684                for i=1,n do
685                    local u = t[i]
686                    if type(u) == "table" then
687                        for i=1,#u do
688                            m = m + 1
689                            l[m] = u[i]
690                        end
691                    else
692                        m = m + 1
693                        l[m] = u
694                    end
695                end
696                missing[d] = false
697                descriptions[d].unicode = l
698                nofmissing = nofmissing - 1
699            end
700        end
701    end
702
703    if nofmissing > 0 then
704        t = { }
705        n = 0
706        local loops = 0
707        while true do
708            loops = loops + 1
709            local old = nofmissing
710            for i=1,#ligatures do
711                recursed(ligatures[i])
712            end
713            if nofmissing <= 0 then
714                if trace_unicodes then
715                    report_unicodes("all missings done in %s loops",loops)
716                end
717                return
718            elseif old == nofmissing then
719                break
720            end
721        end
722        t = nil
723        n = 0
724    end
725
726    if trace_unicodes and nofmissing > 0 then
727        local done = { }
728        for i, r in next, missing do
729            if r then
730                local data = descriptions[i]
731                local name = data and data.name or f_index(i)
732                if not ignore[name] then
733                    done[name] = true
734                end
735            end
736        end
737        if next(done) then
738            report_unicodes("not unicoded: % t",sortedkeys(done))
739        end
740    end
741end
742
743local firstprivate = fonts.privateoffsets and fonts.privateoffsets.textbase or 0xF0000
744local puafirst     = 0xE000
745local pualast      = 0xF8FF
746
747local function unifymissing(fontdata)
748    if not fonts.mappings then
749        require("font-map")
750        require("font-agl")
751    end
752    local unicodes     = { }
753    local resources    = fontdata.resources
754    resources.unicodes = unicodes
755    for unicode, d in next, fontdata.descriptions do
756        if unicode < privateoffset then
757            if unicode >= puafirst and unicode <= pualast then
758                -- report_unicodes("resolving private unicode %U",unicode)
759            else
760                local name = d.name
761                if name then
762                    unicodes[name] = unicode
763                end
764            end
765        else
766            -- report_unicodes("resolving private unicode %U",unicode)
767        end
768    end
769    fonts.mappings.addtounicode(fontdata,fontdata.filename,checklookups)
770    resources.unicodes = nil
771end
772
773local function unifyglyphs(fontdata,usenames)
774    local private      = fontdata.private or privateoffset
775    local glyphs       = fontdata.glyphs
776    local indices      = { }
777    local descriptions = { }
778    local names        = usenames and { }
779    local resources    = fontdata.resources
780    local zero         = glyphs[0]
781    local zerocode     = zero.unicode
782    local nofglyphs    = #glyphs
783    if not zerocode then
784        zerocode       = private
785        zero.unicode   = zerocode
786        private        = private + 1
787    end
788    descriptions[zerocode] = zero
789    if names then
790        local name  = glyphs[0].name or f_private(zerocode)
791        indices[0]  = name
792        names[name] = zerocode
793    else
794        indices[0] = zerocode
795    end
796    --
797    if names then
798        -- seldom uses, we don't issue message ... this branch might even go away
799        for index=1,nofglyphs do
800            local glyph   = glyphs[index]
801            local unicode = glyph.unicode -- this is the primary one
802            if not unicode then
803                unicode        = private
804                local name     = glyph.name or f_private(unicode)
805                indices[index] = name
806                names[name]    = unicode
807                private        = private + 1
808            elseif unicode >= firstprivate then
809                unicode        = private
810                local name     = glyph.name or f_private(unicode)
811                indices[index] = name
812                names[name]    = unicode
813                private        = private + 1
814            elseif unicode >= puafirst and unicode <= pualast then
815                local name     = glyph.name or f_private(unicode)
816                indices[index] = name
817                names[name]    = unicode
818            elseif descriptions[unicode] then
819                unicode        = private
820                local name     = glyph.name or f_private(unicode)
821                indices[index] = name
822                names[name]    = unicode
823                private        = private + 1
824            else
825                local name     = glyph.name or f_unicode(unicode)
826                indices[index] = name
827                names[name]    = unicode
828            end
829            descriptions[unicode] = glyph
830        end
831    elseif trace_unicodes then
832        for index=1,nofglyphs do
833            local glyph   = glyphs[index]
834            local unicode = glyph.unicode -- this is the primary one
835            if not unicode then
836                unicode        = private
837                indices[index] = unicode
838                private        = private + 1
839            elseif unicode >= firstprivate then
840                local name = glyph.name
841                if name then
842                    report_unicodes("moving glyph %a indexed %05X from private %U to %U ",name,index,unicode,private)
843                else
844                    report_unicodes("moving glyph indexed %05X from private %U to %U ",index,unicode,private)
845                end
846                unicode        = private
847                indices[index] = unicode
848                private        = private + 1
849            elseif unicode >= puafirst and unicode <= pualast then
850                local name = glyph.name
851                if name then
852                    report_unicodes("keeping private unicode %U for glyph %a indexed %05X",unicode,name,index)
853                else
854                    report_unicodes("keeping private unicode %U for glyph indexed %05X",unicode,index)
855                end
856                indices[index] = unicode
857            elseif descriptions[unicode] then
858                local name = glyph.name
859                if name then
860                    report_unicodes("assigning duplicate unicode %U to %U for glyph %a indexed %05X ",unicode,private,name,index)
861                else
862                    report_unicodes("assigning duplicate unicode %U to %U for glyph indexed %05X ",unicode,private,index)
863                end
864                unicode        = private
865                indices[index] = unicode
866                private        = private + 1
867            else
868                indices[index] = unicode
869            end
870            descriptions[unicode] = glyph
871        end
872    else
873        for index=1,nofglyphs do
874            local glyph   = glyphs[index]
875            local unicode = glyph.unicode -- this is the primary one
876            if not unicode then
877                unicode        = private
878                indices[index] = unicode
879                private        = private + 1
880            elseif unicode >= firstprivate then
881                local name = glyph.name
882                unicode        = private
883                indices[index] = unicode
884                private        = private + 1
885            elseif unicode >= puafirst and unicode <= pualast then
886                local name = glyph.name
887                indices[index] = unicode
888            elseif descriptions[unicode] then
889                local name = glyph.name
890                unicode        = private
891                indices[index] = unicode
892                private        = private + 1
893            else
894                indices[index] = unicode
895            end
896            descriptions[unicode] = glyph
897        end
898    end
899    --
900    for index=1,nofglyphs do
901        local math = glyphs[index].math
902        if math then
903            local list = math.vparts
904            if list then
905                for i=1,#list do local l = list[i] l.glyph = indices[l.glyph] end
906            end
907            local list = math.hparts
908            if list then
909                for i=1,#list do local l = list[i] l.glyph = indices[l.glyph] end
910            end
911            local list = math.vvariants
912            if list then
913             -- for i=1,#list do local l = list[i] l.glyph = indices[l.glyph] end
914                for i=1,#list do list[i] = indices[list[i]] end
915            end
916            local list = math.hvariants
917            if list then
918             -- for i=1,#list do local l = list[i] l.glyph = indices[l.glyph] end
919                for i=1,#list do list[i] = indices[list[i]] end
920            end
921        end
922    end
923    --
924    local colorpalettes = resources.colorpalettes
925    if colorpalettes then
926        for index=1,nofglyphs do
927            local colors = glyphs[index].colors
928            if colors then
929                for i=1,#colors do
930                    local c = colors[i]
931                    c.slot = indices[c.slot]
932                end
933            end
934        end
935    end
936    --
937    fontdata.private      = private
938    fontdata.glyphs       = nil
939    fontdata.names        = names
940    fontdata.descriptions = descriptions
941    fontdata.hashmethod   = hashmethod
942    fontdata.nofglyphs    = nofglyphs
943    --
944    return indices, names
945end
946
947local p_crappyname  do
948
949    local p_hex   = R("af","AF","09")
950    local p_digit = R("09")
951    local p_done  = S("._-")^0 + P(-1)
952    local p_alpha = R("az","AZ")
953    local p_ALPHA = R("AZ")
954
955    p_crappyname = (
956    -- (P("uni") + P("UNI") + P("Uni") + P("U") + P("u"))
957        lpeg.utfchartabletopattern({ "uni", "u" },true)
958      * S("Xx_")^0
959      * p_hex^1
960   -- + (P("identity") + P("Identity") + P("IDENTITY") + P("glyph") + P("jamo"))
961      + lpeg.utfchartabletopattern({ "identity", "glyph", "jamo" },true)
962      * p_hex^1
963   -- + (P("index") + P("Index") + P("INDEX")+ P("afii"))
964      + lpeg.utfchartabletopattern({ "index", "afii" }, true)
965      * p_digit^1
966      -- also happens l
967      + p_digit
968      * p_hex^3
969      + p_alpha
970      * p_digit^1
971      -- sort of special
972      + P("aj")
973      * p_digit^1
974      + P("eh_")
975      * (p_digit^1 + p_ALPHA * p_digit^1)
976      + (1-P("_"))^1
977      * P("_uni")
978      * p_hex^1
979      + P("_")
980      * P(1)^1
981    ) * p_done
982
983end
984
985-- In context we only keep glyph names because of tracing and access by name
986-- so weird names make no sense.
987
988local forcekeep = false -- only for testing something
989
990directives.register("otf.keepnames",function(v)
991    report_cleanup("keeping weird glyph names, expect larger files and more memory usage")
992    forcekeep = v
993end)
994
995local function stripredundant(fontdata)
996    local descriptions = fontdata.descriptions
997    if descriptions then
998        local n = 0
999        local c = 0
1000        -- in context we always strip
1001        if (not context and fonts.privateoffsets.keepnames) or forcekeep then
1002            for unicode, d in next, descriptions do
1003                if d.class == "base" then
1004                    d.class = nil
1005                    c = c + 1
1006                end
1007            end
1008        else
1009            for unicode, d in next, descriptions do
1010                local name = d.name
1011                if name and lpegmatch(p_crappyname,name) then
1012                    d.name = nil
1013                    n = n + 1
1014                end
1015                if d.class == "base" then
1016                    d.class = nil
1017                    c = c + 1
1018                end
1019            end
1020        end
1021        if trace_cleanup then
1022            if n > 0 then
1023                report_cleanup("%s bogus names removed (verbose unicode)",n)
1024            end
1025            if c > 0 then
1026                report_cleanup("%s base class tags removed (default is base)",c)
1027            end
1028        end
1029    end
1030end
1031
1032readers.stripredundant = stripredundant
1033
1034function readers.getcomponents(fontdata) -- handy for resolving ligatures when names are missing
1035    local resources = fontdata.resources
1036    if resources then
1037        local sequences = resources.sequences
1038        if sequences then
1039            local collected = { }
1040            for i=1,#sequences do
1041                local sequence = sequences[i]
1042                if sequence.type == "gsub_ligature" then
1043                    local steps  = sequence.steps
1044                    if steps then
1045                        local l = { }
1046                        local function traverse(p,k,v)
1047                            if k == "ligature" then
1048                                collected[v] = { unpack(l) }
1049                            elseif tonumber(v) then
1050                                insert(l,k)
1051                                collected[v] = { unpack(l) }
1052                                remove(l)
1053                            else
1054                                insert(l,k)
1055                                for k, vv in next, v do
1056                                    traverse(p,k,vv)
1057                                end
1058                                remove(l)
1059                            end
1060                        end
1061                        for i=1,#steps do
1062                         -- we actually had/have this in base mode
1063                            local c = steps[i].coverage
1064                            if c then
1065                                for k, v in next, c do
1066                                    traverse(k,k,v)
1067                                end
1068                            end
1069                        end
1070                    end
1071                end
1072            end
1073            if next(collected) then
1074                -- remove self referring
1075             -- for k, v in next, collected do
1076             --     for i=1,#v do
1077             --         local vi = v[i]
1078             --         if vi == k then
1079             --          -- report("removing self referring ligature @ slot %5X from collected (1)",k)
1080             --             collected[k] = nil
1081             --         end
1082             --     end
1083             -- end
1084                while true do
1085                    local done = false
1086                    for k, v in next, collected do
1087                        for i=1,#v do
1088                            local vi = v[i]
1089                            if vi == k then
1090                             -- report("removing self referring ligature @ slot %5X from collected (2)",k)
1091                                collected[k] = nil
1092                                break
1093                            else
1094                                local c = collected[vi]
1095                                if c then
1096                                    done = true
1097                                    local t = { }
1098                                    local n = i - 1
1099                                    for j=1,n do
1100                                        t[j] = v[j]
1101                                    end
1102                                    for j=1,#c do
1103                                        n = n + 1
1104                                        t[n] = c[j]
1105                                    end
1106                                    for j=i+1,#v do
1107                                        n = n + 1
1108                                        t[n] = v[j]
1109                                    end
1110                                    collected[k] = t
1111                                    break
1112                                end
1113                            end
1114                        end
1115                    end
1116                    if not done then
1117                        break
1118                    end
1119                end
1120                return collected
1121            end
1122        end
1123    end
1124end
1125
1126readers.unifymissing = unifymissing
1127
1128function readers.rehash(fontdata,hashmethod) -- TODO: combine loops in one
1129    if not (fontdata and fontdata.glyphs) then
1130        return
1131    end
1132    if hashmethod == "indices" then
1133        fontdata.hashmethod = "indices"
1134    elseif hashmethod == "names" then
1135        fontdata.hashmethod = "names"
1136        local indices = unifyglyphs(fontdata,true)
1137        unifyresources(fontdata,indices)
1138        copyduplicates(fontdata)
1139        unifymissing(fontdata)
1140     -- stripredundant(fontdata)
1141    else
1142        fontdata.hashmethod = "unicodes"
1143        local indices = unifyglyphs(fontdata)
1144        unifyresources(fontdata,indices)
1145        copyduplicates(fontdata)
1146        unifymissing(fontdata)
1147        stripredundant(fontdata)
1148    end
1149    -- maybe here components
1150end
1151
1152function readers.checkhash(fontdata)
1153    local hashmethod = fontdata.hashmethod
1154    if hashmethod == "unicodes" then
1155        fontdata.names = nil -- just to be sure
1156    elseif hashmethod == "names" and fontdata.names then
1157        unifyresources(fontdata,fontdata.names)
1158        copyduplicates(fontdata)
1159        fontdata.hashmethod = "unicodes"
1160        fontdata.names = nil -- no need for it
1161    else
1162        readers.rehash(fontdata,"unicodes")
1163    end
1164end
1165
1166function readers.addunicodetable(fontdata)
1167    local resources = fontdata.resources
1168    local unicodes  = resources.unicodes
1169    if not unicodes then
1170        local descriptions = fontdata.descriptions
1171        if descriptions then
1172            unicodes = { }
1173            resources.unicodes = unicodes
1174            for u, d in next, descriptions do
1175                local n = d.name
1176                if n then
1177                    unicodes[n] = u
1178                end
1179            end
1180        end
1181    end
1182end
1183
1184-- for the moment here:
1185
1186local concat, sort = table.concat, table.sort
1187local next, type, tostring = next, type, tostring
1188
1189local criterium     = 1
1190local threshold     = 0
1191
1192local trace_packing = false  trackers.register("otf.packing", function(v) trace_packing = v end)
1193local trace_loading = false  trackers.register("otf.loading", function(v) trace_loading = v end)
1194
1195local report_otf    = logs.reporter("fonts","otf loading")
1196
1197local function tabstr_normal(t)
1198    local s = { }
1199    local n = 0
1200    for k, v in next, t do
1201        n = n + 1
1202        if type(v) == "table" then
1203            s[n] = k .. ">" .. tabstr_normal(v)
1204        elseif v == true then
1205            s[n] = k .. "+" -- "=true"
1206        elseif v then
1207            s[n] = k .. "=" .. v
1208        else
1209            s[n] = k .. "-" -- "=false"
1210        end
1211    end
1212    if n == 0 then
1213        return ""
1214    elseif n == 1 then
1215        return s[1]
1216    else
1217        sort(s) -- costly but needed (occasional wrong hit otherwise)
1218        return concat(s,",")
1219    end
1220end
1221
1222local function tabstr_flat(t)
1223    local s = { }
1224    local n = 0
1225    for k, v in next, t do
1226        n = n + 1
1227        s[n] = k .. "=" .. v
1228    end
1229    if n == 0 then
1230        return ""
1231    elseif n == 1 then
1232        return s[1]
1233    else
1234        sort(s) -- costly but needed (occasional wrong hit otherwise)
1235        return concat(s,",")
1236    end
1237end
1238
1239local function tabstr_mixed(t) -- indexed
1240    local n = #t
1241    if n == 0 then
1242        return ""
1243    elseif n == 1 then
1244        local k = t[1]
1245        if k == true then
1246            return "++" -- we need to distinguish from "true"
1247        elseif k == false then
1248            return "--" -- we need to distinguish from "false"
1249        else
1250            return tostring(k) -- number or string
1251        end
1252    else
1253        local s = { }
1254        for i=1,n do
1255            local k = t[i]
1256            if k == true then
1257                s[i] = "++" -- we need to distinguish from "true"
1258            elseif k == false then
1259                s[i] = "--" -- we need to distinguish from "false"
1260            else
1261                s[i] = k -- number or string
1262            end
1263        end
1264        return concat(s,",")
1265    end
1266end
1267
1268local function tabstr_boolean(t)
1269    local s = { }
1270    local n = 0
1271    for k, v in next, t do
1272        n = n + 1
1273        if v then
1274            s[n] = k .. "+"
1275        else
1276            s[n] = k .. "-"
1277        end
1278    end
1279    if n == 0 then
1280        return ""
1281    elseif n == 1 then
1282        return s[1]
1283    else
1284        sort(s) -- costly but needed (occasional wrong hit otherwise)
1285        return concat(s,",")
1286    end
1287end
1288
1289-- beware: we cannot unpack and repack the same table because then sharing
1290-- interferes (we could catch this if needed) .. so for now: save, reload
1291-- and repack in such cases (never needed anyway) .. a tricky aspect is that
1292-- we then need to sort more thanks to random hashing
1293
1294function readers.pack(data)
1295
1296    if data then
1297
1298        local h, t, c = { }, { }, { }
1299        local hh, tt, cc = { }, { }, { }
1300        local nt, ntt = 0, 0
1301
1302        local function pack_normal(v)
1303            local tag = tabstr_normal(v)
1304            local ht = h[tag]
1305            if ht then
1306                c[ht] = c[ht] + 1
1307                return ht
1308            else
1309                nt = nt + 1
1310                t[nt] = v
1311                h[tag] = nt
1312                c[nt] = 1
1313                return nt
1314            end
1315        end
1316
1317        local function pack_normal_cc(v)
1318            local tag = tabstr_normal(v)
1319            local ht = h[tag]
1320            if ht then
1321                c[ht] = c[ht] + 1
1322                return ht
1323            else
1324                v[1] = 0
1325                nt = nt + 1
1326                t[nt] = v
1327                h[tag] = nt
1328                c[nt] = 1
1329                return nt
1330            end
1331        end
1332
1333        local function pack_flat(v)
1334            local tag = tabstr_flat(v)
1335            local ht = h[tag]
1336            if ht then
1337                c[ht] = c[ht] + 1
1338                return ht
1339            else
1340                nt = nt + 1
1341                t[nt] = v
1342                h[tag] = nt
1343                c[nt] = 1
1344                return nt
1345            end
1346        end
1347
1348        local function pack_indexed(v)
1349            local tag = concat(v," ")
1350            local ht = h[tag]
1351            if ht then
1352                c[ht] = c[ht] + 1
1353                return ht
1354            else
1355                nt = nt + 1
1356                t[nt] = v
1357                h[tag] = nt
1358                c[nt] = 1
1359                return nt
1360            end
1361        end
1362
1363     -- local function pack_indexed(v) -- less code
1364     --     local tag = concat(v," ")
1365     --     local ht = h[tag]
1366     --     if ht then
1367     --         c[ht] = c[ht] + 1
1368     --     else
1369     --         ht = nt + 1
1370     --         t[ht] = v
1371     --         c[ht] = 1
1372     --         h[tag] = ht
1373     --         nt = ht
1374     --     end
1375     --     return ht
1376     -- end
1377
1378        local function pack_mixed(v)
1379            local tag = tabstr_mixed(v)
1380            local ht = h[tag]
1381            if ht then
1382                c[ht] = c[ht] + 1
1383                return ht
1384            else
1385                nt = nt + 1
1386                t[nt] = v
1387                h[tag] = nt
1388                c[nt] = 1
1389                return nt
1390            end
1391        end
1392
1393        -- saves a lot on noto sans
1394
1395        -- can be made more clever
1396
1397        local function pack_boolean(v)
1398            local tag = tabstr_boolean(v)
1399            local ht = h[tag]
1400            if ht then
1401                c[ht] = c[ht] + 1
1402                return ht
1403            else
1404                nt = nt + 1
1405                t[nt] = v
1406                h[tag] = nt
1407                c[nt] = 1
1408                return nt
1409            end
1410        end
1411
1412        local function pack_final(v)
1413            -- v == number
1414            if c[v] <= criterium then
1415                return t[v]
1416            else
1417                -- compact hash
1418                local hv = hh[v]
1419                if hv then
1420                    return hv
1421                else
1422                    ntt = ntt + 1
1423                    tt[ntt] = t[v]
1424                    hh[v] = ntt
1425                    cc[ntt] = c[v]
1426                    return ntt
1427                end
1428            end
1429        end
1430
1431        local function pack_final_cc(v)
1432            -- v == number
1433            if c[v] <= criterium then
1434                return t[v]
1435            else
1436                -- compact hash
1437                local hv = hh[v]
1438                if hv then
1439                    return hv
1440                else
1441                    ntt = ntt + 1
1442                    tt[ntt] = t[v]
1443                    hh[v] = ntt
1444                    cc[ntt] = c[v]
1445                    return ntt
1446                end
1447            end
1448        end
1449
1450        local function success(stage,pass)
1451            if nt == 0 then
1452                if trace_loading or trace_packing then
1453                    report_otf("pack quality: nothing to pack")
1454                end
1455                return false
1456            elseif nt >= threshold then
1457                local one  = 0
1458                local two  = 0
1459                local rest = 0
1460                if pass == 1 then
1461                    for k,v in next, c do
1462                        if v == 1 then
1463                            one = one + 1
1464                        elseif v == 2 then
1465                            two = two + 1
1466                        else
1467                            rest = rest + 1
1468                        end
1469                    end
1470                else
1471                    for k,v in next, cc do
1472                        if v > 20 then
1473                            rest = rest + 1
1474                        elseif v > 10 then
1475                            two = two + 1
1476                        else
1477                            one = one + 1
1478                        end
1479                    end
1480                    data.tables = tt
1481                end
1482                if trace_loading or trace_packing then
1483                    report_otf("pack quality: stage %s, pass %s, %s packed, 1-10:%s, 11-20:%s, rest:%s (criterium: %s)",
1484                        stage, pass, one+two+rest, one, two, rest, criterium)
1485                end
1486                return true
1487            else
1488                if trace_loading or trace_packing then
1489                    report_otf("pack quality: stage %s, pass %s, %s packed, aborting pack (threshold: %s)",
1490                        stage, pass, nt, threshold)
1491                end
1492                return false
1493            end
1494        end
1495
1496        local function packers(pass)
1497            if pass == 1 then
1498                return pack_normal, pack_indexed, pack_flat, pack_boolean, pack_mixed, pack_normal_cc
1499            else
1500                return pack_final, pack_final, pack_final, pack_final, pack_final, pack_final_cc
1501            end
1502        end
1503
1504        local resources  = data.resources
1505        local sequences  = resources.sequences
1506        local sublookups = resources.sublookups
1507        local features   = resources.features
1508        local palettes   = resources.colorpalettes
1509        local variable   = resources.variabledata
1510
1511        local chardata     = characters and characters.data
1512        local descriptions = data.descriptions or data.glyphs
1513
1514        if not descriptions then
1515            return
1516        end
1517
1518        for pass=1,2 do
1519
1520            if trace_packing then
1521                report_otf("start packing: stage 1, pass %s",pass)
1522            end
1523
1524            local pack_normal, pack_indexed, pack_flat, pack_boolean, pack_mixed, pack_normal_cc = packers(pass)
1525
1526            for unicode, description in next, descriptions do
1527                local boundingbox = description.boundingbox
1528                if boundingbox then
1529                    description.boundingbox = pack_indexed(boundingbox)
1530                end
1531                local math = description.math
1532                if math then
1533                    local kerns = math.kerns
1534                    if kerns then
1535                        for tag, kern in next, kerns do
1536                            kerns[tag] = pack_normal(kern)
1537                        end
1538                    end
1539                end
1540             -- if palettes then
1541             --     local color = description.color
1542             --     if color then
1543             --         for i=1,#color do
1544             --             color[i] = pack_normal(color[i])
1545             --         end
1546             --     end
1547             -- end
1548            end
1549
1550            local function packthem(sequences)
1551                for i=1,#sequences do
1552                    local sequence = sequences[i]
1553                    local kind     = sequence.type
1554                    local steps    = sequence.steps
1555                    local order    = sequence.order
1556                    local features = sequence.features
1557                    local flags    = sequence.flags
1558                    if steps then
1559                        for i=1,#steps do
1560                            local step = steps[i]
1561                            if kind == "gpos_pair" then
1562                                local c = step.coverage
1563                                if c then
1564                                    if step.format ~= "pair" then
1565                                        for g1, d1 in next, c do
1566                                            c[g1] = pack_normal(d1)
1567                                        end
1568                                    elseif step.shared then
1569                                        -- This branch results from classes. We already share at the reader end. Maybe
1570                                        -- the sharing should be moved there altogether but it becomes kind of messy
1571                                        -- then. Here we're still wasting time because in the second pass we serialize
1572                                        -- and hash. So we compromise. We could merge the two passes ...
1573                                        local shared = { }
1574                                        for g1, d1 in next, c do
1575                                            for g2, d2 in next, d1 do
1576                                                if not shared[d2] then
1577                                                    local f = d2[1] if f and f ~= true then d2[1] = pack_indexed(f) end
1578                                                    local s = d2[2] if s and s ~= true then d2[2] = pack_indexed(s) end
1579                                                    shared[d2] = true
1580                                                end
1581                                            end
1582                                        end
1583                                        if pass == 2 then
1584                                            step.shared = nil -- weird, so dups
1585                                        end
1586                                    else
1587                                        for g1, d1 in next, c do
1588                                            for g2, d2 in next, d1 do
1589                                                local f = d2[1] if f and f ~= true then d2[1] = pack_indexed(f) end
1590                                                local s = d2[2] if s and s ~= true then d2[2] = pack_indexed(s) end
1591                                            end
1592                                        end
1593                                    end
1594                                end
1595                            elseif kind == "gpos_single" then
1596                                local c = step.coverage
1597                                if c then
1598                                    if step.format == "single" then
1599                                        for g1, d1 in next, c do
1600                                            if d1 and d1 ~= true then
1601                                                c[g1] = pack_indexed(d1)
1602                                            end
1603                                        end
1604                                    else
1605                                        step.coverage = pack_normal(c)
1606                                    end
1607                                end
1608                            elseif kind == "gpos_cursive" then
1609                                local c = step.coverage
1610                                if c then
1611                                    for g1, d1 in next, c do
1612                                        local f = d1[2] if f then d1[2] = pack_indexed(f) end
1613                                        local s = d1[3] if s then d1[3] = pack_indexed(s) end
1614                                    end
1615                                end
1616                            elseif kind == "gpos_mark2base" or kind == "gpos_mark2mark" then
1617                                local c = step.baseclasses
1618                                if c then
1619                                    for g1, d1 in next, c do
1620                                        for g2, d2 in next, d1 do
1621                                            d1[g2] = pack_indexed(d2)
1622                                        end
1623                                    end
1624                                end
1625                                local c = step.coverage
1626                                if c then
1627                                    for g1, d1 in next, c do
1628                                        d1[2] = pack_indexed(d1[2])
1629                                    end
1630                                end
1631                            elseif kind == "gpos_mark2ligature" then
1632                                local c = step.baseclasses
1633                                if c then
1634                                    for g1, d1 in next, c do
1635                                        for g2, d2 in next, d1 do
1636                                            for g3, d3 in next, d2 do
1637                                                d2[g3] = pack_indexed(d3)
1638                                            end
1639                                        end
1640                                    end
1641                                end
1642                                local c = step.coverage
1643                                if c then
1644                                    for g1, d1 in next, c do
1645                                        d1[2] = pack_indexed(d1[2])
1646                                    end
1647                                end
1648                            end
1649                            -- if ... chain ...
1650                            local rules = step.rules
1651                            if rules then
1652                                for i=1,#rules do
1653                                    local rule = rules[i]
1654                                    local r = rule.before       if r then for i=1,#r do r[i] = pack_boolean(r[i]) end end
1655                                    local r = rule.after        if r then for i=1,#r do r[i] = pack_boolean(r[i]) end end
1656                                    local r = rule.current      if r then for i=1,#r do r[i] = pack_boolean(r[i]) end end
1657                                 -- local r = rule.lookups      if r then rule.lookups       = pack_mixed  (r)    end
1658                                    local r = rule.replacements if r then rule.replacements  = pack_flat   (r)    end
1659                                end
1660                            end
1661                        end
1662                    end
1663                    if order then
1664                        sequence.order = pack_indexed(order)
1665                    end
1666                    if features then
1667                        for script, feature in next, features do
1668                            features[script] = pack_normal(feature)
1669                        end
1670                    end
1671                    if flags then
1672                        sequence.flags = pack_normal(flags)
1673                    end
1674               end
1675            end
1676
1677            if sequences then
1678                packthem(sequences)
1679            end
1680
1681            if sublookups then
1682                packthem(sublookups)
1683            end
1684
1685            if features then
1686                for k, list in next, features do
1687                    for feature, spec in next, list do
1688                        list[feature] = pack_normal(spec)
1689                    end
1690                end
1691            end
1692
1693            if palettes then
1694                for i=1,#palettes do
1695                    local p = palettes[i]
1696                    for j=1,#p do
1697                        p[j] = pack_indexed(p[j])
1698                    end
1699                end
1700
1701            end
1702
1703            if variable then
1704
1705                -- todo: segments
1706
1707                local instances = variable.instances
1708                if instances then
1709                    for i=1,#instances do
1710                        local v = instances[i].values
1711                        for j=1,#v do
1712                            v[j] = pack_normal(v[j])
1713                        end
1714                    end
1715                end
1716
1717                local function packdeltas(main)
1718                    if main then
1719                        local deltas = main.deltas
1720                        if deltas then
1721                            for i=1,#deltas do
1722                                local di = deltas[i]
1723                                local d  = di.deltas
1724                             -- local r  = di.regions
1725                                for j=1,#d do
1726                                    d[j] = pack_indexed(d[j])
1727                                end
1728                                di.regions = pack_indexed(di.regions)
1729                            end
1730                        end
1731                        local regions = main.regions
1732                        if regions then
1733                            for i=1,#regions do
1734                                local r = regions[i]
1735                                for j=1,#r do
1736                                    r[j] = pack_normal(r[j])
1737                                end
1738                            end
1739                        end
1740                    end
1741                end
1742
1743                packdeltas(variable.global)
1744                packdeltas(variable.horizontal)
1745                packdeltas(variable.vertical)
1746                packdeltas(variable.metrics)
1747
1748            end
1749
1750            if not success(1,pass) then
1751                return
1752            end
1753
1754        end
1755
1756        if nt > 0 then
1757
1758            for pass=1,2 do
1759
1760                if trace_packing then
1761                    report_otf("start packing: stage 2, pass %s",pass)
1762                end
1763
1764                local pack_normal, pack_indexed, pack_flat, pack_boolean, pack_mixed, pack_normal_cc = packers(pass)
1765
1766                for unicode, description in next, descriptions do
1767                    local math = description.math
1768                    if math then
1769                        local kerns = math.kerns
1770                        if kerns then
1771                            math.kerns = pack_normal(kerns)
1772                        end
1773                    end
1774                end
1775
1776                local function packthem(sequences)
1777                    for i=1,#sequences do
1778                        local sequence = sequences[i]
1779                        local kind     = sequence.type
1780                        local steps    = sequence.steps
1781                        local features = sequence.features
1782                        if steps then
1783                            for i=1,#steps do
1784                                local step = steps[i]
1785                                if kind == "gpos_pair" then
1786                                    local c = step.coverage
1787                                    if c then
1788                                        if step.format == "pair" then
1789                                            for g1, d1 in next, c do
1790                                                for g2, d2 in next, d1 do
1791                                                    d1[g2] = pack_normal(d2)
1792                                                end
1793                                            end
1794                                        end
1795                                    end
1796                             -- elseif kind == "gpos_cursive" then
1797                             --     local c = step.coverage -- new
1798                             --     if c then
1799                             --         for g1, d1 in next, c do
1800                             --             c[g1] = pack_normal_cc(d1)
1801                             --         end
1802                             --     end
1803                                elseif kind == "gpos_mark2ligature" then
1804                                    local c = step.baseclasses -- new
1805                                    if c then
1806                                        for g1, d1 in next, c do
1807                                            for g2, d2 in next, d1 do
1808                                                d1[g2] = pack_normal(d2)
1809                                            end
1810                                        end
1811                                    end
1812                                end
1813                                local rules = step.rules
1814                                if rules then
1815                                    for i=1,#rules do
1816                                        local rule = rules[i]
1817                                        local r = rule.before  if r then rule.before  = pack_normal(r) end
1818                                        local r = rule.after   if r then rule.after   = pack_normal(r) end
1819                                        local r = rule.current if r then rule.current = pack_normal(r) end
1820                                    end
1821                                end
1822                            end
1823                        end
1824                        if features then
1825                            sequence.features = pack_normal(features)
1826                        end
1827                   end
1828                end
1829                if sequences then
1830                    packthem(sequences)
1831                end
1832                if sublookups then
1833                    packthem(sublookups)
1834                end
1835                if variable then
1836                    local function unpackdeltas(main)
1837                        if main then
1838                            local regions = main.regions
1839                            if regions then
1840                                main.regions = pack_normal(regions)
1841                            end
1842                        end
1843                    end
1844                    unpackdeltas(variable.global)
1845                    unpackdeltas(variable.horizontal)
1846                    unpackdeltas(variable.vertical)
1847                    unpackdeltas(variable.metrics)
1848                end
1849             -- if not success(2,pass) then
1850             --  -- return
1851             -- end
1852            end
1853
1854            for pass=1,2 do
1855                if trace_packing then
1856                    report_otf("start packing: stage 3, pass %s",pass)
1857                end
1858
1859                local pack_normal, pack_indexed, pack_flat, pack_boolean, pack_mixed, pack_normal_cc = packers(pass)
1860
1861                local function packthem(sequences)
1862                    for i=1,#sequences do
1863                        local sequence = sequences[i]
1864                        local kind     = sequence.type
1865                        local steps    = sequence.steps
1866                        local features = sequence.features
1867                        if steps then
1868                            for i=1,#steps do
1869                                local step = steps[i]
1870                                if kind == "gpos_pair" then
1871                                    local c = step.coverage
1872                                    if c then
1873                                        if step.format == "pair" then
1874                                            for g1, d1 in next, c do
1875                                                c[g1] = pack_normal(d1)
1876                                            end
1877                                        end
1878                                    end
1879                                elseif kind == "gpos_cursive" then
1880                                    local c = step.coverage
1881                                    if c then
1882                                        for g1, d1 in next, c do
1883                                            c[g1] = pack_normal_cc(d1)
1884                                        end
1885                                    end
1886                                end
1887                            end
1888                        end
1889                    end
1890                end
1891
1892                if sequences then
1893                    packthem(sequences)
1894                end
1895                if sublookups then
1896                    packthem(sublookups)
1897                end
1898
1899            end
1900
1901        end
1902
1903    end
1904end
1905
1906local unpacked_mt = {
1907    __index =
1908        function(t,k)
1909            t[k] = false
1910            return k -- next time true
1911        end
1912}
1913
1914function readers.unpack(data)
1915
1916    if data then
1917        local tables = data.tables
1918        if tables then
1919            local resources    = data.resources
1920            local descriptions = data.descriptions or data.glyphs
1921            local sequences    = resources.sequences
1922            local sublookups   = resources.sublookups
1923            local features     = resources.features
1924            local palettes     = resources.colorpalettes
1925            local variable     = resources.variabledata
1926            local unpacked     = { }
1927            setmetatable(unpacked,unpacked_mt)
1928            for unicode, description in next, descriptions do
1929                local tv = tables[description.boundingbox]
1930                if tv then
1931                    description.boundingbox = tv
1932                end
1933                local math = description.math
1934                if math then
1935                    local kerns = math.kerns
1936                    if kerns then
1937                        local tm = tables[kerns]
1938                        if tm then
1939                            math.kerns = tm
1940                            kerns = unpacked[tm]
1941                        end
1942                        if kerns then
1943                            for k, kern in next, kerns do
1944                                local tv = tables[kern]
1945                                if tv then
1946                                    kerns[k] = tv
1947                                end
1948                            end
1949                        end
1950                    end
1951                end
1952             -- if palettes then
1953             --     local color = description.color
1954             --     if color then
1955             --         for i=1,#color do
1956             --             local tv = tables[color[i]]
1957             --             if tv then
1958             --                 color[i] = tv
1959             --             end
1960             --         end
1961             --     end
1962             -- end
1963            end
1964
1965         -- local function expandranges(t,ranges)
1966         --     for i=1,#ranges do
1967         --         local r = ranges[i]
1968         --         for k=r[1],r[2] do
1969         --             t[k] = true
1970         --         end
1971         --     end
1972         -- end
1973
1974            local function unpackthem(sequences)
1975                for i=1,#sequences do
1976                    local sequence  = sequences[i]
1977                    local kind      = sequence.type
1978                    local steps     = sequence.steps
1979                    local order     = sequence.order
1980                    local features  = sequence.features
1981                    local flags     = sequence.flags
1982                    local markclass = sequence.markclass
1983                    if features then
1984                        local tv = tables[features]
1985                        if tv then
1986                            sequence.features = tv
1987                            features = tv
1988                        end
1989                        for script, feature in next, features do
1990                            local tv = tables[feature]
1991                            if tv then
1992                                features[script] = tv
1993                            end
1994                        end
1995                    end
1996                    if steps then
1997                        for i=1,#steps do
1998                            local step = steps[i]
1999                            if kind == "gpos_pair" then
2000                                local c = step.coverage
2001                                if c then
2002                                    if step.format == "pair" then
2003                                        for g1, d1 in next, c do
2004                                            local tv = tables[d1]
2005                                            if tv then
2006                                                c[g1] = tv
2007                                                d1 = tv
2008                                            end
2009                                            for g2, d2 in next, d1 do
2010                                                local tv = tables[d2]
2011                                                if tv then
2012                                                    d1[g2] = tv
2013                                                    d2 = tv
2014                                                end
2015                                                local f = tables[d2[1]] if f then d2[1] = f end
2016                                                local s = tables[d2[2]] if s then d2[2] = s end
2017                                            end
2018                                        end
2019                                    else
2020                                        for g1, d1 in next, c do
2021                                            local tv = tables[d1]
2022                                            if tv then
2023                                                c[g1] = tv
2024                                            end
2025                                        end
2026                                    end
2027                                end
2028                            elseif kind == "gpos_single" then
2029                                local c = step.coverage
2030                                if c then
2031                                    if step.format == "single" then
2032                                        for g1, d1 in next, c do
2033                                            local tv = tables[d1]
2034                                            if tv then
2035                                                c[g1] = tv
2036                                            end
2037                                        end
2038                                    else
2039                                        local tv = tables[c]
2040                                        if tv then
2041                                            step.coverage = tv
2042                                        end
2043                                    end
2044                                end
2045                            elseif kind == "gpos_cursive" then
2046                                local c = step.coverage
2047                                if c then
2048                                    for g1, d1 in next, c do
2049                                        local tv = tables[d1]
2050                                        if tv then
2051                                            d1 = tv
2052                                            c[g1] = d1
2053                                        end
2054                                        local f = tables[d1[2]] if f then d1[2] = f end
2055                                        local s = tables[d1[3]] if s then d1[3] = s end
2056                                    end
2057                                end
2058                            elseif kind == "gpos_mark2base" or kind == "gpos_mark2mark" then
2059                                local c = step.baseclasses
2060                                if c then
2061                                    for g1, d1 in next, c do
2062                                        for g2, d2 in next, d1 do
2063                                            local tv = tables[d2]
2064                                            if tv then
2065                                                d1[g2] = tv
2066                                            end
2067                                        end
2068                                    end
2069                                end
2070                                local c = step.coverage
2071                                if c then
2072                                    for g1, d1 in next, c do
2073                                        local tv = tables[d1[2]]
2074                                        if tv then
2075                                            d1[2] = tv
2076                                        end
2077                                    end
2078                                end
2079                            elseif kind == "gpos_mark2ligature" then
2080                                local c = step.baseclasses
2081                                if c then
2082                                    for g1, d1 in next, c do
2083                                        for g2, d2 in next, d1 do
2084                                            local tv = tables[d2] -- new
2085                                            if tv then
2086                                                d2 = tv
2087                                                d1[g2] = d2
2088                                            end
2089                                            for g3, d3 in next, d2 do
2090                                                local tv = tables[d2[g3]]
2091                                                if tv then
2092                                                    d2[g3] = tv
2093                                                end
2094                                            end
2095                                        end
2096                                    end
2097                                end
2098                                local c = step.coverage
2099                                if c then
2100                                    for g1, d1 in next, c do
2101                                        local tv = tables[d1[2]]
2102                                        if tv then
2103                                            d1[2] = tv
2104                                        end
2105                                    end
2106                                end
2107                            end
2108                            local rules = step.rules
2109                            if rules then
2110                                for i=1,#rules do
2111                                    local rule = rules[i]
2112                                    local before = rule.before
2113                                    if before then
2114                                        local tv = tables[before]
2115                                        if tv then
2116                                            rule.before = tv
2117                                            before = tv
2118                                        end
2119                                        for i=1,#before do
2120                                            local tv = tables[before[i]]
2121                                            if tv then
2122                                                before[i] = tv
2123                                            end
2124                                        end
2125                                     -- for i=1,#before do
2126                                     --     local bi = before[i]
2127                                     --     local tv = tables[bi]
2128                                     --     if tv then
2129                                     --         bi = tv
2130                                     --         before[i] = bi
2131                                     --     end
2132                                     --     local ranges = bi.ranges
2133                                     --     if ranges then
2134                                     --         expandranges(bi,ranges)
2135                                     --     end
2136                                     -- end
2137                                    end
2138                                    local after = rule.after
2139                                    if after then
2140                                        local tv = tables[after]
2141                                        if tv then
2142                                            rule.after = tv
2143                                            after = tv
2144                                        end
2145                                        for i=1,#after do
2146                                            local tv = tables[after[i]]
2147                                            if tv then
2148                                                after[i] = tv
2149                                            end
2150                                        end
2151                                     -- for i=1,#after do
2152                                     --     local ai = after[i]
2153                                     --     local tv = tables[ai]
2154                                     --     if tv then
2155                                     --         ai = tv
2156                                     --         after[i] = ai
2157                                     --     end
2158                                     --     local ranges = ai.ranges
2159                                     --     if ranges then
2160                                     --         expandranges(ai,ranges)
2161                                     --     end
2162                                     -- end
2163                                    end
2164                                    local current = rule.current
2165                                    if current then
2166                                        local tv = tables[current]
2167                                        if tv then
2168                                            rule.current = tv
2169                                            current = tv
2170                                        end
2171                                        for i=1,#current do
2172                                            local tv = tables[current[i]]
2173                                            if tv then
2174                                                current[i] = tv
2175                                            end
2176                                        end
2177                                     -- for i=1,#current do
2178                                     --     local ci = current[i]
2179                                     --     local tv = tables[ci]
2180                                     --     if tv then
2181                                     --         ci = tv
2182                                     --         current[i] = ci
2183                                     --     end
2184                                     --     local ranges = ci.ranges
2185                                     --     if ranges then
2186                                     --         expandranges(ci,ranges)
2187                                     --     end
2188                                     -- end
2189                                    end
2190                                 -- local lookups = rule.lookups
2191                                 -- if lookups then
2192                                 --     local tv = tables[lookups]
2193                                 --     if tv then
2194                                 --         rule.lookups = tv
2195                                 --     end
2196                                 -- end
2197                                    local replacements = rule.replacements
2198                                    if replacements then
2199                                        local tv = tables[replacements]
2200                                        if tv then
2201                                            rule.replacements = tv
2202                                        end
2203                                    end
2204                                end
2205                            end
2206                        end
2207                    end
2208                    if order then
2209                        local tv = tables[order]
2210                        if tv then
2211                            sequence.order = tv
2212                        end
2213                    end
2214                    if flags then
2215                        local tv = tables[flags]
2216                        if tv then
2217                            sequence.flags = tv
2218                        end
2219                    end
2220               end
2221            end
2222
2223            if sequences then
2224                unpackthem(sequences)
2225            end
2226
2227            if sublookups then
2228                unpackthem(sublookups)
2229            end
2230
2231            if features then
2232                for k, list in next, features do
2233                    for feature, spec in next, list do
2234                        local tv = tables[spec]
2235                        if tv then
2236                            list[feature] = tv
2237                        end
2238                    end
2239                end
2240            end
2241
2242            if palettes then
2243                for i=1,#palettes do
2244                    local p = palettes[i]
2245                    for j=1,#p do
2246                        local tv = tables[p[j]]
2247                        if tv then
2248                            p[j] = tv
2249                        end
2250                    end
2251                end
2252            end
2253
2254            if variable then
2255
2256                -- todo: segments
2257
2258                local instances = variable.instances
2259                if instances then
2260                    for i=1,#instances do
2261                        local v = instances[i].values
2262                        for j=1,#v do
2263                            local tv = tables[v[j]]
2264                            if tv then
2265                                v[j] = tv
2266                            end
2267                        end
2268                    end
2269                end
2270
2271                local function unpackdeltas(main)
2272                    if main then
2273                        local deltas = main.deltas
2274                        if deltas then
2275                            for i=1,#deltas do
2276                                local di = deltas[i]
2277                                local d  = di.deltas
2278                                local r  = di.regions
2279                                for j=1,#d do
2280                                    local tv = tables[d[j]]
2281                                    if tv then
2282                                        d[j] = tv
2283                                    end
2284                                end
2285                                local tv = di.regions
2286                                if tv then
2287                                    di.regions = tv
2288                                end
2289                            end
2290                        end
2291                        local regions = main.regions
2292                        if regions then
2293                            local tv = tables[regions]
2294                            if tv then
2295                                main.regions = tv
2296                                regions = tv
2297                            end
2298                            for i=1,#regions do
2299                                local r = regions[i]
2300                                for j=1,#r do
2301                                    local tv = tables[r[j]]
2302                                    if tv then
2303                                        r[j] = tv
2304                                    end
2305                                end
2306                            end
2307                        end
2308                    end
2309                end
2310
2311                unpackdeltas(variable.global)
2312                unpackdeltas(variable.horizontal)
2313                unpackdeltas(variable.vertical)
2314                unpackdeltas(variable.metrics)
2315
2316            end
2317
2318            data.tables = nil
2319        end
2320    end
2321end
2322
2323local mt = {
2324    __index = function(t,k) -- maybe set it
2325        if k == "height" then
2326            local ht = t.boundingbox[4]
2327            return ht < 0 and 0 or ht
2328        elseif k == "depth" then
2329            local dp = -t.boundingbox[2]
2330            return dp < 0 and 0 or dp
2331        elseif k == "width" then
2332            return 0
2333        elseif k == "name" then -- or maybe uni*
2334            return forcenotdef and ".notdef"
2335        end
2336    end
2337}
2338
2339local function sameformat(sequence,steps,first,nofsteps,kind)
2340    return true
2341end
2342
2343local function mergesteps_1(lookup,strict)
2344    local steps    = lookup.steps
2345    local nofsteps = lookup.nofsteps
2346    local first    = steps[1]
2347    if strict then
2348        local f = first.format
2349        for i=2,nofsteps do
2350            if steps[i].format ~= f then
2351                if trace_optimizations then
2352                    report_optimizations("not merging %a steps of %a lookup %a, different formats",nofsteps,lookup.type,lookup.name)
2353                end
2354                return 0
2355            end
2356        end
2357    end
2358    if trace_optimizations then
2359        report_optimizations("merging %a steps of %a lookup %a",nofsteps,lookup.type,lookup.name)
2360    end
2361    local target = first.coverage
2362    for i=2,nofsteps do
2363        local c = steps[i].coverage
2364        if c then
2365            for k, v in next, c do
2366                if not target[k] then
2367                    target[k] = v
2368                end
2369            end
2370        end
2371    end
2372    lookup.nofsteps = 1
2373    lookup.merged   = true
2374    lookup.steps    = { first }
2375    return nofsteps - 1
2376end
2377
2378local function mergesteps_2(lookup) -- pairs
2379    -- this can be tricky as we can have a match on a mark with no marks skip flag
2380    -- in which case with multiple steps a hit can prevent a next step while in the
2381    -- merged case we can hit differently (a messy font then anyway)
2382    local steps    = lookup.steps
2383    local nofsteps = lookup.nofsteps
2384    local first    = steps[1]
2385    if strict then
2386        local f = first.format
2387        for i=2,nofsteps do
2388            if steps[i].format ~= f then
2389                if trace_optimizations then
2390                    report_optimizations("not merging %a steps of %a lookup %a, different formats",nofsteps,lookup.type,lookup.name)
2391                end
2392                return 0
2393            end
2394        end
2395    end
2396    if trace_optimizations then
2397        report_optimizations("merging %a steps of %a lookup %a",nofsteps,lookup.type,lookup.name)
2398    end
2399    local target = first.coverage
2400    for i=2,nofsteps do
2401        local c = steps[i].coverage
2402        if c then
2403            for k, v in next, c do
2404                local tk = target[k]
2405                if tk then
2406                    for kk, vv in next, v do
2407                        if tk[kk] == nil then
2408                            tk[kk] = vv
2409                        end
2410                    end
2411                else
2412                    target[k] = v
2413                end
2414            end
2415        end
2416    end
2417    lookup.nofsteps = 1
2418    lookup.merged   = true
2419    lookup.steps    = { first }
2420    return nofsteps - 1
2421end
2422
2423-- we could have a coverage[first][second] = { } already here (because eventually
2424-- we also have something like that after loading)
2425
2426local function mergesteps_3(lookup,strict) -- marks
2427    local steps    = lookup.steps
2428    local nofsteps = lookup.nofsteps
2429    if trace_optimizations then
2430        report_optimizations("merging %a steps of %a lookup %a",nofsteps,lookup.type,lookup.name)
2431    end
2432    -- check first
2433    local coverage = { }
2434    for i=1,nofsteps do
2435        local c = steps[i].coverage
2436        if c then
2437            for k, v in next, c do
2438                local tk = coverage[k] -- { class, { x, y } }
2439                if tk then
2440                    if trace_optimizations then
2441                        report_optimizations("quitting merge due to multiple checks")
2442                    end
2443                    return nofsteps
2444                else
2445                    coverage[k] = v
2446                end
2447            end
2448        end
2449    end
2450    -- merge indeed
2451    local first       = steps[1]
2452    local baseclasses = { } -- let's assume sparse step.baseclasses
2453    for i=1,nofsteps do
2454        local offset = i*10  -- we assume max 10 classes per step
2455        local step   = steps[i]
2456        for k, v in sortedhash(step.baseclasses) do
2457            baseclasses[offset+k] = v
2458        end
2459        for k, v in next, step.coverage do
2460            v[1] = offset + v[1]
2461        end
2462    end
2463    first.baseclasses = baseclasses
2464    first.coverage    = coverage
2465    lookup.nofsteps   = 1
2466    lookup.merged     = true
2467    lookup.steps      = { first }
2468    return nofsteps - 1
2469end
2470
2471local function nested(old,new)
2472    for k, v in next, old do
2473        if k == "ligature" then
2474            if not new.ligature then
2475                new.ligature = v
2476            end
2477        else
2478            local n = new[k]
2479            if n then
2480                nested(v,n)
2481            else
2482                new[k] = v
2483            end
2484        end
2485    end
2486end
2487
2488local function mergesteps_4(lookup) -- ligatures
2489    local steps    = lookup.steps
2490    local nofsteps = lookup.nofsteps
2491    local first    = steps[1]
2492    if trace_optimizations then
2493        report_optimizations("merging %a steps of %a lookup %a",nofsteps,lookup.type,lookup.name)
2494    end
2495    local target = first.coverage
2496    for i=2,nofsteps do
2497        local c = steps[i].coverage
2498        if c then
2499            for k, v in next, c do
2500                local tk = target[k]
2501                if tk then
2502                    nested(v,tk)
2503                else
2504                    target[k] = v
2505                end
2506            end
2507        end
2508    end
2509    lookup.nofsteps = 1
2510    lookup.steps = { first }
2511    return nofsteps - 1
2512end
2513
2514-- so we assume only one cursive entry and exit and even then the first one seems
2515-- to win anyway: no exit or entry quite the lookup match and then we take the
2516-- next step; this means that we can as well merge them
2517
2518local function mergesteps_5(lookup) -- cursive
2519    local steps    = lookup.steps
2520    local nofsteps = lookup.nofsteps
2521    local first    = steps[1]
2522    if trace_optimizations then
2523        report_optimizations("merging %a steps of %a lookup %a",nofsteps,lookup.type,lookup.name)
2524    end
2525    local target = first.coverage
2526    local hash   = nil
2527    for k, v in next, target do
2528        hash = v[1]
2529        break
2530    end
2531    for i=2,nofsteps do
2532        local c = steps[i].coverage
2533        if c then
2534            for k, v in next, c do
2535                local tk = target[k]
2536                if tk then
2537                    if not tk[2] then
2538                        tk[2] = v[2]
2539                    end
2540                    if not tk[3] then
2541                        tk[3] = v[3]
2542                    end
2543                else
2544                    target[k] = v
2545                    v[1] = hash
2546                end
2547            end
2548        end
2549    end
2550    lookup.nofsteps = 1
2551    lookup.merged   = true
2552    lookup.steps    = { first }
2553    return nofsteps - 1
2554end
2555
2556local function checkkerns(lookup)
2557    local steps    = lookup.steps
2558    local nofsteps = lookup.nofsteps
2559    local kerned   = 0
2560    for i=1,nofsteps do
2561        local step = steps[i]
2562        if step.format == "pair" then
2563            local coverage = step.coverage
2564            local kerns    = true
2565            for g1, d1 in next, coverage do
2566                if d1 == true then
2567                    -- all zero
2568                elseif not d1 then
2569                    -- null
2570                elseif d1[1] ~= 0 or d1[2] ~= 0 or d1[4] ~= 0 then
2571                    kerns = false
2572                    break
2573                end
2574            end
2575            if kerns then
2576                if trace_optimizations then
2577                    report_optimizations("turning pairs of step %a of %a lookup %a into kerns",i,lookup.type,lookup.name)
2578                end
2579                local c = { }
2580                for g1, d1 in next, coverage do
2581                    if d1 and d1 ~= true then
2582                        c[g1] = d1[3]
2583                    end
2584                end
2585                step.coverage = c
2586                step.format = "move"
2587                kerned = kerned + 1
2588            end
2589        end
2590    end
2591    return kerned
2592end
2593
2594-- There are several options to optimize but we have this somewhat fuzzy aspect of
2595-- advancing (depending on the second of a pair) so we need to retain that information.
2596--
2597-- We can have:
2598--
2599--     true, nil|false
2600--
2601-- which effectively means: nothing to be done and advance to next (so not next of
2602-- next) and because coverage should be not overlapping we can wipe these. However,
2603-- checking for (true,nil) (false,nil) and omitting them doesn't gain much.
2604
2605-- Because we pack we cannot mix tables and numbers so we can only turn a whole set in
2606-- format kern instead of pair.
2607
2608local strip_pairs         = true
2609
2610local compact_pairs       = true
2611local compact_singles     = true
2612
2613local merge_pairs         = true
2614local merge_singles       = true
2615local merge_substitutions = true
2616local merge_alternates    = true
2617local merge_multiples     = true
2618local merge_ligatures     = true
2619local merge_cursives      = true
2620local merge_marks         = true
2621
2622directives.register("otf.strip.pairs",         function(v) strip_pairs     = v end)
2623
2624directives.register("otf.compact.pairs",       function(v) compact_pairs   = v end)
2625directives.register("otf.compact.singles",     function(v) compact_singles = v end)
2626
2627directives.register("otf.merge.pairs",         function(v) merge_pairs         = v end)
2628directives.register("otf.merge.singles",       function(v) merge_singles       = v end)
2629directives.register("otf.merge.substitutions", function(v) merge_substitutions = v end)
2630directives.register("otf.merge.alternates",    function(v) merge_alternates    = v end)
2631directives.register("otf.merge.multiples",     function(v) merge_multiples     = v end)
2632directives.register("otf.merge.ligatures",     function(v) merge_ligatures     = v end)
2633directives.register("otf.merge.cursives",      function(v) merge_cursives      = v end)
2634directives.register("otf.merge.marks",         function(v) merge_marks         = v end)
2635
2636local function checkpairs(lookup)
2637    local steps    = lookup.steps
2638    local nofsteps = lookup.nofsteps
2639    local kerned   = 0
2640
2641    local function onlykerns(step)
2642        local coverage = step.coverage
2643        for g1, d1 in next, coverage do
2644            for g2, d2 in next, d1 do
2645                if d2[2] then
2646                    --- true or { a, b, c, d }
2647                    return false
2648                else
2649                    local v = d2[1]
2650                    if v == true then
2651                        -- all zero
2652                    elseif v and (v[1] ~= 0 or v[2] ~= 0 or v[4] ~= 0) then
2653                        -- complex kerns
2654                        return false
2655                    end
2656                end
2657            end
2658        end
2659        return coverage
2660    end
2661
2662    for i=1,nofsteps do
2663        local step = steps[i]
2664        if step.format == "pair" then
2665            local coverage = onlykerns(step)
2666            if coverage then
2667                if trace_optimizations then
2668                    report_optimizations("turning pairs of step %a of %a lookup %a into kerns",i,lookup.type,lookup.name)
2669                end
2670                for g1, d1 in next, coverage do
2671                    local d = { }
2672                    for g2, d2 in next, d1 do
2673                        local v = d2[1]
2674                        if v == true then
2675                            -- ignore    -- d1[g2] = nil
2676                        elseif v then
2677                            d[g2] = v[3] -- d1[g2] = v[3]
2678                        end
2679                    end
2680                    coverage[g1] = d
2681                end
2682                step.format = "move"
2683                kerned = kerned + 1
2684            end
2685        end
2686    end
2687    return kerned
2688end
2689
2690local function strippairs(lookup)
2691    local steps    = lookup.steps
2692    local nofsteps = lookup.nofsteps
2693    local stripped = 0
2694
2695    for i=1,nofsteps do
2696        local step = steps[i]
2697        if step.format == "pair" then
2698            local coverage = step.coverage
2699            for g1, d1 in next, coverage do
2700                for g2, d2 in next, d1 do
2701                    if d2[2] then
2702                        --- true or { a, b, c, d }
2703                 -- else
2704                 --     local v = d2[1]
2705                 --     if v == true then
2706                 --         d1[g2] = nil
2707                 --         stripped = stripped + 1
2708                 --     elseif v and (v[1] == 0 and v[2] == 0 and v[4] == 0) then -- vkrn can have v[3] ~= 0
2709                 --         d1[g2] = nil
2710                 --         stripped = stripped + 1
2711                 --     end
2712                 -- end
2713                    elseif d2[1] == true then
2714                        d1[g2] = nil
2715                        stripped = stripped + 1
2716                    end
2717                end
2718            end
2719        end
2720    end
2721    return stripped
2722end
2723
2724function readers.compact(data)
2725    if not data or data.compacted then
2726        return
2727    else
2728        data.compacted = true
2729    end
2730    local resources = data.resources
2731    local stripped  = 0
2732    local merged    = 0
2733    local kerned    = 0
2734    local allsteps  = 0
2735    local function compact(what)
2736        local lookups = resources[what]
2737        if lookups then
2738            for i=1,#lookups do
2739                local lookup   = lookups[i]
2740                local nofsteps = lookup.nofsteps
2741                local kind     = lookup.type
2742                allsteps = allsteps + nofsteps
2743                if nofsteps > 1 then
2744                    local merg = merged
2745                    if kind == "gsub_single" then
2746                        if merge_substitutions then
2747                            merged = merged + mergesteps_1(lookup)
2748                        end
2749                    elseif kind == "gsub_alternate" then
2750                        if merge_alternates then
2751                            merged = merged + mergesteps_1(lookup)
2752                        end
2753                    elseif kind == "gsub_multiple" then
2754                        if merge_multiples then
2755                            merged = merged + mergesteps_1(lookup)
2756                        end
2757                    elseif kind == "gsub_ligature" then
2758                        if merge_ligatures then
2759                            merged = merged + mergesteps_4(lookup)
2760                        end
2761                    elseif kind == "gpos_single" then
2762                        -- maybe also strip zeros here
2763                        if merge_singles then
2764                            merged = merged + mergesteps_1(lookup,true)
2765                        end
2766                        if compact_singles then
2767                            kerned = kerned + checkkerns(lookup)
2768                        end
2769                    elseif kind == "gpos_pair" then
2770                        if strip_pairs then
2771                            stripped = stripped + strippairs(lookup) -- noto cjk from 24M -> 8 M
2772                        end
2773                        if merge_pairs then
2774                            merged = merged + mergesteps_2(lookup)
2775                        end
2776                        if compact_pairs then
2777                            kerned = kerned + checkpairs(lookup)
2778                        end
2779                    elseif kind == "gpos_cursive" then
2780                        if merge_cursives then
2781                            merged = merged + mergesteps_5(lookup)
2782                        end
2783                    elseif kind == "gpos_mark2mark" or kind == "gpos_mark2base" or kind == "gpos_mark2ligature" then
2784                        if merge_marks then
2785                            merged = merged + mergesteps_3(lookup)
2786                        end
2787                    end
2788                    if merg ~= merged then
2789                        lookup.merged = true
2790                    end
2791                elseif nofsteps == 1 then
2792                    local kern = kerned
2793                    if kind == "gpos_single" then
2794                        if compact_singles then
2795                            kerned = kerned + checkkerns(lookup)
2796                        end
2797                    elseif kind == "gpos_pair" then
2798                        if compact_pairs then
2799                            kerned = kerned + checkpairs(lookup)
2800                        end
2801                    end
2802                    if kern ~= kerned then
2803                     -- lookup.kerned = true
2804                    end
2805                end
2806            end
2807        elseif trace_optimizations then
2808            report_optimizations("no lookups in %a",what)
2809        end
2810    end
2811    compact("sequences")
2812    compact("sublookups")
2813    if trace_optimizations then
2814        if stripped > 0 then
2815            report_optimizations("%i zero positions stripped before merging",stripped)
2816        end
2817        if merged > 0 then
2818            report_optimizations("%i steps of %i removed due to merging",merged,allsteps)
2819        end
2820        if kerned > 0 then
2821            report_optimizations("%i steps of %i steps turned from pairs into kerns",kerned,allsteps)
2822        end
2823    end
2824end
2825
2826if CONTEXTLMTXMODE and CONTEXTLMTXMODE > 0 then
2827
2828    local done = 0
2829
2830    local function condense_1(k,v,t)
2831        if type(v) == "table" then
2832            local u = false
2833            local l = false
2834            for k, v in next, v do
2835                if k == "ligature" then
2836                    l = v
2837                    if u then
2838                        break
2839                    end
2840                elseif u then
2841                    break
2842                else
2843                    u = true
2844                end
2845            end
2846            if l and not u then
2847                t[k] = l
2848                done = done + 1
2849            end
2850            if u then
2851                for k, vv in next, v do
2852                    if k ~= "ligature" then
2853                        condense_1(k,vv,v)
2854                    end
2855                end
2856            end
2857        end
2858    end
2859
2860    local function condensesteps_1(lookup)
2861        done = 0
2862        if lookup.type == "gsub_ligature" then
2863            local steps = lookup.steps
2864            if steps then
2865                for i=1,#steps do
2866                    local step     = steps[i]
2867                    local coverage = step.coverage
2868                    if coverage then
2869                        for k, v in next, coverage do
2870                            if condense_1(k,v,coverage) then
2871                                coverage[k] = v.ligature
2872                                done = done + 1
2873                            end
2874                        end
2875                    end
2876                end
2877            end
2878        end
2879        return done
2880    end
2881
2882    function readers.condense(data)
2883        if not data or data.condensed then
2884            return
2885        else
2886            data.condensed = true
2887        end
2888        local resources = data.resources
2889        local condensed = 0
2890        local function condense(what)
2891            local lookups = resources[what]
2892            if lookups then
2893                for i=1,#lookups do
2894                    condensed = condensed + condensesteps_1(lookups[i])
2895                end
2896            elseif trace_optimizations then
2897                report_optimizations("no lookups in %a",what)
2898            end
2899        end
2900        condense("sequences")
2901        condense("sublookups")
2902        if trace_optimizations then
2903            if condensed > 0 then
2904                report_optimizations("%i ligatures condensed",condensed)
2905            end
2906        end
2907    end
2908
2909end
2910
2911local function mergesteps(t,k)
2912    if k == "merged" then
2913        local merged = { }
2914        for i=1,#t do
2915            local step     = t[i]
2916            local coverage = step.coverage
2917            for k in next, coverage do
2918                local m = merged[k]
2919                if m then
2920                    m[2] = i
2921                 -- m[#m+1] = step
2922                else
2923                    merged[k] = { i, i }
2924                 -- merged[k] = { step }
2925                end
2926            end
2927        end
2928        t.merged = merged
2929        return merged
2930    end
2931end
2932
2933local function checkmerge(sequence)
2934    local steps = sequence.steps
2935    if steps then
2936        setmetatableindex(steps,mergesteps)
2937    end
2938end
2939
2940local function checkflags(sequence,resources)
2941    if not sequence.skiphash then
2942        local flags = sequence.flags
2943        if flags then
2944            local skipmark     = flags[1]
2945            local skipligature = flags[2]
2946            local skipbase     = flags[3]
2947            local markclass    = sequence.markclass
2948            local skipsome     = skipmark or skipligature or skipbase or markclass or false
2949            if skipsome then
2950                sequence.skiphash = setmetatableindex(function(t,k)
2951                    local c = resources.classes[k] -- delayed table
2952                    local v = c == skipmark
2953                           or (markclass and c == "mark" and not markclass[k])
2954                           or c == skipligature
2955                           or c == skipbase
2956                           or false
2957                    t[k] = v
2958                    return v
2959                end)
2960            else
2961                sequence.skiphash = false
2962            end
2963        else
2964            sequence.skiphash = false
2965        end
2966    end
2967end
2968
2969local function checksteps(sequence)
2970    local steps = sequence.steps
2971    if steps then
2972        for i=1,#steps do
2973            steps[i].index = i
2974        end
2975    end
2976end
2977
2978if fonts.helpers then
2979    fonts.helpers.checkmerge = checkmerge
2980    fonts.helpers.checkflags = checkflags
2981    fonts.helpers.checksteps = checksteps -- has to happen last
2982end
2983
2984function readers.expand(data)
2985    if not data or data.expanded then
2986        return
2987    else
2988        data.expanded = true
2989    end
2990    local resources    = data.resources
2991    local sublookups   = resources.sublookups
2992    local sequences    = resources.sequences -- were one level up
2993    local markclasses  = resources.markclasses
2994    local descriptions = data.descriptions
2995    if descriptions then
2996        local defaultwidth  = resources.defaultwidth  or 0
2997        local defaultheight = resources.defaultheight or 0
2998        local defaultdepth  = resources.defaultdepth  or 0
2999        local basename      = trace_markwidth and file.basename(resources.filename)
3000        for u, d in next, descriptions do
3001            local bb = d.boundingbox
3002            local wd = d.width
3003            if not wd then
3004                -- or bb?
3005                d.width = defaultwidth
3006            elseif trace_markwidth and wd ~= 0 and d.class == "mark" then
3007                report_markwidth("mark %a with width %b found in %a",d.name or "<noname>",wd,basename)
3008            end
3009            if bb then
3010                local ht =  bb[4]
3011                local dp = -bb[2]
3012                if ht == 0 or ht < 0 then
3013                    -- not set
3014                else
3015                    d.height = ht
3016                end
3017                if dp == 0 or dp < 0 then
3018                    -- not set
3019                else
3020                    d.depth  = dp
3021                end
3022            end
3023        end
3024    end
3025
3026    -- using a merged combined hash as first test saves some 30% on ebgaramond and
3027    -- about 15% on arabtype .. then moving the a test also saves a bit (even when
3028    -- often a is not set at all so that one is a bit debatable
3029
3030    local function expandlookups(sequences)
3031        if sequences then
3032            -- we also need to do sublookups
3033            for i=1,#sequences do
3034                local sequence = sequences[i]
3035                local steps    = sequence.steps
3036                if steps then
3037                    local nofsteps = sequence.nofsteps
3038
3039                    local kind = sequence.type
3040                    local markclass = sequence.markclass
3041                    if markclass then
3042                        if not markclasses then
3043                            report_warning("missing markclasses")
3044                            sequence.markclass = false
3045                        else
3046                            sequence.markclass = markclasses[markclass]
3047                        end
3048                    end
3049
3050                    for i=1,nofsteps do
3051                        local step = steps[i]
3052                        local baseclasses = step.baseclasses
3053                        if baseclasses then
3054                            local coverage = step.coverage
3055                            for k, v in next, coverage do
3056                                v[1] = baseclasses[v[1]] -- slot 1 is a placeholder
3057                            end
3058                        elseif kind == "gpos_cursive" then
3059                            local coverage = step.coverage
3060                            for k, v in next, coverage do
3061                                v[1] = coverage -- slot 1 is a placeholder
3062                            end
3063                        end
3064                        local rules = step.rules
3065                        if rules then
3066                            local rulehash   = { n = 0 } -- is contexts in font-ots
3067                            local rulesize   = 0
3068                            local coverage   = { }
3069                            local lookuptype = sequence.type
3070                            local nofrules   = #rules
3071                            step.coverage    = coverage -- combined hits
3072                            for currentrule=1,nofrules do
3073                                local rule         = rules[currentrule]
3074                                local current      = rule.current
3075                                local before       = rule.before
3076                                local after        = rule.after
3077                                local replacements = rule.replacements or false
3078                                local sequence     = { }
3079                                local nofsequences = 0
3080                                if before then
3081                                    for n=1,#before do
3082                                        nofsequences = nofsequences + 1
3083                                        sequence[nofsequences] = before[n]
3084                                    end
3085                                end
3086                                local start = nofsequences + 1
3087                                for n=1,#current do
3088                                    nofsequences = nofsequences + 1
3089                                    sequence[nofsequences] = current[n]
3090                                end
3091                                local stop = nofsequences
3092                                if after then
3093                                    for n=1,#after do
3094                                        nofsequences = nofsequences + 1
3095                                        sequence[nofsequences] = after[n]
3096                                    end
3097                                end
3098                                local lookups = rule.lookups or false
3099                                local subtype = nil
3100                                if lookups then
3101                                    for i=1,#lookups do
3102                                        local lookups = lookups[i]
3103                                        if lookups then
3104                                            for k, v in next, lookups do -- actually this one is indexed
3105                                                local lookup = sublookups[v]
3106                                                if lookup then
3107                                                    lookups[k] = lookup
3108                                                    if not subtype then
3109                                                        subtype = lookup.type
3110                                                    end
3111                                                else
3112                                                    -- already expanded
3113                                                end
3114                                            end
3115                                        end
3116                                    end
3117                                end
3118                                if sequence[1] then -- we merge coverage into one
3119                                    sequence.n = #sequence -- tiny speedup
3120                                    local ruledata = {
3121                                        currentrule,  -- 1 -- original rule number, only use this for tracing!
3122                                        lookuptype,   -- 2
3123                                        sequence,     -- 3
3124                                        start,        -- 4
3125                                        stop,         -- 5
3126                                        lookups,      -- 6 (6/7 also signal of what to do)
3127                                        replacements, -- 7
3128                                        subtype,      -- 8
3129                                    }
3130                                    --
3131                                    -- possible optimization: per [unic] a rulehash, but beware:
3132                                    -- contexts have unique coverage and chains can have multiple
3133                                    -- hits (rules) per coverage entry
3134                                    --
3135                                    -- so: we can combine multiple steps as well as multiple rules
3136                                    -- but that takes careful checking, in which case we can go the
3137                                    -- step list approach and turn contexts into steps .. in fact,
3138                                    -- if we turn multiple contexts into steps we're already ok as
3139                                    -- steps gets a coverage hash by metatable
3140                                    --
3141                                    rulesize = rulesize + 1
3142                                    rulehash[rulesize] = ruledata
3143                                    rulehash.n = rulesize -- tiny speedup
3144                                    --
3145                                    if true then -- nofrules > 1
3146
3147                                        for unic in next, sequence[start] do
3148                                            local cu = coverage[unic]
3149                                            if cu then
3150                                                local n = #cu+1
3151                                                cu[n] = ruledata
3152                                                cu.n = n
3153                                            else
3154                                                coverage[unic] = { ruledata, n = 1 }
3155                                            end
3156                                        end
3157
3158                                    else
3159
3160                                        for unic in next, sequence[start] do
3161                                            local cu = coverage[unic]
3162                                            if cu then
3163                                                -- we can have a contextchains with many matches which we
3164                                                -- can actually optimize
3165                                            else
3166                                                coverage[unic] = rulehash
3167                                            end
3168                                        end
3169
3170                                    end
3171                                end
3172                            end
3173                        end
3174                    end
3175
3176                    checkmerge(sequence)
3177                    checkflags(sequence,resources)
3178                    checksteps(sequence)
3179
3180                end
3181            end
3182        end
3183    end
3184
3185    expandlookups(sequences)
3186    expandlookups(sublookups)
3187end
3188