font-chk.lmt /size: 18 Kb    last modification: 2021-10-28 13:51
1if not modules then modules = { } end modules ['font-chk'] = {
2    version   = 1.001,
3    comment   = "companion to font-ini.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- This is kind of old but it makes no real sense to upgrade it to for instance
10-- using delayed type 3 fonts in order to be lean and mean and cut'n'paste
11-- compliant. When this kicks one needs to fix the choice of fonts anyway! So,
12-- instead we just keep the method we use but slightly adapted to the backend
13-- of lmtx.
14
15local type, next = type, next
16local find, lower, gmatch = string.find, string.lower, string.gmatch
17local floor = math.floor
18
19local context              = context
20
21local formatters           = string.formatters
22local bpfactor             = number.dimenfactors.bp
23local fastcopy             = table.fastcopy
24local sortedkeys           = table.sortedkeys
25local sortedhash           = table.sortedhash
26local contains             = table.contains
27
28local report               = logs.reporter("fonts")
29local report_checking      = logs.reporter("fonts","checking")
30
31local allocate             = utilities.storage.allocate
32
33local getmacro             = tokens.getters.macro
34
35local fonts                = fonts
36
37fonts.checkers             = fonts.checkers or { }
38local checkers             = fonts.checkers
39
40local fonthashes           = fonts.hashes
41local fontdata             = fonthashes.identifiers
42local fontcharacters       = fonthashes.characters
43
44local currentfont          = font.current
45
46local definers             = fonts.definers
47local helpers              = fonts.helpers
48
49local addprivate           = helpers.addprivate
50local hasprivate           = helpers.hasprivate
51local getprivateslot       = helpers.getprivateslot
52local getprivatecharornode = helpers.getprivatecharornode
53
54local otffeatures          = fonts.constructors.features.otf
55local afmfeatures          = fonts.constructors.features.afm
56
57local registerotffeature   = otffeatures.register
58local registerafmfeature   = afmfeatures.register
59
60local is_character         = characters.is_character
61local chardata             = characters.data
62
63local tasks                = nodes.tasks
64local enableaction         = tasks.enableaction
65local disableaction        = tasks.disableaction
66
67local implement            = interfaces.implement
68
69local glyph_code           = nodes.nodecodes.glyph
70
71local hpack_node           = nodes.hpack
72
73local nuts                 = nodes.nuts
74local tonut                = nuts.tonut
75
76local isglyph              = nuts.isglyph
77local setchar              = nuts.setchar
78
79local nextglyph            = nuts.traversers.glyph
80
81local remove_node          = nuts.remove
82local insertnodeafter      = nuts.insertafter
83local insertnodebefore     = nuts.insertbefore
84local copy_node            = nuts.copy
85
86local actions = false
87
88-- to tfmdata.properties ?
89
90local function onetimemessage(font,char,message) -- char == false returns table
91    local tfmdata = fontdata[font]
92    local shared  = tfmdata.shared
93    if not shared then
94        shared = { }
95        tfmdata.shared = shared
96    end
97    local messages = shared.messages
98    if not messages then
99        messages = { }
100        shared.messages = messages
101    end
102    local category = messages[message]
103    if not category then
104        category = { }
105        messages[message] = category
106    end
107    if char == false then
108        return sortedkeys(category), category
109    end
110    local cc = category[char]
111    if not cc then
112        report_checking("char %C in font %a with id %a: %s",char,tfmdata.properties.fullname,font,message)
113        category[char] = 1
114    else
115        category[char] = cc + 1
116    end
117end
118
119fonts.loggers.onetimemessage = onetimemessage
120
121local fakes = {
122    MissingLowercase   = { width  = .45, height = .55, depth  =  .20 },
123    MissingUppercase   = { width  = .65, height = .70, depth  =  .25 },
124    MissingMark        = { width  = .15, height = .70, depth  = -.50 },
125    MissingPunctuation = { width  = .15, height = .55, depth  =  .20 },
126    MissingUnknown     = { width  = .45, height = .20, depth  = 0    },
127}
128
129local mapping = allocate {
130    lu = { "MissingUppercase",    "darkred"     },
131    ll = { "MissingLowercase",    "darkred"     },
132    lt = { "MissingUppercase",    "darkred"     },
133    lm = { "MissingLowercase",    "darkred"     },
134    lo = { "MissingLowercase",    "darkred"     },
135    mn = { "MissingMark",         "darkgreen"   },
136    mc = { "MissingMark",         "darkgreen"   },
137    me = { "MissingMark",         "darkgreen"   },
138    nd = { "MissingLowercase",    "darkblue"    },
139    nl = { "MissingLowercase",    "darkblue"    },
140    no = { "MissingLowercase",    "darkblue"    },
141    pc = { "MissingPunctuation",  "darkcyan"    },
142    pd = { "MissingPunctuation",  "darkcyan"    },
143    ps = { "MissingPunctuation",  "darkcyan"    },
144    pe = { "MissingPunctuation",  "darkcyan"    },
145    pi = { "MissingPunctuation",  "darkcyan"    },
146    pf = { "MissingPunctuation",  "darkcyan"    },
147    po = { "MissingPunctuation",  "darkcyan"    },
148    sm = { "MissingLowercase",    "darkmagenta" },
149    sc = { "MissingLowercase",    "darkyellow"  },
150    sk = { "MissingLowercase",    "darkyellow"  },
151    so = { "MissingLowercase",    "darkyellow"  },
152}
153
154table.setmetatableindex(mapping, { "MissingUnknown", "darkgray" })
155
156checkers.mapping = mapping
157
158-- We provide access by (private) name for tracing purposes. We also need to make
159-- sure the dimensions are known at the lua and tex end. For previous variants see
160-- the mkiv files or older lmtx files. I decided to just drop the old stuff here.
161
162function checkers.placeholder(font,char,category)
163    local category = category or chardata[char].category or "lu" -- todo: unknown
164    local fakedata = mapping[category] or mapping.lu
165    local tfmdata  = fontdata[font]
166    local units    = tfmdata.parameters.units or 1000
167    local slant    = (tfmdata.parameters.slant or 0)/65536
168    local scale    = units/1000
169    local rawdata  = tfmdata.shared and tfmdata.shared.rawdata
170    local weight   = (rawdata and rawdata.metadata and rawdata.metadata.pfmweight or 400)/400
171--     if slant then
172--         slant = 0.2
173--     end
174    local specification = {
175        code      = "MissingGlyph",
176        scale     = scale,
177        slant     = slant,
178        weight    = weight,
179        namespace = font,
180        shapes    = { { shape = fakedata[1], color = fakedata[2] } },
181    }
182    fonts.helpers.setmetaglyphs("missing", font, char, specification)
183end
184
185function checkers.missing(head)
186    local lastfont       = nil
187    local characters     = nil
188    local found          = nil
189    local addplaceholder = checkers.placeholder -- so we can overload
190    if actions.replace or actions.decompose then
191        for n, char, font in nextglyph, head do
192            if font ~= lastfont then
193                lastfont   = font
194                characters = fontcharacters[font]
195            end
196            if font > 0 and not characters[char] and is_character[chardata[char].category or "unknown"] then
197                if actions.decompose then
198                    local c = chardata[char]
199                    if c then
200                        local s = c.specials
201                        if s and (s[1] == "char" or s[1] == "with") then -- with added
202                            local l = #s
203                            if l > 2 then
204                                -- check first
205                                local okay = true
206                                for i=2,l do
207                                    if not characters[s[i]] then
208                                        okay = false
209                                        break
210                                    end
211                                end
212                                if okay then
213                                    -- we work backward
214                                    local o = n -- we're not supposed to change n (lua 5.4+)
215                                    onetimemessage(font,char,"missing (decomposed)")
216                                    setchar(n,s[l])
217                                    for i=l-1,2,-1 do
218                                        head, o = insertnodebefore(head,o,copy_node(n))
219                                        setchar(o,s[i])
220                                    end
221                                    goto DONE
222                                end
223                            end
224                        end
225                    end
226                end
227                if actions.replace then
228                    onetimemessage(font,char,"missing (replaced)")
229                    local f, c = addplaceholder(font,char)
230                    if f and c then
231                        setchar(n, c, f)
232                    end
233                    goto DONE
234                end
235                if actions.remove then
236                    onetimemessage(font,char,"missing (deleted)")
237                    if not found then
238                        found = { n }
239                    else
240                        found[#found+1] = n
241                    end
242                    goto DONE
243                end
244                onetimemessage(font,char,"missing (checked)")
245              ::DONE::
246            end
247        end
248    end
249    if found then
250        for i=1,#found do
251            head = remove_node(head,found[i],true)
252        end
253    end
254    return head
255end
256
257local relevant = {
258    "missing (decomposed)",
259    "missing (replaced)",
260    "missing (deleted)",
261    "missing (checked)",
262    "missing",
263}
264
265local function getmissing(id)
266    if id then
267        local list = getmissing(currentfont())
268        if list then
269            local _, list = next(getmissing(currentfont()))
270            return list
271        else
272            return { }
273        end
274    else
275        local t = { }
276        for id, d in next, fontdata do
277            local shared   = d.shared
278            local messages = shared and shared.messages
279            if messages then
280                local filename = d.properties.filename
281                if not filename then
282                    filename = tostring(d)
283                end
284                local tf = t[filename] or { }
285                for i=1,#relevant do
286                    local tm = messages[relevant[i]]
287                    if tm then
288                        for k, v in next, tm do
289                            tf[k] = (tf[k] or 0) + v
290                        end
291                    end
292                end
293                if next(tf) then
294                    t[filename] = tf
295                end
296            end
297        end
298        local l = { }
299        for k, v in next, t do
300            l[k] = sortedkeys(v)
301        end
302        return l, t
303    end
304end
305
306checkers.getmissing = getmissing
307
308do
309
310    local reported = true
311
312    callback.register("glyph_not_found",function(font,char)
313        if font > 0 then
314            if char > 0 then
315                onetimemessage(font,char,"missing")
316            else
317                -- we have a special case
318            end
319        elseif not reported then
320            report("nullfont is used, maybe no bodyfont is defined")
321            reported = true
322        end
323    end)
324
325    local loaded = false
326
327    trackers.register("fonts.missing", function(v)
328        if v then
329            enableaction("processors","fonts.checkers.missing")
330            if v == true then
331                actions = { check = true }
332            else
333                actions = utilities.parsers.settings_to_set(v)
334                if not loaded and actions.replace then
335                    metapost.simple("simplefun",'loadfile("mp-miss.mpxl");')
336                    loaded = true
337                end
338            end
339        else
340            disableaction("processors","fonts.checkers.missing")
341            actions = false
342        end
343    end)
344
345    logs.registerfinalactions(function()
346        local collected, details = getmissing()
347        if next(collected) then
348            for filename, list in sortedhash(details) do
349                logs.startfilelogging(report,"missing characters",filename)
350                for u, v in sortedhash(list) do
351                    report("%4i  %U  %c  %s",v,u,u,chardata[u].description)
352                end
353                logs.stopfilelogging()
354            end
355            if logs.loggingerrors() then
356                for filename, list in sortedhash(details) do
357                    logs.starterrorlogging(report,"missing characters",filename)
358                    for u, v in sortedhash(list) do
359                        report("%4i  %U  %c  %s",v,u,u,chardata[u].description)
360                    end
361                    logs.stoperrorlogging()
362                end
363            end
364        end
365    end)
366
367end
368
369-- for the moment here
370
371local function expandglyph(characters,index,done)
372    done = done or { }
373    if not done[index] then
374        local data = characters[index]
375        if data then
376            done[index] = true
377            local d = fastcopy(data)
378            local n = d.next
379            if n then
380                d.next = expandglyph(characters,n,done)
381            end
382            local h = d.horiz_variants
383            if h then
384                for i=1,#h do
385                    h[i].glyph = expandglyph(characters,h[i].glyph,done)
386                end
387            end
388            local v = d.vert_variants
389            if v then
390                for i=1,#v do
391                    v[i].glyph = expandglyph(characters,v[i].glyph,done)
392                end
393            end
394            return d
395        end
396    end
397end
398
399helpers.expandglyph = expandglyph
400
401-- should not be needed as we add .notdef in the engine
402
403local dummyzero = {
404 -- width    = 0,
405 -- height   = 0,
406 -- depth    = 0,
407    commands = { { "special", "" } },
408}
409
410local function adddummysymbols(tfmdata)
411    local characters = tfmdata.characters
412    if not characters[0] then
413        characters[0] = dummyzero
414    end
415 -- if not characters[1] then
416 --     characters[1] = dummyzero -- test only
417 -- end
418end
419
420local dummies_specification = {
421    name        = "dummies",
422    description = "dummy symbols",
423    default     = true,
424    manipulators = {
425        base = adddummysymbols,
426        node = adddummysymbols,
427    }
428}
429
430registerotffeature(dummies_specification)
431registerafmfeature(dummies_specification)
432
433--
434
435local function addvisualspace(tfmdata)
436    local spacechar = tfmdata.characters[32]
437    if spacechar and not spacechar.commands then
438        local w = spacechar.width
439        local h = tfmdata.parameters.xheight
440     -- local h = tfmdata.parameters.xheight / 4 -- could be "visualspace=large" or so
441        local c = {
442            width    = w,
443            commands = { { "rule", h, w } },
444         -- commands = { { "line", w, 5*h, h } },
445        }
446        local u = addprivate(tfmdata, "visualspace", c)
447    end
448end
449
450local visualspace_specification = {
451    name        = "visualspace",
452    description = "visual space",
453    default     = true,
454    manipulators = {
455        base = addvisualspace,
456        node = addvisualspace,
457    }
458}
459
460registerotffeature(visualspace_specification)
461registerafmfeature(visualspace_specification)
462
463do
464
465
466    local reference = 88 -- string.byte("X")
467    local mapping   = { ss = "sans", rm = "serif", tt = "mono" }
468    local order     = {      "sans",      "serif",      "mono" }
469    local fallbacks = {       sans = { },  serif = { },  mono = { } }
470
471    local function locate(fallbacks,n,f,c)
472        for i=1,#fallbacks do
473            local id = fallbacks[i]
474            if type(id) == "string" then
475                local fid = definers.define { name = id }
476                report("using fallback font %!font:name! (id: %i)",fid,fid)
477                fallbacks[i] = fid
478                id = fid
479            end
480            if type(id) == "number" then
481                local cid = fontcharacters[id]
482                if cid[c] then
483                    local fc = fontcharacters[f]
484                    local sc = (fc[reference].height / cid[reference].height) * (n.scale or 1000)
485                    report("character %C in font %!font:name! (id: %i) is taken from fallback font %!font:name! (id: %i)",c,f,f,id,id)
486                    return { id, sc }
487                end
488            end
489        end
490        return false
491    end
492
493    local cache = table.setmetatableindex("table")
494
495    callback.register("missing_character", function(n,f,c)
496        local cached = cache[f]
497        local found  = cached[c]
498        if found == nil then
499            -- we can use fonts.helpers.name(f) but we need the monospace flag anyway so:
500            local metadata = fontdata[f].shared
501            if metadata then
502                metadata = metadata.rawdata
503                if metadata then
504                    metadata = metadata.metadata
505                    if metadata then
506                        if metadata.monospaced then
507                            found = locate(fallbacks.mono,n,f,c)
508                            if found then
509                                cached[c] = found
510                                goto done
511                            end
512                        end
513                        local fn = lower(metadata.fullname)
514                        for i=1,3 do
515                            local o = order[i]
516                            if find(fn,o) then
517                                found = locate(fallbacks[o],n,f,c)
518                                if found then
519                                    cached[c] = found
520                                    goto done
521                                end
522                            end
523                        end
524                    end
525                end
526            end
527            found = locate(fallbacks[mapping[getmacro("fontstyle")] or "mono"],n,f,c)
528            if found then
529                cached[c] = found
530                goto done
531            end
532        end
533      ::done::
534        if found then
535            n.font  = found[1]
536            n.scale = found[2]
537        end
538    end)
539
540    function definers.registerfallbackfont(style,list)
541        local l = fallbacks[style]
542        if l then
543            for s in gmatch(list,"[^, ]+") do
544                if not contains(l,s) then
545                    l[#l+1] = s
546                end
547            end
548        end
549    end
550
551    implement {
552        name      = "registerfallbackfont",
553        public    = true,
554        protected = true,
555        arguments = { "optional", "optional" },
556        actions   = definers.registerfallbackfont,
557    }
558
559end
560