font-chk.lua /size: 15 Kb    last modification: 2021-10-28 13:50
1if not modules then modules = { } end modules ['font-chk'] = {
2    version   = 1.001,
3    comment   = "companion to font-ini.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- possible optimization: delayed initialization of vectors
10-- move to the nodes namespace
11
12-- This is old code and I'll make a nicer one for lmtx some day.
13
14local next = next
15local floor = math.floor
16
17local context              = context
18
19local formatters           = string.formatters
20local bpfactor             = number.dimenfactors.bp
21local fastcopy             = table.fastcopy
22local sortedkeys           = table.sortedkeys
23local sortedhash           = table.sortedhash
24
25local report               = logs.reporter("fonts")
26local report_checking      = logs.reporter("fonts","checking")
27
28local allocate             = utilities.storage.allocate
29
30local fonts                = fonts
31
32fonts.checkers             = fonts.checkers or { }
33local checkers             = fonts.checkers
34
35local fonthashes           = fonts.hashes
36local fontdata             = fonthashes.identifiers
37local fontcharacters       = fonthashes.characters
38
39local currentfont          = font.current
40local addcharacters        = font.addcharacters
41
42local helpers              = fonts.helpers
43
44local addprivate           = helpers.addprivate
45local hasprivate           = helpers.hasprivate
46local getprivateslot       = helpers.getprivateslot
47local getprivatecharornode = helpers.getprivatecharornode
48
49local otffeatures          = fonts.constructors.features.otf
50local afmfeatures          = fonts.constructors.features.afm
51
52local registerotffeature   = otffeatures.register
53local registerafmfeature   = afmfeatures.register
54
55local is_character         = characters.is_character
56local chardata             = characters.data
57
58local tasks                = nodes.tasks
59local enableaction         = tasks.enableaction
60local disableaction        = tasks.disableaction
61
62local implement            = interfaces.implement
63
64local glyph_code           = nodes.nodecodes.glyph
65
66local new_special          = nodes.pool.special -- todo: literal
67local hpack_node           = nodes.hpack
68
69local nuts                 = nodes.nuts
70local tonut                = nuts.tonut
71
72local isglyph              = nuts.isglyph
73local setchar              = nuts.setchar
74
75local nextglyph            = nuts.traversers.glyph
76
77local remove_node          = nuts.remove
78local insertnodeafter      = nuts.insertafter
79
80-- maybe in fonts namespace
81-- deletion can be option
82
83local action = false
84
85-- to tfmdata.properties ?
86
87local function onetimemessage(font,char,message) -- char == false returns table
88    local tfmdata = fontdata[font]
89    local shared  = tfmdata.shared
90    if not shared then
91        shared = { }
92        tfmdata.shared = shared
93    end
94    local messages = shared.messages
95    if not messages then
96        messages = { }
97        shared.messages = messages
98    end
99    local category = messages[message]
100    if not category then
101        category = { }
102        messages[message] = category
103    end
104    if char == false then
105        return sortedkeys(category), category
106    end
107    local cc = category[char]
108    if not cc then
109        report_checking("char %C in font %a with id %a: %s",char,tfmdata.properties.fullname,font,message)
110        category[char] = 1
111    else
112        category[char] = cc + 1
113    end
114end
115
116fonts.loggers.onetimemessage = onetimemessage
117
118local mapping = allocate { -- this is just an experiment to illustrate some principles elsewhere
119    lu = "placeholder uppercase red",
120    ll = "placeholder lowercase red",
121    lt = "placeholder uppercase red",
122    lm = "placeholder lowercase red",
123    lo = "placeholder lowercase red",
124    mn = "placeholder mark green",
125    mc = "placeholder mark green",
126    me = "placeholder mark green",
127    nd = "placeholder lowercase blue",
128    nl = "placeholder lowercase blue",
129    no = "placeholder lowercase blue",
130    pc = "placeholder punctuation cyan",
131    pd = "placeholder punctuation cyan",
132    ps = "placeholder punctuation cyan",
133    pe = "placeholder punctuation cyan",
134    pi = "placeholder punctuation cyan",
135    pf = "placeholder punctuation cyan",
136    po = "placeholder punctuation cyan",
137    sm = "placeholder lowercase magenta",
138    sc = "placeholder lowercase yellow",
139    sk = "placeholder lowercase yellow",
140    so = "placeholder lowercase yellow",
141}
142
143table.setmetatableindex(mapping,
144    function(t,k)
145        local v = "placeholder unknown gray"
146        t[k] = v
147        return v
148    end
149)
150
151local fakes = allocate {
152    {
153        name   = "lowercase",
154        code   = ".025 -.175 m .425 -.175 l .425 .525 l .025 .525 l .025 -.175 l .025 0 l .425 0 l .025 -.175 m h S",
155        width  = .45,
156        height = .55,
157        depth  = .20,
158    },
159    {
160        name   = "uppercase",
161        code   = ".025 -.225 m .625 -.225 l .625 .675 l .025 .675 l .025 -.225 l .025 0 l .625 0 l .025 -.225 m h S",
162        width  = .65,
163        height = .70,
164        depth  = .25,
165    },
166    {
167        name   = "mark",
168        code   = ".025  .475 m .125  .475 l .125 .675 l .025 .675 l .025  .475 l h B",
169        width  = .15,
170        height = .70,
171        depth  = -.50,
172    },
173    {
174        name   = "punctuation",
175        code   = ".025 -.175 m .125 -.175 l .125 .525 l .025 .525 l .025 -.175 l h B",
176        width  = .15,
177        height = .55,
178        depth  = .20,
179    },
180    {
181        name   = "unknown",
182        code   = ".025 0 m .425 0 l .425 .175 l .025 .175 l .025 0 l h B",
183        width  = .45,
184        height = .20,
185        depth  = 0,
186    },
187}
188
189local variants = allocate {
190    { tag = "gray",    r = .6, g = .6, b = .6 },
191    { tag = "red",     r = .6, g =  0, b =  0 },
192    { tag = "green",   r =  0, g = .6, b =  0 },
193    { tag = "blue",    r =  0, g =  0, b = .6 },
194    { tag = "cyan",    r =  0, g = .6, b = .6 },
195    { tag = "magenta", r = .6, g =  0, b = .6 },
196    { tag = "yellow",  r = .6, g = .6, b =  0 },
197}
198
199-- bah .. low level pdf ... should be a rule or plugged in
200
201----- pdf_blob = "pdf: q %.6F 0 0 %.6F 0 0 cm %s %s %s rg %s %s %s RG 10 M 1 j 1 J 0.05 w %s Q"
202local pdf_blob = "q %.6F 0 0 %.6F 0 0 cm %s %s %s rg %s %s %s RG 10 M 1 j 1 J 0.05 w %s Q"
203
204local cache = { } -- saves some tables but not that impressive
205
206local function missingtonode(tfmdata,character)
207    local commands = character.commands
208    local fake  = hpack_node(new_special(commands[1][2])) -- todo: literal
209    fake.width  = character.width
210    fake.height = character.height
211    fake.depth  = character.depth
212    return fake
213end
214
215local function addmissingsymbols(tfmdata) -- we can have an alternative with rules
216    local characters = tfmdata.characters
217    local properties = tfmdata.properties
218    local size       = tfmdata.parameters.size
219    local scale      = size * bpfactor
220    local tonode     = nil
221    local collected  = { }
222    if properties.finalized and not addcharacters then
223        tonode = missingtonode
224    end
225    for i=1,#variants do
226        local v = variants[i]
227        local tag, r, g, b = v.tag, v.r, v.g, v.b
228        for i =1, #fakes do
229            local fake = fakes[i]
230            local name = fake.name
231            local privatename = formatters["placeholder %s %s"](name,tag)
232            if not hasprivate(tfmdata,privatename) then
233                local hash = formatters["%s_%s_%1.3f_%1.3f_%1.3f_%i"](name,tag,r,g,b,floor(size))
234                local char = cache[hash]
235                if not char then
236                    char = {
237                        tonode   = tonode,
238                        width    = size*fake.width,
239                        height   = size*fake.height,
240                        depth    = size*fake.depth,
241                     -- commands = { { "special", formatters[pdf_blob](scale,scale,r,g,b,r,g,b,fake.code) } }
242                        commands = { { "pdf", formatters[pdf_blob](scale,scale,r,g,b,r,g,b,fake.code) } }
243                    }
244                    cache[hash] = char
245                end
246                local u = addprivate(tfmdata, privatename, char)
247                if not tonode then
248                    collected[u] = char
249                end
250            end
251        end
252    end
253    if next(collected) then
254        local id = properties.id
255        if id then
256            addcharacters(properties.id, {
257                type       = "real",
258                characters = collected,
259            })
260        end
261    end
262end
263
264registerotffeature {
265    name        = "missing",
266    description = "missing symbols",
267    manipulators = {
268        base = addmissingsymbols,
269        node = addmissingsymbols,
270    }
271}
272
273fonts.loggers.add_placeholders        = function(id) addmissingsymbols(fontdata[id or true]) end
274fonts.loggers.category_to_placeholder = mapping
275
276-- todo in luatex: option to add characters (just slots, no kerns etc)
277-- we can do that now so ...
278
279local function placeholder(font,char)
280    local tfmdata  = fontdata[font]
281    local category = chardata[char].category or "unknown"
282    local fakechar = mapping[category]
283    local slot     = getprivateslot(font,fakechar)
284    if not slot then
285        addmissingsymbols(tfmdata)
286        slot = getprivateslot(font,fakechar)
287    end
288    return getprivatecharornode(tfmdata,fakechar)
289end
290
291checkers.placeholder = placeholder
292
293function checkers.missing(head)
294    local lastfont, characters, found = nil, nil, nil
295    for n, char, font in nextglyph, head do -- faster than while loop so we delay removal
296        if font ~= lastfont then
297            characters = fontcharacters[font]
298            lastfont   = font
299        end
300        if font > 0 and not characters[char] and is_character[chardata[char].category or "unknown"] then
301            if action == "remove" then
302                onetimemessage(font,char,"missing (will be deleted)")
303            elseif action == "replace" then
304                onetimemessage(font,char,"missing (will be flagged)")
305            else
306                onetimemessage(font,char,"missing")
307            end
308            if not found then
309                found = { n }
310            else
311                found[#found+1] = n
312            end
313        end
314    end
315    if not found then
316        -- all well
317    elseif action == "remove" then
318        for i=1,#found do
319            head = remove_node(head,found[i],true)
320        end
321    elseif action == "replace" then
322        for i=1,#found do
323            local node = found[i]
324            local char, font = isglyph(node)
325            local kind, char = placeholder(font,char)
326            if kind == "node" then
327                insertnodeafter(head,node,tonut(char))
328                head = remove_node(head,node,true)
329            elseif kind == "char" then
330                setchar(node,char)
331            else
332                -- error
333            end
334        end
335    else
336        -- maye write a report to the log
337    end
338    return head
339end
340
341local relevant = {
342    "missing (will be deleted)",
343    "missing (will be flagged)",
344    "missing"
345}
346
347local function getmissing(id)
348    if id then
349        local list = getmissing(currentfont())
350        if list then
351            local _, list = next(getmissing(currentfont()))
352            return list
353        else
354            return { }
355        end
356    else
357        local t = { }
358        for id, d in next, fontdata do
359            local shared   = d.shared
360            local messages = shared and shared.messages
361            if messages then
362                local filename = d.properties.filename
363                if not filename then
364                    filename = tostring(d)
365                end
366                local tf = t[filename] or { }
367                for i=1,#relevant do
368                    local tm = messages[relevant[i]]
369                    if tm then
370                        for k, v in next, tm do
371                            tf[k] = (tf[k] or 0) + v
372                        end
373                    end
374                end
375                if next(tf) then
376                    t[filename] = tf
377                end
378            end
379        end
380        local l = { }
381        for k, v in next, t do
382            l[k] = sortedkeys(v)
383        end
384        return l, t
385    end
386end
387
388checkers.getmissing = getmissing
389
390
391do
392
393    local reported = true
394
395    callback.register("glyph_not_found",function(font,char)
396        if font > 0 then
397            if char > 0 then
398                onetimemessage(font,char,"missing")
399            else
400                -- we have a special case
401            end
402        elseif not reported then
403            report("nullfont is used, maybe no bodyfont is defined")
404            reported = true
405        end
406    end)
407
408    trackers.register("fonts.missing", function(v)
409        if v then
410            enableaction("processors","fonts.checkers.missing")
411        else
412            disableaction("processors","fonts.checkers.missing")
413        end
414        if v == "replace" then
415            otffeatures.defaults.missing = true
416        end
417        action = v
418    end)
419
420    logs.registerfinalactions(function()
421        local collected, details = getmissing()
422        if next(collected) then
423            for filename, list in sortedhash(details) do
424                logs.startfilelogging(report,"missing characters",filename)
425                for u, v in sortedhash(list) do
426                    report("%4i  %U  %c  %s",v,u,u,chardata[u].description)
427                end
428                logs.stopfilelogging()
429            end
430            if logs.loggingerrors() then
431                for filename, list in sortedhash(details) do
432                    logs.starterrorlogging(report,"missing characters",filename)
433                    for u, v in sortedhash(list) do
434                        report("%4i  %U  %c  %s",v,u,u,chardata[u].description)
435                    end
436                    logs.stoperrorlogging()
437                end
438            end
439        end
440    end)
441
442end
443
444-- for the moment here
445
446local function expandglyph(characters,index,done)
447    done = done or { }
448    if not done[index] then
449        local data = characters[index]
450        if data then
451            done[index] = true
452            local d = fastcopy(data)
453            local n = d.next
454            if n then
455                d.next = expandglyph(characters,n,done)
456            end
457            local h = d.horiz_variants
458            if h then
459                for i=1,#h do
460                    h[i].glyph = expandglyph(characters,h[i].glyph,done)
461                end
462            end
463            local v = d.vert_variants
464            if v then
465                for i=1,#v do
466                    v[i].glyph = expandglyph(characters,v[i].glyph,done)
467                end
468            end
469            return d
470        end
471    end
472end
473
474helpers.expandglyph = expandglyph
475
476-- should not be needed as we add .notdef in the engine
477
478local dummyzero = {
479 -- width    = 0,
480 -- height   = 0,
481 -- depth    = 0,
482    commands = { { "special", "" } },
483}
484
485local function adddummysymbols(tfmdata)
486    local characters = tfmdata.characters
487    if not characters[0] then
488        characters[0] = dummyzero
489    end
490 -- if not characters[1] then
491 --     characters[1] = dummyzero -- test only
492 -- end
493end
494
495local dummies_specification = {
496    name        = "dummies",
497    description = "dummy symbols",
498    default     = true,
499    manipulators = {
500        base = adddummysymbols,
501        node = adddummysymbols,
502    }
503}
504
505registerotffeature(dummies_specification)
506registerafmfeature(dummies_specification)
507
508--
509
510local function addvisualspace(tfmdata)
511    local spacechar = tfmdata.characters[32]
512    if spacechar and not spacechar.commands then
513        local w = spacechar.width
514        local h = tfmdata.parameters.xheight
515        local c = {
516            width    = w,
517            commands = { { "rule", h, w } }
518        }
519        local u = addprivate(tfmdata, "visualspace", c)
520    end
521end
522
523local visualspace_specification = {
524    name        = "visualspace",
525    description = "visual space",
526    default     = true,
527    manipulators = {
528        base = addvisualspace,
529        node = addvisualspace,
530    }
531}
532
533registerotffeature(visualspace_specification)
534registerafmfeature(visualspace_specification)
535