font-ocl.lua /size: 24 Kb    last modification: 2021-10-28 13:50
1if not modules then modules = { } end modules ['font-ocl'] = {
2    version   = 1.001,
3    comment   = "companion to font-ini.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- todo : user list of colors
10
11local tostring, tonumber, next = tostring, tonumber, next
12local round, max = math.round, math.round
13local gsub, find = string.gsub, string.find
14local sortedkeys, sortedhash, concat = table.sortedkeys, table.sortedhash, table.concat
15local setmetatableindex = table.setmetatableindex
16
17local formatters   = string.formatters
18local tounicode    = fonts.mappings.tounicode
19
20local helpers      = fonts.helpers
21
22local charcommand  = helpers.commands.char
23local rightcommand = helpers.commands.right
24local leftcommand  = helpers.commands.left
25local downcommand  = helpers.commands.down
26
27local otf          = fonts.handlers.otf
28local otfregister  = otf.features.register
29
30local f_color      = formatters["%.3f %.3f %.3f rg"]
31local f_gray       = formatters["%.3f g"]
32
33if context then
34
35    local startactualtext = nil
36    local stopactualtext  = nil
37
38    function otf.getactualtext(s)
39        if not startactualtext then
40            startactualtext = backends.codeinjections.startunicodetoactualtextdirect
41            stopactualtext  = backends.codeinjections.stopunicodetoactualtextdirect
42        end
43        return startactualtext(s), stopactualtext()
44    end
45
46else
47
48    -- Actually we don't need a generic branch at all because (according the the
49    -- internet) other macro packages rely on hb for emoji etc and never used this
50    -- feature of the font loader. So maybe I should just remove this from generic.
51
52    local tounicode = fonts.mappings.tounicode16
53
54    function otf.getactualtext(s)
55        return
56            "/Span << /ActualText <feff" .. s .. "> >> BDC",
57            "EMC"
58    end
59
60end
61
62local sharedpalettes = { }
63
64local hash = setmetatableindex(function(t,k)
65    local v = { "pdf", "direct", k }
66    t[k] = v
67    return v
68end)
69
70if context then
71
72    -- \definefontcolorpalette [emoji-r] [emoji-red,emoji-gray,textcolor] -- looks bad
73    -- \definefontcolorpalette [emoji-r] [emoji-red,emoji-gray]           -- looks okay
74
75    local colors          = attributes.list[attributes.private('color')] or { }
76    local transparencies  = attributes.list[attributes.private('transparency')] or { }
77
78    function otf.registerpalette(name,values)
79        sharedpalettes[name] = values
80        local color          = lpdf.color
81        local transparency   = lpdf.transparency
82        local register       = colors.register
83        for i=1,#values do
84            local v = values[i]
85            if v == "textcolor" then
86                values[i] = false
87            else
88                local c = nil
89                local t = nil
90                if type(v) == "table" then
91                    c = register(name,"rgb",
92                        max(round((v.r or 0)*255),255)/255,
93                        max(round((v.g or 0)*255),255)/255,
94                        max(round((v.b or 0)*255),255)/255
95                    )
96                else
97                    c = colors[v]
98                    t = transparencies[v]
99                end
100                if c and t then
101                    values[i] = hash[color(1,c) .. " " .. transparency(t)]
102                elseif c then
103                    values[i] = hash[color(1,c)]
104                elseif t then
105                    values[i] = hash[color(1,t)]
106                end
107            end
108        end
109    end
110
111else -- for generic
112
113    function otf.registerpalette(name,values)
114        sharedpalettes[name] = values
115        for i=1,#values do
116            local v = values[i]
117            if v then
118                values[i] = hash[f_color(
119                    max(round((v.r or 0)*255),255)/255,
120                    max(round((v.g or 0)*255),255)/255,
121                    max(round((v.b or 0)*255),255)/255
122                )]
123            end
124        end
125    end
126
127end
128
129-- We need to force page first because otherwise the q's get outside the font switch and
130-- as a consequence the next character has no font set (well, it has: the preceding one). As
131-- a consequence these fonts are somewhat inefficient as each glyph gets the font set. It's
132-- a side effect of the fact that a font is handled when a character gets flushed. Okay, from
133-- now on we can use text as literal mode.
134
135local function convert(t,k)
136    local v = { }
137    for i=1,#k do
138        local p = k[i]
139        local r, g, b = p[1], p[2], p[3]
140        if r == g and g == b then
141            v[i] = hash[f_gray(r/255)]
142        else
143            v[i] = hash[f_color(r/255,g/255,b/255)]
144        end
145    end
146    t[k] = v
147    return v
148end
149
150-- At some point 'font' mode was added to the engine and we can assume that most distributions
151-- ship a luatex that has it; ancient versions are no longer supported anyway. Begin 2020 there
152-- was an actualtext related mail exchange with RM etc. that might result in similar mode keys
153-- in other tex->pdf programs because there is a bit of inconsistency in the way this is dealt
154-- with. Best is not to touch this code too much.
155
156local mode = { "pdf", "mode", "font" }
157local push = { "pdf", "page", "q" }
158local pop  = { "pdf", "page", "Q" }
159
160-- see context git repository for older variant (pre 20200501 cleanup)
161
162local function initializeoverlay(tfmdata,kind,value)
163    if value then
164        local resources = tfmdata.resources
165        local palettes  = resources.colorpalettes
166        if palettes then
167            --
168            local converted = resources.converted
169            if not converted then
170                converted = setmetatableindex(convert)
171                resources.converted = converted
172            end
173            local colorvalues = sharedpalettes[value]
174            local default     = false -- so the text color (bad for icon overloads)
175            if colorvalues then
176                default = colorvalues[#colorvalues]
177            else
178                colorvalues = converted[palettes[tonumber(value) or 1] or palettes[1]] or { }
179            end
180            local classes = #colorvalues
181            if classes == 0 then
182                return
183            end
184            --
185            local characters   = tfmdata.characters
186            local descriptions = tfmdata.descriptions
187            local properties   = tfmdata.properties
188            --
189            properties.virtualized = true
190            tfmdata.fonts = {
191                { id = 0 }
192            }
193            --
194            local getactualtext = otf.getactualtext
195            local b, e          = getactualtext(tounicode(0xFFFD))
196            local actualb       = { "pdf", "page", b } -- saves tables
197            local actuale       = { "pdf", "page", e } -- saves tables
198            --
199            for unicode, character in next, characters do
200                local description = descriptions[unicode]
201                if description then
202                    local colorlist = description.colors
203                    if colorlist then
204                        local u = description.unicode or characters[unicode].unicode
205                        local w = character.width or 0
206                        local s = #colorlist
207                        local goback = w ~= 0 and leftcommand[w] or nil -- needs checking: are widths the same
208                        local t = {
209                            mode,
210                            not u and actualb or { "pdf", "page", (getactualtext(tounicode(u))) },
211                            push,
212                        }
213                        local n = 3
214                        local l = nil
215                        for i=1,s do
216                            local entry = colorlist[i]
217                            local v = colorvalues[entry.class] or default
218                            if v and l ~= v then
219                                n = n + 1 t[n] = v
220                                l = v
221                            end
222                            n = n + 1 t[n] = charcommand[entry.slot]
223                            if s > 1 and i < s and goback then
224                                n = n + 1 t[n] = goback
225                            end
226                        end
227                        n = n + 1 t[n] = pop
228                        n = n + 1 t[n] = actuale
229                        character.commands = t
230                    end
231                end
232            end
233            return true
234        end
235    end
236end
237
238otfregister {
239    name         = "colr",
240    description  = "color glyphs",
241    manipulators = {
242        base = initializeoverlay,
243        node = initializeoverlay,
244    }
245}
246
247do
248
249    local nofstreams = 0
250    local f_name     = formatters[ [[pdf-glyph-%05i]] ]
251    local f_used     = context and formatters[ [[original:///%s]] ] or formatters[ [[%s]] ]
252    local hashed     = { }
253    local cache      = { }
254
255    local openpdf = pdfe.new
256    ----- prefix  = "data:application/pdf,"
257
258    function otf.storepdfdata(pdf)
259        local done = hashed[pdf]
260        if not done then
261            nofstreams = nofstreams + 1
262            local f = f_name(nofstreams)
263            local n = openpdf(pdf,#pdf,f)
264            done = f_used(n)
265            hashed[pdf] = done
266        end
267        return done
268    end
269
270end
271
272-- I'll probably make a variant for context as we can do it more efficient there than in
273-- generic.
274
275local function pdftovirtual(tfmdata,pdfshapes,kind) -- kind = png|svg
276    if not tfmdata or not pdfshapes or not kind then
277        return
278    end
279    --
280    local characters = tfmdata.characters
281    local properties = tfmdata.properties
282    local parameters = tfmdata.parameters
283    local hfactor    = parameters.hfactor
284    --
285    properties.virtualized = true
286    --
287    tfmdata.fonts = {
288        { id = 0 } -- not really needed
289    }
290        --
291    local getactualtext = otf.getactualtext
292    local storepdfdata  = otf.storepdfdata
293    --
294    local b, e          = getactualtext(tounicode(0xFFFD))
295    local actualb       = { "pdf", "page", b } -- saves tables
296    local actuale       = { "pdf", "page", e } -- saves tables
297    --
298    local vfimage = lpdf and lpdf.vfimage or function(wd,ht,dp,data,name)
299        local name = storepdfdata(data)
300        return { "image", { filename = name, width = wd, height = ht, depth = dp } }
301    end
302    --
303    for unicode, character in sortedhash(characters) do  -- sort is nicer for svg
304        local index = character.index
305        if index then
306            local pdf   = pdfshapes[index]
307            local typ   = type(pdf)
308            local data  = nil
309            local dx    = nil
310            local dy    = nil
311            local scale = 1
312            if typ == "table" then
313                data  = pdf.data
314                dx    = pdf.x or pdf.dx or 0
315                dy    = pdf.y or pdf.dy or 0
316                scale = pdf.scale or 1
317            elseif typ == "string" then
318                data = pdf
319                dx   = 0
320                dy   = 0
321            elseif typ == "number" then
322                data = pdf
323                dx   = 0
324                dy   = 0
325            end
326            if data then
327                -- We can delay storage by a lua function in commands: but then we need to
328                -- be able to provide our own mem stream name (so that we can reserve it).
329                -- Anyway, we will do this different in a future version of context.
330                local bt = unicode and getactualtext(unicode)
331                local wd = character.width  or 0
332                local ht = character.height or 0
333                local dp = character.depth  or 0
334                -- The down and right will change too (we can move that elsewhere). We have
335                -- a different treatment in lmtx but the next kind of works. These images are
336                -- a mess anyway as in svg the bbox can be messed up absent). A png image
337                -- needs the x/y. I might normalize this once we move to lmtx exlusively.
338                character.commands = {
339                    not unicode and actualb or { "pdf", "page", (getactualtext(unicode)) },
340                    -- lmtx (when we deal with depth in vfimage, currently disabled):
341                 -- downcommand [dy * hfactor],
342                 -- rightcommand[dx * hfactor],
343                 -- vfimage(wd,ht,dp,data,name),
344                    -- mkiv
345                    downcommand [dp + dy * hfactor],
346                    rightcommand[     dx * hfactor],
347                    vfimage(scale*wd,ht,dp,data,pdfshapes.filename or ""),
348                    actuale,
349                }
350                character[kind] = true
351            end
352        end
353    end
354end
355
356local otfsvg   = otf.svg or { }
357otf.svg        = otfsvg
358otf.svgenabled = true
359
360do
361
362    local report_svg = logs.reporter("fonts","svg conversion")
363
364    local loaddata   = io.loaddata
365    local savedata   = io.savedata
366    local remove     = os.remove
367
368if context then
369
370        local xmlconvert = xml.convert
371        local xmlfirst   = xml.first
372
373     -- function otfsvg.filterglyph(entry,index)
374     --     -- we only support decompression in lmtx, so one needs to wipe the
375     --     -- cache when invalid xml is reported
376     --     local svg  = xmlconvert(entry.data)
377     --     local root = svg and xmlfirst(svg,"/svg[@id='glyph"..index.."']")
378     --     local data = root and tostring(root)
379     --  -- report_svg("data for glyph %04X: %s",index,data)
380     --     return data
381     -- end
382
383        function otfsvg.filterglyph(entry,index)
384            local d = entry.data
385            if gzip.compressed(d) then
386                d = gzip.decompress(d) or d
387            end
388            local svg  = xmlconvert(d)
389            local root = svg and xmlfirst(svg,"/svg[@id='glyph"..index.."']")
390            local data = root and tostring(root)
391            return data
392        end
393
394else
395
396        function otfsvg.filterglyph(entry,index) -- can be overloaded
397            return entry.data
398        end
399
400end
401
402    local runner = sandbox and sandbox.registerrunner {
403        name     = "otfsvg",
404        program  = "inkscape",
405        method   = "pipeto",
406        template = "--export-area-drawing --shell > temp-otf-svg-shape.log",
407        reporter = report_svg,
408    }
409
410    if not runner then
411        --
412        -- poor mans variant for generic:
413        --
414        runner = function()
415            return io.popen("inkscape --export-area-drawing --shell > temp-otf-svg-shape.log","w")
416        end
417    end
418
419    -- There are svg out there with bad viewBox specifications where shapes lay outside that area,
420    -- but trying to correct that didn't work out well enough so I discarded that code. BTW, we
421    -- decouple the inskape run and the loading run because inkscape is working in the background
422    -- in the files so we need to have unique files.
423    --
424    -- Because a generic setup can be flawed we need to catch bad inkscape runs which add a bit of
425    -- ugly overhead. Bah.
426    --
427    -- In the long run this method is a dead end because we cannot rely on command line arguments
428    -- etc to be upward compatible (so no real batch tool).
429
430    local new = nil
431
432    local function inkscapeformat(suffix)
433        if new == nil then
434            new = os.resultof("inkscape --version") or ""
435            new = new == "" or not find(new,"Inkscape%s*0")
436        end
437        return new and "filename" or suffix
438    end
439
440    function otfsvg.topdf(svgshapes,tfmdata)
441        local pdfshapes = { }
442        local inkscape  = runner()
443        if inkscape then
444         -- local indices      = fonts.getindices(tfmdata)
445            local descriptions = tfmdata.descriptions
446            local nofshapes    = #svgshapes
447            local s_format     = inkscapeformat("pdf") -- hack, this will go away when is >= 0 is everywhere
448            local f_svgfile    = formatters["temp-otf-svg-shape-%i.svg"]
449            local f_pdffile    = formatters["temp-otf-svg-shape-%i.pdf"]
450            local f_convert    = formatters[new and "file-open:%s; export-%s:%s; export-do\n" or "%s --export-%s=%s\n"]
451            local filterglyph  = otfsvg.filterglyph
452            local nofdone      = 0
453            local processed    = { }
454            report_svg("processing %i svg containers",nofshapes)
455            statistics.starttiming()
456            for i=1,nofshapes do
457                local entry = svgshapes[i]
458                for index=entry.first,entry.last do
459                    local data = filterglyph(entry,index)
460                    if data and data ~= "" then
461                        local svgfile = f_svgfile(index)
462                        local pdffile = f_pdffile(index)
463                        savedata(svgfile,data)
464                        inkscape:write(f_convert(svgfile,s_format,pdffile))
465                        processed[index] = true
466                        nofdone = nofdone + 1
467                        if nofdone % 25 == 0 then
468                            report_svg("%i shapes submitted",nofdone)
469                        end
470                    end
471                end
472            end
473            if nofdone % 25 ~= 0 then
474                report_svg("%i shapes submitted",nofdone)
475            end
476            report_svg("processing can be going on for a while")
477            inkscape:write("quit\n")
478            inkscape:close()
479            report_svg("processing %i pdf results",nofshapes)
480            for index in next, processed do
481                local svgfile = f_svgfile(index)
482                local pdffile = f_pdffile(index)
483             -- local fntdata = descriptions[indices[index]]
484             -- local bounds  = fntdata and fntdata.boundingbox
485                local pdfdata = loaddata(pdffile)
486                if pdfdata and pdfdata ~= "" then
487                    pdfshapes[index] = {
488                        data = pdfdata,
489                     -- x    = bounds and bounds[1] or 0,
490                     -- y    = bounds and bounds[2] or 0,
491                    }
492                end
493                remove(svgfile)
494                remove(pdffile)
495            end
496            local characters = tfmdata.characters
497            for k, v in next, characters do
498                local d = descriptions[k]
499                local i = d.index
500                if i then
501                    local p = pdfshapes[i]
502                    if p then
503                        local w = d.width
504                        local l = d.boundingbox[1]
505                        local r = d.boundingbox[3]
506                        p.scale = (r - l) / w
507                        p.x     = l
508                    end
509                end
510            end
511            if not next(pdfshapes) then
512                report_svg("there are no converted shapes, fix your setup")
513            end
514            statistics.stoptiming()
515            if statistics.elapsedseconds then
516                report_svg("svg conversion time %s",statistics.elapsedseconds() or "-")
517            end
518        end
519        return pdfshapes
520    end
521
522end
523
524local function initializesvg(tfmdata,kind,value) -- hm, always value
525    if value and otf.svgenabled then
526        local svg       = tfmdata.properties.svg
527        local hash      = svg and svg.hash
528        local timestamp = svg and svg.timestamp
529        if not hash then
530            return
531        end
532        local pdffile   = containers.read(otf.pdfcache,hash)
533        local pdfshapes = pdffile and pdffile.pdfshapes
534        if not pdfshapes or pdffile.timestamp ~= timestamp or not next(pdfshapes) then
535            -- the next test tries to catch errors in generic usage but of course can result
536            -- in running again and again
537            local svgfile   = containers.read(otf.svgcache,hash)
538            local svgshapes = svgfile and svgfile.svgshapes
539            pdfshapes = svgshapes and otfsvg.topdf(svgshapes,tfmdata,otf.pdfcache.writable,hash) or { }
540            containers.write(otf.pdfcache, hash, {
541                pdfshapes = pdfshapes,
542                timestamp = timestamp,
543            })
544        end
545        pdftovirtual(tfmdata,pdfshapes,"svg")
546        return true
547    end
548end
549
550otfregister {
551    name         = "svg",
552    description  = "svg glyphs",
553    manipulators = {
554        base = initializesvg,
555        node = initializesvg,
556    }
557}
558
559-- This can be done differently e.g. with ffi and gm and we can share code anway. Using
560-- batchmode in gm is not faster and as it accumulates we would need to flush all
561-- individual shapes. But ... in context lmtx (and maybe the backport) we will use
562-- a different and more efficient method anyway. I'm still wondering if I should
563-- keep color code in generic. Maybe it should be optional.
564
565local otfpng   = otf.png or { }
566otf.png        = otfpng
567otf.pngenabled = true
568
569do
570
571    local report_png = logs.reporter("fonts","png conversion")
572
573    local loaddata   = io.loaddata
574    local savedata   = io.savedata
575    local remove     = os.remove
576
577    local runner = sandbox and sandbox.registerrunner {
578        name     = "otfpng",
579        program  = "gm",
580        template = "convert -quality 100 temp-otf-png-shape.png temp-otf-png-shape.pdf > temp-otf-svg-shape.log",
581     -- reporter = report_png,
582    }
583
584    if not runner then
585        --
586        -- poor mans variant for generic:
587        --
588        runner = function()
589            return os.execute("gm convert -quality 100 temp-otf-png-shape.png temp-otf-png-shape.pdf > temp-otf-svg-shape.log")
590        end
591    end
592
593    -- Alternatively we can create a single pdf file with -adjoin and then pick up pages from
594    -- that file but creating thousands of small files is no fun either.
595
596    function otfpng.topdf(pngshapes)
597        local pdfshapes  = { }
598        local pngfile    = "temp-otf-png-shape.png"
599        local pdffile    = "temp-otf-png-shape.pdf"
600        local nofdone    = 0
601        local indices    = sortedkeys(pngshapes) -- can be sparse
602        local nofindices = #indices
603        report_png("processing %i png containers",nofindices)
604        statistics.starttiming()
605        for i=1,nofindices do
606            local index = indices[i]
607            local entry = pngshapes[index]
608            local data  = entry.data -- or placeholder
609            local x     = entry.x
610            local y     = entry.y
611            savedata(pngfile,data)
612            runner()
613            pdfshapes[index] = {
614                x     = x ~= 0 and x or nil,
615                y     = y ~= 0 and y or nil,
616                data  = loaddata(pdffile),
617            }
618            nofdone = nofdone + 1
619            if nofdone % 100 == 0 then
620                report_png("%i shapes processed",nofdone)
621            end
622        end
623        report_png("processing %i pdf results",nofindices)
624        remove(pngfile)
625        remove(pdffile)
626        statistics.stoptiming()
627        if statistics.elapsedseconds then
628            report_png("png conversion time %s",statistics.elapsedseconds() or "-")
629        end
630        return pdfshapes
631    end
632
633end
634
635-- This will change in a future version of context. More direct.
636
637local function initializepng(tfmdata,kind,value) -- hm, always value
638    if value and otf.pngenabled then
639        local png       = tfmdata.properties.png
640        local hash      = png and png.hash
641        local timestamp = png and png.timestamp
642        if not hash then
643            return
644        end
645        local pdffile   = containers.read(otf.pdfcache,hash)
646        local pdfshapes = pdffile and pdffile.pdfshapes
647        if not pdfshapes or pdffile.timestamp ~= timestamp then
648            local pngfile   = containers.read(otf.pngcache,hash)
649            local pngshapes = pngfile and pngfile.pngshapes
650            pdfshapes = pngshapes and otfpng.topdf(pngshapes) or { }
651            containers.write(otf.pdfcache, hash, {
652                pdfshapes = pdfshapes,
653                timestamp = timestamp,
654            })
655        end
656        --
657        pdftovirtual(tfmdata,pdfshapes,"png")
658        return true
659    end
660end
661
662otfregister {
663    name         = "sbix",
664    description  = "sbix glyphs",
665    manipulators = {
666        base = initializepng,
667        node = initializepng,
668    }
669}
670
671otfregister {
672    name         = "cblc",
673    description  = "cblc glyphs",
674    manipulators = {
675        base = initializepng,
676        node = initializepng,
677    }
678}
679
680if context then
681
682    -- untested in generic and might clash with other color trickery
683    -- anyway so let's stick to context only
684
685    local function initializecolor(tfmdata,kind,value)
686        if value == "auto" then
687            return
688                initializeoverlay(tfmdata,kind,value) or
689                initializesvg(tfmdata,kind,value) or
690                initializepng(tfmdata,kind,value)
691        elseif value == "overlay" then
692            return initializeoverlay(tfmdata,kind,value)
693        elseif value == "svg" then
694            return initializesvg(tfmdata,kind,value)
695        elseif value == "png" or value == "bitmap" then
696            return initializepng(tfmdata,kind,value)
697        else
698            -- forget about it
699        end
700    end
701
702    otfregister {
703        name         = "color",
704        description  = "color glyphs",
705        manipulators = {
706            base = initializecolor,
707            node = initializecolor,
708        }
709    }
710
711end
712