lpdf-lmt.lmt /size: 119 Kb    last modification: 2024-01-16 09:02
1if not modules then modules = { } end modules ['lpdf-lmt'] = {
2    version   = 1.001,
3    optimize  = true,
4    comment   = "companion to lpdf-ini.mkiv",
5    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
6    copyright = "PRAGMA ADE / ConTeXt Development Team",
7    license   = "see context related readme files"
8}
9
10-- The code below was originally in back-lpd.lua but it makes more sense in this
11-- namespace. I will rename variables. Because we run into the 200 locals this file
12-- has to be split.
13
14-- There is no way that a lua based backend can compete performance wise with the
15-- original one for relative simple text runs. And we're talking seconds here on say
16-- 500 pages with paragraphs alternativng between three fonts and colors. But such
17-- documents are rare so in practice we are quite okay, especially because in
18-- ConTeXt we can gain quite a bit elsewhere. So, when we loose 30% on such simple
19-- documents, we break even on for instance the manual, and gain 30% on Thomas's
20-- turture test (also for other reasons). But .. who knows what magic I can cook up
21-- in due time.
22
23-- If you consider this complex, watch:
24--
25-- https://www.youtube.com/watch?v=6H-cAzfB2qo
26--
27-- or in distractionmode:
28--
29-- https://www.youtube.com/watch?v=TYuTE_1jvvE
30-- https://www.youtube.com/watch?v=nnicGKX3lvM
31--
32-- For the moment we have to support the built in backend as well as the alternative. So
33-- the next interface is suboptimal and will change at some time. At that moment I will
34-- also optimize and extend.
35
36local type, next, unpack, tonumber, rawget = type, next, unpack, tonumber, rawget
37local char, rep, find = string.char, string.rep, string.find
38local formatters, splitupstring = string.formatters, string.splitup
39local concat, sortedhash = table.concat, table.sortedhash
40local setmetatableindex = table.setmetatableindex
41local loaddata = io.loaddata
42local ceil = math.ceil
43
44local bpfactor       <const> = number.dimenfactors.bp
45
46local osuuid                 = os.uuid
47local zlibcompresssize       = xzip.compresssize
48
49local nuts                   = nodes.nuts
50
51local pdfreference           = lpdf.reference
52local pdfdictionary          = lpdf.dictionary
53local pdfarray               = lpdf.array
54local pdfconstant            = lpdf.constant
55local pdfliteral             = lpdf.literal -- not to be confused with a whatsit!
56
57local pdfreserveobject       -- forward reference
58local pdfpagereference       -- forward reference
59local pdfgetpagereference    -- forward reference
60local pdfsharedobject        -- forward reference
61local pdfflushobject         -- forward reference
62local pdfflushstreamobject   -- forward reference
63local pdfdeferredobject      -- forward reference
64local pdfimmediateobject     -- forward reference
65
66local pdfincludeimage        -- forward reference
67
68local pdf_pages         = pdfconstant("Pages")
69local pdf_page          = pdfconstant("Page")
70local pdf_xobject       = pdfconstant("XObject")
71local pdf_form          = pdfconstant("Form")
72local pdf_pattern       = pdfconstant("Pattern")
73
74local fonthashes        = fonts.hashes
75local characters        = fonthashes.characters
76local descriptions      = fonthashes.descriptions
77local parameters        = fonthashes.parameters
78local properties        = fonthashes.properties
79
80local report            = logs.reporter("backend")
81local report_objects    = logs.reporter("backend","objects")
82local report_fonts      = logs.reporter("backend","fonts")
83local report_encryption = logs.reporter("backend","encryption")
84
85local trace_objects     = false  trackers.register("backend.objects",       function(v) trace_objects = v end)
86local trace_details     = false  trackers.register("backend.details",       function(v) trace_details = v end)
87local trace_indices     = false  trackers.register("backend.fonts.details", function(v) trace_indices = v end)
88
89-- These two tables used a font id as index and will be metatabled in lpdf-emb.lmt:
90
91local usedfontnames     = { }
92local usedfontobjects   = { }
93
94lpdf.usedfontnames      = usedfontnames
95lpdf.usedfontobjects    = usedfontobjects
96
97-- experiment:
98
99local function compressdata(data,size)
100    local guess = ((size // 4096) + 1) * 2048
101    local comp  = zlibcompresssize(data,guess,3)
102 -- if comp then
103 --     report()
104 --     report("size %i, guess %i, result %i => %s / %s",size,guess,#comp,guess>=#comp and "hit" or "miss")
105 --     report()
106 -- end
107    return comp
108end
109
110-- local function compressdata(data,size)
111--     return zlibcompress(data,3)
112-- end
113
114-- we collect them:
115
116local flushers = { }
117
118-- used variables
119
120local pdf_h = 0
121local pdf_v = 0
122
123local need_tm
124local need_tf
125local need_font
126----- cur_tmrx
127local need_width
128local need_mode
129local done_width
130local done_mode
131local mode
132local current_pdf
133----- f_pdf_cur
134local current_font
135local current_effect
136local current_slant
137local current_weight
138local current_sx
139local current_sy
140local current_factor
141local f_x_scale
142local f_y_scale
143local tj_scale
144local tj_delta
145local tj_position
146----- tmrx, tmry, tmsx, tmsy, tmtx, tmty
147local tmrx, tmry,       tmsy, tmtx, tmty
148----- cmrx, cmry, cmsx, cmsy, cmtx, cmty
149local cmrx, cmry,             cmtx, cmty
150local tmef
151local tmef_f_x_scale
152local tmef_f_w_scale
153
154local usedfonts, usedxforms, usedximages, usedxgroups
155local getxformname, getximagename
156local boundingbox, shippingmode, objectnumber
157
158local function usefont(t,k) -- a bit redundant hash
159 -- local v = pdfgetfontname(k)
160    local v = usedfontnames[k]
161    t[k] = v
162    return v
163end
164
165local function reset_variables(specification)
166    pdf_h, pdf_v   = 0, 0
167    cmrx, cmry     = 1.0, 1.0
168 -- cmsx, cmsy     = 0.0, 0.0
169    cmtx, cmty     = 0.0, 0.0
170    tmrx, tmry     = 1.0, 1.0
171 -- tmsx, tmsy     = 0.0, 0.0
172          tmsy     =      0.0 -- sy makes no sense
173    tmtx, tmty     = 0.0, 0.0
174    tmef           = 1.0
175    need_tm        = false
176    need_tf        = false
177    need_font      = true
178    need_width     = 0
179    need_mode      = 0
180    done_width     = false
181    done_mode      = false
182    mode           = "page"
183    shippingmode   = specification.shippingmode
184    objectnumber   = specification.objectnumber
185 -- cur_tmrx       = 0.0
186 -- f_pdf_cur      = 0 -- nullfont
187    tj_scale       = 1
188    tj_delta       = 0.0
189    tj_position    = 0.0
190    current_font   = 0
191    current_pdf    = 0 -- nullfont
192    current_effect = nil
193    current_slant  = 0
194    current_weight = 0
195    current_factor = 0
196    current_sx     = 1
197    current_sy     = 1
198    f_x_scale      = 1.0
199    f_y_scale      = 1.0
200    tmef_f_x_scale = 1
201    tmef_f_w_scale = 1
202    usedfonts      = setmetatableindex(usefont)
203    usedxforms     = { }
204    usedximages    = { }
205 -- usedxgroups    = { }
206    boundingbox    = specification.boundingbox
207end
208
209-- buffer
210
211local buffer = lua.newtable(1024,0) -- { }
212local b      = 0
213
214local function reset_buffer()
215    b = 0
216end
217
218-- fonts
219
220-- The text in a PDF file ends up in the page stream. Right from the start PDF files
221-- were supposed to be efficient but over time that concept was messed up by for
222-- instance mixed in tagging. One can argue if that was a good idea: just embed the
223-- source and use that as for special purposes instead of trying to turn something
224-- visual into something structure. We have these angle bracket formats for that
225-- already. Anyway, here we assume an uninterupted stream.
226--
227-- A sequence of characters in the same font and with the same scale (and other
228-- rendering properties). When we started with the backend in Lua I just followed
229-- the same approach as in LuaTeX (which was modelled after pdfTeX) but at some
230-- point a (stepwise) transition took place. For instance, we always use a 10bp font
231-- size (instead of the scale in the document) and delegate all scaling to the font
232-- transform matrix. Because the text stream uses a different coordinate system
233-- (font units combined with 1000 based kerning) we need to keep track of where we
234-- are and when we drift too much we need to restart. In the engines this is
235-- complicated by integer math so there drift really plays a role, but in our case
236-- we can now approach it a bit different, although we don't really know the
237-- acceptable thresholds. This code evolved over time, also because we introduced
238-- additional scaling options.
239--
240-- So, in the backend we have to deal with this:
241--
242-- - positioning in target space (page)
243-- - positioning in the stream (text)
244-- - glyph scale (!), glyph xscale (!), glyph yscale
245-- - font scale (!)
246-- - horizontal extend (!)
247-- - vertical squeeze
248-- - slant
249-- - expansion factor (!)
250-- - font effects (mode and width/line)
251-- - temporary glyph effects (extend, squeeze, slant, mode, width/line)
252-- - drift (synchronization and check for threshold)
253--
254-- This means that we have to check if one of these properties has changed and the
255-- horizontal scale is the most demanding one. We also need to mix the check for
256-- font specific changes and glyph specific effects. For now we assume that slanted
257-- font is just a copy but in the future we could just make it a glyph property like
258-- scale, xscale and yscale. An even more advanced approach is to integrate
259-- boldening but then we also need 4 offsets and runtime calculation, while it might
260-- also complicate e.g. extensibles in math. There is much more overhead so it will
261-- also impact performance.
262
263local fontcharacters
264----- fontdescriptions
265local fontparameters
266local fontproperties
267local pdfcharacters
268
269local getstreamhash = fonts.handlers.otf.getstreamhash
270
271local usedfontstreams = utilities.storage.allocate()
272
273local usedindices = setmetatableindex(function(t,k)
274    local n = 0
275-- n = 31
276    local v = setmetatableindex(function(tt,kk)
277        if n >= 0xFFFF then
278            report_fonts("registering character index: overflow in hash %a, todo: use overflow font")
279        else
280            n = n + 1
281        end
282        if trace_indices then
283            report_fonts("registering character index: hash %a, charindex 0x%05X, slotindex 0x%04X",k,kk,n)
284        end
285        local vv = n
286        tt[kk] = vv
287        return vv
288    end)
289    t[k] = v
290    return v
291end)
292
293local usedcharacters = setmetatableindex(function(t,k)
294    local h, d = getstreamhash(k)
295    if trace_indices then
296        report_fonts("registering index table: hash %a, fontid %i",h,k)
297    end
298    usedfontstreams[h] = d
299    local v = usedindices[h]
300    t[k] = v
301    return v
302end)
303
304lpdf.usedfontstreams = usedfontstreams -- [streamhash]        -> fontdata
305lpdf.usedcharacters  = usedcharacters  -- [fontid]            -> indices
306lpdf.usedindices     = usedindices     -- [streamhash][index] -> realindex (can also be dupindex)
307
308local horizontalmode = true
309local scalefactor    = 1
310local threshold      = 655360
311local tjfactor       = 100 / 65536
312
313function flushers.updatefontstate(font)
314    -- virtual t3 fonts have negative font index
315    fontcharacters   = characters[font]
316 -- fontdescriptions = descriptions[font]
317    fontparameters   = parameters[font]
318    fontproperties   = properties[font]
319    local size       = fontparameters.size -- or bad news
320    local designsize = fontparameters.designsize or size
321    pdfcharacters    = usedcharacters[font]
322    horizontalmode   = fontparameters.writingmode ~= "vertical"
323    scalefactor      = (designsize/size) * tjfactor
324end
325
326local f_cm = formatters["%.6N %.6N %.6N %.6N %.6N %.6N cm"]
327local f_cz = formatters["%.6N 0 0 %.6N %.6N %.6N cm"]
328----- f_tm = formatters["%.6N %.6N %.6N %.6N %.6N %.6N Tm"]
329local f_tm = formatters["%.6N 0 %.6N %.6N %.6N %.6N Tm"]
330
331directives.register("backend.pdf.accurate", function()
332    f_cm = formatters["%.9N %.9N %.9N %.9N %.9N %.9N cm"]
333    f_cz = formatters["%.9N 0 0 %.9N %.9N %.9N cm"]
334 -- f_tm = formatters["%.9N %.9N %.9N %.9N %.9N %.9N Tm"]
335    f_tm = formatters["%.9N 0 %.9N %.9N %.9N %.9N Tm"]
336end)
337
338local saved_text_pos_v = 0
339local saved_text_pos_h = 0
340
341local function begin_text()
342    saved_text_pos_h = pdf_h
343    saved_text_pos_v = pdf_v
344    b = b + 1 ; buffer[b] = "BT"
345    need_tf        = true
346    need_font      = true
347    need_width     = 0
348    need_mode      = 0
349    current_effect = nil
350    current_slant  = 0
351    current_weight = 0
352    mode           = "text"
353end
354
355local function end_text()
356    if done_width then
357        b = b + 1 ; buffer[b] = "0 w"
358        done_width = false
359    end
360    if done_mode then
361        b = b + 1 ; buffer[b] = "0 Tr"
362        done_mode = false
363    end
364    b = b + 1 ; buffer[b] = "ET"
365    pdf_h = saved_text_pos_h
366    pdf_v = saved_text_pos_v
367    mode  = "page"
368end
369
370local begin_chararray, end_chararray do
371
372    local saved_chararray_pos_h
373    local saved_chararray_pos_v
374
375    local saved_b = 0
376
377    begin_chararray = function()
378        saved_chararray_pos_h = pdf_h
379        saved_chararray_pos_v = pdf_v
380        tj_position = horizontalmode and saved_chararray_pos_h or - saved_chararray_pos_v
381        tj_delta = 0
382        saved_b = b
383        b = b + 1 ; buffer[b] = " ["
384        mode = "chararray"
385    end
386
387    end_chararray = function()
388        b = b + 1 ; buffer[b] = "] TJ"
389        buffer[saved_b] = concat(buffer,"",saved_b,b)
390        b = saved_b
391        pdf_h = saved_chararray_pos_h
392        pdf_v = saved_chararray_pos_v
393        mode  = "text"
394    end
395
396end
397
398local function begin_charmode()
399    b = b + 1 ; buffer[b] = "<"
400    mode = "char"
401end
402
403local function end_charmode()
404    b = b + 1 ; buffer[b] = ">"
405    mode = "chararray"
406end
407
408local function calc_pdfpos(h,v)
409    -- mostly char
410    if mode == "page" then
411        cmtx = h - pdf_h
412        cmty = v - pdf_v
413        return h ~= pdf_h or v ~= pdf_v
414    elseif mode == "text" then
415        tmtx = h - saved_text_pos_h
416        tmty = v - saved_text_pos_v
417        return h ~= pdf_h or v ~= pdf_v
418    elseif horizontalmode then
419        tmty = v - saved_text_pos_v
420        tj_delta = tj_position - h
421        return tj_delta ~= 0 or v ~= pdf_v
422    else
423        tmtx = h - saved_text_pos_h
424        tj_delta = tj_position + v
425        return tj_delta ~= 0 or h ~= pdf_h
426    end
427end
428
429local function pdf_set_pos(h,v)
430    local move = calc_pdfpos(h,v)
431    if move then
432     -- b = b + 1 ; buffer[b] = f_cm(cmrx, cmsx, cmsy, cmry, cmtx*bpfactor, cmty*bpfactor)
433        b = b + 1 ; buffer[b] = f_cz(cmrx, cmry,             cmtx*bpfactor, cmty*bpfactor)
434        pdf_h = pdf_h + cmtx
435        pdf_v = pdf_v + cmty
436    end
437end
438
439local function pdf_reset_pos()
440    if mode == "page" then
441        cmtx = - pdf_h
442        cmty = - pdf_v
443        if pdf_h == 0 and pdf_v == 0 then
444            return
445        end
446    elseif mode == "text" then
447        tmtx = - saved_text_pos_h
448        tmty = - saved_text_pos_v
449        if pdf_h == 0 and pdf_v == 0 then
450            return
451        end
452    elseif horizontalmode then
453        tmty = - saved_text_pos_v
454        tj_delta = tj_position
455        if tj_delta == 0 and pdf_v == 0 then
456            return
457        end
458    else
459        tmtx = - saved_text_pos_h
460        tj_delta = tj_position
461        if tj_delta == 0 and pdf_h == 0 then
462            return
463        end
464    end
465 -- b = b + 1 ; buffer[b] = f_cm(cmrx, cmsx, cmsy, cmry, cmtx*bpfactor, cmty*bpfactor)
466    b = b + 1 ; buffer[b] = f_cz(cmrx,             cmry, cmtx*bpfactor, cmty*bpfactor)
467    pdf_h = pdf_h + cmtx
468    pdf_v = pdf_v + cmty
469end
470
471local function pdf_set_pos_temp(h,v)
472    local move = calc_pdfpos(h,v)
473    if move then
474     -- b = b + 1 ; buffer[b] = f_cm(cmrx, cmsx, cmsy, cmry, cmtx*bpfactor, cmty*bpfactor)
475        b = b + 1 ; buffer[b] = f_cz(cmrx,             cmry, cmtx*bpfactor, cmty*bpfactor)
476    end
477end
478
479-- these dummy returns makes using them a bit faster
480
481local function pdf_end_string_nl()
482    if mode == "char" then
483        end_charmode()
484        return end_chararray()
485    elseif mode == "chararray" then
486        return end_chararray()
487    end
488end
489
490local function pdf_goto_textmode()
491    if mode == "page" then
492        pdf_reset_pos()
493        return begin_text()
494    elseif mode ~= "text" then
495        if mode == "char" then
496            end_charmode()
497            return end_chararray()
498        else -- if mode == "chararray" then
499            return end_chararray()
500        end
501    end
502end
503
504local function pdf_goto_pagemode()
505    if mode ~= "page" then
506        if mode == "char" then
507            end_charmode()
508            end_chararray()
509            return end_text()
510        elseif mode == "chararray" then
511            end_chararray()
512            return end_text()
513        elseif mode == "text" then
514            return end_text()
515        end
516    end
517end
518
519local function pdf_goto_fontmode()
520    if mode == "char" then
521        end_charmode()
522        end_chararray()
523        end_text()
524    elseif mode == "chararray" then
525        end_chararray()
526        end_text()
527    elseif mode == "text" then
528        end_text()
529    end
530    pdf_reset_pos()
531    mode = "page"
532end
533
534-- characters
535do
536
537    local round = math.round
538
539    -- across pages ... todo: clean up because we don't need to pass the font
540    -- as fontparameters already has checked / set it we can also have a variable
541    -- for it so
542
543    local characterwidth   = nil
544 -- local descriptionwidth = nil
545    local hshift           = false
546    local vshift           = false
547
548    local dupx = 100 / bpfactor -- 6578176.0
549    local dumx = - dupx
550    local dupy = dupx
551    local dumy = dumx
552
553    directives.register("backend.pdf.drift", function(v)
554        dupx = tonumber(v) or 100
555        if dupx < 10 then
556            dupx = 10
557        elseif dupx > 100 then
558            dupx = 100
559        end
560        dupx = dupx / bpfactor
561    end)
562
563    local h_hex_2 = lpdf.h_hex_2
564    local h_hex_4 = lpdf.h_hex_4
565
566    -- The width array uses the original dimensions! This is different from e.g.
567    -- luatex where we have more widths arrays and these reflect the cheated
568    -- widths (goes wrong elsewhere).
569
570    -- when changing this, check math: compact-001.tex (rule width)
571
572    local characterwidths = setmetatableindex(function(t,font)
573        local d = descriptions[font]
574        local c = characters[font]
575        local f = parameters[font].hfactor or parameters[font].factor
576        local v = setmetatableindex(function(t,char)
577            local w
578            local e = c and c[char]
579            if e then
580                local a = e.advance
581                if a then
582                    w = a
583                else
584                    w = e.width or 0
585                end
586            end
587            if not w then
588                e = d and d[char]
589                if e then
590                    w = e.width
591                    if w then
592                        w =  w * f
593                    end
594                end
595            end
596            if not w then
597                w = 0
598            end
599            t[char] = w
600            return w
601        end)
602        t[font] = v
603        return v
604    end)
605
606    -- the descriptions are used for the width array
607
608    local extend  = 1 -- some more can move here
609    local squeeze = 1
610    local tohex   = h_hex_4
611
612    local function setup_fontparameters(font,factor,sx,sy,slant,weight,effect)
613        --
614        current_sx = sx
615        current_sy = sy
616        --
617        if fontproperties.bitmapped then
618            tohex = h_hex_2
619        elseif font < 0 then
620            tohex = h_hex_2
621        else
622            tohex = h_hex_4
623        end
624
625        local format = fontproperties.format
626        local expand = 1 + factor / 1000000
627        if effect then
628            -- We have glyph specific effects and these have a higher priority than
629            -- the font specific effects.
630            if effect ~= current_effect then
631                current_effect   = effect
632                tmrx             = 1
633                tmry             = 1
634                tmsy             = effect.slant   or fontparameters.slantfactor   or slant or 0
635                extend           = effect.extend  or fontparameters.extendfactor  or 1
636                squeeze          = effect.squeeze or fontparameters.squeezefactor or 1
637                need_mode        = effect.mode    or fontparameters.mode          or 0
638                need_width       = effect.weight  or fontparameters.weight        or 0
639                sx               = extend  * sx
640                sy               = squeeze * sy
641            else
642             -- we could check if effects have changed but effects use unique tables; for
643             -- now they win over font effects (only used in math)
644            end
645        else
646            -- These are the font specific effects and the scales are part of the main
647            -- font setup, so we use the factors and not the properties.effect data.
648            current_effect = nil
649            tmrx           = 1
650            tmry           = 1
651            --
652            local e = fontproperties.effect
653            if e then
654                tmsy       = e.slant   or slant or 0
655                extend     = e.extend  or 1
656                squeeze    = e.squeeze or 1
657                need_mode  = e.mode    or 0
658                need_width = e.weight  or 0
659                sx         = extend  * sx
660                sy         = squeeze * sy
661            else
662                tmsy       = slant or 0
663                extend     = 1
664                squeeze    = 1
665                need_mode  = 0
666                need_width = weight
667            end
668        end
669
670        tmef = expand
671        tmrx = expand * tmrx
672
673        current_font   = font
674        current_pdf    = usedfonts[font] -- cache
675        current_factor = factor
676        current_slant  = tmsy
677        current_weight = weight
678
679        local sc = fontparameters.size * bpfactor / 10
680        -- kind of special:
681        if format == "opentype" or format == "type1" then
682            sc = sc * 1000 / fontparameters.units
683            tj_scale = fontparameters.units / 1000
684        else
685            tj_scale = 1
686        end
687        tj_delta    = 0
688        tj_position = 0
689        --
690        tmsy = tmsy * sc
691        tmrx = tmrx * sc
692        tmry = tmry * sc
693        --
694        f_x_scale = sx
695        if f_x_scale ~= 1.0 then
696            tmrx = tmrx * f_x_scale
697        end
698        f_y_scale = sy
699        if f_y_scale ~= 1.0 then
700            tmsy = tmsy * f_y_scale
701            tmry = tmry * f_y_scale
702        end
703        --
704        tmef_f_x_scale = tmef * f_x_scale
705        --
706        if tmef_f_x_scale == 1.0 then
707            tmef_f_x_scale = 1
708        end
709        --
710        tj_scale = tj_scale * scalefactor / tmef_f_x_scale
711        --
712        tmef_f_w_scale = tmef_f_x_scale / extend
713        --
714        characterwidth   = characterwidths[font]
715     -- descriptionwidth = descriptionwidths[font]
716        --
717        hshift = fontparameters.hshift
718        vshift = fontparameters.vshift
719
720        if need_width > 0 then
721            if not need_mode or need_mode == 0 then
722                need_mode = 2
723            end
724            need_width = need_width * 2
725need_width = need_width * tmrx
726-- if hshift then
727--     hshift = hshift  * tmrx
728-- end
729        end
730
731        if current_weight > 0 then
732--             hshift  = (hshift or 0) + current_weight
733        end
734
735    end
736
737    local f_width = formatters["%.6N w"]
738    local f_mode  = formatters["%i Tr"]        -- can be hash
739    local f_font  = formatters["/F%i 10 Tf"]
740
741    local s_width = "0 w"
742    local s_mode  = "0 Tr"
743
744    local width_factor = 72.27 / 72.0
745
746    local last_fpdf
747
748    local function set_font()
749     -- if need_width and need_width ~= 0 then
750        if need_width ~= 0 then
751            b = b + 1 ; buffer[b] = f_width(width_factor*need_width)
752            done_width = true
753        elseif done_width then
754            b = b + 1 ; buffer[b] = s_width
755            done_width = false
756        end
757     -- if need_mode and need_mode ~= 0 then
758        if need_mode ~= 0 then
759            b = b + 1 ; buffer[b] = f_mode(need_mode)
760            done_mode = true
761        elseif done_mode then
762            b = b + 1 ; buffer[b] = s_mode
763            done_mode = false
764        end
765        -- no need when the same
766        if need_font or last_pdf ~= current_pdf then
767            b = b + 1 ; buffer[b] = f_font(current_pdf)
768            last_pdf  = current_pdf
769            need_font = false
770        end
771     -- f_pdf_cur = current_pdf
772        need_tf = false
773        need_tm = true
774    end
775
776    local function set_textmatrix(h,v)
777       local move = calc_pdfpos(h,v)
778       if need_tm or move then
779         -- b = b + 1 ; buffer[b] = f_tm(tmrx, tmsx, tmsy, tmry, tmtx*bpfactor, tmty*bpfactor)
780            b = b + 1 ; buffer[b] = f_tm(tmrx,       tmsy, tmry, tmtx*bpfactor, tmty*bpfactor)
781            pdf_h = saved_text_pos_h + tmtx
782            pdf_v = saved_text_pos_v + tmty
783            need_tm = false
784        end
785     -- cur_tmrx = tmrx
786    end
787
788    ----- f_skip = formatters["%.2N"]
789
790    -- I will redo this mess ... we no longer have the mkiv pdf generator that we used in
791    -- luatex (a precursor to lmtx and also for comparison) but only in lmtx now so ...
792    -- time to move on I guess.
793
794    -- factor is for hz
795
796    flushers.character = function(current,pos_h,pos_v,pos_r,font,char,data,csx,csy,factor,sx,sy,slant,weight) -- ,naturalwidth,width)
797
798        local s = data.scale
799        local x = data.xoffset
800        local y = data.yoffset
801        local effect = data.effect
802
803        if s then
804            sx = s * sx
805            sy = s * sy
806        end
807        if csx then
808            sx = sx * csx
809            csx = 1
810        end
811        if csy then
812            sy = sy * csy
813            csy = 1
814        end
815        if     sx ~= current_sx -- was: f_x_scale
816            or sy ~= current_sy -- was: f_x_scale
817            or need_tf
818            or font ~= current_font
819         -- or current_pdf ~= f_pdf_cur
820            or mode == "page"
821            or slant  ~= current_slant
822            or weight ~= current_weight
823            or effect ~= current_effect
824        then
825            pdf_goto_textmode()
826            setup_fontparameters(font,factor,sx,sy,slant,weight,effect) -- too often due to page
827            set_font()
828     -- elseif mode == "page" then
829     --     pdf_goto_textmode()
830     --     set_font()
831        elseif current_factor ~= factor
832         -- or cur_tmrx ~= tmrx
833        then
834            -- check if this happens and when
835            setup_fontparameters(font,factor,sx,sy,slant,weight,effect)
836            need_tm = true
837        end
838
839        if x then
840            pos_h = pos_h + x * tmef_f_x_scale
841        end
842        if y then
843            pos_v = pos_v + y * f_y_scale
844        end
845
846        local move = calc_pdfpos(pos_h,pos_v)
847
848        if move or need_tm then
849            if not need_tm then
850                if horizontalmode then
851                    if (saved_text_pos_v + tmty) ~= pdf_v then
852                        need_tm = true
853                    elseif tj_delta >= dupx or tj_delta <= dumx then
854                        need_tm = true
855                    end
856                else
857                    if (saved_text_pos_h + tmtx) ~= pdf_h then
858                        need_tm = true
859                    elseif tj_delta >= dupy or tj_delta <= dumy then
860                        need_tm = true
861                    end
862                end
863            end
864            if hshift then pos_h = pos_h + hshift end
865            if vshift then pos_v = pos_v - vshift end
866            if need_tm then
867                pdf_goto_textmode()
868                set_textmatrix(pos_h,pos_v)
869                begin_chararray()
870                move = calc_pdfpos(pos_h,pos_v)
871            end
872            if move then
873                local d = tj_delta * tj_scale
874                if d <= -0.5 or d >= 0.5 then
875                    if mode == "char" then
876                        end_charmode()
877                    end
878                    b = b + 1 ; buffer[b] = round(d) -- or f_skip(d)
879-- print(d,buffer[b])
880                end
881                tj_position = tj_position - tj_delta
882             -- tj_position = tj_position - tj_delta * tmef_f_x_scale
883            end
884        end
885
886        if mode == "chararray" then
887            begin_charmode()
888        end
889
890        -- When we have say 100 x without spaces we do accumulate some error but in practice
891        -- we have spaces so we sync. This is the same as in pdftex and luatex.
892
893        tj_position = tj_position + characterwidth[char] * tmef_f_w_scale
894
895        local slot = pdfcharacters[data.index or char] -- registers usage
896
897--         b = b + 1 ; buffer[b] = font > 0 and h_hex_4[slot] or h_hex_2[slot]
898        b = b + 1 ; buffer[b] = tohex[slot]
899
900    end
901
902    do
903
904        -- We like a little bit of optimization, just for the fun of it.
905
906        local spaces = setmetatableindex(function(t,font)
907            local bytes = false
908            local data  = characters[font]
909            if data then
910                data = data[32]
911                if data then
912                    local index = data.index
913                    if index then
914                        local dummy = usedfonts[font]
915                        local slot  = usedcharacters[font][index]
916                        if slot then
917--                             bytes = font > 0 and h_hex_4[slot] or h_hex_2[slot]
918                            bytes = tohex[slot]
919                        end
920                    end
921                end
922            end
923            t[font] = bytes
924            return bytes
925        end)
926
927        -- We have to be in the current font stream.
928
929        flushers.space = function(font)
930            if mode == "char" and font == current_font then
931                local slot = spaces[font]
932                if slot then
933                    tj_position = tj_position + characterwidth[32] * tmef_f_w_scale
934                    b = b + 1 ; buffer[b] = slot
935                end
936            end
937        end
938
939    end
940
941    flushers.fontchar = function(font,char,data)
942        local dummy = usedfonts[font]
943        local slot  = pdfcharacters[data.index or char] -- registers usage
944        return dummy, slot
945    end
946
947end
948
949-- literals
950
951local flushliteral  do
952
953    local nodeproperties      = nodes.properties.data
954    local literalvalues       = nodes.literalvalues
955
956    local originliteral_code  = literalvalues.origin
957    local pageliteral_code    = literalvalues.page
958    local alwaysliteral_code  = literalvalues.always
959    local rawliteral_code     = literalvalues.raw
960    local textliteral_code    = literalvalues.text
961    local fontliteral_code    = literalvalues.font
962
963    flushliteral = function(current,pos_h,pos_v)
964        local p = nodeproperties[current]
965        if p then
966            local str = p.data
967            if str and str ~= "" then
968                local mode = p.mode
969                if mode == originliteral_code then
970                    pdf_goto_pagemode()
971                    pdf_set_pos(pos_h,pos_v)
972                elseif mode == pageliteral_code then
973                    pdf_goto_pagemode()
974                elseif mode == textliteral_code then
975                    pdf_goto_textmode()
976                elseif mode == fontliteral_code then
977                    pdf_goto_fontmode()
978                elseif mode == alwaysliteral_code then -- aka direct
979                    pdf_end_string_nl()
980                    need_tm = true
981                elseif mode == rawliteral_code then
982                    pdf_end_string_nl()
983                else
984                    report("invalid literal mode %a when flushing %a",mode,str)
985                    return
986                end
987                b = b + 1 ; buffer[b] = str
988            end
989        end
990    end
991
992    flushers.literal = flushliteral
993
994    function lpdf.print(mode,str)
995        -- This only works inside objects, don't change this to flush
996        -- in between. It's different from luatex but okay.
997        if str then
998            mode = literalvalues[mode]
999        else
1000            mode, str = originliteral_code, mode
1001        end
1002        if str and str ~= "" then
1003            if mode == originliteral_code then
1004                pdf_goto_pagemode()
1005             -- pdf_set_pos(pdf_h,pdf_v)
1006            elseif mode == pageliteral_code then
1007                pdf_goto_pagemode()
1008            elseif mode == textliteral_code then
1009                pdf_goto_textmode()
1010            elseif mode == fontliteral_code then
1011                pdf_goto_fontmode()
1012            elseif mode == alwaysliteral_code then
1013                pdf_end_string_nl()
1014                need_tm = true
1015            elseif mode == rawliteral_code then
1016                pdf_end_string_nl()
1017            else
1018                report("invalid literal mode %a when flushing %a",mode,str)
1019                return
1020            end
1021            b = b + 1 ; buffer[b] = str
1022        end
1023    end
1024
1025end
1026
1027-- grouping & orientation
1028
1029do
1030
1031    local matrices     = { }
1032    local positions    = { }
1033    local nofpositions = 0
1034    local nofmatrices  = 0
1035
1036    local flushsave = function(current,pos_h,pos_v)
1037        nofpositions = nofpositions + 1
1038        positions[nofpositions] = { pos_h, pos_v, nofmatrices }
1039        pdf_goto_pagemode()
1040        pdf_set_pos(pos_h,pos_v)
1041        b = b + 1 ; buffer[b] = "q"
1042    end
1043
1044    local flushrestore = function(current,pos_h,pos_v)
1045        if nofpositions < 1 then
1046            return
1047        end
1048        local t = positions[nofpositions]
1049     -- local h = pos_h - t[1]
1050     -- local v = pos_v - t[2]
1051        if shippingmode == "page" then
1052            nofmatrices = t[3]
1053        end
1054        pdf_goto_pagemode()
1055        pdf_set_pos(pos_h,pos_v)
1056        b = b + 1 ; buffer[b] = "Q"
1057        nofpositions = nofpositions - 1
1058    end
1059
1060    local nodeproperties = nodes.properties.data
1061
1062    local s_matrix_0 <const> = "1 0 0 1 0 0 cm"
1063    local f_matrix_2         = formatters["%.6N 0 0 %.6N 0 0 cm"]
1064    local f_matrix_4         = formatters["%.6N %.6N %.6N %.6N 0 0 cm"]
1065
1066    local flushsetmatrix = function(current,pos_h,pos_v)
1067        local p = nodeproperties[current]
1068        if p then
1069            local m = p.matrix
1070            if m then
1071                local rx, sx, sy, ry = unpack(m)
1072                local s
1073                if not rx then
1074                    rx = 1
1075                elseif rx == 0 then
1076                    rx = 0.0001
1077                end
1078                if not ry then
1079                    ry = 1
1080                elseif ry == 0 then
1081                    ry = 0.0001
1082                end
1083                if not sx then
1084                    sx = 0
1085                end
1086                if not sy then
1087                    sy = 0
1088                end
1089                --
1090                if sx == 0 and sy == 0 then
1091                    if rx == 1 and ry == 1 then
1092                        s = s_matrix_0
1093                    else
1094                        s = f_matrix_2(rx,ry)
1095                    end
1096                else
1097                    s = f_matrix_4(rx,sx,sy,ry)
1098                end
1099                --
1100                if shippingmode == "page" then
1101                    local tx = pos_h * (1 - rx) - pos_v * sy
1102                    local ty = pos_v * (1 - ry) - pos_h * sx
1103                    if nofmatrices > 0 then
1104                        local t = matrices[nofmatrices]
1105                        local r_x, s_x, s_y, r_y, te, tf = t[1], t[2], t[3], t[4], t[5], t[6]
1106                        rx, sx = rx * r_x + sx * s_y, rx * s_x + sx * r_y
1107                        sy, ry = sy * r_x + ry * s_y, sy * s_x + ry * r_y
1108                        tx, ty = tx * r_x + ty * s_y, tx * s_x + ty * r_y
1109                    end
1110                    nofmatrices = nofmatrices + 1
1111                    matrices[nofmatrices] = { rx, sx, sy, ry, tx, ty }
1112                end
1113                --
1114                pdf_goto_pagemode()
1115                pdf_set_pos(pos_h,pos_v)
1116                --
1117                b = b + 1
1118                buffer[b] = s
1119            end
1120        end
1121    end
1122
1123    flushers.setmatrix = flushsetmatrix
1124    flushers.save      = flushsave
1125    flushers.restore   = flushrestore
1126
1127    function lpdf.hasmatrix()
1128        return nofmatrices > 0
1129    end
1130
1131    function lpdf.getmatrix()
1132        if nofmatrices > 0 then
1133            return unpack(matrices[nofmatrices])
1134        else
1135            return 1, 0, 0, 1, 0, 0
1136        end
1137    end
1138
1139    flushers.pushorientation = function(orientation,pos_h,pos_v,pos_r)
1140        pdf_goto_pagemode()
1141        pdf_set_pos(pos_h,pos_v)
1142        b = b + 1 ; buffer[b] = "q"
1143        if orientation == 1 then
1144            b = b + 1 ; buffer[b] = "0 -1 1 0 0 0 cm"  --  90
1145        elseif orientation == 2 then
1146            b = b + 1 ; buffer[b] = "-1 0 0 -1 0 0 cm" -- 180
1147        elseif orientation == 3 then
1148            b = b + 1 ; buffer[b] = "0 1 -1 0 0 0 cm"  -- 270
1149        end
1150    end
1151
1152    flushers.poporientation = function(orientation,pos_h,pos_v,pos_r)
1153        pdf_goto_pagemode()
1154        pdf_set_pos(pos_h,pos_v)
1155        b = b + 1 ; buffer[b] = "Q"
1156    end
1157
1158    --
1159
1160    flushers.startmatrix = function(current,pos_h,pos_v)
1161        flushsave(current,pos_h,pos_v)
1162        flushsetmatrix(current,pos_h,pos_v)
1163    end
1164
1165    flushers.stopmatrix = function(current,pos_h,pos_v)
1166        flushrestore(current,pos_h,pos_v)
1167    end
1168
1169    flushers.startscaling = function(current,pos_h,pos_v)
1170        flushsave(current,pos_h,pos_v)
1171        flushsetmatrix(current,pos_h,pos_v)
1172    end
1173
1174    flushers.stopscaling = function(current,pos_h,pos_v)
1175        flushrestore(current,pos_h,pos_v)
1176    end
1177
1178    flushers.startrotation = function(current,pos_h,pos_v)
1179        flushsave(current,pos_h,pos_v)
1180        flushsetmatrix(current,pos_h,pos_v)
1181    end
1182
1183    flushers.stoprotation = function(current,pos_h,pos_v)
1184        flushrestore(current,pos_h,pos_v)
1185    end
1186
1187    flushers.startmirroring = function(current,pos_h,pos_v)
1188        flushsave(current,pos_h,pos_v)
1189        flushsetmatrix(current,pos_h,pos_v)
1190    end
1191
1192    flushers.stopmirroring = function(current,pos_h,pos_v)
1193        flushrestore(current,pos_h,pos_v)
1194    end
1195
1196    flushers.startclipping = function(current,pos_h,pos_v)
1197        flushsave(current,pos_h,pos_v)
1198     -- lpdf.print("origin",formatters["0 w %s W n"](nodeproperties[current].path))
1199        pdf_goto_pagemode()
1200        b = b + 1 ; buffer[b] = formatters["0 w %s W n"](nodeproperties[current].path)
1201    end
1202
1203    flushers.stopclipping = function(current,pos_h,pos_v)
1204        flushrestore(current,pos_h,pos_v)
1205    end
1206
1207end
1208
1209do
1210
1211    local nodeproperties = nodes.properties.data
1212
1213    flushers.setstate = function(current,pos_h,pos_v)
1214        local p = nodeproperties[current]
1215        if p then
1216            local d = p.data
1217            if d and d ~= "" then
1218                pdf_goto_pagemode()
1219                b = b + 1 ; buffer[b] = d
1220            end
1221        end
1222    end
1223
1224end
1225
1226-- rules
1227
1228local flushedxforms  = { } -- actually box resources but can also be direct
1229local localconverter = nil -- will be set
1230
1231local flushimage  do
1232
1233    local tonut             = nodes.tonut
1234    local tonode            = nuts.tonode
1235
1236    local pdfbackend        = backends.registered.pdf
1237    local nodeinjections    = pdfbackend.nodeinjections
1238    local codeinjections    = pdfbackend.codeinjections
1239
1240    local newimagerule      = nuts.pool.imagerule
1241    local newboxrule        = nuts.pool.boxrule
1242
1243    local setprop           = nuts.setprop
1244    local getprop           = nuts.getprop
1245    local setattrlist       = nuts.setattrlist
1246
1247    local getwhd            = nuts.getwhd
1248    local flushlist         = nuts.flushlist
1249    local getdata           = nuts.getdata
1250
1251    local rulecodes         = nodes.rulecodes
1252    local normalrule_code   = rulecodes.normal
1253    local boxrule_code      = rulecodes.box
1254    local imagerule_code    = rulecodes.image
1255    local emptyrule_code    = rulecodes.empty
1256    local userrule_code     = rulecodes.user
1257    local overrule_code     = rulecodes.over
1258    local underrule_code    = rulecodes.under
1259    local fractionrule_code = rulecodes.fraction
1260    local radicalrule_code  = rulecodes.radical
1261    local outlinerule_code  = rulecodes.outline
1262    ----- virtualrule_code  = rulecodes.virtual
1263
1264    local processrule       = nodes.rules.process
1265
1266    local f_fm = formatters["/Fm%d Do"]
1267    local f_im = formatters["/Im%d Do"]
1268    local f_gr = formatters["/Gp%d Do"]
1269
1270    local s_b <const> = "q"
1271    local s_e <const> = "Q"
1272
1273    local f_v  = formatters["[] 0 d 0 J %.6N w 0 0 m %.6N 0 l S"]
1274    local f_h  = formatters["[] 0 d 0 J %.6N w 0 0 m 0 %.6N l S"]
1275
1276    local f_f  = formatters["0 0 %.6N %.6N re f"]
1277    local f_o  = formatters["[] 0 d 0 J 0 0 %.6N %.6N re S"]
1278    local f_w  = formatters["[] 0 d 0 J %.6N w 0 0 %.6N %.6N re S"]
1279
1280    local f_b  = formatters["%.6N w 0 %.6N %.6N %.6N re f"]
1281    local f_x  = formatters["[] 0 d 0 J %.6N w %.6N %.6N %.6N %.6N re S"]
1282    local f_y  = formatters["[] 0 d 0 J %.6N w %.6N %.6N %.6N %.6N re S %.6N 0 m %.6N 0 l S"]
1283
1284    -- Historically the index is an object which is kind of bad.
1285
1286    local boxresources, n = { }, 0
1287
1288    getxformname = function(index)
1289        local l = boxresources[index]
1290        if l then
1291            return l.name
1292        else
1293            report("no box resource %S",index)
1294        end
1295    end
1296
1297    lpdf.getxformname = getxformname
1298
1299    local pdfcollectedresources = lpdf.collectedresources
1300
1301    function codeinjections.saveboxresource(box,attributes,resources,immediate,kind,margin,onum)
1302        n = n + 1
1303        local immediate = true
1304        local margin    = margin or 0 -- or dimension
1305        local objnum    = onum or pdfreserveobject()
1306        local list      = tonut(type(box) == "number" and tex.takebox(box) or box)
1307        --
1308        if resources == true then
1309            resources = pdfcollectedresources()
1310        end
1311        --
1312        local width, height, depth = getwhd(list)
1313        --
1314        local l = {
1315            width      = width,
1316            height     = height,
1317            depth      = depth,
1318            margin     = margin,
1319            attributes = attributes,
1320            resources  = resources,
1321            list       = nil,
1322            type       = kind,
1323            name       = n,
1324            index      = objnum,
1325            objnum     = objnum,
1326        }
1327        local r = boxresources[objnum]
1328        if r then
1329            flushlist(l.list)
1330            l.list = nil -- added
1331        end
1332        boxresources[objnum] = l
1333        if immediate then
1334            localconverter(list,"xform",objnum,l)
1335            flushedxforms[objnum] = { true , objnum }
1336            flushlist(list)
1337        else
1338            l.list = list
1339        end
1340        return objnum
1341    end
1342
1343    function nodeinjections.useboxresource(index,wd,ht,dp)
1344        local l = boxresources[index]
1345        if l then
1346            if wd or ht or dp then
1347                wd, ht, dp = wd or 0, ht or 0, dp or 0
1348            else
1349                wd, ht, dp = l.width, l.height, l.depth
1350            end
1351            local rule = newboxrule(wd,ht,dp)
1352            setattrlist(rule,true)
1353            setprop(rule,"index",index)
1354            return tonode(rule), wd, ht, dp
1355        else
1356            report("no box resource %S",index)
1357        end
1358    end
1359
1360    local function getboxresourcedimensions(index)
1361        local l = boxresources[index]
1362        if l then
1363            return l.width, l.height, l.depth, l.margin
1364        else
1365            report("no box resource %S",index)
1366        end
1367    end
1368
1369    nodeinjections.getboxresourcedimensions = getboxresourcedimensions
1370
1371    function codeinjections.getboxresourcebox(index)
1372        local l = boxresources[index]
1373        if l then
1374            return l.list
1375        end
1376    end
1377
1378    -- a bit of a mess: index is now objnum but that has to change to a proper index
1379    -- ... an engine inheritance
1380
1381    local function flushpdfxform(current,pos_h,pos_v,pos_r,size_h,size_v)
1382        -- object properties
1383        local objnum = getprop(current,"index")
1384        local name   = getxformname(objnum)
1385        local info   = flushedxforms[objnum]
1386        local r      = boxresources[objnum]
1387        if not info then
1388            info = { false , objnum }
1389            flushedxforms[objnum] = info
1390        end
1391        local wd, ht, dp = getboxresourcedimensions(objnum)
1392     -- or:   wd, ht, dp = r.width, r.height, r.depth
1393        -- sanity check
1394        local htdp = ht + dp
1395        if wd == 0 or size_h == 0 or htdp == 0 or size_v == 0 then
1396            return
1397        end
1398        -- calculate scale
1399        local rx, ry = 1, 1
1400        if wd ~= size_h or htdp ~= size_v then
1401            rx = size_h / wd
1402            ry = size_v / htdp
1403        end
1404        -- flush the reference
1405        usedxforms[objnum] = true
1406        pdf_goto_pagemode()
1407        calc_pdfpos(pos_h,pos_v)
1408        local tx = cmtx * bpfactor
1409        local ty = cmty * bpfactor
1410        b = b + 1 ; buffer[b] = s_b
1411     -- b = b + 1 ; buffer[b] = f_cm(rx,0,0,ry,tx,ty)
1412        b = b + 1 ; buffer[b] = f_cz(rx,    ry,tx,ty)
1413        b = b + 1 ; buffer[b] = f_fm(name)
1414        b = b + 1 ; buffer[b] = s_e
1415    end
1416
1417    -- place image also used in vf but we can use a different one if we need it
1418
1419    local imagetypes     = images.types -- pdf png jpg jp2 jbig2 stream
1420    local img_none       = imagetypes.none
1421    local img_pdf        = imagetypes.pdf
1422    local img_stream     = imagetypes.stream
1423
1424    local one_bp = 65536 * bpfactor
1425
1426    local imageresources, n = { }, 0
1427
1428    getximagename = function(index) -- not used
1429        local l = imageresources[index]
1430        if l then
1431            return l.name
1432        else
1433            report("no image resource %S",index)
1434        end
1435    end
1436
1437    -- Groups are flushed immediately but we can decide to make them into a
1438    -- specific whatsit ... but not now. We could hash them if needed when
1439    -- we use lot sof them in mp ... but not now.
1440
1441          usedxgroups = { }
1442    local groups      = 0
1443    local group       = nil
1444
1445    local flushgroup = function(content,bbox)
1446        if not group then
1447            group = pdfdictionary {
1448                Type = pdfconstant("Group"),
1449                S    = pdfconstant("Transparency"),
1450            }
1451        end
1452        local wrapper = pdfdictionary {
1453            Type      = pdf_xobject,
1454            Subtype   = pdf_form,
1455            FormType  = 1,
1456            Group     = group,
1457            BBox      = pdfarray(bbox),
1458            Resources = lpdf.collectedresources { serialize = false },
1459        }
1460        local objnum = pdfflushstreamobject(content,wrapper,false) -- why not compressed ?
1461        groups = groups + 1
1462        usedxgroups[groups] = objnum
1463        return f_gr(groups)
1464    end
1465
1466    flushers.group  = flushgroup
1467    lpdf.flushgroup = flushgroup -- todo: access via driver in mlib-pps
1468
1469    -- end of experiment
1470
1471    local function flushpdfximage(current,pos_h,pos_v,pos_r,size_h,size_v)
1472
1473        local width,
1474              height,
1475              depth     = getwhd(current)
1476        local total     = height + depth
1477        local transform = getprop(current,"transform") or 0  -- we never set it ... so just use rotation then
1478        local index     = getprop(current,"index") or 0
1479        local kind,
1480              xorigin,
1481              yorigin,
1482              xsize,
1483              ysize,
1484              rotation, -- transform / orientation / rotation : it's a mess (i need to redo this)
1485              objnum,
1486              groupref  = pdfincludeimage(index)  -- needs to be sorted out, bad name (no longer mixed anyway)
1487
1488        if not kind then
1489            report("invalid image %S",index)
1490            return
1491        end
1492
1493        local rx, sx, sy, ry, tx, ty = 1, 0, 0, 1, 0, 0
1494
1495        -- tricky: xsize and ysize swapped
1496
1497        if kind == img_pdf or kind == img_stream then
1498            rx, ry, tx, ty = 1/xsize, 1/ysize, xorigin/xsize, yorigin/ysize
1499        else
1500         -- if kind == img_png then
1501         --  -- if groupref > 0 and img_page_group_val == 0 then
1502         --  --     img_page_group_val = groupref
1503         --  -- end
1504         -- end
1505            rx, ry = bpfactor, bpfactor
1506        end
1507
1508        if (transform & 7) > 3 then
1509            -- mirror
1510            rx, tx = -rx, -tx
1511        end
1512        local t = (transform + rotation) & 3
1513        if t == 0 then
1514            -- nothing
1515        elseif t == 1 then
1516            -- rotation over 90 degrees (counterclockwise)
1517            rx, sx, sy, ry, tx, ty = 0, rx, -ry, 0, -ty, tx
1518        elseif t == 2 then
1519            -- rotation over 180 degrees (counterclockwise)
1520            rx, ry, tx, ty = -rx, -ry, -tx, -ty
1521        elseif t == 3 then
1522            -- rotation over 270 degrees (counterclockwise)
1523            rx, sx, sy, ry, tx, ty = 0, -rx, ry, 0, ty, -tx
1524        end
1525
1526        rx = rx * width
1527        sx = sx * total
1528        sy = sy * width
1529        ry = ry * total
1530        tx = pos_h - tx * width
1531        ty = pos_v - ty * total
1532
1533        local t = transform + rotation
1534
1535        if (transform & 7) > 3 then
1536            t = t + 1
1537        end
1538
1539        t = t & 3
1540
1541        if t == 0 then
1542            -- no transform
1543        elseif t == 1 then
1544            -- rotation over 90 degrees (counterclockwise)
1545            tx = tx + width
1546        elseif t == 2 then
1547            -- rotation over 180 degrees (counterclockwise)
1548            tx = tx + width
1549            ty = ty + total
1550        elseif t == 3 then
1551            -- rotation over 270 degrees (counterclockwise)
1552            ty = ty + total
1553        end
1554
1555     -- a flaw in original, can go:
1556     --
1557     -- if img_page_group_val == 0 then
1558     --     img_page_group_val = group_ref
1559     -- end
1560
1561        usedximages[index] = objnum -- hm
1562
1563        pdf_goto_pagemode()
1564
1565        calc_pdfpos(tx,ty)
1566
1567        tx = cmtx * bpfactor
1568        ty = cmty * bpfactor
1569
1570        b = b + 1 ; buffer[b] = s_b
1571        b = b + 1 ; buffer[b] = f_cm(rx,sx,sy,ry,tx,ty)
1572        b = b + 1 ; buffer[b] = f_im(index)
1573        b = b + 1 ; buffer[b] = s_e
1574    end
1575
1576    flushimage = function(index,width,height,depth,pos_h,pos_v)
1577
1578        -- used in vf characters
1579
1580        local total = height + depth
1581        local kind,
1582              xorigin, yorigin,
1583              xsize, ysize,
1584              rotation,
1585              objnum,
1586              groupref = pdfincludeimage(index)
1587
1588        local rx = width / xsize
1589        local sx = 0
1590        local sy = 0
1591        local ry = total / ysize
1592        local tx = pos_h
1593        -- to be sorted out
1594     -- local ty = pos_v - depth
1595        local ty = pos_v -- we assume that depth is dealt with in the caller (for now)
1596        usedximages[index] = objnum
1597
1598        pdf_goto_pagemode()
1599
1600        calc_pdfpos(tx,ty)
1601
1602        tx = cmtx * bpfactor
1603        ty = cmty * bpfactor
1604
1605        b = b + 1 ; buffer[b] = s_b
1606        b = b + 1 ; buffer[b] = f_cm(rx,sx,sy,ry,tx,ty)
1607        b = b + 1 ; buffer[b] = f_im(index)
1608        b = b + 1 ; buffer[b] = s_e
1609    end
1610
1611    flushers.image = flushimage
1612
1613    -- For the moment we need this hack because the engine checks the 'image'
1614    -- command in virtual fonts (so we use lua instead).
1615    --
1616    -- These will be replaced by a new more advanced one ... some day ... or
1617    -- never because the next are like the other engines and compensate for
1618    -- small sizes which is needed for inaccurate viewers.
1619
1620    flushers.rule = function(current,pos_h,pos_v,pos_r,size_h,size_v,subtype)
1621        if subtype == emptyrule_code then
1622            return
1623        elseif subtype == boxrule_code then
1624            return flushpdfxform(current,pos_h,pos_v,pos_r,size_h,size_v)
1625        elseif subtype == imagerule_code then
1626            return flushpdfximage(current,pos_h,pos_v,pos_r,size_h,size_v)
1627        elseif subtype == userrule_code or (subtype >= overrule_code and subtype <= radicalrule_code) then
1628            pdf_goto_pagemode()
1629            b = b + 1 ; buffer[b] = s_b
1630            pdf_set_pos_temp(pos_h,pos_v)
1631            processrule(current,size_h,size_v,pos_r) -- so we pass direction
1632            b = b + 1 ; buffer[b] = s_e
1633            return
1634        end
1635
1636        pdf_goto_pagemode()
1637
1638        b = b + 1 ; buffer[b] = s_b
1639
1640        local dim_h = size_h * bpfactor
1641        local dim_v = size_v * bpfactor
1642        local rule
1643        --
1644        -- this fails for showglyphs so and i have no reason to look into it now and rectangles
1645        -- do a better job anyway
1646        --
1647        if subtype == outlinerule_code then
1648            local linewidth = getdata(current)
1649            pdf_set_pos_temp(pos_h,pos_v)
1650            if linewidth > 0 then
1651                rule = f_w(linewidth * bpfactor,dim_h,dim_v)
1652            else
1653                rule = f_o(dim_h,dim_v)
1654            end
1655        elseif dim_v <= one_bp then
1656            pdf_set_pos_temp(pos_h,pos_v + 0.5 * size_v)
1657            rule = f_v(dim_v,dim_h)
1658        elseif dim_h <= one_bp then
1659            pdf_set_pos_temp(pos_h + 0.5 * size_h,pos_v)
1660            rule = f_h(dim_h,dim_v)
1661        else
1662            pdf_set_pos_temp(pos_h,pos_v)
1663            rule = f_f(dim_h,dim_v)
1664        end
1665
1666        b = b + 1 ; buffer[b] = rule
1667        b = b + 1 ; buffer[b] = s_e
1668
1669    end
1670
1671    flushers.simplerule = function(pos_h,pos_v,pos_r,size_h,size_v)
1672        pdf_goto_pagemode()
1673
1674        b = b + 1 ; buffer[b] = s_b
1675
1676        local dim_h = size_h * bpfactor
1677        local dim_v = size_v * bpfactor
1678        local rule
1679
1680        if dim_v <= one_bp then
1681            pdf_set_pos_temp(pos_h,pos_v + 0.5 * size_v)
1682            rule = f_v(dim_v,dim_h)
1683        elseif dim_h <= one_bp then
1684            pdf_set_pos_temp(pos_h + 0.5 * size_h,pos_v)
1685            rule = f_h(dim_h,dim_v)
1686        else
1687            pdf_set_pos_temp(pos_h,pos_v)
1688            rule = f_f(dim_h,dim_v)
1689        end
1690
1691        b = b + 1 ; buffer[b] = rule
1692        b = b + 1 ; buffer[b] = s_e
1693    end
1694
1695    flushers.specialrule = function(pos_h,pos_v,pos_r,width,height,depth,line,outline,baseline)
1696        pdf_goto_pagemode()
1697
1698        b = b + 1 ; buffer[b] = s_b
1699
1700        local width  = bpfactor * width
1701        local height = bpfactor * height
1702        local depth  = bpfactor * depth
1703        local total  = height + depth
1704        local line   = bpfactor * line
1705        local half   = line / 2
1706        local rule
1707
1708        if outline then
1709            local d = -depth + half
1710            local w =  width - line
1711            local t =  total - line
1712            if baseline and w > 0 then
1713                rule = f_y(line,half,d,w,t,half,w)
1714            else
1715                rule = f_x(line,half,d,w,t)
1716            end
1717        else
1718            rule = f_b(line,-depth,width,total)
1719        end
1720        pdf_set_pos_temp(pos_h,pos_v)
1721
1722        b = b + 1 ; buffer[b] = rule
1723        b = b + 1 ; buffer[b] = s_e
1724    end
1725
1726end
1727
1728--- basics
1729
1730local wrapupdocument, registerpage  do
1731
1732    local pages    = { }
1733    local maxkids  = 10
1734    local nofpages = 0
1735    local pagetag  = "unset"
1736
1737    registerpage = function(object)
1738        nofpages = nofpages + 1
1739        local objnum = pdfpagereference(nofpages)
1740        pages[nofpages] = {
1741            page   = nofpages, -- original number, only for diagnostics
1742            objnum = objnum,
1743            object = object,
1744            tag    = pagetag,
1745        }
1746    end
1747
1748    function lpdf.setpagetag(tag)
1749        pagetag = tag or "unset"
1750    end
1751
1752    function lpdf.getnofpages()
1753        return nofpages
1754    end
1755
1756    function lpdf.getpagetags()
1757        local list = { }
1758        for i=1,nofpages do
1759            list[i] = pages[i].tag
1760        end
1761        return list
1762    end
1763
1764    function lpdf.setpageorder(mapping,p)
1765        -- mapping can be a hash so:
1766        local list = table.sortedkeys(mapping)
1767        local n    = #list
1768        local nop  = p or nofpages
1769        if n == nop then
1770            local done = { }
1771            local hash = { }
1772            for i=1,n do
1773                local order = mapping[list[i]]
1774                if hash[order] then
1775                    report("invalid page order, duplicate entry %i",order)
1776                    return
1777                elseif order < 1 or order > nofpages then
1778                    report("invalid page order, no page %i",order)
1779                    return
1780                else
1781                    done[i]     = pages[order]
1782                    hash[order] = true
1783                end
1784            end
1785            pages = done
1786        else
1787            report("invalid page order, %i entries expected",nop)
1788        end
1789    end
1790
1791    -- We can have this, but then via codeinjections etc. Later.
1792
1793 -- function structures.pages.swapthem()
1794 --     local n = lpdf.getnofpages()
1795 --     local t = { }
1796 --     for i=1,n do
1797 --         t[i] = i
1798 --     end
1799 --     for i=2,math.odd(n) and n or (n-1),2 do
1800 --         t[i]   = i+1
1801 --         t[i+1] = i
1802 --     end
1803 --     lpdf.setpageorder(t)
1804 -- end
1805
1806    wrapupdocument = function(driver)
1807
1808        -- hook (to reshuffle pages)
1809        local pagetree = { }
1810        local parent   = nil
1811        local minimum  = 0
1812        local maximum  = 0
1813        local current  = 0
1814        if #pages > 1.5 * maxkids then
1815            repeat
1816                local plist, pnode
1817                if current == 0 then
1818                    plist, minimum = pages, 1
1819                elseif current == 1 then
1820                    plist, minimum = pagetree, 1
1821                else
1822                    plist, minimum = pagetree, maximum + 1
1823                end
1824                maximum = #plist
1825                if maximum > minimum then
1826                    local kids
1827                    for i=minimum,maximum do
1828                        local p = plist[i]
1829                        if not pnode or #kids == maxkids then
1830                            kids   = pdfarray()
1831                            parent = pdfreserveobject()
1832                            pnode  = pdfdictionary {
1833                                objnum = parent,
1834                                Type   = pdf_pages,
1835                                Kids   = kids,
1836                                Count  = 0,
1837                            }
1838                            pagetree[#pagetree+1] = pnode
1839                        end
1840                        kids[#kids+1] = pdfreference(p.objnum)
1841                        pnode.Count = pnode.Count + (p.Count or 1)
1842                        p.Parent = pdfreference(parent)
1843                    end
1844                end
1845                current = current + 1
1846            until maximum == minimum
1847            -- flush page tree
1848            for i=1,#pagetree do
1849                local entry  = pagetree[i]
1850                local objnum = entry.objnum
1851                entry.objnum = nil
1852                pdfflushobject(objnum,entry)
1853            end
1854        else
1855            -- ugly
1856            local kids = pdfarray()
1857            local list = pdfdictionary {
1858                Type  = pdf_pages,
1859                Kids  = kids,
1860                Count = nofpages,
1861            }
1862            parent = pdfreserveobject()
1863            for i=1,nofpages do
1864                local page = pages[i]
1865                kids[#kids+1] = pdfreference(page.objnum)
1866                page.Parent = pdfreference(parent)
1867            end
1868            pdfflushobject(parent,list)
1869        end
1870        for i=1,nofpages do
1871            local page   = pages[i]
1872            local object = page.object
1873            object.Parent = page.Parent
1874            pdfflushobject(page.objnum,object)
1875        end
1876        lpdf.addtocatalog("Pages",pdfreference(parent))
1877    end
1878
1879end
1880
1881local function initialize(driver,details)
1882    reset_variables(details)
1883    reset_buffer()
1884end
1885
1886-- This will all move and be merged and become less messy.
1887
1888-- todo: more clever resource management: a bit tricky as we can inject
1889-- stuff in the page stream
1890
1891local compact       = false
1892local encryptstream = false
1893local encryptobject = false
1894local encdict       = nil
1895local majorversion  = 1
1896local minorversion  = 7
1897
1898-- Encryption
1899
1900-- This stuff is poorly documented so it took a while to figure out a way that made
1901-- loading in a few programe working. Of course one you see the solution one can
1902-- claim that it's easy and trivial. In the end we could even make acrobat accepting
1903-- the file: it doesn't like the catalog to be in an object stream which to me
1904-- smells like a bug.
1905
1906do
1907
1908    -- move up (some already) or better: lpdf-aes.lmt or so
1909
1910    local byte, sub, bytes, tohex, tobytes = string.byte, string.sub, string.bytes, string.tohex, string.tobytes
1911    local P, S, V, Cs, lpegmatch, patterns = lpeg.P, lpeg.S, lpeg.V, lpeg.Cs, lpeg.match, lpeg.patterns
1912
1913    local digest256 = sha2.digest256
1914    local digest384 = sha2.digest384
1915    local digest512 = sha2.digest512
1916
1917    local aesencode = aes.encode
1918    local aesdecode = aes.decode
1919    local aesrandom = aes.random
1920
1921    -- random and padding functions are gone here
1922
1923    local function validpassword(str)
1924        return #str > 127 and sub(str,1,127) or str
1925    end
1926
1927    local encryptionkey = false
1928    local objectparser  = false
1929
1930    do
1931
1932        local function ps_encrypt(str)
1933            -- string is already unescaped
1934            str = aesencode(str,encryptionkey,true,true,true)
1935            return "<" .. tohex(str) .. ">"
1936        end
1937
1938        local function hex_encrypt(str)
1939            -- string needs to be decoded
1940            str = tobytes(str)
1941            str = aesencode(str,encryptionkey,true,true,true)
1942            return "<" .. tohex(str) .. ">"
1943        end
1944
1945        local whitespace  = S("\000\009\010\012\013\032")^1
1946        local anything    = patterns.anything
1947        local space       = patterns.space
1948        local spacing     = whitespace^0
1949        local newline     = patterns.eol
1950        local cardinal    = patterns.cardinal
1951
1952        local p_psstring  = (
1953                              P("(")
1954                            * Cs(P { ( P("\\")/"" * anything + P("(") * V(1) * P(")") + (1 - P(")")) )^0 })
1955                            * P(")")
1956                          ) / ps_encrypt
1957
1958        local p_hexstring = (
1959                              P("<")
1960                            * Cs((1-P(">"))^1)
1961                            * P(">")
1962                          ) / hex_encrypt
1963
1964        local p_comment   = P("%") * (1-newline)^1 * newline^1
1965        local p_name      = P("/") * (1 - whitespace - S("<>/[]()"))^1
1966        local p_number    = patterns.number
1967        local p_boolean   = P("true") + P("false")
1968        local p_null      = P("null")
1969        local p_reference = cardinal * spacing * cardinal * spacing * P("R")
1970
1971        local p_other     = p_name + p_reference + p_psstring + p_hexstring + p_number
1972                          + p_boolean + p_null + p_comment
1973
1974        local p_dictionary  = { "dictionary",
1975            dictionary = (
1976                P("<<")
1977              * (spacing * p_name * spacing * V("whatever"))^0
1978              * spacing
1979              * P(">>")
1980            ),
1981            array = (
1982                P("[")
1983              * (spacing * V("whatever"))^0
1984              * spacing
1985              * P("]")
1986            ),
1987            whatever = (
1988                V("dictionary")
1989              + V("array")
1990              + p_other
1991            ),
1992        }
1993
1994        local p_object = P { "object",
1995            dictionary = p_dictionary.dictionary,
1996            array      = p_dictionary.array,
1997            whatever   = p_dictionary.whatever,
1998            object     = spacing * (V("dictionary") + V("array") + p_other)
1999        }
2000
2001     -- local p_object = cardinal
2002     --                * spacing
2003     --                * cardinal
2004     --                * spacing
2005     --                * P("obj")
2006     --                * p_object
2007     --                * P(1)^0
2008     --
2009     -- objectparser = Cs(p_object^1)
2010
2011        objectparser = Cs(p_object^1)
2012
2013    end
2014
2015    local function makehash(password,salt,userkey)
2016        local k = digest256(password .. salt .. (userkey or ""))
2017        local n = 0
2018        while true do
2019            local k1 = rep(password .. k .. (userkey or ""),64)
2020            local k2 = sub(k,1,16)
2021            local iv = sub(k,17,32)
2022            local e = aesencode(k1,k2,iv)
2023            local m = 0
2024            local i = 1
2025            for b in bytes(e) do
2026                m = m + b
2027                if i == 16 then
2028                    break
2029                else
2030                    i = i + 1
2031                end
2032            end
2033            m = m % 3
2034            if m == 0 then
2035                k = digest256(e)
2036            elseif m == 1 then
2037                k = digest384(e)
2038            else
2039                k = digest512(e)
2040            end
2041            n = n + 1
2042            if n >= 64 and byte(sub(e,-1)) <= (n - 32) then
2043                break
2044           end
2045        end
2046        return sub(k,1,32)
2047    end
2048
2049    local options = {
2050     -- unknown  = 0x0001, -- bit  1
2051     -- unknown  = 0x0002, -- bit  2
2052        print    = 0x0004, -- bit  3
2053        modify   = 0x0008, -- bit  4
2054        extract  = 0x0010, -- bit  5
2055        add      = 0x0020, -- bit  6
2056     -- unknown  = 0x0040, -- bit  7
2057     -- unknown  = 0x0080, -- bit  8
2058        fillin   = 0x0100, -- bit  9
2059        access   = 0x0200, -- bit 10
2060        assemble = 0x0400, -- bit 11
2061        quality  = 0x0800, -- bit 12
2062     -- unknown  = 0x1000, -- bit 13
2063     -- unknown  = 0x2000, -- bit 14
2064     -- unknown  = 0x4000, -- bit 15
2065     -- unknown  = 0x8000, -- bit 16
2066    }
2067
2068    -- 1111 0000 1100 0011
2069
2070    local mandate  = 0x0200
2071    local defaults = options.print | options.extract | options.quality
2072
2073    -- majorversion = 2
2074    -- minorversion = 0
2075
2076    function lpdf.setencryption(specification)
2077        if not encryptstream then
2078            local ownerpassword = specification.ownerpassword
2079            local userpassword  = specification.userpassword
2080            local optionlist    = specification.permissions
2081            if type(ownerpassword) == "string" and ownerpassword ~= "" then
2082                --
2083                if type(userpassword) ~= "string" then
2084                    userpassword = ""
2085                end
2086                userpassword  = validpassword(userpassword)
2087                ownerpassword = validpassword(ownerpassword)
2088                --
2089                encryptionkey = aesrandom(32) -- used earlier on
2090                --
2091                local permissions = mandate
2092                if optionlist then
2093                    optionlist = utilities.parsers.settings_to_array(optionlist)
2094                    for i=1,#optionlist do
2095                        local p = options[optionlist[i]]
2096                        if p then
2097                            permissions = permissions | p
2098                        end
2099                    end
2100                else
2101                    permissions = permissions | defaults
2102                end
2103                --
2104                permissions = permissions | 0xF0C3 -- needs work
2105                --
2106                optionlist = { }
2107                for k, v in sortedhash(options) do
2108                    if permissions & v == v then
2109                        optionlist[#optionlist+1] = k
2110                    end
2111                end
2112                --
2113                local uservalidationsalt  = aesrandom(8)
2114                local userkeysalt         = aesrandom(8)
2115                local userhash            = makehash(userpassword,uservalidationsalt)
2116                local userkey             = userhash .. uservalidationsalt .. userkeysalt -- U
2117                local userintermediate    = makehash(userpassword,userkeysalt)
2118                local useraes             = aesencode(encryptionkey,userintermediate) -- UE
2119                --
2120                local ownervalidationsalt = aesrandom(8)
2121                local ownerkeysalt        = aesrandom(8)
2122                local ownerhash           = makehash(ownerpassword,ownervalidationsalt,userkey)
2123                local ownerkey            = ownerhash .. ownervalidationsalt .. ownerkeysalt -- O
2124                local ownerintermediate   = makehash(ownerpassword,ownerkeysalt,userkey)
2125                local owneraes            = aesencode(encryptionkey,ownerintermediate) -- OE
2126                --
2127                -- still not ok test in qpdf
2128                --
2129                local permissionsstring   = sio.tocardinal4(0xFFFFFFFF)
2130                                         .. sio.tocardinal4(permissions)
2131                                         .. "T" -- EncryptMetadata
2132                                         .. "adb"
2133                                         .. aesrandom(4)
2134                local permissionsaes      = aesencode(permissionsstring,encryptionkey)
2135                --
2136                permissionsaes = tohex(permissionsaes)
2137                userkey        = tohex(userkey)
2138                ownerkey       = tohex(ownerkey)
2139                useraes        = tohex(useraes)
2140                owneraes       = tohex(owneraes)
2141                --
2142                encdict  = pdfdictionary {
2143                    Filter = pdfconstant("Standard"),
2144                    V      = 5,   -- variant
2145                    R      = 6,   -- revision
2146                    Length = 256, -- not needed
2147                    StmF   = pdfconstant("StdCF"),
2148                    StrF   = pdfconstant("StdCF"),
2149                    P      = permissions,
2150                    Perms  = pdfliteral(permissionsaes,true), -- #16
2151                    U      = pdfliteral(userkey,       true), -- #48
2152                    O      = pdfliteral(ownerkey,      true), -- #48
2153                    UE     = pdfliteral(useraes,       true), -- #32
2154                    OE     = pdfliteral(owneraes,      true), -- #32
2155                    CF     = {
2156                        StdCF = {
2157                            AuthEvent = pdfconstant("DocOpen"),
2158                            CFM       = pdfconstant("AESV3"),
2159                            Length    = 32, -- #encryptionkey
2160                        }
2161                    },
2162                    -- bonus
2163                    EncryptMetadata = true,
2164                }
2165                --
2166                encryptstream = function(str)
2167                    return aesencode(str,encryptionkey,true,true,true) -- random-iv add-iv add-padding
2168                end
2169                encryptobject = function(obj)
2170                    if obj then
2171                        if type(obj) == "table" then
2172                            obj = obj()
2173                        end
2174                        return lpegmatch(objectparser,obj) or obj
2175                    end
2176                end
2177                --
2178                report_encryption("stream objects get encrypted")
2179                if not objectstream then
2180                    report_encryption("strings are not encrypted, enable object streams")
2181                end
2182                report_encryption("permissions: % t",optionlist)
2183                if userpassword == "" then
2184                    report_encryption("no user password")
2185                end
2186                --
2187            end
2188        end
2189    end
2190
2191    backends.registered.pdf.codeinjections.setencryption = lpdf.setencryption
2192
2193end
2194
2195do
2196
2197    -- This is more a convenience feature and it might even be not entirely robust.
2198    -- It removes redundant color directives which makes the page stream look a bit
2199    -- nicer (also when figuring out issues). I might add more here but there is
2200    -- some additional overhead involved so runtime can be impacted.
2201
2202    local P, R, S, Cs, lpegmatch = lpeg.P, lpeg.R, lpeg.S, lpeg.Cs, lpeg.match
2203
2204    local p_ds    = (R("09") + S(" ."))^1
2205    ----- p_nl    = S("\n\r")^1
2206    local p_nl    = S("\n")^1
2207    local p_eg    = P("Q")
2208
2209    local p_cl    = p_ds * (P("rg") + P("g") + P("k")) * p_ds * (P("RG") + P("G") + P("K"))
2210    ----- p_cl    = (p_ds * (P("rg") + P("g") + P("k") + P("RG") + P("G") + P("K")))^1
2211    local p_tr    = P("/Tr") * p_ds * P("gs")
2212
2213    local p_no_cl = (p_cl * p_nl) / ""
2214    local p_no_tr = (p_tr * p_nl) / ""
2215    local p_no_nl = 1 - p_nl
2216
2217    local p_do_cl = p_cl * p_nl
2218    local p_do_tr = p_tr * p_nl
2219
2220    local p_do_eg = p_eg * p_nl
2221
2222    local pattern = Cs( (
2223        (p_no_cl + p_no_tr)^0 * p_do_eg           -- transparencies and colors before Q
2224      +  p_no_tr * p_no_cl    * p_do_tr * p_do_cl -- transparencies and colors before others
2225      +  p_no_cl              * p_do_cl           -- successive colors
2226      +  p_no_tr              * p_do_tr           -- successive transparencies
2227      +  p_no_nl^1
2228      +  1
2229    )^1 )
2230
2231    local oldsize = 0
2232    local newsize = 0
2233
2234    directives.register("backend.pdf.compact", function(v)
2235        compact = v and function(s)
2236            oldsize = oldsize + #s
2237            s = lpegmatch(pattern,s) or s
2238            newsize = newsize + #s
2239            return s
2240        end
2241    end)
2242
2243    statistics.register("pdf pagestream",function()
2244        if oldsize ~= newsize then
2245            return string.format("old size: %i, new size %i",oldsize,newsize)
2246        end
2247    end)
2248
2249end
2250
2251local flushdeferred -- defined later
2252
2253local level = 0
2254local state = true
2255
2256function lpdf.setpagestate(s)
2257    state = s
2258end
2259
2260local finalize  do
2261
2262    local f_font  = formatters["F%d"]
2263
2264    local f_form  = formatters["Fm%d"]
2265    local f_group = formatters["Gp%d"]
2266    local f_image = formatters["Im%d"]
2267
2268    local function checkedbox(mediabox,otherbox,what)
2269        if otherbox and #mediabox == 4 and #otherbox == 4 then
2270            local done = false
2271            if otherbox[1] < mediabox[1] then done = true ; otherbox[1] = mediabox[1] end
2272            if otherbox[2] < mediabox[2] then done = true ; otherbox[2] = mediabox[2] end
2273            if otherbox[3] > mediabox[3] then done = true ; otherbox[3] = mediabox[3] end
2274            if otherbox[4] > mediabox[4] then done = true ; otherbox[4] = mediabox[4] end
2275            if done then
2276                report("limiting %a to 'MediaBox'",what)
2277            end
2278        end
2279        return otherbox
2280    end
2281
2282    finalize = function(driver,details)
2283
2284        if not details then
2285            report("something is wrong, no details in 'finalize'")
2286        end
2287
2288        level = level + 1
2289
2290        pdf_goto_pagemode() -- for now
2291
2292        local objnum        = details.objnum
2293        local specification = details.specification or { }
2294
2295        local content = concat(buffer,"\n",1,b)
2296
2297        if compact then
2298            content = compact(content)
2299        end
2300
2301        local fonts   = nil
2302        local xforms  = nil
2303
2304        if next(usedfonts) then
2305            fonts = pdfdictionary { }
2306            for k, v in next, usedfonts do
2307                fonts[f_font(v)] = pdfreference(usedfontobjects[k]) -- we can overload for testing
2308            end
2309        end
2310
2311        -- messy: use real indexes for both ... so we need to change some in the
2312        -- full luatex part
2313
2314        if next(usedxforms) or next(usedximages) or next(usedxgroups) then
2315            xforms = pdfdictionary { }
2316            for k in sortedhash(usedxforms) do
2317                xforms[f_form(getxformname(k))] = pdfreference(k)
2318            end
2319            for k, v in sortedhash(usedximages) do
2320                xforms[f_image(k)] = pdfreference(v)
2321            end
2322            for k, v in sortedhash(usedxgroups) do
2323                xforms[f_group(k)] = pdfreference(v)
2324            end
2325        end
2326
2327        reset_buffer()
2328
2329     -- finish_pdfpage_callback(shippingmode == "page")
2330
2331        if shippingmode == "page" then
2332
2333            local pageproperties  = lpdf.getpageproperties()
2334
2335            local pageresources   = pageproperties.pageresources
2336            local pageattributes  = pageproperties.pageattributes
2337            local pagesattributes = pageproperties.pagesattributes
2338
2339            pageresources.Font    = fonts
2340            pageresources.XObject = xforms
2341            pageresources.ProcSet = lpdf.procset()
2342
2343            local bbox = pdfarray {
2344                boundingbox[1] * bpfactor,
2345                boundingbox[2] * bpfactor,
2346                boundingbox[3] * bpfactor,
2347                boundingbox[4] * bpfactor,
2348            }
2349
2350            local contentsobj = pdfflushstreamobject(content,false,true)
2351
2352            pageattributes.Type      = pdf_page
2353            pageattributes.Contents  = pdfreference(contentsobj)
2354            pageattributes.Resources = pageresources
2355         -- pageattributes.Resources = pdfreference(pdfflushobject(pageresources))
2356         -- pageattributes.MediaBox  = bbox
2357            pageattributes.MediaBox  = pdfsharedobject(bbox)
2358            pageattributes.Parent    = nil -- precalculate
2359            pageattributes.Group     = nil -- todo
2360
2361            -- resources can be indirect
2362
2363            if state == "ignore" or state == false then
2364
2365            else
2366
2367                registerpage(pageattributes)
2368
2369                lpdf.finalizepage(true)
2370
2371                local TrimBox  = pageattributes.TrimBox
2372                local CropBox  = pageattributes.CropBox
2373                local BleedBox = pageattributes.BleedBox
2374
2375                -- Indirect objects don't work in all viewers.
2376
2377                if TrimBox  then pageattributes.TrimBox  = pdfsharedobject(checkedbox(bbox,TrimBox,"TrimBox")) end
2378                if CropBox  then pageattributes.CropBox  = pdfsharedobject(checkedbox(bbox,CropBox,"CropBox")) end
2379                if BleedBox then pageattributes.BleedBox = pdfsharedobject(checkedbox(bbox,BleedBox,"BleedBox")) end
2380
2381            end
2382
2383        else
2384
2385            local xformtype  = specification.type or 0
2386            local margin     = specification.margin or 0
2387            local attributes = specification.attributes or ""
2388            local resources  = specification.resources or ""
2389
2390            local wrapper    = nil
2391
2392            if xformtype == 0 then
2393                wrapper = pdfdictionary {
2394                    Type      = pdf_xobject,
2395                    Subtype   = pdf_form,
2396                    FormType  = 1,
2397                    BBox      = nil,
2398                    Matrix    = nil,
2399                    Resources = nil,
2400                }
2401            else
2402                wrapper = pdfdictionary {
2403                    BBox      = nil,
2404                    Matrix    = nil,
2405                    Resources = nil,
2406                }
2407            end
2408            if xformtype == 0 or xformtype == 1 or xformtype == 3 then
2409--                 wrapper.BBox = pdfarray {
2410--                     -margin * bpfactor,
2411--                     -margin * bpfactor,
2412--                     (boundingbox[3] + margin) * bpfactor,
2413--                     (boundingbox[4] + margin) * bpfactor,
2414                wrapper.BBox = pdfarray {
2415                    -ceil(                  margin  * bpfactor),
2416                    -ceil(                  margin  * bpfactor),
2417                     ceil((boundingbox[3] + margin) * bpfactor),
2418                     ceil((boundingbox[4] + margin) * bpfactor),
2419                }
2420            end
2421            if xformtype == 0 or xformtype == 2 or xformtype == 3 then
2422                -- can be shared too
2423                wrapper.Matrix = pdfarray { 1, 0, 0, 1, 0, 0 }
2424            end
2425
2426            local patterns = true
2427
2428            if attributes.Type and attributes.Type == pdf_pattern then
2429                patterns = false
2430            end
2431
2432            local boxresources   = lpdf.collectedresources {
2433                patterns  = patterns,
2434                serialize = false,
2435            }
2436            boxresources.Font    = fonts
2437            boxresources.XObject = xforms
2438
2439         -- todo: maybe share them
2440         -- wrapper.Resources = pdfreference(pdfflushobject(boxresources))
2441
2442            if resources ~= "" then
2443                boxresources = boxresources + resources
2444            end
2445            if attributes ~= "" then
2446                wrapper = wrapper + attributes
2447            end
2448
2449            wrapper.Resources = next(boxresources) and boxresources or nil
2450            wrapper.ProcSet   = lpdf.procset()
2451
2452            pdfflushstreamobject(content,wrapper,true,specification.objnum)
2453
2454        end
2455
2456        for objnum in sortedhash(usedxforms) do
2457            local f = flushedxforms[objnum]
2458            if f[1] == false then
2459                f[1] = true
2460                local objnum        = f[2] -- specification.objnum
2461                local specification = boxresources[objnum]
2462                local list          = specification.list
2463                localconverter(list,"xform",f[2],specification)
2464            end
2465        end
2466
2467        pdf_h, pdf_v  = 0, 0
2468
2469        if level == 1 then
2470            flushdeferred()
2471        end
2472        level = level - 1
2473
2474    end
2475
2476end
2477
2478-- now comes the pdf file handling
2479
2480local objects       = { }
2481local streams       = { } -- maybe just parallel to objects (no holes)
2482local nofobjects    = 0
2483local offset        = 0
2484local f             = false
2485local flush         = false
2486local objectstream  = true
2487local compress      = true
2488local cache         = false
2489local info          = ""
2490local catalog       = ""
2491local lastdeferred  = false
2492
2493local f_object       = formatters["%i 0 obj\010%s\010endobj\010"]
2494local f_stream_n_u   = formatters["%i 0 obj\010<< /Length %i >>\010stream\010%s\010endstream\010endobj\010"]
2495local f_stream_n_c   = formatters["%i 0 obj\010<< /Filter /FlateDecode /Length %i >>\010stream\010%s\010endstream\010endobj\010"]
2496local f_stream_d_u   = formatters["%i 0 obj\010<< %s /Length %i >>\010stream\010%s\010endstream\010endobj\010"]
2497local f_stream_d_c   = formatters["%i 0 obj\010<< %s /Filter /FlateDecode /Length %i >>\010stream\010%s\010endstream\010endobj\010"]
2498local f_stream_d_r   = formatters["%i 0 obj\010<< %s >>\010stream\010%s\010endstream\010endobj\010"]
2499
2500----- f_object_b     = formatters["%i 0 obj\010"]
2501local f_stream_b_n_u = formatters["%i 0 obj\010<< /Length %i >>\010stream\010"]
2502local f_stream_b_n_c = formatters["%i 0 obj\010<< /Filter /FlateDecode /Length %i >>\010stream\010"]
2503local f_stream_b_d_u = formatters["%i 0 obj\010<< %s /Length %i >>\010stream\010"]
2504local f_stream_b_d_c = formatters["%i 0 obj\010<< %s /Filter /FlateDecode /Length %i >>\010stream\010"]
2505local f_stream_b_d_r = formatters["%i 0 obj\010<< %s >>\010stream\010"]
2506
2507----- s_object_e <const> = "\010endobj\010"
2508local s_stream_e <const> = "\010endstream\010endobj\010"
2509
2510do
2511
2512    -- Versions can be set but normally are managed by the official standards. When possible
2513    -- reading and writing should look at these values.
2514
2515    function lpdf.setversion(major,minor)
2516        majorversion = tonumber(major) or majorversion
2517        minorversion = tonumber(minor) or minorversion
2518    end
2519
2520    function lpdf.getversion(major,minor)
2521        return majorversion, minorversion
2522    end
2523
2524    function lpdf.majorversion() return majorversion end
2525    function lpdf.minorversion() return minorversion end
2526
2527    -- It makes no sense to support levels so we only enable and disable and stick to level 3
2528    -- which is both fast and efficient.
2529
2530    local frozen = false
2531    local clevel = 3
2532    local olevel = 1
2533
2534    function lpdf.setcompression(level,objectlevel,freeze)
2535        if not frozen then
2536            compress     = level       and level       ~= 0 and true or false
2537            objectstream = objectlevel and objectlevel ~= 0 and true or false
2538            frozen       = freeze
2539        end
2540    end
2541
2542    function lpdf.getcompression()
2543        return compress and olevel or 0, objectstream and clevel or 0
2544    end
2545
2546    function lpdf.compresslevel()
2547        return compress and olevel or 0
2548    end
2549
2550    function lpdf.objectcompresslevel()
2551        return objectstream and clevel or 0
2552    end
2553
2554    if environment.arguments.nocompression then
2555        lpdf.setcompression(0,0,true)
2556    end
2557
2558end
2559
2560local addtocache, flushcache, cache do
2561
2562    local data, d  = { }, 0
2563    local list, l  = { }, 0
2564    local coffset  = 0
2565    local indices  = { }
2566
2567    local maxsize  = 32 * 1024 -- uncompressed
2568    local maxcount = 0xFF
2569
2570    addtocache = function(n,str)
2571        local size = #str
2572        if size == 0 then
2573            -- todo: message
2574            return
2575        end
2576        if coffset + size > maxsize or d == maxcount then
2577            flushcache()
2578        end
2579        if d == 0 then
2580            nofobjects = nofobjects + 1
2581            objects[nofobjects] = false
2582            streams[nofobjects] = indices
2583            cache = nofobjects
2584        end
2585        objects[n] = - cache
2586        indices[n] = d
2587        d = d + 1
2588        -- can have a comment n 0 obj as in luatex
2589        data[d] = str
2590        l = l + 1 ; list[l] = n
2591        l = l + 1 ; list[l] = coffset
2592        coffset = coffset + size + 1
2593    end
2594
2595    local p_ObjStm = pdfconstant("ObjStm")
2596
2597    flushcache = function() -- references cannot be stored
2598        if l > 0 then
2599            list = concat(list," ")
2600            data[0] = list
2601            data = concat(data,"\010",0,d)
2602            local strobj = pdfdictionary {
2603                Type  = p_ObjStm,
2604                N     = d,
2605                First = #list + 1,
2606            }
2607            objects[cache] = offset
2608            local fb
2609            if compress then
2610                local size = #data
2611                local comp = compressdata(data,size)
2612                if comp and #comp < size then
2613                    data = comp
2614                    fb = f_stream_b_d_c
2615                else
2616                    fb = f_stream_b_d_u
2617                end
2618            else
2619                fb = f_stream_b_d_u
2620            end
2621            local size = #data
2622            if encryptstream then
2623                data = encryptstream(data)
2624                size = #data
2625            end
2626            local b = fb(cache,strobj(),size)
2627            local e = s_stream_e
2628            flush(f,b)
2629            flush(f,data)
2630            flush(f,e)
2631            offset = offset + #b + size + #e
2632            data, d = { }, 0
2633            list, l = { }, 0
2634            coffset = 0
2635            indices = { }
2636        end
2637    end
2638
2639end
2640
2641do
2642
2643    local names        = { }
2644    local cache        = { }
2645    local nofpages     = 0
2646
2647    local texgetcount  = tex.getcount
2648    local c_realpageno = tex.iscount("realpageno")
2649
2650    pdfreserveobject = function(name)
2651        nofobjects = nofobjects + 1
2652        objects[nofobjects] = false
2653        if name then
2654            names[name] = nofobjects
2655            if trace_objects then
2656                report_objects("reserving number %a under name %a",nofobjects,name)
2657            end
2658        elseif trace_objects then
2659            report_objects("reserving number %a",nofobjects)
2660        end
2661        return nofobjects
2662    end
2663
2664    pdfpagereference = function(n,complete) -- true | false | nil | n [true,false]
2665        if n == true or not n then
2666            complete = n
2667            n = texgetcount(c_realpageno)
2668        end
2669        if n > nofpages then
2670            nofpages = n
2671        end
2672        local r = pdfgetpagereference(n)
2673        return complete and pdfreference(r) or r
2674    end
2675
2676    lpdf.reserveobject = pdfreserveobject
2677    lpdf.pagereference = pdfpagereference
2678
2679    function lpdf.lastreferredpage()
2680        return nofpages
2681    end
2682
2683    function lpdf.nofpages() -- this will change: document nofpages
2684        return structures.pages.nofpages
2685    end
2686
2687    function lpdf.object(...)
2688        pdfdeferredobject(...)
2689    end
2690
2691    function lpdf.delayedobject(data,n)
2692        if n then
2693            pdfdeferredobject(n,data)
2694        else
2695            n = pdfdeferredobject(data)
2696        end
2697--         pdfreferenceobject(n)
2698        return n
2699    end
2700
2701    pdfflushobject = function(name,data)
2702        if data then
2703            local named = names[name]
2704            if named then
2705                if not trace_objects then
2706                elseif trace_details then
2707                    report_objects("flushing data to reserved object with name %a, data: %S",name,data)
2708                else
2709                    report_objects("flushing data to reserved object with name %a",name)
2710                end
2711                return pdfimmediateobject(named,tostring(data))
2712            else
2713                if not trace_objects then
2714                elseif trace_details then
2715                    report_objects("flushing data to reserved object with number %s, data: %S",name,data)
2716                else
2717                    report_objects("flushing data to reserved object with number %s",name)
2718                end
2719                return pdfimmediateobject(name,tostring(data))
2720            end
2721        else
2722            if trace_objects and trace_details then
2723                report_objects("flushing data: %S",name)
2724            end
2725            return pdfimmediateobject(tostring(name))
2726        end
2727    end
2728
2729    pdfflushstreamobject = function(data,dict,compressed,objnum) -- default compressed
2730        if trace_objects then
2731            report_objects("flushing stream object of %s bytes",#data)
2732        end
2733        local dtype    = type(dict)
2734        local kind     = compressed == "raw" and "raw" or "stream"
2735        local nolength = nil
2736        if compressed == "raw" then
2737            compressed = false
2738            nolength   = true
2739         -- data       = string.formatters["<< %s >>stream\n%s\nendstream"](attr,data)
2740        end
2741
2742        return pdfdeferredobject {
2743            objnum        = objnum,
2744            immediate     = true,
2745            nolength      = nolength,
2746            compresslevel = compressed,
2747            type          = "stream",
2748            string        = data,
2749            attr          = (dtype == "string" and dict) or (dtype == "table" and dict()) or nil,
2750        }
2751    end
2752
2753    function lpdf.flushstreamfileobject(filename,dict,compressed,objnum) -- default compressed
2754        if trace_objects then
2755            report_objects("flushing stream file object %a",filename)
2756        end
2757        local dtype = type(dict)
2758        return pdfdeferredobject {
2759            objnum        = objnum,
2760            immediate     = true,
2761            compresslevel = compressed,
2762            type          = "stream",
2763            file          = filename,
2764            attr          = (dtype == "string" and dict) or (dtype == "table" and dict()) or nil,
2765        }
2766    end
2767
2768    local shareobjectcache, shareobjectreferencecache = { }, { }
2769
2770    function lpdf.shareobject(content)
2771        if content == nil then
2772            -- invalid object not created
2773        else
2774            content = tostring(content)
2775            local o = shareobjectcache[content]
2776            if not o then
2777                o = pdfimmediateobject(content)
2778                shareobjectcache[content] = o
2779            end
2780            return o
2781        end
2782    end
2783
2784    pdfsharedobject = function(content)
2785        if content == nil then
2786            -- invalid object not created
2787        else
2788            content = tostring(content)
2789            local r = shareobjectreferencecache[content]
2790            if not r then
2791                local o = shareobjectcache[content]
2792                if not o then
2793                    o = pdfimmediateobject(content)
2794                    shareobjectcache[content] = o
2795                end
2796                r = pdfreference(o)
2797                shareobjectreferencecache[content] = r
2798            end
2799            return r
2800        end
2801    end
2802
2803    lpdf.flushobject          = pdfflushobject
2804    lpdf.flushstreamobject    = pdfflushstreamobject
2805    lpdf.shareobjectreference = pdfsharedobject
2806    lpdf.sharedobject         = pdfsharedobject
2807
2808end
2809
2810local pages = table.setmetatableindex(function(t,k)
2811    local v = pdfreserveobject()
2812    t[k] = v
2813    return v
2814end)
2815
2816pdfgetpagereference = function(n)
2817    return pages[n]
2818end
2819
2820lpdf.getpagereference = pdfgetpagereference
2821
2822local function flushnormalobj(data,n)
2823    if not n then
2824        nofobjects = nofobjects + 1
2825        n = nofobjects
2826    end
2827    if encryptobject then
2828        data = encryptobject(data)
2829    end
2830    data = f_object(n,data)
2831    if level == 0 then
2832        objects[n] = offset
2833        offset = offset + #data
2834        flush(f,data)
2835    else
2836        if not lastdeferred then
2837            lastdeferred = n
2838        elseif n < lastdeferred then
2839            lastdeferred = n
2840        end
2841        objects[n] = data
2842    end
2843    return n
2844end
2845
2846local flushstreamobj, streamstatus do
2847
2848    local uncompressed  = 0
2849    local compressed    = 0
2850    local notcompressed = 0
2851
2852    local threshold     = 40 -- also #("/Filter /FlateDecode") (compression threshold)
2853
2854    -- directives.register("backend.pdf.threshold",function(v)
2855    --     if v then
2856    --         threshold = tonumber(v) or 40
2857    --     else
2858    --         threshold = -1000
2859    --     end
2860    -- end)
2861
2862    streamstatus = function()
2863        return {
2864            nofstreams          = uncompressed + compressed + notcompressed,
2865            uncompressed        = uncompressed,
2866            compressed          = compressed,
2867            notcompressed       = notcompressed,
2868            threshold           = threshold,
2869            compresslevel       = lpdf.compresslevel(),
2870            objectcompresslevel = lpdf.objectcompresslevel(),
2871
2872        }
2873    end
2874
2875    flushstreamobj = function(data,n,dict,comp,nolength)
2876        if not data then
2877            report("no data for %S",dict)
2878            return
2879        end
2880        if not n then
2881            nofobjects = nofobjects + 1
2882            n = nofobjects
2883        end
2884        local size = #data
2885        if comp ~= false then
2886            comp = compress and size > threshold
2887        end
2888        if encryptobject then
2889            dict = encryptobject(dict)
2890        end
2891        if level == 0 then
2892            local b = nil
2893            local e = s_stream_e
2894            if nolength then
2895                -- probleem: we need to adapt length!
2896                b = f_stream_b_d_r(n,dict) -- raw object, already treated
2897                if encryptstream then
2898                    print("check length")
2899                    data = encryptstream(data)
2900                    size = #data
2901                end
2902                uncompressed = uncompressed + 1
2903            else
2904                if comp then
2905                    local compdata = compressdata(data,size)
2906                    if compdata then
2907                        local compsize = #compdata
2908                        if compsize <= size - threshold then
2909                            data = compdata
2910                            size = compsize
2911                        else
2912                            comp = false
2913                        end
2914                    else
2915                        comp = false
2916                    end
2917                end
2918                if encryptstream then
2919                    data = encryptstream(data)
2920                    size = #data
2921                end
2922                if comp then
2923                    b = dict and f_stream_b_d_c(n,dict,size) or f_stream_b_n_c(n,size)
2924                    compressed = compressed + 1
2925                else
2926                    b = dict and f_stream_b_d_u(n,dict,size) or f_stream_b_n_u(n,size)
2927                    notcompressed = notcompressed + 1
2928                end
2929            end
2930            flush(f,b)
2931            flush(f,data)
2932            flush(f,e)
2933            objects[n] = offset
2934            offset = offset + #b + size + #e
2935        else
2936            if nolength then
2937                if encryptstream then
2938                    print("check length")
2939                    data = encryptstream(data)
2940                end
2941                data = f_stream_d_r(n,dict,data) -- raw object, already treated
2942                uncompressed = uncompressed + 1
2943            else
2944                if comp then
2945                    local compdata = compressdata(data,size)
2946                    if compdata then
2947                        local compsize = #compdata
2948                        if compsize <= size - threshold then
2949                            data = compdata
2950                            size = compsize
2951                        else
2952                            comp = false
2953                        end
2954                    else
2955                        comp = false
2956                    end
2957                end
2958                if encryptstream then
2959                    data = encryptstream(data)
2960                    size = #data
2961                end
2962                if comp then
2963                    data = dict and f_stream_d_c(n,dict,size,data) or f_stream_n_c(n,size,data)
2964                    compressed = compressed + 1
2965                else
2966                    data = dict and f_stream_d_u(n,dict,size,data) or f_stream_n_u(n,size,data)
2967                    notcompressed = notcompressed + 1
2968                end
2969            end
2970            if not lastdeferred then
2971                lastdeferred = n
2972            elseif n < lastdeferred then
2973                lastdeferred = n
2974            end
2975            objects[n] = data
2976        end
2977        return n
2978    end
2979
2980end
2981
2982flushdeferred = function() -- was forward defined
2983    if lastdeferred then
2984        for n=lastdeferred,nofobjects do
2985            local o = objects[n]
2986            if type(o) == "string" then
2987                objects[n] = offset
2988                offset = offset + #o
2989                flush(f,o)
2990            end
2991        end
2992        lastdeferred = false
2993    end
2994end
2995
2996pdfimmediateobject = function(a,b,c,d)
2997    local kind --, immediate
2998    local objnum, data, attr, filename
2999    local compresslevel, objcompression, nolength
3000    local argtype = type(a)
3001    if argtype == "table" then
3002        kind           = a.type          -- raw | stream
3003     -- immediate      = a.immediate
3004        objnum         = a.objnum
3005        attr           = a.attr
3006        compresslevel  = a.compresslevel
3007        objcompression = a.objcompression
3008        filename       = a.file
3009        data           = a.string or a.stream or ""
3010        nolength       = a.nolength
3011        if kind == "stream" then
3012            if filename then
3013                data = loaddata(filename) or ""
3014            end
3015        elseif kind == "raw"then
3016            if filename then
3017                data = loaddata(filename) or ""
3018            end
3019        elseif kind == "file"then
3020            kind = "raw"
3021            data = filename and loaddata(filename) or ""
3022        elseif kind == "streamfile" then
3023            kind = "stream"
3024            data = filename and loaddata(filename) or ""
3025        end
3026    else
3027        if argtype == "number" then
3028            objnum = a
3029            a, b, c = b, c, d
3030        else
3031            nofobjects = nofobjects + 1
3032            objnum = nofobjects
3033        end
3034        if b then
3035            if a == "stream" then
3036                kind = "stream"
3037                data = b
3038            elseif a == "file" then
3039             -- kind = "raw"
3040                data = loaddata(b)
3041            elseif a == "streamfile" then
3042                kind = "stream"
3043                data = loaddata(b)
3044            else
3045                data = "" -- invalid object
3046            end
3047            attr = c
3048        else
3049         -- kind = "raw"
3050            data = a
3051        end
3052    end
3053    if not objnum then
3054        nofobjects = nofobjects + 1
3055        objnum = nofobjects
3056    end
3057    -- todo: immediate
3058    if kind == "stream" then
3059        flushstreamobj(data,objnum,attr,compresslevel,nolength) -- nil == auto
3060    elseif objectstream and objcompression ~= false then
3061        addtocache(objnum,data)
3062    else
3063        flushnormalobj(data,objnum)
3064    end
3065    return objnum
3066end
3067
3068pdfdeferredobject    = pdfimmediateobject
3069
3070lpdf.deferredobject  = pdfimmediateobject
3071lpdf.immediateobject = pdfimmediateobject
3072
3073-- In lua 5.4 the methods are now moved one metalevel deeper so we need to get them
3074-- from mt.__index instead. (I did get that at first.) It makes for a slightly (imo)
3075-- nicer interface but no real gain in speed as we don't flush that often.
3076
3077local openfile, closefile  do
3078
3079    -- I used to do <space><lf> but then figured out that when I open and save a file in a mode
3080    -- that removes trailing spaces, the xref becomes invalid. The problem was then that a
3081    -- reconstruction of the file by a viewer gives weird effects probably because percent symbols
3082    -- gets interpreted then. Thanks to Ross Moore for noticing this side effect!
3083
3084    local f_used       = formatters["%010i 00000 n\013\010"]
3085    local f_link       = formatters["%010i 00000 f\013\010"]
3086    local f_first      = formatters["%010i 65535 f\013\010"]
3087
3088    local f_pdf_tag    = formatters["%%PDF-%i.%i\010"]
3089    local f_xref       = formatters["xref\0100 %i\010"]
3090    local f_trailer_id = formatters["trailer\010<< %s /ID [ <%s> <%s> ] >>\010startxref\010%i\010%%%%EOF"]
3091    local f_trailer_no = formatters["trailer\010<< %s >>\010startxref\010%i\010%%%%EOF"]
3092    local f_startxref  = formatters["startxref\010%i\010%%%%EOF"]
3093
3094    local inmemory = false
3095    local close    = false
3096    local update   = false
3097    local usedname = false
3098    local usedsize = false
3099
3100    directives.enable("backend.pdf.inmemory", function(v) inmemory = true end)
3101
3102 -- local banner <const> = "%\xCC\xD5\xC1\xD4\xC5\xD8\xD0\xC4\xC6\010"     -- LUATEXPDF  (+128)
3103    local banner <const> = "%\xC3\xCF\xCE\xD4\xC5\xD8\xD4\xD0\xC4\xC6\010" -- CONTEXTPDF (+128)
3104
3105    openfile = function(filename)
3106        --
3107        local arguments = environment.arguments
3108        if arguments.ownerpassword then
3109            lpdf.setencryption {
3110                ownerpassword = arguments.ownerpassword,
3111                userpassword  = arguments.userpassword,
3112                permissions   = arguments.permissions,
3113            }
3114        end
3115        --
3116        if inmemory then
3117            local n = 0
3118            f = { }
3119            flush = function(f,s)
3120                n = n + 1 f[n] = s
3121             -- offset = offset + #s
3122            end
3123            close = function(f)
3124                f = concat(f)
3125                usedsize = #f
3126                io.savedata(filename,f)
3127                f = false
3128            end
3129            update = function(f,s)
3130                f[1] = s
3131            end
3132         -- local n = 0
3133         -- f = {
3134         --     write = function(self,s)
3135         --         n = n + 1 f[n] = s
3136         --     end,
3137         --     close = function(self)
3138         --         f = concat(f)
3139         --         io.savedata(filename,f)
3140         --         f = false
3141         --     end,
3142         -- }
3143         else
3144            f = io.open(filename,"wb")
3145            if not f then
3146                report()
3147                report("quitting because file %a cannot be opened for writing",filename)
3148                report()
3149                os.exit()
3150            end
3151         -- f:setvbuf("full",64*1024)
3152            local m = getmetatable(f)
3153            flush = m.write or m.__index.write
3154            close = m.close or m.__index.close
3155            update = function(f,s)
3156                f:seek("set",0)
3157                f:write(s)
3158            end
3159        end
3160        local version = f_pdf_tag(majorversion,minorversion)
3161        flush(f,version)
3162        flush(f,banner)
3163        offset = offset + #version + #banner
3164        usedname = filename
3165    end
3166
3167    closefile = function(abort)
3168        if abort then
3169            close(f)
3170            if not environment.arguments.nodummy then
3171                f = io.open(abort,"wb")
3172                if f then
3173                    local name = resolvers.findfile("context-lmtx-error.pdf")
3174                    if name then
3175                        local data = io.loaddata(name)
3176                        if data then
3177                            f:write(data)
3178                            f:close()
3179                            return
3180                        end
3181                    end
3182                    f:close()
3183                end
3184            end
3185            os.remove(abort)
3186        else
3187            local xrefoffset = offset
3188            local lastfree   = 0
3189            local noffree    = 0
3190            --
3191            local os = objectstream
3192            if encryptstream then
3193                objectstream = false
3194            end
3195            local catalog    = lpdf.getcatalog()
3196            objectstream = os
3197            --
3198            local info       = lpdf.getinfo()
3199            local trailerid  = lpdf.gettrailerid()
3200            --
3201            if objectstream then
3202                flushdeferred()
3203                flushcache()
3204                --
3205                offset = lpdf.preparesignature and lpdf.preparesignature(flush,f,offset,objects) or offset
3206                --
3207                xrefoffset = offset
3208                --
3209                nofobjects = nofobjects + 1
3210                objects[nofobjects] = offset -- + 1
3211                --
3212                -- combine these three in one doesn't really give less code so
3213                -- we go for the efficient ones
3214                --
3215                local nofbytes  = 4
3216                local c1, c2, c3, c4
3217                if offset <= 0xFFFF then
3218                    nofbytes = 2
3219                    for i=1,nofobjects do
3220                        local o = objects[i]
3221                        if not o then
3222                            noffree = noffree + 1
3223                        else
3224                            local strm = o < 0
3225                            if strm then
3226                                o = -o
3227                            end
3228                            c1 = (o>>8)&0xFF
3229                            c2 = (o>>0)&0xFF
3230                            if strm then
3231                                objects[i] = char(2,c1,c2,streams[o][i])
3232                            else
3233                                objects[i] = char(1,c1,c2,0)
3234                            end
3235                        end
3236                    end
3237                    if noffree > 0 then
3238                        for i=nofobjects,1,-1 do
3239                            local o = objects[i]
3240                            if not o then
3241                                local f1 = (lastfree>>8)&0xFF
3242                                local f2 = (lastfree>>0)&0xFF
3243                                objects[i] = char(0,f1,f2,0)
3244                                lastfree   = i
3245                            end
3246                        end
3247                    end
3248                elseif offset <= 0xFFFFFF then
3249                    nofbytes = 3
3250                    for i=1,nofobjects do
3251                        local o = objects[i]
3252                        if not o then
3253                            noffree = noffree + 1
3254                        else
3255                            local strm = o < 0
3256                            if strm then
3257                                o = -o
3258                            end
3259                            c1 = (o>>16)&0xFF
3260                            c2 = (o>> 8)&0xFF
3261                            c3 = (o>> 0)&0xFF
3262                            if strm then
3263                                objects[i] = char(2,c1,c2,c3,streams[o][i])
3264                            else
3265                                objects[i] = char(1,c1,c2,c3,0)
3266                            end
3267                        end
3268                    end
3269                    if noffree > 0 then
3270                        for i=nofobjects,1,-1 do
3271                            local o = objects[i]
3272                            if not o then
3273                                local f1 = (lastfree>>16)&0xFF
3274                                local f2 = (lastfree>> 8)&0xFF
3275                                local f3 = (lastfree>> 0)&0xFF
3276                                objects[i] = char(0,f1,f2,f3,0)
3277                                lastfree   = i
3278                            end
3279                        end
3280                    end
3281                else
3282                    nofbytes = 4
3283                    for i=1,nofobjects do
3284                        local o = objects[i]
3285                        if not o then
3286                            noffree = noffree + 1
3287                        else
3288                            local strm = o < 0
3289                            if strm then
3290                                o = -o
3291                            end
3292                            c1 = (o>>24)&0xFF
3293                            c2 = (o>>16)&0xFF
3294                            c3 = (o>> 8)&0xFF
3295                            c4 = (o>> 0)&0xFF
3296                            if strm then
3297                                objects[i] = char(2,c1,c2,c3,c4,streams[o][i])
3298                            else
3299                                objects[i] = char(1,c1,c2,c3,c4,0)
3300                            end
3301                        end
3302                    end
3303                    if noffree > 0 then
3304                        for i=nofobjects,1,-1 do
3305                            local o = objects[i]
3306                            if not o then
3307                                local f1 = (lastfree>>24)&0xFF
3308                                local f2 = (lastfree>>16)&0xFF
3309                                local f3 = (lastfree>> 8)&0xFF
3310                                local f4 = (lastfree>> 0)&0xFF
3311                                objects[i] = char(0,f1,f2,f3,f4,0)
3312                                lastfree   = i
3313                            end
3314                        end
3315                    end
3316                end
3317                objects[0] = rep("\0",1+nofbytes+1)
3318                local data = concat(objects,"",0,nofobjects)
3319                local size = #data
3320                local xref = pdfdictionary {
3321                    Type    = pdfconstant("XRef"),
3322                    Size    = nofobjects + 1,
3323                    W       = pdfarray { 1, nofbytes, 1 },
3324                    Root    = catalog,
3325                    Info    = info,
3326                    ID      = trailerid and pdfarray { pdfliteral(trailerid,true), pdfliteral(trailerid,true) } or nil,
3327                    Encrypt = encdict or nil,
3328                }
3329                local fb
3330             -- if encryptstream then
3331             --     if compress then
3332             --         local comp = compressdata(data,size)
3333             --         if comp then
3334             --             data = comp
3335             --             size = #data
3336             --             fb = f_stream_b_d_c
3337             --             xref.Filter = pdfarray {
3338             --                 pdfconstant("Crypt"), -- identity
3339             --                 pdfconstant("FlateDecode")
3340             --             }
3341             --         else
3342             --             xref.Filter = pdfconstant("Crypt") -- identity
3343             --         end
3344             --     else
3345             --         xref.Filter = pdfconstant("Crypt") -- identity
3346             --     end
3347             --     fb = f_stream_b_d_u
3348             -- else
3349                    if compress then
3350                        local comp = compressdata(data,size)
3351                        if comp then
3352                            data = comp
3353                            size = #data
3354                            fb = f_stream_b_d_c
3355                        else
3356                            fb = f_stream_b_d_u
3357                        end
3358                    else
3359                        fb = f_stream_b_d_u
3360                    end
3361             -- end
3362                -- no encryption of data here
3363                flush(f,fb(nofobjects,xref(),size))
3364                flush(f,data)
3365                flush(f,s_stream_e)
3366                flush(f,f_startxref(xrefoffset))
3367            else
3368                flushdeferred()
3369                --
3370                offset = lpdf.preparesignature and lpdf.preparesignature(flush,f,offset,objects) or offset
3371                --
3372             -- if encryptstream then
3373             --     -- unencrypted !
3374             --     local eo = encryptobject
3375             --     encryptobject = false
3376             --     encdict = pdfreference(pdfimmediateobject(tostring(encdict)))
3377             --     encryptobject = eo
3378             -- end
3379                --
3380                xrefoffset = offset
3381                flush(f,f_xref(nofobjects+1))
3382                local trailer = pdfdictionary {
3383                    Size    = nofobjects + 1,
3384                    Root    = catalog,
3385                    Info    = info,
3386                    Encrypt = encdict or nil,
3387                }
3388                for i=1,nofobjects do
3389                    local o = objects[i]
3390                    if o then
3391                        objects[i] = f_used(o)
3392                    end
3393                end
3394                for i=nofobjects,1,-1 do
3395                    local o = objects[i]
3396                    if not o then
3397                        objects[i] = f_link(lastfree)
3398                        lastfree   = i
3399                    end
3400                end
3401                objects[0] = f_first(lastfree)
3402                flush(f,concat(objects,"",0,nofobjects))
3403                trailer.Size = nofobjects + 1
3404                if trailerid then
3405                    flush(f,f_trailer_id(trailer(),trailerid,trailerid,xrefoffset))
3406                else
3407                    flush(f,f_trailer_no(trailer(),xrefoffset))
3408                end
3409            end
3410            update(f,f_pdf_tag(majorversion,minorversion)) -- check this
3411            usedsize = f:seek("end")
3412            close(f)
3413            io.flush()
3414            if lpdf.finalizesignature then lpdf.finalizesignature(usedname,usedsize) end
3415        end
3416        io.flush()
3417        closefile = function() end
3418    end
3419
3420end
3421
3422-- For the moment we overload it here, although back-fil.lua eventually will
3423-- be merged with back-pdf as it's pdf specific, or maybe back-imp-pdf or so.
3424
3425do
3426
3427    -- We overload img but at some point it will even go away, so we just
3428    -- reimplement what we need in context. This will change completely i.e.
3429    -- we will drop the low level interface!
3430
3431    local pdfbackend     = backends.registered.pdf
3432    local nodeinjections = pdfbackend.nodeinjections
3433    local codeinjections = pdfbackend.codeinjections
3434
3435    local imagetypes     = images.types -- pdf png jpg jp2 jbig2 stream
3436    local img_none       = imagetypes.none
3437
3438    local tonode         = nuts.tonode
3439    local newimagerule   = nuts.pool.imagerule
3440    local setattrlist    = nuts.setattrlist
3441    local setprop        = nuts.setprop
3442
3443    local report_images  = logs.reporter("backend","images")
3444
3445    local lastindex      = 0
3446    local indices        = { }
3447
3448    local bpfactor       = number.dimenfactors.bp
3449
3450    function codeinjections.newimage(specification)
3451        return specification
3452    end
3453
3454    function codeinjections.copyimage(original)
3455        return setmetatableindex(original)
3456    end
3457
3458    function codeinjections.scanimage(specification)
3459        -- placeholder, doesn't give back dimensions etc but will be plugged in
3460        return specification
3461    end
3462
3463    local function embedimage(specification)
3464        if specification then
3465            lastindex = lastindex + 1
3466            index     = lastindex
3467            specification.index = index
3468            local xobject = pdfdictionary { }
3469            if not specification.notype then
3470                xobject.Type     = pdf_xobject
3471                xobject.Subtype  = pdf_form
3472                xobject.FormType = 1
3473            end
3474            local bbox = specification.bbox
3475            if bbox and not specification.nobbox then
3476                xobject.BBox = pdfarray {
3477                    bbox[1] * bpfactor,
3478                    bbox[2] * bpfactor,
3479                    bbox[3] * bpfactor,
3480                    bbox[4] * bpfactor,
3481                }
3482            end
3483            xobject = xobject + specification.attr
3484            if bbox and not specification.width then
3485                specification.width = bbox[3]
3486            end
3487            if bbox and not specification.height then
3488                specification.height = bbox[4]
3489            end
3490            local dict = xobject()
3491            --
3492            nofobjects     = nofobjects + 1
3493            local objnum   = nofobjects
3494            local nolength = specification.nolength
3495            local stream   = specification.stream or specification.string
3496            --
3497            -- We cannot set type in native img so we need this hack or
3498            -- otherwise we need to patch too much. Better that i write
3499            -- a wrapper then. Anyway, it has to be done better: a key that
3500            -- tells either or not to scale by xsize/ysize when flushing.
3501            --
3502            if not specification.type then
3503                local kind = specification.kind
3504                if kind then
3505                    -- take that one
3506                elseif attr and find(attr,"BBox") then
3507                    kind = img_stream
3508                else
3509                    -- hack: a bitmap
3510                    kind = img_none
3511                end
3512                specification.type = kind
3513                specification.kind = kind
3514            end
3515            flushstreamobj(stream,objnum,dict,compresslevel,nolength)
3516            specification.objnum      = objnum
3517            specification.rotation    = specification.rotation or 0
3518            specification.orientation = specification.orientation or 0
3519            specification.transform   = specification.transform or 0
3520            specification.stream      = nil
3521            specification.attr        = nil
3522            specification.type        = specification.kind or specification.type or img_none
3523            indices[index]            = specification -- better create a real specification
3524            return specification
3525        end
3526    end
3527
3528    codeinjections.embedimage = embedimage
3529
3530    function codeinjections.wrapimage(specification)
3531        --
3532        local index = specification.index
3533        if not index then
3534            embedimage(specification)
3535        end
3536        --
3537        local n = newimagerule(
3538            specification.width  or 0,
3539            specification.height or 0,
3540            specification.depth  or 0
3541        )
3542        setattrlist(n,true)
3543        setprop(n,"index",specification.index)
3544        return tonode(n)
3545    end
3546
3547    pdfincludeimage = function(index)
3548        local specification = indices[index]
3549        if specification then
3550            local bbox      = specification.bbox
3551            local xorigin   = bbox[1]
3552            local yorigin   = bbox[2]
3553            local xsize     = bbox[3] - xorigin -- we need the original ones, not the 'rotated' ones
3554            local ysize     = bbox[4] - yorigin -- we need the original ones, not the 'rotated' ones
3555            local transform = specification.transform or 0
3556            local objnum    = specification.objnum or pdfreserveobject()
3557            local groupref  = nil
3558            local kind      = specification.kind or specification.type or img_none -- determines scaling type
3559            return
3560                kind,
3561                xorigin, yorigin,
3562                xsize, ysize,
3563                transform,
3564                objnum,
3565                groupref
3566        end
3567    end
3568
3569    lpdf.includeimage = pdfincludeimage
3570
3571end
3572
3573-- The driver.
3574
3575do
3576
3577 -- local addsuffix  = file.addsuffix
3578    local texgetbox  = tex.getbox
3579
3580    local pdfname    = nil
3581    local converter  = nil
3582    local useddriver = nil -- a bit of a hack
3583
3584    local function outputfilename(driver)
3585        return pdfname
3586    end
3587
3588-- local outputfilename ; do -- old todo usedname in ^^
3589--     local filename = nil
3590--     outputfilename = function(driver,usedname)
3591--         if usedname and usedname ~= "" then
3592--             filename = addsuffix(usedname,"pdf")
3593--         elseif not filename or filename == "" then
3594--             filename = addsuffix(tex.jobname,"pdf")
3595--         end
3596--         return filename
3597--     end
3598-- end
3599
3600    -- todo: prevent twice
3601    local function prepare(driver)
3602        if not environment.initex then
3603            --
3604            backends.initialize("pdf") -- also does bindings
3605            --
3606            pdfname = tex.jobname .. ".pdf"
3607            openfile(pdfname)
3608            --
3609            luatex.registerstopactions(1,function()
3610                if pdfname then
3611                    lpdf.finalizedocument()
3612                    closefile()
3613                    pdfname = nil
3614                end
3615            end)
3616            --
3617            luatex.registerpageactions(1,function()
3618                if pdfname then
3619                    lpdf.finalizepage(true)
3620                end
3621            end)
3622            --
3623            lpdf.registerdocumentfinalizer(wrapupdocument,nil,"wrapping up")
3624            --
3625            statistics.register("result saved in file", function()
3626                local status = streamstatus()
3627                local outputfilename = environment.outputfilename or environment.jobname or tex.jobname or "<unset>"
3628                outputfilename = string.gsub(outputfilename,"^%./+","") -- todo: make/use a helper
3629                return string.format(
3630                    "%s.pdf, compresslevel %s, objectcompresslevel %s, %i streams, %i uncompressed, %i compressed, %i not compressed, threshold %i",
3631                    outputfilename,
3632                    status.compresslevel,
3633                    status.objectcompresslevel,
3634                    status.nofstreams or 0,
3635                    status.uncompressed or 0,
3636                    status.compressed or 0,
3637                    status.notcompressed or 0,
3638                    status.threshold or 0
3639                )
3640            end)
3641            --
3642            luatex.registerstopactions(function()
3643                if pdfname then
3644                    local r = lpdf.lastreferredpage() -- somehow referenced
3645                    local s = lpdf.getnofpages()      -- in page tree, saved in file
3646                    local t = lpdf.nofpages()         -- in tuc file
3647                    if r > s then
3648                        report()
3649                        report("referred pages: %i, saved pages %i, pages from tuc file: %i, possible corrupt file",r,s,t)
3650                        report()
3651                    end
3652                end
3653            end)
3654        end
3655        converter  = drivers.converters.lmtx
3656        useddriver = driver
3657    end
3658
3659    local function wrapup(driver)
3660        if pdfname then
3661            closefile()
3662            pdfname = nil
3663        end
3664    end
3665
3666    local function cleanup(driver)
3667        if pdfname then
3668            closefile(pdfname)
3669            pdfname = nil
3670        end
3671    end
3672
3673    local function convert(driver,boxnumber)
3674        converter(driver,texgetbox(boxnumber),"page")
3675    end
3676
3677--     localconverter = function(...)
3678--         print(...)                -- ok when we add this
3679--         converter(useddriver,...) -- otherwise nil .. lua bug
3680--     end
3681
3682    localconverter = function(a,b,c,d)
3683        converter(useddriver,a,b,c,d)
3684    end
3685
3686    drivers.install {
3687        name     = "pdf",
3688        flushers = flushers,
3689        actions  = {
3690            prepare         = prepare,
3691            wrapup          = wrapup,
3692            cleanup         = cleanup,
3693            --
3694            initialize      = initialize,
3695            convert         = convert,
3696            finalize        = finalize,
3697            --
3698            outputfilename  = outputfilename,
3699        },
3700    }
3701
3702end
3703