core-uti.lmt /size: 15 Kb    last modification: 2023-12-21 09:44
1if not modules then modules = { } end modules ['core-uti'] = {
2    version   = 1.001,
3    comment   = "companion to core-uti.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- A utility file has always been part of ConTeXt and with the move to LuaTeX we
10-- also moved a lot of multi-pass info to a Lua table. Instead of loading a TeX
11-- based utility file under different setups, we now load a table once. This saves
12-- much runtime but at the cost of more memory usage.
13--
14-- In the meantime the overhead is a bit more due to the amount of data being saved
15-- and more agressive compacting.
16
17local math = math
18local next, type, tostring, tonumber, setmetatable, load = next, type, tostring, tonumber, setmetatable, load
19local format, match = string.format, string.match
20local concat, sortedkeys = table.concat, table.sortedkeys
21
22local definetable    = utilities.tables.definetable
23local accesstable    = utilities.tables.accesstable
24local migratetable   = utilities.tables.migratetable
25local serialize      = table.serialize
26local packers        = utilities.packers
27local allocate       = utilities.storage.allocate
28local mark           = utilities.storage.mark
29
30local getrandom      = utilities.randomizer.get
31local setrandomseedi = utilities.randomizer.setseedi
32local getrandomseed  = utilities.randomizer.getseed
33
34local implement      = interfaces.implement
35
36local texgetcount    = tex.getcount
37
38local report_passes  = logs.reporter("job","passes")
39
40job                  = job or { }
41local job            = job
42
43job.version          = 1.33
44job.packversion      = 1.02
45
46-- Variables are saved using in the previously defined table and passed onto TeX
47-- using the following method. Of course one can also directly access the variable
48-- using a Lua call.
49
50local savelist, comment = { }, { }
51
52function job.comment(key,value)
53    if type(key) == "table" then
54        for k, v in next, key do
55            comment[k] = v
56        end
57    else
58        comment[key] = value
59    end
60end
61
62job.comment("version",job.version)
63
64local enabled     = true
65local initialized = false
66
67directives.register("job.save",function(v) enabled = v end)
68
69function job.disablesave()
70    enabled = false -- for instance called when an error
71end
72
73function job.initialize(loadname,savename)
74    if not initialized then
75        if not loadname or loadname == "" then
76            loadname = tex.jobname .. ".tuc"
77        end
78        if not savename or savename == "" then
79            savename = tex.jobname .. ".tua"
80        end
81        job.load(loadname) -- has to come after structure is defined !
82        luatex.registerstopactions(function()
83            if enabled then
84                job.save(savename)
85            end
86        end)
87        initialized = true
88    end
89end
90
91function job.register(collected, tobesaved, initializer, finalizer, serializer)
92    savelist[#savelist+1] = { collected, tobesaved, initializer, finalizer, serializer }
93end
94
95-- as an example we implement variables
96
97local tobesaved, collected, checksums = allocate(), allocate(), allocate()
98
99local jobvariables = {
100    collected = collected,
101    tobesaved = tobesaved,
102    checksums = checksums,
103}
104
105-- if not checksums.old then checksums.old = md5.HEX("old") end -- used in experiment
106-- if not checksums.new then checksums.new = md5.HEX("new") end -- used in experiment
107
108job.variables = jobvariables
109
110local function initializer()
111    checksums = jobvariables.checksums
112end
113
114job.register('job.variables.checksums', 'job.variables.checksums', initializer)
115
116local rmethod, rvalue
117local collectedmacros, tobesavedmacros
118
119local ctx_setxvalue = context.setxvalue
120
121local function initializer()
122    tobesaved = jobvariables.tobesaved
123    collected = jobvariables.collected
124    --
125    rvalue = collected.randomseed
126    if not rvalue then
127        rvalue = getrandom("initialize")
128        setrandomseedi(rvalue)
129        rmethod = "initialized"
130    else
131        setrandomseedi(rvalue)
132        rmethod = "resumed"
133    end
134    tobesaved.randomseed = rvalue
135    --
136    collectedmacros = collected.macros
137    tobesavedmacros = tobesaved.macros
138    if not collectedmacros then
139        collectedmacros  = { }
140        collected.macros = collectedmacros
141    end
142    if not tobesavedmacros then
143        tobesavedmacros  = { }
144        tobesaved.macros = tobesavedmacros
145    end
146    -- will become collected.macros
147    for cs, value in next, collectedmacros do
148        if type(value) == "string" then -- safeguard
149            ctx_setxvalue(cs,value)
150        end
151    end
152end
153
154job.register('job.variables.collected', tobesaved, initializer)
155
156function jobvariables.save(cs,value)
157    tobesavedmacros[cs] = value
158end
159
160function jobvariables.restore(cs)
161    return collectedmacros[cs] or tobesavedmacros[cs]
162end
163
164function job.getrandomseed()
165    return tobesaved.randomseed or getrandomseed()
166end
167
168-- checksums
169
170function jobvariables.getchecksum(tag)
171    return checksums[tag] -- no default
172end
173
174function jobvariables.makechecksum(data)
175    return data and md5.HEX(data) -- no default
176end
177
178function jobvariables.setchecksum(tag,checksum)
179    checksums[tag] = checksum
180end
181
182--
183
184local packlist = {
185    "numbers",
186    "ownnumbers",
187    "metadata",
188    "sectiondata",
189    "prefixdata",
190    "numberdata",
191    "pagedata",
192    "directives",
193    "specification",
194    "processors", -- might become key under directives or metadata
195--  "references", -- we need to rename of them as only one packs (not structures.lists.references)
196}
197
198local skiplist = {
199    "datasets",
200    "userdata",
201    "positions",
202}
203
204-- I'm not that impressed by the savings. It's some 5 percent on the luametatex
205-- manual and probably some more on the m4 files (if so I might enable it).
206
207local deltapacking = false
208-- local deltapacking = true
209
210local function packnumberdata(tobesaved)
211    if deltapacking and tobesaved[1] then
212        local last
213        local current
214        for i=1,#tobesaved do
215            current = tobesaved[i]
216            if last then
217                if last.numbers and last.block then
218                    for k, v in next, last do
219                        if k ~= "numbers" and v ~= current[k] then
220                            goto DIFFERENT
221                        end
222                    end
223                    for k, v in next, current do
224                        if k ~= "numbers" and v ~= last[k] then
225                            goto DIFFERENT
226                        end
227                    end
228                    tobesaved[i] = {
229                        numbers = current.numbers,
230                    }
231                    goto CONTINUE
232                else
233                    current = nil
234                end
235            end
236          ::DIFFERENT::
237            last = current
238          ::CONTINUE::
239        end
240    end
241end
242
243local function unpacknumberdata(collected)
244    if deltapacking and collected[1] then
245        local key  = "numbers"
246        local last = collected[1]
247        local meta = false
248        for i=2,#collected do
249            local c = collected[i]
250            if c.block then
251                last = c
252                meta = false
253            elseif c.numbers then
254                if not meta then
255                    meta = { __index = last }
256                end
257                setmetatable(c, meta)
258            end
259        end
260    end
261end
262
263-- -- --
264
265-- not ok as we can have arbitrary keys in userdata and dataset so some day we
266-- might need a bit more granularity, like skippers
267
268local jobpacker = packers.new(packlist,job.packversion,skiplist) -- jump number when changs in hash
269
270job.pack = true
271-- job.pack = false
272
273directives.register("job.pack",function(v) job.pack = v end)
274
275local savedfiles  = { }
276local loadedfiles = { } -- for now only timers
277local othercache  = { }
278
279function job.save(filename) -- we could return a table but it can get pretty large
280    statistics.starttiming(savedfiles)
281    local f = io.open(filename,'w')
282    if f then
283        f:write("local utilitydata = { }\n\n")
284        f:write(serialize(comment,"utilitydata.comment",true),"\n\n")
285        for l=1,#savelist do
286            local list       = savelist[l]
287            local target     = format("utilitydata.%s",list[1])
288            local data       = list[2]
289            local finalizer  = list[4]
290            local serializer = list[5]
291            if type(data) == "string" then
292                data = utilities.tables.accesstable(data)
293            end
294            if type(finalizer) == "function" then
295                finalizer()
296            end
297            if job.pack then
298                packers.pack(data,jobpacker,true)
299            end
300            local definer, name = definetable(target,true,true) -- no first and no last
301            if serializer then
302                f:write(definer,"\n\n",serializer(data,name,true),"\n\n")
303            else
304                f:write(definer,"\n\n",serialize(data,name,true),"\n\n")
305            end
306        end
307        if job.pack then
308            packers.strip(jobpacker)
309            packnumberdata(jobpacker.index)
310            f:write(serialize(jobpacker,"utilitydata.job.packed",true),"\n\n")
311        end
312        f:write("return utilitydata")
313        f:close()
314    end
315    statistics.stoptiming(savedfiles)
316end
317
318local function load(filename)
319    if lfs.isfile(filename) then
320        local function dofile(filename)
321            local result = loadstring(io.loaddata(filename))
322            if result then
323                return result()
324            else
325                return nil
326            end
327        end
328        local okay, data = pcall(dofile,filename)
329        if okay and type(data) == "table" then
330            local jobversion  = job.version
331            local datacomment = data.comment
332            local dataversion = datacomment and datacomment.version or "?"
333            if dataversion ~= jobversion then
334                report_passes("version mismatch: %s <> %s",dataversion,jobversion)
335            else
336                return data
337            end
338        else
339            os.remove(filename)
340            report_passes("removing stale job data file %a, restart job, message: %s",filename,tostring(data))
341            os.exit(true) -- trigger second run
342        end
343    end
344end
345
346function job.load(filename)
347    statistics.starttiming(loadedfiles)
348    local utilitydata = load(filename)
349    if utilitydata then
350        local jobpacker = utilitydata.job.packed
351        unpacknumberdata(jobpacker.index)
352        for i=1,#savelist do
353            local list   = savelist[i]
354            local target = list[1]
355            local result = accesstable(target,utilitydata)
356            if result then
357                local done = packers.unpack(result,jobpacker,true)
358                if done then
359                    migratetable(target,mark(result))
360                else
361                    report_passes("pack version mismatch")
362                end
363            end
364        end
365    end
366    for i=1,#savelist do
367        local list        = savelist[i]
368        local initializer = list[3]
369        if type(initializer) == "function" then
370            initializer(utilitydata and accesstable(list[1],utilitydata) or nil)
371        end
372    end
373    statistics.stoptiming(loadedfiles)
374end
375
376function job.loadother(filename)
377    local jobname = environment.jobname
378    if filename == jobname then
379        return
380    end
381    filename = file.addsuffix(filename,"tuc")
382    local unpacked = othercache[filename]
383    if not unpacked then
384        -- so we can register the same name twice (in loading order) ... needs checking if we want this
385        statistics.starttiming(loadedfiles)
386        local utilitydata = load(filename)
387        if utilitydata then
388            report_passes("integrating list %a into %a",filename,jobname)
389            local jobpacker = utilitydata.job.packed
390            unpacknumberdata(jobpacker.index)
391            unpacked = { }
392            for l=1,#savelist do
393                local list   = savelist[l]
394                local target = list[1]
395                local result = accesstable(target,utilitydata)
396                local done   = packers.unpack(result,jobpacker,true)
397                if done then
398                    migratetable(target,result,unpacked)
399                end
400            end
401            unpacked.job.packed = nil -- nicer in inspecting
402            othercache[filename] = unpacked
403            --
404            utilitydata.components, utilitydata.namestack = collectstructure(utilitydata.job.structure.collected)
405            --
406            structures.lists     .integrate(utilitydata)
407            structures.registers .integrate(utilitydata)
408            structures.references.integrate(utilitydata)
409        end
410        statistics.stoptiming(loadedfiles)
411    end
412    return unpacked
413end
414
415statistics.register("startup time", function()
416    return statistics.elapsedseconds(statistics,"including runtime option file processing")
417end)
418
419statistics.register("jobdata time",function()
420    local elapsedsave = statistics.elapsedtime(savedfiles)
421    local elapsedload = statistics.elapsedtime(loadedfiles)
422    if enabled then
423        if next(othercache) then
424            return format("%s seconds saving, %s seconds loading, other files: %s",elapsedsave,elapsedload,concat(sortedkeys(othercache), ", "))
425        else
426            return format("%s seconds saving, %s seconds loading",elapsedsave,elapsedload)
427        end
428    else
429        if next(othercache) then
430            return format("nothing saved, %s seconds loading, other files: %s",elapsedload,concat(sortedkeys(othercache), ", "))
431        else
432            return format("nothing saved, %s seconds loading",elapsedload)
433        end
434    end
435end)
436
437statistics.register("callbacks", function()
438    local backend  = backends.getcallbackstate()
439    local frontend = status.getcallbackstate()
440    local pages    = structures.pages.nofpages or 0
441    local total    = frontend.count + backend.count
442    local average  = pages > 0 and math.round(total/pages) or 0
443    local result   = format (
444        "file: %s, saved: %s, direct: %s, function: %s, value: %s, message: %s, bytecode: %s, late %s, total: %s (%s per page)",
445        frontend.file,  frontend.saved,   frontend.direct,   frontend["function"],
446        frontend.value, frontend.message, frontend.bytecode, backend.count,
447        total, average
448    )
449    statistics.callbacks = function()
450        return result
451    end
452    return result
453end)
454
455statistics.register("randomizer", function()
456    if rmethod and rvalue then
457        return format("%s with value %s",rmethod,rvalue)
458    end
459end)
460
461function statistics.formatruntime(runtime)
462    if not environment.initex then -- else error when testing as not counters yet
463     -- stoptiming(statistics) -- to be sure
464        local shipped = texgetcount("nofshipouts")
465        local pages = texgetcount("realpageno")
466        if pages > shipped then
467            pages = shipped
468        end
469        runtime = tonumber(runtime)
470        if shipped > 0 or pages > 0 then
471            local persecond = (runtime > 0) and (shipped/runtime) or pages
472            if pages == 0 then
473                pages = shipped
474            end
475            return format("%0.3f seconds, %i processed pages, %i shipped pages, %.3f pages/second",runtime,pages,shipped,persecond)
476        else
477            return format("%0.3f seconds",runtime)
478        end
479    end
480end
481
482implement {
483    name      = "savecurrentvalue",
484    public    = true,
485    actions   = job.variables.save,
486    arguments = { "csname", "argument" },
487}
488
489implement {
490    name      = "setjobcomment",
491    actions   = job.comment,
492    arguments = { { "*" } }
493}
494
495implement {
496    name      = "initializejob",
497    actions   = job.initialize
498}
499
500implement {
501    name      = "disablejobsave",
502    actions   = job.disablesave
503}
504