SourceBrowser

core-uti.lua /size: 14 Kb last modification: 2023-12-21 09:44
1if not modules then modules = { } end modules ['core-uti'] = {
2    version   = 1.001,
3    comment   = "companion to core-uti.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- A utility file has always been part of ConTeXt and with the move to LuaTeX we
10-- also moved a lot of multi-pass info to a Lua table. Instead of loading a TeX
11-- based utility file under different setups, we now load a table once. This saves
12-- much runtime but at the cost of more memory usage.
13--
14-- In the meantime the overhead is a bit more due to the amount of data being saved
15-- and more agressive compacting.
16
17local math = math
18local format, match = string.format, string.match
19local next, type, tostring, tonumber = next, type, tostring, tonumber
20local concat = table.concat
21
22local definetable    = utilities.tables.definetable
23local accesstable    = utilities.tables.accesstable
24local migratetable   = utilities.tables.migratetable
25local serialize      = table.serialize
26local packers        = utilities.packers
27local allocate       = utilities.storage.allocate
28local mark           = utilities.storage.mark
29
30local getrandom      = utilities.randomizer.get
31local setrandomseedi = utilities.randomizer.setseedi
32local getrandomseed  = utilities.randomizer.getseed
33
34local implement      = interfaces.implement
35
36local texgetcount    = tex.getcount
37
38local report_passes  = logs.reporter("job","passes")
39
40job                  = job or { }
41local job            = job
42
43job.version          = 1.32
44job.packversion      = 1.02
45
46-- Variables are saved using in the previously defined table and passed onto TeX
47-- using the following method. Of course one can also directly access the variable
48-- using a Lua call.
49
50local savelist, comment = { }, { }
51
52function job.comment(key,value)
53    if type(key) == "table" then
54        for k, v in next, key do
55            comment[k] = v
56        end
57    else
58        comment[key] = value
59    end
60end
61
62job.comment("version",job.version)
63
64local enabled     = true
65local initialized = false
66
67directives.register("job.save",function(v) enabled = v end)
68
69function job.disablesave()
70    enabled = false -- for instance called when an error
71end
72
73function job.initialize(loadname,savename)
74    if not initialized then
75        if not loadname or loadname == "" then
76            loadname = tex.jobname .. ".tuc"
77        end
78        if not savename or savename == "" then
79            savename = tex.jobname .. ".tua"
80        end
81        job.load(loadname) -- has to come after structure is defined !
82        luatex.registerstopactions(function()
83            if enabled then
84                job.save(savename)
85            end
86        end)
87        initialized = true
88    end
89end
90
91function job.register(collected, tobesaved, initializer, finalizer, serializer)
92    savelist[#savelist+1] = { collected, tobesaved, initializer, finalizer, serializer }
93end
94
95-- as an example we implement variables
96
97local tobesaved, collected, checksums = allocate(), allocate(), allocate()
98
99local jobvariables = {
100    collected = collected,
101    tobesaved = tobesaved,
102    checksums = checksums,
103}
104
105-- if not checksums.old then checksums.old = md5.HEX("old") end -- used in experiment
106-- if not checksums.new then checksums.new = md5.HEX("new") end -- used in experiment
107
108job.variables = jobvariables
109
110local function initializer()
111    checksums = jobvariables.checksums
112end
113
114job.register('job.variables.checksums', 'job.variables.checksums', initializer)
115
116local rmethod, rvalue
117local collectedmacros, tobesavedmacros
118
119local ctx_setxvalue = context.setxvalue
120
121local function initializer()
122    tobesaved = jobvariables.tobesaved
123    collected = jobvariables.collected
124    --
125    rvalue = collected.randomseed
126    if not rvalue then
127        rvalue = getrandom("initialize")
128        setrandomseedi(rvalue)
129        rmethod = "initialized"
130    else
131        setrandomseedi(rvalue)
132        rmethod = "resumed"
133    end
134    tobesaved.randomseed = rvalue
135    --
136    collectedmacros = collected.macros
137    tobesavedmacros = tobesaved.macros
138    if not collectedmacros then
139        collectedmacros  = { }
140        collected.macros = collectedmacros
141    end
142    if not tobesavedmacros then
143        tobesavedmacros  = { }
144        tobesaved.macros = tobesavedmacros
145    end
146    -- will become collected.macros
147    for cs, value in next, collectedmacros do
148        if type(value) == "string" then -- safeguard
149            ctx_setxvalue(cs,value)
150        end
151    end
152end
153
154job.register('job.variables.collected', tobesaved, initializer)
155
156function jobvariables.save(cs,value)
157    tobesavedmacros[cs] = value
158end
159
160function jobvariables.restore(cs)
161    return collectedmacros[cs] or tobesavedmacros[cs]
162end
163
164function job.getrandomseed()
165    return tobesaved.randomseed or getrandomseed()
166end
167
168-- checksums
169
170function jobvariables.getchecksum(tag)
171    return checksums[tag] -- no default
172end
173
174function jobvariables.makechecksum(data)
175    return data and md5.HEX(data) -- no default
176end
177
178function jobvariables.setchecksum(tag,checksum)
179    checksums[tag] = checksum
180end
181
182--
183
184local packlist = {
185    "numbers",
186    "ownnumbers",
187    "metadata",
188    "sectiondata",
189    "prefixdata",
190    "numberdata",
191    "pagedata",
192    "directives",
193    "specification",
194    "processors", -- might become key under directives or metadata
195--  "references", -- we need to rename of them as only one packs (not structures.lists.references)
196}
197
198local skiplist = {
199    "datasets",
200    "userdata",
201    "positions",
202}
203
204-- not ok as we can have arbitrary keys in userdata and dataset so some day we
205-- might need a bit more granularity, like skippers
206
207local jobpacker = packers.new(packlist,job.packversion,skiplist) -- jump number when changs in hash
208
209job.pack = true
210-- job.pack = false
211
212directives.register("job.pack",function(v) job.pack = v end)
213
214local _save_, _load_, _others_ = { }, { }, { } -- registers timing
215
216function job.save(filename) -- we could return a table but it can get pretty large
217    statistics.starttiming(_save_)
218    local f = io.open(filename,'w')
219    if f then
220        f:write("local utilitydata = { }\n\n")
221        f:write(serialize(comment,"utilitydata.comment",true),"\n\n")
222        for l=1,#savelist do
223         -- f:write("do\n\n") -- no solution for the jit limitatione either
224            local list       = savelist[l]
225            local target     = format("utilitydata.%s",list[1])
226            local data       = list[2]
227            local finalizer  = list[4]
228            local serializer = list[5]
229            if type(data) == "string" then
230                data = utilities.tables.accesstable(data)
231            end
232            if type(finalizer) == "function" then
233                finalizer()
234            end
235            if job.pack then
236                packers.pack(data,jobpacker,true)
237            end
238            local definer, name = definetable(target,true,true) -- no first and no last
239            if serializer then
240                f:write(definer,"\n\n",serializer(data,name,true),"\n\n")
241            else
242                f:write(definer,"\n\n",serialize(data,name,true),"\n\n")
243            end
244         -- f:write("end\n\n")
245        end
246        if job.pack then
247            packers.strip(jobpacker)
248         -- f:write("do\n\n")
249            f:write(serialize(jobpacker,"utilitydata.job.packed",true),"\n\n")
250         -- f:write("end\n\n")
251        end
252        f:write("return utilitydata")
253        f:close()
254    end
255    statistics.stoptiming(_save_)
256end
257
258local function load(filename)
259    if lfs.isfile(filename) then
260
261        local function dofile(filename)
262            local result = loadstring(io.loaddata(filename))
263            if result then
264                return result()
265            else
266                return nil
267            end
268        end
269
270        local okay, data = pcall(dofile,filename)
271        if okay and type(data) == "table" then
272            local jobversion  = job.version
273            local datacomment = data.comment
274            local dataversion = datacomment and datacomment.version or "?"
275            if dataversion ~= jobversion then
276                report_passes("version mismatch: %s <> %s",dataversion,jobversion)
277            else
278                return data
279            end
280        else
281            os.remove(filename) -- probably a bad file (or luajit overflow as it cannot handle large tables well)
282            report_passes("removing stale job data file %a, restart job, message: %s%s",filename,tostring(data),
283                jit and " (try luatex instead of luajittex)" or "")
284            os.exit(true) -- trigger second run
285        end
286    end
287end
288
289function job.load(filename)
290    statistics.starttiming(_load_)
291    local utilitydata = load(filename)
292    if utilitydata then
293        local jobpacker = utilitydata.job.packed
294        local handlers  = { }
295        for i=1,#savelist do
296            local list        = savelist[i]
297            local target      = list[1]
298            local initializer = list[3]
299            local result      = accesstable(target,utilitydata)
300            if result then
301                local done = packers.unpack(result,jobpacker,true)
302                if done then
303                    migratetable(target,mark(result))
304                    if type(initializer) == "function" then
305                        handlers[#handlers+1] = { initializer, result }
306                    end
307                else
308                    report_passes("pack version mismatch")
309                end
310            end
311        end
312        -- so we have all tables available (unpacked)
313        for i=1,#handlers do
314            local handler = handlers[i]
315            handler[1](handler[2])
316        end
317    end
318    statistics.stoptiming(_load_)
319end
320
321function job.loadother(filename)
322    statistics.starttiming(_load_)
323    _others_[#_others_+1] = file.nameonly(filename)
324    local utilitydata = load(filename)
325    if utilitydata then
326        local jobpacker = utilitydata.job.packed
327        local unpacked = { }
328        for l=1,#savelist do
329            local list   = savelist[l]
330            local target = list[1]
331            local result = accesstable(target,utilitydata)
332            local done = packers.unpack(result,jobpacker,true)
333            if done then
334                migratetable(target,result,unpacked)
335            end
336        end
337        unpacked.job.packed = nil -- nicer in inspecting
338        return unpacked
339    end
340    statistics.stoptiming(_load_)
341end
342
343-- function job.keep(filename)
344--     local suffix = file.suffix(filename)
345--     local base   = file.removesuffix(filename)
346--     if suffix == "" then
347--         suffix = "tuc"
348--     end
349--     for i=1,10 do
350--         local tmpname = format("%s-%s-%02d.tmp",base,suffix,i)
351--         if lfs.isfile(tmpname) then
352--             os.remove(tmpname)
353--             report_passes("removing %a",tmpname)
354--         end
355--     end
356--     if lfs.isfile(filename) then
357--         local tmpname = format("%s-%s-%02d.tmp",base,suffix,environment.currentrun or 1)
358--         report_passes("copying %a into %a",filename,tmpname)
359--         file.copy(filename,tmpname)
360--     else
361--         report_passes("no file %a, nothing kept",filename)
362--     end
363-- end
364
365-- eventually this will end up in strc-ini
366
367statistics.register("startup time", function()
368    return statistics.elapsedseconds(statistics,"including runtime option file processing")
369end)
370
371statistics.register("jobdata time",function()
372    if enabled then
373        if #_others_ > 0 then
374            return format("%s seconds saving, %s seconds loading, other files: %s",statistics.elapsedtime(_save_),statistics.elapsedtime(_load_),concat(_others_," "))
375        else
376            return format("%s seconds saving, %s seconds loading",statistics.elapsedtime(_save_),statistics.elapsedtime(_load_))
377        end
378    else
379        if #_others_ > 0 then
380            return format("nothing saved, %s seconds loading, other files: %s",statistics.elapsedtime(_load_),concat(_others_," "))
381        else
382            return format("nothing saved, %s seconds loading",statistics.elapsedtime(_load_))
383        end
384    end
385end)
386
387statistics.register("callbacks", function()
388    local c_internal = status.callbacks or 0
389    local c_file     = status.indirect_callbacks or 0
390    local c_direct   = status.direct_callbacks or 0
391    local c_late     = backends.getcallbackstate().count
392    local c_function = status.function_callbacks or 0
393    local c_total    = c_internal + c_file + c_direct + c_late + c_function
394    local n_pages    = structures.pages.nofpages or 0
395    local c_average  = n_pages > 0 and math.round(c_total/n_pages) or 0
396    local result     = format (
397        "internal: %s, file: %s, direct: %s, late: %s, function %s, total: %s (%s per page)",
398        c_internal, c_file, c_direct, c_late, c_function, c_total, c_average
399    )
400    statistics.callbacks = function()
401        return result
402    end
403    return result
404end)
405
406statistics.register("randomizer", function()
407    if rmethod and rvalue then
408        return format("%s with value %s",rmethod,rvalue)
409    end
410end)
411
412-- a sort of joke (for ctx meeting)
413
414-- local kg_per_watt_per_second  = 1 / 15000000
415-- local watts_per_core          = 50
416-- local speedup_by_other_engine = 1.2
417-- local used_wood_factor        = watts_per_core * kg_per_watt_per_second / speedup_by_other_engine
418-- local used_wood_factor        = (50 / 15000000) / 1.2
419
420
421function statistics.formatruntime(runtime)
422    if not environment.initex then -- else error when testing as not counters yet
423     -- stoptiming(statistics) -- to be sure
424        local shipped = texgetcount('nofshipouts')
425        local pages = texgetcount('realpageno')
426        if pages > shipped then
427            pages = shipped
428        end
429        runtime = tonumber(runtime)
430        if shipped > 0 or pages > 0 then
431            local persecond = (runtime > 0) and (shipped/runtime) or pages
432            if pages == 0 then
433                pages = shipped
434            end
435            return format("%0.3f seconds, %i processed pages, %i shipped pages, %.3f pages/second",runtime,pages,shipped,persecond)
436        else
437            return format("%0.3f seconds",runtime)
438        end
439    end
440end
441
442implement {
443    name      = "savevariable",
444    actions   = job.variables.save,
445    arguments = "2 strings",
446}
447
448implement {
449    name      = "setjobcomment",
450    actions   = job.comment,
451    arguments = { { "*" } }
452}
453
454implement {
455    name      = "initializejob",
456    actions   = job.initialize
457}
458
459implement {
460    name      = "disablejobsave",
461    actions   = job.disablesave
462}
463
Source Browser ?
