SourceBrowser

core-uti.lua /size: 14 Kb last modification: 2021-10-28 13:51
1if not modules then modules = { } end modules ['core-uti'] = {
2    version   = 1.001,
3    comment   = "companion to core-uti.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- todo: keep track of changes here (hm, track access, and only true when
10-- accessed and changed)
11
12--[[ldx--
13<p>A utility file has always been part of <l n='context'/> and with
14the move to <l n='luatex'/> we also moved a lot of multi-pass info
15to a <l n='lua'/> table. Instead of loading a <l n='tex'/> based
16utility file under different setups, we now load a table once. This
17saves much runtime but at the cost of more memory usage.</p>
18--ldx]]--
19
20local math = math
21local format, match = string.format, string.match
22local next, type, tostring, tonumber = next, type, tostring, tonumber
23local concat = table.concat
24
25local definetable    = utilities.tables.definetable
26local accesstable    = utilities.tables.accesstable
27local migratetable   = utilities.tables.migratetable
28local serialize      = table.serialize
29local packers        = utilities.packers
30local allocate       = utilities.storage.allocate
31local mark           = utilities.storage.mark
32
33local getrandom      = utilities.randomizer.get
34local setrandomseedi = utilities.randomizer.setseedi
35local getrandomseed  = utilities.randomizer.getseed
36
37local implement      = interfaces.implement
38
39local texgetcount    = tex.getcount
40
41local report_passes  = logs.reporter("job","passes")
42
43job                  = job or { }
44local job            = job
45
46job.version          = 1.32
47job.packversion      = 1.02
48
49-- some day we will implement loading of other jobs and then we need
50-- job.jobs
51
52--[[ldx--
53<p>Variables are saved using in the previously defined table and passed
54onto <l n='tex'/> using the following method. Of course one can also
55directly access the variable using a <l n='lua'/> call.</p>
56--ldx]]--
57
58local savelist, comment = { }, { }
59
60function job.comment(key,value)
61    if type(key) == "table" then
62        for k, v in next, key do
63            comment[k] = v
64        end
65    else
66        comment[key] = value
67    end
68end
69
70job.comment("version",job.version)
71
72local enabled     = true
73local initialized = false
74
75directives.register("job.save",function(v) enabled = v end)
76
77function job.disablesave()
78    enabled = false -- for instance called when an error
79end
80
81function job.initialize(loadname,savename)
82    if not initialized then
83        if not loadname or loadname == "" then
84            loadname = tex.jobname .. ".tuc"
85        end
86        if not savename or savename == "" then
87            savename = tex.jobname .. ".tua"
88        end
89        job.load(loadname) -- has to come after structure is defined !
90        luatex.registerstopactions(function()
91            if enabled then
92                job.save(savename)
93            end
94        end)
95        initialized = true
96    end
97end
98
99function job.register(collected, tobesaved, initializer, finalizer, serializer)
100    savelist[#savelist+1] = { collected, tobesaved, initializer, finalizer, serializer }
101end
102
103-- as an example we implement variables
104
105local tobesaved, collected, checksums = allocate(), allocate(), allocate()
106
107local jobvariables = {
108    collected = collected,
109    tobesaved = tobesaved,
110    checksums = checksums,
111}
112
113-- if not checksums.old then checksums.old = md5.HEX("old") end -- used in experiment
114-- if not checksums.new then checksums.new = md5.HEX("new") end -- used in experiment
115
116job.variables = jobvariables
117
118local function initializer()
119    checksums = jobvariables.checksums
120end
121
122job.register('job.variables.checksums', 'job.variables.checksums', initializer)
123
124local rmethod, rvalue
125local collectedmacros, tobesavedmacros
126
127local ctx_setxvalue = context.setxvalue
128
129local function initializer()
130    tobesaved = jobvariables.tobesaved
131    collected = jobvariables.collected
132    --
133    rvalue = collected.randomseed
134    if not rvalue then
135        rvalue = getrandom("initialize")
136        setrandomseedi(rvalue)
137        rmethod = "initialized"
138    else
139        setrandomseedi(rvalue)
140        rmethod = "resumed"
141    end
142    tobesaved.randomseed = rvalue
143    --
144    collectedmacros = collected.macros
145    tobesavedmacros = tobesaved.macros
146    if not collectedmacros then
147        collectedmacros  = { }
148        collected.macros = collectedmacros
149    end
150    if not tobesavedmacros then
151        tobesavedmacros  = { }
152        tobesaved.macros = tobesavedmacros
153    end
154    -- will become collected.macros
155    for cs, value in next, collectedmacros do
156        if type(value) == "string" then -- safeguard
157            ctx_setxvalue(cs,value)
158        end
159    end
160end
161
162job.register('job.variables.collected', tobesaved, initializer)
163
164function jobvariables.save(cs,value)
165    tobesavedmacros[cs] = value
166end
167
168function jobvariables.restore(cs)
169    return collectedmacros[cs] or tobesavedmacros[cs]
170end
171
172function job.getrandomseed()
173    return tobesaved.randomseed or getrandomseed()
174end
175
176-- checksums
177
178function jobvariables.getchecksum(tag)
179    return checksums[tag] -- no default
180end
181
182function jobvariables.makechecksum(data)
183    return data and md5.HEX(data) -- no default
184end
185
186function jobvariables.setchecksum(tag,checksum)
187    checksums[tag] = checksum
188end
189
190--
191
192local packlist = {
193    "numbers",
194    "ownnumbers",
195    "metadata",
196    "sectiondata",
197    "prefixdata",
198    "numberdata",
199    "pagedata",
200    "directives",
201    "specification",
202    "processors", -- might become key under directives or metadata
203--  "references", -- we need to rename of them as only one packs (not structures.lists.references)
204}
205
206local skiplist = {
207    "datasets",
208    "userdata",
209    "positions",
210}
211
212-- not ok as we can have arbitrary keys in userdata and dataset so some day we
213-- might need a bit more granularity, like skippers
214
215local jobpacker = packers.new(packlist,job.packversion,skiplist) -- jump number when changs in hash
216
217job.pack = true
218-- job.pack = false
219
220directives.register("job.pack",function(v) job.pack = v end)
221
222local _save_, _load_, _others_ = { }, { }, { } -- registers timing
223
224function job.save(filename) -- we could return a table but it can get pretty large
225    statistics.starttiming(_save_)
226    local f = io.open(filename,'w')
227    if f then
228        f:write("local utilitydata = { }\n\n")
229        f:write(serialize(comment,"utilitydata.comment",true),"\n\n")
230        for l=1,#savelist do
231         -- f:write("do\n\n") -- no solution for the jit limitatione either
232            local list       = savelist[l]
233            local target     = format("utilitydata.%s",list[1])
234            local data       = list[2]
235            local finalizer  = list[4]
236            local serializer = list[5]
237            if type(data) == "string" then
238                data = utilities.tables.accesstable(data)
239            end
240            if type(finalizer) == "function" then
241                finalizer()
242            end
243            if job.pack then
244                packers.pack(data,jobpacker,true)
245            end
246            local definer, name = definetable(target,true,true) -- no first and no last
247            if serializer then
248                f:write(definer,"\n\n",serializer(data,name,true),"\n\n")
249            else
250                f:write(definer,"\n\n",serialize(data,name,true),"\n\n")
251            end
252         -- f:write("end\n\n")
253        end
254        if job.pack then
255            packers.strip(jobpacker)
256         -- f:write("do\n\n")
257            f:write(serialize(jobpacker,"utilitydata.job.packed",true),"\n\n")
258         -- f:write("end\n\n")
259        end
260        f:write("return utilitydata")
261        f:close()
262    end
263    statistics.stoptiming(_save_)
264end
265
266local function load(filename)
267    if lfs.isfile(filename) then
268
269        local function dofile(filename)
270            local result = loadstring(io.loaddata(filename))
271            if result then
272                return result()
273            else
274                return nil
275            end
276        end
277
278        local okay, data = pcall(dofile,filename)
279        if okay and type(data) == "table" then
280            local jobversion  = job.version
281            local datacomment = data.comment
282            local dataversion = datacomment and datacomment.version or "?"
283            if dataversion ~= jobversion then
284                report_passes("version mismatch: %s <> %s",dataversion,jobversion)
285            else
286                return data
287            end
288        else
289            os.remove(filename) -- probably a bad file (or luajit overflow as it cannot handle large tables well)
290            report_passes("removing stale job data file %a, restart job, message: %s%s",filename,tostring(data),
291                jit and " (try luatex instead of luajittex)" or "")
292            os.exit(true) -- trigger second run
293        end
294    end
295end
296
297function job.load(filename)
298    statistics.starttiming(_load_)
299    local utilitydata = load(filename)
300    if utilitydata then
301        local jobpacker = utilitydata.job.packed
302        local handlers  = { }
303        for i=1,#savelist do
304            local list        = savelist[i]
305            local target      = list[1]
306            local initializer = list[3]
307            local result      = accesstable(target,utilitydata)
308            if result then
309                local done = packers.unpack(result,jobpacker,true)
310                if done then
311                    migratetable(target,mark(result))
312                    if type(initializer) == "function" then
313                        handlers[#handlers+1] = { initializer, result }
314                    end
315                else
316                    report_passes("pack version mismatch")
317                end
318            end
319        end
320        -- so we have all tables available (unpacked)
321        for i=1,#handlers do
322            local handler = handlers[i]
323            handler[1](handler[2])
324        end
325    end
326    statistics.stoptiming(_load_)
327end
328
329function job.loadother(filename)
330    statistics.starttiming(_load_)
331    _others_[#_others_+1] = file.nameonly(filename)
332    local utilitydata = load(filename)
333    if utilitydata then
334        local jobpacker = utilitydata.job.packed
335        local unpacked = { }
336        for l=1,#savelist do
337            local list   = savelist[l]
338            local target = list[1]
339            local result = accesstable(target,utilitydata)
340            local done = packers.unpack(result,jobpacker,true)
341            if done then
342                migratetable(target,result,unpacked)
343            end
344        end
345        unpacked.job.packed = nil -- nicer in inspecting
346        return unpacked
347    end
348    statistics.stoptiming(_load_)
349end
350
351-- function job.keep(filename)
352--     local suffix = file.suffix(filename)
353--     local base   = file.removesuffix(filename)
354--     if suffix == "" then
355--         suffix = "tuc"
356--     end
357--     for i=1,10 do
358--         local tmpname = format("%s-%s-%02d.tmp",base,suffix,i)
359--         if lfs.isfile(tmpname) then
360--             os.remove(tmpname)
361--             report_passes("removing %a",tmpname)
362--         end
363--     end
364--     if lfs.isfile(filename) then
365--         local tmpname = format("%s-%s-%02d.tmp",base,suffix,environment.currentrun or 1)
366--         report_passes("copying %a into %a",filename,tmpname)
367--         file.copy(filename,tmpname)
368--     else
369--         report_passes("no file %a, nothing kept",filename)
370--     end
371-- end
372
373-- eventually this will end up in strc-ini
374
375statistics.register("startup time", function()
376    return statistics.elapsedseconds(statistics,"including runtime option file processing")
377end)
378
379statistics.register("jobdata time",function()
380    if enabled then
381        if #_others_ > 0 then
382            return format("%s seconds saving, %s seconds loading, other files: %s",statistics.elapsedtime(_save_),statistics.elapsedtime(_load_),concat(_others_," "))
383        else
384            return format("%s seconds saving, %s seconds loading",statistics.elapsedtime(_save_),statistics.elapsedtime(_load_))
385        end
386    else
387        if #_others_ > 0 then
388            return format("nothing saved, %s seconds loading, other files: %s",statistics.elapsedtime(_load_),concat(_others_," "))
389        else
390            return format("nothing saved, %s seconds loading",statistics.elapsedtime(_load_))
391        end
392    end
393end)
394
395statistics.register("callbacks", function()
396    local c_internal = status.callbacks or 0
397    local c_file     = status.indirect_callbacks or 0
398    local c_direct   = status.direct_callbacks or 0
399    local c_late     = backends.getcallbackstate().count
400    local c_function = status.function_callbacks or 0
401    local c_total    = c_internal + c_file + c_direct + c_late + c_function
402    local n_pages    = structures.pages.nofpages or 0
403    local c_average  = n_pages > 0 and math.round(c_total/n_pages) or 0
404    local result     = format (
405        "internal: %s, file: %s, direct: %s, late: %s, function %s, total: %s (%s per page)",
406        c_internal, c_file, c_direct, c_late, c_function, c_total, c_average
407    )
408    statistics.callbacks = function()
409        return result
410    end
411    return result
412end)
413
414statistics.register("randomizer", function()
415    if rmethod and rvalue then
416        return format("%s with value %s",rmethod,rvalue)
417    end
418end)
419
420-- a sort of joke (for ctx meeting)
421
422-- local kg_per_watt_per_second  = 1 / 15000000
423-- local watts_per_core          = 50
424-- local speedup_by_other_engine = 1.2
425-- local used_wood_factor        = watts_per_core * kg_per_watt_per_second / speedup_by_other_engine
426-- local used_wood_factor        = (50 / 15000000) / 1.2
427
428
429function statistics.formatruntime(runtime)
430    if not environment.initex then -- else error when testing as not counters yet
431     -- stoptiming(statistics) -- to be sure
432        local shipped = texgetcount('nofshipouts')
433        local pages = texgetcount('realpageno')
434        if pages > shipped then
435            pages = shipped
436        end
437        runtime = tonumber(runtime)
438        if shipped > 0 or pages > 0 then
439            local persecond = (runtime > 0) and (shipped/runtime) or pages
440            if pages == 0 then
441                pages = shipped
442            end
443            return format("%0.3f seconds, %i processed pages, %i shipped pages, %.3f pages/second",runtime,pages,shipped,persecond)
444        else
445            return format("%0.3f seconds",runtime)
446        end
447    end
448end
449
450implement {
451    name      = "savevariable",
452    actions   = job.variables.save,
453    arguments = "2 strings",
454}
455
456implement {
457    name      = "setjobcomment",
458    actions   = job.comment,
459    arguments = { { "*" } }
460}
461
462implement {
463    name      = "initializejob",
464    actions   = job.initialize
465}
466
467implement {
468    name      = "disablejobsave",
469    actions   = job.disablesave
470}
471
Source Browser ?
