Source Browser ?

1if not modules then modules = { } end modules ['core-uti'] = {
2    version   = 1.001,
3    comment   = "companion to core-uti.mkiv",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE / ConTeXt Development Team",
6    license   = "see context related readme files"
7}
8
9-- A utility file has always been part of ConTeXt and with the move to LuaTeX we
10-- also moved a lot of multi-pass info to a Lua table. Instead of loading a TeX
11-- based utility file under different setups, we now load a table once. This saves
12-- much runtime but at the cost of more memory usage.
13--
14-- In the meantime the overhead is a bit more due to the amount of data being saved
15-- and more agressive compacting.
16
17local math = math
18local next, type, tostring, tonumber, setmetatable, load = next, type, tostring, tonumber, setmetatable, load
19local format, match = string.format, string.match
20local concat, sortedkeys = table.concat, table.sortedkeys
21
22local definetable    = utilities.tables.definetable
23local accesstable    = utilities.tables.accesstable
24local migratetable   = utilities.tables.migratetable
25local serialize      = table.serialize
26local packers        = utilities.packers
27local allocate       = utilities.storage.allocate
28local mark           = utilities.storage.mark
29
30local getrandom      = utilities.randomizer.get
31local setrandomseedi = utilities.randomizer.setseedi
32local getrandomseed  = utilities.randomizer.getseed
33
34local implement      = interfaces.implement
35
36local texgetcount    = tex.getcount
37
38local report_passes  = logs.reporter("job","passes")
39
40job                  = job or { }
41local job            = job
42
43job.version          = 1.33
44job.packversion      = 1.02
45
46-- Variables are saved using in the previously defined table and passed onto TeX
47-- using the following method. Of course one can also directly access the variable
48-- using a Lua call.
49
50local savelist, comment = { }, { }
51
52function job.comment(key,value)
53    if type(key) == "table" then
54        for k, v in next, key do
55            comment[k] = v
56        end
57    else
58        comment[key] = value
59    end
60end
61
62job.comment("version",job.version)
63
64local enabled     = true
65local initialized = false
66
67directives.register("job.save",function(v) enabled = v end)
68
69function job.disablesave()
70    enabled = false -- for instance called when an error
71end
72
73function job.initialize(loadname,savename)
74    if not initialized then
75        if not loadname or loadname == "" then
76            loadname = tex.jobname .. ".tuc"
77        end
78        if not savename or savename == "" then
79            savename = tex.jobname .. ".tua"
80        end
81        job.load(loadname) -- has to come after structure is defined !
82        luatex.registerstopactions(function()
83            if enabled then
84                job.save(savename)
85            end
86        end)
87        initialized = true
88    end
89end
90
91function job.register(collected, tobesaved, initializer, finalizer, serializer)
92    savelist[#savelist+1] = { collected, tobesaved, initializer, finalizer, serializer }
93end
94
95-- as an example we implement variables
96--
97-- if not checksums.old then checksums.old = md5.HEX("old") end -- used in experiment
98-- if not checksums.new then checksums.new = md5.HEX("new") end -- used in experiment
99
100do
101
102    local tobesaved = allocate()
103    local collected = allocate()
104    local checksums = allocate()
105
106    local jobvariables = {
107        collected = collected,
108        tobesaved = tobesaved,
109        checksums = checksums,
110    }
111
112    job.variables = jobvariables
113
114    local function initializer()
115        checksums = jobvariables.checksums
116    end
117
118    job.register('job.variables.checksums', 'job.variables.checksums', initializer)
119
120    local rmethod, rvalue
121
122    statistics.register("randomizer", function()
123        if rmethod and rvalue then
124            return format("%s with value %s",rmethod,rvalue)
125        end
126    end)
127
128    local collectedmacros
129    local tobesavedmacros
130
131 -- local setmacro = context.setxvalue
132    local setmacro = token.setmacro
133
134    local function initializer(indeed)
135        if indeed then
136            -- can be changed due to packing
137            collected              = indeed
138            jobvariables.collected = indeed
139        end
140        --
141        rvalue = collected.randomseed
142        if not rvalue then
143            rvalue = getrandom("initialize")
144            setrandomseedi(rvalue)
145            rmethod = "initialized"
146        else
147            setrandomseedi(rvalue)
148            rmethod = "resumed"
149        end
150        collected.randomseed = rvalue
151        tobesaved.randomseed = rvalue
152        --
153        collectedmacros = collected.macros
154        tobesavedmacros = tobesaved.macros
155        if not collectedmacros then
156            collectedmacros  = { }
157            collected.macros = collectedmacros
158        end
159        if not tobesavedmacros then
160            tobesavedmacros  = { }
161            tobesaved.macros = tobesavedmacros
162        end
163        -- will become collected.macros
164        for cs, value in next, collectedmacros do
165            if type(value) == "string" then -- safeguard
166                setmacro(cs,value,"global")
167            end
168        end
169    end
170
171    job.register('job.variables.collected', tobesaved, initializer)
172
173    function jobvariables.save(cs,value)
174        tobesavedmacros[cs] = value
175    end
176
177    function jobvariables.restore(cs)
178        return collectedmacros[cs] or tobesavedmacros[cs]
179    end
180
181    function job.getrandomseed()
182        return tobesaved.randomseed or getrandomseed()
183    end
184
185    -- checksums
186
187    function jobvariables.getchecksum(tag)
188        return checksums[tag] -- no default
189    end
190
191    function jobvariables.makechecksum(data)
192        return data and md5.HEX(data) -- no default
193    end
194
195    function jobvariables.setchecksum(tag,checksum)
196        checksums[tag] = checksum
197    end
198
199end
200
201--
202
203local packlist = {
204    "numbers",
205    "ownnumbers",
206    "metadata",
207    "sectiondata",
208    "prefixdata",
209    "numberdata",
210    "pagedata",
211    "directives",
212    "specification",
213    "processors", -- might become key under directives or metadata
214--  "references", -- we need to rename of them as only one packs (not structures.lists.references)
215}
216
217local skiplist = {
218    "datasets",
219    "userdata",
220    "positions",
221--     "variables",
222}
223
224-- I'm not that impressed by the savings. It's some 5 percent on the luametatex
225-- manual and probably some more on the m4 files (if so I might enable it).
226
227local deltapacking = false
228-- local deltapacking = true
229
230local function packnumberdata(tobesaved)
231    if deltapacking and tobesaved[1] then
232        local last
233        local current
234        for i=1,#tobesaved do
235            current = tobesaved[i]
236            if last then
237                if last.numbers and last.block then
238                    for k, v in next, last do
239                        if k ~= "numbers" and v ~= current[k] then
240                            goto DIFFERENT
241                        end
242                    end
243                    for k, v in next, current do
244                        if k ~= "numbers" and v ~= last[k] then
245                            goto DIFFERENT
246                        end
247                    end
248                    tobesaved[i] = {
249                        numbers = current.numbers,
250                    }
251                    goto CONTINUE
252                else
253                    current = nil
254                end
255            end
256          ::DIFFERENT::
257            last = current
258          ::CONTINUE::
259        end
260    end
261end
262
263local function unpacknumberdata(collected)
264    if deltapacking and collected[1] then
265        local key  = "numbers"
266        local last = collected[1]
267        local meta = false
268        for i=2,#collected do
269            local c = collected[i]
270            if c.block then
271                last = c
272                meta = false
273            elseif c.numbers then
274                if not meta then
275                    meta = { __index = last }
276                end
277                setmetatable(c, meta)
278            end
279        end
280    end
281end
282
283-- -- --
284
285-- not ok as we can have arbitrary keys in userdata and dataset so some day we
286-- might need a bit more granularity, like skippers
287
288local jobpacker = packers.new(packlist,job.packversion,skiplist) -- jump number when changs in hash
289
290job.pack = true
291-- job.pack = false
292
293directives.register("job.pack",function(v) job.pack = v end)
294
295local savedfiles  = { }
296local loadedfiles = { } -- for now only timers
297local othercache  = { }
298
299function job.save(filename) -- we could return a table but it can get pretty large
300    statistics.starttiming(savedfiles)
301    local f = io.open(filename,'w')
302    if f then
303        f:write("local utilitydata = { }\n\n")
304        f:write(serialize(comment,"utilitydata.comment",true),"\n\n")
305        for l=1,#savelist do
306            local list       = savelist[l]
307            local target     = format("utilitydata.%s",list[1])
308            local data       = list[2]
309            local finalizer  = list[4]
310            local serializer = list[5]
311            if type(data) == "string" then
312                data = utilities.tables.accesstable(data)
313            end
314            if type(finalizer) == "function" then
315                finalizer()
316            end
317            if job.pack then
318                packers.pack(data,jobpacker,true)
319            end
320            local definer, name = definetable(target,true,true) -- no first and no last
321            if serializer then
322                f:write(definer,"\n\n",serializer(data,name,true),"\n\n")
323            else
324                f:write(definer,"\n\n",serialize(data,name,true),"\n\n")
325            end
326        end
327        if job.pack then
328            packers.strip(jobpacker)
329            packnumberdata(jobpacker.index)
330            f:write(serialize(jobpacker,"utilitydata.job.packed",true),"\n\n")
331        end
332        f:write("return utilitydata")
333        f:close()
334    end
335    statistics.stoptiming(savedfiles)
336end
337
338local function load(filename)
339    if lfs.isfile(filename) then
340        local function dofile(filename)
341            local result = loadstring(io.loaddata(filename))
342            if result then
343                return result()
344            else
345                return nil
346            end
347        end
348        local okay, data = pcall(dofile,filename)
349        if okay and type(data) == "table" then
350            local jobversion  = job.version
351            local datacomment = data.comment
352            local dataversion = datacomment and datacomment.version or "?"
353            if dataversion ~= jobversion then
354                report_passes("version mismatch: %s <> %s",dataversion,jobversion)
355            else
356                return data
357            end
358        else
359            os.remove(filename)
360            report_passes("removing stale job data file %a, restart job, message: %s",filename,tostring(data))
361            os.exit(true) -- trigger second run
362        end
363    end
364end
365
366function job.load(filename)
367    statistics.starttiming(loadedfiles)
368    local utilitydata = load(filename)
369    if utilitydata then
370        local jobpacker = utilitydata.job.packed
371        if jobpacker then
372            unpacknumberdata(jobpacker.index)
373            for i=1,#savelist do
374                local list   = savelist[i]
375                local target = list[1]
376                local result = accesstable(target,utilitydata)
377                if result then
378                    local done = packers.unpack(result,jobpacker,true)
379                    if done then
380                        migratetable(target,mark(result))
381                    else
382                        report_passes("pack version mismatch")
383                    end
384                end
385            end
386        end
387    end
388    -- why not inside previous if
389    for i=1,#savelist do
390        local list        = savelist[i]
391        local target      = list[1]
392        local initializer = list[3]
393        if type(initializer) == "function" then
394            initializer(utilitydata and accesstable(target,utilitydata) or nil)
395        end
396    end
397    statistics.stoptiming(loadedfiles)
398end
399
400function job.loadother(filename)
401    local jobname = environment.jobname
402    if filename == jobname then
403        return
404    end
405    filename = file.addsuffix(filename,"tuc")
406    local unpacked = othercache[filename]
407    if not unpacked then
408        -- so we can register the same name twice (in loading order) ... needs checking if we want this
409        statistics.starttiming(loadedfiles)
410        local utilitydata = load(filename)
411        if utilitydata then
412            report_passes("integrating list %a into %a",filename,jobname)
413            local jobpacker = utilitydata.job.packed
414            unpacknumberdata(jobpacker.index)
415            unpacked = { }
416            for l=1,#savelist do
417                local list   = savelist[l]
418                local target = list[1]
419                local result = accesstable(target,utilitydata)
420                local done   = packers.unpack(result,jobpacker,true)
421                if done then
422                    migratetable(target,result,unpacked)
423                end
424            end
425            unpacked.job.packed = nil -- nicer in inspecting
426            othercache[filename] = unpacked
427            --
428            utilitydata.components, utilitydata.namestack = collectstructure(utilitydata.job.structure.collected)
429            --
430            structures.lists     .integrate(utilitydata)
431            structures.registers .integrate(utilitydata)
432            structures.references.integrate(utilitydata)
433        end
434        statistics.stoptiming(loadedfiles)
435    end
436    return unpacked
437end
438
439statistics.register("startup time", function()
440    return statistics.elapsedseconds(statistics,"including runtime option file processing")
441end)
442
443statistics.register("jobdata time",function()
444    local elapsedsave = statistics.elapsedtime(savedfiles)
445    local elapsedload = statistics.elapsedtime(loadedfiles)
446    if enabled then
447        if next(othercache) then
448            return format("%s seconds saving, %s seconds loading, other files: %s",elapsedsave,elapsedload,concat(sortedkeys(othercache), ", "))
449        else
450            return format("%s seconds saving, %s seconds loading",elapsedsave,elapsedload)
451        end
452    else
453        if next(othercache) then
454            return format("nothing saved, %s seconds loading, other files: %s",elapsedload,concat(sortedkeys(othercache), ", "))
455        else
456            return format("nothing saved, %s seconds loading",elapsedload)
457        end
458    end
459end)
460
461statistics.register("callbacks", function()
462    local backend  = backends.getcallbackstate()
463    local frontend = status.getcallbackstate()
464    local pages    = structures.pages.nofpages or 0
465    local total    = frontend.count + backend.count
466    local average  = pages > 0 and math.round(total/pages) or 0
467    local result   = format (
468        "file: %s, saved: %s, direct: %s, function: %s, value: %s, message: %s, bytecode: %s, late %s, total: %s (%s per page)",
469        frontend.file,  frontend.saved,   frontend.direct,   frontend["function"],
470        frontend.value, frontend.message, frontend.bytecode, backend.count,
471        total, average
472    )
473    statistics.callbacks = function()
474        return result
475    end
476    return result
477end)
478
479function statistics.formatruntime(runtime)
480    if not environment.initex then -- else error when testing as not counters yet
481     -- stoptiming(statistics) -- to be sure
482        local shipped = texgetcount("nofshipouts")
483        local pages = texgetcount("realpageno")
484        if pages > shipped then
485            pages = shipped
486        end
487        runtime = tonumber(runtime)
488        if shipped > 0 or pages > 0 then
489            local persecond = (runtime > 0) and (shipped/runtime) or pages
490            if pages == 0 then
491                pages = shipped
492            end
493            return format("%0.3f seconds, %i processed pages, %i shipped pages, %.3f pages/second",runtime,pages,shipped,persecond)
494        else
495            return format("%0.3f seconds",runtime)
496        end
497    end
498end
499
500implement {
501    name      = "savecurrentvalue",
502    public    = true,
503    actions   = job.variables.save,
504    arguments = { "csname", "argument" },
505}
506
507implement {
508    name      = "setjobcomment",
509    actions   = job.comment,
510    arguments = { { "*" } }
511}
512
513implement {
514    name      = "initializejob",
515    actions   = job.initialize
516}
517
518implement {
519    name      = "disablejobsave",
520    actions   = job.disablesave
521}
522