if not modules then modules = { } end modules ['core-uti'] = {
    version   = 1.001,
    comment   = "companion to core-uti.mkiv",
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

-- A utility file has always been part of ConTeXt and with the move to LuaTeX we
-- also moved a lot of multi-pass info to a Lua table. Instead of loading a TeX
-- based utility file under different setups, we now load a table once. This saves
-- much runtime but at the cost of more memory usage.
--
-- In the meantime the overhead is a bit more due to the amount of data being saved
-- and more agressive compacting.

local math = math
local next, type, tostring, tonumber, setmetatable, load = next, type, tostring, tonumber, setmetatable, load
local format, match = string.format, string.match
local concat, sortedkeys = table.concat, table.sortedkeys

local definetable    = utilities.tables.definetable
local accesstable    = utilities.tables.accesstable
local migratetable   = utilities.tables.migratetable
local serialize      = table.serialize
local packers        = utilities.packers
local allocate       = utilities.storage.allocate
local mark           = utilities.storage.mark

local getrandom      = utilities.randomizer.get
local setrandomseedi = utilities.randomizer.setseedi
local getrandomseed  = utilities.randomizer.getseed

local implement      = interfaces.implement

local texgetcount    = tex.getcount

local report_passes  = logs.reporter("job","passes")

job                  = job or { }
local job            = job

job.version          = 1.33
job.packversion      = 1.02

-- Variables are saved using in the previously defined table and passed onto TeX
-- using the following method. Of course one can also directly access the variable
-- using a Lua call.

local savelist, comment = { }, { }

function job.comment(key,value)
    if type(key) == "table" then
        for k, v in next, key do
            comment[k] = v
        end
    else
        comment[key] = value
    end
end

job.comment("version",job.version)

local enabled     = true
local initialized = false

directives.register("job.save",function(v) enabled = v end)

function job.disablesave()
    enabled = false -- for instance called when an error
end

function job.initialize(loadname,savename)
    if not initialized then
        if not loadname or loadname == "" then
            loadname = tex.jobname .. ".tuc"
        end
        if not savename or savename == "" then
            savename = tex.jobname .. ".tua"
        end
        job.load(loadname) -- has to come after structure is defined !
        luatex.registerstopactions(function()
            if enabled then
                job.save(savename)
            end
        end)
        initialized = true
    end
end

function job.register(collected, tobesaved, initializer, finalizer, serializer)
    savelist[#savelist+1] = { collected, tobesaved, initializer, finalizer, serializer }
end

-- as an example we implement variables

local tobesaved, collected, checksums = allocate(), allocate(), allocate()

local jobvariables = {
    collected = collected,
    tobesaved = tobesaved,
    checksums = checksums,
}

-- if not checksums.old then checksums.old = md5.HEX("old") end -- used in experiment
-- if not checksums.new then checksums.new = md5.HEX("new") end -- used in experiment

job.variables = jobvariables

local function initializer()
    checksums = jobvariables.checksums
end

job.register('job.variables.checksums', 'job.variables.checksums', initializer)

local rmethod, rvalue
local collectedmacros, tobesavedmacros

local ctx_setxvalue = context.setxvalue

local function initializer()
    tobesaved = jobvariables.tobesaved
    collected = jobvariables.collected
    --
    rvalue = collected.randomseed
    if not rvalue then
        rvalue = getrandom("initialize")
        setrandomseedi(rvalue)
        rmethod = "initialized"
    else
        setrandomseedi(rvalue)
        rmethod = "resumed"
    end
    tobesaved.randomseed = rvalue
    --
    collectedmacros = collected.macros
    tobesavedmacros = tobesaved.macros
    if not collectedmacros then
        collectedmacros  = { }
        collected.macros = collectedmacros
    end
    if not tobesavedmacros then
        tobesavedmacros  = { }
        tobesaved.macros = tobesavedmacros
    end
    -- will become collected.macros
    for cs, value in next, collectedmacros do
        if type(value) == "string" then -- safeguard
            ctx_setxvalue(cs,value)
        end
    end
end

job.register('job.variables.collected', tobesaved, initializer)

function jobvariables.save(cs,value)
    tobesavedmacros[cs] = value
end

function jobvariables.restore(cs)
    return collectedmacros[cs] or tobesavedmacros[cs]
end

function job.getrandomseed()
    return tobesaved.randomseed or getrandomseed()
end

-- checksums

function jobvariables.getchecksum(tag)
    return checksums[tag] -- no default
end

function jobvariables.makechecksum(data)
    return data and md5.HEX(data) -- no default
end

function jobvariables.setchecksum(tag,checksum)
    checksums[tag] = checksum
end

--

local packlist = {
    "numbers",
    "ownnumbers",
    "metadata",
    "sectiondata",
    "prefixdata",
    "numberdata",
    "pagedata",
    "directives",
    "specification",
    "processors", -- might become key under directives or metadata
--  "references", -- we need to rename of them as only one packs (not structures.lists.references)
}

local skiplist = {
    "datasets",
    "userdata",
    "positions",
}

-- I'm not that impressed by the savings. It's some 5 percent on the luametatex
-- manual and probably some more on the m4 files (if so I might enable it).

local deltapacking = false
-- local deltapacking = true

local function packnumberdata(tobesaved)
    if deltapacking and tobesaved[1] then
        local last
        local current
        for i=1,#tobesaved do
            current = tobesaved[i]
            if last then
                if last.numbers and last.block then
                    for k, v in next, last do
                        if k ~= "numbers" and v ~= current[k] then
                            goto DIFFERENT
                        end
                    end
                    for k, v in next, current do
                        if k ~= "numbers" and v ~= last[k] then
                            goto DIFFERENT
                        end
                    end
                    tobesaved[i] = {
                        numbers = current.numbers,
                    }
                    goto CONTINUE
                else
                    current = nil
                end
            end
          ::DIFFERENT::
            last = current
          ::CONTINUE::
        end
    end
end

local function unpacknumberdata(collected)
    if deltapacking and collected[1] then
        local key  = "numbers"
        local last = collected[1]
        local meta = false
        for i=2,#collected do
            local c = collected[i]
            if c.block then
                last = c
                meta = false
            elseif c.numbers then
                if not meta then
                    meta = { __index = last }
                end
                setmetatable(c, meta)
            end
        end
    end
end

-- -- --

-- not ok as we can have arbitrary keys in userdata and dataset so some day we
-- might need a bit more granularity, like skippers

local jobpacker = packers.new(packlist,job.packversion,skiplist) -- jump number when changs in hash

job.pack = true
-- job.pack = false

directives.register("job.pack",function(v) job.pack = v end)

local savedfiles  = { }
local loadedfiles = { } -- for now only timers
local othercache  = { }

function job.save(filename) -- we could return a table but it can get pretty large
    statistics.starttiming(savedfiles)
    local f = io.open(filename,'w')
    if f then
        f:write("local utilitydata = { }\n\n")
        f:write(serialize(comment,"utilitydata.comment",true),"\n\n")
        for l=1,#savelist do
            local list       = savelist[l]
            local target     = format("utilitydata.%s",list[1])
            local data       = list[2]
            local finalizer  = list[4]
            local serializer = list[5]
            if type(data) == "string" then
                data = utilities.tables.accesstable(data)
            end
            if type(finalizer) == "function" then
                finalizer()
            end
            if job.pack then
                packers.pack(data,jobpacker,true)
            end
            local definer, name = definetable(target,true,true) -- no first and no last
            if serializer then
                f:write(definer,"\n\n",serializer(data,name,true),"\n\n")
            else
                f:write(definer,"\n\n",serialize(data,name,true),"\n\n")
            end
        end
        if job.pack then
            packers.strip(jobpacker)
            packnumberdata(jobpacker.index)
            f:write(serialize(jobpacker,"utilitydata.job.packed",true),"\n\n")
        end
        f:write("return utilitydata")
        f:close()
    end
    statistics.stoptiming(savedfiles)
end

local function load(filename)
    if lfs.isfile(filename) then
        local function dofile(filename)
            local result = loadstring(io.loaddata(filename))
            if result then
                return result()
            else
                return nil
            end
        end
        local okay, data = pcall(dofile,filename)
        if okay and type(data) == "table" then
            local jobversion  = job.version
            local datacomment = data.comment
            local dataversion = datacomment and datacomment.version or "?"
            if dataversion ~= jobversion then
                report_passes("version mismatch: %s <> %s",dataversion,jobversion)
            else
                return data
            end
        else
            os.remove(filename)
            report_passes("removing stale job data file %a, restart job, message: %s",filename,tostring(data))
            os.exit(true) -- trigger second run
        end
    end
end

function job.load(filename)
    statistics.starttiming(loadedfiles)
    local utilitydata = load(filename)
    if utilitydata then
        local jobpacker = utilitydata.job.packed
        unpacknumberdata(jobpacker.index)
        for i=1,#savelist do
            local list   = savelist[i]
            local target = list[1]
            local result = accesstable(target,utilitydata)
            if result then
                local done = packers.unpack(result,jobpacker,true)
                if done then
                    migratetable(target,mark(result))
                else
                    report_passes("pack version mismatch")
                end
            end
        end
    end
    for i=1,#savelist do
        local list        = savelist[i]
        local initializer = list[3]
        if type(initializer) == "function" then
            initializer(utilitydata and accesstable(list[1],utilitydata) or nil)
        end
    end
    statistics.stoptiming(loadedfiles)
end

function job.loadother(filename)
    local jobname = environment.jobname
    if filename == jobname then
        return
    end
    filename = file.addsuffix(filename,"tuc")
    local unpacked = othercache[filename]
    if not unpacked then
        -- so we can register the same name twice (in loading order) ... needs checking if we want this
        statistics.starttiming(loadedfiles)
        local utilitydata = load(filename)
        if utilitydata then
            report_passes("integrating list %a into %a",filename,jobname)
            local jobpacker = utilitydata.job.packed
            unpacknumberdata(jobpacker.index)
            unpacked = { }
            for l=1,#savelist do
                local list   = savelist[l]
                local target = list[1]
                local result = accesstable(target,utilitydata)
                local done   = packers.unpack(result,jobpacker,true)
                if done then
                    migratetable(target,result,unpacked)
                end
            end
            unpacked.job.packed = nil -- nicer in inspecting
            othercache[filename] = unpacked
            --
            utilitydata.components, utilitydata.namestack = collectstructure(utilitydata.job.structure.collected)
            --
            structures.lists     .integrate(utilitydata)
            structures.registers .integrate(utilitydata)
            structures.references.integrate(utilitydata)
        end
        statistics.stoptiming(loadedfiles)
    end
    return unpacked
end

statistics.register("startup time", function()
    return statistics.elapsedseconds(statistics,"including runtime option file processing")
end)

statistics.register("jobdata time",function()
    local elapsedsave = statistics.elapsedtime(savedfiles)
    local elapsedload = statistics.elapsedtime(loadedfiles)
    if enabled then
        if next(othercache) then
            return format("%s seconds saving, %s seconds loading, other files: %s",elapsedsave,elapsedload,concat(sortedkeys(othercache), ", "))
        else
            return format("%s seconds saving, %s seconds loading",elapsedsave,elapsedload)
        end
    else
        if next(othercache) then
            return format("nothing saved, %s seconds loading, other files: %s",elapsedload,concat(sortedkeys(othercache), ", "))
        else
            return format("nothing saved, %s seconds loading",elapsedload)
        end
    end
end)

statistics.register("callbacks", function()
    local backend  = backends.getcallbackstate()
    local frontend = status.getcallbackstate()
    local pages    = structures.pages.nofpages or 0
    local total    = frontend.count + backend.count
    local average  = pages > 0 and math.round(total/pages) or 0
    local result   = format (
        "file: %s, saved: %s, direct: %s, function: %s, value: %s, message: %s, bytecode: %s, late %s, total: %s (%s per page)",
        frontend.file,  frontend.saved,   frontend.direct,   frontend["function"],
        frontend.value, frontend.message, frontend.bytecode, backend.count,
        total, average
    )
    statistics.callbacks = function()
        return result
    end
    return result
end)

statistics.register("randomizer", function()
    if rmethod and rvalue then
        return format("%s with value %s",rmethod,rvalue)
    end
end)

function statistics.formatruntime(runtime)
    if not environment.initex then -- else error when testing as not counters yet
     -- stoptiming(statistics) -- to be sure
        local shipped = texgetcount("nofshipouts")
        local pages = texgetcount("realpageno")
        if pages > shipped then
            pages = shipped
        end
        runtime = tonumber(runtime)
        if shipped > 0 or pages > 0 then
            local persecond = (runtime > 0) and (shipped/runtime) or pages
            if pages == 0 then
                pages = shipped
            end
            return format("%0.3f seconds, %i processed pages, %i shipped pages, %.3f pages/second",runtime,pages,shipped,persecond)
        else
            return format("%0.3f seconds",runtime)
        end
    end
end

implement {
    name      = "savecurrentvalue",
    public    = true,
    actions   = job.variables.save,
    arguments = { "csname", "argument" },
}

implement {
    name      = "setjobcomment",
    actions   = job.comment,
    arguments = { { "*" } }
}

implement {
    name      = "initializejob",
    actions   = job.initialize
}

implement {
    name      = "disablejobsave",
    actions   = job.disablesave
}