if not modules then modules = { } end modules ['data-tmp'] = {
version = 1.100,
comment = "companion to luat-lib.mkiv",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files"
}
--[[ldx--
<p>This module deals with caching data. It sets up the paths and implements
loaders and savers for tables. Best is to set the following variable. When not
set, the usual paths will be checked. Personally I prefer the (users) temporary
path.</p>
</code>
TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;.
</code>
<p>Currently we do no locking when we write files. This is no real problem
because most caching involves fonts and the chance of them being written at the
same time is small. We also need to extend luatools with a recache feature.</p>
--ldx]]--
local next, type = next, type
local pcall, loadfile, collectgarbage = pcall, loadfile, collectgarbage
local format, lower, gsub = string.format, string.lower, string.gsub
local concat, serialize, fastserialize, serializetofile = table.concat, table.serialize, table.fastserialize, table.tofile
local mkdirs, expanddirname, isdir, isfile = dir.mkdirs, dir.expandname, lfs.isdir, lfs.isfile
local is_writable, is_readable = file.is_writable, file.is_readable
local collapsepath, joinfile, addsuffix, dirname = file.collapsepath, file.join, file.addsuffix, file.dirname
local savedata = file.savedata
local formatters = string.formatters
local osexit, osdate, osuuid = os.exit, os.date, os.uuid
local removefile = os.remove
local md5hex = md5.hex
local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end)
local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end)
local report_caches = logs.reporter("resolvers","caches")
local report_resolvers = logs.reporter("resolvers","caching")
local resolvers = resolvers
local cleanpath = resolvers.cleanpath
local resolvepath = resolvers.resolve
local luautilities = utilities.lua
-- intermezzo
do
local directive_cleanup = false directives.register("system.compile.cleanup", function(v) directive_cleanup = v end)
local directive_strip = false directives.register("system.compile.strip", function(v) directive_strip = v end)
local compilelua = luautilities.compile
function luautilities.compile(luafile,lucfile,cleanup,strip)
if cleanup == nil then cleanup = directive_cleanup end
if strip == nil then strip = directive_strip end
return compilelua(luafile,lucfile,cleanup,strip)
end
end
-- end of intermezzo
caches = caches or { }
local caches = caches
local writable = nil
local readables = { }
local usedreadables = { }
local compilelua = luautilities.compile
local luasuffixes = luautilities.suffixes
caches.base = caches.base or "luatex-cache" -- can be local
caches.more = caches.more or "context" -- can be local
caches.defaults = { "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" }
local direct_cache = false -- true is faster but may need huge amounts of memory
local fast_cache = false
local cache_tree = false
directives.register("system.caches.direct",function(v) direct_cache = true end)
directives.register("system.caches.fast", function(v) fast_cache = true end)
-- we could use a metatable for writable and readable but not yet
local function configfiles()
return concat(resolvers.configurationfiles(),";")
end
local function hashed(tree)
tree = gsub(tree,"[\\/]+$","")
tree = lower(tree)
local hash = md5hex(tree)
if trace_cache or trace_locating then
report_caches("hashing tree %a, hash %a",tree,hash)
end
return hash
end
local function treehash()
local tree = configfiles()
if not tree or tree == "" then
return false
else
return hashed(tree)
end
end
caches.hashed = hashed
caches.treehash = treehash
caches.configfiles = configfiles
local function identify()
-- Combining the loops makes it messy. First we check the format cache path
-- and when the last component is not present we try to create it.
local texmfcaches = resolvers.cleanpathlist("TEXMFCACHE") -- forward ref
if texmfcaches then
for k=1,#texmfcaches do
local cachepath = texmfcaches[k]
if cachepath ~= "" then
cachepath = resolvepath(cachepath)
cachepath = cleanpath(cachepath)
cachepath = collapsepath(cachepath)
local valid = isdir(cachepath)
if valid then
if is_readable(cachepath) then
readables[#readables+1] = cachepath
if not writable and is_writable(cachepath) then
writable = cachepath
end
end
elseif not writable then
local cacheparent = dirname(cachepath)
if is_writable(cacheparent) then -- we go on anyway (needed for mojca's kind of paths)
mkdirs(cachepath)
if isdir(cachepath) and is_writable(cachepath) then
report_caches("path %a created",cachepath)
writable = cachepath
readables[#readables+1] = cachepath
end
end
end
end
end
end
-- As a last resort we check some temporary paths but this time we don't
-- create them.
local texmfcaches = caches.defaults
if texmfcaches then
for k=1,#texmfcaches do
local cachepath = texmfcaches[k]
cachepath = resolvers.expansion(cachepath) -- was getenv
if cachepath ~= "" then
cachepath = resolvepath(cachepath)
cachepath = cleanpath(cachepath)
local valid = isdir(cachepath)
if valid and is_readable(cachepath) then
if not writable and is_writable(cachepath) then
readables[#readables+1] = cachepath
writable = cachepath
break
end
end
end
end
end
-- Some extra checking. If we have no writable or readable path then we simply
-- quit.
if not writable then
report_caches("fatal error: there is no valid writable cache path defined")
osexit()
elseif #readables == 0 then
report_caches("fatal error: there is no valid readable cache path defined")
osexit()
end
-- why here
writable = expanddirname(cleanpath(writable)) -- just in case
-- moved here ( we have only one writable tree)
local base = caches.base
local more = caches.more
local tree = cache_tree or treehash() -- we have only one writable tree
if tree then
cache_tree = tree
writable = mkdirs(writable,base,more,tree)
for i=1,#readables do
readables[i] = joinfile(readables[i],base,more,tree)
end
else
writable = mkdirs(writable,base,more)
for i=1,#readables do
readables[i] = joinfile(readables[i],base,more)
end
end
-- end
if trace_cache then
for i=1,#readables do
report_caches("using readable path %a (order %s)",readables[i],i)
end
report_caches("using writable path %a",writable)
end
identify = function()
return writable, readables
end
return writable, readables
end
function caches.usedpaths(separator)
local writable, readables = identify()
if #readables > 1 then
local result = { }
local done = { }
for i=1,#readables do
local readable = readables[i]
if readable == writable then
done[readable] = true
result[#result+1] = formatters["readable+writable: %a"](readable)
elseif usedreadables[i] then
done[readable] = true
result[#result+1] = formatters["readable: %a"](readable)
end
end
if not done[writable] then
result[#result+1] = formatters["writable: %a"](writable)
end
return concat(result,separator or " | ")
else
return writable or "?"
end
end
local r_cache = { }
local w_cache = { }
local function getreadablepaths(...)
local tags = { ... }
local hash = concat(tags,"/")
local done = r_cache[hash]
if not done then
local writable, readables = identify() -- exit if not found
if #tags > 0 then
done = { }
for i=1,#readables do
done[i] = joinfile(readables[i],...)
end
else
done = readables
end
r_cache[hash] = done
end
return done
end
local function getwritablepath(...)
local tags = { ... }
local hash = concat(tags,"/")
local done = w_cache[hash]
if not done then
local writable, readables = identify() -- exit if not found
if #tags > 0 then
done = mkdirs(writable,...)
else
done = writable
end
w_cache[hash] = done
end
return done
end
local function setfirstwritablefile(filename,...)
local wr = getwritablepath(...)
local fullname = joinfile(wr,filename)
return fullname, wr
end
local function setluanames(path,name)
return
format("%s/%s.%s",path,name,luasuffixes.tma),
format("%s/%s.%s",path,name,luasuffixes.tmc)
end
local function getfirstreadablefile(filename,...)
-- check if we have already written once
local fullname, path = setfirstwritablefile(filename,...)
if is_readable(fullname) then
return fullname, path -- , true
end
-- otherwise search for pregenerated
local rd = getreadablepaths(...)
for i=1,#rd do
local path = rd[i]
local fullname = joinfile(path,filename)
if is_readable(fullname) then
usedreadables[i] = true
return fullname, path -- , false
end
end
-- else assume new written
return fullname, path -- , true
end
caches.getreadablepaths = getreadablepaths
caches.getwritablepath = getwritablepath
caches.setfirstwritablefile = setfirstwritablefile
caches.getfirstreadablefile = getfirstreadablefile
caches.setluanames = setluanames
-- -- not used:
--
-- function caches.define(category,subcategory)
-- return function()
-- return getwritablepath(category,subcategory)
-- end
-- end
-- This works best if the first writable is the first readable too. In practice
-- we can have these situations for file databases:
--
-- tma in readable
-- tma + tmb/c in readable
--
-- runtime files like fonts are written to the writable cache anyway
function caches.loaddata(readables,name,writable)
if type(readables) == "string" then
readables = { readables }
end
for i=1,#readables do
local path = readables[i]
local loader = false
local state = false
local tmaname, tmcname = setluanames(path,name)
if isfile(tmcname) then
state, loader = pcall(loadfile,tmcname)
end
if not loader and isfile(tmaname) then
-- can be different paths when we read a file database from disk
local tmacrap, tmcname = setluanames(writable,name)
if isfile(tmcname) then
state, loader = pcall(loadfile,tmcname)
end
compilelua(tmaname,tmcname)
if isfile(tmcname) then
state, loader = pcall(loadfile,tmcname)
end
if not loader then
state, loader = pcall(loadfile,tmaname)
end
end
if loader then
loader = loader()
collectgarbage("step")
return loader
end
end
return false
end
function caches.is_writable(filepath,filename)
local tmaname, tmcname = setluanames(filepath,filename)
return is_writable(tmaname)
end
local saveoptions = { compact = true }
function caches.savedata(filepath,filename,data,fast)
local tmaname, tmcname = setluanames(filepath,filename)
data.cache_uuid = osuuid()
if fast or fast_cache then
savedata(tmaname,fastserialize(data,true))
elseif direct_cache then
savedata(tmaname,serialize(data,true,saveoptions))
else
serializetofile(tmaname,data,true,saveoptions)
end
compilelua(tmaname,tmcname)
end
-- moved from data-res:
local content_state = { }
function caches.contentstate()
return content_state or { }
end
function caches.loadcontent(cachename,dataname,filename)
if not filename then
local name = hashed(cachename)
local full, path = getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees")
filename = joinfile(path,name)
end
local state, blob = pcall(loadfile,addsuffix(filename,luasuffixes.luc))
if not blob then
state, blob = pcall(loadfile,addsuffix(filename,luasuffixes.lua))
end
if blob then
local data = blob()
if data and data.content then
if data.type == dataname then
if data.version == resolvers.cacheversion then
content_state[#content_state+1] = data.uuid
if trace_locating then
report_resolvers("loading %a for %a from %a",dataname,cachename,filename)
end
return data.content
else
report_resolvers("skipping %a for %a from %a (version mismatch)",dataname,cachename,filename)
end
else
report_resolvers("skipping %a for %a from %a (datatype mismatch)",dataname,cachename,filename)
end
elseif trace_locating then
report_resolvers("skipping %a for %a from %a (no content)",dataname,cachename,filename)
end
elseif trace_locating then
report_resolvers("skipping %a for %a from %a (invalid file)",dataname,cachename,filename)
end
end
function caches.collapsecontent(content)
for k, v in next, content do
if type(v) == "table" and #v == 1 then
content[k] = v[1]
end
end
end
function caches.savecontent(cachename,dataname,content,filename)
if not filename then
local name = hashed(cachename)
local full, path = setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees")
filename = joinfile(path,name) -- is full
end
local luaname = addsuffix(filename,luasuffixes.lua)
local lucname = addsuffix(filename,luasuffixes.luc)
if trace_locating then
report_resolvers("preparing %a for %a",dataname,cachename)
end
local data = {
type = dataname,
root = cachename,
version = resolvers.cacheversion,
date = osdate("%Y-%m-%d"),
time = osdate("%H:%M:%S"),
content = content,
uuid = osuuid(),
}
local ok = savedata(luaname,serialize(data,true))
if ok then
if trace_locating then
report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname)
end
if compilelua(luaname,lucname) then
if trace_locating then
report_resolvers("%a compiled to %a",dataname,lucname)
end
return true
else
if trace_locating then
report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname)
end
removefile(lucname)
end
elseif trace_locating then
report_resolvers("unable to save %a in %a (access error)",dataname,luaname)
end
end
