1if not modules then modules = { } end modules ['mtx-testsuite'] = {
2 version = 1.002,
3 comment = "companion to mtxrun.lua",
4 author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5 copyright = "PRAGMA ADE / ConTeXt Development Team",
6 license = "see context related readme files"
7}
8
9
10
11
12
13
14
15local helpinfo = [[
16<?xml version="1.0"?>
17<application>
18 <metadata>
19 <entry name="name">mtx-testsuite</entry>
20 <entry name="detail">Experiments with the testsuite</entry>
21 <entry name="version">1.00</entry>
22 </metadata>
23 <flags>
24 <category name="basic">
25 <subcategory>
26 <flag name="process"><short>process files (<ref name="pattern"/>)"/></short></flag>
27 </subcategory>
28 <subcategory>
29 <flag name="compare"><short>compare files (<ref name="pattern"/> <ref name="newname"/> <ref name="oldname"/> <ref name="collect)"/></short></flag>
30 </subcategory>
31 </category>
32 </flags>
33 <examples>
34 <category>
35 <title>Example</title>
36 <subcategory>
37 <example><command>mtxrun --script testsuite --compare --objects --oldname=cld-compare-old --newname=cld-compare-new --pattern=**/*.cld</command></example>
38 <example><command>mtxrun --script testsuite --compare --objects --oldname=mkiv-compare-old --newname=mkiv-compare-new --pattern=**/*.mkiv</command></example>
39 <example><command>mtxrun --script testsuite --compare --objects --oldname=mkvi-compare-old --newname=mkvi-compare-new --pattern=**/*.mkvi</command></example>
40 <example><command>mtxrun --script testsuite --compare --objects --oldname=tex-compare-old --newname=tex-compare-new --pattern=**/*.tex</command></example>
41 </subcategory>
42 </category>
43 </examples>
44</application>
45]]
46
47local application = logs.application {
48 name = "mtx-testsuite",
49 banner = "Experiments with the testsuite 1.00",
50 helpinfo = helpinfo,
51}
52
53local gmatch, match, gsub, find, lower, format = string.gmatch, string.match, string.gsub, string.find, string.lower, string.format
54local concat = table.concat
55local split = string.split
56local are_equal = table.are_equal
57local tonumber = tonumber
58local formatters = string.formatters
59
60local report = application.report
61
62scripts = scripts or { }
63scripts.testsuite = scripts.testsuite or { }
64
65local f_runner = formatters['%s --batch --nocompression --nodates --trailerid=1 --randomseed=1234 %s "%s"']
66
67
68function scripts.testsuite.process()
69 local pattern = environment.argument("pattern")
70 if pattern then
71 local cleanup = environment.argument("cleanup")
72 local jit = environment.argument("jit")
73 local engine = environment.argument("luatex") and "--luatex" or ""
74 local results = { }
75 local start = statistics.starttiming(scripts.testsuite.process)
76 local files = dir.glob(pattern)
77 local start = tonumber(environment.argument("start"))
78 local suffix = start and ("-" ..start) or ""
79 local start = start or 1
80 local stop = tonumber(environment.argument("stop")) or #files
81 local step = tonumber(environment.argument("step")) or 1
82 local luaname = "testsuite-process" .. suffix .. ".lua"
83 for i=start,stop,step do
84 local filename = files[i]
85 if filename then
86 local dirname = file.dirname(filename)
87 local basename = file.basename(filename)
88 local texname = basename
89 local pdfname = file.replacesuffix(basename,"pdf")
90 local tucname = file.replacesuffix(basename,"tuc")
91 local workdir = lfs.currentdir()
92 lfs.chdir(dirname)
93 os.remove(pdfname)
94 if cleanup then
95 os.remove(tucname)
96 end
97 if lfs.isfile(texname) then
98 local command = f_runner(jit and "contextjit" or "context",engine,texname)
99 local result = tonumber(os.execute(command)) or 0
100 if result > 0 then
101 results[filename] = result
102 end
103 end
104 lfs.chdir(workdir)
105 else
106 break
107 end
108 end
109 statistics.stoptiming(scripts.testsuite.process)
110 results.runtime = statistics.elapsedtime(scripts.testsuite.process)
111 io.savedata(luaname,table.serialize(results,true))
112 report()
113 report("files: %i, runtime: %s, overview: %s",#files,results.runtime,luaname)
114 report()
115 end
116end
117
118local popen = io.popen
119local close = io.close
120local read = io.read
121local gobble = io.gobble
122local clock = os.clock
123
124function scripts.testsuite.parallel()
125 local pattern = environment.argument("pattern")
126 if pattern then
127 local cleanup = environment.argument("cleanup")
128 local engine = environment.argument("luatex") and "--luatex" or ""
129 local results = { }
130 local start = statistics.starttiming(scripts.testsuite.process)
131 local files = dir.glob(pattern)
132 local luaname = "testsuite-process.lua"
133 local process = { }
134 local total = #files
135 local runners = tonumber(environment.argument("parallel")) or 8
136 local count = 0
137 while true do
138 local done = false
139 for i=1,runners do
140 local pi = process[i]
141 if pi then
142
143 local s = gobble(pi[1])
144 if s then
145
146 done = true
147 goto done
148 else
149 local r, detail, n = close(pi[1])
150 local bad = not r or n > 0
151 if bad then
152 results[pi[2]] = { detail, n }
153 end
154 report("%02i : %04i : %s : %s : %0.3f ",i,pi[3],bad and "error" or "done ",pi[2],clock()-pi[4])
155 process[i] = false
156 end
157 end
158 count = count + 1
159 if count > total then
160
161 else
162 local filename = files[count]
163 local dirname = file.dirname(filename)
164 local basename = file.basename(filename)
165 local texname = basename
166 local pdfname = file.replacesuffix(basename,"pdf")
167 local tucname = file.replacesuffix(basename,"tuc")
168 local workdir = lfs.currentdir()
169 lfs.chdir(dirname)
170 os.remove(pdfname)
171 if cleanup then
172 os.remove(tucname)
173 end
174 if lfs.isfile(texname) then
175 local command = f_runner("context",engine,texname)
176 local result = popen(command)
177 if result then
178
179 process[i] = { result, filename, count, clock() }
180 else
181 results[filename] = "error"
182 end
183 report("%02i : %04i : %s : %s",i,count,result and "start" or "error",filename)
184 end
185 lfs.chdir(workdir)
186 done = true
187 end
188 ::done::
189 end
190 if not done then
191 break
192 end
193 end
194 statistics.stoptiming(scripts.testsuite.process)
195 results.runtime = statistics.elapsedtime(scripts.testsuite.process)
196 io.savedata(luaname,table.serialize(results,true))
197 report()
198 report("files: %i, runtime: %s, overview: %s",total,results.runtime,luaname)
199 report()
200 end
201end
202
203function scripts.testsuite.compare()
204 local pattern = environment.argument("pattern")
205 local oldname = environment.argument("oldname")
206 local newname = environment.argument("newname")
207 local collect = environment.argument("collect")
208 local bitmaps = environment.argument("bitmaps")
209 local objects = environment.argument("objects")
210 local cleanup = environment.argument("cleanup")
211 local jit = environment.argument("jit")
212 local engine = environment.argument("luatex") and "--luatex" or ""
213 if pattern and newname then
214 oldname = oldname and file.addsuffix(oldname,"lua")
215 newname = file.addsuffix(newname,"lua")
216 local files = dir.glob(pattern)
217 local info = table.load("testsuite-info.lua")
218 local skip = info and info.exceptions or { }
219 local oldhashes = oldname and lfs.isfile(oldname) and dofile(oldname) or { }
220 local newhashes = {
221 version = 0.01,
222 files = { },
223 data = os.date(),
224 }
225 local old = oldhashes and oldhashes.files or {}
226 local new = newhashes and newhashes.files or {}
227 local err = { }
228
229 local function compare(filename,olddata,name)
230 local newhash = md5.HEX(io.loaddata(name))
231 local oldhash = olddata and olddata.hash
232 if not oldhash then
233 new[filename] = { status = "new", hash = newhash }
234 elseif oldhash == newhash then
235 new[filename] = { status = "unchanged", hash = oldhash }
236 else
237 new[filename] = { status = "changed", hash = newhash }
238 end
239 end
240
241 for i=1,#files do
242 local filename = files[i]
243 local olddata = old[filename]
244 if collect then
245 new[filename] = olddata or { status = "collected" }
246 elseif olddata and olddata.status == "skip" then
247 new[filename] = olddata
248 else
249 local dirname = file.dirname(filename)
250 local basename = file.basename(filename)
251 local texname = basename
252 local pdfname = file.replacesuffix(basename,"pdf")
253 local tucname = file.replacesuffix(basename,"tuc")
254 local pngname = "temp.png"
255 local oldname = "old-" .. pdfname
256 local workdir = lfs.currentdir()
257 lfs.chdir(dirname)
258 os.remove(oldname)
259 os.rename(pdfname,oldname)
260 if cleanup then
261 os.remove(tucname)
262 end
263 os.remove(pngname)
264 if lfs.isfile(texname) then
265 local command = f_runner(jit and "contextjit" or "context",engine,texname)
266 local result = os.execute(command)
267 if result > 0 then
268 new[filename] = { status = "error", comment = "error code: " .. result }
269 err[filename] = result
270 elseif lfs.isfile(pdfname) then
271 local fullname = gsub(filename,"^%./","")
272 if skip[fullname] then
273 new[filename] = { status = "okay", comment = (bitmaps or objects) and "not compared" or nil }
274 elseif bitmaps then
275 local command = string.format('mutool draw -o %s -r 600 %s',pngname,pdfname)
276 local result = os.execute(command)
277 if lfs.isfile(pngname) then
278 compare(filename,olddata,pngname)
279 else
280 new[filename] = { status = "error", comment = "no png file" }
281 end
282 elseif objects then
283 compare(filename,olddata,pdfname)
284 else
285 new[filename] = { status = "okay" }
286 end
287 else
288 new[filename] = { status = "error", comment = "no pdf file" }
289 end
290 else
291 new[filename] = { status = "error", comment = "no tex file" }
292 end
293 os.remove(pngname)
294 lfs.chdir(workdir)
295 end
296 end
297 io.savedata(newname,table.serialize(newhashes,true))
298 if next(err) then
299 for filename, data in table.sortedhash(err) do
300 report("fatal error in file %a",filename)
301 end
302 else
303 report("no fatal errors")
304 end
305 else
306 report("provide --pattern --oldname --newname [--cleanup] [--bitmaps | --objects]")
307 end
308end
309
310if environment.argument("compare") then
311 scripts.testsuite.compare()
312elseif environment.argument("process") then
313 scripts.testsuite.process()
314elseif environment.argument("parallel") then
315 scripts.testsuite.parallel()
316elseif environment.argument("exporthelp") then
317 application.export(environment.argument("exporthelp"),environment.files[1])
318else
319 application.help()
320end
321
322 |