m-scite.mkiv /size: 14 Kb    last modification: 2021-10-28 13:51
1%D \module
2%D   [       file=m-scite,
3%D        version=2014.04.28,
4%D          title=\CONTEXT\ Extra Modules,
5%D       subtitle=\SCITE\ lexers,
6%D         author=Hans Hagen,
7%D           date=\currentdate,
8%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
9%C
10%C This module is part of the \CONTEXT\ macro||package and is
11%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
12%C details.
13
14% We can simplify the scite lexers, as long as we're able to return the
15% lexed result table and provide alexer module with the functions that
16% the lexer expects (so I need to decipher the cxx file).
17%
18% lexer._TOKENSTYLES : table
19% lexer._CHILDREN    : flag
20% lexer._EXTRASTYLES : table
21% lexer._GRAMMAR     : flag
22%
23% lexers.load        : function
24% lexers.lex         : function
25%
26% And some properties that map styles onto scintilla styling. I get the
27% impression that we end up with something simpler, a hybrid between the
28% scite lexing and the current context way, so we get an intermediate
29% step, with some penalty for context, but at least I don't have to
30% maintain two sets (three sets as we also have a line based series).
31
32% TODO: as these files are in tds we can locate them and set the lexer root
33% to that one. Currently we're on: we're on context/documents.
34
35% TODO: tab
36
37% This is an experiment: eventually we need to hook it into the verbatim code
38% and deal with widow lines and so.
39
40\startluacode
41
42-- todo: merge with collapse
43-- todo: prehash whitespaces
44
45-- todo: hook into the pretty print code
46-- todo: a simple catcode regime with only \ { }
47
48local gsub, sub, find, lower = string.gsub, string.sub, string.find, string.lower
49local concat = table.concat
50local formatters = string.formatters
51local lpegmatch = lpeg.match
52local setmetatableindex = table.setmetatableindex
53
54local scite   = require("util-sci")
55buffers.scite = scite
56
57-- context output:
58
59local f_def_color = formatters["\\definecolor[slxc%s][h=%02X%02X%02X]%%"]
60local f_fore_none = formatters["\\unexpanded\\def\\slx%s#1{{\\slxc%s#1}\\slxbreak}%%"]
61local f_fore_bold = formatters["\\unexpanded\\def\\slx%s#1{{\\slxc%s\\bf#1}\\slxbreak}%%"]
62local f_none_bold = formatters["\\unexpanded\\def\\slx%s#1{{\\bf#1}\\slxbreak}%%"]
63local f_none_none = formatters["\\unexpanded\\def\\slx%s#1{{#1}\\slxbreak}%%"]
64local f_texstyled = formatters["\\slx%s{%s}\\slxbreak"]
65local f_hanging   = formatters["\\slxb{%r}%s\\slxe"] -- we need to round: lua 5.3
66
67local v_none      = interfaces.variables.none
68
69local f_mapping   = [[
70\let\string\slxL\string\letterleftbrace
71\let\string\slxR\string\letterrightbrace
72\let\string\slxM\string\letterdollar
73\let\string\slxV\string\letterbar
74\let\string\slxU\string\letterhat
75\let\string\slxD\string\letterunderscore
76\let\string\slxH\string\letterhash
77\let\string\slxB\string\letterbackslash
78\let\string\slxP\string\letterpercent
79\let\string\slxT\string\lettertilde
80\let\string\slxS\string\fixedspace
81%]]
82
83local replacer = lpeg.replacer {
84    ["{"]  = "\\slxL ",
85    ["}"]  = "\\slxR ",
86    ["$"]  = "\\slxM ",
87    ["^"]  = "\\slxU ",
88    ["_"]  = "\\slxD ",
89    ["|"]  = "\\slxV ",
90    ["#"]  = "\\slxH ",
91    ["\\"] = "\\slxB ",
92    ["%"]  = "\\slxP ",
93    ["~"]  = "\\slxT ",
94    [" "]  = "\\slxS ", -- can be made more efficient: \\slxF{n}
95}
96
97local colors = nil
98
99local function exportcolors()
100    if not colors then
101        scite.loadscitelexer()
102        local function black(f)
103            return (f[1] == f[2]) and (f[2] == f[3]) and (f[3] == 0)
104        end
105     -- local result, r = { f_mapping }, 1
106        local result, r = { }, 0
107        for k, v in table.sortedhash(lexers.styles) do
108            local fore = v.fore
109            if fore and not black(fore) then
110                r = r + 1
111                result[r] = f_def_color(k,fore[1],fore[2] or fore[1],fore[3] or fore[1])
112            end
113        end
114        r = r + 1
115        result[r] = "%"
116        for k, v in table.sortedhash(lexers.styles) do
117            local bold = v.bold
118            local fore = v.fore
119            r = r + 1
120            if fore and not black(fore) then
121                if bold then
122                    result[r] = f_fore_bold(k,k)
123                else
124                    result[r] = f_fore_none(k,k)
125                end
126            else
127                if bold then
128                    result[r] = f_none_bold(k)
129                else
130                    result[r] = f_none_none(k)
131                end
132            end
133        end
134        colors = concat(result,"\n")
135    end
136    return colors
137end
138
139local function exportwhites()
140    return setmetatableindex(function(t,k)
141        local v = find(k,"white") and true or false
142        t[k] = v
143        return v
144    end)
145end
146
147local function exportstyled(lexer,text)
148    local result = lexers.lex(lexer,text,0)
149    local start = 1
150    local whites = exportwhites()
151    local buffer = { }
152    for i=1,#result,2 do
153        local style = result[i]
154        local position = result[i+1]
155        local txt = sub(text,start,position-1)
156        txt = lpegmatch(replacer,txt)
157        if txt == "" then
158            -- skip
159        elseif whites[style] then
160            buffer[#buffer+1] = txt
161        else
162            buffer[#buffer+1] = f_texstyled(style,txt)
163        end
164        start = position
165    end
166    buffer = concat(buffer)
167    return buffer
168end
169
170function scite.installcommands()
171    context(exportcolors())
172end
173
174local function slxF(b,e)
175    local d = (e - b)/6
176    if d > 0 then
177        return "\\slxF{" .. d .. "}"
178    else
179        return " "
180    end
181end
182
183local p1 = lpeg.tsplitat(lpeg.patterns.newline)
184local p2 = lpeg.P("\\slxS ")
185local p3 = p2^1
186local p4 = lpeg.Cs( ( (lpeg.Cp() * p2 * p2^1 * lpeg.Cp()) / slxF + lpeg.P(1) )^0 )
187
188local function indent(str)
189    local l = lpegmatch(p1,str)
190    for i=1,#l do
191        local s = l[i]
192        if #s > 0 then
193            local n = lpegmatch(p3,s)
194            if n then
195                n = (n-1)/6 -- length of "\\slxS "
196            else
197                n = 0
198            end
199            if n > 0 then
200                s = sub(s,n*6+1)
201            end
202            s = lpegmatch(p4,s)   -- can be combined
203            l[i] = f_hanging(n,s) -- "\\slxb{%s}%s\\slxe "
204        end
205    end
206    return concat(l,"\n")
207end
208
209local assignbuffer = buffers.assign
210local getcontent   = buffers.getcontent
211local loaddata     = io.loaddata
212local loadedlexers = scite.loadedlexers
213
214local function lexdata(data,lexname)
215    if not data then
216        data = ""
217    elseif data ~= "" then
218        if data and not find(data,"[\r\n]$") then
219            -- fix for lexbyline
220            data = data .. "\r"
221        end
222        if lexname == v_none then
223--             data = string.formatters["%!tex!"](data)
224            data = exportstyled(loadedlexers["dummy"] or loadedlexers.tex,data)
225        else
226            data = exportstyled(loadedlexers[lexname] or loadedlexers.tex,data)
227        end
228        data = indent(data)
229    end
230 -- io.savedata("temp.log",data)
231    assignbuffer("lex",data)
232end
233
234scite.lexdata = lexdata
235
236function scite.lexbuffer(name,lexname)
237    lexdata(getcontent(name) or "",lexname or "tex")
238end
239
240function scite.lexfile(filename,lexname)
241    lexdata(loaddata(filename) or "",lexname or file.suffix(filename))
242end
243
244interfaces.implement {
245    name      = "scitelexfile",
246    arguments = "string",
247    actions   = scite.lexfile,
248}
249
250interfaces.implement {
251    name      = "scitelexbuffer",
252    arguments = { "string", "string" },
253    actions   = scite.lexbuffer,
254}
255
256interfaces.implement {
257    name      = "sciteinstallcommands",
258    actions   = scite.installcommands,
259}
260
261local startInlineScite  = context.startInlineScite
262local stopInlineScite   = context.stopInlineScite
263local startDisplayScite = context.startDisplayScite
264local stopDisplayScite  = context.stopDisplayScite
265
266local lexdata           = buffers.scite.lexdata
267
268local handler = visualizers.newhandler {
269    startinline  = function(settings) startInlineScite(settings.name or "") end,
270    stopinline   = function(settings) stopInlineScite() end,
271    startdisplay = function(settings) startDisplayScite(settings.name or "") end,
272    stopdisplay  = function(settings) stopDisplayScite() end,
273    noescape     = true,
274}
275
276local knownlexers = scite.knownlexers
277
278local parser = function(content,specification)
279    local method = lower(specification.method)
280    lexdata(content,knownlexers[method] or method)
281end
282
283local visualizer = {
284    handler = handler,
285    parser  = parser,
286}
287
288visualizers.register("cld",   visualizer)
289visualizers.register("tex",   visualizer)
290visualizers.register("lua",   visualizer)
291visualizers.register("mps",   visualizer)
292visualizers.register("mp",    visualizer)
293visualizers.register("pdf",   visualizer)
294visualizers.register("xml",   visualizer)
295visualizers.register("bibtex",visualizer)
296visualizers.register("btx",   visualizer)
297visualizers.register("web",   visualizer)
298visualizers.register("cpp",   visualizer)
299visualizers.register("txt",   visualizer)
300visualizers.register("bnf",   visualizer)
301visualizers.register("sql",   visualizer)
302visualizers.register("json",  visualizer)
303
304visualizers.register("sas",   visualizer)
305
306function scite.register(name)
307    visualizers.register(name,visualizer)
308end
309
310moduledata.scite = scite
311
312\stopluacode
313
314%definetyping[TEX] [option=cld]
315\setuptyping [TEX] [option=cld]
316%definetyping[LUA] [option=lua]
317\setuptyping [LUA] [option=lua]
318\definetyping[BTX] [option=bibtex]
319\definetyping[MPS] [option=mps]
320%definetyping[MP]  [option=mp]
321\setuptyping [MP]  [option=mp]
322\definetyping[PDF] [option=pdf]
323\definetyping[CPP] [option=cpp]    % Which is kind of like the web one.
324\definetyping[WEB] [option=web]
325\definetyping[TXT] [option=txt]
326\definetyping[BNF] [option=bnf]    % I might use this in the metafun manual.
327\definetyping[SQL] [option=sql]    % To be tested in an upcoming manual.
328\definetyping[JSON][option=json]   % To be tested in an upcoming manual.
329\definetyping[NONE][option=none]
330
331\definetyping[SAS] [option=sas]
332
333% This is a preliminary interface.
334
335\unprotect
336
337\newdimen\scitespaceskip
338
339\unexpanded\def\buff_scite_slxb#1%
340  {% we can have a side float
341   \advance\hangindent\numexpr#1+2\relax\scitespaceskip
342   \begstrut\hskip#1\scitespaceskip\relax
343   \ifcase\hangafter
344     \hangafter\plusone\relax
345   \fi}
346
347\unexpanded\def\buff_scite_slxe
348  {\endstrut\par}
349
350\unexpanded\def\buff_scite_slxs  {\hskip\scitespaceskip\relax}
351\unexpanded\def\buff_scite_slxf#1{\hskip#1\scitespaceskip\relax}
352
353\unexpanded\def\installscitecommandsinline
354  {\scitespaceskip\interwordspace % \fontcharwd\font`0\relax % brrrrr
355   \let\slxb\gobbleoneargument
356   \let\slxe\space
357   \let\slxbreak\relax
358   \let\installscitecommandsinline\relax}
359
360\unexpanded\def\installscitecommandsdisplay
361  {\scitespaceskip\interwordspace % \fontcharwd\font`0\relax % brrrrr
362   \let\slxb\buff_scite_slxb
363   \let\slxe\buff_scite_slxe
364   \let\installscitecommandsdisplay\relax}
365
366\clf_sciteinstallcommands
367
368\let\slxb    \gobbleoneargument
369\let\slxe    \space
370\let\slxbreak\relax
371
372\let\slxL    \letterleftbrace
373\let\slxR    \letterrightbrace
374\let\slxM    \letterdollar
375\let\slxV    \letterbar
376\let\slxU    \letterhat
377\let\slxD    \letterunderscore
378\let\slxH    \letterhash
379\let\slxB    \letterbackslash
380\let\slxP    \letterpercent
381\let\slxT    \lettertilde
382\let\slxS    \fixedspace
383
384%let\slxS    \buff_scite_slxs
385\let\slxF    \buff_scite_slxf
386
387\def\module_scite_inherit_typing
388  {\buff_verbatim_initialize_typing_one
389   \buff_verbatim_set_line_margin}
390
391\unexpanded\def\startscite
392  {\begingroup
393   \dosingleempty\buff_scite_start}
394
395\def\buff_scite_start[#1]%
396  {\edef\currentscitelexer{#1}%
397   \dostartbuffer[@scite@][startscite][stopscite]}
398
399\unexpanded\def\stopscite
400  {\scitebuffer[\ifx\currentscitelexer\empty tex\else\currentscitelexer\fi][@scite@]%
401   \endgroup}
402
403\definelines
404  [scitelines]
405
406\setuplines
407  [scitelines]
408  [\c!before=,
409   \c!after=]
410
411\unexpanded\def\scitefile
412  {\dosingleargument\module_scite_file}
413
414\unexpanded\def\module_scite_file[#1]%
415  {\begingroup
416   \setcatcodetable\ctxcatcodes % needed in xml
417   \clf_scitelexfile{#1}%
418   \tt
419   \installscitecommandsdisplay
420   \module_scite_inherit_typing
421   \dontcomplain
422   \raggedright
423   \startcontextcode
424   \startscitelines
425   \getbuffer[lex]
426   \stopscitelines
427   \stopcontextcode
428   \endgroup}
429
430\unexpanded\def\scitebuffer
431  {\dodoubleempty\module_scite_buffer}
432
433\unexpanded\def\module_scite_buffer[#1][#2]%
434  {\begingroup
435   \setcatcodetable\ctxcatcodes % needed in xml
436   \ifsecondargument
437     \clf_scitelexbuffer{#2}{#1}%
438   \else
439     \clf_scitelexbuffer{#1}{tex}%
440   \fi
441   \tt
442   \installscitecommandsdisplay
443   \module_scite_inherit_typing
444   \dontcomplain
445   \raggedright
446   \startscitelines
447   \getbuffer[lex]
448   \stopscitelines
449   \endgroup}
450
451\unexpanded\def\sciteinlinebuffer
452  {\dodoubleempty\module_scite_buffer_inline}
453
454\unexpanded\def\module_scite_buffer_inline[#1][#2]%
455  {\dontleavehmode
456   \begingroup
457   \setcatcodetable\ctxcatcodes % needed in xml
458   \ifsecondargument
459     \clf_scitelexbuffer{#2}{#1}%
460   \else
461     \clf_scitelexbuffer{#1}{tex}%
462   \fi
463   \tt
464   \installscitecommandsinline
465   \module_scite_inherit_typing
466   \dontcomplain
467   \getbuffer[lex]%
468   \endgroup}
469
470\unexpanded\def\startInlineScite#1%
471  {\dontleavehmode
472   \begingroup
473   \setcatcodetable\ctxcatcodes % needed in xml
474   \tt
475   \installscitecommandsinline
476   \dontcomplain
477   \ignorespaces
478   \clf_getbuffercontent{lex}% much faster than getbuffer
479   \removeunwantedspaces}
480
481\unexpanded\def\stopInlineScite
482  {\endgroup}
483
484\unexpanded\def\startDisplayScite#1%
485  {\begingroup
486   \setcatcodetable\ctxcatcodes % needed in xml
487   \tt
488   \installscitecommandsdisplay
489   \dontcomplain
490   \buff_verbatim_initialize_typing_one
491   \buff_verbatim_set_line_margin
492   \raggedright
493   \startscitelines
494   \clf_getbuffer{lex}}
495
496\unexpanded\def\stopDisplayScite
497  {\stopscitelines
498   \endgroup}
499
500\unexpanded\def\slxbreak
501  {\allowbreak}
502
503\protect
504
505\continueifinputfile{m-scite.mkiv}
506
507\setupbodyfont[dejavu,8pt]
508
509\setuplayout
510  [width=middle,
511   height=middle,
512   header=1cm,
513   footer=1cm,
514   topspace=1cm,
515   bottomspace=1cm,
516   backspace=1cm]
517
518\startbuffer[demo]
519\startsubsubject[title={oeps}]
520
521\startMPcode
522    draw fullcircle
523        scaled 2cm
524        withpen pencircle scaled 1mm
525        withcolor .5green;
526    draw textext (
527        lua (
528            "local function f(s) return string.upper(s) end mp.quoted(f('foo'))"
529        )
530    ) withcolor .5red ;
531\stopMPcode
532
533\startluacode
534    context("foo")
535\stopluacode
536
537\stopsubsubject
538\stopbuffer
539
540\starttext
541
542% \scitefile[../lexers/scite-context-lexer.lua]  \page
543% \scitefile[t:/manuals/about/about-metafun.tex] \page
544% \scitefile[t:/sources/strc-sec.mkiv]           \page
545% \scitefile[e:/tmp/mp.w]                        \page
546% \scitefile[t:/manuals/hybrid/tugboat.bib]        \page
547\scitefile[e:/tmp/test.bib]        \page
548
549% \getbuffer[demo] \scitebuffer[demo]
550
551\startbuffer[foo]
552This is text. % line 1
553This is text. % line 2
554\stopbuffer
555
556\scitebuffer[none][foo]
557
558\sciteinlinebuffer[none][foo]
559
560\stoptext
561