SourceBrowser

m-scite.mkiv /size: 16 Kb last modification: 2025-02-21 11:03
1%D \module
2%D   [       file=m-scite,
3%D        version=2014.04.28,
4%D          title=\CONTEXT\ Extra Modules,
5%D       subtitle=\SCITE\ lexers,
6%D         author=Hans Hagen,
7%D           date=\currentdate,
8%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
9%C
10%C This module is part of the \CONTEXT\ macro||package and is
11%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
12%C details.
13
14% We can simplify the scite lexers, as long as we're able to return the
15% lexed result table and provide alexer module with the functions that
16% the lexer expects (so I need to decipher the cxx file).
17%
18% lexer._TOKENSTYLES : table
19% lexer._CHILDREN    : flag
20% lexer._EXTRASTYLES : table
21% lexer._GRAMMAR     : flag
22%
23% lexers.load        : function
24% lexers.lex         : function
25%
26% And some properties that map styles onto scintilla styling. I get the
27% impression that we end up with something simpler, a hybrid between the
28% scite lexing and the current context way, so we get an intermediate
29% step, with some penalty for context, but at least I don't have to
30% maintain two sets (three sets as we also have a line based series).
31
32% TODO: as these files are in tds we can locate them and set the lexer root
33% to that one. Currently we're on: we're on context/documents.
34
35% TODO: tab
36
37% This is an experiment: eventually we need to hook it into the verbatim code
38% and deal with widow lines and so.
39
40\startluacode
41
42-- todo: merge with collapse
43-- todo: prehash whitespaces
44
45-- todo: hook into the pretty print code
46-- todo: a simple catcode regime with only \ { }
47
48local gsub, sub, find, lower = string.gsub, string.sub, string.find, string.lower
49local concat = table.concat
50local formatters = string.formatters
51local lpegmatch = lpeg.match
52local setmetatableindex = table.setmetatableindex
53
54local scite   = require("util-sci")
55buffers.scite = scite
56
57-- context output:
58
59local f_def_color = formatters["\\definecolor[slxc%s][h=%02X%02X%02X]%%"]
60local f_fore_none = formatters["\\unexpanded\\def\\slx%s#1{{\\slxc%s#1}\\slxbreak}%%"]
61local f_fore_bold = formatters["\\unexpanded\\def\\slx%s#1{{\\slxc%s\\slxbold#1}\\slxbreak}%%"]
62local f_none_bold = formatters["\\unexpanded\\def\\slx%s#1{{\\slxbold#1}\\slxbreak}%%"]
63local f_none_none = formatters["\\unexpanded\\def\\slx%s#1{{#1}\\slxbreak}%%"]
64local f_texstyled = formatters["\\slx%s{%s}\\slxbreak"]
65local f_hanging   = formatters["\\slxb{%r}%s\\slxe"] -- we need to round: lua 5.3
66
67local v_none      = interfaces.variables.none
68
69local f_mapping   = [[
70\let\string\slxL\string\letterleftbrace
71\let\string\slxR\string\letterrightbrace
72\let\string\slxM\string\letterdollar
73\let\string\slxV\string\letterbar
74\let\string\slxU\string\letterhat
75\let\string\slxD\string\letterunderscore
76\let\string\slxH\string\letterhash
77\let\string\slxB\string\letterbackslash
78\let\string\slxP\string\letterpercent
79\let\string\slxT\string\lettertilde
80\let\string\slxS\string\fixedspace
81%]]
82
83local replacer = lpeg.replacer {
84    ["{"]  = "\\slxL ",
85    ["}"]  = "\\slxR ",
86    ["$"]  = "\\slxM ",
87    ["^"]  = "\\slxU ",
88    ["_"]  = "\\slxD ",
89    ["|"]  = "\\slxV ",
90    ["#"]  = "\\slxH ",
91    ["\\"] = "\\slxB ",
92    ["%"]  = "\\slxP ",
93    ["~"]  = "\\slxT ",
94    [" "]  = "\\slxS ", -- can be made more efficient: \\slxF{n}
95}
96
97local colors = nil
98local escape = false
99
100local function exportcolors()
101    if not colors then
102        scite.loadscitelexer()
103        local function black(f)
104            return (f[1] == f[2]) and (f[2] == f[3]) and (f[3] == 0)
105        end
106     -- local result, r = { f_mapping }, 1
107        local result, r = { }, 0
108        for k, v in table.sortedhash(lexers.styles) do
109            local fore = v.fore
110            if fore and not black(fore) then
111                r = r + 1
112                result[r] = f_def_color(k,fore[1],fore[2] or fore[1],fore[3] or fore[1])
113            end
114        end
115        r = r + 1
116        result[r] = "%"
117        for k, v in table.sortedhash(lexers.styles) do
118            local bold = v.bold
119            local fore = v.fore
120            r = r + 1
121            if fore and not black(fore) then
122                if bold then
123                    result[r] = f_fore_bold(k,k)
124                else
125                    result[r] = f_fore_none(k,k)
126                end
127            else
128                if bold then
129                    result[r] = f_none_bold(k)
130                else
131                    result[r] = f_none_none(k)
132                end
133            end
134        end
135        colors = concat(result,"\n")
136    end
137    return colors
138end
139
140local function exportwhites()
141    return setmetatableindex(function(t,k)
142        local v = find(k,"white") and true or false
143        t[k] = v
144        return v
145    end)
146end
147
148local commenthandlers = {
149    tex = function(txt)
150        if find(txt,"^%%tex ") then
151            return "{" .. gsub(txt,"^%%tex ","") .. "}"
152        end
153    end,
154    mps = function(txt)
155        if find(txt,"^%%tex ") then
156            return "{" .. gsub(txt,"^%%tex ","") .. "}"
157        end
158    end,
159    lua = function(txt)
160        if find(txt,"^%-%-tex ") then
161            return "{" .. gsub(txt,"^%-%-tex ","") .. "}"
162        end
163    end,
164}
165
166local function exportstyled(lexer,text,commenthandler)
167    local result = lexers.lex(lexer,text,0)
168    local start = 1
169    local whites = exportwhites()
170    local buffer, b = { }, 0
171    for i=1,#result,2 do
172        local style = result[i]
173        local position = result[i+1]
174        local txt = sub(text,start,position-1)
175        txt = commenthandler and style == "comment" and commenthandler(txt) or lpegmatch(replacer,txt)
176        if txt == "" then
177            -- skip
178        elseif whites[style] then
179            b = b + 1 ; buffer[b] = txt
180        else
181            b = b + 1 ; buffer[b] = f_texstyled(style,txt)
182        end
183        start = position
184    end
185    buffer = concat(buffer)
186    return buffer
187end
188
189function scite.installcommands()
190    context(exportcolors())
191end
192
193local function slxF(b,e)
194    local d = (e - b)/6
195    if d > 0 then
196        return "\\slxF{" .. d .. "}"
197    else
198        return " "
199    end
200end
201
202local p1 = lpeg.tsplitat(lpeg.patterns.newline)
203local p2 = lpeg.P("\\slxS ")
204local p3 = p2^1
205local p4 = lpeg.Cs( ( (lpeg.Cp() * p2 * p2^1 * lpeg.Cp()) / slxF + lpeg.P(1) )^0 )
206
207local function indent(str)
208    local l = lpegmatch(p1,str)
209    for i=1,#l do
210        local s = l[i]
211        if #s > 0 then
212            local n = lpegmatch(p3,s)
213            if n then
214                n = (n-1)/6 -- length of "\\slxS "
215            else
216                n = 0
217            end
218            if n > 0 then
219                s = sub(s,n*6+1)
220            end
221            s = lpegmatch(p4,s)   -- can be combined
222            l[i] = f_hanging(n,s) -- "\\slxb{%s}%s\\slxe "
223        end
224    end
225    return concat(l,"\n")
226end
227
228local assignbuffer = buffers.assign
229local getcontent   = buffers.getcontent
230local loaddata     = io.loaddata
231local loadedlexers = scite.loadedlexers
232
233local function lexdata(data,lexname,handlecomment)
234    if not data then
235        data = ""
236    elseif data ~= "" then
237        if data and not find(data,"[\r\n]$") then
238            -- fix for lexbyline
239            data = data .. "\r"
240        end
241        local commenthandler = (escape or handlecomment) and commenthandlers[lexname]
242        if lexname == v_none then
243         -- data = string.formatters["%!tex!"](data)
244            data = exportstyled(loadedlexers["dummy"] or loadedlexers.tex,data,commenthandler)
245        else
246            data = exportstyled(loadedlexers[lexname] or loadedlexers.tex,data,commenthandler)
247        end
248        data = indent(data)
249    end
250 -- io.savedata("temp.log",data)
251    assignbuffer("lex",data)
252end
253
254scite.lexdata = lexdata
255
256function scite.lexbuffer(name,lexname,options)
257    lexdata(getcontent(name) or "",lexname or "tex",options == "escape" and handlecomment)
258end
259
260function scite.lexfile(filename,lexname,options)
261    lexdata(loaddata(filename) or "",lexname or file.suffix(filename),options == "escape" and handlecomment)
262end
263
264interfaces.implement {
265    name      = "scitelexfile",
266    arguments = "string",
267    actions   = scite.lexfile,
268}
269
270interfaces.implement {
271    name      = "scitelexbuffer",
272    arguments = "2 strings",
273    actions   = scite.lexbuffer,
274}
275
276interfaces.implement {
277    name      = "sciteinstallcommands",
278    actions   = scite.installcommands,
279}
280
281local startInlineScite  = context.startInlineScite
282local stopInlineScite   = context.stopInlineScite
283local startDisplayScite = context.startDisplayScite
284local stopDisplayScite  = context.stopDisplayScite
285
286local lexdata           = buffers.scite.lexdata
287
288local handler = visualizers.newhandler {
289    startinline  = function(settings)
290        startInlineScite(settings.name or "")
291    end,
292    stopinline   = function(settings)
293        stopInlineScite()
294    end,
295    startdisplay = function(settings)
296        escape = settings.escape == interfaces.variables.yes and true or false
297        startDisplayScite(settings.name or "")
298    end,
299    stopdisplay  = function(settings)
300        stopDisplayScite()
301        escape = false
302    end,
303    noescape     = true,
304}
305
306local knownlexers = scite.knownlexers
307
308local parser = function(content,specification)
309    local method = lower(specification.method)
310    lexdata(content,knownlexers[method] or method)
311end
312
313local visualizer = {
314    handler = handler,
315    parser  = parser,
316}
317
318visualizers.register("cld",   visualizer)
319visualizers.register("tex",   visualizer)
320visualizers.register("lua",   visualizer)
321visualizers.register("mps",   visualizer)
322visualizers.register("mp",    visualizer)
323visualizers.register("pdf",   visualizer)
324visualizers.register("xml",   visualizer)
325visualizers.register("bibtex",visualizer)
326visualizers.register("btx",   visualizer)
327visualizers.register("web",   visualizer)
328visualizers.register("cpp",   visualizer)
329visualizers.register("txt",   visualizer)
330visualizers.register("bnf",   visualizer)
331visualizers.register("sql",   visualizer)
332visualizers.register("json",  visualizer)
333
334visualizers.register("sas",   visualizer)
335
336function scite.register(name)
337    visualizers.register(name,visualizer)
338end
339
340moduledata.scite = scite
341
342\stopluacode
343
344%definetyping[TEX] [option=cld]
345\setuptyping [TEX] [option=cld]
346%definetyping[LUA] [option=lua]
347\setuptyping [LUA] [option=lua]
348\definetyping[BTX] [option=bibtex]
349\definetyping[MPS] [option=mps]
350%definetyping[MP]  [option=mp]
351\setuptyping [MP]  [option=mp]
352\definetyping[PDF] [option=pdf]
353\definetyping[CPP] [option=cpp]    % Which is kind of like the web one.
354\definetyping[WEB] [option=web]
355\definetyping[TXT] [option=txt]
356\definetyping[BNF] [option=bnf]    % I might use this in the metafun manual.
357\definetyping[SQL] [option=sql]    % To be tested in an upcoming manual.
358\definetyping[JSON][option=json]   % To be tested in an upcoming manual.
359\definetyping[NONE][option=none]
360
361\definetyping[SAS] [option=sas]
362
363% This is a preliminary interface.
364
365\unprotect
366
367\newdimen\scitespaceskip
368
369\unexpanded\def\buff_scite_slxb#1%
370  {% we can have a side float
371   \advance\hangindent\numexpr#1+2\relax\scitespaceskip
372   \begstrut\hskip#1\scitespaceskip\relax
373   \ifcase\hangafter
374     \hangafter\plusone\relax
375   \fi}
376
377\unexpanded\def\buff_scite_slxe
378  {\endstrut\par}
379
380\unexpanded\def\buff_scite_slxs  {\hskip\scitespaceskip\relax}
381\unexpanded\def\buff_scite_slxf#1{\hskip#1\scitespaceskip\relax}
382
383\unexpanded\def\installscitecommandsinline
384  {\scitespaceskip\interwordspace % \fontcharwd\font`0\relax % brrrrr
385   \let\slxb\gobbleoneargument
386   \let\slxe\space
387   \let\slxbreak\relax
388   \let\installscitecommandsinline\relax}
389
390\unexpanded\def\installscitecommandsdisplay
391  {\scitespaceskip\interwordspace % \fontcharwd\font`0\relax % brrrrr
392   \let\slxb\buff_scite_slxb
393   \let\slxe\buff_scite_slxe
394   \let\installscitecommandsdisplay\relax}
395
396\clf_sciteinstallcommands
397
398\let\slxb    \gobbleoneargument
399\let\slxe    \space
400\let\slxbreak\relax
401
402\def\slxbold {\bf} % maybe contraststyle in verbatim
403
404\let\slxL    \letterleftbrace
405\let\slxR    \letterrightbrace
406\let\slxM    \letterdollar
407\let\slxV    \letterbar
408\let\slxU    \letterhat
409\let\slxD    \letterunderscore
410\let\slxH    \letterhash
411\let\slxB    \letterbackslash
412\let\slxP    \letterpercent
413\let\slxT    \lettertilde
414\let\slxS    \fixedspace
415
416%let\slxS    \buff_scite_slxs
417\let\slxF    \buff_scite_slxf
418
419\def\module_scite_inherit_typing
420  {\buff_verbatim_initialize_typing_one
421   \buff_verbatim_set_line_margin}
422
423\unexpanded\def\startscite
424  {\begingroup
425   \dosingleempty\buff_scite_start}
426
427\def\buff_scite_start[#1]%
428  {\edef\currentscitelexer{#1}%
429   \dostartbuffer[@scite@][startscite][stopscite]}
430
431\unexpanded\def\stopscite
432  {\scitebuffer[\ifx\currentscitelexer\empty tex\else\currentscitelexer\fi][@scite@]%
433   \endgroup}
434
435\definelines
436  [scitelines]
437
438\setuplines
439  [scitelines]
440  [\c!before=,
441   \c!after=]
442
443\unexpanded\def\scitefile
444  {\dosingleargument\module_scite_file}
445
446\unexpanded\def\module_scite_file[#1]%
447  {\begingroup
448   \setcatcodetable\ctxcatcodes % needed in xml
449   \clf_scitelexfile{#1}%
450   \tt
451   \installscitecommandsdisplay
452   \module_scite_inherit_typing
453   \dontcomplain
454   \raggedright
455   \startcontextcode
456   \startscitelines
457   \getbuffer[lex]
458   \stopscitelines
459   \stopcontextcode
460   \endgroup}
461
462\unexpanded\def\scitebuffer
463  {\dodoubleempty\module_scite_buffer}
464
465\unexpanded\def\module_scite_buffer[#1][#2]%
466  {\begingroup
467   \setcatcodetable\ctxcatcodes % needed in xml
468   \ifsecondargument
469     \clf_scitelexbuffer{#2}{#1}%
470   \else
471     \clf_scitelexbuffer{#1}{tex}%
472   \fi
473   \tt
474   \installscitecommandsdisplay
475   \module_scite_inherit_typing
476   \dontcomplain
477   \raggedright
478   \startscitelines
479   \getbuffer[lex]
480   \stopscitelines
481   \endgroup}
482
483\unexpanded\def\sciteinlinebuffer
484  {\dodoubleempty\module_scite_buffer_inline}
485
486\unexpanded\def\module_scite_buffer_inline[#1][#2]%
487  {\dontleavehmode
488   \begingroup
489   \lettypingparameter\c!margin\zeropoint
490   \setcatcodetable\ctxcatcodes % needed in xml
491   \ifsecondargument
492     \clf_scitelexbuffer{#2}{#1}%
493   \else
494     \clf_scitelexbuffer{#1}{tex}%
495   \fi
496   \tt
497   \installscitecommandsinline
498   \module_scite_inherit_typing
499   \dontcomplain
500   \getbuffer[lex]%
501   \removeunwantedspaces
502   \endgroup}
503
504\unexpanded\def\startInlineScite#1%
505  {\dontleavehmode
506   \begingroup
507   \setcatcodetable\ctxcatcodes % needed in xml
508   \tt
509   \installscitecommandsinline
510   \dontcomplain
511   \ignorespaces
512   \clf_getbuffercontent{lex}% much faster than getbuffer
513   \removeunwantedspaces}
514
515\unexpanded\def\stopInlineScite
516  {\endgroup}
517
518\unexpanded\def\startDisplayScite#1%
519  {\begingroup
520   \setcatcodetable\ctxcatcodes % needed in xml
521   \tt
522   \installscitecommandsdisplay
523   \dontcomplain
524   \buff_verbatim_initialize_typing_one
525   \buff_verbatim_set_line_margin
526   \raggedright
527   \startscitelines
528   \clf_getbuffer{lex}}
529
530\unexpanded\def\stopDisplayScite
531  {\stopscitelines
532   \endgroup}
533
534\unexpanded\def\slxbreak
535  {\allowbreak}
536
537\protect
538
539\continueifinputfile{m-scite.mkiv}
540
541\setupbodyfont[dejavu,8pt]
542
543\setuplayout
544  [width=middle,
545   height=middle,
546   header=1cm,
547   footer=1cm,
548   topspace=1cm,
549   bottomspace=1cm,
550   backspace=1cm]
551
552\startbuffer[demo]
553\startsubsubject[title={oeps}]
554
555\startMPcode
556    draw fullcircle
557        scaled 2cm
558        withpen pencircle scaled 1mm
559        withcolor .5green;
560    draw textext (
561        lua (
562            "local function f(s) return string.upper(s) end mp.quoted(f('foo'))"
563        )
564    ) withcolor .5red ;
565\stopMPcode
566
567\startluacode
568    context("foo")
569\stopluacode
570
571\stopsubsubject
572\stopbuffer
573
574\starttext
575
576% \scitefile[../lexers/scite-context-lexer.lua]  \page
577% \scitefile[t:/manuals/about/about-metafun.tex] \page
578% \scitefile[t:/sources/strc-sec.mkiv]           \page
579% \scitefile[e:/tmp/mp.w]                        \page
580% \scitefile[t:/manuals/hybrid/tugboat.bib]        \page
581% \scitefile[e:/tmp/test.bib]        \page
582
583% \getbuffer[demo] \scitebuffer[demo]
584
585% \startbuffer[foo]
586% This is text. % line 1
587% This is text. % line 2
588% \stopbuffer
589
590% \scitebuffer[none][foo]
591
592% \sciteinlinebuffer[none][foo]
593
594\starttyping[option=TEX,escape=yes]
595\framed
596  [width=4cm] % options
597  {some text}
598
599\framed
600  [width=4cm] % \letterpercent\ \rm\bf these are options
601  {some text}
602
603\framed
604  [width=4cm] %tex \letterpercent\ \rm\bf these are options
605  {some text}
606\stoptyping
607
608\starttyping[option=LUA,escape=yes]
609function test(x)
610    return x * x  --tex -- \rm\bf these are options
611end
612\stoptyping
613
614\stoptext
615
Source Browser ?
