m-scite.mkiv /size: 14 Kb    last modification: 2020-07-01 14:35
1
%D \module
2
%D [ file=m-scite,
3
%D version=2014.04.28,
4
%D title=\CONTEXT\ Extra Modules,
5
%D subtitle=\SCITE\ lexers,
6
%D author=Hans Hagen,
7
%D date=\currentdate,
8
%D copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
9
%C
10
%C This module is part of the \CONTEXT\ macro||package and is
11
%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
12
%C details.
13 14
% We can simplify the scite lexers, as long as we're able to return the
15
% lexed result table and provide alexer module with the functions that
16
% the lexer expects (so I need to decipher the cxx file).
17
%
18
% lexer._TOKENSTYLES : table
19
% lexer._CHILDREN : flag
20
% lexer._EXTRASTYLES : table
21
% lexer._GRAMMAR : flag
22
%
23
% lexers.load : function
24
% lexers.lex : function
25
%
26
% And some properties that map styles onto scintilla styling. I get the
27
% impression that we end up with something simpler, a hybrid between the
28
% scite lexing and the current context way, so we get an intermediate
29
% step, with some penalty for context, but at least I don't have to
30
% maintain two sets (three sets as we also have a line based series).
31 32
% TODO: as these files are in tds we can locate them and set the lexer root
33
% to that one. Currently we're on: we're on context/documents.
34 35
% TODO: tab
36 37
% This is an experiment: eventually we need to hook it into the verbatim code
38
% and deal with widow lines and so.
39 40
\startluacode
41 42
-- todo: merge with collapse
43
-- todo: prehash whitespaces
44 45
-- todo: hook into the pretty print code
46
-- todo: a simple catcode regime with only \ { }
47 48
local
gsub
,
sub
,
find
,
lower
=
string
.
gsub
,
string
.
sub
,
string
.
find
,
string
.
lower
49
local
concat
=
table
.
concat
50
local
formatters
=
string
.
formatters
51
local
lpegmatch
=
lpeg
.
match
52
local
setmetatableindex
=
table
.
setmetatableindex
53 54
local
scite
=
require
(
"
util-sci
"
)
55
buffers
.
scite
=
scite
56 57
-- context output:
58 59
local
f_def_color
=
formatters
[
"
\\definecolor[slxc%s][h=%02X%02X%02X]%%
"
]
60
local
f_fore_none
=
formatters
[
"
\\unexpanded\\def\\slx%s#1{{\\slxc%s#1}\\slxbreak}%%
"
]
61
local
f_fore_bold
=
formatters
[
"
\\unexpanded\\def\\slx%s#1{{\\slxc%s\\bf#1}\\slxbreak}%%
"
]
62
local
f_none_bold
=
formatters
[
"
\\unexpanded\\def\\slx%s#1{{\\bf#1}\\slxbreak}%%
"
]
63
local
f_none_none
=
formatters
[
"
\\unexpanded\\def\\slx%s#1{{#1}\\slxbreak}%%
"
]
64
local
f_texstyled
=
formatters
[
"
\\slx%s{%s}\\slxbreak
"
]
65
local
f_hanging
=
formatters
[
"
\\slxb{%r}%s\\slxe
"
]
-- we need to round: lua 5.3
66 67
local
v_none
=
interfaces
.
variables
.
none
68 69
local
f_mapping
=
[
[
70
\let
\string
\slxL
\string
\letterleftbrace
71
\let
\string
\slxR
\string
\letterrightbrace
72
\let
\string
\slxM
\string
\letterdollar
73
\let
\string
\slxV
\string
\letterbar
74
\let
\string
\slxU
\string
\letterhat
75
\let
\string
\slxD
\string
\letterunderscore
76
\let
\string
\slxH
\string
\letterhash
77
\let
\string
\slxB
\string
\letterbackslash
78
\let
\string
\slxP
\string
\letterpercent
79
\let
\string
\slxT
\string
\lettertilde
80
\let
\string
\slxS
\string
\fixedspace
81
%
]
]
82 83
local
replacer
=
lpeg
.
replacer
{
84
[
"
{
"
]
=
"
\\slxL
"
,
85
[
"
}
"
]
=
"
\\slxR
"
,
86
[
"
$
"
]
=
"
\\slxM
"
,
87
[
"
^
"
]
=
"
\\slxU
"
,
88
[
"
_
"
]
=
"
\\slxD
"
,
89
[
"
|
"
]
=
"
\\slxV
"
,
90
[
"
#
"
]
=
"
\\slxH
"
,
91
[
"
\\
"
]
=
"
\\slxB
"
,
92
[
"
%
"
]
=
"
\\slxP
"
,
93
[
"
~
"
]
=
"
\\slxT
"
,
94
[
"
"
]
=
"
\\slxS
"
,
-- can be made more efficient: \\slxF{n}
95
}
96 97
local
colors
=
nil
98 99
local
function
exportcolors
(
)
100
if
not
colors
then
101
scite
.
loadscitelexer
(
)
102
local
function
black
(
f
)
103
return
(
f
[
1
]
=
=
f
[
2
]
)
and
(
f
[
2
]
=
=
f
[
3
]
)
and
(
f
[
3
]
=
=
0
)
104
end
105
-- local result, r = { f_mapping }, 1
106
local
result
,
r
=
{
}
,
0
107
for
k
,
v
in
table
.
sortedhash
(
lexer
.
context
.
styles
)
do
108
local
fore
=
v
.
fore
109
if
fore
and
not
black
(
fore
)
then
110
r
=
r
+
1
111
result
[
r
]
=
f_def_color
(
k
,
fore
[
1
]
,
fore
[
2
]
or
fore
[
1
]
,
fore
[
3
]
or
fore
[
1
]
)
112
end
113
end
114
r
=
r
+
1
115
result
[
r
]
=
"
%
"
116
for
k
,
v
in
table
.
sortedhash
(
lexer
.
context
.
styles
)
do
117
local
bold
=
v
.
bold
118
local
fore
=
v
.
fore
119
r
=
r
+
1
120
if
fore
and
not
black
(
fore
)
then
121
if
bold
then
122
result
[
r
]
=
f_fore_bold
(
k
,
k
)
123
else
124
result
[
r
]
=
f_fore_none
(
k
,
k
)
125
end
126
else
127
if
bold
then
128
result
[
r
]
=
f_none_bold
(
k
)
129
else
130
result
[
r
]
=
f_none_none
(
k
)
131
end
132
end
133
end
134
colors
=
concat
(
result
,
"
\n
"
)
135
end
136
return
colors
137
end
138 139
local
function
exportwhites
(
)
140
return
setmetatableindex
(
function
(
t
,
k
)
141
local
v
=
find
(
k
,
"
white
"
)
and
true
or
false
142
t
[
k
]
=
v
143
return
v
144
end
)
145
end
146 147
local
function
exportstyled
(
lexer
,
text
)
148
local
result
=
lexer
.
lex
(
lexer
,
text
,
0
)
149
local
start
=
1
150
local
whites
=
exportwhites
(
)
151
local
buffer
=
{
}
152
for
i
=
1
,
#
result
,
2
do
153
local
style
=
result
[
i
]
154
local
position
=
result
[
i
+
1
]
155
local
txt
=
sub
(
text
,
start
,
position
-1
)
156
txt
=
lpegmatch
(
replacer
,
txt
)
157
if
txt
=
=
"
"
then
158
-- skip
159
elseif
whites
[
style
]
then
160
buffer
[
#
buffer
+
1
]
=
txt
161
else
162
buffer
[
#
buffer
+
1
]
=
f_texstyled
(
style
,
txt
)
163
end
164
start
=
position
165
end
166
buffer
=
concat
(
buffer
)
167
return
buffer
168
end
169 170
function
scite
.
installcommands
(
)
171
context
(
exportcolors
(
)
)
172
end
173 174
local
function
slxF
(
b
,
e
)
175
local
d
=
(
e
-
b
)
/
6
176
if
d
>
0
then
177
return
"
\\slxF{
"
.
.
d
.
.
"
}
"
178
else
179
return
"
"
180
end
181
end
182 183
local
p1
=
lpeg
.
tsplitat
(
lpeg
.
patterns
.
newline
)
184
local
p2
=
lpeg
.
P
(
"
\\slxS
"
)
185
local
p3
=
p2
^
1
186
local
p4
=
lpeg
.
Cs
(
(
(
lpeg
.
Cp
(
)
*
p2
*
p2
^
1
*
lpeg
.
Cp
(
)
)
/
slxF
+
lpeg
.
P
(
1
)
)
^
0
)
187 188
local
function
indent
(
str
)
189
local
l
=
lpegmatch
(
p1
,
str
)
190
for
i
=
1
,
#
l
do
191
local
s
=
l
[
i
]
192
if
#
s
>
0
then
193
local
n
=
lpegmatch
(
p3
,
s
)
194
if
n
then
195
n
=
(
n
-1
)
/
6
-- length of "\\slxS "
196
else
197
n
=
0
198
end
199
if
n
>
0
then
200
s
=
sub
(
s
,
n
*
6
+
1
)
201
end
202
s
=
lpegmatch
(
p4
,
s
)
-- can be combined
203
l
[
i
]
=
f_hanging
(
n
,
s
)
-- "\\slxb{%s}%s\\slxe "
204
end
205
end
206
return
concat
(
l
,
"
\n
"
)
207
end
208 209
local
assignbuffer
=
buffers
.
assign
210
local
getcontent
=
buffers
.
getcontent
211
local
loaddata
=
io
.
loaddata
212
local
loadedlexers
=
scite
.
loadedlexers
213 214
local
function
lexdata
(
data
,
lexname
)
215
if
not
data
then
216
data
=
"
"
217
elseif
data
~
=
"
"
then
218
if
data
and
not
find
(
data
,
"
[\r\n]$
"
)
then
219
-- fix for lexbyline
220
data
=
data
.
.
"
\r
"
221
end
222
if
lexname
=
=
v_none
then
223
-- data = string.formatters["%!tex!"](data)
224
data
=
exportstyled
(
loadedlexers
[
"
dummy
"
]
or
loadedlexers
.
tex
,
data
)
225
else
226
data
=
exportstyled
(
loadedlexers
[
lexname
]
or
loadedlexers
.
tex
,
data
)
227
end
228
data
=
indent
(
data
)
229
end
230
-- io.savedata("temp.log",data)
231
assignbuffer
(
"
lex
"
,
data
)
232
end
233 234
scite
.
lexdata
=
lexdata
235 236
function
scite
.
lexbuffer
(
name
,
lexname
)
237
lexdata
(
getcontent
(
name
)
or
"
"
,
lexname
or
"
tex
"
)
238
end
239 240
function
scite
.
lexfile
(
filename
,
lexname
)
241
lexdata
(
loaddata
(
filename
)
or
"
"
,
lexname
or
file
.
suffix
(
filename
)
)
242
end
243 244
interfaces
.
implement
{
245
name
=
"
scitelexfile
"
,
246
arguments
=
"
string
"
,
247
actions
=
scite
.
lexfile
,
248
}
249 250
interfaces
.
implement
{
251
name
=
"
scitelexbuffer
"
,
252
arguments
=
{
"
string
"
,
"
string
"
}
,
253
actions
=
scite
.
lexbuffer
,
254
}
255 256
interfaces
.
implement
{
257
name
=
"
sciteinstallcommands
"
,
258
actions
=
scite
.
installcommands
,
259
}
260 261
local
startInlineScite
=
context
.
startInlineScite
262
local
stopInlineScite
=
context
.
stopInlineScite
263
local
startDisplayScite
=
context
.
startDisplayScite
264
local
stopDisplayScite
=
context
.
stopDisplayScite
265 266
local
lexdata
=
buffers
.
scite
.
lexdata
267 268
local
handler
=
visualizers
.
newhandler
{
269
startinline
=
function
(
settings
)
startInlineScite
(
settings
.
name
or
"
"
)
end
,
270
stopinline
=
function
(
settings
)
stopInlineScite
(
)
end
,
271
startdisplay
=
function
(
settings
)
startDisplayScite
(
settings
.
name
or
"
"
)
end
,
272
stopdisplay
=
function
(
settings
)
stopDisplayScite
(
)
end
,
273
noescape
=
true
,
274
}
275 276
local
knownlexers
=
scite
.
knownlexers
277 278
local
parser
=
function
(
content
,
specification
)
279
local
method
=
lower
(
specification
.
method
)
280
lexdata
(
content
,
knownlexers
[
method
]
or
method
)
281
end
282 283
local
visualizer
=
{
284
handler
=
handler
,
285
parser
=
parser
,
286
}
287 288
visualizers
.
register
(
"
cld
"
,
visualizer
)
289
visualizers
.
register
(
"
tex
"
,
visualizer
)
290
visualizers
.
register
(
"
lua
"
,
visualizer
)
291
visualizers
.
register
(
"
mps
"
,
visualizer
)
292
visualizers
.
register
(
"
mp
"
,
visualizer
)
293
visualizers
.
register
(
"
pdf
"
,
visualizer
)
294
visualizers
.
register
(
"
xml
"
,
visualizer
)
295
visualizers
.
register
(
"
bibtex
"
,
visualizer
)
296
visualizers
.
register
(
"
btx
"
,
visualizer
)
297
visualizers
.
register
(
"
web
"
,
visualizer
)
298
visualizers
.
register
(
"
cpp
"
,
visualizer
)
299
visualizers
.
register
(
"
txt
"
,
visualizer
)
300
visualizers
.
register
(
"
bnf
"
,
visualizer
)
301
visualizers
.
register
(
"
sql
"
,
visualizer
)
302
visualizers
.
register
(
"
json
"
,
visualizer
)
303 304
visualizers
.
register
(
"
sas
"
,
visualizer
)
305 306
function
scite
.
register
(
name
)
307
visualizers
.
register
(
name
,
visualizer
)
308
end
309 310
moduledata
.
scite
=
scite
311 312
\stopluacode
313 314
\definetyping
[
TEX
]
[
option
=
cld
]
315
\definetyping
[
LUA
]
[
option
=
lua
]
316
\definetyping
[
BTX
]
[
option
=
bibtex
]
317
\definetyping
[
MPS
]
[
option
=
mps
]
318
\definetyping
[
MP
]
[
option
=
mp
]
319
\definetyping
[
PDF
]
[
option
=
pdf
]
320
\definetyping
[
CPP
]
[
option
=
cpp
]
% Which is kind of like the web one.
321
\definetyping
[
WEB
]
[
option
=
web
]
322
\definetyping
[
TXT
]
[
option
=
txt
]
323
\definetyping
[
BNF
]
[
option
=
bnf
]
% I might use this in the metafun manual.
324
\definetyping
[
SQL
]
[
option
=
sql
]
% To be tested in an upcoming manual.
325
\definetyping
[
JSON
][
option
=
json
]
% To be tested in an upcoming manual.
326
\definetyping
[
NONE
][
option
=
none
]
327 328
\definetyping
[
SAS
]
[
option
=
sas
]
329 330
% This is a preliminary interface.
331 332
\unprotect
333 334
\newdimen
\scitespaceskip
335 336
\unexpanded
\def
\buff_scite_slxb
#
1
%
337
{
% we can have a side float
338
\advance\hangindent\numexpr
#
1
+
2
\relax
\scitespaceskip
339
\begstrut
\hskip
#
1
\scitespaceskip
340
\ifcase
\hangafter
341
\hangafter
\plusone
\relax
342
\fi
}
343 344
\unexpanded
\def
\buff_scite_slxe
345
{
\endstrut
\par
}
346 347
\unexpanded
\def
\buff_scite_slxs
{
\hskip
\scitespaceskip
\relax
}
348
\unexpanded
\def
\buff_scite_slxf
#
1
{
\hskip
#
1
\scitespaceskip
\relax
}
349 350
\unexpanded
\def
\installscitecommandsinline
351
{
\scitespaceskip
\interwordspace
% \fontcharwd\font`0\relax % brrrrr
352
\let
\slxb
\gobbleoneargument
353
\let
\slxe
\space
354
\let
\slxbreak
\relax
355
\let
\installscitecommandsinline
\relax
}
356 357
\unexpanded
\def
\installscitecommandsdisplay
358
{
\scitespaceskip
\interwordspace
% \fontcharwd\font`0\relax % brrrrr
359
\let
\slxb
\buff_scite_slxb
360
\let
\slxe
\buff_scite_slxe
361
\let
\installscitecommandsdisplay
\relax
}
362 363
\clf_sciteinstallcommands
364 365
\let
\slxb
\gobbleoneargument
366
\let
\slxe
\space
367
\let
\slxbreak
\relax
368 369
\let
\slxL
\letterleftbrace
370
\let
\slxR
\letterrightbrace
371
\let
\slxM
\letterdollar
372
\let
\slxV
\letterbar
373
\let
\slxU
\letterhat
374
\let
\slxD
\letterunderscore
375
\let
\slxH
\letterhash
376
\let
\slxB
\letterbackslash
377
\let
\slxP
\letterpercent
378
\let
\slxT
\lettertilde
379
\let
\slxS
\fixedspace
380 381
\let
\slxS
\buff_scite_slxs
382
\let
\slxF
\buff_scite_slxf
383 384
\def
\module_scite_inherit_typing
385
{
\buff_verbatim_initialize_typing_one
386
\buff_verbatim_set_line_margin
}
387 388
\unexpanded
\def
\startscite
389
{
\begingroup
390
\dosingleempty
\buff_scite_start
}
391 392
\def
\buff_scite_start
[#
1
]
%
393
{
\edef
\currentscitelexer
{
#
1
}
%
394
\dostartbuffer
[
@
scite
@
][
startscite
][
stopscite
]
}
395 396
\unexpanded
\def
\stopscite
397
{
\scitebuffer
[
\ifx
\currentscitelexer
\empty
tex
\else
\currentscitelexer
\fi
][
@
scite
@
]
%
398
\endgroup
}
399 400
\definelines
401
[
scitelines
]
402 403
\setuplines
404
[
scitelines
]
405
[
\c!before
=
,
406
\c!after
=]
407 408
\unexpanded
\def
\scitefile
409
{
\dosingleargument
\module_scite_file
}
410 411
\unexpanded
\def
\module_scite_file
[#
1
]
%
412
{
\begingroup
413
\setcatcodetable
\ctxcatcodes
% needed in xml
414
\clf_scitelexfile
{
#
1
}
%
415
\tt
416
\installscitecommandsdisplay
417
\module_scite_inherit_typing
418
\dontcomplain
419
\raggedright
420
\startcontextcode
421
\startscitelines
422
\getbuffer
[
lex
]
423
\stopscitelines
424
\stopcontextcode
425
\endgroup
}
426 427
\unexpanded
\def
\scitebuffer
428
{
\dodoubleempty
\module_scite_buffer
}
429 430
\unexpanded
\def
\module_scite_buffer
[#
1
][#
2
]
%
431
{
\begingroup
432
\setcatcodetable
\ctxcatcodes
% needed in xml
433
\ifsecondargument
434
\clf_scitelexbuffer
{
#
2
}{
#
1
}
%
435
\else
436
\clf_scitelexbuffer
{
#
1
}{
tex
}
%
437
\fi
438
\tt
439
\installscitecommandsdisplay
440
\module_scite_inherit_typing
441
\dontcomplain
442
\raggedright
443
\startscitelines
444
\getbuffer
[
lex
]
445
\stopscitelines
446
\endgroup
}
447 448
\unexpanded
\def
\sciteinlinebuffer
449
{
\dodoubleempty
\module_scite_buffer_inline
}
450 451
\unexpanded
\def
\module_scite_buffer_inline
[#
1
][#
2
]
%
452
{
\dontleavehmode
453
\begingroup
454
\setcatcodetable
\ctxcatcodes
% needed in xml
455
\ifsecondargument
456
\clf_scitelexbuffer
{
#
2
}{
#
1
}
%
457
\else
458
\clf_scitelexbuffer
{
#
1
}{
tex
}
%
459
\fi
460
\tt
461
\installscitecommandsinline
462
\module_scite_inherit_typing
463
\dontcomplain
464
\getbuffer
[
lex
]
%
465
\endgroup
}
466 467
\unexpanded
\def
\startInlineScite
#
1
%
468
{
\dontleavehmode
469
\begingroup
470
\setcatcodetable
\ctxcatcodes
% needed in xml
471
\tt
472
\installscitecommandsinline
473
\dontcomplain
474
\ignorespaces
475
\clf_getbuffercontent
{
lex
}
% much faster than getbuffer
476
\removeunwantedspaces
}
477 478
\unexpanded
\def
\stopInlineScite
479
{
\endgroup
}
480 481
\unexpanded
\def
\startDisplayScite
#
1
%
482
{
\begingroup
483
\setcatcodetable
\ctxcatcodes
% needed in xml
484
\tt
485
\installscitecommandsdisplay
486
\dontcomplain
487
\buff_verbatim_initialize_typing_one
488
\buff_verbatim_set_line_margin
489
\raggedright
490
\startscitelines
491
\clf_getbuffer
{
lex
}}
492 493
\unexpanded
\def
\stopDisplayScite
494
{
\stopscitelines
495
\endgroup
}
496 497
\unexpanded
\def
\slxbreak
498
{
\allowbreak
}
499 500
\protect
501 502
\continueifinputfile
{
m
-
scite
.
mkiv
}
503 504
\setupbodyfont
[
dejavu
,
8
pt
]
505 506
\setuplayout
507
[
width
=
middle
,
508
height
=
middle
,
509
header
=
1
cm
,
510
footer
=
1
cm
,
511
topspace
=
1
cm
,
512
bottomspace
=
1
cm
,
513
backspace
=
1
cm
]
514 515
\startbuffer
[
demo
]
516
\startsubsubject
[
title
=
{
oeps
}
]
517 518
\startMPcode
519
draw
fullcircle
520
scaled
2
cm
521
withpen
pencircle
scaled
1
mm
522
withcolor
.5
green
;
523
draw
textext
(
524
lua
(
525
"
local function f(s) return string.upper(s) end mp.quoted(f('foo'))
"
526
)
527
)
withcolor
.5
red
;
528
\stopMPcode
529 530
\startluacode
531
context
(
"
foo
"
)
532
\stopluacode
533 534
\stopsubsubject
535
\stopbuffer
536 537
\starttext
538 539
% \scitefile[../lexers/scite-context-lexer.lua] \page
540
% \scitefile[t:/manuals/about/about-metafun.tex] \page
541
% \scitefile[t:/sources/strc-sec.mkiv] \page
542
% \scitefile[e:/tmp/mp.w] \page
543
% \scitefile[t:/manuals/hybrid/tugboat.bib] \page
544
\scitefile
[
e
:
/
tmp
/
test
.
bib
]
\page
545 546
% \getbuffer[demo] \scitebuffer[demo]
547 548
\startbuffer
[
foo
]
549
This
is
text
.
% line 1
550
This
is
text
.
% line 2
551
\stopbuffer
552 553
\scitebuffer
[
none
][
foo
]
554 555
\sciteinlinebuffer
[
none
][
foo
]
556 557
\stoptext
558