mtx-vscode.lua /size: 126 Kb    last modification: 2021-10-28 13:50
1if not modules then modules = { } end modules ['mtx-vscode'] = {
2    version   = 1.000,
3    comment   = "this script is experimental",
4    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5    copyright = "PRAGMA ADE",
6    license   = "see context related readme files"
7}
8
9-- todo: folding and comments
10-- todo: runners (awaiting global script setup)
11-- todo: dark theme
12
13-- Already for quite a while lexing in ConTeXt is kind of standardized and the way
14-- the format evolved also relates to how I see the source. We started with lexing
15-- beginning of 1990 with texedit/wdt in Modula2 and went via perltk (texwork),
16-- Scite (native), Scite (lpeg) as well as different stages in verbatim. So, as
17-- github uses VSCODE I decided to convert the couple of base lexers to the format
18-- that this editor likes. It's all about habits and consistency, not about tons of
19-- fancy features that I don't need and would not use anyway.
20--
21-- I use a lua script to generate the lexer definitions simply because that way the
22-- update will be in sync with the updates in the context distribution.
23--
24--   code.exe --extensions-dir e:\vscode\extensions --install-extension context
25--
26-- In the end all these systems look alike so again we we have these token onto
27-- styling mappings. We even have embedded lexers. Actually, when reading the
28-- explanations it has become internally more close to what scintilla does with
29-- tokens and numbers related to it but then mapped back onto css.
30--
31-- Multiline lexing is a pain here, so I just assume that stuff belonging together is
32-- on one line (like keys to simple values). I'm not in the mood for ugly multistep
33-- parsing now. Here the lpeg lexers win.
34--
35-- We can optimize these expressions if needed but it all looks fast enough. Anyway,
36-- we do start from the already old lexers that we use in SciTe. The lexing as well
37-- as use of colors is kind of consistent and standardized in context and I don't
38-- want to change it. The number of colors is not that large and (at least to me) it
39-- looks less extreme. We also use a gray background because over time we figured
40-- out that this works best (1) for long sessions, and (2) for colors. We have quite
41-- some embedding so that is another reason for consistency.
42--
43-- I do remember generating plist files many years ago but stopped doing that
44-- because I never could check them in practice. We're now kind of back to that. The
45-- reason for using a lua script to generate the json file is that it is easier to
46-- keep in sync with context and also because then a user can just generate the
47-- extension her/himself.
48--
49-- There are nice examples of lexer definitions in the vc extensions path. My regexp
50-- experiences are somewhat rusted and I don't really have intentions to spend too
51-- much time on them. Compared to the lpeg lexers the regexp based ones are often
52-- more compact. It's just a different concept. Anyway, I might improve things after
53-- I've read more of the specs (it seems like the regexp engine is the one from ruby).
54
55-- We normally use a light gray background with rather dark colors which at least
56-- for us is less tiresome. The problem with dark backgrounds is that one needs to
57-- use light colors from pastel palettes. I need to figure out a mapping that works
58-- for a dark background so that optionally one can install without color theme.
59
60-- It is possible to define tasks and even relate them to languages but for some reason
61-- it can not be done global but per workspace which makes using vscode no option for
62-- me (too many different folders with source code, documentation etc). It's kind of
63-- strange because simple runners are provided by many editors. I don't want to program
64-- a lot to get such simple things done so, awaiting global tasks I stick to using the
65-- terminal. Also, having .vscode folderd in every place where a file sits makes no
66-- sense and clutters my disk too much. There is not much progress in this area but we
67-- are prepared.
68
69-- Another showstopper is the fact that we cannot disable utf8 for languages (like pdf,
70-- which is just bytes). I couldn't figure out how to set it in the extension.
71
72-- {
73--     "window.zoomLevel": 2,
74--     "editor.renderWhitespace": "all",
75--     "telemetry.enableCrashReporter": false,
76--     "telemetry.enableTelemetry": false,
77--     "editor.fontFamily": "Dejavu Sans Mono, Consolas, 'Courier New', monospace",
78--     "window.autoDetectHighContrast": false,
79--     "zenMode.hideLineNumbers": false,
80--     "zenMode.centerLayout": false,
81--     "zenMode.fullScreen": false,
82--     "zenMode.hideTabs": false,
83--     "workbench.editor.showIcons": false,
84--     "workbench.settings.enableNaturalLanguageSearch": false,
85--     "window.enableMenuBarMnemonics": false,
86--     "search.location": "panel",
87--     "breadcrumbs.enabled": false,
88--     "workbench.activityBar.visible": false,
89--     "editor.minimap.enabled": false,
90--     "workbench.iconTheme": null,
91--     "extensions.ignoreRecommendations": true,
92--     "editor.renderControlCharacters": true,
93--     "terminal.integrated.scrollback": 5000,
94--     "workbench.colorTheme": "ConTeXt",
95--     "[context.cld]": {},
96--     "terminal.integrated.fontSize": 10,
97--     "terminal.integrated.rendererType": "dom",
98--     "workbench.colorCustomizations": {
99--         "terminal.ansiBlack":         "#000000",
100--         "terminal.ansiWhite":         "#FFFFFF",
101--         "terminal.ansiRed":           "#7F0000",
102--         "terminal.ansiGreen":         "#007F00",
103--         "terminal.ansiBlue":          "#00007F",
104--         "terminal.ansiMagenta":       "#7F007F",
105--         "terminal.ansiCyan":          "#007F7F",
106--         "terminal.ansiYellow":        "#7F7F00",
107--         "terminal.ansiBrightBlack":   "#000000",
108--         "terminal.ansiBrightWhite":   "#FFFFFF",
109--         "terminal.ansiBrightRed":     "#7F0000",
110--         "terminal.ansiBrightGreen":   "#007F00",
111--         "terminal.ansiBrightBlue":    "#00007F",
112--         "terminal.ansiBrightMagenta": "#7F007F",
113--         "terminal.ansiBrightCyan":    "#007F7F",
114--         "terminal.ansiBrightYellow":  "#7F7F00",
115--     }
116-- }
117
118-- kind of done:
119--
120--   tex mps lua cld bibtex sql bnf(untested) pdf xml json c(pp)(simplified)
121--
122-- unlikely to be done (ok, i'm not interested in all this ide stuff anyway):
123--
124--   cpp-web tex-web web web-snippets txt
125--
126-- still todo:
127--
128--   xml: preamble and dtd
129--   pdf: nested string (..(..)..)
130
131local helpinfo = [[
132<?xml version="1.0"?>
133<application>
134 <metadata>
135  <entry name="name">mtx-vscode</entry>
136  <entry name="detail">vscode extension generator</entry>
137  <entry name="version">1.00</entry>
138 </metadata>
139 <flags>
140  <category name="basic">
141   <subcategory>
142    <flag name="generate"><short>generate extension in sync with current version</short></flag>
143    <flag name="program"><short>use the given binary (e.g. codium, default: code)</short></flag>
144    <flag name="start"><short>start vscode with extension context</short></flag>
145   </subcategory>
146  </category>
147 </flags>
148 <examples>
149  <category>
150   <title>Example</title>
151   <subcategory>
152    <example><command>mtxrun --script vscode --generate e:/vscode/extensions</command></example>
153    <example><command>mtxrun --script vscode --generate</command></example>
154    <example><command>mtxrun --script vscode --start</command></example>
155    <example><command>mtxrun --script vscode --program=codium --start</command></example>
156   </subcategory>
157  </category>
158 </examples>
159</application>
160]]
161
162local application = logs.application {
163    name     = "mtx-vscode",
164    banner   = "vscode extension generator",
165    helpinfo = helpinfo,
166}
167
168local report = application.report
169
170require("util-jsn")
171
172scripts        = scripts        or { }
173scripts.vscode = scripts.vscode or { }
174
175local readmedata = [[
176These files are generated. You can use these extensions with for instance:
177
178  code.exe --extensions-dir <someplace>/tex/texmf-context/context/data/vscode/extensions --install-extension context
179
180There are examples of scripts and keybindings too.
181]]
182
183local function locate()
184    local name = resolvers.findfile("vscode-context.readme")
185    if name and name ~= "" then
186        local path = file.dirname(file.dirname(name))
187        if lfs.isdir(path) then
188            return path
189        end
190    end
191end
192
193function scripts.vscode.generate(targetpath)
194
195    local targetpath = targetpath or environment.files[1] or locate()
196
197    if not targetpath or targetpath == "" or not lfs.isdir(targetpath) then
198        report("invalid targetpath %a",targetpath)
199        return
200    end
201
202    local contextpath = string.gsub(targetpath,"\\","/")  .. "/context"
203
204    dir.makedirs(contextpath)
205
206    if not lfs.chdir(contextpath) then
207        return
208    end
209
210    local syntaxpath     = contextpath .. "/syntaxes"
211    local themepath      = contextpath .. "/themes"
212    local taskpath       = contextpath .. "/tasks"
213    local keybindingpath = contextpath .. "/keybindings"
214    local settingspath   = contextpath .. "/settings"
215
216    dir.makedirs(syntaxpath)
217    dir.makedirs(themepath)
218    dir.makedirs(taskpath)
219    dir.makedirs(keybindingpath)
220    dir.makedirs(settingspath)
221
222    if not lfs.isdir(syntaxpath)     then return end
223    if not lfs.isdir(themepath)      then return end
224    if not lfs.isdir(taskpath)       then return end
225    if not lfs.isdir(keybindingpath) then return end
226    if not lfs.isdir(settingspath)   then return end
227
228    -- The package.
229
230    local languages    = { }
231    local grammars     = { }
232    local themes       = { }
233    local tasks        = { }
234    local keybindings  = { }
235
236    local function registerlexer(lexer)
237
238        local category    = lexer.category
239        local contextid   = "context." .. category
240        local scope       = "source." .. contextid
241
242        local setupfile   = "./settings/context-settings-" .. category .. ".json"
243        local grammarfile = "./syntaxes/context-syntax-" .. category .. ".json"
244
245        local grammar = utilities.json.tojson {
246            name       = contextid,
247            scopeName  = scope,
248            version    = lexer.version,
249            repository = lexer.repository,
250            patterns   = lexer.patterns,
251        }
252
253        local setup = utilities.json.tojson(lexer.setup)
254
255        local suffixes   = lexer.suffixes or { }
256        local extensions = { }
257
258        for i=1,#suffixes do
259            extensions[i] = "." .. string.gsub(suffixes[i],"%.","")
260        end
261
262        table.sort(extensions)
263
264        languages[#languages+1] = {
265            id            = contextid,
266            extensions    = #extensions > 0 and extensions or nil,
267            aliases       = { lexer.description },
268            configuration = setupfile,
269        }
270
271        grammars[#grammars+1] = {
272            language  = contextid,
273            scopeName = "source." .. contextid,
274            path      = grammarfile,
275        }
276
277        report("saving grammar for %a in %a",category,grammarfile)
278        report("saving setup for %a in %a",category,setupfile)
279
280        io.savedata(grammarfile, grammar)
281        io.savedata(setupfile, setup)
282
283    end
284
285    local function registertheme(theme)
286
287        local category = theme.category
288        local filename = "./themes/" .. category .. ".json"
289
290        themes[#themes+1] = {
291            label   = theme.description,
292            uiTheme = "vs",
293            path    = filename,
294        }
295
296        local data = utilities.json.tojson {
297            ["$schema"]     = "vscode://schemas/color-theme",
298            ["name"]        = category,
299            ["colors"]      = theme.colors,
300            ["tokenColors"] = theme.styles,
301        }
302
303        report("saving theme %a in %a",category,filename)
304
305        io.savedata(filename,data)
306
307    end
308
309    local function registertask(task)
310
311        local category = task.category
312        local filename = "./tasks/" .. category .. ".json"
313
314        tasks[#tasks+1] = {
315            label = task.description,
316            path  = filename,
317        }
318
319        local data = utilities.json.tojson {
320            ["name"]  = category,
321            ["tasks"] = task.tasks,
322        }
323
324        report("saving task %a in %a",category,filename)
325        io.savedata(filename,data)
326
327    end
328
329    local function registerkeybinding(keybinding)
330
331        local bindings = keybinding.keybindings
332
333        if bindings then
334
335            local category = keybinding.category
336            local filename = "./keybindings/" .. category .. ".json"
337
338            report("saving keybinding %a in %a",category,filename)
339
340            io.savedata(filename,utilities.json.tojson(bindings))
341
342            for i=1,#bindings do
343                 keybindings[#keybindings+1] = bindings[i]
344            end
345
346        end
347    end
348
349    local function savepackage()
350
351        local packagefile  = "package.json"
352        local whateverfile = "package.nls.json"
353        local readmefile   = "vscode-context.readme"
354
355        local specification = utilities.json.tojson {
356            name        = "context",
357            displayName = "ConTeXt",
358            description = "ConTeXt Syntax Highlighting",
359            publisher   = "ConTeXt Development Team",
360            version     = "1.0.0",
361            engines     = {
362                vscode = "*"
363            },
364            categories = {
365                "Lexers",
366                "Syntaxes"
367            },
368            contributes = {
369                languages   = languages,
370                grammars    = grammars,
371                themes      = themes,
372                tasks       = tasks,
373                keybindings = keybindings,
374            },
375
376        }
377
378        report("saving package in %a",packagefile)
379
380        io.savedata(packagefile,specification)
381
382        local whatever = utilities.json.tojson {
383            displayName = "ConTeXt",
384            description = "Provides syntax highlighting and bracket matching in ConTeXt files.",
385        }
386
387        report("saving whatever in %a",whateverfile)
388
389        io.savedata(whateverfile,whatever)
390
391        report("saving readme in %a",readmefile)
392
393        io.savedata(readmefile,readmedata)
394
395    end
396
397    -- themes
398
399    do
400
401        local mycolors = {
402            red       = "#7F0000",
403            green     = "#007F00",
404            blue      = "#00007F",
405            cyan      = "#007F7F",
406            magenta   = "#7F007F",
407            yellow    = "#7F7F00",
408            orange    = "#B07F00",
409            white     = "#FFFFFF",
410            light     = "#CFCFCF",
411            grey      = "#808080",
412            dark      = "#4F4F4F",
413            black     = "#000000",
414            selection = "#F7F7F7",
415            logpanel  = "#E7E7E7",
416            textpanel = "#CFCFCF",
417            linepanel = "#A7A7A7",
418            tippanel  = "#444444",
419            right     = "#0000FF",
420            wrong     = "#FF0000",
421            default   = "#000000",
422            reverse   = "#FFFFFF",
423
424            -- some day a dark:
425
426--     red       = "#CC4444",
427--     green     = "#44CC44",
428--     blue      = "#4444FF",
429--     cyan      = "#55BBBB",
430--     magenta   = "#BB55BB",
431--     yellow    = "#BBBB55",
432--     orange    = "#B07F00",
433--     white     = "#FFFFFF",
434--     light     = "#CFCFCF",
435--     grey      = "#808080",
436--     dark      = "#4F4F4F",
437--     black     = "#000000",
438--     selection = "#F7F7F7",
439--     logpanel  = "#E7E7E7",
440--     textpanel = "#1E1E1E", -- VS "#101010",
441--     linepanel = "#A7A7A7",
442--     tippanel  = "#444444",
443--     right     = "#0000FF",
444--     wrong     = "#FF0000",
445--     default   = "#D4D4D4",
446--     reverse   = "#000000",
447
448        }
449
450        local colors = {
451            ["editor.background"]               = mycolors.textpanel,
452            ["editor.foreground"]               = mycolors.default,
453            ["editorLineNumber.foreground"]     = mycolors.default,
454            ["editorIndentGuide.background"]    = mycolors.textpanel,
455            ["editorBracketMatch.background"]   = mycolors.textpanel,
456            ["editorBracketMatch.border"]       = mycolors.orange,
457            ["editor.lineHighlightBackground"]  = mycolors.textpanel,
458            ["focusBorder"]                     = mycolors.default,
459
460            ["activityBar.background"]          = mycolors.default,
461
462            ["editorGutter.background"]         = mycolors.linepanel,
463            ["editorGutter.foreground"]         = mycolors.default,
464            ["editorGutter.border"]             = mycolors.reverse,
465            ["sideBarTitle.foreground"]         = mycolors.default,
466            ["sideBarSectionHeader.background"] = mycolors.linepanel,
467            ["sideBarSectionHeader.foreground"] = mycolors.default,
468
469            ["statusBar.foreground"]            = mycolors.default,
470            ["statusBar.background"]            = mycolors.linepanel,
471            ["statusBar.border"]                = mycolors.reverse,
472            ["statusBar.noFolderForeground"]    = mycolors.default,
473            ["statusBar.noFolderBackground"]    = mycolors.linepanel,
474            ["statusBar.debuggingForeground"]   = mycolors.default,
475            ["statusBar.debuggingBackground"]   = mycolors.linepanel,
476
477            ["notification.background"]         = mycolors.default,
478        }
479
480        local styles = {
481
482            { scope = "context.whitespace",              settings = { } },
483            { scope = "context.default",                 settings = { foreground = mycolors.default } },
484            { scope = "context.number",                  settings = { foreground = mycolors.cyan } },
485            { scope = "context.comment",                 settings = { foreground = mycolors.yellow } },
486            { scope = "context.keyword",                 settings = { foreground = mycolors.blue, fontStyle = "bold" } },
487            { scope = "context.string",                  settings = { foreground = mycolors.magenta } },
488            { scope = "context.error",                   settings = { foreground = mycolors.red } },
489            { scope = "context.label",                   settings = { foreground = mycolors.red, fontStyle = "bold"  } },
490            { scope = "context.nothing",                 settings = { } },
491            { scope = "context.class",                   settings = { foreground = mycolors.default, fontStyle = "bold" } },
492            { scope = "context.function",                settings = { foreground = mycolors.default, fontStyle = "bold" } },
493            { scope = "context.constant",                settings = { foreground = mycolors.cyan, fontStyle = "bold" } },
494            { scope = "context.operator",                settings = { foreground = mycolors.blue } },
495            { scope = "context.regex",                   settings = { foreground = mycolors.magenta } },
496            { scope = "context.preprocessor",            settings = { foreground = mycolors.yellow, fontStyle = "bold" } },
497            { scope = "context.tag",                     settings = { foreground = mycolors.cyan } },
498            { scope = "context.type",                    settings = { foreground = mycolors.blue } },
499            { scope = "context.variable",                settings = { foreground = mycolors.default } },
500            { scope = "context.identifier",              settings = { } },
501            { scope = "context.linenumber",              settings = { background = mycolors.linepanel } },
502            { scope = "context.bracelight",              settings = { foreground = mycolors.orange, fontStyle = "bold" } },
503            { scope = "context.bracebad",                settings = { foreground = mycolors.orange, fontStyle = "bold" } },
504            { scope = "context.controlchar",             settings = { } },
505            { scope = "context.indentguide",             settings = { foreground = mycolors.linepanel, back = colors.reverse } },
506            { scope = "context.calltip",                 settings = { foreground = mycolors.reverse, back = colors.tippanel } },
507            { scope = "context.invisible",               settings = { background = mycolors.orange } },
508            { scope = "context.quote",                   settings = { foreground = mycolors.blue, fontStyle = "bold" } },
509            { scope = "context.special",                 settings = { foreground = mycolors.blue } },
510            { scope = "context.extra",                   settings = { foreground = mycolors.yellow } },
511            { scope = "context.embedded",                settings = { foreground = mycolors.default, fontStyle = "bold" } },
512            { scope = "context.char",                    settings = { foreground = mycolors.magenta } },
513            { scope = "context.reserved",                settings = { foreground = mycolors.magenta, fontStyle = "bold" } },
514            { scope = "context.definition",              settings = { foreground = mycolors.default, fontStyle = "bold" } },
515            { scope = "context.okay",                    settings = { foreground = mycolors.dark } },
516            { scope = "context.warning",                 settings = { foreground = mycolors.orange } },
517            { scope = "context.standout",                settings = { foreground = mycolors.orange, fontStyle = "bold" } },
518            { scope = "context.command",                 settings = { foreground = mycolors.green, fontStyle = "bold" } },
519            { scope = "context.internal",                settings = { foreground = mycolors.orange, fontStyle = "bold" } },
520            { scope = "context.preamble",                settings = { foreground = mycolors.yellow } },
521            { scope = "context.grouping",                settings = { foreground = mycolors.red } },
522            { scope = "context.primitive",               settings = { foreground = mycolors.blue, fontStyle = "bold" } },
523            { scope = "context.plain",                   settings = { foreground = mycolors.dark, fontStyle = "bold" } },
524            { scope = "context.user",                    settings = { foreground = mycolors.green } },
525            { scope = "context.data",                    settings = { foreground = mycolors.cyan, fontStyle = "bold" } },
526            { scope = "context.text",                    settings = { foreground = mycolors.default } },
527
528            { scope = { "emphasis" }, settings = { fontStyle = "italic" } },
529            { scope = { "strong"   }, settings = { fontStyle = "bold"   } },
530
531            { scope = { "comment"  }, settings = { foreground = mycolors.default  } },
532            { scope = { "string"   }, settings = { foreground = mycolors.magenta } },
533
534            {
535                scope = {
536                    "constant.numeric",
537                    "constant.language.null",
538                    "variable.language.this",
539                    "support.type.primitive",
540                    "support.function",
541                    "support.variable.dom",
542                    "support.variable.property",
543                    "support.variable.property",
544                    "meta.property-name",
545                    "meta.property-value",
546                    "support.constant.handlebars"
547                },
548                settings = {
549                    foreground = mycolors.cyan,
550                }
551            },
552
553            {
554                scope = {
555                    "keyword",
556                    "storage.modifier",
557                    "storage.type",
558                    "variable.parameter"
559                },
560                settings = {
561                    foreground = mycolors.blue,
562                    fontStyle  = "bold",
563                }
564            },
565
566            {
567                scope = {
568                    "entity.name.type",
569                    "entity.other.inherited-class",
570                    "meta.function-call",
571                    "entity.other.attribute-name",
572                    "entity.name.function.shell"
573                },
574                settings = {
575                    foreground = mycolors.default,
576                }
577            },
578
579            {
580                scope = {
581                    "entity.name.tag",
582                },
583                settings = {
584                    foreground = mycolors.default,
585                }
586            },
587
588        }
589
590        registertheme {
591            category    = "context",
592            description = "ConTeXt",
593            colors      = colors,
594            styles      = styles,
595        }
596
597    end
598
599    do
600
601        local presentation = {
602            echo             = true,
603            reveal           = "always",
604            focus            = false,
605            panel            = "shared",
606            showReuseMessage = false,
607            clear            = true,
608        }
609
610        -- chcp 65001 ; ...
611
612        local tasks = {
613            {
614                group   = "build",
615                label   = "process tex file",
616                type    = "shell",
617                command =             "context     --autogenerate --autopdf ${file}",
618                windows = { command = "context.exe --autogenerate --autopdf ${file}" },
619            },
620            {
621                group   = "build",
622                label   = "check tex file",
623                type    = "shell",
624                command =             "mtxrun     --autogenerate --script check ${file}",
625                windows = { command = "mtxrun.exe --autogenerate --script check ${file}" },
626            },
627            {
628                group   = "build",
629                label   = "identify fonts",
630                type    = "shell",
631                command =             "mtxrun     --script fonts --reload --force",
632                windows = { command = "mtxrun.exe --script fonts --reload --force" },
633            },
634            {
635                group   = "build",
636                label   = "process lua file",
637                type    = "shell",
638                command =             "mtxrun     --script ${file}",
639                windows = { command = "mtxrun.exe --script ${file}" },
640            },
641        }
642
643        for i=1,#tasks do
644            local task = tasks[i]
645            if not task.windows then
646                task.windows = {  command = task.command }
647            end
648            if not task.presentation then
649                task.presentation = presentation
650            end
651        end
652
653        registertask {
654            category    = "context",
655            description = "ConTeXt Tasks",
656            tasks       = tasks,
657        }
658
659    end
660
661    do
662
663         local keybindings = {
664            {
665             -- runner  = "context --autogenerate --autopdf ${file}",
666                key     = "ctrl-F12",
667                command = "workbench.action.tasks.runTask",
668                args    = "process tex file",
669                when    = "editorTextFocus && editorLangId == context.tex",
670            },
671            {
672             -- runner  = "mtxrun --autogenerate --script check ${file}",
673                key     = "F12",
674                command = "workbench.action.tasks.runTask",
675                args    = "check tex file",
676                when    = "editorTextFocus && editorLangId == context.tex",
677            },
678            {
679             -- runner  = "mtxrun --script ${file}",
680                key     = "ctrl-F12",
681                command = "workbench.action.tasks.runTask",
682                args    = "process lua file",
683                when    = "editorTextFocus && editorLangId == context.cld",
684            }
685        }
686
687        registerkeybinding {
688            category    = "context",
689            description = "ConTeXt Keybindings",
690            keybindings = keybindings,
691        }
692
693    end
694
695    -- helpers
696
697    local function loaddefinitions(name)
698        return table.load(resolvers.findfile(name))
699    end
700
701    local escapes = {
702        ["."]  = "\\.",
703        ["-"]  = "\\-",
704        ["+"]  = "\\+",
705        ["*"]  = "\\*",
706        ['"']  = '\\"',
707        ["'"]  = "\\'",
708        ['^']  = '\\^',
709        ['$']  = '\\$',
710        ["|"]  = "\\|",
711        ["\\"] = "\\\\",
712        ["["]  = "\\[",
713        ["]"]  = "\\]",
714        ["("]  = "\\(",
715        [")"]  = "\\)",
716        ["%"]  = "\\%",
717        ["!"]  = "\\!",
718        ["&"]  = "\\&",
719        ["?"]  = "\\?",
720        ["~"]  = "\\~",
721    }
722
723    local function sorter(a,b)
724        return a > b
725    end
726
727    local function oneof(t)
728        local result = { }
729        table.sort(t,sorter)
730        for i=1,#t do
731            result[i] = string.gsub(t[i],".",escapes)
732        end
733        return table.concat(result,"|")
734    end
735
736    local function capture(str)
737        return "(" .. str .. ")"
738    end
739
740    local function captures(str)
741        return "\\*(" .. str .. ")\\*"
742    end
743
744    local function include(str)
745        return { include = str }
746    end
747
748    local function configuration(s)
749        if s then
750            local pairs    = s.pairs
751            local comments = s.comments
752            return {
753                brackets         = pairs,
754                autoClosingPairs = pairs,
755                surroundingPairs = pairs,
756                comments = {
757                    lineComment  = comments and comments.inline or nil,
758                    blockComment = comments and comments.display or nil,
759                },
760            }
761        else
762            return { }
763        end
764    end
765
766    -- I need to figure out a decent mapping for dark as the defaults are just
767    -- not to my taste and we also have a different categorization.
768
769    local mapping = {
770        ["context.default"]      = "text source",
771        ["context.number"]       = "constant.numeric",
772        ["context.comment"]      = "comment",
773        ["context.keyword"]      = "keyword",
774        ["context.string"]       = "string source",
775        ["context.label"]        = "meta.tag",
776        ["context.constant"]     = "support.constant",
777        ["context.operator"]     = "keyword.operator.js",
778        ["context.identifier"]   = "support.variable",
779        ["context.quote"]        = "string",
780        ["context.special"]      =      "unset",
781        ["context.extra"]        =      "unset",
782        ["context.embedded"]     = "meta.embedded",
783        ["context.reserved"]     =      "unset",
784        ["context.definition"]   = "keyword",
785        ["context.warning"]      = "invalid",
786        ["context.command"]      =      "unset",
787        ["context.grouping"]     =      "unset",
788        ["context.primitive"]    = "keyword",
789        ["context.plain"]        =      "unset",
790        ["context.user"]         =      "unset",
791        ["context.data"]         = "text source",
792        ["context.text"]         = "text source",
793    }
794
795    local function styler(namespace)
796        local done  = { }
797        local style = function(what,where)
798            if not what or not where then
799                report()
800                report("?  %-5s  %-20s  %s",namespace,what or "?",where or "?")
801                report()
802                os.exit()
803            end
804-- if mapping then
805--     what = mapping[what] or what
806-- end
807            local hash = what .. "." .. where
808            if done[hash] then
809                report("-  %-5s  %-20s  %s",namespace,what,where)
810            else
811             -- report("+  %-5s  %-20s  %s",namespace,what,where)
812                done[hash] = true
813            end
814            return hash .. "." .. namespace
815        end
816        return style, function(what,where) return { name = style(what, where) } end
817    end
818
819    local function embedded(name)
820        return { { include = "source.context." .. name } }
821    end
822
823    -- The tex lexer.
824
825    do
826
827        local interface_lowlevel   = loaddefinitions("scite-context-data-context.lua")
828        local interface_interfaces = loaddefinitions("scite-context-data-interfaces.lua")
829        local interface_tex        = loaddefinitions("scite-context-data-tex.lua")
830
831        local constants  = interface_lowlevel.constants
832        local helpers    = interface_lowlevel.helpers
833        local interfaces = interface_interfaces.common
834        local primitives = { }
835        local overloaded = { }
836
837        for i=1,#helpers do
838            overloaded[helpers[i]] = true
839        end
840        for i=1,#constants do
841            overloaded[constants[i]] = true
842        end
843
844        local function add(data)
845            for k, v in next, data do
846                if v ~= "/" and v ~= "-" then
847                    if not overloaded[v] then
848                        primitives[#primitives+1] = v
849                    end
850                    v = "normal" .. v
851                    if not overloaded[v] then
852                        primitives[#primitives+1] = v
853                    end
854                end
855            end
856        end
857
858        add(interface_tex.tex)
859        add(interface_tex.etex)
860        add(interface_tex.pdftex)
861        add(interface_tex.aleph)
862        add(interface_tex.omega)
863        add(interface_tex.luatex)
864        add(interface_tex.xetex)
865
866        local luacommands = {
867            "ctxlua", "ctxcommand", "ctxfunction",
868            "ctxlatelua", "ctxlatecommand",
869            "cldcommand", "cldcontext",
870            "luaexpr", "luascript", "luathread",
871            "directlua", "latelua",
872        }
873
874        local luaenvironments = {
875            "luacode",
876            "luasetups", "luaparameterset",
877            "ctxfunction", "ctxfunctiondefinition",
878        }
879
880        local mpscommands = {
881            "reusableMPgraphic", "usableMPgraphic",
882            "uniqueMPgraphic", "uniqueMPpagegraphic",
883            "useMPgraphic", "reuseMPgraphic",
884            "MPpositiongraphic",
885        }
886
887        local mpsenvironments_o = {
888            "MPpage"
889        }
890
891        local mpsenvironments_a = {
892            "MPcode", "useMPgraphic", "reuseMPgraphic",
893            "MPinclusions", "MPinitializations", "MPdefinitions", "MPextensions",
894            "MPgraphic", "MPcalculation",
895        }
896
897        -- clf_a-zA-z_
898
899        -- btx|xml a-z
900        -- a-z btx|xml a-z
901
902        -- mp argument {...text}
903        local function words(list)
904            table.sort(list,sorter)
905            return "\\\\(" .. table.concat(list,"|") .. ")" .. "(?=[^a-zA-Z])"
906        end
907
908        local function bwords(list)
909            table.sort(list,sorter)
910            return "(\\\\(" .. table.concat(list,"|") .. "))\\s*(\\{)"
911        end
912
913        local function ewords()
914            return "(\\})"
915        end
916
917        local function environments(list)
918            table.sort(list,sorter)
919            last = table.concat(list,"|")
920            if #list > 1 then
921                last = "(?:" .. last .. ")"
922            end
923            return capture("\\\\start" .. last), capture("\\\\stop" .. last)
924        end
925
926        local capturedconstants     = words(constants)
927        local capturedprimitives    = words(primitives)
928        local capturedhelpers       = words(helpers)
929        local capturedcommands      = words(interfaces)
930        local capturedmpscommands   = words(mpscommands)
931
932        local spaces                = "\\s*"
933        local identifier            = "[a-zA-Z\\_@!?\127-\255]+"
934
935        local comment               = "%.*$\\n?"
936        local ifprimitive           = "\\\\if[a-zA-Z\\_@!?\127-\255]*"
937        local csname                = "\\\\[a-zA-Z\\_@!?\127-\255]+"
938        local csprefix              = "\\\\(btx|xml)[a-z]+"
939        local cssuffix              = "\\\\[a-z]+(btx|xml)[a-z]*"
940        local csreserved            = "\\\\(\\?\\?|[a-z]\\!)[a-zA-Z\\_@!?\127-\255]+"
941
942        local luaenvironmentopen,
943              luaenvironmentclose   = environments(luaenvironments)
944        local luacommandopen,
945              luacommandclose       = environments(luacommands)
946
947        local mpsenvironmentopen_o,
948              mpsenvironmentclose_o = environments(mpsenvironments_o)
949        local mpsenvironmentopen_a,
950              mpsenvironmentclose_a = environments(mpsenvironments_a)
951
952        local argumentopen          = capture("\\{")
953        local argumentclose         = capture("\\}")
954        local argumentcontent       = capture("[^\\}]*")
955
956        local optionopen            = capture("\\[")
957        local optionclose           = capture("\\]")
958        local optioncontent         = capture("[^\\]]*")
959
960        -- not ok yet, todo: equal in settings .. but it would become quite ugly, lpeg wins here
961        -- so instead we color differently
962
963        local option                = "(?:" .. optionopen   .. optioncontent   .. optionclose   ..  ")?"
964        local argument              = "(?:" .. argumentopen .. argumentcontent .. argumentclose ..  ")?"
965
966        local mpsenvironmentopen_o  = mpsenvironmentopen_o .. spaces .. option   .. spaces .. option
967        local mpsenvironmentopen_a  = mpsenvironmentopen_a .. spaces .. argument .. spaces .. argument
968
969        local style, styled         = styler("tex")
970
971        local capturedgroupings = oneof {
972            "{", "}", "$"
973        }
974
975        local capturedextras = oneof {
976            "~", "%", "^", "&", "_",
977            "-", "+", "/",
978            "'", "`",
979            "\\", "|",
980        }
981
982        local capturedspecials = oneof {
983            "(", ")", "[", "]", "<", ">",
984            "#", "=", '"',
985        }
986
987        local capturedescaped = "\\\\."
988
989        registerlexer {
990
991            category    = "tex",
992            description = "ConTeXt TEX",
993            suffixes    = { "tex", "mkiv", "mkvi", "mkix", "mkxi", "mkil", "mkxl", "mklx" },
994            version     = "1.0.0",
995
996            setup       = configuration {
997                pairs = {
998                    { "{", "}" },
999                    { "[", "]" },
1000                    { "(", ")" },
1001                },
1002                comments = {
1003                    inline  = "%",
1004                },
1005            },
1006
1007            repository  = {
1008
1009                comment = {
1010                    name  = style("context.comment", "comment"),
1011                    match = comment,
1012                },
1013
1014                constant = {
1015                    name  = style("context.constant", "commands.constant"),
1016                    match = capturedconstants,
1017                },
1018
1019                ifprimitive = {
1020                    name  = style("context.primitive", "commands.if"),
1021                    match = ifprimitive,
1022                },
1023
1024                primitive = {
1025                    name  = style("context.primitive", "commands.primitive"),
1026                    match = capturedprimitives,
1027                },
1028
1029                helper = {
1030                    name  = style("context.plain", "commands.plain"),
1031                    match = capturedhelpers,
1032                },
1033
1034                command = {
1035                    name  = style("context.command", "commands.context"),
1036                    match = capturedcommands,
1037                },
1038
1039                csname = {
1040                    name  = style("context.user", "commands.user"),
1041                    match = csname,
1042                },
1043
1044                escaped = {
1045                    name  = style("context.command", "commands.escaped"),
1046                    match = capturedescaped,
1047                },
1048
1049                subsystem_prefix = {
1050                    name  = style("context.embedded", "subsystem.prefix"),
1051                    match = csprefix,
1052                },
1053
1054                subsystem_suffix = {
1055                    name  = style("context.embedded", "subsystem.suffix"),
1056                    match = cssuffix,
1057                },
1058
1059                grouping = {
1060                    name  = style("context.grouping", "symbols.groups"),
1061                    match = capturedgroupings,
1062                },
1063
1064                extra = {
1065                    name  = style("context.extra", "symbols.extras"),
1066                    match = capturedextras,
1067                },
1068
1069                special = {
1070                    name  = style("context.special", "symbols.special"),
1071                    match = capturedspecials,
1072                },
1073
1074                reserved = {
1075                    name  = style("context.reserved", "commands.reserved"),
1076                    match = csreserved,
1077                },
1078
1079                lua_environment = {
1080                    ["begin"]     = luaenvironmentopen,
1081                    ["end"]       = luaenvironmentclose,
1082                    patterns      = embedded("cld"),
1083                    beginCaptures = { ["0"] = styled("context.embedded", "lua.environment.open") },
1084                    endCaptures   = { ["0"] = styled("context.embedded", "lua.environment.close") },
1085                },
1086
1087                lua_command = {
1088                    ["begin"]     = luacommandopen,
1089                    ["end"]       = luacommandclose,
1090                    patterns      = embedded("cld"),
1091                    beginCaptures = {
1092                        ["1"] = styled("context.embedded", "lua.command.open"),
1093                        ["2"] = styled("context.grouping", "lua.command.open"),
1094                    },
1095                    endCaptures   = {
1096                        ["1"] = styled("context.grouping", "lua.command.close"),
1097                    },
1098
1099                },
1100
1101                metafun_environment_o = {
1102                    ["begin"]     = mpsenvironmentopen_o,
1103                    ["end"]       = mpsenvironmentclose_o,
1104                    patterns      = embedded("mps"),
1105                    beginCaptures = {
1106                        ["1"] = styled("context.embedded", "metafun.environment.start.o"),
1107                        ["2"] = styled("context.embedded", "metafun.environment.open.o.1"),
1108                        ["3"] = styled("context.warning", "metafun.environment.content.o.1"),
1109                        ["4"] = styled("context.embedded", "metafun.environment.close.o.1"),
1110                        ["5"] = styled("context.embedded", "metafun.environment.open.o.2"),
1111                        ["6"] = styled("context.warning", "metafun.environment.content.o.2"),
1112                        ["7"] = styled("context.embedded", "metafun.environment.close.o.2"),
1113                    },
1114                    endCaptures   = {
1115                        ["0"] = styled("context.embedded", "metafun.environment.stop.o")
1116                    },
1117                },
1118
1119                metafun_environment_a = {
1120                    ["begin"]     = mpsenvironmentopen_a,
1121                    ["end"]       = mpsenvironmentclose_a,
1122                    patterns      = embedded("mps"),
1123                    beginCaptures = {
1124                        ["1"] = styled("context.embedded", "metafun.environment.start.a"),
1125                        ["2"] = styled("context.embedded", "metafun.environment.open.a.1"),
1126                        ["3"] = styled("context.warning", "metafun.environment.content.a.1"),
1127                        ["4"] = styled("context.embedded", "metafun.environment.close.a.1"),
1128                        ["5"] = styled("context.embedded", "metafun.environment.open.a.2"),
1129                        ["6"] = styled("context.warning", "metafun.environment.content.a.2"),
1130                        ["7"] = styled("context.embedded", "metafun.environment.close.a.2"),
1131                    },
1132                    endCaptures   = {
1133                        ["0"] = styled("context.embedded", "metafun.environment.stop.a")
1134                    },
1135                },
1136
1137                metafun_command = {
1138                    name  = style("context.embedded", "metafun.command"),
1139                    match = capturedmpscommands,
1140                },
1141
1142            },
1143
1144            patterns = {
1145                include("#comment"),
1146                include("#constant"),
1147                include("#lua_environment"),
1148                include("#lua_command"),
1149                include("#metafun_environment_o"),
1150                include("#metafun_environment_a"),
1151                include("#metafun_command"),
1152                include("#subsystem_prefix"),
1153                include("#subsystem_suffix"),
1154                include("#ifprimitive"),
1155                include("#helper"),
1156                include("#command"),
1157                include("#primitive"),
1158                include("#reserved"),
1159                include("#csname"),
1160                include("#escaped"),
1161                include("#grouping"),
1162                include("#special"),
1163                include("#extra"),
1164            },
1165
1166        }
1167
1168    end
1169
1170    -- The metafun lexer.
1171
1172    do
1173
1174        local metapostprimitives = { }
1175        local metapostinternals  = { }
1176        local metapostshortcuts  = { }
1177        local metapostcommands   = { }
1178
1179        local metafuninternals   = { }
1180        local metafunshortcuts   = { }
1181        local metafuncommands    = { }
1182
1183        local mergedshortcuts    = { }
1184        local mergedinternals    = { }
1185
1186        do
1187
1188            local definitions = loaddefinitions("scite-context-data-metapost.lua")
1189
1190            if definitions then
1191                metapostprimitives = definitions.primitives or { }
1192                metapostinternals  = definitions.internals  or { }
1193                metapostshortcuts  = definitions.shortcuts  or { }
1194                metapostcommands   = definitions.commands   or { }
1195            end
1196
1197            local definitions = loaddefinitions("scite-context-data-metafun.lua")
1198
1199            if definitions then
1200                metafuninternals  = definitions.internals or { }
1201                metafunshortcuts  = definitions.shortcuts or { }
1202                metafuncommands   = definitions.commands  or { }
1203            end
1204
1205            for i=1,#metapostshortcuts do
1206                mergedshortcuts[#mergedshortcuts+1] = metapostshortcuts[i]
1207            end
1208            for i=1,#metafunshortcuts do
1209                mergedshortcuts[#mergedshortcuts+1] = metafunshortcuts[i]
1210            end
1211
1212            for i=1,#metapostinternals do
1213                mergedinternals[#mergedinternals+1] = metapostinternals[i]
1214            end
1215            for i=1,#metafuninternals do
1216                mergedinternals[#mergedinternals+1] = metafuninternals[i]
1217            end
1218
1219
1220        end
1221
1222        local function words(list)
1223            table.sort(list,sorter)
1224            return "(" .. table.concat(list,"|") .. ")" .. "(?=[^a-zA-Z\\_@!?\127-\255])"
1225        end
1226
1227        local capturedshortcuts          = oneof(mergedshortcuts)
1228        local capturedinternals          = words(mergedinternals)
1229        local capturedmetapostcommands   = words(metapostcommands)
1230        local capturedmetafuncommands    = words(metafuncommands)
1231        local capturedmetapostprimitives = words(metapostprimitives)
1232
1233        local capturedsuffixes = oneof {
1234            "#@", "@#", "#"
1235        }
1236        local capturedspecials = oneof {
1237            "#@", "@#", "#",
1238            "(", ")", "[", "]", "{", "}",
1239            "<", ">", "=", ":",
1240            '"',
1241        }
1242        local capturedexatras = oneof {
1243            "+-+", "++",
1244            "~", "%", "^", "&",
1245            "_", "-", "+", "*", "/",
1246            "`", "'", "|", "\\",
1247        }
1248
1249        local spaces              = "\\s*"
1250        local mandatespaces       = "\\s+"
1251
1252        local identifier          = "[a-zA-Z\\_@!?\127-\255]+"
1253
1254        local decnumber           = "[\\-]?[0-9]+(\\.[0-9]+)?([eE]\\-?[0-9]+)?"
1255
1256        local comment             = "%.*$\\n?"
1257
1258        local stringopen          = "\""
1259        local stringclose         = stringopen
1260
1261        local qualifier           = "[\\.]"
1262        local optionalqualifier   = spaces .. qualifier .. spaces
1263
1264        local capturedstringopen  = capture(stringopen)
1265        local capturedstringclose = capture(stringclose)
1266
1267        local capturedlua         = capture("lua")
1268
1269        local capturedopen        = capture("\\(")
1270        local capturedclose       = capture("\\)")
1271
1272        local capturedtexopen     = capture("(?:b|verbatim)tex") .. mandatespaces
1273        local capturedtexclose    = mandatespaces .. capture("etex")
1274
1275        local texcommand          = "\\[a-zA-Z\\_@!?\127-\255]+"
1276
1277        local style, styled       = styler("mps")
1278
1279        registerlexer {
1280
1281            category    = "mps",
1282            description = "ConTeXt MetaFun",
1283            suffixes    = { "mp", "mpii", "mpiv", "mpxl" },
1284            version     = "1.0.0",
1285
1286            setup       = configuration {
1287                pairs = {
1288                    { "{", "}" },
1289                    { "[", "]" },
1290                    { "(", ")" },
1291                },
1292                comments = {
1293                    inline  = "%",
1294                },
1295            },
1296
1297            repository  = {
1298
1299                comment = {
1300                    name  = style("context.comment", "comment"),
1301                    match = comment,
1302                },
1303
1304                internal = {
1305                    name  = style("context.reserved", "internal"),
1306                    match = capturedshortcuts,
1307                },
1308
1309                shortcut = {
1310                    name  = style("context.data", "shortcut"),
1311                    match = capturedinternals,
1312                },
1313
1314                helper = {
1315                    name  = style("context.command.metafun", "helper"),
1316                    match = capturedmetafuncommands,
1317                },
1318
1319                plain = {
1320                    name  = style("context.plain", "plain"),
1321                    match = capturedmetapostcommands,
1322                },
1323
1324                primitive = {
1325                    name  = style("context.primitive", "primitive"),
1326                    match = capturedmetapostprimitives,
1327                },
1328
1329                quoted = {
1330                    name          = style("context.string", "string.text"),
1331                    ["begin"]     = stringopen,
1332                    ["end"]       = stringclose,
1333                    beginCaptures = { ["0"] = styled("context.special", "string.open") },
1334                    endCaptures   = { ["0"] = styled("context.special", "string.close") },
1335                },
1336
1337                identifier = {
1338                    name  = style("context.default", "identifier"),
1339                    match = identifier,
1340                },
1341
1342                suffix = {
1343                    name  = style("context.number", "suffix"),
1344                    match = capturedsuffixes,
1345                },
1346
1347                special = {
1348                    name  = style("context.special", "special"),
1349                    match = capturedspecials,
1350                },
1351
1352                number = {
1353                    name  = style("context.number", "number"),
1354                    match = decnumber,
1355                },
1356
1357                extra = {
1358                    name  = "context.extra",
1359                    match = capturedexatras,
1360                },
1361
1362                luacall = {
1363                    ["begin"]     = capturedlua .. spaces .. capturedopen .. spaces .. capturedstringopen,
1364                    ["end"]       = capturedstringclose .. spaces .. capturedclose,
1365                    patterns      = embedded("cld"),
1366                    beginCaptures =  {
1367                        ["1"] = styled("context.embedded", "lua.command"),
1368                        ["2"] = styled("context.special",  "lua.open"),
1369                        ["3"] = styled("context.special",  "lua.text.open"),
1370                    },
1371                    endCaptures   =  {
1372                        ["1"] = styled("context.special", "lua.text.close"),
1373                        ["2"] = styled("context.special", "lua.close"),
1374                    },
1375                },
1376
1377                -- default and embedded have the same color but differ in boldness
1378
1379                luacall_suffixed = {
1380                    name      = style("context.embedded", "luacall"),
1381                    ["begin"] = capturedlua,
1382                    ["end"]   = "(?!(" .. optionalqualifier .. identifier .. "))",
1383                    patterns  = {
1384                        {
1385                            match = qualifier,
1386                         -- name  = style("context.operator", "luacall.qualifier"),
1387                            name  = style("context.default", "luacall.qualifier"),
1388                        },
1389                    }
1390                },
1391
1392                texlike = { -- simplified variant
1393                    name  = style("context.warning","unexpected.tex"),
1394                    match = texcommand,
1395                },
1396
1397                texstuff = {
1398                    name          = style("context.string", "tex"),
1399                    ["begin"]     = capturedtexopen,
1400                    ["end"]       = capturedtexclose,
1401                    patterns      = embedded("tex"),
1402                    beginCaptures = { ["1"] = styled("context.primitive", "tex.open") },
1403                    endCaptures   = { ["1"] = styled("context.primitive", "tex.close") },
1404                },
1405
1406            },
1407
1408            patterns = {
1409                include("#comment"),
1410                include("#internal"),
1411                include("#shortcut"),
1412                include("#luacall_suffixed"),
1413                include("#luacall"),
1414                include("#helper"),
1415                include("#plain"),
1416                include("#primitive"),
1417                include("#texstuff"),
1418                include("#suffix"),
1419                include("#identifier"),
1420                include("#number"),
1421                include("#quoted"),
1422                include("#special"),
1423                include("#texlike"),
1424                include("#extra"),
1425            },
1426
1427        }
1428
1429    end
1430
1431    -- The lua lexer.
1432
1433    do
1434
1435        local function words(list)
1436            table.sort(list,sorter)
1437            return "(" .. table.concat(list,"|") .. ")" .. "(?=[^a-zA-Z])"
1438        end
1439
1440        local capturedkeywords = words {
1441            "and", "break", "do", "else", "elseif", "end", "false", "for", "function", -- "goto",
1442            "if", "in", "local", "nil", "not", "or", "repeat", "return", "then", "true",
1443            "until", "while",
1444        }
1445
1446        local capturedbuiltin = words {
1447            "assert", "collectgarbage", "dofile", "error", "getmetatable",
1448            "ipairs", "load", "loadfile", "module", "next", "pairs",
1449            "pcall", "print", "rawequal", "rawget", "rawset", "require",
1450            "setmetatable", "tonumber", "tostring", "type", "unpack", "xpcall", "select",
1451            "string", "table", "coroutine", "debug", "file", "io", "lpeg", "math", "os", "package", "bit32", "utf8",
1452            -- todo: also extra luametatex ones
1453        }
1454
1455        local capturedconstants = words {
1456            "_G", "_VERSION", "_M", "\\.\\.\\.", "_ENV",
1457            "__add", "__call", "__concat", "__div", "__idiv", "__eq", "__gc", "__index",
1458            "__le", "__lt", "__metatable", "__mode", "__mul", "__newindex",
1459            "__pow", "__sub", "__tostring", "__unm", "__len",
1460            "__pairs", "__ipairs",
1461            "__close",
1462            "NaN",
1463           "<const>", "<toclose>",
1464        }
1465
1466        local capturedcsnames = words { -- todo: option
1467            "commands",
1468            "context",
1469         -- "ctxcmd",
1470         -- "ctx",
1471            "metafun",
1472            "metapost",
1473            "ctx[A-Za-z_]*",
1474        }
1475
1476        local capturedoperators = oneof {
1477            "+", "-", "*", "/", "%", "^",
1478            "#", "=", "<", ">",
1479            ";", ":", ",", ".",
1480            "{", "}", "[", "]", "(", ")",
1481            "|", "~", "'"
1482        }
1483
1484        local spaces                = "\\s*"
1485
1486        local identifier            = "[_\\w][_\\w0-9]*"
1487        local qualifier             = "[\\.\\:]"
1488        local optionalqualifier     = spaces .. "[\\.\\:]*" .. spaces
1489
1490        local doublequote           = "\""
1491        local singlequote           = "\'"
1492
1493        local doublecontent         = "(?:\\\\\"|[^\"])*"
1494        local singlecontent         = "(?:\\\\\'|[^\'])*"
1495
1496        local captureddouble        = capture(doublequote) .. capture(doublecontent) .. capture(doublequote)
1497        local capturedsingle        = capture(singlequote) .. capture(singlecontent) .. capture(singlequote)
1498
1499        local longcommentopen       = "--\\[\\["
1500        local longcommentclose      = "\\]\\]"
1501
1502        local longstringopen        = "\\[(=*)\\["
1503        local longstringclose       = "\\](\\2)\\]"
1504
1505        local shortcomment          = "--.*$\\n?"
1506
1507        local hexnumber             = "[\\-]?0[xX][A-Fa-f0-9]+(\\.[A-Fa-f0-9]+)?([eEpP]\\-?[A-Fa-f0-9]+)?"
1508        local decnumber             = "[\\-]?[0-9]+(\\.[0-9]+)?([eEpP]\\-?[0-9]+)?"
1509
1510        local capturedidentifier    = capture(identifier)
1511        local capturedgotodelimiter = capture("::")
1512        local capturedqualifier     = capture(qualifier)
1513        local capturedgoto          = capture("goto")
1514
1515        local style, styled         = styler("lua")
1516
1517        local lualexer = {
1518
1519            category    = "lua",
1520            description = "ConTeXt Lua",
1521         -- suffixes    = { "lua", "luc", "cld", "tuc", "luj", "lum", "tma", "lfg", "luv", "lui" },
1522            version     = "1.0.0",
1523
1524            setup       = configuration {
1525                pairs = {
1526                    { "(", ")" },
1527                    { "{", "}" },
1528                    { "[", "]" },
1529                },
1530                comments = {
1531                    inline  = "--",
1532                    display = { "--[[", "]]" },
1533                },
1534            },
1535
1536            repository  = {
1537
1538                shortcomment = {
1539                    name  = style("context.comment", "comment.short"),
1540                    match = shortcomment,
1541                },
1542
1543                longcomment = {
1544                    name      = style("context.comment", "comment.long"),
1545                    ["begin"] = longcommentopen,
1546                    ["end"]   = longcommentclose,
1547                },
1548
1549                keyword = {
1550                    name  = style("context.keyword", "reserved.keyword"),
1551                    match = capturedkeywords,
1552                },
1553
1554                builtin = {
1555                    name  = style("context.plain", "reserved.builtin"),
1556                    match = capturedbuiltin,
1557                },
1558
1559                constant = {
1560                    name  = style("context.data", "reserved.constants"),
1561                    match = capturedconstants,
1562                },
1563
1564                csname = {
1565                    name      = style("context.user", "csname"),
1566                    ["begin"] = capturedcsnames,
1567                    ["end"]   = "(?!(" .. optionalqualifier .. identifier .. "))",
1568                    patterns  = {
1569                        {
1570                            match = qualifier,
1571                            name  = style("context.operator", "csname.qualifier")
1572                        },
1573                    }
1574                },
1575
1576                identifier_keyword = {
1577                    match  = spaces .. capturedqualifier .. spaces .. capturedkeywords,
1578                    captures = {
1579                        ["1"] = styled("context.operator", "identifier.keyword"),
1580                        ["2"] = styled("context.warning", "identifier.keyword"),
1581                    },
1582                },
1583
1584                identifier_valid = {
1585                    name  = style("context.default", "identifier.valid"),
1586                    match = identifier,
1587                },
1588
1589                ["goto"] = {
1590                    match    = capturedgoto .. spaces .. capturedidentifier,
1591                    captures = {
1592                        ["1"] = styled("context.keyword",  "goto.keyword"),
1593                        ["2"] = styled("context.grouping", "goto.target"),
1594                    }
1595                },
1596
1597                label = {
1598                    match    = capturedgotodelimiter .. capturedidentifier .. capturedgotodelimiter,
1599                    captures = {
1600                        ["1"] = styled("context.keyword",  "label.open"),
1601                        ["2"] = styled("context.grouping", "label.target"),
1602                        ["3"] = styled("context.keyword",  "label.close"),
1603                    }
1604                },
1605
1606                operator = {
1607                    name  = style("context.special", "operator"),
1608                    match = capturedoperators,
1609                },
1610
1611                string_double = {
1612                    match    = captureddouble,
1613                    captures = {
1614                        ["1"] = styled("context.special", "doublequoted.open"),
1615                        ["2"] = styled("context.string",  "doublequoted.text"),
1616                        ["3"] = styled("context.special", "doublequoted.close"),
1617                    },
1618                },
1619
1620                string_single = {
1621                    match    = capturedsingle,
1622                    captures = {
1623                        ["1"] = styled("context.special", "singlequoted.open"),
1624                        ["2"] = styled("context.string",  "singlequoted.text"),
1625                        ["3"] = styled("context.special", "singlequoted.close"),
1626                    },
1627                },
1628
1629                string_long = {
1630                    name          = style("context.string", "long.text"),
1631                    ["begin"]     = longstringopen,
1632                    ["end"]       = longstringclose,
1633                    beginCaptures = { ["0"] = styled("context.special", "string.long.open") },
1634                    endCaptures   = { ["0"] = styled("context.special", "string.long.close") },
1635                },
1636
1637                number_hex = {
1638                    name  = style("context.number", "hexnumber"),
1639                    match = hexnumber,
1640                },
1641
1642                number = {
1643                    name  = style("context.number", "decnumber"),
1644                    match = decnumber,
1645                },
1646
1647            },
1648
1649            patterns   = {
1650                include("#keyword"),
1651                include("#buildin"),
1652                include("#constant"),
1653                include("#csname"),
1654                include("#goto"),
1655                include("#number_hex"),
1656                include("#number"),
1657                include("#identifier_keyword"),
1658                include("#identifier_valid"),
1659                include("#longcomment"),
1660                include("#string_long"),
1661                include("#string_double"),
1662                include("#string_single"),
1663                include("#shortcomment"),
1664                include("#label"),
1665                include("#operator"),
1666            },
1667
1668        }
1669
1670        local texstringopen  = "\\\\!!bs"
1671        local texstringclose = "\\\\!!es"
1672        local texcommand     = "\\\\[A-Za-z\127-\255@\\!\\?_]*"
1673
1674        local cldlexer = {
1675
1676            category    = "cld",
1677            description = "ConTeXt CLD",
1678            suffixes    = { "lmt", "lua", "luc", "cld", "tuc", "luj", "lum", "tma", "lfg", "luv", "lui" },
1679            version     = lualexer.version,
1680            setup       = lualexer.setup,
1681
1682            repository  = {
1683
1684                texstring = {
1685                    name          = style("context.string", "texstring.text"),
1686                    ["begin"]     = texstringopen,
1687                    ["end"]       = texstringclose,
1688                    beginCaptures = { ["0"] = styled("context.special", "texstring.open") },
1689                    endCaptures   = { ["0"] = styled("context.special", "texstring.close") },
1690                },
1691
1692             -- texcomment = {
1693             --     -- maybe some day
1694             -- },
1695
1696                texcommand = {
1697                    name  = style("context.warning", "texcommand"),
1698                    match = texcommand
1699                },
1700
1701            },
1702
1703            patterns = {
1704                include("#texstring"),
1705             -- include("#texcomment"),
1706                include("#texcommand"),
1707            },
1708
1709        }
1710
1711        table.merge (cldlexer.repository,lualexer.repository)
1712        table.imerge(cldlexer.patterns,  lualexer.patterns)
1713
1714        registerlexer(lualexer)
1715        registerlexer(cldlexer)
1716
1717    end
1718
1719    -- The xml lexer.
1720
1721    local xmllexer, xmlconfiguration  do
1722
1723        local spaces            = "\\s*"
1724
1725        local namespace         = "(?:[-\\w.]+:)?"
1726        local name              = "[-\\w.:]+"
1727
1728        local equal             = "="
1729
1730        local elementopen       = "<"
1731        local elementclose      = ">"
1732        local elementopenend    = "</"
1733        local elementempty      = "/?"
1734        local elementnoclose    = "?:([^>]*)"
1735
1736        local entity            = "&.*?;"
1737
1738        local doublequote       = "\""
1739        local singlequote       = "\'"
1740
1741        local doublecontent     = "(?:\\\\\"|[^\"])*"
1742        local singlecontent     = "(?:\\\\\'|[^\'])*"
1743
1744        local captureddouble    = capture(doublequote) .. capture(doublecontent) .. capture(doublequote)
1745        local capturedsingle    = capture(singlequote) .. capture(singlecontent) .. capture(singlequote)
1746
1747        local capturednamespace = capture(namespace)
1748        local capturedname      = capture(name)
1749        local capturedopen      = capture(elementopen)
1750        local capturedclose     = capture(elementclose)
1751        local capturedempty     = capture(elementempty)
1752        local capturedopenend   = capture(elementopenend)
1753
1754        local cdataopen        = "<!\\[CDATA\\["
1755        local cdataclose       = "]]>"
1756
1757        local commentopen      = "<!--"
1758        local commentclose     = "-->"
1759
1760        local processingopen   = "<\\?"
1761        local processingclose  = "\\?>"
1762
1763        local instructionopen  = processingopen .. name
1764        local instructionclose = processingclose
1765
1766        local xmlopen          = processingopen .. "xml"
1767        local xmlclose         = processingclose
1768
1769        local luaopen          = processingopen .. "lua"
1770        local luaclose         = processingclose
1771
1772        local style, styled    = styler("xml")
1773
1774        registerlexer {
1775
1776            category    = "xml",
1777            description = "ConTeXt XML",
1778            suffixes    = {
1779                "xml", "xsl", "xsd", "fo", "exa", "rlb", "rlg", "rlv", "rng",
1780                "xfdf", "xslt", "dtd", "lmx", "htm", "html", "xhtml", "ctx",
1781                "export", "svg", "xul",
1782            },
1783            version     = "1.0.0",
1784
1785            setup       = configuration {
1786                comments = {
1787                    display = { "<!--", "-->" },
1788                },
1789            },
1790
1791            repository  = {
1792
1793                attribute_double = {
1794                    match    = capturednamespace .. capturedname .. spaces .. equal .. spaces .. captureddouble,
1795                    captures = {
1796                        ["1"] = styled("context.plain",    "attribute.double.namespace"),
1797                        ["2"] = styled("context.constant", "attribute.double.name"),
1798                        ["3"] = styled("context.special",  "attribute.double.open"),
1799                        ["4"] = styled("context.string",   "attribute.double.text"),
1800                        ["5"] = styled("context.special",  "attribute.double.close"),
1801                    },
1802                },
1803
1804                attribute_single = {
1805                    match    = capturednamespace .. capturedname .. spaces .. equal .. spaces .. capturedsingle,
1806                    captures = {
1807                        ["1"] = styled("context.plain",    "attribute.single.namespace"),
1808                        ["2"] = styled("context.constant", "attribute.single.name"),
1809                        ["3"] = styled("context.special",  "attribute.single.open"),
1810                        ["4"] = styled("context.string",   "attribute.single.text"),
1811                        ["5"] = styled("context.special",  "attribute.single.close"),
1812                    },
1813                },
1814
1815                attributes = {
1816                    patterns = {
1817                        include("#attribute_double"),
1818                        include("#attribute_single"),
1819                    }
1820                },
1821
1822                entity = {
1823                    name  = style("context.constant", "entity"),
1824                    match = entity,
1825                },
1826
1827                instruction = {
1828                    name          = style("context.default", "instruction.text"),
1829                    ["begin"]     = instructionopen,
1830                    ["end"]       = instructionclose,
1831                    beginCaptures = { ["0"] = styled("context.command", "instruction.open") },
1832                    endCaptures   = { ["0"] = styled("context.command", "instruction.close") },
1833                },
1834
1835                instruction_xml = {
1836                    ["begin"]     = xmlopen,
1837                    ["end"]       = xmlclose,
1838                    beginCaptures = { ["0"] = styled("context.command", "instruction.xml.open") },
1839                    endCaptures   = { ["0"] = styled("context.command", "instruction.xml.close") },
1840                    patterns      = { include("#attributes") }
1841                },
1842
1843                instruction_lua = {
1844                    ["begin"]     = luaopen,
1845                    ["end"]       = luaclose,
1846                    patterns      = embedded("cld"),
1847                    beginCaptures = { ["0"] = styled("context.command", "instruction.lua.open") },
1848                    endCaptures   = { ["0"] = styled("context.command", "instruction.lua.close") },
1849                },
1850
1851                cdata = {
1852                    name          = style("context.default", "cdata.text"),
1853                    ["begin"]     = cdataopen,
1854                    ["end"]       = cdataclose,
1855                    beginCaptures = { ["0"] = styled("context.command", "cdata.open") },
1856                    endCaptures   = { ["0"] = styled("context.command", "cdata.close") },
1857                },
1858
1859                comment = {
1860                    name          = style("context.comment", "comment.text"),
1861                    ["begin"]     = commentopen,
1862                    ["end"]       = commentclose,
1863                    beginCaptures = { ["0"] = styled("context.command", "comment.open") },
1864                    endCaptures   = { ["0"] = styled("context.command", "comment.close") },
1865                },
1866
1867                open = {
1868                    ["begin"]     = capturedopen .. capturednamespace .. capturedname,
1869                    ["end"]       = capturedempty .. capturedclose,
1870                    patterns      = { include("#attributes") },
1871                    beginCaptures = {
1872                        ["1"] = styled("context.keyword", "open.open"),
1873                        ["2"] = styled("context.plain",   "open.namespace"),
1874                        ["3"] = styled("context.keyword", "open.name"),
1875                    },
1876                    endCaptures   = {
1877                        ["1"] = styled("context.keyword", "open.empty"),
1878                        ["2"] = styled("context.keyword", "open.close"),
1879                    },
1880                },
1881
1882                close = {
1883                    match    = capturedopenend .. capturednamespace .. capturedname .. spaces .. capturedclose,
1884                    captures = {
1885                        ["1"] = styled("context.keyword", "close.open"),
1886                        ["2"] = styled("context.plain",   "close.namespace"),
1887                        ["3"] = styled("context.keyword", "close.name"),
1888                        ["4"] = styled("context.keyword", "close.close"),
1889                    },
1890                },
1891
1892                element_error = {
1893                    name  = style("context.error","error"),
1894                    match = elementopen .. elementnoclose .. elementclose,
1895                },
1896
1897            },
1898
1899            patterns = {
1900             -- include("#preamble"),
1901                include("#comment"),
1902                include("#cdata"),
1903             -- include("#doctype"),
1904                include("#instruction_xml"),
1905                include("#instruction_lua"),
1906                include("#instruction"),
1907                include("#close"),
1908                include("#open"),
1909                include("#element_error"),
1910                include("#entity"),
1911            },
1912
1913        }
1914
1915    end
1916
1917    -- The bibtex lexer. Again we assume the keys to be on the same line as the
1918    -- first snippet of the value.
1919
1920    do
1921
1922        local spaces            = "\\s*"
1923        local open              = "{"
1924        local close             = "}"
1925        local hash              = "#"
1926        local equal             = "="
1927        local comma             = ","
1928
1929        local doublequote       = "\""
1930        local doublecontent     = "(?:\\\\\"|[^\"])*"
1931
1932        local singlequote       = "\'"
1933        local singlecontent     = "(?:\\\\\'|[^\'])*"
1934
1935        local groupopen         = "{"
1936        local groupclose        = "}"
1937        local groupcontent      = "(?:\\\\{|\\\\}|[^\\{\\}])*"
1938
1939        local shortcut          = "@(?:string|String|STRING)"    -- enforce consistency
1940        local comment           = "@(?:comment|Comment|COMMENT)" -- enforce consistency
1941
1942        local keyword           = "[a-zA-Z0-9\\_@:\\-]+"
1943
1944        local capturedcomment   = spaces .. capture(comment) .. spaces
1945        local capturedshortcut  = spaces .. capture(shortcut) .. spaces
1946        local capturedkeyword   = spaces .. capture(keyword) .. spaces
1947        local capturedopen      = spaces .. capture(open) .. spaces
1948        local capturedclose     = spaces .. capture(close) .. spaces
1949        local capturedequal     = spaces .. capture(equal) .. spaces
1950        local capturedcomma     = spaces .. capture(comma) .. spaces
1951        local capturedhash      = spaces .. capture(hash) .. spaces
1952
1953        local captureddouble    = spaces .. capture(doublequote) .. capture(doublecontent) .. capture(doublequote) .. spaces
1954        local capturedsingle    = spaces .. capture(singlequote) .. capture(singlecontent) .. capture(singlequote) .. spaces
1955        local capturedgroup     = spaces .. capture(groupopen) .. capture(groupcontent) .. capture(groupclose) .. spaces
1956
1957        local forget            = "%.*$\\n?"
1958
1959        local style, styled     = styler("bibtex")
1960
1961        registerlexer {
1962
1963            category    = "bibtex",
1964            description = "ConTeXt bibTeX",
1965            suffixes    = { "bib", "btx" },
1966            version     = "1.0.0",
1967
1968            setup       = configuration {
1969                pairs = {
1970                    { "{", "}" },
1971                },
1972                comments = {
1973                    inline  = "%",
1974                },
1975            },
1976
1977            repository  = {
1978
1979                forget = {
1980                    name  = style("context.comment", "comment.comment.inline"),
1981                    match = forget,
1982                },
1983
1984                comment = {
1985                    name  = style("context.comment", "comment.comment.content"),
1986                    ["begin"]     = capturedcomment .. capturedopen,
1987                    ["end"]       = capturedclose,
1988                    beginCaptures = {
1989                        ["1"] = styled("context.keyword", "comment.name"),
1990                        ["2"] = styled("context.grouping", "comment.open"),
1991                    },
1992                    endCaptures   = {
1993                        ["1"] = styled("context.grouping", "comment.close"),
1994                    },
1995                },
1996
1997                -- a bit inefficient but good enough
1998
1999                string_double = {
2000                    match    = capturedkeyword .. capturedequal .. captureddouble,
2001                    captures = {
2002                        ["1"] = styled("context.command","doublequoted.key"),
2003                        ["2"] = styled("context.operator","doublequoted.equal"),
2004                        ["3"] = styled("context.special", "doublequoted.open"),
2005                        ["4"] = styled("context.text", "doublequoted.text"),
2006                        ["5"] = styled("context.special", "doublequoted.close"),
2007                    },
2008                },
2009
2010                string_single = {
2011                    match    = capturedkeyword .. capturedequal .. capturedsingle,
2012                    captures = {
2013                        ["1"] = styled("context.command","singlequoted.key"),
2014                        ["2"] = styled("context.operator","singlequoted.equal"),
2015                        ["3"] = styled("context.special", "singlequoted.open"),
2016                        ["4"] = styled("context.text", "singlequoted.text"),
2017                        ["5"] = styled("context.special", "singlequoted.close"),
2018                    },
2019                },
2020
2021                string_grouped = {
2022                    match    = capturedkeyword .. capturedequal .. capturedgroup,
2023                    captures = {
2024                        ["1"] = styled("context.command","grouped.key"),
2025                        ["2"] = styled("context.operator","grouped.equal"),
2026                        ["3"] = styled("context.operator", "grouped.open"),
2027                        ["4"] = styled("context.text", "grouped.text"),
2028                        ["5"] = styled("context.operator", "grouped.close"),
2029                    },
2030                },
2031
2032                string_value = {
2033                    match    = capturedkeyword .. capturedequal .. capturedkeyword,
2034                    captures = {
2035                        ["1"] = styled("context.command", "value.key"),
2036                        ["2"] = styled("context.operator", "value.equal"),
2037                        ["3"] = styled("context.text", "value.text"),
2038                    },
2039                },
2040
2041                string_concat = {
2042                    patterns = {
2043                        {
2044                            match    = capturedhash .. captureddouble,
2045                            captures = {
2046                                ["1"] = styled("context.operator","concat.doublequoted.concatinator"),
2047                                ["2"] = styled("context.special", "concat.doublequoted.open"),
2048                                ["3"] = styled("context.text", "concat.doublequoted.text"),
2049                                ["4"] = styled("context.special", "concat.doublequoted.close"),
2050                            }
2051                        },
2052                        {
2053                            match    = capturedhash .. capturedsingle,
2054                            captures = {
2055                                ["1"] = styled("context.operator","concat.singlequoted.concatinator"),
2056                                ["2"] = styled("context.special", "concat.singlequoted.open"),
2057                                ["3"] = styled("context.text", "concat.singlequoted.text"),
2058                                ["4"] = styled("context.special", "concat.singlequoted.close"),
2059                            },
2060                        },
2061                        {
2062                            match    = capturedhash .. capturedgroup,
2063                            captures = {
2064                                ["1"] = styled("context.operator","concat.grouped.concatinator"),
2065                                ["2"] = styled("context.operator", "concat.grouped.open"),
2066                                ["3"] = styled("context.text", "concat.grouped.text"),
2067                                ["4"] = styled("context.operator", "concat.grouped.close"),
2068                            },
2069                        },
2070                        {
2071                            match    = capturedhash .. capturedkeyword,
2072                            captured = {
2073                                ["1"] = styled("context.operator","concat.value.concatinator"),
2074                                ["2"] = styled("context.text", "concat.value.text"),
2075                            },
2076                        },
2077                    },
2078                },
2079
2080                separator = {
2081                    match = capturedcomma,
2082                    name  = style("context.operator","definition.separator"),
2083                },
2084
2085                definition = {
2086                    name      = style("context.warning","definition.error"),
2087                    ["begin"] = capturedkeyword .. capturedopen .. capturedkeyword .. capturedcomma,
2088                    ["end"]   = capturedclose,
2089                    beginCaptures = {
2090                        ["1"] = styled("context.keyword", "definition.category"),
2091                        ["2"] = styled("context.grouping", "definition.open"),
2092                        ["3"] = styled("context.warning", "definition.label.text"),
2093                        ["3"] = styled("context.operator", "definition.label.separator"),
2094                    },
2095                    endCaptures = {
2096                        ["1"] = styled("context.grouping", "definition.close"),
2097                    },
2098                    patterns  = {
2099                        include("#string_double"),
2100                        include("#string_single"),
2101                        include("#string_grouped"),
2102                        include("#string_value"),
2103                        include("#string_concat"),
2104                        include("#separator"),
2105                    },
2106                },
2107
2108                concatinator = {
2109                    match = capturedhash,
2110                    name  = style("context.operator","definition.concatinator"),
2111                },
2112
2113                shortcut = {
2114                    name      =  style("context.warning","shortcut.error"),
2115                    ["begin"] = capturedshortcut .. capturedopen,
2116                    ["end"]   = capturedclose,
2117                    beginCaptures = {
2118                        ["1"] = styled("context.keyword", "shortcut.name"),
2119                        ["2"] = styled("context.grouping", "shortcut.open"),
2120                    },
2121                    endCaptures = {
2122                        ["1"] = styled("context.grouping", "shortcut.close"),
2123                    },
2124                    patterns  = {
2125                        include("#string_double"),
2126                        include("#string_single"),
2127                        include("#string_grouped"),
2128                        include("#string_value"),
2129                        include("#string_concat"),
2130                    },
2131                },
2132
2133            },
2134
2135            patterns = {
2136                include("#forget"),
2137                include("#comment"),
2138                include("#shortcut"),
2139                include("#definition"),
2140            },
2141
2142        }
2143
2144    end
2145
2146    -- The sql lexer (only needed occasionally in documentation and so).
2147
2148    do
2149
2150        -- ANSI SQL 92 | 99 | 2003
2151
2152        local function words(list)
2153            table.sort(list,sorter)
2154            local str = table.concat(list,"|")
2155            return "(" .. str .. "|" .. string.upper(str) .. ")" .. "(?=[^a-zA-Z])"
2156        end
2157
2158        local capturedkeywords = words {
2159            "absolute", "action", "add", "after", "all", "allocate", "alter", "and", "any",
2160            "are", "array", "as", "asc", "asensitive", "assertion", "asymmetric", "at",
2161            "atomic", "authorization", "avg", "before", "begin", "between", "bigint",
2162            "binary", "bit", "bit_length", "blob", "boolean", "both", "breadth", "by",
2163            "call", "called", "cascade", "cascaded", "case", "cast", "catalog", "char",
2164            "char_length", "character", "character_length", "check", "clob", "close",
2165            "coalesce", "collate", "collation", "column", "commit", "condition", "connect",
2166            "connection", "constraint", "constraints", "constructor", "contains", "continue",
2167            "convert", "corresponding", "count", "create", "cross", "cube", "current",
2168            "current_date", "current_default_transform_group", "current_path",
2169            "current_role", "current_time", "current_timestamp",
2170            "current_transform_group_for_type", "current_user", "cursor", "cycle", "data",
2171            "date", "day", "deallocate", "dec", "decimal", "declare", "default",
2172            "deferrable", "deferred", "delete", "depth", "deref", "desc", "describe",
2173            "descriptor", "deterministic", "diagnostics", "disconnect", "distinct", "do",
2174            "domain", "double", "drop", "dynamic", "each", "element", "else", "elseif",
2175            "end", "equals", "escape", "except", "exception", "exec", "execute", "exists",
2176            "exit", "external", "extract", "false", "fetch", "filter", "first", "float",
2177            "for", "foreign", "found", "free", "from", "full", "function", "general", "get",
2178            "global", "go", "goto", "grant", "group", "grouping", "handler", "having",
2179            "hold", "hour", "identity", "if", "immediate", "in", "indicator", "initially",
2180            "inner", "inout", "input", "insensitive", "insert", "int", "integer",
2181            "intersect", "interval", "into", "is", "isolation", "iterate", "join", "key",
2182            "language", "large", "last", "lateral", "leading", "leave", "left", "level",
2183            "like", "local", "localtime", "localtimestamp", "locator", "loop", "lower",
2184            "map", "match", "max", "member", "merge", "method", "min", "minute", "modifies",
2185            "module", "month", "multiset", "names", "national", "natural", "nchar", "nclob",
2186            "new", "next", "no", "none", "not", "null", "nullif", "numeric", "object",
2187            "octet_length", "of", "old", "on", "only", "open", "option", "or", "order",
2188            "ordinality", "out", "outer", "output", "over", "overlaps", "pad", "parameter",
2189            "partial", "partition", "path", "position", "precision", "prepare", "preserve",
2190            "primary", "prior", "privileges", "procedure", "public", "range", "read",
2191            "reads", "real", "recursive", "ref", "references", "referencing", "relative",
2192            "release", "repeat", "resignal", "restrict", "result", "return", "returns",
2193            "revoke", "right", "role", "rollback", "rollup", "routine", "row", "rows",
2194            "savepoint", "schema", "scope", "scroll", "search", "second", "section",
2195            "select", "sensitive", "session", "session_user", "set", "sets", "signal",
2196            "similar", "size", "smallint", "some", "space", "specific", "specifictype",
2197            "sql", "sqlcode", "sqlerror", "sqlexception", "sqlstate", "sqlwarning", "start",
2198            "state", "static", "submultiset", "substring", "sum", "symmetric", "system",
2199            "system_user", "table", "tablesample", "temporary", "then", "time", "timestamp",
2200            "timezone_hour", "timezone_minute", "to", "trailing", "transaction", "translate",
2201            "translation", "treat", "trigger", "trim", "true", "under", "undo", "union",
2202            "unique", "unknown", "unnest", "until", "update", "upper", "usage", "user",
2203            "using", "value", "values", "varchar", "varying", "view", "when", "whenever",
2204            "where", "while", "window", "with", "within", "without", "work", "write", "year",
2205            "zone",
2206        }
2207
2208        -- The dialects list is taken from drupal.org with standard subtracted.
2209        --
2210        -- MySQL 3.23.x | 4.x | 5.x
2211        -- PostGreSQL 8.1
2212        -- MS SQL Server 2000
2213        -- MS ODBC
2214        -- Oracle 10.2
2215
2216        local captureddialects = words {
2217            "a", "abort", "abs", "access", "ada", "admin", "aggregate", "alias", "also",
2218            "always", "analyse", "analyze", "assignment", "attribute", "attributes", "audit",
2219            "auto_increment", "avg_row_length", "backup", "backward", "bernoulli", "bitvar",
2220            "bool", "break", "browse", "bulk", "c", "cache", "cardinality", "catalog_name",
2221            "ceil", "ceiling", "chain", "change", "character_set_catalog",
2222            "character_set_name", "character_set_schema", "characteristics", "characters",
2223            "checked", "checkpoint", "checksum", "class", "class_origin", "cluster",
2224            "clustered", "cobol", "collation_catalog", "collation_name", "collation_schema",
2225            "collect", "column_name", "columns", "command_function", "command_function_code",
2226            "comment", "committed", "completion", "compress", "compute", "condition_number",
2227            "connection_name", "constraint_catalog", "constraint_name", "constraint_schema",
2228            "containstable", "conversion", "copy", "corr", "covar_pop", "covar_samp",
2229            "createdb", "createrole", "createuser", "csv", "cume_dist", "cursor_name",
2230            "database", "databases", "datetime", "datetime_interval_code",
2231            "datetime_interval_precision", "day_hour", "day_microsecond", "day_minute",
2232            "day_second", "dayofmonth", "dayofweek", "dayofyear", "dbcc", "defaults",
2233            "defined", "definer", "degree", "delay_key_write", "delayed", "delimiter",
2234            "delimiters", "dense_rank", "deny", "derived", "destroy", "destructor",
2235            "dictionary", "disable", "disk", "dispatch", "distinctrow", "distributed", "div",
2236            "dual", "dummy", "dump", "dynamic_function", "dynamic_function_code", "enable",
2237            "enclosed", "encoding", "encrypted", "end-exec", "enum", "errlvl", "escaped",
2238            "every", "exclude", "excluding", "exclusive", "existing", "exp", "explain",
2239            "fields", "file", "fillfactor", "final", "float4", "float8", "floor", "flush",
2240            "following", "force", "fortran", "forward", "freetext", "freetexttable",
2241            "freeze", "fulltext", "fusion", "g", "generated", "granted", "grants",
2242            "greatest", "header", "heap", "hierarchy", "high_priority", "holdlock", "host",
2243            "hosts", "hour_microsecond", "hour_minute", "hour_second", "identified",
2244            "identity_insert", "identitycol", "ignore", "ilike", "immutable",
2245            "implementation", "implicit", "include", "including", "increment", "index",
2246            "infile", "infix", "inherit", "inherits", "initial", "initialize", "insert_id",
2247            "instance", "instantiable", "instead", "int1", "int2", "int3", "int4", "int8",
2248            "intersection", "invoker", "isam", "isnull", "k", "key_member", "key_type",
2249            "keys", "kill", "lancompiler", "last_insert_id", "least", "length", "less",
2250            "limit", "lineno", "lines", "listen", "ln", "load", "location", "lock", "login",
2251            "logs", "long", "longblob", "longtext", "low_priority", "m", "matched",
2252            "max_rows", "maxextents", "maxvalue", "mediumblob", "mediumint", "mediumtext",
2253            "message_length", "message_octet_length", "message_text", "middleint",
2254            "min_rows", "minus", "minute_microsecond", "minute_second", "minvalue",
2255            "mlslabel", "mod", "mode", "modify", "monthname", "more", "move", "mumps",
2256            "myisam", "name", "nesting", "no_write_to_binlog", "noaudit", "nocheck",
2257            "nocompress", "nocreatedb", "nocreaterole", "nocreateuser", "noinherit",
2258            "nologin", "nonclustered", "normalize", "normalized", "nosuperuser", "nothing",
2259            "notify", "notnull", "nowait", "nullable", "nulls", "number", "octets", "off",
2260            "offline", "offset", "offsets", "oids", "online", "opendatasource", "openquery",
2261            "openrowset", "openxml", "operation", "operator", "optimize", "optionally",
2262            "options", "ordering", "others", "outfile", "overlay", "overriding", "owner",
2263            "pack_keys", "parameter_mode", "parameter_name", "parameter_ordinal_position",
2264            "parameter_specific_catalog", "parameter_specific_name",
2265            "parameter_specific_schema", "parameters", "pascal", "password", "pctfree",
2266            "percent", "percent_rank", "percentile_cont", "percentile_disc", "placing",
2267            "plan", "pli", "postfix", "power", "preceding", "prefix", "preorder", "prepared",
2268            "print", "proc", "procedural", "process", "processlist", "purge", "quote",
2269            "raid0", "raiserror", "rank", "raw", "readtext", "recheck", "reconfigure",
2270            "regexp", "regr_avgx", "regr_avgy", "regr_count", "regr_intercept", "regr_r2",
2271            "regr_slope", "regr_sxx", "regr_sxy", "regr_syy", "reindex", "reload", "rename",
2272            "repeatable", "replace", "replication", "require", "reset", "resource",
2273            "restart", "restore", "returned_cardinality", "returned_length",
2274            "returned_octet_length", "returned_sqlstate", "rlike", "routine_catalog",
2275            "routine_name", "routine_schema", "row_count", "row_number", "rowcount",
2276            "rowguidcol", "rowid", "rownum", "rule", "save", "scale", "schema_name",
2277            "schemas", "scope_catalog", "scope_name", "scope_schema", "second_microsecond",
2278            "security", "self", "separator", "sequence", "serializable", "server_name",
2279            "setof", "setuser", "share", "show", "shutdown", "simple", "soname", "source",
2280            "spatial", "specific_name", "sql_big_result", "sql_big_selects",
2281            "sql_big_tables", "sql_calc_found_rows", "sql_log_off", "sql_log_update",
2282            "sql_low_priority_updates", "sql_select_limit", "sql_small_result",
2283            "sql_warnings", "sqlca", "sqrt", "ssl", "stable", "starting", "statement",
2284            "statistics", "status", "stddev_pop", "stddev_samp", "stdin", "stdout",
2285            "storage", "straight_join", "strict", "string", "structure", "style",
2286            "subclass_origin", "sublist", "successful", "superuser", "synonym", "sysdate",
2287            "sysid", "table_name", "tables", "tablespace", "temp", "template", "terminate",
2288            "terminated", "text", "textsize", "than", "ties", "tinyblob", "tinyint",
2289            "tinytext", "toast", "top", "top_level_count", "tran", "transaction_active",
2290            "transactions_committed", "transactions_rolled_back", "transform", "transforms",
2291            "trigger_catalog", "trigger_name", "trigger_schema", "truncate", "trusted",
2292            "tsequal", "type", "uescape", "uid", "unbounded", "uncommitted", "unencrypted",
2293            "unlisten", "unlock", "unnamed", "unsigned", "updatetext", "use",
2294            "user_defined_type_catalog", "user_defined_type_code", "user_defined_type_name",
2295            "user_defined_type_schema", "utc_date", "utc_time", "utc_timestamp", "vacuum",
2296            "valid", "validate", "validator", "var_pop", "var_samp", "varbinary", "varchar2",
2297            "varcharacter", "variable", "variables", "verbose", "volatile", "waitfor",
2298            "width_bucket", "writetext", "x509", "xor", "year_month", "zerofill",
2299        }
2300
2301        local capturedoperators = oneof {
2302            "+", "-", "*", "/",
2303            "%", "^", "!", "&", "|", "?", "~",
2304            "=", "<", ">",
2305            ";", ":", ".",
2306            "{", "}", "[", "]", "(", ")",
2307        }
2308
2309        local spaces          = "\\s*"
2310        local identifier      = "[a-zA-Z\\_][a-zA-Z0-9\\_]*"
2311
2312        local comment         = "%.*$\\n?"
2313        local commentopen     = "/\\*"
2314        local commentclose    = "\\*/"
2315
2316        local doublequote     = "\""
2317        local singlequote     = "\'"
2318        local reversequote    = "`"
2319
2320        local doublecontent   = "(?:\\\\\"|[^\"])*"
2321        local singlecontent   = "(?:\\\\\'|[^\'])*"
2322        local reversecontent  = "(?:\\\\`|[^`])*"
2323
2324        local decnumber       = "[\\-]?[0-9]+(\\.[0-9]+)?([eEpP]\\-?[0-9]+)?"
2325
2326        local captureddouble  = capture(doublequote) .. capture(doublecontent) .. capture(doublequote)
2327        local capturedsingle  = capture(singlequote) .. capture(singlecontent) .. capture(singlequote)
2328        local capturedreverse = capture(reversequote) .. capture(reversecontent) .. capture(reversequote)
2329
2330        local style, styled   = styler("sql")
2331
2332        registerlexer {
2333
2334            category    = "sql",
2335            description = "ConTeXt SQL",
2336            suffixes    = { "sql" },
2337            version     = "1.0.0",
2338
2339            setup       = configuration {
2340--                 comments = {
2341--                     inline  = "...",
2342--                     display = { "...", "..." },
2343--                 },
2344            },
2345
2346            repository  = {
2347
2348                comment_short = {
2349                    name  = style("context.comment", "comment.comment"),
2350                    match = comment,
2351                },
2352
2353                comment_long = {
2354                    name          = style("context.comment", "comment.text"),
2355                    ["begin"]     = commentopen,
2356                    ["end"]       = commentclose,
2357                    beginCaptures = { ["0"] = styled("context.command", "comment.open") },
2358                    endCaptures   = { ["0"] = styled("context.command", "comment.close") },
2359                },
2360
2361                keyword_standard = {
2362                    name  = style("context.keyword", "reserved.standard"),
2363                    match = capturedkeywords,
2364                },
2365
2366                keyword_dialect = {
2367                    name  = style("context.keyword", "reserved.dialect"),
2368                    match = captureddialects,
2369                },
2370
2371                operator = {
2372                    name  = style("context.special", "operator"),
2373                    match = capturedoperators,
2374
2375                },
2376
2377                identifier = {
2378                    name  = style("context.text", "identifier"),
2379                    match = identifier,
2380                },
2381
2382                string_double = {
2383                    match    = captureddouble,
2384                    captures = {
2385                        ["1"] = styled("context.special", "doublequoted.open"),
2386                        ["2"] = styled("context.text", "doublequoted.text"),
2387                        ["3"] = styled("context.special", "doublequoted.close"),
2388                    },
2389                },
2390
2391                string_single = {
2392                    match    = capturedsingle,
2393                    captures = {
2394                        ["1"] = styled("context.special", "singlequoted.open"),
2395                        ["2"] = styled("context.text", "singlequoted.text"),
2396                        ["3"] = styled("context.special", "singlequoted.close"),
2397                    },
2398                },
2399
2400                string_reverse = {
2401                    match    = capturedreverse,
2402                    captures = {
2403                        ["1"] = styled("context.special", "reversequoted.open"),
2404                        ["2"] = styled("context.text", "reversequoted.text"),
2405                        ["3"] = styled("context.special", "reversequoted.close"),
2406                    },
2407                },
2408
2409                number = {
2410                    name  = style("context.number", "number"),
2411                    match = decnumber,
2412                },
2413
2414            },
2415
2416            patterns = {
2417                include("#keyword_standard"),
2418                include("#keyword_dialect"),
2419                include("#identifier"),
2420                include("#string_double"),
2421                include("#string_single"),
2422                include("#string_reverse"),
2423                include("#comment_long"),
2424                include("#comment_short"),
2425                include("#number"),
2426                include("#operator"),
2427            },
2428
2429        }
2430
2431    end
2432
2433    -- The bnf lexer (only used for documentation, untested).
2434
2435    do
2436
2437        local operators = oneof {
2438            "*", "+", "-", "/",
2439            ",", ".", ":", ";",
2440            "(", ")", "<", ">", "{", "}", "[",  "]",
2441             "#", "=", "?", "@", "|", " ", "!","$", "%", "&", "\\", "^", "-", "_", "`", "~",
2442        }
2443
2444        local spaces          = "\\s*"
2445
2446        local text            = "[a-zA-Z0-9]|" .. operators
2447
2448        local doublequote     = "\""
2449        local singlequote     = "\'"
2450
2451        local termopen        = "<"
2452        local termclose       = ">"
2453        local termcontent     = "([a-zA-Z][a-zA-Z0-9\\-]*)"
2454
2455        local becomes         = "::="
2456        local extra           = "|"
2457
2458        local captureddouble  = capture(doublequote) .. capture(text) .. capture(doublequote)
2459        local capturedsingle  = capture(singlequote) .. capture(text) .. capture(singlequote)
2460        local capturedterm    = capture(termopen) .. capture(termcontent) .. capture(termclose)
2461
2462        local style, styled   = styler("bnf")
2463
2464        registerlexer {
2465
2466            category    = "bnf",
2467            description = "ConTeXt BNF",
2468            suffixes    = { "bnf" },
2469            version     = "1.0.0",
2470
2471            setup       = configuration {
2472                pairs = {
2473                    { "<", ">" },
2474                },
2475            },
2476
2477            repository  = {
2478
2479                term = {
2480                    match    = capturedterm,
2481                    captures = {
2482                        ["1"] = styled("context.command", "term.open"),
2483                        ["2"] = styled("context.text", "term.text"),
2484                        ["3"] = styled("context.command", "term.close"),
2485                    },
2486                },
2487
2488                text_single = {
2489                    match    = capturedsingle,
2490                    captures = {
2491                        ["1"] = styled("context.special", "singlequoted.open"),
2492                        ["2"] = styled("context.text", "singlequoted.text"),
2493                        ["3"] = styled("context.special", "singlequoted.close"),
2494                    },
2495                },
2496
2497                text_double = {
2498                    match    = captureddouble,
2499                    captures = {
2500                        ["1"] = styled("context.special", "doublequoted.open"),
2501                        ["2"] = styled("context.text", "doublequoted.text"),
2502                        ["3"] = styled("context.special", "doublequoted.close"),
2503                    },
2504                },
2505
2506                becomes = {
2507                    name  = style("context.operator", "symbol.becomes"),
2508                    match = becomes,
2509                },
2510
2511                extra = {
2512                    name  = style("context.extra", "symbol.extra"),
2513                    match = extra,
2514                },
2515
2516            },
2517
2518            patterns = {
2519                include("#term"),
2520                include("#text_single"),
2521                include("#text_reverse"),
2522                include("#becomes"),
2523                include("#extra"),
2524            },
2525
2526        }
2527
2528    end
2529
2530    do
2531
2532        -- A rather simple one, but consistent with the rest. I don't use an IDE or fancy
2533        -- features. No tricks for me.
2534
2535        local function words(list)
2536            table.sort(list,sorter)
2537            return "\\b(" .. table.concat(list,"|") .. ")\\b"
2538        end
2539
2540        local capturedkeywords = words { -- copied from cpp.lua
2541            -- c
2542            "asm", "auto", "break", "case", "const", "continue", "default", "do", "else",
2543            "extern", "false", "for", "goto", "if", "inline", "register", "return",
2544            "sizeof", "static", "switch", "true", "typedef", "volatile", "while",
2545            "restrict",
2546            -- hm
2547            "_Bool", "_Complex", "_Pragma", "_Imaginary",
2548            -- c++.
2549            "catch", "class", "const_cast", "delete", "dynamic_cast", "explicit",
2550            "export", "friend", "mutable", "namespace", "new", "operator", "private",
2551            "protected", "public", "signals", "slots", "reinterpret_cast",
2552            "static_assert", "static_cast", "template", "this", "throw", "try", "typeid",
2553            "typename", "using", "virtual"
2554        }
2555
2556        local captureddatatypes = words { -- copied from cpp.lua
2557            "bool", "char", "double", "enum", "float", "int", "long", "short", "signed",
2558            "struct", "union", "unsigned", "void"
2559        }
2560
2561        local capturedluatex = words { -- new
2562            "word", "halfword", "quarterword", "scaled", "pointer", "glueratio",
2563        }
2564
2565        local capturedmacros = words { -- copied from cpp.lua
2566            "define", "elif", "else", "endif", "error", "if", "ifdef", "ifndef", "import",
2567            "include", "line", "pragma", "undef", "using", "warning"
2568        }
2569
2570        local operators = oneof {
2571            "*", "+", "-", "/",
2572            "%", "^", "!", "&", "?", "~", "|",
2573            "=", "<", ">",
2574            ";", ":", ".",
2575            "{", "}", "[", "]", "(", ")",
2576        }
2577
2578        local spaces          = "\\s*"
2579
2580        local identifier      = "[A-Za-z_][A-Za-z_0-9]*"
2581
2582        local comment         = "//.*$\\n?"
2583        local commentopen     = "/\\*"
2584        local commentclose    = "\\*/"
2585
2586        local doublequote     = "\""
2587        local singlequote     = "\'"
2588        local reversequote    = "`"
2589
2590        local doublecontent   = "(?:\\\\\"|[^\"])*"
2591        local singlecontent   = "(?:\\\\\'|[^\'])*"
2592
2593        local captureddouble  = capture(doublequote) .. capture(doublecontent) .. capture(doublequote)
2594        local capturedsingle  = capture(singlequote) .. capture(singlecontent) .. capture(singlequote)
2595
2596        local texopen         = "/\\*tex"
2597        local texclose        = "\\*/"
2598
2599        local hexnumber       = "[\\-]?0[xX][A-Fa-f0-9]+(\\.[A-Fa-f0-9]+)?([eEpP]\\-?[A-Fa-f0-9]+)?"
2600        local decnumber       = "[\\-]?[0-9]+(\\.[0-9]+)?([eEpP]\\-?[0-9]+)?"
2601
2602        local capturedmacros  = spaces .. capture("#") .. spaces .. capturedmacros
2603
2604        local style, styled   = styler("c")
2605
2606        registerlexer {
2607
2608            category    = "cpp",
2609            description = "ConTeXt C",
2610            suffixes    = { "c", "h", "cpp", "hpp" },
2611            version     = "1.0.0",
2612
2613            setup       = configuration {
2614                pairs = {
2615                    { "{", "}" },
2616                    { "[", "]" },
2617                    { "(", ")" },
2618                },
2619            },
2620
2621            repository  = {
2622
2623                keyword = {
2624                    match = capturedkeywords,
2625                    name  = style("context.keyword","c"),
2626                },
2627
2628                datatype = {
2629                    match = captureddatatypes,
2630                    name  = style("context.keyword","datatype"),
2631                },
2632
2633                luatex = {
2634                    match = capturedluatex,
2635                    name  = style("context.command","luatex"),
2636                },
2637
2638                macro = {
2639                    match = capturedmacros,
2640                    captures = {
2641                        ["1"] = styled("context.data","macro.tag"),
2642                        ["2"] = styled("context.data","macro.name"),
2643                    }
2644                },
2645
2646                texcomment = {
2647                    ["begin"]     = texopen,
2648                    ["end"]       = texclose,
2649                    patterns      = embedded("tex"),
2650                    beginCaptures = { ["0"] = styled("context.comment", "tex.open") },
2651                    endCaptures   = { ["0"] = styled("context.comment", "tex.close") },
2652                },
2653
2654                longcomment = {
2655                    name      = style("context.comment","long"),
2656                    ["begin"] = commentopen,
2657                    ["end"]   = commentclose,
2658                },
2659
2660                shortcomment = {
2661                    name  = style("context.comment","short"),
2662                    match = comment,
2663                },
2664
2665                identifier = {
2666                    name  = style("context.default","identifier"),
2667                    match = identifier,
2668                },
2669
2670                operator = {
2671                    name  = style("context.operator","any"),
2672                    match = operators,
2673                },
2674
2675                string_double = {
2676                    match    = captureddouble,
2677                    captures = {
2678                        ["1"] = styled("context.special", "doublequoted.open"),
2679                        ["2"] = styled("context.string",  "doublequoted.text"),
2680                        ["3"] = styled("context.special", "doublequoted.close"),
2681                    },
2682                },
2683
2684                string_single = {
2685                    match    = capturedsingle,
2686                    captures = {
2687                        ["1"] = styled("context.special", "singlequoted.open"),
2688                        ["2"] = styled("context.string",  "singlequoted.text"),
2689                        ["3"] = styled("context.special", "singlequoted.close"),
2690                    },
2691                },
2692
2693                hexnumber = {
2694                    name  = style("context.number","hex"),
2695                    match = hexnumber,
2696                },
2697
2698                decnumber = {
2699                    name  = style("context.number","dec"),
2700                    match = decnumber,
2701                },
2702
2703            },
2704
2705            patterns = {
2706                include("#keyword"),
2707                include("#datatype"),
2708                include("#luatex"),
2709                include("#identifier"),
2710                include("#macro"),
2711                include("#string_double"),
2712                include("#string_single"),
2713                include("#texcomment"),
2714                include("#longcomment"),
2715                include("#shortcomment"),
2716                include("#hexnumber"),
2717                include("#decnumber"),
2718                include("#operator"),
2719            },
2720
2721        }
2722
2723    end
2724
2725    -- The pdf lexer.
2726
2727    do
2728
2729        -- we can assume no errors in the syntax
2730
2731        local spaces                  = "\\s*"
2732
2733        local reserved                = oneof { "true" ,"false" , "null" }
2734        local reference               = "R"
2735
2736        local dictionaryopen          = "<<"
2737        local dictionaryclose         = ">>"
2738
2739        local arrayopen               = "\\["
2740        local arrayclose              = "\\]"
2741
2742        local stringopen              = "\\("
2743        local stringcontent           = "(?:\\\\[\\(\\)]|[^\\(\\)])*"
2744        local stringclose             = "\\)"
2745
2746        local hexstringopen           = "<"
2747        local hexstringcontent        = "[^>]*"
2748        local hexstringclose          = ">"
2749
2750        local unicodebomb             = "feff"
2751
2752        local objectopen              = "obj"    -- maybe also ^ $
2753        local objectclose             = "endobj" -- maybe also ^ $
2754
2755        local streamopen              = "^stream$"
2756        local streamclose             = "^endstream$"
2757
2758        local name                    = "/[^\\s<>/\\[\\]\\(\\)]+"   -- no need to be more clever than this
2759        local integer                 = "[\\-]?[0-9]+"              -- no need to be more clever than this
2760        local real                    = "[\\-]?[0-9]*[\\.]?[0-9]+"  -- no need to be more clever than this
2761
2762        local capturedcardinal        = "([0-9]+)"
2763
2764        local captureddictionaryopen  = capture(dictionaryopen)
2765        local captureddictionaryclose = capture(dictionaryclose)
2766
2767        local capturedarrayopen       = capture(arrayopen)
2768        local capturedarrayclose      = capture(arrayclose)
2769
2770        local capturedobjectopen      = capture(objectopen)
2771        local capturedobjectclose     = capture(objectclose)
2772
2773        local capturedname            = capture(name)
2774        local capturedinteger         = capture(integer)
2775        local capturedreal            = capture(real)
2776        local capturedreserved        = capture(reserved)
2777        local capturedreference       = capture(reference)
2778
2779        local capturedunicode         = capture(hexstringopen) .. capture(unicodebomb) .. capture(hexstringcontent) .. capture(hexstringclose)
2780        local capturedunicode         = capture(hexstringopen) .. capture(unicodebomb) .. capture(hexstringcontent) .. capture(hexstringclose)
2781        local capturedwhatsit         = capture(hexstringopen) .. capture(hexstringcontent) .. capture(hexstringclose)
2782        local capturedstring          = capture(stringopen) .. capture(stringcontent) .. capture(stringclose)
2783
2784        local style, styled           = styler("pdf")
2785
2786        -- strings are not ok yet: there can be nested unescaped () but not critical now
2787
2788        registerlexer {
2789
2790            category    = "pdf",
2791            description = "ConTeXt PDF",
2792            suffixes    = { "pdf" },
2793            version     = "1.0.0",
2794
2795            setup       = configuration {
2796                pairs = {
2797                    { "<", ">" },
2798                    { "[", "]" },
2799                    { "(", ")" },
2800                },
2801            },
2802
2803            repository  = {
2804
2805                comment = {
2806                    name  = style("context.comment","comment"),
2807                    match = "%.*$\\n?",
2808                },
2809
2810                content = {
2811                    patterns = {
2812                        { include = "#dictionary" },
2813                        { include = "#stream" },
2814                        { include = "#array" },
2815                        {
2816                            name  = style("context.constant","object.content.name"),
2817                            match = capturedname,
2818                        },
2819                        {
2820                            match    = capturedcardinal .. spaces .. capturedcardinal .. spaces .. capturedreference,
2821                            captures = {
2822                                ["1"] = styled("context.warning","content.reference.1"),
2823                                ["2"] = styled("context.warning","content.reference.2"),
2824                                ["3"] = styled("context.command","content.reference.3"),
2825                            }
2826                        },
2827                        {
2828                            name  = style("context.number","content.real"),
2829                            match = capturedreal,
2830                        },
2831                        {
2832                            name  = style("context.number","content.integer"),
2833                            match = capturedinteger,
2834                        },
2835                        {
2836                            match    = capturedstring,
2837                            captures = {
2838                                ["1"] = styled("context.quote","content.string.open"),
2839                                ["2"] = styled("context.string","content.string.text"),
2840                                ["3"] = styled("context.quote","content.string.close"),
2841                            }
2842                        },
2843                        {
2844                            name  = style("context.number","content.reserved"),
2845                            match = capturedreserved,
2846                        },
2847                        {
2848                            match    = capturedunicode,
2849                            captures = {
2850                                ["1"] = styled("context.quote","content.unicode.open"),
2851                                ["2"] = styled("context.plain","content.unicode.bomb"),
2852                                ["3"] = styled("context.string","content.unicode.text"),
2853                                ["4"] = styled("context.quote","content.unicode.close"),
2854                            }
2855                        },
2856                        {
2857                            match    = capturedwhatsit,
2858                            captures = {
2859                                ["1"] = styled("context.quote","content.whatsit.open"),
2860                                ["2"] = styled("context.string","content.whatsit.text"),
2861                                ["3"] = styled("context.quote","content.whatsit.close"),
2862                            }
2863                        },
2864                    },
2865                },
2866
2867                object = {
2868                    ["begin"]     = capturedcardinal .. spaces .. capturedcardinal .. spaces .. capturedobjectopen,
2869                    ["end"]       = capturedobjectclose,
2870                    patterns      = { { include = "#content" } },
2871                    beginCaptures = {
2872                        ["1"] = styled("context.warning","object.1"),
2873                        ["2"] = styled("context.warning","object.2"),
2874                        ["3"] = styled("context.keyword", "object.open")
2875                    },
2876                    endCaptures   = {
2877                        ["1"] = styled("context.keyword", "object.close")
2878                    },
2879                },
2880
2881                array = {
2882                    ["begin"]     = capturedarrayopen,
2883                    ["end"]       = capturedarrayclose,
2884                    patterns      = { { include = "#content" } },
2885                    beginCaptures = { ["1"] = styled("context.grouping", "array.open") },
2886                    endCaptures   = { ["1"] = styled("context.grouping", "array.close") },
2887                },
2888
2889                dictionary = {
2890                    ["begin"]     = captureddictionaryopen,
2891                    ["end"]       = captureddictionaryclose,
2892                    beginCaptures = { ["1"] = styled("context.grouping", "dictionary.open") },
2893                    endCaptures   = { ["1"] = styled("context.grouping", "dictionary.close") },
2894                    patterns      = {
2895                        {
2896                            ["begin"]     = capturedname .. spaces,
2897                            ["end"]       = "(?=[>])",
2898                            beginCaptures = { ["1"] = styled("context.command", "dictionary.name") },
2899                            patterns = { { include = "#content" } },
2900                        },
2901                    },
2902                },
2903
2904                xref = {
2905                    ["begin"] = "xref" .. spaces,
2906                    ["end"]   = "(?=[^0-9])",
2907                    captures  = {
2908                        ["0"] = styled("context.keyword", "xref.1"),
2909                    },
2910                    patterns = {
2911                        {
2912                            ["begin"] = capturedcardinal .. spaces .. capturedcardinal .. spaces,
2913                            ["end"]   = "(?=[^0-9])",
2914                            captures  = {
2915                                ["1"] = styled("context.number", "xref.2"),
2916                                ["2"] = styled("context.number", "xref.3"),
2917                            },
2918                            patterns = {
2919                                {
2920                                    ["begin"] = capturedcardinal .. spaces .. capturedcardinal .. spaces .. "([fn])" .. spaces,
2921                                    ["end"]   = "(?=.)",
2922                                    captures = {
2923                                        ["1"] = styled("context.number", "xref.4"),
2924                                        ["2"] = styled("context.number", "xref.5"),
2925                                        ["3"] = styled("context.keyword", "xref.6"),
2926                                    },
2927                                },
2928                            },
2929                        },
2930                    },
2931                },
2932
2933                startxref = {
2934                    ["begin"] = "startxref" .. spaces,
2935                    ["end"]   = "(?=[^0-9])",
2936                    captures  = {
2937                        ["0"] = styled("context.keyword", "startxref.1"),
2938                    },
2939                    patterns = {
2940                        {
2941                            ["begin"] = capturedcardinal .. spaces,
2942                            ["end"]   = "(?=.)",
2943                            captures  = {
2944                                ["1"] = styled("context.number", "startxref.2"),
2945                            },
2946                        },
2947                    },
2948                },
2949
2950                trailer = {
2951                    name  = style("context.keyword", "trailer"),
2952                    match = "trailer",
2953                },
2954
2955                stream = {
2956                    ["begin"]     = streamopen,
2957                    ["end"]       = streamclose,
2958                    beginCaptures = { ["0"] = styled("context.keyword", "stream.open") },
2959                    endCaptures   = { ["0"] = styled("context.keyword", "stream.close") },
2960                },
2961
2962            },
2963
2964            patterns = {
2965                include("#object"),
2966                include("#comment"),
2967                include("#trailer"),
2968                include("#dictionary"), -- cheat: trailer dict
2969                include("#startxref"),
2970                include("#xref"),
2971            },
2972
2973        }
2974
2975    end
2976
2977    -- The JSON lexer. I don't want to spend time on (and mess up the lexer) with
2978    -- some ugly multistage key/value parser so we just assume that the key is on
2979    -- the same line as the colon and the value. It looks bad otherwise anyway.
2980
2981    do
2982
2983        local spaces             = "\\s*"
2984        local separator          = "\\,"
2985        local becomes            = "\\:"
2986
2987        local arrayopen          = "\\["
2988        local arrayclose         = "\\]"
2989
2990        local hashopen           = "\\{"
2991        local hashclose          = "\\}"
2992
2993        local stringopen         = "\""
2994        local stringcontent      = "(?:\\\\\"|[^\"])*"
2995        local stringclose        = stringopen
2996
2997        local reserved           = oneof { "true", "false", "null" }
2998
2999        local hexnumber          = "[\\-]?0[xX][A-Fa-f0-9]+(\\.[A-Fa-f0-9]+)?([eEpP]\\-?[A-Fa-f0-9]+)?"
3000        local decnumber          = "[\\-]?[0-9]+(\\.[0-9]+)?([eEpP]\\-?[0-9]+)?"
3001
3002        local capturedarrayopen  = capture(arrayopen)
3003        local capturedarrayclose = capture(arrayclose)
3004        local capturedhashopen   = capture(hashopen)
3005        local capturedhashclose  = capture(hashclose)
3006
3007        local capturedreserved   = capture(reserved)
3008        local capturedbecomes    = capture(becomes)
3009        local capturedseparator  = capture(separator)
3010        local capturedstring     = capture(stringopen) .. capture(stringcontent) .. capture(stringclose)
3011        local capturedhexnumber  = capture(hexnumber)
3012        local captureddecnumber  = capture(decnumber)
3013
3014        local style, styled      = styler("json")
3015
3016        registerlexer {
3017
3018            category    = "json",
3019            description = "ConTeXt JSON",
3020            suffixes    = { "json" },
3021            version     = "1.0.0",
3022
3023            setup       = configuration {
3024                pairs = {
3025                    { "{", "}" },
3026                    { "[", "]" },
3027                },
3028            },
3029
3030            repository = {
3031
3032                separator = {
3033                    name  = style("context.operator","separator"),
3034                    match = spaces .. capturedseparator,
3035                },
3036
3037                reserved = {
3038                    name  = style("context.primitive","reserved"),
3039                    match = spaces .. capturedreserved,
3040                },
3041
3042                hexnumber = {
3043                    name  = style("context.number","hex"),
3044                    match = spaces .. capturedhexnumber,
3045                },
3046
3047                decnumber = {
3048                    name  = style("context.number","dec"),
3049                    match = spaces .. captureddecnumber,
3050                },
3051
3052                string = {
3053                    match    = spaces .. capturedstring,
3054                    captures = {
3055                        ["1"] = styled("context.quote","string.open"),
3056                        ["2"] = styled("context.string","string.text"),
3057                        ["3"] = styled("context.quote","string.close"),
3058                    },
3059                },
3060
3061                kv_reserved = {
3062                    match    = capturedstring .. spaces .. capturedbecomes .. spaces .. capturedreserved,
3063                    captures = {
3064                        ["1"] = styled("context.quote",    "reserved.key.open"),
3065                        ["2"] = styled("context.text",     "reserved.key.text"),
3066                        ["3"] = styled("context.quote",    "reserved.key.close"),
3067                        ["4"] = styled("context.operator", "reserved.becomes"),
3068                        ["5"] = styled("context.primitive","reserved.value"),
3069                    }
3070                },
3071
3072                kv_hexnumber = {
3073                    match = capturedstring .. spaces .. capturedbecomes .. spaces .. capturedhexnumber,
3074                    captures = {
3075                        ["1"] = styled("context.quote",   "hex.key.open"),
3076                        ["2"] = styled("context.text",    "hex.key.text"),
3077                        ["3"] = styled("context.quote",   "hex.key.close"),
3078                        ["4"] = styled("context.operator","hex.becomes"),
3079                        ["5"] = styled("context.number",  "hex.value"),
3080                    }
3081                },
3082
3083                kv_decnumber = {
3084                    match = capturedstring .. spaces .. capturedbecomes .. spaces .. captureddecnumber,
3085                    captures = {
3086                        ["1"] = styled("context.quote",   "dec.key.open"),
3087                        ["2"] = styled("context.text",    "dec.key.text"),
3088                        ["3"] = styled("context.quote",   "dec.key.close"),
3089                        ["4"] = styled("context.operator","dec.becomes"),
3090                        ["5"] = styled("context.number",  "dec.value"),
3091                    }
3092                },
3093
3094                kv_string = {
3095                    match = capturedstring .. spaces .. capturedbecomes .. spaces .. capturedstring,
3096                    captures = {
3097                        ["1"] = styled("context.quote",   "string.key.open"),
3098                        ["2"] = styled("context.text",    "string.key.text"),
3099                        ["3"] = styled("context.quote",   "string.key.close"),
3100                        ["4"] = styled("context.operator","string.becomes"),
3101                        ["5"] = styled("context.quote",   "string.value.open"),
3102                        ["6"] = styled("context.string",  "string.value.text"),
3103                        ["7"] = styled("context.quote",   "string.value.close"),
3104                    },
3105                },
3106
3107                kv_array = {
3108                    ["begin"]     = capturedstring .. spaces .. capturedbecomes .. spaces .. capturedarrayopen,
3109                    ["end"]       = arrayclose,
3110                    beginCaptures = {
3111                        ["1"] = styled("context.quote",   "array.key.open"),
3112                        ["2"] = styled("context.text",    "array.key.text"),
3113                        ["3"] = styled("context.quote",   "array.key.close"),
3114                        ["4"] = styled("context.operator","array.becomes"),
3115                        ["5"] = styled("context.grouping","array.value.open")
3116                    },
3117                    endCaptures   = {
3118                        ["0"] = styled("context.grouping","array.value.close")
3119                    },
3120                    patterns      = { include("#content") },
3121                },
3122
3123                kv_hash = {
3124                    ["begin"]     = capturedstring .. spaces .. capturedbecomes .. spaces .. capturedhashopen,
3125                    ["end"]       = hashclose,
3126                    beginCaptures = {
3127                        ["1"] = styled("context.quote",   "hash.key.open"),
3128                        ["2"] = styled("context.text",    "hash.key.text"),
3129                        ["3"] = styled("context.quote",   "hash.key.close"),
3130                        ["4"] = styled("context.operator","hash.becomes"),
3131                        ["5"] = styled("context.grouping","hash.value.open")
3132                    },
3133                    endCaptures   = {
3134                        ["0"] = styled("context.grouping","hash.value.close")
3135                    },
3136                    patterns      = { include("#kv_content") },
3137                },
3138
3139                content = {
3140                    patterns = {
3141                        include("#string"),
3142                        include("#hexnumber"),
3143                        include("#decnumber"),
3144                        include("#reserved"),
3145                        include("#hash"),
3146                        include("#array"),
3147                        include("#separator"),
3148                    },
3149                },
3150
3151                kv_content = {
3152                    patterns = {
3153                        include("#kv_string"),
3154                        include("#kv_hexnumber"),
3155                        include("#kv_decnumber"),
3156                        include("#kv_reserved"),
3157                        include("#kv_hash"),
3158                        include("#kv_array"),
3159                        include("#separator"),
3160                    },
3161                },
3162
3163                array = {
3164                    ["begin"]     = arrayopen,
3165                    ["end"]       = arrayclose,
3166                    beginCaptures = { ["0"] = styled("context.grouping","array.open") },
3167                    endCaptures   = { ["0"] = styled("context.grouping","array.close") },
3168                    patterns      = { include("#content") },
3169                },
3170
3171                hash = {
3172                    ["begin"]     = hashopen,
3173                    ["end"]       = hashclose,
3174                    beginCaptures = { ["0"] = styled("context.grouping","hash.open") },
3175                    endCaptures   = { ["0"] = styled("context.grouping","hash.close") },
3176                    patterns      = { include("#kv_content") },
3177                },
3178
3179            },
3180
3181            patterns = {
3182                include("#content"),
3183            },
3184
3185        }
3186
3187    end
3188
3189    savepackage()
3190
3191end
3192
3193function scripts.vscode.start()
3194    local path = locate()
3195    if path then
3196        local codecmd = environment.arguments.program or "code" -- can be codium
3197     -- local command = 'start "vs code context" ' .. codecmd .. ' --reuse-window --ignore-gpu-blacklist --extensions-dir "' .. path .. '" --install-extension context'
3198        local command = 'start "vs code context" ' .. codecmd .. ' --reuse-window --extensions-dir "' .. path .. '" --install-extension context'
3199        report("running command: %s",command)
3200        os.execute(command)
3201    end
3202end
3203
3204if environment.arguments.generate then
3205    scripts.vscode.generate()
3206elseif environment.arguments.start then
3207    scripts.vscode.start()
3208elseif environment.arguments.exporthelp then
3209    application.export(environment.arguments.exporthelp,environment.files[1])
3210else
3211    application.help()
3212end
3213
3214scripts.vscode.generate([[t:/vscode/data/context/extensions]])
3215