SourceBrowser

s-obsolete-tokens.mkiv /size: 9 Kb last modification: 2021-10-28 13:51
1%D \module
2%D   [       file=toks-tra, % was toks-ini
3%D        version=2007.03.03,
4%D          title=\CONTEXT\ Obsolete Modules,
5%D       subtitle=Tokens,
6%D         author=Hans Hagen,
7%D           date=\currentdate,
8%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
9%C
10%C This module is part of the \CONTEXT\ macro||package and is
11%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
12%C details.
13
14%D The code here used to be in the \type {toks-tra} files which were made in
15%D the real early days of \LUATEX\ and used in articles and presentations
16%D about this engine. Because the code is used in manuals we keep it around
17%D as module.
18
19\writestatus{loading}{ConTeXt Obsolete Modules / Tokens}
20
21%D This used to be in \type {toks-tra.lua}:
22
23\startluacode
24
25if not modules then modules = { } end modules ['s-obsolete-tokens'] = {
26    version   = 1.001,
27    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
28    copyright = "PRAGMA ADE / ConTeXt Development Team",
29    license   = "see context related readme files"
30}
31
32local utfbyte, utfchar, utfvalues = utf.byte, utf.char, utf.values
33local format, gsub = string.format, string.gsub
34local tostring = tostring
35
36local tokens   = tokens
37local token    = token -- the built in one
38local tex      = tex
39local context  = context
40local commands = commands
41
42tokens.collectors     = tokens.collectors or { }
43local collectors      = tokens.collectors
44
45collectors.data       = collectors.data or { }
46local collectordata   = collectors.data
47
48collectors.registered = collectors.registered or { }
49local registered      = collectors.registered
50
51local report          = logs.reporter("tokens","collectors")
52
53-- todo:
54--
55-- register : macros that will be expanded (only for demo-ing)
56-- flush    : print back to tex
57-- test     : fancy stuff
58
59local get_next_token = tokens.scanners.next
60local create_token   = tokens.create
61
62function collectors.install(tag,end_cs)
63    local data, d = { }, 0
64    collectordata[tag] = data
65    end_cs = gsub(end_cs,"^\\","")
66    while true do
67        local t = get_next_token()
68        if t.csname == end_cs then
69            context[end_cs]()
70            return
71        else
72            d = d + 1
73            data[d] = t
74        end
75    end
76end
77
78local simple = { letter = "letter", other_char = "other" }
79
80function collectors.show(data)
81    -- We no longer have methods as we only used (in demos) method a
82    -- so there is no need to burden the core with this. We have a
83    -- different table anyway.
84    if type(data) == "string" then
85        data = collectordata[data]
86    end
87    if not data then
88        return
89    end
90    local ctx_NC       = context.NC
91    local ctx_NR       = context.NR
92    local ctx_bold     = context.bold
93    local ctx_verbatim = context.verbatim
94    local function show(data)
95        for i=1,#data do
96            local tok = data[i]
97            if type(tok) == "table" then
98                show(tok)
99            else
100                local cmdname = tok.cmdname
101                local simple  = simple[cmdname]
102                ctx_NC()
103                ctx_verbatim(simple or cmdname)
104                ctx_NC()
105                ctx_verbatim(simple and utfchar(tok.index) or tok.csname)
106                ctx_NC()
107                if tok.active     then context("active ") end
108                if tok.expandable then context("expandable ") end
109                if tok.protected  then context("protected ") end
110                ctx_NC()
111                ctx_NR()
112            end
113        end
114    end
115    context.starttabulate { "|Tl|Tc|Tl|" }
116        ctx_NC() ctx_bold("cmd")
117        ctx_NC() ctx_bold("meaning")
118        ctx_NC() ctx_bold("properties")
119        ctx_NC() ctx_NR()
120        context.HL()
121        show(data)
122    context.stoptabulate()
123end
124
125local function printlist(data)
126    if data and #data > 0 then
127        report("not supported (yet): printing back to tex")
128    end
129end
130
131tokens.printlist = printlist -- will change to another namespace
132
133function collectors.flush(tag)
134    printlist(collectordata[tag])
135end
136
137function collectors.test(tag,handle)
138    report("not supported (yet): testing")
139end
140
141function collectors.register(name)
142    report("not supported (yet): registering")
143end
144
145-- -- old token code
146--
147--  -- 1 = command, 2 = modifier (char), 3 = controlsequence id
148--
149--  local create       = token.create
150--  local csname_id    = token.csname_id
151--  local command_id   = token.command_id
152--  local command_name = token.command_name
153--  local get_next     = token.get_next
154--  local expand       = token.expand
155--  local csname_name  = token.csname_name
156--
157--  local function printlist(data)
158--      if data and #data > 0 then
159--          callbacks.push('token_filter', function ()
160--             callbacks.pop('token_filter') -- tricky but the nil assignment helps
161--             return data
162--          end)
163--      end
164--  end
165--
166--  tokens.printlist = printlist -- will change to another namespace
167--
168--  function collectors.flush(tag)
169--      printlist(collectordata[tag])
170--  end
171--
172--  function collectors.register(name)
173--      registered[csname_id(name)] = name
174--  end
175--
176--  local call   = command_id("call")
177--  local letter = command_id("letter")
178--  local other  = command_id("other_char")
179--
180--  function collectors.install(tag,end_cs)
181--      local data, d = { }, 0
182--      collectordata[tag] = data
183--      end_cs = gsub(end_cs,"^\\","")
184--      local endcs = csname_id(end_cs)
185--      while true do
186--          local t = get_next()
187--          local a, b = t[1], t[3]
188--          if b == endcs then
189--              context[end_cs]()
190--              return
191--          elseif a == call and registered[b] then
192--              expand()
193--          else
194--              d = d + 1
195--              data[d] = t
196--          end
197--      end
198--  end
199--
200--  function collectors.show(data)
201--      -- We no longer have methods as we only used (in demos) method a
202--      -- so there is no need to burden the core with this.
203--      if type(data) == "string" then
204--          data = collectordata[data]
205--      end
206--      if not data then
207--          return
208--      end
209--      local ctx_NC       = context.NC
210--      local ctx_NR       = context.NR
211--      local ctx_bold     = context.bold
212--      local ctx_verbatim = context.verbatim
213--      context.starttabulate { "|T|Tr|cT|Tr|T|" }
214--      ctx_NC() ctx_bold("cmd")
215--      ctx_NC() ctx_bold("chr")
216--      ctx_NC()
217--      ctx_NC() ctx_bold("id")
218--      ctx_NC() ctx_bold("name")
219--      ctx_NC() ctx_NR()
220--      context.HL()
221--      for i=1,#data do
222--          local token = data[i]
223--          local cmd   = token[1]
224--          local chr   = token[2]
225--          local id    = token[3]
226--          local name  = command_name(token)
227--          ctx_NC()
228--          ctx_verbatim(name)
229--          ctx_NC()
230--          if tonumber(chr) >= 0 then
231--              ctx_verbatim(chr)
232--          end
233--          ctx_NC()
234--          if cmd == letter or cmd == other then
235--              ctx_verbatim(utfchar(chr))
236--          end
237--          ctx_NC()
238--          if id > 0 then
239--              ctx_verbatim(id)
240--          end
241--          ctx_NC()
242--          if id > 0 then
243--              ctx_verbatim(csname_name(token) or "")
244--          end
245--          ctx_NC() ctx_NR()
246--      end
247--      context.stoptabulate()
248--  end
249--
250--  function collectors.test(tag,handle)
251--      local t, w, tn, wn = { }, { }, 0, 0
252--      handle = handle or collectors.defaultwords
253--      local tagdata = collectordata[tag]
254--      for k=1,#tagdata do
255--          local v = tagdata[k]
256--          if v[1] == letter then
257--              wn = wn + 1
258--              w[wn] = v[2]
259--          else
260--              if wn > 0 then
261--                  handle(t,w)
262--                  wn = 0
263--              end
264--              tn = tn + 1
265--              t[tn] = v
266--          end
267--      end
268--      if wn > 0 then
269--          handle(t,w)
270--      end
271--      collectordata[tag] = t
272--  end
273
274-- Interfacing:
275
276commands.collecttokens = collectors.install
277commands.showtokens    = collectors.show
278commands.flushtokens   = collectors.flush
279commands.testtokens    = collectors.test
280commands.registertoken = collectors.register
281
282-- Redundant:
283
284-- function collectors.test(tag)
285--     printlist(collectordata[tag])
286-- end
287
288-- For old times sake:
289
290collectors.dowithwords = collectors.test
291
292-- This is only used in old articles ... will move to a module:
293
294tokens.vbox   = create_token("vbox")
295tokens.hbox   = create_token("hbox")
296tokens.vtop   = create_token("vtop")
297tokens.bgroup = create_token(utfbyte("{"),1)
298tokens.egroup = create_token(utfbyte("}"),2)
299
300tokens.letter = function(chr) return create_token(utfbyte(chr),11) end
301tokens.other  = function(chr) return create_token(utfbyte(chr),12) end
302
303tokens.letters = function(str)
304    local t, n = { }, 0
305    for chr in utfvalues(str) do
306        n = n + 1
307        t[n] = create_token(chr, 11)
308    end
309    return t
310end
311
312function collectors.defaultwords(t,str)
313    if t then
314        local n = #t
315        n = n + 1 ; t[n] = tokens.bgroup
316        n = n + 1 ; t[n] = create_token("red")
317        for i=1,#str do
318            n = n + 1 ; t[n] = tokens.other('*')
319        end
320        n = n + 1 ; t[n] = tokens.egroup
321    end
322end
323
324\stopluacode
325
326%D This used to be in \type {toks-tra.mkiv}:
327
328% used to be: \registerctxluafile{toks-tra}{}
329
330\unprotect
331
332%D Handy for manuals \unknown\ but not really used in practice, so it might
333%D become a runtime loaded module instead.
334
335\unexpanded\def\starttokens  [#1]{\ctxcommand{collecttokens("#1","stoptokens")}}
336           \let\stoptokens        \relax
337\unexpanded\def\flushtokens  [#1]{\ctxcommand{flushtokens("#1")}}
338\unexpanded\def\showtokens   [#1]{\ctxcommand{showtokens("#1")}}
339\unexpanded\def\testtokens   [#1]{\ctxcommand{testtokens("#1")}}
340\unexpanded\def\registertoken  #1{\ctxcommand{registertoken("#1")}}
341
342\let\toks_show\showtokens % we also support the primitive
343
344\unexpanded\def\showtokens{\doifelsenextoptional\toks_show\normalshowtokens}
345
346\protect \endinput
347
Source Browser ?
