Source Browser ?

1%D \module
2%D   [       file=toks-tra, % was toks-ini
3%D        version=2007.03.03,
4%D          title=\CONTEXT\ Obsolete Modules,
5%D       subtitle=Tokens,
6%D         author=Hans Hagen,
7%D           date=\currentdate,
8%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
9%C
10%C This module is part of the \CONTEXT\ macro||package and is
11%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
12%C details.
13
14%D The code here used to be in the \type {toks-tra} files which were made in
15%D the real early days of \LUATEX\ and used in articles and presentations
16%D about this engine. Because the code is used in manuals we keep it around
17%D as module.
18
19\writestatus{loading}{ConTeXt Obsolete Modules / Tokens}
20
21%D This used to be in \type {toks-tra.lua}:
22
23\continuewhenmkivmode
24
25\startluacode
26
27if not modules then modules = { } end modules ['s-obsolete-tokens'] = {
28    version   = 1.001,
29    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
30    copyright = "PRAGMA ADE / ConTeXt Development Team",
31    license   = "see context related readme files"
32}
33
34local utfbyte, utfchar, utfvalues = utf.byte, utf.char, utf.values
35local format, gsub = string.format, string.gsub
36local tostring = tostring
37
38local tokens   = tokens
39local token    = token -- the built in one
40local tex      = tex
41local context  = context
42local commands = commands
43
44tokens.collectors     = tokens.collectors or { }
45local collectors      = tokens.collectors
46
47collectors.data       = collectors.data or { }
48local collectordata   = collectors.data
49
50collectors.registered = collectors.registered or { }
51local registered      = collectors.registered
52
53local report          = logs.reporter("tokens","collectors")
54
55-- todo:
56--
57-- register : macros that will be expanded (only for demo-ing)
58-- flush    : print back to tex
59-- test     : fancy stuff
60
61local get_next_token = tokens.scanners.next
62local create_token   = tokens.create
63
64function collectors.install(tag,end_cs)
65    local data, d = { }, 0
66    collectordata[tag] = data
67    end_cs = gsub(end_cs,"^\\","")
68    while true do
69        local t = get_next_token()
70        if t.csname == end_cs then
71            context[end_cs]()
72            return
73        else
74            d = d + 1
75            data[d] = t
76        end
77    end
78end
79
80local simple = { letter = "letter", other_char = "other" }
81
82function collectors.show(data)
83    -- We no longer have methods as we only used (in demos) method a
84    -- so there is no need to burden the core with this. We have a
85    -- different table anyway.
86    if type(data) == "string" then
87        data = collectordata[data]
88    end
89    if not data then
90        return
91    end
92    local ctx_NC       = context.NC
93    local ctx_NR       = context.NR
94    local ctx_bold     = context.bold
95    local ctx_verbatim = context.verbatim
96    local function show(data)
97        for i=1,#data do
98            local tok = data[i]
99            if type(tok) == "table" then
100                show(tok)
101            else
102                local cmdname = tok.cmdname
103                local simple  = simple[cmdname]
104                ctx_NC()
105                ctx_verbatim(simple or cmdname)
106                ctx_NC()
107                ctx_verbatim(simple and utfchar(tok.index) or tok.csname)
108                ctx_NC()
109                if tok.active     then context("active ") end
110                if tok.expandable then context("expandable ") end
111                if tok.protected  then context("protected ") end
112                ctx_NC()
113                ctx_NR()
114            end
115        end
116    end
117    context.starttabulate { "|Tl|Tc|Tl|" }
118        ctx_NC() ctx_bold("cmd")
119        ctx_NC() ctx_bold("meaning")
120        ctx_NC() ctx_bold("properties")
121        ctx_NC() ctx_NR()
122        context.HL()
123        show(data)
124    context.stoptabulate()
125end
126
127local function printlist(data)
128    if data and #data > 0 then
129        report("not supported (yet): printing back to tex")
130    end
131end
132
133tokens.printlist = printlist -- will change to another namespace
134
135function collectors.flush(tag)
136    printlist(collectordata[tag])
137end
138
139function collectors.test(tag,handle)
140    report("not supported (yet): testing")
141end
142
143function collectors.register(name)
144    report("not supported (yet): registering")
145end
146
147-- -- old token code
148--
149--  -- 1 = command, 2 = modifier (char), 3 = controlsequence id
150--
151--  local create       = token.create
152--  local csname_id    = token.csname_id
153--  local command_id   = token.command_id
154--  local command_name = token.command_name
155--  local get_next     = token.get_next
156--  local expand       = token.expand
157--  local csname_name  = token.csname_name
158--
159--  local function printlist(data)
160--      if data and #data > 0 then
161--          callbacks.push('token_filter', function ()
162--             callbacks.pop('token_filter') -- tricky but the nil assignment helps
163--             return data
164--          end)
165--      end
166--  end
167--
168--  tokens.printlist = printlist -- will change to another namespace
169--
170--  function collectors.flush(tag)
171--      printlist(collectordata[tag])
172--  end
173--
174--  function collectors.register(name)
175--      registered[csname_id(name)] = name
176--  end
177--
178--  local call   = command_id("call")
179--  local letter = command_id("letter")
180--  local other  = command_id("other_char")
181--
182--  function collectors.install(tag,end_cs)
183--      local data, d = { }, 0
184--      collectordata[tag] = data
185--      end_cs = gsub(end_cs,"^\\","")
186--      local endcs = csname_id(end_cs)
187--      while true do
188--          local t = get_next()
189--          local a, b = t[1], t[3]
190--          if b == endcs then
191--              context[end_cs]()
192--              return
193--          elseif a == call and registered[b] then
194--              expand()
195--          else
196--              d = d + 1
197--              data[d] = t
198--          end
199--      end
200--  end
201--
202--  function collectors.show(data)
203--      -- We no longer have methods as we only used (in demos) method a
204--      -- so there is no need to burden the core with this.
205--      if type(data) == "string" then
206--          data = collectordata[data]
207--      end
208--      if not data then
209--          return
210--      end
211--      local ctx_NC       = context.NC
212--      local ctx_NR       = context.NR
213--      local ctx_bold     = context.bold
214--      local ctx_verbatim = context.verbatim
215--      context.starttabulate { "|T|Tr|cT|Tr|T|" }
216--      ctx_NC() ctx_bold("cmd")
217--      ctx_NC() ctx_bold("chr")
218--      ctx_NC()
219--      ctx_NC() ctx_bold("id")
220--      ctx_NC() ctx_bold("name")
221--      ctx_NC() ctx_NR()
222--      context.HL()
223--      for i=1,#data do
224--          local token = data[i]
225--          local cmd   = token[1]
226--          local chr   = token[2]
227--          local id    = token[3]
228--          local name  = command_name(token)
229--          ctx_NC()
230--          ctx_verbatim(name)
231--          ctx_NC()
232--          if tonumber(chr) >= 0 then
233--              ctx_verbatim(chr)
234--          end
235--          ctx_NC()
236--          if cmd == letter or cmd == other then
237--              ctx_verbatim(utfchar(chr))
238--          end
239--          ctx_NC()
240--          if id > 0 then
241--              ctx_verbatim(id)
242--          end
243--          ctx_NC()
244--          if id > 0 then
245--              ctx_verbatim(csname_name(token) or "")
246--          end
247--          ctx_NC() ctx_NR()
248--      end
249--      context.stoptabulate()
250--  end
251--
252--  function collectors.test(tag,handle)
253--      local t, w, tn, wn = { }, { }, 0, 0
254--      handle = handle or collectors.defaultwords
255--      local tagdata = collectordata[tag]
256--      for k=1,#tagdata do
257--          local v = tagdata[k]
258--          if v[1] == letter then
259--              wn = wn + 1
260--              w[wn] = v[2]
261--          else
262--              if wn > 0 then
263--                  handle(t,w)
264--                  wn = 0
265--              end
266--              tn = tn + 1
267--              t[tn] = v
268--          end
269--      end
270--      if wn > 0 then
271--          handle(t,w)
272--      end
273--      collectordata[tag] = t
274--  end
275
276-- Interfacing:
277
278commands.collecttokens = collectors.install
279commands.showtokens    = collectors.show
280commands.flushtokens   = collectors.flush
281commands.testtokens    = collectors.test
282commands.registertoken = collectors.register
283
284-- Redundant:
285
286-- function collectors.test(tag)
287--     printlist(collectordata[tag])
288-- end
289
290-- For old times sake:
291
292collectors.dowithwords = collectors.test
293
294-- This is only used in old articles ... will move to a module:
295
296tokens.vbox   = create_token("vbox")
297tokens.hbox   = create_token("hbox")
298tokens.vtop   = create_token("vtop")
299tokens.bgroup = create_token(utfbyte("{"),1)
300tokens.egroup = create_token(utfbyte("}"),2)
301
302tokens.letter = function(chr) return create_token(utfbyte(chr),11) end
303tokens.other  = function(chr) return create_token(utfbyte(chr),12) end
304
305tokens.letters = function(str)
306    local t, n = { }, 0
307    for chr in utfvalues(str) do
308        n = n + 1
309        t[n] = create_token(chr, 11)
310    end
311    return t
312end
313
314function collectors.defaultwords(t,str)
315    if t then
316        local n = #t
317        n = n + 1 ; t[n] = tokens.bgroup
318        n = n + 1 ; t[n] = create_token("red")
319        for i=1,#str do
320            n = n + 1 ; t[n] = tokens.other('*')
321        end
322        n = n + 1 ; t[n] = tokens.egroup
323    end
324end
325
326\stopluacode
327
328%D This used to be in \type {toks-tra.mkiv}:
329
330% used to be: \registerctxluafile{toks-tra}{}
331
332\unprotect
333
334%D Handy for manuals \unknown\ but not really used in practice, so it might
335%D become a runtime loaded module instead.
336
337\unexpanded\def\starttokens  [#1]{\ctxcommand{collecttokens("#1","stoptokens")}}
338           \let\stoptokens        \relax
339\unexpanded\def\flushtokens  [#1]{\ctxcommand{flushtokens("#1")}}
340\unexpanded\def\showtokens   [#1]{\ctxcommand{showtokens("#1")}}
341\unexpanded\def\testtokens   [#1]{\ctxcommand{testtokens("#1")}}
342\unexpanded\def\registertoken  #1{\ctxcommand{registertoken("#1")}}
343
344\let\toks_show\showtokens % we also support the primitive
345
346\unexpanded\def\showtokens{\doifelsenextoptional\toks_show\normalshowtokens}
347
348\protect \endinput
349