1if not modules then modules = { } end modules ['lang-rep'] = {
2 version = 1.001,
3 comment = "companion to lang-rep.mkiv",
4 author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
5 copyright = "PRAGMA ADE / ConTeXt Development Team",
6 license = "see context related readme files"
7}
8
9
10
11
12
13
14
15
16
17
18local type, tonumber, next = type, tonumber, next
19local gmatch, gsub = string.gmatch, string.gsub
20local utfbyte, utfsplit = utf.byte, utf.split
21local P, C, U, Cc, Ct, Cs, lpegmatch = lpeg.P, lpeg.C, lpeg.patterns.utf8character, lpeg.Cc, lpeg.Ct, lpeg.Cs, lpeg.match
22local find = string.find
23
24local zwnj = 0x200C
25local grouped = P("{") * ( Ct((U/utfbyte-P("}"))^1) + Cc(false) ) * P("}")
26local splitter = Ct((
27 #P("{") * (
28 P("{}") / function() return zwnj end
29 + Ct(Cc("discretionary") * grouped * grouped * grouped)
30 + Ct(Cc("noligature") * grouped)
31 )
32 + U/utfbyte
33 )^1)
34
35local stripper = P("{") * Cs((1-P(-2))^0) * P("}") * P(-1)
36
37local trace_replacements = false trackers.register("languages.replacements", function(v) trace_replacements = v end)
38local trace_details = false trackers.register("languages.replacements.details", function(v) trace_details = v end)
39
40local report_replacement = logs.reporter("languages","replacements")
41
42local glyph_code = nodes.nodecodes.glyph
43local glue_code = nodes.nodecodes.glue
44
45local spaceskip_code = nodes.gluecodes.spaceskip
46local xspaceskip_code = nodes.gluecodes.xspaceskip
47
48local nuts = nodes.nuts
49
50local getnext = nuts.getnext
51local getprev = nuts.getprev
52local getattr = nuts.getattr
53local getid = nuts.getid
54local getsubtype = nuts.getsubtype
55local getchar = nuts.getchar
56local isglyph = nuts.isglyph
57
58local setattr = nuts.setattr
59local setlink = nuts.setlink
60local setnext = nuts.setnext
61local setprev = nuts.setprev
62local setchar = nuts.setchar
63local setattrlist = nuts.setattrlist
64
65local insertbefore = nuts.insertbefore
66local insertafter = nuts.insertafter
67local remove_node = nuts.remove
68local copy_node = nuts.copy
69local flushlist = nuts.flushlist
70
71local nodepool = nuts.pool
72local new_disc = nodepool.disc
73
74local texsetattribute = tex.setattribute
75local unsetvalue = attributes.unsetvalue
76
77local enableaction = nodes.tasks.enableaction
78
79local v_reset = interfaces.variables.reset
80
81local implement = interfaces.implement
82
83local processors = typesetters.processors
84local splitprocessor = processors.split
85
86local replacements = languages.replacements or { }
87languages.replacements = replacements
88
89local a_replacements = attributes.private("replacements")
90local a_noligature = attributes.private("noligature")
91
92local lists = { }
93local last = 0
94local trees = { }
95
96table.setmetatableindex(lists,function(lists,name)
97 last = last + 1
98 local list = { }
99 local data = { name = name, list = list, attribute = last }
100 lists[last] = data
101 lists[name] = data
102 trees[last] = list
103 return data
104end)
105
106lists[v_reset].attribute = unsetvalue
107
108
109
110local function add(root,word,replacement)
111 local processor, replacement = splitprocessor(replacement,true)
112 replacement = lpegmatch(stripper,replacement) or replacement
113 local list = utfsplit(word)
114 local size = #list
115 for i=1,size do
116 local l = utfbyte(list[i])
117 if not root[l] then
118 root[l] = { }
119 end
120 if i == size then
121 local special = find(replacement,"{",1,true)
122 local newlist = lpegmatch(splitter,replacement)
123 root[l].final = {
124 word = word,
125 replacement = replacement,
126 processor = processor,
127 oldlength = size,
128 newcodes = newlist,
129 special = special,
130 }
131 end
132 root = root[l]
133 end
134end
135
136function replacements.add(category,word,replacement)
137 local root = lists[category].list
138 if type(word) == "table" then
139 for word, replacement in next, word do
140 add(root,word,replacement)
141 end
142 else
143 add(root,word,replacement or "")
144 end
145end
146
147
148
149function languages.replacements.addlist(category,list)
150 local root = lists[category].list
151 if type(list) == "string" then
152 for new in gmatch(list,"%S+") do
153 local old = gsub(new,"[{}]","")
154
155 add(root,old,new)
156 end
157 else
158 for i=1,#list do
159 local new = list[i]
160 local old = gsub(new,"[{}]","")
161
162 add(root,old,new)
163 end
164 end
165end
166
167local function tonodes(list,template)
168 local head, current
169 for i=1,#list do
170 local new = copy_node(template)
171 setchar(new,list[i])
172 if head then
173 head, current = insertafter(head,current,new)
174 else
175 head, current = new, new
176 end
177 end
178 return head
179end
180
181local is_punctuation = characters.is_punctuation
182
183
184
185
186
187
188
189
190
191local function replace(head,first,last,final,hasspace,overload)
192 local current = first
193 local prefirst = getprev(first) or head
194 local postlast = getnext(last)
195 local oldlength = final.oldlength
196 local newcodes = final.newcodes
197 local newlength = newcodes and #newcodes or 0
198 if trace_replacements then
199 report_replacement("replacing word %a by %a",final.word,final.replacement)
200 end
201 if hasspace or final.special then
202
203
204
205 local prev = getprev(current)
206 local next = getnext(last)
207 local list = current
208 setnext(last)
209 setlink(prev,next)
210 current = prev
211 if not current then
212 head = nil
213 end
214 local i = 1
215 while i <= newlength do
216 local codes = newcodes[i]
217 if type(codes) == "table" then
218 local method = codes[1]
219 if method == "discretionary" then
220 local pre, post, replace = codes[2], codes[3], codes[4]
221 if pre then
222 pre = tonodes(pre,first)
223 end
224 if post then
225 post = tonodes(post,first)
226 end
227 if replace then
228 replace = tonodes(replace,first)
229 end
230
231 local new = new_disc(pre,post,replace)
232 setattrlist(new,first)
233 head, current = insertafter(head,current,new)
234 elseif method == "noligature" then
235
236 local list = codes[2]
237 if list then
238 for i=1,#list do
239 local new = copy_node(first)
240 setchar(new,list[i])
241 setattr(new,a_noligature,1)
242 head, current = insertafter(head,current,new)
243 end
244 else
245 local new = copy_node(first)
246 setchar(new,zwnj)
247 head, current = insertafter(head,current,new)
248 end
249 else
250 report_replacement("unknown method %a",method or "?")
251 end
252 else
253 local new = copy_node(first)
254 setchar(new,codes)
255 head, current = insertafter(head,current,new)
256 end
257 i = i + 1
258 end
259 flushlist(list)
260 elseif newlength == 0 then
261
262 elseif oldlength == newlength then
263 if final.word ~= final.replacement then
264 for i=1,newlength do
265 setchar(current,newcodes[i])
266 current = getnext(current)
267 end
268 end
269
270 elseif oldlength < newlength then
271 for i=1,newlength-oldlength do
272 local n = copy_node(current)
273 setchar(n,newcodes[i])
274 head, current = insertbefore(head,current,n)
275 current = getnext(current)
276 end
277 for i=newlength-oldlength+1,newlength do
278 setchar(current,newcodes[i])
279 current = getnext(current)
280 end
281 else
282 for i=1,oldlength-newlength do
283 head, current = remove_node(head,current,true)
284 end
285 for i=1,newlength do
286 setchar(current,newcodes[i])
287 current = getnext(current)
288 end
289 end
290 if overload then
291 overload(final,getnext(prefirst),getprev(postlast))
292 end
293 return head, postlast
294end
295
296
297
298function replacements.handler(head)
299 local current = head
300 local overload = attributes.applyoverloads
301 local mode = false
302 local wordstart = false
303 local wordend = false
304 local prevend = false
305 local prevfinal = false
306 local tree = false
307 local root = false
308 local hasspace = false
309 while current do
310 local id = getid(current)
311 if id == glyph_code then
312 local a = getattr(current,a_replacements)
313 if a then
314
315 tree = trees[a]
316 if tree then
317 local char = getchar(current)
318 local punc = is_punctuation[char]
319 if mode == "punc" then
320 if not punc then
321 if root then
322 local final = root.final
323 if final then
324 head = replace(head,wordstart,wordend,final,hasspace,overload)
325 elseif prevfinal then
326 head = replace(head,wordstart,prevend,prevfinal,hasspace,overload)
327 end
328 prevfinal = false
329 root = false
330 end
331 mode = "word"
332 end
333 elseif mode == "word" then
334 if punc then
335 if root then
336 local final = root.final
337 if final then
338 head = replace(head,wordstart,wordend,final,hasspace,overload)
339 elseif prevfinal then
340 head = replace(head,wordstart,prevend,prevfinal,hasspace,overload)
341 end
342 prevfinal = false
343 root = false
344 end
345 mode = "punc"
346 end
347 else
348 mode = punc and "punc" or "word"
349 end
350 if root then
351 root = root[char]
352 if root then
353 wordend = current
354 end
355 else
356 if prevfinal then
357 head = replace(head,wordstart,prevend,prevfinal,hasspace,overload)
358 prevfinal = false
359 end
360 root = tree[char]
361 if root then
362 wordstart = current
363 wordend = current
364 prevend = false
365 hasspace = false
366 end
367 end
368 else
369 root= false
370 end
371 else
372 tree = false
373 end
374 current = getnext(current)
375 elseif root then
376 local final = root.final
377 if mode == "word" and id == glue_code then
378 local s = getsubtype(current)
379 if s == spaceskip_code or s == xspaceskip_code then
380 local r = root[32]
381 if r then
382 if not prevend then
383 local f = root.final
384 if f then
385 prevend = wordend
386 prevfinal = f
387 end
388 end
389 wordend = current
390 root = r
391 hasspace = true
392 goto moveon
393 end
394 end
395 end
396 if final then
397 head, current = replace(head,wordstart,wordend,final,hasspace,overload)
398 elseif prevfinal then
399 head, current = replace(head,wordstart,prevend,prevfinal,hasspace,overload)
400 end
401 prevfinal = false
402 root = false
403 ::moveon::
404 current = getnext(current)
405 else
406 current = getnext(current)
407 end
408 end
409 if root then
410 local final = root.final
411 if final then
412 head = replace(head,wordstart,wordend,final,hasspace,overload)
413 elseif prevfinal then
414 head = replace(head,wordstart,prevend,prevfinal,hasspace,overload)
415 end
416 end
417 return head
418end
419
420local enabled = false
421
422function replacements.set(n)
423 if n == v_reset then
424 n = unsetvalue
425 else
426 n = lists[n].attribute
427 if not enabled then
428 enableaction("processors","languages.replacements.handler")
429 if trace_replacements then
430 report_replacement("enabling replacement handler")
431 end
432 enabled = true
433 end
434 end
435 texsetattribute(a_replacements,n)
436end
437
438
439
440implement {
441 name = "setreplacements",
442 actions = replacements.set,
443 arguments = "string"
444}
445
446implement {
447 name = "addreplacements",
448 actions = replacements.add,
449 arguments = "3 strings",
450}
451
452implement {
453 name = "addreplacementslist",
454 actions = replacements.addlist,
455 arguments = "2 strings",
456}
457 |
