util-soc-imp-http.lua /size: 12 Kb    last modification: 2025-02-21 11:03
1-- original file : http.lua
2-- for more into : see util-soc.lua
3
4local tostring, tonumber, setmetatable, next, type = tostring, tonumber, setmetatable, next, type
5local find, lower, format, gsub, match  = string.find, string.lower, string.format, string.gsub, string.match
6local concat = table.concat
7
8local socket  = socket         or require("socket")
9local url     = socket.url     or require("socket.url")
10local ltn12   = ltn12          or require("ltn12")
11local mime    = mime           or require("mime")
12local headers = socket.headers or require("socket.headers")
13
14local normalizeheaders = headers.normalize
15
16local parseurl         = url.parse
17local buildurl         = url.build
18local absoluteurl      = url.absolute
19local unescapeurl      = url.unescape
20
21local skipsocket       = socket.skip
22local sinksocket       = socket.sink
23local sourcesocket     = socket.source
24local trysocket        = socket.try
25local tcpsocket        = socket.tcp
26local newtrysocket     = socket.newtry
27local protectsocket    = socket.protect
28
29local emptysource      = ltn12.source.empty
30local stringsource     = ltn12.source.string
31local rewindsource     = ltn12.source.rewind
32local pumpstep         = ltn12.pump.step
33local pumpall          = ltn12.pump.all
34local sinknull         = ltn12.sink.null
35local sinktable        = ltn12.sink.table
36
37local lowerheaders     = headers.lower
38
39local mimeb64          = mime.b64
40
41-- todo: localize ltn12
42
43local http  = {
44    TIMEOUT   = 60,               -- connection timeout in seconds
45    USERAGENT = socket._VERSION,  -- user agent field sent in request
46}
47
48socket.http = http
49
50local PORT    = 80
51local SCHEMES = {
52    http = true,
53}
54
55-- Reads MIME headers from a connection, unfolding where needed
56
57local function receiveheaders(sock, headers)
58    if not headers then
59        headers = { }
60    end
61    -- get first line
62    local line, err = sock:receive("*l") -- this seems to be wrong!
63    if err then
64        return nil, err
65    end
66    -- headers go until a blank line is found
67    while line ~= "" do
68        -- get field-name and value
69        local name, value = skipsocket(2, find(line, "^(.-):%s*(.*)"))
70        if not (name and value) then
71            return nil, "malformed response headers"
72        end
73        name = lower(name)
74        -- get next line (value might be folded)
75        line, err  = sock:receive("*l")
76        if err then
77            return nil, err
78        end
79        -- unfold any folded values
80        while find(line, "^%s") do
81            value = value .. line
82            line  = sock:receive("*l")
83            if err then
84                return nil, err
85            end
86        end
87        -- save pair in table
88        local found = headers[name]
89        if found then
90            value = found .. ", " .. value
91        end
92        headers[name] = value
93    end
94    return headers
95end
96
97-- Extra sources and sinks
98
99socket.sourcet["http-chunked"] = function(sock, headers)
100    return setmetatable (
101        {
102            getfd = function() return sock:getfd() end,
103            dirty = function() return sock:dirty() end,
104        }, {
105            __call = function()
106                local line, err = sock:receive("*l")
107                if err then
108                    return nil, err
109                end
110                local size = tonumber(gsub(line, ";.*", ""), 16)
111                if not size then
112                    return nil, "invalid chunk size"
113                end
114                if size > 0 then
115                    local chunk, err, part = sock:receive(size)
116                    if chunk then
117                        sock:receive("*a")
118                    end
119                    return chunk, err
120                else
121                    headers, err = receiveheaders(sock, headers)
122                    if not headers then
123                        return nil, err
124                    end
125                end
126            end
127        }
128    )
129end
130
131socket.sinkt["http-chunked"] = function(sock)
132    return setmetatable(
133        {
134            getfd = function() return sock:getfd() end,
135            dirty = function() return sock:dirty() end,
136        },
137        {
138            __call = function(self, chunk, err)
139                if not chunk then
140                    chunk = ""
141                end
142                return sock:send(format("%X\r\n%s\r\n",#chunk,chunk))
143            end
144    })
145end
146
147-- Low level HTTP API
148
149local methods = { }
150local mt      = { __index = methods }
151
152local function openhttp(host, port, create)
153    local c = trysocket((create or tcpsocket)())
154    local h = setmetatable({ c = c }, mt)
155    local try = newtrysocket(function() h:close() end)
156    h.try = try
157    try(c:settimeout(http.TIMEOUT))
158    try(c:connect(host, port or PORT))
159    return h
160end
161
162http.open = openhttp
163
164function methods.sendrequestline(self, method, uri)
165    local requestline = format("%s %s HTTP/1.1\r\n", method or "GET", uri)
166    return self.try(self.c:send(requestline))
167end
168
169function methods.sendheaders(self,headers)
170    self.try(self.c:send(normalizeheaders(headers)))
171    return 1
172end
173
174function methods.sendbody(self, headers, source, step)
175    if not source then
176        source = emptysource()
177    end
178    if not step then
179        step = pumpstep
180    end
181    local mode = "http-chunked"
182    if headers["content-length"] then
183        mode = "keep-open"
184    end
185    return self.try(pumpall(source, sinksocket(mode, self.c), step))
186end
187
188function methods.receivestatusline(self)
189    local try         = self.try
190    local status, err = try(self.c:receive(5))
191    if status ~= "HTTP/" then
192        if err == "timeout" then
193            return 408
194        else
195            return nil, status -- HTTP/0.9
196        end
197    end
198    status = try(self.c:receive("*l", status))
199    local code = skipsocket(2, find(status, "HTTP/%d*%.%d* (%d%d%d)"))
200    return try(tonumber(code), status)
201end
202
203function methods.receiveheaders(self)
204    return self.try(receiveheaders(self.c))
205end
206
207-- part of request:
208--
209-- Accept-Encoding: gzip
210
211-- part if body:
212--
213-- Content-Encoding: gzip
214-- Vary: Accept-Encoding
215
216function methods.receivebody(self, headers, sink, step)
217    if not sink then
218        sink = sinknull()
219    end
220    if not step then
221        step = pumpstep
222    end
223    local length   = tonumber(headers["content-length"])
224    local encoding = headers["transfer-encoding"] -- shortcut
225    local mode     = "default" -- connection close
226    if encoding and encoding ~= "identity" then
227        mode = "http-chunked"
228    elseif length then
229        mode = "by-length"
230    end
231    --hh: so length can be nil
232    return self.try(pumpall(sourcesocket(mode, self.c, length), sink, step))
233end
234
235function methods.receive09body(self, status, sink, step)
236    local source = rewindsource(sourcesocket("until-closed", self.c))
237    source(status)
238    return self.try(pumpall(source, sink, step))
239end
240
241function methods.close(self)
242    return self.c:close()
243end
244
245-- High level HTTP API
246
247local function adjusturi(request)
248    if not request.proxy and not http.PROXY then
249        request = {
250           path     = trysocket(request.path, "invalid path 'nil'"),
251           params   = request.params,
252           query    = request.query,
253           fragment = request.fragment,
254        }
255    end
256    return buildurl(request)
257end
258
259local function adjustheaders(request)
260    local headers = {
261        ["user-agent"] = http.USERAGENT,
262        ["host"]       = gsub(request.authority, "^.-@", ""),
263        ["connection"] = "close, TE",
264        ["te"]         = "trailers"
265    }
266    local username = request.user
267    local password = request.password
268    if username and password then
269        headers["authorization"] = "Basic " ..  (mimeb64(username .. ":" .. unescapeurl(password)))
270    end
271    local proxy = request.proxy or http.PROXY
272    if proxy then
273        proxy = parseurl(proxy)
274        local username = proxy.user
275        local password = proxy.password
276        if username and password then
277            headers["proxy-authorization"] = "Basic " ..  (mimeb64(username .. ":" .. password))
278        end
279    end
280    local requestheaders = request.headers
281    if requestheaders then
282        headers = lowerheaders(headers,requestheaders)
283    end
284    return headers
285end
286
287-- default url parts
288
289local default = {
290    host   = "",
291    port   = PORT,
292    path   = "/",
293    scheme = "http"
294}
295
296local function adjustrequest(originalrequest)
297    local url     = originalrequest.url
298    local request = url and parseurl(url,default) or { }
299    for k, v in next, originalrequest do
300        request[k] = v
301    end
302    local host = request.host
303    local port = request.port
304    local uri  = request.uri
305    if not host or host == "" then
306        trysocket(nil, "invalid host '" .. tostring(host) .. "'")
307    end
308    if port == "" then
309        request.port = PORT
310    end
311    if not uri or uri == "" then
312        request.uri = adjusturi(request)
313    end
314    request.headers = adjustheaders(request)
315    local proxy = request.proxy or http.PROXY
316    if proxy then
317        proxy        = parseurl(proxy)
318        request.host = proxy.host
319        request.port = proxy.port or 3128
320    end
321    return request
322end
323
324local maxredericts   = 5
325local validredirects = { [301] = true, [302] = true, [303] = true, [307] = true }
326local validmethods   = { [false] = true, GET = true, HEAD = true }
327
328local function shouldredirect(request, code, headers)
329    local location = headers.location
330    if not location then
331        return false
332    end
333    location = gsub(location, "%s", "")
334    if location == "" then
335        return false
336    end
337    local scheme = match(location, "^([%w][%w%+%-%.]*)%:")
338    if scheme and not SCHEMES[scheme] then
339        return false
340    end
341    local method       = request.method
342    local redirect     = request.redirect
343    local redirects    = request.nredirects or 0
344    local maxredirects = request.maxredirects or maxredirects
345    return redirect and validredirects[code] and validmethods[method] and redirects <= maxredirects
346end
347
348local function shouldreceivebody(request, code)
349    if request.method == "HEAD" then
350        return nil
351    end
352    if code == 204 or code == 304 then
353        return nil
354    end
355    if code >= 100 and code < 200 then
356        return nil
357    end
358    return 1
359end
360
361local tredirect, trequest, srequest
362
363tredirect = function(request, location)
364    local result, code, headers, status = trequest {
365        url          = absoluteurl(request.url,location),
366        source       = request.source,
367        sink         = request.sink,
368        headers      = request.headers,
369        proxy        = request.proxy,
370        nredirects   = (request.nredirects or 0) + 1,
371        maxredirects = request.maxredirects or maxredirects,
372        create       = request.create,
373    }
374    if not headers then
375        headers = { }
376    end
377    if not headers.location then
378        headers.location = location
379    end
380    return result, code, headers, status
381end
382
383trequest = function(originalrequest)
384    local request    = adjustrequest(originalrequest)
385    local connection = openhttp(request.host, request.port, request.create)
386    local headers    = request.headers
387    connection:sendrequestline(request.method, request.uri)
388    connection:sendheaders(headers)
389    if request.source then
390        connection:sendbody(headers, request.source, request.step)
391    end
392    local code, status = connection:receivestatusline()
393    if not code then
394        connection:receive09body(status, request.sink, request.step)
395        connection:close()
396        return 1, 200
397    elseif code == 408 then
398        return 1, code
399    end
400    while code == 100 do
401        connection:receiveheaders()
402        code, status = connection:receivestatusline()
403    end
404    headers = connection:receiveheaders()
405    if shouldredirect(request, code, headers) and not request.source then
406        connection:close()
407        return tredirect(originalrequest,headers.location)
408    end
409    if shouldreceivebody(request, code) then
410        connection:receivebody(headers, request.sink, request.step)
411    end
412    connection:close()
413    return 1, code, headers, status
414end
415
416-- turns an url and a body into a generic request
417
418local function genericform(url, body)
419    local buffer  = { }
420    local request = {
421        url    = url,
422        sink   = sinktable(buffer),
423        target = buffer,
424    }
425    if body then
426        request.source  = stringsource(body)
427        request.method  = "POST"
428        request.headers = {
429            ["content-length"] = #body,
430            ["content-type"]   = "application/x-www-form-urlencoded"
431        }
432    end
433    return request
434end
435
436http.genericform = genericform
437
438srequest = function(url, body)
439    local request = genericform(url, body)
440    local _, code, headers, status = trequest(request)
441    return concat(request.target), code, headers, status
442end
443
444http.request = protectsocket(function(request, body)
445    if type(request) == "string" then
446        return srequest(request, body)
447    else
448        return trequest(request)
449    end
450end)
451
452package.loaded["socket.http"] = http
453
454return http
455