Source Browser ?

1-- original file : http.lua
2-- for more into : see util-soc.lua
3
4local tostring, tonumber, setmetatable, next, type = tostring, tonumber, setmetatable, next, type
5local find, lower, format, gsub, match  = string.find, string.lower, string.format, string.gsub, string.match
6local concat = table.concat
7
8local socket  = socket         or require("socket")
9local url     = socket.url     or require("socket.url")
10local ltn12   = ltn12          or require("ltn12")
11local mime    = mime           or require("mime")
12local headers = socket.headers or require("socket.headers")
13
14local normalizeheaders = headers.normalize
15
16local parseurl         = url.parse
17local buildurl         = url.build
18local absoluteurl      = url.absolute
19local unescapeurl      = url.unescape
20
21local skipsocket       = socket.skip
22local sinksocket       = socket.sink
23local sourcesocket     = socket.source
24local trysocket        = socket.try
25local tcpsocket        = socket.tcp
26local newtrysocket     = socket.newtry
27local protectsocket    = socket.protect
28
29local emptysource      = ltn12.source.empty
30local stringsource     = ltn12.source.string
31local rewindsource     = ltn12.source.rewind
32local pumpstep         = ltn12.pump.step
33local pumpall          = ltn12.pump.all
34local sinknull         = ltn12.sink.null
35local sinktable        = ltn12.sink.table
36
37local lowerheaders     = headers.lower
38
39local mimeb64          = mime.b64
40
41-- todo: localize ltn12
42
43local http  = {
44    TIMEOUT   = 60,               -- connection timeout in seconds
45    USERAGENT = socket._VERSION,  -- user agent field sent in request
46}
47
48socket.http = http
49
50local PORT    = 80
51local SCHEMES = {
52    http = true,
53}
54
55-- Reads MIME headers from a connection, unfolding where needed
56
57local function receiveheaders(sock, headers)
58    if not headers then
59        headers = { }
60    end
61    -- get first line
62    local line, err = sock:receive("*l") -- this seems to be wrong!
63    if err then
64        return nil, err
65    end
66    -- headers go until a blank line is found
67    while line ~= "" do
68        -- get field-name and value
69        local name, value = skipsocket(2, find(line, "^(.-):%s*(.*)"))
70        if not (name and value) then
71            return nil, "malformed reponse headers"
72        end
73        name = lower(name)
74        -- get next line (value might be folded)
75        line, err  = sock:receive("*l")
76        if err then
77            return nil, err
78        end
79        -- unfold any folded values
80        while find(line, "^%s") do
81            value = value .. line
82            line  = sock:receive("*l")
83            if err then
84                return nil, err
85            end
86        end
87        -- save pair in table
88        local found = headers[name]
89        if found then
90            value = found .. ", " .. value
91        end
92        headers[name] = value
93    end
94    return headers
95end
96
97-- Extra sources and sinks
98
99socket.sourcet["http-chunked"] = function(sock, headers)
100    return setmetatable (
101        {
102            getfd = function() return sock:getfd() end,
103            dirty = function() return sock:dirty() end,
104        }, {
105            __call = function()
106                local line, err = sock:receive("*l")
107                if err then
108                    return nil, err
109                end
110                local size = tonumber(gsub(line, ";.*", ""), 16)
111                if not size then
112                    return nil, "invalid chunk size"
113                end
114                if size > 0 then
115                    local chunk, err, part = sock:receive(size)
116                    if chunk then
117                        sock:receive("*a")
118                    end
119                    return chunk, err
120                else
121                    headers, err = receiveheaders(sock, headers)
122                    if not headers then
123                        return nil, err
124                    end
125                end
126            end
127        }
128    )
129end
130
131socket.sinkt["http-chunked"] = function(sock)
132    return setmetatable(
133        {
134            getfd = function() return sock:getfd() end,
135            dirty = function() return sock:dirty() end,
136        },
137        {
138            __call = function(self, chunk, err)
139                if not chunk then
140                    chunk = ""
141                end
142                return sock:send(format("%X\r\n%s\r\n",#chunk,chunk))
143            end
144    })
145end
146
147-- Low level HTTP API
148
149local methods = { }
150local mt      = { __index = methods }
151
152local function openhttp(host, port, create)
153    local c = trysocket((create or tcpsocket)())
154    local h = setmetatable({ c = c }, mt)
155    local try = newtrysocket(function() h:close() end)
156    h.try = try
157    try(c:settimeout(http.TIMEOUT))
158    try(c:connect(host, port or PORT))
159    return h
160end
161
162http.open = openhttp
163
164function methods.sendrequestline(self, method, uri)
165    local requestline = format("%s %s HTTP/1.1\r\n", method or "GET", uri)
166    return self.try(self.c:send(requestline))
167end
168
169function methods.sendheaders(self,headers)
170    self.try(self.c:send(normalizeheaders(headers)))
171    return 1
172end
173
174function methods.sendbody(self, headers, source, step)
175    if not source then
176        source = emptysource()
177    end
178    if not step then
179        step = pumpstep
180    end
181    local mode = "http-chunked"
182    if headers["content-length"] then
183        mode = "keep-open"
184    end
185    return self.try(pumpall(source, sinksocket(mode, self.c), step))
186end
187
188function methods.receivestatusline(self)
189    local try    = self.try
190    local status = try(self.c:receive(5))
191    if status ~= "HTTP/" then
192        return nil, status -- HTTP/0.9
193    end
194    status = try(self.c:receive("*l", status))
195    local code = skipsocket(2, find(status, "HTTP/%d*%.%d* (%d%d%d)"))
196    return try(tonumber(code), status)
197end
198
199function methods.receiveheaders(self)
200    return self.try(receiveheaders(self.c))
201end
202
203-- part of request:
204--
205-- Accept-Encoding: gzip
206
207-- part if body:
208--
209-- Content-Encoding: gzip
210-- Vary: Accept-Encoding
211
212function methods.receivebody(self, headers, sink, step)
213    if not sink then
214        sink = sinknull()
215    end
216    if not step then
217        step = pumpstep
218    end
219    local length   = tonumber(headers["content-length"])
220    local encoding = headers["transfer-encoding"] -- shortcut
221    local mode     = "default" -- connection close
222    if encoding and encoding ~= "identity" then
223        mode = "http-chunked"
224    elseif length then
225        mode = "by-length"
226    end
227    --hh: so length can be nil
228    return self.try(pumpall(sourcesocket(mode, self.c, length), sink, step))
229end
230
231function methods.receive09body(self, status, sink, step)
232    local source = rewindsource(sourcesocket("until-closed", self.c))
233    source(status)
234    return self.try(pumpall(source, sink, step))
235end
236
237function methods.close(self)
238    return self.c:close()
239end
240
241-- High level HTTP API
242
243local function adjusturi(request)
244    if not request.proxy and not http.PROXY then
245        request = {
246           path     = trysocket(request.path, "invalid path 'nil'"),
247           params   = request.params,
248           query    = request.query,
249           fragment = request.fragment,
250        }
251    end
252    return buildurl(request)
253end
254
255local function adjustheaders(request)
256    local headers = {
257        ["user-agent"] = http.USERAGENT,
258        ["host"]       = gsub(request.authority, "^.-@", ""),
259        ["connection"] = "close, TE",
260        ["te"]         = "trailers"
261    }
262    local username = request.user
263    local password = request.password
264    if username and password then
265        headers["authorization"] = "Basic " ..  (mimeb64(username .. ":" .. unescapeurl(password)))
266    end
267    local proxy = request.proxy or http.PROXY
268    if proxy then
269        proxy = parseurl(proxy)
270        local username = proxy.user
271        local password = proxy.password
272        if username and password then
273            headers["proxy-authorization"] = "Basic " ..  (mimeb64(username .. ":" .. password))
274        end
275    end
276    local requestheaders = request.headers
277    if requestheaders then
278        headers = lowerheaders(headers,requestheaders)
279    end
280    return headers
281end
282
283-- default url parts
284
285local default = {
286    host   = "",
287    port   = PORT,
288    path   = "/",
289    scheme = "http"
290}
291
292local function adjustrequest(originalrequest)
293    local url     = originalrequest.url
294    local request = url and parseurl(url,default) or { }
295    for k, v in next, originalrequest do
296        request[k] = v
297    end
298    local host = request.host
299    local port = request.port
300    local uri  = request.uri
301    if not host or host == "" then
302        trysocket(nil, "invalid host '" .. tostring(host) .. "'")
303    end
304    if port == "" then
305        request.port = PORT
306    end
307    if not uri or uri == "" then
308        request.uri = adjusturi(request)
309    end
310    request.headers = adjustheaders(request)
311    local proxy = request.proxy or http.PROXY
312    if proxy then
313        proxy        = parseurl(proxy)
314        request.host = proxy.host
315        request.port = proxy.port or 3128
316    end
317    return request
318end
319
320local maxredericts   = 4
321local validredirects = { [301] = true, [302] = true, [303] = true, [307] = true }
322local validmethods   = { [false] = true, GET = true, HEAD = true }
323
324local function shouldredirect(request, code, headers)
325    local location = headers.location
326    if not location then
327        return false
328    end
329    location = gsub(location, "%s", "")
330    if location == "" then
331        return false
332    end
333    local scheme = match(location, "^([%w][%w%+%-%.]*)%:")
334    if scheme and not SCHEMES[scheme] then
335        return false
336    end
337    local method    = request.method
338    local redirect  = request.redirect
339    local redirects = request.nredirects or 0
340    return redirect and validredirects[code] and validmethods[method] and redirects <= maxredericts
341end
342
343local function shouldreceivebody(request, code)
344    if request.method == "HEAD" then
345        return nil
346    end
347    if code == 204 or code == 304 then
348        return nil
349    end
350    if code >= 100 and code < 200 then
351        return nil
352    end
353    return 1
354end
355
356local tredirect, trequest, srequest
357
358tredirect = function(request, location)
359    local result, code, headers, status = trequest {
360        url        = absoluteurl(request.url,location),
361        source     = request.source,
362        sink       = request.sink,
363        headers    = request.headers,
364        proxy      = request.proxy,
365        nredirects = (request.nredirects or 0) + 1,
366        create     = request.create,
367    }
368    if not headers then
369        headers = { }
370    end
371    if not headers.location then
372        headers.location = location
373    end
374    return result, code, headers, status
375end
376
377trequest = function(originalrequest)
378    local request    = adjustrequest(originalrequest)
379    local connection = openhttp(request.host, request.port, request.create)
380    local headers    = request.headers
381    connection:sendrequestline(request.method, request.uri)
382    connection:sendheaders(headers)
383    if request.source then
384        connection:sendbody(headers, request.source, request.step)
385    end
386    local code, status = connection:receivestatusline()
387    if not code then
388        connection:receive09body(status, request.sink, request.step)
389        connection:close()
390        return 1, 200
391    end
392    while code == 100 do
393        headers = connection:receiveheaders()
394        code, status = connection:receivestatusline()
395    end
396    headers = connection:receiveheaders()
397    if shouldredirect(request, code, headers) and not request.source then
398        connection:close()
399        return tredirect(originalrequest,headers.location)
400    end
401    if shouldreceivebody(request, code) then
402        connection:receivebody(headers, request.sink, request.step)
403    end
404    connection:close()
405    return 1, code, headers, status
406end
407
408-- turns an url and a body into a generic request
409
410local function genericform(url, body)
411    local buffer  = { }
412    local request = {
413        url    = url,
414        sink   = sinktable(buffer),
415        target = buffer,
416    }
417    if body then
418        request.source  = stringsource(body)
419        request.method  = "POST"
420        request.headers = {
421            ["content-length"] = #body,
422            ["content-type"]   = "application/x-www-form-urlencoded"
423        }
424    end
425    return request
426end
427
428http.genericform = genericform
429
430srequest = function(url, body)
431    local request = genericform(url, body)
432    local _, code, headers, status = trequest(request)
433    return concat(request.target), code, headers, status
434end
435
436http.request = protectsocket(function(request, body)
437    if type(request) == "string" then
438        return srequest(request, body)
439    else
440        return trequest(request)
441    end
442end)
443
444package.loaded["socket.http"] = http
445
446return http
447