Source Browser ?

1-- original file : http.lua
2-- for more into : see util-soc.lua
3
4local tostring, tonumber, setmetatable, next, type = tostring, tonumber, setmetatable, next, type
5local find, lower, format, gsub, match  = string.find, string.lower, string.format, string.gsub, string.match
6local concat = table.concat
7
8local socket  = socket         or require("socket")
9local url     = socket.url     or require("socket.url")
10local ltn12   = ltn12          or require("ltn12")
11local mime    = mime           or require("mime")
12local headers = socket.headers or require("socket.headers")
13
14local normalizeheaders = headers.normalize
15
16local parseurl         = url.parse
17local buildurl         = url.build
18local absoluteurl      = url.absolute
19local unescapeurl      = url.unescape
20
21local skipsocket       = socket.skip
22local sinksocket       = socket.sink
23local sourcesocket     = socket.source
24local trysocket        = socket.try
25local tcpsocket        = socket.tcp
26local newtrysocket     = socket.newtry
27local protectsocket    = socket.protect
28
29local emptysource      = ltn12.source.empty
30local stringsource     = ltn12.source.string
31local rewindsource     = ltn12.source.rewind
32local pumpstep         = ltn12.pump.step
33local pumpall          = ltn12.pump.all
34local sinknull         = ltn12.sink.null
35local sinktable        = ltn12.sink.table
36
37local lowerheaders     = headers.lower
38
39local mimeb64          = mime.b64
40
41-- todo: localize ltn12
42
43local http  = {
44    TIMEOUT   = 60,               -- connection timeout in seconds
45    USERAGENT = socket._VERSION,  -- user agent field sent in request
46}
47
48socket.http = http
49
50local PORT    = 80
51local SCHEMES = {
52    http = true,
53}
54
55-- Reads MIME headers from a connection, unfolding where needed
56
57local function receiveheaders(sock, headers)
58    if not headers then
59        headers = { }
60    end
61    -- get first line
62    local line, err = sock:receive("*l") -- this seems to be wrong!
63    if err then
64        return nil, err
65    end
66    -- headers go until a blank line is found
67    while line ~= "" do
68        -- get field-name and value
69        local name, value = skipsocket(2, find(line, "^(.-):%s*(.*)"))
70        if not (name and value) then
71            return nil, "malformed reponse headers"
72        end
73        name = lower(name)
74        -- get next line (value might be folded)
75        line, err  = sock:receive("*l")
76        if err then
77            return nil, err
78        end
79        -- unfold any folded values
80        while find(line, "^%s") do
81            value = value .. line
82            line  = sock:receive("*l")
83            if err then
84                return nil, err
85            end
86        end
87        -- save pair in table
88        local found = headers[name]
89        if found then
90            value = found .. ", " .. value
91        end
92        headers[name] = value
93    end
94    return headers
95end
96
97-- Extra sources and sinks
98
99socket.sourcet["http-chunked"] = function(sock, headers)
100    return setmetatable (
101        {
102            getfd = function() return sock:getfd() end,
103            dirty = function() return sock:dirty() end,
104        }, {
105            __call = function()
106                local line, err = sock:receive("*l")
107                if err then
108                    return nil, err
109                end
110                local size = tonumber(gsub(line, ";.*", ""), 16)
111                if not size then
112                    return nil, "invalid chunk size"
113                end
114                if size > 0 then
115                    local chunk, err, part = sock:receive(size)
116                    if chunk then
117                        sock:receive("*a")
118                    end
119                    return chunk, err
120                else
121                    headers, err = receiveheaders(sock, headers)
122                    if not headers then
123                        return nil, err
124                    end
125                end
126            end
127        }
128    )
129end
130
131socket.sinkt["http-chunked"] = function(sock)
132    return setmetatable(
133        {
134            getfd = function() return sock:getfd() end,
135            dirty = function() return sock:dirty() end,
136        },
137        {
138            __call = function(self, chunk, err)
139                if not chunk then
140                    chunk = ""
141                end
142                return sock:send(format("%X\r\n%s\r\n",#chunk,chunk))
143            end
144    })
145end
146
147-- Low level HTTP API
148
149local methods = { }
150local mt      = { __index = methods }
151
152local function openhttp(host, port, create)
153    local c = trysocket((create or tcpsocket)())
154    local h = setmetatable({ c = c }, mt)
155    local try = newtrysocket(function() h:close() end)
156    h.try = try
157    try(c:settimeout(http.TIMEOUT))
158    try(c:connect(host, port or PORT))
159    return h
160end
161
162http.open = openhttp
163
164function methods.sendrequestline(self, method, uri)
165    local requestline = format("%s %s HTTP/1.1\r\n", method or "GET", uri)
166    return self.try(self.c:send(requestline))
167end
168
169function methods.sendheaders(self,headers)
170    self.try(self.c:send(normalizeheaders(headers)))
171    return 1
172end
173
174function methods.sendbody(self, headers, source, step)
175    if not source then
176        source = emptysource()
177    end
178    if not step then
179        step = pumpstep
180    end
181    local mode = "http-chunked"
182    if headers["content-length"] then
183        mode = "keep-open"
184    end
185    return self.try(pumpall(source, sinksocket(mode, self.c), step))
186end
187
188function methods.receivestatusline(self)
189    local try    = self.try
190    local status = try(self.c:receive(5))
191    if status ~= "HTTP/" then
192        return nil, status -- HTTP/0.9
193    end
194    status = try(self.c:receive("*l", status))
195    local code = skipsocket(2, find(status, "HTTP/%d*%.%d* (%d%d%d)"))
196    return try(tonumber(code), status)
197end
198
199function methods.receiveheaders(self)
200    return self.try(receiveheaders(self.c))
201end
202
203function methods.receivebody(self, headers, sink, step)
204    if not sink then
205        sink = sinknull()
206    end
207    if not step then
208        step = pumpstep
209    end
210    local length   = tonumber(headers["content-length"])
211    local encoding = headers["transfer-encoding"] -- shortcut
212    local mode     = "default" -- connection close
213    if encoding and encoding ~= "identity" then
214        mode = "http-chunked"
215    elseif length then
216        mode = "by-length"
217    end
218    --hh: so length can be nil
219    return self.try(pumpall(sourcesocket(mode, self.c, length), sink, step))
220end
221
222function methods.receive09body(self, status, sink, step)
223    local source = rewindsource(sourcesocket("until-closed", self.c))
224    source(status)
225    return self.try(pumpall(source, sink, step))
226end
227
228function methods.close(self)
229    return self.c:close()
230end
231
232-- High level HTTP API
233
234local function adjusturi(request)
235    if not request.proxy and not http.PROXY then
236        request = {
237           path     = trysocket(request.path, "invalid path 'nil'"),
238           params   = request.params,
239           query    = request.query,
240           fragment = request.fragment,
241        }
242    end
243    return buildurl(request)
244end
245
246local function adjustheaders(request)
247    local headers = {
248        ["user-agent"] = http.USERAGENT,
249        ["host"]       = gsub(request.authority, "^.-@", ""),
250        ["connection"] = "close, TE",
251        ["te"]         = "trailers"
252    }
253    local username = request.user
254    local password = request.password
255    if username and password then
256        headers["authorization"] = "Basic " ..  (mimeb64(username .. ":" .. unescapeurl(password)))
257    end
258    local proxy = request.proxy or http.PROXY
259    if proxy then
260        proxy = parseurl(proxy)
261        local username = proxy.user
262        local password = proxy.password
263        if username and password then
264            headers["proxy-authorization"] = "Basic " ..  (mimeb64(username .. ":" .. password))
265        end
266    end
267    local requestheaders = request.headers
268    if requestheaders then
269        headers = lowerheaders(headers,requestheaders)
270    end
271    return headers
272end
273
274-- default url parts
275
276local default = {
277    host   = "",
278    port   = PORT,
279    path   = "/",
280    scheme = "http"
281}
282
283local function adjustrequest(originalrequest)
284    local url     = originalrequest.url
285    local request = url and parseurl(url,default) or { }
286    for k, v in next, originalrequest do
287        request[k] = v
288    end
289    local host = request.host
290    local port = request.port
291    local uri  = request.uri
292    if not host or host == "" then
293        trysocket(nil, "invalid host '" .. tostring(host) .. "'")
294    end
295    if port == "" then
296        request.port = PORT
297    end
298    if not uri or uri == "" then
299        request.uri = adjusturi(request)
300    end
301    request.headers = adjustheaders(request)
302    local proxy = request.proxy or http.PROXY
303    if proxy then
304        proxy        = parseurl(proxy)
305        request.host = proxy.host
306        request.port = proxy.port or 3128
307    end
308    return request
309end
310
311local maxredericts   = 4
312local validredirects = { [301] = true, [302] = true, [303] = true, [307] = true }
313local validmethods   = { [false] = true, GET = true, HEAD = true }
314
315local function shouldredirect(request, code, headers)
316    local location = headers.location
317    if not location then
318        return false
319    end
320    location = gsub(location, "%s", "")
321    if location == "" then
322        return false
323    end
324    local scheme = match(location, "^([%w][%w%+%-%.]*)%:")
325    if scheme and not SCHEMES[scheme] then
326        return false
327    end
328    local method    = request.method
329    local redirect  = request.redirect
330    local redirects = request.nredirects or 0
331    return redirect and validredirects[code] and validmethods[method] and redirects <= maxredericts
332end
333
334local function shouldreceivebody(request, code)
335    if request.method == "HEAD" then
336        return nil
337    end
338    if code == 204 or code == 304 then
339        return nil
340    end
341    if code >= 100 and code < 200 then
342        return nil
343    end
344    return 1
345end
346
347local tredirect, trequest, srequest
348
349tredirect = function(request, location)
350    local result, code, headers, status = trequest {
351        url        = absoluteurl(request.url,location),
352        source     = request.source,
353        sink       = request.sink,
354        headers    = request.headers,
355        proxy      = request.proxy,
356        nredirects = (request.nredirects or 0) + 1,
357        create     = request.create,
358    }
359    if not headers then
360        headers = { }
361    end
362    if not headers.location then
363        headers.location = location
364    end
365    return result, code, headers, status
366end
367
368trequest = function(originalrequest)
369    local request    = adjustrequest(originalrequest)
370    local connection = openhttp(request.host, request.port, request.create)
371    local headers    = request.headers
372    connection:sendrequestline(request.method, request.uri)
373    connection:sendheaders(headers)
374    if request.source then
375        connection:sendbody(headers, request.source, request.step)
376    end
377    local code, status = connection:receivestatusline()
378    if not code then
379        connection:receive09body(status, request.sink, request.step)
380        return 1, 200
381    end
382    while code == 100 do
383        headers = connection:receiveheaders()
384        code, status = connection:receivestatusline()
385    end
386    headers = connection:receiveheaders()
387    if shouldredirect(request, code, headers) and not request.source then
388        connection:close()
389        return tredirect(originalrequest,headers.location)
390    end
391    if shouldreceivebody(request, code) then
392        connection:receivebody(headers, request.sink, request.step)
393    end
394    connection:close()
395    return 1, code, headers, status
396end
397
398-- turns an url and a body into a generic request
399
400local function genericform(url, body)
401    local buffer  = { }
402    local request = {
403        url    = url,
404        sink   = sinktable(buffer),
405        target = buffer,
406    }
407    if body then
408        request.source  = stringsource(body)
409        request.method  = "POST"
410        request.headers = {
411            ["content-length"] = #body,
412            ["content-type"]   = "application/x-www-form-urlencoded"
413        }
414    end
415    return request
416end
417
418http.genericform = genericform
419
420srequest = function(url, body)
421    local request = genericform(url, body)
422    local _, code, headers, status = trequest(request)
423    return concat(request.target), code, headers, status
424end
425
426http.request = protectsocket(function(request, body)
427    if type(request) == "string" then
428        return srequest(request, body)
429    else
430        return trequest(request)
431    end
432end)
433
434package.loaded["socket.http"] = http
435
436return http
437