luasocket: get.lua patch


Forum: Programming and Scripting
Topic: luasocket: get.lua patch
started by: ^thehatsrule^
Posted by ^thehatsrule^ on June 23 2008,04:58
This is forked off from < this thread >.  The recent events reminded me of lua... and I took the suggestion from MaxiJavi in that thread and decided to take a look at it - and then I ended up modifying it.  I added resuming downloads and to use the basename of the url as the filename by default if arg2 is not specified (I messed up my terminal many times since the original prints to stdout by default). This code is not robust - it just mostly shows that it can work (hopefully).  It didn't take a lot of time, but I did get hung up on the ftp resume part - "fixed" after some trial and error.  Does anyone know if there's a real fix to this, or if this is a bug in the platform?

The original can be found in [luasocket]/etc/get.lua. I've only tested this on local httpd/ftpd's and a bit on the DSL download mirrors.  But I haven't tried it on the lua environment on the latest 4.4.x yet.

Going through this made me think that the use of ltn12 is pretty neat.

get2-003.lua:
Code Sample
-----------------------------------------------------------------------------
-- Little program to download files from URLs
-- LuaSocket sample files
-- Author: Diego Nehab
-- Extended to add resume by ^thehatsrule^
-- RCS ID: $Id: get.lua,v 1.25 2007/03/12 04:08:40 diego Exp $
-----------------------------------------------------------------------------
local socket = require("socket")
local http = require("socket.http")
local ftp = require("socket.ftp")
local url = require("socket.url")
local ltn12 = require("ltn12")
local lfs = require("lfs")

-- formats a number of seconds into human readable form
function nicetime(s)
   local l = "s"
   if s > 60 then
       s = s / 60
       l = "m"
       if s > 60 then
           s = s / 60
           l = "h"
           if s > 24 then
               s = s / 24
               l = "d" -- hmmm
           end
       end
   end
   if l == "s" then return string.format("%5.0f%s", s, l)
   else return string.format("%5.2f%s", s, l) end
end

-- formats a number of bytes into human readable form
function nicesize(b)
   local l = "B"
   if b > 1024 then
       b = b / 1024
       l = "KB"
       if b > 1024 then
           b = b / 1024
           l = "MB"
           if b > 1024 then
               b = b / 1024
               l = "GB" -- hmmm
           end
       end
   end
   return string.format("%7.2f%2s", b, l)
end

-- returns a string with the current state of the download
local remaining_s = "%s received, %s/s throughput, %2.0f%% done, %s remaining"
local elapsed_s =   "%s received, %s/s throughput, %s elapsed                "
function gauge(got, delta, size)
   local rate = got / delta
   if size and size >= 1 then
       return string.format(remaining_s, nicesize(got),  nicesize(rate),
           100*got/size, nicetime((size-got)/rate))
   else
       return string.format(elapsed_s, nicesize(got),
           nicesize(rate), nicetime(delta))
   end
end

-- creates a new instance of a receive_cb that saves to disk
-- kind of copied from luasocket's manual callback examples
function stats(size)
   local start = socket.gettime()
   local last = start
   local got = 0
   return function(chunk)
       -- elapsed time since start
       local current = socket.gettime()
       if chunk then
           -- total bytes received
           got = got + string.len(chunk)  
           -- not enough time for estimate
           if current - last > 1 then
               io.stderr:write("\r", gauge(got, current - start, size))
               io.stderr:flush()
               last = current
           end
       else
           -- close up
           io.stderr:write("\r", gauge(got, current - start), "\n")
       end
       return chunk
   end
end

-- determines the size of a http file
function gethttpsize(u)
   local r, c, h = http.request {method = "HEAD", url = u}
   if c == 200 then
       return tonumber(h["content-length"])
   end
end

-- downloads a file using the http protocol
function getbyhttp(u, file, sz)
   local save = ltn12.sink.file(file or io.stdout)
   local size2get = gethttpsize(u)
   if not size2get then
       io.stderr:write("Error contacting remote host.")
       --if not sz then do_something() end
       os.exit(0)
   end
   if size2get == sz then
       print("File already completed.")
       os.exit(0)
   end
   -- only print feedback if output is not stdout
   if file then save = ltn12.sink.chain(stats(size2get), save) end
   local hdrs
   if sz then
       -- resume header
       hdrs = { ["Range"] = "bytes=" .. sz .."-" }
   end
   local r, c, h, s = http.request {url = u, sink = save, headers = hdrs }
   -- todo: check for 206; fallback?
   if c ~= 200 then io.stderr:write(s or c, "\n") end
end

-- downloads a file using the ftp protocol
function getbyftp(u, file, sz)
   local save = ltn12.sink.file(file or io.stdout)
   -- only print feedback if output is not stdout
   -- and we don't know how big the file is
   -- todo: can use SIZE file to get file size, but is not RFC
   --       would probably be easier to do 2 ftp sessions
   if file then save = ltn12.sink.chain(stats(), save) end
   local gett = url.parse(u)
   gett.sink = save
   gett.type = "i"
   if sz then
       -- try to resume; fallback?
       --  filler command (see below) + restart offset + retrieve file
       --  bug? having REST first will return 350, ftp.get returns early
       gett.command = 'SYST' .. '\n' .. 'REST ' .. sz .. '\nRETR'
   end
   local ret, err = ftp.get(gett)
   if err then print(err) end
end

-- determines the scheme
function getscheme(u)
   -- this is an heuristic to solve a common invalid url poblem
   if not string.find(u, "//") then u = "//" .. u end
   local parsed = url.parse(u, {scheme = "http"})
   return parsed.scheme
end

-- gets a file either by http or ftp, saving as <name>
-- todo: check filesize etc before opening file
function get(u, name)
   if not name then
       -- use the basename from the url, assuming no spaces
       -- better to exec `basename` instead?
       name = string.gsub(u, '.*/', "")
       -- todo: change this
       if not name then
           name = os.tmpname()
           print ("Using " .. name .. " to store data")
       end
   end
   local sz = lfs.attributes(name, "size")
   local mode
   if sz then
       print ("File " .. name .. " found with a size of " .. sz)
       print ("Trying to resume ...")
       mode = "a+b"
   else
       mode = "wb"
   end
   local fout = name and io.open(name, mode)
   local scheme = getscheme(u)
   if scheme == "ftp" then getbyftp(u, fout, sz)
   elseif scheme == "http" then getbyhttp(u, fout, sz)
   else print("unknown scheme" .. scheme) end
end

-- main program
arg = arg or {}
if table.getn(arg) < 1 then
   io.write("Usage:\n  lua get.lua <remote-url> [<local-file>]\n")
   os.exit(1)
else get(arg[1], arg[2]) end
Posted by mikshaw on June 25 2008,19:05
Speaking as someone who knows nothing about luasocket (and has not tried this script yet)...

Do you have any thoughts about how well this might work as a replacement for wget in scripts such as mydslBrowser?  Any idea about including a progress bar? Do you know what license this script has?
Posted by ^thehatsrule^ on June 25 2008,21:45
As it stands, it could work "well enough" - I've already seen some issues with 301 redirection (even if you specify http.request to use it).

As for a progress bar, do you mean the one from fltk?  Currently this is console-only.  I haven't tried any kind of modification to the stats yet though, but should be doable.  And you'd probably want the ftp SIZE operation to be implemented.

iirc these example scripts were released under the same MIT license of the project which is the same as Lua itself.  I could double check, if you'd like.
Powered by Ikonboard 3.1.2a
Ikonboard © 2001 Jarvis Entertainment Group, Inc.