Programming and Scripting :: MyDSL info parser



Another thing that concerns me is the size of the list file. It's now 700kb and will only get larger over time.
This is a pretty large download for a dialup user to have every time the app is loaded.
You might want to consider the possibility of storing the file locally, maybe downloading it automatically if it doesn't already exist.
After that, the user could have the option of either automatically downloading it or updating the file at the user's leisure.


I've implemented a kind of update function, although I'm not sure how reliable it is
Code Sample
#!/bin/murgaLua
-- mydsl info browser, 2008 mikshaw
--
-- Changelog (yyyy/mm/dd)
--   2008/01/14: Fixed display of incorrect duplicate info file
--               Check for trailing spaces in title string
--               Added "Update" function
--               Code and interface tweaks
--   2008/01/13: start of project

listfile="/home/dsl/WORK/mydsl_master_list/list.txt"
listurl="ftp://ibiblio.org/pub/Linux/distributions/damnsmall/mydsl/list.txt"

function load_list()
inputfile=io.open(listfile)
if not inputfile then
 print("can't open "..listfile)
 os.exit(1)
end

data=inputfile:read("*a")
inputfile:close()

info={}
info_count=0
-- breaks up each info file into a separate string
for s in string.gmatch(data,"(Location:.-\n)Location:") do
 info_count=info_count+1
 info[info_count]={
 location=string.match(s,"Location:%s*(.-)%s*\n"),
 title=string.match(s,"Title:%s*(.-)%s*\n"),
 text=s
 }
 b_loc:add(info[info_count].location)
end

b_loc:value(0)
b_loc:do_callback()
end

ww=420; wh=360; bh=25; bw=ww/2-5
w=fltk:Fl_Window(ww,wh,"MyDSL Info Browser")

b_loc=fltk:Fl_Choice(5,5,bw,bh)
b_loc:callback(
function()
 while b_files:size()>1 do b_files:remove(0) end
 b_files:redraw()
   for i=1,info_count do
     if info[i].location==b_loc:text() then
       b_files:add(info[i].title)
     end
   end
 b_files:value(0)
 b_files:do_callback()
end
)

b_files=fltk:Fl_Choice(ww/2,5,bw,bh)
b_files:callback(
function()
 for i=1,info_count do
   if info[i].title == b_files:text() and info[i].location == b_loc:text() then
     info_display_buffer:text(info[i].text)
     break
   end
 end
end
)

info_display=fltk:Fl_Text_Display(5,10+bh,ww-10,wh-(20+bh*2))
info_display:textfont(fltk.FL_SCREEN)
info_display:textsize(12)
info_display_buffer=fltk:Fl_Text_Buffer()
info_display:buffer(info_display_buffer)

install=fltk:Fl_Button(5,wh-bh-5,bw,bh,"Install Selected Package")
install:callback(
function()
 os.execute("mydsl-load "..b_files:text().." "..b_loc:text())
end
)

updatedb=fltk:Fl_Button(bw+5,wh-bh-5,bw,bh,"Update Database")
updatedb:callback(
function()
 backup=os.rename(listfile,listfile..".bak")
 if backup then
   wget=os.execute("wget "..listurl.." -O "..listfile)
   if wget==0 then
     while b_loc:size()>1 do b_loc:remove(0) end
     load_list()
   else os.rename(listfile..".bak",listfile)
   end
 end
end
)

load_list()
w:resizable(info_display)
w:show()
Fl:run()
Quote
Another thing that concerns me is the size of the list file. It's now 700kb and will only get larger over time.
This is a pretty large download for a dialup user to have every time the app is loaded.
You might want to consider the possibility of storing the file locally, maybe downloading it automatically if it doesn't already exist.
After that, the user could have the option of either automatically downloading it or updating the file at the user's leisure.
I'm pretty sure a gzip'd version could also be uploaded... and since it's all text, the compression ratio should be good enough to just replace the file if there is a requested update.

I was originally using a shell script for utilizing the list.txt, but I'm glad that a lua version is being developed since it will be more streamlined.
Quote
I'm pretty sure a gzip'd version could also be uploaded... and since it's all text, the compression ratio should be good enough to just replace the file if there is a requested update.
I like that idea. A test on the current file compresses it from 704k to 187k

Also edited the previous post. I made some cosmetic changes to the interface for better use of visual space.
Added text search:
EDIT: made text search case insensitive (string.lower on both the match string and the info text contents)
Code Sample
#!/bin/murgaLua
-- mydsl info browser, 2008 mikshaw
--
-- Changelog (yyyy/mm/dd)
--   2008/01/15: Added text search
--   2008/01/14: Fixed display of incorrect duplicate info file
--               Check for trailing spaces in title string
--               Added "Update" function
--               Code and interface tweaks
--   2008/01/13: start of project

listfile="/home/dsl/WORK/mydsl_master_list/list.txt"
listurl="ftp://ibiblio.org/pub/Linux/distributions/damnsmall/mydsl/list.txt"

function load_list()
inputfile=io.open(listfile)
if not inputfile then
 print("can't open "..listfile)
 os.exit(1)
end

data=inputfile:read("*a")
inputfile:close()

info={}
info_count=0
-- breaks up each info file into a separate string
for s in string.gmatch(data,"(Location:.-\n)Location:") do
 info_count=info_count+1
 info[info_count]={
 location=string.match(s,"Location:%s*(.-)%s*\n"),
 title=string.match(s,"Title:%s*(.-)%s*\n"),
 text=s
 }
 b_loc:add(info[info_count].location)
end

b_loc:value(0)
b_loc:do_callback()
end

ww=420; wh=360; bh=25; bw=ww/2-5
w=fltk:Fl_Window(ww,wh,"MyDSL Info Browser")

tabs=fltk:Fl_Tabs(0,0,ww,wh)

tab1=fltk:Fl_Group(0,0,ww,wh-bh,"browse")
b_loc=fltk:Fl_Choice(5,5,bw,bh)
b_loc:callback(
function()
 while b_files:size()>1 do b_files:remove(0) end
 b_files:redraw()
   for i=1,info_count do
     if info[i].location==b_loc:text() then
       b_files:add(info[i].title)
     end
   end
 b_files:value(0)
 b_files:do_callback()
end
)

b_files=fltk:Fl_Choice(ww/2,5,bw,bh)
b_files:callback(
function()
 for i=1,info_count do
   if info[i].title == b_files:text() and info[i].location == b_loc:text() then
     info_display_buffer:text(info[i].text)
     break
   end
 end
end
)

info_display=fltk:Fl_Text_Display(5,10+bh,ww-10,wh-(20+bh*3))
info_display:textfont(fltk.FL_SCREEN)
info_display:textsize(12)
info_display_buffer=fltk:Fl_Text_Buffer()
info_display:buffer(info_display_buffer)

install=fltk:Fl_Button(5,wh-bh*2-5,bw,bh,"Install Selected Package")
install:callback(
function()
 os.execute("mydsl-load "..b_files:text().." "..b_loc:text())
end
)

updatedb=fltk:Fl_Button(bw+5,wh-bh*2-5,bw,bh,"Update Database")
updatedb:callback(
function()
 backup=os.rename(listfile,listfile..".bak")
 if backup then
   wget=os.execute("wget "..listurl.." -O "..listfile)
   if wget==0 then
     while b_loc:size()>1 do b_loc:remove(0) end
     load_list()
   else os.rename(listfile..".bak",listfile)
   end
 end
end
)
fltk:Fl_End()

tab2=fltk:Fl_Group(0,0,ww,wh-bh,"search text")
search_b=fltk:Fl_Return_Button(5,5,80,bh,"search")
search_b:callback(
function()
 if search_text:value() ~= "" then
   results:clear()
   for i=1,info_count do
     if string.find(string.lower(info[i].text),string.lower(search_text:value()),1,true) then
       results:add(info[i].title.."\t\t"..info[i].location)
     end
   end
 end
end
)
search_text=fltk:Fl_Input(85,5,ww-90,bh)
results=fltk:Fl_Hold_Browser(5,10+bh,ww-10,wh-(20+bh*2))
results:textfont(fltk.FL_SCREEN)
results:textsize(12)
results:callback(
function()
-- show selected info file in the browse tab
if results:value() > 0 then
 get_loc=string.match(results:text(results:value()),"\t\t(.*)")
 get_fname=string.match(results:text(results:value()),"(.*)\t\t")
 for i=0,b_loc:size() do
   if get_loc == b_loc:text(i) then
     b_loc:value(i)
     break
   end
 end
 b_loc:do_callback()
 for i=0,b_files:size() do
   if get_fname == b_files:text(i) then
     b_files:value(i)
     break
   end
 end
 b_files:do_callback()
tabs:value(tab1)
end
end
)
fltk:Fl_End()

load_list()
--w:resizable(info_display)
--w:resizable(results)
w:show()
Fl:run()
Looking good, mikshaw.

I have bzip2 compresed the text database list.txt and renamed it to mydslinfo.bz2

Here are my requests:

1. Use the contents of /opt/.mydsl_dir to store the mydslinfo.bz2 database file.
   I am trying to keep all mydsl related items together.

2. Display a count of items in database and date last updated (downloaded).

3. When initially starting, call get (update) database to download as per item 1,
   as I don't want to place an initial mydslinfo.bz2 in the iso.
Next Page...
original here.