From 1bbc51725e2a3e0979f63a7c4821523b884c1236 Mon Sep 17 00:00:00 2001
From: Daniil Medvedev <medvdanil@gmail.com>
Date: Mon, 20 Jul 2015 12:22:09 +0300
Subject: [PATCH] csv skipping rows and documentation
---
csv.documentation | 35 +++++++++++++++++++++++++++++------
src/lua/csv.lua | 13 +++++++++++--
test/app/csv.result | 4 +---
test/app/csv.test.lua | 16 +++++++---------
4 files changed, 48 insertions(+), 20 deletions(-)
diff --git a/csv.documentation b/csv.documentation
index 3c96cd7ab1..912f78b48d 100644
--- a/csv.documentation
+++ b/csv.documentation
@@ -6,11 +6,14 @@ fio,pathjoin,string
csv,load,table
none,",comma in field", and ""quote""
-Commas in fields must be in quotes
-If there are quotes in a field, it must be double-quotes.
+Commas an linebreaks in fields must be in quotes
+Quotes in fields is repeated two times quote character.
You can set delimiter and quote character:
csv.delimiter = ','
csv.quote = '"'
+Input/output works through readable/writable objects, for example files or sockets.
+Readable object has method read(N), which returns N or less bytes as string.
+Writable object has method write(string), which sends string to output.
csv.iterate = function(readable[, csv_chunk_size])
--@brief parse csv string by string
@@ -29,12 +32,32 @@ Output:
csv load table
none ,comma in field and "quote"
-csv.load = function(readable, csv_chunk_size)
+csv.load = function(readable[, skip_lines, csv_chunk_size])
--@brief parse csv and make table
+--@param skip_lines is number of lines to skip.
--@return table
+If csv file has a header, it may be skipped.
-
-csv.dump = function(t, writable)
+csv.dump = function(t[, writable])
--@brief dumps tuple or table as csv
+--@param t is table or tuple. Fields may be of any type, but it will be converted to string by method tostring(field).
--@param writable must be object with method write(string) like file or socket
---@return there is no writable it returns csv as string
\ No newline at end of file
+--@return there is no writable it returns csv as string
+
+Example:
+ f = require("fio").open("dump.csv", { "O_WRONLY", "O_TRUNC" , "O_CREAT"}, 0x1FF)
+ multiline_header = {{'csv example'}, {'3 numbers per string:'}}
+ csv.dump(multiline_header, f)
+ for i=0,14,3 do
+ t = {i, i + 1, i + 2}
+ s = csv.dump(t, f)
+ end
+dump.csv:
+ csv example
+ 3 numbers per string:
+ 0,1,2
+ 3,4,5
+ 6,7,8
+ 9,10,11
+ 12,13,14
+
diff --git a/src/lua/csv.lua b/src/lua/csv.lua
index 58b0783866..8c0183e7f7 100644
--- a/src/lua/csv.lua
+++ b/src/lua/csv.lua
@@ -69,6 +69,7 @@ local iter = function(csvstate)
log.warn("CSV file has errors")
break
elseif st == ffi.C.CSV_IT_EOF then
+ ffi.C.csv_destroy(csv)
break
end
st = ffi.C.csv_next(it)
@@ -129,12 +130,19 @@ module.iterate = function(readable, csv_chunk_size)
end
--@brief parse csv and make table
+--@param skip_lines is number of lines to skip.
--@return table
-module.load = function(readable, csv_chunk_size)
+module.load = function(readable, skip_lines, csv_chunk_size)
+ skip_lines = skip_lines or 0
csv_chunk_size = csv_chunk_size or 4096
result = {}
+ i = 0
for tup in module.iterate(readable, csv_chunk_size) do
- table.insert(result, tup)
+ if i < skip_lines then
+ i = i + 1
+ else
+ table.insert(result, tup)
+ end
end
return result
@@ -177,6 +185,7 @@ module.dump = function(t, writable)
end
writable:write('\n')
end
+ ffi.C.csv_destroy(csv)
csv[0].csv_realloc(buf, 0)
if writable.returnstring then
return writable.returnstring
diff --git a/test/app/csv.result b/test/app/csv.result
index fb51b0d409..73f58e2b21 100644
--- a/test/app/csv.result
+++ b/test/app/csv.result
@@ -20,7 +20,6 @@ fio test1:
|ok|
fio test2:
-fio test3:
|1|
|23| |456| |abcac| |'multiword field 4'|
|none| |none| |0|
@@ -30,8 +29,7 @@ fio test3:
|iflag = bit.bor(iflag| |fio.c.flag[ flag ])|
|| || ||
-fio test4:
-|1|
+fio test3:
|23| |456| |abcac| |'multiword field 4'|
|none| |none| |0|
|| || ||
diff --git a/test/app/csv.test.lua b/test/app/csv.test.lua
index fbdb2bd466..3bb6a54bf4 100755
--- a/test/app/csv.test.lua
+++ b/test/app/csv.test.lua
@@ -29,12 +29,12 @@ print(table2str(csv.load(readable)))
print("obj test2:")
readable.v = ", ,\n , \n\n"
readable.i = 0
-print(table2str(csv.load(readable, 1)))
+print(table2str(csv.load(readable, 0, 1)))
print("obj test3:")
readable.v = ", \r\nkp\"\"v"
readable.i = 0
-print(table2str(csv.load(readable, 3)))
+print(table2str(csv.load(readable, 0, 3)))
tmpdir = fio.tempdir()
file1 = fio.pathjoin(tmpdir, 'file.1')
@@ -47,7 +47,7 @@ f:write("123 , 5 , 92 , 0, 0\n" ..
"1, 12 34, 56, \"quote , \", 66\nok")
f:close()
f = fio.open(file1, {'O_RDONLY'})
-print(table2str(csv.load(f,10)))
+print(table2str(csv.load(f,0,10)))
f:close()
@@ -62,15 +62,13 @@ f:write("1\n23,456,abcac,\'multiword field 4\'\n" ..
",,"
)
f:close()
-
-print("fio test3:")
f = fio.open(file2, {'O_RDONLY'})
-print(table2str(csv.load(f, 1))) --symbol by symbol reading
+print(table2str(csv.load(f, 0, 1))) --symbol by symbol reading
f:close()
-print("fio test4:")
+print("fio test3:")
f = fio.open(file2, {'O_RDONLY'})
-print(table2str(csv.load(f, 7))) --7 symbols per chunk
+print(table2str(csv.load(f, 1, 7))) --7 symbols per chunk
f:close()
@@ -87,7 +85,7 @@ f = require("fio").open(file3, { "O_WRONLY", "O_TRUNC" , "O_CREAT"}, 0x1FF)
csv.dump(t, f)
f:close()
f = fio.open(file3, {'O_RDONLY'})
-t2 = csv.load(f, 5)
+t2 = csv.load(f, 0, 5)
f:close()
print("test roundtrip: ")
--
GitLab