From 133405fff878f8ff16caf6776a8407f27c4d6182 Mon Sep 17 00:00:00 2001
From: Alexandr Lyapunov <a.lyapunov@corp.mail.ru>
Date: Wed, 15 Jun 2016 11:50:50 +0300
Subject: [PATCH] fixed gh-1509, fixed gh-1210 : problems with csv parser
---
src/lib/csv/csv.c | 7 ++++---
test/app-tap/csv.result | 4 +++-
test/app-tap/csv.test.lua | 18 +++++++++++++++++-
test/unit/csv.c | 32 ++++++++++++++++++++++++++++++++
test/unit/csv.result | 4 ++++
5 files changed, 60 insertions(+), 5 deletions(-)
diff --git a/src/lib/csv/csv.c b/src/lib/csv/csv.c
index e2d85d79c4..46e32bfa92 100644
--- a/src/lib/csv/csv.c
+++ b/src/lib/csv/csv.c
@@ -328,13 +328,14 @@ csv_next(struct csv_iterator *it)
return CSV_IT_ERROR;
it->buf_begin = tail;
- /* bufp == NULL means end of line */
- if (it->csv->bufp == NULL)
- return CSV_IT_EOL;
if (tail == it->buf_end) /* buffer is empty */
return CSV_IT_NEEDMORE;
+ /* bufp == NULL means end of line */
+ if (it->csv->bufp == NULL)
+ return CSV_IT_EOL;
+
/* return field via iterator structure */
it->field = it->csv->buf;
it->field_len = it->csv->bufp - it->csv->buf;
diff --git a/test/app-tap/csv.result b/test/app-tap/csv.result
index 8b6f866b04..1596c0d10b 100644
--- a/test/app-tap/csv.result
+++ b/test/app-tap/csv.result
@@ -1,5 +1,5 @@
TAP version 13
-1..9
+1..11
ok - obj test1
ok - obj test2
ok - obj test3
@@ -9,3 +9,5 @@ ok - fio test3
ok - test roundtrip
ok - test load(dump(t))
ok - final comma
+ok - gh-1210 (1)
+ok - gh-1210 (2)
diff --git a/test/app-tap/csv.test.lua b/test/app-tap/csv.test.lua
index a20024ae6f..472a8d23e4 100755
--- a/test/app-tap/csv.test.lua
+++ b/test/app-tap/csv.test.lua
@@ -36,7 +36,7 @@ local test6_ans = "|23|\t|456|\t|abcac|\t|'multiword field 4'|\t\n|none|" ..
"lag[ flag ])|\t\n||\t||\t||\t\n"
test = tap.test("csv")
-test:plan(9)
+test:plan(11)
readable = {}
readable.read = myread
@@ -56,6 +56,7 @@ tmpdir = fio.tempdir()
file1 = fio.pathjoin(tmpdir, 'file.1')
file2 = fio.pathjoin(tmpdir, 'file.2')
file3 = fio.pathjoin(tmpdir, 'file.3')
+file4 = fio.pathjoin(tmpdir, 'file.4')
local f = fio.open(file1, { 'O_WRONLY', 'O_TRUNC', 'O_CREAT' }, 0777)
f:write("123 , 5 , 92 , 0, 0\n" ..
@@ -109,7 +110,22 @@ test:is(table2str(t), table2str(csv.load(csv.dump(t))), "test load(dump(t))")
test:is(table2str(csv.load('a,b,c,')), '|a|\t|b|\t|c|\t||\t\n', "final comma")
+local str = "ÑчÑмитьб-Pincall;79031111111\r\n"
+str = str .. str .. str .. str .. str .. str
+str = "Vendor;Prefix\r\n" .. str
+f = fio.open(file4, { "O_WRONLY", "O_TRUNC" , "O_CREAT"}, 0x1FF)
+f:write(str)
+f:close()
+
+test:is(#csv.load(fio.open(file4, {'O_RDONLY'}),
+ {separator = ';', chunk_size = 3}), 7, "gh-1210 (1)")
+test:is(#csv.load(fio.open(file4, {'O_RDONLY'}),
+ {separator = ';', chunk_size = 4}), 7, "gh-1210 (2)")
+
fio.unlink(file1)
fio.unlink(file2)
fio.unlink(file3)
+fio.unlink(file4)
fio.rmdir(tmpdir)
+
+test:check()
diff --git a/test/unit/csv.c b/test/unit/csv.c
index 0749f00dbb..52d2e385d1 100644
--- a/test/unit/csv.c
+++ b/test/unit/csv.c
@@ -310,6 +310,37 @@ void iter_test2() {
footer();
}
+void iter_test3() {
+ header();
+ struct csv_iterator it;
+ struct csv csv;
+ csv_create(&csv);
+ csv_iterator_create(&it, &csv);
+ int st = 0;
+ const char *ar[] = {"1,2,3\r\n", "4,5,6", ""};
+ int i = 0;
+ const char *buf = ar[i++];
+ while((st = csv_next(&it)) != CSV_IT_EOF) {
+ switch(st) {
+ case CSV_IT_NEEDMORE:
+ csv_feed(&it, buf, strlen(buf));
+ buf = ar[i++];
+ break;
+ case CSV_IT_EOL:
+ print_endl(0);
+ break;
+ case CSV_IT_OK:
+ print_field(0, it.field, it.field + it.field_len);
+ break;
+ case CSV_IT_ERROR:
+ printf("\nerror");
+ break;
+ }
+ }
+ csv_destroy(&csv);
+ footer();
+}
+
void csv_out() {
header();
@@ -392,6 +423,7 @@ int main() {
//iterator tests
iter_test1();
iter_test2();
+ iter_test3();
//output test
csv_out();
diff --git a/test/unit/csv.result b/test/unit/csv.result
index ecf954d2af..8f9dab5b22 100644
--- a/test/unit/csv.result
+++ b/test/unit/csv.result
@@ -93,6 +93,10 @@ ha|
|1|
|23|
*** iter_test2: done ***
+ *** iter_test3 ***
+|1| |2| |3|
+|4| |5| |6|
+ *** iter_test3: done ***
*** csv_out ***
abc<len=3>,"with,comma"<len=12>,""in quotes""<len=13>,1 "" quote<len=10>
*** csv_out: done ***
--
GitLab