fixed gh-1509, fixed gh-1210 : problems with csv parser

src/lib/csv/csv.c

@@ -328,13 +328,14 @@ csv_next(struct csv_iterator *it)
 		return CSV_IT_ERROR;
 
 	it->buf_begin = tail;
-	/* bufp == NULL means end of line */
-	if (it->csv->bufp == NULL)
-		return CSV_IT_EOL;
 
 	if (tail == it->buf_end) /* buffer is empty */
 		return CSV_IT_NEEDMORE;
 
+	/* bufp == NULL means end of line */
+	if (it->csv->bufp == NULL)
+		return CSV_IT_EOL;
+
 	/* return field via iterator structure */
 	it->field = it->csv->buf;
 	it->field_len = it->csv->bufp - it->csv->buf;

test/app-tap/csv.result

 TAP version 13
-1..9
+1..11
 ok - obj test1
 ok - obj test2
 ok - obj test3
@@ -9,3 +9,5 @@ ok - fio test3
 ok - test roundtrip
 ok - test load(dump(t))
 ok - final comma
+ok - gh-1210 (1)
+ok - gh-1210 (2)

test/app-tap/csv.test.lua

@@ -36,7 +36,7 @@ local test6_ans = "|23|\t|456|\t|abcac|\t|'multiword field 4'|\t\n|none|" ..
                   "lag[ flag ])|\t\n||\t||\t||\t\n"
 
 test = tap.test("csv")
-test:plan(9)
+test:plan(11)
 
 readable = {}
 readable.read = myread
@@ -56,6 +56,7 @@ tmpdir = fio.tempdir()
 file1 = fio.pathjoin(tmpdir, 'file.1')
 file2 = fio.pathjoin(tmpdir, 'file.2')
 file3 = fio.pathjoin(tmpdir, 'file.3')
+file4 = fio.pathjoin(tmpdir, 'file.4')
 
 local f = fio.open(file1, { 'O_WRONLY', 'O_TRUNC', 'O_CREAT' }, 0777)
 f:write("123 , 5  ,       92    , 0, 0\n" ..
@@ -109,7 +110,22 @@ test:is(table2str(t), table2str(csv.load(csv.dump(t))), "test load(dump(t))")
 
 test:is(table2str(csv.load('a,b,c,')), '|a|\t|b|\t|c|\t||\t\n', "final comma")
 
+local str = "ячсмитьб-Pincall;79031111111\r\n"
+str = str .. str .. str .. str .. str .. str
+str = "Vendor;Prefix\r\n" .. str
+f = fio.open(file4, { "O_WRONLY", "O_TRUNC" , "O_CREAT"}, 0x1FF)
+f:write(str)
+f:close()
+
+test:is(#csv.load(fio.open(file4, {'O_RDONLY'}),
+                  {separator = ';', chunk_size = 3}), 7, "gh-1210 (1)")
+test:is(#csv.load(fio.open(file4, {'O_RDONLY'}),
+                  {separator = ';', chunk_size = 4}), 7, "gh-1210 (2)")
+
 fio.unlink(file1)
 fio.unlink(file2)
 fio.unlink(file3)
+fio.unlink(file4)
 fio.rmdir(tmpdir)
+
+test:check()

test/unit/csv.c

@@ -310,6 +310,37 @@ void iter_test2() {
 	footer();
 }
 
+void iter_test3() {
+	header();
+	struct csv_iterator it;
+	struct csv csv;
+	csv_create(&csv);
+	csv_iterator_create(&it, &csv);
+	int st = 0;
+	const char *ar[] = {"1,2,3\r\n", "4,5,6", ""};
+	int i = 0;
+	const char *buf = ar[i++];
+	while((st = csv_next(&it)) != CSV_IT_EOF) {
+		switch(st) {
+		case CSV_IT_NEEDMORE:
+			csv_feed(&it, buf, strlen(buf));
+			buf = ar[i++];
+			break;
+		case CSV_IT_EOL:
+			print_endl(0);
+			break;
+		case CSV_IT_OK:
+			print_field(0, it.field, it.field + it.field_len);
+			break;
+		case CSV_IT_ERROR:
+			printf("\nerror");
+			break;
+		}
+	}
+	csv_destroy(&csv);
+	footer();
+}
+
 void csv_out() {
 	header();
 
@@ -392,6 +423,7 @@ int main() {
 	//iterator tests
 	iter_test1();
 	iter_test2();
+	iter_test3();
 
 	//output test
 	csv_out();

test/unit/csv.result

@@ -93,6 +93,10 @@ ha|
 |1|
 |23|
 	*** iter_test2: done ***
+	*** iter_test3 ***
+|1|	|2|	|3|
+|4|	|5|	|6|
+	*** iter_test3: done ***
 	*** csv_out ***
 abc<len=3>,"with,comma"<len=12>,""in quotes""<len=13>,1 "" quote<len=10>
 	*** csv_out: done ***