fio: extend functionality

* Add listdir function based on dirent:readdir
* Add mktree, rmtree, copyfile, copytree to fio module

Closes #2751

src/coio_file.c

@@ -32,9 +32,10 @@
 #include "coio_task.h"
 #include "fiber.h"
 #include "say.h"
+#include "fio.h"
 #include <stdio.h>
 #include <stdlib.h>
-
+#include <dirent.h>
 
 /**
  * A context of libeio request for any
@@ -91,6 +92,16 @@ struct coio_file_task {
 		struct {
 			char *tpl;
 		} tempdir;
+
+		struct {
+			char **bufp;
+			const char *pathname;
+		} readdir;
+
+		struct {
+			const char *source;
+			const char *dest;
+		} copyfile;
 	};
 };
 
@@ -490,3 +501,130 @@ coio_fdatasync(int fd)
 	eio_req *req = eio_fdatasync(fd, 0, coio_complete, &eio);
 	return coio_wait_done(req, &eio);
 }
+
+static void
+coio_do_readdir(eio_req *req)
+{
+	struct coio_file_task *eio = (struct coio_file_task *)req->data;
+	DIR *dirp = opendir(eio->readdir.pathname);
+	if (dirp == NULL)
+		goto error;
+	size_t capacity = 128;
+	size_t len = 0;
+	struct dirent *entry;
+	char *buf = (char *) malloc(capacity);
+	if (buf == NULL)
+		goto mem_error;
+	req->result = 0;
+	do {
+		entry = readdir(dirp);
+		if (entry == NULL ||
+		    strcmp(entry->d_name, ".") == 0 ||
+		    strcmp(entry->d_name, "..") == 0)
+			continue;
+		size_t namlen = strlen(entry->d_name);
+		size_t needed = len + namlen + 1;
+		if (needed > capacity) {
+			if (needed <= capacity * 2)
+				capacity *= 2;
+			else
+				capacity = needed * 2;
+			char *new_buf = (char *) realloc(buf, capacity);
+			if (new_buf == NULL)
+				goto mem_error;
+			buf = new_buf;
+		}
+		memcpy(&buf[len], entry->d_name, namlen);
+		len += namlen;
+		buf[len++] = '\n';
+		req->result++;
+	} while(entry != NULL);
+
+	if (len > 0)
+		buf[len - 1] = 0;
+	else
+		buf[0] = 0;
+
+	*eio->readdir.bufp = buf;
+	closedir(dirp);
+	return;
+
+mem_error:
+	free(buf);
+	closedir(dirp);
+error:
+	req->result = -1;
+	req->errorno = errno;
+}
+
+int
+coio_readdir(const char *dir_path, char **buf)
+{
+	INIT_COEIO_FILE(eio)
+	eio.readdir.bufp = buf;
+	eio.readdir.pathname = dir_path;
+	eio_req *req = eio_custom(coio_do_readdir, 0, coio_complete, &eio);
+	return coio_wait_done(req, &eio);
+}
+
+static void
+coio_do_copyfile(eio_req *req)
+{
+	struct coio_file_task *eio = (struct coio_file_task *)req->data;
+
+	struct stat st;
+	if (stat(eio->copyfile.source, &st) < 0) {
+		goto error;
+	}
+
+	int source_fd = open(eio->copyfile.source, O_RDONLY);
+	if (source_fd < 0) {
+		goto error;
+	}
+
+	int dest_fd = open(eio->copyfile.dest, O_WRONLY | O_CREAT,
+			   st.st_mode & 0777);
+	if (dest_fd < 0) {
+		goto error_dest;
+	}
+
+	enum { COPY_FILE_BUF_SIZE = 4096 };
+
+	char buf[COPY_FILE_BUF_SIZE];
+
+	while (true) {
+		ssize_t nread = fio_read(source_fd, buf, sizeof(buf));
+		if (nread < 0)
+			goto error_copy;
+
+		if (nread == 0)
+			break; /* eof */
+
+		ssize_t nwritten = fio_writen(dest_fd, buf, nread);
+		if (nwritten < 0)
+			goto error_copy;
+	}
+	req->result = 0;
+	close(source_fd);
+	close(dest_fd);
+	return;
+
+error_copy:
+	close(dest_fd);
+error_dest:
+	close(source_fd);
+error:
+	req->errorno = errno;
+	req->result = -1;
+	return;
+}
+
+int
+coio_copyfile(const char *source, const char *dest)
+{
+	INIT_COEIO_FILE(eio)
+	eio.copyfile.source = source;
+	eio.copyfile.dest = dest;
+	eio_req *req = eio_custom(coio_do_copyfile, 0, coio_complete, &eio);
+	return coio_wait_done(req, &eio);
+}

src/coio_file.h

@@ -82,6 +82,8 @@ int     coio_fdatasync(int fd);
 
 int	coio_tempdir(char *path, size_t path_len);
 
+int	coio_readdir(const char *path, char **buf);
+int	coio_copyfile(const char *source, const char *dest);
 #if defined(__cplusplus)
 } /* extern "C" */
 #endif /* defined(__cplusplus) */

src/lua/fio.c

@@ -507,6 +507,24 @@ lbox_fio_rmdir(struct lua_State *L)
 	return 1;
 }
 
+static int
+lbox_fio_listdir(struct lua_State *L) {
+	const char *pathname;
+	if (lua_gettop(L) < 1) {
+		luaL_error(L, "Usage: fio.listdir(pathname)");
+	}
+	pathname = lua_tostring(L, 1);
+	char *buf;
+	if (coio_readdir(pathname, &buf) >= 0) {
+		lua_pushstring(L, buf);
+		free(buf);
+	} else {
+		lua_pushnil(L);
+	}
+	return 1;
+}
+
+
 static int
 lbox_fio_glob(struct lua_State *L)
 {
@@ -674,6 +692,15 @@ lbox_fio_close(struct lua_State *L)
 	return 1;
 }
 
+static int
+lbox_fio_copyfile(struct lua_State *L)
+{
+	const char *source = lua_tostring(L, -2);
+	const char *dest = lua_tostring(L, -1);
+	assert(source != NULL && dest != NULL);
+	lua_pushboolean(L, coio_copyfile(source, dest) == 0);
+	return 1;
+}
 
 
 
@@ -716,7 +743,9 @@ tarantool_lua_fio_init(struct lua_State *L)
 		{ "ftruncate",		lbox_fio_ftruncate		},
 		{ "fsync",		lbox_fio_fsync			},
 		{ "fdatasync",		lbox_fio_fdatasync		},
+		{ "listdir",		lbox_fio_listdir		},
 		{ "fstat",		lbox_fio_fstat			},
+		{ "copyfile",		lbox_fio_copyfile,		},
 		{ NULL,			NULL				}
 	};
 	luaL_register(L, NULL, internal_methods);

src/lua/fio.lua

@@ -262,4 +262,136 @@ fio.chdir = function(path)
     return ffi.C.chdir(path) == 0
 end
 
+fio.listdir = function(path)
+    if path == nil or type(path) ~= 'string' then
+        return nil
+    end
+    local str = internal.listdir(path)
+    if str == nil then
+        return nil
+    end
+    local t = {}
+    if str == "" then
+        return t
+    end
+    local names = string.split(str, "\n")
+    for i, name in ipairs(names) do
+        table.insert(t, name)
+    end
+    return t
+end
+
+fio.mktree = function(path, mode)
+    path = fio.abspath(path)
+    if path == nil then
+        return false
+    end
+    local path = string.gsub(path, '^/', '')
+    local dirs = string.split(path, "/")
+
+    if #dirs == 1 then
+        if mode then
+            return fio.mkdir(path, mode)
+        else
+            return fio.mkdir(path)
+        end
+    end
+
+    local current_dir = "/"
+    for i, dir in ipairs(dirs) do
+        current_dir = fio.pathjoin(current_dir, dir)
+        if not fio.stat(current_dir) then
+            local res
+            if mode then
+                if not fio.mkdir(current_dir, mode) then
+                    res = false
+                end
+            else
+                if not fio.mkdir(current_dir) then
+                    res = false
+                end
+            end
+        end
+    end
+    return true
+end
+
+fio.rmtree = function(path)
+    if path == nil then
+        return false
+    end
+    local path = tostring(path)
+    path = fio.abspath(path)
+    local ls = fio.listdir(path)
+
+    for i, f in ipairs(ls) do
+        local tmppath = fio.pathjoin(path, f)
+        if fio.stat(tmppath):is_dir() then
+            if not fio.rmtree(tmppath) then
+                return false
+            end
+        end
+    end
+    return fio.rmdir(path)
+end
+
+fio.copyfile = function(from, to)
+    if type(from) ~= 'string' or type(to) ~= 'string' then
+        error('Usage: fio.copyfile(from, to)')
+    end
+    local st = fio.stat(to)
+    if st and st:is_dir() then
+        to = fio.pathjoin(to, fio.basename(from))
+    end
+    return internal.copyfile(from, to)
+end
+
+fio.copytree = function(from, to)
+    if type(from) ~= 'string' or type(to) ~= 'string' then
+        error('Usage: fio.copytree(from, to)')
+    end
+    local status, reason
+    from = fio.abspath(from)
+    local st = fio.stat(from)
+    if st == nil or not st:is_dir() then
+        return false, errno.strerror(errno.ENOTDIR)
+    end
+    local ls = fio.listdir(from)
+    to = fio.abspath(to)
+
+    -- create tree of destination
+    local status, reason = fio.mktree(to)
+    if not status then
+        return status, reason
+    end
+    for i, f in ipairs(ls) do
+        local ffrom = fio.pathjoin(from, f)
+        local fto = fio.pathjoin(to, f)
+        local st = fio.lstat(ffrom)
+        if st:is_dir() then
+            status, reason = fio.copytree(ffrom, fto)
+            if not status then
+                return status, reason
+            end
+        end
+        if st:is_reg() then
+            status, reason = fio.copyfile(ffrom, fto)
+            if not status then
+                return status, reason
+            end
+        end
+        if st:is_link() then
+            local link_to, reason = fio.readlink(ffrom)
+            if not link_to then
+                return status, reason
+            end
+            status, reason = fio.symlink(link_to, fto)
+            if not status then
+                return nil, "can't create symlink in place of existing file "..fto
+            end
+        end
+    end
+    return true
+end
+
 return fio

test/app/fio.result

@@ -505,3 +505,144 @@ fio.cwd() == old_cwd
 ---
 - true
 ...
+-- listdir
+tmpdir = fio.tempdir()
+---
+...
+dir3 = fio.pathjoin(tmpdir, "dir3")
+---
+...
+fio.mkdir(dir3)
+---
+- true
+...
+fio.mkdir(fio.pathjoin(dir3, "1"))
+---
+- true
+...
+fio.mkdir(fio.pathjoin(dir3, "2"))
+---
+- true
+...
+fio.mkdir(fio.pathjoin(dir3, "3"))
+---
+- true
+...
+ls = fio.listdir(dir3)
+---
+...
+table.sort(ls, function(a, b) return tonumber(a) < tonumber(b) end)
+---
+...
+ls
+---
+- - '1'
+  - '2'
+  - '3'
+...
+-- rmtree
+fio.stat(dir3) ~= nil
+---
+- true
+...
+fio.rmtree(dir3)
+---
+- true
+...
+fio.stat(dir3) == nil
+---
+- true
+...
+-- mktree
+tmp1 = fio.pathjoin(tmpdir, "1")
+---
+...
+tmp2 = fio.pathjoin(tmp1, "2")
+---
+...
+tree = fio.pathjoin(tmp2, "3")
+---
+...
+fio.mktree(tree)
+---
+- true
+...
+fio.stat(tree) ~= nil
+---
+- true
+...
+fio.stat(tmp2) ~= nil
+---
+- true
+...
+-- copy and copytree
+file1 = fio.pathjoin(tmp1, 'file.1')
+---
+...
+file2 = fio.pathjoin(tmp2, 'file.2')
+---
+...
+file3 = fio.pathjoin(tree, 'file.3')
+---
+...
+fh1 = fio.open(file1, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
+---
+...
+fh1:write("gogo")
+---
+- true
+...
+fh1:close()
+---
+- true
+...
+fh1 = fio.open(file2, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
+---
+...
+fh1:write("lolo")
+---
+- true
+...
+fh1:close()
+---
+- true
+...
+fio.symlink(file1, file3)
+---
+- true
+...
+fio.copyfile(file1, tmp2)
+---
+- true
+...
+fio.stat(fio.pathjoin(tmp2, "file.1")) ~= nil
+---
+- true
+...
+fio.copyfile(fio.pathjoin(tmp1, 'not_exists.txt'), tmp1)
+---
+- false
+...
+newdir = fio.pathjoin(tmpdir, "newdir")
+---
+...
+fio.copytree(fio.pathjoin(tmpdir, "1"), newdir)
+---
+- true
+...
+fio.stat(fio.pathjoin(newdir, "file.1")) ~= nil
+---
+- true
+...
+fio.stat(fio.pathjoin(newdir, "2", "file.2")) ~= nil
+---
+- true
+...
+fio.stat(fio.pathjoin(newdir, "2", "3", "file.3")) ~= nil
+---
+- true
+...
+fio.readlink(fio.pathjoin(newdir, "2", "3", "file.3")) == file1
+---
+- true
+...

test/app/fio.test.lua

@@ -163,3 +163,52 @@ fio.chdir('/')
 fio.cwd()
 fio.chdir(old_cwd)
 fio.cwd() == old_cwd
+
+-- listdir
+tmpdir = fio.tempdir()
+dir3 = fio.pathjoin(tmpdir, "dir3")
+fio.mkdir(dir3)
+fio.mkdir(fio.pathjoin(dir3, "1"))
+fio.mkdir(fio.pathjoin(dir3, "2"))
+fio.mkdir(fio.pathjoin(dir3, "3"))
+ls = fio.listdir(dir3)
+table.sort(ls, function(a, b) return tonumber(a) < tonumber(b) end)
+ls
+
+-- rmtree
+fio.stat(dir3) ~= nil
+fio.rmtree(dir3)
+fio.stat(dir3) == nil
+
+-- mktree
+tmp1 = fio.pathjoin(tmpdir, "1")
+tmp2 = fio.pathjoin(tmp1, "2")
+tree = fio.pathjoin(tmp2, "3")
+fio.mktree(tree)
+fio.stat(tree) ~= nil
+fio.stat(tmp2) ~= nil
+
+
+-- copy and copytree
+file1 = fio.pathjoin(tmp1, 'file.1')
+file2 = fio.pathjoin(tmp2, 'file.2')
+file3 = fio.pathjoin(tree, 'file.3')
+
+fh1 = fio.open(file1, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
+fh1:write("gogo")
+fh1:close()
+fh1 = fio.open(file2, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
+fh1:write("lolo")
+fh1:close()
+fio.symlink(file1, file3)
+fio.copyfile(file1, tmp2)
+fio.stat(fio.pathjoin(tmp2, "file.1")) ~= nil
+
+fio.copyfile(fio.pathjoin(tmp1, 'not_exists.txt'), tmp1)
+
+newdir = fio.pathjoin(tmpdir, "newdir")
+fio.copytree(fio.pathjoin(tmpdir, "1"), newdir)
+fio.stat(fio.pathjoin(newdir, "file.1")) ~= nil
+fio.stat(fio.pathjoin(newdir, "2", "file.2")) ~= nil
+fio.stat(fio.pathjoin(newdir, "2", "3", "file.3")) ~= nil
+fio.readlink(fio.pathjoin(newdir, "2", "3", "file.3")) == file1