From d7135403a8341e865b80519f13587ed32a73a359 Mon Sep 17 00:00:00 2001
From: Ilya <markovilya197@gmail.com>
Date: Thu, 28 Sep 2017 13:34:22 +0300
Subject: [PATCH] fio: extend functionality
* Add listdir function based on dirent:readdir
* Add mktree, rmtree, copyfile, copytree to fio module
Closes #2751
---
src/coio_file.c | 140 ++++++++++++++++++++++++++++++++++++++++-
src/coio_file.h | 2 +
src/lua/fio.c | 29 +++++++++
src/lua/fio.lua | 132 +++++++++++++++++++++++++++++++++++++++
test/app/fio.result | 141 ++++++++++++++++++++++++++++++++++++++++++
test/app/fio.test.lua | 49 +++++++++++++++
6 files changed, 492 insertions(+), 1 deletion(-)
diff --git a/src/coio_file.c b/src/coio_file.c
index 8b5bbde919..c5b2db781e 100644
--- a/src/coio_file.c
+++ b/src/coio_file.c
@@ -32,9 +32,10 @@
#include "coio_task.h"
#include "fiber.h"
#include "say.h"
+#include "fio.h"
#include <stdio.h>
#include <stdlib.h>
-
+#include <dirent.h>
/**
* A context of libeio request for any
@@ -91,6 +92,16 @@ struct coio_file_task {
struct {
char *tpl;
} tempdir;
+
+ struct {
+ char **bufp;
+ const char *pathname;
+ } readdir;
+
+ struct {
+ const char *source;
+ const char *dest;
+ } copyfile;
};
};
@@ -490,3 +501,130 @@ coio_fdatasync(int fd)
eio_req *req = eio_fdatasync(fd, 0, coio_complete, &eio);
return coio_wait_done(req, &eio);
}
+
+static void
+coio_do_readdir(eio_req *req)
+{
+ struct coio_file_task *eio = (struct coio_file_task *)req->data;
+ DIR *dirp = opendir(eio->readdir.pathname);
+ if (dirp == NULL)
+ goto error;
+ size_t capacity = 128;
+ size_t len = 0;
+ struct dirent *entry;
+ char *buf = (char *) malloc(capacity);
+ if (buf == NULL)
+ goto mem_error;
+ req->result = 0;
+ do {
+ entry = readdir(dirp);
+ if (entry == NULL ||
+ strcmp(entry->d_name, ".") == 0 ||
+ strcmp(entry->d_name, "..") == 0)
+ continue;
+ size_t namlen = strlen(entry->d_name);
+ size_t needed = len + namlen + 1;
+ if (needed > capacity) {
+ if (needed <= capacity * 2)
+ capacity *= 2;
+ else
+ capacity = needed * 2;
+ char *new_buf = (char *) realloc(buf, capacity);
+ if (new_buf == NULL)
+ goto mem_error;
+ buf = new_buf;
+ }
+ memcpy(&buf[len], entry->d_name, namlen);
+ len += namlen;
+ buf[len++] = '\n';
+ req->result++;
+ } while(entry != NULL);
+
+ if (len > 0)
+ buf[len - 1] = 0;
+ else
+ buf[0] = 0;
+
+ *eio->readdir.bufp = buf;
+ closedir(dirp);
+ return;
+
+mem_error:
+ free(buf);
+ closedir(dirp);
+error:
+ req->result = -1;
+ req->errorno = errno;
+}
+
+int
+coio_readdir(const char *dir_path, char **buf)
+{
+ INIT_COEIO_FILE(eio)
+ eio.readdir.bufp = buf;
+ eio.readdir.pathname = dir_path;
+ eio_req *req = eio_custom(coio_do_readdir, 0, coio_complete, &eio);
+ return coio_wait_done(req, &eio);
+}
+
+static void
+coio_do_copyfile(eio_req *req)
+{
+ struct coio_file_task *eio = (struct coio_file_task *)req->data;
+
+ struct stat st;
+ if (stat(eio->copyfile.source, &st) < 0) {
+ goto error;
+ }
+
+ int source_fd = open(eio->copyfile.source, O_RDONLY);
+ if (source_fd < 0) {
+ goto error;
+ }
+
+ int dest_fd = open(eio->copyfile.dest, O_WRONLY | O_CREAT,
+ st.st_mode & 0777);
+ if (dest_fd < 0) {
+ goto error_dest;
+ }
+
+ enum { COPY_FILE_BUF_SIZE = 4096 };
+
+ char buf[COPY_FILE_BUF_SIZE];
+
+ while (true) {
+ ssize_t nread = fio_read(source_fd, buf, sizeof(buf));
+ if (nread < 0)
+ goto error_copy;
+
+ if (nread == 0)
+ break; /* eof */
+
+ ssize_t nwritten = fio_writen(dest_fd, buf, nread);
+ if (nwritten < 0)
+ goto error_copy;
+ }
+ req->result = 0;
+ close(source_fd);
+ close(dest_fd);
+ return;
+
+error_copy:
+ close(dest_fd);
+error_dest:
+ close(source_fd);
+error:
+ req->errorno = errno;
+ req->result = -1;
+ return;
+}
+
+int
+coio_copyfile(const char *source, const char *dest)
+{
+ INIT_COEIO_FILE(eio)
+ eio.copyfile.source = source;
+ eio.copyfile.dest = dest;
+ eio_req *req = eio_custom(coio_do_copyfile, 0, coio_complete, &eio);
+ return coio_wait_done(req, &eio);
+}
diff --git a/src/coio_file.h b/src/coio_file.h
index 4bb295dad2..7db5611d4e 100644
--- a/src/coio_file.h
+++ b/src/coio_file.h
@@ -82,6 +82,8 @@ int coio_fdatasync(int fd);
int coio_tempdir(char *path, size_t path_len);
+int coio_readdir(const char *path, char **buf);
+int coio_copyfile(const char *source, const char *dest);
#if defined(__cplusplus)
} /* extern "C" */
#endif /* defined(__cplusplus) */
diff --git a/src/lua/fio.c b/src/lua/fio.c
index 7113b15748..ccbd9aaed5 100644
--- a/src/lua/fio.c
+++ b/src/lua/fio.c
@@ -507,6 +507,24 @@ lbox_fio_rmdir(struct lua_State *L)
return 1;
}
+static int
+lbox_fio_listdir(struct lua_State *L) {
+ const char *pathname;
+ if (lua_gettop(L) < 1) {
+ luaL_error(L, "Usage: fio.listdir(pathname)");
+ }
+ pathname = lua_tostring(L, 1);
+ char *buf;
+ if (coio_readdir(pathname, &buf) >= 0) {
+ lua_pushstring(L, buf);
+ free(buf);
+ } else {
+ lua_pushnil(L);
+ }
+ return 1;
+}
+
+
static int
lbox_fio_glob(struct lua_State *L)
{
@@ -674,6 +692,15 @@ lbox_fio_close(struct lua_State *L)
return 1;
}
+static int
+lbox_fio_copyfile(struct lua_State *L)
+{
+ const char *source = lua_tostring(L, -2);
+ const char *dest = lua_tostring(L, -1);
+ assert(source != NULL && dest != NULL);
+ lua_pushboolean(L, coio_copyfile(source, dest) == 0);
+ return 1;
+}
@@ -716,7 +743,9 @@ tarantool_lua_fio_init(struct lua_State *L)
{ "ftruncate", lbox_fio_ftruncate },
{ "fsync", lbox_fio_fsync },
{ "fdatasync", lbox_fio_fdatasync },
+ { "listdir", lbox_fio_listdir },
{ "fstat", lbox_fio_fstat },
+ { "copyfile", lbox_fio_copyfile, },
{ NULL, NULL }
};
luaL_register(L, NULL, internal_methods);
diff --git a/src/lua/fio.lua b/src/lua/fio.lua
index 32b23ef83b..15a178ab22 100644
--- a/src/lua/fio.lua
+++ b/src/lua/fio.lua
@@ -262,4 +262,136 @@ fio.chdir = function(path)
return ffi.C.chdir(path) == 0
end
+fio.listdir = function(path)
+ if path == nil or type(path) ~= 'string' then
+ return nil
+ end
+ local str = internal.listdir(path)
+ if str == nil then
+ return nil
+ end
+ local t = {}
+ if str == "" then
+ return t
+ end
+ local names = string.split(str, "\n")
+ for i, name in ipairs(names) do
+ table.insert(t, name)
+ end
+ return t
+end
+
+fio.mktree = function(path, mode)
+ path = fio.abspath(path)
+ if path == nil then
+ return false
+ end
+ local path = string.gsub(path, '^/', '')
+ local dirs = string.split(path, "/")
+
+ if #dirs == 1 then
+ if mode then
+ return fio.mkdir(path, mode)
+ else
+ return fio.mkdir(path)
+ end
+ end
+
+ local current_dir = "/"
+ for i, dir in ipairs(dirs) do
+ current_dir = fio.pathjoin(current_dir, dir)
+ if not fio.stat(current_dir) then
+ local res
+ if mode then
+ if not fio.mkdir(current_dir, mode) then
+ res = false
+ end
+ else
+ if not fio.mkdir(current_dir) then
+ res = false
+ end
+ end
+ end
+ end
+ return true
+end
+
+fio.rmtree = function(path)
+ if path == nil then
+ return false
+ end
+ local path = tostring(path)
+ path = fio.abspath(path)
+ local ls = fio.listdir(path)
+
+ for i, f in ipairs(ls) do
+ local tmppath = fio.pathjoin(path, f)
+ if fio.stat(tmppath):is_dir() then
+ if not fio.rmtree(tmppath) then
+ return false
+ end
+ end
+ end
+ return fio.rmdir(path)
+end
+
+fio.copyfile = function(from, to)
+ if type(from) ~= 'string' or type(to) ~= 'string' then
+ error('Usage: fio.copyfile(from, to)')
+ end
+ local st = fio.stat(to)
+ if st and st:is_dir() then
+ to = fio.pathjoin(to, fio.basename(from))
+ end
+ return internal.copyfile(from, to)
+end
+
+fio.copytree = function(from, to)
+ if type(from) ~= 'string' or type(to) ~= 'string' then
+ error('Usage: fio.copytree(from, to)')
+ end
+ local status, reason
+ from = fio.abspath(from)
+ local st = fio.stat(from)
+ if st == nil or not st:is_dir() then
+ return false, errno.strerror(errno.ENOTDIR)
+ end
+ local ls = fio.listdir(from)
+ to = fio.abspath(to)
+
+ -- create tree of destination
+ local status, reason = fio.mktree(to)
+ if not status then
+ return status, reason
+ end
+ for i, f in ipairs(ls) do
+ local ffrom = fio.pathjoin(from, f)
+ local fto = fio.pathjoin(to, f)
+ local st = fio.lstat(ffrom)
+ if st:is_dir() then
+ status, reason = fio.copytree(ffrom, fto)
+ if not status then
+ return status, reason
+ end
+ end
+ if st:is_reg() then
+ status, reason = fio.copyfile(ffrom, fto)
+ if not status then
+ return status, reason
+ end
+ end
+ if st:is_link() then
+ local link_to, reason = fio.readlink(ffrom)
+ if not link_to then
+ return status, reason
+ end
+ status, reason = fio.symlink(link_to, fto)
+ if not status then
+ return nil, "can't create symlink in place of existing file "..fto
+ end
+ end
+ end
+ return true
+end
+
return fio
diff --git a/test/app/fio.result b/test/app/fio.result
index 60d69cd161..4266f4fe7a 100644
--- a/test/app/fio.result
+++ b/test/app/fio.result
@@ -505,3 +505,144 @@ fio.cwd() == old_cwd
---
- true
...
+-- listdir
+tmpdir = fio.tempdir()
+---
+...
+dir3 = fio.pathjoin(tmpdir, "dir3")
+---
+...
+fio.mkdir(dir3)
+---
+- true
+...
+fio.mkdir(fio.pathjoin(dir3, "1"))
+---
+- true
+...
+fio.mkdir(fio.pathjoin(dir3, "2"))
+---
+- true
+...
+fio.mkdir(fio.pathjoin(dir3, "3"))
+---
+- true
+...
+ls = fio.listdir(dir3)
+---
+...
+table.sort(ls, function(a, b) return tonumber(a) < tonumber(b) end)
+---
+...
+ls
+---
+- - '1'
+ - '2'
+ - '3'
+...
+-- rmtree
+fio.stat(dir3) ~= nil
+---
+- true
+...
+fio.rmtree(dir3)
+---
+- true
+...
+fio.stat(dir3) == nil
+---
+- true
+...
+-- mktree
+tmp1 = fio.pathjoin(tmpdir, "1")
+---
+...
+tmp2 = fio.pathjoin(tmp1, "2")
+---
+...
+tree = fio.pathjoin(tmp2, "3")
+---
+...
+fio.mktree(tree)
+---
+- true
+...
+fio.stat(tree) ~= nil
+---
+- true
+...
+fio.stat(tmp2) ~= nil
+---
+- true
+...
+-- copy and copytree
+file1 = fio.pathjoin(tmp1, 'file.1')
+---
+...
+file2 = fio.pathjoin(tmp2, 'file.2')
+---
+...
+file3 = fio.pathjoin(tree, 'file.3')
+---
+...
+fh1 = fio.open(file1, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
+---
+...
+fh1:write("gogo")
+---
+- true
+...
+fh1:close()
+---
+- true
+...
+fh1 = fio.open(file2, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
+---
+...
+fh1:write("lolo")
+---
+- true
+...
+fh1:close()
+---
+- true
+...
+fio.symlink(file1, file3)
+---
+- true
+...
+fio.copyfile(file1, tmp2)
+---
+- true
+...
+fio.stat(fio.pathjoin(tmp2, "file.1")) ~= nil
+---
+- true
+...
+fio.copyfile(fio.pathjoin(tmp1, 'not_exists.txt'), tmp1)
+---
+- false
+...
+newdir = fio.pathjoin(tmpdir, "newdir")
+---
+...
+fio.copytree(fio.pathjoin(tmpdir, "1"), newdir)
+---
+- true
+...
+fio.stat(fio.pathjoin(newdir, "file.1")) ~= nil
+---
+- true
+...
+fio.stat(fio.pathjoin(newdir, "2", "file.2")) ~= nil
+---
+- true
+...
+fio.stat(fio.pathjoin(newdir, "2", "3", "file.3")) ~= nil
+---
+- true
+...
+fio.readlink(fio.pathjoin(newdir, "2", "3", "file.3")) == file1
+---
+- true
+...
diff --git a/test/app/fio.test.lua b/test/app/fio.test.lua
index e95f6397b2..7826993131 100644
--- a/test/app/fio.test.lua
+++ b/test/app/fio.test.lua
@@ -163,3 +163,52 @@ fio.chdir('/')
fio.cwd()
fio.chdir(old_cwd)
fio.cwd() == old_cwd
+
+-- listdir
+tmpdir = fio.tempdir()
+dir3 = fio.pathjoin(tmpdir, "dir3")
+fio.mkdir(dir3)
+fio.mkdir(fio.pathjoin(dir3, "1"))
+fio.mkdir(fio.pathjoin(dir3, "2"))
+fio.mkdir(fio.pathjoin(dir3, "3"))
+ls = fio.listdir(dir3)
+table.sort(ls, function(a, b) return tonumber(a) < tonumber(b) end)
+ls
+
+-- rmtree
+fio.stat(dir3) ~= nil
+fio.rmtree(dir3)
+fio.stat(dir3) == nil
+
+-- mktree
+tmp1 = fio.pathjoin(tmpdir, "1")
+tmp2 = fio.pathjoin(tmp1, "2")
+tree = fio.pathjoin(tmp2, "3")
+fio.mktree(tree)
+fio.stat(tree) ~= nil
+fio.stat(tmp2) ~= nil
+
+
+-- copy and copytree
+file1 = fio.pathjoin(tmp1, 'file.1')
+file2 = fio.pathjoin(tmp2, 'file.2')
+file3 = fio.pathjoin(tree, 'file.3')
+
+fh1 = fio.open(file1, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
+fh1:write("gogo")
+fh1:close()
+fh1 = fio.open(file2, { 'O_RDWR', 'O_TRUNC', 'O_CREAT' }, 0777)
+fh1:write("lolo")
+fh1:close()
+fio.symlink(file1, file3)
+fio.copyfile(file1, tmp2)
+fio.stat(fio.pathjoin(tmp2, "file.1")) ~= nil
+
+fio.copyfile(fio.pathjoin(tmp1, 'not_exists.txt'), tmp1)
+
+newdir = fio.pathjoin(tmpdir, "newdir")
+fio.copytree(fio.pathjoin(tmpdir, "1"), newdir)
+fio.stat(fio.pathjoin(newdir, "file.1")) ~= nil
+fio.stat(fio.pathjoin(newdir, "2", "file.2")) ~= nil
+fio.stat(fio.pathjoin(newdir, "2", "3", "file.3")) ~= nil
+fio.readlink(fio.pathjoin(newdir, "2", "3", "file.3")) == file1
--
GitLab