From 26b3b3e368a79ce29df732ea04e72a4c002ae2ce Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 6 Mar 2018 17:38:59 +0000 Subject: rename into ioriot --- ioreplay/src/generate/generate.c | 235 ----------- ioreplay/src/generate/generate.h | 112 ------ ioreplay/src/generate/gioop.c | 838 --------------------------------------- ioreplay/src/generate/gioop.h | 102 ----- ioreplay/src/generate/gparser.c | 356 ----------------- ioreplay/src/generate/gparser.h | 113 ------ ioreplay/src/generate/gprocess.c | 101 ----- ioreplay/src/generate/gprocess.h | 90 ----- ioreplay/src/generate/gtask.c | 91 ----- ioreplay/src/generate/gtask.h | 100 ----- ioreplay/src/generate/gwriter.c | 85 ---- ioreplay/src/generate/gwriter.h | 86 ---- ioreplay/src/generate/vsize.c | 247 ------------ ioreplay/src/generate/vsize.h | 180 --------- 14 files changed, 2736 deletions(-) delete mode 100644 ioreplay/src/generate/generate.c delete mode 100644 ioreplay/src/generate/generate.h delete mode 100644 ioreplay/src/generate/gioop.c delete mode 100644 ioreplay/src/generate/gioop.h delete mode 100644 ioreplay/src/generate/gparser.c delete mode 100644 ioreplay/src/generate/gparser.h delete mode 100644 ioreplay/src/generate/gprocess.c delete mode 100644 ioreplay/src/generate/gprocess.h delete mode 100644 ioreplay/src/generate/gtask.c delete mode 100644 ioreplay/src/generate/gtask.h delete mode 100644 ioreplay/src/generate/gwriter.c delete mode 100644 ioreplay/src/generate/gwriter.h delete mode 100644 ioreplay/src/generate/vsize.c delete mode 100644 ioreplay/src/generate/vsize.h (limited to 'ioreplay/src/generate') diff --git a/ioreplay/src/generate/generate.c b/ioreplay/src/generate/generate.c deleted file mode 100644 index 05445ae..0000000 --- a/ioreplay/src/generate/generate.c +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generate.h" - -#include "../meta/meta.h" -#include "gtask.h" -#include "gwriter.h" -#include "gparser.h" - -#include - -#define _MAX_PROCESSES 1024*1024*10 - -#define _Perc_filtered (g->num_lines_filtered / (g->lineno/100.0)) - -generate_s* generate_new(options_s *opts) -{ - generate_s *g = Malloc(generate_s); - - g->writer = NULL; - g->lineno = 0; - g->name = opts->name; - g->replay_fd = NULL; - g->mps = mounts_new(opts); - g->num_lines_filtered = 0; - g->num_vsizes = 0; - g->start_time = -1; - g->pid_map = amap_new(_MAX_PROCESSES); - g->vsize_map = hmap_new(_MAX_PROCESSES); - g->mmap_map = hmap_new(1024*1024); - g->vfd_buffer = rbuffer_new(1024); - g->num_mapped_pids = 0; - g->num_mapped_fds = 10; - g->opts = opts; - g->reuse_queue = rbuffer_new(1024); - g->replay_fd = Fopen(opts->replay_file, "w"); - - return g; -} - -void generate_destroy(generate_s *g) -{ - // TODO: Also clean the contets of these maps - amap_destroy(g->pid_map); - hmap_destroy(g->vsize_map); - hmap_destroy(g->mmap_map); - rbuffer_destroy(g->vfd_buffer); - mounts_destroy(g->mps); - - gtask_s *task = NULL; - while (NULL != (task = rbuffer_get_next(g->reuse_queue))) - gtask_destroy(task); - rbuffer_destroy(g->reuse_queue); - - fclose(g->replay_fd); - free(g); -} - - -status_e generate_run(options_s *opts) -{ - generate_s *g = generate_new(opts); - Put("Parsing file %s, writing output to %s", opts->capture_file, - opts->replay_file); - FILE *capture_fd = Fopen(opts->capture_file, "r"); - - size_t len = 0; - ssize_t read; - char *line = NULL; - - set_limits_drop_root(opts->user); - - // Reserve first few bytes for meta information - meta_s *meta = meta_new(g->replay_fd); - meta_reserve(meta); - - // The writer will write the .replay file - gwriter_s *writer = gwriter_new(g); - - // The parser will parse every line of the .capture file - gparser_s *parser = gparser_new(g); - - g->writer = writer; - - // Start one writer and one parser thread! - gparser_start(parser); - gwriter_start(writer); - - Out("Processing, it may take a while: "); - - // Process each line of the .capture file. Determine line by line whether - // the I/O operation makes sense or not. It might be that SystemTap skipped - // some I/O ops due to system overload or other issues. The result is that - // some lines may be corrupt or contain I/O operations on unknown file - // handles. It could also be that there are operations on unknown - // file handles such as sockets etc. These will be all filtered out by - // either the parser or the writer thread! - - while ((read = getline(&line, &len, capture_fd)) != -1) { - if (0 > ++g->lineno) { - Error("lineno:%lu Line number overflow", g->lineno); - } - if (strlen(line) >= MAX_LINE_LEN) { - Error("lineno:%lu Exceeded max line length", g->lineno); - } - - // Create a new generate task (try to reuse a task object)... - gtask_s *t = rbuffer_get_next(g->reuse_queue); - if (!t) { - t = gtask_new(g); - } else if (t->ret != 0) { - g->num_lines_filtered++; - } - gtask_init(t, line, g->lineno); - - // ...pass it to the parser queue - while (!rbuffer_insert(parser->queue, t)) - usleep(100); - - if (g->lineno % 1000000 == 0) { - Out(" %lu (filtered:%.2lf%%)", g->lineno, _Perc_filtered); - } - } - - Put("\nDone reading input file!"); - - Put("Waiting for parser thread..."); - gparser_terminate(parser); - gparser_destroy(parser); - - Put("Waiting for writer thread..."); - gwriter_terminate(writer); - gwriter_destroy(writer); - - // Retrieve all left over processed tasks to collect the - // statistics! - gtask_s *t; - while (NULL != (t = rbuffer_get_next(g->reuse_queue))) { - if (t->ret != 0) - g->num_lines_filtered++; - gtask_destroy(t); - } - - Put("Processed %lu lines in total, had to filter out %.2lf%%", - g->lineno, _Perc_filtered); - - Put("Writing init section to '%s'...", opts->replay_file); - fprintf(g->replay_fd, "#INIT\n"); - off_t init_offset = ftello(g->replay_fd); - hmap_run_cb(g->vsize_map, generate_write_init_cb); - - Put("Writing meta header to '%s'...", opts->replay_file); - meta_write_start(meta); - - // The meta header is being written to the first line of the .replay - // file and used by ioreplay to do various things (e.g. initializing - // the test correctly, creating the internal data structures with the - // correct sizes etc. - - meta_write_l(meta, "replay_version", REPLAY_VERSION); - meta_write_l(meta, "init_offset", init_offset); - - meta_write_s(meta, "user", opts->user); - meta_write_s(meta, "name", opts->name); - - meta_write_l(meta, "num_vsizes", g->num_vsizes); - meta_write_l(meta, "num_mapped_pids", g->num_mapped_pids); - meta_write_l(meta, "num_mapped_fds", g->num_mapped_fds); - meta_write_l(meta, "num_lines", g->lineno - g->num_lines_filtered); - - meta_destroy(meta); - fclose(capture_fd); - - Put("Generating '%s' done", opts->replay_file); - generate_destroy(g); - - return SUCCESS; -} - -void generate_write_init_cb(void *data) -{ - vsize_s *l = data; - generate_s *g = l->generate; - - if (l->required && strlen(l->path) > 0) { - fprintf(g->replay_fd, "%d|%d|%ld|%s|\n", - l->is_dir, l->is_file, -l->vsize_deficit, l->path); - } -} - -vsize_s* generate_vsize_by_path(generate_s *g, gtask_s *t, - char *path) -{ - vsize_s *v = NULL; - - if (!path && t) - path = t->path; - - Error_if(!path, "No path specified"); - v = hmap_get(g->vsize_map, path); - - if (!v) { - v = vsize_new(path, ++g->num_vsizes, g); - hmap_insert(g->vsize_map, path, v); - } - - if (t) - t->vsize = v; - - return v; -} - -void generate_gprocess_by_realpid(generate_s *g, gtask_s *t) -{ - // Get the virtual process data object from the virtual PID space. - t->gprocess = amap_get(g->pid_map, t->pid); - if (t->gprocess == NULL) { - t->gprocess = gprocess_new(t->pid, ++g->num_mapped_pids); - if (amap_set(g->pid_map, t->pid, t->gprocess)) { - Error("lineno:%lu Can not insert PID %ld", t->lineno, t->pid); - } - } -} diff --git a/ioreplay/src/generate/generate.h b/ioreplay/src/generate/generate.h deleted file mode 100644 index cf096d2..0000000 --- a/ioreplay/src/generate/generate.h +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GENERATE_H -#define GENERATE_H - -#include "gwriter.h" -#include "../datas/amap.h" -#include "../datas/hmap.h" -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "../mounts.h" -#include "../options.h" - -// Forward declarations (header include hell) -struct gtask_s_; - -/** - * @brief The generate object definition - * - * This is the general data structure required to generate a .replay file from - * the .capture file. - */ -typedef struct generate_s_ { - long lineno; /**< The current line number */ - long num_lines_filtered; /**< The amount of lines filtered out */ - long start_time; /**< The start time from the .capture file */ - char *name; /**< The name of the test specified by the user */ - FILE *replay_fd; /**< The fd of the .replay file */ - mounts_s *mps; /**< The mounts object */ - hmap_s *mmap_map; /**< mmap address mappings */ - amap_s *pid_map; /**< A map of all virtual process objects */ - unsigned long num_mapped_pids; /**< The amount of mapped PIDs */ - unsigned long num_mapped_fds; /**< The amount of mapped FDs */ - hmap_s *vsize_map; /**< A hash map of all virtual size objects */ - unsigned long num_vsizes; /**< The amount of virtual sizes */ - options_s *opts; /**< A pointer to the options object */ - rbuffer_s *vfd_buffer; /**< A virtual fd buffer, for reusing these */ - rbuffer_s *reuse_queue; /**< A task buffer, for reusing these */ - struct gwriter_s_ *writer; /**< A pointer to the writer object */ -} generate_s; - -/** - * @brief Creates a new generate object - * - * @param opts The options object - * @return The new generate object - */ -generate_s* generate_new(options_s *opts); - -/** - * @brief Destroys a generate object - * - * @param g The generate object to destroy - */ -void generate_destroy(generate_s* g); - -/** - * @brief Generates a .replay file from a .capture file - * - * @param opts The options object - * @return SUCCESS on success - */ -status_e generate_run(options_s *opts); - -/** - * @brief Callback to write the INIT section to the .replay file - * - * This function writes a list of all pre-required - * paths to the .replay file. That then can be used - * by ioreplay to initialise the test enironment. - * - * @param data A pointer to the vsize timestamp object - */ -void generate_write_init_cb(void *data); - -/** - * @brief Retrieves the virtual size object of a given path - * - * A new one will be created in case there is no such virtual size object yet. - * - * @param g The generate object - * @param t The task object (vfd will be stored to t->vfd) - * @param path The file path - * @return The virtual size object - */ -vsize_s* generate_vsize_by_path(generate_s *g, struct gtask_s_ *t, - char *path); - -/** - * @brief Retrieves the virtual process object of a given real PID - * - * A new one will be created in case there is no such virtual process object - * yet. - * - * @param g The generate object - * @param t The task object (vfd will be stored to t->gprocess) - */ -void generate_gprocess_by_realpid(generate_s *g, struct gtask_s_ *t); - -#endif // GENERATE_H diff --git a/ioreplay/src/generate/gioop.c b/ioreplay/src/generate/gioop.c deleted file mode 100644 index 01701bc..0000000 --- a/ioreplay/src/generate/gioop.c +++ /dev/null @@ -1,838 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gioop.h" - -status_e gioop_run(gwriter_s *w, gtask_s *t) -{ - status_e ret = SUCCESS; - - // There was already an error in the parser (parser.c) processing this - // task! Don't process it futher. - if (t->ret != SUCCESS) { - Cleanup(t->ret); - } - - generate_s *g = w->generate; - - // Get the virtual process data object from the virtual PID space and store - // a pointer to it to t->gprocess - generate_gprocess_by_realpid(g, t); - - // One of the open syscalls may openes a file handle succesfully - if (Eq(t->op, "open")) { - Cleanup(gioop_open(w, t, g)); - - } else if (Eq(t->op, "openat")) { - Cleanup(gioop_openat(w, t, g)); - - } else if (Eq(t->op, "creat")) { - Cleanup(gioop_creat(w, t, g)); - } - - // Get the virtual file descriptor of a given real fd and store a pointer - // to it to t->vfd. - if (t->has_fd) { - ret = gprocess_vfd_by_realfd(t->gprocess, t); - Cleanup_unless(SUCCESS, ret); - } - - - if (Eq(t->op, "close")) { - Cleanup(gioop_close(w, t, g)); - - } else if (Eq(t->op, "stat")) { - Cleanup(gioop_stat(w, t, g)); - - } else if (Eq(t->op, "statfs")) { - Cleanup(gioop_statfs(w, t, g)); - - } else if (Eq(t->op, "statfs64")) { - Cleanup(gioop_statfs64(w, t, g)); - - } else if (Eq(t->op, "fstat")) { - Cleanup(gioop_fstat(w, t, g)); - - } else if (Eq(t->op, "fstatat")) { - Cleanup(gioop_fstatat(w, t, g)); - - } else if (Eq(t->op, "fstatfs")) { - Cleanup(gioop_fstatfs(w, t, g)); - - } else if (Eq(t->op, "fstatfs64")) { - Cleanup(gioop_fstatfs64(w, t, g)); - - } else if (Eq(t->op, "rename")) { - Cleanup(gioop_rename(w, t, g)); - - } else if (Eq(t->op, "renameat")) { - Cleanup(gioop_renameat(w, t, g)); - - } else if (Eq(t->op, "renameat2")) { - Cleanup(gioop_renameat2(w, t, g)); - - } else if (Eq(t->op, "read")) { - Cleanup(gioop_read(w, t, g)); - - } else if (Eq(t->op, "readv")) { - Cleanup(gioop_readv(w, t, g)); - - } else if (Eq(t->op, "readahead")) { - Cleanup(gioop_readahead(w, t, g)); - - } else if (Eq(t->op, "readdir")) { - Cleanup(gioop_readdir(w, t, g)); - - } else if (Eq(t->op, "readlink")) { - Cleanup(gioop_readlink(w, t, g)); - - } else if (Eq(t->op, "readlinkat")) { - Cleanup(gioop_readlinkat(w, t, g)); - - } else if (Eq(t->op, "write")) { - Cleanup(gioop_write(w, t, g)); - - } else if (Eq(t->op, "writev")) { - Cleanup(gioop_writev(w, t, g)); - - } else if (Eq(t->op, "lseek")) { - Cleanup(gioop_lseek(w, t, g)); - - } else if (Eq(t->op, "getdents")) { - Cleanup(gioop_getdents(w, t, g)); - - } else if (Eq(t->op, "mkdir")) { - Cleanup(gioop_mkdir(w, t, g)); - - } else if (Eq(t->op, "rmdir")) { - Cleanup(gioop_rmdir(w, t, g)); - - } else if (Eq(t->op, "mkdirat")) { - Cleanup(gioop_mkdirat(w, t, g)); - - } else if (Eq(t->op, "unlink")) { - Cleanup(gioop_unlink(w, t, g)); - - } else if (Eq(t->op, "unlinkat")) { - Cleanup(gioop_unlinkat(w, t, g)); - - } else if (Eq(t->op, "lstat")) { - Cleanup(gioop_lstat(w, t, g)); - - } else if (Eq(t->op, "fsync")) { - Cleanup(gioop_fsync(w, t, g)); - - } else if (Eq(t->op, "fdatasync")) { - Cleanup(gioop_fdatasync(w, t, g)); - - } else if (Eq(t->op, "sync")) { - Cleanup(gioop_sync(w, t, g)); - - } else if (Eq(t->op, "syncfs")) { - Cleanup(gioop_syncfs(w, t, g)); - - } else if (Eq(t->op, "sync_file_range")) { - Cleanup(gioop_sync_file_range(w, t, g)); - - } else if (Eq(t->op, "fcntl")) { - Cleanup(gioop_fcntl(w, t, g)); - - } else if (Eq(t->op, "fcntl")) { - Cleanup(gioop_fcntl(w, t, g)); - - } else if (Eq(t->op, "mmap2")) { - // Support for mmap added later - - } else if (Eq(t->op, "munmap")) { - // Support for mmap added later - - } else if (Eq(t->op, "mremap")) { - // Support for mmap added later - - } else if (Eq(t->op, "msync")) { - // Support for mmap added later - - } else if (Eq(t->op, "chmod")) { - Cleanup(gioop_chmod(w, t, g)); - - } else if (Eq(t->op, "fchmodat")) { - Cleanup(gioop_chmod(w, t, g)); - - } else if (Eq(t->op, "fchmod")) { - Cleanup(gioop_fchmod(w, t, g)); - - } else if (Eq(t->op, "chown")) { - Cleanup(gioop_chown(w, t, g)); - - } else if (Eq(t->op, "chown16")) { - Cleanup(gioop_chown(w, t, g)); - - } else if (Eq(t->op, "lchown")) { - Cleanup(gioop_lchown(w, t, g)); - - } else if (Eq(t->op, "lchown16")) { - Cleanup(gioop_lchown(w, t, g)); - - } else if (Eq(t->op, "fchown")) { - Cleanup(gioop_fchown(w, t, g)); - - } else if (Eq(t->op, "fchownat")) { - Cleanup(gioop_chown(w, t, g)); - - } else if (Eq(t->op, "exit_group")) { - Cleanup(gioop_exit_group(w, t, g)); - - } else { - Cleanup(ERROR;); - } - -cleanup: - -#ifdef LOG_FILTERED - if (ret != SUCCESS) - t->filtered_where = __FILE__; -#endif - - t->ret = ret; - return ret; -} - -status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd || t->path == NULL || t->flags == -1) { - return ERROR; - } - - gprocess_create_vfd_by_realfd(t->gprocess, t, g); - generate_vsize_by_path(g, t, NULL); - - Gioop_write(OPEN, "%ld|%s|%d|%d|open", - t->mapped_fd, t->path, t->mode, t->flags); - - if (t->fd > 0) - vsize_open(t->vsize, t->vfd, t->path, t->flags); - - return SUCCESS; -} - -status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd || t->path == NULL || t->flags == -1) { - return ERROR; - } - - gprocess_create_vfd_by_realfd(t->gprocess, t, g); - generate_vsize_by_path(g, t, NULL); - Gioop_write(OPEN_AT, "%ld|%s|%d|%d|openat", - t->mapped_fd,t->path, t->mode, t->flags); - if (t->fd > 0) - vsize_open(t->vsize, t->vfd, t->path, t->flags); - - return SUCCESS; -} - -status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd || t->path == NULL || t->flags == -1) { - return ERROR; - } - - gprocess_create_vfd_by_realfd(t->gprocess, t, g); - generate_vsize_by_path(g, t, NULL); - - Gioop_write(CREAT, "%ld|%s|%d|%d|creat", - t->mapped_fd, t->path, t->mode, t->flags); - if (t->fd > 0) - vsize_open(t->vsize, t->vfd, t->path, t->flags); - - return SUCCESS; -} - - -status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(CLOSE, "%ld|%d|close", t->mapped_fd, t->status); - - if (t->status == 0) - vsize_close(t->vsize, t->vfd); - - hmap_remove_l(t->gprocess->fd_map, t->fd); - hmap_remove_l(t->gprocess->vfd_map, t->mapped_fd); - - if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) - vfd_destroy(t->vfd); - - return SUCCESS; -} - -status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(STAT, "%s|%d|stat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(STATFS, "%s|%d|statfs", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(STATFS64, "%s|%d|statfs64", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSTAT, "%ld|%d|fstat", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(FSTAT_AT, "%s|%d|fstatat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSTATFS, "%ld|%d|fstatfs", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSTATFS64, "%ld|%d|fstatfs64", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL || t->path2 == NULL ) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(RENAME, "%s|%s|%d|rename", t->path, t->path2, t->status); - - if (t->status == 0) { - t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); - vsize_rename(t->vsize, t->vsize2, t->path, t->path2); - } - - return SUCCESS; -} - -status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL || t->path2 == NULL ) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(RENAME_AT, "%s|%s|%d|renameat", t->path, t->path2, t->status); - - if (t->status == 0) { - t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); - vsize_rename(t->vsize, t->vsize2, t->path, t->path2); - } - - return SUCCESS; -} -status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL || t->path2 == NULL ) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(RENAME_AT2, "%s|%s|%d|renameat2", - t->path, t->path2, t->status); - - if (t->status == 0) { - t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); - vsize_rename(t->vsize, t->vsize2, t->path, t->path2); - } - - return SUCCESS; -} - -status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READ, "%ld|%ld|read", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READ, "%ld|%ld|readv", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READAHEAD, "%ld|%ld|%ld|readahead", - t->mapped_fd, t->offset, t->count); - - return SUCCESS; -} - -status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READDIR, "%ld|%d|readdir", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(READLINK, "%s|%d|readlink", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(READLINK_AT, "%s|%d|readlinkat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(WRITE, "%ld|%ld|write", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_write(t->vsize, t->vfd, t->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(WRITEV, "%ld|%ld|writev", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_write(t->vsize, t->vfd, t->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(LSEEK, "%ld|%ld|%ld|%ld|lseek", - t->mapped_fd, t->offset, t->whence, t->bytes); - - if (t->bytes >= 0) - vsize_seek(t->vsize, t->vfd, t->bytes); - - return SUCCESS; -} - -status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(GETDENTS, "%ld|%ld|%ld|getdents", - t->mapped_fd, t->count, t->bytes); - - return SUCCESS; -} - -status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(MKDIR, "%s|%d|%d|mkdir", t->path, t->mode, t->status); - - if (t->status == 0) - vsize_mkdir(t->vsize, t->path); - - return SUCCESS; -} -status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(MKDIR, "%s|%d|rmdir", t->path, t->status); - - if (t->status == 0) - vsize_rmdir(t->vsize, t->path); - - return SUCCESS; -} -status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(MKDIR_AT, "%s|%d|%d|mkdirat", t->path, t->mode, t->status); - - if (t->status == 0) - vsize_mkdir(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(UNLINK, "%s|%d|unlink", t->path, t->status); - - if (t->status == 0) - vsize_unlink(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(UNLINK_AT, "%s|%d|unlinkat", t->path, t->status); - - if (t->status == 0) - vsize_unlink(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(LSTAT, "%s|%d|lstat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSYNC, "%ld|%d|fsync", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FDATASYNC, "%ld|%d|fdatasync", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g) -{ - Gioop_write(SYNC, "%d|sync", t->status); - - return SUCCESS; -} - -status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(SYNCFS, "%ld|%d|syncfs", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(SYNC_FILE_RANGE, "%ld|%ld|%ld|%d|sync_file_range", - t->mapped_fd, t->offset, t->bytes, t->status); - - return SUCCESS; -} - -status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - switch (t->F) { - case F_GETFD: - case F_GETFL: - case F_SETFD: - case F_SETFL: - break; - default: - return ERROR; - break; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FCNTL, "%ld|%d|%d|%d|fcntl", - t->mapped_fd, t->F, t->G, t->status); - - return SUCCESS; -} - -status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(CHMOD, "%s|%d|%d|chmod", t->path, t->mode, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FCHMOD, "%ld|%d|%d|fchmod", t->mapped_fd, t->mode, t->status); - - return SUCCESS; -} - -status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - // Hmm, maybe rename t->offset, because here it is used for the user UID - Gioop_write(CHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - // Hmm, maybe rename t->offset, because here it is used for the user UID - Gioop_write(FCHOWN, "%ld|%ld|%d|%d|fchown", t->mapped_fd, t->offset, t->G, t->status); - - return SUCCESS; -} - -status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - // Hmm, maybe rename t->offset, because here it is used for the user UID - Gioop_write(LCHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g) -{ - // It means that the process and all its threads terminate. - // Therefore close all file handles of that process! - hmap_run_cb2(t->gprocess->vfd_map, gioop_close_all_vfd_cb, t); - - // Remove virtual process from pid map and destroy it - amap_unset(g->pid_map, t->pid); - gprocess_destroy(t->gprocess); - - return SUCCESS; -} - -void gioop_close_all_vfd_cb(void *data, void *data2) -{ - gtask_s *t = data2; - t->vfd = data; - generate_s *g = t->generate; - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(CLOSE, "%ld|%d|close on exit_group", t->vfd->mapped_fd, 0); - vsize_close(t->vsize, t->vfd); -} - diff --git a/ioreplay/src/generate/gioop.h b/ioreplay/src/generate/gioop.h deleted file mode 100644 index ad49713..0000000 --- a/ioreplay/src/generate/gioop.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GIOOP_H -#define GIOOP_H - -#include "../defaults.h" -#include "gwriter.h" -#include "gtask.h" -#include "generate.h" - - -// Helper macro regarding writing the .replay file! - -#define Gioop_write(op, ...) \ - fprintf(g->replay_fd, "%ld|%ld|%ld|0|0|%d|", \ - t->mapped_time, \ - (t->vsize ? t->vsize->id : 0),\ - t->gprocess->mapped_pid, \ - op); \ - fprintf(g->replay_fd, __VA_ARGS__); \ - fprintf(g->replay_fd, "@%ld", t->lineno); \ - fprintf(g->replay_fd, "|\n") - -/** - * @brief Function used when closing all virtual FDs of a virtual process - * - * This function is run on all virtual file handles whenever a virtual generate - * process object (gprocess_s) gets destroyed. This is on an exit_group - * syscall (a thread group, a process with all its threads, terminates). Upon - * process termination Linux also closes all its file descriptors! This is what - * we simulate here! - * - * @param data The pointer to the virtual file descriptor object - * @param data2 The pointer to the corresponding generate task object. - */ -void gioop_close_all_vfd_cb(void *data, void *data2); - -/** - * @brief Run a generate I/O operation on a given task - * - * @param w The writer object - * @param t The task object - * @return SUCCESS if everything went fine - */ -status_e gioop_run(gwriter_s *w, gtask_s *t); - -status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g); - -#endif // GIOOP_H diff --git a/ioreplay/src/generate/gparser.c b/ioreplay/src/generate/gparser.c deleted file mode 100644 index 514128f..0000000 --- a/ioreplay/src/generate/gparser.c +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gparser.h" - -#include "gtask.h" -#include "gwriter.h" - -void* gparser_pthread_start(void *data) -{ - gparser_s *p = data; - generate_s *g = p->generate; - gwriter_s *w = g->writer; - gtask_s *t = NULL; - - do { - while (NULL != (t = rbuffer_get_next(p->queue))) { - // First extract - gparser_extract(p, t); - // Second, pass the task to the writer thread - rbuffer_insert(w->queue, t); - } - usleep(100); - } while (!p->terminate); - - while (NULL != (t = rbuffer_get_next(p->queue))) { - gparser_extract(p, t); - rbuffer_insert(w->queue, t); - } - - return NULL; -} - -gparser_s* gparser_new(generate_s *g) -{ - gparser_s *p = Malloc(gparser_s); - - p->generate = g; - p->terminate = false; - p->queue = rbuffer_new(1024); - - return p; -} - -void gparser_start(gparser_s *p) -{ - start_pthread(&p->pthread, gparser_pthread_start, (void*)p); -} - -void gparser_destroy(gparser_s *p) -{ - rbuffer_destroy(p->queue); - free(p); -} - -void gparser_terminate(gparser_s *p) -{ - p->terminate = true; - pthread_join(p->pthread, NULL); -} - -void gparser_extract(gparser_s *p, gtask_s *t) -{ - status_e ret = SUCCESS; - generate_s *g = p->generate; - - char *saveptr; - char* tok = strtok2_r(t->line, ";:,", &saveptr); - int ntoks = 0; - - while (tok) { - if (++ntoks > MAX_TOKENS) { - ret = ERROR; - break; - } - ret = gparser_extract_tok(p, t, tok); - if (ret != SUCCESS) - break; - - tok = strtok2_r(NULL, ";:,", &saveptr); - } - - if (ret == SUCCESS) { - - // Check for the existance of mandatory values! - if (t->pid < 0 || t->tid < 0) { - Cleanup(ERROR); - - } else if (t->op == NULL) { - Cleanup(ERROR); - - } else if (t->mapped_time == -1) { - Cleanup(ERROR); - } - - // We are inserting ".ioreplay/NAME" to the paths. This enables us to - // run multiple tests simoultaneously. - - if (t->path) { - if (!mounts_transform_path(g->mps, g->name, - t->path, &t->path_r)) { - Cleanup(ERROR); - } - if (t->path_r) - t->path = t->path_r; - } - - if (t->path2) { - if (!mounts_transform_path(g->mps, g->name, - t->path2, &t->path2_r)) { - Cleanup(ERROR); - } - if (t->path2_r) - t->path2 = t->path2_r; - } - - } - -cleanup: - - t->ret = ret; - -#ifdef LOG_FILTERED - t->filtered_where = __FILE__; -#endif -} - -status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok) -{ - status_e ret = SUCCESS; - - if (gparser_token_not_ok(p, tok)) { - Cleanup(ERROR); - } - - generate_s *g = t->generate; - - char key = tok[0]; - char *value = tok; - value += 2; - - switch (key) { - case 'a': - // Address - t->address = strtol(value, NULL, 10); - break; - - case 'A': - // Address 2 - t->address2 = strtol(value, NULL, 10); - break; - - case 'b': - // Bytes - if (t->bytes != -1) { - Cleanup(ERROR); - } - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->bytes = strtol(value, NULL, 10); - break; - - case 'c': - // Count - if (t->count != -1) { - Cleanup(ERROR); - } - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->count = strtol(value, NULL, 10); - break; - - case 'd': - // Descriptor - if (t->fd != -1) { - Cleanup(ERROR); - } - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->fd = atoi(value); - if (t->fd > 0) - t->has_fd = true; - break; - - case 'f': - // Flags - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->flags = atoi(value); - break; - - case 'i': - // PID:TID - t->pidtid = value; - // Extract PID and TID from "PID:TID" - if (!gparser_get_pidtid(p, t->pidtid, &t->pid, &t->tid)) { - Cleanup(ERROR); - } - break; - - case 'm': - // Mode - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->mode = atoi(value); - break; - - case 'o': - // Operation - t->op = value; - break; - - case 'O': - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->offset = strtol(value, NULL, 10); - break; - - case 'W': - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->whence = strtol(value, NULL, 10); - break; - - case 'p': - // File path - t->path = value; - chreplace(t->path, '|', '_'); - strunquote(t->path); - break; - - case 'P': - // File path 2 - t->path2 = value; - chreplace(t->path2, '|', '_'); - strunquote(t->path2); - break; - - case 's': - // Cleanup status - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->status = atoi(value); - break; - - case 't': - // Time - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->mapped_time = strtol(value, NULL, 10); - // Start replay time from 0 - if (g->start_time == -1) { - g->start_time = t->mapped_time; - } - t->mapped_time -= g->start_time; - break; - - case 'F': - // FCNTL function - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->F = atoi(value); - break; - - case 'G': - // FCNTL argument - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->G = atoi(value); - break; - - case 'T': - break; - - default: - // Unknown key - { - Cleanup(ERROR); - } - } - -cleanup: - if (t->path_r) { - free(t->path_r); - t->path_r = NULL; - } - if (t->path2_r) { - free(t->path2_r); - t->path2_r = NULL; - } - - return ret; -} - -bool gparser_token_not_ok(gparser_s *p, char *tok) -{ - if (strlen(tok) < 3) { - return true; - - } else if (tok[1] != '=') { - return true; - } - - return false; -} - -bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid) -{ - char *pos = strchr(pidtid, ':'); - - if (pos) { - char *tmp = pos; - tmp++; - - if (is_number(tmp)) { - *tid = atol(tmp); - } else { - return false; - } - - pos[0] = '\0'; - if (is_number(pidtid)) { - *pid = atol(pidtid); - } else { - return false; - } - } - - else { - return false; - } - - return (*pid >= 0 && *tid >= 0); -} diff --git a/ioreplay/src/generate/gparser.h b/ioreplay/src/generate/gparser.h deleted file mode 100644 index f3e204a..0000000 --- a/ioreplay/src/generate/gparser.h +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GPARSER_H -#define GPARSER_H - -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "generate.h" -#include "gtask.h" - -/** - * @brief The parser definition - * - * The parser is to extract all information from the .capture file. - */ -typedef struct gparser_s_ { - bool terminate; /**< The parser thread will terminate if set to true */ - generate_s *generate; /**< The generate object */ - pthread_t pthread; /**< The posix thread */ - rbuffer_s *queue; /**< A queue of task objects */ -} gparser_s; - -/** - * @brief Creates a new parser - * - * @param g The generate object - * @return The new parser object - */ -gparser_s* gparser_new(generate_s *g); - -/** - * @brief Starts the parser thread - * - * @param p The parser object - */ -void gparser_start(gparser_s *p); - -/** - * @brief Terminates the parser thread - * - * @param p The parser object - */ -void gparser_terminate(gparser_s *p); - -/** - * @brief Destroys the parser thread - * - * @param p The parser object - */ -void gparser_destroy(gparser_s *p); - -/** - * @brief Extracts information a .capture line - * - * Extracts information from a .capture line and stores it into the task - * object. - * - * @param p The parser object - * @param t The task object - */ -void gparser_extract(gparser_s *p, gtask_s *t); - -/** - * @brief Extracts information from a specific token string - * - * @param p The parser object - * @param t The task object - * @param tok The token string - * @return Returns with SUCCESS on success - */ -status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok); - -/** - * @brief Verifies the correctness of a token - * - * @param p The parser object - * @param tok The token to be verified - * @return true if token verified successfully - */ -bool gparser_token_not_ok(gparser_s *p, char *tok); - -/** - * @brief Checks whether the pidtid string is correct or not - * - * @param p The parser object - * @param pidtid The string to check - * @param pid The pointer to the resulting pid - * @param tid The pointer to the resulting tid - * @return true on success - */ -bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid); - -/** - * @brief Entry point of the parser POSIX thread - * - * @param data A pointer to the parser object - * return Always NULL - */ -void* gparser_pthread_start(void *data); - -#endif // GPARSER_H diff --git a/ioreplay/src/generate/gprocess.c b/ioreplay/src/generate/gprocess.c deleted file mode 100644 index 6a0b37a..0000000 --- a/ioreplay/src/generate/gprocess.c +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gprocess.h" - -#include "../vfd.h" -#include "gioop.h" - -void _gprocess_vfd_map_destroy_cb(void *data) -{ - vfd_destroy(data); -} - -gprocess_s* gprocess_new(const long pid, const long mapped_pid) -{ - gprocess_s* gp = Malloc(gprocess_s); - - gp->pid = pid; - gp->mapped_pid = mapped_pid; - gp->max_mapped_fd = 0; - gp->fd_map = hmap_new_l(1024); - gp->vfd_map = hmap_new_l(1024); - gp->vfd_map->data_destroy = _gprocess_vfd_map_destroy_cb; - - return gp; -} - -void gprocess_destroy(gprocess_s *gp) -{ - hmap_destroy(gp->vfd_map); - hmap_destroy(gp->fd_map); - free(gp); -} - -void gprocess_create_vfd_by_realfd(gprocess_s *gp, gtask_s *t, generate_s *g) -{ - if (t->fd < 0) - return; - - // Check whether the real FD is still open according to the .capture log - long old_mapped = (long) hmap_get_l(gp->fd_map, t->fd); - if (old_mapped) { - - // That real file descriptor is already with a mapping to a virtual - // file descriptor. This may happen when SystemTap missed to trace a - // 'close' syscall. We are inserting a close now... - - t->vfd = hmap_get_l(gp->vfd_map, old_mapped); - - hmap_remove_l(gp->fd_map, t->fd); - hmap_remove_l(gp->vfd_map, old_mapped); - - if (t->vfd) { - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(CLOSE, "%ld|%d|close inserted", old_mapped, 0); - vsize_close(t->vsize, t->vfd); - if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) - vfd_destroy(t->vfd); - } - } - - t->vfd = rbuffer_get_next(g->vfd_buffer); - t->mapped_fd = ++g->num_mapped_fds; - if (!t->vfd) - t->vfd = vfd_new(t->fd, t->mapped_fd, t->path); - else - vfd_update(t->vfd, t->fd, t->mapped_fd, t->path); - t->vfd->free_path = t->path_r != NULL; - - hmap_insert_l(gp->vfd_map, t->mapped_fd, t->vfd); - hmap_insert_l(gp->fd_map, t->fd, (void*)t->mapped_fd); -} - -status_e gprocess_vfd_by_realfd(gprocess_s *gp, gtask_s *t) -{ - t->mapped_fd = (long) hmap_get_l(gp->fd_map, t->fd); - if (t->mapped_fd == 0) { - // No corresponding virtual fd number mapping - t->has_fd = false; - - } else { - t->vfd = hmap_get_l(gp->vfd_map, t->mapped_fd); - if (!t->vfd) { - return ERROR; - } - t->mapped_fd = t->vfd->mapped_fd; - } - - return SUCCESS; -} diff --git a/ioreplay/src/generate/gprocess.h b/ioreplay/src/generate/gprocess.h deleted file mode 100644 index 47e5037..0000000 --- a/ioreplay/src/generate/gprocess.h +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GPROCESS_H -#define GPROCESS_H - -#include "../datas/hmap.h" -#include "../defaults.h" -#include "gtask.h" -#include "generate.h" - -// Forward declarations (header include hell) -struct gtask_s_; -struct generate_s_; - -/** - * @brief Virtual process object used for generating .replay file - * - * An object of this represents a Linux process in generate context. - * In Linux every process owns * its own file descriptor table which is - * simulated here. Usually, a Linux process re-uses a FD number once not used - * anymore (e.g. after a close). However, as we want to increase concurrency - * while replaying the I/O we want * to ensure to always use unique file - * descriptor IDs for every open. Thats why we use max_mapped_fd to always - * map a real FD number to a uniq virtual FD number. - */ -typedef struct gprocess_s_ { - long pid; /**< The real PID */ - long mapped_pid; /**< The mapped PID */ - hmap_s *vfd_map; /**< All virtual file descriptors of that process */ - hmap_s *fd_map; /**< All mappings from real fd to virtual fd */ - long max_mapped_fd; /**< The max mapped fd number */ -} gprocess_s; - -/** - * @brief Creates a new gprocess object - * - * @param pid The process ID - * @param mapped_pid the mapped PID - * @return The new gprocess object - */ -gprocess_s* gprocess_new(const long pid, const long mapped_pid); - -/** - * @brief Destroys a gprocess object - * - * @param gp The gprocess object - */ -void gprocess_destroy(gprocess_s *gp); - -/** - * @brief Creates a new virtual FD from a given real FD number - * - * In ioreplay we map the real file descriptor (the fd number protocolled in - * the.capture file) to a virtual file descriptor (the fd numner written to the - * .replay file). The purpose is to increase concurrency of the I/O during - * replay. Normally, a process would reuse the same file descriptor number - * once closed earlier. However, when replaying we can't reuse the number if - * we want to replay the I/O on multiple paths in parallel. Therefore, it is - * ensured that the virtual file descriptor number in the .replay file is - * always * unique for every open! - * - * @param gp The process object - * @param t The task object (the vfd pointer will be stored to * t->vfd) - * @param g The generate object - */ -void gprocess_create_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t, - struct generate_s_ *g); - -/** - * @brief Retrieves a virtual FD from a given real FD number - * - * @param gp The process object - * @param t The task object (the vfd pointer will be stored to * t->vfd) - * @return SUCCESS if everything went smothly! - */ -status_e gprocess_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t); - -#endif // GPROCESS_H diff --git a/ioreplay/src/generate/gtask.c b/ioreplay/src/generate/gtask.c deleted file mode 100644 index 55a1124..0000000 --- a/ioreplay/src/generate/gtask.c +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gtask.h" - -gtask_s* gtask_new(void *generate) -{ - gtask_s *t = Malloc(gtask_s); - - t->generate = generate; - t->line = NULL; - t->path_r = NULL; - t->path2_r = NULL; -#ifdef LOG_FILTERED - t->original_line = NULL; -#endif - - return t; -} - -void gtask_init(gtask_s *t, char *line, const unsigned long lineno) -{ - if (t->line) - free(t->line); - t->line = Clone(line); - - if (t->path_r) - free(t->path_r); - if (t->path2_r) - free(t->path2_r); - -#ifdef LOG_FILTERED - if (t->original_line) - free(t->original_line); - t->original_line = Clone(line); - t->filtered_where = NULL; -#endif - - t->bytes = -1; - t->address = 0; - t->address2 = 0; - t->count = -1; - t->F = -1; - t->fd = -1; - t->flags = -1; - t->G = -1; - t->has_fd = false; - t->vsize = NULL; - t->vsize2 = NULL; - t->lineno = lineno; - t->mapped_fd = -1; - t->mapped_time = -1; - t->mode = -1; - t->offset = -1; - t->op = NULL; - t->path2 = NULL; - t->path2_r = NULL; - t->path = NULL; - t->path_r = NULL; - t->gprocess = NULL; - t->pid = -1; - t->pidtid = NULL; - t->ret = 0; - t->status = -1; - t->tid = -1; - t->vfd = NULL; - t->whence = -1; -} - -void gtask_destroy(gtask_s *t) -{ - if (t->line) - free(t->line); - if (t->path_r) - free(t->path_r); - if (t->path2_r) - free(t->path2_r); - free(t); -} - diff --git a/ioreplay/src/generate/gtask.h b/ioreplay/src/generate/gtask.h deleted file mode 100644 index 2f364d3..0000000 --- a/ioreplay/src/generate/gtask.h +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GTASK_H -#define GTASK_H - -#include "vsize.h" - -#include "gprocess.h" -#include "../vfd.h" -#include "../datas/amap.h" -#include "../datas/hmap.h" -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "../mounts.h" -#include "../options.h" - -/** - * @brief The generate task definition - * - * The gtask holds all possible variables required to process a particular - * .capture line and to generate the corresponding .replay line - */ -typedef struct gtask_s_ { - bool has_fd; /**< True if task has a file descriptor number */ - char *line; /**< A pointer to the remaining part of the .capture line */ - char *op; /**< Operation/syscall name */ - char *path2; /**< A second path name (e.g. for rename) */ - char *path2_r; /**< Work around to track mallocs, so it can be freed */ - char *path; /**< Path name */ - char *path_r; /**< Work around to track mallocs, so it can be freed */ - char *pidtid; /**< String representing pid:tid */ - int F; /**< Arguments for fcntl syscall */ - int G; /**< Arguments for fcntl syscall */ - int fd; /**< File descriptor number */ - int flags; /**< File open flags */ - int mode; /**< File open mode */ - int ret; /**< ioreplay process status, SUCCESS if everything is alright */ - int status; /**< Operation/syscall return status */ - long address2; /**< Another address (used by mmap related syscalls) */ - long address; /**< An address (used by mmap related syscalls) */ - long bytes; /**< Amount of bytes */ - long count; /**< A count */ - long lineno; /**< The current line number */ - long mapped_fd; /**< The mapped file descriptor number */ - long mapped_time; /**< The mapped time */ - long offset; /**< A offset */ - long pid; /**< The process ID */ - long tid; /**< The thread ID */ - long whence; /**< Whence */ - vfd_s *vfd; /**< A pointer to the virtual file descriptor */ - struct gprocess_s_ *gprocess; /**< A pointer to the process object */ - void *generate; /**< A pointer to the generate object */ - vsize_s *vsize2; /**< Pointer to a second virtual size object */ - vsize_s *vsize; /**< Pointer to the virtual size object */ -#ifdef LOG_FILTERED - char *original_line; /**< Only used for debugging purposes */ - char *filtered_where; /**< Only used for debugging purposes */ -#endif -} gtask_s; - -/** - * @brief Creates a new task object - * - * @param generate A pointer to the generate object - * @return The new task object - */ -gtask_s* gtask_new(void *generate); - -/** - * @brief Initialises a taks object - * - * This function is used in particular when we recycle/reuse an old - * gtask object. - * - * @param t The gtask object - * @param line The corresponding line from the .capture file - * @param lineno The line number - */ -void gtask_init(gtask_s *t, char *line, const unsigned long lineno); - -/** - * @brief Destroys a given task object - * - * @param t The task object - */ -void gtask_destroy(gtask_s *t); - -#endif // GTASK_H diff --git a/ioreplay/src/generate/gwriter.c b/ioreplay/src/generate/gwriter.c deleted file mode 100644 index e0d448e..0000000 --- a/ioreplay/src/generate/gwriter.c +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gwriter.h" - -#include "gtask.h" -#include "generate.h" -#include "gioop.h" -#include "../opcodes.h" - -void* gwriter_pthread_start(void *data) -{ - gwriter_s *w = data; - generate_s *g = w->generate; - gtask_s *t = NULL; - - do { - while (NULL != (t = rbuffer_get_next(w->queue))) { -#ifdef LOG_FILTERED - // Logging filtered lines - if (SUCCESS != gioop_run(w, t)) { - fprintf(g->replay_fd, "#FILTERED @%ld %s", t->lineno, - t->original_line); - } -#else - gioop_run(w, t); -#endif - rbuffer_insert(g->reuse_queue, t); - } - usleep(100); - } while (!w->terminate); - - while (NULL != (t = rbuffer_get_next(w->queue))) { -#ifdef LOG_FILTERED - if (SUCCESS != gioop_run(w, t)) { - fprintf(g->replay_fd, "#FILTERED @%ld %s\n", t->lineno, - t->original_line); - } -#else - gioop_run(w, t); -#endif - rbuffer_insert(g->reuse_queue, t); - } - - return NULL; -} - -gwriter_s* gwriter_new(generate_s *g) -{ - gwriter_s *w = Malloc(gwriter_s); - - w->generate = g; - w->terminate = false; - w->queue = rbuffer_new(1024); - - return w;