diff options
Diffstat (limited to 'ioreplay')
70 files changed, 10172 insertions, 0 deletions
diff --git a/ioreplay/Makefile b/ioreplay/Makefile new file mode 100644 index 0000000..67f4421 --- /dev/null +++ b/ioreplay/Makefile @@ -0,0 +1,36 @@ +#DEBUG=-g3 -ggdb3 -pg +NAME=ioreplay +LIBS=-pthread +CFLAGS=-Wall -std=gnu99 -pedantic +STATIC=#-static +DESTDIR=/opt/ioreplay/bin +SRCS=$(wildcard src/*.c src/*/*.c) +HDRS=$(SRCS:.c=.h) +OBJS=$(SRCS:.c=.o) +all: compile +quick: clean ctags compile sudo_install +cshell: compile + gdb -ex='break main; run' --args ./$(NAME) +test: compile + gdb -ex=run --args ./$(NAME) -U +compile: $(OBJS) + $(CC) $(STATIC) $(DEBUG) $(LIBS) $(OBJS) -o $(NAME) +%.o: %.c %.h + $(CC) $(STATIC) $(DEBUG) $(LIBS) -c $(CFLAGS) $< -o $@ +clean: + rm -v ioreplay ./src/*.o ./src/*/*.o 2>/dev/null || exit 0 +install: + test ! -d $(DESTDIR) && mkdir -p $(DESTDIR) || exit 0 + cp -v $(NAME) $(DESTDIR) + @echo "Don't forget to add $(DESTDIR) to your PATH as follows:" + @echo " export PATH=\$$PATH:$(DESTDIR)" +uninstall: + test ! -z "$(DESTDIR)" && test -f $(DESTDIR)/$(NAME) && rm -v $(DESTDIR)/$(NAME) || exit 0 +deinstall: uninstall +astyle: + astyle -n --style=linux src/*.h src/*/*.h + astyle -n --style=linux src/*.c src/*/*.c +todo: + fgrep ../TODO ./src/* +ctags: + ctags ./src/*.{h,c} ./src/*/*.{h,c} diff --git a/ioreplay/src/capture/capture.c b/ioreplay/src/capture/capture.c new file mode 100644 index 0000000..0ac336b --- /dev/null +++ b/ioreplay/src/capture/capture.c @@ -0,0 +1,99 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "capture.h" + +#include <sys/utsname.h> + +status_e capture_run(options_s *opts) +{ + int status = 0; + struct utsname uts; + + // To make it nicer we should iterate over PATH instead + char *staprun_paths[3] = { + "/usr/bin/staprun", + "/usr/local/bin/staprun", + "/bin/staprun" + }; + int num_staprun_paths = 3; + + if (0 != uname(&uts)) { + Errno("Could not identify release of currently running Kernel!"); + } + + Put("Release of currently running Kernel: %s", uts.release); + char modules_dir[128]; + sprintf(modules_dir, "/opt/ioreplay/systemtap/%s", uts.release); + Put("Changing directory to module path: %s/", modules_dir); + + if (0 != chdir(modules_dir)) { + Errno("Could not change into '%s', please ensure that the compiled " + "SystemTap modules correspond to the currently running Kernel " + "and that these are installed properly!\n", + modules_dir); + } + + if (0 != access(opts->module, R_OK)) { + Errno("Module '%s/%s' can't be read, please make sure that the " + "SystemTap Kernel modules are installed!", + modules_dir, opts->module); + } + + char *staprun_path = NULL; + for (int i = 0; i < num_staprun_paths; ++i) { + if (0 == access(staprun_paths[i], X_OK)) { + staprun_path = staprun_paths[i]; + //Put("SystemTap command path: %s", staprun_path); + break; + } + } + + if (staprun_path == NULL) { + Errno("Can't find 'staprun' command, please ensure to have the SystemTap " + "runtime (usually package 'systemtap-runtime') installed!"); + } + + char staprun_command[128]; + if (opts->pid >= 0) { + sprintf(staprun_command, "%s %s -v -o %s -x %d", staprun_path, opts->module, + opts->capture_file, opts->pid); + } else { + sprintf(staprun_command, "%s %s -v -o %s", staprun_path, opts->module, + opts->capture_file); + } + + Out("NOTICE: It is good practise first to stop all processes, then to "); + Out("start capturing, and then to start all processes again. The reason "); + Out("is that processes may have already open file handles. In that case "); + Out("I/O Replay would be unable to replay these! This may be improved "); + Put("in a future release!"); + Put("To abort capturing now send Ctrl+C, otherwise wait 1h"); + Put("Capturing I/O via: '%s'", staprun_command); + + char buf[1024]; + FILE *fp; + + if ((fp = popen(staprun_command, "r")) == NULL) { + Errno("Unable to invoke staprun command!"); + } + while (fgets(buf, 1024, fp) != NULL) + Out("stapio: %s", buf); + + if (0 != pclose(fp)) { + Error("Problems invoking staprun command!"); + } + + return status; +} diff --git a/ioreplay/src/capture/capture.h b/ioreplay/src/capture/capture.h new file mode 100644 index 0000000..7718d3e --- /dev/null +++ b/ioreplay/src/capture/capture.h @@ -0,0 +1,30 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CAPTURE_H +#define CAPTURE_H + +#include "../defaults.h" +#include "../utils/futils.h" +#include "../options.h" + +/** + * @brief Captures I/O to a .capture file by using stap from SystemTap + * + * @param opts The options object + * @return SUCCESS if everything went fine + */ +status_e capture_run(options_s *opts); + +#endif // CAPTURE_H diff --git a/ioreplay/src/cleanup/cleanup.c b/ioreplay/src/cleanup/cleanup.c new file mode 100644 index 0000000..13c557c --- /dev/null +++ b/ioreplay/src/cleanup/cleanup.c @@ -0,0 +1,30 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cleanup.h" + +#include "../mounts.h" + +status_e cleanup_run(options_s *opts) +{ + drop_root(opts->user); + mounts_s *m = mounts_new(opts); + + if (opts->purge) + mounts_purge(m); + else + mounts_trash(m); + + return SUCCESS; +} diff --git a/ioreplay/src/cleanup/cleanup.h b/ioreplay/src/cleanup/cleanup.h new file mode 100644 index 0000000..127badf --- /dev/null +++ b/ioreplay/src/cleanup/cleanup.h @@ -0,0 +1,29 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CLEANUP_H +#define CLEANUP_H + +#include "../defaults.h" +#include "../options.h" + +/** + * @brief Cleans up all files and directories of a given test + * + * @brief opts The options object + * @return SUCCESS in case everything went fine + */ +status_e cleanup_run(options_s *opts); + +#endif // CLEANUP_H diff --git a/ioreplay/src/datas/amap.c b/ioreplay/src/datas/amap.c new file mode 100644 index 0000000..806a3f8 --- /dev/null +++ b/ioreplay/src/datas/amap.c @@ -0,0 +1,264 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "amap.h" + +/** + * @brief Creates a new array map + * + * @param size The array map size + * @param mmapped true if the memory should be mmapped + * @return The new amap object + */ +static amap_s *_amap_new(long size, bool mmapped) +{ + amap_s *a = NULL; + void ***arrays = NULL; + + // Calculate a multiple of 1024, but at least in size of 'size'. + if (size % 1024 != 0) { + size = 1024*(1+(long)(size/1024)); + } + + if (size < 1) { + Error("Size overflow"); + } + + int num_arrays = size / AMAP_MAX_ARRAY_LENGTH; + + if (mmapped) { + a = Mmapshared(amap_s); + arrays = Cmapshared(num_arrays, void**); + } else { + a = Malloc(amap_s); + arrays = Calloc(num_arrays, void**); + } + + for (int i = 0; i < num_arrays; ++i) { + if (mmapped) { + //Put("%d", AMAP_MAX_ARRAY_LENGTH); + arrays[i] = Cmapshared(AMAP_MAX_ARRAY_LENGTH, void*); + } else { + arrays[i] = Calloc(AMAP_MAX_ARRAY_LENGTH, void*); + } + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + arrays[i][j] = NULL; + } + } + + a->arrays = arrays; + a->num_arrays = num_arrays; + a->size = size; + a->data_destroy = NULL; + a->mmapped = mmapped; + + return a; +} + +/** + * @brief Creates a new array map + * + * @param size The array map size + * @return The new amap object + */ +amap_s* amap_new(const long size) +{ + return _amap_new(size, false); +} + +/** + * @brief Creates a new mmapped array map + * + * @param size The array map size + * @return The new amap object + */ +amap_s* amap_new_mmapped(const long size) +{ + return _amap_new(size, true); +} + +/** + * @brief Destroys a mmap object + * + * @a The new amap object + */ +void amap_destroy(amap_s* a) +{ + if (!a) { + return; + } + + // Don't bother, the mmapped version of amap will stay alive until + // process terminations. And after process termination everything + // will be cleaned up automatically by Linux. + if (a->mmapped) { + return; + } + + for (int i = 0; i < a->num_arrays; ++i) { + if (a->data_destroy) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) + if (a->arrays[i][j]) { + a->data_destroy(a->arrays[i][j]); + } + } + free(a->arrays[i]); + } + free(a->arrays); + free(a); +} + +/** + * @brief Resets a mmap object + * + * This resets all entries to NULL. + * + * @a The new amap object + */ +void amap_reset(amap_s* a) +{ + for (int i = 0; i < a->num_arrays; ++i) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + if (a->data_destroy) { + if (a->arrays[i][j]) { + a->data_destroy(a->arrays[i][j]); + } + } + a->arrays[i][j] = NULL; + } + } +} + +int amap_set(amap_s *a, const long position, void* value) +{ + if (position >= a->size) + return -1; + int which_array = position / AMAP_MAX_ARRAY_LENGTH; + int array_pos = position % AMAP_MAX_ARRAY_LENGTH; + a->arrays[which_array][array_pos] = value; + return 0; +} + +void* amap_get(amap_s *a, const long position) +{ + if (position >= a->size) + return NULL; + int which_array = position / AMAP_MAX_ARRAY_LENGTH; + int array_pos = position % AMAP_MAX_ARRAY_LENGTH; + return a->arrays[which_array][array_pos]; +} + +void* amap_unset(amap_s *a, const long position) +{ + if (position >= a->size) + return NULL; + int which_array = position / AMAP_MAX_ARRAY_LENGTH; + int array_pos = position % AMAP_MAX_ARRAY_LENGTH; + void *value = a->arrays[which_array][array_pos]; + a->arrays[which_array][array_pos] = NULL; + return value; +} + +void amap_run_cb(amap_s *a, void (*cb)(void *data)) +{ + for (int i = 0; i < a->num_arrays; ++i) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + if (a->arrays[i][j]) + cb(a->arrays[i][j]); + } + } +} + +void amap_print(amap_s* a) +{ + Put("amap_s (%p):", (void*)a); + Put("\tmmapped: %d", a->mmapped); + Put("\tmax_array_length: %d", AMAP_MAX_ARRAY_LENGTH); + Put("\tnum_arrays: %d", a->num_arrays); + Put("\tsize: %lu", a->size); + Out("\toccupied slots: "); + for (int i = 0; i < a->num_arrays; ++i) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + if (a->arrays[i][j] != NULL) { + Out("%d:%d ", i, j); + } + } + } + Out("\n"); +} + +void _amap_test(amap_s *a) +{ + assert(0 == amap_set(a, 0, (void*)10)); + assert(0 == amap_set(a, 1, (void*)11)); + assert(0 == amap_set(a, 2, (void*)12)); + assert(0 == amap_set(a, 3, (void*)a)); + assert(10 == (long) amap_get(a, 0)); + assert(11 == (long) amap_get(a, 1)); + assert(12 == (long) amap_get(a, 2)); + assert(a == amap_get(a, 3)); + + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH-1, (void*) 23)); + assert(23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH-1)); + + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH, (void*) 42)); + assert(42 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH)); + + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2-1, (void*) (23+42))); + assert(42+23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH*2-1)); + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2, (void*) 23)); + + + assert(NULL == amap_get(a, 1024*1024*9-1)); + assert(0 == amap_set(a, 1024*1024*9-1, (void*) 0x1)); + assert(0x1 == (long) amap_get(a, 1024*1024*9-1)); + assert(0x1 == (long) amap_unset(a, 1024*1024*9-1)); + assert(NULL == amap_get(a, 1024*1024*9-1)); + + assert(0 == amap_set(a, 1024*1024*9, (void*) 100)); + assert(100 == (long) amap_get(a, 1024*1024*9)); + + assert(0 == amap_set(a, 1024*1024*9+1, (void*) 101)); + assert(101 == (long) amap_get(a, 1024*1024*9+1)); + + assert(0 == amap_set(a, 1024*1024*10-2, (void*) 102)); + assert(102 == (long) amap_get(a, 1024*1024*10-2)); + + assert(0 == amap_set(a, 1024*1024*10-1, a)); + assert(a == amap_get(a, 1024*1024*10-1)); + //amap_print(a); + + assert(a == amap_unset(a, 1024*1024*10-1)); + assert(a != amap_unset(a, 1024*1024*10-1)); + //amap_print(a); +} + +void amap_test(void) +{ + // First test the non-mmapped version + amap_s* a = amap_new(1024*1024*10); + _amap_test(a); + amap_destroy(a); + + // Now test the mapped version + a = amap_new_mmapped(1024*1024*10); + _amap_test(a); + amap_destroy(a); + + // Another test with non-alligned size + a = amap_new(1024*1024*10+1); + _amap_test(a); + amap_destroy(a); +} + diff --git a/ioreplay/src/datas/amap.h b/ioreplay/src/datas/amap.h new file mode 100644 index 0000000..882a7c5 --- /dev/null +++ b/ioreplay/src/datas/amap.h @@ -0,0 +1,49 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef AMAP_H +#define AMAP_H + +#include "../defaults.h" + +#define AMAP_MAX_ARRAY_LENGTH 1024*8 + +/** + * @brief Implements an array map data structure + * + * This array map can hold a HUGE amount of entries by allocating multiple + * smaller arrays. There are two version of the amap data structure available: + * a memory mapped (mmap) and a normal version. The memory mapped version can + * be used for IPC between various processes. + */ +typedef struct amap_s_ { + void*** arrays; /**< The pointers to the amap arrays */ + int num_arrays; /**< The amount of arrays used in the amap */ + long size; /**< The total size/capacity of the amap */ + bool mmapped; /**< True if amap is memory mapped */ + void (*data_destroy)(void *data); /**< Callback to destroy all elements */ +} amap_s; + +amap_s* amap_new(const long size); +amap_s* amap_new_mmapped(const long size); +int amap_set(amap_s *a, const long position, void* value); +void* amap_get(amap_s *a, const long position); +void* amap_unset(amap_s *a, const long position); +void amap_print(amap_s *a); +void amap_destroy(amap_s *a); +void amap_reset(amap_s *a); +void amap_run_cb(amap_s *a, void (*cb)(void *data)); +void amap_test(void); + +#endif // AMAP_H diff --git a/ioreplay/src/datas/btree.c b/ioreplay/src/datas/btree.c new file mode 100644 index 0000000..da5da48 --- /dev/null +++ b/ioreplay/src/datas/btree.c @@ -0,0 +1,169 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "btree.h" + +btree_s* btree_new() +{ + btree_s *b = Malloc(btree_s); + *b = (btree_s) { + .root = NULL, .size = 0 + }; + return b; +} + +void btree_destroy(btree_s* b) +{ + if (b->root) + btreelem_destroy_r(b->root); + free(b); +} + +void btree_destroy2(btree_s* b) +{ + if (b->root) + btreelem_destroy_r2(b->root); + free(b); +} + +int btree_insert(btree_s* b, int key, void *data) +{ + int ret = 1; + + if (b->root == NULL) { + b->root = btreelem_new(key, data); + ret = 0; + } else { + ret = btreelem_insert_r(b->root, key, data); + } + + if (ret == 0) { + b->size++; + } + + return ret; +} + +void* btree_get(btree_s* b, int key) +{ + if (b->root == NULL) + return NULL; + + return btreelem_get_r(b->root, key); +} + +void btree_print(btree_s* b) +{ + btreelem_print_r(b->root, 0); +} + +btreelem_s* btreelem_new(int key, void *data) +{ + btreelem_s *e = Malloc(btreelem_s); + *e = (btreelem_s) { + .key = key, .data = data, .left = NULL, .right = NULL + }; + return e; +} + +void btreelem_destroy_r(btreelem_s* e) +{ + if (e->left) { + btreelem_destroy_r(e->left); + } + if (e->right) { + btreelem_destroy_r(e->right); + } + + free(e); +} + +void btreelem_destroy_r2(btreelem_s* e) +{ + if (e->left) + btreelem_destroy_r(e->left); + if (e->right) + btreelem_destroy_r(e->right); + if (e->data) + btree_destroy(e->data); + + free(e); +} + +int btreelem_insert_r(btreelem_s* e, int key, void *data) +{ + int ret = 0; + + if (e->key == key) { + ret = 1; + } + + else if (e->key > key) { + if (e->left == NULL) { + e->left = btreelem_new(key, data); + } else { + ret = btreelem_insert_r(e->left, key, data); + } + } + + else { + if (e->right == NULL) { + e->right = btreelem_new(key, data); + } else { + ret = btreelem_insert_r(e->right, key, data); + } + } + + return ret; +} + +void* btreelem_get_r(btreelem_s* e, int key) +{ + void *data = NULL; + + if (e->key == key) { + data = e->data; + } + + else if (e->key > key) { + if (e->left) { + data = btreelem_get_r(e->left, key); + } + } + + else { + if (e->right) { + data = btreelem_get_r(e->right, key); + } + } + + return data; +} + +void btreelem_print_r(btreelem_s* e, int depth) +{ + for (int i = 0; i < depth; ++i) { + Out(" "); + } + Put("%d\n", e->key); + + if (e->left) { + btreelem_print_r(e->left, depth); + } + + if (e->right) { + btreelem_print_r(e->right, depth+1); + } +} + diff --git a/ioreplay/src/datas/btree.h b/ioreplay/src/datas/btree.h new file mode 100644 index 0000000..55da560 --- /dev/null +++ b/ioreplay/src/datas/btree.h @@ -0,0 +1,52 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BTREE_H +#define BTREE_H + +#include "../defaults.h" + +/** + * @brief This defines an element of the binary tree data structure + */ +typedef struct btreelem_ { + struct btreelem_ *left; /**< The next element to the left */ + struct btreelem_ *right; /**< The next element to the right */ + int key; /**< The key of the element */ + void *data; /**< A pointer to the data stored in this element */ +} btreelem_s; + +/** + * @brief This defines a binary tree data structure. + */ +typedef struct btree_s_ { + btreelem_s *root; /**< The root element */ + int size; /**< The current size of the binary tree */ +} btree_s; + +btree_s* btree_new(); +void btree_destroy(btree_s *b); +void btree_destroy2(btree_s *b); +int btree_insert(btree_s *b, int key, void *data); +void* btree_get(btree_s *b, int key); +void btree_print(btree_s *b); + +btreelem_s* btreelem_new(int key, void *data); +void btreelem_destroy_r(btreelem_s *e); +void btreelem_destroy_r2(btreelem_s *e); +int btreelem_insert_r(btreelem_s *e, int key, void *data); +void* btreelem_get_r(btreelem_s *e, int key); +void btreelem_print_r(btreelem_s *e, int depth); + +#endif // BTREE_H diff --git a/ioreplay/src/datas/hmap.c b/ioreplay/src/datas/hmap.c new file mode 100644 index 0000000..d0e1d70 --- /dev/null +++ b/ioreplay/src/datas/hmap.c @@ -0,0 +1,364 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hmap.h" + +#define _Using_string_keys h->keys != NULL + +unsigned int hmap_get_addr(hmap_s *h, char *key) +{ + unsigned long hash = 5381; + int len = strlen(key); + + for (int i = 0; i < len; ++i) { + hash = ((hash << 5) + hash) + key[i]; /* hash * 33 + c */ + } + + return (unsigned int) (hash % h->size); +} + +unsigned int hmap_get_addr_l(hmap_s *h, const long key) +{ + return (unsigned int) (key % h->size); +} + +hmap_s *_hmap_new(unsigned int init_size) +{ + hmap_s *h = Malloc(hmap_s); + + h->size = init_size; + h->data = Calloc(init_size, void*); + h->l = Calloc(init_size, list_s*); + h->data_destroy = NULL; + h->keys = NULL; + h->keys_l = NULL; + + Mset(h->data, 0, init_size, void*); + Mset(h->l, 0, init_size, list_s*); + + return h; +} + +hmap_s *hmap_new(unsigned int init_size) +{ + hmap_s *h = _hmap_new(init_size); + h->keys = Calloc(init_size, char*); + Mset(h->keys, 0, init_size, char*); + + return h; +} + +hmap_s *hmap_new_l(unsigned int init_size) +{ + hmap_s *h = _hmap_new(init_size); + h->keys_l = Calloc(init_size, int); + Mset(h->keys_l, -1, init_size, int); + + return h; +} + +void hmap_destroy(hmap_s *h) +{ + for (int i = 0; i < h->size; ++i) { + if (h->l[i]) { + list_s *l = h->l[i]; + if (h->data_destroy) + l->data_destroy = h->data_destroy; + list_destroy(h->l[i]); + } + if (h->data[i] && h->data_destroy) { + h->data_destroy(h->data[i]); + } + } + + free(h->data); + if (h->keys) + free(h->keys); + if (h->keys_l) + free(h->keys_l); + free(h->l); + free(h); + + return; +} + +int hmap_insert(hmap_s *h, char *key, void *data) +{ + if (data == NULL) { + Error("insert data can not be NULL"); + } + + int addr = hmap_get_addr(h, key); + + if (h->data[addr]) { + + if (strcmp(key, h->keys[addr]) == 0) { + // Key already exists + return 0; + } + + // There is already data, collision, create a linked list + list_s *l = h->l[addr] = list_new(); + list_key_insert(l, h->keys[addr], h->data[addr]); + list_key_insert(l, key, data); + + // Not needed anymore, as the elements are in the linked list now. + free(h->keys[addr]); + h->data[addr] = h->keys[addr] = NULL; + + return 1; + + } else if (h->l[addr]) { + // There was a collision at this address before. Insert + // the element to the linked list. Returns 0 if key is already + // in the list (no additional insert made) or 1 otherwise. + return list_key_insert(h->l[addr], key, data); + } + + // New entry on a collision free address + h->data[addr] = data; + h->keys[addr] = Clone(key); + + return 1; +} + +int hmap_insert_l(hmap_s *h, const long key, void *data) +{ + if (data == NULL) { + Error("insert data can not be NULL"); + } + + int addr = hmap_get_addr_l(h, key); + + if (h->data[addr]) { + + if (key == h->keys_l[addr]) { + // Key already exists + return 0; + } + + // There is already data, collision, create a linked list + list_s *l = h->l[addr] = list_new_l(); + list_key_insert_l(l, h->keys_l[addr], h->data[addr]); + list_key_insert_l(l, key, data); + + // Not needed anymore, as the elements are in the linked list now. + h->data[addr] = NULL; + h->keys_l[addr] = -1; + + return 1; + + } else if (h->l[addr]) { + // There was a collision at this address before. Insert + // the element to the linked list. Returns 0 if key is already + // in the list (no additional insert made) or 1 otherwise. + return list_key_insert_l(h->l[addr], key, data); + } + + // New entry on a collision free address + h->data[addr] = data; + h->keys_l[addr] = key; + + return 1; +} + +void* hmap_remove(hmap_s *h, char *key) +{ + int addr = hmap_get_addr(h, key); + + if (h->data[addr] != NULL) { + void *data = h->data[addr]; + free(h->keys[addr]); + h->data[addr] = h->keys[addr] = NULL; + return data; + + } else if (h->l[addr] != NULL) { + // There was a collision at this address before. Remove + // the element to the linked list. Returns the object if key is + // already in the list (no additional insert made) or NULL + // otherwise. + return list_key_remove(h->l[addr], key); + } + + // Key is not present + return NULL; +} + +void* hmap_remove_l(hmap_s *h, const long key) +{ + int addr = hmap_get_addr_l(h, key); + + if (h->data[addr] != NULL) { + void *data = h->data[addr]; + h->data[addr] = NULL; + h->keys_l[addr] = -1; + return data; + + } else if (h->l[addr] != NULL) { + // There was a collision at this address before. Remove + // the element to the linked list. Returns the object if key is + // already in the list (no additional insert made) or NULL + // otherwise. + return list_key_remove_l(h->l[addr], key); + } + + // Key is not present + return NULL; +} + +void* hmap_get(hmap_s *h, char *key) +{ + int addr = hmap_get_addr(h, key); + if (h->data[addr] && strcmp(h->keys[addr], key) == 0) { + return h->data[addr]; + + } else if (h->l[addr]) { + return list_key_get(h->l[addr], key); + } + + return NULL; +} + +void* hmap_get_l(hmap_s *h, const long key) +{ + int addr = hmap_get_addr_l(h, key); + if (h->data[addr] && h->keys_l[addr] == key) { + return h->data[addr]; + + } else if (h->l[addr]) { + return list_key_get_l(h->l[addr], key); + } + + return NULL; +} + +void hmap_run_cb(hmap_s* h, void (*cb)(void *data)) +{ + for (int i = 0; i < h->size; ++i) { + if (h->l[i]) { + list_s *l = h->l[i]; + list_run_cb(l, cb); + } + if (h->data[i]) { + cb(h->data[i]); + } + } +} + +void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_) +{ + for (int i = 0; i < h->size; ++i) { + if (h->l[i]) { + list_s *l = h->l[i]; + list_run_cb2(l, cb, data_); + } + if (h->data[i]) { + cb(h->data[i], data_); + } + } +} + +void hmap_print(hmap_s *h) +{ + for (int i = 0; i < h->size; ++i) { + if (h->data[i]) { + if (_Using_string_keys) { + Put("hmap:%p addr:%d key:'%s'", (void*)h, i, h->keys[i]); + } else { + Put("hmap:%p addr:%d key:%d", (void*)h, i, h->keys_l[i]); + } + } else if (h->l[i]) { + Put("hmap:%p addr:%d LIST", (void*)h, i); + list_print(h->l[i]); + } + } +} + +static void _hmap_test(hmap_s *h) +{ + void* somedata = (void*)h; + + assert(1 == hmap_insert(h, "someval", (void*)23)); + assert(1 == hmap_insert(h, "another value", (void*)123)); + + assert(1 == hmap_insert(h, "mimecast", somedata)); + assert(0 == hmap_insert(h, "mimecast", somedata)); + assert(1 == hmap_insert(h, "is", somedata)); + assert(1 == hmap_insert(h, "hiring", somedata)); + + assert(NULL != hmap_get(h, "mimecast")); + assert(NULL == hmap_get(h, "Mimecast")); + + assert(NULL != hmap_remove(h, "mimecast")); + assert(NULL == hmap_remove(h, "mimecast")); + + assert(1 == hmap_insert(h, "mimecast", somedata)); + assert(NULL != hmap_get(h, "mimecast")); + + assert(23 == (long)hmap_get(h, "someval")); + assert(23 == (long)hmap_get(h, "someval")); + + assert(123 == (long)hmap_remove(h, "another value")); + assert(0 == (long)hmap_remove(h, "another value")); + assert(NULL == hmap_get(h, "another value")); + + //hmap_print(h); +} + +static void _hmap_test_l(hmap_s *h) +{ + void* somedata = (void*)h; + + assert(1 == hmap_insert_l(h, 1, (void*)23)); + + assert(1 == hmap_insert_l(h, 1, (void*)23)); + assert(1 == hmap_insert_l(h, 5, (void*)123)); + + assert(1 == hmap_insert_l(h, 3, somedata)); + assert(0 == hmap_insert_l(h, 3, somedata)); + assert(1 == hmap_insert_l(h, 4, somedata)); + assert(1 == hmap_insert_l(h, 6, somedata)); + + assert(NULL != hmap_get_l(h, 3)); + assert(NULL == hmap_get_l(h, 7)); + + assert(NULL != hmap_remove_l(h, 3)); + assert(NULL == hmap_remove_l(h, 3)); + + assert(1 == hmap_insert_l(h, 3, somedata)); + assert(NULL != hmap_get_l(h, 3)); + + assert(23 == (long)hmap_get_l(h, 1)); + assert(23 == (long)hmap_get_l(h, 1)); + + assert(123 == (long)hmap_remove_l(h, 5)); + assert(0 == (long)hmap_remove_l(h, 5)); + assert(NULL == hmap_get_l(h, 5)); +} + +void hmap_test(void) +{ + hmap_s* h = hmap_new(1024); + _hmap_test(h); + hmap_destroy(h); + + h = hmap_new(2); + _hmap_test(h); + hmap_destroy(h); + + h = hmap_new_l(1024); + _hmap_test_l(h); + hmap_print(h); + hmap_destroy(h); +} diff --git a/ioreplay/src/datas/hmap.h b/ioreplay/src/datas/hmap.h new file mode 100644 index 0000000..9d1978b --- /dev/null +++ b/ioreplay/src/datas/hmap.h @@ -0,0 +1,56 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HMAP_H +#define HMAP_H + +#include "../defaults.h" +#include "list.h" + +/** + * @brief A hash map data structure + * + * There are two version of this hmap data structure. One version is utilising + * string keys and the other one is utilising long keys. + * + * On hash collision the data structure will make use of a "named" linked list, + * whereas every member of the linked list has either a string key or a long + * key associated. + */ +typedef struct hmap_s_ { + char **keys; /**< List of all keys, NULL if nothing at a address */ + int *keys_l; /**< Same as keys, but for long keys */ + void **data; /**< Pointers to the stored data, NULL if nothing there */ + list_s **l; /**< Pointers to the linked lists, used on hash collision */ + void (*data_destroy)(void *data); /**< Callback to destroy all data */ + unsigned int size; /**< Size of the hmap */ +} hmap_s; + +hmap_s* hmap_new(unsigned int init_size); +hmap_s* hmap_new_l(unsigned int init_size); +void hmap_destroy(hmap_s* h); +void hmap_run_cb(hmap_s* h, void (*cb)(void *data)); +void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_); +int hmap_insert_l(hmap_s* h, const long key, void *data); +int hmap_insert(hmap_s* h, char* key, void *data); +void* hmap_remove_l(hmap_s* h, const long key); +void* hmap_remove(hmap_s* h, char* key); +void* hmap_get_l(hmap_s* h, const long key); +void* hmap_get(hmap_s* h, char* key); +unsigned int hmap_get_addr_l(hmap_s* h, const long key); +unsigned int hmap_get_addr(hmap_s* h, char* key); +void hmap_print(hmap_s* h); +void hmap_test(void); + +#endif // HMAP_H diff --git a/ioreplay/src/datas/list.c b/ioreplay/src/datas/list.c new file mode 100644 index 0000000..9cc78db --- /dev/null +++ b/ioreplay/src/datas/list.c @@ -0,0 +1,279 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "list.h" + + +list_s *list_new() +{ + list_s *l = Malloc(list_s); + *l = (list_s) { + .first = NULL, .data_destroy = NULL + }; + return l; +} + +list_s *list_new_l() +{ + return list_new(); +} + +void list_destroy(list_s *l) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key) + free(current->key); + if (current->data && l->data_destroy) + l->data_destroy(current->data); + list_elem_s *next = current->next; + free(current); + current = next; + } + + free(l); +} + +int list_key_insert(list_s *l, char *key, void *data) +{ + list_elem_s *current = l->first; + + while (current) { + // Already in the list + if (strcmp(current->key, key) == 0) + return 0; + current = current->next; + } + + list_elem_s *e = Malloc(list_elem_s); + + e->prev = NULL; + e->next = l->first; + e->key = Clone(key); + e->key_l = -1; + e->data = data; + + if (l->first) { + l->first->prev = e; + l->first = e; + + } else { + l->first = e; + } + + return 1; +} + +int list_key_insert_l(list_s *l, const long key, void *data) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key_l == key) + return 0; + current = current->next; + } + + list_elem_s *e = Malloc(list_elem_s); + + e->prev = NULL; + e->next = l->first; + e->key = NULL; + e->key_l = key; + e->data = data; + + if (l->first) { + l->first->prev = e; + l->first = e; + + } else { + l->first = e; + } + + return 1; +} + +void _list_elem_remove(list_s *l, list_elem_s *e) +{ + if (l->first == e) { + list_elem_s *first = e->next; + if (first) + first->prev = NULL; + l->first = first; + + } else { + list_elem_s *prev = e->prev; + list_elem_s *next = e->next; + + prev->next = next; + if (next) + next->prev = prev; + } + + if (e->key) + free(e->key); + free(e); +} + +void* list_key_remove(list_s *l, char *key) +{ + list_elem_s *current = l->first; + + while (current) { + if (strcmp(current->key, key) == 0) { + void *data = current->data; + _list_elem_remove(l, current); + return data; + } + current = current->next; + } + + return NULL; +} + +void* list_key_remove_l(list_s *l, const long key) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key_l == key) { + void *data = current->data; + _list_elem_remove(l, current); + return data; + } + current = current->next; + } + + return NULL; +} + +void* list_key_get(list_s *l, char *key) +{ + list_elem_s *current = l->first; + + while (current) { + if (strcmp(current->key, key) == 0) + return current->data; + current = current->next; + } + + return NULL; +} + +void* list_key_get_l(list_s *l, const long key) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key_l == key) + return current->data; + current = current->next; + } + + return NULL; +} + +void list_run_cb(list_s* l, void (*cb)(void *data)) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->data) + cb(current->data); + current = current->next; + } +} + +void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->data) + cb(current->data, data_); + current = current->next; + } +} + +void list_print(list_s *l) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key != NULL) { + Put("list:%p key:'%s' data:%p", (void*)l, + current->key, current->data); + } else { + Put("list:%p key:%ld data:%p", (void*)l, + current->key_l, current->data); + } + current = current->next; + } +} + +void list_test(void) +{ + list_s *l = list_new(); + void* somedata = (void*)l; + + assert(1 == list_key_insert(l, "foo", (void*)1)); + assert(1 == list_key_insert(l, "bar", (void*)2)); + assert(1 == list_key_insert(l, "baz", (void*)3)); + assert(2 == (long)list_key_remove(l, "bar")); + assert(1 == (long)list_key_remove(l, "foo")); + assert(3 == (long)list_key_remove(l, "baz")); + + assert(1 == list_key_insert(l, "I/O replay", somedata)); + assert(1 == list_key_insert(l, "for", somedata)); + assert(1 == list_key_insert(l, "benchmarking your server", somedata)); + assert(0 == list_key_insert(l, "for", somedata)); + + assert(NULL != list_key_get(l, "benchmarking your server")); + assert(NULL == list_key_get(l, "Mimecast")); + + assert(NULL != list_key_remove(l, "benchmarking your server")); + assert(NULL == list_key_remove(l, "benchmarking your server")); + assert(1 == list_key_insert(l, "benchmarking your server", somedata)); + + assert(1 == list_key_insert(l, "MiMecast", (void*)42)); + assert(42 == (long)list_key_get(l, "MiMecast")); + + l = list_new_l(); + + assert(1 == list_key_insert_l(l, 1, (void*)1)); + assert(1 == list_key_insert_l(l, 2, (void*)2)); + assert(1 == list_key_insert_l(l, 3, (void*)3)); + assert(1 == (long)list_key_get_l(l, 1)); + assert(1 == (long)list_key_remove_l(l, 1)); + assert(1 != (long)list_key_remove_l(l, 1)); + assert(3 == (long)list_key_remove_l(l, 3)); + + assert(1 == list_key_insert_l(l, 1234, somedata)); + assert(1 == list_key_insert_l(l, 13, somedata)); + assert(1 == list_key_insert_l(l, 666, somedata)); + assert(0 == list_key_insert_l(l, 13, somedata)); + + assert(NULL != list_key_get_l(l, 666)); + assert(NULL == list_key_get_l(l, 777)); + + assert(NULL != list_key_remove_l(l, 666)); + assert(NULL == list_key_remove_l(l, 666)); + assert(1 == list_key_insert_l(l, 666, somedata)); + + assert(1 == list_key_insert_l(l, 42, (void*)42)); + assert(42 == (long)list_key_get_l(l, 42)); + + //list_print(l); +} diff --git a/ioreplay/src/datas/list.h b/ioreplay/src/datas/list.h new file mode 100644 index 0000000..385333c --- /dev/null +++ b/ioreplay/src/datas/list.h @@ -0,0 +1,56 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef LIST_H +#define LIST_H + +#include "../defaults.h" + +/** + * @brief Definition of a linked list element + */ +typedef struct list_elem_s_ { + struct list_elem_s_ *prev; /**< The previous element */ + struct list_elem_s_ *next; /**< The next element */ + char *key; /**< The key of the lemenet */ + long key_l; /**< The same as key, but for long keys */ + void *data; /**< Pointer to the stored data */ +} list_elem_s; + +/** + * @brief Definition of a named linked list data structure + * + * There are two version of this list data structure. One version is utilising + * string keys and the other one is utilising long keys. + */ +typedef struct list_s_ { + list_elem_s *first; /**< The first element, NULL if list empty */ + void (*data_destroy)(void *data); /**< Callback to destroy all data */ +} list_s; + +list_s* list_new(); +list_s* list_new_l(); +void list_destroy(list_s* l); +void list_run_cb(list_s* l, void (*cb)(void *data)); +void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_); +int list_key_insert(list_s* l, char *key, void *data); +int list_key_insert_l(list_s* l, const long key, void *data); +void* list_key_remove(list_s* l, char *key); +void* list_key_remove_l(list_s* l, const long key); +void* list_key_get(list_s* l, char *key); +void* list_key_get_l(list_s* l, const long key); +void list_print(list_s* l); +void list_test(); + +#endif // LIST_H diff --git a/ioreplay/src/datas/rbuffer.c b/ioreplay/src/datas/rbuffer.c new file mode 100644 index 0000000..c019e6c --- /dev/null +++ b/ioreplay/src/datas/rbuffer.c @@ -0,0 +1,147 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rbuffer.h" + +rbuffer_s *rbuffer_new(const int size) +{ + rbuffer_s *r = Malloc(rbuffer_s); + + r->size = size; + r->read_pos = size-1; + r->write_pos = 0; + r->ring = Calloc(size, void*); + + Mset(r->ring, 0, size, void*); + + return r; +} + +void rbuffer_destroy(rbuffer_s *r) +{ + if (r) { + free(r->ring); + free(r); + } +} + +bool rbuffer_insert(rbuffer_s* r, void *data) +{ + if (r->write_pos == r->read_pos) + // Ring buffer is full + return false; + + r->ring[r->write_pos] = data; + r->write_pos = (r->write_pos+1) % r->size; + + return true; +} + +bool rbuffer_has_next(rbuffer_s* r) +{ + sig_atomic_t read_pos = (r->read_pos+1) % r->size; + + if (read_pos == r->write_pos) + // No more items to read, buffer is empty + { + return false; + } + + return true; +} + +void* rbuffer_get_next(rbuffer_s* r) +{ + sig_atomic_t read_pos = (r->read_pos+1) % r->size; + + if (read_pos == r->write_pos) + // No more items to read, buffer is empty + { + return NULL; + } + + void *data = r->ring[read_pos]; + r->ring[read_pos] = NULL; + r->read_pos = read_pos; + + return data; +} + +void rbuffer_print(rbuffer_s* r) +{ + Put("rbuffer_s (%p):", (void*)r); + Put("\tsize: %d", (int)r->size); + Put("\tread_pos: %d", r->read_pos); + Put("\twrite_pos: %d", r->write_pos); + Out("\toccupied slots: "); + for (int i = 0; i < r->size; ++i) + if (r->ring[i]) { + Out("%d:%p ", i, r->ring[i]); + } + Out("\n"); +} + +void rbuffer_test(void) +{ + rbuffer_s *r = rbuffer_new(5); + assert(NULL == rbuffer_get_next(r)); + + assert(rbuffer_insert(r, (void*)1)); + assert(rbuffer_insert(r, (void*)2)); + assert(rbuffer_insert(r, (void*)3)); + assert(rbuffer_insert(r, (void*)4)); + assert(!rbuffer_insert(r, (void*)5)); + rbuffer_print(r); + + assert(rbuffer_has_next(r)); + assert(1 == (long) rbuffer_get_next(r)); + assert(2 == (long) rbuffer_get_next(r)); + assert(3 == (long) rbuffer_get_next(r)); + assert(4 == (long) rbuffer_get_next(r)); + assert(!rbuffer_has_next(r)); + assert(NULL == rbuffer_get_next(r)); + + assert(rbuffer_insert(r, (void*)1)); + assert(1 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)2)); + assert(2 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)3)); + assert(3 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)4)); + assert(4 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)5)); + assert(5 == (long) rbuffer_get_next(r)); + assert(NULL == rbuffer_get_next(r)); + rbuffer_print(r); + + assert(rbuffer_insert(r, (void*)1)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)2)); + assert(1 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)3)); + assert(2 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)4)); + assert(3 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)5)); + rbuffer_print(r); + assert(4 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(5 == (long) rbuffer_get_next(r)); + assert(NULL == rbuffer_get_next(r)); + + rbuffer_destroy(r); +} diff --git a/ioreplay/src/datas/rbuffer.h b/ioreplay/src/datas/rbuffer.h new file mode 100644 index 0000000..fa634de --- /dev/null +++ b/ioreplay/src/datas/rbuffer.h @@ -0,0 +1,102 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RBUFFER_H +#define RBUFFER_H + +#include "signal.h" + +#include "../defaults.h" + +/** + * @brief An atomic ring buffer data type definition + * + * This data structure can be used for the common producer/consumer problem. + * As long as there is only max one producer thread and max one consumer thread + * it can be used without any mutex locking. All the operations are atomic. + */ +typedef struct rbuffer_s_ { + /** + * The positions are atomic, means the ring buffer can be accessed from + * multiple threads concurrently (one producer and one consumer thread). + * This is the current read position. + */ + sig_atomic_t read_pos; + /** + * This is the current write position. + */ + sig_atomic_t write_pos; + /** + * Holds the pointers to the actual ring data stored in the ring buffer + */ + void **ring; + /** + * Determines how many elements the ring buffer can hold. The capacity + * will be size-1 though, as we need one empty slot. + */ + int size; +} rbuffer_s; + +/** + * @brief Creates a new ring buffer + * + * @param size The size of the ring buffer + * @return The new ring buffer object + */ +rbuffer_s* rbuffer_new(const int size); + +/** + * @brief Destroys a ring buffer + * + * @param r The ring buffer object + */ +void rbuffer_destroy(rbuffer_s* r); + +/** + * @brief Inserts data pointer to the ring buffer + * + * @param r The ring buffer object + * @param data The data pointer + */ +bool rbuffer_insert(rbuffer_s* r, void *data); + +/** + * @brief Determines whether there is any data in the ring buffer + * + * @param r The ring buffer object + * @return True if there is any data, false otherwise + */ +bool rbuffer_has_next(rbuffer_s* r); + +/** + * @brief Returns and removes the next element from the ring buffer + * + * @param r The ring buffer object + * @return The data pointer + */ +void* rbuffer_get_next(rbuffer_s* r); + +/** + * @brief Prints a ring buffer + * + * @param r The ring buffer object + */ +void rbuffer_print(rbuffer_s* r); + +/** + * @brief Unit tests the ring buffer + */ +void rbuffer_test(void); + +#endif // RBUFFER_H diff --git a/ioreplay/src/datas/stack.c b/ioreplay/src/datas/stack.c new file mode 100644 index 0000000..94e83e3 --- /dev/null +++ b/ioreplay/src/datas/stack.c @@ -0,0 +1,85 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "stack.h" + + +stack_s *stack_new() +{ + stack_s *s = Malloc(stack_s); + *s = (stack_s) { + .top = NULL, .size = 0 + }; + return s; +} + +void stack_destroy(stack_s *s) +{ + stack_elem_s *current = s->top; + + while (current) { + stack_elem_s *next = current->next; + free(current); + current = next; + } + + free(s); +} + +void stack_push(stack_s *s, void *data) +{ + stack_elem_s *new_top = Malloc(stack_elem_s); + + *new_top = (stack_elem_s) { + .next = s->top, + .data = data + }; + + s->top = new_top; + s->size++; +} + +void* stack_pop(stack_s *s) +{ + if (s->top == NULL) { + return NULL; + } + + stack_elem_s *old_top = s->top; + + void *data = old_top->data; + s->top = old_top->next; + free(old_top); + s->size--; + + return data; +} + +int stack_is_empty(stack_s *s) +{ + return s->top == NULL; +} + +stack_s* stack_new_reverse_from(stack_s *s) +{ + stack_s* r = stack_new(); + + while (!stack_is_empty(s)) { + stack_push(r, stack_pop(s)); + } + + stack_destroy(s); + + return r; +} diff --git a/ioreplay/src/datas/stack.h b/ioreplay/src/datas/stack.h new file mode 100644 index 0000000..87e0974 --- /dev/null +++ b/ioreplay/src/datas/stack.h @@ -0,0 +1,43 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef STACK_H +#define STACK_H + +#include "../defaults.h" + +/** + * @brief Definition of a stack element + */ +typedef struct stack_elem_s_ { + struct stack_elem_s_ *next; /**< The next element */ + void *data; /**< Pointer to the stored data in the current element */ +} stack_elem_s; + +/** + * @brief Definition of a stack data structure + */ +typedef struct stack_s_ { + stack_elem_s *top; /**< The top element of the stack, NULL if empty */ + unsigned long size; /**< A count how many elements are in the stack */ +} stack_s; + +stack_s* stack_new(); +stack_s* stack_new_reverse_from(stack_s* s); +void stack_destroy(stack_s* s); +void stack_push(stack_s* s, void *data); +void* stack_pop(stack_s* s); +int stack_is_empty(stack_s* s); + +#endif // STACK_H diff --git a/ioreplay/src/defaults.h b/ioreplay/src/defaults.h new file mode 100644 index 0000000..c607b95 --- /dev/null +++ b/ioreplay/src/defaults.h @@ -0,0 +1,50 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DEFAULTS_H +#define DEFAULTS_H + +#include "utils/utils.h" + +/** Version of the supported .capture format */ +#define CAPTURE_VERSION 1 +/** Version of the supported .replay format */ +#define REPLAY_VERSION 1 +/** Max amount of tokens per line in the .capture file */ +#define MAX_TOKENS 10 +/** Max line length in either .capture or .replay file */ +#define MAX_LINE_LEN 1024*8 +/** Controls how many tasks can be queued and buffered per worker thread */ +#define TASK_BUFFER_PER_THREAD 512 +/** Version of I/O Replay */ +#define IOREPLAY_VERSION "0.1" +/** Copyright information */ +#define IOREPLAY_COPYRIGHT "Mimecast 2017, 2018 (c)" + +// The following are for debugging purposes only + +//#define NO_IOOP +//#define THREAD_DEBUG +//#define LOG_FILTERED + +/** + * @brief Return status codes + */ +typedef enum status_e_ { + SUCCESS, /**< Great success! */ + UNKNOWN, /**< Unknown return status :-/ */ + ERROR, /**< An error happened :-( */ +} status_e; + +#endif // DEFAULTS_H diff --git a/ioreplay/src/generate/generate.c b/ioreplay/src/generate/generate.c new file mode 100644 index 0000000..ff1be94 --- /dev/null +++ b/ioreplay/src/generate/generate.c @@ -0,0 +1,235 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generate.h" + +#include "../meta/meta.h" +#include "gtask.h" +#include "gwriter.h" +#include "gparser.h" + +#include <fcntl.h> + +#define _MAX_PROCESSES 1024*1024*10 + +#define _Perc_filtered (g->num_lines_filtered / (g->lineno/100.0)) + +generate_s* generate_new(options_s *opts) +{ + generate_s *g = Malloc(generate_s); + + g->writer = NULL; + g->lineno = 0; + g->name = opts->name; + g->replay_fd = NULL; + g->mps = mounts_new(opts); + g->num_lines_filtered = 0; + g->num_vsizes = 0; + g->start_time = -1; + g->pid_map = amap_new(_MAX_PROCESSES); + g->vsize_map = hmap_new(_MAX_PROCESSES); + g->mmap_map = hmap_new(1024*1024); + g->vfd_buffer = rbuffer_new(1024); + g->num_mapped_pids = 0; + g->num_mapped_fds = 10; + g->opts = opts; + g->reuse_queue = rbuffer_new(1024); + g->replay_fd = Fopen(opts->replay_file, "w"); + + return g; +} + +void generate_destroy(generate_s *g) +{ + // TODO: Also clean the contets of these maps + amap_destroy(g->pid_map); + hmap_destroy(g->vsize_map); + hmap_destroy(g->mmap_map); + rbuffer_destroy(g->vfd_buffer); + mounts_destroy(g->mps); + + gtask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(g->reuse_queue))) + gtask_destroy(task); + rbuffer_destroy(g->reuse_queue); + + fclose(g->replay_fd); + free(g); +} + + +status_e generate_run(options_s *opts) +{ + generate_s *g = generate_new(opts); + Put("Parsing file %s, writing output to %s", opts->capture_file, + opts->replay_file); + FILE *capture_fd = Fopen(opts->capture_file, "r"); + + size_t len = 0; + ssize_t read; + char *line = NULL; + + drop_root(opts->user); + + // Reserve first few bytes for meta information + meta_s *meta = meta_new(g->replay_fd); + meta_reserve(meta); + + // The writer will write the .replay file + gwriter_s *writer = gwriter_new(g); + + // The parser will parse every line of the .capture file + gparser_s *parser = gparser_new(g); + + g->writer = writer; + + // Start one writer and one parser thread! + gparser_start(parser); + gwriter_start(writer); + + Out("Processing, it may take a while: "); + + // Process each line of the .capture file. Determine line by line whether + // the I/O operation makes sense or not. It might be that SystemTap skipped + // some I/O ops due to system overload or other issues. The result is that + // some lines may be corrupt or contain I/O operations on unknown file + // handles. It could also be that there are operations on unknown + // file handles such as sockets etc. These will be all filtered out by + // either the parser or the writer thread! + + while ((read = getline(&line, &len, capture_fd)) != -1) { + if (0 > ++g->lineno) { + Error("lineno:%lu Line number overflow", g->lineno); + } + if (strlen(line) >= MAX_LINE_LEN) { + Error("lineno:%lu Exceeded max line length", g->lineno); + } + + // Create a new generate task (try to reuse a task object)... + gtask_s *t = rbuffer_get_next(g->reuse_queue); + if (!t) { + t = gtask_new(g); + } else if (t->ret != 0) { + g->num_lines_filtered++; + } + gtask_init(t, line, g->lineno); + + // ...pass it to the parser queue + while (!rbuffer_insert(parser->queue, t)) + usleep(100); + + if (g->lineno % 1000000 == 0) { + Out(" %lu (filtered:%.2lf%%)", g->lineno, _Perc_filtered); + } + } + + Put("\nDone reading input file!"); + + Put("Waiting for parser thread..."); + gparser_terminate(parser); + gparser_destroy(parser); + + Put("Waiting for writer thread..."); + gwriter_terminate(writer); + gwriter_destroy(writer); + + // Retrieve all left over processed tasks to collect the + // statistics! + gtask_s *t; + while (NULL != (t = rbuffer_get_next(g->reuse_queue))) { + if (t->ret != 0) + g->num_lines_filtered++; + gtask_destroy(t); + } + + Put("Processed %lu lines in total, had to filter out %.2lf%%", + g->lineno, _Perc_filtered); + + Put("Writing init section to '%s'...", opts->replay_file); + fprintf(g->replay_fd, "#INIT\n"); + off_t init_offset = ftello(g->replay_fd); + hmap_run_cb(g->vsize_map, generate_write_init_cb); + + Put("Writing meta header to '%s'...", opts->replay_file); + meta_write_start(meta); + + // The meta header is being written to the first line of the .replay + // file and used by ioreplay to do various things (e.g. initializing + // the test correctly, creating the internal data structures with the + // correct sizes etc. + + meta_write_l(meta, "replay_version", REPLAY_VERSION); + meta_write_l(meta, "init_offset", init_offset); + + meta_write_s(meta, "user", opts->user); + meta_write_s(meta, "name", opts->name); + + meta_write_l(meta, "num_vsizes", g->num_vsizes); + meta_write_l(meta, "num_mapped_pids", g->num_mapped_pids); + meta_write_l(meta, "num_mapped_fds", g->num_mapped_fds); + meta_write_l(meta, "num_lines", g->lineno - g->num_lines_filtered); + + meta_destroy(meta); + fclose(capture_fd); + + Put("Generating '%s' done", opts->replay_file); + generate_destroy(g); + + return SUCCESS; +} + +void generate_write_init_cb(void *data) +{ + vsize_s *l = data; + generate_s *g = l->generate; + + if (l->required && strlen(l->path) > 0) { + fprintf(g->replay_fd, "%d|%d|%ld|%s|\n", + l->is_dir, l->is_file, -l->vsize_deficit, l->path); + } +} + +vsize_s* generate_vsize_by_path(generate_s *g, gtask_s *t, + char *path) +{ + vsize_s *v = NULL; + + if (!path && t) + path = t->path; + + Error_if(!path, "No path specified"); + v = hmap_get(g->vsize_map, path); + + if (!v) { + v = vsize_new(path, ++g->num_vsizes, g); + hmap_insert(g->vsize_map, path, v); + } + + if (t) + t->vsize = v; + + return v; +} + +void generate_gprocess_by_realpid(generate_s *g, gtask_s *t) +{ + // Get the virtual process data object from the virtual PID space. + t->gprocess = amap_get(g->pid_map, t->pid); + if (t->gprocess == NULL) { + t->gprocess = gprocess_new(t->pid, ++g->num_mapped_pids); + if (amap_set(g->pid_map, t->pid, t->gprocess)) { + Error("lineno:%lu Can not insert PID %ld", t->lineno, t->pid); + } + } +} diff --git a/ioreplay/src/generate/generate.h b/ioreplay/src/generate/generate.h new file mode 100644 index 0000000..cf096d2 --- /dev/null +++ b/ioreplay/src/generate/generate.h @@ -0,0 +1,112 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GENERATE_H +#define GENERATE_H + +#include "gwriter.h" +#include "../datas/amap.h" +#include "../datas/hmap.h" +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "../mounts.h" +#include "../options.h" + +// Forward declarations (header include hell) +struct gtask_s_; + +/** + * @brief The generate object definition + * + * This is the general data structure required to generate a .replay file from + * the .capture file. + */ +typedef struct generate_s_ { + long lineno; /**< The current line number */ + long num_lines_filtered; /**< The amount of lines filtered out */ + long start_time; /**< The start time from the .capture file */ + char *name; /**< The name of the test specified by the user */ + FILE *replay_fd; /**< The fd of the .replay file */ + mounts_s *mps; /**< The mounts object */ + hmap_s *mmap_map; /**< mmap address mappings */ + amap_s *pid_map; /**< A map of all virtual process objects */ + unsigned long num_mapped_pids; /**< The amount of mapped PIDs */ + unsigned long num_mapped_fds; /**< The amount of mapped FDs */ + hmap_s *vsize_map; /**< A hash map of all virtual size objects */ + unsigned long num_vsizes; /**< The amount of virtual sizes */ + options_s *opts; /**< A pointer to the options object */ + rbuffer_s *vfd_buffer; /**< A virtual fd buffer, for reusing these */ + rbuffer_s *reuse_queue; /**< A task buffer, for reusing these */ + struct gwriter_s_ *writer; /**< A pointer to the writer object */ +} generate_s; + +/** + * @brief Creates a new generate object + * + * @param opts The options object + * @return The new generate object + */ +generate_s* generate_new(options_s *opts); + +/** + * @brief Destroys a generate object + * + * @param g The generate object to destroy + */ +void generate_destroy(generate_s* g); + +/** + * @brief Generates a .replay file from a .capture file + * + * @param opts The options object + * @return SUCCESS on success + */ +status_e generate_run(options_s *opts); + +/** + * @brief Callback to write the INIT section to the .replay file + * + * This function writes a list of all pre-required + * paths to the .replay file. That then can be used + * by ioreplay to initialise the test enironment. + * + * @param data A pointer to the vsize timestamp object + */ +void generate_write_init_cb(void *data); + +/** + * @brief Retrieves the virtual size object of a given path + * + * A new one will be created in case there is no such virtual size object yet. + * + * @param g The generate object + * @param t The task object (vfd will be stored to t->vfd) + * @param path The file path + * @return The virtual size object + */ +vsize_s* generate_vsize_by_path(generate_s *g, struct gtask_s_ *t, + char *path); + +/** + * @brief Retrieves the virtual process object of a given real PID + * + * A new one will be created in case there is no such virtual process object + * yet. + * + * @param g The generate object + * @param t The task object (vfd will be stored to t->gprocess) + */ +void generate_gprocess_by_realpid(generate_s *g, struct gtask_s_ *t); + +#endif // GENERATE_H diff --git a/ioreplay/src/generate/gioop.c b/ioreplay/src/generate/gioop.c new file mode 100644 index 0000000..01701bc --- /dev/null +++ b/ioreplay/src/generate/gioop.c @@ -0,0 +1,838 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gioop.h" + +status_e gioop_run(gwriter_s *w, gtask_s *t) +{ + status_e ret = SUCCESS; + + // There was already an error in the parser (parser.c) processing this + // task! Don't process it futher. + if (t->ret != SUCCESS) { + Cleanup(t->ret); + } + + generate_s *g = w->generate; + + // Get the virtual process data object from the virtual PID space and store + // a pointer to it to t->gprocess + generate_gprocess_by_realpid(g, t); + + // One of the open syscalls may openes a file handle succesfully + if (Eq(t->op, "open")) { + Cleanup(gioop_open(w, t, g)); + + } else if (Eq(t->op, "openat")) { + Cleanup(gioop_openat(w, t, g)); + + } else if (Eq(t->op, "creat")) { + Cleanup(gioop_creat(w, t, g)); + } + + // Get the virtual file descriptor of a given real fd and store a pointer + // to it to t->vfd. + if (t->has_fd) { + ret = gprocess_vfd_by_realfd(t->gprocess, t); + Cleanup_unless(SUCCESS, ret); + } + + + if (Eq(t->op, "close")) { + Cleanup(gioop_close(w, t, g)); + + } else if (Eq(t->op, "stat")) { + Cleanup(gioop_stat(w, t, g)); + + } else if (Eq(t->op, "statfs")) { + Cleanup(gioop_statfs(w, t, g)); + + } else if (Eq(t->op, "statfs64")) { + Cleanup(gioop_statfs64(w, t, g)); + + } else if (Eq(t->op, "fstat")) { + Cleanup(gioop_fstat(w, t, g)); + + } else if (Eq(t->op, "fstatat")) { + Cleanup(gioop_fstatat(w, t, g)); + + } else if (Eq(t->op, "fstatfs")) { + Cleanup(gioop_fstatfs(w, t, g)); + + } else if (Eq(t->op, "fstatfs64")) { + Cleanup(gioop_fstatfs64(w, t, g)); + + } else if (Eq(t->op, "rename")) { + Cleanup(gioop_rename(w, t, g)); + + } else if (Eq(t->op, "renameat")) { + Cleanup(gioop_renameat(w, t, g)); + + } else if (Eq(t->op, "renameat2")) { + Cleanup(gioop_renameat2(w, t, g)); + + } else if (Eq(t->op, "read")) { + Cleanup(gioop_read(w, t, g)); + + } else if (Eq(t->op, "readv")) { + Cleanup(gioop_readv(w, t, g)); + + } else if (Eq(t->op, "readahead")) { + Cleanup(gioop_readahead(w, t, g)); + + } else if (Eq(t->op, "readdir")) { + Cleanup(gioop_readdir(w, t, g)); + + } else if (Eq(t->op, "readlink")) { + Cleanup(gioop_readlink(w, t, g)); + + } else if (Eq(t->op, "readlinkat")) { + Cleanup(gioop_readlinkat(w, t, g)); + + } else if (Eq(t->op, "write")) { + Cleanup(gioop_write(w, t, g)); + + } else if (Eq(t->op, "writev")) { + Cleanup(gioop_writev(w, t, g)); + + } else if (Eq(t->op, "lseek")) { + Cleanup(gioop_lseek(w, t, g)); + + } else if (Eq(t->op, "getdents")) { + Cleanup(gioop_getdents(w, t, g)); + + } else if (Eq(t->op, "mkdir")) { + Cleanup(gioop_mkdir(w, t, g)); + + } else if (Eq(t->op, "rmdir")) { + Cleanup(gioop_rmdir(w, t, g)); + + } else if (Eq(t->op, "mkdirat")) { + Cleanup(gioop_mkdirat(w, t, g)); + + } else if (Eq(t->op, "unlink")) { + Cleanup(gioop_unlink(w, t, g)); + + } else if (Eq(t->op, "unlinkat")) { + Cleanup(gioop_unlinkat(w, t, g)); + + } else if (Eq(t->op, "lstat")) { + Cleanup(gioop_lstat(w, t, g)); + + } else if (Eq(t->op, "fsync")) { + Cleanup(gioop_fsync(w, t, g)); + + } else if (Eq(t->op, "fdatasync")) { + Cleanup(gioop_fdatasync(w, t, g)); + + } else if (Eq(t->op, "sync")) { + Cleanup(gioop_sync(w, t, g)); + + } else if (Eq(t->op, "syncfs")) { + Cleanup(gioop_syncfs(w, t, g)); + + } else if (Eq(t->op, "sync_file_range")) { + Cleanup(gioop_sync_file_range(w, t, g)); + + } else if (Eq(t->op, "fcntl")) { + Cleanup(gioop_fcntl(w, t, g)); + + } else if (Eq(t->op, "fcntl")) { + Cleanup(gioop_fcntl(w, t, g)); + + } else if (Eq(t->op, "mmap2")) { + // Support for mmap added later + + } else if (Eq(t->op, "munmap")) { + // Support for mmap added later + + } else if (Eq(t->op, "mremap")) { + // Support for mmap added later + + } else if (Eq(t->op, "msync")) { + // Support for mmap added later + + } else if (Eq(t->op, "chmod")) { + Cleanup(gioop_chmod(w, t, g)); + + } else if (Eq(t->op, "fchmodat")) { + Cleanup(gioop_chmod(w, t, g)); + + } else if (Eq(t->op, "fchmod")) { + Cleanup(gioop_fchmod(w, t, g)); + + } else if (Eq(t->op, "chown")) { + Cleanup(gioop_chown(w, t, g)); + + } else if (Eq(t->op, "chown16")) { + Cleanup(gioop_chown(w, t, g)); + + } else if (Eq(t->op, "lchown")) { + Cleanup(gioop_lchown(w, t, g)); + + } else if (Eq(t->op, "lchown16")) { + Cleanup(gioop_lchown(w, t, g)); + + } else if (Eq(t->op, "fchown")) { + Cleanup(gioop_fchown(w, t, g)); + + } else if (Eq(t->op, "fchownat")) { + Cleanup(gioop_chown(w, t, g)); + + } else if (Eq(t->op, "exit_group")) { + Cleanup(gioop_exit_group(w, t, g)); + + } else { + Cleanup(ERROR;); + } + +cleanup: + +#ifdef LOG_FILTERED + if (ret != SUCCESS) + t->filtered_where = __FILE__; +#endif + + t->ret = ret; + return ret; +} + +status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd || t->path == NULL || t->flags == -1) { + return ERROR; + } + + gprocess_create_vfd_by_realfd(t->gprocess, t, g); + generate_vsize_by_path(g, t, NULL); + + Gioop_write(OPEN, "%ld|%s|%d|%d|open", + t->mapped_fd, t->path, t->mode, t->flags); + + if (t->fd > 0) + vsize_open(t->vsize, t->vfd, t->path, t->flags); + + return SUCCESS; +} + +status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd || t->path == NULL || t->flags == -1) { + return ERROR; + } + + gprocess_create_vfd_by_realfd(t->gprocess, t, g); + generate_vsize_by_path(g, t, NULL); + Gioop_write(OPEN_AT, "%ld|%s|%d|%d|openat", + t->mapped_fd,t->path, t->mode, t->flags); + if (t->fd > 0) + vsize_open(t->vsize, t->vfd, t->path, t->flags); + + return SUCCESS; +} + +status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd || t->path == NULL || t->flags == -1) { + return ERROR; + } + + gprocess_create_vfd_by_realfd(t->gprocess, t, g); + generate_vsize_by_path(g, t, NULL); + + Gioop_write(CREAT, "%ld|%s|%d|%d|creat", + t->mapped_fd, t->path, t->mode, t->flags); + if (t->fd > 0) + vsize_open(t->vsize, t->vfd, t->path, t->flags); + + return SUCCESS; +} + + +status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(CLOSE, "%ld|%d|close", t->mapped_fd, t->status); + + if (t->status == 0) + vsize_close(t->vsize, t->vfd); + + hmap_remove_l(t->gprocess->fd_map, t->fd); + hmap_remove_l(t->gprocess->vfd_map, t->mapped_fd); + + if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) + vfd_destroy(t->vfd); + + return SUCCESS; +} + +status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(STAT, "%s|%d|stat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(STATFS, "%s|%d|statfs", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(STATFS64, "%s|%d|statfs64", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSTAT, "%ld|%d|fstat", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(FSTAT_AT, "%s|%d|fstatat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSTATFS, "%ld|%d|fstatfs", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSTATFS64, "%ld|%d|fstatfs64", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL || t->path2 == NULL ) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(RENAME, "%s|%s|%d|rename", t->path, t->path2, t->status); + + if (t->status == 0) { + t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); + vsize_rename(t->vsize, t->vsize2, t->path, t->path2); + } + + return SUCCESS; +} + +status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL || t->path2 == NULL ) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(RENAME_AT, "%s|%s|%d|renameat", t->path, t->path2, t->status); + + if (t->status == 0) { + t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); + vsize_rename(t->vsize, t->vsize2, t->path, t->path2); + } + + return SUCCESS; +} +status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL || t->path2 == NULL ) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(RENAME_AT2, "%s|%s|%d|renameat2", + t->path, t->path2, t->status); + + if (t->status == 0) { + t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); + vsize_rename(t->vsize, t->vsize2, t->path, t->path2); + } + + return SUCCESS; +} + +status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READ, "%ld|%ld|read", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READ, "%ld|%ld|readv", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READAHEAD, "%ld|%ld|%ld|readahead", + t->mapped_fd, t->offset, t->count); + + return SUCCESS; +} + +status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READDIR, "%ld|%d|readdir", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(READLINK, "%s|%d|readlink", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(READLINK_AT, "%s|%d|readlinkat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(WRITE, "%ld|%ld|write", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_write(t->vsize, t->vfd, t->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(WRITEV, "%ld|%ld|writev", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_write(t->vsize, t->vfd, t->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(LSEEK, "%ld|%ld|%ld|%ld|lseek", + t->mapped_fd, t->offset, t->whence, t->bytes); + + if (t->bytes >= 0) + vsize_seek(t->vsize, t->vfd, t->bytes); + + return SUCCESS; +} + +status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(GETDENTS, "%ld|%ld|%ld|getdents", + t->mapped_fd, t->count, t->bytes); + + return SUCCESS; +} + +status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(MKDIR, "%s|%d|%d|mkdir", t->path, t->mode, t->status); + + if (t->status == 0) + vsize_mkdir(t->vsize, t->path); + + return SUCCESS; +} +status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(MKDIR, "%s|%d|rmdir", t->path, t->status); + + if (t->status == 0) + vsize_rmdir(t->vsize, t->path); + + return SUCCESS; +} +status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(MKDIR_AT, "%s|%d|%d|mkdirat", t->path, t->mode, t->status); + + if (t->status == 0) + vsize_mkdir(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(UNLINK, "%s|%d|unlink", t->path, t->status); + + if (t->status == 0) + vsize_unlink(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(UNLINK_AT, "%s|%d|unlinkat", t->path, t->status); + + if (t->status == 0) + vsize_unlink(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(LSTAT, "%s|%d|lstat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSYNC, "%ld|%d|fsync", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FDATASYNC, "%ld|%d|fdatasync", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g) +{ + Gioop_write(SYNC, "%d|sync", t->status); + + return SUCCESS; +} + +status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(SYNCFS, "%ld|%d|syncfs", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(SYNC_FILE_RANGE, "%ld|%ld|%ld|%d|sync_file_range", + t->mapped_fd, t->offset, t->bytes, t->status); + + return SUCCESS; +} + +status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + switch (t->F) { + case F_GETFD: + case F_GETFL: + case F_SETFD: + case F_SETFL: + break; + default: + return ERROR; + break; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FCNTL, "%ld|%d|%d|%d|fcntl", + t->mapped_fd, t->F, t->G, t->status); + + return SUCCESS; +} + +status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(CHMOD, "%s|%d|%d|chmod", t->path, t->mode, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FCHMOD, "%ld|%d|%d|fchmod", t->mapped_fd, t->mode, t->status); + + return SUCCESS; +} + +status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + // Hmm, maybe rename t->offset, because here it is used for the user UID + Gioop_write(CHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + // Hmm, maybe rename t->offset, because here it is used for the user UID + Gioop_write(FCHOWN, "%ld|%ld|%d|%d|fchown", t->mapped_fd, t->offset, t->G, t->status); + + return SUCCESS; +} + +status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + // Hmm, maybe rename t->offset, because here it is used for the user UID + Gioop_write(LCHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g) +{ + // It means that the process and all its threads terminate. + // Therefore close all file handles of that process! + hmap_run_cb2(t->gprocess->vfd_map, gioop_close_all_vfd_cb, t); + + // Remove virtual process from pid map and destroy it + amap_unset(g->pid_map, t->pid); + gprocess_destroy(t->gprocess); + + return SUCCESS; +} + +void gioop_close_all_vfd_cb(void *data, void *data2) +{ + gtask_s *t = data2; + t->vfd = data; + generate_s *g = t->generate; + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(CLOSE, "%ld|%d|close on exit_group", t->vfd->mapped_fd, 0); + vsize_close(t->vsize, t->vfd); +} + diff --git a/ioreplay/src/generate/gioop.h b/ioreplay/src/generate/gioop.h new file mode 100644 index 0000000..ad49713 --- /dev/null +++ b/ioreplay/src/generate/gioop.h @@ -0,0 +1,102 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GIOOP_H +#define GIOOP_H + +#include "../defaults.h" +#include "gwriter.h" +#include "gtask.h" +#include "generate.h" + + +// Helper macro regarding writing the .replay file! + +#define Gioop_write(op, ...) \ + fprintf(g->replay_fd, "%ld|%ld|%ld|0|0|%d|", \ + t->mapped_time, \ + (t->vsize ? t->vsize->id : 0),\ + t->gprocess->mapped_pid, \ + op); \ + fprintf(g->replay_fd, __VA_ARGS__); \ + fprintf(g->replay_fd, "@%ld", t->lineno); \ + fprintf(g->replay_fd, "|\n") + +/** + * @brief Function used when closing all virtual FDs of a virtual process + * + * This function is run on all virtual file handles whenever a virtual generate + * process object (gprocess_s) gets destroyed. This is on an exit_group + * syscall (a thread group, a process with all its threads, terminates). Upon + * process termination Linux also closes all its file descriptors! This is what + * we simulate here! + * + * @param data The pointer to the virtual file descriptor object + * @param data2 The pointer to the corresponding generate task object. + */ +void gioop_close_all_vfd_cb(void *data, void *data2); + +/** + * @brief Run a generate I/O operation on a given task + * + * @param w The writer object + * @param t The task object + * @return SUCCESS if everything went fine + */ +status_e gioop_run(gwriter_s *w, gtask_s *t); + +status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g); + +#endif // GIOOP_H diff --git a/ioreplay/src/generate/gparser.c b/ioreplay/src/generate/gparser.c new file mode 100644 index 0000000..514128f --- /dev/null +++ b/ioreplay/src/generate/gparser.c @@ -0,0 +1,356 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gparser.h" + +#include "gtask.h" +#include "gwriter.h" + +void* gparser_pthread_start(void *data) +{ + gparser_s *p = data; + generate_s *g = p->generate; + gwriter_s *w = g->writer; + gtask_s *t = NULL; + + do { + while (NULL != (t = rbuffer_get_next(p->queue))) { + // First extract + gparser_extract(p, t); + // Second, pass the task to the writer thread + rbuffer_insert(w->queue, t); + } + usleep(100); + } while (!p->terminate); + + while (NULL != (t = rbuffer_get_next(p->queue))) { + gparser_extract(p, t); + rbuffer_insert(w->queue, t); + } + + return NULL; +} + +gparser_s* gparser_new(generate_s *g) +{ + gparser_s *p = Malloc(gparser_s); + + p->generate = g; + p->terminate = false; + p->queue = rbuffer_new(1024); + + return p; +} + +void gparser_start(gparser_s *p) +{ + start_pthread(&p->pthread, gparser_pthread_start, (void*)p); +} + +void gparser_destroy(gparser_s *p) +{ + rbuffer_destroy(p->queue); + free(p); +} + +void gparser_terminate(gparser_s *p) +{ + p->terminate = true; + pthread_join(p->pthread, NULL); +} + +void gparser_extract(gparser_s *p, gtask_s *t) +{ + status_e ret = SUCCESS; + generate_s *g = p->generate; + + char *saveptr; + char* tok = strtok2_r(t->line, ";:,", &saveptr); + int ntoks = 0; + + while (tok) { + if (++ntoks > MAX_TOKENS) { + ret = ERROR; + break; + } + ret = gparser_extract_tok(p, t, tok); + if (ret != SUCCESS) + break; + + tok = strtok2_r(NULL, ";:,", &saveptr); + } + + if (ret == SUCCESS) { + + // Check for the existance of mandatory values! + if (t->pid < 0 || t->tid < 0) { + Cleanup(ERROR); + + } else if (t->op == NULL) { + Cleanup(ERROR); + + } else if (t->mapped_time == -1) { + Cleanup(ERROR); + } + + // We are inserting ".ioreplay/NAME" to the paths. This enables us to + // run multiple tests simoultaneously. + + if (t->path) { + if (!mounts_transform_path(g->mps, g->name, + t->path, &t->path_r)) { + Cleanup(ERROR); + } + if (t->path_r) + t->path = t->path_r; + } + + if (t->path2) { + if (!mounts_transform_path(g->mps, g->name, + t->path2, &t->path2_r)) { + Cleanup(ERROR); + } + if (t->path2_r) + t->path2 = t->path2_r; + } + + } + +cleanup: + + t->ret = ret; + +#ifdef LOG_FILTERED + t->filtered_where = __FILE__; +#endif +} + +status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok) +{ + status_e ret = SUCCESS; + + if (gparser_token_not_ok(p, tok)) { + Cleanup(ERROR); + } + + generate_s *g = t->generate; + + char key = tok[0]; + char *value = tok; + value += 2; + + switch (key) { + case 'a': + // Address + t->address = strtol(value, NULL, 10); + break; + + case 'A': + // Address 2 + t->address2 = strtol(value, NULL, 10); + break; + + case 'b': + // Bytes + if (t->bytes != -1) { + Cleanup(ERROR); + } + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->bytes = strtol(value, NULL, 10); + break; + + case 'c': + // Count + if (t->count != -1) { + Cleanup(ERROR); + } + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->count = strtol(value, NULL, 10); + break; + + case 'd': + // Descriptor + if (t->fd != -1) { + Cleanup(ERROR); + } + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->fd = atoi(value); + if (t->fd > 0) + t->has_fd = true; + break; + + case 'f': + // Flags + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->flags = atoi(value); + break; + + case 'i': + // PID:TID + t->pidtid = value; + // Extract PID and TID from "PID:TID" + if (!gparser_get_pidtid(p, t->pidtid, &t->pid, &t->tid)) { + Cleanup(ERROR); + } + break; + + case 'm': + // Mode + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->mode = atoi(value); + break; + + case 'o': + // Operation + t->op = value; + break; + + case 'O': + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->offset = strtol(value, NULL, 10); + break; + + case 'W': + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->whence = strtol(value, NULL, 10); + break; + + case 'p': + // File path + t->path = value; + chreplace(t->path, '|', '_'); + strunquote(t->path); + break; + + case 'P': + // File path 2 + t->path2 = value; + chreplace(t->path2, '|', '_'); + strunquote(t->path2); + break; + + case 's': + // Cleanup status + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->status = atoi(value); + break; + + case 't': + // Time + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->mapped_time = strtol(value, NULL, 10); + // Start replay time from 0 + if (g->start_time == -1) { + g->start_time = t->mapped_time; + } + t->mapped_time -= g->start_time; + break; + + case 'F': + // FCNTL function + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->F = atoi(value); + break; + + case 'G': + // FCNTL argument + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->G = atoi(value); + break; + + case 'T': + break; + + default: + // Unknown key + { + Cleanup(ERROR); + } + } + +cleanup: + if (t->path_r) { + free(t->path_r); + t->path_r = NULL; + } + if (t->path2_r) { + free(t->path2_r); + t->path2_r = NULL; + } + + return ret; +} + +bool gparser_token_not_ok(gparser_s *p, char *tok) +{ + if (strlen(tok) < 3) { + return true; + + } else if (tok[1] != '=') { + return true; + } + + return false; +} + +bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid) +{ + char *pos = strchr(pidtid, ':'); + + if (pos) { + char *tmp = pos; + tmp++; + + if (is_number(tmp)) { + *tid = atol(tmp); + } else { + return false; + } + + pos[0] = '\0'; + if (is_number(pidtid)) { + *pid = atol(pidtid); + } else { + return false; + } + } + + else { + return false; + } + + return (*pid >= 0 && *tid >= 0); +} diff --git a/ioreplay/src/generate/gparser.h b/ioreplay/src/generate/gparser.h new file mode 100644 index 0000000..f3e204a --- /dev/null +++ b/ioreplay/src/generate/gparser.h @@ -0,0 +1,113 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GPARSER_H +#define GPARSER_H + +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "generate.h" +#include "gtask.h" + +/** + * @brief The parser definition + * + * The parser is to extract all information from the .capture file. + */ +typedef struct gparser_s_ { + bool terminate; /**< The parser thread will terminate if set to true */ + generate_s *generate; /**< The generate object */ + pthread_t pthread; /**< The posix thread */ + rbuffer_s *queue; /**< A queue of task objects */ +} gparser_s; + +/** + * @brief Creates a new parser + * + * @param g The generate object + * @return The new parser object + */ +gparser_s* gparser_new(generate_s *g); + +/** + * @brief Starts the parser thread + * + * @param p The parser object + */ +void gparser_start(gparser_s *p); + +/** + * @brief Terminates the parser thread + * + * @param p The parser object + */ +void gparser_terminate(gparser_s *p); + +/** + * @brief Destroys the parser thread + * + * @param p The parser object + */ +void gparser_destroy(gparser_s *p); + +/** + * @brief Extracts information a .capture line + * + * Extracts information from a .capture line and stores it into the task + * object. + * + * @param p The parser object + * @param t The task object + */ +void gparser_extract(gparser_s *p, gtask_s *t); + +/** + * @brief Extracts information from a specific token string + * + * @param p The parser object + * @param t The task object + * @param tok The token string + * @return Returns with SUCCESS on success + */ +status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok); + +/** + * @brief Verifies the correctness of a token + * + * @param p The parser object + * @param tok The token to be verified + * @return true if token verified successfully + */ +bool gparser_token_not_ok(gparser_s *p, char *tok); + +/** + * @brief Checks whether the pidtid string is correct or not + * + * @param p The parser object + * @param pidtid The string to check + * @param pid The pointer to the resulting pid + * @param tid The pointer to the resulting tid + * @return true on success + */ +bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid); + +/** + * @brief Entry point of the parser POSIX thread + * + * @param data A pointer to the parser object + * return Always NULL + */ +void* gparser_pthread_start(void *data); + +#endif // GPARSER_H diff --git a/ioreplay/src/generate/gprocess.c b/ioreplay/src/generate/gprocess.c new file mode 100644 index 0000000..6a0b37a --- /dev/null +++ b/ioreplay/src/generate/gprocess.c @@ -0,0 +1,101 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gprocess.h" + +#include "../vfd.h" +#include "gioop.h" + +void _gprocess_vfd_map_destroy_cb(void *data) +{ + vfd_destroy(data); +} + +gprocess_s* gprocess_new(const long pid, const long mapped_pid) +{ + gprocess_s* gp = Malloc(gprocess_s); + + gp->pid = pid; + gp->mapped_pid = mapped_pid; + gp->max_mapped_fd = 0; + gp->fd_map = hmap_new_l(1024); + gp->vfd_map = hmap_new_l(1024); + gp->vfd_map->data_destroy = _gprocess_vfd_map_destroy_cb; + + return gp; +} + +void gprocess_destroy(gprocess_s *gp) +{ + hmap_destroy(gp->vfd_map); + hmap_destroy(gp->fd_map); + free(gp); +} + +void gprocess_create_vfd_by_realfd(gprocess_s *gp, gtask_s *t, generate_s *g) +{ + if (t->fd < 0) + return; + + // Check whether the real FD is still open according to the .capture log + long old_mapped = (long) hmap_get_l(gp->fd_map, t->fd); + if (old_mapped) { + + // That real file descriptor is already with a mapping to a virtual + // file descriptor. This may happen when SystemTap missed to trace a + // 'close' syscall. We are inserting a close now... + + t->vfd = hmap_get_l(gp->vfd_map, old_mapped); + + hmap_remove_l(gp->fd_map, t->fd); + hmap_remove_l(gp->vfd_map, old_mapped); + + if (t->vfd) { + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(CLOSE, "%ld|%d|close inserted", old_mapped, 0); + vsize_close(t->vsize, t->vfd); + if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) + vfd_destroy(t->vfd); + } + } + + t->vfd = rbuffer_get_next(g->vfd_buffer); + t->mapped_fd = ++g->num_mapped_fds; + if (!t->vfd) + t->vfd = vfd_new(t->fd, t->mapped_fd, t->path); + else + vfd_update(t->vfd, t->fd, t->mapped_fd, t->path); + t->vfd->free_path = t->path_r != NULL; + + hmap_insert_l(gp->vfd_map, t->mapped_fd, t->vfd); + hmap_insert_l(gp->fd_map, t->fd, (void*)t->mapped_fd); +} + +status_e gprocess_vfd_by_realfd(gprocess_s *gp, gtask_s *t) +{ + t->mapped_fd = (long) hmap_get_l(gp->fd_map, t->fd); + if (t->mapped_fd == 0) { + // No corresponding virtual fd number mapping + t->has_fd = false; + + } else { + t->vfd = hmap_get_l(gp->vfd_map, t->mapped_fd); + if (!t->vfd) { + return ERROR; + } + t->mapped_fd = t->vfd->mapped_fd; + } + + return SUCCESS; +} diff --git a/ioreplay/src/generate/gprocess.h b/ioreplay/src/generate/gprocess.h new file mode 100644 index 0000000..47e5037 --- /dev/null +++ b/ioreplay/src/generate/gprocess.h @@ -0,0 +1,90 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GPROCESS_H +#define GPROCESS_H + +#include "../datas/hmap.h" +#include "../defaults.h" +#include "gtask.h" +#include "generate.h" + +// Forward declarations (header include hell) +struct gtask_s_; +struct generate_s_; + +/** + * @brief Virtual process object used for generating .replay file + * + * An object of this represents a Linux process in generate context. + * In Linux every process owns * its own file descriptor table which is + * simulated here. Usually, a Linux process re-uses a FD number once not used + * anymore (e.g. after a close). However, as we want to increase concurrency + * while replaying the I/O we want * to ensure to always use unique file + * descriptor IDs for every open. Thats why we use max_mapped_fd to always + * map a real FD number to a uniq virtual FD number. + */ +typedef struct gprocess_s_ { + long pid; /**< The real PID */ + long mapped_pid; /**< The mapped PID */ + hmap_s *vfd_map; /**< All virtual file descriptors of that process */ + hmap_s *fd_map; /**< All mappings from real fd to virtual fd */ + long max_mapped_fd; /**< The max mapped fd number */ +} gprocess_s; + +/** + * @brief Creates a new gprocess object + * + * @param pid The process ID + * @param mapped_pid the mapped PID + * @return The new gprocess object + */ +gprocess_s* gprocess_new(const long pid, const long mapped_pid); + +/** + * @brief Destroys a gprocess object + * + * @param gp The gprocess object + */ +void gprocess_destroy(gprocess_s *gp); + +/** + * @brief Creates a new virtual FD from a given real FD number + * + * In ioreplay we map the real file descriptor (the fd number protocolled in + * the.capture file) to a virtual file descriptor (the fd numner written to the + * .replay file). The purpose is to increase concurrency of the I/O during + * replay. Normally, a process would reuse the same file descriptor number + * once closed earlier. However, when replaying we can't reuse the number if + * we want to replay the I/O on multiple paths in parallel. Therefore, it is + * ensured that the virtual file descriptor number in the .replay file is + * always * unique for every open! + * + * @param gp The process object + * @param t The task object (the vfd pointer will be stored to * t->vfd) + * @param g The generate object + */ +void gprocess_create_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t, + struct generate_s_ *g); + +/** + * @brief Retrieves a virtual FD from a given real FD number + * + * @param gp The process object + * @param t The task object (the vfd pointer will be stored to * t->vfd) + * @return SUCCESS if everything went smothly! + */ +status_e gprocess_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t); + +#endif // GPROCESS_H diff --git a/ioreplay/src/generate/gtask.c b/ioreplay/src/generate/gtask.c new file mode 100644 index 0000000..55a1124 --- /dev/null +++ b/ioreplay/src/generate/gtask.c @@ -0,0 +1,91 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtask.h" + +gtask_s* gtask_new(void *generate) +{ + gtask_s *t = Malloc(gtask_s); + + t->generate = generate; + t->line = NULL; + t->path_r = NULL; + t->path2_r = NULL; +#ifdef LOG_FILTERED + t->original_line = NULL; +#endif + + return t; +} + +void gtask_init(gtask_s *t, char *line, const unsigned long lineno) +{ + if (t->line) + free(t->line); + t->line = Clone(line); + + if (t->path_r) + free(t->path_r); + if (t->path2_r) + free(t->path2_r); + +#ifdef LOG_FILTERED + if (t->original_line) + free(t->original_line); + t->original_line = Clone(line); + t->filtered_where = NULL; +#endif + + t->bytes = -1; + t->address = 0; + t->address2 = 0; + t->count = -1; + t->F = -1; + t->fd = -1; + t->flags = -1; + t->G = -1; + t->has_fd = false; + t->vsize = NULL; + t->vsize2 = NULL; + t->lineno = lineno; + t->mapped_fd = -1; + t->mapped_time = -1; + t->mode = -1; + t->offset = -1; + t->op = NULL; + t->path2 = NULL; + t->path2_r = NULL; + t->path = NULL; + t->path_r = NULL; + t->gprocess = NULL; + t->pid = -1; + t->pidtid = NULL; + t->ret = 0; + t->status = -1; + t->tid = -1; + t->vfd = NULL; + t->whence = -1; +} + +void gtask_destroy(gtask_s *t) +{ + if (t->line) + free(t->line); + if (t->path_r) + free(t->path_r); + if (t->path2_r) + free(t->path2_r); + free(t); +} + diff --git a/ioreplay/src/generate/gtask.h b/ioreplay/src/generate/gtask.h new file mode 100644 index 0000000..2f364d3 --- /dev/null +++ b/ioreplay/src/generate/gtask.h @@ -0,0 +1,100 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GTASK_H +#define GTASK_H + +#include "vsize.h" + +#include "gprocess.h" +#include "../vfd.h" +#include "../datas/amap.h" +#include "../datas/hmap.h" +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "../mounts.h" +#include "../options.h" + +/** + * @brief The generate task definition + * + * The gtask holds all possible variables required to process a particular + * .capture line and to generate the corresponding .replay line + */ +typedef struct gtask_s_ { + bool has_fd; /**< True if task has a file descriptor number */ + char *line; /**< A pointer to the remaining part of the .capture line */ + char *op; /**< Operation/syscall name */ + char *path2; /**< A second path name (e.g. for rename) */ + char *path2_r; /**< Work around to track mallocs, so it can be freed */ + char *path; /**< Path name */ + char *path_r; /**< Work around to track mallocs, so it can be freed */ + char *pidtid; /**< String representing pid:tid */ + int F; /**< Arguments for fcntl syscall */ + int G; /**< Arguments for fcntl syscall */ + int fd; /**< File descriptor number */ + int flags; /**< File open flags */ + int mode; /**< File open mode */ + int ret; /**< ioreplay process status, SUCCESS if everything is alright */ + int status; /**< Operation/syscall return status */ + long address2; /**< Another address (used by mmap related syscalls) */ + long address; /**< An address (used by mmap related syscalls) */ + long bytes; /**< Amount of bytes */ + long count; /**< A count */ + long lineno; /**< The current line number */ + long mapped_fd; /**< The mapped file descriptor number */ + long mapped_time; /**< The mapped time */ + long offset; /**< A offset */ + long pid; /**< The process ID */ + long tid; /**< The thread ID */ + long whence; /**< Whence */ + vfd_s *vfd; /**< A pointer to the virtual file descriptor */ + struct gprocess_s_ *gprocess; /**< A pointer to the process object */ + void *generate; /**< A pointer to the generate object */ + vsize_s *vsize2; /**< Pointer to a second virtual size object */ + vsize_s *vsize; /**< Pointer to the virtual size object */ +#ifdef LOG_FILTERED + char *original_line; /**< Only used for debugging purposes */ + char *filtered_where; /**< Only used for debugging purposes */ +#endif +} gtask_s; + +/** + * @brief Creates a new task object + * + * @param generate A pointer to the generate object + * @return The new task object + */ +gtask_s* gtask_new(void *generate); + +/** + * @brief Initialises a taks object + * + * This function is used in particular when we recycle/reuse an old + * gtask object. + * + * @param t The gtask object + * @param line The corresponding line from the .capture file + * @param lineno The line number + */ +void gtask_init(gtask_s *t, char *line, const unsigned long lineno); + +/** + * @brief Destroys a given task object + * + * @param t The task object + */ +void gtask_destroy(gtask_s *t); + +#endif // GTASK_H diff --git a/ioreplay/src/generate/gwriter.c b/ioreplay/src/generate/gwriter.c new file mode 100644 index 0000000..e0d448e --- /dev/null +++ b/ioreplay/src/generate/gwriter.c @@ -0,0 +1,85 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gwriter.h" + +#include "gtask.h" +#include "generate.h" +#include "gioop.h" +#include "../opcodes.h" + +void* gwriter_pthread_start(void *data) +{ + gwriter_s *w = data; + generate_s *g = w->generate; + gtask_s *t = NULL; + + do { + while (NULL != (t = rbuffer_get_next(w->queue))) { +#ifdef LOG_FILTERED + // Logging filtered lines + if (SUCCESS != gioop_run(w, t)) { + fprintf(g->replay_fd, "#FILTERED @%ld %s", t->lineno, + t->original_line); + } +#else + gioop_run(w, t); +#endif + rbuffer_insert(g->reuse_queue, t); + } + usleep(100); + } while (!w->terminate); + + while (NULL != (t = rbuffer_get_next(w->queue))) { +#ifdef LOG_FILTERED + if (SUCCESS != gioop_run(w, t)) { + fprintf(g->replay_fd, "#FILTERED @%ld %s\n", t->lineno, + t->original_line); + } +#else + gioop_run(w, t); +#endif + rbuffer_insert(g->reuse_queue, t); + } + + return NULL; +} + +gwriter_s* gwriter_new(generate_s *g) +{ + gwriter_s *w = Malloc(gwriter_s); + + w->generate = g; + w->terminate = false; + w->queue = rbuffer_new(1024); + + return w; +} + +void gwriter_start(gwriter_s *w) +{ + start_pthread(&w->pthread, gwriter_pthread_start, (void*)w); +} + +void gwriter_destroy(gwriter_s *w) +{ + rbuffer_destroy(w->queue); + free(w); +} + +void gwriter_terminate(gwriter_s *w) +{ + w->terminate = true; + pthread_join(w->pthread, NULL); +} diff --git a/ioreplay/src/generate/gwriter.h b/ioreplay/src/generate/gwriter.h new file mode 100644 index 0000000..4295580 --- /dev/null +++ b/ioreplay/src/generate/gwriter.h @@ -0,0 +1,86 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GWRITER_H +#define GWRITER_H + +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "vsize.h" +#include "gtask.h" +#include "generate.h" + +// Forward declaration (header include hell) +struct gtask_s_; +struct generate_s_; + +/** + * @brief Definition of the writer object + * + * The writer utilises the information extracted by the parser to actually + * write the .replay file. + */ +typedef struct gwriter_s_ { + bool terminate; /**< The writer thread will terminate if set to true */ + struct generate_s_ *generate; /**< The generate object */ + pthread_t pthread; /**< The posix thread */ + rbuffer_s *queue; /**< A queue of task objects */ +} gwriter_s; + +/** + * @brief Creates a new writer + * + * @param g The generate object + * @return The new writer object + */ +gwriter_s* gwriter_new(struct generate_s_ *g); + +/** + * @brief Starts the writer thread + * + * @param w The writer object + */ +void gwriter_start(gwriter_s *w); + +/** + * @brief Terminates the writer thread + * + * @param w The writer object + */ +void gwriter_terminate(gwriter_s *w); + +/** + * @brief Destroys the writer thread + * + * @param w The writer object + */ +void gwriter_destroy(gwriter_s *w); + +/** + * @brief Writes a line to the .replay file + * + * @param w The writer object + * @param t The task object + */ +void gwriter_write(gwriter_s *w, struct gtask_s_ *t); + +/** + * @brief Entry function of the writer pthread + * + * @param data A pointer to the writer object + * @return Always returns a NULL pointer if it doesnt crash! + */ +void* gwriter_pthread_start(void *data); + +#endif // GWRITER_H diff --git a/ioreplay/src/generate/vsize.c b/ioreplay/src/generate/vsize.c new file mode 100644 index 0000000..f2d56ba --- /dev/null +++ b/ioreplay/src/generate/vsize.c @@ -0,0 +1,247 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "vsize.h" + +#include "generate.h" + +// Helper macros + +#define _Set_file(v) v->is_file = true; v->unsure = v->is_dir = false +#define _Set_dir(v) v->is_dir = true; v->unsure = v->is_file = false +#define _Set_unsure(v) v->unsure = true +#define _Set_inserted(v) v->inserted = true +#define _Set_renamed(v) v->renamed = true +#define _Set_required(v) v->required = true + +vsize_s* vsize_new(char *file_path, const unsigned long id, + void *generate) +{ + vsize_s *v = Malloc(vsize_s); + + v->generate = generate; + v->id = id; + v->inserted = false; + v->is_dir = false; + v->is_file = false; + v->offset = -1; + v->path = Clone(file_path); + v->renamed = false; + v->required = false; + v->unsure = false; + v->updates = 0; + v->vsize = 0; + v->vsize_deficit = 0; + + return v; +} + +void vsize_destroy(vsize_s *v) +{ + if (!v) + return; + + free(v->path); + free(v); +} + +void init_parent_dir(vsize_s *v, const char *path) +{ + generate_s *g = v->generate; + char *clone = Clone(path); + char *parent = dirname(clone); + + vsize_s *v_parent = hmap_get(g->vsize_map, parent); + if (!v_parent) { + + // Parent directory does not yet have a vsize! + // Create a vsize object for it and set it as a pre-requirement + // so that the directory can be created during init mode. + + v_parent = vsize_new(parent, ++g->num_vsizes, g); + hmap_insert(g->vsize_map, parent, v_parent); + + _Set_required(v_parent); + _Set_dir(v_parent); + + // This is for debugging purposes only + _Set_inserted(v_parent); + v_parent->updates++; + + } else if (v_parent->unsure) { + // We now know for sure that this path must be a directory! + _Set_dir(v_parent); + v_parent->updates++; + } + + free(clone); +} + +void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags) +{ + + // v->first_encounter == false means, that this is the first occurance of + // this path and we didn't initialise it (means we didn't ensure that + // we want to create all parent directories etc. + + if (v->updates == 0) { + // We may use a recycled vfd object! When opening a file we always + // assume that the offset is 0! + vfd_s *vfd_ = vfd; + vfd_->offset = 0; + init_parent_dir(v, path); + + if (Has(flags, O_DIRECTORY)) { + _Set_required(v); + _Set_dir(v); + + } else if (Hasnt(flags, O_CREAT)) { + _Set_required(v); + _Set_file(v); + _Set_unsure(v); + } + v->updates++; + + } else if (v->unsure) { + if (Has(flags, O_DIRECTORY)) { + // Now we know for sure that this path must be a directory! + _Set_dir(v); + v->updates++; + } + } +} + +void vsize_close(vsize_s *v, void* vfd) +{ + vfd_s *vfd_ = vfd; + vfd_->offset = 0; + v->updates++; +} + +void vsize_stat(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + _Set_file(v); + + // We are not 100% sure that this is really a file, + // the path might be still a directory though! + _Set_unsure(v); + v->updates++; + } +} + +void vsize_rename(vsize_s *v, vsize_s *v2, + const char *path, const char *path2) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + _Set_file(v); + _Set_unsure(v); + v->updates++; + } + + if (v2->updates == 0) { + init_parent_dir(v2, path2); + _Set_file(v2); + + // We are not 100% sure that this is really a file, + // the path might be still a directory though! + _Set_unsure(v2); + + // For debugging purposes only + _Set_renamed(v2); + v2->updates++; + } +} + +void vsize_adjust(vsize_s *v, vfd_s* vfd) +{ + if (v->vsize >= vfd->offset) + return; + + long deficit = v->vsize - vfd->offset; + if (deficit < v->vsize_deficit) { + v->vsize_deficit = deficit; + _Set_required(v); + _Set_file(v); + } +} + +void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes) +{ + vfd_s *vfd_ = vfd; + vfd_->offset += bytes; + vsize_adjust(v, vfd_); + v->updates++; +} + +void vsize_seek(vsize_s *v, void *vfd, const long new_offset) +{ + //vfd_s *vfd_ = vfd; + + // The file's offset can be greater than the file's current size, in which + // case the next write to the file will extend the file. This is referred + // to as creating a hole in a file and is allowed. However, this behaviour + // does not suit the estimation of the file size before we want to run the + // test. + + // TODO: Implement file hole support! + //v->updates++; +} + +void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes) +{ + vfd_s *vfd_ = vfd; + vfd_->offset += bytes; + + if (v->vsize < vfd_->offset) + v->vsize = vfd_->offset; + + v->updates++; +} + +void vsize_mkdir(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_dir(v); + v->updates++; + } +} + +void vsize_rmdir(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + _Set_dir(v); + v->updates++; + } +} + +void vsize_unlink(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + if (!v->is_dir) { + _Set_file(v); + _Set_unsure(v); + } + v->updates++; + } +} diff --git a/ioreplay/src/generate/vsize.h b/ioreplay/src/generate/vsize.h new file mode 100644 index 0000000..bb1008e --- /dev/null +++ b/ioreplay/src/generate/vsize.h @@ -0,0 +1,180 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef VSIZE_H +#define VSIZE_H + +#include "../utils/utils.h" +#include "../datas/hmap.h" +#include "../vfd.h" + +/** + * @brief Definition of a virtual size object + * + * The virtual size is used to determine the expected type and size of a file. + * This piece of information will be added to the INIT section of the the + * .replay file. That file then will be created during test initialisation. + * before running the test. It is very likely the case that the test requires + * a file of a certain size already to be present, so it can be read from disk. + */ +typedef struct vsize_s_ { + char *path; /**< The path to the file/directory */ + off_t offset; /**< The current file offset */ + unsigned long id; /**< The vsize id */ + void *generate; /**< A pointer to the generate object */ + long vsize; /**< The virtual size */ + long vsize_deficit; /**< Size to use for file creating during init mode */ + bool renamed; /**< True if file/dir has been renamed */ + bool required; /**< True if init mode will create this file/dir */ + bool is_dir; /**< True if this file/dir is a directory */ + bool is_file; /**< True if this file/dir is a regular file */ + bool unsure; /**< True if the file type is not fully clear */ + long updates; /**< Amount of times this vsize has been updated */ + bool inserted; /**< For debugging purposes only */ +} vsize_s; + +/** + * @brief Creates a new vsize object + * + * @param file_path The corresponding file path + * @param id The vsize vsize aka ID + * @param generate The generate object + * @return The new vsize object + */ +vsize_s* vsize_new(char *file_path, const unsigned long id, void *generate); + +/** + * @brief Destroys a vsize object + * + * @param v The vsize object + */ +void vsize_destroy(vsize_s *v); + +/** + * @brief Ensures that the parent directory exists + * + * This function ensures that the parent directory exists as a vsize object! + * + * @param v The vsize object + * @param path The given path + */ +void init_parent_dir(vsize_s *v, const char *path); + +/** + * @brief Adjusts the vsize + * + * Compares the virtual file size of the file in the vsize + * object to the the offset in the virtual file descriptor. + * In case the offset is higher we have a size deficit and + * we need to mark it. That way ioreplay can ensure that + * during init mode it will create a file with the correct + * size prior of running the test! + * + * @param v The virtual size object + * @param vfd The virtual file descriptor object + */ +void vsize_adjust(vsize_s *v, vfd_s* vfd); + +/** + * @brief Adjust vsize on open + * + * @param v The virtual size object + * @param vfd The virtual file descriptor object + * @param path The file open path + * @param flags The file open flags + */ +void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags); + +/** + * @brief Adjust vsize on close + * + * @param v The virtual size object + * @param vfd The virtual file descriptor object + */ +void vsize_close(vsize_s *v, void *vfd); + +/** + * @brief Adjust vsize on stat + * + * @param v The virtual size object + * @param path The stat path + */ +void vsize_stat(vsize_s *v, const char *path); + +/** + * @brief Adjust vsize on rename + * + * @param v The virtual size object + * @param v2 The virtual size object of path2 + * @param path The first file path + * @param path2 The second file path + */ +void vsize_rename(vsize_s *v, vsize_s *v2, + const char *path, const char *path2); + +/** + * @brief Adjust vsize on read + * + * @param v The virtual size object + * @param vfd The virtual vile descriptor object + * @param path The file path + * @param bytes The amount of bytes read + */ +void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes); + +/** + * @brief Adjust vsize on seek + * + * @param v The virtual size object + * @param vfd The virtual vile descriptor object + * @param new_offset The new file offset after seek + */ +void vsize_seek(vsize_s *v, void *vfd, const long new_offset); + +/** + * @brief Adjust vsize on write + * + * @param v The virtual size object + * @param vfd The virtual vile descriptor object + * @param path The file path + * @param bytes The amount of bytes written + */ +void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes); + +/** + * @brief Adjust vsize on mkdir + * + * @param v The virtual size object + * @param path The directory path + */ +void vsize_mkdir(vsize_s *v, const char *path); + +/** + * @brief Adjust vsize on rmdir + * + * @param v The virtual size object + * @param path The directory path + */ +void vsize_rmdir(vsize_s *v, const char *path); + +/** + * @brief Adjust vsize on unlink + * + * @param v The virtual size object + * @param path The file path + */ +void vsize_unlink(vsize_s *v, const char *path); + +#endif // VSIZE_H + diff --git a/ioreplay/src/init/init.c b/ioreplay/src/init/init.c new file mode 100644 index 0000000..988729e --- /dev/null +++ b/ioreplay/src/init/init.c @@ -0,0 +1,226 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "init.h" + +#include "../datas/stack.h" +#include "itask.h" +#include "ithread.h" +#include "../meta/meta.h" +#include "../mounts.h" +#include "../utils/futils.h" + + +init_s *init_new(options_s *opts) +{ + init_s *i = Malloc(init_s); + + i->opts = opts; + i->mounts = mounts_new(opts); + i->threads_map = amap_new(i->mounts->count); + i->reuse_queue = rbuffer_new(4096); + i->replay_fd = Fopen(opts->replay_file, "r"); + + pthread_mutex_init(&i->reuse_queue_mutex, NULL); + + return i; +} + +void init_destroy(init_s *i) +{ + amap_destroy(i->threads_map); + mounts_destroy(i->mounts); + + itask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(i->reuse_queue))) { + itask_destroy(task); + } + rbuffer_destroy(i->reuse_queue); + + fclose(i->replay_fd); + pthread_mutex_destroy(&i->reuse_queue_mutex); + + free(i); +} + +void init_extract_header(init_s *i, off_t *init_offset) +{ + options_s *opts = i->opts; + meta_s *m = meta_new(i->replay_fd); + meta_read_start(m); + + long version = 0; + if (meta_read_l(m, "version", &version)) { + Put("Replay version is '%ld'", version); + if (version != REPLAY_VERSION) { + Error(".replay file of incompatible version, got %x, expected %x", + (int)version, REPLAY_VERSION); + } + } + + char *user; + if (meta_read_s(m, "user", &user)) { + Put("Setting user to '%s'", user); + opts->user = user; + } + + char *name; + if (meta_read_s(m, "name", &name)) { + Put("Setting name to '%s'", name); + opts->name = name; + } + + if (meta_read_l(m, "init_offset", init_offset)) { + if (*init_offset < 0) { + Error("Offset overflow (init offset too large in .replay)"); + } + Put("Setting init offset to '%ld'", *init_offset); + } + + meta_destroy(m); +} + +status_e init_run(options_s *opts) +{ + status_e ret = SUCCESS; + init_s *i = init_new(opts); + + off_t init_offset; + init_extract_header(i, &init_offset); + + // Ensure that all ./replay/NAME directories exist + mounts_init(i->mounts); + + // Don't do messy stuff as super user + drop_root(opts->user); + + // We need to clean up garbish from previous runs! + if (opts->purge) + mounts_purge(i->mounts); + else + mounts_trash(i->mounts); + + Out("Creating all files and directories requried for test '%s'...", + opts->name); + + // Seek to the INIT section + fseeko(i->replay_fd, init_offset, SEEK_SET); + + bool is_file = false, is_dir = false; + long vsize = 0; + char *path; + + // Stats + long dirs_created = 0; + long files_created = 0; + long files_total_size = 0; + + // Helper variables for getline + char *line = NULL; + size_t len = 0, read = 0; + char *saveptr; + + stack_s *all_threads = stack_new(); + + // Process the INIT section of the .replay file line by line. + + while ((read = getline(&line, &len, i->replay_fd)) != -1) { + char *tok = strtok_r(line, "|", &saveptr); + + for (int ntok = 0; tok; ntok++) { + switch (ntok) { + case 0: + is_dir = atoi(tok) == 1; + break; + case 1: + is_file = atoi(tok) == 1; + break; + case 2: + vsize = atol(tok); + if (vsize < 0) { + Error("Size overflow"); + } + break; + case 3: + path = tok; + break; + default: + break; + } + + tok = strtok_r(NULL, "|", &saveptr); + } + + itask_s *task = rbuffer_get_next(i->reuse_queue); + + if (!task) { + task = itask_new(); + + } else { + itask_extract_stats(task, &dirs_created, &files_created, + &files_total_size); + } + + // Set new task values + if (is_dir) { + task->is_dir = true; + + } else if (is_file) { + task->is_file = true; + task->vsize = vsize; + } + task->path = Clone(path); + + // We run one init thread per mount point + int mnr = mounts_get_mountnumber(i->mounts, path); + ithread_s *t = amap_get(i->threads_map, mnr); + + if (!t) { + t = ithread_new(i); + amap_set(i->threads_map, mnr, t); + stack_push(all_threads, t); + ithread_start(t); + } + + //itask_print(task); + while (!rbuffer_insert(t->queue, task)) + usleep(1000); + } + + ithread_s *t = NULL; + while (NULL != (t = stack_pop(all_threads))) { + ithread_terminate(t); + ithread_destroy(t); + } + stack_destroy(all_threads); + + itask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(i->reuse_queue))) { + itask_extract_stats(task, &dirs_created, &files_created, + &files_total_size); + itask_destroy(task); + } + + Put("Done!"); + + Put("Created %ld files (net total size: %.2fg) and %ld directories!", + files_created, files_total_size/(1024*1024*1024.0), + dirs_created); + + init_destroy(i); + + Put("You are ready to fire up the test now"); + + return ret; +} diff --git a/ioreplay/src/init/init.h b/ioreplay/src/init/init.h new file mode 100644 index 0000000..3d9f9e9 --- /dev/null +++ b/ioreplay/src/init/init.h @@ -0,0 +1,64 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INIT_H +#define INIT_H + +#include "../defaults.h" +#include "../options.h" +#include "../datas/amap.h" +#include "../datas/rbuffer.h" +#include "../mounts.h" + +typedef struct init_s_ { + amap_s *threads_map; + rbuffer_s *reuse_queue; + options_s *opts; + mounts_s *mounts; + FILE *replay_fd; + pthread_mutex_t reuse_queue_mutex; +} init_s; + +/** + * @brief Creates a new init object + * + * @param opts The options object + * @return The new mounts object + */ +init_s* init_new(options_s *opts); + +/** + * @brief Destroys the init object + * + * @param i The init object + */ +void init_destroy(init_s *i); + +/** + * @brief Initialises the test environment + * + * @param opts The options object + * @return SUCCESS if initialised without any issues + */ +status_e init_run(options_s *opts); + +/** + * @brief Extracts some useful information from the .replay meta header + * + * @param i The init object + * @param init_offset To store the offset of the init section + */ +void init_extract_header(init_s *i, off_t *init_offset); + +#endif // INIT_H diff --git a/ioreplay/src/init/itask.c b/ioreplay/src/init/itask.c new file mode 100644 index 0000000..f04ce33 --- /dev/null +++ b/ioreplay/src/init/itask.c @@ -0,0 +1,66 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "itask.h" + +itask_s* itask_new() +{ + itask_s *task = Malloc(itask_s); + + task->path = NULL; + itask_reset_stats(task); + + return task; +} + +void itask_destroy(itask_s *task) +{ + if (task->path) + free(task->path); + + free(task); +} + +void itask_reset_stats(itask_s *task) +{ + task->is_dir = task->is_file = false; + task->sizes_created = task->vsize = 0; + task->dirs_created = task->files_created = 0; + + if (task->path) { + free(task->path); + task->path = NULL; + } +} + +void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created, + long *files_total_size) +{ + *dirs_created += task->dirs_created; + *files_created += task->files_created; + *files_total_size += task->sizes_created; + + if (*dirs_created < 0 || *files_created < 0 || *files_total_size < 0) { + Error("Size overflow"); + } + + itask_reset_stats(task); +} + +void itask_print(itask_s *task) +{ + Put("itask(%p): is_dir:%d is_file:%d vsize:%ld path:%s", + (void*)task, task->is_dir, task->is_file, + task->vsize, task->path); +} diff --git a/ioreplay/src/init/itask.h b/ioreplay/src/init/itask.h new file mode 100644 index 0000000..b10d515 --- /dev/null +++ b/ioreplay/src/init/itask.h @@ -0,0 +1,72 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ITASK_H +#define ITASK_H + +#include "../defaults.h" + +/** + * @brief The initialise task definition + */ +typedef struct itask_s_ { + bool is_dir; + bool is_file; + long vsize; + char *path; + long dirs_created; + long files_created; + long sizes_created; +} itask_s; + +/** + * @brief Creates a new task object + * + * @return The new task object + */ +itask_s* itask_new(); + +/** + * @brief Resets the task stats + * + * @param task The itask object + */ +void itask_reset_stats(itask_s *task); + +/** + * @brief Extract stats from a task object + * + * @param task The itask object + * @param dirs_created Adds count of dirs created to that variable + * @param files_created Adds count of files created to that variable + * @param files_total_size Adds size of files created to that variable + */ +void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created, + long *files_total_size); + +/** + * @brief Destroys a given task object + * + * @param task The task object + */ +void itask_destroy(itask_s *task); + +/** + * @brief Prints a task to stdout + * + * @param task The task object + */ +void itask_print(itask_s *task); + +#endif // ITASK_H diff --git a/ioreplay/src/init/ithread.c b/ioreplay/src/init/ithread.c new file mode 100644 index 0000000..a580e70 --- /dev/null +++ b/ioreplay/src/init/ithread.c @@ -0,0 +1,99 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ithread.h" + +#include "itask.h" +#include "../utils/futils.h" + + +void* ithread_pthread_start(void *data) +{ + ithread_s *t = data; + init_s *i = t->init; + itask_s *task = NULL; + + do { + while (NULL != (task = rbuffer_get_next(t->queue))) { + ithread_run_task(t, task); + + // We need to mutex lock the reuse_queue as multiple threads + // can insert into it + pthread_mutex_lock(&i->reuse_queue_mutex); + int ret = rbuffer_insert(i->reuse_queue, task); + pthread_mutex_unlock(&i->reuse_queue_mutex); + if (!ret) + itask_destroy(task); + } + usleep(100); + } while (!t->terminate); + + while (NULL != (task = rbuffer_get_next(t->queue))) { + ithread_run_task(t, task); + if (!rbuffer_insert(i->reuse_queue, task)) + itask_destroy(task); + + pthread_mutex_lock(&i->reuse_queue_mutex); + int ret = rbuffer_insert(i->reuse_queue, task); + pthread_mutex_unlock(&i->reuse_queue_mutex); + if (!ret) + itask_destroy(task); + } + + return NULL; +} + +ithread_s* ithread_new(init_s *i) +{ + ithread_s *t = Malloc(ithread_s); + + t->init = i; + t->queue = rbuffer_new(1024); + t->terminate = false; + + return t; +} + +void ithread_start(ithread_s *t) +{ + start_pthread(&t->pthread, ithread_pthread_start, (void*)t); +} + +void ithread_destroy(ithread_s *t) +{ + rbuffer_destroy(t->queue); + free(t); +} + +void ithread_terminate(ithread_s *t) +{ + t->terminate = true; + pthread_join(t->pthread, NULL); +} + +void ithread_run_task(ithread_s *t, itask_s *task) +{ + if (task->is_dir) { + task->dirs_created += ensure_dir_exists(task->path); + + } else if (task->is_file) { + if (!ensure_file_exists(task->path, &task->dirs_created)) { + task->files_created++; + if (task->vsize > 0) { + append_random_to_file(task->path, task->vsize); + task->sizes_created += task->vsize; + } + } + } +} diff --git a/ioreplay/src/init/ithread.h b/ioreplay/src/init/ithread.h new file mode 100644 index 0000000..0884519 --- /dev/null +++ b/ioreplay/src/init/ithread.h @@ -0,0 +1,86 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ITHREAD_H +#define ITHREAD_H + +#include "../defaults.h" +#include "../datas/rbuffer.h" + +#include "init.h" +#include "itask.h" + +#include <pthread.h> + +/** + * @brief Definition of an init thread + * + */ +typedef struct ithread_s_ { + pthread_t pthread; /**< We run the init tasks in concurrent pthreads */ + rbuffer_s *queue; /**< The thread's task queue */ + init_s *init; /**< The responsible init object */ + bool terminate; /**< Indicates that thread can terminate */ +} ithread_s; + +/** + * @brief Creates a new thread object + * + * @param i The init object + * @return The new thread object + */ +ithread_s* ithread_new(init_s *i); + +/** + * @brief Terminates the thread + * + * This function waits (via join) for the pthread to complete all its + * current tasks from the queue. + * + * @param t The thread object + */ +void ithread_terminate(ithread_s* t); + +/** + * @brief Destroys the thread object + * + * @param t The thread object + */ +void ithread_destroy(ithread_s* t); + +/** + * @brief Executes the init task + * + * @param t The thread object + * @param task The task object + */ +void ithread_run_task(ithread_s* t, itask_s *task); + +/** + * @brief Starts the POSIX thread + * + * @param t The responsible thread object + */ +void ithread_start(ithread_s *t); + +/** + * @brief Entry point of the POSIX thread + * + * @param data Data passed to the pthread + * @return Always NULL on success + */ + +void* ithread_pthread_start(void *data); + +#endif // ITHREAD_H diff --git a/ioreplay/src/macros.h b/ioreplay/src/macros.h new file mode 100644 index 0000000..45e5a10 --- /dev/null +++ b/ioreplay/src/macros.h @@ -0,0 +1,116 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MACROS_H +#define MACROS_H + +#define Cleanup(code) ret = code; goto cleanup +#define Cleanup_unless(expr, code) \ + if (expr != code) { ret = code; goto cleanup; } + +// String helpers +#define Clone(str) notnull(strdup(str),__FILE__,__LINE__,0) +#define Eq(str1,str2) strcmp(str1,str2) == 0 + +// Number helpers +#define Abs(num) num >= 0 ? num : -num +#define Readhex(str) strtol(str, NULL, 16) +#define Perc(a, b) a > b ? b/(a/100.) : a/(b/100.) + +// Bitwise helpers +#define Has(flags, what) (flags & (what)) == (what) +#define Hasnt(flags, what) (flags & (what)) != (what) + +// Memory helpers +#define Malloc(what) \ + notnull(malloc(sizeof(what)),__FILE__,__LINE__,1) +#define Calloc(count,what) \ + notnull(calloc(count,sizeof(what)),__FILE__,__LINE__,count) +#define Mset(where,value,count,what) \ + memset(where,value,count*sizeof(what)) + +// Open helpers +#define Fopen(path, mode) fnotnull(fopen(path, mode), path, __FILE__, __LINE__) + +// Mmap helpers +#define Mmapshared(what) \ + mmapok(mmap(NULL, sizeof(what), \ + PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0), \ + __FILE__,__LINE__) +#define Cmapshared(count,what) \ + mmapok(mmap(NULL, count*sizeof(what), \ + PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0), \ + __FILE__,__LINE__) + +// Printing messages +#define Out(...) \ + fprintf(stdout, __VA_ARGS__); \ + fflush(stdout); +#define Put(...) \ + fprintf(stdout, __VA_ARGS__); \ + fprintf(stdout, "\n"); \ + fflush(stdout); + +// Printing debug messages +#define Debug(...) \ + fprintf(stderr, "%s:%d DEBUG: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + fflush(stderr); + +// Printing error messages +#define Error(...) \ + fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define Error_if(expr, ...) if (expr) { Error(__VA_ARGS__); } + +#define Errno(...) \ + fprintf(stderr, "%s:%d ERROR: %s (%d). ", __FILE__, __LINE__, \ + strerror(errno), errno); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define Errno_if(expr, ...) if (expr) { Errno(__VA_ARGS__); } + +#define Segfault(...) \ + fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); \ + *(int*)0 = 0; + +// Printing warn messages +#define Warn(...) \ + fprintf(stderr, "WARN: "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); + +#define Warn_if(expr, ...) if (expr) { Warn(__VA_ARGS__); } + +// Other helpers +#define Fill_with_stuff(buf, len) \ + for (int i = 0; i<len-1; ++i) { buf[i] = 'X'; } + +#endif // MACROS_H diff --git a/ioreplay/src/main.c b/ioreplay/src/main.c new file mode 100644 index 0000000..4a65de3 --- /dev/null +++ b/ioreplay/src/main.c @@ -0,0 +1,275 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file main.c + * @author Paul Buetow + * + * @brief The entry point of the I/O Replay program. + */ + +#include <signal.h> +#include <fcntl.h> + +#include "capture/capture.h" +#include "cleanup/cleanup.h" +#include "generate/generate.h" +#include "init/init.h" +#include "mounts.h" +#include "options.h" +#include "replay/replay.h" +#include "utests.h" +#include "utils/utils.h" + +/** + * @brief Do some architecture checks + * + * To ensure that I/O replay works correctly we have to check whether some + * data types are atomic or not. This is what this function does! + */ +static void _arch_check_atomic(void) +{ + if (sizeof(int) > sizeof(sig_atomic_t)) { + Error("int data type is not atomic on this architecture: %ld > %ld", + sizeof(int), sizeof(sig_atomic_t)); + + } else if (sizeof(bool) > sizeof(sig_atomic_t)) { + Error("bool data type is not atomic on this architecture: %ld > %ld", + sizeof(bool), sizeof(sig_atomic_t)); + } +} + +/** + * @brief Prints out version and copyright information + */ +static void _print_version(void) +{ + Put("This is I/O Replay %s - %s", IOREPLAY_VERSION, IOREPLAY_COPYRIGHT); +} + +/** + * @brief Print the synopsis + */ +static void _print_synopsis(void) +{ + _print_version(); + + Put("Synopsis:"); + Put("\tioreplay -c io.capture [-x PID] [-m MODULE]"); + Put("\tioreplay -c io.capture -r io.replay [-n str] [-u str] [-w str]"); + Put("\tioreplay -i io.replay"); + Put("\tioreplay -r io.replay [-p #] [-t #] [-D] [-s #]"); + Put("\tioreplay -R io.replay [-p #] [-t #] [-D] [-s #]"); + Put("\tioreplay -d"); + Put("\tioreplay -P"); + Put("\tioreplay -T [-n NAME]"); + Put("\tioreplay -V"); +} + +/** + * @brief Print a brief help + */ +static void _print_help(void) +{ + _print_synopsis(); + + Put("Help:"); + Put("\t-d Drop all Linux/FS caches and exit ioreplay"); + Put("\t-D Don't drop all caches (in conjunction with -r/-R):"); + Put("\t-s SPEED The speed factor (default: 0 [as fast as possible])"); + Put("\t-h Print this help"); + Put("\t-c FILE The capture file"); + Put("\t-n NAME The name (default: test0)"); + Put("\t-u USER The test run user (default: mcuser)"); + Put("\t-p #WORKERS Amount of of parallel worker processes (default: 4)"); + Put("\t-t #THREADS Threads per worker process (default: 128)"); + Put("\t-i REPLAYFILE The replay file to be initialised"); + Put("\t-r REPLAYFILE The replay file to be replayed"); + Put("\t-R REPLAYFILE Init and replay in one run (-i and -r combined)"); + Put("\t-S STATSFILE Write a stats file at the end of a test"); + Put("\t-T Trash data directories"); + Put("\t-P Purge all trash directories of all tests)"); + Put("\t-V Print I/O replay program version"); + Put("\t-w WD_BASE The working directory's base path"); + Put("\t (default: /usr/local/ioreplay)"); + Put("\t-x PID To specify a process ID (in conjunction with -c)"); + Put("\t-m MODULE To specify a module (in conjunction with -c)"); + Put("\nExample (run these commands one after another):"); + Put("\t 1.) sudo ioreplay -c io.capture"); + Put("\t 2.) sudo ioreplay -r io.replay -c io.capture -u paul -n test1"); + Put("\t 3.) sudo ioreplay -i io.replay"); + Put("\t 4.) sudo ioreplay -r io.replay -S"); +} + +/** + * @brief I/O Replay's entry point + * + * Not much more to document here though! + * @return The exit code + */ +int main(int argc, char **argv) +{ + _arch_check_atomic(); + status_e ret = UNKNOWN; + + bool dont_drop_caches = false; + options_s *opts = options_new(); + int opt = 0; + + while ((opt = getopt(argc, argv, "Vr:R:S:c:u:i:hw:n:dDs:w:p:t:UPTx:m:")) != -1) { + switch (opt) { + case 'U': + utests_run(); + Cleanup(SUCCESS); + break; + case 'V': + _print_version(); + Cleanup(SUCCESS); + break; + case 'd': + drop_caches(); + Cleanup(SUCCESS); + break; + case 'D': + dont_drop_caches = true; + break; + case 'c': + opts->capture_file = absolute_path(optarg); + Put("Capture file: %s", opts->capture_file); + break; + case 'P': + opts->purge = true; + Put("Purge option set"); + break; + case 'T': + opts->trash = true; + Put("Trash option set"); + break; + case 'i': + opts->init = true; + if (!opts->replay_file) { + opts->replay_file = absolute_path(optarg); + Put("Replay file: %s", opts->replay_file); + } + break; + case 'R': + opts->init = true; + opts->replay = true; + if (!opts->replay_file) { + opts->replay_file = absolute_path(optarg); + Put("Replay file: %s", opts->replay_file); + } + break; + case 'r': + opts->replay = true; + if (!opts->replay_file) { + opts->replay_file = absolute_path(optarg); + Put("Replay file: %s", opts->replay_file); + } + break; + case 'S': + opts->stats_file = Clone(optarg); + Put("Stats output file: %s", opts->stats_file); + break; + case 'w': + opts->wd_base = optarg; + Put("WD base: %s", opts->wd_base); + break; + case 'u': + opts->user = optarg; + Put("User: %s", opts->user); + break; + case 'm': + opts->module = Clone(optarg); + Put("Module: %s", opts->module); + break; + case 'n': + opts->name = optarg; + Put("Name: %s", opts->name); + break; + case 'h': + _print_help(); + Cleanup(SUCCESS); + case 's': + sscanf(optarg, "%lf", &opts->speed_factor); + Put("Speed factor: %lf", opts->speed_factor); + break; + case 'p': + opts->num_workers = atoi(optarg); + if (opts->num_workers < 1) + opts->num_workers = 1; + Put("Num worker processes: %d", opts->num_workers); + break; + case 't': + opts->num_threads_per_worker = atoi(optarg); + if (opts->num_threads_per_worker < 1) + opts->num_threads_per_worker = 1; + Put("Num threads per worker: %d", opts->num_threads_per_worker); + break; + case 'x': + opts->pid = atoi(optarg); + Put("PID: %d", opts->pid); + break; + default: + _print_help(); + Cleanup(ERROR); + } + } + + if (opts->purge || opts->trash) { + // Clean up all temp data of previous test runs + Cleanup(cleanup_run(opts)); + + } else if (opts->capture_file && !opts->replay_file) { + // We are going to capture I/O + Cleanup(capture_run(opts)); + + } else if (opts->capture_file && opts->replay_file) { + // We are going to generate a .replay file from the .capture file + Cleanup(generate_run(opts)); + + } else if (opts->replay_file && opts->init && !opts->replay) { + // We are going to initialise the test from the .replay file! + Cleanup(init_run(opts)); + + } else if (opts->replay_file && opts->init && opts->replay) { + // We are going to initialise the test and run the test! Run the + // initialiser in a sub-process, as it drops root privileges! + pid_t pid = fork(); + if (pid == 0) { + Cleanup(init_run(opts)); + } else { + opts->drop_caches = !dont_drop_caches; + int init_status; + waitpid(pid, &init_status, 0); + // Only proceed if initialisation was successfull! + Cleanup_unless(SUCCESS, init_status); + Cleanup(replay_run(opts)); + } + + } else if (opts->replay_file && !opts->init && opts->replay) { + // We are going to replay the I/O + opts->drop_caches = !dont_drop_caches; + Cleanup(replay_run(opts)); + + } else { + _print_help(); + Cleanup(ERROR); + } + +cleanup: + options_destroy(opts); + + return ret; +} diff --git a/ioreplay/src/meta/meta.c b/ioreplay/src/meta/meta.c new file mode 100644 index 0000000..d56c17e --- /dev/null +++ b/ioreplay/src/meta/meta.c @@ -0,0 +1,111 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "meta.h" + +#define _MAX_META_LEN 256 + +meta_s* meta_new(FILE *replay_fd) +{ + meta_s *m = Malloc(meta_s); + + m->replay_fd = replay_fd; + m->offset = ftello(replay_fd); + m->read_buf = NULL; + + return m; +} + +void meta_destroy(meta_s *m) +{ + if (!m) + return; + + if (m->read_buf) + free(m->read_buf); + + free(m); +} + +void meta_reserve(meta_s *m) +{ + // TODO: Use a hole in the .replay file to reserve space + char buf[_MAX_META_LEN]; + Mset(&buf, '#', _MAX_META_LEN-1, char); + fprintf(m->replay_fd, "%s\n", buf); +} + +void meta_write_start(meta_s *m) +{ + fseeko(m->replay_fd, m->offset, SEEK_SET); + // Write required '#' so that the regular worker processes + // will ignore that meta line. + fprintf(m->replay_fd, "#"); + + // Required for parsing in 'meta_read_s' + fprintf(m->replay_fd, "|"); +} + +void meta_write_s(meta_s *m, char *key, char *val) +{ + fprintf(m->replay_fd, "%s=%s|", key, val); +} + +void meta_write_l(meta_s *m, char *key, long val) +{ + char buf[1024]; + sprintf(buf, "%ld", val); + fprintf(m->replay_fd, "%s=%ld|", key, val); +} + +void meta_read_start(meta_s *m) +{ + size_t len = 0; + m->read_buf = Calloc(_MAX_META_LEN, char); + getline(&m->read_buf, &len, m->replay_fd); +} + +bool meta_read_s(meta_s *m, char *key, char **val) +{ + char *saveptr = NULL; + char *iterate_buf = Clone(m->read_buf); + int keylen = strlen(key); + + char *tok = strtok_r(iterate_buf, "|", &saveptr); + + while (tok) { + if (strncmp(tok, key, keylen) == 0 && tok[keylen] == '=') { + asprintf(val, "%s", tok+keylen+1); + free(iterate_buf); + return true; + } + tok = strtok_r(NULL, "|", &saveptr); + } + + free(iterate_buf); + return false; +} + +bool meta_read_l(meta_s *m, char *key, long *val) +{ + char *buf = NULL; + + if (meta_read_s(m, key, &buf)) { + *val = atol(buf); + free(buf); + return true; + } + + return false; +} diff --git a/ioreplay/src/meta/meta.h b/ioreplay/src/meta/meta.h new file mode 100644 index 0000000..10002cc --- /dev/null +++ b/ioreplay/src/meta/meta.h @@ -0,0 +1,107 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef META_H +#define META_H + +#include "../defaults.h" + +/** + * @brief The meta information definition + * + * This is used to write or read meta information to/from the header + * of the .replay file. This information then is used by ioreplay + * in other steps. E.g. reading the amount of used file descriptors + * from the meta header in order to allocate data structures of the + * correct sizes before running the test! + */ +typedef struct meta_s_ { + FILE* replay_fd; /**< The FS of the .replay file */ + off_t offset; /**< The meta offset (usually 0) */ + char* read_buf; /**< Pointer to a read buffer */ +} meta_s; + +/** + * @brief Creates a new meta bject + * + * @return The new meta object + */ +meta_s* meta_new(); + +/** + * @brief Destroys a meta object + * + * @param m The meta object + */ +void meta_destroy(meta_s *m); + +/** + * @brief Reserves space in the .replay file for the meta header + * + * @param m The meta object + */ +void meta_reserve(meta_s *m); + +/** + * @brief Indicates that we start writing the meta header to the .replay file + * + * @param m The meta object + */ +void meta_write_start(meta_s *m); + +/** + * @brief Writes a string to the meta header + * + * @param m The meta object + * @param key The key + * @param val The string value + */ +void meta_write_s(meta_s *m, char *key, char *val); + +/** + * @brief Writes a long to the meta header + * + * @param m The meta object + * @param key The key + * @param val The long value + */ +void meta_write_l(meta_s *m, char *key, long val); + +/** + * @brief indicates that we start reading from the meta header + * + * @param m The meta object + */ +void meta_read_start(meta_s *m); + +/** + * @brief Reads a string from the meta header + * + * @param m The meta object + * @param key The key + * @param val The string val read + */ +bool meta_read_s(meta_s *m, char *key, char **val); + +/** + * @brief Reads a long from the meta header + * + * @param m The meta object + * @param key The key + * @param val The long val read + */ +bool meta_read_l(meta_s *m, char *key, long *val); + +#endif // META_H + diff --git a/ioreplay/src/mounts.c b/ioreplay/src/mounts.c new file mode 100644 index 0000000..ac6f1d4 --- /dev/null +++ b/ioreplay/src/mounts.c @@ -0,0 +1,400 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mounts.h" + +#include "utils/futils.h" + +#define _PATH_INSERT "/.ioreplay/" +#define _PATH_INSERT_LEN 11 // strlen of _PATH_INSERT + +void mounts_read(mounts_s *m) +{ + char *mounts = "/proc/mounts"; + size_t len = 0; + char *line = NULL; + char *saveptr = NULL; + + Put("Reading '%s'", mounts); + + FILE *fp = Fopen(mounts, "r"); + Out("Adding supported file systems to replay paths:"); + + while (getline(&line, &len, fp) != -1) { + bool ignore = true; + + char *dev = strtok_r(line, " ", &saveptr); + if (dev == NULL) { + Error("Could not parse device from %s", mounts); + } + + char *mp = strtok_r(NULL, " ", &saveptr); + if (mp == NULL) { + Error("Could not parse mountpoint from %s", mounts); + } + + char *fs = strtok_r(NULL, " ", &saveptr); + if (fs == NULL) { + Error("Could not parse file system from %s", mounts); + } +#ifdef MP_DEBUG + Debug("fs:%s", fs); +#endif + // TODO: Make file system types configurable + if (Eq(fs, "ext2")) { + ignore = false; + } else if (Eq(fs, "ext5")) { + ignore = false; + } else if (Eq(fs, "ext4")) { + ignore = false; + } else if (Eq(fs, "xfs")) { + ignore = false; + } else if (Eq(fs, "zfs")) { + ignore = false; + } else if (Eq(fs, "btrfs")) { + ignore = false; + } + + if (ignore) { + if (strcmp(mp, "/") != 0) { + m->ignore_mps[m->ignore_count] = Clone(mp); + m->ignore_count++; + } + + } else if (m->count >= MAX_MOUNTPOINTS) { + Error("Exceeded max mount points: %d\n", m->count); + + } else { + Out(" %s (%s)", mp, fs); + m->mps[m->count] = Clone(mp); + m->lengths[m->count] = strlen(mp); + m->count++; + } + } + + fclose(fp); + Out("\n"); +} + +mounts_s *mounts_new(options_s *opts) +{ + mounts_s *m = Malloc(mounts_s); + + m->opts = opts; + m->count = 0; + m->ignore_count = 0; + mounts_read(m); + + return m; +} + +void mounts_destroy(mounts_s *m) +{ + if (!m) + return; + for (int i = 0; i < m->count; i++) + free(m->mps[i]); + free(m); +} + +void mounts_trash(mounts_s *m) +{ + options_s *opts = m->opts; + drop_root(opts->user); + Put("Moving all old files to trash (of previous tests)..."); + + struct timeval tv; + gettimeofday(&tv, NULL); + + char *wd_path = NULL; + asprintf(&wd_path, "%s/%s", opts->wd_base, opts->name); + + char *trash_path = NULL; + asprintf(&trash_path, "%s/.trash/%ld", opts->wd_base, tv.tv_sec); + + if (is_dir(wd_path)) { + ensure_dir_exists(trash_path); + chown_path(opts->user, trash_path); + if (rename(wd_path, trash_path)) { + Errno("Could not move '%s' to '%s'", wd_path, trash_path); + } + } + free(wd_path); + free(trash_path); + + for (int i = 0; i < m->count; i++) { + char *mp = m->mps[i]; + char *path = NULL; + asprintf(&path, "%s/%s/%s", mp, _PATH_INSERT, opts->name); + asprintf(&trash_path, "%s/%s/.trash/%ld", + mp, _PATH_INSERT, tv.tv_sec); + + if (is_dir(path)) { + ensure_dir_exists(trash_path); + chown_path(opts->user, trash_path); + if (rename(path, trash_path)) { + Errno("Could not move '%s' to '%s'", path, trash_path); + } + } + + free(path); + free(trash_path); + } + + Put("Done trashing!"); + Put("Once the drives fill up you may want to purge old data (-P)"); +} + +void mounts_purge(mounts_s *m) +{ + options_s *opts = m->opts; + drop_root(opts->user); + + Out("Purging all data from the following directories:"); + + int active_purgers = 0, max_purgers = 16; + if (opts->num_workers > max_purgers) + max_purgers = opts->num_workers; + + char *purge_path = NULL; + asprintf(&purge_path, "%s", opts->wd_base); + if (is_dir(purge_path)) { + Out(" %s", purge_path); + pid_t pid = fork(); + + if (pid == 0) { + ensure_dir_empty(purge_path); + free(purge_path); + exit(0); + + } else if (pid < 0) { + Errno("\nUnable to create cleaner process! :'-("); + } + active_purgers++; + } + free(purge_path); + + int cleaner_status = SUCCESS; + + for (int i = 0; i < m->count; i++) { + char *mp = m->mps[i]; + char *purge_path = NULL; + asprintf(&purge_path, "%s/%s", mp, _PATH_INSERT); + + if (is_dir(purge_path)) { + if (active_purgers+1 >= max_purgers) { + wait(&cleaner_status); + active_purgers--; + } + + // TODO: Use threading model same way as in init/init.c + pid_t pid = fork(); + if (pid == 0) { + Out(" %s", purge_path); + ensure_dir_empty(purge_path); + free(purge_path); + exit(0); + } else if (pid < 0) { + Errno("Unable to create cleaner process! :'-("); + } + active_purgers++; + } + free(purge_path); + } + + while (wait(&cleaner_status) > 0) + active_purgers--; + Put("\nCleaning done!"); +} + +void mounts_init(mounts_s *m) +{ + options_s *opts = m->opts; + char *wd_path = NULL; + asprintf(&wd_path, "%s/%s", opts->wd_base, opts->name); + ensure_dir_exists(wd_path); + chown_path(opts->user, opts->wd_base); + chown_path(opts->user, wd_path); + + if (chdir(wd_path)) { + Errno("Could not chdir into '%s'!", wd_path); + + } else { + Put("Chdir into '%s'", wd_path); + } + + free(wd_path); + + for (int i = 0; i < m->count; i++) { + char *mp = m->mps[i]; + char *path = NULL; + + // Create .ioreplay/ directory on MP + asprintf(&path, "%s/%s", mp, _PATH_INSERT); + ensure_dir_exists(path); + chown_path(m->opts->user, path); + free(path); + path = NULL; + + // Create .ioreplay/NAME directory on MP + asprintf(&path, "%s/%s/%s", mp, _PATH_INSERT, opts->name); + ensure_dir_exists(path); + chown_path(m->opts->user, path); + free(path); + } +} + +bool mounts_ignore_path(mounts_s *m, const char *path) +{ + // CentOS 7 specific, ignore temp namespace mounts! + char *pos = strstr(path, "/tmp/namespace-"); + if (pos == path) + return true; + + // iterate backwards through all mount points. + for (int i = m->ignore_count-1; i >= 0; --i) { + char *mountpoint = m->ignore_mps[i]; + pos = strstr(path, mountpoint); + // Ignore this path as it is in the ignore mp list + if (pos == path) + return true; + } + + return false; +} + +bool mounts_transform_path(mounts_s *m, const char *name, + char *path, char **path_r) +{ + char *tmp = NULL; +#ifdef DEBUG_TRANSFORM_PATH + char *original_path = path; +#endif + bool line_ok = true; + + // First figure out whether there are '..' in any paths. If so we have to + // tokenize the path and remove '..'. Example: + // transform '/foo/bar/../' into '/foo/'. + // Also remove double '/' from paths. + + if (strstr(path, "..") || strstr(path, "//")) { + // tmp will be freed under label 'cleanup' at end of function. + tmp = Calloc(strlen(path)+1, char); + + // stack to put the tokens on + stack_s *s = stack_new(); + + // we need a copy of the path, so we can tokenize it into the stack + char* clone = Clone(path); + + char *saveptr = NULL; + char *tok = strtok_r(clone, "/", &saveptr); + + // Add each part of the path to the stack. + while (tok) { + if (strcmp(tok, "..") == 0) { + stack_pop(s); + } else { + stack_push(s, tok); + } + tok = strtok_r(NULL, "/", &saveptr); + } + + if (stack_is_empty(s)) { + strcpy(tmp, "."); + + } else { + s = stack_new_reverse_from(s); + strcpy(tmp, "/"); + strcat(tmp, (char*)stack_pop(s)); + + while(!stack_is_empty(s)) { + strcat(tmp, "/"); + strcat(tmp, (char*)stack_pop(s)); + } + } + + stack_destroy(s); + free(clone); + + // This is the path without '..' and '//' (and '///' ... etc') + path = tmp; + } + + // Now heck whether the path is on a supported file system. If not, ignore! + if (mounts_ignore_path(m, path)) { + line_ok = false; + goto cleanup; + } + + // So the path is on a valid mount point! Now we need to insert + // .ioreplay/NAME to each mount point, e.g. /usr/local/.ioreplay/NAME/... + + // Iterate backwards through all mount points. + for (int i = m->count-1; i >= 0; --i) { + char *mountpoint = m->mps[i]; + int mp_len = m->lengths[i]; + + if (strncmp(path, mountpoint, mp_len) == 0) { + // Found a path to replace + // Now insert .ioreplay/NAME/ into the file path. + *path_r = Calloc(strlen(path) + strlen(name)+1 + + _PATH_INSERT_LEN+1, char); + + if (strcmp(mountpoint, "/") == 0) { + // Root path + strcpy(*path_r, _PATH_INSERT); + strcat(*path_r, name); + strcat(*path_r, path); + + } else { + strcpy(*path_r, mountpoint); + strcat(*path_r, _PATH_INSERT); + strcat(*path_r, name); + char *pos = path; + pos += mp_len * (int) sizeof(char); + strcat(*path_r, pos); + } + + goto cleanup; + } + } + + if (tmp) + free(tmp); + + return line_ok; + +cleanup: +#ifdef DEBUG_TRANSFORM_PATH + Debug("Transform path '%s' -> '%s' -> '%s'", original_path, path, *path_r); +#endif + if (tmp) + free(tmp); + + return line_ok; +} + +int mounts_get_mountnumber(mounts_s *m, const char *path) +{ + for (int i = m->count-1; i >= 0; --i) { + char *mountpoint = m->mps[i]; + int mp_len = m->lengths[i]; + + if (strncmp(path, mountpoint, mp_len) == 0) + return i; + } + + return 0; +} diff --git a/ioreplay/src/mounts.h b/ioreplay/src/mounts.h new file mode 100644 index 0000000..a644ddb --- /dev/null +++ b/ioreplay/src/mounts.h @@ -0,0 +1,154 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MOUNTPOINTS_H +#define MOUNTPOINTS_H + +#include "datas/stack.h" +#include "defaults.h" +#include "options.h" + +#define MAX_MOUNTPOINTS 1024 + +/** + * @brief Represents data parsed from /proc/mounts + * + * This is used to determine the file systems and the file system types + * currently mounted on the Linux system. I/O replay only replays I/O + * on specific file systems such as ext4 or xfs and will ignore any special + * or pseudo file systems such as tmpfs, devfs, sysfs. It does not make sense + * to replay I/O on these because there is actually no underlying block device + * attached to these. + * + * A mounts object helps to determine whether a path relies on a valid file + * system or not. All I/O operations on invalid file systems are being filtered + * out! + * + * The mounts object also does more things such as purging temp test data from + * the mountpoints etc... + */ +typedef struct mounts_s_ { + int count; /**< The amount of mount points */ + char *mps[MAX_MOUNTPOINTS]; /**< The mp paths */ + int lengths[MAX_MOUNTPOINTS]; /**< The mp lenghts */ + int ignore_count; /**< The amount of ignored mount points */ + char *ignore_mps[MAX_MOUNTPOINTS]; /**< The ignored mp paths */ + options_s *opts; /**< A pointer to the options object */ +} mounts_s; + +/** + * @brief Creates a new mounts object + * + * @param opts The options object + * @return The new mounts object + */ +mounts_s *mounts_new(options_s *opts); + +/** + * @brief Destroys the mounts object + * + * @param m The mounts object + */ +void mounts_destroy(mounts_s *m); + +/** + * @brief moves all files within replay mounts to trash + * + * It moves all files of the .ioreplay/NAME directories to + * .ioreplay/NAME.trashEPOCH directories for all available mount points. + * It does the same for the working dorectory of the current test. + * + * @param m The responsible mounts object + */ +void mounts_trash(mounts_s *m); + +/** + * @brief Deletes all files within replay mounts + * + * It deletes all files from the .ioreplay/ directories for all availabe + * mount points. It also deletes the working directory of all tests. The + * function forks one sub-process per mount point, so it is cleaning all drives + * in parallel. + * + * It can take a significant amount of time to actually delete all these files. + * That's why there is also a mounts_trash function, which will not delete the + * files but move them to trash folders so they can be deleted at a later + * point. + * + * @param m The responsible mounts object + */ +void mounts_purge(mounts_s *m); + +/** + * @brief Ensures all mounts have a .ioreplay/NAME directory + * + * These directories are used by ioreplay to run the I/O replay tests in. + * The function also ensures to have the correct user permissions for these + * directories. + * + * @param m The responsible mounts object + */ +void mounts_init(mounts_s *m); + +/** + * @brief Reads /proc/mounts to determine which mounts are available + * + * @param m The mounts object + */ +void mounts_read(mounts_s *m); + +/** + * @brief Determines whether a path should be ignored + * + * ioreplay replays I/O only on known mount points of known + * file system types. This function helps to determine whether + * a path is on a valid mount point or not. + * + * @param m The responsible mounts object + * @param path The path to check + * @return true if path has to be ignored + */ +bool mounts_ignore_path(mounts_s *m, const char *path); + +/** + * @brief Inserts ./ioreplay/NAME into a path + * + * This function inserts ./ioreplay/NAME into a given file path. + * The function also checks whether the path is on a supported replay + * path or not. E.g. we want to ignore file systems such as devfs, sysfs, + * procfs.. etc. + * + * @param m The responsible mountpoint object + * @param name The name of the test + * @param path The original path + * @param path_r The tansformed path (has to be freed if not NULL) + * @return False if this path is to be ignored + */ +bool mounts_transform_path(mounts_s *m, const char *name, + char *path, char **path_r); + + +/** + * @brief Get's the mount point number of a path + * + * Used by init.c to determine which thread to use to initialise a file + * or directory on a given path. + * + * @param m The responsible mountpoint object + * @param path The file/directory path + * @return The mountpoint number + */ +int mounts_get_mountnumber(mounts_s *m, const char *path); + +#endif // MOUNTPOINTS_H diff --git a/ioreplay/src/opcodes.h b/ioreplay/src/opcodes.h new file mode 100644 index 0000000..3d5c114 --- /dev/null +++ b/ioreplay/src/opcodes.h @@ -0,0 +1,103 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPCODES_H +#define OPCODES_H + +typedef enum { + // stat() syscalls + FSTAT = 0, + FSTAT_AT, + FSTATFS, + FSTATFS64, + LSTAT, + STAT, + STATFS, + STATFS64, + + // read() syscalls + READ = 10, + READV, + READAHEAD, + READDIR, + READLINK, + READLINK_AT, + + // write() syscalls + WRITE = 20, + WRITEV, + + // open() and other syscalls which may create files + OPEN = 30, + OPEN_AT, + CREAT, + MKDIR, + MKDIR_AT, + NAME_TO_HANDLE_AT, + OPEN_BY_HANDLE_AT, + + // rename() syscalls + RENAME = 40, + RENAME_AT, + RENAME_AT2, + + // close() and unlink() syscalls + CLOSE = 50, + UNLINK, + UNLINK_AT, + RMDIR, + + // sync() syscalls + FSYNC = 60, + FDATASYNC, + SYNC, + SYNCFS, + SYNC_FILE_RANGE, + + // other syscalls + FCNTL = 70, + GETDENTS, + LSEEK, + + // mmap syscalls + MMAP2 = 80, + MUNMAP, + REMAP, + MSYNC, + + // chmod() syscalls + CHMOD = 100, + FCHMOD, + FCHMODAT, + + // chown() syscalls + CHOWN = 110, + CHOWN16, + LCHOWN, + LCOWN16, + FCHOWN, + FCHOWN16, + FCHOWNAT, + + // Meta operations (I/O replay internal use only) + // A single thread terminates + META_EXIT = 900, + // All threads of a process termiate (process termination) + META_EXIT_GROUP, + // Meta operation for lamport synchronisation (currently unused) + META_TIMELINE + +} opcode_e; + +#endif // OPCODES_H diff --git a/ioreplay/src/options.c b/ioreplay/src/options.c new file mode 100644 index 0000000..c1dcdb9 --- /dev/null +++ b/ioreplay/src/options.c @@ -0,0 +1,51 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "options.h" + +options_s *options_new() +{ + options_s *o = Malloc(options_s); + + o->capture_file = NULL; + o->replay_file = NULL; + o->stats_file = NULL; + o->wd_base = "/usr/local/ioreplay"; + o->num_workers = 4; + o->num_threads_per_worker = 128; + o->user = "mcuser"; + o->name = "test0"; + o->init = false; + o->replay = false; + o->speed_factor = 0; + o->drop_caches = false; + o->purge = false; + o->trash = false; + o->pid = -1; + o->module = "ioreplay.ko"; + + return o; +} + +void options_destroy(options_s *o) +{ + if (o->capture_file) + free(o->capture_file); + if (o->replay_file) + free(o->replay_file); + if (o->stats_file) + free(o->stats_file); + + free(o); +} diff --git a/ioreplay/src/options.h b/ioreplay/src/options.h new file mode 100644 index 0000000..66cb0f7 --- /dev/null +++ b/ioreplay/src/options.h @@ -0,0 +1,61 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPTIONS_H +#define OPTIONS_H + +#include <stdbool.h> +#include "defaults.h" + +/** + * @brief The options definition, used to store user input + */ +typedef struct options_s_ { + char *capture_file; /**< The name of the .capture file */ + char *replay_file; /**< The name of the .replay file */ + char *stats_file; /**< The name of the .stats file */ + bool write_stats_file; /**< Write a stats file at the end of the test */ + char *user; /**< The user name to run the test as */ + char *name; /**< The name of the test (found in .ioreplay/name sub-dirs) */ + char *wd_base; /**< The working directory base */ + int num_workers; /**< The amount of worker processes */ + int num_threads_per_worker; /**< Max threads per worker processes */ + bool init; /**< If set ioreplay will initialise the environment */ + bool replay; /**< If set ioreplay will run/replay the test */ + bool purge; /**< If set ioreplay will purge the environment */ + bool trash; /**< If set ioreplay will trash the environment */ + bool drop_caches; /**< True if ioreplay should drop all Linux caches */ + double speed_factor; /**< Specifies how fast the test is replayed */ + int pid; /**< Specifies a process id to capture */ + char *module; /**< Specifies the kernel module for capturing */ +} options_s; + +/** + * @brief Creates a new options object + * + * The options object contains all options specified by the user as a command + * line option. It is filled with default values during creation. + * + * @return The options object + */ +options_s *options_new(); + +/** + * @brief Destroys the options object + * + * @param o The options object + */ +void options_destroy(options_s *o); + +#endif // OPTIONS_H diff --git a/ioreplay/src/replay/replay.c b/ioreplay/src/replay/replay.c new file mode 100644 index 0000000..89f5fee --- /dev/null +++ b/ioreplay/src/replay/replay.c @@ -0,0 +1,191 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "replay.h" + +#include "../datas/amap.h" +#include "../meta/meta.h" +#include "../mounts.h" +#include "rworker.h" +#include "rstats.h" + +void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes, + long *num_pids, long *num_fds, long *num_lines) +{ + meta_s *m = meta_new(replay_fd); + meta_read_start(m); + + long version = 0; + if (meta_read_l(m, "version", &version)) { + Put("Replay version is '%ld'", version); + if (version != REPLAY_VERSION) { + Error(".replay file of incompatible version, got %x, expected %x", + (int)version, REPLAY_VERSION); + } + } + + char *user; + if (meta_read_s(m, "user", &user)) { + Put("Setting user to '%s'", user); + opts->user = user; + } + + char *name; + if (meta_read_s(m, "name", &name)) { + Put("Setting name to '%s'", name); + opts->name = name; + } + + if (meta_read_l(m, "num_vsizes", num_vsizes)) { + if (*num_vsizes < 0) { + Error("Lamport vsize overflow"); + } + Put("Setting num of vsizes to '%ld'", *num_vsizes); + } + + if (meta_read_l(m, "num_mapped_pids", num_pids)) { + if (*num_pids < 0) { + Error("Process overflow (too many process IDs in .replay)"); + } + Put("Setting num of PIDs to '%ld'", *num_pids); + } + + if (meta_read_l(m, "num_mapped_fds", num_fds)) { + if (*num_fds < 0) { + Error("FD overflow (too many FDs in .replay)"); + } + Put("Setting num of FDs to '%ld'", *num_fds); + } + + if (meta_read_l(m, "num_lines", num_lines)) { + if (*num_fds < 0) { + Error("Overflow (too many lines in .replay)"); + } + Put("Setting num of lines to '%ld'", *num_lines); + } + + meta_destroy(m); +} + +status_e replay_run(options_s *opts) +{ + status_e status = SUCCESS; + + if (opts->drop_caches) { + drop_caches(); + //cache_file(opts->replay_file); + } + + // Extract information from the meta header + FILE *replay_fd = Fopen(opts->replay_file, "r"); + long num_vsizes = 0, num_pids = 0, num_fds = 0, num_lines = 0; + replay_extract_header(opts, replay_fd, &num_vsizes, &num_pids, + &num_fds, &num_lines); + fclose(replay_fd); + + // A map of all file descriptors used. + Out("Creating FD map..."); + amap_s *fds_map = NULL; + if (opts->num_workers > 1) { + fds_map = amap_new_mmapped(num_fds); + } else { + fds_map = amap_new(num_fds); + } + Put("done"); + + // To collect all individual worker's stats into the global + // stats object. + stack_s *all_worker_stats = stack_new(); + + // The global stats object + rstats_s *stats = rstats_new(opts); + rstats_start(stats); + + // Fork worker processes, each worker process will read the .replay file + // individually. + + if (opts->num_workers > 1) { + for (int i = 0; i < opts->num_workers; ++i) { + rworker_stats_s *worker_stats = rworker_stats_new_mmap(); + stack_push(all_worker_stats, worker_stats); + + pid_t pid = fork(); + + if (pid == 0) { + // One worker object per fork + rworker_s *w = rworker_new(i, fds_map, num_vsizes, num_pids, opts, + worker_stats); + + // Process the .replay journal line by line + status_e status = rworker_process_lines(w, num_lines); + Put("worker(%d): Exiting from %d with status %d", i, + pid, status); + rworker_destroy(w); + + // Exit sub-process + exit(status); + + } else if (pid < 0) { + Errno("worker(%d): Unable to create worker process! :'-(", i); + + } else { + Put("worker(%d): Process with pid %d forked", i, pid); + } + } + + drop_root(opts->user); + + Put("Waiting for worker processes to finish"); + pid_t pid; + int rworker_status = SUCCESS; + + while ((pid = wait(&rworker_status)) > 0) { + if (rworker_status != SUCCESS) + status = rworker_status; + + Put("Process with pid %d exited with status %d", + pid, rworker_status); + } + + Put("All workers finished (%d)!", status); + + } else { + Put("Only one worker, don't fork sub-processes"); + + rworker_stats_s *worker_stats = rworker_stats_new_mmap(); + stack_push(all_worker_stats, worker_stats); + + rworker_s *w = rworker_new(0, fds_map, num_vsizes, num_pids, + opts, worker_stats); + status = rworker_process_lines(w, num_lines); + rworker_destroy(w); + + Put("Worker finished work!"); + } + + // Collect all statistics + rstats_stop(stats); + while (!stack_is_empty(all_worker_stats)) { + rworker_stats_s *worker_stats = stack_pop(all_worker_stats); + rstats_add_from_worker(stats, worker_stats); + rworker_stats_destroy(worker_stats); + } + stack_destroy(all_worker_stats); + + rstats_print(stats); + rstats_destroy(stats); + + amap_destroy(fds_map); + return status; +} diff --git a/ioreplay/src/replay/replay.h b/ioreplay/src/replay/replay.h new file mode 100644 index 0000000..dcc3d84 --- /dev/null +++ b/ioreplay/src/replay/replay.h @@ -0,0 +1,46 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef REPLAY_H +#define REPLAY_H + +#include "../defaults.h" +#include "../utils/futils.h" +#include "../opcodes.h" +#include "../options.h" +#include "rioop.h" +#include "rprocess.h" + +/** + * @brief Replays the given .replay file + * + * @param opts The options object + * @return SUCCESS if everything went fine + */ +status_e replay_run(options_s *opts); + +/** + * @brief Extract required meta data from .replay's meta header + * + * @param opts The options object + * @param replay_fd The file handle to the .replay file + * @param num_vsizes The amount of virtual sizes/paths + * @param num_pids The amount of process IDs + * @param num_fds The amount of virtual file descriptors + * @param num_lines The amount of .replay lines with I/O ops + */ +void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes, + long *num_pids, long *num_fds,long *num_lines); + +#endif // REPLAY_H diff --git a/ioreplay/src/replay/rioop.c b/ioreplay/src/replay/rioop.c new file mode 100644 index 0000000..2e16c94 --- /dev/null +++ b/ioreplay/src/replay/rioop.c @@ -0,0 +1,425 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rioop.h" + +#include "../vfd.h" +#include "rworker.h" + +// Printing error messages +#define _Error(...) \ + fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\nlineno:%ld path:%s\n", task->lineno, vfd->path); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define _Errno(...) \ + fprintf(stderr, "%s:%d ERROR: %s (%d). ", __FILE__, __LINE__, \ + strerror(errno), errno); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\nlineno:%ld path:%s\n", task->lineno, vfd->path); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define _Init_arg(num) int arg = atoi(task->toks[num]) +#define _Init_cmd(num) int cmd = atoi(task->toks[num]) +#define _Init_fd(num) long fd = atol(task->toks[num]) +#define _Init_flags(num) int flags = atoi(task->toks[num]) +//#define _Init_mode(num) int mode = atoi(task->toks[num]) +#define _Init_offset(num) long offset = atol(task->toks[num]) +#define _Init_op(num) int op = atoi(task->toks[num]) +#define _Init_path2(num) char *path2 = task->toks[num] +#define _Init_path(num) char *path = task->toks[num] +#define _Init_rc(num) int rc = atoi(task->toks[num]) +#define _Init_whence(num) long whence = atol(task->toks[num]) + +#define _Init_bytes(num) \ + int bytes = atoi(task->toks[num]); \ + if (bytes <= 0) return + +#define _Init_virtfd \ + vfd_s *vfd = amap_get(p->fds_map, fd); \ + if (vfd == NULL) return + +void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_op(2); + + switch (op) { + // stat() syscalls + case FSTAT: + rioop_fstat(p, t, task); + break; + case FSTATFS: + case FSTATFS64: + //Error("op(%d) not implemented", op); + break; + case FSTAT_AT: + case LSTAT: + case STAT: + rioop_stat(p, t, task); + break; + case STATFS: + case STATFS64: + //Error("op(%d) not implemented", op); + break; + + // read() syscalls + case READ: + case READV: + rioop_read(p, t, task); + break; + case READAHEAD: + //Error("op(%d) not implemented", op); + break; + case READLINK: + case READLINK_AT: + //Error("op(%d) not implemented", op); + break; + + // write() syscalls + case WRITE: + case WRITEV: + rioop_write(p, t, task); + break; + + // open() and other syscalls which may creat + case OPEN: + case OPEN_AT: + rioop_open(p, t, task, -1); + break; + case CREAT: + // A call to crat() is equivalent to calling open() with flags.. + rioop_open(p, t, task, O_CREAT|O_WRONLY|O_TRUNC); + break; + case MKDIR: + case MKDIR_AT: + rioop_mkdir(p, t, task); + break; + + // rename() syscalls + case RENAME: + case RENAME_AT: + case RENAME_AT2: + rioop_rename(p, t, task); + break; + + // close() and unlink() syscalls + case CLOSE: + rioop_close(p, t, task); + break; + case UNLINK: + case UNLINK_AT: + rioop_unlink(p, t, task); + break; + case RMDIR: + rioop_rmdir(p, t, task); + break; + + // sync() syscalls + case FSYNC: + rioop_fsync(p, t, task); + break; + case FDATASYNC: + rioop_fdatasync(p, t, task); + break; + case SYNC: + case SYNCFS: + case SYNC_FILE_RANGE: + //Error("op(%d) not implemented", op); + break; + + // Other syscalls + case FCNTL: + rioop_fcntl(p, t, task); + break; + case GETDENTS: + rioop_getdents(p, t, task); + break; + case LSEEK: + rioop_lseek(p, t, task); + break; + + // chmod() syscalls + case CHMOD: + rioop_chmod(p, t, task); + break; + case FCHMOD: + rioop_fchmod(p, t, task); + break; + + // chown() syscalls + case CHOWN: + rioop_chown(p, t, task); + break; + case FCHOWN: + case FCHOWNAT: + rioop_fchown(p, t, task); + break; + case LCHOWN: + rioop_lchown(p, t, task); + break; + + // Meta operations (I/O replay internal use only). + case META_EXIT_GROUP: + break; + case META_TIMELINE: + break; + + default: + Error("op(%d) not implemented", op); + break; + } +} + +void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + struct stat buf; + stat(path, &buf); +} + +void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + struct stat buf; + fstat(vfd->fd, &buf); +} + +void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + _Init_path2(4); + rename(path, path2); +} + +void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_bytes(4); + _Init_virtfd; + + char *buf = Calloc(bytes+1, char); + read(vfd->fd, buf, bytes); + free(buf); +} + +void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_bytes(4); + _Init_virtfd; + + char *buf = Calloc(bytes+1, char); + sprintf(buf, "%ld", task->lineno); + Fill_with_stuff(buf, bytes); + if (vfd->fd == 0) { + Debug("%d %d %ld", vfd->fd, vfd->debug, task->lineno); + _Error("ERROR"); + } + write(vfd->fd, buf, bytes); + free(buf); +} + +void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_) +{ + _Init_fd(3); + _Init_path(4); + _Init_flags(6); + + // Special case as this is creat() now + if (flags_ != -1) + flags = flags_; + + bool directory = Has(flags, O_DIRECTORY); + + if (fd > 0) { + if (directory) { + // We can not open a directory via open() otherwise! + flags &= (O_RDONLY & ~(O_RDWR|O_WRONLY|O_CREAT)); + } else { + // We don't want to open the file in read only mode. + // SystemTap could have skipped syscalls to fcntl or open + flags &= ~O_RDONLY; + } + // flags |= O_DIRECT|O_SYNC; + flags &= ~O_EXCL; + } + + int ret = open(path, flags, S_IRWXU|S_IRWXG|S_IRWXO); + + if (fd < 0 && ret > 0) { + close(ret); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|open+close|%s|\n", path); + fflush(t->rthread_fd); +#endif + } + + if (fd > 0 && ret > 0) { + vfd_s *vfd = vfd_new(ret, fd, path); + amap_set(p->fds_map, fd, vfd); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|open|%s|\n", path); + fflush(t->rthread_fd); +#endif + } +} + +void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + + amap_unset(p->fds_map, fd); + if (vfd->dirfd) { + closedir(vfd->dirfd); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|closedir|%s|\n", vfd->path); + fflush(t->rthread_fd); +#endif + } else { + close(vfd->fd); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|close|%s|\n", vfd->path); + fflush(t->rthread_fd); +#endif + } + vfd_destroy(vfd); +} + +void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + + // getdents expects a dirfd + DIR *dirfd = fdopendir(vfd->fd); + if (dirfd) { + vfd->dirfd = dirfd; + readdir(dirfd); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|fdopendir|%s|\n", vfd->path); + fflush(t->rthread_fd); +#endif + } +} + +void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + mkdir(path, S_IRWXU|S_IRWXG|S_IRWXO); +} + +void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + unlink(path); +} + +void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + rmdir(path); +} + +void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_bytes(6); + _Init_virtfd; + lseek(vfd->fd, bytes, SEEK_SET); +} + +void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + fsync(vfd->fd); +} + +void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + fdatasync(vfd->fd); +} + +void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_cmd(4); + _Init_arg(5); + _Init_virtfd; + + switch (cmd) { + case F_GETFD: + case F_GETFL: + fcntl(vfd->fd, cmd); + break; + case F_SETFD: + case F_SETFL: + fcntl(vfd->fd, cmd, arg); + break; + default: + break; + } +} + +void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + chmod(path, S_IRWXU|S_IRWXG|S_IRWXO); +} + +void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + fchmod(vfd->fd, S_IRWXU|S_IRWXG|S_IRWXO); +} + +void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + rworker_s *w = t->worker; + options_s *opts = w->opts; + struct passwd *pwd = getpwnam(opts->user); + chown(path, pwd->pw_uid, -1); +} + +void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + rworker_s *w = t->worker; + options_s *opts = w->opts; + struct passwd *pwd = getpwnam(opts->user); + fchown(vfd->fd, pwd->pw_uid, -1); +} + +void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + rworker_s *w = t->worker; + options_s *opts = w->opts; + struct passwd *pwd = getpwnam(opts->user); + lchown(path, pwd->pw_uid, -1); +} + diff --git a/ioreplay/src/replay/rioop.h b/ioreplay/src/replay/rioop.h new file mode 100644 index 0000000..4db4284 --- /dev/null +++ b/ioreplay/src/replay/rioop.h @@ -0,0 +1,54 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RIOOP_H +#define RIOOP_H + +#include "../defaults.h" +#include "../utils/futils.h" +#include "../opcodes.h" +#include "rprocess.h" +#include "rthread.h" + +/** + * @brief Replays the responsible I/O operation of a given task + * + * @param p The virtual replay process object + * @param t The thread object + * @param task The replay task object + */ +void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task); + +void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_); +void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task); + +#endif // RIOOP_H diff --git a/ioreplay/src/replay/rprocess.c b/ioreplay/src/replay/rprocess.c new file mode 100644 index 0000000..4efd835 --- /dev/null +++ b/ioreplay/src/replay/rprocess.c @@ -0,0 +1,34 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rprocess.h" + +rprocess_s* rprocess_new(const int pid, amap_s *fds_map) +{ + rprocess_s *p = Malloc(rprocess_s); + + p->fds_map = fds_map; + p->pid = pid; + p->terminate = 0; + p->lineno = 0; + + return p; +} + +void rprocess_destroy(rprocess_s *p) +{ + if (!p) + return; + free(p); +} diff --git a/ioreplay/src/replay/rprocess.h b/ioreplay/src/replay/rprocess.h new file mode 100644 index 0000000..739dd89 --- /dev/null +++ b/ioreplay/src/replay/rprocess.h @@ -0,0 +1,40 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RPROCESS_H +#define RPROCESS_H + +#include "../datas/hmap.h" +#include "../datas/amap.h" +#include "../defaults.h" +#include "rthread.h" + +/** + * @brief The virtual replay process object definition + * + * This defines a virtual process in replay context. + */ +typedef struct rprocess_s_ { + int terminate; /**< Indicates whether the worker is terminating or not */ + int rworker_num; /**< The worker number of the responsible worker */ + int pid; /**< The virtual process ID */ + unsigned long lineno; /**< Holding the current .replay line number */ + bool initm; /**< Indicates whether ioreplay is in init mode or not */ + amap_s *fds_map; /**< Holding all file descriptors */ +} rprocess_s; + +rprocess_s* rprocess_new(const int pid, amap_s *fds_map); +void rprocess_destroy(rprocess_s* p); + +#endif // RPROCESS_H diff --git a/ioreplay/src/replay/rstats.c b/ioreplay/src/replay/rstats.c new file mode 100644 index 0000000..c3e6e38 --- /dev/null +++ b/ioreplay/src/replay/rstats.c @@ -0,0 +1,108 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rstats.h" + +#include <sys/types.h> + +rstats_s* rstats_new(options_s *opts) +{ + rstats_s *s = Malloc(rstats_s); + + s->opts = opts; + s->loadavg_high = 0; + s->ioops = 0; + s->duration = 0; + s->time_ahead = -1; + + if (opts->stats_file) + s->stats_fd = Fopen(opts->stats_file, "w"); + else + s->stats_fd = stdout; + + return s; +} + +void rstats_destroy(rstats_s *s) +{ + if (s->stats_fd != stdout) + fclose(s->stats_fd); + + free(s); +} + +rworker_stats_s* rworker_stats_new_mmap(options_s *opts) +{ + // Share this object between processes, so that the stats cann be + // collected by the master process! + rworker_stats_s *s = Mmapshared(rworker_stats_s); + + s->loadavg_high = 0; + s->ioops = 0; + s->time_ahead = -1; + + return s; +} + +void rworker_stats_destroy(rworker_stats_s *s) +{ + munmap(s, sizeof(rworker_stats_s)); +} + + +void rstats_start(rstats_s* s) +{ + gettimeofday(&s->start_time, NULL); +} + +void rstats_stop(rstats_s* s) +{ + gettimeofday(&s->end_time, NULL); + s->duration= ((s->end_time.tv_sec - s->start_time.tv_sec) * 1000 + + (s->end_time.tv_usec - s->start_time.tv_usec) / 1000) / 1000; + +} + +void rstats_add_from_worker(rstats_s* s, rworker_stats_s* w) +{ + if (s->loadavg_high < w->loadavg_high) + s->loadavg_high = w->loadavg_high; + + if (s->time_ahead == -1 || s->time_ahead > w->time_ahead) + s->time_ahead = w->time_ahead; + + s->ioops += w->ioops; +} + +void rstats_print(rstats_s* s) +{ + options_s *opts = s->opts; + + if (opts->stats_file) { + Put("Writing stats to '%s'", opts->stats_file); + } + + fprintf(s->stats_fd, "Stats of test '%s':\n", opts->name); + fprintf(s->stats_fd, "\tNum workers: %d\n", opts->num_workers); + fprintf(s->stats_fd, "\tThreads per worker: %d\n", opts->num_threads_per_worker); + fprintf(s->stats_fd, "\tThreads total: %d\n", + opts->num_threads_per_worker * opts->num_workers); + fprintf(s->stats_fd, "\tHighest loadavg: %.2f\n", s->loadavg_high); + fprintf(s->stats_fd, "\tPerformed ioops: %ld\n", s->ioops); + if (s->duration > 0) + fprintf(s->stats_fd, "\tAverage ioops/s: %.2f\n", s->ioops/s->duration); + fprintf(s->stats_fd, "\tTime ahead: %lds\n", s->time_ahead/1000); + fprintf(s->stats_fd, "\tTotal time: %.2fs\n", s->duration); +} + diff --git a/ioreplay/src/replay/rstats.h b/ioreplay/src/replay/rstats.h new file mode 100644 index 0000000..1ce3f27 --- /dev/null +++ b/ioreplay/src/replay/rstats.h @@ -0,0 +1,117 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file rstats.h + * @author Paul Buetow + * + * @brief For collecting replay stats + */ + +#ifndef RSTATS_H +#define RSTATS_H + +#include "../defaults.h" +#include "../options.h" + +#include <pthread.h> + +/** + * @brief Definition of the rstats object + * + * Used to store global statistics. + */ +typedef struct rstats_s_ { + double loadavg_high; /**< Highest load average */ + long ioops; /**< Total amount if io operations */ + double duration; /**< Duration of the test */ + long time_ahead; /**< Time ahead of the original speed */ + struct timeval start_time; /**< Start time of the test */ + struct timeval end_time; /**< End time of the test */ + options_s *opts; /**< The I/O replay options object */ + FILE *stats_fd; /**< The file descriptor for writing the stats */ +} rstats_s; + +/** + * @brief Definition of the per worker stats object + * + * Used to store per worker process I/O stats + */ +typedef struct rworker_stats_s_ { + double loadavg_high; /**< Highest amount of io ops per second */ + long ioops; /**< Total amount if io operations */ + long time_ahead; /**< Time ahead of the original speed */ +} rworker_stats_s; + +/** + * @brief Creates a new stats object + * + * @return The new stats object + */ +rstats_s* rstats_new(options_s *opts); + +/** + * @brief Destroys the stats object + * + * @param s The stats object + */ +void rstats_destroy(rstats_s* s); + +/** + * @brief Creates a new per worker stats object + * + * The memory is mapped into shared memory so it can be shared across multiple + * processes. + * + * @return The new stats object + */ +rworker_stats_s* rworker_stats_new_mmap(); + +/** + * @brief Destroys the per worker stats object + * + * @param s The stats object + */ +void rworker_stats_destroy(rworker_stats_s* s); + +/** + * @brief Starts the stats + * + * @param s The stats object + */ +void rstats_start(rstats_s* s); + +/** + * @brief Finalises the stats + * + * @param s The stats object + */ +void rstats_stop(rstats_s* s); + +/** + * @brief Prints the stats + * + * @param s The stats object + */ +void rstats_print(rstats_s* s); + +/** + * @brief Adds per worker stats to the global stats object + * + * @param s The global stats object + * @param w The worker stats object + */ +void rstats_add_from_worker(rstats_s* s, rworker_stats_s* w); + +#endif diff --git a/ioreplay/src/replay/rtask.c b/ioreplay/src/replay/rtask.c new file mode 100644 index 0000000..b1afb92 --- /dev/null +++ b/ioreplay/src/replay/rtask.c @@ -0,0 +1,50 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rtask.h" + +#include "rthread.h" +#include "rworker.h" + +rtask_s* rtask_new() +{ + rtask_s *task = Malloc(rtask_s); + + *task = (rtask_s) { + .worker = NULL, .process = NULL + }; + task->line[0] = '\0'; + +#ifdef THREAD_DEBUG + task->clone = NULL; +#endif + + return task; +} + +void rtask_destroy(rtask_s *task) +{ + if (task) + free(task); +} + +void rtask_update(rtask_s *task, void *worker, void *process, char *line, + const long lineno, const long vsize) +{ + task->worker = worker; + task->process = process; + task->lineno = lineno; + task->vsize = vsize; + strcpy(task->line, line); +} diff --git a/ioreplay/src/replay/rtask.h b/ioreplay/src/replay/rtask.h new file mode 100644 index 0000000..35c5714 --- /dev/null +++ b/ioreplay/src/replay/rtask.h @@ -0,0 +1,69 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RTASK_H +#define RTASK_H + +#include "../defaults.h" + +/** + * @brief The replay task definition + * + * The rtask holds all possible variables required to process a particular + * .replay line and to replay the corresponding I/O operation. + */ +typedef struct rtask_s_ { + void *worker; /* The responsible worker object */ + void *process; /* The responsible process object */ + unsigned long lineno; /**< The current line number */ + unsigned long vsize; /**< The vsize */ + char *toks[MAX_TOKENS+1]; /**< The tokens parsed from the .replay line */ + char line[MAX_LINE_LEN]; /**< The remaining part of the .replay line */ +#ifdef RTASK_DEBUG + char *clone; /**< Used for debug purposes only */ +#endif +} rtask_s; + +/** + * @brief Creates a new thread task object + * + * This function creates a new thread task object. Such a task object is used + * by the worker to hand over I/O tasks to the corresponding threads. The + * actual I/O work is performed by the threads then. + * + * @return The new thread task object + */ +rtask_s* rtask_new(); + +/** + * @brief Destroys the replay task object + * + * @param t The thread task object to be destroyed + */ +void rtask_destroy(rtask_s* t); + +/** + * @brief Updates a reused/recycle task object + * + * @param task The task object to be updated + * @param worker The responsibe worker object + * @param process The responsible process object + * @param line The remaining line of the .replay file + * @param lineno The current line number of the .replay file + * @param vsize The vsize/path id + */ +void rtask_update(rtask_s *task, void *worker, void *process, char *line, + const long lineno, const long vsize); + +#endif // RTASK_H diff --git a/ioreplay/src/replay/rthread.c b/ioreplay/src/replay/rthread.c new file mode 100644 index 0000000..55364ec --- /dev/null +++ b/ioreplay/src/replay/rthread.c @@ -0,0 +1,216 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rthread.h" + +#include <sys/types.h> + +#include "rworker.h" +#include "rprocess.h" + +#include "rioop.h" + +#ifdef THREAD_DEBUG +/** + * @brief For debugging purposes only + * + * @param t The responsible thread object + */ +static void _rthread_init_log(rthread_s *t) +{ + rworker_s *w = t->worker; + char *rthread_log = Calloc(1024, char); + snprintf(rthread_log, 1023, "/tmp/ioreplay/worker%d.thread%ld.debuglog", + w->rworker_num, (long)pthread_self()); + + ensure_dir_exists("/tmp/ioreplay"); + t->rthread_fd = Fopen(rthread_log, "a"); + + free(rthread_log); + fprintf(t->rthread_fd, "%ld: DEBUG: Created thread log\n", t->tid); +} +#endif + +void rthread_process_task(rthread_s* t, rtask_s *task, + pid_t pthread_id) +{ + char *next = task->line; + rworker_s *w = (rworker_s*) task->worker; + + // Tokenize the remaining elements of the line. + int ntoks = 0; + char *saveptr; + char *tok = strtok_r(next, "|", &saveptr); + + while (tok) { + if (ntoks > MAX_TOKENS) { + Error("worker(%d) pthread(%d): lineno:%lu, missing newline?", + w->rworker_num, pthread_id, task->lineno); + } + task->toks[ntoks++] = tok; + tok = strtok_r(NULL, "|", &saveptr); + } + // NULL marker (no more token from here) + task->toks[ntoks] = NULL; + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld(%ld): %s", + t->tid, (long)pthread_self(), task->clone); + fflush(t->rthread_fd); + free(task->clone); + task->clone = NULL; +#endif +#ifndef NO_RIOOP + // Perform the corresponding I/O operation! + rioop_run(task->process, t, task); +#endif + + // Make the task object recyclable/reusable + pthread_mutex_lock(&w->task_buffer_mutex); + if (!rbuffer_insert(w->task_buffer, task)) + // We can't recycle the task object if the buffer is full! + rtask_destroy(task); + pthread_mutex_unlock(&w->task_buffer_mutex); +} + +void *rthread_pthread_start(void *data) +{ + rthread_s* t = (rthread_s*) data; + rworker_s *w = t->worker; + rtask_s *task = NULL; + pid_t pthread_id = pthread_self(); + +#ifdef THREAD_DEBUG + _rthread_init_log(t); +#endif + + do { + while (!rbuffer_has_next(t->tasks) && !t->terminate) + usleep(100); + + while ((task = rbuffer_get_next(t->tasks)) != NULL) + rthread_process_task(t, task, pthread_id); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld: DEBUG: Idling\n", t->tid); + fflush(t->rthread_fd); +#endif + + // Tell rworker_s that thread is not doing any work! + int inserted = false; + while (!inserted && !t->terminate) { + if (rbuffer_has_next(t->tasks)) + break; + + usleep(1000); + + if (rbuffer_has_next(t->tasks)) + break; + + // Make the rthread reusable, he is without any tasks + // for some time. + pthread_mutex_lock(&w->rthread_buffer_mutex); + inserted = rbuffer_insert(w->rthread_buffer, t); + pthread_mutex_unlock(&w->rthread_buffer_mutex); + } + +#ifdef THREAD_DEBUG + if (inserted) { + fprintf(t->rthread_fd, "%ld: DEBUG: Added to thread buffer\n", + t->tid); + } else { + fprintf(t->rthread_fd, "%ld: DEBUG: Idling thread recovered\n", + t->tid); + } + fflush(t->rthread_fd); +#endif + + } while (!t->terminate); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld: DEBUG: Terminating\n", t->tid); + fflush(t->rthread_fd); +#endif + + // Process the very last tasks + while (NULL != (task = rbuffer_get_next(t->tasks))) + rthread_process_task(t, task, pthread_id); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld: DEBUG: Done terminating\n", t->tid); + fflush(t->rthread_fd); +#endif + + return NULL; +} + +rthread_s* rthread_new(const long tid, void *worker) +{ + rthread_s *t = Malloc(rthread_s); + rworker_s *w = worker; + + t->single_threaded = w->opts->num_threads_per_worker == 1; + t->tasks = rbuffer_new(TASK_BUFFER_PER_THREAD); + t->terminate = false; + t->worker = worker; + rthread_update(t, tid); + + if (t->single_threaded) { +#ifdef THREAD_DEBUG + _rthread_init_log(t); +#endif + return t; + } + + start_pthread(&t->pthread, rthread_pthread_start, (void*)t); + return t; +} + +long rthread_update(rthread_s *t, const long tid) +{ + long prev_tid = t->tid; + t->tid = tid; + + return prev_tid; +} + +void rthread_destroy(rthread_s *t) +{ + if (rbuffer_has_next(t->tasks)) { + Error("Didn't expect to have any tasks left!"); + } + rbuffer_destroy(t->tasks); + +#ifdef THREAD_DEBUG + if (t->rthread_fd) + fclose(t->rthread_fd); +#endif + + free(t); +} + +bool rthread_insert_task(rthread_s* t, rtask_s* task) +{ + if (t->single_threaded) { + rthread_process_task(t, task, pthread_self()); + return true; + } + return rbuffer_insert(t->tasks, task); +} + +void rthread_terminate(rthread_s* t) +{ + t->terminate = true; + pthread_join(t->pthread, NULL); +} diff --git a/ioreplay/src/replay/rthread.h b/ioreplay/src/replay/rthread.h new file mode 100644 index 0000000..9971e49 --- /dev/null +++ b/ioreplay/src/replay/rthread.h @@ -0,0 +1,123 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file rthread.h + * @author Paul Buetow + * + * @brief The replay thread definitiion + */ + +#ifndef RTHREAD_H +#define RTHREAD_H + +#include "../defaults.h" +#include "../datas/rbuffer.h" +#include "../datas/amap.h" +#include "../vfd.h" +#include "rtask.h" + +#include <pthread.h> + +/** + * @brief Definition of a worker thread + * + * Every worker utilises a set of worker threads in order to parallelise the + * replaying of the I/O! Every thread comes with its own task queue. It is + * filled by the repsonsible worker. + * + * The user can specify the max amount of threads per worker per -t command + * line switch. + */ +typedef struct rthread_s_ { + void *worker; /**< The responsible worker object */ + long tid; /**< The virtual thread id */ + rbuffer_s* tasks; /**< Holds all outstanding tasks */ + bool terminate; /**< True if thread shall terminate */ + bool single_threaded; /**< Worker is single threaded or not */ + pthread_t pthread; /**< We run the tasks in concurrent pthreads */ +#ifdef RTHREAD_DEBUG + FILE *rthread_fd; /**< Used for debugging purposes only */ +#endif +} rthread_s; + +/** + * @brief Creates a new thread object + * + * @param tid The thread ID + * @param worker The worker object managing this thread + * @return The new thread object + */ +rthread_s* rthread_new(const long tid, void *worker); + +/** + * @brief Updates a thread object after recycling it + * + * @param t The thread object + * @param tid The new thread ID + */ +long rthread_update(rthread_s *t, const long tid); + +/** + * @brief Terminates the thread + * + * This function waits (via join) for the pthread to complete all its + * current tasks from the queue. + * + * @param t The thread object + */ +void rthread_terminate(rthread_s* t); + +/** + * @brief Destroys the thread object + * + * @param t The thread object + */ +void rthread_destroy(rthread_s* t); + +/** + * @brief Inserts a task into the threads work queue + * + * Inserts a task into the threads work queue. We use an atomic ring buffer + * data structure for the work queue. The ring buffer does not require any + * mutex locks. + * + * @param t The thread object + * @param task The task to be inserted + * @return Returns true on success, returns false if the task queue is full + */ +bool rthread_insert_task(rthread_s* t, rtask_s* task); + +/** + * @brief Used by the pthread to process a task + * + * In this function the pthread will attempt to process a task. It extracts all + * required information from the task object and invokes the corresponding I/O + * syscalls. + * + * @param t The responsible thread object + * @param task The task object + * @param pthread_id The current pthread id + */ +void rthread_process_task(rthread_s* t, rtask_s *task, pid_t pthread_id); + +/** + * @brief The entry function for the pthreads + * + * @param data The data structure passed to the pthread + * @return The exit code of the pthread. + */ +void *rthread_pthread_start(void *data); + +#endif // RTHREAD_H diff --git a/ioreplay/src/replay/rworker.c b/ioreplay/src/replay/rworker.c new file mode 100644 index 0000000..5a50ada --- /dev/null +++ b/ioreplay/src/replay/rworker.c @@ -0,0 +1,360 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rworker.h" + +#include "../datas/stack.h" +#include "rprocess.h" +#include "rthread.h" + +#define _Compute_current_time(now) \ + (now.tv_sec - start_time.tv_sec) * 1000 \ + + (now.tv_usec - start_time.tv_usec) / 1000 + + +/** + * @brief A callback helper function for destroying all virtual process objects + * + * @param data The process object. + */ +static void _rprocess_destroy_cb(void *data) +{ + rprocess_destroy(data); +} + +rworker_s* rworker_new(const int rworker_num, amap_s *fds_map, + const long num_vsizes, const long num_pids, + options_s *opts, rworker_stats_s *worker_stats) +{ + rworker_s *w = Malloc(rworker_s); + +#ifdef THREAD_DEBUG + char *rworker_log = Calloc(1024, char); + snprintf(rworker_log, 1023, "/tmp/ioreplay/_worker%d.debuglog", + rworker_num); + + w->rworker_fd = Fopen(rworker_log, "a"); + free(rworker_log); + fprintf(w->rworker_fd, "DEBUG: Started worker\n"); +#endif + + w->rworker_num = rworker_num; + w->opts = opts; + w->fds_map = fds_map; + + w->rprocess_map = amap_new(num_pids); + w->rthread_map = amap_new(num_vsizes); + w->task_buffer = rbuffer_new(opts->num_threads_per_worker + *TASK_BUFFER_PER_THREAD); + w->rthread_buffer = rbuffer_new(opts->num_threads_per_worker); + w->worker_stats = worker_stats; + + // Attach a cleanup callback function to the worker map. + w->rprocess_map->data_destroy = _rprocess_destroy_cb; + + pthread_mutex_init(&w->rthread_buffer_mutex, NULL); + pthread_mutex_init(&w->task_buffer_mutex, NULL); + + // TODO: Check in the program whether the ulimit is high enough + // or not! (ulimit -n) + + return w; +} + +/** + * @brief Destroys the object + * + * Destroys the worker object (frees all memory allocated by the worker) + * + * @param w The worker object + */ +void rworker_destroy(rworker_s *w) +{ + if (!w) + return; + + if (w->rprocess_map) + amap_destroy(w->rprocess_map); + if (w->rthread_map) + amap_destroy(w->rthread_map); + + if (w->task_buffer) { + rtask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(w->task_buffer))) + rtask_destroy(task); + rbuffer_destroy(w->task_buffer); + } + + if (w->rthread_buffer) + rbuffer_destroy(w->rthread_buffer); + + pthread_mutex_destroy(&w->task_buffer_mutex); + pthread_mutex_destroy(&w->rthread_buffer_mutex); + +#ifdef THREAD_DEBUG + if (w->rworker_fd) + fclose(w->rworker_fd); +#endif + + free(w); +} + +status_e rworker_process_lines(rworker_s* w, const long num_lines) +{ + Out("worker(%d): Starting to process replay lines\n", w->rworker_num); + + options_s *opts = w->opts; + FILE *replay_fd = Fopen(opts->replay_file, "r"); + + // Drop root privileges, otherwise we may overwrite other system + // files by accident in case of a bug or user error! + drop_root(opts->user); + + // Variables required for the time based caluclations + struct timeval now, start_time; + long current_time = 0, stats_time = 0; + gettimeofday(&start_time, NULL); + + // Helper variables required for reading lines + char *line = NULL; + char *next = NULL, *next2 = NULL; + size_t len = 0, read = 0; + + // Helpers required for threading + rthread_s *t = NULL; + stack_s *all_threads = stack_new(); + rworker_stats_s *s = w->worker_stats; + + // More helper variables + //unsigned long lineno = 0, stats_ioop = 0, vsize_id = 0; + unsigned long lineno = 0, vsize_id = 0; + long pid = -1, time = -1; + + // Process the .replay file line by line. + while ((read = getline(&line, &len, replay_fd)) != -1) { + lineno++; + + if (read >= MAX_LINE_LEN) { + Error("line:%lu Exceeded max line len", lineno); + } + + // If the line begins with #: Ignore that line, it contains + // debug or meta information or comments. + + if (line[0] == '#') { + if (line[1] == 'I') { + // We stop replaying I/O once we reach the line '#INIT' + // which incitates the begin of the INIT section. + break; + } + continue; + } + +#ifdef THREAD_DEBUG + char *clone = Clone(line); +#endif + + next = strchr(line, '|'); + Error_if(!next, "lineno:%ld Could not parse time from input file", + lineno); + next[0] = '\0'; + next++; + time = atol(line); + + next2 = strchr(next, '|'); + Error_if(!next2, "Could not parse vsize_id from input file"); + next2[0] = '\0'; + next2++; + vsize_id = atol(next); + + // This worker is not responsible for this line, skip it! + if ((vsize_id % opts->num_workers) != w->rworker_num) { +#ifdef THREAD_DEBUG + free(clone); +#endif + continue; + } + + next = strchr(next2, '|'); + Error_if(!next, "Could not parse PID from input file"); + next[0] = '\0'; + next++; + pid = atol(next2); + + gettimeofday(&now, NULL); + current_time = _Compute_current_time(now); + + // Check whether the user specified a replay speed factor. If so, we + // may need to throttle down a bit. + + if (opts->speed_factor) { + s->time_ahead = time / opts->speed_factor - current_time; + if (s->time_ahead > 0) + usleep(s->time_ahead*1000); + + } else { + s->time_ahead = time - current_time; + } + + // Get the responsible process object. The process object holds data + // structures usually found in a Linux process, e.g. a table of open + // file descriptors. + + rprocess_s *p = amap_get(w->rprocess_map, pid); + if (p == NULL) { + p = rprocess_new(pid, w->fds_map); + amap_set(w->rprocess_map, pid, p); + } + p->lineno = lineno; + + if (opts->num_threads_per_worker == 1) { + // Single threaded mode? + if (!t) + t = rthread_new(vsize_id, w); + else + rthread_update(t, vsize_id); + + } else { + t = amap_get(w->rthread_map, vsize_id); + } + + if (t == NULL) { + + // First try to recycle an old (likely unused) thread + if (NULL != (t = rbuffer_get_next(w->rthread_buffer))) { + rthread_update(t, vsize_id); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Reused an idling thread\n"); + fflush(w->rworker_fd); +#endif + + } else if (opts->num_threads_per_worker <= all_threads->size) { + // Reached max threads, waiting until one becomes available + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Reached max threads\n"); + fflush(w->rworker_fd); +#endif + while (NULL == (t = rbuffer_get_next(w->rthread_buffer))) + usleep(1000); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Reused an idling thread\n"); + fflush(w->rworker_fd); +#endif + + rthread_update(t, vsize_id); + + } else { + t = rthread_new(vsize_id, w); + + // We hold a pointer to all created threads in a stack. This + // stack is later used to terminate/join all therads. + stack_push(all_threads, t); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Created a new thread\n"); + fflush(w->rworker_fd); +#endif + } + + amap_set(w->rthread_map, vsize_id, t); + } + + // Create a new task for the thread. The task contains all required + // information to run an I/O operation. However, first try to + // reuse/recycle a task object! If there is no such, create a new one. + + rtask_s *task = rbuffer_get_next(w->task_buffer); + if (!task) + task = rtask_new(); + rtask_update(task, w, p, next, lineno, vsize_id); + s->ioops++; + + +#ifdef THREAD_DEBUG + task->clone = clone; + fprintf(w->rworker_fd, "DEBUG: Inserting new task\n"); + fflush(w->rworker_fd); +#endif + + // Insert that task to a ring buffer to pass it to the pthread without + // much synchronisation overhead! + + while (!rthread_insert_task(t, task)) + // The ring buffer is full. This may happen if the pthread didn't + // manage to process tasks fast enough. re-try after a short period! + usleep(1000); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Task inserted\n"); + fflush(w->rworker_fd); +#endif + + // The worker prints out stats every 3s + if (current_time - stats_time > 3000) { + // IDEA: Maybe refactor this block to be implemented in rstats.c + + double loadavg = get_loadavg(); + + // Determines whether we replay the I/O faster or slower than + // original speed! + char *a_b = s->time_ahead >= 0 ? "ahead" : "behind"; + + Put("worker(%d): threads:%ld %s:%lds progress:%0.2f%% " + "loadavg:%0.2f", + w->rworker_num, all_threads->size, a_b, Abs(s->time_ahead/1000), + Perc(lineno,num_lines), loadavg); + + stats_time = current_time; + //stats_ioop = lineno; + + if (s->loadavg_high < loadavg) + s->loadavg_high = loadavg; + } + } + + Put("worker(%d): Waiting for all threads to finish business...", + w->rworker_num); + + // This will wait (join) all threads one after another until all threads + // have finished their work and have terminated. + + while (!stack_is_empty(all_threads)) { + rthread_s *t = stack_pop(all_threads); + rthread_terminate(t); + rthread_destroy(t); + } + stack_destroy(all_threads); + + // Collect some stats last time + double loadavg = get_loadavg(); + if (s->loadavg_high < loadavg) + s->loadavg_high = loadavg; + + gettimeofday(&now, NULL); + current_time = _Compute_current_time(now); + if (opts->speed_factor) { + s->time_ahead = time / opts->speed_factor - current_time; + } else { + s->time_ahead = time - current_time; + } + + + Put("worker(%d): All threads terminated!", w->rworker_num); + fclose(replay_fd); + + return SUCCESS; +} diff --git a/ioreplay/src/replay/rworker.h b/ioreplay/src/replay/rworker.h new file mode 100644 index 0000000..26a1300 --- /dev/null +++ b/ioreplay/src/replay/rworker.h @@ -0,0 +1,82 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RWORKER_H +#define RWORKER_H + +#include <pthread.h> + +#include "../datas/amap.h" +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "../options.h" +#include "rstats.h" + +/** + * @brief Represents a worker process. + * + * This represents an I/O replay worker process. The user can specify the + * amount of worker processes via the -p command line switch. This is not + * to confuse with rprocess_s, which represents an original captured process + * and we now want to replay the I/O for! + */ +typedef struct { + int rworker_num; /**< The current worker ID */ + amap_s* fds_map; /**< Holding all file descriptors */ + amap_s* rprocess_map; /**< Holding all processes handled by this worker */ + amap_s* rthread_map; /**< Holding all threads handled by this worker */ + rbuffer_s *task_buffer; /**< Buffering thread tasks to be reused */ + pthread_mutex_t task_buffer_mutex; /**< To sync access to task_buffer */ + rbuffer_s *rthread_buffer; /**< Buffering idle threads to be reused */ + pthread_mutex_t rthread_buffer_mutex; /**< Sync access to rthread_buffer */ + options_s *opts; /**< To synchronise access to rthread_buffer */ + rworker_stats_s *worker_stats; /**< Object holding per worker statistics */ +#ifdef RTHREAD_DEBUG + FILE *rworker_fd; /**< For debugging purposes only */ +#endif +} rworker_s; + +/** + * @brief Creates a new worker object + * + * @param rworker_num The worker number + * @param fds_map A map of all virtual file descriptor objects + * @param num_vsizes The amount of virtual sizes/total file paths of the test + * @param num_pids The total amount of virtual process IDs used in this test + * @param opts A pointer to the options object + * @param worker_stats A pointer to the worker stats object + + * @return The new worker object + */ +rworker_s* rworker_new(const int rworker_num, amap_s *fds_map, + const long num_vsizes, const long num_pids, + options_s* opts, rworker_stats_s *worker_stats); + +/** + * @brief Destroys a worker object + * + * @param w The worker object to be destroyed + */ +void rworker_destroy(rworker_s* w); + +/** + * @brief Makes the worker to process all .replay lines + * + * @param w The responsible worker object + * @param num_lines The total amount of I/O op lines in the .replay file + * @return SUCCESS if everything went fine + */ +status_e rworker_process_lines(rworker_s* w, const long num_lines); + +#endif // RWORKER_H diff --git a/ioreplay/src/utests.c b/ioreplay/src/utests.c new file mode 100644 index 0000000..5812a66 --- /dev/null +++ b/ioreplay/src/utests.c @@ -0,0 +1,30 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utests.h" + +#include "datas/amap.h" +#include "datas/hmap.h" +#include "datas/list.h" +#include "datas/rbuffer.h" + +void utests_run() +{ + fprintf(stderr, "Running unit tests\n"); + amap_test(); + hmap_test(); + list_test(); + rbuffer_test(); + fprintf(stderr, "Great success, run all unit tests without any errors!\n"); +} diff --git a/ioreplay/src/utests.h b/ioreplay/src/utests.h new file mode 100644 index 0000000..4ad6973 --- /dev/null +++ b/ioreplay/src/utests.h @@ -0,0 +1,25 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef UTESTS_H +#define UTESTS_H + +#include "utils/utils.h" + +/** + * @brief This function runs all currently implemented unit tests + */ +void utests_run(); + +#endif // UTESTS_H diff --git a/ioreplay/src/utils/futils.c b/ioreplay/src/utils/futils.c new file mode 100644 index 0000000..5b35618 --- /dev/null +++ b/ioreplay/src/utils/futils.c @@ -0,0 +1,291 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "futils.h" + +#include <libgen.h> +#include <pwd.h> +#include <unistd.h> +#include <limits.h> + +#include "../macros.h" + +void append_random_to_file(char *path, unsigned long bytes) +{ + char *buf = NULL; + int max_chunk = 50000000; // 50 mebibyetes + FILE *fp = Fopen(path, "a"); + + for (;;) { + if (bytes > max_chunk) { + if (!buf) + buf = Calloc(max_chunk+1, char); + + Fill_with_stuff(buf, max_chunk); + buf[max_chunk] = '\0'; + fprintf(fp, "%s", buf); + bytes -= max_chunk; + + // Print out a dot every time we wrote 'much' data to a file + Out("."); + + } else { + if (!buf) + buf = Calloc(bytes+1, char); + + Fill_with_stuff(buf, bytes); + buf[bytes] = '\0'; + fprintf(fp, "%s", buf); + + break; + } + } + + if (buf) + free(buf); + fclose(fp); +} + +long ensure_dir_exists(const char *path) +{ + long num_dirs_created = 0; + int ret = mkdir_p(path, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, + &num_dirs_created); + if (ret != 0) { + Errno("Could not create dir '%s'", path); + } + + return num_dirs_created; +} + +void ensure_parent_dir_exists(const char *path) +{ + char *clone = Clone(path); + char *parent = dirname(clone); + + int ret = mkdir_p(parent, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, NULL); + if (ret != 0) { + Errno("Could not create dir %s", parent); + } + + free(clone); +} + +void ensure_dir_empty(const char *path) +{ + DIR *dh = opendir(path); + + if (!dh) { + Errno("Unable to empty %s", path); + } + + struct dirent *de; + + while ((de = readdir(dh))) { + if (0 == strcmp(de->d_name, ".") || + 0 == strcmp(de->d_name, "..")) + continue; + + char *absolute; + asprintf(&absolute, "%s/%s", path, de->d_name); + + if (is_dir(absolute)) + ensure_dir_empty(absolute); + + if (remove(absolute) == -1) + // Don't throw an error if there is no such file or directory + if (errno != 2) { + Errno("Unable to remove %s", absolute); + } + + free(absolute); + } + + closedir(dh); +} + +int ensure_file_exists(char *path, long *num_dirs_created) +{ + if (is_reg(path)) + return SUCCESS; + + char *dirname = dirname_r(Clone(path)); + *num_dirs_created += ensure_dir_exists(dirname); + free(dirname); + + FILE *fp = fopen(path, "a"); + if (fp) { + // We only need some data, less than 1 block in size, this is answer: + fprintf(fp, "42"); + fclose(fp); + return SUCCESS; + } + + return ERROR; +} + +char* dirname_r(char *path) +{ + int len = strlen(path); + int has = 0; + int i = len-1; + + if (strcmp(path, "..") == 0) { + return path; + } + + if (path[i] == '/') { + // Root directory + if (len == 1) + return path; + + // Remove all trailing / + for (; i >= 0; --i) { + if (path[i] == '/') { + path[i] = '\0'; + has = 1; + } else { + break; + } + } + } + + // Find next / + for (; i >= 0; --i) { + if (path[i] == '/') { + path[i] = '\0'; + has = 1; + break; + } + } + + // If no / + if (has == 0) { + path[0] = '.'; + path[1] = '\0'; + } + + return path; +} + +bool is_dir(const char *path) +{ + struct stat path_stat; + if (stat(path, &path_stat) == 0 && S_ISDIR(path_stat.st_mode)) + return true; + return false; +} + +bool is_reg(const char *path) +{ + struct stat path_stat; + if (stat(path, &path_stat) == 0 && S_ISREG(path_stat.st_mode)) + return true; + return false; +} + +bool exists(const char *path) +{ + struct stat path_stat; + if (stat(path, &path_stat) == 0) + return true; + return false; +} + +int mkdir_p(const char *path, mode_t mode, long *num_dirs_created) +{ + int res = 0; + + if (is_dir(path)) + return 0; + + if (is_reg(path)) + unlink(path); + + char *top = dirname_r(Clone(path)); + if (0 != mkdir_p(top, mode, num_dirs_created)) + goto cleanup; + + if ((mkdir(path, mode) == -1) && (errno != EEXIST)) + res = -1; + + if (res != -1) + *num_dirs_created = *num_dirs_created+1; + +cleanup: + free(top); + + return res; +} + +void cache_file(const char *file) +{ + Out("Caching file %s... it can take a while", file); + FILE *fd = Fopen(file, "r"); + char *line = NULL; + size_t len = 0, read = 0; + + while ((read = getline(&line, &len, fd)) != -1); + fclose(fd); +} + +void drop_caches(void) +{ + Out("Dropping all Linux caches..."); + + if (getuid() != 0) { + Out("\n"); + Error("I need to be root to do this, aborting!"); + } + + // echo 3 > /proc/sys/vm/drop_caches + char *drop_caches = "/proc/sys/vm/drop_caches"; + FILE *fd = Fopen(drop_caches, "w"); + fprintf(fd, "3"); + fclose(fd); + + Put("done"); +} + +void chown_path(const char *user, const char *path) +{ + struct passwd *pwd = getpwnam(user); + if (!pwd) { + Errno("Unable to retrieve information about system user %s!", user); + } + + if (chown(path, pwd->pw_uid, -1) == -1) { + Errno("Could not change ownership of '%s' to '%s'!", path, user); + } +} + +char *absolute_path(const char *path) +{ + if (path[0] == '/') + return Clone(path); + + char cwd[MAX_LINE_LEN]; + getcwd(cwd, sizeof(char)*MAX_LINE_LEN); + + if (!getcwd(cwd, sizeof(cwd))) { + Errno("Could not get current working directory"); + } + + char *absolute = NULL; + if (-1 == asprintf(&absolute, "%s/%s", cwd, path)) { + Error("Could not get absolute path of '%s'", path); + } + + return absolute; +} diff --git a/ioreplay/src/utils/futils.h b/ioreplay/src/utils/futils.h new file mode 100644 index 0000000..9afde1a --- /dev/null +++ b/ioreplay/src/utils/futils.h @@ -0,0 +1,134 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef FUTILS_H +#define FUTILS_H + +#include "../defaults.h" + +/** + * @brief Thread safe version of dirname() + * + * @param path The full file path + * @return The directory path + */ +char* dirname_r(char *path); + +/** + * @brief Ensures that a file exists + * + * @param path The file path + * @param num_dirs_created Holds a count of how many sub dirs have been created + * @return -1 on error, 0 on success. + */ +int ensure_file_exists(char *path, long *num_dirs_created); + +/** + * @brief Checks whether path exists + * + * @param path The path + * @return true if the path exists + */ +bool exists(const char *path); + +/** + * @brief Check if path is a directory + * + * @param path The directory path + * @return true if the path is a directory, false otherwise + */ +bool is_dir(const char *path); + +/** + * @brief Check if path is a regular file + * + * @param path The file path + * @return true if the file at path is a regular fike + */ +bool is_reg(const char *path); + +/** + * @brief Create a directory recursively + * + * @param path The directory path + * @param mode The mode + * @param num_dirs_created Counts how many directories have been created + * @return -1 on error + */ +int mkdir_p(const char *path, mode_t mode, long *num_dirs_created); + +/** + * @brief Appends data to a file + * + * @param path The file path + * @param bytes The amount of bytes + */ +void append_random_to_file(char *path, unsigned long bytes); + +/** + * @brief Ensures that a directory exists + * + * @param path The directory path + * @return The amount of directories created (including parent directories) + */ +long ensure_dir_exists(const char *path); + +/** + * @brief Ensures that a parent directory exists + * + * @param path The directory path + */ +void ensure_parent_dir_exists(const char *path); + +/** + * @brief Ensures that a directory is empty + * + * @param path The directory path + */ +void ensure_dir_empty(const char *path); + +/** + * @brief Loading a file into the file system cache + * + * @param file The path to the file + */ +void cache_file(const char *file); + +/** + * @brief Drop all Linux caches + * + * This function drops all Linux caches, which includes all file + * system caches. + */ +void drop_caches(void); + +/** + * @brief Changes owner of a path + * + * Terminates the process with an error message if failed. + * + * @param user The new owner + * @param path The path + */ +void chown_path(const char *user, const char *path); + +/** + * @brief Retrieves the absolute path of a given path + * + * @param path The path + * @return The absolute path. It must be freed manually. + */ +char *absolute_path(const char *path); + +#endif // FUTILS_H diff --git a/ioreplay/src/utils/utils.c b/ioreplay/src/utils/utils.c new file mode 100644 index 0000000..57d6737 --- /dev/null +++ b/ioreplay/src/utils/utils.c @@ -0,0 +1,152 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utils.h" + +#include <sys/resource.h> +#include <sys/time.h> + +void* notnull(void *p, char *file, int line, int count) +{ + if (p == NULL) { + Errno("%s:%d count:%d Could not allocate memory", file, line, count); + } + return p; +} + + +FILE* fnotnull(FILE *fd, const char *path, char *file, int line) +{ + if (fd == NULL) { + Errno("%s:%d Could not open file '%s'", file, line, path); + } + return fd; +} + +void* mmapok(void *p, char *file, int line) +{ + if (p == MAP_FAILED) { + Errno("%s:%d: Mmap failed", file, line); + } + return p; +} + +char* strtok2_r(char *str, char *delim, char **saveptr) +{ + int len = strlen(delim); + + if (str == NULL) + str = *saveptr; + + char *next = strstr(str, delim); + if (next) { + next[0] = '\0'; + for (int i = 0; i < len; ++i) + next++; + *saveptr = next; + return str; + } + + return NULL; +} + +void chreplace(char *str, char replace, char with) +{ + for (int i = 0; ; ++i) { + if (str[i] == '\0') + break; + if (str[i] == replace) + str[i] = with; + } +} + +void strunquote(char *str) +{ + int len = strlen(str); + + if (str[0] == '"') { + if (str[len-1] == '"') + str[len-1] = '\0'; + for (int i = 1; i < len; ++i) + str[i-1] = str[i]; + } +} + +void drop_root(const char *user) +{ + if (getuid() == 0) { + Put("Dropping root privileges to user %s", user); + + struct passwd *pw = getpwnam(user); + + /* process is running as root, drop privileges */ + if (setgid(pw->pw_gid) != 0) { + Errno("setgid: Unable to drop group privileges!"); + } + if (setuid(pw->pw_uid) != 0) { + Errno("setuid: Unable to drop user privileges!"); + } + } +} + +void get_loadavg_s(char *readbuf) +{ + FILE *fp = Fopen("/proc/loadavg", "r"); + fgets(readbuf, 128, fp); + char *pos = strchr(readbuf, ' '); + pos[0] = '\0'; + fclose(fp); +} + +double get_loadavg() +{ + // Not thread safe, but multi processing safe + static char buf[128]; + get_loadavg_s(buf); + + return atof(buf); +} + +bool is_number(char *str) +{ + for (int i = 0; ; ++i) { + if (str[i] == '\0') + return true; + if (isdigit(str[i]) == 0 && str[i] != '-') + return false; + } + + return true; +} + +void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data) +{ + int rc = pthread_create(thread, NULL, cb, data); + + switch (rc) { + case 0: + break; + case EAGAIN: + Error("Out of resources while creating pthread (%d)", rc); + break; + case EINVAL: + Error("Ivalid settings while creating pthread (%d)", rc); + break; + case EPERM: + Error("No permissions to configure pthread (%d)", rc); + default: + Error("Unknown error while creating pthread (%d)", rc); + break; + } +} diff --git a/ioreplay/src/utils/utils.h b/ioreplay/src/utils/utils.h new file mode 100644 index 0000000..cfe4dbc --- /dev/null +++ b/ioreplay/src/utils/utils.h @@ -0,0 +1,165 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef UTILS_H +#define UTILS_H + +// For asprintf in stdio.h +#define _GNU_SOURCE + +#include <assert.h> +#include <ctype.h> +#include <dirent.h> +#include <errno.h> +#include <fcntl.h> +#include <libgen.h> +#include <linux/types.h> +#include <linux/unistd.h> +#include <pthread.h> +#include <pwd.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <time.h> +#include <unistd.h> + +#include "../macros.h" +#include "../defaults.h" + +/** + * @brief Check whether allocated memory is not NULL + * + * This function is used in conjunction with malloc() and co. It + * introduces an extra sanity check whether the memory could be + * allocated successfully or not. If not it will print out a error + * message stating the position in the source code where the memory + * allocation failed! + * + * @param p The pointer being checked + * @param file The source file name the memory was allocated in + * @param line The source line number the memory was allocated at + * @param count The amount of memory being allocated + * @return The pointer to the allocated memory + */ +void* notnull(void *p, char *file, int line, int count); + +/** + * @brief Check whether opened file handle is not NULL + * + * This function is used in conjunction with fopen(). It + * introduces an extra sanity check whether the file could be + * opened successfully or not. If not it will print out a error + * message stating the position in the source code where the open + * failed! + * + * @param fd The fd stream to be checked. + * @param path The file path opened + * @param file The source file name + * @param line The source line number + * @return The pointer to the allocated memory + */ +FILE* fnotnull(FILE *fd, const char *path, char *file, int line); + +/** + * @brief Check whether allocated memory via mmap is not null + * + * This function is used in conjunction with mmap() and co. It + * introduces an extra sanity check whether the memory could be + * allocated successfully or not. If not it will print out a error + * message stating the position in the source code where the memory + * allocation failed! + * + * @param addr The pointer being checked + * @param file The source file name the memory was allocated in + * @param line The source line number the memory was allocated at + * @return The pointer to the allocated memory + */ +void* mmapok(void *addr, char *file, int line); + +/** + * @brief A version of strtok_r supporting multi char delims + * + * @param str The input string + * @param delim The multi-char delimiter + * @param saveptr A temp storage location + * @return The next match if != NULL + */ +char* strtok2_r(char *str, char *delim, char **saveptr); + +/** + * @brief Replaces a character with another one in a string + * + * @param str The input string + * @param replace The character to be replaced + * @param with The character to replace with + */ +void chreplace(char *str, char replace, char with); + +/** + * @brief Removes quotes from a string + * + * @param str The input sting + */ +void strunquote(char *str); + +/** + * @brief Drop root privileges + * + * @param user The user to switch to + */ +void drop_root(const char *user); + +/** + * @brief Retrieve current 1 min Linux load average + * + * @param readbuf The buffer to store the load average as a string + */ +void get_loadavg_s(char *readbuf); + +/** + * @brief Retrieve current 1 min Linux load average + * + * This function is not thread safe! + * + * @return The 1 minute load average of the system + */ +double get_loadavg(); + +/** + * @brief Check whether a string represents a number + * + * @param str The input string + * @return true if all characters of the input string are a digits + */ +bool is_number(char *str); + +/** + * @brief Wrapper around pthread_create + * + * The wrapper also checks whether the thread has been created successfully + * or not! It will exit the process if not. + * + * @param thread The POSIX thread variable + * @param cb The threadss start callback routine + * @param data A data pointer passed to the thread. + */ +void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data); + +#endif // UTILS_H diff --git a/ioreplay/src/vfd.c b/ioreplay/src/vfd.c new file mode 100644 index 0000000..6e86f61 --- /dev/null +++ b/ioreplay/src/vfd.c @@ -0,0 +1,55 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "vfd.h" + +vfd_s* vfd_new(const int fd, const long mapped_fd, char *path) +{ + vfd_s *vfd = Malloc(vfd_s); + vfd->path = NULL; + vfd->debug = false; + vfd_update(vfd, fd, mapped_fd, path); + + return vfd; +} + +void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path) +{ + vfd->fd = fd; + vfd->dirfd = NULL; + vfd->mapped_fd = mapped_fd; + vfd->offset = 0; + + if (path) + free(vfd->path); + + vfd->path = Clone(path); +} + +void vfd_destroy(vfd_s *vfd) +{ + if (!vfd) + return; + + if (vfd->path) + free(vfd->path); + + free(vfd); +} + +void vfd_print(vfd_s *vfd) +{ + fprintf(stderr, "virtfd(%p) fd:%x mapped_fd:%lx path:%s\n", + (void*)vfd, vfd->fd, vfd->mapped_fd, vfd->path); +} diff --git a/ioreplay/src/vfd.h b/ioreplay/src/vfd.h new file mode 100644 index 0000000..fd0c4fb --- /dev/null +++ b/ioreplay/src/vfd.h @@ -0,0 +1,77 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef VFD_H +#define VFD_H + +#include "opcodes.h" +#include "defaults.h" + +/** + * @brief The virtual file descriptor definition + * + * A virtual file descriptor represents a file descriptor from ioreplay's + * point of view and is being used in various ways to simulate the real I/O + * protocolled to the .capture/.replay files. + * + * Generally speaking I/O replay maps the real FD numbers (the ones logged to + * the .capture file) to virtual FD numbers (a uniqe FD number for every open + * to increase concurrency). + */ +typedef struct vfd_s_ { + int fd; /**< the real fd */ + DIR *dirfd; /**< The real dirfd */ + long mapped_fd; /**< The mapped fd (virtual fd) */ + char *path; /**< The file path belonging to that fd */ + bool free_path; /**< True if path has to be freed or not */ + unsigned long offset; /**< The current virtual file offset in bytes */ + int debug; /**< Used for debugging purposes only */ +} vfd_s; + +/** + * @brief Creates a new virtual file descriptor object + * + * @param fd The file descriptor + * @param mapped_fd The mapped file descriptor + * @param path The path name + * @return The new fd object + */ +vfd_s* vfd_new(const int fd, const long mapped_fd, char *path); + +/** + * @brief Updates the virtfd object + * + * @param vfd The virtfd object + * @param fd The (real) file descriptor + * @param mapped_fd The mapped (virtual) file descriptor + * @param path The path name + * @return The new fd object + */ +void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path); + +/** + * @brief Destroys a file descriptor object + * + * @param vfd The file descriptor object + */ +void vfd_destroy(vfd_s *vfd); + +/** + * @brief Prints the virtual file descriptor + * @param vfd The virtual file descriptor + */ +void vfd_print(vfd_s *vfd); + +#endif // VFD_H + diff --git a/ioreplay/tags b/ioreplay/tags new file mode 100644 index 0000000..cbad51a --- /dev/null +++ b/ioreplay/tags @@ -0,0 +1,661 @@ +!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ +!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ +!_TAG_PROGRAM_AUTHOR Darren Hiebert /dhiebert@users.sourceforge.net/ +!_TAG_PROGRAM_NAME Exuberant Ctags // +!_TAG_PROGRAM_URL http://ctags.sourceforge.net /official site/ +!_TAG_PROGRAM_VERSION 5.8 // +AMAP_H ./src/datas/amap.h 2;" d +AMAP_MAX_ARRAY_LENGTH ./src/datas/amap.h 6;" d +Abs ./src/macros.h 13;" d +BTREE_H ./src/datas/btree.h 2;" d +CAPTURE_VERSION ./src/defaults.h 7;" d +CHMOD ./src/opcodes.h /^ CHMOD = 100,$/;" e enum:__anon1 +CHOWN ./src/opcodes.h /^ CHOWN = 110,$/;" e enum:__anon1 +CHOWN16 ./src/opcodes.h /^ CHOWN16,$/;" e enum:__anon1 +CLEANUP_H ./src/cleanup/cleanup.h 2;" d +CLOSE ./src/opcodes.h /^ CLOSE = 50,$/;" e enum:__anon1 +CREAT ./src/opcodes.h /^ CREAT,$/;" e enum:__anon1 +Calloc ./src/macros.h 23;" d +Cleanup ./src/macros.h 4;" d +Cleanup_unless ./src/macros.h 5;" d +Clone ./src/macros.h 9;" d +Cmapshared ./src/macros.h 36;" d +DEFAULTS_H ./src/defaults.h 2;" d +Debug ./src/macros.h 51;" d +ERROR ./src/defaults.h /^ ERROR, \/**< An error happened :-( *\/$/;" e enum:status_e_ +Eq ./src/macros.h 10;" d +Errno ./src/macros.h 68;" d +Errno_if ./src/macros.h 77;" d +Error ./src/macros.h 58;" d +Error_if ./src/macros.h 66;" d +F ./src/generate/gtask.h /^ int F; \/**< Arguments for fcntl syscall *\/$/;" m struct:gtask_s_ +FCHMOD ./src/opcodes.h /^ FCHMOD,$/;" e enum:__anon1 +FCHMODAT ./src/opcodes.h /^ FCHMODAT,$/;" e enum:__anon1 +FCHOWN ./src/opcodes.h /^ FCHOWN,$/;" e enum:__anon1 +FCHOWN16 ./src/opcodes.h /^ FCHOWN16,$/;" e enum:__anon1 +FCHOWNAT ./src/opcodes.h /^ FCHOWNAT,$/;" e enum:__anon1 +FCNTL ./src/opcodes.h /^ FCNTL = 70,$/;" e enum:__anon1 +FDATASYNC ./src/opcodes.h /^ FDATASYNC,$/;" e enum:__anon1 +FSTAT ./src/opcodes.h /^ FSTAT = 0,$/;" e enum:__anon1 +FSTATFS ./src/opcodes.h /^ FSTATFS,$/;" e enum:__anon1 +FSTATFS64 ./src/opcodes.h /^ FSTATFS64,$/;" e enum:__anon1 +FSTAT_AT ./src/opcodes.h /^ FSTAT_AT,$/;" e enum:__anon1 +FSYNC ./src/opcodes.h /^ FSYNC = 60,$/;" e enum:__anon1 +FUTILS_H ./src/utils/futils.h 2;" d +Fill_with_stuff ./src/macros.h 98;" d +Fopen ./src/macros.h 29;" d +G ./src/generate/gtask.h /^ int G; \/**< Arguments for fcntl syscall *\/$/;" m struct:gtask_s_ +GENERATE_H ./src/generate/generate.h 2;" d +GETDENTS ./src/opcodes.h /^ GETDENTS,$/;" e enum:__anon1 +GIOOP_H ./src/generate/gioop.h 2;" d +GPARSER_H ./src/generate/gparser.h 2;" d +GPROCESS_H ./src/generate/gprocess.h 2;" d +GTASK_H ./src/generate/gtask.h 2;" d +GWRITER_H ./src/generate/gwriter.h 2;" d +Gioop_write ./src/generate/gioop.h 12;" d +HMAP_H ./src/datas/hmap.h 2;" d +Has ./src/macros.h 17;" d +Hasnt ./src/macros.h 18;" d +INIT_H ./src/init/init.h 2;" d +IOREPLAY_COPYRIGHT ./src/defaults.h 19;" d +IOREPLAY_VERSION ./src/defaults.h 17;" d +ITASK_H ./src/init/itask.h 2;" d +ITHREAD_H ./src/init/ithread.h 2;" d +LCHOWN ./src/opcodes.h /^ LCHOWN,$/;" e enum:__anon1 +LCOWN16 ./src/opcodes.h /^ LCOWN16,$/;" e enum:__anon1 +LIST_H ./src/datas/list.h 2;" d +LSEEK ./src/opcodes.h /^ LSEEK,$/;" e enum:__anon1 +LSTAT ./src/opcodes.h /^ LSTAT,$/;" e enum:__anon1 +MACROS_H ./src/macros.h 2;" d +MAX_LINE_LEN ./src/defaults.h 13;" d +MAX_MOUNTPOINTS ./src/mounts.h 8;" d +MAX_TOKENS ./src/defaults.h 11;" d +META_EXIT ./src/opcodes.h /^ META_EXIT = 900,$/;" e enum:__anon1 +META_EXIT_GROUP ./src/opcodes.h /^ META_EXIT_GROUP,$/;" e enum:__anon1 +META_H ./src/meta/meta.h 2;" d +META_TIMELINE ./src/opcodes.h /^ META_TIMELINE$/;" e enum:__anon1 +MKDIR ./src/opcodes.h /^ MKDIR,$/;" e enum:__anon1 +MKDIR_AT ./src/opcodes.h /^ MKDIR_AT,$/;" e enum:__anon1 +MMAP2 ./src/opcodes.h /^ MMAP2 = 80,$/;" e enum:__anon1 +MOUNTPOINTS_H ./src/mounts.h 2;" d +MSYNC ./src/opcodes.h /^ MSYNC,$/;" e enum:__anon1 +MUNMAP ./src/opcodes.h /^ MUNMAP,$/;" e enum:__anon1 +Malloc ./src/macros.h 21;" d +Mmapshared ./src/macros.h 32;" d +Mset ./src/macros.h 25;" d +NAME_TO_HANDLE_AT ./src/opcodes.h /^ NAME_TO_HANDLE_AT,$/;" e enum:__anon1 +OPCODES_H ./src/opcodes.h 2;" d +OPEN ./src/opcodes.h /^ OPEN = 30,$/;" e enum:__anon1 +OPEN_AT ./src/opcodes.h /^ OPEN_AT,$/;" e enum:__anon1 +OPEN_BY_HANDLE_AT ./src/opcodes.h /^ OPEN_BY_HANDLE_AT,$/;" e enum:__anon1 +OPTIONS_H ./src/options.h 2;" d +Out ./src/macros.h 42;" d +Put ./src/macros.h 45;" d +RBUFFER_H ./src/datas/rbuffer.h 2;" d +READ ./src/opcodes.h /^ READ = 10,$/;" e enum:__anon1 +READAHEAD ./src/opcodes.h /^ READAHEAD,$/;" e enum:__anon1 +READDIR ./src/opcodes.h /^ READDIR,$/;" e enum:__anon1 +READLINK ./src/opcodes.h /^ READLINK,$/;" e enum:__anon1 +READLINK_AT ./src/opcodes.h /^ READLINK_AT,$/;" e enum:__anon1 +READV ./src/opcodes.h /^ READV,$/;" e enum:__anon1 +REMAP ./src/opcodes.h /^ REMAP,$/;" e enum:__anon1 +RENAME ./src/opcodes.h /^ RENAME = 40,$/;" e enum:__anon1 +RENAME_AT ./src/opcodes.h /^ RENAME_AT,$/;" e enum:__anon1 +RENAME_AT2 ./src/opcodes.h /^ RENAME_AT2,$/;" e enum:__anon1 +REPLAY_H ./src/replay/replay.h 2;" d +REPLAY_VERSION ./src/defaults.h 9;" d +RIOOP_H ./src/replay/rioop.h 2;" d +RMDIR ./src/opcodes.h /^ RMDIR,$/;" e enum:__anon1 +RPROCESS_H ./src/replay/rprocess.h 2;" d +RTASK_H ./src/replay/rtask.h 2;" d +RTHREAD_H ./src/replay/rthread.h 9;" d +RWORKER_H ./src/replay/rworker.h 2;" d +Readhex ./src/macros.h 14;" d +STACK_H ./src/datas/stack.h 2;" d +STAT ./src/opcodes.h /^ STAT,$/;" e enum:__anon1 +STATFS ./src/opcodes.h /^ STATFS,$/;" e enum:__anon1 +STATFS64 ./src/opcodes.h /^ STATFS64,$/;" e enum:__anon1 +SUCCESS ./src/defaults.h /^ SUCCESS, \/**< Great success! *\/$/;" e enum:status_e_ +SYNC ./src/opcodes.h /^ SYNC,$/;" e enum:__anon1 +SYNCFS ./src/opcodes.h /^ SYNCFS,$/;" e enum:__anon1 +SYNC_FILE_RANGE ./src/opcodes.h /^ SYNC_FILE_RANGE,$/;" e enum:__anon1 +Segfault ./src/macros.h 79;" d +TASK_BUFFER_PER_THREAD ./src/defaults.h 15;" d +UNKNOWN ./src/defaults.h /^ UNKNOWN, \/**< Unknown return status :-\/ *\/$/;" e enum:status_e_ +UNLINK ./src/opcodes.h /^ UNLINK,$/;" e enum:__anon1 +UNLINK_AT ./src/opcodes.h /^ UNLINK_AT,$/;" e enum:__anon1 +UTESTS_H ./src/utests.h 2;" d +UTILS_H ./src/utils/utils.h 2;" d +VFD_H ./src/vfd.h 2;" d +VSIZE_H ./src/generate/vsize.h 2;" d +WRITE ./src/opcodes.h /^ WRITE = 20,$/;" e enum:__anon1 +WRITEV ./src/opcodes.h /^ WRITEV,$/;" e enum:__anon1 +Warn ./src/macros.h 88;" d +Warn_if ./src/macros.h 95;" d +_Errno ./src/replay/rioop.c 15;" d file: +_Error ./src/replay/rioop.c 7;" d file: +_GNU_SOURCE ./src/utils/utils.h 5;" d +_Init_arg ./src/replay/rioop.c 24;" d file: +_Init_bytes ./src/replay/rioop.c 36;" d file: +_Init_cmd ./src/replay/rioop.c 25;" d file: +_Init_fd ./src/replay/rioop.c 26;" d file: +_Init_flags ./src/replay/rioop.c 27;" d file: +_Init_offset ./src/replay/rioop.c 29;" d file: +_Init_op ./src/replay/rioop.c 30;" d file: +_Init_path ./src/replay/rioop.c 32;" d file: +_Init_path2 ./src/replay/rioop.c 31;" d file: +_Init_rc ./src/replay/rioop.c 33;" d file: +_Init_virtfd ./src/replay/rioop.c 40;" d file: +_Init_whence ./src/replay/rioop.c 34;" d file: +_MAX_META_LEN ./src/meta/meta.c 3;" d file: +_MAX_PROCESSES ./src/generate/generate.c 10;" d file: +_PATH_INSERT ./src/mounts.c 5;" d file: +_PATH_INSERT_LEN ./src/mounts.c 6;" d file: +_Perc_filtered ./src/generate/generate.c 12;" d file: +_Set_dir ./src/generate/vsize.c 8;" d file: +_Set_file ./src/generate/vsize.c 7;" d file: +_Set_inserted ./src/generate/vsize.c 10;" d file: +_Set_renamed ./src/generate/vsize.c 11;" d file: +_Set_required ./src/generate/vsize.c 12;" d file: +_Set_unsure ./src/generate/vsize.c 9;" d file: +_Using_string_keys ./src/datas/hmap.c 3;" d file: +_amap_new ./src/datas/amap.c /^static amap_s *_amap_new(long size, bool mmapped)$/;" f file: +_amap_test ./src/datas/amap.c /^void _amap_test(amap_s *a)$/;" f +_arch_check_atomic ./src/main.c /^static void _arch_check_atomic(void)$/;" f file: +_gprocess_vfd_map_destroy_cb ./src/generate/gprocess.c /^void _gprocess_vfd_map_destroy_cb(void *data)$/;" f +_hmap_new ./src/datas/hmap.c /^hmap_s *_hmap_new(unsigned int init_size)$/;" f +_hmap_test ./src/datas/hmap.c /^static void _hmap_test(hmap_s *h)$/;" f file: +_hmap_test_l ./src/datas/hmap.c /^static void _hmap_test_l(hmap_s *h)$/;" f file: +_list_elem_remove ./src/datas/list.c /^void _list_elem_remove(list_s *l, list_elem_s *e)$/;" f +_print_help ./src/main.c /^static void _print_help(void)$/;" f file: +_print_synopsis ./src/main.c /^static void _print_synopsis(void)$/;" f file: +_print_version ./src/main.c /^static void _print_version(void)$/;" f file: +_rprocess_destroy_cb ./src/replay/rworker.c /^static void _rprocess_destroy_cb(void *data)$/;" f file: +_rthread_init_log ./src/replay/rthread.c /^static void _rthread_init_log(rthread_s *t)$/;" f file: +absolute_path ./src/utils/futils.c /^char *absolute_path(const char *path)$/;" f +address ./src/generate/gtask.h /^ long address; \/**< An address (used by mmap related syscalls) *\/$/;" m struct:gtask_s_ +address2 ./src/generate/gtask.h /^ long address2; \/**< Another address (used by mmap related syscalls) *\/$/;" m struct:gtask_s_ +amap_destroy ./src/datas/amap.c /^void amap_destroy(amap_s* a)$/;" f +amap_get ./src/datas/amap.c /^void* amap_get(amap_s *a, const long position)$/;" f +amap_new ./src/datas/amap.c /^amap_s* amap_new(const long size)$/;" f +amap_new_mmapped ./src/datas/amap.c /^amap_s* amap_new_mmapped(const long size)$/;" f +amap_print ./src/datas/amap.c /^void amap_print(amap_s* a)$/;" f +amap_reset ./src/datas/amap.c /^void amap_reset(amap_s* a)$/;" f +amap_run_cb ./src/datas/amap.c /^void amap_run_cb(amap_s *a, void (*cb)(void *data))$/;" f +amap_s ./src/datas/amap.h /^} amap_s;$/;" t typeref:struct:amap_s_ +amap_s_ ./src/datas/amap.h /^typedef struct amap_s_ {$/;" s +amap_set ./src/datas/amap.c /^int amap_set(amap_s *a, const long position, void* value)$/;" f +amap_test ./src/datas/amap.c /^void amap_test(void)$/;" f +amap_unset ./src/datas/amap.c /^void* amap_unset(amap_s *a, const long position)$/;" f +append_random_to_file ./src/utils/futils.c /^void append_random_to_file(char *path, unsigned long bytes)$/;" f +arrays ./src/datas/amap.h /^ void*** arrays; \/**< The pointers to the amap arrays *\/$/;" m struct:amap_s_ +btree_destroy ./src/datas/btree.c /^void btree_destroy(btree_s* b)$/;" f +btree_destroy2 ./src/datas/btree.c /^void btree_destroy2(btree_s* b)$/;" f +btree_get ./src/datas/btree.c /^void* btree_get(btree_s* b, int key)$/;" f +btree_insert ./src/datas/btree.c /^int btree_insert(btree_s* b, int key, void *data)$/;" f +btree_new ./src/datas/btree.c /^btree_s* btree_new()$/;" f +btree_print ./src/datas/btree.c /^void btree_print(btree_s* b)$/;" f +btree_s ./src/datas/btree.h /^} btree_s;$/;" t typeref:struct:btree_s_ +btree_s_ ./src/datas/btree.h /^typedef struct btree_s_ {$/;" s +btreelem_ ./src/datas/btree.h /^typedef struct btreelem_ {$/;" s +btreelem_destroy_r ./src/datas/btree.c /^void btreelem_destroy_r(btreelem_s* e)$/;" f +btreelem_destroy_r2 ./src/datas/btree.c /^void btreelem_destroy_r2(btreelem_s* e)$/;" f +btreelem_get_r ./src/datas/btree.c /^void* btreelem_get_r(btreelem_s* e, int key)$/;" f +btreelem_insert_r ./src/datas/btree.c /^int btreelem_insert_r(btreelem_s* e, int key, void *data)$/;" f +btreelem_new ./src/datas/btree.c /^btreelem_s* btreelem_new(int key, void *data)$/;" f +btreelem_print_r ./src/datas/btree.c /^void btreelem_print_r(btreelem_s* e, int depth)$/;" f +btreelem_s ./src/datas/btree.h /^} btreelem_s;$/;" t typeref:struct:btreelem_ +bytes ./src/generate/gtask.h /^ long bytes; \/**< Amount of bytes *\/$/;" m struct:gtask_s_ +cache_file ./src/utils/futils.c /^void cache_file(const char *file)$/;" f +capture_file ./src/options.h /^ char *capture_file; \/**< The name of the .capture file *\/$/;" m struct:options_s_ +chown_path ./src/utils/futils.c /^void chown_path(const char *user, const char *path)$/;" f +chreplace ./src/utils/utils.c /^void chreplace(char *str, char replace, char with)$/;" f +cleanup_run ./src/cleanup/cleanup.c /^status_e cleanup_run(options_s *opts)$/;" f +clone ./src/replay/rtask.h /^ char *clone; \/**< Used for debug purposes only *\/$/;" m struct:rtask_s_ +count ./src/generate/gtask.h /^ long count; \/**< A count *\/$/;" m struct:gtask_s_ +count ./src/mounts.h /^ int count; \/**< The amount of mount points *\/$/;" m struct:mounts_s_ +data ./src/datas/btree.h /^ void *data; \/**< A pointer to the data stored in this element *\/$/;" m struct:btreelem_ +data ./src/datas/hmap.h /^ void **data; \/**< Pointers to the stored data, NULL if nothing there *\/$/;" m struct:hmap_s_ +data ./src/datas/list.h /^ void *data; \/**< Pointer to the stored data *\/$/;" m struct:list_elem_s_ +data ./src/datas/stack.h /^ void *data; \/**< Pointer to the stored data in the current element *\/$/;" m struct:stack_elem_s_ +data_destroy ./src/datas/amap.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all elements *\/$/;" m struct:amap_s_ +data_destroy ./src/datas/hmap.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all data *\/$/;" m struct:hmap_s_ +data_destroy ./src/datas/list.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all data *\/$/;" m struct:list_s_ +debug ./src/vfd.h /^ int debug; \/**< Used for debugging purposes only *\/$/;" m struct:vfd_s_ +dirfd ./src/vfd.h /^ DIR *dirfd; \/**< The real dirfd *\/$/;" m struct:vfd_s_ +dirname_r ./src/utils/futils.c /^char* dirname_r(char *path)$/;" f +dirs_created ./src/init/itask.h /^ long dirs_created;$/;" m struct:itask_s_ +drop_caches ./src/options.h /^ bool drop_caches; \/**< True if ioreplay should drop all Linux caches *\/$/;" m struct:options_s_ +drop_caches ./src/utils/futils.c /^void drop_caches(void)$/;" f +drop_root ./src/utils/utils.c /^void drop_root(const char *user)$/;" f +ensure_dir_empty ./src/utils/futils.c /^void ensure_dir_empty(const char *path)$/;" f +ensure_dir_exists ./src/utils/futils.c /^long ensure_dir_exists(const char *path)$/;" f +ensure_file_exists ./src/utils/futils.c /^int ensure_file_exists(char *path, long *num_dirs_created)$/;" f +ensure_parent_dir_exists ./src/utils/futils.c /^void ensure_parent_dir_exists(const char *path)$/;" f +exists ./src/utils/futils.c /^bool exists(const char *path)$/;" f +fd ./src/generate/gtask.h /^ int fd; \/**< File descriptor number *\/$/;" m struct:gtask_s_ +fd ./src/vfd.h /^ int fd; \/**< the real fd *\/$/;" m struct:vfd_s_ +fd_map ./src/generate/gprocess.h /^ hmap_s *fd_map; \/**< All mappings from real fd to virtual fd *\/$/;" m struct:gprocess_s_ +fds_map ./src/replay/rprocess.h /^ amap_s *fds_map; \/**< Holding all file descriptors *\/$/;" m struct:rprocess_s_ +fds_map ./src/replay/rworker.h /^ amap_s* fds_map; \/**< Holding all file descriptors *\/$/;" m struct:__anon2 +files_created ./src/init/itask.h /^ long files_created;$/;" m struct:itask_s_ +filtered_where ./src/generate/gtask.h /^ char *filtered_where; \/**< Only used for debugging purposes *\/$/;" m struct:gtask_s_ +first ./src/datas/list.h /^ list_elem_s *first; \/**< The first element, NULL if list empty *\/$/;" m struct:list_s_ +flags ./src/generate/gtask.h /^ int flags; \/**< File open flags *\/$/;" m struct:gtask_s_ +fnotnull ./src/utils/utils.c /^FILE* fnotnull(FILE *fd, const char *path, char *file, int line)$/;" f +free_path ./src/vfd.h /^ bool free_path; \/**< True if path has to be freed or not *\/$/;" m struct:vfd_s_ +generate ./src/generate/gparser.h /^ generate_s *generate; \/**< The generate object *\/$/;" m struct:gparser_s_ +generate ./src/generate/gtask.h /^ void *generate; \/**< A pointer to the generate object *\/$/;" m struct:gtask_s_ +generate ./src/generate/gwriter.h /^ struct generate_s_ *generate; \/**< The generate object *\/$/;" m struct:gwriter_s_ typeref:struct:gwriter_s_::generate_s_ +generate ./src/generate/vsize.h /^ void *generate; \/**< A pointer to the generate object *\/$/;" m struct:vsize_s_ +generate_destroy ./src/generate/generate.c /^void generate_destroy(generate_s *g)$/;" f +generate_gprocess_by_realpid ./src/generate/generate.c /^void generate_gprocess_by_realpid(generate_s *g, gtask_s *t)$/;" f +generate_new ./src/generate/generate.c /^generate_s* generate_new(options_s *opts)$/;" f +generate_run ./src/generate/generate.c /^status_e generate_run(options_s *opts)$/;" f +generate_s ./src/generate/generate.h /^} generate_s;$/;" t typeref:struct:generate_s_ +generate_s_ ./src/generate/generate.h /^typedef struct generate_s_ {$/;" s +generate_vsize_by_path ./src/generate/generate.c /^vsize_s* generate_vsize_by_path(generate_s *g, gtask_s *t,$/;" f +generate_write_init_cb ./src/generate/generate.c /^void generate_write_init_cb(void *data)$/;" f +get_loadavg ./src/utils/utils.c /^void get_loadavg(char *readbuf)$/;" f +gioop_chmod ./src/generate/gioop.c /^status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_chown ./src/generate/gioop.c /^status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_close ./src/generate/gioop.c /^status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_close_all_vfd_cb ./src/generate/gioop.c /^void gioop_close_all_vfd_cb(void *data, void *data2)$/;" f +gioop_creat ./src/generate/gioop.c /^status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_exit_group ./src/generate/gioop.c /^status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fchmod ./src/generate/gioop.c /^status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fchown ./src/generate/gioop.c /^status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fcntl ./src/generate/gioop.c /^status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fdatasync ./src/generate/gioop.c /^status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstat ./src/generate/gioop.c /^status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstatat ./src/generate/gioop.c /^status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstatfs ./src/generate/gioop.c /^status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstatfs64 ./src/generate/gioop.c /^status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fsync ./src/generate/gioop.c /^status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_getdents ./src/generate/gioop.c /^status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_lchown ./src/generate/gioop.c /^status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_lseek ./src/generate/gioop.c /^status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_lstat ./src/generate/gioop.c /^status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_mkdir ./src/generate/gioop.c /^status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_mkdirat ./src/generate/gioop.c /^status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_open ./src/generate/gioop.c /^status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_openat ./src/generate/gioop.c /^status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_read ./src/generate/gioop.c /^status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readahead ./src/generate/gioop.c /^status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readdir ./src/generate/gioop.c /^status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readlink ./src/generate/gioop.c /^status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readlinkat ./src/generate/gioop.c /^status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readv ./src/generate/gioop.c /^status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_rename ./src/generate/gioop.c /^status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_renameat ./src/generate/gioop.c /^status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_renameat2 ./src/generate/gioop.c /^status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_rmdir ./src/generate/gioop.c /^status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_run ./src/generate/gioop.c /^status_e gioop_run(gwriter_s *w, gtask_s *t)$/;" f +gioop_stat ./src/generate/gioop.c /^status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_statfs ./src/generate/gioop.c /^status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_statfs64 ./src/generate/gioop.c /^status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_sync ./src/generate/gioop.c /^status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_sync_file_range ./src/generate/gioop.c /^status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_syncfs ./src/generate/gioop.c /^status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_unlink ./src/generate/gioop.c /^status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_unlinkat ./src/generate/gioop.c /^status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_write ./src/generate/gioop.c /^status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_writev ./src/generate/gioop.c /^status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gparser_destroy ./src/generate/gparser.c /^void gparser_destroy(gparser_s *p)$/;" f +gparser_extract ./src/generate/gparser.c /^void gparser_extract(gparser_s *p, gtask_s *t)$/;" f +gparser_extract_tok ./src/generate/gparser.c /^status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok)$/;" f +gparser_get_pidtid ./src/generate/gparser.c /^bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid)$/;" f +gparser_new ./src/generate/gparser.c /^gparser_s* gparser_new(generate_s *g)$/;" f +gparser_pthread_start ./src/generate/gparser.c /^void* gparser_pthread_start(void *data)$/;" f +gparser_s ./src/generate/gparser.h /^} gparser_s;$/;" t typeref:struct:gparser_s_ +gparser_s_ ./src/generate/gparser.h /^typedef struct gparser_s_ {$/;" s +gparser_start ./src/generate/gparser.c /^void gparser_start(gparser_s *p)$/;" f +gparser_terminate ./src/generate/gparser.c /^void gparser_terminate(gparser_s *p)$/;" f +gparser_token_not_ok ./src/generate/gparser.c /^bool gparser_token_not_ok(gparser_s *p, char *tok)$/;" f +gprocess ./src/generate/gtask.h /^ struct gprocess_s_ *gprocess; \/**< A pointer to the process object *\/$/;" m struct:gtask_s_ typeref:struct:gtask_s_::gprocess_s_ +gprocess_create_vfd_by_realfd ./src/generate/gprocess.c /^void gprocess_create_vfd_by_realfd(gprocess_s *gp, gtask_s *t, generate_s *g)$/;" f +gprocess_destroy ./src/generate/gprocess.c /^void gprocess_destroy(gprocess_s *gp)$/;" f +gprocess_new ./src/generate/gprocess.c /^gprocess_s* gprocess_new(const long pid, const long mapped_pid)$/;" f +gprocess_s ./src/generate/gprocess.h /^} gprocess_s;$/;" t typeref:struct:gprocess_s_ +gprocess_s_ ./src/generate/gprocess.h /^typedef struct gprocess_s_ {$/;" s +gprocess_vfd_by_realfd ./src/generate/gprocess.c /^status_e gprocess_vfd_by_realfd(gprocess_s *gp, gtask_s *t)$/;" f +gtask_destroy ./src/generate/gtask.c /^void gtask_destroy(gtask_s *t)$/;" f +gtask_init ./src/generate/gtask.c /^void gtask_init(gtask_s *t, char *line, const unsigned long lineno)$/;" f +gtask_new ./src/generate/gtask.c /^gtask_s* gtask_new(void *generate)$/;" f +gtask_s ./src/generate/gtask.h /^} gtask_s;$/;" t typeref:struct:gtask_s_ +gtask_s_ ./src/generate/gtask.h /^typedef struct gtask_s_ {$/;" s +gwriter_destroy ./src/generate/gwriter.c /^void gwriter_destroy(gwriter_s *w)$/;" f +gwriter_new ./src/generate/gwriter.c /^gwriter_s* gwriter_new(generate_s *g)$/;" f +gwriter_pthread_start ./src/generate/gwriter.c /^void* gwriter_pthread_start(void *data)$/;" f +gwriter_s ./src/generate/gwriter.h /^} gwriter_s;$/;" t typeref:struct:gwriter_s_ +gwriter_s_ ./src/generate/gwriter.h /^typedef struct gwriter_s_ {$/;" s +gwriter_start ./src/generate/gwriter.c /^void gwriter_start(gwriter_s *w)$/;" f +gwriter_terminate ./src/generate/gwriter.c /^void gwriter_terminate(gwriter_s *w)$/;" f +has_fd ./src/generate/gtask.h /^ bool has_fd; \/**< True if task has a file descriptor number *\/$/;" m struct:gtask_s_ +hmap_destroy ./src/datas/hmap.c /^void hmap_destroy(hmap_s *h)$/;" f +hmap_get ./src/datas/hmap.c /^void* hmap_get(hmap_s *h, char *key)$/;" f +hmap_get_addr ./src/datas/hmap.c /^unsigned int hmap_get_addr(hmap_s *h, char *key)$/;" f +hmap_get_addr_l ./src/datas/hmap.c /^unsigned int hmap_get_addr_l(hmap_s *h, const long key)$/;" f +hmap_get_l ./src/datas/hmap.c /^void* hmap_get_l(hmap_s *h, const long key)$/;" f +hmap_insert ./src/datas/hmap.c /^int hmap_insert(hmap_s *h, char *key, void *data)$/;" f +hmap_insert_l ./src/datas/hmap.c /^int hmap_insert_l(hmap_s *h, const long key, void *data)$/;" f +hmap_new ./src/datas/hmap.c /^hmap_s *hmap_new(unsigned int init_size)$/;" f +hmap_new_l ./src/datas/hmap.c /^hmap_s *hmap_new_l(unsigned int init_size)$/;" f +hmap_print ./src/datas/hmap.c /^void hmap_print(hmap_s *h)$/;" f +hmap_remove ./src/datas/hmap.c /^void* hmap_remove(hmap_s *h, char *key)$/;" f +hmap_remove_l ./src/datas/hmap.c /^void* hmap_remove_l(hmap_s *h, const long key)$/;" f +hmap_run_cb ./src/datas/hmap.c /^void hmap_run_cb(hmap_s* h, void (*cb)(void *data))$/;" f +hmap_run_cb2 ./src/datas/hmap.c /^void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_)$/;" f +hmap_s ./src/datas/hmap.h /^} hmap_s;$/;" t typeref:struct:hmap_s_ +hmap_s_ ./src/datas/hmap.h /^typedef struct hmap_s_ {$/;" s +hmap_test ./src/datas/hmap.c /^void hmap_test(void)$/;" f +id ./src/generate/vsize.h /^ unsigned long id; \/**< The vsize id *\/$/;" m struct:vsize_s_ +ignore_count ./src/mounts.h /^ int ignore_count; \/**< The amount of ignored mount points *\/$/;" m struct:mounts_s_ +ignore_mps ./src/mounts.h /^ char *ignore_mps[MAX_MOUNTPOINTS]; \/**< The ignored mp paths *\/$/;" m struct:mounts_s_ +init ./src/init/ithread.h /^ init_s *init; \/**< The responsible init object *\/$/;" m struct:ithread_s_ +init ./src/options.h /^ bool init; \/**< If set ioreplay will initialise the environment *\/$/;" m struct:options_s_ +init_destroy ./src/init/init.c /^void init_destroy(init_s *i)$/;" f +init_extract_header ./src/init/init.c /^void init_extract_header(init_s *i, off_t *init_offset)$/;" f +init_new ./src/init/init.c /^init_s *init_new(options_s *opts)$/;" f +init_parent_dir ./src/generate/vsize.c /^void init_parent_dir(vsize_s *v, const char *path)$/;" f +init_run ./src/init/init.c /^status_e init_run(options_s *opts)$/;" f +init_s ./src/init/init.h /^} init_s;$/;" t typeref:struct:init_s_ +init_s_ ./src/init/init.h /^typedef struct init_s_ {$/;" s +initm ./src/replay/rprocess.h /^ bool initm; \/**< Indicates whether ioreplay is in init mode or not *\/$/;" m struct:rprocess_s_ +inserted ./src/generate/vsize.h /^ bool inserted; \/**< For debugging purposes only *\/$/;" m struct:vsize_s_ +is_dir ./src/generate/vsize.h /^ bool is_dir; \/**< True if this file\/dir is a directory *\/$/;" m struct:vsize_s_ +is_dir ./src/init/itask.h /^ bool is_dir;$/;" m struct:itask_s_ +is_dir ./src/utils/futils.c /^bool is_dir(const char *path)$/;" f +is_file ./src/generate/vsize.h /^ bool is_file; \/**< True if this file\/dir is a regular file *\/$/;" m struct:vsize_s_ +is_file ./src/init/itask.h /^ bool is_file;$/;" m struct:itask_s_ +is_number ./src/utils/utils.c /^bool is_number(char *str)$/;" f +is_reg ./src/utils/futils.c /^bool is_reg(const char *path)$/;" f +itask_destroy ./src/init/itask.c /^void itask_destroy(itask_s *task)$/;" f +itask_extract_stats ./src/init/itask.c /^void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created,$/;" f +itask_new ./src/init/itask.c /^itask_s* itask_new()$/;" f +itask_print ./src/init/itask.c /^void itask_print(itask_s *task)$/;" f +itask_reset_stats ./src/init/itask.c /^void itask_reset_stats(itask_s *task)$/;" f +itask_s ./src/init/itask.h /^} itask_s;$/;" t typeref:struct:itask_s_ +itask_s_ ./src/init/itask.h /^typedef struct itask_s_ {$/;" s +ithread_destroy ./src/init/ithread.c /^void ithread_destroy(ithread_s *t)$/;" f +ithread_new ./src/init/ithread.c /^ithread_s* ithread_new(init_s *i)$/;" f +ithread_pthread_start ./src/init/ithread.c /^void* ithread_pthread_start(void *data)$/;" f +ithread_run_task ./src/init/ithread.c /^void ithread_run_task(ithread_s *t, itask_s *task)$/;" f +ithread_s ./src/init/ithread.h /^} ithread_s;$/;" t typeref:struct:ithread_s_ +ithread_s_ ./src/init/ithread.h /^typedef struct ithread_s_ {$/;" s +ithread_start ./src/init/ithread.c /^void ithread_start(ithread_s *t)$/;" f +ithread_terminate ./src/init/ithread.c /^void ithread_terminate(ithread_s *t)$/;" f +key ./src/datas/btree.h /^ int key; \/**< The key of the element *\/$/;" m struct:btreelem_ +key ./src/datas/list.h /^ char *key; \/**< The key of the lemenet *\/$/;" m struct:list_elem_s_ +key_l ./src/datas/list.h /^ long key_l; \/**< The same as key, but for long keys *\/$/;" m struct:list_elem_s_ +keys ./src/datas/hmap.h /^ char **keys; \/**< List of all keys, NULL if nothing at a address *\/$/;" m struct:hmap_s_ +keys_l ./src/datas/hmap.h /^ int *keys_l; \/**< Same as keys, but for long keys *\/$/;" m struct:hmap_s_ +l ./src/datas/hmap.h /^ list_s **l; \/**< Pointers to the linked lists, used on hash collision *\/$/;" m struct:hmap_s_ +left ./src/datas/btree.h /^ struct btreelem_ *left; \/**< The next element to the left *\/$/;" m struct:btreelem_ typeref:struct:btreelem_::btreelem_ +lengths ./src/mounts.h /^ int lengths[MAX_MOUNTPOINTS]; \/**< The mp lenghts *\/$/;" m struct:mounts_s_ +line ./src/generate/gtask.h /^ char *line; \/**< A pointer to the remaining part of the .capture line *\/$/;" m struct:gtask_s_ +line ./src/replay/rtask.h /^ char line[MAX_LINE_LEN]; \/**< The remaining part of the .replay line *\/$/;" m struct:rtask_s_ +lineno ./src/generate/generate.h /^ long lineno; \/**< The current line number *\/$/;" m struct:generate_s_ +lineno ./src/generate/gtask.h /^ long lineno; \/**< The current line number *\/$/;" m struct:gtask_s_ +lineno ./src/replay/rprocess.h /^ unsigned long lineno; \/**< Holding the current .replay line number *\/$/;" m struct:rprocess_s_ +lineno ./src/replay/rtask.h /^ unsigned long lineno; \/**< The current line number *\/$/;" m struct:rtask_s_ +list_destroy ./src/datas/list.c /^void list_destroy(list_s *l)$/;" f +list_elem_s ./src/datas/list.h /^} list_elem_s;$/;" t typeref:struct:list_elem_s_ +list_elem_s_ ./src/datas/list.h /^typedef struct list_elem_s_ {$/;" s +list_key_get ./src/datas/list.c /^void* list_key_get(list_s *l, char *key)$/;" f +list_key_get_l ./src/datas/list.c /^void* list_key_get_l(list_s *l, const long key)$/;" f +list_key_insert ./src/datas/list.c /^int list_key_insert(list_s *l, char *key, void *data)$/;" f +list_key_insert_l ./src/datas/list.c /^int list_key_insert_l(list_s *l, const long key, void *data)$/;" f +list_key_remove ./src/datas/list.c /^void* list_key_remove(list_s *l, char *key)$/;" f +list_key_remove_l ./src/datas/list.c /^void* list_key_remove_l(list_s *l, const long key)$/;" f +list_new ./src/datas/list.c /^list_s *list_new()$/;" f +list_new_l ./src/datas/list.c /^list_s *list_new_l()$/;" f +list_print ./src/datas/list.c /^void list_print(list_s *l)$/;" f +list_run_cb ./src/datas/list.c /^void list_run_cb(list_s* l, void (*cb)(void *data))$/;" f +list_run_cb2 ./src/datas/list.c /^void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_)$/;" f +list_s ./src/datas/list.h /^} list_s;$/;" t typeref:struct:list_s_ +list_s_ ./src/datas/list.h /^typedef struct list_s_ {$/;" s +list_test ./src/datas/list.c /^void list_test(void)$/;" f +main ./src/main.c /^int main(int argc, char **argv)$/;" f +mapped_fd ./src/generate/gtask.h /^ long mapped_fd; \/**< The mapped file descriptor number *\/$/;" m struct:gtask_s_ +mapped_fd ./src/vfd.h /^ long mapped_fd; \/**< The mapped fd (virtual fd) *\/$/;" m struct:vfd_s_ +mapped_pid ./src/generate/gprocess.h /^ long mapped_pid; \/**< The mapped PID *\/$/;" m struct:gprocess_s_ +mapped_time ./src/generate/gtask.h /^ long mapped_time; \/**< The mapped time *\/$/;" m struct:gtask_s_ +max_mapped_fd ./src/generate/gprocess.h /^ long max_mapped_fd; \/**< The max mapped fd number *\/$/;" m struct:gprocess_s_ +meta_destroy ./src/meta/meta.c /^void meta_destroy(meta_s *m)$/;" f +meta_new ./src/meta/meta.c /^meta_s* meta_new(FILE *replay_fd)$/;" f +meta_read_l ./src/meta/meta.c /^bool meta_read_l(meta_s *m, char *key, long *val)$/;" f +meta_read_s ./src/meta/meta.c /^bool meta_read_s(meta_s *m, char *key, char **val)$/;" f +meta_read_start ./src/meta/meta.c /^void meta_read_start(meta_s *m)$/;" f +meta_reserve ./src/meta/meta.c /^void meta_reserve(meta_s *m)$/;" f +meta_s ./src/meta/meta.h /^} meta_s;$/;" t typeref:struct:meta_s_ +meta_s_ ./src/meta/meta.h /^typedef struct meta_s_ {$/;" s +meta_write_l ./src/meta/meta.c /^void meta_write_l(meta_s *m, char *key, long val)$/;" f +meta_write_s ./src/meta/meta.c /^void meta_write_s(meta_s *m, char *key, char *val)$/;" f +meta_write_start ./src/meta/meta.c /^void meta_write_start(meta_s *m)$/;" f +mkdir_p ./src/utils/futils.c /^int mkdir_p(const char *path, mode_t mode, long *num_dirs_created)$/;" f +mmap_map ./src/generate/generate.h /^ hmap_s *mmap_map; \/**< mmap address mappings *\/$/;" m struct:generate_s_ +mmapok ./src/utils/utils.c /^void* mmapok(void *p, char *file, int line)$/;" f +mmapped ./src/datas/amap.h /^ bool mmapped; \/**< True if amap is memory mapped *\/$/;" m struct:amap_s_ +mode ./src/generate/gtask.h /^ int mode; \/**< File open mode *\/$/;" m struct:gtask_s_ +mounts ./src/init/init.h /^ mounts_s *mounts;$/;" m struct:init_s_ +mounts_destroy ./src/mounts.c /^void mounts_destroy(mounts_s *m)$/;" f +mounts_get_mountnumber ./src/mounts.c /^int mounts_get_mountnumber(mounts_s *m, const char *path)$/;" f +mounts_ignore_path ./src/mounts.c /^bool mounts_ignore_path(mounts_s *m, const char *path)$/;" f +mounts_init ./src/mounts.c /^void mounts_init(mounts_s *m)$/;" f +mounts_new ./src/mounts.c /^mounts_s *mounts_new(options_s *opts)$/;" f +mounts_purge ./src/mounts.c /^void mounts_purge(mounts_s *m)$/;" f +mounts_read ./src/mounts.c /^void mounts_read(mounts_s *m)$/;" f +mounts_s ./src/mounts.h /^} mounts_s;$/;" t typeref:struct:mounts_s_ +mounts_s_ ./src/mounts.h /^typedef struct mounts_s_ {$/;" s +mounts_transform_path ./src/mounts.c /^bool mounts_transform_path(mounts_s *m, const char *name,$/;" f +mounts_trash ./src/mounts.c /^void mounts_trash(mounts_s *m)$/;" f +mps ./src/generate/generate.h /^ mounts_s *mps; \/**< The mounts object *\/$/;" m struct:generate_s_ +mps ./src/mounts.h /^ char *mps[MAX_MOUNTPOINTS]; \/**< The mp paths *\/$/;" m struct:mounts_s_ +name ./src/generate/generate.h /^ char *name; \/**< The name of the test specified by the user *\/$/;" m struct:generate_s_ +name ./src/options.h /^ char *name; \/**< The name of the test (found in .ioreplay\/name sub-dirs) *\/$/;" m struct:options_s_ +next ./src/datas/list.h /^ struct list_elem_s_ *next; \/**< The next element *\/$/;" m struct:list_elem_s_ typeref:struct:list_elem_s_::list_elem_s_ +next ./src/datas/stack.h /^ struct stack_elem_s_ *next; \/**< The next element *\/$/;" m struct:stack_elem_s_ typeref:struct:stack_elem_s_::stack_elem_s_ +notnull ./src/utils/utils.c /^void* notnull(void *p, char *file, int line, int count)$/;" f +num_arrays ./src/datas/amap.h /^ int num_arrays; \/**< The amount of arrays used in the amap *\/$/;" m struct:amap_s_ +num_lines_filtered ./src/generate/generate.h /^ long num_lines_filtered; \/**< The amount of lines filtered out *\/$/;" m struct:generate_s_ +num_mapped_fds ./src/generate/generate.h /^ unsigned long num_mapped_fds; \/**< The amount of mapped FDs *\/$/;" m struct:generate_s_ +num_mapped_pids ./src/generate/generate.h /^ unsigned long num_mapped_pids; \/**< The amount of mapped PIDs *\/$/;" m struct:generate_s_ +num_threads_per_worker ./src/options.h /^ int num_threads_per_worker; \/**< Max threads per worker processes *\/$/;" m struct:options_s_ +num_vsizes ./src/generate/generate.h /^ unsigned long num_vsizes; \/**< The amount of virtual sizes *\/$/;" m struct:generate_s_ +num_workers ./src/options.h /^ int num_workers; \/**< The amount of worker processes *\/$/;" m struct:options_s_ +offset ./src/generate/gtask.h /^ long offset; \/**< A offset *\/$/;" m struct:gtask_s_ +offset ./src/generate/vsize.h /^ off_t offset; \/**< The current file offset *\/$/;" m struct:vsize_s_ +offset ./src/meta/meta.h /^ off_t offset; \/**< The meta offset (usually 0) *\/$/;" m struct:meta_s_ +offset ./src/vfd.h /^ unsigned long offset; \/**< The current virtual file offset in bytes *\/$/;" m struct:vfd_s_ +op ./src/generate/gtask.h /^ char *op; \/**< Operation\/syscall name *\/$/;" m struct:gtask_s_ +opcode_e ./src/opcodes.h /^} opcode_e;$/;" t typeref:enum:__anon1 +options_destroy ./src/options.c /^void options_destroy(options_s *o)$/;" f +options_new ./src/options.c /^options_s *options_new()$/;" f +options_s ./src/options.h /^} options_s;$/;" t typeref:struct:options_s_ +options_s_ ./src/options.h /^typedef struct options_s_ {$/;" s +opts ./src/generate/generate.h /^ options_s *opts; \/**< A pointer to the options object *\/$/;" m struct:generate_s_ +opts ./src/init/init.h /^ options_s *opts;$/;" m struct:init_s_ +opts ./src/mounts.h /^ options_s *opts; \/**< A pointer to the options object *\/$/;" m struct:mounts_s_ +opts ./src/replay/rworker.h /^ options_s *opts; \/**< To synchronise access to rthread_buffer *\/$/;" m struct:__anon2 +original_line ./src/generate/gtask.h /^ char *original_line; \/**< Only used for debugging purposes *\/$/;" m struct:gtask_s_ +path ./src/generate/gtask.h /^ char *path; \/**< Path name *\/$/;" m struct:gtask_s_ +path ./src/generate/vsize.h /^ char *path; \/**< The path to the file\/directory *\/$/;" m struct:vsize_s_ +path ./src/init/itask.h /^ char *path;$/;" m struct:itask_s_ +path ./src/vfd.h /^ char *path; \/**< The file path belonging to that fd *\/$/;" m struct:vfd_s_ +path2 ./src/generate/gtask.h /^ char *path2; \/**< A second path name (e.g. for rename) *\/$/;" m struct:gtask_s_ +path2_r ./src/generate/gtask.h /^ char *path2_r; \/**< Work around to track mallocs, so it can be freed *\/$/;" m struct:gtask_s_ +path_r ./src/generate/gtask.h /^ char *path_r; \/**< Work around to track mallocs, so it can be freed *\/$/;" m struct:gtask_s_ +pid ./src/generate/gprocess.h /^ long pid; \/**< The real PID *\/$/;" m struct:gprocess_s_ +pid ./src/generate/gtask.h /^ long pid; \/**< The process ID *\/$/;" m struct:gtask_s_ +pid ./src/replay/rprocess.h /^ int pid; \/**< The virtual process ID *\/$/;" m struct:rprocess_s_ +pid_map ./src/generate/generate.h /^ amap_s *pid_map; \/**< A map of all virtual process objects *\/$/;" m struct:generate_s_ +pidtid ./src/generate/gtask.h /^ char *pidtid; \/**< String representing pid:tid *\/$/;" m struct:gtask_s_ +prev ./src/datas/list.h /^ struct list_elem_s_ *prev; \/**< The previous element *\/$/;" m struct:list_elem_s_ typeref:struct:list_elem_s_::list_elem_s_ +process ./src/replay/rtask.h /^ void *process; \/* The responsible process object *\/$/;" m struct:rtask_s_ +pthread ./src/generate/gparser.h /^ pthread_t pthread; \/**< The posix thread *\/$/;" m struct:gparser_s_ +pthread ./src/generate/gwriter.h /^ pthread_t pthread; \/**< The posix thread *\/$/;" m struct:gwriter_s_ +pthread ./src/init/ithread.h /^ pthread_t pthread; \/**< We run the init tasks in concurrent pthreads *\/$/;" m struct:ithread_s_ +pthread ./src/replay/rthread.h /^ pthread_t pthread; \/**< We run the tasks in concurrent pthreads *\/$/;" m struct:rthread_s_ +purge ./src/options.h /^ bool purge; \/**< If set ioreplay will purge the environment *\/$/;" m struct:options_s_ +queue ./src/generate/gparser.h /^ rbuffer_s *queue; \/**< A queue of task objects *\/$/;" m struct:gparser_s_ +queue ./src/generate/gwriter.h /^ rbuffer_s *queue; \/**< A queue of task objects *\/$/;" m struct:gwriter_s_ +queue ./src/init/ithread.h /^ rbuffer_s *queue; \/**< The thread's task queue *\/$/;" m struct:ithread_s_ +rbuffer_destroy ./src/datas/rbuffer.c /^void rbuffer_destroy(rbuffer_s *r)$/;" f +rbuffer_get_next ./src/datas/rbuffer.c /^void* rbuffer_get_next(rbuffer_s* r)$/;" f +rbuffer_has_next ./src/datas/rbuffer.c /^bool rbuffer_has_next(rbuffer_s* r)$/;" f +rbuffer_insert ./src/datas/rbuffer.c /^bool rbuffer_insert(rbuffer_s* r, void *data)$/;" f +rbuffer_new ./src/datas/rbuffer.c /^rbuffer_s *rbuffer_new(const int size)$/;" f +rbuffer_print ./src/datas/rbuffer.c /^void rbuffer_print(rbuffer_s* r)$/;" f +rbuffer_s ./src/datas/rbuffer.h /^} rbuffer_s;$/;" t typeref:struct:rbuffer_s_ +rbuffer_s_ ./src/datas/rbuffer.h /^typedef struct rbuffer_s_ {$/;" s +rbuffer_test ./src/datas/rbuffer.c /^void rbuffer_test(void)$/;" f +read_buf ./src/meta/meta.h /^ char* read_buf; \/**< Pointer to a read buffer *\/$/;" m struct:meta_s_ +read_pos ./src/datas/rbuffer.h /^ sig_atomic_t read_pos;$/;" m struct:rbuffer_s_ +renamed ./src/generate/vsize.h /^ bool renamed; \/**< True if file\/dir has been renamed *\/$/;" m struct:vsize_s_ +replay ./src/options.h /^ bool replay; \/**< If set ioreplay will run\/replay the test *\/$/;" m struct:options_s_ +replay_extract_header ./src/replay/replay.c /^void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes,$/;" f +replay_fd ./src/generate/generate.h /^ FILE *replay_fd; \/**< The fd of the .replay file *\/$/;" m struct:generate_s_ +replay_fd ./src/init/init.h /^ FILE *replay_fd;$/;" m struct:init_s_ +replay_fd ./src/meta/meta.h /^ FILE* replay_fd; \/**< The FS of the .replay file *\/$/;" m struct:meta_s_ +replay_file ./src/options.h /^ char *replay_file; \/**< The name of the .replay file *\/$/;" m struct:options_s_ +replay_run ./src/replay/replay.c /^status_e replay_run(options_s *opts)$/;" f +required ./src/generate/vsize.h /^ bool required; \/**< True if init mode will create this file\/dir *\/$/;" m struct:vsize_s_ +ret ./src/generate/gtask.h /^ int ret; \/**< ioreplay process status, SUCCESS if everything is alright *\/$/;" m struct:gtask_s_ +reuse_queue ./src/generate/generate.h /^ rbuffer_s *reuse_queue; \/**< A task buffer, for reusing these *\/$/;" m struct:generate_s_ +reuse_queue ./src/init/init.h /^ rbuffer_s *reuse_queue;$/;" m struct:init_s_ +reuse_queue_mutex ./src/init/init.h /^ pthread_mutex_t reuse_queue_mutex;$/;" m struct:init_s_ +right ./src/datas/btree.h /^ struct btreelem_ *right; \/**< The next element to the right *\/$/;" m struct:btreelem_ typeref:struct:btreelem_::btreelem_ +ring ./src/datas/rbuffer.h /^ void **ring;$/;" m struct:rbuffer_s_ +rioop_chmod ./src/replay/rioop.c /^void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_chown ./src/replay/rioop.c /^void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_close ./src/replay/rioop.c /^void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fchmod ./src/replay/rioop.c /^void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fchown ./src/replay/rioop.c /^void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fcntl ./src/replay/rioop.c /^void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fdatasync ./src/replay/rioop.c /^void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fstat ./src/replay/rioop.c /^void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fsync ./src/replay/rioop.c /^void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_getdents ./src/replay/rioop.c /^void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_lchown ./src/replay/rioop.c /^void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_lseek ./src/replay/rioop.c /^void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_mkdir ./src/replay/rioop.c /^void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_open ./src/replay/rioop.c /^void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_)$/;" f +rioop_read ./src/replay/rioop.c /^void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_rename ./src/replay/rioop.c /^void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_rmdir ./src/replay/rioop.c /^void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_run ./src/replay/rioop.c /^void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_stat ./src/replay/rioop.c /^void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_unlink ./src/replay/rioop.c /^void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_write ./src/replay/rioop.c /^void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +root ./src/datas/btree.h /^ btreelem_s *root; \/**< The root element *\/$/;" m struct:btree_s_ +rprocess_destroy ./src/replay/rprocess.c /^void rprocess_destroy(rprocess_s *p)$/;" f +rprocess_map ./src/replay/rworker.h /^ amap_s* rprocess_map; \/**< Holding all processes handled by this worker *\/$/;" m struct:__anon2 +rprocess_new ./src/replay/rprocess.c /^rprocess_s* rprocess_new(const int pid, amap_s *fds_map)$/;" f +rprocess_s ./src/replay/rprocess.h /^} rprocess_s;$/;" t typeref:struct:rprocess_s_ +rprocess_s_ ./src/replay/rprocess.h /^typedef struct rprocess_s_ {$/;" s +rtask_destroy ./src/replay/rtask.c /^void rtask_destroy(rtask_s *task)$/;" f +rtask_new ./src/replay/rtask.c /^rtask_s* rtask_new()$/;" f +rtask_s ./src/replay/rtask.h /^} rtask_s;$/;" t typeref:struct:rtask_s_ +rtask_s_ ./src/replay/rtask.h /^typedef struct rtask_s_ {$/;" s +rtask_update ./src/replay/rtask.c /^void rtask_update(rtask_s *task, void *worker, void *process, char *line,$/;" f +rthread_buffer ./src/replay/rworker.h /^ rbuffer_s *rthread_buffer; \/**< Buffering idle threads to be reused *\/$/;" m struct:__anon2 +rthread_buffer_mutex ./src/replay/rworker.h /^ pthread_mutex_t rthread_buffer_mutex; \/**< Sync access to rthread_buffer *\/$/;" m struct:__anon2 +rthread_destroy ./src/replay/rthread.c /^void rthread_destroy(rthread_s *t)$/;" f +rthread_fd ./src/replay/rthread.h /^ FILE *rthread_fd; \/**< Used for debugging purposes only *\/$/;" m struct:rthread_s_ +rthread_insert_task ./src/replay/rthread.c /^bool rthread_insert_task(rthread_s* t, rtask_s* task)$/;" f +rthread_map ./src/replay/rworker.h /^ amap_s* rthread_map; \/**< Holding all threads handled by this worker *\/$/;" m struct:__anon2 +rthread_new ./src/replay/rthread.c /^rthread_s* rthread_new(const long tid, void *worker)$/;" f +rthread_process_task ./src/replay/rthread.c /^void rthread_process_task(rthread_s* t, rtask_s *task,$/;" f +rthread_pthread_start ./src/replay/rthread.c /^void *rthread_pthread_start(void *data)$/;" f +rthread_s ./src/replay/rthread.h /^} rthread_s;$/;" t typeref:struct:rthread_s_ +rthread_s_ ./src/replay/rthread.h /^typedef struct rthread_s_ {$/;" s +rthread_terminate ./src/replay/rthread.c /^void rthread_terminate(rthread_s* t)$/;" f +rthread_update ./src/replay/rthread.c /^long rthread_update(rthread_s *t, const long tid)$/;" f +rworker_destroy ./src/replay/rworker.c /^void rworker_destroy(rworker_s *w)$/;" f +rworker_fd ./src/replay/rworker.h /^ FILE *rworker_fd; \/**< For debugging purposes only *\/$/;" m struct:__anon2 +rworker_new ./src/replay/rworker.c /^rworker_s* rworker_new(const int rworker_num, amap_s *fds_map,$/;" f +rworker_num ./src/replay/rprocess.h /^ int rworker_num; \/**< The worker number of the responsible worker *\/$/;" m struct:rprocess_s_ +rworker_num ./src/replay/rworker.h /^ int rworker_num; \/**< The current worker ID *\/$/;" m struct:__anon2 +rworker_process_lines ./src/replay/rworker.c /^status_e rworker_process_lines(rworker_s* w, const long num_lines)$/;" f +rworker_s ./src/replay/rworker.h /^} rworker_s;$/;" t typeref:struct:__anon2 +single_threaded ./src/replay/rthread.h /^ bool single_threaded; \/**< Worker is single threaded or not *\/$/;" m struct:rthread_s_ +size ./src/datas/amap.h /^ long size; \/**< The total size\/capacity of the amap *\/$/;" m struct:amap_s_ +size ./src/datas/btree.h /^ int size; \/**< The current size of the binary tree *\/$/;" m struct:btree_s_ +size ./src/datas/hmap.h /^ unsigned int size; \/**< Size of the hmap *\/$/;" m struct:hmap_s_ +size ./src/datas/rbuffer.h /^ int size;$/;" m struct:rbuffer_s_ +size ./src/datas/stack.h /^ unsigned long size; \/**< A count how many elements are in the stack *\/$/;" m struct:stack_s_ +sizes_created ./src/init/itask.h /^ long sizes_created;$/;" m struct:itask_s_ +speed_factor ./src/options.h /^ double speed_factor; \/**< Specifies how fast the test is replayed *\/$/;" m struct:options_s_ +stack_destroy ./src/datas/stack.c /^void stack_destroy(stack_s *s)$/;" f +stack_elem_s ./src/datas/stack.h /^} stack_elem_s;$/;" t typeref:struct:stack_elem_s_ +stack_elem_s_ ./src/datas/stack.h /^typedef struct stack_elem_s_ {$/;" s +stack_is_empty ./src/datas/stack.c /^int stack_is_empty(stack_s *s)$/;" f +stack_new ./src/datas/stack.c /^stack_s *stack_new()$/;" f +stack_new_reverse_from ./src/datas/stack.c /^stack_s* stack_new_reverse_from(stack_s *s)$/;" f +stack_pop ./src/datas/stack.c /^void* stack_pop(stack_s *s)$/;" f +stack_push ./src/datas/stack.c /^void stack_push(stack_s *s, void *data)$/;" f +stack_s ./src/datas/stack.h /^} stack_s;$/;" t typeref:struct:stack_s_ +stack_s_ ./src/datas/stack.h /^typedef struct stack_s_ {$/;" s +start_pthread ./src/utils/utils.c /^void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data)$/;" f +start_time ./src/generate/generate.h /^ long start_time; \/**< The start time from the .capture file *\/$/;" m struct:generate_s_ +status ./src/generate/gtask.h /^ int status; \/**< Operation\/syscall return status *\/$/;" m struct:gtask_s_ +status_e ./src/defaults.h /^} status_e;$/;" t typeref:enum:status_e_ +status_e_ ./src/defaults.h /^typedef enum status_e_ {$/;" g +strtok2_r ./src/utils/utils.c /^char* strtok2_r(char *str, char *delim, char **saveptr)$/;" f +strunquote ./src/utils/utils.c /^void strunquote(char *str)$/;" f +task_buffer ./src/replay/rworker.h /^ rbuffer_s *task_buffer; \/**< Buffering thread tasks to be reused *\/$/;" m struct:__anon2 +task_buffer_mutex ./src/replay/rworker.h /^ pthread_mutex_t task_buffer_mutex; \/**< To sync access to task_buffer *\/$/;" m struct:__anon2 +tasks ./src/replay/rthread.h /^ rbuffer_s* tasks; \/**< Holds all outstanding tasks *\/$/;" m struct:rthread_s_ +terminate ./src/generate/gparser.h /^ bool terminate; \/**< The parser thread will terminate if set to true *\/$/;" m struct:gparser_s_ +terminate ./src/generate/gwriter.h /^ bool terminate; \/**< The writer thread will terminate if set to true *\/$/;" m struct:gwriter_s_ +terminate ./src/init/ithread.h /^ bool terminate; \/**< Indicates that thread can terminate *\/$/;" m struct:ithread_s_ +terminate ./src/replay/rprocess.h /^ int terminate; \/**< Indicates whether the worker is terminating or not *\/$/;" m struct:rprocess_s_ +terminate ./src/replay/rthread.h /^ bool terminate; \/**< True if thread shall terminate *\/$/;" m struct:rthread_s_ +threads_map ./src/init/init.h /^ amap_s *threads_map;$/;" m struct:init_s_ +tid ./src/generate/gtask.h /^ long tid; \/**< The thread ID *\/$/;" m struct:gtask_s_ +tid ./src/replay/rthread.h /^ long tid; \/**< The virtual thread id *\/$/;" m struct:rthread_s_ +toks ./src/replay/rtask.h /^ char *toks[MAX_TOKENS+1]; \/**< The tokens parsed from the .replay line *\/$/;" m struct:rtask_s_ +top ./src/datas/stack.h /^ stack_elem_s *top; \/**< The top element of the stack, NULL if empty *\/$/;" m struct:stack_s_ +trash ./src/options.h /^ bool trash; \/**< If set ioreplay will trash the environment *\/$/;" m struct:options_s_ +unsure ./src/generate/vsize.h /^ bool unsure; \/**< True if the file type is not fully clear *\/$/;" m struct:vsize_s_ +updates ./src/generate/vsize.h /^ long updates; \/**< Amount of times this vsize has been updated *\/$/;" m struct:vsize_s_ +user ./src/options.h /^ char *user; \/**< The user name to run the test as *\/$/;" m struct:options_s_ +utests_run ./src/utests.c /^void utests_run()$/;" f +vfd ./src/generate/gtask.h /^ vfd_s *vfd; \/**< A pointer to the virtual file descriptor *\/$/;" m struct:gtask_s_ +vfd_buffer ./src/generate/generate.h /^ rbuffer_s *vfd_buffer; \/**< A virtual fd buffer, for reusing these *\/$/;" m struct:generate_s_ +vfd_destroy ./src/vfd.c /^void vfd_destroy(vfd_s *vfd)$/;" f +vfd_map ./src/generate/gprocess.h /^ hmap_s *vfd_map; \/**< All virtual file descriptors of that process *\/$/;" m struct:gprocess_s_ +vfd_new ./src/vfd.c /^vfd_s* vfd_new(const int fd, const long mapped_fd, char *path)$/;" f +vfd_print ./src/vfd.c /^void vfd_print(vfd_s *vfd)$/;" f +vfd_s ./src/vfd.h /^} vfd_s;$/;" t typeref:struct:vfd_s_ +vfd_s_ ./src/vfd.h /^typedef struct vfd_s_ {$/;" s +vfd_update ./src/vfd.c /^void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path)$/;" f +vsize ./src/generate/gtask.h /^ vsize_s *vsize; \/**< Pointer to the virtual size object *\/$/;" m struct:gtask_s_ +vsize ./src/generate/vsize.h /^ long vsize; \/**< The virtual size *\/$/;" m struct:vsize_s_ +vsize ./src/init/itask.h /^ long vsize;$/;" m struct:itask_s_ +vsize ./src/replay/rtask.h /^ unsigned long vsize; \/**< The vsize *\/$/;" m struct:rtask_s_ +vsize2 ./src/generate/gtask.h /^ vsize_s *vsize2; \/**< Pointer to a second virtual size object *\/$/;" m struct:gtask_s_ +vsize_adjust ./src/generate/vsize.c /^void vsize_adjust(vsize_s *v, vfd_s* vfd)$/;" f +vsize_close ./src/generate/vsize.c /^void vsize_close(vsize_s *v, void* vfd)$/;" f +vsize_deficit ./src/generate/vsize.h /^ long vsize_deficit; \/**< Size to use for file creating during init mode *\/$/;" m struct:vsize_s_ +vsize_destroy ./src/generate/vsize.c /^void vsize_destroy(vsize_s *v)$/;" f +vsize_map ./src/generate/generate.h /^ hmap_s *vsize_map; \/**< A hash map of all virtual size objects *\/$/;" m struct:generate_s_ +vsize_mkdir ./src/generate/vsize.c /^void vsize_mkdir(vsize_s *v, const char *path)$/;" f +vsize_new ./src/generate/vsize.c /^vsize_s* vsize_new(char *file_path, const unsigned long id,$/;" f +vsize_open ./src/generate/vsize.c /^void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags)$/;" f +vsize_read ./src/generate/vsize.c /^void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes)$/;" f +vsize_rename ./src/generate/vsize.c /^void vsize_rename(vsize_s *v, vsize_s *v2,$/;" f +vsize_rmdir ./src/generate/vsize.c /^void vsize_rmdir(vsize_s *v, const char *path)$/;" f +vsize_s ./src/generate/vsize.h /^} vsize_s;$/;" t typeref:struct:vsize_s_ +vsize_s_ ./src/generate/vsize.h /^typedef struct vsize_s_ {$/;" s +vsize_seek ./src/generate/vsize.c /^void vsize_seek(vsize_s *v, void *vfd, const long new_offset)$/;" f +vsize_stat ./src/generate/vsize.c /^void vsize_stat(vsize_s *v, const char *path)$/;" f +vsize_unlink ./src/generate/vsize.c /^void vsize_unlink(vsize_s *v, const char *path)$/;" f +vsize_write ./src/generate/vsize.c /^void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes)$/;" f +wd_base ./src/options.h /^ char *wd_base; \/**< The working directory base *\/$/;" m struct:options_s_ +whence ./src/generate/gtask.h /^ long whence; \/**< Whence *\/$/;" m struct:gtask_s_ +worker ./src/replay/rtask.h /^ void *worker; \/* The responsible worker object *\/$/;" m struct:rtask_s_ +worker ./src/replay/rthread.h /^ void *worker; \/**< The responsible worker object *\/$/;" m struct:rthread_s_ +write_pos ./src/datas/rbuffer.h /^ sig_atomic_t write_pos;$/;" m struct:rbuffer_s_ +writer ./src/generate/generate.h /^ struct gwriter_s_ *writer; \/**< A pointer to the writer object *\/$/;" m struct:generate_s_ typeref:struct:generate_s_::gwriter_s_ |