diff options
| author | Paul Buetow <pbuetow@mimecast.com> | 2018-03-06 17:38:59 +0000 |
|---|---|---|
| committer | Paul Buetow <pbuetow@mimecast.com> | 2018-03-06 17:38:59 +0000 |
| commit | 26b3b3e368a79ce29df732ea04e72a4c002ae2ce (patch) | |
| tree | e3fc8d7461ab371279f7bf9c692096cd39cc92f6 /ioreplay/src | |
| parent | ae2221660f9b411fa78cdf8034f0803e9a870cde (diff) | |
rename into ioriot
Diffstat (limited to 'ioreplay/src')
68 files changed, 0 insertions, 9531 deletions
diff --git a/ioreplay/src/capture/capture.c b/ioreplay/src/capture/capture.c deleted file mode 100644 index 0ac336b..0000000 --- a/ioreplay/src/capture/capture.c +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "capture.h" - -#include <sys/utsname.h> - -status_e capture_run(options_s *opts) -{ - int status = 0; - struct utsname uts; - - // To make it nicer we should iterate over PATH instead - char *staprun_paths[3] = { - "/usr/bin/staprun", - "/usr/local/bin/staprun", - "/bin/staprun" - }; - int num_staprun_paths = 3; - - if (0 != uname(&uts)) { - Errno("Could not identify release of currently running Kernel!"); - } - - Put("Release of currently running Kernel: %s", uts.release); - char modules_dir[128]; - sprintf(modules_dir, "/opt/ioreplay/systemtap/%s", uts.release); - Put("Changing directory to module path: %s/", modules_dir); - - if (0 != chdir(modules_dir)) { - Errno("Could not change into '%s', please ensure that the compiled " - "SystemTap modules correspond to the currently running Kernel " - "and that these are installed properly!\n", - modules_dir); - } - - if (0 != access(opts->module, R_OK)) { - Errno("Module '%s/%s' can't be read, please make sure that the " - "SystemTap Kernel modules are installed!", - modules_dir, opts->module); - } - - char *staprun_path = NULL; - for (int i = 0; i < num_staprun_paths; ++i) { - if (0 == access(staprun_paths[i], X_OK)) { - staprun_path = staprun_paths[i]; - //Put("SystemTap command path: %s", staprun_path); - break; - } - } - - if (staprun_path == NULL) { - Errno("Can't find 'staprun' command, please ensure to have the SystemTap " - "runtime (usually package 'systemtap-runtime') installed!"); - } - - char staprun_command[128]; - if (opts->pid >= 0) { - sprintf(staprun_command, "%s %s -v -o %s -x %d", staprun_path, opts->module, - opts->capture_file, opts->pid); - } else { - sprintf(staprun_command, "%s %s -v -o %s", staprun_path, opts->module, - opts->capture_file); - } - - Out("NOTICE: It is good practise first to stop all processes, then to "); - Out("start capturing, and then to start all processes again. The reason "); - Out("is that processes may have already open file handles. In that case "); - Out("I/O Replay would be unable to replay these! This may be improved "); - Put("in a future release!"); - Put("To abort capturing now send Ctrl+C, otherwise wait 1h"); - Put("Capturing I/O via: '%s'", staprun_command); - - char buf[1024]; - FILE *fp; - - if ((fp = popen(staprun_command, "r")) == NULL) { - Errno("Unable to invoke staprun command!"); - } - while (fgets(buf, 1024, fp) != NULL) - Out("stapio: %s", buf); - - if (0 != pclose(fp)) { - Error("Problems invoking staprun command!"); - } - - return status; -} diff --git a/ioreplay/src/capture/capture.h b/ioreplay/src/capture/capture.h deleted file mode 100644 index 7718d3e..0000000 --- a/ioreplay/src/capture/capture.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef CAPTURE_H -#define CAPTURE_H - -#include "../defaults.h" -#include "../utils/futils.h" -#include "../options.h" - -/** - * @brief Captures I/O to a .capture file by using stap from SystemTap - * - * @param opts The options object - * @return SUCCESS if everything went fine - */ -status_e capture_run(options_s *opts); - -#endif // CAPTURE_H diff --git a/ioreplay/src/cleanup/cleanup.c b/ioreplay/src/cleanup/cleanup.c deleted file mode 100644 index 570f8a7..0000000 --- a/ioreplay/src/cleanup/cleanup.c +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "cleanup.h" - -#include "../mounts.h" - -status_e cleanup_run(options_s *opts) -{ - set_limits_drop_root(opts->user); - mounts_s *m = mounts_new(opts); - - if (opts->purge) - mounts_purge(m); - else - mounts_trash(m); - - return SUCCESS; -} diff --git a/ioreplay/src/cleanup/cleanup.h b/ioreplay/src/cleanup/cleanup.h deleted file mode 100644 index 127badf..0000000 --- a/ioreplay/src/cleanup/cleanup.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef CLEANUP_H -#define CLEANUP_H - -#include "../defaults.h" -#include "../options.h" - -/** - * @brief Cleans up all files and directories of a given test - * - * @brief opts The options object - * @return SUCCESS in case everything went fine - */ -status_e cleanup_run(options_s *opts); - -#endif // CLEANUP_H diff --git a/ioreplay/src/datas/amap.c b/ioreplay/src/datas/amap.c deleted file mode 100644 index 806a3f8..0000000 --- a/ioreplay/src/datas/amap.c +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "amap.h" - -/** - * @brief Creates a new array map - * - * @param size The array map size - * @param mmapped true if the memory should be mmapped - * @return The new amap object - */ -static amap_s *_amap_new(long size, bool mmapped) -{ - amap_s *a = NULL; - void ***arrays = NULL; - - // Calculate a multiple of 1024, but at least in size of 'size'. - if (size % 1024 != 0) { - size = 1024*(1+(long)(size/1024)); - } - - if (size < 1) { - Error("Size overflow"); - } - - int num_arrays = size / AMAP_MAX_ARRAY_LENGTH; - - if (mmapped) { - a = Mmapshared(amap_s); - arrays = Cmapshared(num_arrays, void**); - } else { - a = Malloc(amap_s); - arrays = Calloc(num_arrays, void**); - } - - for (int i = 0; i < num_arrays; ++i) { - if (mmapped) { - //Put("%d", AMAP_MAX_ARRAY_LENGTH); - arrays[i] = Cmapshared(AMAP_MAX_ARRAY_LENGTH, void*); - } else { - arrays[i] = Calloc(AMAP_MAX_ARRAY_LENGTH, void*); - } - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - arrays[i][j] = NULL; - } - } - - a->arrays = arrays; - a->num_arrays = num_arrays; - a->size = size; - a->data_destroy = NULL; - a->mmapped = mmapped; - - return a; -} - -/** - * @brief Creates a new array map - * - * @param size The array map size - * @return The new amap object - */ -amap_s* amap_new(const long size) -{ - return _amap_new(size, false); -} - -/** - * @brief Creates a new mmapped array map - * - * @param size The array map size - * @return The new amap object - */ -amap_s* amap_new_mmapped(const long size) -{ - return _amap_new(size, true); -} - -/** - * @brief Destroys a mmap object - * - * @a The new amap object - */ -void amap_destroy(amap_s* a) -{ - if (!a) { - return; - } - - // Don't bother, the mmapped version of amap will stay alive until - // process terminations. And after process termination everything - // will be cleaned up automatically by Linux. - if (a->mmapped) { - return; - } - - for (int i = 0; i < a->num_arrays; ++i) { - if (a->data_destroy) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) - if (a->arrays[i][j]) { - a->data_destroy(a->arrays[i][j]); - } - } - free(a->arrays[i]); - } - free(a->arrays); - free(a); -} - -/** - * @brief Resets a mmap object - * - * This resets all entries to NULL. - * - * @a The new amap object - */ -void amap_reset(amap_s* a) -{ - for (int i = 0; i < a->num_arrays; ++i) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - if (a->data_destroy) { - if (a->arrays[i][j]) { - a->data_destroy(a->arrays[i][j]); - } - } - a->arrays[i][j] = NULL; - } - } -} - -int amap_set(amap_s *a, const long position, void* value) -{ - if (position >= a->size) - return -1; - int which_array = position / AMAP_MAX_ARRAY_LENGTH; - int array_pos = position % AMAP_MAX_ARRAY_LENGTH; - a->arrays[which_array][array_pos] = value; - return 0; -} - -void* amap_get(amap_s *a, const long position) -{ - if (position >= a->size) - return NULL; - int which_array = position / AMAP_MAX_ARRAY_LENGTH; - int array_pos = position % AMAP_MAX_ARRAY_LENGTH; - return a->arrays[which_array][array_pos]; -} - -void* amap_unset(amap_s *a, const long position) -{ - if (position >= a->size) - return NULL; - int which_array = position / AMAP_MAX_ARRAY_LENGTH; - int array_pos = position % AMAP_MAX_ARRAY_LENGTH; - void *value = a->arrays[which_array][array_pos]; - a->arrays[which_array][array_pos] = NULL; - return value; -} - -void amap_run_cb(amap_s *a, void (*cb)(void *data)) -{ - for (int i = 0; i < a->num_arrays; ++i) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - if (a->arrays[i][j]) - cb(a->arrays[i][j]); - } - } -} - -void amap_print(amap_s* a) -{ - Put("amap_s (%p):", (void*)a); - Put("\tmmapped: %d", a->mmapped); - Put("\tmax_array_length: %d", AMAP_MAX_ARRAY_LENGTH); - Put("\tnum_arrays: %d", a->num_arrays); - Put("\tsize: %lu", a->size); - Out("\toccupied slots: "); - for (int i = 0; i < a->num_arrays; ++i) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - if (a->arrays[i][j] != NULL) { - Out("%d:%d ", i, j); - } - } - } - Out("\n"); -} - -void _amap_test(amap_s *a) -{ - assert(0 == amap_set(a, 0, (void*)10)); - assert(0 == amap_set(a, 1, (void*)11)); - assert(0 == amap_set(a, 2, (void*)12)); - assert(0 == amap_set(a, 3, (void*)a)); - assert(10 == (long) amap_get(a, 0)); - assert(11 == (long) amap_get(a, 1)); - assert(12 == (long) amap_get(a, 2)); - assert(a == amap_get(a, 3)); - - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH-1, (void*) 23)); - assert(23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH-1)); - - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH, (void*) 42)); - assert(42 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH)); - - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2-1, (void*) (23+42))); - assert(42+23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH*2-1)); - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2, (void*) 23)); - - - assert(NULL == amap_get(a, 1024*1024*9-1)); - assert(0 == amap_set(a, 1024*1024*9-1, (void*) 0x1)); - assert(0x1 == (long) amap_get(a, 1024*1024*9-1)); - assert(0x1 == (long) amap_unset(a, 1024*1024*9-1)); - assert(NULL == amap_get(a, 1024*1024*9-1)); - - assert(0 == amap_set(a, 1024*1024*9, (void*) 100)); - assert(100 == (long) amap_get(a, 1024*1024*9)); - - assert(0 == amap_set(a, 1024*1024*9+1, (void*) 101)); - assert(101 == (long) amap_get(a, 1024*1024*9+1)); - - assert(0 == amap_set(a, 1024*1024*10-2, (void*) 102)); - assert(102 == (long) amap_get(a, 1024*1024*10-2)); - - assert(0 == amap_set(a, 1024*1024*10-1, a)); - assert(a == amap_get(a, 1024*1024*10-1)); - //amap_print(a); - - assert(a == amap_unset(a, 1024*1024*10-1)); - assert(a != amap_unset(a, 1024*1024*10-1)); - //amap_print(a); -} - -void amap_test(void) -{ - // First test the non-mmapped version - amap_s* a = amap_new(1024*1024*10); - _amap_test(a); - amap_destroy(a); - - // Now test the mapped version - a = amap_new_mmapped(1024*1024*10); - _amap_test(a); - amap_destroy(a); - - // Another test with non-alligned size - a = amap_new(1024*1024*10+1); - _amap_test(a); - amap_destroy(a); -} - diff --git a/ioreplay/src/datas/amap.h b/ioreplay/src/datas/amap.h deleted file mode 100644 index 882a7c5..0000000 --- a/ioreplay/src/datas/amap.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef AMAP_H -#define AMAP_H - -#include "../defaults.h" - -#define AMAP_MAX_ARRAY_LENGTH 1024*8 - -/** - * @brief Implements an array map data structure - * - * This array map can hold a HUGE amount of entries by allocating multiple - * smaller arrays. There are two version of the amap data structure available: - * a memory mapped (mmap) and a normal version. The memory mapped version can - * be used for IPC between various processes. - */ -typedef struct amap_s_ { - void*** arrays; /**< The pointers to the amap arrays */ - int num_arrays; /**< The amount of arrays used in the amap */ - long size; /**< The total size/capacity of the amap */ - bool mmapped; /**< True if amap is memory mapped */ - void (*data_destroy)(void *data); /**< Callback to destroy all elements */ -} amap_s; - -amap_s* amap_new(const long size); -amap_s* amap_new_mmapped(const long size); -int amap_set(amap_s *a, const long position, void* value); -void* amap_get(amap_s *a, const long position); -void* amap_unset(amap_s *a, const long position); -void amap_print(amap_s *a); -void amap_destroy(amap_s *a); -void amap_reset(amap_s *a); -void amap_run_cb(amap_s *a, void (*cb)(void *data)); -void amap_test(void); - -#endif // AMAP_H diff --git a/ioreplay/src/datas/btree.c b/ioreplay/src/datas/btree.c deleted file mode 100644 index da5da48..0000000 --- a/ioreplay/src/datas/btree.c +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "btree.h" - -btree_s* btree_new() -{ - btree_s *b = Malloc(btree_s); - *b = (btree_s) { - .root = NULL, .size = 0 - }; - return b; -} - -void btree_destroy(btree_s* b) -{ - if (b->root) - btreelem_destroy_r(b->root); - free(b); -} - -void btree_destroy2(btree_s* b) -{ - if (b->root) - btreelem_destroy_r2(b->root); - free(b); -} - -int btree_insert(btree_s* b, int key, void *data) -{ - int ret = 1; - - if (b->root == NULL) { - b->root = btreelem_new(key, data); - ret = 0; - } else { - ret = btreelem_insert_r(b->root, key, data); - } - - if (ret == 0) { - b->size++; - } - - return ret; -} - -void* btree_get(btree_s* b, int key) -{ - if (b->root == NULL) - return NULL; - - return btreelem_get_r(b->root, key); -} - -void btree_print(btree_s* b) -{ - btreelem_print_r(b->root, 0); -} - -btreelem_s* btreelem_new(int key, void *data) -{ - btreelem_s *e = Malloc(btreelem_s); - *e = (btreelem_s) { - .key = key, .data = data, .left = NULL, .right = NULL - }; - return e; -} - -void btreelem_destroy_r(btreelem_s* e) -{ - if (e->left) { - btreelem_destroy_r(e->left); - } - if (e->right) { - btreelem_destroy_r(e->right); - } - - free(e); -} - -void btreelem_destroy_r2(btreelem_s* e) -{ - if (e->left) - btreelem_destroy_r(e->left); - if (e->right) - btreelem_destroy_r(e->right); - if (e->data) - btree_destroy(e->data); - - free(e); -} - -int btreelem_insert_r(btreelem_s* e, int key, void *data) -{ - int ret = 0; - - if (e->key == key) { - ret = 1; - } - - else if (e->key > key) { - if (e->left == NULL) { - e->left = btreelem_new(key, data); - } else { - ret = btreelem_insert_r(e->left, key, data); - } - } - - else { - if (e->right == NULL) { - e->right = btreelem_new(key, data); - } else { - ret = btreelem_insert_r(e->right, key, data); - } - } - - return ret; -} - -void* btreelem_get_r(btreelem_s* e, int key) -{ - void *data = NULL; - - if (e->key == key) { - data = e->data; - } - - else if (e->key > key) { - if (e->left) { - data = btreelem_get_r(e->left, key); - } - } - - else { - if (e->right) { - data = btreelem_get_r(e->right, key); - } - } - - return data; -} - -void btreelem_print_r(btreelem_s* e, int depth) -{ - for (int i = 0; i < depth; ++i) { - Out(" "); - } - Put("%d\n", e->key); - - if (e->left) { - btreelem_print_r(e->left, depth); - } - - if (e->right) { - btreelem_print_r(e->right, depth+1); - } -} - diff --git a/ioreplay/src/datas/btree.h b/ioreplay/src/datas/btree.h deleted file mode 100644 index 55da560..0000000 --- a/ioreplay/src/datas/btree.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef BTREE_H -#define BTREE_H - -#include "../defaults.h" - -/** - * @brief This defines an element of the binary tree data structure - */ -typedef struct btreelem_ { - struct btreelem_ *left; /**< The next element to the left */ - struct btreelem_ *right; /**< The next element to the right */ - int key; /**< The key of the element */ - void *data; /**< A pointer to the data stored in this element */ -} btreelem_s; - -/** - * @brief This defines a binary tree data structure. - */ -typedef struct btree_s_ { - btreelem_s *root; /**< The root element */ - int size; /**< The current size of the binary tree */ -} btree_s; - -btree_s* btree_new(); -void btree_destroy(btree_s *b); -void btree_destroy2(btree_s *b); -int btree_insert(btree_s *b, int key, void *data); -void* btree_get(btree_s *b, int key); -void btree_print(btree_s *b); - -btreelem_s* btreelem_new(int key, void *data); -void btreelem_destroy_r(btreelem_s *e); -void btreelem_destroy_r2(btreelem_s *e); -int btreelem_insert_r(btreelem_s *e, int key, void *data); -void* btreelem_get_r(btreelem_s *e, int key); -void btreelem_print_r(btreelem_s *e, int depth); - -#endif // BTREE_H diff --git a/ioreplay/src/datas/hmap.c b/ioreplay/src/datas/hmap.c deleted file mode 100644 index 96c373e..0000000 --- a/ioreplay/src/datas/hmap.c +++ /dev/null @@ -1,362 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "hmap.h" - -#define _Using_string_keys h->keys != NULL - -unsigned int hmap_get_addr(hmap_s *h, char *key) -{ - unsigned long hash = 5381; - int len = strlen(key); - - for (int i = 0; i < len; ++i) { - hash = ((hash << 5) + hash) + key[i]; /* hash * 33 + c */ - } - - return (unsigned int) (hash % h->size); -} - -unsigned int hmap_get_addr_l(hmap_s *h, const long key) -{ - return (unsigned int) (key % h->size); -} - -hmap_s *_hmap_new(unsigned int init_size) -{ - hmap_s *h = Malloc(hmap_s); - - h->size = init_size; - h->data = Calloc(init_size, void*); - h->l = Calloc(init_size, list_s*); - h->data_destroy = NULL; - h->keys = NULL; - h->keys_l = NULL; - - Mset(h->data, 0, init_size, void*); - Mset(h->l, 0, init_size, list_s*); - - return h; -} - -hmap_s *hmap_new(unsigned int init_size) -{ - hmap_s *h = _hmap_new(init_size); - h->keys = Calloc(init_size, char*); - Mset(h->keys, 0, init_size, char*); - - return h; -} - -hmap_s *hmap_new_l(unsigned int init_size) -{ - hmap_s *h = _hmap_new(init_size); - h->keys_l = Calloc(init_size, int); - Mset(h->keys_l, -1, init_size, int); - - return h; -} - -void hmap_destroy(hmap_s *h) -{ - for (int i = 0; i < h->size; ++i) { - if (h->l[i]) { - list_s *l = h->l[i]; - if (h->data_destroy) - l->data_destroy = h->data_destroy; - list_destroy(h->l[i]); - } - if (h->data[i] && h->data_destroy) { - h->data_destroy(h->data[i]); - } - } - - free(h->data); - if (h->keys) - free(h->keys); - if (h->keys_l) - free(h->keys_l); - free(h->l); - free(h); - - return; -} - -int hmap_insert(hmap_s *h, char *key, void *data) -{ - if (data == NULL) { - Error("insert data can not be NULL"); - } - - int addr = hmap_get_addr(h, key); - - if (h->data[addr]) { - - if (strcmp(key, h->keys[addr]) == 0) { - // Key already exists - return 0; - } - - // There is already data, collision, create a linked list - list_s *l = h->l[addr] = list_new(); - list_key_insert(l, h->keys[addr], h->data[addr]); - list_key_insert(l, key, data); - - // Not needed anymore, as the elements are in the linked list now. - free(h->keys[addr]); - h->data[addr] = h->keys[addr] = NULL; - - return 1; - - } else if (h->l[addr]) { - // There was a collision at this address before. Insert - // the element to the linked list. Returns 0 if key is already - // in the list (no additional insert made) or 1 otherwise. - return list_key_insert(h->l[addr], key, data); - } - - // New entry on a collision free address - h->data[addr] = data; - h->keys[addr] = Clone(key); - - return 1; -} - -int hmap_insert_l(hmap_s *h, const long key, void *data) -{ - if (data == NULL) { - Error("insert data can not be NULL"); - } - - int addr = hmap_get_addr_l(h, key); - - if (h->data[addr]) { - - if (key == h->keys_l[addr]) { - // Key already exists - return 0; - } - - // There is already data, collision, create a linked list - list_s *l = h->l[addr] = list_new_l(); - list_key_insert_l(l, h->keys_l[addr], h->data[addr]); - list_key_insert_l(l, key, data); - - // Not needed anymore, as the elements are in the linked list now. - h->data[addr] = NULL; - h->keys_l[addr] = -1; - - return 1; - - } else if (h->l[addr]) { - // There was a collision at this address before. Insert - // the element to the linked list. Returns 0 if key is already - // in the list (no additional insert made) or 1 otherwise. - return list_key_insert_l(h->l[addr], key, data); - } - - // New entry on a collision free address - h->data[addr] = data; - h->keys_l[addr] = key; - - return 1; -} - -void* hmap_remove(hmap_s *h, char *key) -{ - int addr = hmap_get_addr(h, key); - - if (h->data[addr] != NULL) { - void *data = h->data[addr]; - free(h->keys[addr]); - h->data[addr] = h->keys[addr] = NULL; - return data; - - } else if (h->l[addr] != NULL) { - // There was a collision at this address before. Remove - // the element to the linked list. Returns the object if key is - // already in the list (no additional insert made) or NULL - // otherwise. - return list_key_remove(h->l[addr], key); - } - - // Key is not present - return NULL; -} - -void* hmap_remove_l(hmap_s *h, const long key) -{ - int addr = hmap_get_addr_l(h, key); - - if (h->data[addr] != NULL) { - void *data = h->data[addr]; - h->data[addr] = NULL; - h->keys_l[addr] = -1; - return data; - - } else if (h->l[addr] != NULL) { - // There was a collision at this address before. Remove - // the element to the linked list. Returns the object if key is - // already in the list (no additional insert made) or NULL - // otherwise. - return list_key_remove_l(h->l[addr], key); - } - - // Key is not present - return NULL; -} - -void* hmap_get(hmap_s *h, char *key) -{ - int addr = hmap_get_addr(h, key); - if (h->data[addr] && strcmp(h->keys[addr], key) == 0) { - return h->data[addr]; - - } else if (h->l[addr]) { - return list_key_get(h->l[addr], key); - } - - return NULL; -} - -void* hmap_get_l(hmap_s *h, const long key) -{ - int addr = hmap_get_addr_l(h, key); - if (h->data[addr] && h->keys_l[addr] == key) { - return h->data[addr]; - - } else if (h->l[addr]) { - return list_key_get_l(h->l[addr], key); - } - - return NULL; -} - -void hmap_run_cb(hmap_s* h, void (*cb)(void *data)) -{ - for (int i = 0; i < h->size; ++i) { - if (h->l[i]) { - list_s *l = h->l[i]; - list_run_cb(l, cb); - } - if (h->data[i]) { - cb(h->data[i]); - } - } -} - -void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_) -{ - for (int i = 0; i < h->size; ++i) { - if (h->l[i]) { - list_s *l = h->l[i]; - list_run_cb2(l, cb, data_); - } - if (h->data[i]) { - cb(h->data[i], data_); - } - } -} - -void hmap_print(hmap_s *h) -{ - for (int i = 0; i < h->size; ++i) { - if (h->data[i]) { - if (_Using_string_keys) { - Put("hmap:%p addr:%d key:'%s'", (void*)h, i, h->keys[i]); - } else { - Put("hmap:%p addr:%d key:%d", (void*)h, i, h->keys_l[i]); - } - } else if (h->l[i]) { - Put("hmap:%p addr:%d LIST", (void*)h, i); - list_print(h->l[i]); - } - } -} - -static void _hmap_test(hmap_s *h) -{ - void* somedata = (void*)h; - - assert(1 == hmap_insert(h, "someval", (void*)23)); - assert(1 == hmap_insert(h, "another value", (void*)123)); - - assert(1 == hmap_insert(h, "mimecast", somedata)); - assert(0 == hmap_insert(h, "mimecast", somedata)); - assert(1 == hmap_insert(h, "is", somedata)); - assert(1 == hmap_insert(h, "hiring", somedata)); - - assert(NULL != hmap_get(h, "mimecast")); - assert(NULL == hmap_get(h, "Mimecast")); - - assert(NULL != hmap_remove(h, "mimecast")); - assert(NULL == hmap_remove(h, "mimecast")); - - assert(1 == hmap_insert(h, "mimecast", somedata)); - assert(NULL != hmap_get(h, "mimecast")); - - assert(23 == (long)hmap_get(h, "someval")); - assert(23 == (long)hmap_get(h, "someval")); - - assert(123 == (long)hmap_remove(h, "another value")); - assert(0 == (long)hmap_remove(h, "another value")); - assert(NULL == hmap_get(h, "another value")); - - //hmap_print(h); -} - -static void _hmap_test_l(hmap_s *h) -{ - void* somedata = (void*)h; - - assert(1 == hmap_insert_l(h, 1, (void*)23)); - assert(1 == hmap_insert_l(h, 5, (void*)123)); - - assert(1 == hmap_insert_l(h, 3, somedata)); - assert(0 == hmap_insert_l(h, 3, somedata)); - assert(1 == hmap_insert_l(h, 4, somedata)); - assert(1 == hmap_insert_l(h, 6, somedata)); - - assert(NULL != hmap_get_l(h, 3)); - assert(NULL == hmap_get_l(h, 7)); - - assert(NULL != hmap_remove_l(h, 3)); - assert(NULL == hmap_remove_l(h, 3)); - - assert(1 == hmap_insert_l(h, 3, somedata)); - assert(NULL != hmap_get_l(h, 3)); - - assert(23 == (long)hmap_get_l(h, 1)); - assert(23 == (long)hmap_get_l(h, 1)); - - assert(123 == (long)hmap_remove_l(h, 5)); - assert(0 == (long)hmap_remove_l(h, 5)); - assert(NULL == hmap_get_l(h, 5)); -} - -void hmap_test(void) -{ - hmap_s* h = hmap_new(1024); - _hmap_test(h); - hmap_destroy(h); - - h = hmap_new(2); - _hmap_test(h); - hmap_destroy(h); - - h = hmap_new_l(1024); - _hmap_test_l(h); - hmap_print(h); - hmap_destroy(h); -} diff --git a/ioreplay/src/datas/hmap.h b/ioreplay/src/datas/hmap.h deleted file mode 100644 index 9d1978b..0000000 --- a/ioreplay/src/datas/hmap.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef HMAP_H -#define HMAP_H - -#include "../defaults.h" -#include "list.h" - -/** - * @brief A hash map data structure - * - * There are two version of this hmap data structure. One version is utilising - * string keys and the other one is utilising long keys. - * - * On hash collision the data structure will make use of a "named" linked list, - * whereas every member of the linked list has either a string key or a long - * key associated. - */ -typedef struct hmap_s_ { - char **keys; /**< List of all keys, NULL if nothing at a address */ - int *keys_l; /**< Same as keys, but for long keys */ - void **data; /**< Pointers to the stored data, NULL if nothing there */ - list_s **l; /**< Pointers to the linked lists, used on hash collision */ - void (*data_destroy)(void *data); /**< Callback to destroy all data */ - unsigned int size; /**< Size of the hmap */ -} hmap_s; - -hmap_s* hmap_new(unsigned int init_size); -hmap_s* hmap_new_l(unsigned int init_size); -void hmap_destroy(hmap_s* h); -void hmap_run_cb(hmap_s* h, void (*cb)(void *data)); -void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_); -int hmap_insert_l(hmap_s* h, const long key, void *data); -int hmap_insert(hmap_s* h, char* key, void *data); -void* hmap_remove_l(hmap_s* h, const long key); -void* hmap_remove(hmap_s* h, char* key); -void* hmap_get_l(hmap_s* h, const long key); -void* hmap_get(hmap_s* h, char* key); -unsigned int hmap_get_addr_l(hmap_s* h, const long key); -unsigned int hmap_get_addr(hmap_s* h, char* key); -void hmap_print(hmap_s* h); -void hmap_test(void); - -#endif // HMAP_H diff --git a/ioreplay/src/datas/list.c b/ioreplay/src/datas/list.c deleted file mode 100644 index 9cc78db..0000000 --- a/ioreplay/src/datas/list.c +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "list.h" - - -list_s *list_new() -{ - list_s *l = Malloc(list_s); - *l = (list_s) { - .first = NULL, .data_destroy = NULL - }; - return l; -} - -list_s *list_new_l() -{ - return list_new(); -} - -void list_destroy(list_s *l) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->key) - free(current->key); - if (current->data && l->data_destroy) - l->data_destroy(current->data); - list_elem_s *next = current->next; - free(current); - current = next; - } - - free(l); -} - -int list_key_insert(list_s *l, char *key, void *data) -{ - list_elem_s *current = l->first; - - while (current) { - // Already in the list - if (strcmp(current->key, key) == 0) - return 0; - current = current->next; - } - - list_elem_s *e = Malloc(list_elem_s); - - e->prev = NULL; - e->next = l->first; - e->key = Clone(key); - e->key_l = -1; - e->data = data; - - if (l->first) { - l->first->prev = e; - l->first = e; - - } else { - l->first = e; - } - - return 1; -} - -int list_key_insert_l(list_s *l, const long key, void *data) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->key_l == key) - return 0; - current = current->next; - } - - list_elem_s *e = Malloc(list_elem_s); - - e->prev = NULL; - e->next = l->first; - e->key = NULL; - e->key_l = key; - e->data = data; - - if (l->first) { - l->first->prev = e; - l->first = e; - - } else { - l->first = e; - } - - return 1; -} - -void _list_elem_remove(list_s *l, list_elem_s *e) -{ - if (l->first == e) { - list_elem_s *first = e->next; - if (first) - first->prev = NULL; - l->first = first; - - } else { - list_elem_s *prev = e->prev; - list_elem_s *next = e->next; - - prev->next = next; - if (next) - next->prev = prev; - } - - if (e->key) - free(e->key); - free(e); -} - -void* list_key_remove(list_s *l, char *key) -{ - list_elem_s *current = l->first; - - while (current) { - if (strcmp(current->key, key) == 0) { - void *data = current->data; - _list_elem_remove(l, current); - return data; - } - current = current->next; - } - - return NULL; -} - -void* list_key_remove_l(list_s *l, const long key) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->key_l == key) { - void *data = current->data; - _list_elem_remove(l, current); - return data; - } - current = current->next; - } - - return NULL; -} - -void* list_key_get(list_s *l, char *key) -{ - list_elem_s *current = l->first; - - while (current) { - if (strcmp(current->key, key) == 0) - return current->data; - current = current->next; - } - - return NULL; -} - -void* list_key_get_l(list_s *l, const long key) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->key_l == key) - return current->data; - current = current->next; - } - - return NULL; -} - -void list_run_cb(list_s* l, void (*cb)(void *data)) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->data) - cb(current->data); - current = current->next; - } -} - -void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->data) - cb(current->data, data_); - current = current->next; - } -} - -void list_print(list_s *l) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->key != NULL) { - Put("list:%p key:'%s' data:%p", (void*)l, - current->key, current->data); - } else { - Put("list:%p key:%ld data:%p", (void*)l, - current->key_l, current->data); - } - current = current->next; - } -} - -void list_test(void) -{ - list_s *l = list_new(); - void* somedata = (void*)l; - - assert(1 == list_key_insert(l, "foo", (void*)1)); - assert(1 == list_key_insert(l, "bar", (void*)2)); - assert(1 == list_key_insert(l, "baz", (void*)3)); - assert(2 == (long)list_key_remove(l, "bar")); - assert(1 == (long)list_key_remove(l, "foo")); - assert(3 == (long)list_key_remove(l, "baz")); - - assert(1 == list_key_insert(l, "I/O replay", somedata)); - assert(1 == list_key_insert(l, "for", somedata)); - assert(1 == list_key_insert(l, "benchmarking your server", somedata)); - assert(0 == list_key_insert(l, "for", somedata)); - - assert(NULL != list_key_get(l, "benchmarking your server")); - assert(NULL == list_key_get(l, "Mimecast")); - - assert(NULL != list_key_remove(l, "benchmarking your server")); - assert(NULL == list_key_remove(l, "benchmarking your server")); - assert(1 == list_key_insert(l, "benchmarking your server", somedata)); - - assert(1 == list_key_insert(l, "MiMecast", (void*)42)); - assert(42 == (long)list_key_get(l, "MiMecast")); - - l = list_new_l(); - - assert(1 == list_key_insert_l(l, 1, (void*)1)); - assert(1 == list_key_insert_l(l, 2, (void*)2)); - assert(1 == list_key_insert_l(l, 3, (void*)3)); - assert(1 == (long)list_key_get_l(l, 1)); - assert(1 == (long)list_key_remove_l(l, 1)); - assert(1 != (long)list_key_remove_l(l, 1)); - assert(3 == (long)list_key_remove_l(l, 3)); - - assert(1 == list_key_insert_l(l, 1234, somedata)); - assert(1 == list_key_insert_l(l, 13, somedata)); - assert(1 == list_key_insert_l(l, 666, somedata)); - assert(0 == list_key_insert_l(l, 13, somedata)); - - assert(NULL != list_key_get_l(l, 666)); - assert(NULL == list_key_get_l(l, 777)); - - assert(NULL != list_key_remove_l(l, 666)); - assert(NULL == list_key_remove_l(l, 666)); - assert(1 == list_key_insert_l(l, 666, somedata)); - - assert(1 == list_key_insert_l(l, 42, (void*)42)); - assert(42 == (long)list_key_get_l(l, 42)); - - //list_print(l); -} diff --git a/ioreplay/src/datas/list.h b/ioreplay/src/datas/list.h deleted file mode 100644 index 385333c..0000000 --- a/ioreplay/src/datas/list.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef LIST_H -#define LIST_H - -#include "../defaults.h" - -/** - * @brief Definition of a linked list element - */ -typedef struct list_elem_s_ { - struct list_elem_s_ *prev; /**< The previous element */ - struct list_elem_s_ *next; /**< The next element */ - char *key; /**< The key of the lemenet */ - long key_l; /**< The same as key, but for long keys */ - void *data; /**< Pointer to the stored data */ -} list_elem_s; - -/** - * @brief Definition of a named linked list data structure - * - * There are two version of this list data structure. One version is utilising - * string keys and the other one is utilising long keys. - */ -typedef struct list_s_ { - list_elem_s *first; /**< The first element, NULL if list empty */ - void (*data_destroy)(void *data); /**< Callback to destroy all data */ -} list_s; - -list_s* list_new(); -list_s* list_new_l(); -void list_destroy(list_s* l); -void list_run_cb(list_s* l, void (*cb)(void *data)); -void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_); -int list_key_insert(list_s* l, char *key, void *data); -int list_key_insert_l(list_s* l, const long key, void *data); -void* list_key_remove(list_s* l, char *key); -void* list_key_remove_l(list_s* l, const long key); -void* list_key_get(list_s* l, char *key); -void* list_key_get_l(list_s* l, const long key); -void list_print(list_s* l); -void list_test(); - -#endif // LIST_H diff --git a/ioreplay/src/datas/rbuffer.c b/ioreplay/src/datas/rbuffer.c deleted file mode 100644 index c019e6c..0000000 --- a/ioreplay/src/datas/rbuffer.c +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rbuffer.h" - -rbuffer_s *rbuffer_new(const int size) -{ - rbuffer_s *r = Malloc(rbuffer_s); - - r->size = size; - r->read_pos = size-1; - r->write_pos = 0; - r->ring = Calloc(size, void*); - - Mset(r->ring, 0, size, void*); - - return r; -} - -void rbuffer_destroy(rbuffer_s *r) -{ - if (r) { - free(r->ring); - free(r); - } -} - -bool rbuffer_insert(rbuffer_s* r, void *data) -{ - if (r->write_pos == r->read_pos) - // Ring buffer is full - return false; - - r->ring[r->write_pos] = data; - r->write_pos = (r->write_pos+1) % r->size; - - return true; -} - -bool rbuffer_has_next(rbuffer_s* r) -{ - sig_atomic_t read_pos = (r->read_pos+1) % r->size; - - if (read_pos == r->write_pos) - // No more items to read, buffer is empty - { - return false; - } - - return true; -} - -void* rbuffer_get_next(rbuffer_s* r) -{ - sig_atomic_t read_pos = (r->read_pos+1) % r->size; - - if (read_pos == r->write_pos) - // No more items to read, buffer is empty - { - return NULL; - } - - void *data = r->ring[read_pos]; - r->ring[read_pos] = NULL; - r->read_pos = read_pos; - - return data; -} - -void rbuffer_print(rbuffer_s* r) -{ - Put("rbuffer_s (%p):", (void*)r); - Put("\tsize: %d", (int)r->size); - Put("\tread_pos: %d", r->read_pos); - Put("\twrite_pos: %d", r->write_pos); - Out("\toccupied slots: "); - for (int i = 0; i < r->size; ++i) - if (r->ring[i]) { - Out("%d:%p ", i, r->ring[i]); - } - Out("\n"); -} - -void rbuffer_test(void) -{ - rbuffer_s *r = rbuffer_new(5); - assert(NULL == rbuffer_get_next(r)); - - assert(rbuffer_insert(r, (void*)1)); - assert(rbuffer_insert(r, (void*)2)); - assert(rbuffer_insert(r, (void*)3)); - assert(rbuffer_insert(r, (void*)4)); - assert(!rbuffer_insert(r, (void*)5)); - rbuffer_print(r); - - assert(rbuffer_has_next(r)); - assert(1 == (long) rbuffer_get_next(r)); - assert(2 == (long) rbuffer_get_next(r)); - assert(3 == (long) rbuffer_get_next(r)); - assert(4 == (long) rbuffer_get_next(r)); - assert(!rbuffer_has_next(r)); - assert(NULL == rbuffer_get_next(r)); - - assert(rbuffer_insert(r, (void*)1)); - assert(1 == (long) rbuffer_get_next(r)); - assert(rbuffer_insert(r, (void*)2)); - assert(2 == (long) rbuffer_get_next(r)); - assert(rbuffer_insert(r, (void*)3)); - assert(3 == (long) rbuffer_get_next(r)); - assert(rbuffer_insert(r, (void*)4)); - assert(4 == (long) rbuffer_get_next(r)); - assert(rbuffer_insert(r, (void*)5)); - assert(5 == (long) rbuffer_get_next(r)); - assert(NULL == rbuffer_get_next(r)); - rbuffer_print(r); - - assert(rbuffer_insert(r, (void*)1)); - rbuffer_print(r); - assert(rbuffer_insert(r, (void*)2)); - assert(1 == (long) rbuffer_get_next(r)); - rbuffer_print(r); - assert(rbuffer_insert(r, (void*)3)); - assert(2 == (long) rbuffer_get_next(r)); - rbuffer_print(r); - assert(rbuffer_insert(r, (void*)4)); - assert(3 == (long) rbuffer_get_next(r)); - rbuffer_print(r); - assert(rbuffer_insert(r, (void*)5)); - rbuffer_print(r); - assert(4 == (long) rbuffer_get_next(r)); - rbuffer_print(r); - assert(5 == (long) rbuffer_get_next(r)); - assert(NULL == rbuffer_get_next(r)); - - rbuffer_destroy(r); -} diff --git a/ioreplay/src/datas/rbuffer.h b/ioreplay/src/datas/rbuffer.h deleted file mode 100644 index fa634de..0000000 --- a/ioreplay/src/datas/rbuffer.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef RBUFFER_H -#define RBUFFER_H - -#include "signal.h" - -#include "../defaults.h" - -/** - * @brief An atomic ring buffer data type definition - * - * This data structure can be used for the common producer/consumer problem. - * As long as there is only max one producer thread and max one consumer thread - * it can be used without any mutex locking. All the operations are atomic. - */ -typedef struct rbuffer_s_ { - /** - * The positions are atomic, means the ring buffer can be accessed from - * multiple threads concurrently (one producer and one consumer thread). - * This is the current read position. - */ - sig_atomic_t read_pos; - /** - * This is the current write position. - */ - sig_atomic_t write_pos; - /** - * Holds the pointers to the actual ring data stored in the ring buffer - */ - void **ring; - /** - * Determines how many elements the ring buffer can hold. The capacity - * will be size-1 though, as we need one empty slot. - */ - int size; -} rbuffer_s; - -/** - * @brief Creates a new ring buffer - * - * @param size The size of the ring buffer - * @return The new ring buffer object - */ -rbuffer_s* rbuffer_new(const int size); - -/** - * @brief Destroys a ring buffer - * - * @param r The ring buffer object - */ -void rbuffer_destroy(rbuffer_s* r); - -/** - * @brief Inserts data pointer to the ring buffer - * - * @param r The ring buffer object - * @param data The data pointer - */ -bool rbuffer_insert(rbuffer_s* r, void *data); - -/** - * @brief Determines whether there is any data in the ring buffer - * - * @param r The ring buffer object - * @return True if there is any data, false otherwise - */ -bool rbuffer_has_next(rbuffer_s* r); - -/** - * @brief Returns and removes the next element from the ring buffer - * - * @param r The ring buffer object - * @return The data pointer - */ -void* rbuffer_get_next(rbuffer_s* r); - -/** - * @brief Prints a ring buffer - * - * @param r The ring buffer object - */ -void rbuffer_print(rbuffer_s* r); - -/** - * @brief Unit tests the ring buffer - */ -void rbuffer_test(void); - -#endif // RBUFFER_H diff --git a/ioreplay/src/datas/stack.c b/ioreplay/src/datas/stack.c deleted file mode 100644 index 94e83e3..0000000 --- a/ioreplay/src/datas/stack.c +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "stack.h" - - -stack_s *stack_new() -{ - stack_s *s = Malloc(stack_s); - *s = (stack_s) { - .top = NULL, .size = 0 - }; - return s; -} - -void stack_destroy(stack_s *s) -{ - stack_elem_s *current = s->top; - - while (current) { - stack_elem_s *next = current->next; - free(current); - current = next; - } - - free(s); -} - -void stack_push(stack_s *s, void *data) -{ - stack_elem_s *new_top = Malloc(stack_elem_s); - - *new_top = (stack_elem_s) { - .next = s->top, - .data = data - }; - - s->top = new_top; - s->size++; -} - -void* stack_pop(stack_s *s) -{ - if (s->top == NULL) { - return NULL; - } - - stack_elem_s *old_top = s->top; - - void *data = old_top->data; - s->top = old_top->next; - free(old_top); - s->size--; - - return data; -} - -int stack_is_empty(stack_s *s) -{ - return s->top == NULL; -} - -stack_s* stack_new_reverse_from(stack_s *s) -{ - stack_s* r = stack_new(); - - while (!stack_is_empty(s)) { - stack_push(r, stack_pop(s)); - } - - stack_destroy(s); - - return r; -} diff --git a/ioreplay/src/datas/stack.h b/ioreplay/src/datas/stack.h deleted file mode 100644 index 87e0974..0000000 --- a/ioreplay/src/datas/stack.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef STACK_H -#define STACK_H - -#include "../defaults.h" - -/** - * @brief Definition of a stack element - */ -typedef struct stack_elem_s_ { - struct stack_elem_s_ *next; /**< The next element */ - void *data; /**< Pointer to the stored data in the current element */ -} stack_elem_s; - -/** - * @brief Definition of a stack data structure - */ -typedef struct stack_s_ { - stack_elem_s *top; /**< The top element of the stack, NULL if empty */ - unsigned long size; /**< A count how many elements are in the stack */ -} stack_s; - -stack_s* stack_new(); -stack_s* stack_new_reverse_from(stack_s* s); -void stack_destroy(stack_s* s); -void stack_push(stack_s* s, void *data); -void* stack_pop(stack_s* s); -int stack_is_empty(stack_s* s); - -#endif // STACK_H diff --git a/ioreplay/src/defaults.h b/ioreplay/src/defaults.h deleted file mode 100644 index de7910e..0000000 --- a/ioreplay/src/defaults.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef DEFAULTS_H -#define DEFAULTS_H - -#include "utils/utils.h" - -/** Version of the supported .capture format */ -#define CAPTURE_VERSION 1 -/** Version of the supported .replay format */ -#define REPLAY_VERSION 1 -/** Max amount of tokens per line in the .capture file */ -#define MAX_TOKENS 10 -/** Max line length in either .capture or .replay file */ -#define MAX_LINE_LEN 1024*8 -/** Controls how many tasks can be queued and buffered per worker thread */ -#define TASK_BUFFER_PER_THREAD 512 -/** Version of I/O Replay */ -#define IOREPLAY_VERSION "0.3-develop" -/** Copyright information */ -#define IOREPLAY_COPYRIGHT "Mimecast 2017, 2018 (c)" -/** Max open files resource user limit */ -#define SET_RLIMIT_NOFILE 369216 -/** Max processes resource user limit */ -#define SET_RLIMIT_NPROC 30768 - -// The following are for debugging purposes only - -//#define NO_IOOP -//#define THREAD_DEBUG -//#define LOG_FILTERED - -/** - * @brief Return status codes - */ -typedef enum status_e_ { - SUCCESS, /**< Great success! */ - UNKNOWN, /**< Unknown return status :-/ */ - ERROR, /**< An error happened :-( */ -} status_e; - -#endif // DEFAULTS_H diff --git a/ioreplay/src/generate/generate.c b/ioreplay/src/generate/generate.c deleted file mode 100644 index 05445ae..0000000 --- a/ioreplay/src/generate/generate.c +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generate.h" - -#include "../meta/meta.h" -#include "gtask.h" -#include "gwriter.h" -#include "gparser.h" - -#include <fcntl.h> - -#define _MAX_PROCESSES 1024*1024*10 - -#define _Perc_filtered (g->num_lines_filtered / (g->lineno/100.0)) - -generate_s* generate_new(options_s *opts) -{ - generate_s *g = Malloc(generate_s); - - g->writer = NULL; - g->lineno = 0; - g->name = opts->name; - g->replay_fd = NULL; - g->mps = mounts_new(opts); - g->num_lines_filtered = 0; - g->num_vsizes = 0; - g->start_time = -1; - g->pid_map = amap_new(_MAX_PROCESSES); - g->vsize_map = hmap_new(_MAX_PROCESSES); - g->mmap_map = hmap_new(1024*1024); - g->vfd_buffer = rbuffer_new(1024); - g->num_mapped_pids = 0; - g->num_mapped_fds = 10; - g->opts = opts; - g->reuse_queue = rbuffer_new(1024); - g->replay_fd = Fopen(opts->replay_file, "w"); - - return g; -} - -void generate_destroy(generate_s *g) -{ - // TODO: Also clean the contets of these maps - amap_destroy(g->pid_map); - hmap_destroy(g->vsize_map); - hmap_destroy(g->mmap_map); - rbuffer_destroy(g->vfd_buffer); - mounts_destroy(g->mps); - - gtask_s *task = NULL; - while (NULL != (task = rbuffer_get_next(g->reuse_queue))) - gtask_destroy(task); - rbuffer_destroy(g->reuse_queue); - - fclose(g->replay_fd); - free(g); -} - - -status_e generate_run(options_s *opts) -{ - generate_s *g = generate_new(opts); - Put("Parsing file %s, writing output to %s", opts->capture_file, - opts->replay_file); - FILE *capture_fd = Fopen(opts->capture_file, "r"); - - size_t len = 0; - ssize_t read; - char *line = NULL; - - set_limits_drop_root(opts->user); - - // Reserve first few bytes for meta information - meta_s *meta = meta_new(g->replay_fd); - meta_reserve(meta); - - // The writer will write the .replay file - gwriter_s *writer = gwriter_new(g); - - // The parser will parse every line of the .capture file - gparser_s *parser = gparser_new(g); - - g->writer = writer; - - // Start one writer and one parser thread! - gparser_start(parser); - gwriter_start(writer); - - Out("Processing, it may take a while: "); - - // Process each line of the .capture file. Determine line by line whether - // the I/O operation makes sense or not. It might be that SystemTap skipped - // some I/O ops due to system overload or other issues. The result is that - // some lines may be corrupt or contain I/O operations on unknown file - // handles. It could also be that there are operations on unknown - // file handles such as sockets etc. These will be all filtered out by - // either the parser or the writer thread! - - while ((read = getline(&line, &len, capture_fd)) != -1) { - if (0 > ++g->lineno) { - Error("lineno:%lu Line number overflow", g->lineno); - } - if (strlen(line) >= MAX_LINE_LEN) { - Error("lineno:%lu Exceeded max line length", g->lineno); - } - - // Create a new generate task (try to reuse a task object)... - gtask_s *t = rbuffer_get_next(g->reuse_queue); - if (!t) { - t = gtask_new(g); - } else if (t->ret != 0) { - g->num_lines_filtered++; - } - gtask_init(t, line, g->lineno); - - // ...pass it to the parser queue - while (!rbuffer_insert(parser->queue, t)) - usleep(100); - - if (g->lineno % 1000000 == 0) { - Out(" %lu (filtered:%.2lf%%)", g->lineno, _Perc_filtered); - } - } - - Put("\nDone reading input file!"); - - Put("Waiting for parser thread..."); - gparser_terminate(parser); - gparser_destroy(parser); - - Put("Waiting for writer thread..."); - gwriter_terminate(writer); - gwriter_destroy(writer); - - // Retrieve all left over processed tasks to collect the - // statistics! - gtask_s *t; - while (NULL != (t = rbuffer_get_next(g->reuse_queue))) { - if (t->ret != 0) - g->num_lines_filtered++; - gtask_destroy(t); - } - - Put("Processed %lu lines in total, had to filter out %.2lf%%", - g->lineno, _Perc_filtered); - - Put("Writing init section to '%s'...", opts->replay_file); - fprintf(g->replay_fd, "#INIT\n"); - off_t init_offset = ftello(g->replay_fd); - hmap_run_cb(g->vsize_map, generate_write_init_cb); - - Put("Writing meta header to '%s'...", opts->replay_file); - meta_write_start(meta); - - // The meta header is being written to the first line of the .replay - // file and used by ioreplay to do various things (e.g. initializing - // the test correctly, creating the internal data structures with the - // correct sizes etc. - - meta_write_l(meta, "replay_version", REPLAY_VERSION); - meta_write_l(meta, "init_offset", init_offset); - - meta_write_s(meta, "user", opts->user); - meta_write_s(meta, "name", opts->name); - - meta_write_l(meta, "num_vsizes", g->num_vsizes); - meta_write_l(meta, "num_mapped_pids", g->num_mapped_pids); - meta_write_l(meta, "num_mapped_fds", g->num_mapped_fds); - meta_write_l(meta, "num_lines", g->lineno - g->num_lines_filtered); - - meta_destroy(meta); - fclose(capture_fd); - - Put("Generating '%s' done", opts->replay_file); - generate_destroy(g); - - return SUCCESS; -} - -void generate_write_init_cb(void *data) -{ - vsize_s *l = data; - generate_s *g = l->generate; - - if (l->required && strlen(l->path) > 0) { - fprintf(g->replay_fd, "%d|%d|%ld|%s|\n", - l->is_dir, l->is_file, -l->vsize_deficit, l->path); - } -} - -vsize_s* generate_vsize_by_path(generate_s *g, gtask_s *t, - char *path) -{ - vsize_s *v = NULL; - - if (!path && t) - path = t->path; - - Error_if(!path, "No path specified"); - v = hmap_get(g->vsize_map, path); - - if (!v) { - v = vsize_new(path, ++g->num_vsizes, g); - hmap_insert(g->vsize_map, path, v); - } - - if (t) - t->vsize = v; - - return v; -} - -void generate_gprocess_by_realpid(generate_s *g, gtask_s *t) -{ - // Get the virtual process data object from the virtual PID space. - t->gprocess = amap_get(g->pid_map, t->pid); - if (t->gprocess == NULL) { - t->gprocess = gprocess_new(t->pid, ++g->num_mapped_pids); - if (amap_set(g->pid_map, t->pid, t->gprocess)) { - Error("lineno:%lu Can not insert PID %ld", t->lineno, t->pid); - } - } -} diff --git a/ioreplay/src/generate/generate.h b/ioreplay/src/generate/generate.h deleted file mode 100644 index cf096d2..0000000 --- a/ioreplay/src/generate/generate.h +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GENERATE_H -#define GENERATE_H - -#include "gwriter.h" -#include "../datas/amap.h" -#include "../datas/hmap.h" -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "../mounts.h" -#include "../options.h" - -// Forward declarations (header include hell) -struct gtask_s_; - -/** - * @brief The generate object definition - * - * This is the general data structure required to generate a .replay file from - * the .capture file. - */ -typedef struct generate_s_ { - long lineno; /**< The current line number */ - long num_lines_filtered; /**< The amount of lines filtered out */ - long start_time; /**< The start time from the .capture file */ - char *name; /**< The name of the test specified by the user */ - FILE *replay_fd; /**< The fd of the .replay file */ - mounts_s *mps; /**< The mounts object */ - hmap_s *mmap_map; /**< mmap address mappings */ - amap_s *pid_map; /**< A map of all virtual process objects */ - unsigned long num_mapped_pids; /**< The amount of mapped PIDs */ - unsigned long num_mapped_fds; /**< The amount of mapped FDs */ - hmap_s *vsize_map; /**< A hash map of all virtual size objects */ - unsigned long num_vsizes; /**< The amount of virtual sizes */ - options_s *opts; /**< A pointer to the options object */ - rbuffer_s *vfd_buffer; /**< A virtual fd buffer, for reusing these */ - rbuffer_s *reuse_queue; /**< A task buffer, for reusing these */ - struct gwriter_s_ *writer; /**< A pointer to the writer object */ -} generate_s; - -/** - * @brief Creates a new generate object - * - * @param opts The options object - * @return The new generate object - */ -generate_s* generate_new(options_s *opts); - -/** - * @brief Destroys a generate object - * - * @param g The generate object to destroy - */ -void generate_destroy(generate_s* g); - -/** - * @brief Generates a .replay file from a .capture file - * - * @param opts The options object - * @return SUCCESS on success - */ -status_e generate_run(options_s *opts); - -/** - * @brief Callback to write the INIT section to the .replay file - * - * This function writes a list of all pre-required - * paths to the .replay file. That then can be used - * by ioreplay to initialise the test enironment. - * - * @param data A pointer to the vsize timestamp object - */ -void generate_write_init_cb(void *data); - -/** - * @brief Retrieves the virtual size object of a given path - * - * A new one will be created in case there is no such virtual size object yet. - * - * @param g The generate object - * @param t The task object (vfd will be stored to t->vfd) - * @param path The file path - * @return The virtual size object - */ -vsize_s* generate_vsize_by_path(generate_s *g, struct gtask_s_ *t, - char *path); - -/** - * @brief Retrieves the virtual process object of a given real PID - * - * A new one will be created in case there is no such virtual process object - * yet. - * - * @param g The generate object - * @param t The task object (vfd will be stored to t->gprocess) - */ -void generate_gprocess_by_realpid(generate_s *g, struct gtask_s_ *t); - -#endif // GENERATE_H diff --git a/ioreplay/src/generate/gioop.c b/ioreplay/src/generate/gioop.c deleted file mode 100644 index 01701bc..0000000 --- a/ioreplay/src/generate/gioop.c +++ /dev/null @@ -1,838 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gioop.h" - -status_e gioop_run(gwriter_s *w, gtask_s *t) -{ - status_e ret = SUCCESS; - - // There was already an error in the parser (parser.c) processing this - // task! Don't process it futher. - if (t->ret != SUCCESS) { - Cleanup(t->ret); - } - - generate_s *g = w->generate; - - // Get the virtual process data object from the virtual PID space and store - // a pointer to it to t->gprocess - generate_gprocess_by_realpid(g, t); - - // One of the open syscalls may openes a file handle succesfully - if (Eq(t->op, "open")) { - Cleanup(gioop_open(w, t, g)); - - } else if (Eq(t->op, "openat")) { - Cleanup(gioop_openat(w, t, g)); - - } else if (Eq(t->op, "creat")) { - Cleanup(gioop_creat(w, t, g)); - } - - // Get the virtual file descriptor of a given real fd and store a pointer - // to it to t->vfd. - if (t->has_fd) { - ret = gprocess_vfd_by_realfd(t->gprocess, t); - Cleanup_unless(SUCCESS, ret); - } - - - if (Eq(t->op, "close")) { - Cleanup(gioop_close(w, t, g)); - - } else if (Eq(t->op, "stat")) { - Cleanup(gioop_stat(w, t, g)); - - } else if (Eq(t->op, "statfs")) { - Cleanup(gioop_statfs(w, t, g)); - - } else if (Eq(t->op, "statfs64")) { - Cleanup(gioop_statfs64(w, t, g)); - - } else if (Eq(t->op, "fstat")) { - Cleanup(gioop_fstat(w, t, g)); - - } else if (Eq(t->op, "fstatat")) { - Cleanup(gioop_fstatat(w, t, g)); - - } else if (Eq(t->op, "fstatfs")) { - Cleanup(gioop_fstatfs(w, t, g)); - - } else if (Eq(t->op, "fstatfs64")) { - Cleanup(gioop_fstatfs64(w, t, g)); - - } else if (Eq(t->op, "rename")) { - Cleanup(gioop_rename(w, t, g)); - - } else if (Eq(t->op, "renameat")) { - Cleanup(gioop_renameat(w, t, g)); - - } else if (Eq(t->op, "renameat2")) { - Cleanup(gioop_renameat2(w, t, g)); - - } else if (Eq(t->op, "read")) { - Cleanup(gioop_read(w, t, g)); - - } else if (Eq(t->op, "readv")) { - Cleanup(gioop_readv(w, t, g)); - - } else if (Eq(t->op, "readahead")) { - Cleanup(gioop_readahead(w, t, g)); - - } else if (Eq(t->op, "readdir")) { - Cleanup(gioop_readdir(w, t, g)); - - } else if (Eq(t->op, "readlink")) { - Cleanup(gioop_readlink(w, t, g)); - - } else if (Eq(t->op, "readlinkat")) { - Cleanup(gioop_readlinkat(w, t, g)); - - } else if (Eq(t->op, "write")) { - Cleanup(gioop_write(w, t, g)); - - } else if (Eq(t->op, "writev")) { - Cleanup(gioop_writev(w, t, g)); - - } else if (Eq(t->op, "lseek")) { - Cleanup(gioop_lseek(w, t, g)); - - } else if (Eq(t->op, "getdents")) { - Cleanup(gioop_getdents(w, t, g)); - - } else if (Eq(t->op, "mkdir")) { - Cleanup(gioop_mkdir(w, t, g)); - - } else if (Eq(t->op, "rmdir")) { - Cleanup(gioop_rmdir(w, t, g)); - - } else if (Eq(t->op, "mkdirat")) { - Cleanup(gioop_mkdirat(w, t, g)); - - } else if (Eq(t->op, "unlink")) { - Cleanup(gioop_unlink(w, t, g)); - - } else if (Eq(t->op, "unlinkat")) { - Cleanup(gioop_unlinkat(w, t, g)); - - } else if (Eq(t->op, "lstat")) { - Cleanup(gioop_lstat(w, t, g)); - - } else if (Eq(t->op, "fsync")) { - Cleanup(gioop_fsync(w, t, g)); - - } else if (Eq(t->op, "fdatasync")) { - Cleanup(gioop_fdatasync(w, t, g)); - - } else if (Eq(t->op, "sync")) { - Cleanup(gioop_sync(w, t, g)); - - } else if (Eq(t->op, "syncfs")) { - Cleanup(gioop_syncfs(w, t, g)); - - } else if (Eq(t->op, "sync_file_range")) { - Cleanup(gioop_sync_file_range(w, t, g)); - - } else if (Eq(t->op, "fcntl")) { - Cleanup(gioop_fcntl(w, t, g)); - - } else if (Eq(t->op, "fcntl")) { - Cleanup(gioop_fcntl(w, t, g)); - - } else if (Eq(t->op, "mmap2")) { - // Support for mmap added later - - } else if (Eq(t->op, "munmap")) { - // Support for mmap added later - - } else if (Eq(t->op, "mremap")) { - // Support for mmap added later - - } else if (Eq(t->op, "msync")) { - // Support for mmap added later - - } else if (Eq(t->op, "chmod")) { - Cleanup(gioop_chmod(w, t, g)); - - } else if (Eq(t->op, "fchmodat")) { - Cleanup(gioop_chmod(w, t, g)); - - } else if (Eq(t->op, "fchmod")) { - Cleanup(gioop_fchmod(w, t, g)); - - } else if (Eq(t->op, "chown")) { - Cleanup(gioop_chown(w, t, g)); - - } else if (Eq(t->op, "chown16")) { - Cleanup(gioop_chown(w, t, g)); - - } else if (Eq(t->op, "lchown")) { - Cleanup(gioop_lchown(w, t, g)); - - } else if (Eq(t->op, "lchown16")) { - Cleanup(gioop_lchown(w, t, g)); - - } else if (Eq(t->op, "fchown")) { - Cleanup(gioop_fchown(w, t, g)); - - } else if (Eq(t->op, "fchownat")) { - Cleanup(gioop_chown(w, t, g)); - - } else if (Eq(t->op, "exit_group")) { - Cleanup(gioop_exit_group(w, t, g)); - - } else { - Cleanup(ERROR;); - } - -cleanup: - -#ifdef LOG_FILTERED - if (ret != SUCCESS) - t->filtered_where = __FILE__; -#endif - - t->ret = ret; - return ret; -} - -status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd || t->path == NULL || t->flags == -1) { - return ERROR; - } - - gprocess_create_vfd_by_realfd(t->gprocess, t, g); - generate_vsize_by_path(g, t, NULL); - - Gioop_write(OPEN, "%ld|%s|%d|%d|open", - t->mapped_fd, t->path, t->mode, t->flags); - - if (t->fd > 0) - vsize_open(t->vsize, t->vfd, t->path, t->flags); - - return SUCCESS; -} - -status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd || t->path == NULL || t->flags == -1) { - return ERROR; - } - - gprocess_create_vfd_by_realfd(t->gprocess, t, g); - generate_vsize_by_path(g, t, NULL); - Gioop_write(OPEN_AT, "%ld|%s|%d|%d|openat", - t->mapped_fd,t->path, t->mode, t->flags); - if (t->fd > 0) - vsize_open(t->vsize, t->vfd, t->path, t->flags); - - return SUCCESS; -} - -status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd || t->path == NULL || t->flags == -1) { - return ERROR; - } - - gprocess_create_vfd_by_realfd(t->gprocess, t, g); - generate_vsize_by_path(g, t, NULL); - - Gioop_write(CREAT, "%ld|%s|%d|%d|creat", - t->mapped_fd, t->path, t->mode, t->flags); - if (t->fd > 0) - vsize_open(t->vsize, t->vfd, t->path, t->flags); - - return SUCCESS; -} - - -status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(CLOSE, "%ld|%d|close", t->mapped_fd, t->status); - - if (t->status == 0) - vsize_close(t->vsize, t->vfd); - - hmap_remove_l(t->gprocess->fd_map, t->fd); - hmap_remove_l(t->gprocess->vfd_map, t->mapped_fd); - - if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) - vfd_destroy(t->vfd); - - return SUCCESS; -} - -status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(STAT, "%s|%d|stat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(STATFS, "%s|%d|statfs", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(STATFS64, "%s|%d|statfs64", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSTAT, "%ld|%d|fstat", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(FSTAT_AT, "%s|%d|fstatat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSTATFS, "%ld|%d|fstatfs", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSTATFS64, "%ld|%d|fstatfs64", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL || t->path2 == NULL ) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(RENAME, "%s|%s|%d|rename", t->path, t->path2, t->status); - - if (t->status == 0) { - t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); - vsize_rename(t->vsize, t->vsize2, t->path, t->path2); - } - - return SUCCESS; -} - -status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL || t->path2 == NULL ) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(RENAME_AT, "%s|%s|%d|renameat", t->path, t->path2, t->status); - - if (t->status == 0) { - t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); - vsize_rename(t->vsize, t->vsize2, t->path, t->path2); - } - - return SUCCESS; -} -status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL || t->path2 == NULL ) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(RENAME_AT2, "%s|%s|%d|renameat2", - t->path, t->path2, t->status); - - if (t->status == 0) { - t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); - vsize_rename(t->vsize, t->vsize2, t->path, t->path2); - } - - return SUCCESS; -} - -status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READ, "%ld|%ld|read", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READ, "%ld|%ld|readv", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READAHEAD, "%ld|%ld|%ld|readahead", - t->mapped_fd, t->offset, t->count); - - return SUCCESS; -} - -status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READDIR, "%ld|%d|readdir", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(READLINK, "%s|%d|readlink", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(READLINK_AT, "%s|%d|readlinkat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(WRITE, "%ld|%ld|write", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_write(t->vsize, t->vfd, t->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(WRITEV, "%ld|%ld|writev", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_write(t->vsize, t->vfd, t->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(LSEEK, "%ld|%ld|%ld|%ld|lseek", - t->mapped_fd, t->offset, t->whence, t->bytes); - - if (t->bytes >= 0) - vsize_seek(t->vsize, t->vfd, t->bytes); - - return SUCCESS; -} - -status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(GETDENTS, "%ld|%ld|%ld|getdents", - t->mapped_fd, t->count, t->bytes); - - return SUCCESS; -} - -status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(MKDIR, "%s|%d|%d|mkdir", t->path, t->mode, t->status); - - if (t->status == 0) - vsize_mkdir(t->vsize, t->path); - - return SUCCESS; -} -status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(MKDIR, "%s|%d|rmdir", t->path, t->status); - - if (t->status == 0) - vsize_rmdir(t->vsize, t->path); - - return SUCCESS; -} -status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(MKDIR_AT, "%s|%d|%d|mkdirat", t->path, t->mode, t->status); - - if (t->status == 0) - vsize_mkdir(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(UNLINK, "%s|%d|unlink", t->path, t->status); - - if (t->status == 0) - vsize_unlink(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(UNLINK_AT, "%s|%d|unlinkat", t->path, t->status); - - if (t->status == 0) - vsize_unlink(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(LSTAT, "%s|%d|lstat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSYNC, "%ld|%d|fsync", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FDATASYNC, "%ld|%d|fdatasync", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g) -{ - Gioop_write(SYNC, "%d|sync", t->status); - - return SUCCESS; -} - -status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(SYNCFS, "%ld|%d|syncfs", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(SYNC_FILE_RANGE, "%ld|%ld|%ld|%d|sync_file_range", - t->mapped_fd, t->offset, t->bytes, t->status); - - return SUCCESS; -} - -status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - switch (t->F) { - case F_GETFD: - case F_GETFL: - case F_SETFD: - case F_SETFL: - break; - default: - return ERROR; - break; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FCNTL, "%ld|%d|%d|%d|fcntl", - t->mapped_fd, t->F, t->G, t->status); - - return SUCCESS; -} - -status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(CHMOD, "%s|%d|%d|chmod", t->path, t->mode, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FCHMOD, "%ld|%d|%d|fchmod", t->mapped_fd, t->mode, t->status); - - return SUCCESS; -} - -status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - // Hmm, maybe rename t->offset, because here it is used for the user UID - Gioop_write(CHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - // Hmm, maybe rename t->offset, because here it is used for the user UID - Gioop_write(FCHOWN, "%ld|%ld|%d|%d|fchown", t->mapped_fd, t->offset, t->G, t->status); - - return SUCCESS; -} - -status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - // Hmm, maybe rename t->offset, because here it is used for the user UID - Gioop_write(LCHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g) -{ - // It means that the process and all its threads terminate. - // Therefore close all file handles of that process! - hmap_run_cb2(t->gprocess->vfd_map, gioop_close_all_vfd_cb, t); - - // Remove virtual process from pid map and destroy it - amap_unset(g->pid_map, t->pid); - gprocess_destroy(t->gprocess); - - return SUCCESS; -} - -void gioop_close_all_vfd_cb(void *data, void *data2) -{ - gtask_s *t = data2; - t->vfd = data; - generate_s *g = t->generate; - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(CLOSE, "%ld|%d|close on exit_group", t->vfd->mapped_fd, 0); - vsize_close(t->vsize, t->vfd); -} - diff --git a/ioreplay/src/generate/gioop.h b/ioreplay/src/generate/gioop.h deleted file mode 100644 index ad49713..0000000 --- a/ioreplay/src/generate/gioop.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GIOOP_H -#define GIOOP_H - -#include "../defaults.h" -#include "gwriter.h" -#include "gtask.h" -#include "generate.h" - - -// Helper macro regarding writing the .replay file! - -#define Gioop_write(op, ...) \ - fprintf(g->replay_fd, "%ld|%ld|%ld|0|0|%d|", \ - t->mapped_time, \ - (t->vsize ? t->vsize->id : 0),\ - t->gprocess->mapped_pid, \ - op); \ - fprintf(g->replay_fd, __VA_ARGS__); \ - fprintf(g->replay_fd, "@%ld", t->lineno); \ - fprintf(g->replay_fd, "|\n") - -/** - * @brief Function used when closing all virtual FDs of a virtual process - * - * This function is run on all virtual file handles whenever a virtual generate - * process object (gprocess_s) gets destroyed. This is on an exit_group - * syscall (a thread group, a process with all its threads, terminates). Upon - * process termination Linux also closes all its file descriptors! This is what - * we simulate here! - * - * @param data The pointer to the virtual file descriptor object - * @param data2 The pointer to the corresponding generate task object. - */ -void gioop_close_all_vfd_cb(void *data, void *data2); - -/** - * @brief Run a generate I/O operation on a given task - * - * @param w The writer object - * @param t The task object - * @return SUCCESS if everything went fine - */ -status_e gioop_run(gwriter_s *w, gtask_s *t); - -status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g); - -#endif // GIOOP_H diff --git a/ioreplay/src/generate/gparser.c b/ioreplay/src/generate/gparser.c deleted file mode 100644 index 514128f..0000000 --- a/ioreplay/src/generate/gparser.c +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gparser.h" - -#include "gtask.h" -#include "gwriter.h" - -void* gparser_pthread_start(void *data) -{ - gparser_s *p = data; - generate_s *g = p->generate; - gwriter_s *w = g->writer; - gtask_s *t = NULL; - - do { - while (NULL != (t = rbuffer_get_next(p->queue))) { - // First extract - gparser_extract(p, t); - // Second, pass the task to the writer thread - rbuffer_insert(w->queue, t); - } - usleep(100); - } while (!p->terminate); - - while (NULL != (t = rbuffer_get_next(p->queue))) { - gparser_extract(p, t); - rbuffer_insert(w->queue, t); - } - - return NULL; -} - -gparser_s* gparser_new(generate_s *g) -{ - gparser_s *p = Malloc(gparser_s); - - p->generate = g; - p->terminate = false; - p->queue = rbuffer_new(1024); - - return p; -} - -void gparser_start(gparser_s *p) -{ - start_pthread(&p->pthread, gparser_pthread_start, (void*)p); -} - -void gparser_destroy(gparser_s *p) -{ - rbuffer_destroy(p->queue); - free(p); -} - -void gparser_terminate(gparser_s *p) -{ - p->terminate = true; - pthread_join(p->pthread, NULL); -} - -void gparser_extract(gparser_s *p, gtask_s *t) -{ - status_e ret = SUCCESS; - generate_s *g = p->generate; - - char *saveptr; - char* tok = strtok2_r(t->line, ";:,", &saveptr); - int ntoks = 0; - - while (tok) { - if (++ntoks > MAX_TOKENS) { - ret = ERROR; - break; - } - ret = gparser_extract_tok(p, t, tok); - if (ret != SUCCESS) - break; - - tok = strtok2_r(NULL, ";:,", &saveptr); - } - - if (ret == SUCCESS) { - - // Check for the existance of mandatory values! - if (t->pid < 0 || t->tid < 0) { - Cleanup(ERROR); - - } else if (t->op == NULL) { - Cleanup(ERROR); - - } else if (t->mapped_time == -1) { - Cleanup(ERROR); - } - - // We are inserting ".ioreplay/NAME" to the paths. This enables us to - // run multiple tests simoultaneously. - - if (t->path) { - if (!mounts_transform_path(g->mps, g->name, - t->path, &t->path_r)) { - Cleanup(ERROR); - } - if (t->path_r) - t->path = t->path_r; - } - - if (t->path2) { - if (!mounts_transform_path(g->mps, g->name, - t->path2, &t->path2_r)) { - Cleanup(ERROR); - } - if (t->path2_r) - t->path2 = t->path2_r; - } - - } - -cleanup: - - t->ret = ret; - -#ifdef LOG_FILTERED - t->filtered_where = __FILE__; -#endif -} - -status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok) -{ - status_e ret = SUCCESS; - - if (gparser_token_not_ok(p, tok)) { - Cleanup(ERROR); - } - - generate_s *g = t->generate; - - char key = tok[0]; - char *value = tok; - value += 2; - - switch (key) { - case 'a': - // Address - t->address = strtol(value, NULL, 10); - break; - - case 'A': - // Address 2 - t->address2 = strtol(value, NULL, 10); - break; - - case 'b': - // Bytes - if (t->bytes != -1) { - Cleanup(ERROR); - } - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->bytes = strtol(value, NULL, 10); - break; - - case 'c': - // Count - if (t->count != -1) { - Cleanup(ERROR); - } - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->count = strtol(value, NULL, 10); - break; - - case 'd': - // Descriptor - if (t->fd != -1) { - Cleanup(ERROR); - } - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->fd = atoi(value); - if (t->fd > 0) - t->has_fd = true; - break; - - case 'f': - // Flags - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->flags = atoi(value); - break; - - case 'i': - // PID:TID - t->pidtid = value; - // Extract PID and TID from "PID:TID" - if (!gparser_get_pidtid(p, t->pidtid, &t->pid, &t->tid)) { - Cleanup(ERROR); - } - break; - - case 'm': - // Mode - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->mode = atoi(value); - break; - - case 'o': - // Operation - t->op = value; - break; - - case 'O': - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->offset = strtol(value, NULL, 10); - break; - - case 'W': - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->whence = strtol(value, NULL, 10); - break; - - case 'p': - // File path - t->path = value; - chreplace(t->path, '|', '_'); - strunquote(t->path); - break; - - case 'P': - // File path 2 - t->path2 = value; - chreplace(t->path2, '|', '_'); - strunquote(t->path2); - break; - - case 's': - // Cleanup status - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->status = atoi(value); - break; - - case 't': - // Time - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->mapped_time = strtol(value, NULL, 10); - // Start replay time from 0 - if (g->start_time == -1) { - g->start_time = t->mapped_time; - } - t->mapped_time -= g->start_time; - break; - - case 'F': - // FCNTL function - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->F = atoi(value); - break; - - case 'G': - // FCNTL argument - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->G = atoi(value); - break; - - case 'T': - break; - - default: - // Unknown key - { - Cleanup(ERROR); - } - } - -cleanup: - if (t->path_r) { - free(t->path_r); - t->path_r = NULL; - } - if (t->path2_r) { - free(t->path2_r); - t->path2_r = NULL; - } - - return ret; -} - -bool gparser_token_not_ok(gparser_s *p, char *tok) -{ - if (strlen(tok) < 3) { - return true; - - } else if (tok[1] != '=') { - return true; - } - - return false; -} - -bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid) -{ - char *pos = strchr(pidtid, ':'); - - if (pos) { - char *tmp = pos; - tmp++; - - if (is_number(tmp)) { - *tid = atol(tmp); - } else { - return false; - } - - pos[0] = '\0'; - if (is_number(pidtid)) { - *pid = atol(pidtid); - } else { - return false; - } - } - - else { - return false; - } - - return (*pid >= 0 && *tid >= 0); -} diff --git a/ioreplay/src/generate/gparser.h b/ioreplay/src/generate/gparser.h deleted file mode 100644 index f3e204a..0000000 --- a/ioreplay/src/generate/gparser.h +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GPARSER_H -#define GPARSER_H - -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "generate.h" -#include "gtask.h" - -/** - * @brief The parser definition - * - * The parser is to extract all information from the .capture file. - */ -typedef struct gparser_s_ { - bool terminate; /**< The parser thread will terminate if set to true */ - generate_s *generate; /**< The generate object */ - pthread_t pthread; /**< The posix thread */ - rbuffer_s *queue; /**< A queue of task objects */ -} gparser_s; - -/** - * @brief Creates a new parser - * - * @param g The generate object - * @return The new parser object - */ -gparser_s* gparser_new(generate_s *g); - -/** - * @brief Starts the parser thread - * - * @param p The parser object - */ -void gparser_start(gparser_s *p); - -/** - * @brief Terminates the parser thread - * - * @param p The parser object - */ -void gparser_terminate(gparser_s *p); - -/** - * @brief Destroys the parser thread - * - * @param p The parser object - */ -void gparser_destroy(gparser_s *p); - -/** - * @brief Extracts information a .capture line - * - * Extracts information from a .capture line and stores it into the task - * object. - * - * @param p The parser object - * @param t The task object - */ -void gparser_extract(gparser_s *p, gtask_s *t); - -/** - * @brief Extracts information from a specific token string - * - * @param p The parser object - * @param t The task object - * @param tok The token string - * @return Returns with SUCCESS on success - */ -status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok); - -/** - * @brief Verifies the correctness of a token - * - * @param p The parser object - * @param tok The token to be verified - * @return true if token verified successfully - */ -bool gparser_token_not_ok(gparser_s *p, char *tok); - -/** - * @brief Checks whether the pidtid string is correct or not - * - * @param p The parser object - * @param pidtid The string to check - * @param pid The pointer to the resulting pid - * @param tid The pointer to the resulting tid - * @return true on success - */ -bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid); - -/** - * @brief Entry point of the parser POSIX thread - * - * @param data A pointer to the parser object - * return Always NULL - */ -void* gparser_pthread_start(void *data); - -#endif // GPARSER_H diff --git a/ioreplay/src/generate/gprocess.c b/ioreplay/src/generate/gprocess.c deleted file mode 100644 index 6a0b37a..0000000 --- a/ioreplay/src/generate/gprocess.c +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gprocess.h" - -#include "../vfd.h" -#include "gioop.h" - -void _gprocess_vfd_map_destroy_cb(void *data) -{ - vfd_destroy(data); -} - -gprocess_s* gprocess_new(const long pid, const long mapped_pid) -{ - gprocess_s* gp = Malloc(gprocess_s); - - gp->pid = pid; - gp->mapped_pid = mapped_pid; - gp->max_mapped_fd = 0; - gp->fd_map = hmap_new_l(1024); - gp->vfd_map = hmap_new_l(1024); - gp->vfd_map->data_destroy = _gprocess_vfd_map_destroy_cb; - - return gp; -} - -void gprocess_destroy(gprocess_s *gp) -{ - hmap_destroy(gp->vfd_map); - hmap_destroy(gp->fd_map); - free(gp); -} - -void gprocess_create_vfd_by_realfd(gprocess_s *gp, gtask_s *t, generate_s *g) -{ - if (t->fd < 0) - return; - - // Check whether the real FD is still open according to the .capture log - long old_mapped = (long) hmap_get_l(gp->fd_map, t->fd); - if (old_mapped) { - - // That real file descriptor is already with a mapping to a virtual - // file descriptor. This may happen when SystemTap missed to trace a - // 'close' syscall. We are inserting a close now... - - t->vfd = hmap_get_l(gp->vfd_map, old_mapped); - - hmap_remove_l(gp->fd_map, t->fd); - hmap_remove_l(gp->vfd_map, old_mapped); - - if (t->vfd) { - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(CLOSE, "%ld|%d|close inserted", old_mapped, 0); - vsize_close(t->vsize, t->vfd); - if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) - vfd_destroy(t->vfd); - } - } - - t->vfd = rbuffer_get_next(g->vfd_buffer); - t->mapped_fd = ++g->num_mapped_fds; - if (!t->vfd) - t->vfd = vfd_new(t->fd, t->mapped_fd, t->path); - else - vfd_update(t->vfd, t->fd, t->mapped_fd, t->path); - t->vfd->free_path = t->path_r != NULL; - - hmap_insert_l(gp->vfd_map, t->mapped_fd, t->vfd); - hmap_insert_l(gp->fd_map, t->fd, (void*)t->mapped_fd); -} - -status_e gprocess_vfd_by_realfd(gprocess_s *gp, gtask_s *t) -{ - t->mapped_fd = (long) hmap_get_l(gp->fd_map, t->fd); - if (t->mapped_fd == 0) { - // No corresponding virtual fd number mapping - t->has_fd = false; - - } else { - t->vfd = hmap_get_l(gp->vfd_map, t->mapped_fd); - if (!t->vfd) { - return ERROR; - } - t->mapped_fd = t->vfd->mapped_fd; - } - - return SUCCESS; -} diff --git a/ioreplay/src/generate/gprocess.h b/ioreplay/src/generate/gprocess.h deleted file mode 100644 index 47e5037..0000000 --- a/ioreplay/src/generate/gprocess.h +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GPROCESS_H -#define GPROCESS_H - -#include "../datas/hmap.h" -#include "../defaults.h" -#include "gtask.h" -#include "generate.h" - -// Forward declarations (header include hell) -struct gtask_s_; -struct generate_s_; - -/** - * @brief Virtual process object used for generating .replay file - * - * An object of this represents a Linux process in generate context. - * In Linux every process owns * its own file descriptor table which is - * simulated here. Usually, a Linux process re-uses a FD number once not used - * anymore (e.g. after a close). However, as we want to increase concurrency - * while replaying the I/O we want * to ensure to always use unique file - * descriptor IDs for every open. Thats why we use max_mapped_fd to always - * map a real FD number to a uniq virtual FD number. - */ -typedef struct gprocess_s_ { - long pid; /**< The real PID */ - long mapped_pid; /**< The mapped PID */ - hmap_s *vfd_map; /**< All virtual file descriptors of that process */ - hmap_s *fd_map; /**< All mappings from real fd to virtual fd */ - long max_mapped_fd; /**< The max mapped fd number */ -} gprocess_s; - -/** - * @brief Creates a new gprocess object - * - * @param pid The process ID - * @param mapped_pid the mapped PID - * @return The new gprocess object - */ -gprocess_s* gprocess_new(const long pid, const long mapped_pid); - -/** - * @brief Destroys a gprocess object - * - * @param gp The gprocess object - */ -void gprocess_destroy(gprocess_s *gp); - -/** - * @brief Creates a new virtual FD from a given real FD number - * - * In ioreplay we map the real file descriptor (the fd number protocolled in - * the.capture file) to a virtual file descriptor (the fd numner written to the - * .replay file). The purpose is to increase concurrency of the I/O during - * replay. Normally, a process would reuse the same file descriptor number - * once closed earlier. However, when replaying we can't reuse the number if - * we want to replay the I/O on multiple paths in parallel. Therefore, it is - * ensured that the virtual file descriptor number in the .replay file is - * always * unique for every open! - * - * @param gp The process object - * @param t The task object (the vfd pointer will be stored to * t->vfd) - * @param g The generate object - */ -void gprocess_create_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t, - struct generate_s_ *g); - -/** - * @brief Retrieves a virtual FD from a given real FD number - * - * @param gp The process object - * @param t The task object (the vfd pointer will be stored to * t->vfd) - * @return SUCCESS if everything went smothly! - */ -status_e gprocess_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t); - -#endif // GPROCESS_H diff --git a/ioreplay/src/generate/gtask.c b/ioreplay/src/generate/gtask.c deleted file mode 100644 index 55a1124..0000000 --- a/ioreplay/src/generate/gtask.c +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gtask.h" - -gtask_s* gtask_new(void *generate) -{ - gtask_s *t = Malloc(gtask_s); - - t->generate = generate; - t->line = NULL; - t->path_r = NULL; - t->path2_r = NULL; -#ifdef LOG_FILTERED - t->original_line = NULL; -#endif - - return t; -} - -void gtask_init(gtask_s *t, char *line, const unsigned long lineno) -{ - if (t->line) - free(t->line); - t->line = Clone(line); - - if (t->path_r) - free(t->path_r); - if (t->path2_r) - free(t->path2_r); - -#ifdef LOG_FILTERED - if (t->original_line) - free(t->original_line); - t->original_line = Clone(line); - t->filtered_where = NULL; -#endif - - t->bytes = -1; - t->address = 0; - t->address2 = 0; - t->count = -1; - t->F = -1; - t->fd = -1; - t->flags = -1; - t->G = -1; - t->has_fd = false; - t->vsize = NULL; - t->vsize2 = NULL; - t->lineno = lineno; - t->mapped_fd = -1; - t->mapped_time = -1; - t->mode = -1; - t->offset = -1; - t->op = NULL; - t->path2 = NULL; - t->path2_r = NULL; - t->path = NULL; - t->path_r = NULL; - t->gprocess = NULL; - t->pid = -1; - t->pidtid = NULL; - t->ret = 0; - t->status = -1; - t->tid = -1; - t->vfd = NULL; - t->whence = -1; -} - -void gtask_destroy(gtask_s *t) -{ - if (t->line) - free(t->line); - if (t->path_r) - free(t->path_r); - if (t->path2_r) - free(t->path2_r); - free(t); -} - diff --git a/ioreplay/src/generate/gtask.h b/ioreplay/src/generate/gtask.h deleted file mode 100644 index 2f364d3..0000000 --- a/ioreplay/src/generate/gtask.h +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GTASK_H -#define GTASK_H - -#include "vsize.h" - -#include "gprocess.h" -#include "../vfd.h" -#include "../datas/amap.h" -#include "../datas/hmap.h" -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "../mounts.h" -#include "../options.h" - -/** - * @brief The generate task definition - * - * The gtask holds all possible variables required to process a particular - * .capture line and to generate the corresponding .replay line - */ -typedef struct gtask_s_ { - bool has_fd; /**< True if task has a file descriptor number */ - char *line; /**< A pointer to the remaining part of the .capture line */ - char *op; /**< Operation/syscall name */ - char *path2; /**< A second path name (e.g. for rename) */ - char *path2_r; /**< Work around to track mallocs, so it can be freed */ - char *path; /**< Path name */ - char *path_r; /**< Work around to track mallocs, so it can be freed */ - char *pidtid; /**< String representing pid:tid */ - int F; /**< Arguments for fcntl syscall */ - int G; /**< Arguments for fcntl syscall */ - int fd; /**< File descriptor number */ - int flags; /**< File open flags */ - int mode; /**< File open mode */ - int ret; /**< ioreplay process status, SUCCESS if everything is alright */ - int status; /**< Operation/syscall return status */ - long address2; /**< Another address (used by mmap related syscalls) */ - long address; /**< An address (used by mmap related syscalls) */ - long bytes; /**< Amount of bytes */ - long count; /**< A count */ - long lineno; /**< The current line number */ - long mapped_fd; /**< The mapped file descriptor number */ - long mapped_time; /**< The mapped time */ - long offset; /**< A offset */ - long pid; /**< The process ID */ - long tid; /**< The thread ID */ - long whence; /**< Whence */ - vfd_s *vfd; /**< A pointer to the virtual file descriptor */ - struct gprocess_s_ *gprocess; /**< A pointer to the process object */ - void *generate; /**< A pointer to the generate object */ - vsize_s *vsize2; /**< Pointer to a second virtual size object */ - vsize_s *vsize; /**< Pointer to the virtual size object */ -#ifdef LOG_FILTERED - char *original_line; /**< Only used for debugging purposes */ - char *filtered_where; /**< Only used for debugging purposes */ -#endif -} gtask_s; - -/** - * @brief Creates a new task object - * - * @param generate A pointer to the generate object - * @return The new task object - */ -gtask_s* gtask_new(void *generate); - -/** - * @brief Initialises a taks object - * - * This function is used in particular when we recycle/reuse an old - * gtask object. - * - * @param t The gtask object - * @param line The corresponding line from the .capture file - * @param lineno The line number - */ -void gtask_init(gtask_s *t, char *line, const unsigned long lineno); - -/** - * @brief Destroys a given task object - * - * @param t The task object - */ -void gtask_destroy(gtask_s *t); - -#endif // GTASK_H diff --git a/ioreplay/src/generate/gwriter.c b/ioreplay/src/generate/gwriter.c deleted file mode 100644 index e0d448e..0000000 --- a/ioreplay/src/generate/gwriter.c +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gwriter.h" - -#include "gtask.h" -#include "generate.h" -#include "gioop.h" -#include "../opcodes.h" - -void* gwriter_pthread_start(void *data) -{ - gwriter_s *w = data; - generate_s *g = w->generate; - gtask_s *t = NULL; - - do { - while (NULL != (t = rbuffer_get_next(w->queue))) { -#ifdef LOG_FILTERED - // Logging filtered lines - if (SUCCESS != gioop_run(w, t)) { - fprintf(g->replay_fd, "#FILTERED @%ld %s", t->lineno, - t->original_line); - } -#else - gioop_run(w, t); -#endif - rbuffer_insert(g->reuse_queue, t); - } - usleep(100); - } while (!w->terminate); - - while (NULL != (t = rbuffer_get_next(w->queue))) { -#ifdef LOG_FILTERED - if (SUCCESS != gioop_run(w, t)) { - fprintf(g->replay_fd, "#FILTERED @%ld %s\n", t->lineno, - t->original_line); - } -#else - gioop_run(w, t); -#endif - rbuffer_insert(g->reuse_queue, t); - } - - return NULL; -} - -gwriter_s* gwriter_new(generate_s *g) -{ - gwriter_s *w = Malloc(gwriter_s); - - w->generate = g; - w->terminate = false; - w->queue = rbuffer_new(1024); - - return w; -} - -void gwriter_start(gwriter_s *w) -{ - start_pthread(&w->pthread, gwriter_pthread_start, (void*)w); -} - -void gwriter_destroy(gwriter_s *w) -{ - rbuffer_destroy(w->queue); - free(w); -} - -void gwriter_terminate(gwriter_s *w) -{ - w->terminate = true; - pthread_join(w->pthread, NULL); -} diff --git a/ioreplay/src/generate/gwriter.h b/ioreplay/src/generate/gwriter.h deleted file mode 100644 index 4295580..0000000 --- a/ioreplay/src/generate/gwriter.h +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GWRITER_H -#define GWRITER_H - -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "vsize.h" -#include "gtask.h" -#include "generate.h" - -// Forward declaration (header include hell) -struct gtask_s_; -struct generate_s_; - -/** - * @brief Definition of the writer object - * - * The writer utilises the information extracted by the parser to actually - * write the .replay file. - */ -typedef struct gwriter_s_ { - bool terminate; /**< The writer thread will terminate if set to true */ - struct generate_s_ *generate; /**< The generate object */ - pthread_t pthread; /**< The posix thread */ - rbuffer_s *queue; /**< A queue of task objects */ -} gwriter_s; - -/** - * @brief Creates a new writer - * - * @param g The generate object - * @return The new writer object - */ -gwriter_s* gwriter_new(struct generate_s_ *g); - -/** - * @brief Starts the writer thread - * - * @param w The writer object - */ -void gwriter_start(gwriter_s *w); - -/** - * @brief Terminates the writer thread - * - * @param w The writer object - */ -void gwriter_terminate(gwriter_s *w); - -/** - * @brief Destroys the writer thread - * - * @param w The writer object - */ -void gwriter_destroy(gwriter_s *w); - -/** - * @brief Writes a line to the .replay file - * - * @param w The writer object - * @param t The task object - */ -void gwriter_write(gwriter_s *w, struct gtask_s_ *t); - -/** - * @brief Entry function of the writer pthread - * - * @param data A pointer to the writer object - * @return Always returns a NULL pointer if it doesnt crash! - */ -void* gwriter_pthread_start(void *data); - -#endif // GWRITER_H diff --git a/ioreplay/src/generate/vsize.c b/ioreplay/src/generate/vsize.c deleted file mode 100644 index f2d56ba..0000000 --- a/ioreplay/src/generate/vsize.c +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "vsize.h" - -#include "generate.h" - -// Helper macros - -#define _Set_file(v) v->is_file = true; v->unsure = v->is_dir = false -#define _Set_dir(v) v->is_dir = true; v->unsure = v->is_file = false -#define _Set_unsure(v) v->unsure = true -#define _Set_inserted(v) v->inserted = true -#define _Set_renamed(v) v->renamed = true -#define _Set_required(v) v->required = true - -vsize_s* vsize_new(char *file_path, const unsigned long id, - void *generate) -{ - vsize_s *v = Malloc(vsize_s); - - v->generate = generate; - v->id = id; - v->inserted = false; - v->is_dir = false; - v->is_file = false; - v->offset = -1; - v->path = Clone(file_path); - v->renamed = false; - v->required = false; - v->unsure = false; - v->updates = 0; - v->vsize = 0; - v->vsize_deficit = 0; - - return v; -} - -void vsize_destroy(vsize_s *v) -{ - if (!v) - return; - - free(v->path); - free(v); -} - -void init_parent_dir(vsize_s *v, const char *path) -{ - generate_s *g = v->generate; - char *clone = Clone(path); - char *parent = dirname(clone); - - vsize_s *v_parent = hmap_get(g->vsize_map, parent); - if (!v_parent) { - - // Parent directory does not yet have a vsize! - // Create a vsize object for it and set it as a pre-requirement - // so that the directory can be created during init mode. - - v_parent = vsize_new(parent, ++g->num_vsizes, g); - hmap_insert(g->vsize_map, parent, v_parent); - - _Set_required(v_parent); - _Set_dir(v_parent); - - // This is for debugging purposes only - _Set_inserted(v_parent); - v_parent->updates++; - - } else if (v_parent->unsure) { - // We now know for sure that this path must be a directory! - _Set_dir(v_parent); - v_parent->updates++; - } - - free(clone); -} - -void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags) -{ - - // v->first_encounter == false means, that this is the first occurance of - // this path and we didn't initialise it (means we didn't ensure that - // we want to create all parent directories etc. - - if (v->updates == 0) { - // We may use a recycled vfd object! When opening a file we always - // assume that the offset is 0! - vfd_s *vfd_ = vfd; - vfd_->offset = 0; - init_parent_dir(v, path); - - if (Has(flags, O_DIRECTORY)) { - _Set_required(v); - _Set_dir(v); - - } else if (Hasnt(flags, O_CREAT)) { - _Set_required(v); - _Set_file(v); - _Set_unsure(v); - } - v->updates++; - - } else if (v->unsure) { - if (Has(flags, O_DIRECTORY)) { - // Now we know for sure that this path must be a directory! - _Set_dir(v); - v->updates++; - } - } -} - -void vsize_close(vsize_s *v, void* vfd) -{ - vfd_s *vfd_ = vfd; - vfd_->offset = 0; - v->updates++; -} - -void vsize_stat(vsize_s *v, const char *path) -{ - if (v->updates == 0) { - init_parent_dir(v, path); - _Set_required(v); - _Set_file(v); - - // We are not 100% sure that this is really a file, - // the path might be still a directory though! - _Set_unsure(v); - v->updates++; - } -} - -void vsize_rename(vsize_s *v, vsize_s *v2, - const char *path, const char *path2) -{ - if (v->updates == 0) { - init_parent_dir(v, path); - _Set_required(v); - _Set_file(v); - _Set_unsure(v); - v->updates++; - } - - if (v2->updates == 0) { - init_parent_dir(v2, path2); - _Set_file(v2); - - // We are not 100% sure that this is really a file, - // the path might be still a directory though! - _Set_unsure(v2); - - // For debugging purposes only - _Set_renamed(v2); - v2->updates++; - } -} - -void vsize_adjust(vsize_s *v, vfd_s* vfd) -{ - if (v->vsize >= vfd->offset) - return; - - long deficit = v->vsize - vfd->offset; - if (deficit < v->vsize_deficit) { - v->vsize_deficit = deficit; - _Set_required(v); - _Set_file(v); - } -} - -void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes) -{ - vfd_s *vfd_ = vfd; - vfd_->offset += bytes; - vsize_adjust(v, vfd_); - v->updates++; -} - -void vsize_seek(vsize_s *v, void *vfd, const long new_offset) -{ - //vfd_s *vfd_ = vfd; - - // The file's offset can be greater than the file's current size, in which - // case the next write to the file will extend the file. This is referred - // to as creating a hole in a file and is allowed. However, this behaviour - // does not suit the estimation of the file size before we want to run the - // test. - - // TODO: Implement file hole support! - //v->updates++; -} - -void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes) -{ - vfd_s *vfd_ = vfd; - vfd_->offset += bytes; - - if (v->vsize < vfd_->offset) - v->vsize = vfd_->offset; - - v->updates++; -} - -void vsize_mkdir(vsize_s *v, const char *path) -{ - if (v->updates == 0) { - init_parent_dir(v, path); - _Set_dir(v); - v->updates++; - } -} - -void vsize_rmdir(vsize_s *v, const char *path) -{ - if (v->updates == 0) { - init_parent_dir(v, path); - _Set_required(v); - _Set_dir(v); - v->updates++; - } -} - -void vsize_unlink(vsize_s *v, const char *path) -{ - if (v->updates == 0) { - init_parent_dir(v, path); - _Set_required(v); - if (!v->is_dir) { - _Set_file(v); - _Set_unsure(v); - } - v->updates++; - } -} diff --git a/ioreplay/src/generate/vsize.h b/ioreplay/src/generate/vsize.h deleted file mode 100644 index bb1008e..0000000 --- a/ioreplay/src/generate/vsize.h +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef VSIZE_H -#define VSIZE_H - -#include "../utils/utils.h" -#include "../datas/hmap.h" -#include "../vfd.h" - -/** - * @brief Definition of a virtual size object - * - * The virtual size is used to determine the expected type and size of a file. - * This piece of information will be added to the INIT section of the the - * .replay file. That file then will be created during test initialisation. - * before running the test. It is very likely the case that the test requires - * a file of a certain size already to be present, so it can be read from disk. - */ -typedef struct vsize_s_ { - char *path; /**< The path to the file/directory */ - off_t offset; /**< The current file offset */ - unsigned long id; /**< The vsize id */ - void *generate; /**< A pointer to the generate object */ - long vsize; /**< The virtual size */ - long vsize_deficit; /**< Size to use for file creating during init mode */ - bool renamed; /**< True if file/dir has been renamed */ - bool required; /**< True if init mode will create this file/dir */ - bool is_dir; /**< True if this file/dir is a directory */ - bool is_file; /**< True if this file/dir is a regular file */ - bool unsure; /**< True if the file type is not fully clear */ - long updates; /**< Amount of times this vsize has been updated */ - bool inserted; /**< For debugging purposes only */ -} vsize_s; - -/** - * @brief Creates a new vsize object - * - * @param file_path The corresponding file path - * @param id The vsize vsize aka ID - * @param generate The generate object - * @return The new vsize object - */ -vsize_s* vsize_new(char *file_path, const unsigned long id, void *generate); - -/** - * @brief Destroys a vsize object - * - * @param v The vsize object - */ -void vsize_destroy(vsize_s *v); - -/** - * @brief Ensures that the parent directory exists - * - * This function ensures that the parent directory exists as a vsize object! - * - * @param v The vsize object - * @param path The given path - */ -void init_parent_dir(vsize_s *v, const char *path); - -/** - * @brief Adjusts the vsize - * - * Compares the virtual file size of the file in the vsize - * object to the the offset in the virtual file descriptor. - * In case the offset is higher we have a size deficit and - * we need to mark it. That way ioreplay can ensure that - * during init mode it will create a file with the correct - * size prior of running the test! - * - * @param v The virtual size object - * @param vfd The virtual file descriptor object - */ -void vsize_adjust(vsize_s *v, vfd_s* vfd); - -/** - * @brief Adjust vsize on open - * - * @param v The virtual size object - * @param vfd The virtual file descriptor object - * @param path The file open path - * @param flags The file open flags - */ -void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags); - -/** - * @brief Adjust vsize on close - * - * @param v The virtual size object - * @param vfd The virtual file descriptor object - */ -void vsize_close(vsize_s *v, void *vfd); - -/** - * @brief Adjust vsize on stat - * - * @param v The virtual size object - * @param path The stat path - */ -void vsize_stat(vsize_s *v, const char *path); - -/** - * @brief Adjust vsize on rename - * - * @param v The virtual size object - * @param v2 The virtual size object of path2 - * @param path The first file path - * @param path2 The second file path - */ -void vsize_rename(vsize_s *v, vsize_s *v2, - const char *path, const char *path2); - -/** - * @brief Adjust vsize on read - * - * @param v The virtual size object - * @param vfd The virtual vile descriptor object - * @param path The file path - * @param bytes The amount of bytes read - */ -void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes); - -/** - * @brief Adjust vsize on seek - * - * @param v The virtual size object - * @param vfd The virtual vile descriptor object - * @param new_offset The new file offset after seek - */ -void vsize_seek(vsize_s *v, void *vfd, const long new_offset); - -/** - * @brief Adjust vsize on write - * - * @param v The virtual size object - * @param vfd The virtual vile descriptor object - * @param path The file path - * @param bytes The amount of bytes written - */ -void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes); - -/** - * @brief Adjust vsize on mkdir - * - * @param v The virtual size object - * @param path The directory path - */ -void vsize_mkdir(vsize_s *v, const char *path); - -/** - * @brief Adjust vsize on rmdir - * - * @param v The virtual size object - * @param path The directory path - */ -void vsize_rmdir(vsize_s *v, const char *path); - -/** - * @brief Adjust vsize on unlink - * - * @param v The virtual size object - * @param path The file path - */ -void vsize_unlink(vsize_s *v, const char *path); - -#endif // VSIZE_H - diff --git a/ioreplay/src/init/init.c b/ioreplay/src/init/init.c deleted file mode 100644 index e375379..0000000 --- a/ioreplay/src/init/init.c +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "init.h" - -#include "../datas/stack.h" -#include "itask.h" -#include "ithread.h" -#include "../meta/meta.h" -#include "../mounts.h" -#include "../utils/futils.h" - - -init_s *init_new(options_s *opts) -{ - init_s *i = Malloc(init_s); - - i->opts = opts; - i->mounts = mounts_new(opts); - i->threads_map = amap_new(i->mounts->count); - i->reuse_queue = rbuffer_new(4096); - i->replay_fd = Fopen(opts->replay_file, "r"); - - pthread_mutex_init(&i->reuse_queue_mutex, NULL); - - return i; -} - -void init_destroy(init_s *i) -{ - amap_destroy(i->threads_map); - mounts_destroy(i->mounts); - - itask_s *task = NULL; - while (NULL != (task = rbuffer_get_next(i->reuse_queue))) { - itask_destroy(task); - } - rbuffer_destroy(i->reuse_queue); - - fclose(i->replay_fd); - pthread_mutex_destroy(&i->reuse_queue_mutex); - - free(i); -} - -void init_extract_header(init_s *i, off_t *init_offset) -{ - options_s *opts = i->opts; - meta_s *m = meta_new(i->replay_fd); - meta_read_start(m); - - long version = 0; - if (meta_read_l(m, "version", &version)) { - Put("Replay version is '%ld'", version); - if (version != REPLAY_VERSION) { - Error(".replay file of incompatible version, got %x, expected %x", - (int)version, REPLAY_VERSION); - } - } - - char *user; - if (meta_read_s(m, "user", &user)) { - Put("Setting user to '%s'", user); - opts->user = user; - } - - char *name; - if (meta_read_s(m, "name", &name)) { - Put("Setting name to '%s'", name); - opts->name = name; - } - - if (meta_read_l(m, "init_offset", init_offset)) { - if (*init_offset < 0) { - Error("Offset overflow (init offset too large in .replay)"); - } - Put("Setting init offset to '%ld'", *init_offset); - } - - meta_destroy(m); -} - -status_e init_run(options_s *opts) -{ - status_e ret = SUCCESS; - init_s *i = init_new(opts); - - off_t init_offset; - init_extract_header(i, &init_offset); - - // Ensure that all ./replay/NAME directories exist - mounts_init(i->mounts); - - // Don't do messy stuff as super user - set_limits_drop_root(opts->user); - - // We need to clean up garbish from previous runs! - if (opts->purge) - mounts_purge(i->mounts); - else - mounts_trash(i->mounts); - - Out("Creating all files and directories requried for test '%s'...", - opts->name); - - // Seek to the INIT section - fseeko(i->replay_fd, init_offset, SEEK_SET); - - bool is_file = false, is_dir = false; - long vsize = 0; - char *path; - - // Stats - long dirs_created = 0; - long files_created = 0; - long files_total_size = 0; - - // Helper variables for getline - char *line = NULL; - size_t len = 0, read = 0; - char *saveptr; - - stack_s *all_threads = stack_new(); - - // Process the INIT section of the .replay file line by line. - - while ((read = getline(&line, &len, i->replay_fd)) != -1) { - char *tok = strtok_r(line, "|", &saveptr); - - for (int ntok = 0; tok; ntok++) { - switch (ntok) { - case 0: - is_dir = atoi(tok) == 1; - break; - case 1: - is_file = atoi(tok) == 1; - break; - case 2: - vsize = atol(tok); - if (vsize < 0) { - Error("Size overflow"); - } - break; - case 3: - path = tok; - break; - default: - break; - } - - tok = strtok_r(NULL, "|", &saveptr); - } - - itask_s *task = rbuffer_get_next(i->reuse_queue); - - if (!task) { - task = itask_new(); - - } else { - itask_extract_stats(task, &dirs_created, &files_created, - &files_total_size); - } - - // Set new task values - if (is_dir) { - task->is_dir = true; - - } else if (is_file) { - task->is_file = true; - task->vsize = vsize; - } - task->path = Clone(path); - - // We run one init thread per mount point - int mnr = mounts_get_mountnumber(i->mounts, path); - ithread_s *t = amap_get(i->threads_map, mnr); - - if (!t) { - t = ithread_new(i); - amap_set(i->threads_map, mnr, t); - stack_push(all_threads, t); - ithread_start(t); - } - - //itask_print(task); - while (!rbuffer_insert(t->queue, task)) - usleep(1000); - } - - ithread_s *t = NULL; - while (NULL != (t = stack_pop(all_threads))) { - ithread_terminate(t); - ithread_destroy(t); - } - stack_destroy(all_threads); - - itask_s *task = NULL; - while (NULL != (task = rbuffer_get_next(i->reuse_queue))) { - itask_extract_stats(task, &dirs_created, &files_created, - &files_total_size); - itask_destroy(task); - } - - Put("Done!"); - - Put("Created %ld files (net total size: %.2fg) and %ld directories!", - files_created, files_total_size/(1024*1024*1024.0), - dirs_created); - - init_destroy(i); - - Put("You are ready to fire up the test now"); - - return ret; -} diff --git a/ioreplay/src/init/init.h b/ioreplay/src/init/init.h deleted file mode 100644 index 3d9f9e9..0000000 --- a/ioreplay/src/init/init.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INIT_H -#define INIT_H - -#include "../defaults.h" -#include "../options.h" -#include "../datas/amap.h" -#include "../datas/rbuffer.h" -#include "../mounts.h" - -typedef struct init_s_ { - amap_s *threads_map; - rbuffer_s *reuse_queue; - options_s *opts; - mounts_s *mounts; - FILE *replay_fd; - pthread_mutex_t reuse_queue_mutex; -} init_s; - -/** - * @brief Creates a new init object - * - * @param opts The options object - * @return The new mounts object - */ -init_s* init_new(options_s *opts); - -/** - * @brief Destroys the init object - * - * @param i The init object - */ -void init_destroy(init_s *i); - -/** - * @brief Initialises the test environment - * - * @param opts The options object - * @return SUCCESS if initialised without any issues - */ -status_e init_run(options_s *opts); - -/** - * @brief Extracts some useful information from the .replay meta header - * - * @param i The init object - * @param init_offset To store the offset of the init section - */ -void init_extract_header(init_s *i, off_t *init_offset); - -#endif // INIT_H diff --git a/ioreplay/src/init/itask.c b/ioreplay/src/init/itask.c deleted file mode 100644 index f04ce33..0000000 --- a/ioreplay/src/init/itask.c +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "itask.h" - -itask_s* itask_new() -{ - itask_s *task = Malloc(itask_s); - - task->path = NULL; - itask_reset_stats(task); - - return task; -} - -void itask_destroy(itask_s *task) -{ - if (task->path) - free(task->path); - - free(task); -} - -void itask_reset_stats(itask_s *task) -{ - task->is_dir = task->is_file = false; - task->sizes_created = task->vsize = 0; - task->dirs_created = task->files_created = 0; - - if (task->path) { - free(task->path); - task->path = NULL; - } -} - -void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created, - long *files_total_size) -{ - *dirs_created += task->dirs_created; - *files_created += task->files_created; - *files_total_size += task->sizes_created; - - if (*dirs_created < 0 || *files_created < 0 || *files_total_size < 0) { - Error("Size overflow"); - } - - itask_reset_stats(task); -} - -void itask_print(itask_s *task) -{ - Put("itask(%p): is_dir:%d is_file:%d vsize:%ld path:%s", - (void*)task, task->is_dir, task->is_file, - task->vsize, task->path); -} diff --git a/ioreplay/src/init/itask.h b/ioreplay/src/init/itask.h deleted file mode 100644 index b10d515..0000000 --- a/ioreplay/src/init/itask.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef ITASK_H -#define ITASK_H - -#include "../defaults.h" - -/** - * @brief The initialise task definition - */ -typedef struct itask_s_ { - bool is_dir; - bool is_file; - long vsize; - char *path; - long dirs_created; - long files_created; - long sizes_created; -} itask_s; - -/** - * @brief Creates a new task object - * - * @return The new task object - */ -itask_s* itask_new(); - -/** - * @brief Resets the task stats - * - * @param task The itask object - */ -void itask_reset_stats(itask_s *task); - -/** - * @brief Extract stats from a task object - * - * @param task The itask object - * @param dirs_created Adds count of dirs created to that variable - * @param files_created Adds count of files created to that variable - * @param files_total_size Adds size of files created to that variable - */ -void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created, - long *files_total_size); - -/** - * @brief Destroys a given task object - * - * @param task The task object - */ -void itask_destroy(itask_s *task); - -/** - * @brief Prints a task to stdout - * - * @param task The task object - */ -void itask_print(itask_s *task); - -#endif // ITASK_H diff --git a/ioreplay/src/init/ithread.c b/ioreplay/src/init/ithread.c deleted file mode 100644 index a580e70..0000000 --- a/ioreplay/src/init/ithread.c +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ithread.h" - -#include "itask.h" -#include "../utils/futils.h" - - -void* ithread_pthread_start(void *data) -{ - ithread_s *t = data; - init_s *i = t->init; - itask_s *task = NULL; - - do { - while (NULL != (task = rbuffer_get_next(t->queue))) { - ithread_run_task(t, task); - - // We need to mutex lock the reuse_queue as multiple threads - // can insert into it - pthread_mutex_lock(&i->reuse_queue_mutex); - int ret = rbuffer_insert(i->reuse_queue, task); - pthread_mutex_unlock(&i->reuse_queue_mutex); - if (!ret) - itask_destroy(task); - } - usleep(100); - } while (!t->terminate); - - while (NULL != (task = rbuffer_get_next(t->queue))) { - ithread_run_task(t, task); - if (!rbuffer_insert(i->reuse_queue, task)) - itask_destroy(task); - - pthread_mutex_lock(&i->reuse_queue_mutex); - int ret = rbuffer_insert(i->reuse_queue, task); - pthread_mutex_unlock(&i->reuse_queue_mutex); - if (!ret) - itask_destroy(task); - } - - return NULL; -} - -ithread_s* ithread_new(init_s *i) -{ - ithread_s *t = Malloc(ithread_s); - - t->init = i; - t->queue = rbuffer_new(1024); - t->terminate = false; - - return t; -} - -void ithread_start(ithread_s *t) -{ - start_pthread(&t->pthread, ithread_pthread_start, (void*)t); -} - -void ithread_destroy(ithread_s *t) -{ - rbuffer_destroy(t->queue); - free(t); -} - -void ithread_terminate(ithread_s *t) -{ - t->terminate = true; - pthread_join(t->pthread, NULL); -} - -void ithread_run_task(ithread_s *t, itask_s *task) -{ - if (task->is_dir) { - task->dirs_created += ensure_dir_exists(task->path); - - } else if (task->is_file) { - if (!ensure_file_exists(task->path, &task->dirs_created)) { - task->files_created++; - if (task->vsize > 0) { - append_random_to_file(task->path, task->vsize); - task->sizes_created += task->vsize; - } - } - } -} diff --git a/ioreplay/src/init/ithread.h b/ioreplay/src/init/ithread.h deleted file mode 100644 index 0884519..0000000 --- a/ioreplay/src/init/ithread.h +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef ITHREAD_H -#define ITHREAD_H - -#include "../defaults.h" -#include "../datas/rbuffer.h" - -#include "init.h" -#include "itask.h" - -#include <pthread.h> - -/** - * @brief Definition of an init thread - * - */ -typedef struct ithread_s_ { - pthread_t pthread; /**< We run the init tasks in concurrent pthreads */ - rbuffer_s *queue; /**< The thread's task queue */ - init_s *init; /**< The responsible init object */ - bool terminate; /**< Indicates that thread can terminate */ -} ithread_s; - -/** - * @brief Creates a new thread object - * - * @param i The init object - * @return The new thread object - */ -ithread_s* ithread_new(init_s *i); - -/** - * @brief Terminates the thread - * - * This function waits (via join) for the pthread to complete all its - * current tasks from the queue. - * - * @param t The thread object - */ -void ithread_terminate(ithread_s* t); - -/** - * @brief Destroys the thread object - * - * @param t The thread object - */ -void ithread_destroy(ithread_s* t); - -/** - * @brief Executes the init task - * - * @param t The thread object - * @param task The task object - */ -void ithread_run_task(ithread_s* t, itask_s *task); - -/** - * @brief Starts the POSIX thread - * - * @param t The responsible thread object - */ -void ithread_start(ithread_s *t); - -/** - * @brief Entry point of the POSIX thread - * - * @param data Data passed to the pthread - * @return Always NULL on success - */ - -void* ithread_pthread_start(void *data); - -#endif // ITHREAD_H diff --git a/ioreplay/src/macros.h b/ioreplay/src/macros.h deleted file mode 100644 index 45e5a10..0000000 --- a/ioreplay/src/macros.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MACROS_H -#define MACROS_H - -#define Cleanup(code) ret = code; goto cleanup -#define Cleanup_unless(expr, code) \ - if (expr != code) { ret = code; goto cleanup; } - -// String helpers -#define Clone(str) notnull(strdup(str),__FILE__,__LINE__,0) -#define Eq(str1,str2) strcmp(str1,str2) == 0 - -// Number helpers -#define Abs(num) num >= 0 ? num : -num -#define Readhex(str) strtol(str, NULL, 16) -#define Perc(a, b) a > b ? b/(a/100.) : a/(b/100.) - -// Bitwise helpers -#define Has(flags, what) (flags & (what)) == (what) -#define Hasnt(flags, what) (flags & (what)) != (what) - -// Memory helpers -#define Malloc(what) \ - notnull(malloc(sizeof(what)),__FILE__,__LINE__,1) -#define Calloc(count,what) \ - notnull(calloc(count,sizeof(what)),__FILE__,__LINE__,count) -#define Mset(where,value,count,what) \ - memset(where,value,count*sizeof(what)) - -// Open helpers -#define Fopen(path, mode) fnotnull(fopen(path, mode), path, __FILE__, __LINE__) - -// Mmap helpers -#define Mmapshared(what) \ - mmapok(mmap(NULL, sizeof(what), \ - PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0), \ - __FILE__,__LINE__) -#define Cmapshared(count,what) \ - mmapok(mmap(NULL, count*sizeof(what), \ - PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0), \ - __FILE__,__LINE__) - -// Printing messages -#define Out(...) \ - fprintf(stdout, __VA_ARGS__); \ - fflush(stdout); -#define Put(...) \ - fprintf(stdout, __VA_ARGS__); \ - fprintf(stdout, "\n"); \ - fflush(stdout); - -// Printing debug messages -#define Debug(...) \ - fprintf(stderr, "%s:%d DEBUG: ", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - fflush(stderr); - -// Printing error messages -#define Error(...) \ - fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr,"\n"); \ - fflush(stdout); \ - fflush(stderr); \ - exit(ERROR); - -#define Error_if(expr, ...) if (expr) { Error(__VA_ARGS__); } - -#define Errno(...) \ - fprintf(stderr, "%s:%d ERROR: %s (%d). ", __FILE__, __LINE__, \ - strerror(errno), errno); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr,"\n"); \ - fflush(stdout); \ - fflush(stderr); \ - exit(ERROR); - -#define Errno_if(expr, ...) if (expr) { Errno(__VA_ARGS__); } - -#define Segfault(...) \ - fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr,"\n"); \ - fflush(stdout); \ - fflush(stderr); \ - *(int*)0 = 0; - -// Printing warn messages -#define Warn(...) \ - fprintf(stderr, "WARN: "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr,"\n"); \ - fflush(stdout); \ - fflush(stderr); - -#define Warn_if(expr, ...) if (expr) { Warn(__VA_ARGS__); } - -// Other helpers -#define Fill_with_stuff(buf, len) \ - for (int i = 0; i<len-1; ++i) { buf[i] = 'X'; } - -#endif // MACROS_H diff --git a/ioreplay/src/main.c b/ioreplay/src/main.c deleted file mode 100644 index 4a65de3..0000000 --- a/ioreplay/src/main.c +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * @file main.c - * @author Paul Buetow - * - * @brief The entry point of the I/O Replay program. - */ - -#include <signal.h> -#include <fcntl.h> - -#include "capture/capture.h" -#include "cleanup/cleanup.h" -#include "generate/generate.h" -#include "init/init.h" -#include "mounts.h" -#include "options.h" -#include "replay/replay.h" -#include "utests.h" -#include "utils/utils.h" - -/** - * @brief Do some architecture checks - * - * To ensure that I/O replay works correctly we have to check whether some - * data types are atomic or not. This is what this function does! - */ -static void _arch_check_atomic(void) -{ - if (sizeof(int) > sizeof(sig_atomic_t)) { - Error("int data type is not atomic on this architecture: %ld > %ld", - sizeof(int), sizeof(sig_atomic_t)); - - } else if (sizeof(bool) > sizeof(sig_atomic_t)) { - Error("bool data type is not atomic on this architecture: %ld > %ld", - sizeof(bool), sizeof(sig_atomic_t)); - } -} - -/** - * @brief Prints out version and copyright information - */ -static void _print_version(void) -{ - Put("This is I/O Replay %s - %s", IOREPLAY_VERSION, IOREPLAY_COPYRIGHT); -} - -/** - * @brief Print the synopsis - */ -static void _print_synopsis(void) -{ - _print_version(); - - Put("Synopsis:"); - Put("\tioreplay -c io.capture [-x PID] [-m MODULE]"); - Put("\tioreplay -c io.capture -r io.replay [-n str] [-u str] [-w str]"); - Put("\tioreplay -i io.replay"); - Put("\tioreplay -r io.replay [-p #] [-t #] [-D] [-s #]"); - Put("\tioreplay -R io.replay [-p #] [-t #] [-D] [-s #]"); - Put("\tioreplay -d"); - Put("\tioreplay -P"); - Put("\tioreplay -T [-n NAME]"); - Put("\tioreplay -V"); -} - -/** - * @brief Print a brief help - */ -static void _print_help(void) -{ - _print_synopsis(); - - Put("Help:"); - Put("\t-d Drop all Linux/FS caches and exit ioreplay"); - Put("\t-D Don't drop all caches (in conjunction with -r/-R):"); - Put("\t-s SPEED The speed factor (default: 0 [as fast as possible])"); - Put("\t-h Print this help"); - Put("\t-c FILE The capture file"); - Put("\t-n NAME The name (default: test0)"); - Put("\t-u USER The test run user (default: mcuser)"); - Put("\t-p #WORKERS Amount of of parallel worker processes (default: 4)"); - Put("\t-t #THREADS Threads per worker process (default: 128)"); - Put("\t-i REPLAYFILE The replay file to be initialised"); - Put("\t-r REPLAYFILE The replay file to be replayed"); - Put("\t-R REPLAYFILE Init and replay in one run (-i and -r combined)"); - Put("\t-S STATSFILE Write a stats file at the end of a test"); - Put("\t-T Trash data directories"); - Put("\t-P Purge all trash directories of all tests)"); - Put("\t-V Print I/O replay program version"); - Put("\t-w WD_BASE The working directory's base path"); - Put("\t (default: /usr/local/ioreplay)"); - Put("\t-x PID To specify a process ID (in conjunction with -c)"); - Put("\t-m MODULE To specify a module (in conjunction with -c)"); - Put("\nExample (run these commands one after another):"); - Put("\t 1.) sudo ioreplay -c io.capture"); - Put("\t 2.) sudo ioreplay -r io.replay -c io.capture -u paul -n test1"); - Put("\t 3.) sudo ioreplay -i io.replay"); - Put("\t 4.) sudo ioreplay -r io.replay -S"); -} - -/** - * @brief I/O Replay's entry point - * - * Not much more to document here though! - * @return The exit code - */ -int main(int argc, char **argv) -{ - _arch_check_atomic(); - status_e ret = UNKNOWN; - - bool dont_drop_caches = false; - options_s *opts = options_new(); - int opt = 0; - - while ((opt = getopt(argc, argv, "Vr:R:S:c:u:i:hw:n:dDs:w:p:t:UPTx:m:")) != -1) { - switch (opt) { - case 'U': - utests_run(); - Cleanup(SUCCESS); - break; - case 'V': - _print_version(); - Cleanup(SUCCESS); - break; - case 'd': - drop_caches(); - Cleanup(SUCCESS); - break; - case 'D': - dont_drop_caches = true; - break; - case 'c': - opts->capture_file = absolute_path(optarg); - Put("Capture file: %s", opts->capture_file); - break; - case 'P': - opts->purge = true; - Put("Purge option set"); - break; - case 'T': - opts->trash = true; - Put("Trash option set"); - break; - case 'i': - opts->init = true; - if (!opts->replay_file) { - opts->replay_file = absolute_path(optarg); - Put("Replay file: %s", opts->replay_file); - } - break; - case 'R': - opts->init = true; - opts->replay = true; - if (!opts->replay_file) { - opts->replay_file = absolute_path(optarg); - Put("Replay file: %s", opts->replay_file); - } - break; - case 'r': - opts->replay = true; - if (!opts->replay_file) { - opts->replay_file = absolute_path(optarg); - Put("Replay file: %s", opts->replay_file); - } - break; - case 'S': - opts->stats_file = Clone(optarg); - Put("Stats output file: %s", opts->stats_file); - break; - case 'w': - opts->wd_base = optarg; - Put("WD base: %s", opts->wd_base); - break; - case 'u': - opts->user = optarg; - Put("User: %s", opts->user); - break; - case 'm': - opts->module = Clone(optarg); - Put("Module: %s", opts->module); - break; - case 'n': - opts->name = optarg; - Put("Name: %s", opts->name); - break; - case 'h': - _print_help(); - Cleanup(SUCCESS); - case 's': - sscanf(optarg, "%lf", &opts->speed_factor); - Put("Speed factor: %lf", opts->speed_factor); - break; - case 'p': - opts->num_workers = atoi(optarg); - if (opts->num_workers < 1) - opts->num_workers = 1; - Put("Num worker processes: %d", opts->num_workers); - break; - case 't': - opts->num_threads_per_worker = atoi(optarg); - if (opts->num_threads_per_worker < 1) - opts->num_threads_per_worker = 1; - Put("Num threads per worker: %d", opts->num_threads_per_worker); - break; - case 'x': - opts->pid = atoi(optarg); - Put("PID: %d", opts->pid); - break; - default: - _print_help(); - Cleanup(ERROR); - } - } - - if (opts->purge || opts->trash) { - // Clean up all temp data of previous test runs - Cleanup(cleanup_run(opts)); - - } else if (opts->capture_file && !opts->replay_file) { - // We are going to capture I/O - Cleanup(capture_run(opts)); - - } else if (opts->capture_file && opts->replay_file) { - // We are going to generate a .replay file from the .capture file - Cleanup(generate_run(opts)); - - } else if (opts->replay_file && opts->init && !opts->replay) { - // We are going to initialise the test from the .replay file! - Cleanup(init_run(opts)); - - } else if (opts->replay_file && opts->init && opts->replay) { - // We are going to initialise the test and run the test! Run the - // initialiser in a sub-process, as it drops root privileges! - pid_t pid = fork(); - if (pid == 0) { - Cleanup(init_run(opts)); - } else { - opts->drop_caches = !dont_drop_caches; - int init_status; - waitpid(pid, &init_status, 0); - // Only proceed if initialisation was successfull! - Cleanup_unless(SUCCESS, init_status); - Cleanup(replay_run(opts)); - } - - } else if (opts->replay_file && !opts->init && opts->replay) { - // We are going to replay the I/O - opts->drop_caches = !dont_drop_caches; - Cleanup(replay_run(opts)); - - } else { - _print_help(); - Cleanup(ERROR); - } - -cleanup: - options_destroy(opts); - - return ret; -} diff --git a/ioreplay/src/meta/meta.c b/ioreplay/src/meta/meta.c deleted file mode 100644 index d56c17e..0000000 --- a/ioreplay/src/meta/meta.c +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "meta.h" - -#define _MAX_META_LEN 256 - -meta_s* meta_new(FILE *replay_fd) -{ - meta_s *m = Malloc(meta_s); - - m->replay_fd = replay_fd; - m->offset = ftello(replay_fd); - m->read_buf = NULL; - - return m; -} - -void meta_destroy(meta_s *m) -{ - if (!m) - return; - - if (m->read_buf) - free(m->read_buf); - - free(m); -} - -void meta_reserve(meta_s *m) -{ - // TODO: Use a hole in the .replay file to reserve space - char buf[_MAX_META_LEN]; - Mset(&buf, '#', _MAX_META_LEN-1, char); - fprintf(m->replay_fd, "%s\n", buf); -} - -void meta_write_start(meta_s *m) -{ - fseeko(m->replay_fd, m->offset, SEEK_SET); - // Write required '#' so that the regular worker processes - // will ignore that meta line. - fprintf(m->replay_fd, "#"); - - // Required for parsing in 'meta_read_s' - fprintf(m->replay_fd, "|"); -} - -void meta_write_s(meta_s *m, char *key, char *val) -{ - fprintf(m->replay_fd, "%s=%s|", key, val); -} - -void meta_write_l(meta_s *m, char *key, long val) -{ - char buf[1024]; - sprintf(buf, "%ld", val); - fprintf(m->replay_fd, "%s=%ld|", key, val); -} - -void meta_read_start(meta_s *m) -{ - size_t len = 0; - m->read_buf = Calloc(_MAX_META_LEN, char); - getline(&m->read_buf, &len, m->replay_fd); -} - -bool meta_read_s(meta_s *m, char *key, char **val) -{ - char *saveptr = NULL; - char *iterate_buf = Clone(m->read_buf); - int keylen = strlen(key); - - char *tok = strtok_r(iterate_buf, "|", &saveptr); - - while (tok) { - if (strncmp(tok, key, keylen) == 0 && tok[keylen] == '=') { - asprintf(val, "%s", tok+keylen+1); - free(iterate_buf); - return true; - } - tok = strtok_r(NULL, "|", &saveptr); - } - - free(iterate_buf); - return false; -} - -bool meta_read_l(meta_s *m, char *key, long *val) -{ - char *buf = NULL; - - if (meta_read_s(m, key, &buf)) { - *val = atol(buf); - free(buf); - return true; - } - - return false; -} diff --git a/ioreplay/src/meta/meta.h b/ioreplay/src/meta/meta.h deleted file mode 100644 index 10002cc..0000000 --- a/ioreplay/src/meta/meta.h +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef META_H -#define META_H - -#include "../defaults.h" - -/** - * @brief The meta information definition - * - * This is used to write or read meta information to/from the header - * of the .replay file. This information then is used by ioreplay - * in other steps. E.g. reading the amount of used file descriptors - * from the meta header in order to allocate data structures of the - * correct sizes before running the test! - */ -typedef struct meta_s_ { - FILE* replay_fd; /**< The FS of the .replay file */ - off_t offset; /**< The meta offset (usually 0) */ - char* read_buf; /**< Pointer to a read buffer */ -} meta_s; - -/** - * @brief Creates a new meta bject - * - * @return The new meta object - */ -meta_s* meta_new(); - -/** - * @brief Destroys a meta object - * - * @param m The meta object - */ -void meta_destroy(meta_s *m); - -/** - * @brief Reserves space in the .replay file for the meta header - * - * @param m The meta object - */ -void meta_reserve(meta_s *m); - -/** - * @brief Indicates that we start writing the meta header to the .replay file - * - * @param m The meta object - */ -void meta_write_start(meta_s *m); - -/** - * @brief Writes a string to the meta header - * - * @param m The meta object - * @param key The key - * @param val The string value - */ -void meta_write_s(meta_s *m, char *key, char *val); - -/** - * @brief Writes a long to the meta header - * - * @param m The meta object - * @param key The key - * @param val The long value - */ -void meta_write_l(meta_s *m, char *key, long val); - -/** - * @brief indicates that we start reading from the meta header - * - * @param m The meta object - */ -void meta_read_start(meta_s *m); - -/** - * @brief Reads a string from the meta header - * - * @param m The meta object - * @param key The key - * @param val The string val read - */ -bool meta_read_s(meta_s *m, char *key, char **val); - -/** - * @brief Reads a long from the meta header - * - * @param m The meta object - * @param key The key - * @param val The long val read - */ -bool meta_read_l(meta_s *m, char *key, long *val); - -#endif // META_H - diff --git a/ioreplay/src/mounts.c b/ioreplay/src/mounts.c deleted file mode 100644 index afa8376..0000000 --- a/ioreplay/src/mounts.c +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mounts.h" - -#include "utils/futils.h" - -#define _PATH_INSERT "/.ioreplay/" -#define _PATH_INSERT_LEN 11 // strlen of _PATH_INSERT - -void mounts_read(mounts_s *m) -{ - char *mounts = "/proc/mounts"; - size_t len = 0; - char *line = NULL; - char *saveptr = NULL; - - Put("Reading '%s'", mounts); - - FILE *fp = Fopen(mounts, "r"); - Out("Adding supported file systems to replay paths:"); - - while (getline(&line, &len, fp) != -1) { - bool ignore = true; - - char *dev = strtok_r(line, " ", &saveptr); - if (dev == NULL) { - Error("Could not parse device from %s", mounts); - } - - char *mp = strtok_r(NULL, " ", &saveptr); - if (mp == NULL) { - Error("Could not parse mountpoint from %s", mounts); - } - - char *fs = strtok_r(NULL, " ", &saveptr); - if (fs == NULL) { - Error("Could not parse file system from %s", mounts); - } -#ifdef MP_DEBUG - Debug("fs:%s", fs); -#endif - // TODO: Make file system types configurable - if (Eq(fs, "ext2")) { - ignore = false; - } else if (Eq(fs, "ext5")) { - ignore = false; - } else if (Eq(fs, "ext4")) { - ignore = false; - } else if (Eq(fs, "xfs")) { - ignore = false; - } else if (Eq(fs, "zfs")) { - ignore = false; - } else if (Eq(fs, "btrfs")) { - ignore = false; - } - - if (ignore) { - if (strcmp(mp, "/") != 0) { - m->ignore_mps[m->ignore_count] = Clone(mp); - m->ignore_count++; - } - - } else if (m->count >= MAX_MOUNTPOINTS) { - Error("Exceeded max mount points: %d\n", m->count); - - } else { - Out(" %s (%s)", mp, fs); - m->mps[m->count] = Clone(mp); - m->lengths[m->count] = strlen(mp); - m->count++; - } - } - - fclose(fp); - Out("\n"); -} - -mounts_s *mounts_new(options_s *opts) -{ - mounts_s *m = Malloc(mounts_s); - - m->opts = opts; - m->count = 0; - m->ignore_count = 0; - mounts_read(m); - - return m; -} - -void mounts_destroy(mounts_s *m) -{ - if (!m) - return; - for (int i = 0; i < m->count; i++) - free(m->mps[i]); - free(m); -} - -void mounts_trash(mounts_s *m) -{ - options_s *opts = m->opts; - set_limits_drop_root(opts->user); - Put("Moving all old files to trash (of previous tests)..."); - - struct timeval tv; - gettimeofday(&tv, NULL); - - char *wd_path = NULL; - asprintf(&wd_path, "%s/%s", opts->wd_base, opts->name); - - char *trash_path = NULL; - asprintf(&trash_path, "%s/.trash/%ld", opts->wd_base, tv.tv_sec); - - if (is_dir(wd_path)) { - ensure_dir_exists(trash_path); - chown_path(opts->user, trash_path); - if (rename(wd_path, trash_path)) { - Errno("Could not move '%s' to '%s'", wd_path, trash_path); - } - } - free(wd_path); - free(trash_path); - - for (int i = 0; i < m->count; i++) { - char *mp = m->mps[i]; - char *path = NULL; - asprintf(&path, "%s/%s/%s", mp, _PATH_INSERT, opts->name); - asprintf(&trash_path, "%s/%s/.trash/%ld", - mp, _PATH_INSERT, tv.tv_sec); - - if (is_dir(path)) { - ensure_dir_exists(trash_path); - chown_path(opts->user, trash_path); - if (rename(path, trash_path)) { - Errno("Could not move '%s' to '%s'", path, trash_path); - } - } - - free(path); - free(trash_path); - } - - Put("Done trashing!"); - Put("Once the drives fill up you may want to purge old data (-P)"); -} - -void mounts_purge(mounts_s *m) -{ - options_s *opts = m->opts; - set_limits_drop_root(opts->user); - - Out("Purging all data from the following directories:"); - - int active_purgers = 0, max_purgers = 16; - if (opts->num_workers > max_purgers) - max_purgers = opts->num_workers; - - char *purge_path = NULL; - asprintf(&purge_path, "%s", opts->wd_base); - if (is_dir(purge_path)) { - Out(" %s", purge_path); - pid_t pid = fork(); - - if (pid == 0) { - ensure_dir_empty(purge_path); - free(purge_path); - exit(0); - - } else if (pid < 0) { - Errno("\nUnable to create cleaner process! :'-("); - } - active_purgers++; - } - free(purge_path); - - int cleaner_status = SUCCESS; - - for (int i = 0; i < m->count; i++) { - char *mp = m->mps[i]; - char *purge_path = NULL; - asprintf(&purge_path, "%s/%s", mp, _PATH_INSERT); - - if (is_dir(purge_path)) { - if (active_purgers+1 >= max_purgers) { - wait(&cleaner_status); - active_purgers--; - } - - // TODO: Use threading model same way as in init/init.c - pid_t pid = fork(); - if (pid == 0) { - Out(" %s", purge_path); - ensure_dir_empty(purge_path); - free(purge_path); - exit(0); - } else if (pid < 0) { - Errno("Unable to create cleaner process! :'-("); - } - active_purgers++; - } - free(purge_path); - } - - while (wait(&cleaner_status) > 0) - active_purgers--; - Put("\nCleaning done!"); -} - -void mounts_init(mounts_s *m) -{ - options_s *opts = m->opts; - char *wd_path = NULL; - asprintf(&wd_path, "%s/%s", opts->wd_base, opts->name); - ensure_dir_exists(wd_path); - chown_path(opts->user, opts->wd_base); - chown_path(opts->user, wd_path); - - if (chdir(wd_path)) { - Errno("Could not chdir into '%s'!", wd_path); - - } else { - Put("Chdir into '%s'", wd_path); - } - - free(wd_path); - - for (int i = 0; i < m->count; i++) { - char *mp = m->mps[i]; - char *path = NULL; - - // Create .ioreplay/ directory on MP - asprintf(&path, "%s/%s", mp, _PATH_INSERT); - ensure_dir_exists(path); - chown_path(m->opts->user, path); - free(path); - path = NULL; - - // Create .ioreplay/NAME directory on MP - asprintf(&path, "%s/%s/%s", mp, _PATH_INSERT, opts->name); - ensure_dir_exists(path); - chown_path(m->opts->user, path); - free(path); - } -} - -bool mounts_ignore_path(mounts_s *m, const char *path) -{ - // CentOS 7 specific, ignore temp namespace mounts! - char *pos = strstr(path, "/tmp/namespace-"); - if (pos == path) - return true; - - // iterate backwards through all mount points. - for (int i = m->ignore_count-1; i >= 0; --i) { - char *mountpoint = m->ignore_mps[i]; - pos = strstr(path, mountpoint); - // Ignore this path as it is in the ignore mp list - if (pos == path) - return true; - } - - return false; -} - -bool mounts_transform_path(mounts_s *m, const char *name, - char *path, char **path_r) -{ - char *tmp = NULL; -#ifdef DEBUG_TRANSFORM_PATH - char *original_path = path; -#endif - bool line_ok = true; - - // First figure out whether there are '..' in any paths. If so we have to - // tokenize the path and remove '..'. Example: - // transform '/foo/bar/../' into '/foo/'. - // Also remove double '/' from paths. - - if (strstr(path, "..") || strstr(path, "//")) { - // tmp will be freed under label 'cleanup' at end of function. - tmp = Calloc(strlen(path)+1, char); - - // stack to put the tokens on - stack_s *s = stack_new(); - - // we need a copy of the path, so we can tokenize it into the stack - char* clone = Clone(path); - - char *saveptr = NULL; - char *tok = strtok_r(clone, "/", &saveptr); - - // Add each part of the path to the stack. - while (tok) { - if (strcmp(tok, "..") == 0) { - stack_pop(s); - } else { - stack_push(s, tok); - } - tok = strtok_r(NULL, "/", &saveptr); - } - - if (stack_is_empty(s)) { - strcpy(tmp, "."); - - } else { - s = stack_new_reverse_from(s); - strcpy(tmp, "/"); - strcat(tmp, (char*)stack_pop(s)); - - while(!stack_is_empty(s)) { - strcat(tmp, "/"); - strcat(tmp, (char*)stack_pop(s)); - } - } - - stack_destroy(s); - free(clone); - - // This is the path without '..' and '//' (and '///' ... etc') - path = tmp; - } - - // Now heck whether the path is on a supported file system. If not, ignore! - if (mounts_ignore_path(m, path)) { - line_ok = false; - goto cleanup; - } - - // So the path is on a valid mount point! Now we need to insert - // .ioreplay/NAME to each mount point, e.g. /usr/local/.ioreplay/NAME/... - - // Iterate backwards through all mount points. - for (int i = m->count-1; i >= 0; --i) { - char *mountpoint = m->mps[i]; - int mp_len = m->lengths[i]; - - if (strncmp(path, mountpoint, mp_len) == 0) { - // Found a path to replace - // Now insert .ioreplay/NAME/ into the file path. - *path_r = Calloc(strlen(path) + strlen(name)+1 - + _PATH_INSERT_LEN+1, char); - - if (strcmp(mountpoint, "/") == 0) { - // Root path - strcpy(*path_r, _PATH_INSERT); - strcat(*path_r, name); - strcat(*path_r, path); - - } else { - strcpy(*path_r, mountpoint); - strcat(*path_r, _PATH_INSERT); - strcat(*path_r, name); - char *pos = path; - pos += mp_len * (int) sizeof(char); - strcat(*path_r, pos); - } - - goto cleanup; - } - } - - if (tmp) - free(tmp); - - return line_ok; - -cleanup: -#ifdef DEBUG_TRANSFORM_PATH - Debug("Transform path '%s' -> '%s' -> '%s'", original_path, path, *path_r); -#endif - if (tmp) - free(tmp); - - return line_ok; -} - -int mounts_get_mountnumber(mounts_s *m, const char *path) -{ - for (int i = m->count-1; i >= 0; --i) { - char *mountpoint = m->mps[i]; - int mp_len = m->lengths[i]; - - if (strncmp(path, mountpoint, mp_len) == 0) - return i; - } - - return 0; -} diff --git a/ioreplay/src/mounts.h b/ioreplay/src/mounts.h deleted file mode 100644 index a644ddb..0000000 --- a/ioreplay/src/mounts.h +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MOUNTPOINTS_H -#define MOUNTPOINTS_H - -#include "datas/stack.h" -#include "defaults.h" -#include "options.h" - -#define MAX_MOUNTPOINTS 1024 - -/** - * @brief Represents data parsed from /proc/mounts - * - * This is used to determine the file systems and the file system types - * currently mounted on the Linux system. I/O replay only replays I/O - * on specific file systems such as ext4 or xfs and will ignore any special - * or pseudo file systems such as tmpfs, devfs, sysfs. It does not make sense - * to replay I/O on these because there is actually no underlying block device - * attached to these. - * - * A mounts object helps to determine whether a path relies on a valid file - * system or not. All I/O operations on invalid file systems are being filtered - * out! - * - * The mounts object also does more things such as purging temp test data from - * the mountpoints etc... - */ -typedef struct mounts_s_ { - int count; /**< The amount of mount points */ - char *mps[MAX_MOUNTPOINTS]; /**< The mp paths */ - int lengths[MAX_MOUNTPOINTS]; /**< The mp lenghts */ - int ignore_count; /**< The amount of ignored mount points */ - char *ignore_mps[MAX_MOUNTPOINTS]; /**< The ignored mp paths */ - options_s *opts; /**< A pointer to the options object */ -} mounts_s; - -/** - * @brief Creates a new mounts object - * - * @param opts The options object - * @return The new mounts object - */ -mounts_s *mounts_new(options_s *opts); - -/** - * @brief Destroys the mounts object - * - * @param m The mounts object - */ -void mounts_destroy(mounts_s *m); - -/** - * @brief moves all files within replay mounts to trash - * - * It moves all files of the .ioreplay/NAME directories to - * .ioreplay/NAME.trashEPOCH directories for all available mount points. - * It does the same for the working dorectory of the current test. - * - * @param m The responsible mounts object - */ -void mounts_trash(mounts_s *m); - -/** - * @brief Deletes all files within replay mounts - * - * It deletes all files from the .ioreplay/ directories for all availabe - * mount points. It also deletes the working directory of all tests. The - * function forks one sub-process per mount point, so it is cleaning all drives - * in parallel. - * - * It can take a significant amount of time to actually delete all these files. - * That's why there is also a mounts_trash function, which will not delete the - * files but move them to trash folders so they can be deleted at a later - * point. - * - * @param m The responsible mounts object - */ -void mounts_purge(mounts_s *m); - -/** - * @brief Ensures all mounts have a .ioreplay/NAME directory - * - * These directories are used by ioreplay to run the I/O replay tests in. - * The function also ensures to have the correct user permissions for these - * directories. - * - * @param m The responsible mounts object - */ -void mounts_init(mounts_s *m); - -/** - * @brief Reads /proc/mounts to determine which mounts are available - * - * @param m The mounts object - */ -void mounts_read(mounts_s *m); - -/** - * @brief Determines whether a path should be ignored - * - * ioreplay replays I/O only on known mount points of known - * file system types. This function helps to determine whether - * a path is on a valid mount point or not. - * - * @param m The responsible mounts object - * @param path The path to check - * @return true if path has to be ignored - */ -bool mounts_ignore_path(mounts_s *m, const char *path); - -/** - * @brief Inserts ./ioreplay/NAME into a path - * - * This function inserts ./ioreplay/NAME into a given file path. - * The function also checks whether the path is on a supported replay - * path or not. E.g. we want to ignore file systems such as devfs, sysfs, - * procfs.. etc. - * - * @param m The responsible mountpoint object - * @param name The name of the test - * @param path The original path - * @param path_r The tansformed path (has to be freed if not NULL) - * @return False if this path is to be ignored - */ -bool mounts_transform_path(mounts_s *m, const char *name, - char *path, char **path_r); - - -/** - * @brief Get's the mount point number of a path - * - * Used by init.c to determine which thread to use to initialise a file - * or directory on a given path. - * - * @param m The responsible mountpoint object - * @param path The file/directory path - * @return The mountpoint number - */ -int mounts_get_mountnumber(mounts_s *m, const char *path); - -#endif // MOUNTPOINTS_H diff --git a/ioreplay/src/opcodes.h b/ioreplay/src/opcodes.h deleted file mode 100644 index 3d5c114..0000000 --- a/ioreplay/src/opcodes.h +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef OPCODES_H -#define OPCODES_H - -typedef enum { - // stat() syscalls - FSTAT = 0, - FSTAT_AT, - FSTATFS, - FSTATFS64, - LSTAT, - STAT, - STATFS, - STATFS64, - - // read() syscalls - READ = 10, - READV, - READAHEAD, - READDIR, - READLINK, - READLINK_AT, - - // write() syscalls - WRITE = 20, - WRITEV, - - // open() and other syscalls which may create files - OPEN = 30, - OPEN_AT, - CREAT, - MKDIR, - MKDIR_AT, - NAME_TO_HANDLE_AT, - OPEN_BY_HANDLE_AT, - - // rename() syscalls - RENAME = 40, - RENAME_AT, - RENAME_AT2, - - // close() and unlink() syscalls - CLOSE = 50, - UNLINK, - UNLINK_AT, - RMDIR, - - // sync() syscalls - FSYNC = 60, - FDATASYNC, - SYNC, - SYNCFS, - SYNC_FILE_RANGE, - - // other syscalls - FCNTL = 70, - GETDENTS, - LSEEK, - - // mmap syscalls - MMAP2 = 80, - MUNMAP, - REMAP, - MSYNC, - - // chmod() syscalls - CHMOD = 100, - FCHMOD, - FCHMODAT, - - // chown() syscalls - CHOWN = 110, - CHOWN16, - LCHOWN, - LCOWN16, - FCHOWN, - FCHOWN16, - FCHOWNAT, - - // Meta operations (I/O replay internal use only) - // A single thread terminates - META_EXIT = 900, - // All threads of a process termiate (process termination) - META_EXIT_GROUP, - // Meta operation for lamport synchronisation (currently unused) - META_TIMELINE - -} opcode_e; - -#endif // OPCODES_H diff --git a/ioreplay/src/options.c b/ioreplay/src/options.c deleted file mode 100644 index c1dcdb9..0000000 --- a/ioreplay/src/options.c +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "options.h" - -options_s *options_new() -{ - options_s *o = Malloc(options_s); - - o->capture_file = NULL; - o->replay_file = NULL; - o->stats_file = NULL; - o->wd_base = "/usr/local/ioreplay"; - o->num_workers = 4; - o->num_threads_per_worker = 128; - o->user = "mcuser"; - o->name = "test0"; - o->init = false; - o->replay = false; - o->speed_factor = 0; - o->drop_caches = false; - o->purge = false; - o->trash = false; - o->pid = -1; - o->module = "ioreplay.ko"; - - return o; -} - -void options_destroy(options_s *o) -{ - if (o->capture_file) - free(o->capture_file); - if (o->replay_file) - free(o->replay_file); - if (o->stats_file) - free(o->stats_file); - - free(o); -} diff --git a/ioreplay/src/options.h b/ioreplay/src/options.h deleted file mode 100644 index 66cb0f7..0000000 --- a/ioreplay/src/options.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef OPTIONS_H -#define OPTIONS_H - -#include <stdbool.h> -#include "defaults.h" - -/** - * @brief The options definition, used to store user input - */ -typedef struct options_s_ { - char *capture_file; /**< The name of the .capture file */ - char *replay_file; /**< The name of the .replay file */ - char *stats_file; /**< The name of the .stats file */ - bool write_stats_file; /**< Write a stats file at the end of the test */ - char *user; /**< The user name to run the test as */ - char *name; /**< The name of the test (found in .ioreplay/name sub-dirs) */ - char *wd_base; /**< The working directory base */ - int num_workers; /**< The amount of worker processes */ - int num_threads_per_worker; /**< Max threads per worker processes */ - bool init; /**< If set ioreplay will initialise the environment */ - bool replay; /**< If set ioreplay will run/replay the test */ - bool purge; /**< If set ioreplay will purge the environment */ - bool trash; /**< If set ioreplay will trash the environment */ - bool drop_caches; /**< True if ioreplay should drop all Linux caches */ - double speed_factor; /**< Specifies how fast the test is replayed */ - int pid; /**< Specifies a process id to capture */ - char *module; /**< Specifies the kernel module for capturing */ -} options_s; - -/** - * @brief Creates a new options object - * - * The options object contains all options specified by the user as a command - * line option. It is filled with default values during creation. - * - * @return The options object - */ -options_s *options_new(); - -/** - * @brief Destroys the options object - * - * @param o The options object - */ -void options_destroy(options_s *o); - -#endif // OPTIONS_H diff --git a/ioreplay/src/replay/replay.c b/ioreplay/src/replay/replay.c deleted file mode 100644 index e4606d1..0000000 --- a/ioreplay/src/replay/replay.c +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "replay.h" - -#include "../datas/amap.h" -#include "../meta/meta.h" -#include "../mounts.h" -#include "rworker.h" -#include "rstats.h" - -void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes, - long *num_pids, long *num_fds, long *num_lines) -{ - meta_s *m = meta_new(replay_fd); - meta_read_start(m); - - long version = 0; - if (meta_read_l(m, "version", &version)) { - Put("Replay version is '%ld'", version); - if (version != REPLAY_VERSION) { - Error(".replay file of incompatible version, got %x, expected %x", - (int)version, REPLAY_VERSION); - } - } - - char *user; - if (meta_read_s(m, "user", &user)) { - Put("Setting user to '%s'", user); - opts->user = user; - } - - char *name; - if (meta_read_s(m, "name", &name)) { - Put("Setting name to '%s'", name); - opts->name = name; - } - - if (meta_read_l(m, "num_vsizes", num_vsizes)) { - if (*num_vsizes < 0) { - Error("Lamport vsize overflow"); - } - Put("Setting num of vsizes to '%ld'", *num_vsizes); - } - - if (meta_read_l(m, "num_mapped_pids", num_pids)) { - if (*num_pids < 0) { - Error("Process overflow (too many process IDs in .replay)"); - } - Put("Setting num of PIDs to '%ld'", *num_pids); - } - - if (meta_read_l(m, "num_mapped_fds", num_fds)) { - if (*num_fds < 0) { - Error("FD overflow (too many FDs in .replay)"); - } - Put("Setting num of FDs to '%ld'", *num_fds); - } - - if (meta_read_l(m, "num_lines", num_lines)) { - if (*num_fds < 0) { - Error("Overflow (too many lines in .replay)"); - } - Put("Setting num of lines to '%ld'", *num_lines); - } - - meta_destroy(m); -} - -status_e replay_run(options_s *opts) -{ - status_e status = SUCCESS; - - if (opts->drop_caches) { - drop_caches(); - //cache_file(opts->replay_file); - } - - // Extract information from the meta header - FILE *replay_fd = Fopen(opts->replay_file, "r"); - long num_vsizes = 0, num_pids = 0, num_fds = 0, num_lines = 0; - replay_extract_header(opts, replay_fd, &num_vsizes, &num_pids, - &num_fds, &num_lines); - fclose(replay_fd); - - // A map of all file descriptors used. - Out("Creating FD map..."); - amap_s *fds_map = NULL; - if (opts->num_workers > 1) { - fds_map = amap_new_mmapped(num_fds); - } else { - fds_map = amap_new(num_fds); - } - Put("done"); - - // To collect all individual worker's stats into the global - // stats object. - stack_s *all_worker_stats = stack_new(); - - // The global stats object - rstats_s *stats = rstats_new(opts); - rstats_start(stats); - - // Fork worker processes, each worker process will read the .replay file - // individually. - - if (opts->num_workers > 1) { - for (int i = 0; i < opts->num_workers; ++i) { - rworker_stats_s *worker_stats = rworker_stats_new_mmap(); - stack_push(all_worker_stats, worker_stats); - - pid_t pid = fork(); - - if (pid == 0) { - // One worker object per fork - rworker_s *w = rworker_new(i, fds_map, num_vsizes, num_pids, opts, - worker_stats); - - // Process the .replay journal line by line - status_e status = rworker_process_lines(w, num_lines); - Put("worker(%d): Exiting from %d with status %d", i, - pid, status); - rworker_destroy(w); - - // Exit sub-process - exit(status); - - } else if (pid < 0) { - Errno("worker(%d): Unable to create worker process! :'-(", i); - - } else { - Put("worker(%d): Process with pid %d forked", i, pid); - } - } - - set_limits_drop_root(opts->user); - - Put("Waiting for worker processes to finish"); - pid_t pid; - int rworker_status = SUCCESS; - - while ((pid = wait(&rworker_status)) > 0) { - if (rworker_status != SUCCESS) - status = rworker_status; - - Put("Process with pid %d exited with status %d", - pid, rworker_status); - } - - Put("All workers finished (%d)!", status); - - } else { - Put("Only one worker, don't fork sub-processes"); - - rworker_stats_s *worker_stats = rworker_stats_new_mmap(); - stack_push(all_worker_stats, worker_stats); - - rworker_s *w = rworker_new(0, fds_map, num_vsizes, num_pids, - opts, worker_stats); - status = rworker_process_lines(w, num_lines); - rworker_destroy(w); - - Put("Worker finished work!"); - } - - // Collect all statistics - rstats_stop(stats); - while (!stack_is_empty(all_worker_stats)) { - rworker_stats_s *worker_stats = stack_pop(all_worker_stats); - rstats_add_from_worker(stats, worker_stats); - rworker_stats_destroy(worker_stats); - } - stack_destroy(all_worker_stats); - - rstats_print(stats); - rstats_destroy(stats); - - amap_destroy(fds_map); - return status; -} diff --git a/ioreplay/src/replay/replay.h b/ioreplay/src/replay/replay.h deleted file mode 100644 index dcc3d84..0000000 --- a/ioreplay/src/replay/replay.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef REPLAY_H -#define REPLAY_H - -#include "../defaults.h" -#include "../utils/futils.h" -#include "../opcodes.h" -#include "../options.h" -#include "rioop.h" -#include "rprocess.h" - -/** - * @brief Replays the given .replay file - * - * @param opts The options object - * @return SUCCESS if everything went fine - */ -status_e replay_run(options_s *opts); - -/** - * @brief Extract required meta data from .replay's meta header - * - * @param opts The options object - * @param replay_fd The file handle to the .replay file - * @param num_vsizes The amount of virtual sizes/paths - * @param num_pids The amount of process IDs - * @param num_fds The amount of virtual file descriptors - * @param num_lines The amount of .replay lines with I/O ops - */ -void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes, - long *num_pids, long *num_fds,long *num_lines); - -#endif // REPLAY_H diff --git a/ioreplay/src/replay/rioop.c b/ioreplay/src/replay/rioop.c deleted file mode 100644 index 2e16c94..0000000 --- a/ioreplay/src/replay/rioop.c +++ /dev/null @@ -1,425 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rioop.h" - -#include "../vfd.h" -#include "rworker.h" - -// Printing error messages -#define _Error(...) \ - fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\nlineno:%ld path:%s\n", task->lineno, vfd->path); \ - fflush(stdout); \ - fflush(stderr); \ - exit(ERROR); - -#define _Errno(...) \ - fprintf(stderr, "%s:%d ERROR: %s (%d). ", __FILE__, __LINE__, \ - strerror(errno), errno); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\nlineno:%ld path:%s\n", task->lineno, vfd->path); \ - fflush(stdout); \ - fflush(stderr); \ - exit(ERROR); - -#define _Init_arg(num) int arg = atoi(task->toks[num]) -#define _Init_cmd(num) int cmd = atoi(task->toks[num]) -#define _Init_fd(num) long fd = atol(task->toks[num]) -#define _Init_flags(num) int flags = atoi(task->toks[num]) -//#define _Init_mode(num) int mode = atoi(task->toks[num]) -#define _Init_offset(num) long offset = atol(task->toks[num]) -#define _Init_op(num) int op = atoi(task->toks[num]) -#define _Init_path2(num) char *path2 = task->toks[num] -#define _Init_path(num) char *path = task->toks[num] -#define _Init_rc(num) int rc = atoi(task->toks[num]) -#define _Init_whence(num) long whence = atol(task->toks[num]) - -#define _Init_bytes(num) \ - int bytes = atoi(task->toks[num]); \ - if (bytes <= 0) return - -#define _Init_virtfd \ - vfd_s *vfd = amap_get(p->fds_map, fd); \ - if (vfd == NULL) return - -void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_op(2); - - switch (op) { - // stat() syscalls - case FSTAT: - rioop_fstat(p, t, task); - break; - case FSTATFS: - case FSTATFS64: - //Error("op(%d) not implemented", op); - break; - case FSTAT_AT: - case LSTAT: - case STAT: - rioop_stat(p, t, task); - break; - case STATFS: - case STATFS64: - //Error("op(%d) not implemented", op); - break; - - // read() syscalls - case READ: - case READV: - rioop_read(p, t, task); - break; - case READAHEAD: - //Error("op(%d) not implemented", op); - break; - case READLINK: - case READLINK_AT: - //Error("op(%d) not implemented", op); - break; - - // write() syscalls - case WRITE: - case WRITEV: - rioop_write(p, t, task); - break; - - // open() and other syscalls which may creat - case OPEN: - case OPEN_AT: - rioop_open(p, t, task, -1); - break; - case CREAT: - // A call to crat() is equivalent to calling open() with flags.. - rioop_open(p, t, task, O_CREAT|O_WRONLY|O_TRUNC); - break; - case MKDIR: - case MKDIR_AT: - rioop_mkdir(p, t, task); - break; - - // rename() syscalls - case RENAME: - case RENAME_AT: - case RENAME_AT2: - rioop_rename(p, t, task); - break; - - // close() and unlink() syscalls - case CLOSE: - rioop_close(p, t, task); - break; - case UNLINK: - case UNLINK_AT: - rioop_unlink(p, t, task); - break; - case RMDIR: - rioop_rmdir(p, t, task); - break; - - // sync() syscalls - case FSYNC: - rioop_fsync(p, t, task); - break; - case FDATASYNC: - rioop_fdatasync(p, t, task); - break; - case SYNC: - case SYNCFS: - case SYNC_FILE_RANGE: - //Error("op(%d) not implemented", op); - break; - - // Other syscalls - case FCNTL: - rioop_fcntl(p, t, task); - break; - case GETDENTS: - rioop_getdents(p, t, task); - break; - case LSEEK: - rioop_lseek(p, t, task); - break; - - // chmod() syscalls - case CHMOD: - rioop_chmod(p, t, task); - break; - case FCHMOD: - rioop_fchmod(p, t, task); - break; - - // chown() syscalls - case CHOWN: - rioop_chown(p, t, task); - break; - case FCHOWN: - case FCHOWNAT: - rioop_fchown(p, t, task); - break; - case LCHOWN: - rioop_lchown(p, t, task); - break; - - // Meta operations (I/O replay internal use only). - case META_EXIT_GROUP: - break; - case META_TIMELINE: - break; - - default: - Error("op(%d) not implemented", op); - break; - } -} - -void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - struct stat buf; - stat(path, &buf); -} - -void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - struct stat buf; - fstat(vfd->fd, &buf); -} - -void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - _Init_path2(4); - rename(path, path2); -} - -void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_bytes(4); - _Init_virtfd; - - char *buf = Calloc(bytes+1, char); - read(vfd->fd, buf, bytes); - free(buf); -} - -void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_bytes(4); - _Init_virtfd; - - char *buf = Calloc(bytes+1, char); - sprintf(buf, "%ld", task->lineno); - Fill_with_stuff(buf, bytes); - if (vfd->fd == 0) { - Debug("%d %d %ld", vfd->fd, vfd->debug, task->lineno); - _Error("ERROR"); - } - write(vfd->fd, buf, bytes); - free(buf); -} - -void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_) -{ - _Init_fd(3); - _Init_path(4); - _Init_flags(6); - - // Special case as this is creat() now - if (flags_ != -1) - flags = flags_; - - bool directory = Has(flags, O_DIRECTORY); - - if (fd > 0) { - if (directory) { - // We can not open a directory via open() otherwise! - flags &= (O_RDONLY & ~(O_RDWR|O_WRONLY|O_CREAT)); - } else { - // We don't want to open the file in read only mode. - // SystemTap could have skipped syscalls to fcntl or open - flags &= ~O_RDONLY; - } - // flags |= O_DIRECT|O_SYNC; - flags &= ~O_EXCL; - } - - int ret = open(path, flags, S_IRWXU|S_IRWXG|S_IRWXO); - - if (fd < 0 && ret > 0) { - close(ret); -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "TRACE OPEN|open+close|%s|\n", path); - fflush(t->rthread_fd); -#endif - } - - if (fd > 0 && ret > 0) { - vfd_s *vfd = vfd_new(ret, fd, path); - amap_set(p->fds_map, fd, vfd); - -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "TRACE OPEN|open|%s|\n", path); - fflush(t->rthread_fd); -#endif - } -} - -void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - - amap_unset(p->fds_map, fd); - if (vfd->dirfd) { - closedir(vfd->dirfd); -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "TRACE OPEN|closedir|%s|\n", vfd->path); - fflush(t->rthread_fd); -#endif - } else { - close(vfd->fd); -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "TRACE OPEN|close|%s|\n", vfd->path); - fflush(t->rthread_fd); -#endif - } - vfd_destroy(vfd); -} - -void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - - // getdents expects a dirfd - DIR *dirfd = fdopendir(vfd->fd); - if (dirfd) { - vfd->dirfd = dirfd; - readdir(dirfd); -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "TRACE OPEN|fdopendir|%s|\n", vfd->path); - fflush(t->rthread_fd); -#endif - } -} - -void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - mkdir(path, S_IRWXU|S_IRWXG|S_IRWXO); -} - -void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - unlink(path); -} - -void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - rmdir(path); -} - -void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_bytes(6); - _Init_virtfd; - lseek(vfd->fd, bytes, SEEK_SET); -} - -void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - fsync(vfd->fd); -} - -void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - fdatasync(vfd->fd); -} - -void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_cmd(4); - _Init_arg(5); - _Init_virtfd; - - switch (cmd) { - case F_GETFD: - case F_GETFL: - fcntl(vfd->fd, cmd); - break; - case F_SETFD: - case F_SETFL: - fcntl(vfd->fd, cmd, arg); - break; - default: - break; - } -} - -void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - chmod(path, S_IRWXU|S_IRWXG|S_IRWXO); -} - -void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - fchmod(vfd->fd, S_IRWXU|S_IRWXG|S_IRWXO); -} - -void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - rworker_s *w = t->worker; - options_s *opts = w->opts; - struct passwd *pwd = getpwnam(opts->user); - chown(path, pwd->pw_uid, -1); -} - -void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - rworker_s *w = t->worker; - options_s *opts = w->opts; - struct passwd *pwd = getpwnam(opts->user); - fchown(vfd->fd, pwd->pw_uid, -1); -} - -void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - rworker_s *w = t->worker; - options_s *opts = w->opts; - struct passwd *pwd = getpwnam(opts->user); - lchown(path, pwd->pw_uid, -1); -} - diff --git a/ioreplay/src/replay/rioop.h b/ioreplay/src/replay/rioop.h deleted file mode 100644 index 4db4284..0000000 --- a/ioreplay/src/replay/rioop.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef RIOOP_H -#define RIOOP_H - -#include "../defaults.h" -#include "../utils/futils.h" -#include "../opcodes.h" -#include "rprocess.h" -#include "rthread.h" - -/** - * @brief Replays the responsible I/O operation of a given task - * - * @param p The virtual replay process object - * @param t The thread object - * @param task The replay task object - */ -void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task); - -void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_); -void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task); - -#endif // RIOOP_H diff --git a/ioreplay/src/replay/rprocess.c b/ioreplay/src/replay/rprocess.c deleted file mode 100644 index 4efd835..0000000 --- a/ioreplay/src/replay/rprocess.c +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rprocess.h" - -rprocess_s* rprocess_new(const int pid, amap_s *fds_map) -{ - rprocess_s *p = Malloc(rprocess_s); - - p->fds_map = fds_map; - p->pid = pid; - p->terminate = 0; - p->lineno = 0; - - return p; -} - -void rprocess_destroy(rprocess_s *p) -{ - if (!p) - return; - free(p); -} diff --git a/ioreplay/src/replay/rprocess.h b/ioreplay/src/replay/rprocess.h deleted file mode 100644 index 739dd89..0000000 --- a/ioreplay/src/replay/rprocess.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef RPROCESS_H -#define RPROCESS_H - -#include "../datas/hmap.h" -#include "../datas/amap.h" -#include "../defaults.h" -#include "rthread.h" - -/** - * @brief The virtual replay process object definition - * - * This defines a virtual process in replay context. - */ -typedef struct rprocess_s_ { - int terminate; /**< Indicates whether the worker is terminating or not */ - int rworker_num; /**< The worker number of the responsible worker */ - int pid; /**< The virtual process ID */ - unsigned long lineno; /**< Holding the current .replay line number */ - bool initm; /**< Indicates whether ioreplay is in init mode or not */ - amap_s *fds_map; /**< Holding all file descriptors */ -} rprocess_s; - -rprocess_s* rprocess_new(const int pid, amap_s *fds_map); -void rprocess_destroy(rprocess_s* p); - -#endif // RPROCESS_H diff --git a/ioreplay/src/replay/rstats.c b/ioreplay/src/replay/rstats.c deleted file mode 100644 index c3e6e38..0000000 --- a/ioreplay/src/replay/rstats.c +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rstats.h" - -#include <sys/types.h> - -rstats_s* rstats_new(options_s *opts) -{ - rstats_s *s = Malloc(rstats_s); - - s->opts = opts; - s->loadavg_high = 0; - s->ioops = 0; - s->duration = 0; - s->time_ahead = -1; - - if (opts->stats_file) - s->stats_fd = Fopen(opts->stats_file, "w"); - else - s->stats_fd = stdout; - - return s; -} - -void rstats_destroy(rstats_s *s) -{ - if (s->stats_fd != stdout) - fclose(s->stats_fd); - - free(s); -} - -rworker_stats_s* rworker_stats_new_mmap(options_s *opts) -{ - // Share this object between processes, so that the stats cann be - // collected by the master process! - rworker_stats_s *s = Mmapshared(rworker_stats_s); - - s->loadavg_high = 0; - s->ioops = 0; - s->time_ahead = -1; - - return s; -} - -void rworker_stats_destroy(rworker_stats_s *s) -{ - munmap(s, sizeof(rworker_stats_s)); -} - - -void rstats_start(rstats_s* s) -{ - gettimeofday(&s->start_time, NULL); -} - -void rstats_stop(rstats_s* s) -{ - gettimeofday(&s->end_time, NULL); - s->duration= ((s->end_time.tv_sec - s->start_time.tv_sec) * 1000 - + (s->end_time.tv_usec - s->start_time.tv_usec) / 1000) / 1000; - -} - -void rstats_add_from_worker(rstats_s* s, rworker_stats_s* w) -{ - if (s->loadavg_high < w->loadavg_high) - s->loadavg_high = w->loadavg_high; - - if (s->time_ahead == -1 || s->time_ahead > w->time_ahead) - s->time_ahead = w->time_ahead; - - s->ioops += w->ioops; -} - -void rstats_print(rstats_s* s) -{ - options_s *opts = s->opts; - - if (opts->stats_file) { - Put("Writing stats to '%s'", opts->stats_file); - } - - fprintf(s->stats_fd, "Stats of test '%s':\n", opts->name); - fprintf(s->stats_fd, "\tNum workers: %d\n", opts->num_workers); - fprintf(s->stats_fd, "\tThreads per worker: %d\n", opts->num_threads_per_worker); - fprintf(s->stats_fd, "\tThreads total: %d\n", - opts->num_threads_per_worker * opts->num_workers); - fprintf(s->stats_fd, "\tHighest loadavg: %.2f\n", s->loadavg_high); - fprintf(s->stats_fd, "\tPerformed ioops: %ld\n", s->ioops); - if (s->duration > 0) - fprintf(s->stats_fd, "\tAverage ioops/s: %.2f\n", s->ioops/s->duration); - fprintf(s->stats_fd, "\tTime ahead: %lds\n", s->time_ahead/1000); - fprintf(s->stats_fd, "\tTotal time: %.2fs\n", s->duration); -} - diff --git a/ioreplay/src/replay/rstats.h b/ioreplay/src/replay/rstats.h deleted file mode 100644 index 1ce3f27..0000000 --- a/ioreplay/src/replay/rstats.h +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * @file rstats.h - * @author Paul Buetow - * - * @brief For collecting replay stats - */ - -#ifndef RSTATS_H -#define RSTATS_H - -#include "../defaults.h" -#include "../options.h" - -#include <pthread.h> - -/** - * @brief Definition of the rstats object - * - * Used to store global statistics. - */ -typedef struct rstats_s_ { - double loadavg_high; /**< Highest load average */ - long ioops; /**< Total amount if io operations */ - double duration; /**< Duration of the test */ - long time_ahead; /**< Time ahead of the original speed */ - struct timeval start_time; /**< Start time of the test */ - struct timeval end_time; /**< End time of the test */ - options_s *opts; /**< The I/O replay options object */ - FILE *stats_fd; /**< The file descriptor for writing the stats */ -} rstats_s; - -/** - * @brief Definition of the per worker stats object - * - * Used to store per worker process I/O stats - */ -typedef struct rworker_stats_s_ { - double loadavg_high; /**< Highest amount of io ops per second */ - long ioops; /**< Total amount if io operations */ - long time_ahead; /**< Time ahead of the original speed */ -} rworker_stats_s; - -/** - * @brief Creates a new stats object - * - * @return The new stats object - */ -rstats_s* rstats_new(options_s *opts); - -/** - * @brief Destroys the stats object - * - * @param s The stats object - */ -void rstats_destroy(rstats_s* s); - -/** - * @brief Creates a new per worker stats object - * - * The memory is mapped into shared memory so it can be shared across multiple - * processes. - * - * @return The new stats object - */ -rworker_stats_s* rworker_stats_new_mmap(); - -/** - * @brief Destroys the per worker stats object - * - * @param s The stats object - */ -void rworker_stats_destroy(rworker_stats_s* s); - -/** - * @brief Starts the stats - * - * @param s The stats object - */ -void rstats_start(rstats_s* s); - -/** - * @brief Finalises the stats - * - * @param s The stats object - */ -void rstats_stop(rstats_s* s); - -/** - * @brief Prints the stats - * - * @param s The stats object - */ -void rstats_print(rstats_s* s); - -/** - * @brief Adds per worker stats to the global stats object - * - * @param s The global stats object - * @param w The worker stats object - */ -void rstats_add_from_worker(rstats_s* s, rworker_stats_s* w); - -#endif diff --git a/ioreplay/src/replay/rtask.c b/ioreplay/src/replay/rtask.c deleted file mode 100644 index b1afb92..0000000 --- a/ioreplay/src/replay/rtask.c +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rtask.h" - -#include "rthread.h" -#include "rworker.h" - -rtask_s* rtask_new() -{ - rtask_s *task = Malloc(rtask_s); - - *task = (rtask_s) { - .worker = NULL, .process = NULL - }; - task->line[0] = '\0'; - -#ifdef THREAD_DEBUG - task->clone = NULL; -#endif - - return task; -} - -void rtask_destroy(rtask_s *task) -{ - if (task) - free(task); -} - -void rtask_update(rtask_s *task, void *worker, void *process, char *line, - const long lineno, const long vsize) -{ - task->worker = worker; - task->process = process; - task->lineno = lineno; - task->vsize = vsize; - strcpy(task->line, line); -} diff --git a/ioreplay/src/replay/rtask.h b/ioreplay/src/replay/rtask.h deleted file mode 100644 index 35c5714..0000000 --- a/ioreplay/src/replay/rtask.h +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef RTASK_H -#define RTASK_H - -#include "../defaults.h" - -/** - * @brief The replay task definition - * - * The rtask holds all possible variables required to process a particular - * .replay line and to replay the corresponding I/O operation. - */ -typedef struct rtask_s_ { - void *worker; /* The responsible worker object */ - void *process; /* The responsible process object */ - unsigned long lineno; /**< The current line number */ - unsigned long vsize; /**< The vsize */ - char *toks[MAX_TOKENS+1]; /**< The tokens parsed from the .replay line */ - char line[MAX_LINE_LEN]; /**< The remaining part of the .replay line */ -#ifdef RTASK_DEBUG - char *clone; /**< Used for debug purposes only */ -#endif -} rtask_s; - -/** - * @brief Creates a new thread task object - * - * This function creates a new thread task object. Such a task object is used - * by the worker to hand over I/O tasks to the corresponding threads. The - * actual I/O work is performed by the threads then. - * - * @return The new thread task object - */ -rtask_s* rtask_new(); - -/** - * @brief Destroys the replay task object - * - * @param t The thread task object to be destroyed - */ -void rtask_destroy(rtask_s* t); - -/** - * @brief Updates a reused/recycle task object - * - * @param task The task object to be updated - * @param worker The responsibe worker object - * @param process The responsible process object - * @param line The remaining line of the .replay file - * @param lineno The current line number of the .replay file - * @param vsize The vsize/path id - */ -void rtask_update(rtask_s *task, void *worker, void *process, char *line, - const long lineno, const long vsize); - -#endif // RTASK_H diff --git a/ioreplay/src/replay/rthread.c b/ioreplay/src/replay/rthread.c deleted file mode 100644 index 55364ec..0000000 --- a/ioreplay/src/replay/rthread.c +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rthread.h" - -#include <sys/types.h> - -#include "rworker.h" -#include "rprocess.h" - -#include "rioop.h" - -#ifdef THREAD_DEBUG -/** - * @brief For debugging purposes only - * - * @param t The responsible thread object - */ -static void _rthread_init_log(rthread_s *t) -{ - rworker_s *w = t->worker; - char *rthread_log = Calloc(1024, char); - snprintf(rthread_log, 1023, "/tmp/ioreplay/worker%d.thread%ld.debuglog", - w->rworker_num, (long)pthread_self()); - - ensure_dir_exists("/tmp/ioreplay"); - t->rthread_fd = Fopen(rthread_log, "a"); - - free(rthread_log); - fprintf(t->rthread_fd, "%ld: DEBUG: Created thread log\n", t->tid); -} -#endif - -void rthread_process_task(rthread_s* t, rtask_s *task, - pid_t pthread_id) -{ - char *next = task->line; - rworker_s *w = (rworker_s*) task->worker; - - // Tokenize the remaining elements of the line. - int ntoks = 0; - char *saveptr; - char *tok = strtok_r(next, "|", &saveptr); - - while (tok) { - if (ntoks > MAX_TOKENS) { - Error("worker(%d) pthread(%d): lineno:%lu, missing newline?", - w->rworker_num, pthread_id, task->lineno); - } - task->toks[ntoks++] = tok; - tok = strtok_r(NULL, "|", &saveptr); - } - // NULL marker (no more token from here) - task->toks[ntoks] = NULL; - -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "%ld(%ld): %s", - t->tid, (long)pthread_self(), task->clone); - fflush(t->rthread_fd); - free(task->clone); - task->clone = NULL; -#endif -#ifndef NO_RIOOP - // Perform the corresponding I/O operation! - rioop_run(task->process, t, task); -#endif - - // Make the task object recyclable/reusable - pthread_mutex_lock(&w->task_buffer_mutex); - if (!rbuffer_insert(w->task_buffer, task)) - // We can't recycle the task object if the buffer is full! - rtask_destroy(task); - pthread_mutex_unlock(&w->task_buffer_mutex); -} - -void *rthread_pthread_start(void *data) -{ - rthread_s* t = (rthread_s*) data; - rworker_s *w = t->worker; - rtask_s *task = NULL; - pid_t pthread_id = pthread_self(); - -#ifdef THREAD_DEBUG - _rthread_init_log(t); -#endif - - do { - while (!rbuffer_has_next(t->tasks) && !t->terminate) - usleep(100); - - while ((task = rbuffer_get_next(t->tasks)) != NULL) - rthread_process_task(t, task, pthread_id); - -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "%ld: DEBUG: Idling\n", t->tid); - fflush(t->rthread_fd); -#endif - - // Tell rworker_s that thread is not doing any work! - int inserted = false; - while (!inserted && !t->terminate) { - if (rbuffer_has_next(t->tasks)) - break; - - usleep(1000); - - if (rbuffer_has_next(t->tasks)) - break; - - // Make the rthread reusable, he is without any tasks - // for some time. - pthread_mutex_lock(&w->rthread_buffer_mutex); - inserted = rbuffer_insert(w->rthread_buffer, t); - pthread_mutex_unlock(&w->rthread_buffer_mutex); - } - -#ifdef THREAD_DEBUG - if (inserted) { - fprintf(t->rthread_fd, "%ld: DEBUG: Added to thread buffer\n", - t->tid); - } else { - fprintf(t->rthread_fd, "%ld: DEBUG: Idling thread recovered\n", - t->tid); - } - fflush(t->rthread_fd); -#endif - - } while (!t->terminate); - -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "%ld: DEBUG: Terminating\n", t->tid); - fflush(t->rthread_fd); -#endif - - // Process the very last tasks - while (NULL != (task = rbuffer_get_next(t->tasks))) - rthread_process_task(t, task, pthread_id); - -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "%ld: DEBUG: Done terminating\n", t->tid); - fflush(t->rthread_fd); -#endif - - return NULL; -} - -rthread_s* rthread_new(const long tid, void *worker) -{ - rthread_s *t = Malloc(rthread_s); - rworker_s *w = worker; - - t->single_threaded = w->opts->num_threads_per_worker == 1; - t->tasks = rbuffer_new(TASK_BUFFER_PER_THREAD); - t->terminate = false; - t->worker = worker; - rthread_update(t, tid); - - if (t->single_threaded) { -#ifdef THREAD_DEBUG - _rthread_init_log(t); -#endif - return t; - } - - start_pthread(&t->pthread, rthread_pthread_start, (void*)t); - return t; -} - -long rthread_update(rthread_s *t, const long tid) -{ - long prev_tid = t->tid; - t->tid = tid; - - return prev_tid; -} - -void rthread_destroy(rthread_s *t) -{ - if (rbuffer_has_next(t->tasks)) { - Error("Didn't expect to have any tasks left!"); - } - rbuffer_destroy(t->tasks); - -#ifdef THREAD_DEBUG - if (t->rthread_fd) - fclose(t->rthread_fd); -#endif - - free(t); -} - -bool rthread_insert_task(rthread_s* t, rtask_s* task) -{ - if (t->single_threaded) { - rthread_process_task(t, task, pthread_self()); - return true; - } - return rbuffer_insert(t->tasks, task); -} - -void rthread_terminate(rthread_s* t) -{ - t->terminate = true; - pthread_join(t->pthread, NULL); -} diff --git a/ioreplay/src/replay/rthread.h b/ioreplay/src/replay/rthread.h deleted file mode 100644 index 9971e49..0000000 --- a/ioreplay/src/replay/rthread.h +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * @file rthread.h - * @author Paul Buetow - * - * @brief The replay thread definitiion - */ - -#ifndef RTHREAD_H -#define RTHREAD_H - -#include "../defaults.h" -#include "../datas/rbuffer.h" -#include "../datas/amap.h" -#include "../vfd.h" -#include "rtask.h" - -#include <pthread.h> - -/** - * @brief Definition of a worker thread - * - * Every worker utilises a set of worker threads in order to parallelise the - * replaying of the I/O! Every thread comes with its own task queue. It is - * filled by the repsonsible worker. - * - * The user can specify the max amount of threads per worker per -t command - * line switch. - */ -typedef struct rthread_s_ { - void *worker; /**< The responsible worker object */ - long tid; /**< The virtual thread id */ - rbuffer_s* tasks; /**< Holds all outstanding tasks */ - bool terminate; /**< True if thread shall terminate */ - bool single_threaded; /**< Worker is single threaded or not */ - pthread_t pthread; /**< We run the tasks in concurrent pthreads */ -#ifdef RTHREAD_DEBUG - FILE *rthread_fd; /**< Used for debugging purposes only */ -#endif -} rthread_s; - -/** - * @brief Creates a new thread object - * - * @param tid The thread ID - * @param worker The worker object managing this thread - * @return The new thread object - */ -rthread_s* rthread_new(const long tid, void *worker); - -/** - * @brief Updates a thread object after recycling it - * - * @param t The thread object - * @param tid The new thread ID - */ -long rthread_update(rthread_s *t, const long tid); - -/** - * @brief Terminates the thread - * - * This function waits (via join) for the pthread to complete all its - * current tasks from the queue. - * - * @param t The thread object - */ -void rthread_terminate(rthread_s* t); - -/** - * @brief Destroys the thread object - * - * @param t The thread object - */ -void rthread_destroy(rthread_s* t); - -/** - * @brief Inserts a task into the threads work queue - * - * Inserts a task into the threads work queue. We use an atomic ring buffer - * data structure for the work queue. The ring buffer does not require any - * mutex locks. - * - * @param t The thread object - * @param task The task to be inserted - * @return Returns true on success, returns false if the task queue is full - */ -bool rthread_insert_task(rthread_s* t, rtask_s* task); - -/** - * @brief Used by the pthread to process a task - * - * In this function the pthread will attempt to process a task. It extracts all - * required information from the task object and invokes the corresponding I/O - * syscalls. - * - * @param t The responsible thread object - * @param task The task object - * @param pthread_id The current pthread id - */ -void rthread_process_task(rthread_s* t, rtask_s *task, pid_t pthread_id); - -/** - * @brief The entry function for the pthreads - * - * @param data The data structure passed to the pthread - * @return The exit code of the pthread. - */ -void *rthread_pthread_start(void *data); - -#endif // RTHREAD_H diff --git a/ioreplay/src/replay/rworker.c b/ioreplay/src/replay/rworker.c deleted file mode 100644 index 0e3fbe9..0000000 --- a/ioreplay/src/replay/rworker.c +++ /dev/null @@ -1,360 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rworker.h" - -#include "../datas/stack.h" -#include "rprocess.h" -#include "rthread.h" - -#define _Compute_current_time(now) \ - (now.tv_sec - start_time.tv_sec) * 1000 \ - + (now.tv_usec - start_time.tv_usec) / 1000 - - -/** - * @brief A callback helper function for destroying all virtual process objects - * - * @param data The process object. - */ -static void _rprocess_destroy_cb(void *data) -{ - rprocess_destroy(data); -} - -rworker_s* rworker_new(const int rworker_num, amap_s *fds_map, - const long num_vsizes, const long num_pids, - options_s *opts, rworker_stats_s *worker_stats) -{ - rworker_s *w = Malloc(rworker_s); - -#ifdef THREAD_DEBUG - char *rworker_log = Calloc(1024, char); - snprintf(rworker_log, 1023, "/tmp/ioreplay/_worker%d.debuglog", - rworker_num); - - w->rworker_fd = Fopen(rworker_log, "a"); - free(rworker_log); - fprintf(w->rworker_fd, "DEBUG: Started worker\n"); -#endif - - w->rworker_num = rworker_num; - w->opts = opts; - w->fds_map = fds_map; - - w->rprocess_map = amap_new(num_pids); - w->rthread_map = amap_new(num_vsizes); - w->task_buffer = rbuffer_new(opts->num_threads_per_worker - *TASK_BUFFER_PER_THREAD); - w->rthread_buffer = rbuffer_new(opts->num_threads_per_worker); - w->worker_stats = worker_stats; - - // Attach a cleanup callback function to the worker map. - w->rprocess_map->data_destroy = _rprocess_destroy_cb; - - pthread_mutex_init(&w->rthread_buffer_mutex, NULL); - pthread_mutex_init(&w->task_buffer_mutex, NULL); - - // TODO: Check in the program whether the ulimit is high enough - // or not! (ulimit -n) - - return w; -} - -/** - * @brief Destroys the object - * - * Destroys the worker object (frees all memory allocated by the worker) - * - * @param w The worker object - */ -void rworker_destroy(rworker_s *w) -{ - if (!w) - return; - - if (w->rprocess_map) - amap_destroy(w->rprocess_map); - if (w->rthread_map) - amap_destroy(w->rthread_map); - - if (w->task_buffer) { - rtask_s *task = NULL; - while (NULL != (task = rbuffer_get_next(w->task_buffer))) - rtask_destroy(task); - rbuffer_destroy(w->task_buffer); - } - - if (w->rthread_buffer) - rbuffer_destroy(w->rthread_buffer); - - pthread_mutex_destroy(&w->task_buffer_mutex); - pthread_mutex_destroy(&w->rthread_buffer_mutex); - -#ifdef THREAD_DEBUG - if (w->rworker_fd) - fclose(w->rworker_fd); -#endif - - free(w); -} - -status_e rworker_process_lines(rworker_s* w, const long num_lines) -{ - Out("worker(%d): Starting to process replay lines\n", w->rworker_num); - - options_s *opts = w->opts; - FILE *replay_fd = Fopen(opts->replay_file, "r"); - - // Drop root privileges, otherwise we may overwrite other system - // files by accident in case of a bug or user error! - set_limits_drop_root(opts->user); - - // Variables required for the time based caluclations - struct timeval now, start_time; - long current_time = 0, stats_time = 0; - gettimeofday(&start_time, NULL); - - // Helper variables required for reading lines - char *line = NULL; - char *next = NULL, *next2 = NULL; - size_t len = 0, read = 0; - - // Helpers required for threading - rthread_s *t = NULL; - stack_s *all_threads = stack_new(); - rworker_stats_s *s = w->worker_stats; - - // More helper variables - //unsigned long lineno = 0, stats_ioop = 0, vsize_id = 0; - unsigned long lineno = 0, vsize_id = 0; - long pid = -1, time = -1; - - // Process the .replay file line by line. - while ((read = getline(&line, &len, replay_fd)) != -1) { - lineno++; - - if (read >= MAX_LINE_LEN) { - Error("line:%lu Exceeded max line len", lineno); - } - - // If the line begins with #: Ignore that line, it contains - // debug or meta information or comments. - - if (line[0] == '#') { - if (line[1] == 'I') { - // We stop replaying I/O once we reach the line '#INIT' - // which incitates the begin of the INIT section. - break; - } - continue; - } - -#ifdef THREAD_DEBUG - char *clone = Clone(line); -#endif - - next = strchr(line, '|'); - Error_if(!next, "lineno:%ld Could not parse time from input file", - lineno); - next[0] = '\0'; - next++; - time = atol(line); - - next2 = strchr(next, '|'); - Error_if(!next2, "Could not parse vsize_id from input file"); - next2[0] = '\0'; - next2++; - vsize_id = atol(next); - - // This worker is not responsible for this line, skip it! - if ((vsize_id % opts->num_workers) != w->rworker_num) { -#ifdef THREAD_DEBUG - free(clone); -#endif - continue; - } - - next = strchr(next2, '|'); - Error_if(!next, "Could not parse PID from input file"); - next[0] = '\0'; - next++; - pid = atol(next2); - - gettimeofday(&now, NULL); - current_time = _Compute_current_time(now); - - // Check whether the user specified a replay speed factor. If so, we - // may need to throttle down a bit. - - if (opts->speed_factor) { - s->time_ahead = time / opts->speed_factor - current_time; - if (s->time_ahead > 0) - usleep(s->time_ahead*1000); - - } else { - s->time_ahead = time - current_time; - } - - // Get the responsible process object. The process object holds data - // structures usually found in a Linux process, e.g. a table of open - // file descriptors. - - rprocess_s *p = amap_get(w->rprocess_map, pid); - if (p == NULL) { - p = rprocess_new(pid, w->fds_map); - amap_set(w->rprocess_map, pid, p); - } - p->lineno = lineno; - - if (opts->num_threads_per_worker == 1) { - // Single threaded mode? - if (!t) - t = rthread_new(vsize_id, w); - else - rthread_update(t, vsize_id); - - } else { - t = amap_get(w->rthread_map, vsize_id); - } - - if (t == NULL) { - - // First try to recycle an old (likely unused) thread - if (NULL != (t = rbuffer_get_next(w->rthread_buffer))) { - rthread_update(t, vsize_id); - -#ifdef THREAD_DEBUG - fprintf(w->rworker_fd, "DEBUG: Reused an idling thread\n"); - fflush(w->rworker_fd); -#endif - - } else if (opts->num_threads_per_worker <= all_threads->size) { - // Reached max threads, waiting until one becomes available - -#ifdef THREAD_DEBUG - fprintf(w->rworker_fd, "DEBUG: Reached max threads\n"); - fflush(w->rworker_fd); -#endif - while (NULL == (t = rbuffer_get_next(w->rthread_buffer))) - usleep(1000); - -#ifdef THREAD_DEBUG - fprintf(w->rworker_fd, "DEBUG: Reused an idling thread\n"); - fflush(w->rworker_fd); -#endif - - rthread_update(t, vsize_id); - - } else { - t = rthread_new(vsize_id, w); - - // We hold a pointer to all created threads in a stack. This - // stack is later used to terminate/join all therads. - stack_push(all_threads, t); - -#ifdef THREAD_DEBUG - fprintf(w->rworker_fd, "DEBUG: Created a new thread\n"); - fflush(w->rworker_fd); -#endif - } - - amap_set(w->rthread_map, vsize_id, t); - } - - // Create a new task for the thread. The task contains all required - // information to run an I/O operation. However, first try to - // reuse/recycle a task object! If there is no such, create a new one. - - rtask_s *task = rbuffer_get_next(w->task_buffer); - if (!task) - task = rtask_new(); - rtask_update(task, w, p, next, lineno, vsize_id); - s->ioops++; - - -#ifdef THREAD_DEBUG - task->clone = clone; - fprintf(w->rworker_fd, "DEBUG: Inserting new task\n"); - fflush(w->rworker_fd); -#endif - - // Insert that task to a ring buffer to pass it to the pthread without - // much synchronisation overhead! - - while (!rthread_insert_task(t, task)) - // The ring buffer is full. This may happen if the pthread didn't - // manage to process tasks fast enough. re-try after a short period! - usleep(1000); - -#ifdef THREAD_DEBUG - fprintf(w->rworker_fd, "DEBUG: Task inserted\n"); - fflush(w->rworker_fd); -#endif - - // The worker prints out stats every 3s - if (current_time - stats_time > 3000) { - // IDEA: Maybe refactor this block to be implemented in rstats.c - - double loadavg = get_loadavg(); - - // Determines whether we replay the I/O faster or slower than - // original speed! - char *a_b = s->time_ahead >= 0 ? "ahead" : "behind"; - - Put("worker(%d): threads:%ld %s:%lds progress:%0.2f%% " - "loadavg:%0.2f", - w->rworker_num, all_threads->size, a_b, Abs(s->time_ahead/1000), - Perc(lineno,num_lines), loadavg); - - stats_time = current_time; - //stats_ioop = lineno; - - if (s->loadavg_high < loadavg) - s->loadavg_high = loadavg; - } - } - - Put("worker(%d): Waiting for all threads to finish business...", - w->rworker_num); - - // This will wait (join) all threads one after another until all threads - // have finished their work and have terminated. - - while (!stack_is_empty(all_threads)) { - rthread_s *t = stack_pop(all_threads); - rthread_terminate(t); - rthread_destroy(t); - } - stack_destroy(all_threads); - - // Collect some stats last time - double loadavg = get_loadavg(); - if (s->loadavg_high < loadavg) - s->loadavg_high = loadavg; - - gettimeofday(&now, NULL); - current_time = _Compute_current_time(now); - if (opts->speed_factor) { - s->time_ahead = time / opts->speed_factor - current_time; - } else { - s->time_ahead = time - current_time; - } - - - Put("worker(%d): All threads terminated!", w->rworker_num); - fclose(replay_fd); - - return SUCCESS; -} diff --git a/ioreplay/src/replay/rworker.h b/ioreplay/src/replay/rworker.h deleted file mode 100644 index 26a1300..0000000 --- a/ioreplay/src/replay/rworker.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef RWORKER_H -#define RWORKER_H - -#include <pthread.h> - -#include "../datas/amap.h" -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "../options.h" -#include "rstats.h" - -/** - * @brief Represents a worker process. - * - * This represents an I/O replay worker process. The user can specify the - * amount of worker processes via the -p command line switch. This is not - * to confuse with rprocess_s, which represents an original captured process - * and we now want to replay the I/O for! - */ -typedef struct { - int rworker_num; /**< The current worker ID */ - amap_s* fds_map; /**< Holding all file descriptors */ - amap_s* rprocess_map; /**< Holding all processes handled by this worker */ - amap_s* rthread_map; /**< Holding all threads handled by this worker */ - rbuffer_s *task_buffer; /**< Buffering thread tasks to be reused */ - pthread_mutex_t task_buffer_mutex; /**< To sync access to task_buffer */ - rbuffer_s *rthread_buffer; /**< Buffering idle threads to be reused */ - pthread_mutex_t rthread_buffer_mutex; /**< Sync access to rthread_buffer */ - options_s *opts; /**< To synchronise access to rthread_buffer */ - rworker_stats_s *worker_stats; /**< Object holding per worker statistics */ -#ifdef RTHREAD_DEBUG - FILE *rworker_fd; /**< For debugging purposes only */ -#endif -} rworker_s; - -/** - * @brief Creates a new worker object - * - * @param rworker_num The worker number - * @param fds_map A map of all virtual file descriptor objects - * @param num_vsizes The amount of virtual sizes/total file paths of the test - * @param num_pids The total amount of virtual process IDs used in this test - * @param opts A pointer to the options object - * @param worker_stats A pointer to the worker stats object - - * @return The new worker object - */ -rworker_s* rworker_new(const int rworker_num, amap_s *fds_map, - const long num_vsizes, const long num_pids, - options_s* opts, rworker_stats_s *worker_stats); - -/** - * @brief Destroys a worker object - * - * @param w The worker object to be destroyed - */ -void rworker_destroy(rworker_s* w); - -/** - * @brief Makes the worker to process all .replay lines - * - * @param w The responsible worker object - * @param num_lines The total amount of I/O op lines in the .replay file - * @return SUCCESS if everything went fine - */ -status_e rworker_process_lines(rworker_s* w, const long num_lines); - -#endif // RWORKER_H diff --git a/ioreplay/src/utests.c b/ioreplay/src/utests.c deleted file mode 100644 index 7cc4959..0000000 --- a/ioreplay/src/utests.c +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "utests.h" - -#include "datas/amap.h" -#include "datas/hmap.h" -#include "datas/list.h" -#include "datas/rbuffer.h" -#include "utils/utils.h" - -void utests_run() -{ - fprintf(stderr, "Running utils_test()\n"); - utils_test(); - - fprintf(stderr, "Running amap_test()\n"); - amap_test(); - - fprintf(stderr, "Running hmap_test()\n"); - hmap_test(); - - fprintf(stderr, "Running list_test()\n"); - list_test(); - - fprintf(stderr, "Running rbuffer_test()\n"); - rbuffer_test(); - - fprintf(stderr, "Great success, run all unit tests without any errors!\n"); -} diff --git a/ioreplay/src/utests.h b/ioreplay/src/utests.h deleted file mode 100644 index 4ad6973..0000000 --- a/ioreplay/src/utests.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef UTESTS_H -#define UTESTS_H - -#include "utils/utils.h" - -/** - * @brief This function runs all currently implemented unit tests - */ -void utests_run(); - -#endif // UTESTS_H diff --git a/ioreplay/src/utils/futils.c b/ioreplay/src/utils/futils.c deleted file mode 100644 index 5b35618..0000000 --- a/ioreplay/src/utils/futils.c +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "futils.h" - -#include <libgen.h> -#include <pwd.h> -#include <unistd.h> -#include <limits.h> - -#include "../macros.h" - -void append_random_to_file(char *path, unsigned long bytes) -{ - char *buf = NULL; - int max_chunk = 50000000; // 50 mebibyetes - FILE *fp = Fopen(path, "a"); - - for (;;) { - if (bytes > max_chunk) { - if (!buf) - buf = Calloc(max_chunk+1, char); - - Fill_with_stuff(buf, max_chunk); - buf[max_chunk] = '\0'; - fprintf(fp, "%s", buf); - bytes -= max_chunk; - - // Print out a dot every time we wrote 'much' data to a file - Out("."); - - } else { - if (!buf) - buf = Calloc(bytes+1, char); - - Fill_with_stuff(buf, bytes); - buf[bytes] = '\0'; - fprintf(fp, "%s", buf); - - break; - } - } - - if (buf) - free(buf); - fclose(fp); -} - -long ensure_dir_exists(const char *path) -{ - long num_dirs_created = 0; - int ret = mkdir_p(path, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, - &num_dirs_created); - if (ret != 0) { - Errno("Could not create dir '%s'", path); - } - - return num_dirs_created; -} - -void ensure_parent_dir_exists(const char *path) -{ - char *clone = Clone(path); - char *parent = dirname(clone); - - int ret = mkdir_p(parent, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, NULL); - if (ret != 0) { - Errno("Could not create dir %s", parent); - } - - free(clone); -} - -void ensure_dir_empty(const char *path) -{ - DIR *dh = opendir(path); - - if (!dh) { - Errno("Unable to empty %s", path); - } - - struct dirent *de; - - while ((de = readdir(dh))) { - if (0 == strcmp(de->d_name, ".") || - 0 == strcmp(de->d_name, "..")) - continue; - - char *absolute; - asprintf(&absolute, "%s/%s", path, de->d_name); - - if (is_dir(absolute)) - ensure_dir_empty(absolute); - - if (remove(absolute) == -1) - // Don't throw an error if there is no such file or directory - if (errno != 2) { - Errno("Unable to remove %s", absolute); - } - - free(absolute); - } - - closedir(dh); -} - -int ensure_file_exists(char *path, long *num_dirs_created) -{ - if (is_reg(path)) - return SUCCESS; - - char *dirname = dirname_r(Clone(path)); - *num_dirs_created += ensure_dir_exists(dirname); - free(dirname); - - FILE *fp = fopen(path, "a"); - if (fp) { - // We only need some data, less than 1 block in size, this is answer: - fprintf(fp, "42"); - fclose(fp); - return SUCCESS; - } - - return ERROR; -} - -char* dirname_r(char *path) -{ - int len = strlen(path); - int has = 0; - int i = len-1; - - if (strcmp(path, "..") == 0) { - return path; - } - - if (path[i] == '/') { - // Root directory - if (len == 1) - return path; - - // Remove all trailing / - for (; i >= 0; --i) { - if (path[i] == '/') { - path[i] = '\0'; - has = 1; - } else { - break; - } - } - } - - // Find next / - for (; i >= 0; --i) { - if (path[i] == '/') { - path[i] = '\0'; - has = 1; - break; - } - } - - // If no / - if (has == 0) { - path[0] = '.'; - path[1] = '\0'; - } - - return path; -} - -bool is_dir(const char *path) -{ - struct stat path_stat; - if (stat(path, &path_stat) == 0 && S_ISDIR(path_stat.st_mode)) - return true; - return false; -} - -bool is_reg(const char *path) -{ - struct stat path_stat; - if (stat(path, &path_stat) == 0 && S_ISREG(path_stat.st_mode)) - return true; - return false; -} - -bool exists(const char *path) -{ - struct stat path_stat; - if (stat(path, &path_stat) == 0) - return true; - return false; -} - -int mkdir_p(const char *path, mode_t mode, long *num_dirs_created) -{ - int res = 0; - - if (is_dir(path)) - return 0; - - if (is_reg(path)) - unlink(path); - - char *top = dirname_r(Clone(path)); - if (0 != mkdir_p(top, mode, num_dirs_created)) - goto cleanup; - - if ((mkdir(path, mode) == -1) && (errno != EEXIST)) - res = -1; - - if (res != -1) - *num_dirs_created = *num_dirs_created+1; - -cleanup: - free(top); - - return res; -} - -void cache_file(const char *file) -{ - Out("Caching file %s... it can take a while", file); - FILE *fd = Fopen(file, "r"); - char *line = NULL; - size_t len = 0, read = 0; - - while ((read = getline(&line, &len, fd)) != -1); - fclose(fd); -} - -void drop_caches(void) -{ - Out("Dropping all Linux caches..."); - - if (getuid() != 0) { - Out("\n"); - Error("I need to be root to do this, aborting!"); - } - - // echo 3 > /proc/sys/vm/drop_caches - char *drop_caches = "/proc/sys/vm/drop_caches"; - FILE *fd = Fopen(drop_caches, "w"); - fprintf(fd, "3"); - fclose(fd); - - Put("done"); -} - -void chown_path(const char *user, const char *path) -{ - struct passwd *pwd = getpwnam(user); - if (!pwd) { - Errno("Unable to retrieve information about system user %s!", user); - } - - if (chown(path, pwd->pw_uid, -1) == -1) { - Errno("Could not change ownership of '%s' to '%s'!", path, user); - } -} - -char *absolute_path(const char *path) -{ - if (path[0] == '/') - return Clone(path); - - char cwd[MAX_LINE_LEN]; - getcwd(cwd, sizeof(char)*MAX_LINE_LEN); - - if (!getcwd(cwd, sizeof(cwd))) { - Errno("Could not get current working directory"); - } - - char *absolute = NULL; - if (-1 == asprintf(&absolute, "%s/%s", cwd, path)) { - Error("Could not get absolute path of '%s'", path); - } - - return absolute; -} diff --git a/ioreplay/src/utils/futils.h b/ioreplay/src/utils/futils.h deleted file mode 100644 index 9afde1a..0000000 --- a/ioreplay/src/utils/futils.h +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef FUTILS_H -#define FUTILS_H - -#include "../defaults.h" - -/** - * @brief Thread safe version of dirname() - * - * @param path The full file path - * @return The directory path - */ -char* dirname_r(char *path); - -/** - * @brief Ensures that a file exists - * - * @param path The file path - * @param num_dirs_created Holds a count of how many sub dirs have been created - * @return -1 on error, 0 on success. - */ -int ensure_file_exists(char *path, long *num_dirs_created); - -/** - * @brief Checks whether path exists - * - * @param path The path - * @return true if the path exists - */ -bool exists(const char *path); - -/** - * @brief Check if path is a directory - * - * @param path The directory path - * @return true if the path is a directory, false otherwise - */ -bool is_dir(const char *path); - -/** - * @brief Check if path is a regular file - * - * @param path The file path - * @return true if the file at path is a regular fike - */ -bool is_reg(const char *path); - -/** - * @brief Create a directory recursively - * - * @param path The directory path - * @param mode The mode - * @param num_dirs_created Counts how many directories have been created - * @return -1 on error - */ -int mkdir_p(const char *path, mode_t mode, long *num_dirs_created); - -/** - * @brief Appends data to a file - * - * @param path The file path - * @param bytes The amount of bytes - */ -void append_random_to_file(char *path, unsigned long bytes); - -/** - * @brief Ensures that a directory exists - * - * @param path The directory path - * @return The amount of directories created (including parent directories) - */ -long ensure_dir_exists(const char *path); - -/** - * @brief Ensures that a parent directory exists - * - * @param path The directory path - */ -void ensure_parent_dir_exists(const char *path); - -/** - * @brief Ensures that a directory is empty - * - * @param path The directory path - */ -void ensure_dir_empty(const char *path); - -/** - * @brief Loading a file into the file system cache - * - * @param file The path to the file - */ -void cache_file(const char *file); - -/** - * @brief Drop all Linux caches - * - * This function drops all Linux caches, which includes all file - * system caches. - */ -void drop_caches(void); - -/** - * @brief Changes owner of a path - * - * Terminates the process with an error message if failed. - * - * @param user The new owner - * @param path The path - */ -void chown_path(const char *user, const char *path); - -/** - * @brief Retrieves the absolute path of a given path - * - * @param path The path - * @return The absolute path. It must be freed manually. - */ -char *absolute_path(const char *path); - -#endif // FUTILS_H diff --git a/ioreplay/src/utils/utils.c b/ioreplay/src/utils/utils.c deleted file mode 100644 index 4b41273..0000000 --- a/ioreplay/src/utils/utils.c +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "utils.h" - -#include <sys/resource.h> -#include <sys/time.h> - -void* notnull(void *p, char *file, int line, int count) -{ - if (p == NULL) { - Errno("%s:%d count:%d Could not allocate memory", file, line, count); - } - return p; -} - - -FILE* fnotnull(FILE *fd, const char *path, char *file, int line) -{ - if (fd == NULL) { - Errno("%s:%d Could not open file '%s'", file, line, path); - } - return fd; -} - -void* mmapok(void *p, char *file, int line) -{ - if (p == MAP_FAILED) { - Errno("%s:%d: Mmap failed", file, line); - } - return p; -} - -char* strtok2_r(char *str, char *delim, char **saveptr) -{ - int len = strlen(delim); - - if (str == NULL) - str = *saveptr; - - char *next = strstr(str, delim); - if (next) { - next[0] = '\0'; - for (int i = 0; i < len; ++i) - next++; - *saveptr = next; - return str; - } - - return NULL; -} - -void chreplace(char *str, char replace, char with) -{ - for (int i = 0; ; ++i) { - if (str[i] == '\0') - break; - if (str[i] == replace) - str[i] = with; - } -} - -void strunquote(char *str) -{ - int len = strlen(str); - - if (str[0] == '"') { - if (str[len-1] == '"') - str[len-1] = '\0'; - for (int i = 1; i < len; ++i) - str[i-1] = str[i]; - } -} - -void set_limits_drop_root(const char *user) -{ - if (getuid() == 0) { - struct rlimit rl; - rl.rlim_cur = rl.rlim_max = SET_RLIMIT_NOFILE; - if (0 != setrlimit(RLIMIT_NOFILE, &rl)) { - Errno("Could not set RLIMIT_NOFILE to '%lld'!", - (long long) SET_RLIMIT_NOFILE) - } - rl.rlim_cur = rl.rlim_max = SET_RLIMIT_NPROC; - if (0 != setrlimit(RLIMIT_NPROC, &rl)) { - Errno("Could not set RLIMIT_NPROC to '%lld'!", - (long long) SET_RLIMIT_NPROC) - } - - Put("Dropping root privileges to user '%s'", user); - struct passwd *pw = getpwnam(user); - - /* process is running as root, drop privileges */ - if (setgid(pw->pw_gid) != 0) { - Errno("Unable to drop group privileges!"); - } - if (setuid(pw->pw_uid) != 0) { - Errno("Unable to drop user privileges!"); - } - } - - /* - getrlimit(RLIMIT_NOFILE, &rl); - Put("Max open files: '%lld'", (long long) rl.rlim_cur); - getrlimit(RLIMIT_NPROC, &rl); - Put("Max open processes : '%lld'", (long long) rl.rlim_cur); - */ -} - -void get_loadavg_s(char *readbuf) -{ - FILE *fp = Fopen("/proc/loadavg", "r"); - fgets(readbuf, 128, fp); - char *pos = strchr(readbuf, ' '); - pos[0] = '\0'; - fclose(fp); -} - -double get_loadavg() -{ - // Not thread safe, but multi processing safe - static char buf[128]; - get_loadavg_s(buf); - - return atof(buf); -} - -bool is_number(char *str) -{ - for (int i = 0; ; ++i) { - if (str[i] == '\0') - return true; - if (isdigit(str[i]) == 0 && str[i] != '-') - return false; - } - - return true; -} - -void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data) -{ - int rc = pthread_create(thread, NULL, cb, data); - - switch (rc) { - case 0: - break; - case EAGAIN: - Error("Out of resources while creating pthread (%d)", rc); - break; - case EINVAL: - Error("Ivalid settings while creating pthread (%d)", rc); - break; - case EPERM: - Error("No permissions to configure pthread (%d)", rc); - default: - Error("Unknown error while creating pthread (%d)", rc); - break; - } -} - -void utils_test(void) -{ - if (getuid() == 0) { - set_limits_drop_root("nobody"); - struct rlimit rl; - - getrlimit(RLIMIT_NOFILE, &rl); - assert(rl.rlim_cur == SET_RLIMIT_NOFILE); - assert(rl.rlim_max == SET_RLIMIT_NOFILE); - - getrlimit(RLIMIT_NPROC, &rl); - assert(rl.rlim_cur == SET_RLIMIT_NPROC); - assert(rl.rlim_max == SET_RLIMIT_NPROC); - } -} diff --git a/ioreplay/src/utils/utils.h b/ioreplay/src/utils/utils.h deleted file mode 100644 index 3e86865..0000000 --- a/ioreplay/src/utils/utils.h +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef UTILS_H -#define UTILS_H - -// For asprintf in stdio.h -#define _GNU_SOURCE - -#include <assert.h> -#include <ctype.h> -#include <dirent.h> -#include <errno.h> -#include <fcntl.h> -#include <libgen.h> -#include <linux/types.h> -#include <linux/unistd.h> -#include <pthread.h> -#include <pwd.h> -#include <stdbool.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <sys/mman.h> -#include <sys/stat.h> -#include <sys/time.h> -#include <sys/types.h> -#include <sys/wait.h> -#include <time.h> -#include <unistd.h> - -#include "../macros.h" -#include "../defaults.h" - -/** - * @brief Check whether allocated memory is not NULL - * - * This function is used in conjunction with malloc() and co. It - * introduces an extra sanity check whether the memory could be - * allocated successfully or not. If not it will print out a error - * message stating the position in the source code where the memory - * allocation failed! - * - * @param p The pointer being checked - * @param file The source file name the memory was allocated in - * @param line The source line number the memory was allocated at - * @param count The amount of memory being allocated - * @return The pointer to the allocated memory - */ -void* notnull(void *p, char *file, int line, int count); - -/** - * @brief Check whether opened file handle is not NULL - * - * This function is used in conjunction with fopen(). It - * introduces an extra sanity check whether the file could be - * opened successfully or not. If not it will print out a error - * message stating the position in the source code where the open - * failed! - * - * @param fd The fd stream to be checked. - * @param path The file path opened - * @param file The source file name - * @param line The source line number - * @return The pointer to the allocated memory - */ -FILE* fnotnull(FILE *fd, const char *path, char *file, int line); - -/** - * @brief Check whether allocated memory via mmap is not null - * - * This function is used in conjunction with mmap() and co. It - * introduces an extra sanity check whether the memory could be - * allocated successfully or not. If not it will print out a error - * message stating the position in the source code where the memory - * allocation failed! - * - * @param addr The pointer being checked - * @param file The source file name the memory was allocated in - * @param line The source line number the memory was allocated at - * @return The pointer to the allocated memory - */ -void* mmapok(void *addr, char *file, int line); - -/** - * @brief A version of strtok_r supporting multi char delims - * - * @param str The input string - * @param delim The multi-char delimiter - * @param saveptr A temp storage location - * @return The next match if != NULL - */ -char* strtok2_r(char *str, char *delim, char **saveptr); - -/** - * @brief Replaces a character with another one in a string - * - * @param str The input string - * @param replace The character to be replaced - * @param with The character to replace with - */ -void chreplace(char *str, char replace, char with); - -/** - * @brief Removes quotes from a string - * - * @param str The input sting - */ -void strunquote(char *str); - -/** - * @brief Set rlimits and drop root privileges - * - * This function firsts sets the user resource limits to SET_RLIMIT_NOFILE and - * SET_RLIMIT_NPROC and then attempts to drop the root user to the specified - * one. - * - * @param user The user to switch to - */ -void set_limits_drop_root(const char *user); - -/** - * @brief Retrieve current 1 min Linux load average - * - * @param readbuf The buffer to store the load average as a string - */ -void get_loadavg_s(char *readbuf); - -/** - * @brief Retrieve current 1 min Linux load average - * - * This function is not thread safe! - * - * @return The 1 minute load average of the system - */ -double get_loadavg(); - -/** - * @brief Check whether a string represents a number - * - * @param str The input string - * @return true if all characters of the input string are a digits - */ -bool is_number(char *str); - -/** - * @brief Wrapper around pthread_create - * - * The wrapper also checks whether the thread has been created successfully - * or not! It will exit the process if not. - * - * @param thread The POSIX thread variable - * @param cb The threadss start callback routine - * @param data A data pointer passed to the thread. - */ -void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data); - -/** - * @brief Testing various of the utilities - */ -void utils_test(void); - -#endif // UTILS_H diff --git a/ioreplay/src/vfd.c b/ioreplay/src/vfd.c deleted file mode 100644 index 6e86f61..0000000 --- a/ioreplay/src/vfd.c +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "vfd.h" - -vfd_s* vfd_new(const int fd, const long mapped_fd, char *path) -{ - vfd_s *vfd = Malloc(vfd_s); - vfd->path = NULL; - vfd->debug = false; - vfd_update(vfd, fd, mapped_fd, path); - - return vfd; -} - -void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path) -{ - vfd->fd = fd; - vfd->dirfd = NULL; - vfd->mapped_fd = mapped_fd; - vfd->offset = 0; - - if (path) - free(vfd->path); - - vfd->path = Clone(path); -} - -void vfd_destroy(vfd_s *vfd) -{ - if (!vfd) - return; - - if (vfd->path) - free(vfd->path); - - free(vfd); -} - -void vfd_print(vfd_s *vfd) -{ - fprintf(stderr, "virtfd(%p) fd:%x mapped_fd:%lx path:%s\n", - (void*)vfd, vfd->fd, vfd->mapped_fd, vfd->path); -} diff --git a/ioreplay/src/vfd.h b/ioreplay/src/vfd.h deleted file mode 100644 index fd0c4fb..0000000 --- a/ioreplay/src/vfd.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef VFD_H -#define VFD_H - -#include "opcodes.h" -#include "defaults.h" - -/** - * @brief The virtual file descriptor definition - * - * A virtual file descriptor represents a file descriptor from ioreplay's - * point of view and is being used in various ways to simulate the real I/O - * protocolled to the .capture/.replay files. - * - * Generally speaking I/O replay maps the real FD numbers (the ones logged to - * the .capture file) to virtual FD numbers (a uniqe FD number for every open - * to increase concurrency). - */ -typedef struct vfd_s_ { - int fd; /**< the real fd */ - DIR *dirfd; /**< The real dirfd */ - long mapped_fd; /**< The mapped fd (virtual fd) */ - char *path; /**< The file path belonging to that fd */ - bool free_path; /**< True if path has to be freed or not */ - unsigned long offset; /**< The current virtual file offset in bytes */ - int debug; /**< Used for debugging purposes only */ -} vfd_s; - -/** - * @brief Creates a new virtual file descriptor object - * - * @param fd The file descriptor - * @param mapped_fd The mapped file descriptor - * @param path The path name - * @return The new fd object - */ -vfd_s* vfd_new(const int fd, const long mapped_fd, char *path); - -/** - * @brief Updates the virtfd object - * - * @param vfd The virtfd object - * @param fd The (real) file descriptor - * @param mapped_fd The mapped (virtual) file descriptor - * @param path The path name - * @return The new fd object - */ -void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path); - -/** - * @brief Destroys a file descriptor object - * - * @param vfd The file descriptor object - */ -void vfd_destroy(vfd_s *vfd); - -/** - * @brief Prints the virtual file descriptor - * @param vfd The virtual file descriptor - */ -void vfd_print(vfd_s *vfd); - -#endif // VFD_H - |
