initial commit0.1

68 files changed, 9475 insertions, 0 deletions

diff --git a/ioreplay/src/capture/capture.c b/ioreplay/src/capture/capture.c
new file mode 100644
index 0000000..0ac336b
--- /dev/null
+++ b/ioreplay/src/capture/capture.c
@@ -0,0 +1,99 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "capture.h"
+
+#include <sys/utsname.h>
+
+status_e capture_run(options_s *opts)
+{
+    int status = 0;
+    struct utsname uts;
+
+    // To make it nicer we should iterate over PATH instead
+    char *staprun_paths[3] = {
+        "/usr/bin/staprun",
+        "/usr/local/bin/staprun",
+        "/bin/staprun"
+    };
+    int num_staprun_paths = 3;
+
+    if (0 != uname(&uts)) {
+        Errno("Could not identify release of currently running Kernel!");
+    }
+
+    Put("Release of currently running Kernel: %s", uts.release);
+    char modules_dir[128];
+    sprintf(modules_dir, "/opt/ioreplay/systemtap/%s", uts.release);
+    Put("Changing directory to module path: %s/", modules_dir);
+
+    if (0 != chdir(modules_dir)) {
+        Errno("Could not change into '%s', please ensure that the compiled "
+              "SystemTap modules correspond to the currently running Kernel "
+              "and that these are installed properly!\n",
+              modules_dir);
+    }
+
+    if (0 != access(opts->module, R_OK)) {
+        Errno("Module '%s/%s' can't be read, please make sure that the "
+              "SystemTap Kernel modules are installed!",
+              modules_dir, opts->module);
+    }
+
+    char *staprun_path = NULL;
+    for (int i = 0; i < num_staprun_paths; ++i) {
+        if (0 == access(staprun_paths[i], X_OK)) {
+            staprun_path = staprun_paths[i];
+            //Put("SystemTap command path: %s", staprun_path);
+            break;
+        }
+    }
+
+    if (staprun_path == NULL) {
+        Errno("Can't find 'staprun' command, please ensure to have the SystemTap "
+              "runtime (usually package 'systemtap-runtime') installed!");
+    }
+
+    char staprun_command[128];
+    if (opts->pid >= 0) {
+        sprintf(staprun_command, "%s %s -v -o %s -x %d", staprun_path, opts->module,
+                opts->capture_file, opts->pid);
+    } else {
+        sprintf(staprun_command, "%s %s -v -o %s", staprun_path, opts->module,
+                opts->capture_file);
+    }
+
+    Out("NOTICE: It is good practise first to stop all processes, then to ");
+    Out("start capturing, and then to start all processes again. The reason ");
+    Out("is that processes may have already open file handles. In that case ");
+    Out("I/O Replay would be unable to replay these! This may be improved "); 
+    Put("in a future release!");
+    Put("To abort capturing now send Ctrl+C, otherwise wait 1h");
+    Put("Capturing I/O via: '%s'", staprun_command);
+
+    char buf[1024];
+    FILE *fp;
+
+    if ((fp = popen(staprun_command, "r")) == NULL) {
+        Errno("Unable to invoke staprun command!");
+    }
+    while (fgets(buf, 1024, fp) != NULL)
+        Out("stapio: %s", buf);
+
+    if (0 != pclose(fp)) {
+        Error("Problems invoking staprun command!");
+    }
+
+    return status;
+}
diff --git a/ioreplay/src/capture/capture.h b/ioreplay/src/capture/capture.h
new file mode 100644
index 0000000..7718d3e
--- /dev/null
+++ b/ioreplay/src/capture/capture.h
@@ -0,0 +1,30 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef CAPTURE_H
+#define CAPTURE_H
+
+#include "../defaults.h"
+#include "../utils/futils.h"
+#include "../options.h"
+
+/**
+ * @brief Captures I/O to a .capture file by using stap from SystemTap
+ *
+ * @param opts The options object
+ * @return SUCCESS if everything went fine
+ */
+status_e capture_run(options_s *opts);
+
+#endif // CAPTURE_H
diff --git a/ioreplay/src/cleanup/cleanup.c b/ioreplay/src/cleanup/cleanup.c
new file mode 100644
index 0000000..13c557c
--- /dev/null
+++ b/ioreplay/src/cleanup/cleanup.c
@@ -0,0 +1,30 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cleanup.h"
+
+#include "../mounts.h"
+
+status_e cleanup_run(options_s *opts)
+{
+    drop_root(opts->user);
+    mounts_s *m = mounts_new(opts);
+
+    if (opts->purge)
+        mounts_purge(m);
+    else
+        mounts_trash(m);
+
+    return SUCCESS;
+}
diff --git a/ioreplay/src/cleanup/cleanup.h b/ioreplay/src/cleanup/cleanup.h
new file mode 100644
index 0000000..127badf
--- /dev/null
+++ b/ioreplay/src/cleanup/cleanup.h
@@ -0,0 +1,29 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef CLEANUP_H
+#define CLEANUP_H
+
+#include "../defaults.h"
+#include "../options.h"
+
+/**
+ * @brief Cleans up all files and directories of a given test
+ *
+ * @brief opts The options object
+ * @return SUCCESS in case everything went fine
+ */
+status_e cleanup_run(options_s *opts);
+
+#endif // CLEANUP_H
diff --git a/ioreplay/src/datas/amap.c b/ioreplay/src/datas/amap.c
new file mode 100644
index 0000000..806a3f8
--- /dev/null
+++ b/ioreplay/src/datas/amap.c
@@ -0,0 +1,264 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "amap.h"
+
+/**
+ * @brief Creates a new array map
+ *
+ * @param size The array map size
+ * @param mmapped true if the memory should be mmapped
+ * @return The new amap object
+ */
+static amap_s *_amap_new(long size, bool mmapped)
+{
+    amap_s *a = NULL;
+    void ***arrays = NULL;
+
+    // Calculate a multiple of 1024, but at least in size of 'size'.
+    if (size % 1024 != 0) {
+        size = 1024*(1+(long)(size/1024));
+    }
+
+    if (size < 1) {
+        Error("Size overflow");
+    }
+
+    int num_arrays = size / AMAP_MAX_ARRAY_LENGTH;
+
+    if (mmapped) {
+        a = Mmapshared(amap_s);
+        arrays = Cmapshared(num_arrays, void**);
+    } else {
+        a = Malloc(amap_s);
+        arrays = Calloc(num_arrays, void**);
+    }
+
+    for (int i = 0; i < num_arrays; ++i) {
+        if (mmapped) {
+            //Put("%d", AMAP_MAX_ARRAY_LENGTH);
+            arrays[i] = Cmapshared(AMAP_MAX_ARRAY_LENGTH, void*);
+        } else {
+            arrays[i] = Calloc(AMAP_MAX_ARRAY_LENGTH, void*);
+        }
+        for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) {
+            arrays[i][j] = NULL;
+        }
+    }
+
+    a->arrays = arrays;
+    a->num_arrays = num_arrays;
+    a->size = size;
+    a->data_destroy = NULL;
+    a->mmapped = mmapped;
+
+    return a;
+}
+
+/**
+ * @brief Creates a new array map
+ *
+ * @param size The array map size
+ * @return The new amap object
+ */
+amap_s* amap_new(const long size)
+{
+    return _amap_new(size, false);
+}
+
+/**
+ * @brief Creates a new mmapped array map
+ *
+ * @param size The array map size
+ * @return The new amap object
+ */
+amap_s* amap_new_mmapped(const long size)
+{
+    return _amap_new(size, true);
+}
+
+/**
+ * @brief Destroys a mmap object
+ *
+ * @a The new amap object
+ */
+void amap_destroy(amap_s* a)
+{
+    if (!a) {
+        return;
+    }
+
+    // Don't bother, the mmapped version of amap will stay alive until
+    // process terminations. And after process termination everything
+    // will be cleaned up automatically by Linux.
+    if (a->mmapped) {
+        return;
+    }
+
+    for (int i = 0; i < a->num_arrays; ++i) {
+        if (a->data_destroy) {
+            for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j)
+                if (a->arrays[i][j]) {
+                    a->data_destroy(a->arrays[i][j]);
+                }
+        }
+        free(a->arrays[i]);
+    }
+    free(a->arrays);
+    free(a);
+}
+
+/**
+ * @brief Resets a mmap object
+ *
+ * This resets all entries to NULL.
+ *
+ * @a The new amap object
+ */
+void amap_reset(amap_s* a)
+{
+    for (int i = 0; i < a->num_arrays; ++i) {
+        for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) {
+            if (a->data_destroy) {
+                if (a->arrays[i][j]) {
+                    a->data_destroy(a->arrays[i][j]);
+                }
+            }
+            a->arrays[i][j] = NULL;
+        }
+    }
+}
+
+int amap_set(amap_s *a, const long position, void* value)
+{
+    if (position >= a->size)
+        return -1;
+    int which_array = position / AMAP_MAX_ARRAY_LENGTH;
+    int array_pos = position % AMAP_MAX_ARRAY_LENGTH;
+    a->arrays[which_array][array_pos] = value;
+    return 0;
+}
+
+void* amap_get(amap_s *a, const long position)
+{
+    if (position >= a->size)
+        return NULL;
+    int which_array = position / AMAP_MAX_ARRAY_LENGTH;
+    int array_pos = position % AMAP_MAX_ARRAY_LENGTH;
+    return a->arrays[which_array][array_pos];
+}
+
+void* amap_unset(amap_s *a, const long position)
+{
+    if (position >= a->size)
+        return NULL;
+    int which_array = position / AMAP_MAX_ARRAY_LENGTH;
+    int array_pos = position % AMAP_MAX_ARRAY_LENGTH;
+    void *value = a->arrays[which_array][array_pos];
+    a->arrays[which_array][array_pos] = NULL;
+    return value;
+}
+
+void amap_run_cb(amap_s *a, void (*cb)(void *data))
+{
+    for (int i = 0; i < a->num_arrays; ++i) {
+        for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) {
+            if (a->arrays[i][j])
+                cb(a->arrays[i][j]);
+        }
+    }
+}
+
+void amap_print(amap_s* a)
+{
+    Put("amap_s (%p):", (void*)a);
+    Put("\tmmapped: %d", a->mmapped);
+    Put("\tmax_array_length: %d", AMAP_MAX_ARRAY_LENGTH);
+    Put("\tnum_arrays: %d", a->num_arrays);
+    Put("\tsize: %lu", a->size);
+    Out("\toccupied slots: ");
+    for (int i = 0; i < a->num_arrays; ++i) {
+        for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) {
+            if (a->arrays[i][j] != NULL) {
+                Out("%d:%d ", i, j);
+            }
+        }
+    }
+    Out("\n");
+}
+
+void _amap_test(amap_s *a)
+{
+    assert(0 == amap_set(a, 0, (void*)10));
+    assert(0 == amap_set(a, 1, (void*)11));
+    assert(0 == amap_set(a, 2, (void*)12));
+    assert(0 == amap_set(a, 3, (void*)a));
+    assert(10 == (long) amap_get(a, 0));
+    assert(11 == (long) amap_get(a, 1));
+    assert(12 == (long) amap_get(a, 2));
+    assert(a == amap_get(a, 3));
+
+    assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH-1, (void*) 23));
+    assert(23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH-1));
+
+    assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH, (void*) 42));
+    assert(42 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH));
+
+    assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2-1, (void*) (23+42)));
+    assert(42+23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH*2-1));
+    assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2, (void*) 23));
+
+
+    assert(NULL == amap_get(a, 1024*1024*9-1));
+    assert(0 == amap_set(a, 1024*1024*9-1, (void*) 0x1));
+    assert(0x1 == (long) amap_get(a, 1024*1024*9-1));
+    assert(0x1 == (long) amap_unset(a, 1024*1024*9-1));
+    assert(NULL == amap_get(a, 1024*1024*9-1));
+
+    assert(0 == amap_set(a, 1024*1024*9, (void*) 100));
+    assert(100 == (long) amap_get(a, 1024*1024*9));
+
+    assert(0 == amap_set(a, 1024*1024*9+1, (void*) 101));
+    assert(101 == (long) amap_get(a, 1024*1024*9+1));
+
+    assert(0 == amap_set(a, 1024*1024*10-2, (void*) 102));
+    assert(102 == (long) amap_get(a, 1024*1024*10-2));
+
+    assert(0 == amap_set(a, 1024*1024*10-1, a));
+    assert(a == amap_get(a, 1024*1024*10-1));
+    //amap_print(a);
+
+    assert(a == amap_unset(a, 1024*1024*10-1));
+    assert(a != amap_unset(a, 1024*1024*10-1));
+    //amap_print(a);
+}
+
+void amap_test(void)
+{
+    // First test the non-mmapped version
+    amap_s* a = amap_new(1024*1024*10);
+    _amap_test(a);
+    amap_destroy(a);
+
+    // Now test the mapped version
+    a = amap_new_mmapped(1024*1024*10);
+    _amap_test(a);
+    amap_destroy(a);
+
+    // Another test with non-alligned size
+    a = amap_new(1024*1024*10+1);
+    _amap_test(a);
+    amap_destroy(a);
+}
+
diff --git a/ioreplay/src/datas/amap.h b/ioreplay/src/datas/amap.h
new file mode 100644
index 0000000..882a7c5
--- /dev/null
+++ b/ioreplay/src/datas/amap.h
@@ -0,0 +1,49 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef AMAP_H
+#define AMAP_H
+
+#include "../defaults.h"
+
+#define AMAP_MAX_ARRAY_LENGTH 1024*8
+
+/**
+ * @brief Implements an array map data structure
+ *
+ * This array map can hold a HUGE amount of entries by allocating multiple
+ * smaller arrays. There are two version of the amap data structure available:
+ * a memory mapped (mmap) and a normal version. The memory mapped version can
+ * be used for IPC between various processes.
+ */
+typedef struct amap_s_ {
+    void*** arrays; /**< The pointers to the amap arrays */
+    int num_arrays; /**< The amount of arrays used in the amap */
+    long size; /**< The total size/capacity of the amap */
+    bool mmapped; /**< True if amap is memory mapped */
+    void (*data_destroy)(void *data); /**< Callback to destroy all elements */
+} amap_s;
+
+amap_s* amap_new(const long size);
+amap_s* amap_new_mmapped(const long size);
+int amap_set(amap_s *a, const long position, void* value);
+void* amap_get(amap_s *a, const long position);
+void* amap_unset(amap_s *a, const long position);
+void amap_print(amap_s *a);
+void amap_destroy(amap_s *a);
+void amap_reset(amap_s *a);
+void amap_run_cb(amap_s *a, void (*cb)(void *data));
+void amap_test(void);
+
+#endif // AMAP_H
diff --git a/ioreplay/src/datas/btree.c b/ioreplay/src/datas/btree.c
new file mode 100644
index 0000000..da5da48
--- /dev/null
+++ b/ioreplay/src/datas/btree.c
@@ -0,0 +1,169 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "btree.h"
+
+btree_s* btree_new()
+{
+    btree_s *b = Malloc(btree_s);
+    *b = (btree_s) {
+        .root = NULL, .size = 0
+    };
+    return b;
+}
+
+void btree_destroy(btree_s* b)
+{
+    if (b->root)
+        btreelem_destroy_r(b->root);
+    free(b);
+}
+
+void btree_destroy2(btree_s* b)
+{
+    if (b->root)
+        btreelem_destroy_r2(b->root);
+    free(b);
+}
+
+int btree_insert(btree_s* b, int key, void *data)
+{
+    int ret = 1;
+
+    if (b->root == NULL) {
+        b->root = btreelem_new(key, data);
+        ret = 0;
+    } else {
+        ret = btreelem_insert_r(b->root, key, data);
+    }
+
+    if (ret == 0) {
+        b->size++;
+    }
+
+    return ret;
+}
+
+void* btree_get(btree_s* b, int key)
+{
+    if (b->root == NULL)
+        return NULL;
+
+    return btreelem_get_r(b->root, key);
+}
+
+void btree_print(btree_s* b)
+{
+    btreelem_print_r(b->root, 0);
+}
+
+btreelem_s* btreelem_new(int key, void *data)
+{
+    btreelem_s *e = Malloc(btreelem_s);
+    *e = (btreelem_s) {
+        .key = key, .data = data, .left = NULL, .right = NULL
+    };
+    return e;
+}
+
+void btreelem_destroy_r(btreelem_s* e)
+{
+    if (e->left) {
+        btreelem_destroy_r(e->left);
+    }
+    if (e->right) {
+        btreelem_destroy_r(e->right);
+    }
+
+    free(e);
+}
+
+void btreelem_destroy_r2(btreelem_s* e)
+{
+    if (e->left)
+        btreelem_destroy_r(e->left);
+    if (e->right)
+        btreelem_destroy_r(e->right);
+    if (e->data)
+        btree_destroy(e->data);
+
+    free(e);
+}
+
+int btreelem_insert_r(btreelem_s* e, int key, void *data)
+{
+    int ret = 0;
+
+    if (e->key == key) {
+        ret = 1;
+    }
+
+    else if (e->key > key) {
+        if (e->left == NULL) {
+            e->left = btreelem_new(key, data);
+        } else {
+            ret = btreelem_insert_r(e->left, key, data);
+        }
+    }
+
+    else {
+        if (e->right == NULL) {
+            e->right = btreelem_new(key, data);
+        } else {
+            ret = btreelem_insert_r(e->right, key, data);
+        }
+    }
+
+    return ret;
+}
+
+void* btreelem_get_r(btreelem_s* e, int key)
+{
+    void *data = NULL;
+
+    if (e->key == key) {
+        data = e->data;
+    }
+
+    else if (e->key > key) {
+        if (e->left) {
+            data = btreelem_get_r(e->left, k