initial commit0.1

4 files changed, 742 insertions, 0 deletions

diff --git a/ioreplay/src/utils/futils.c b/ioreplay/src/utils/futils.c
new file mode 100644
index 0000000..5b35618
--- /dev/null
+++ b/ioreplay/src/utils/futils.c
@@ -0,0 +1,291 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "futils.h"
+
+#include <libgen.h>
+#include <pwd.h>
+#include <unistd.h>
+#include <limits.h>
+
+#include "../macros.h"
+
+void append_random_to_file(char *path, unsigned long bytes)
+{
+    char *buf = NULL;
+    int max_chunk = 50000000; // 50 mebibyetes
+    FILE *fp = Fopen(path, "a");
+
+    for (;;) {
+        if (bytes > max_chunk) {
+            if (!buf)
+                buf = Calloc(max_chunk+1, char);
+
+            Fill_with_stuff(buf, max_chunk);
+            buf[max_chunk] = '\0';
+            fprintf(fp, "%s", buf);
+            bytes -= max_chunk;
+
+            // Print out a dot every time we wrote 'much' data to a file
+            Out(".");
+
+        } else {
+            if (!buf)
+                buf = Calloc(bytes+1, char);
+
+            Fill_with_stuff(buf, bytes);
+            buf[bytes] = '\0';
+            fprintf(fp, "%s", buf);
+
+            break;
+        }
+    }
+
+    if (buf)
+        free(buf);
+    fclose(fp);
+}
+
+long ensure_dir_exists(const char *path)
+{
+    long num_dirs_created = 0;
+    int ret = mkdir_p(path, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH,
+                      &num_dirs_created);
+    if (ret != 0) {
+        Errno("Could not create dir '%s'", path);
+    }
+
+    return num_dirs_created;
+}
+
+void ensure_parent_dir_exists(const char *path)
+{
+    char *clone = Clone(path);
+    char *parent = dirname(clone);
+
+    int ret = mkdir_p(parent, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, NULL);
+    if (ret != 0) {
+        Errno("Could not create dir %s", parent);
+    }
+
+    free(clone);
+}
+
+void ensure_dir_empty(const char *path)
+{
+    DIR *dh = opendir(path);
+
+    if (!dh) {
+        Errno("Unable to empty %s", path);
+    }
+
+    struct dirent *de;
+
+    while ((de = readdir(dh))) {
+        if (0 == strcmp(de->d_name, ".") ||
+            0 == strcmp(de->d_name, ".."))
+            continue;
+
+        char *absolute;
+        asprintf(&absolute, "%s/%s", path, de->d_name);
+
+        if (is_dir(absolute))
+            ensure_dir_empty(absolute);
+
+        if (remove(absolute) == -1)
+            // Don't throw an error if there is no such file or directory
+            if (errno != 2) {
+                Errno("Unable to remove %s", absolute);
+            }
+
+        free(absolute);
+    }
+
+    closedir(dh);
+}
+
+int ensure_file_exists(char *path, long *num_dirs_created)
+{
+    if (is_reg(path))
+        return SUCCESS;
+
+    char *dirname = dirname_r(Clone(path));
+    *num_dirs_created += ensure_dir_exists(dirname);
+    free(dirname);
+
+    FILE *fp = fopen(path, "a");
+    if (fp) {
+        // We only need some data, less than 1 block in size, this is answer:
+        fprintf(fp, "42");
+        fclose(fp);
+        return SUCCESS;
+    }
+
+    return ERROR;
+}
+
+char* dirname_r(char *path)
+{
+    int len = strlen(path);
+    int has = 0;
+    int i = len-1;
+
+    if (strcmp(path, "..") == 0) {
+        return path;
+    }
+
+    if (path[i] == '/') {
+        // Root directory
+        if (len == 1)
+            return path;
+
+        // Remove all trailing /
+        for (; i >= 0; --i) {
+            if (path[i] == '/') {
+                path[i] = '\0';
+                has = 1;
+            } else {
+                break;
+            }
+        }
+    }
+
+    // Find next /
+    for (; i >= 0; --i) {
+        if (path[i] == '/') {
+            path[i] = '\0';
+            has = 1;
+            break;
+        }
+    }
+
+    // If no /
+    if (has == 0) {
+        path[0] = '.';
+        path[1] = '\0';
+    }
+
+    return path;
+}
+
+bool is_dir(const char *path)
+{
+    struct stat path_stat;
+    if (stat(path, &path_stat) == 0 && S_ISDIR(path_stat.st_mode))
+        return true;
+    return false;
+}
+
+bool is_reg(const char *path)
+{
+    struct stat path_stat;
+    if (stat(path, &path_stat) == 0 && S_ISREG(path_stat.st_mode))
+        return true;
+    return false;
+}
+
+bool exists(const char *path)
+{
+    struct stat path_stat;
+    if (stat(path, &path_stat) == 0)
+        return true;
+    return false;
+}
+
+int mkdir_p(const char *path, mode_t mode, long *num_dirs_created)
+{
+    int res = 0;
+
+    if (is_dir(path))
+        return 0;
+
+    if (is_reg(path))
+        unlink(path);
+
+    char *top = dirname_r(Clone(path));
+    if (0 != mkdir_p(top, mode, num_dirs_created))
+        goto cleanup;
+
+    if ((mkdir(path, mode) == -1) && (errno != EEXIST))
+        res = -1;
+
+    if (res != -1)
+        *num_dirs_created = *num_dirs_created+1;
+
+cleanup:
+    free(top);
+
+    return res;
+}
+
+void cache_file(const char *file)
+{
+    Out("Caching file %s... it can take a while", file);
+    FILE *fd = Fopen(file, "r");
+    char *line = NULL;
+    size_t len = 0, read = 0;
+
+    while ((read = getline(&line, &len, fd)) != -1);
+    fclose(fd);
+}
+
+void drop_caches(void)
+{
+    Out("Dropping all Linux caches...");
+
+    if (getuid() != 0) {
+        Out("\n");
+        Error("I need to be root to do this, aborting!");
+    }
+
+    // echo 3 > /proc/sys/vm/drop_caches
+    char *drop_caches = "/proc/sys/vm/drop_caches";
+    FILE *fd = Fopen(drop_caches, "w");
+    fprintf(fd, "3");
+    fclose(fd);
+
+    Put("done");
+}
+
+void chown_path(const char *user, const char *path)
+{
+    struct passwd *pwd = getpwnam(user);
+    if (!pwd) {
+        Errno("Unable to retrieve information about system user %s!", user);
+    }
+
+    if (chown(path, pwd->pw_uid, -1) == -1) {
+        Errno("Could not change ownership of '%s' to '%s'!", path, user);
+    }
+}
+
+char *absolute_path(const char *path)
+{
+    if (path[0] == '/')
+        return Clone(path);
+
+    char cwd[MAX_LINE_LEN];
+    getcwd(cwd, sizeof(char)*MAX_LINE_LEN);
+
+    if (!getcwd(cwd, sizeof(cwd))) {
+        Errno("Could not get current working directory");
+    }
+
+    char *absolute = NULL;
+    if (-1 == asprintf(&absolute, "%s/%s", cwd, path)) {
+        Error("Could not get absolute path of '%s'", path);
+    }
+
+    return absolute;
+}
diff --git a/ioreplay/src/utils/futils.h b/ioreplay/src/utils/futils.h
new file mode 100644
index 0000000..9afde1a
--- /dev/null
+++ b/ioreplay/src/utils/futils.h
@@ -0,0 +1,134 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef FUTILS_H
+#define FUTILS_H
+
+#include "../defaults.h"
+
+/**
+ * @brief Thread safe version of dirname()
+ *
+ * @param path The full file path
+ * @return The directory path
+ */
+char* dirname_r(char *path);
+
+/**
+ * @brief Ensures that a file exists
+ *
+ * @param path The file path
+ * @param num_dirs_created Holds a count of how many sub dirs have been created
+ * @return -1 on error, 0 on success.
+ */
+int ensure_file_exists(char *path, long *num_dirs_created);
+
+/**
+ * @brief Checks whether path exists
+ *
+ * @param path The path
+ * @return true if the path exists
+ */
+bool exists(const char *path);
+
+/**
+ * @brief Check if path is a directory
+ *
+ * @param path The directory path
+ * @return true if the path is a directory, false otherwise
+ */
+bool is_dir(const char *path);
+
+/**
+ * @brief Check if path is a regular file
+ *
+ * @param path The file path
+ * @return true if the file at path is a regular fike
+ */
+bool is_reg(const char *path);
+
+/**
+ * @brief Create a directory recursively
+ *
+ * @param path The directory path
+ * @param mode The mode
+ * @param num_dirs_created Counts how many directories have been created
+ * @return -1 on error
+ */
+int mkdir_p(const char *path, mode_t mode, long *num_dirs_created);
+
+/**
+ * @brief Appends data to a file
+ *
+ * @param path The file path
+ * @param bytes The amount of bytes
+ */
+void append_random_to_file(char *path, unsigned long bytes);
+
+/**
+ * @brief Ensures that a directory exists
+ *
+ * @param path The directory path
+ * @return The amount of directories created (including parent directories)
+ */
+long ensure_dir_exists(const char *path);
+
+/**
+ * @brief Ensures that a parent directory exists
+ *
+ * @param path The directory path
+ */
+void ensure_parent_dir_exists(const char *path);
+
+/**
+ * @brief Ensures that a directory is empty
+ *
+ * @param path The directory path
+ */
+void ensure_dir_empty(const char *path);
+
+/**
+ * @brief Loading a file into the file system cache
+ *
+ * @param file The path to the file
+ */
+void cache_file(const char *file);
+
+/**
+ * @brief Drop all Linux caches
+ *
+ * This function drops all Linux caches, which includes all file
+ * system caches.
+ */
+void drop_caches(void);
+
+/**
+ * @brief Changes owner of a path
+ *
+ * Terminates the process with an error message if failed.
+ *
+ * @param user The new owner
+ * @param path The path
+ */
+void chown_path(const char *user, const char *path);
+
+/**
+ * @brief Retrieves the absolute path of a given path
+ *
+ * @param path The path
+ * @return The absolute path. It must be freed manually.
+ */
+char *absolute_path(const char *path);
+
+#endif // FUTILS_H
diff --git a/ioreplay/src/utils/utils.c b/ioreplay/src/utils/utils.c
new file mode 100644
index 0000000..57d6737
--- /dev/null
+++ b/ioreplay/src/utils/utils.c
@@ -0,0 +1,152 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "utils.h"
+
+#include <sys/resource.h>
+#include <sys/time.h>
+
+void* notnull(void *p, char *file, int line, int count)
+{
+    if (p == NULL) {
+        Errno("%s:%d count:%d Could not allocate memory", file, line, count);
+    }
+    return p;
+}
+
+
+FILE* fnotnull(FILE *fd, const char *path, char *file, int line)
+{
+    if (fd == NULL) {
+        Errno("%s:%d Could not open file '%s'", file, line, path);
+    }
+    return fd;
+}
+
+void* mmapok(void *p, char *file, int line)
+{
+    if (p == MAP_FAILED) {
+        Errno("%s:%d: Mmap failed", file, line);
+    }
+    return p;
+}
+
+char* strtok2_r(char *str, char *delim, char **saveptr)
+{
+    int len = strlen(delim);
+
+    if (str == NULL)
+        str = *saveptr;
+
+    char *next = strstr(str, delim);
+    if (next) {
+        next[0] = '\0';
+        for (int i = 0; i < len; ++i)
+            next++;
+        *saveptr = next;
+        return str;
+    }
+
+    return NULL;
+}
+
+void chreplace(char *str, char replace, char with)
+{
+    for (int i = 0; ; ++i) {
+        if (str[i] == '\0')
+            break;
+        if (str[i] == replace)
+            str[i] = with;
+    }
+}
+
+void strunquote(char *str)
+{
+    int len = strlen(str);
+
+    if (str[0] == '"') {
+        if (str[len-1] == '"')
+            str[len-1] = '\0';
+        for (int i = 1; i < len; ++i)
+            str[i-1] = str[i];
+    }
+}
+
+void drop_root(const char *user)
+{
+    if (getuid() == 0) {
+        Put("Dropping root privileges to user %s", user);
+
+        struct passwd *pw = getpwnam(user);
+
+        /* process is running as root, drop privileges */
+        if (setgid(pw->pw_gid) != 0) {
+            Errno("setgid: Unable to drop group privileges!");
+        }
+        if (setuid(pw->pw_uid) != 0) {
+            Errno("setuid: Unable to drop user privileges!");
+        }
+    }
+}
+
+void get_loadavg_s(char *readbuf)
+{
+    FILE *fp = Fopen("/proc/loadavg", "r");
+    fgets(readbuf, 128, fp);
+    char *pos = strchr(readbuf, ' ');
+    pos[0] = '\0';
+    fclose(fp);
+}
+
+double get_loadavg()
+{
+    // Not thread safe, but multi processing safe
+    static char buf[128];
+    get_loadavg_s(buf);
+
+    return atof(buf);
+}
+
+bool is_number(char *str)
+{
+    for (int i = 0; ; ++i) {
+        if (str[i] == '\0')
+            return true;
+        if (isdigit(str[i]) == 0 && str[i] != '-')
+            return false;
+    }
+
+    return true;
+}
+
+void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data)
+{
+    int rc = pthread_create(thread, NULL, cb, data);
+
+    switch (rc) {
+    case 0:
+        break;
+    case EAGAIN:
+        Error("Out of resources while creating pthread (%d)", rc);
+        break;
+    case EINVAL:
+        Error("Ivalid settings while creating pthread (%d)", rc);
+        break;
+    case EPERM:
+        Error("No permissions to configure pthread (%d)", rc);
+    default:
+        Error("Unknown error while creating pthread (%d)", rc);
+        break;
+    }
+}
diff --git a/ioreplay/src/utils/utils.h b/ioreplay/src/utils/utils.h
new file mode 100644
index 0000000..cfe4dbc
--- /dev/null
+++ b/ioreplay/src/utils/utils.h
@@ -0,0 +1,165 @@
+// Copyright 2018 Mimecast Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef UTILS_H
+#define UTILS_H
+
+// For asprintf in stdio.h
+#define _GNU_SOURCE
+
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libgen.h>
+#include <linux/types.h>
+#include <linux/unistd.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "../macros.h"
+#include "../defaults.h"
+
+/**
+ * @brief Check whether allocated memory is not NULL
+ *
+ * This function is used in conjunction with malloc() and co. It
+ * introduces an extra sanity check whether the memory could be
+ * allocated successfully or not. If not it will print out a error
+ * message stating the position in the source code where the memory
+ * allocation failed!
+ *
+ * @param p The pointer being checked
+ * @param file The source file name the memory was allocated in
+ * @param line The source line number the memory was allocated at
+ * @param count The amount of memory being allocated
+ * @return The pointer to the allocated memory
+ */
+void* notnull(void *p, char *file, int line, int count);
+
+/**
+ * @brief Check whether opened file handle is not NULL
+ *
+ * This function is used in conjunction with fopen(). It
+ * introduces an extra sanity check whether the file could be
+ * opened successfully or not. If not it will print out a error
+ * message stating the position in the source code where the open
+ * failed!
+ *
+ * @param fd The fd stream to be checked.
+ * @param path The file path opened
+ * @param file The source file name
+ * @param line The source line number
+ * @return The pointer to the allocated memory
+ */
+FILE* fnotnull(FILE *fd, const char *path, char *file, int line);
+
+/**
+ * @brief Check whether allocated memory via mmap is not null
+ *
+ * This function is used in conjunction with mmap() and co. It
+ * introduces an extra sanity check whether the memory could be
+ * allocated successfully or not. If not it will print out a error
+ * message stating the position in the source code where the memory
+ * allocation failed!
+ *
+ * @param addr The pointer being checked
+ * @param file The source file name the memory was allocated in
+ * @param line The source line number the memory was allocated at
+ * @return The pointer to the allocated memory
+ */
+void* mmapok(void *addr, char *file, int line);
+
+/**
+ * @brief A version of strtok_r supporting multi char delims
+ *
+ * @param str The input string
+ * @param delim The multi-char delimiter
+ * @param saveptr A temp storage location
+ * @return The next match if != NULL
+ */
+char* strtok2_r(char *str, char *delim, char **saveptr);
+
+/**
+ * @brief Replaces a character with another one in a string
+ *
+ * @param str The input string
+ * @param replace The character to be replaced
+ * @param with The character to replace with
+ */
+void chreplace(char *str, char replace, char with);
+
+/**
+ * @brief Removes quotes from a string
+ *
+ * @param str The input sting
+ */
+void strunquote(char *str);
+
+/**
+ * @brief Drop root privileges
+ *
+ * @param user The user to switch to
+ */
+void drop_root(const char *user);
+
+/**
+ * @brief Retrieve current 1 min Linux load average
+ *
+ * @param readbuf The buffer to store the load average as a string
+ */
+void get_loadavg_s(char *readbuf);
+
+/**
+ * @brief Retrieve current 1 min Linux load average
+ *
+ * This function is not thread safe!
+ *
+ * @return The 1 minute load average of the system
+ */
+double get_loadavg();
+
+/**
+ * @brief Check whether a string represents a number
+ *
+ * @param str The input string
+ * @return true if all characters of the input string are a digits
+ */
+bool is_number(char *str);
+
+/**
+ * @brief Wrapper around pthread_create
+ *
+ * The wrapper also checks whether the thread has been created successfully
+ * or not! It will exit the process if not.
+ *
+ * @param thread The POSIX thread variable
+ * @param cb The threadss start callback routine
+ * @param data A data pointer passed to the thread.
+ */
+void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data);
+
+#endif // UTILS_H