From 26b3b3e368a79ce29df732ea04e72a4c002ae2ce Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 6 Mar 2018 17:38:59 +0000 Subject: rename into ioriot --- .gitignore | 2 +- Makefile | 12 +- README.md | 108 ++--- docs/doxygen.conf | 4 +- ioreplay/Makefile | 36 -- ioreplay/src/capture/capture.c | 99 ----- ioreplay/src/capture/capture.h | 30 -- ioreplay/src/cleanup/cleanup.c | 30 -- ioreplay/src/cleanup/cleanup.h | 29 -- ioreplay/src/datas/amap.c | 264 ------------ ioreplay/src/datas/amap.h | 49 --- ioreplay/src/datas/btree.c | 169 -------- ioreplay/src/datas/btree.h | 52 --- ioreplay/src/datas/hmap.c | 362 ---------------- ioreplay/src/datas/hmap.h | 56 --- ioreplay/src/datas/list.c | 279 ------------ ioreplay/src/datas/list.h | 56 --- ioreplay/src/datas/rbuffer.c | 147 ------- ioreplay/src/datas/rbuffer.h | 102 ----- ioreplay/src/datas/stack.c | 85 ---- ioreplay/src/datas/stack.h | 43 -- ioreplay/src/defaults.h | 54 --- ioreplay/src/generate/generate.c | 235 ----------- ioreplay/src/generate/generate.h | 112 ----- ioreplay/src/generate/gioop.c | 838 ------------------------------------- ioreplay/src/generate/gioop.h | 102 ----- ioreplay/src/generate/gparser.c | 356 ---------------- ioreplay/src/generate/gparser.h | 113 ----- ioreplay/src/generate/gprocess.c | 101 ----- ioreplay/src/generate/gprocess.h | 90 ---- ioreplay/src/generate/gtask.c | 91 ---- ioreplay/src/generate/gtask.h | 100 ----- ioreplay/src/generate/gwriter.c | 85 ---- ioreplay/src/generate/gwriter.h | 86 ---- ioreplay/src/generate/vsize.c | 247 ----------- ioreplay/src/generate/vsize.h | 180 -------- ioreplay/src/init/init.c | 226 ---------- ioreplay/src/init/init.h | 64 --- ioreplay/src/init/itask.c | 66 --- ioreplay/src/init/itask.h | 72 ---- ioreplay/src/init/ithread.c | 99 ----- ioreplay/src/init/ithread.h | 86 ---- ioreplay/src/macros.h | 116 ----- ioreplay/src/main.c | 275 ------------ ioreplay/src/meta/meta.c | 111 ----- ioreplay/src/meta/meta.h | 107 ----- ioreplay/src/mounts.c | 400 ------------------ ioreplay/src/mounts.h | 154 ------- ioreplay/src/opcodes.h | 103 ----- ioreplay/src/options.c | 51 --- ioreplay/src/options.h | 61 --- ioreplay/src/replay/replay.c | 191 --------- ioreplay/src/replay/replay.h | 46 -- ioreplay/src/replay/rioop.c | 425 ------------------- ioreplay/src/replay/rioop.h | 54 --- ioreplay/src/replay/rprocess.c | 34 -- ioreplay/src/replay/rprocess.h | 40 -- ioreplay/src/replay/rstats.c | 108 ----- ioreplay/src/replay/rstats.h | 117 ------ ioreplay/src/replay/rtask.c | 50 --- ioreplay/src/replay/rtask.h | 69 --- ioreplay/src/replay/rthread.c | 216 ---------- ioreplay/src/replay/rthread.h | 123 ------ ioreplay/src/replay/rworker.c | 360 ---------------- ioreplay/src/replay/rworker.h | 82 ---- ioreplay/src/utests.c | 41 -- ioreplay/src/utests.h | 25 -- ioreplay/src/utils/futils.c | 291 ------------- ioreplay/src/utils/futils.h | 134 ------ ioreplay/src/utils/utils.c | 186 -------- ioreplay/src/utils/utils.h | 174 -------- ioreplay/src/vfd.c | 55 --- ioreplay/src/vfd.h | 77 ---- ioreplay/tags | 661 ----------------------------- ioriot/Makefile | 36 ++ ioriot/src/capture/capture.c | 99 +++++ ioriot/src/capture/capture.h | 30 ++ ioriot/src/cleanup/cleanup.c | 30 ++ ioriot/src/cleanup/cleanup.h | 29 ++ ioriot/src/datas/amap.c | 264 ++++++++++++ ioriot/src/datas/amap.h | 49 +++ ioriot/src/datas/btree.c | 169 ++++++++ ioriot/src/datas/btree.h | 52 +++ ioriot/src/datas/hmap.c | 362 ++++++++++++++++ ioriot/src/datas/hmap.h | 56 +++ ioriot/src/datas/list.c | 279 ++++++++++++ ioriot/src/datas/list.h | 56 +++ ioriot/src/datas/rbuffer.c | 147 +++++++ ioriot/src/datas/rbuffer.h | 102 +++++ ioriot/src/datas/stack.c | 85 ++++ ioriot/src/datas/stack.h | 43 ++ ioriot/src/defaults.h | 54 +++ ioriot/src/generate/generate.c | 235 +++++++++++ ioriot/src/generate/generate.h | 112 +++++ ioriot/src/generate/gioop.c | 838 +++++++++++++++++++++++++++++++++++++ ioriot/src/generate/gioop.h | 102 +++++ ioriot/src/generate/gparser.c | 356 ++++++++++++++++ ioriot/src/generate/gparser.h | 113 +++++ ioriot/src/generate/gprocess.c | 101 +++++ ioriot/src/generate/gprocess.h | 90 ++++ ioriot/src/generate/gtask.c | 91 ++++ ioriot/src/generate/gtask.h | 100 +++++ ioriot/src/generate/gwriter.c | 85 ++++ ioriot/src/generate/gwriter.h | 86 ++++ ioriot/src/generate/vsize.c | 247 +++++++++++ ioriot/src/generate/vsize.h | 180 ++++++++ ioriot/src/init/init.c | 226 ++++++++++ ioriot/src/init/init.h | 64 +++ ioriot/src/init/itask.c | 66 +++ ioriot/src/init/itask.h | 72 ++++ ioriot/src/init/ithread.c | 99 +++++ ioriot/src/init/ithread.h | 86 ++++ ioriot/src/macros.h | 116 +++++ ioriot/src/main.c | 275 ++++++++++++ ioriot/src/meta/meta.c | 111 +++++ ioriot/src/meta/meta.h | 107 +++++ ioriot/src/mounts.c | 400 ++++++++++++++++++ ioriot/src/mounts.h | 154 +++++++ ioriot/src/opcodes.h | 103 +++++ ioriot/src/options.c | 51 +++ ioriot/src/options.h | 61 +++ ioriot/src/replay/replay.c | 191 +++++++++ ioriot/src/replay/replay.h | 46 ++ ioriot/src/replay/rioop.c | 425 +++++++++++++++++++ ioriot/src/replay/rioop.h | 54 +++ ioriot/src/replay/rprocess.c | 34 ++ ioriot/src/replay/rprocess.h | 40 ++ ioriot/src/replay/rstats.c | 108 +++++ ioriot/src/replay/rstats.h | 117 ++++++ ioriot/src/replay/rtask.c | 50 +++ ioriot/src/replay/rtask.h | 69 +++ ioriot/src/replay/rthread.c | 216 ++++++++++ ioriot/src/replay/rthread.h | 123 ++++++ ioriot/src/replay/rworker.c | 360 ++++++++++++++++ ioriot/src/replay/rworker.h | 82 ++++ ioriot/src/utests.c | 41 ++ ioriot/src/utests.h | 25 ++ ioriot/src/utils/futils.c | 291 +++++++++++++ ioriot/src/utils/futils.h | 134 ++++++ ioriot/src/utils/utils.c | 186 ++++++++ ioriot/src/utils/utils.h | 174 ++++++++ ioriot/src/vfd.c | 55 +++ ioriot/src/vfd.h | 77 ++++ ioriot/tags | 661 +++++++++++++++++++++++++++++ systemtap/Makefile | 20 +- systemtap/src/ioreplay.stp | 591 -------------------------- systemtap/src/ioriot.stp | 591 ++++++++++++++++++++++++++ systemtap/src/javaioreplay.stp | 591 -------------------------- systemtap/src/javaioriot.stp | 591 ++++++++++++++++++++++++++ systemtap/src/targetedioreplay.stp | 591 -------------------------- systemtap/src/targetedioriot.stp | 591 ++++++++++++++++++++++++++ 151 files changed, 12074 insertions(+), 12074 deletions(-) delete mode 100644 ioreplay/Makefile delete mode 100644 ioreplay/src/capture/capture.c delete mode 100644 ioreplay/src/capture/capture.h delete mode 100644 ioreplay/src/cleanup/cleanup.c delete mode 100644 ioreplay/src/cleanup/cleanup.h delete mode 100644 ioreplay/src/datas/amap.c delete mode 100644 ioreplay/src/datas/amap.h delete mode 100644 ioreplay/src/datas/btree.c delete mode 100644 ioreplay/src/datas/btree.h delete mode 100644 ioreplay/src/datas/hmap.c delete mode 100644 ioreplay/src/datas/hmap.h delete mode 100644 ioreplay/src/datas/list.c delete mode 100644 ioreplay/src/datas/list.h delete mode 100644 ioreplay/src/datas/rbuffer.c delete mode 100644 ioreplay/src/datas/rbuffer.h delete mode 100644 ioreplay/src/datas/stack.c delete mode 100644 ioreplay/src/datas/stack.h delete mode 100644 ioreplay/src/defaults.h delete mode 100644 ioreplay/src/generate/generate.c delete mode 100644 ioreplay/src/generate/generate.h delete mode 100644 ioreplay/src/generate/gioop.c delete mode 100644 ioreplay/src/generate/gioop.h delete mode 100644 ioreplay/src/generate/gparser.c delete mode 100644 ioreplay/src/generate/gparser.h delete mode 100644 ioreplay/src/generate/gprocess.c delete mode 100644 ioreplay/src/generate/gprocess.h delete mode 100644 ioreplay/src/generate/gtask.c delete mode 100644 ioreplay/src/generate/gtask.h delete mode 100644 ioreplay/src/generate/gwriter.c delete mode 100644 ioreplay/src/generate/gwriter.h delete mode 100644 ioreplay/src/generate/vsize.c delete mode 100644 ioreplay/src/generate/vsize.h delete mode 100644 ioreplay/src/init/init.c delete mode 100644 ioreplay/src/init/init.h delete mode 100644 ioreplay/src/init/itask.c delete mode 100644 ioreplay/src/init/itask.h delete mode 100644 ioreplay/src/init/ithread.c delete mode 100644 ioreplay/src/init/ithread.h delete mode 100644 ioreplay/src/macros.h delete mode 100644 ioreplay/src/main.c delete mode 100644 ioreplay/src/meta/meta.c delete mode 100644 ioreplay/src/meta/meta.h delete mode 100644 ioreplay/src/mounts.c delete mode 100644 ioreplay/src/mounts.h delete mode 100644 ioreplay/src/opcodes.h delete mode 100644 ioreplay/src/options.c delete mode 100644 ioreplay/src/options.h delete mode 100644 ioreplay/src/replay/replay.c delete mode 100644 ioreplay/src/replay/replay.h delete mode 100644 ioreplay/src/replay/rioop.c delete mode 100644 ioreplay/src/replay/rioop.h delete mode 100644 ioreplay/src/replay/rprocess.c delete mode 100644 ioreplay/src/replay/rprocess.h delete mode 100644 ioreplay/src/replay/rstats.c delete mode 100644 ioreplay/src/replay/rstats.h delete mode 100644 ioreplay/src/replay/rtask.c delete mode 100644 ioreplay/src/replay/rtask.h delete mode 100644 ioreplay/src/replay/rthread.c delete mode 100644 ioreplay/src/replay/rthread.h delete mode 100644 ioreplay/src/replay/rworker.c delete mode 100644 ioreplay/src/replay/rworker.h delete mode 100644 ioreplay/src/utests.c delete mode 100644 ioreplay/src/utests.h delete mode 100644 ioreplay/src/utils/futils.c delete mode 100644 ioreplay/src/utils/futils.h delete mode 100644 ioreplay/src/utils/utils.c delete mode 100644 ioreplay/src/utils/utils.h delete mode 100644 ioreplay/src/vfd.c delete mode 100644 ioreplay/src/vfd.h delete mode 100644 ioreplay/tags create mode 100644 ioriot/Makefile create mode 100644 ioriot/src/capture/capture.c create mode 100644 ioriot/src/capture/capture.h create mode 100644 ioriot/src/cleanup/cleanup.c create mode 100644 ioriot/src/cleanup/cleanup.h create mode 100644 ioriot/src/datas/amap.c create mode 100644 ioriot/src/datas/amap.h create mode 100644 ioriot/src/datas/btree.c create mode 100644 ioriot/src/datas/btree.h create mode 100644 ioriot/src/datas/hmap.c create mode 100644 ioriot/src/datas/hmap.h create mode 100644 ioriot/src/datas/list.c create mode 100644 ioriot/src/datas/list.h create mode 100644 ioriot/src/datas/rbuffer.c create mode 100644 ioriot/src/datas/rbuffer.h create mode 100644 ioriot/src/datas/stack.c create mode 100644 ioriot/src/datas/stack.h create mode 100644 ioriot/src/defaults.h create mode 100644 ioriot/src/generate/generate.c create mode 100644 ioriot/src/generate/generate.h create mode 100644 ioriot/src/generate/gioop.c create mode 100644 ioriot/src/generate/gioop.h create mode 100644 ioriot/src/generate/gparser.c create mode 100644 ioriot/src/generate/gparser.h create mode 100644 ioriot/src/generate/gprocess.c create mode 100644 ioriot/src/generate/gprocess.h create mode 100644 ioriot/src/generate/gtask.c create mode 100644 ioriot/src/generate/gtask.h create mode 100644 ioriot/src/generate/gwriter.c create mode 100644 ioriot/src/generate/gwriter.h create mode 100644 ioriot/src/generate/vsize.c create mode 100644 ioriot/src/generate/vsize.h create mode 100644 ioriot/src/init/init.c create mode 100644 ioriot/src/init/init.h create mode 100644 ioriot/src/init/itask.c create mode 100644 ioriot/src/init/itask.h create mode 100644 ioriot/src/init/ithread.c create mode 100644 ioriot/src/init/ithread.h create mode 100644 ioriot/src/macros.h create mode 100644 ioriot/src/main.c create mode 100644 ioriot/src/meta/meta.c create mode 100644 ioriot/src/meta/meta.h create mode 100644 ioriot/src/mounts.c create mode 100644 ioriot/src/mounts.h create mode 100644 ioriot/src/opcodes.h create mode 100644 ioriot/src/options.c create mode 100644 ioriot/src/options.h create mode 100644 ioriot/src/replay/replay.c create mode 100644 ioriot/src/replay/replay.h create mode 100644 ioriot/src/replay/rioop.c create mode 100644 ioriot/src/replay/rioop.h create mode 100644 ioriot/src/replay/rprocess.c create mode 100644 ioriot/src/replay/rprocess.h create mode 100644 ioriot/src/replay/rstats.c create mode 100644 ioriot/src/replay/rstats.h create mode 100644 ioriot/src/replay/rtask.c create mode 100644 ioriot/src/replay/rtask.h create mode 100644 ioriot/src/replay/rthread.c create mode 100644 ioriot/src/replay/rthread.h create mode 100644 ioriot/src/replay/rworker.c create mode 100644 ioriot/src/replay/rworker.h create mode 100644 ioriot/src/utests.c create mode 100644 ioriot/src/utests.h create mode 100644 ioriot/src/utils/futils.c create mode 100644 ioriot/src/utils/futils.h create mode 100644 ioriot/src/utils/utils.c create mode 100644 ioriot/src/utils/utils.h create mode 100644 ioriot/src/vfd.c create mode 100644 ioriot/src/vfd.h create mode 100644 ioriot/tags delete mode 100644 systemtap/src/ioreplay.stp create mode 100644 systemtap/src/ioriot.stp delete mode 100644 systemtap/src/javaioreplay.stp create mode 100644 systemtap/src/javaioriot.stp delete mode 100644 systemtap/src/targetedioreplay.stp create mode 100644 systemtap/src/targetedioriot.stp diff --git a/.gitignore b/.gitignore index 0a122f0..ab4c720 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ *.o docs/html/ docs/latex/ -ioreplay/ioreplay +ioriot/ioriot systemtap/downloads/ diff --git a/Makefile b/Makefile index 427e7bb..d5ab493 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,19 @@ -DESTDIR=/opt/ioreplay +DESTDIR=/opt/ioriot all: $(MAKE) -C systemtap - $(MAKE) -C ioreplay + $(MAKE) -C ioriot install: $(MAKE) -C systemtap install - $(MAKE) -C ioreplay install + $(MAKE) -C ioriot install uninstall: test ! -z $(DESTDIR) && test -d $(DESTDIR) && rm -Rfv $(DESTDIR) || exit 0 deinstall: uninstall clean: - $(MAKE) -C ioreplay clean + $(MAKE) -C ioriot clean $(MAKE) -C systemtap clean astyle: - $(MAKE) -C ioreplay astyle + $(MAKE) -C ioriot astyle loc: - wc -l ./systemtap/src/*.stp ./ioreplay/src/*.{h,c} ./ioreplay/src/*/*.{h,c} | tail -n 1 + wc -l ./systemtap/src/*.stp ./ioriot/src/*.{h,c} ./ioriot/src/*/*.{h,c} | tail -n 1 doxygen: doxygen ./docs/doxygen.conf diff --git a/README.md b/README.md index db18da5..be6aa8f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# I/O Replay +# I/O Riot ## Overview -I/O Replay is an I/O benchmarking tool for Linux based operating systems which captures I/O operations on a (possibly production) server in order to replay the exact same I/O operations on a load test machine. +I/O Riot is an I/O benchmarking tool for Linux based operating systems which captures I/O operations on a (possibly production) server in order to replay the exact same I/O operations on a load test machine. -I/O Replay is operated in 5 steps: +I/O Riot is operated in 5 steps: 1. Capture: Record all I/O operations over a given period of time to a capture log. 2. Initialize: Copy the log to a load test machine and initialize the load test environment. @@ -23,11 +23,11 @@ The file system fragmentation (depending on the file system type and utilisation ## Benefits -In contrast to traditional I/O benchmarking tools, I/O Replay reproduces real production I/O, and does not rely on a pre-defined set of I/O operations. +In contrast to traditional I/O benchmarking tools, I/O Riot reproduces real production I/O, and does not rely on a pre-defined set of I/O operations. -Also, I/O Replay only requires a server machine for capturing and another server machine for replaying. A traditional load test environment would usually be a distributed system which can consist of many components and machines. Such a distributed system can become quite complex which makes it difficult to isolate possible I/O bottlenecks. For example in order to trigger I/O events a client application would usually have to call a remote server application. The remote server application itself would query a database and the database would trigger the actual I/O operations in Linux. Furthermore, it is not easy to switch forth and back between hardware and OS settings. For example without a backup and restore procedure a database would most likely be corrupt after reformatting the data partitions with a different file system type. +Also, I/O Riot only requires a server machine for capturing and another server machine for replaying. A traditional load test environment would usually be a distributed system which can consist of many components and machines. Such a distributed system can become quite complex which makes it difficult to isolate possible I/O bottlenecks. For example in order to trigger I/O events a client application would usually have to call a remote server application. The remote server application itself would query a database and the database would trigger the actual I/O operations in Linux. Furthermore, it is not easy to switch forth and back between hardware and OS settings. For example without a backup and restore procedure a database would most likely be corrupt after reformatting the data partitions with a different file system type. -The benefits of I/O replay are: +The benefits of I/O Riot are: * It is easy to determine whether a new hardware type is suitable for an already existing application. * It is easy to change OS and hardware for performance tests and optimizations. @@ -41,7 +41,7 @@ The benefits of I/O replay are: # Send in patches -Patches of any kind (bug fixes, new features...) are welcome! I/O Replay is new software and not everything might be perfect yet. Also, I/O Replay is used for a very specific use case at Mimecast. It may need tuning or extension for your use case. It will grow and mature over time. +Patches of any kind (bug fixes, new features...) are welcome! I/O Riot is new software and not everything might be perfect yet. Also, I/O Riot is used for a very specific use case at Mimecast. It may need tuning or extension for your use case. It will grow and mature over time. This is also potentially a great tool just for analysing (not replaying) the I/O, therefore it would be a great opportunity to add more features related to that (e.g. more stats, filters, etc.). @@ -49,11 +49,11 @@ Future work will also include file hole support and I/O support for memory mappe # Getting started -I/O Replay consists of a set of SystemTap kernel modules (capturing I/O) and the tool ``ioreplay`` (replaying I/O). Usually you want to capture I/O from a production machine and want to replay it on a separate load testing machine. +I/O Riot consists of a set of SystemTap kernel modules (capturing I/O) and the tool ``ioriot`` (replaying I/O). Usually you want to capture I/O from a production machine and want to replay it on a separate load testing machine. ## System requirements -I/O replay has been tested on +I/O Riot has been tested on * CentOS 7.4 64Bit (latest version, all packages up to date, booted into the installed Kernel) * SystemTap (from the default CentOS repository) @@ -61,29 +61,29 @@ I/O replay has been tested on Before proceeding please ensure that the latest CentOS 7 kernel is installed and running on all machines involved. It should also be ensured that the capture machine and the load test machine have the same mount points mounted. This is to ensure that I/O is being replayed on the corresponding data drives on the load test machine. -## Compiling and installing ioreplay +## Compiling and installing ioriot -I/O Replay has to be installed on all machines involved. To install I/O Replay perform the following steps: +I/O Riot has to be installed on all machines involved. To install I/O Riot perform the following steps: ```sh sudo yum install gcc systemtap yum-utils kernel-devel-$(uname -r) sudo debuginfo-install kernel-$(uname -r) make && sudo make install -export PATH=$PATH:/opt/ioreplay/bin +export PATH=$PATH:/opt/ioriot/bin ``` -This will install the ``ioreplay`` utility to ``/opt/ioreplay/bin/`` and the SystemTap kernel modules to ``/opt/ioreplay/systemtap/``. Run ``ioreplay -h`` to print out a brief help. +This will install the ``ioriot`` utility to ``/opt/ioriot/bin/`` and the SystemTap kernel modules to ``/opt/ioriot/systemtap/``. Run ``ioriot -h`` to print out a brief help. -However, best practise is not to install any compilers on a production machine. You can either compile I/O Replay from scratch on all machines involved like shown above or only compile it on a build machine and distribute the ``/opt/ioreplay`` directory to the remaining machines. In the latter case you will also need to install the ``systemtap-runtime`` package as an additional dependency. +However, best practise is not to install any compilers on a production machine. You can either compile I/O Riot from scratch on all machines involved like shown above or only compile it on a build machine and distribute the ``/opt/ioriot`` directory to the remaining machines. In the latter case you will also need to install the ``systemtap-runtime`` package as an additional dependency. -In case you decided to deinstall I/O Replay you can do so by running +In case you decided to deinstall I/O Riot you can do so by running ```sh -sudo ioreplay -P # purges all test files created by ioreplay +sudo ioriot -P # purges all test files created by ioriot sudo make uninstall ``` -# Operating I/O Replay +# Operating I/O Riot ## 1. Capture @@ -93,7 +93,7 @@ The following steps are required to capture all I/O operation of the entire (Lin * 2) Run: ```sh -sudo ioreplay -c ~/io.capture +sudo ioriot -c ~/io.capture ``` * 3) Start all applications again. @@ -102,13 +102,13 @@ sudo ioreplay -c ~/io.capture To capture only I/O caused by Java process run: ```sh -sudo ioreplay -c ~/io.capture -m javaioreplay.ko +sudo ioriot -c ~/io.capture -m javaioriot.ko ``` To capture the I/O of a specific process run the following respectively: ```sh -sudo ioreplay -c ~/io.capture -m targetedioreplay.ko -p PID +sudo ioriot -c ~/io.capture -m targetedioriot.ko -p PID ``` The resulting capture log looks like this and can be multiple GB in size: @@ -168,7 +168,7 @@ sudo mount -t tmpfs -o size=32g tmpfs /mnt/ramdisk Make sure that there is enough system memory available for such a RAMdisk and all the processes running on the machine. Eventually, RAM will be taken away from the Linux caches which potentially could decrease system I/O performance. Run the following command to capture to the RAMdisk respectively: ```sh -sudo ioreplay -c /mnt/ramdisk/io.capture +sudo ioriot -c /mnt/ramdisk/io.capture ``` ## 2. Initialize @@ -181,59 +181,59 @@ After producing ``io.capture`` it must be pre-processed. The resulting replay lo * Use of internal opcodes rather than strings (e.g. ``30`` instead of ``open``) for faster parsing. * All operations on unknown file handles are _removed_. * All incomplete or corrupt lines from the capture file are ignored. There may be corrupt lines in the capture file because SystemTap may skips a very few probe points if it decides that capturing I/O is causing too much overhead. -* Rewrite of all file paths. ``ioreplay`` adds ``/.ioreplay/NAME`` to all file paths for each file system mount point. +* Rewrite of all file paths. ``ioriot`` adds ``/.ioriot/NAME`` to all file paths for each file system mount point. To generate the the replay log ``io.replay`` from the capture log ``io.capture`` run: ```sh -sudo ioreplay -c io.capture -r io.replay -n NAME -u USER +sudo ioriot -c io.capture -r io.replay -n NAME -u USER ``` -In which NAME is a freely chosen name and USER must be a valid system user. It is the system user under which the replay test will run. This command also creates all required top level directories such as ``/.ioreplay/NAME/``, ``/mnt/.ioreplay/NAME/`` in all mounted file systems. These are the directories where the replay test will read/write files from/to. These directories will belong to user USER. +In which NAME is a freely chosen name and USER must be a valid system user. It is the system user under which the replay test will run. This command also creates all required top level directories such as ``/.ioriot/NAME/``, ``/mnt/.ioriot/NAME/`` in all mounted file systems. These are the directories where the replay test will read/write files from/to. These directories will belong to user USER. -``ioreplay`` will filter out many operations, especially all operations on pseudo file systems (e.g. sysfs, procfs), as it does not make a lot of sense to replay I/O on these file systems. Also, I/O operations on unknown file handles will be filtered out as well. This can happen when we start capturing the I/O *after* an application already opened a file. As a result we won't see how the application opened that file. The best practise is to stop all applications on the machine first, start capturing the I/O, and start all applications again. This may be improved in future releases of I/O Replay. +``ioriot`` will filter out many operations, especially all operations on pseudo file systems (e.g. sysfs, procfs), as it does not make a lot of sense to replay I/O on these file systems. Also, I/O operations on unknown file handles will be filtered out as well. This can happen when we start capturing the I/O *after* an application already opened a file. As a result we won't see how the application opened that file. The best practise is to stop all applications on the machine first, start capturing the I/O, and start all applications again. This may be improved in future releases of I/O Riot. The resulting replay log will look like this: At the first line there is a meta header. It contains information about the test configuration. The meta header is followed by all the I/O operations. At the end of the file is the INIT section. It lists all files (also their sizes) and directories required to be present before replaying the I/O. ```sh -#|num_timelines=509591|num_mapped_pids=19189|num_mapped_fds=4292067|num_lines=55040114|replay_version=1|user=ioreplayuser|name=test0|init_offset=2578735248| -23|1|1|0|0|30|11|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-9:1|438|0|open@31| +#|num_timelines=509591|num_mapped_pids=19189|num_mapped_fds=4292067|num_lines=55040114|replay_version=1|user=ioriotuser|name=test0|init_offset=2578735248| +23|1|1|0|0|30|11|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-9:1|438|0|open@31| 23|1|1|0|0|0|11|0|fstat@32| 23|1|1|0|0|10|11|12|read@33| 23|1|1|0|0|0|11|0|fstat@34| 23|1|1|0|0|72|11|0|1|12|lseek@35| 23|1|1|0|0|10|11|0|read@36| 23|1|1|0|0|50|11|0|close@37| -23|2|1|0|0|30|12|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-8:1|438|0|open@38| +23|2|1|0|0|30|12|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-8:1|438|0|open@38| 23|2|1|0|0|0|12|0|fstat@39| 23|2|1|0|0|10|12|12|read@40| 23|2|1|0|0|0|12|0|fstat@41| 23|2|1|0|0|72|12|0|1|12|lseek@42| 23|2|1|0|0|10|12|0|read@43| 23|2|1|0|0|50|12|0|close@44| -23|3|1|0|0|30|13|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-9:0|438|0|open@45| +23|3|1|0|0|30|13|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-9:0|438|0|open@45| 23|3|1|0|0|0|13|0|fstat@46| 23|3|1|0|0|10|13|12|read@47| 23|3|1|0|0|0|13|0|fstat@48| 23|3|1|0|0|72|13|0|1|12|lseek@49| 23|3|1|0|0|10|13|0|read@50| 23|3|1|0|0|50|13|0|close@51| -23|4|1|0|0|30|14|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-7:1|438|0|open@52| +23|4|1|0|0|30|14|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-7:1|438|0|open@52| 23|4|1|0|0|0|14|0|fstat@53| 23|4|1|0|0|10|14|12|read@54| 23|4|1|0|0|0|14|0|fstat@55| 23|4|1|0|0|72|14|0|1|12|lseek@56| 23|4|1|0|0|10|14|0|read@57| 23|4|1|0|0|50|14|0|close@58| -23|5|1|0|0|30|15|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-8:0|438|0|open@59| +23|5|1|0|0|30|15|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-8:0|438|0|open@59| 23|5|1|0|0|0|15|0|fstat@60| 23|5|1|0|0|10|15|12|read@61| 23|5|1|0|0|0|15|0|fstat@62| 23|5|1|0|0|72|15|0|1|12|lseek@63| 23|5|1|0|0|10|15|0|read@64| 23|5|1|0|0|50|15|0|close@65| -23|6|1|0|0|30|16|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-6:1|438|0|open@66| +23|6|1|0|0|30|16|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-6:1|438|0|open@66| 23|6|1|0|0|0|16|0|fstat@67| 23|6|1|0|0|10|16|12|read@68| 23|6|1|0|0|0|16|0|fstat@69| @@ -241,18 +241,18 @@ The resulting replay log will look like this: At the first line there is a meta . . #INIT -0|1|688|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/8/b_dv01_11_vd11-11_a|@55290437 -0|1|2592|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/3/b_dv01_11_vd11-11_b|@33907067 -0|1|768|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/d/b_dv01_11_vd11-11_c|@64247527 -0|1|1440|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/0/b_dv01_11_vd11-11_d|@2014896 -0|1|960|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/9/b_dv01_11_vd11-11_e|@17724079 -0|1|928|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/1/b_dv01_11_vd11-11_f|@4534389 -0|1|1712|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/5/b_dv01_11_vd11-11_g|@2738458 -0|1|784|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/b/b_dv01_11_vd11-11_h|@21136612 -0|1|624|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/6/b_dv01_11_vd11-11_i|@24683427 -0|1|672|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/9/b_dv01_11_vd11-11_j|@12584061 -0|1|336|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/5/b_dv01_11_vd11-11_k|@7737434 -0|1|12|/mnt/06/.ioreplay/test0/bmnt/tmp/b|@42498106 +0|1|688|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/8/b_dv01_11_vd11-11_a|@55290437 +0|1|2592|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/3/b_dv01_11_vd11-11_b|@33907067 +0|1|768|/mnt/14/.ioriot/test0/bmnt/2/20171101/b/d/b_dv01_11_vd11-11_c|@64247527 +0|1|1440|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/0/b_dv01_11_vd11-11_d|@2014896 +0|1|960|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/9/b_dv01_11_vd11-11_e|@17724079 +0|1|928|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/1/b_dv01_11_vd11-11_f|@4534389 +0|1|1712|/mnt/14/.ioriot/test0/bmnt/2/20171101/b/5/b_dv01_11_vd11-11_g|@2738458 +0|1|784|/mnt/14/.ioriot/test0/bmnt/2/20171101/b/b/b_dv01_11_vd11-11_h|@21136612 +0|1|624|/mnt/14/.ioriot/test0/bmnt/2/20171101/b/6/b_dv01_11_vd11-11_i|@24683427 +0|1|672|/mnt/14/.ioriot/test0/bmnt/2/20171101/b/9/b_dv01_11_vd11-11_j|@12584061 +0|1|336|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/5/b_dv01_11_vd11-11_k|@7737434 +0|1|12|/mnt/06/.ioriot/test0/bmnt/tmp/b|@42498106 . . . @@ -264,17 +264,17 @@ The resulting replay log will look like this: At the first line there is a meta It is very likely that the replay test wants to access already existing files. Therefore it has to be ensured that all of these exist already before starting the test. To create all files and directories required by the test run the following command: ```sh -sudo ioreplay -i io.replay +sudo ioriot -i io.replay ``` -For that ``ioreplay`` makes use of the INIT section in ``io.replay``. +For that ``ioriot`` makes use of the INIT section in ``io.replay``. ## 3. Replay To replay the log run: ```sh -sudo ioreplay -r io.replay +sudo ioriot -r io.replay ``` It is beneficial to read ``io.replay`` from RAMdisk so that we are not interfering so much with the system I/O. @@ -284,12 +284,12 @@ It is beneficial to read ``io.replay`` from RAMdisk so that we are not interferi It is posisble to initialise the test and run the test with one single command, just replace option `-r` with `-R`: ```sh -sudo ioreplay -R io.replay +sudo ioriot -R io.replay ``` *Speed factor* -By default `ioreplay` tries to replay all I/O operations as fast as it can. To replay the I/O at a different speed it is possible to configure the speed factor by using the `-s` command line option. +By default `ioriot` tries to replay all I/O operations as fast as it can. To replay the I/O at a different speed it is possible to configure the speed factor by using the `-s` command line option. The following pseudo code demonstrates how the speed factor affects the replay speed. Here `current_time` represents the current time while replaying the I/O, `time_in_log` represents the time as logged in `io.replay` and `time_ahead` indicates whether the replay is too quick or not. @@ -306,7 +306,7 @@ A speed factor of `0` is interpreted as "replay as fast as possible". A speed fa In order to replay the I/O in original speed the factor of `1` can be used as follows: ```sh -sudo ioreplay -R io.replay -s 1 +sudo ioriot -R io.replay -s 1 ``` ## 4. Analyse @@ -322,14 +322,14 @@ It is important to understand the I/O statistics observed. It is possible to rep To purge all temporally data files of all tests run ```sh -sudo ioreplay -P +sudo ioriot -P ``` -Note: It's not required to cleanup any test data manually when you intend to re-run a test or run a new test. During initialization (``-i`` or ``-R`` switch) ``ioreplay`` will automatically move all old data to ``.ioreplay/.trash/`` sub-folders. The data will be ignored there. However, once you intend to completely delete all test files and directories (e.g. you run out of disk space or want to deinstall ``ioreplay`` you should purge them with ``-P`` as shown above. +Note: It's not required to cleanup any test data manually when you intend to re-run a test or run a new test. During initialization (``-i`` or ``-R`` switch) ``ioriot`` will automatically move all old data to ``.ioriot/.trash/`` sub-folders. The data will be ignored there. However, once you intend to completely delete all test files and directories (e.g. you run out of disk space or want to deinstall ``ioriot`` you should purge them with ``-P`` as shown above. ## Supported file systems -Currently I/O Replay supports replaying I/O on ``ext2``, ``ext3``, ``ext4`` and ``xfs``. However, it should be straightforward add additional file systems. +Currently I/O Riot supports replaying I/O on ``ext2``, ``ext3``, ``ext4`` and ``xfs``. However, it should be straightforward add additional file systems. ## Supported syscalls @@ -391,4 +391,4 @@ exit_group - To detect process termination (closing all open file handles) ## Source code documentation -The documentation of the source code can be generated via the Doxygen Framework. To install doxygen run ``sudo yum install doxygen`` and to generate the documentation run ``make doxygen`` in the top level source directory. Once done, the resulting documentation can be found in the ``docs/html`` subfolder of the project. It is worthwhile to start from ``ioreplay/src/main.c`` and read your way through. Functions are generally documented in the header files. Exceptions are static functions which don't have any separate declarations. +The documentation of the source code can be generated via the Doxygen Framework. To install doxygen run ``sudo yum install doxygen`` and to generate the documentation run ``make doxygen`` in the top level source directory. Once done, the resulting documentation can be found in the ``docs/html`` subfolder of the project. It is worthwhile to start from ``ioriot/src/main.c`` and read your way through. Functions are generally documented in the header files. Exceptions are static functions which don't have any separate declarations. diff --git a/docs/doxygen.conf b/docs/doxygen.conf index 340f731..8f014b6 100644 --- a/docs/doxygen.conf +++ b/docs/doxygen.conf @@ -32,7 +32,7 @@ DOXYFILE_ENCODING = UTF-8 # title of most generated pages and in a few other places. # The default value is: My Project. -PROJECT_NAME = "I/O Replay" +PROJECT_NAME = "I/O Riot" # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version @@ -791,7 +791,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = README.md ./ioreplay/src +INPUT = README.md ./ioriot/src # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/ioreplay/Makefile b/ioreplay/Makefile deleted file mode 100644 index 67f4421..0000000 --- a/ioreplay/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -#DEBUG=-g3 -ggdb3 -pg -NAME=ioreplay -LIBS=-pthread -CFLAGS=-Wall -std=gnu99 -pedantic -STATIC=#-static -DESTDIR=/opt/ioreplay/bin -SRCS=$(wildcard src/*.c src/*/*.c) -HDRS=$(SRCS:.c=.h) -OBJS=$(SRCS:.c=.o) -all: compile -quick: clean ctags compile sudo_install -cshell: compile - gdb -ex='break main; run' --args ./$(NAME) -test: compile - gdb -ex=run --args ./$(NAME) -U -compile: $(OBJS) - $(CC) $(STATIC) $(DEBUG) $(LIBS) $(OBJS) -o $(NAME) -%.o: %.c %.h - $(CC) $(STATIC) $(DEBUG) $(LIBS) -c $(CFLAGS) $< -o $@ -clean: - rm -v ioreplay ./src/*.o ./src/*/*.o 2>/dev/null || exit 0 -install: - test ! -d $(DESTDIR) && mkdir -p $(DESTDIR) || exit 0 - cp -v $(NAME) $(DESTDIR) - @echo "Don't forget to add $(DESTDIR) to your PATH as follows:" - @echo " export PATH=\$$PATH:$(DESTDIR)" -uninstall: - test ! -z "$(DESTDIR)" && test -f $(DESTDIR)/$(NAME) && rm -v $(DESTDIR)/$(NAME) || exit 0 -deinstall: uninstall -astyle: - astyle -n --style=linux src/*.h src/*/*.h - astyle -n --style=linux src/*.c src/*/*.c -todo: - fgrep ../TODO ./src/* -ctags: - ctags ./src/*.{h,c} ./src/*/*.{h,c} diff --git a/ioreplay/src/capture/capture.c b/ioreplay/src/capture/capture.c deleted file mode 100644 index 0ac336b..0000000 --- a/ioreplay/src/capture/capture.c +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "capture.h" - -#include - -status_e capture_run(options_s *opts) -{ - int status = 0; - struct utsname uts; - - // To make it nicer we should iterate over PATH instead - char *staprun_paths[3] = { - "/usr/bin/staprun", - "/usr/local/bin/staprun", - "/bin/staprun" - }; - int num_staprun_paths = 3; - - if (0 != uname(&uts)) { - Errno("Could not identify release of currently running Kernel!"); - } - - Put("Release of currently running Kernel: %s", uts.release); - char modules_dir[128]; - sprintf(modules_dir, "/opt/ioreplay/systemtap/%s", uts.release); - Put("Changing directory to module path: %s/", modules_dir); - - if (0 != chdir(modules_dir)) { - Errno("Could not change into '%s', please ensure that the compiled " - "SystemTap modules correspond to the currently running Kernel " - "and that these are installed properly!\n", - modules_dir); - } - - if (0 != access(opts->module, R_OK)) { - Errno("Module '%s/%s' can't be read, please make sure that the " - "SystemTap Kernel modules are installed!", - modules_dir, opts->module); - } - - char *staprun_path = NULL; - for (int i = 0; i < num_staprun_paths; ++i) { - if (0 == access(staprun_paths[i], X_OK)) { - staprun_path = staprun_paths[i]; - //Put("SystemTap command path: %s", staprun_path); - break; - } - } - - if (staprun_path == NULL) { - Errno("Can't find 'staprun' command, please ensure to have the SystemTap " - "runtime (usually package 'systemtap-runtime') installed!"); - } - - char staprun_command[128]; - if (opts->pid >= 0) { - sprintf(staprun_command, "%s %s -v -o %s -x %d", staprun_path, opts->module, - opts->capture_file, opts->pid); - } else { - sprintf(staprun_command, "%s %s -v -o %s", staprun_path, opts->module, - opts->capture_file); - } - - Out("NOTICE: It is good practise first to stop all processes, then to "); - Out("start capturing, and then to start all processes again. The reason "); - Out("is that processes may have already open file handles. In that case "); - Out("I/O Replay would be unable to replay these! This may be improved "); - Put("in a future release!"); - Put("To abort capturing now send Ctrl+C, otherwise wait 1h"); - Put("Capturing I/O via: '%s'", staprun_command); - - char buf[1024]; - FILE *fp; - - if ((fp = popen(staprun_command, "r")) == NULL) { - Errno("Unable to invoke staprun command!"); - } - while (fgets(buf, 1024, fp) != NULL) - Out("stapio: %s", buf); - - if (0 != pclose(fp)) { - Error("Problems invoking staprun command!"); - } - - return status; -} diff --git a/ioreplay/src/capture/capture.h b/ioreplay/src/capture/capture.h deleted file mode 100644 index 7718d3e..0000000 --- a/ioreplay/src/capture/capture.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef CAPTURE_H -#define CAPTURE_H - -#include "../defaults.h" -#include "../utils/futils.h" -#include "../options.h" - -/** - * @brief Captures I/O to a .capture file by using stap from SystemTap - * - * @param opts The options object - * @return SUCCESS if everything went fine - */ -status_e capture_run(options_s *opts); - -#endif // CAPTURE_H diff --git a/ioreplay/src/cleanup/cleanup.c b/ioreplay/src/cleanup/cleanup.c deleted file mode 100644 index 570f8a7..0000000 --- a/ioreplay/src/cleanup/cleanup.c +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "cleanup.h" - -#include "../mounts.h" - -status_e cleanup_run(options_s *opts) -{ - set_limits_drop_root(opts->user); - mounts_s *m = mounts_new(opts); - - if (opts->purge) - mounts_purge(m); - else - mounts_trash(m); - - return SUCCESS; -} diff --git a/ioreplay/src/cleanup/cleanup.h b/ioreplay/src/cleanup/cleanup.h deleted file mode 100644 index 127badf..0000000 --- a/ioreplay/src/cleanup/cleanup.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef CLEANUP_H -#define CLEANUP_H - -#include "../defaults.h" -#include "../options.h" - -/** - * @brief Cleans up all files and directories of a given test - * - * @brief opts The options object - * @return SUCCESS in case everything went fine - */ -status_e cleanup_run(options_s *opts); - -#endif // CLEANUP_H diff --git a/ioreplay/src/datas/amap.c b/ioreplay/src/datas/amap.c deleted file mode 100644 index 806a3f8..0000000 --- a/ioreplay/src/datas/amap.c +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "amap.h" - -/** - * @brief Creates a new array map - * - * @param size The array map size - * @param mmapped true if the memory should be mmapped - * @return The new amap object - */ -static amap_s *_amap_new(long size, bool mmapped) -{ - amap_s *a = NULL; - void ***arrays = NULL; - - // Calculate a multiple of 1024, but at least in size of 'size'. - if (size % 1024 != 0) { - size = 1024*(1+(long)(size/1024)); - } - - if (size < 1) { - Error("Size overflow"); - } - - int num_arrays = size / AMAP_MAX_ARRAY_LENGTH; - - if (mmapped) { - a = Mmapshared(amap_s); - arrays = Cmapshared(num_arrays, void**); - } else { - a = Malloc(amap_s); - arrays = Calloc(num_arrays, void**); - } - - for (int i = 0; i < num_arrays; ++i) { - if (mmapped) { - //Put("%d", AMAP_MAX_ARRAY_LENGTH); - arrays[i] = Cmapshared(AMAP_MAX_ARRAY_LENGTH, void*); - } else { - arrays[i] = Calloc(AMAP_MAX_ARRAY_LENGTH, void*); - } - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - arrays[i][j] = NULL; - } - } - - a->arrays = arrays; - a->num_arrays = num_arrays; - a->size = size; - a->data_destroy = NULL; - a->mmapped = mmapped; - - return a; -} - -/** - * @brief Creates a new array map - * - * @param size The array map size - * @return The new amap object - */ -amap_s* amap_new(const long size) -{ - return _amap_new(size, false); -} - -/** - * @brief Creates a new mmapped array map - * - * @param size The array map size - * @return The new amap object - */ -amap_s* amap_new_mmapped(const long size) -{ - return _amap_new(size, true); -} - -/** - * @brief Destroys a mmap object - * - * @a The new amap object - */ -void amap_destroy(amap_s* a) -{ - if (!a) { - return; - } - - // Don't bother, the mmapped version of amap will stay alive until - // process terminations. And after process termination everything - // will be cleaned up automatically by Linux. - if (a->mmapped) { - return; - } - - for (int i = 0; i < a->num_arrays; ++i) { - if (a->data_destroy) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) - if (a->arrays[i][j]) { - a->data_destroy(a->arrays[i][j]); - } - } - free(a->arrays[i]); - } - free(a->arrays); - free(a); -} - -/** - * @brief Resets a mmap object - * - * This resets all entries to NULL. - * - * @a The new amap object - */ -void amap_reset(amap_s* a) -{ - for (int i = 0; i < a->num_arrays; ++i) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - if (a->data_destroy) { - if (a->arrays[i][j]) { - a->data_destroy(a->arrays[i][j]); - } - } - a->arrays[i][j] = NULL; - } - } -} - -int amap_set(amap_s *a, const long position, void* value) -{ - if (position >= a->size) - return -1; - int which_array = position / AMAP_MAX_ARRAY_LENGTH; - int array_pos = position % AMAP_MAX_ARRAY_LENGTH; - a->arrays[which_array][array_pos] = value; - return 0; -} - -void* amap_get(amap_s *a, const long position) -{ - if (position >= a->size) - return NULL; - int which_array = position / AMAP_MAX_ARRAY_LENGTH; - int array_pos = position % AMAP_MAX_ARRAY_LENGTH; - return a->arrays[which_array][array_pos]; -} - -void* amap_unset(amap_s *a, const long position) -{ - if (position >= a->size) - return NULL; - int which_array = position / AMAP_MAX_ARRAY_LENGTH; - int array_pos = position % AMAP_MAX_ARRAY_LENGTH; - void *value = a->arrays[which_array][array_pos]; - a->arrays[which_array][array_pos] = NULL; - return value; -} - -void amap_run_cb(amap_s *a, void (*cb)(void *data)) -{ - for (int i = 0; i < a->num_arrays; ++i) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - if (a->arrays[i][j]) - cb(a->arrays[i][j]); - } - } -} - -void amap_print(amap_s* a) -{ - Put("amap_s (%p):", (void*)a); - Put("\tmmapped: %d", a->mmapped); - Put("\tmax_array_length: %d", AMAP_MAX_ARRAY_LENGTH); - Put("\tnum_arrays: %d", a->num_arrays); - Put("\tsize: %lu", a->size); - Out("\toccupied slots: "); - for (int i = 0; i < a->num_arrays; ++i) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - if (a->arrays[i][j] != NULL) { - Out("%d:%d ", i, j); - } - } - } - Out("\n"); -} - -void _amap_test(amap_s *a) -{ - assert(0 == amap_set(a, 0, (void*)10)); - assert(0 == amap_set(a, 1, (void*)11)); - assert(0 == amap_set(a, 2, (void*)12)); - assert(0 == amap_set(a, 3, (void*)a)); - assert(10 == (long) amap_get(a, 0)); - assert(11 == (long) amap_get(a, 1)); - assert(12 == (long) amap_get(a, 2)); - assert(a == amap_get(a, 3)); - - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH-1, (void*) 23)); - assert(23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH-1)); - - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH, (void*) 42)); - assert(42 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH)); - - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2-1, (void*) (23+42))); - assert(42+23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH*2-1)); - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2, (void*) 23)); - - - assert(NULL == amap_get(a, 1024*1024*9-1)); - assert(0 == amap_set(a, 1024*1024*9-1, (void*) 0x1)); - assert(0x1 == (long) amap_get(a, 1024*1024*9-1)); - assert(0x1 == (long) amap_unset(a, 1024*1024*9-1)); - assert(NULL == amap_get(a, 1024*1024*9-1)); - - assert(0 == amap_set(a, 1024*1024*9, (void*) 100)); - assert(100 == (long) amap_get(a, 1024*1024*9)); - - assert(0 == amap_set(a, 1024*1024*9+1, (void*) 101)); - assert(101 == (long) amap_get(a, 1024*1024*9+1)); - - assert(0 == amap_set(a, 1024*1024*10-2, (void*) 102)); - assert(102 == (long) amap_get(a, 1024*1024*10-2)); - - assert(0 == amap_set(a, 1024*1024*10-1, a)); - assert(a == amap_get(a, 1024*1024*10-1)); - //amap_print(a); - - assert(a == amap_unset(a, 1024*1024*10-1)); - assert(a != amap_unset(a, 1024*1024*10-1)); - //amap_print(a); -} - -void amap_test(void) -{ - // First test the non-mmapped version - amap_s* a = amap_new(1024*1024*10); - _amap_test(a); - amap_destroy(a); - - // Now test the mapped version - a = amap_new_mmapped(1024*1024*10); - _amap_test(a); - amap_destroy(a); - - // Another test with non-alligned size - a = amap_new(1024*1024*10+1); - _amap_test(a); - amap_destroy(a); -} - diff --git a/ioreplay/src/datas/amap.h b/ioreplay/src/datas/amap.h deleted file mode 100644 index 882a7c5..0000000 --- a/ioreplay/src/datas/amap.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef AMAP_H -#define AMAP_H - -#include "../defaults.h" - -#define AMAP_MAX_ARRAY_LENGTH 1024*8 - -/** - * @brief Implements an array map data structure - * - * This array map can hold a HUGE amount of entries by allocating multiple - * smaller arrays. There are two version of the amap data structure available: - * a memory mapped (mmap) and a normal version. The memory mapped version can - * be used for IPC between various processes. - */ -typedef struct amap_s_ { - void*** arrays; /**< The pointers to the amap arrays */ - int num_arrays; /**< The amount of arrays used in the amap */ - long size; /**< The total size/capacity of the amap */ - bool mmapped; /**< True if amap is memory mapped */ - void (*data_destroy)(void *data); /**< Callback to destroy all elements */ -} amap_s; - -amap_s* amap_new(const long size); -amap_s* amap_new_mmapped(const long size); -int amap_set(amap_s *a, const long position, void* value); -void* amap_get(amap_s *a, const long position); -void* amap_unset(amap_s *a, const long position); -void amap_print(amap_s *a); -void amap_destroy(amap_s *a); -void amap_reset(amap_s *a); -void amap_run_cb(amap_s *a, void (*cb)(void *data)); -void amap_test(void); - -#endif // AMAP_H diff --git a/ioreplay/src/datas/btree.c b/ioreplay/src/datas/btree.c deleted file mode 100644 index da5da48..0000000 --- a/ioreplay/src/datas/btree.c +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "btree.h" - -btree_s* btree_new() -{ - btree_s *b = Malloc(btree_s); - *b = (btree_s) { - .root = NULL, .size = 0 - }; - return b; -} - -void btree_destroy(btree_s* b) -{ - if (b->root) - btreelem_destroy_r(b->root); - free(b); -} - -void btree_destroy2(btree_s* b) -{ - if (b->root) - btreelem_destroy_r2(b->root); - free(b); -} - -int btree_insert(btree_s* b, int key, void *data) -{ - int ret = 1; - - if (b->root == NULL) { - b->root = btreelem_new(key, data); - ret = 0; - } else { - ret = btreelem_insert_r(b->root, key, data); - } - - if (ret == 0) { - b->size++; - } - - return ret; -} - -void* btree_get(btree_s* b, int key) -{ - if (b->root == NULL) - return NULL; - - return btreelem_get_r(b->root, key); -} - -void btree_print(btree_s* b) -{ - btreelem_print_r(b->root, 0); -} - -btreelem_s* btreelem_new(int key, void *data) -{ - btreelem_s *e = Malloc(btreelem_s); - *e = (btreelem_s) { - .key = key, .data = data, .left = NULL, .right = NULL - }; - return e; -} - -void btreelem_destroy_r(btreelem_s* e) -{ - if (e->left) { - btreelem_destroy_r(e->left); - } - if (e->right) { - btreelem_destroy_r(e->right); - } - - free(e); -} - -void btreelem_destroy_r2(btreelem_s* e) -{ - if (e->left) - btreelem_destroy_r(e->left); - if (e->right) - btreelem_destroy_r(e->right); - if (e->data) - btree_destroy(e->data); - - free(e); -} - -int btreelem_insert_r(btreelem_s* e, int key, void *data) -{ - int ret = 0; - - if (e->key == key) { - ret = 1; - } - - else if (e->key > key) { - if (e->left == NULL) { - e->left = btreelem_new(key, data); - } else { - ret = btreelem_insert_r(e->left, key, data); - } - } - - else { - if (e->right == NULL) { - e->right = btreelem_new(key, data); - } else { - ret = btreelem_insert_r(e->right, key, data); - } - } - - return ret; -} - -void* btreelem_get_r(btreelem_s* e, int key) -{ - void *data = NULL; - - if (e->key == key) { - data = e->data; - } - - else if (e->key > key) { - if (e->left) { - data = btreelem_get_r(e->left, key); - } - } - - else { - if (e->right) { - data = btreelem_get_r(e->right, key); - } - } - - return data; -} - -void btreelem_print_r(btreelem_s* e, int depth) -{ - for (int i = 0; i < depth; ++i) { - Out(" "); - } - Put("%d\n", e->key); - - if (e->left) { - btreelem_print_r(e->left, depth); - } - - if (e->right) { - btreelem_print_r(e->right, depth+1); - } -} - diff --git a/ioreplay/src/datas/btree.h b/ioreplay/src/datas/btree.h deleted file mode 100644 index 55da560..0000000 --- a/ioreplay/src/datas/btree.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef BTREE_H -#define BTREE_H - -#include "../defaults.h" - -/** - * @brief This defines an element of the binary tree data structure - */ -typedef struct btreelem_ { - struct btreelem_ *left; /**< The next element to the left */ - struct btreelem_ *right; /**< The next element to the right */ - int key; /**< The key of the element */ - void *data; /**< A pointer to the data stored in this element */ -} btreelem_s; - -/** - * @brief This defines a binary tree data structure. - */ -typedef struct btree_s_ { - btreelem_s *root; /**< The root element */ - int size; /**< The current size of the binary tree */ -} btree_s; - -btree_s* btree_new(); -void btree_destroy(btree_s *b); -void btree_destroy2(btree_s *b); -int btree_insert(btree_s *b, int key, void *data); -void* btree_get(btree_s *b, int key); -void btree_print(btree_s *b); - -btreelem_s* btreelem_new(int key, void *data); -void btreelem_destroy_r(btreelem_s *e); -void btreelem_destroy_r2(btreelem_s *e); -int btreelem_insert_r(btreelem_s *e, int key, void *data); -void* btreelem_get_r(btreelem_s *e, int key); -void btreelem_print_r(btreelem_s *e, int depth); - -#endif // BTREE_H diff --git a/ioreplay/src/datas/hmap.c b/ioreplay/src/datas/hmap.c deleted file mode 100644 index 96c373e..0000000 --- a/ioreplay/src/datas/hmap.c +++ /dev/null @@ -1,362 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "hmap.h" - -#define _Using_string_keys h->keys != NULL - -unsigned int hmap_get_addr(hmap_s *h, char *key) -{ - unsigned long hash = 5381; - int len = strlen(key); - - for (int i = 0; i < len; ++i) { - hash = ((hash << 5) + hash) + key[i]; /* hash * 33 + c */ - } - - return (unsigned int) (hash % h->size); -} - -unsigned int hmap_get_addr_l(hmap_s *h, const long key) -{ - return (unsigned int) (key % h->size); -} - -hmap_s *_hmap_new(unsigned int init_size) -{ - hmap_s *h = Malloc(hmap_s); - - h->size = init_size; - h->data = Calloc(init_size, void*); - h->l = Calloc(init_size, list_s*); - h->data_destroy = NULL; - h->keys = NULL; - h->keys_l = NULL; - - Mset(h->data, 0, init_size, void*); - Mset(h->l, 0, init_size, list_s*); - - return h; -} - -hmap_s *hmap_new(unsigned int init_size) -{ - hmap_s *h = _hmap_new(init_size); - h->keys = Calloc(init_size, char*); - Mset(h->keys, 0, init_size, char*); - - return h; -} - -hmap_s *hmap_new_l(unsigned int init_size) -{ - hmap_s *h = _hmap_new(init_size); - h->keys_l = Calloc(init_size, int); - Mset(h->keys_l, -1, init_size, int); - - return h; -} - -void hmap_destroy(hmap_s *h) -{ - for (int i = 0; i < h->size; ++i) { - if (h->l[i]) { - list_s *l = h->l[i]; - if (h->data_destroy) - l->data_destroy = h->data_destroy; - list_destroy(h->l[i]); - } - if (h->data[i] && h->data_destroy) { - h->data_destroy(h->data[i]); - } - } - - free(h->data); - if (h->keys) - free(h->keys); - if (h->keys_l) - free(h->keys_l); - free(h->l); - free(h); - - return; -} - -int hmap_insert(hmap_s *h, char *key, void *data) -{ - if (data == NULL) { - Error("insert data can not be NULL"); - } - - int addr = hmap_get_addr(h, key); - - if (h->data[addr]) { - - if (strcmp(key, h->keys[addr]) == 0) { - // Key already exists - return 0; - } - - // There is already data, collision, create a linked list - list_s *l = h->l[addr] = list_new(); - list_key_insert(l, h->keys[addr], h->data[addr]); - list_key_insert(l, key, data); - - // Not needed anymore, as the elements are in the linked list now. - free(h->keys[addr]); - h->data[addr] = h->keys[addr] = NULL; - - return 1; - - } else if (h->l[addr]) { - // There was a collision at this address before. Insert - // the element to the linked list. Returns 0 if key is already - // in the list (no additional insert made) or 1 otherwise. - return list_key_insert(h->l[addr], key, data); - } - - // New entry on a collision free address - h->data[addr] = data; - h->keys[addr] = Clone(key); - - return 1; -} - -int hmap_insert_l(hmap_s *h, const long key, void *data) -{ - if (data == NULL) { - Error("insert data can not be NULL"); - } - - int addr = hmap_get_addr_l(h, key); - - if (h->data[addr]) { - - if (key == h->keys_l[addr]) { - // Key already exists - ret