From 26b3b3e368a79ce29df732ea04e72a4c002ae2ce Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Tue, 6 Mar 2018 17:38:59 +0000 Subject: rename into ioriot --- .gitignore | 2 +- Makefile | 12 +- README.md | 108 ++--- docs/doxygen.conf | 4 +- ioreplay/Makefile | 36 -- ioreplay/src/capture/capture.c | 99 ----- ioreplay/src/capture/capture.h | 30 -- ioreplay/src/cleanup/cleanup.c | 30 -- ioreplay/src/cleanup/cleanup.h | 29 -- ioreplay/src/datas/amap.c | 264 ------------ ioreplay/src/datas/amap.h | 49 --- ioreplay/src/datas/btree.c | 169 -------- ioreplay/src/datas/btree.h | 52 --- ioreplay/src/datas/hmap.c | 362 ---------------- ioreplay/src/datas/hmap.h | 56 --- ioreplay/src/datas/list.c | 279 ------------ ioreplay/src/datas/list.h | 56 --- ioreplay/src/datas/rbuffer.c | 147 ------- ioreplay/src/datas/rbuffer.h | 102 ----- ioreplay/src/datas/stack.c | 85 ---- ioreplay/src/datas/stack.h | 43 -- ioreplay/src/defaults.h | 54 --- ioreplay/src/generate/generate.c | 235 ----------- ioreplay/src/generate/generate.h | 112 ----- ioreplay/src/generate/gioop.c | 838 ------------------------------------- ioreplay/src/generate/gioop.h | 102 ----- ioreplay/src/generate/gparser.c | 356 ---------------- ioreplay/src/generate/gparser.h | 113 ----- ioreplay/src/generate/gprocess.c | 101 ----- ioreplay/src/generate/gprocess.h | 90 ---- ioreplay/src/generate/gtask.c | 91 ---- ioreplay/src/generate/gtask.h | 100 ----- ioreplay/src/generate/gwriter.c | 85 ---- ioreplay/src/generate/gwriter.h | 86 ---- ioreplay/src/generate/vsize.c | 247 ----------- ioreplay/src/generate/vsize.h | 180 -------- ioreplay/src/init/init.c | 226 ---------- ioreplay/src/init/init.h | 64 --- ioreplay/src/init/itask.c | 66 --- ioreplay/src/init/itask.h | 72 ---- ioreplay/src/init/ithread.c | 99 ----- ioreplay/src/init/ithread.h | 86 ---- ioreplay/src/macros.h | 116 ----- ioreplay/src/main.c | 275 ------------ ioreplay/src/meta/meta.c | 111 ----- ioreplay/src/meta/meta.h | 107 ----- ioreplay/src/mounts.c | 400 ------------------ ioreplay/src/mounts.h | 154 ------- ioreplay/src/opcodes.h | 103 ----- ioreplay/src/options.c | 51 --- ioreplay/src/options.h | 61 --- ioreplay/src/replay/replay.c | 191 --------- ioreplay/src/replay/replay.h | 46 -- ioreplay/src/replay/rioop.c | 425 ------------------- ioreplay/src/replay/rioop.h | 54 --- ioreplay/src/replay/rprocess.c | 34 -- ioreplay/src/replay/rprocess.h | 40 -- ioreplay/src/replay/rstats.c | 108 ----- ioreplay/src/replay/rstats.h | 117 ------ ioreplay/src/replay/rtask.c | 50 --- ioreplay/src/replay/rtask.h | 69 --- ioreplay/src/replay/rthread.c | 216 ---------- ioreplay/src/replay/rthread.h | 123 ------ ioreplay/src/replay/rworker.c | 360 ---------------- ioreplay/src/replay/rworker.h | 82 ---- ioreplay/src/utests.c | 41 -- ioreplay/src/utests.h | 25 -- ioreplay/src/utils/futils.c | 291 ------------- ioreplay/src/utils/futils.h | 134 ------ ioreplay/src/utils/utils.c | 186 -------- ioreplay/src/utils/utils.h | 174 -------- ioreplay/src/vfd.c | 55 --- ioreplay/src/vfd.h | 77 ---- ioreplay/tags | 661 ----------------------------- ioriot/Makefile | 36 ++ ioriot/src/capture/capture.c | 99 +++++ ioriot/src/capture/capture.h | 30 ++ ioriot/src/cleanup/cleanup.c | 30 ++ ioriot/src/cleanup/cleanup.h | 29 ++ ioriot/src/datas/amap.c | 264 ++++++++++++ ioriot/src/datas/amap.h | 49 +++ ioriot/src/datas/btree.c | 169 ++++++++ ioriot/src/datas/btree.h | 52 +++ ioriot/src/datas/hmap.c | 362 ++++++++++++++++ ioriot/src/datas/hmap.h | 56 +++ ioriot/src/datas/list.c | 279 ++++++++++++ ioriot/src/datas/list.h | 56 +++ ioriot/src/datas/rbuffer.c | 147 +++++++ ioriot/src/datas/rbuffer.h | 102 +++++ ioriot/src/datas/stack.c | 85 ++++ ioriot/src/datas/stack.h | 43 ++ ioriot/src/defaults.h | 54 +++ ioriot/src/generate/generate.c | 235 +++++++++++ ioriot/src/generate/generate.h | 112 +++++ ioriot/src/generate/gioop.c | 838 +++++++++++++++++++++++++++++++++++++ ioriot/src/generate/gioop.h | 102 +++++ ioriot/src/generate/gparser.c | 356 ++++++++++++++++ ioriot/src/generate/gparser.h | 113 +++++ ioriot/src/generate/gprocess.c | 101 +++++ ioriot/src/generate/gprocess.h | 90 ++++ ioriot/src/generate/gtask.c | 91 ++++ ioriot/src/generate/gtask.h | 100 +++++ ioriot/src/generate/gwriter.c | 85 ++++ ioriot/src/generate/gwriter.h | 86 ++++ ioriot/src/generate/vsize.c | 247 +++++++++++ ioriot/src/generate/vsize.h | 180 ++++++++ ioriot/src/init/init.c | 226 ++++++++++ ioriot/src/init/init.h | 64 +++ ioriot/src/init/itask.c | 66 +++ ioriot/src/init/itask.h | 72 ++++ ioriot/src/init/ithread.c | 99 +++++ ioriot/src/init/ithread.h | 86 ++++ ioriot/src/macros.h | 116 +++++ ioriot/src/main.c | 275 ++++++++++++ ioriot/src/meta/meta.c | 111 +++++ ioriot/src/meta/meta.h | 107 +++++ ioriot/src/mounts.c | 400 ++++++++++++++++++ ioriot/src/mounts.h | 154 +++++++ ioriot/src/opcodes.h | 103 +++++ ioriot/src/options.c | 51 +++ ioriot/src/options.h | 61 +++ ioriot/src/replay/replay.c | 191 +++++++++ ioriot/src/replay/replay.h | 46 ++ ioriot/src/replay/rioop.c | 425 +++++++++++++++++++ ioriot/src/replay/rioop.h | 54 +++ ioriot/src/replay/rprocess.c | 34 ++ ioriot/src/replay/rprocess.h | 40 ++ ioriot/src/replay/rstats.c | 108 +++++ ioriot/src/replay/rstats.h | 117 ++++++ ioriot/src/replay/rtask.c | 50 +++ ioriot/src/replay/rtask.h | 69 +++ ioriot/src/replay/rthread.c | 216 ++++++++++ ioriot/src/replay/rthread.h | 123 ++++++ ioriot/src/replay/rworker.c | 360 ++++++++++++++++ ioriot/src/replay/rworker.h | 82 ++++ ioriot/src/utests.c | 41 ++ ioriot/src/utests.h | 25 ++ ioriot/src/utils/futils.c | 291 +++++++++++++ ioriot/src/utils/futils.h | 134 ++++++ ioriot/src/utils/utils.c | 186 ++++++++ ioriot/src/utils/utils.h | 174 ++++++++ ioriot/src/vfd.c | 55 +++ ioriot/src/vfd.h | 77 ++++ ioriot/tags | 661 +++++++++++++++++++++++++++++ systemtap/Makefile | 20 +- systemtap/src/ioreplay.stp | 591 -------------------------- systemtap/src/ioriot.stp | 591 ++++++++++++++++++++++++++ systemtap/src/javaioreplay.stp | 591 -------------------------- systemtap/src/javaioriot.stp | 591 ++++++++++++++++++++++++++ systemtap/src/targetedioreplay.stp | 591 -------------------------- systemtap/src/targetedioriot.stp | 591 ++++++++++++++++++++++++++ 151 files changed, 12074 insertions(+), 12074 deletions(-) delete mode 100644 ioreplay/Makefile delete mode 100644 ioreplay/src/capture/capture.c delete mode 100644 ioreplay/src/capture/capture.h delete mode 100644 ioreplay/src/cleanup/cleanup.c delete mode 100644 ioreplay/src/cleanup/cleanup.h delete mode 100644 ioreplay/src/datas/amap.c delete mode 100644 ioreplay/src/datas/amap.h delete mode 100644 ioreplay/src/datas/btree.c delete mode 100644 ioreplay/src/datas/btree.h delete mode 100644 ioreplay/src/datas/hmap.c delete mode 100644 ioreplay/src/datas/hmap.h delete mode 100644 ioreplay/src/datas/list.c delete mode 100644 ioreplay/src/datas/list.h delete mode 100644 ioreplay/src/datas/rbuffer.c delete mode 100644 ioreplay/src/datas/rbuffer.h delete mode 100644 ioreplay/src/datas/stack.c delete mode 100644 ioreplay/src/datas/stack.h delete mode 100644 ioreplay/src/defaults.h delete mode 100644 ioreplay/src/generate/generate.c delete mode 100644 ioreplay/src/generate/generate.h delete mode 100644 ioreplay/src/generate/gioop.c delete mode 100644 ioreplay/src/generate/gioop.h delete mode 100644 ioreplay/src/generate/gparser.c delete mode 100644 ioreplay/src/generate/gparser.h delete mode 100644 ioreplay/src/generate/gprocess.c delete mode 100644 ioreplay/src/generate/gprocess.h delete mode 100644 ioreplay/src/generate/gtask.c delete mode 100644 ioreplay/src/generate/gtask.h delete mode 100644 ioreplay/src/generate/gwriter.c delete mode 100644 ioreplay/src/generate/gwriter.h delete mode 100644 ioreplay/src/generate/vsize.c delete mode 100644 ioreplay/src/generate/vsize.h delete mode 100644 ioreplay/src/init/init.c delete mode 100644 ioreplay/src/init/init.h delete mode 100644 ioreplay/src/init/itask.c delete mode 100644 ioreplay/src/init/itask.h delete mode 100644 ioreplay/src/init/ithread.c delete mode 100644 ioreplay/src/init/ithread.h delete mode 100644 ioreplay/src/macros.h delete mode 100644 ioreplay/src/main.c delete mode 100644 ioreplay/src/meta/meta.c delete mode 100644 ioreplay/src/meta/meta.h delete mode 100644 ioreplay/src/mounts.c delete mode 100644 ioreplay/src/mounts.h delete mode 100644 ioreplay/src/opcodes.h delete mode 100644 ioreplay/src/options.c delete mode 100644 ioreplay/src/options.h delete mode 100644 ioreplay/src/replay/replay.c delete mode 100644 ioreplay/src/replay/replay.h delete mode 100644 ioreplay/src/replay/rioop.c delete mode 100644 ioreplay/src/replay/rioop.h delete mode 100644 ioreplay/src/replay/rprocess.c delete mode 100644 ioreplay/src/replay/rprocess.h delete mode 100644 ioreplay/src/replay/rstats.c delete mode 100644 ioreplay/src/replay/rstats.h delete mode 100644 ioreplay/src/replay/rtask.c delete mode 100644 ioreplay/src/replay/rtask.h delete mode 100644 ioreplay/src/replay/rthread.c delete mode 100644 ioreplay/src/replay/rthread.h delete mode 100644 ioreplay/src/replay/rworker.c delete mode 100644 ioreplay/src/replay/rworker.h delete mode 100644 ioreplay/src/utests.c delete mode 100644 ioreplay/src/utests.h delete mode 100644 ioreplay/src/utils/futils.c delete mode 100644 ioreplay/src/utils/futils.h delete mode 100644 ioreplay/src/utils/utils.c delete mode 100644 ioreplay/src/utils/utils.h delete mode 100644 ioreplay/src/vfd.c delete mode 100644 ioreplay/src/vfd.h delete mode 100644 ioreplay/tags create mode 100644 ioriot/Makefile create mode 100644 ioriot/src/capture/capture.c create mode 100644 ioriot/src/capture/capture.h create mode 100644 ioriot/src/cleanup/cleanup.c create mode 100644 ioriot/src/cleanup/cleanup.h create mode 100644 ioriot/src/datas/amap.c create mode 100644 ioriot/src/datas/amap.h create mode 100644 ioriot/src/datas/btree.c create mode 100644 ioriot/src/datas/btree.h create mode 100644 ioriot/src/datas/hmap.c create mode 100644 ioriot/src/datas/hmap.h create mode 100644 ioriot/src/datas/list.c create mode 100644 ioriot/src/datas/list.h create mode 100644 ioriot/src/datas/rbuffer.c create mode 100644 ioriot/src/datas/rbuffer.h create mode 100644 ioriot/src/datas/stack.c create mode 100644 ioriot/src/datas/stack.h create mode 100644 ioriot/src/defaults.h create mode 100644 ioriot/src/generate/generate.c create mode 100644 ioriot/src/generate/generate.h create mode 100644 ioriot/src/generate/gioop.c create mode 100644 ioriot/src/generate/gioop.h create mode 100644 ioriot/src/generate/gparser.c create mode 100644 ioriot/src/generate/gparser.h create mode 100644 ioriot/src/generate/gprocess.c create mode 100644 ioriot/src/generate/gprocess.h create mode 100644 ioriot/src/generate/gtask.c create mode 100644 ioriot/src/generate/gtask.h create mode 100644 ioriot/src/generate/gwriter.c create mode 100644 ioriot/src/generate/gwriter.h create mode 100644 ioriot/src/generate/vsize.c create mode 100644 ioriot/src/generate/vsize.h create mode 100644 ioriot/src/init/init.c create mode 100644 ioriot/src/init/init.h create mode 100644 ioriot/src/init/itask.c create mode 100644 ioriot/src/init/itask.h create mode 100644 ioriot/src/init/ithread.c create mode 100644 ioriot/src/init/ithread.h create mode 100644 ioriot/src/macros.h create mode 100644 ioriot/src/main.c create mode 100644 ioriot/src/meta/meta.c create mode 100644 ioriot/src/meta/meta.h create mode 100644 ioriot/src/mounts.c create mode 100644 ioriot/src/mounts.h create mode 100644 ioriot/src/opcodes.h create mode 100644 ioriot/src/options.c create mode 100644 ioriot/src/options.h create mode 100644 ioriot/src/replay/replay.c create mode 100644 ioriot/src/replay/replay.h create mode 100644 ioriot/src/replay/rioop.c create mode 100644 ioriot/src/replay/rioop.h create mode 100644 ioriot/src/replay/rprocess.c create mode 100644 ioriot/src/replay/rprocess.h create mode 100644 ioriot/src/replay/rstats.c create mode 100644 ioriot/src/replay/rstats.h create mode 100644 ioriot/src/replay/rtask.c create mode 100644 ioriot/src/replay/rtask.h create mode 100644 ioriot/src/replay/rthread.c create mode 100644 ioriot/src/replay/rthread.h create mode 100644 ioriot/src/replay/rworker.c create mode 100644 ioriot/src/replay/rworker.h create mode 100644 ioriot/src/utests.c create mode 100644 ioriot/src/utests.h create mode 100644 ioriot/src/utils/futils.c create mode 100644 ioriot/src/utils/futils.h create mode 100644 ioriot/src/utils/utils.c create mode 100644 ioriot/src/utils/utils.h create mode 100644 ioriot/src/vfd.c create mode 100644 ioriot/src/vfd.h create mode 100644 ioriot/tags delete mode 100644 systemtap/src/ioreplay.stp create mode 100644 systemtap/src/ioriot.stp delete mode 100644 systemtap/src/javaioreplay.stp create mode 100644 systemtap/src/javaioriot.stp delete mode 100644 systemtap/src/targetedioreplay.stp create mode 100644 systemtap/src/targetedioriot.stp diff --git a/.gitignore b/.gitignore index 0a122f0..ab4c720 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ *.o docs/html/ docs/latex/ -ioreplay/ioreplay +ioriot/ioriot systemtap/downloads/ diff --git a/Makefile b/Makefile index 427e7bb..d5ab493 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,19 @@ -DESTDIR=/opt/ioreplay +DESTDIR=/opt/ioriot all: $(MAKE) -C systemtap - $(MAKE) -C ioreplay + $(MAKE) -C ioriot install: $(MAKE) -C systemtap install - $(MAKE) -C ioreplay install + $(MAKE) -C ioriot install uninstall: test ! -z $(DESTDIR) && test -d $(DESTDIR) && rm -Rfv $(DESTDIR) || exit 0 deinstall: uninstall clean: - $(MAKE) -C ioreplay clean + $(MAKE) -C ioriot clean $(MAKE) -C systemtap clean astyle: - $(MAKE) -C ioreplay astyle + $(MAKE) -C ioriot astyle loc: - wc -l ./systemtap/src/*.stp ./ioreplay/src/*.{h,c} ./ioreplay/src/*/*.{h,c} | tail -n 1 + wc -l ./systemtap/src/*.stp ./ioriot/src/*.{h,c} ./ioriot/src/*/*.{h,c} | tail -n 1 doxygen: doxygen ./docs/doxygen.conf diff --git a/README.md b/README.md index db18da5..be6aa8f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# I/O Replay +# I/O Riot ## Overview -I/O Replay is an I/O benchmarking tool for Linux based operating systems which captures I/O operations on a (possibly production) server in order to replay the exact same I/O operations on a load test machine. +I/O Riot is an I/O benchmarking tool for Linux based operating systems which captures I/O operations on a (possibly production) server in order to replay the exact same I/O operations on a load test machine. -I/O Replay is operated in 5 steps: +I/O Riot is operated in 5 steps: 1. Capture: Record all I/O operations over a given period of time to a capture log. 2. Initialize: Copy the log to a load test machine and initialize the load test environment. @@ -23,11 +23,11 @@ The file system fragmentation (depending on the file system type and utilisation ## Benefits -In contrast to traditional I/O benchmarking tools, I/O Replay reproduces real production I/O, and does not rely on a pre-defined set of I/O operations. +In contrast to traditional I/O benchmarking tools, I/O Riot reproduces real production I/O, and does not rely on a pre-defined set of I/O operations. -Also, I/O Replay only requires a server machine for capturing and another server machine for replaying. A traditional load test environment would usually be a distributed system which can consist of many components and machines. Such a distributed system can become quite complex which makes it difficult to isolate possible I/O bottlenecks. For example in order to trigger I/O events a client application would usually have to call a remote server application. The remote server application itself would query a database and the database would trigger the actual I/O operations in Linux. Furthermore, it is not easy to switch forth and back between hardware and OS settings. For example without a backup and restore procedure a database would most likely be corrupt after reformatting the data partitions with a different file system type. +Also, I/O Riot only requires a server machine for capturing and another server machine for replaying. A traditional load test environment would usually be a distributed system which can consist of many components and machines. Such a distributed system can become quite complex which makes it difficult to isolate possible I/O bottlenecks. For example in order to trigger I/O events a client application would usually have to call a remote server application. The remote server application itself would query a database and the database would trigger the actual I/O operations in Linux. Furthermore, it is not easy to switch forth and back between hardware and OS settings. For example without a backup and restore procedure a database would most likely be corrupt after reformatting the data partitions with a different file system type. -The benefits of I/O replay are: +The benefits of I/O Riot are: * It is easy to determine whether a new hardware type is suitable for an already existing application. * It is easy to change OS and hardware for performance tests and optimizations. @@ -41,7 +41,7 @@ The benefits of I/O replay are: # Send in patches -Patches of any kind (bug fixes, new features...) are welcome! I/O Replay is new software and not everything might be perfect yet. Also, I/O Replay is used for a very specific use case at Mimecast. It may need tuning or extension for your use case. It will grow and mature over time. +Patches of any kind (bug fixes, new features...) are welcome! I/O Riot is new software and not everything might be perfect yet. Also, I/O Riot is used for a very specific use case at Mimecast. It may need tuning or extension for your use case. It will grow and mature over time. This is also potentially a great tool just for analysing (not replaying) the I/O, therefore it would be a great opportunity to add more features related to that (e.g. more stats, filters, etc.). @@ -49,11 +49,11 @@ Future work will also include file hole support and I/O support for memory mappe # Getting started -I/O Replay consists of a set of SystemTap kernel modules (capturing I/O) and the tool ``ioreplay`` (replaying I/O). Usually you want to capture I/O from a production machine and want to replay it on a separate load testing machine. +I/O Riot consists of a set of SystemTap kernel modules (capturing I/O) and the tool ``ioriot`` (replaying I/O). Usually you want to capture I/O from a production machine and want to replay it on a separate load testing machine. ## System requirements -I/O replay has been tested on +I/O Riot has been tested on * CentOS 7.4 64Bit (latest version, all packages up to date, booted into the installed Kernel) * SystemTap (from the default CentOS repository) @@ -61,29 +61,29 @@ I/O replay has been tested on Before proceeding please ensure that the latest CentOS 7 kernel is installed and running on all machines involved. It should also be ensured that the capture machine and the load test machine have the same mount points mounted. This is to ensure that I/O is being replayed on the corresponding data drives on the load test machine. -## Compiling and installing ioreplay +## Compiling and installing ioriot -I/O Replay has to be installed on all machines involved. To install I/O Replay perform the following steps: +I/O Riot has to be installed on all machines involved. To install I/O Riot perform the following steps: ```sh sudo yum install gcc systemtap yum-utils kernel-devel-$(uname -r) sudo debuginfo-install kernel-$(uname -r) make && sudo make install -export PATH=$PATH:/opt/ioreplay/bin +export PATH=$PATH:/opt/ioriot/bin ``` -This will install the ``ioreplay`` utility to ``/opt/ioreplay/bin/`` and the SystemTap kernel modules to ``/opt/ioreplay/systemtap/``. Run ``ioreplay -h`` to print out a brief help. +This will install the ``ioriot`` utility to ``/opt/ioriot/bin/`` and the SystemTap kernel modules to ``/opt/ioriot/systemtap/``. Run ``ioriot -h`` to print out a brief help. -However, best practise is not to install any compilers on a production machine. You can either compile I/O Replay from scratch on all machines involved like shown above or only compile it on a build machine and distribute the ``/opt/ioreplay`` directory to the remaining machines. In the latter case you will also need to install the ``systemtap-runtime`` package as an additional dependency. +However, best practise is not to install any compilers on a production machine. You can either compile I/O Riot from scratch on all machines involved like shown above or only compile it on a build machine and distribute the ``/opt/ioriot`` directory to the remaining machines. In the latter case you will also need to install the ``systemtap-runtime`` package as an additional dependency. -In case you decided to deinstall I/O Replay you can do so by running +In case you decided to deinstall I/O Riot you can do so by running ```sh -sudo ioreplay -P # purges all test files created by ioreplay +sudo ioriot -P # purges all test files created by ioriot sudo make uninstall ``` -# Operating I/O Replay +# Operating I/O Riot ## 1. Capture @@ -93,7 +93,7 @@ The following steps are required to capture all I/O operation of the entire (Lin * 2) Run: ```sh -sudo ioreplay -c ~/io.capture +sudo ioriot -c ~/io.capture ``` * 3) Start all applications again. @@ -102,13 +102,13 @@ sudo ioreplay -c ~/io.capture To capture only I/O caused by Java process run: ```sh -sudo ioreplay -c ~/io.capture -m javaioreplay.ko +sudo ioriot -c ~/io.capture -m javaioriot.ko ``` To capture the I/O of a specific process run the following respectively: ```sh -sudo ioreplay -c ~/io.capture -m targetedioreplay.ko -p PID +sudo ioriot -c ~/io.capture -m targetedioriot.ko -p PID ``` The resulting capture log looks like this and can be multiple GB in size: @@ -168,7 +168,7 @@ sudo mount -t tmpfs -o size=32g tmpfs /mnt/ramdisk Make sure that there is enough system memory available for such a RAMdisk and all the processes running on the machine. Eventually, RAM will be taken away from the Linux caches which potentially could decrease system I/O performance. Run the following command to capture to the RAMdisk respectively: ```sh -sudo ioreplay -c /mnt/ramdisk/io.capture +sudo ioriot -c /mnt/ramdisk/io.capture ``` ## 2. Initialize @@ -181,59 +181,59 @@ After producing ``io.capture`` it must be pre-processed. The resulting replay lo * Use of internal opcodes rather than strings (e.g. ``30`` instead of ``open``) for faster parsing. * All operations on unknown file handles are _removed_. * All incomplete or corrupt lines from the capture file are ignored. There may be corrupt lines in the capture file because SystemTap may skips a very few probe points if it decides that capturing I/O is causing too much overhead. -* Rewrite of all file paths. ``ioreplay`` adds ``/.ioreplay/NAME`` to all file paths for each file system mount point. +* Rewrite of all file paths. ``ioriot`` adds ``/.ioriot/NAME`` to all file paths for each file system mount point. To generate the the replay log ``io.replay`` from the capture log ``io.capture`` run: ```sh -sudo ioreplay -c io.capture -r io.replay -n NAME -u USER +sudo ioriot -c io.capture -r io.replay -n NAME -u USER ``` -In which NAME is a freely chosen name and USER must be a valid system user. It is the system user under which the replay test will run. This command also creates all required top level directories such as ``/.ioreplay/NAME/``, ``/mnt/.ioreplay/NAME/`` in all mounted file systems. These are the directories where the replay test will read/write files from/to. These directories will belong to user USER. +In which NAME is a freely chosen name and USER must be a valid system user. It is the system user under which the replay test will run. This command also creates all required top level directories such as ``/.ioriot/NAME/``, ``/mnt/.ioriot/NAME/`` in all mounted file systems. These are the directories where the replay test will read/write files from/to. These directories will belong to user USER. -``ioreplay`` will filter out many operations, especially all operations on pseudo file systems (e.g. sysfs, procfs), as it does not make a lot of sense to replay I/O on these file systems. Also, I/O operations on unknown file handles will be filtered out as well. This can happen when we start capturing the I/O *after* an application already opened a file. As a result we won't see how the application opened that file. The best practise is to stop all applications on the machine first, start capturing the I/O, and start all applications again. This may be improved in future releases of I/O Replay. +``ioriot`` will filter out many operations, especially all operations on pseudo file systems (e.g. sysfs, procfs), as it does not make a lot of sense to replay I/O on these file systems. Also, I/O operations on unknown file handles will be filtered out as well. This can happen when we start capturing the I/O *after* an application already opened a file. As a result we won't see how the application opened that file. The best practise is to stop all applications on the machine first, start capturing the I/O, and start all applications again. This may be improved in future releases of I/O Riot. The resulting replay log will look like this: At the first line there is a meta header. It contains information about the test configuration. The meta header is followed by all the I/O operations. At the end of the file is the INIT section. It lists all files (also their sizes) and directories required to be present before replaying the I/O. ```sh -#|num_timelines=509591|num_mapped_pids=19189|num_mapped_fds=4292067|num_lines=55040114|replay_version=1|user=ioreplayuser|name=test0|init_offset=2578735248| -23|1|1|0|0|30|11|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-9:1|438|0|open@31| +#|num_timelines=509591|num_mapped_pids=19189|num_mapped_fds=4292067|num_lines=55040114|replay_version=1|user=ioriotuser|name=test0|init_offset=2578735248| +23|1|1|0|0|30|11|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-9:1|438|0|open@31| 23|1|1|0|0|0|11|0|fstat@32| 23|1|1|0|0|10|11|12|read@33| 23|1|1|0|0|0|11|0|fstat@34| 23|1|1|0|0|72|11|0|1|12|lseek@35| 23|1|1|0|0|10|11|0|read@36| 23|1|1|0|0|50|11|0|close@37| -23|2|1|0|0|30|12|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-8:1|438|0|open@38| +23|2|1|0|0|30|12|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-8:1|438|0|open@38| 23|2|1|0|0|0|12|0|fstat@39| 23|2|1|0|0|10|12|12|read@40| 23|2|1|0|0|0|12|0|fstat@41| 23|2|1|0|0|72|12|0|1|12|lseek@42| 23|2|1|0|0|10|12|0|read@43| 23|2|1|0|0|50|12|0|close@44| -23|3|1|0|0|30|13|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-9:0|438|0|open@45| +23|3|1|0|0|30|13|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-9:0|438|0|open@45| 23|3|1|0|0|0|13|0|fstat@46| 23|3|1|0|0|10|13|12|read@47| 23|3|1|0|0|0|13|0|fstat@48| 23|3|1|0|0|72|13|0|1|12|lseek@49| 23|3|1|0|0|10|13|0|read@50| 23|3|1|0|0|50|13|0|close@51| -23|4|1|0|0|30|14|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-7:1|438|0|open@52| +23|4|1|0|0|30|14|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-7:1|438|0|open@52| 23|4|1|0|0|0|14|0|fstat@53| 23|4|1|0|0|10|14|12|read@54| 23|4|1|0|0|0|14|0|fstat@55| 23|4|1|0|0|72|14|0|1|12|lseek@56| 23|4|1|0|0|10|14|0|read@57| 23|4|1|0|0|50|14|0|close@58| -23|5|1|0|0|30|15|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-8:0|438|0|open@59| +23|5|1|0|0|30|15|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-8:0|438|0|open@59| 23|5|1|0|0|0|15|0|fstat@60| 23|5|1|0|0|10|15|12|read@61| 23|5|1|0|0|0|15|0|fstat@62| 23|5|1|0|0|72|15|0|1|12|lseek@63| 23|5|1|0|0|10|15|0|read@64| 23|5|1|0|0|50|15|0|close@65| -23|6|1|0|0|30|16|/usr/local/mimecast/.ioreplay/test0/someapp/somesubdir/vd11-6:1|438|0|open@66| +23|6|1|0|0|30|16|/usr/local/mimecast/.ioriot/test0/someapp/somesubdir/vd11-6:1|438|0|open@66| 23|6|1|0|0|0|16|0|fstat@67| 23|6|1|0|0|10|16|12|read@68| 23|6|1|0|0|0|16|0|fstat@69| @@ -241,18 +241,18 @@ The resulting replay log will look like this: At the first line there is a meta . . #INIT -0|1|688|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/8/b_dv01_11_vd11-11_a|@55290437 -0|1|2592|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/3/b_dv01_11_vd11-11_b|@33907067 -0|1|768|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/d/b_dv01_11_vd11-11_c|@64247527 -0|1|1440|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/0/b_dv01_11_vd11-11_d|@2014896 -0|1|960|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/9/b_dv01_11_vd11-11_e|@17724079 -0|1|928|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/1/b_dv01_11_vd11-11_f|@4534389 -0|1|1712|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/5/b_dv01_11_vd11-11_g|@2738458 -0|1|784|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/b/b_dv01_11_vd11-11_h|@21136612 -0|1|624|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/6/b_dv01_11_vd11-11_i|@24683427 -0|1|672|/mnt/14/.ioreplay/test0/bmnt/2/20171101/b/9/b_dv01_11_vd11-11_j|@12584061 -0|1|336|/mnt/15/.ioreplay/test0/bmnt/2/20171101/b/5/b_dv01_11_vd11-11_k|@7737434 -0|1|12|/mnt/06/.ioreplay/test0/bmnt/tmp/b|@42498106 +0|1|688|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/8/b_dv01_11_vd11-11_a|@55290437 +0|1|2592|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/3/b_dv01_11_vd11-11_b|@33907067 +0|1|768|/mnt/14/.ioriot/test0/bmnt/2/20171101/b/d/b_dv01_11_vd11-11_c|@64247527 +0|1|1440|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/0/b_dv01_11_vd11-11_d|@2014896 +0|1|960|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/9/b_dv01_11_vd11-11_e|@17724079 +0|1|928|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/1/b_dv01_11_vd11-11_f|@4534389 +0|1|1712|/mnt/14/.ioriot/test0/bmnt/2/20171101/b/5/b_dv01_11_vd11-11_g|@2738458 +0|1|784|/mnt/14/.ioriot/test0/bmnt/2/20171101/b/b/b_dv01_11_vd11-11_h|@21136612 +0|1|624|/mnt/14/.ioriot/test0/bmnt/2/20171101/b/6/b_dv01_11_vd11-11_i|@24683427 +0|1|672|/mnt/14/.ioriot/test0/bmnt/2/20171101/b/9/b_dv01_11_vd11-11_j|@12584061 +0|1|336|/mnt/15/.ioriot/test0/bmnt/2/20171101/b/5/b_dv01_11_vd11-11_k|@7737434 +0|1|12|/mnt/06/.ioriot/test0/bmnt/tmp/b|@42498106 . . . @@ -264,17 +264,17 @@ The resulting replay log will look like this: At the first line there is a meta It is very likely that the replay test wants to access already existing files. Therefore it has to be ensured that all of these exist already before starting the test. To create all files and directories required by the test run the following command: ```sh -sudo ioreplay -i io.replay +sudo ioriot -i io.replay ``` -For that ``ioreplay`` makes use of the INIT section in ``io.replay``. +For that ``ioriot`` makes use of the INIT section in ``io.replay``. ## 3. Replay To replay the log run: ```sh -sudo ioreplay -r io.replay +sudo ioriot -r io.replay ``` It is beneficial to read ``io.replay`` from RAMdisk so that we are not interfering so much with the system I/O. @@ -284,12 +284,12 @@ It is beneficial to read ``io.replay`` from RAMdisk so that we are not interferi It is posisble to initialise the test and run the test with one single command, just replace option `-r` with `-R`: ```sh -sudo ioreplay -R io.replay +sudo ioriot -R io.replay ``` *Speed factor* -By default `ioreplay` tries to replay all I/O operations as fast as it can. To replay the I/O at a different speed it is possible to configure the speed factor by using the `-s` command line option. +By default `ioriot` tries to replay all I/O operations as fast as it can. To replay the I/O at a different speed it is possible to configure the speed factor by using the `-s` command line option. The following pseudo code demonstrates how the speed factor affects the replay speed. Here `current_time` represents the current time while replaying the I/O, `time_in_log` represents the time as logged in `io.replay` and `time_ahead` indicates whether the replay is too quick or not. @@ -306,7 +306,7 @@ A speed factor of `0` is interpreted as "replay as fast as possible". A speed fa In order to replay the I/O in original speed the factor of `1` can be used as follows: ```sh -sudo ioreplay -R io.replay -s 1 +sudo ioriot -R io.replay -s 1 ``` ## 4. Analyse @@ -322,14 +322,14 @@ It is important to understand the I/O statistics observed. It is possible to rep To purge all temporally data files of all tests run ```sh -sudo ioreplay -P +sudo ioriot -P ``` -Note: It's not required to cleanup any test data manually when you intend to re-run a test or run a new test. During initialization (``-i`` or ``-R`` switch) ``ioreplay`` will automatically move all old data to ``.ioreplay/.trash/`` sub-folders. The data will be ignored there. However, once you intend to completely delete all test files and directories (e.g. you run out of disk space or want to deinstall ``ioreplay`` you should purge them with ``-P`` as shown above. +Note: It's not required to cleanup any test data manually when you intend to re-run a test or run a new test. During initialization (``-i`` or ``-R`` switch) ``ioriot`` will automatically move all old data to ``.ioriot/.trash/`` sub-folders. The data will be ignored there. However, once you intend to completely delete all test files and directories (e.g. you run out of disk space or want to deinstall ``ioriot`` you should purge them with ``-P`` as shown above. ## Supported file systems -Currently I/O Replay supports replaying I/O on ``ext2``, ``ext3``, ``ext4`` and ``xfs``. However, it should be straightforward add additional file systems. +Currently I/O Riot supports replaying I/O on ``ext2``, ``ext3``, ``ext4`` and ``xfs``. However, it should be straightforward add additional file systems. ## Supported syscalls @@ -391,4 +391,4 @@ exit_group - To detect process termination (closing all open file handles) ## Source code documentation -The documentation of the source code can be generated via the Doxygen Framework. To install doxygen run ``sudo yum install doxygen`` and to generate the documentation run ``make doxygen`` in the top level source directory. Once done, the resulting documentation can be found in the ``docs/html`` subfolder of the project. It is worthwhile to start from ``ioreplay/src/main.c`` and read your way through. Functions are generally documented in the header files. Exceptions are static functions which don't have any separate declarations. +The documentation of the source code can be generated via the Doxygen Framework. To install doxygen run ``sudo yum install doxygen`` and to generate the documentation run ``make doxygen`` in the top level source directory. Once done, the resulting documentation can be found in the ``docs/html`` subfolder of the project. It is worthwhile to start from ``ioriot/src/main.c`` and read your way through. Functions are generally documented in the header files. Exceptions are static functions which don't have any separate declarations. diff --git a/docs/doxygen.conf b/docs/doxygen.conf index 340f731..8f014b6 100644 --- a/docs/doxygen.conf +++ b/docs/doxygen.conf @@ -32,7 +32,7 @@ DOXYFILE_ENCODING = UTF-8 # title of most generated pages and in a few other places. # The default value is: My Project. -PROJECT_NAME = "I/O Replay" +PROJECT_NAME = "I/O Riot" # The PROJECT_NUMBER tag can be used to enter a project or revision number. This # could be handy for archiving the generated documentation or if some version @@ -791,7 +791,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = README.md ./ioreplay/src +INPUT = README.md ./ioriot/src # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/ioreplay/Makefile b/ioreplay/Makefile deleted file mode 100644 index 67f4421..0000000 --- a/ioreplay/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -#DEBUG=-g3 -ggdb3 -pg -NAME=ioreplay -LIBS=-pthread -CFLAGS=-Wall -std=gnu99 -pedantic -STATIC=#-static -DESTDIR=/opt/ioreplay/bin -SRCS=$(wildcard src/*.c src/*/*.c) -HDRS=$(SRCS:.c=.h) -OBJS=$(SRCS:.c=.o) -all: compile -quick: clean ctags compile sudo_install -cshell: compile - gdb -ex='break main; run' --args ./$(NAME) -test: compile - gdb -ex=run --args ./$(NAME) -U -compile: $(OBJS) - $(CC) $(STATIC) $(DEBUG) $(LIBS) $(OBJS) -o $(NAME) -%.o: %.c %.h - $(CC) $(STATIC) $(DEBUG) $(LIBS) -c $(CFLAGS) $< -o $@ -clean: - rm -v ioreplay ./src/*.o ./src/*/*.o 2>/dev/null || exit 0 -install: - test ! -d $(DESTDIR) && mkdir -p $(DESTDIR) || exit 0 - cp -v $(NAME) $(DESTDIR) - @echo "Don't forget to add $(DESTDIR) to your PATH as follows:" - @echo " export PATH=\$$PATH:$(DESTDIR)" -uninstall: - test ! -z "$(DESTDIR)" && test -f $(DESTDIR)/$(NAME) && rm -v $(DESTDIR)/$(NAME) || exit 0 -deinstall: uninstall -astyle: - astyle -n --style=linux src/*.h src/*/*.h - astyle -n --style=linux src/*.c src/*/*.c -todo: - fgrep ../TODO ./src/* -ctags: - ctags ./src/*.{h,c} ./src/*/*.{h,c} diff --git a/ioreplay/src/capture/capture.c b/ioreplay/src/capture/capture.c deleted file mode 100644 index 0ac336b..0000000 --- a/ioreplay/src/capture/capture.c +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "capture.h" - -#include - -status_e capture_run(options_s *opts) -{ - int status = 0; - struct utsname uts; - - // To make it nicer we should iterate over PATH instead - char *staprun_paths[3] = { - "/usr/bin/staprun", - "/usr/local/bin/staprun", - "/bin/staprun" - }; - int num_staprun_paths = 3; - - if (0 != uname(&uts)) { - Errno("Could not identify release of currently running Kernel!"); - } - - Put("Release of currently running Kernel: %s", uts.release); - char modules_dir[128]; - sprintf(modules_dir, "/opt/ioreplay/systemtap/%s", uts.release); - Put("Changing directory to module path: %s/", modules_dir); - - if (0 != chdir(modules_dir)) { - Errno("Could not change into '%s', please ensure that the compiled " - "SystemTap modules correspond to the currently running Kernel " - "and that these are installed properly!\n", - modules_dir); - } - - if (0 != access(opts->module, R_OK)) { - Errno("Module '%s/%s' can't be read, please make sure that the " - "SystemTap Kernel modules are installed!", - modules_dir, opts->module); - } - - char *staprun_path = NULL; - for (int i = 0; i < num_staprun_paths; ++i) { - if (0 == access(staprun_paths[i], X_OK)) { - staprun_path = staprun_paths[i]; - //Put("SystemTap command path: %s", staprun_path); - break; - } - } - - if (staprun_path == NULL) { - Errno("Can't find 'staprun' command, please ensure to have the SystemTap " - "runtime (usually package 'systemtap-runtime') installed!"); - } - - char staprun_command[128]; - if (opts->pid >= 0) { - sprintf(staprun_command, "%s %s -v -o %s -x %d", staprun_path, opts->module, - opts->capture_file, opts->pid); - } else { - sprintf(staprun_command, "%s %s -v -o %s", staprun_path, opts->module, - opts->capture_file); - } - - Out("NOTICE: It is good practise first to stop all processes, then to "); - Out("start capturing, and then to start all processes again. The reason "); - Out("is that processes may have already open file handles. In that case "); - Out("I/O Replay would be unable to replay these! This may be improved "); - Put("in a future release!"); - Put("To abort capturing now send Ctrl+C, otherwise wait 1h"); - Put("Capturing I/O via: '%s'", staprun_command); - - char buf[1024]; - FILE *fp; - - if ((fp = popen(staprun_command, "r")) == NULL) { - Errno("Unable to invoke staprun command!"); - } - while (fgets(buf, 1024, fp) != NULL) - Out("stapio: %s", buf); - - if (0 != pclose(fp)) { - Error("Problems invoking staprun command!"); - } - - return status; -} diff --git a/ioreplay/src/capture/capture.h b/ioreplay/src/capture/capture.h deleted file mode 100644 index 7718d3e..0000000 --- a/ioreplay/src/capture/capture.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef CAPTURE_H -#define CAPTURE_H - -#include "../defaults.h" -#include "../utils/futils.h" -#include "../options.h" - -/** - * @brief Captures I/O to a .capture file by using stap from SystemTap - * - * @param opts The options object - * @return SUCCESS if everything went fine - */ -status_e capture_run(options_s *opts); - -#endif // CAPTURE_H diff --git a/ioreplay/src/cleanup/cleanup.c b/ioreplay/src/cleanup/cleanup.c deleted file mode 100644 index 570f8a7..0000000 --- a/ioreplay/src/cleanup/cleanup.c +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "cleanup.h" - -#include "../mounts.h" - -status_e cleanup_run(options_s *opts) -{ - set_limits_drop_root(opts->user); - mounts_s *m = mounts_new(opts); - - if (opts->purge) - mounts_purge(m); - else - mounts_trash(m); - - return SUCCESS; -} diff --git a/ioreplay/src/cleanup/cleanup.h b/ioreplay/src/cleanup/cleanup.h deleted file mode 100644 index 127badf..0000000 --- a/ioreplay/src/cleanup/cleanup.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef CLEANUP_H -#define CLEANUP_H - -#include "../defaults.h" -#include "../options.h" - -/** - * @brief Cleans up all files and directories of a given test - * - * @brief opts The options object - * @return SUCCESS in case everything went fine - */ -status_e cleanup_run(options_s *opts); - -#endif // CLEANUP_H diff --git a/ioreplay/src/datas/amap.c b/ioreplay/src/datas/amap.c deleted file mode 100644 index 806a3f8..0000000 --- a/ioreplay/src/datas/amap.c +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "amap.h" - -/** - * @brief Creates a new array map - * - * @param size The array map size - * @param mmapped true if the memory should be mmapped - * @return The new amap object - */ -static amap_s *_amap_new(long size, bool mmapped) -{ - amap_s *a = NULL; - void ***arrays = NULL; - - // Calculate a multiple of 1024, but at least in size of 'size'. - if (size % 1024 != 0) { - size = 1024*(1+(long)(size/1024)); - } - - if (size < 1) { - Error("Size overflow"); - } - - int num_arrays = size / AMAP_MAX_ARRAY_LENGTH; - - if (mmapped) { - a = Mmapshared(amap_s); - arrays = Cmapshared(num_arrays, void**); - } else { - a = Malloc(amap_s); - arrays = Calloc(num_arrays, void**); - } - - for (int i = 0; i < num_arrays; ++i) { - if (mmapped) { - //Put("%d", AMAP_MAX_ARRAY_LENGTH); - arrays[i] = Cmapshared(AMAP_MAX_ARRAY_LENGTH, void*); - } else { - arrays[i] = Calloc(AMAP_MAX_ARRAY_LENGTH, void*); - } - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - arrays[i][j] = NULL; - } - } - - a->arrays = arrays; - a->num_arrays = num_arrays; - a->size = size; - a->data_destroy = NULL; - a->mmapped = mmapped; - - return a; -} - -/** - * @brief Creates a new array map - * - * @param size The array map size - * @return The new amap object - */ -amap_s* amap_new(const long size) -{ - return _amap_new(size, false); -} - -/** - * @brief Creates a new mmapped array map - * - * @param size The array map size - * @return The new amap object - */ -amap_s* amap_new_mmapped(const long size) -{ - return _amap_new(size, true); -} - -/** - * @brief Destroys a mmap object - * - * @a The new amap object - */ -void amap_destroy(amap_s* a) -{ - if (!a) { - return; - } - - // Don't bother, the mmapped version of amap will stay alive until - // process terminations. And after process termination everything - // will be cleaned up automatically by Linux. - if (a->mmapped) { - return; - } - - for (int i = 0; i < a->num_arrays; ++i) { - if (a->data_destroy) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) - if (a->arrays[i][j]) { - a->data_destroy(a->arrays[i][j]); - } - } - free(a->arrays[i]); - } - free(a->arrays); - free(a); -} - -/** - * @brief Resets a mmap object - * - * This resets all entries to NULL. - * - * @a The new amap object - */ -void amap_reset(amap_s* a) -{ - for (int i = 0; i < a->num_arrays; ++i) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - if (a->data_destroy) { - if (a->arrays[i][j]) { - a->data_destroy(a->arrays[i][j]); - } - } - a->arrays[i][j] = NULL; - } - } -} - -int amap_set(amap_s *a, const long position, void* value) -{ - if (position >= a->size) - return -1; - int which_array = position / AMAP_MAX_ARRAY_LENGTH; - int array_pos = position % AMAP_MAX_ARRAY_LENGTH; - a->arrays[which_array][array_pos] = value; - return 0; -} - -void* amap_get(amap_s *a, const long position) -{ - if (position >= a->size) - return NULL; - int which_array = position / AMAP_MAX_ARRAY_LENGTH; - int array_pos = position % AMAP_MAX_ARRAY_LENGTH; - return a->arrays[which_array][array_pos]; -} - -void* amap_unset(amap_s *a, const long position) -{ - if (position >= a->size) - return NULL; - int which_array = position / AMAP_MAX_ARRAY_LENGTH; - int array_pos = position % AMAP_MAX_ARRAY_LENGTH; - void *value = a->arrays[which_array][array_pos]; - a->arrays[which_array][array_pos] = NULL; - return value; -} - -void amap_run_cb(amap_s *a, void (*cb)(void *data)) -{ - for (int i = 0; i < a->num_arrays; ++i) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - if (a->arrays[i][j]) - cb(a->arrays[i][j]); - } - } -} - -void amap_print(amap_s* a) -{ - Put("amap_s (%p):", (void*)a); - Put("\tmmapped: %d", a->mmapped); - Put("\tmax_array_length: %d", AMAP_MAX_ARRAY_LENGTH); - Put("\tnum_arrays: %d", a->num_arrays); - Put("\tsize: %lu", a->size); - Out("\toccupied slots: "); - for (int i = 0; i < a->num_arrays; ++i) { - for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { - if (a->arrays[i][j] != NULL) { - Out("%d:%d ", i, j); - } - } - } - Out("\n"); -} - -void _amap_test(amap_s *a) -{ - assert(0 == amap_set(a, 0, (void*)10)); - assert(0 == amap_set(a, 1, (void*)11)); - assert(0 == amap_set(a, 2, (void*)12)); - assert(0 == amap_set(a, 3, (void*)a)); - assert(10 == (long) amap_get(a, 0)); - assert(11 == (long) amap_get(a, 1)); - assert(12 == (long) amap_get(a, 2)); - assert(a == amap_get(a, 3)); - - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH-1, (void*) 23)); - assert(23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH-1)); - - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH, (void*) 42)); - assert(42 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH)); - - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2-1, (void*) (23+42))); - assert(42+23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH*2-1)); - assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2, (void*) 23)); - - - assert(NULL == amap_get(a, 1024*1024*9-1)); - assert(0 == amap_set(a, 1024*1024*9-1, (void*) 0x1)); - assert(0x1 == (long) amap_get(a, 1024*1024*9-1)); - assert(0x1 == (long) amap_unset(a, 1024*1024*9-1)); - assert(NULL == amap_get(a, 1024*1024*9-1)); - - assert(0 == amap_set(a, 1024*1024*9, (void*) 100)); - assert(100 == (long) amap_get(a, 1024*1024*9)); - - assert(0 == amap_set(a, 1024*1024*9+1, (void*) 101)); - assert(101 == (long) amap_get(a, 1024*1024*9+1)); - - assert(0 == amap_set(a, 1024*1024*10-2, (void*) 102)); - assert(102 == (long) amap_get(a, 1024*1024*10-2)); - - assert(0 == amap_set(a, 1024*1024*10-1, a)); - assert(a == amap_get(a, 1024*1024*10-1)); - //amap_print(a); - - assert(a == amap_unset(a, 1024*1024*10-1)); - assert(a != amap_unset(a, 1024*1024*10-1)); - //amap_print(a); -} - -void amap_test(void) -{ - // First test the non-mmapped version - amap_s* a = amap_new(1024*1024*10); - _amap_test(a); - amap_destroy(a); - - // Now test the mapped version - a = amap_new_mmapped(1024*1024*10); - _amap_test(a); - amap_destroy(a); - - // Another test with non-alligned size - a = amap_new(1024*1024*10+1); - _amap_test(a); - amap_destroy(a); -} - diff --git a/ioreplay/src/datas/amap.h b/ioreplay/src/datas/amap.h deleted file mode 100644 index 882a7c5..0000000 --- a/ioreplay/src/datas/amap.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef AMAP_H -#define AMAP_H - -#include "../defaults.h" - -#define AMAP_MAX_ARRAY_LENGTH 1024*8 - -/** - * @brief Implements an array map data structure - * - * This array map can hold a HUGE amount of entries by allocating multiple - * smaller arrays. There are two version of the amap data structure available: - * a memory mapped (mmap) and a normal version. The memory mapped version can - * be used for IPC between various processes. - */ -typedef struct amap_s_ { - void*** arrays; /**< The pointers to the amap arrays */ - int num_arrays; /**< The amount of arrays used in the amap */ - long size; /**< The total size/capacity of the amap */ - bool mmapped; /**< True if amap is memory mapped */ - void (*data_destroy)(void *data); /**< Callback to destroy all elements */ -} amap_s; - -amap_s* amap_new(const long size); -amap_s* amap_new_mmapped(const long size); -int amap_set(amap_s *a, const long position, void* value); -void* amap_get(amap_s *a, const long position); -void* amap_unset(amap_s *a, const long position); -void amap_print(amap_s *a); -void amap_destroy(amap_s *a); -void amap_reset(amap_s *a); -void amap_run_cb(amap_s *a, void (*cb)(void *data)); -void amap_test(void); - -#endif // AMAP_H diff --git a/ioreplay/src/datas/btree.c b/ioreplay/src/datas/btree.c deleted file mode 100644 index da5da48..0000000 --- a/ioreplay/src/datas/btree.c +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "btree.h" - -btree_s* btree_new() -{ - btree_s *b = Malloc(btree_s); - *b = (btree_s) { - .root = NULL, .size = 0 - }; - return b; -} - -void btree_destroy(btree_s* b) -{ - if (b->root) - btreelem_destroy_r(b->root); - free(b); -} - -void btree_destroy2(btree_s* b) -{ - if (b->root) - btreelem_destroy_r2(b->root); - free(b); -} - -int btree_insert(btree_s* b, int key, void *data) -{ - int ret = 1; - - if (b->root == NULL) { - b->root = btreelem_new(key, data); - ret = 0; - } else { - ret = btreelem_insert_r(b->root, key, data); - } - - if (ret == 0) { - b->size++; - } - - return ret; -} - -void* btree_get(btree_s* b, int key) -{ - if (b->root == NULL) - return NULL; - - return btreelem_get_r(b->root, key); -} - -void btree_print(btree_s* b) -{ - btreelem_print_r(b->root, 0); -} - -btreelem_s* btreelem_new(int key, void *data) -{ - btreelem_s *e = Malloc(btreelem_s); - *e = (btreelem_s) { - .key = key, .data = data, .left = NULL, .right = NULL - }; - return e; -} - -void btreelem_destroy_r(btreelem_s* e) -{ - if (e->left) { - btreelem_destroy_r(e->left); - } - if (e->right) { - btreelem_destroy_r(e->right); - } - - free(e); -} - -void btreelem_destroy_r2(btreelem_s* e) -{ - if (e->left) - btreelem_destroy_r(e->left); - if (e->right) - btreelem_destroy_r(e->right); - if (e->data) - btree_destroy(e->data); - - free(e); -} - -int btreelem_insert_r(btreelem_s* e, int key, void *data) -{ - int ret = 0; - - if (e->key == key) { - ret = 1; - } - - else if (e->key > key) { - if (e->left == NULL) { - e->left = btreelem_new(key, data); - } else { - ret = btreelem_insert_r(e->left, key, data); - } - } - - else { - if (e->right == NULL) { - e->right = btreelem_new(key, data); - } else { - ret = btreelem_insert_r(e->right, key, data); - } - } - - return ret; -} - -void* btreelem_get_r(btreelem_s* e, int key) -{ - void *data = NULL; - - if (e->key == key) { - data = e->data; - } - - else if (e->key > key) { - if (e->left) { - data = btreelem_get_r(e->left, key); - } - } - - else { - if (e->right) { - data = btreelem_get_r(e->right, key); - } - } - - return data; -} - -void btreelem_print_r(btreelem_s* e, int depth) -{ - for (int i = 0; i < depth; ++i) { - Out(" "); - } - Put("%d\n", e->key); - - if (e->left) { - btreelem_print_r(e->left, depth); - } - - if (e->right) { - btreelem_print_r(e->right, depth+1); - } -} - diff --git a/ioreplay/src/datas/btree.h b/ioreplay/src/datas/btree.h deleted file mode 100644 index 55da560..0000000 --- a/ioreplay/src/datas/btree.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef BTREE_H -#define BTREE_H - -#include "../defaults.h" - -/** - * @brief This defines an element of the binary tree data structure - */ -typedef struct btreelem_ { - struct btreelem_ *left; /**< The next element to the left */ - struct btreelem_ *right; /**< The next element to the right */ - int key; /**< The key of the element */ - void *data; /**< A pointer to the data stored in this element */ -} btreelem_s; - -/** - * @brief This defines a binary tree data structure. - */ -typedef struct btree_s_ { - btreelem_s *root; /**< The root element */ - int size; /**< The current size of the binary tree */ -} btree_s; - -btree_s* btree_new(); -void btree_destroy(btree_s *b); -void btree_destroy2(btree_s *b); -int btree_insert(btree_s *b, int key, void *data); -void* btree_get(btree_s *b, int key); -void btree_print(btree_s *b); - -btreelem_s* btreelem_new(int key, void *data); -void btreelem_destroy_r(btreelem_s *e); -void btreelem_destroy_r2(btreelem_s *e); -int btreelem_insert_r(btreelem_s *e, int key, void *data); -void* btreelem_get_r(btreelem_s *e, int key); -void btreelem_print_r(btreelem_s *e, int depth); - -#endif // BTREE_H diff --git a/ioreplay/src/datas/hmap.c b/ioreplay/src/datas/hmap.c deleted file mode 100644 index 96c373e..0000000 --- a/ioreplay/src/datas/hmap.c +++ /dev/null @@ -1,362 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "hmap.h" - -#define _Using_string_keys h->keys != NULL - -unsigned int hmap_get_addr(hmap_s *h, char *key) -{ - unsigned long hash = 5381; - int len = strlen(key); - - for (int i = 0; i < len; ++i) { - hash = ((hash << 5) + hash) + key[i]; /* hash * 33 + c */ - } - - return (unsigned int) (hash % h->size); -} - -unsigned int hmap_get_addr_l(hmap_s *h, const long key) -{ - return (unsigned int) (key % h->size); -} - -hmap_s *_hmap_new(unsigned int init_size) -{ - hmap_s *h = Malloc(hmap_s); - - h->size = init_size; - h->data = Calloc(init_size, void*); - h->l = Calloc(init_size, list_s*); - h->data_destroy = NULL; - h->keys = NULL; - h->keys_l = NULL; - - Mset(h->data, 0, init_size, void*); - Mset(h->l, 0, init_size, list_s*); - - return h; -} - -hmap_s *hmap_new(unsigned int init_size) -{ - hmap_s *h = _hmap_new(init_size); - h->keys = Calloc(init_size, char*); - Mset(h->keys, 0, init_size, char*); - - return h; -} - -hmap_s *hmap_new_l(unsigned int init_size) -{ - hmap_s *h = _hmap_new(init_size); - h->keys_l = Calloc(init_size, int); - Mset(h->keys_l, -1, init_size, int); - - return h; -} - -void hmap_destroy(hmap_s *h) -{ - for (int i = 0; i < h->size; ++i) { - if (h->l[i]) { - list_s *l = h->l[i]; - if (h->data_destroy) - l->data_destroy = h->data_destroy; - list_destroy(h->l[i]); - } - if (h->data[i] && h->data_destroy) { - h->data_destroy(h->data[i]); - } - } - - free(h->data); - if (h->keys) - free(h->keys); - if (h->keys_l) - free(h->keys_l); - free(h->l); - free(h); - - return; -} - -int hmap_insert(hmap_s *h, char *key, void *data) -{ - if (data == NULL) { - Error("insert data can not be NULL"); - } - - int addr = hmap_get_addr(h, key); - - if (h->data[addr]) { - - if (strcmp(key, h->keys[addr]) == 0) { - // Key already exists - return 0; - } - - // There is already data, collision, create a linked list - list_s *l = h->l[addr] = list_new(); - list_key_insert(l, h->keys[addr], h->data[addr]); - list_key_insert(l, key, data); - - // Not needed anymore, as the elements are in the linked list now. - free(h->keys[addr]); - h->data[addr] = h->keys[addr] = NULL; - - return 1; - - } else if (h->l[addr]) { - // There was a collision at this address before. Insert - // the element to the linked list. Returns 0 if key is already - // in the list (no additional insert made) or 1 otherwise. - return list_key_insert(h->l[addr], key, data); - } - - // New entry on a collision free address - h->data[addr] = data; - h->keys[addr] = Clone(key); - - return 1; -} - -int hmap_insert_l(hmap_s *h, const long key, void *data) -{ - if (data == NULL) { - Error("insert data can not be NULL"); - } - - int addr = hmap_get_addr_l(h, key); - - if (h->data[addr]) { - - if (key == h->keys_l[addr]) { - // Key already exists - return 0; - } - - // There is already data, collision, create a linked list - list_s *l = h->l[addr] = list_new_l(); - list_key_insert_l(l, h->keys_l[addr], h->data[addr]); - list_key_insert_l(l, key, data); - - // Not needed anymore, as the elements are in the linked list now. - h->data[addr] = NULL; - h->keys_l[addr] = -1; - - return 1; - - } else if (h->l[addr]) { - // There was a collision at this address before. Insert - // the element to the linked list. Returns 0 if key is already - // in the list (no additional insert made) or 1 otherwise. - return list_key_insert_l(h->l[addr], key, data); - } - - // New entry on a collision free address - h->data[addr] = data; - h->keys_l[addr] = key; - - return 1; -} - -void* hmap_remove(hmap_s *h, char *key) -{ - int addr = hmap_get_addr(h, key); - - if (h->data[addr] != NULL) { - void *data = h->data[addr]; - free(h->keys[addr]); - h->data[addr] = h->keys[addr] = NULL; - return data; - - } else if (h->l[addr] != NULL) { - // There was a collision at this address before. Remove - // the element to the linked list. Returns the object if key is - // already in the list (no additional insert made) or NULL - // otherwise. - return list_key_remove(h->l[addr], key); - } - - // Key is not present - return NULL; -} - -void* hmap_remove_l(hmap_s *h, const long key) -{ - int addr = hmap_get_addr_l(h, key); - - if (h->data[addr] != NULL) { - void *data = h->data[addr]; - h->data[addr] = NULL; - h->keys_l[addr] = -1; - return data; - - } else if (h->l[addr] != NULL) { - // There was a collision at this address before. Remove - // the element to the linked list. Returns the object if key is - // already in the list (no additional insert made) or NULL - // otherwise. - return list_key_remove_l(h->l[addr], key); - } - - // Key is not present - return NULL; -} - -void* hmap_get(hmap_s *h, char *key) -{ - int addr = hmap_get_addr(h, key); - if (h->data[addr] && strcmp(h->keys[addr], key) == 0) { - return h->data[addr]; - - } else if (h->l[addr]) { - return list_key_get(h->l[addr], key); - } - - return NULL; -} - -void* hmap_get_l(hmap_s *h, const long key) -{ - int addr = hmap_get_addr_l(h, key); - if (h->data[addr] && h->keys_l[addr] == key) { - return h->data[addr]; - - } else if (h->l[addr]) { - return list_key_get_l(h->l[addr], key); - } - - return NULL; -} - -void hmap_run_cb(hmap_s* h, void (*cb)(void *data)) -{ - for (int i = 0; i < h->size; ++i) { - if (h->l[i]) { - list_s *l = h->l[i]; - list_run_cb(l, cb); - } - if (h->data[i]) { - cb(h->data[i]); - } - } -} - -void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_) -{ - for (int i = 0; i < h->size; ++i) { - if (h->l[i]) { - list_s *l = h->l[i]; - list_run_cb2(l, cb, data_); - } - if (h->data[i]) { - cb(h->data[i], data_); - } - } -} - -void hmap_print(hmap_s *h) -{ - for (int i = 0; i < h->size; ++i) { - if (h->data[i]) { - if (_Using_string_keys) { - Put("hmap:%p addr:%d key:'%s'", (void*)h, i, h->keys[i]); - } else { - Put("hmap:%p addr:%d key:%d", (void*)h, i, h->keys_l[i]); - } - } else if (h->l[i]) { - Put("hmap:%p addr:%d LIST", (void*)h, i); - list_print(h->l[i]); - } - } -} - -static void _hmap_test(hmap_s *h) -{ - void* somedata = (void*)h; - - assert(1 == hmap_insert(h, "someval", (void*)23)); - assert(1 == hmap_insert(h, "another value", (void*)123)); - - assert(1 == hmap_insert(h, "mimecast", somedata)); - assert(0 == hmap_insert(h, "mimecast", somedata)); - assert(1 == hmap_insert(h, "is", somedata)); - assert(1 == hmap_insert(h, "hiring", somedata)); - - assert(NULL != hmap_get(h, "mimecast")); - assert(NULL == hmap_get(h, "Mimecast")); - - assert(NULL != hmap_remove(h, "mimecast")); - assert(NULL == hmap_remove(h, "mimecast")); - - assert(1 == hmap_insert(h, "mimecast", somedata)); - assert(NULL != hmap_get(h, "mimecast")); - - assert(23 == (long)hmap_get(h, "someval")); - assert(23 == (long)hmap_get(h, "someval")); - - assert(123 == (long)hmap_remove(h, "another value")); - assert(0 == (long)hmap_remove(h, "another value")); - assert(NULL == hmap_get(h, "another value")); - - //hmap_print(h); -} - -static void _hmap_test_l(hmap_s *h) -{ - void* somedata = (void*)h; - - assert(1 == hmap_insert_l(h, 1, (void*)23)); - assert(1 == hmap_insert_l(h, 5, (void*)123)); - - assert(1 == hmap_insert_l(h, 3, somedata)); - assert(0 == hmap_insert_l(h, 3, somedata)); - assert(1 == hmap_insert_l(h, 4, somedata)); - assert(1 == hmap_insert_l(h, 6, somedata)); - - assert(NULL != hmap_get_l(h, 3)); - assert(NULL == hmap_get_l(h, 7)); - - assert(NULL != hmap_remove_l(h, 3)); - assert(NULL == hmap_remove_l(h, 3)); - - assert(1 == hmap_insert_l(h, 3, somedata)); - assert(NULL != hmap_get_l(h, 3)); - - assert(23 == (long)hmap_get_l(h, 1)); - assert(23 == (long)hmap_get_l(h, 1)); - - assert(123 == (long)hmap_remove_l(h, 5)); - assert(0 == (long)hmap_remove_l(h, 5)); - assert(NULL == hmap_get_l(h, 5)); -} - -void hmap_test(void) -{ - hmap_s* h = hmap_new(1024); - _hmap_test(h); - hmap_destroy(h); - - h = hmap_new(2); - _hmap_test(h); - hmap_destroy(h); - - h = hmap_new_l(1024); - _hmap_test_l(h); - hmap_print(h); - hmap_destroy(h); -} diff --git a/ioreplay/src/datas/hmap.h b/ioreplay/src/datas/hmap.h deleted file mode 100644 index 9d1978b..0000000 --- a/ioreplay/src/datas/hmap.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef HMAP_H -#define HMAP_H - -#include "../defaults.h" -#include "list.h" - -/** - * @brief A hash map data structure - * - * There are two version of this hmap data structure. One version is utilising - * string keys and the other one is utilising long keys. - * - * On hash collision the data structure will make use of a "named" linked list, - * whereas every member of the linked list has either a string key or a long - * key associated. - */ -typedef struct hmap_s_ { - char **keys; /**< List of all keys, NULL if nothing at a address */ - int *keys_l; /**< Same as keys, but for long keys */ - void **data; /**< Pointers to the stored data, NULL if nothing there */ - list_s **l; /**< Pointers to the linked lists, used on hash collision */ - void (*data_destroy)(void *data); /**< Callback to destroy all data */ - unsigned int size; /**< Size of the hmap */ -} hmap_s; - -hmap_s* hmap_new(unsigned int init_size); -hmap_s* hmap_new_l(unsigned int init_size); -void hmap_destroy(hmap_s* h); -void hmap_run_cb(hmap_s* h, void (*cb)(void *data)); -void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_); -int hmap_insert_l(hmap_s* h, const long key, void *data); -int hmap_insert(hmap_s* h, char* key, void *data); -void* hmap_remove_l(hmap_s* h, const long key); -void* hmap_remove(hmap_s* h, char* key); -void* hmap_get_l(hmap_s* h, const long key); -void* hmap_get(hmap_s* h, char* key); -unsigned int hmap_get_addr_l(hmap_s* h, const long key); -unsigned int hmap_get_addr(hmap_s* h, char* key); -void hmap_print(hmap_s* h); -void hmap_test(void); - -#endif // HMAP_H diff --git a/ioreplay/src/datas/list.c b/ioreplay/src/datas/list.c deleted file mode 100644 index 9cc78db..0000000 --- a/ioreplay/src/datas/list.c +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "list.h" - - -list_s *list_new() -{ - list_s *l = Malloc(list_s); - *l = (list_s) { - .first = NULL, .data_destroy = NULL - }; - return l; -} - -list_s *list_new_l() -{ - return list_new(); -} - -void list_destroy(list_s *l) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->key) - free(current->key); - if (current->data && l->data_destroy) - l->data_destroy(current->data); - list_elem_s *next = current->next; - free(current); - current = next; - } - - free(l); -} - -int list_key_insert(list_s *l, char *key, void *data) -{ - list_elem_s *current = l->first; - - while (current) { - // Already in the list - if (strcmp(current->key, key) == 0) - return 0; - current = current->next; - } - - list_elem_s *e = Malloc(list_elem_s); - - e->prev = NULL; - e->next = l->first; - e->key = Clone(key); - e->key_l = -1; - e->data = data; - - if (l->first) { - l->first->prev = e; - l->first = e; - - } else { - l->first = e; - } - - return 1; -} - -int list_key_insert_l(list_s *l, const long key, void *data) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->key_l == key) - return 0; - current = current->next; - } - - list_elem_s *e = Malloc(list_elem_s); - - e->prev = NULL; - e->next = l->first; - e->key = NULL; - e->key_l = key; - e->data = data; - - if (l->first) { - l->first->prev = e; - l->first = e; - - } else { - l->first = e; - } - - return 1; -} - -void _list_elem_remove(list_s *l, list_elem_s *e) -{ - if (l->first == e) { - list_elem_s *first = e->next; - if (first) - first->prev = NULL; - l->first = first; - - } else { - list_elem_s *prev = e->prev; - list_elem_s *next = e->next; - - prev->next = next; - if (next) - next->prev = prev; - } - - if (e->key) - free(e->key); - free(e); -} - -void* list_key_remove(list_s *l, char *key) -{ - list_elem_s *current = l->first; - - while (current) { - if (strcmp(current->key, key) == 0) { - void *data = current->data; - _list_elem_remove(l, current); - return data; - } - current = current->next; - } - - return NULL; -} - -void* list_key_remove_l(list_s *l, const long key) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->key_l == key) { - void *data = current->data; - _list_elem_remove(l, current); - return data; - } - current = current->next; - } - - return NULL; -} - -void* list_key_get(list_s *l, char *key) -{ - list_elem_s *current = l->first; - - while (current) { - if (strcmp(current->key, key) == 0) - return current->data; - current = current->next; - } - - return NULL; -} - -void* list_key_get_l(list_s *l, const long key) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->key_l == key) - return current->data; - current = current->next; - } - - return NULL; -} - -void list_run_cb(list_s* l, void (*cb)(void *data)) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->data) - cb(current->data); - current = current->next; - } -} - -void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->data) - cb(current->data, data_); - current = current->next; - } -} - -void list_print(list_s *l) -{ - list_elem_s *current = l->first; - - while (current) { - if (current->key != NULL) { - Put("list:%p key:'%s' data:%p", (void*)l, - current->key, current->data); - } else { - Put("list:%p key:%ld data:%p", (void*)l, - current->key_l, current->data); - } - current = current->next; - } -} - -void list_test(void) -{ - list_s *l = list_new(); - void* somedata = (void*)l; - - assert(1 == list_key_insert(l, "foo", (void*)1)); - assert(1 == list_key_insert(l, "bar", (void*)2)); - assert(1 == list_key_insert(l, "baz", (void*)3)); - assert(2 == (long)list_key_remove(l, "bar")); - assert(1 == (long)list_key_remove(l, "foo")); - assert(3 == (long)list_key_remove(l, "baz")); - - assert(1 == list_key_insert(l, "I/O replay", somedata)); - assert(1 == list_key_insert(l, "for", somedata)); - assert(1 == list_key_insert(l, "benchmarking your server", somedata)); - assert(0 == list_key_insert(l, "for", somedata)); - - assert(NULL != list_key_get(l, "benchmarking your server")); - assert(NULL == list_key_get(l, "Mimecast")); - - assert(NULL != list_key_remove(l, "benchmarking your server")); - assert(NULL == list_key_remove(l, "benchmarking your server")); - assert(1 == list_key_insert(l, "benchmarking your server", somedata)); - - assert(1 == list_key_insert(l, "MiMecast", (void*)42)); - assert(42 == (long)list_key_get(l, "MiMecast")); - - l = list_new_l(); - - assert(1 == list_key_insert_l(l, 1, (void*)1)); - assert(1 == list_key_insert_l(l, 2, (void*)2)); - assert(1 == list_key_insert_l(l, 3, (void*)3)); - assert(1 == (long)list_key_get_l(l, 1)); - assert(1 == (long)list_key_remove_l(l, 1)); - assert(1 != (long)list_key_remove_l(l, 1)); - assert(3 == (long)list_key_remove_l(l, 3)); - - assert(1 == list_key_insert_l(l, 1234, somedata)); - assert(1 == list_key_insert_l(l, 13, somedata)); - assert(1 == list_key_insert_l(l, 666, somedata)); - assert(0 == list_key_insert_l(l, 13, somedata)); - - assert(NULL != list_key_get_l(l, 666)); - assert(NULL == list_key_get_l(l, 777)); - - assert(NULL != list_key_remove_l(l, 666)); - assert(NULL == list_key_remove_l(l, 666)); - assert(1 == list_key_insert_l(l, 666, somedata)); - - assert(1 == list_key_insert_l(l, 42, (void*)42)); - assert(42 == (long)list_key_get_l(l, 42)); - - //list_print(l); -} diff --git a/ioreplay/src/datas/list.h b/ioreplay/src/datas/list.h deleted file mode 100644 index 385333c..0000000 --- a/ioreplay/src/datas/list.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef LIST_H -#define LIST_H - -#include "../defaults.h" - -/** - * @brief Definition of a linked list element - */ -typedef struct list_elem_s_ { - struct list_elem_s_ *prev; /**< The previous element */ - struct list_elem_s_ *next; /**< The next element */ - char *key; /**< The key of the lemenet */ - long key_l; /**< The same as key, but for long keys */ - void *data; /**< Pointer to the stored data */ -} list_elem_s; - -/** - * @brief Definition of a named linked list data structure - * - * There are two version of this list data structure. One version is utilising - * string keys and the other one is utilising long keys. - */ -typedef struct list_s_ { - list_elem_s *first; /**< The first element, NULL if list empty */ - void (*data_destroy)(void *data); /**< Callback to destroy all data */ -} list_s; - -list_s* list_new(); -list_s* list_new_l(); -void list_destroy(list_s* l); -void list_run_cb(list_s* l, void (*cb)(void *data)); -void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_); -int list_key_insert(list_s* l, char *key, void *data); -int list_key_insert_l(list_s* l, const long key, void *data); -void* list_key_remove(list_s* l, char *key); -void* list_key_remove_l(list_s* l, const long key); -void* list_key_get(list_s* l, char *key); -void* list_key_get_l(list_s* l, const long key); -void list_print(list_s* l); -void list_test(); - -#endif // LIST_H diff --git a/ioreplay/src/datas/rbuffer.c b/ioreplay/src/datas/rbuffer.c deleted file mode 100644 index c019e6c..0000000 --- a/ioreplay/src/datas/rbuffer.c +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rbuffer.h" - -rbuffer_s *rbuffer_new(const int size) -{ - rbuffer_s *r = Malloc(rbuffer_s); - - r->size = size; - r->read_pos = size-1; - r->write_pos = 0; - r->ring = Calloc(size, void*); - - Mset(r->ring, 0, size, void*); - - return r; -} - -void rbuffer_destroy(rbuffer_s *r) -{ - if (r) { - free(r->ring); - free(r); - } -} - -bool rbuffer_insert(rbuffer_s* r, void *data) -{ - if (r->write_pos == r->read_pos) - // Ring buffer is full - return false; - - r->ring[r->write_pos] = data; - r->write_pos = (r->write_pos+1) % r->size; - - return true; -} - -bool rbuffer_has_next(rbuffer_s* r) -{ - sig_atomic_t read_pos = (r->read_pos+1) % r->size; - - if (read_pos == r->write_pos) - // No more items to read, buffer is empty - { - return false; - } - - return true; -} - -void* rbuffer_get_next(rbuffer_s* r) -{ - sig_atomic_t read_pos = (r->read_pos+1) % r->size; - - if (read_pos == r->write_pos) - // No more items to read, buffer is empty - { - return NULL; - } - - void *data = r->ring[read_pos]; - r->ring[read_pos] = NULL; - r->read_pos = read_pos; - - return data; -} - -void rbuffer_print(rbuffer_s* r) -{ - Put("rbuffer_s (%p):", (void*)r); - Put("\tsize: %d", (int)r->size); - Put("\tread_pos: %d", r->read_pos); - Put("\twrite_pos: %d", r->write_pos); - Out("\toccupied slots: "); - for (int i = 0; i < r->size; ++i) - if (r->ring[i]) { - Out("%d:%p ", i, r->ring[i]); - } - Out("\n"); -} - -void rbuffer_test(void) -{ - rbuffer_s *r = rbuffer_new(5); - assert(NULL == rbuffer_get_next(r)); - - assert(rbuffer_insert(r, (void*)1)); - assert(rbuffer_insert(r, (void*)2)); - assert(rbuffer_insert(r, (void*)3)); - assert(rbuffer_insert(r, (void*)4)); - assert(!rbuffer_insert(r, (void*)5)); - rbuffer_print(r); - - assert(rbuffer_has_next(r)); - assert(1 == (long) rbuffer_get_next(r)); - assert(2 == (long) rbuffer_get_next(r)); - assert(3 == (long) rbuffer_get_next(r)); - assert(4 == (long) rbuffer_get_next(r)); - assert(!rbuffer_has_next(r)); - assert(NULL == rbuffer_get_next(r)); - - assert(rbuffer_insert(r, (void*)1)); - assert(1 == (long) rbuffer_get_next(r)); - assert(rbuffer_insert(r, (void*)2)); - assert(2 == (long) rbuffer_get_next(r)); - assert(rbuffer_insert(r, (void*)3)); - assert(3 == (long) rbuffer_get_next(r)); - assert(rbuffer_insert(r, (void*)4)); - assert(4 == (long) rbuffer_get_next(r)); - assert(rbuffer_insert(r, (void*)5)); - assert(5 == (long) rbuffer_get_next(r)); - assert(NULL == rbuffer_get_next(r)); - rbuffer_print(r); - - assert(rbuffer_insert(r, (void*)1)); - rbuffer_print(r); - assert(rbuffer_insert(r, (void*)2)); - assert(1 == (long) rbuffer_get_next(r)); - rbuffer_print(r); - assert(rbuffer_insert(r, (void*)3)); - assert(2 == (long) rbuffer_get_next(r)); - rbuffer_print(r); - assert(rbuffer_insert(r, (void*)4)); - assert(3 == (long) rbuffer_get_next(r)); - rbuffer_print(r); - assert(rbuffer_insert(r, (void*)5)); - rbuffer_print(r); - assert(4 == (long) rbuffer_get_next(r)); - rbuffer_print(r); - assert(5 == (long) rbuffer_get_next(r)); - assert(NULL == rbuffer_get_next(r)); - - rbuffer_destroy(r); -} diff --git a/ioreplay/src/datas/rbuffer.h b/ioreplay/src/datas/rbuffer.h deleted file mode 100644 index fa634de..0000000 --- a/ioreplay/src/datas/rbuffer.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef RBUFFER_H -#define RBUFFER_H - -#include "signal.h" - -#include "../defaults.h" - -/** - * @brief An atomic ring buffer data type definition - * - * This data structure can be used for the common producer/consumer problem. - * As long as there is only max one producer thread and max one consumer thread - * it can be used without any mutex locking. All the operations are atomic. - */ -typedef struct rbuffer_s_ { - /** - * The positions are atomic, means the ring buffer can be accessed from - * multiple threads concurrently (one producer and one consumer thread). - * This is the current read position. - */ - sig_atomic_t read_pos; - /** - * This is the current write position. - */ - sig_atomic_t write_pos; - /** - * Holds the pointers to the actual ring data stored in the ring buffer - */ - void **ring; - /** - * Determines how many elements the ring buffer can hold. The capacity - * will be size-1 though, as we need one empty slot. - */ - int size; -} rbuffer_s; - -/** - * @brief Creates a new ring buffer - * - * @param size The size of the ring buffer - * @return The new ring buffer object - */ -rbuffer_s* rbuffer_new(const int size); - -/** - * @brief Destroys a ring buffer - * - * @param r The ring buffer object - */ -void rbuffer_destroy(rbuffer_s* r); - -/** - * @brief Inserts data pointer to the ring buffer - * - * @param r The ring buffer object - * @param data The data pointer - */ -bool rbuffer_insert(rbuffer_s* r, void *data); - -/** - * @brief Determines whether there is any data in the ring buffer - * - * @param r The ring buffer object - * @return True if there is any data, false otherwise - */ -bool rbuffer_has_next(rbuffer_s* r); - -/** - * @brief Returns and removes the next element from the ring buffer - * - * @param r The ring buffer object - * @return The data pointer - */ -void* rbuffer_get_next(rbuffer_s* r); - -/** - * @brief Prints a ring buffer - * - * @param r The ring buffer object - */ -void rbuffer_print(rbuffer_s* r); - -/** - * @brief Unit tests the ring buffer - */ -void rbuffer_test(void); - -#endif // RBUFFER_H diff --git a/ioreplay/src/datas/stack.c b/ioreplay/src/datas/stack.c deleted file mode 100644 index 94e83e3..0000000 --- a/ioreplay/src/datas/stack.c +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "stack.h" - - -stack_s *stack_new() -{ - stack_s *s = Malloc(stack_s); - *s = (stack_s) { - .top = NULL, .size = 0 - }; - return s; -} - -void stack_destroy(stack_s *s) -{ - stack_elem_s *current = s->top; - - while (current) { - stack_elem_s *next = current->next; - free(current); - current = next; - } - - free(s); -} - -void stack_push(stack_s *s, void *data) -{ - stack_elem_s *new_top = Malloc(stack_elem_s); - - *new_top = (stack_elem_s) { - .next = s->top, - .data = data - }; - - s->top = new_top; - s->size++; -} - -void* stack_pop(stack_s *s) -{ - if (s->top == NULL) { - return NULL; - } - - stack_elem_s *old_top = s->top; - - void *data = old_top->data; - s->top = old_top->next; - free(old_top); - s->size--; - - return data; -} - -int stack_is_empty(stack_s *s) -{ - return s->top == NULL; -} - -stack_s* stack_new_reverse_from(stack_s *s) -{ - stack_s* r = stack_new(); - - while (!stack_is_empty(s)) { - stack_push(r, stack_pop(s)); - } - - stack_destroy(s); - - return r; -} diff --git a/ioreplay/src/datas/stack.h b/ioreplay/src/datas/stack.h deleted file mode 100644 index 87e0974..0000000 --- a/ioreplay/src/datas/stack.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef STACK_H -#define STACK_H - -#include "../defaults.h" - -/** - * @brief Definition of a stack element - */ -typedef struct stack_elem_s_ { - struct stack_elem_s_ *next; /**< The next element */ - void *data; /**< Pointer to the stored data in the current element */ -} stack_elem_s; - -/** - * @brief Definition of a stack data structure - */ -typedef struct stack_s_ { - stack_elem_s *top; /**< The top element of the stack, NULL if empty */ - unsigned long size; /**< A count how many elements are in the stack */ -} stack_s; - -stack_s* stack_new(); -stack_s* stack_new_reverse_from(stack_s* s); -void stack_destroy(stack_s* s); -void stack_push(stack_s* s, void *data); -void* stack_pop(stack_s* s); -int stack_is_empty(stack_s* s); - -#endif // STACK_H diff --git a/ioreplay/src/defaults.h b/ioreplay/src/defaults.h deleted file mode 100644 index de7910e..0000000 --- a/ioreplay/src/defaults.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef DEFAULTS_H -#define DEFAULTS_H - -#include "utils/utils.h" - -/** Version of the supported .capture format */ -#define CAPTURE_VERSION 1 -/** Version of the supported .replay format */ -#define REPLAY_VERSION 1 -/** Max amount of tokens per line in the .capture file */ -#define MAX_TOKENS 10 -/** Max line length in either .capture or .replay file */ -#define MAX_LINE_LEN 1024*8 -/** Controls how many tasks can be queued and buffered per worker thread */ -#define TASK_BUFFER_PER_THREAD 512 -/** Version of I/O Replay */ -#define IOREPLAY_VERSION "0.3-develop" -/** Copyright information */ -#define IOREPLAY_COPYRIGHT "Mimecast 2017, 2018 (c)" -/** Max open files resource user limit */ -#define SET_RLIMIT_NOFILE 369216 -/** Max processes resource user limit */ -#define SET_RLIMIT_NPROC 30768 - -// The following are for debugging purposes only - -//#define NO_IOOP -//#define THREAD_DEBUG -//#define LOG_FILTERED - -/** - * @brief Return status codes - */ -typedef enum status_e_ { - SUCCESS, /**< Great success! */ - UNKNOWN, /**< Unknown return status :-/ */ - ERROR, /**< An error happened :-( */ -} status_e; - -#endif // DEFAULTS_H diff --git a/ioreplay/src/generate/generate.c b/ioreplay/src/generate/generate.c deleted file mode 100644 index 05445ae..0000000 --- a/ioreplay/src/generate/generate.c +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "generate.h" - -#include "../meta/meta.h" -#include "gtask.h" -#include "gwriter.h" -#include "gparser.h" - -#include - -#define _MAX_PROCESSES 1024*1024*10 - -#define _Perc_filtered (g->num_lines_filtered / (g->lineno/100.0)) - -generate_s* generate_new(options_s *opts) -{ - generate_s *g = Malloc(generate_s); - - g->writer = NULL; - g->lineno = 0; - g->name = opts->name; - g->replay_fd = NULL; - g->mps = mounts_new(opts); - g->num_lines_filtered = 0; - g->num_vsizes = 0; - g->start_time = -1; - g->pid_map = amap_new(_MAX_PROCESSES); - g->vsize_map = hmap_new(_MAX_PROCESSES); - g->mmap_map = hmap_new(1024*1024); - g->vfd_buffer = rbuffer_new(1024); - g->num_mapped_pids = 0; - g->num_mapped_fds = 10; - g->opts = opts; - g->reuse_queue = rbuffer_new(1024); - g->replay_fd = Fopen(opts->replay_file, "w"); - - return g; -} - -void generate_destroy(generate_s *g) -{ - // TODO: Also clean the contets of these maps - amap_destroy(g->pid_map); - hmap_destroy(g->vsize_map); - hmap_destroy(g->mmap_map); - rbuffer_destroy(g->vfd_buffer); - mounts_destroy(g->mps); - - gtask_s *task = NULL; - while (NULL != (task = rbuffer_get_next(g->reuse_queue))) - gtask_destroy(task); - rbuffer_destroy(g->reuse_queue); - - fclose(g->replay_fd); - free(g); -} - - -status_e generate_run(options_s *opts) -{ - generate_s *g = generate_new(opts); - Put("Parsing file %s, writing output to %s", opts->capture_file, - opts->replay_file); - FILE *capture_fd = Fopen(opts->capture_file, "r"); - - size_t len = 0; - ssize_t read; - char *line = NULL; - - set_limits_drop_root(opts->user); - - // Reserve first few bytes for meta information - meta_s *meta = meta_new(g->replay_fd); - meta_reserve(meta); - - // The writer will write the .replay file - gwriter_s *writer = gwriter_new(g); - - // The parser will parse every line of the .capture file - gparser_s *parser = gparser_new(g); - - g->writer = writer; - - // Start one writer and one parser thread! - gparser_start(parser); - gwriter_start(writer); - - Out("Processing, it may take a while: "); - - // Process each line of the .capture file. Determine line by line whether - // the I/O operation makes sense or not. It might be that SystemTap skipped - // some I/O ops due to system overload or other issues. The result is that - // some lines may be corrupt or contain I/O operations on unknown file - // handles. It could also be that there are operations on unknown - // file handles such as sockets etc. These will be all filtered out by - // either the parser or the writer thread! - - while ((read = getline(&line, &len, capture_fd)) != -1) { - if (0 > ++g->lineno) { - Error("lineno:%lu Line number overflow", g->lineno); - } - if (strlen(line) >= MAX_LINE_LEN) { - Error("lineno:%lu Exceeded max line length", g->lineno); - } - - // Create a new generate task (try to reuse a task object)... - gtask_s *t = rbuffer_get_next(g->reuse_queue); - if (!t) { - t = gtask_new(g); - } else if (t->ret != 0) { - g->num_lines_filtered++; - } - gtask_init(t, line, g->lineno); - - // ...pass it to the parser queue - while (!rbuffer_insert(parser->queue, t)) - usleep(100); - - if (g->lineno % 1000000 == 0) { - Out(" %lu (filtered:%.2lf%%)", g->lineno, _Perc_filtered); - } - } - - Put("\nDone reading input file!"); - - Put("Waiting for parser thread..."); - gparser_terminate(parser); - gparser_destroy(parser); - - Put("Waiting for writer thread..."); - gwriter_terminate(writer); - gwriter_destroy(writer); - - // Retrieve all left over processed tasks to collect the - // statistics! - gtask_s *t; - while (NULL != (t = rbuffer_get_next(g->reuse_queue))) { - if (t->ret != 0) - g->num_lines_filtered++; - gtask_destroy(t); - } - - Put("Processed %lu lines in total, had to filter out %.2lf%%", - g->lineno, _Perc_filtered); - - Put("Writing init section to '%s'...", opts->replay_file); - fprintf(g->replay_fd, "#INIT\n"); - off_t init_offset = ftello(g->replay_fd); - hmap_run_cb(g->vsize_map, generate_write_init_cb); - - Put("Writing meta header to '%s'...", opts->replay_file); - meta_write_start(meta); - - // The meta header is being written to the first line of the .replay - // file and used by ioreplay to do various things (e.g. initializing - // the test correctly, creating the internal data structures with the - // correct sizes etc. - - meta_write_l(meta, "replay_version", REPLAY_VERSION); - meta_write_l(meta, "init_offset", init_offset); - - meta_write_s(meta, "user", opts->user); - meta_write_s(meta, "name", opts->name); - - meta_write_l(meta, "num_vsizes", g->num_vsizes); - meta_write_l(meta, "num_mapped_pids", g->num_mapped_pids); - meta_write_l(meta, "num_mapped_fds", g->num_mapped_fds); - meta_write_l(meta, "num_lines", g->lineno - g->num_lines_filtered); - - meta_destroy(meta); - fclose(capture_fd); - - Put("Generating '%s' done", opts->replay_file); - generate_destroy(g); - - return SUCCESS; -} - -void generate_write_init_cb(void *data) -{ - vsize_s *l = data; - generate_s *g = l->generate; - - if (l->required && strlen(l->path) > 0) { - fprintf(g->replay_fd, "%d|%d|%ld|%s|\n", - l->is_dir, l->is_file, -l->vsize_deficit, l->path); - } -} - -vsize_s* generate_vsize_by_path(generate_s *g, gtask_s *t, - char *path) -{ - vsize_s *v = NULL; - - if (!path && t) - path = t->path; - - Error_if(!path, "No path specified"); - v = hmap_get(g->vsize_map, path); - - if (!v) { - v = vsize_new(path, ++g->num_vsizes, g); - hmap_insert(g->vsize_map, path, v); - } - - if (t) - t->vsize = v; - - return v; -} - -void generate_gprocess_by_realpid(generate_s *g, gtask_s *t) -{ - // Get the virtual process data object from the virtual PID space. - t->gprocess = amap_get(g->pid_map, t->pid); - if (t->gprocess == NULL) { - t->gprocess = gprocess_new(t->pid, ++g->num_mapped_pids); - if (amap_set(g->pid_map, t->pid, t->gprocess)) { - Error("lineno:%lu Can not insert PID %ld", t->lineno, t->pid); - } - } -} diff --git a/ioreplay/src/generate/generate.h b/ioreplay/src/generate/generate.h deleted file mode 100644 index cf096d2..0000000 --- a/ioreplay/src/generate/generate.h +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GENERATE_H -#define GENERATE_H - -#include "gwriter.h" -#include "../datas/amap.h" -#include "../datas/hmap.h" -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "../mounts.h" -#include "../options.h" - -// Forward declarations (header include hell) -struct gtask_s_; - -/** - * @brief The generate object definition - * - * This is the general data structure required to generate a .replay file from - * the .capture file. - */ -typedef struct generate_s_ { - long lineno; /**< The current line number */ - long num_lines_filtered; /**< The amount of lines filtered out */ - long start_time; /**< The start time from the .capture file */ - char *name; /**< The name of the test specified by the user */ - FILE *replay_fd; /**< The fd of the .replay file */ - mounts_s *mps; /**< The mounts object */ - hmap_s *mmap_map; /**< mmap address mappings */ - amap_s *pid_map; /**< A map of all virtual process objects */ - unsigned long num_mapped_pids; /**< The amount of mapped PIDs */ - unsigned long num_mapped_fds; /**< The amount of mapped FDs */ - hmap_s *vsize_map; /**< A hash map of all virtual size objects */ - unsigned long num_vsizes; /**< The amount of virtual sizes */ - options_s *opts; /**< A pointer to the options object */ - rbuffer_s *vfd_buffer; /**< A virtual fd buffer, for reusing these */ - rbuffer_s *reuse_queue; /**< A task buffer, for reusing these */ - struct gwriter_s_ *writer; /**< A pointer to the writer object */ -} generate_s; - -/** - * @brief Creates a new generate object - * - * @param opts The options object - * @return The new generate object - */ -generate_s* generate_new(options_s *opts); - -/** - * @brief Destroys a generate object - * - * @param g The generate object to destroy - */ -void generate_destroy(generate_s* g); - -/** - * @brief Generates a .replay file from a .capture file - * - * @param opts The options object - * @return SUCCESS on success - */ -status_e generate_run(options_s *opts); - -/** - * @brief Callback to write the INIT section to the .replay file - * - * This function writes a list of all pre-required - * paths to the .replay file. That then can be used - * by ioreplay to initialise the test enironment. - * - * @param data A pointer to the vsize timestamp object - */ -void generate_write_init_cb(void *data); - -/** - * @brief Retrieves the virtual size object of a given path - * - * A new one will be created in case there is no such virtual size object yet. - * - * @param g The generate object - * @param t The task object (vfd will be stored to t->vfd) - * @param path The file path - * @return The virtual size object - */ -vsize_s* generate_vsize_by_path(generate_s *g, struct gtask_s_ *t, - char *path); - -/** - * @brief Retrieves the virtual process object of a given real PID - * - * A new one will be created in case there is no such virtual process object - * yet. - * - * @param g The generate object - * @param t The task object (vfd will be stored to t->gprocess) - */ -void generate_gprocess_by_realpid(generate_s *g, struct gtask_s_ *t); - -#endif // GENERATE_H diff --git a/ioreplay/src/generate/gioop.c b/ioreplay/src/generate/gioop.c deleted file mode 100644 index 01701bc..0000000 --- a/ioreplay/src/generate/gioop.c +++ /dev/null @@ -1,838 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gioop.h" - -status_e gioop_run(gwriter_s *w, gtask_s *t) -{ - status_e ret = SUCCESS; - - // There was already an error in the parser (parser.c) processing this - // task! Don't process it futher. - if (t->ret != SUCCESS) { - Cleanup(t->ret); - } - - generate_s *g = w->generate; - - // Get the virtual process data object from the virtual PID space and store - // a pointer to it to t->gprocess - generate_gprocess_by_realpid(g, t); - - // One of the open syscalls may openes a file handle succesfully - if (Eq(t->op, "open")) { - Cleanup(gioop_open(w, t, g)); - - } else if (Eq(t->op, "openat")) { - Cleanup(gioop_openat(w, t, g)); - - } else if (Eq(t->op, "creat")) { - Cleanup(gioop_creat(w, t, g)); - } - - // Get the virtual file descriptor of a given real fd and store a pointer - // to it to t->vfd. - if (t->has_fd) { - ret = gprocess_vfd_by_realfd(t->gprocess, t); - Cleanup_unless(SUCCESS, ret); - } - - - if (Eq(t->op, "close")) { - Cleanup(gioop_close(w, t, g)); - - } else if (Eq(t->op, "stat")) { - Cleanup(gioop_stat(w, t, g)); - - } else if (Eq(t->op, "statfs")) { - Cleanup(gioop_statfs(w, t, g)); - - } else if (Eq(t->op, "statfs64")) { - Cleanup(gioop_statfs64(w, t, g)); - - } else if (Eq(t->op, "fstat")) { - Cleanup(gioop_fstat(w, t, g)); - - } else if (Eq(t->op, "fstatat")) { - Cleanup(gioop_fstatat(w, t, g)); - - } else if (Eq(t->op, "fstatfs")) { - Cleanup(gioop_fstatfs(w, t, g)); - - } else if (Eq(t->op, "fstatfs64")) { - Cleanup(gioop_fstatfs64(w, t, g)); - - } else if (Eq(t->op, "rename")) { - Cleanup(gioop_rename(w, t, g)); - - } else if (Eq(t->op, "renameat")) { - Cleanup(gioop_renameat(w, t, g)); - - } else if (Eq(t->op, "renameat2")) { - Cleanup(gioop_renameat2(w, t, g)); - - } else if (Eq(t->op, "read")) { - Cleanup(gioop_read(w, t, g)); - - } else if (Eq(t->op, "readv")) { - Cleanup(gioop_readv(w, t, g)); - - } else if (Eq(t->op, "readahead")) { - Cleanup(gioop_readahead(w, t, g)); - - } else if (Eq(t->op, "readdir")) { - Cleanup(gioop_readdir(w, t, g)); - - } else if (Eq(t->op, "readlink")) { - Cleanup(gioop_readlink(w, t, g)); - - } else if (Eq(t->op, "readlinkat")) { - Cleanup(gioop_readlinkat(w, t, g)); - - } else if (Eq(t->op, "write")) { - Cleanup(gioop_write(w, t, g)); - - } else if (Eq(t->op, "writev")) { - Cleanup(gioop_writev(w, t, g)); - - } else if (Eq(t->op, "lseek")) { - Cleanup(gioop_lseek(w, t, g)); - - } else if (Eq(t->op, "getdents")) { - Cleanup(gioop_getdents(w, t, g)); - - } else if (Eq(t->op, "mkdir")) { - Cleanup(gioop_mkdir(w, t, g)); - - } else if (Eq(t->op, "rmdir")) { - Cleanup(gioop_rmdir(w, t, g)); - - } else if (Eq(t->op, "mkdirat")) { - Cleanup(gioop_mkdirat(w, t, g)); - - } else if (Eq(t->op, "unlink")) { - Cleanup(gioop_unlink(w, t, g)); - - } else if (Eq(t->op, "unlinkat")) { - Cleanup(gioop_unlinkat(w, t, g)); - - } else if (Eq(t->op, "lstat")) { - Cleanup(gioop_lstat(w, t, g)); - - } else if (Eq(t->op, "fsync")) { - Cleanup(gioop_fsync(w, t, g)); - - } else if (Eq(t->op, "fdatasync")) { - Cleanup(gioop_fdatasync(w, t, g)); - - } else if (Eq(t->op, "sync")) { - Cleanup(gioop_sync(w, t, g)); - - } else if (Eq(t->op, "syncfs")) { - Cleanup(gioop_syncfs(w, t, g)); - - } else if (Eq(t->op, "sync_file_range")) { - Cleanup(gioop_sync_file_range(w, t, g)); - - } else if (Eq(t->op, "fcntl")) { - Cleanup(gioop_fcntl(w, t, g)); - - } else if (Eq(t->op, "fcntl")) { - Cleanup(gioop_fcntl(w, t, g)); - - } else if (Eq(t->op, "mmap2")) { - // Support for mmap added later - - } else if (Eq(t->op, "munmap")) { - // Support for mmap added later - - } else if (Eq(t->op, "mremap")) { - // Support for mmap added later - - } else if (Eq(t->op, "msync")) { - // Support for mmap added later - - } else if (Eq(t->op, "chmod")) { - Cleanup(gioop_chmod(w, t, g)); - - } else if (Eq(t->op, "fchmodat")) { - Cleanup(gioop_chmod(w, t, g)); - - } else if (Eq(t->op, "fchmod")) { - Cleanup(gioop_fchmod(w, t, g)); - - } else if (Eq(t->op, "chown")) { - Cleanup(gioop_chown(w, t, g)); - - } else if (Eq(t->op, "chown16")) { - Cleanup(gioop_chown(w, t, g)); - - } else if (Eq(t->op, "lchown")) { - Cleanup(gioop_lchown(w, t, g)); - - } else if (Eq(t->op, "lchown16")) { - Cleanup(gioop_lchown(w, t, g)); - - } else if (Eq(t->op, "fchown")) { - Cleanup(gioop_fchown(w, t, g)); - - } else if (Eq(t->op, "fchownat")) { - Cleanup(gioop_chown(w, t, g)); - - } else if (Eq(t->op, "exit_group")) { - Cleanup(gioop_exit_group(w, t, g)); - - } else { - Cleanup(ERROR;); - } - -cleanup: - -#ifdef LOG_FILTERED - if (ret != SUCCESS) - t->filtered_where = __FILE__; -#endif - - t->ret = ret; - return ret; -} - -status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd || t->path == NULL || t->flags == -1) { - return ERROR; - } - - gprocess_create_vfd_by_realfd(t->gprocess, t, g); - generate_vsize_by_path(g, t, NULL); - - Gioop_write(OPEN, "%ld|%s|%d|%d|open", - t->mapped_fd, t->path, t->mode, t->flags); - - if (t->fd > 0) - vsize_open(t->vsize, t->vfd, t->path, t->flags); - - return SUCCESS; -} - -status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd || t->path == NULL || t->flags == -1) { - return ERROR; - } - - gprocess_create_vfd_by_realfd(t->gprocess, t, g); - generate_vsize_by_path(g, t, NULL); - Gioop_write(OPEN_AT, "%ld|%s|%d|%d|openat", - t->mapped_fd,t->path, t->mode, t->flags); - if (t->fd > 0) - vsize_open(t->vsize, t->vfd, t->path, t->flags); - - return SUCCESS; -} - -status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd || t->path == NULL || t->flags == -1) { - return ERROR; - } - - gprocess_create_vfd_by_realfd(t->gprocess, t, g); - generate_vsize_by_path(g, t, NULL); - - Gioop_write(CREAT, "%ld|%s|%d|%d|creat", - t->mapped_fd, t->path, t->mode, t->flags); - if (t->fd > 0) - vsize_open(t->vsize, t->vfd, t->path, t->flags); - - return SUCCESS; -} - - -status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(CLOSE, "%ld|%d|close", t->mapped_fd, t->status); - - if (t->status == 0) - vsize_close(t->vsize, t->vfd); - - hmap_remove_l(t->gprocess->fd_map, t->fd); - hmap_remove_l(t->gprocess->vfd_map, t->mapped_fd); - - if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) - vfd_destroy(t->vfd); - - return SUCCESS; -} - -status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(STAT, "%s|%d|stat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(STATFS, "%s|%d|statfs", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(STATFS64, "%s|%d|statfs64", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSTAT, "%ld|%d|fstat", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(FSTAT_AT, "%s|%d|fstatat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSTATFS, "%ld|%d|fstatfs", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSTATFS64, "%ld|%d|fstatfs64", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL || t->path2 == NULL ) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(RENAME, "%s|%s|%d|rename", t->path, t->path2, t->status); - - if (t->status == 0) { - t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); - vsize_rename(t->vsize, t->vsize2, t->path, t->path2); - } - - return SUCCESS; -} - -status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL || t->path2 == NULL ) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(RENAME_AT, "%s|%s|%d|renameat", t->path, t->path2, t->status); - - if (t->status == 0) { - t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); - vsize_rename(t->vsize, t->vsize2, t->path, t->path2); - } - - return SUCCESS; -} -status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL || t->path2 == NULL ) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(RENAME_AT2, "%s|%s|%d|renameat2", - t->path, t->path2, t->status); - - if (t->status == 0) { - t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); - vsize_rename(t->vsize, t->vsize2, t->path, t->path2); - } - - return SUCCESS; -} - -status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READ, "%ld|%ld|read", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READ, "%ld|%ld|readv", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READAHEAD, "%ld|%ld|%ld|readahead", - t->mapped_fd, t->offset, t->count); - - return SUCCESS; -} - -status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(READDIR, "%ld|%d|readdir", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(READLINK, "%s|%d|readlink", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(READLINK_AT, "%s|%d|readlinkat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(WRITE, "%ld|%ld|write", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_write(t->vsize, t->vfd, t->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(WRITEV, "%ld|%ld|writev", t->mapped_fd, t->bytes); - - if (t->bytes > 0) - vsize_write(t->vsize, t->vfd, t->path, t->bytes); - - return SUCCESS; -} - -status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(LSEEK, "%ld|%ld|%ld|%ld|lseek", - t->mapped_fd, t->offset, t->whence, t->bytes); - - if (t->bytes >= 0) - vsize_seek(t->vsize, t->vfd, t->bytes); - - return SUCCESS; -} - -status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(GETDENTS, "%ld|%ld|%ld|getdents", - t->mapped_fd, t->count, t->bytes); - - return SUCCESS; -} - -status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(MKDIR, "%s|%d|%d|mkdir", t->path, t->mode, t->status); - - if (t->status == 0) - vsize_mkdir(t->vsize, t->path); - - return SUCCESS; -} -status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(MKDIR, "%s|%d|rmdir", t->path, t->status); - - if (t->status == 0) - vsize_rmdir(t->vsize, t->path); - - return SUCCESS; -} -status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(MKDIR_AT, "%s|%d|%d|mkdirat", t->path, t->mode, t->status); - - if (t->status == 0) - vsize_mkdir(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(UNLINK, "%s|%d|unlink", t->path, t->status); - - if (t->status == 0) - vsize_unlink(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(UNLINK_AT, "%s|%d|unlinkat", t->path, t->status); - - if (t->status == 0) - vsize_unlink(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(LSTAT, "%s|%d|lstat", t->path, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FSYNC, "%ld|%d|fsync", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FDATASYNC, "%ld|%d|fdatasync", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g) -{ - Gioop_write(SYNC, "%d|sync", t->status); - - return SUCCESS; -} - -status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(SYNCFS, "%ld|%d|syncfs", t->mapped_fd, t->status); - - return SUCCESS; -} - -status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(SYNC_FILE_RANGE, "%ld|%ld|%ld|%d|sync_file_range", - t->mapped_fd, t->offset, t->bytes, t->status); - - return SUCCESS; -} - -status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - switch (t->F) { - case F_GETFD: - case F_GETFL: - case F_SETFD: - case F_SETFL: - break; - default: - return ERROR; - break; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FCNTL, "%ld|%d|%d|%d|fcntl", - t->mapped_fd, t->F, t->G, t->status); - - return SUCCESS; -} - -status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - Gioop_write(CHMOD, "%s|%d|%d|chmod", t->path, t->mode, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(FCHMOD, "%ld|%d|%d|fchmod", t->mapped_fd, t->mode, t->status); - - return SUCCESS; -} - -status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - // Hmm, maybe rename t->offset, because here it is used for the user UID - Gioop_write(CHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (!t->has_fd) { - return ERROR; - } - - generate_vsize_by_path(g, t, t->vfd->path); - // Hmm, maybe rename t->offset, because here it is used for the user UID - Gioop_write(FCHOWN, "%ld|%ld|%d|%d|fchown", t->mapped_fd, t->offset, t->G, t->status); - - return SUCCESS; -} - -status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g) -{ - if (t->path == NULL) { - return ERROR; - } - - generate_vsize_by_path(g, t, NULL); - // Hmm, maybe rename t->offset, because here it is used for the user UID - Gioop_write(LCHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); - - if (t->status == 0) - vsize_stat(t->vsize, t->path); - - return SUCCESS; -} - -status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g) -{ - // It means that the process and all its threads terminate. - // Therefore close all file handles of that process! - hmap_run_cb2(t->gprocess->vfd_map, gioop_close_all_vfd_cb, t); - - // Remove virtual process from pid map and destroy it - amap_unset(g->pid_map, t->pid); - gprocess_destroy(t->gprocess); - - return SUCCESS; -} - -void gioop_close_all_vfd_cb(void *data, void *data2) -{ - gtask_s *t = data2; - t->vfd = data; - generate_s *g = t->generate; - - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(CLOSE, "%ld|%d|close on exit_group", t->vfd->mapped_fd, 0); - vsize_close(t->vsize, t->vfd); -} - diff --git a/ioreplay/src/generate/gioop.h b/ioreplay/src/generate/gioop.h deleted file mode 100644 index ad49713..0000000 --- a/ioreplay/src/generate/gioop.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GIOOP_H -#define GIOOP_H - -#include "../defaults.h" -#include "gwriter.h" -#include "gtask.h" -#include "generate.h" - - -// Helper macro regarding writing the .replay file! - -#define Gioop_write(op, ...) \ - fprintf(g->replay_fd, "%ld|%ld|%ld|0|0|%d|", \ - t->mapped_time, \ - (t->vsize ? t->vsize->id : 0),\ - t->gprocess->mapped_pid, \ - op); \ - fprintf(g->replay_fd, __VA_ARGS__); \ - fprintf(g->replay_fd, "@%ld", t->lineno); \ - fprintf(g->replay_fd, "|\n") - -/** - * @brief Function used when closing all virtual FDs of a virtual process - * - * This function is run on all virtual file handles whenever a virtual generate - * process object (gprocess_s) gets destroyed. This is on an exit_group - * syscall (a thread group, a process with all its threads, terminates). Upon - * process termination Linux also closes all its file descriptors! This is what - * we simulate here! - * - * @param data The pointer to the virtual file descriptor object - * @param data2 The pointer to the corresponding generate task object. - */ -void gioop_close_all_vfd_cb(void *data, void *data2); - -/** - * @brief Run a generate I/O operation on a given task - * - * @param w The writer object - * @param t The task object - * @return SUCCESS if everything went fine - */ -status_e gioop_run(gwriter_s *w, gtask_s *t); - -status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g); -status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g); - -#endif // GIOOP_H diff --git a/ioreplay/src/generate/gparser.c b/ioreplay/src/generate/gparser.c deleted file mode 100644 index 514128f..0000000 --- a/ioreplay/src/generate/gparser.c +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gparser.h" - -#include "gtask.h" -#include "gwriter.h" - -void* gparser_pthread_start(void *data) -{ - gparser_s *p = data; - generate_s *g = p->generate; - gwriter_s *w = g->writer; - gtask_s *t = NULL; - - do { - while (NULL != (t = rbuffer_get_next(p->queue))) { - // First extract - gparser_extract(p, t); - // Second, pass the task to the writer thread - rbuffer_insert(w->queue, t); - } - usleep(100); - } while (!p->terminate); - - while (NULL != (t = rbuffer_get_next(p->queue))) { - gparser_extract(p, t); - rbuffer_insert(w->queue, t); - } - - return NULL; -} - -gparser_s* gparser_new(generate_s *g) -{ - gparser_s *p = Malloc(gparser_s); - - p->generate = g; - p->terminate = false; - p->queue = rbuffer_new(1024); - - return p; -} - -void gparser_start(gparser_s *p) -{ - start_pthread(&p->pthread, gparser_pthread_start, (void*)p); -} - -void gparser_destroy(gparser_s *p) -{ - rbuffer_destroy(p->queue); - free(p); -} - -void gparser_terminate(gparser_s *p) -{ - p->terminate = true; - pthread_join(p->pthread, NULL); -} - -void gparser_extract(gparser_s *p, gtask_s *t) -{ - status_e ret = SUCCESS; - generate_s *g = p->generate; - - char *saveptr; - char* tok = strtok2_r(t->line, ";:,", &saveptr); - int ntoks = 0; - - while (tok) { - if (++ntoks > MAX_TOKENS) { - ret = ERROR; - break; - } - ret = gparser_extract_tok(p, t, tok); - if (ret != SUCCESS) - break; - - tok = strtok2_r(NULL, ";:,", &saveptr); - } - - if (ret == SUCCESS) { - - // Check for the existance of mandatory values! - if (t->pid < 0 || t->tid < 0) { - Cleanup(ERROR); - - } else if (t->op == NULL) { - Cleanup(ERROR); - - } else if (t->mapped_time == -1) { - Cleanup(ERROR); - } - - // We are inserting ".ioreplay/NAME" to the paths. This enables us to - // run multiple tests simoultaneously. - - if (t->path) { - if (!mounts_transform_path(g->mps, g->name, - t->path, &t->path_r)) { - Cleanup(ERROR); - } - if (t->path_r) - t->path = t->path_r; - } - - if (t->path2) { - if (!mounts_transform_path(g->mps, g->name, - t->path2, &t->path2_r)) { - Cleanup(ERROR); - } - if (t->path2_r) - t->path2 = t->path2_r; - } - - } - -cleanup: - - t->ret = ret; - -#ifdef LOG_FILTERED - t->filtered_where = __FILE__; -#endif -} - -status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok) -{ - status_e ret = SUCCESS; - - if (gparser_token_not_ok(p, tok)) { - Cleanup(ERROR); - } - - generate_s *g = t->generate; - - char key = tok[0]; - char *value = tok; - value += 2; - - switch (key) { - case 'a': - // Address - t->address = strtol(value, NULL, 10); - break; - - case 'A': - // Address 2 - t->address2 = strtol(value, NULL, 10); - break; - - case 'b': - // Bytes - if (t->bytes != -1) { - Cleanup(ERROR); - } - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->bytes = strtol(value, NULL, 10); - break; - - case 'c': - // Count - if (t->count != -1) { - Cleanup(ERROR); - } - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->count = strtol(value, NULL, 10); - break; - - case 'd': - // Descriptor - if (t->fd != -1) { - Cleanup(ERROR); - } - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->fd = atoi(value); - if (t->fd > 0) - t->has_fd = true; - break; - - case 'f': - // Flags - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->flags = atoi(value); - break; - - case 'i': - // PID:TID - t->pidtid = value; - // Extract PID and TID from "PID:TID" - if (!gparser_get_pidtid(p, t->pidtid, &t->pid, &t->tid)) { - Cleanup(ERROR); - } - break; - - case 'm': - // Mode - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->mode = atoi(value); - break; - - case 'o': - // Operation - t->op = value; - break; - - case 'O': - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->offset = strtol(value, NULL, 10); - break; - - case 'W': - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->whence = strtol(value, NULL, 10); - break; - - case 'p': - // File path - t->path = value; - chreplace(t->path, '|', '_'); - strunquote(t->path); - break; - - case 'P': - // File path 2 - t->path2 = value; - chreplace(t->path2, '|', '_'); - strunquote(t->path2); - break; - - case 's': - // Cleanup status - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->status = atoi(value); - break; - - case 't': - // Time - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->mapped_time = strtol(value, NULL, 10); - // Start replay time from 0 - if (g->start_time == -1) { - g->start_time = t->mapped_time; - } - t->mapped_time -= g->start_time; - break; - - case 'F': - // FCNTL function - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->F = atoi(value); - break; - - case 'G': - // FCNTL argument - if (is_number(value) == 0) { - Cleanup(ERROR); - } - t->G = atoi(value); - break; - - case 'T': - break; - - default: - // Unknown key - { - Cleanup(ERROR); - } - } - -cleanup: - if (t->path_r) { - free(t->path_r); - t->path_r = NULL; - } - if (t->path2_r) { - free(t->path2_r); - t->path2_r = NULL; - } - - return ret; -} - -bool gparser_token_not_ok(gparser_s *p, char *tok) -{ - if (strlen(tok) < 3) { - return true; - - } else if (tok[1] != '=') { - return true; - } - - return false; -} - -bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid) -{ - char *pos = strchr(pidtid, ':'); - - if (pos) { - char *tmp = pos; - tmp++; - - if (is_number(tmp)) { - *tid = atol(tmp); - } else { - return false; - } - - pos[0] = '\0'; - if (is_number(pidtid)) { - *pid = atol(pidtid); - } else { - return false; - } - } - - else { - return false; - } - - return (*pid >= 0 && *tid >= 0); -} diff --git a/ioreplay/src/generate/gparser.h b/ioreplay/src/generate/gparser.h deleted file mode 100644 index f3e204a..0000000 --- a/ioreplay/src/generate/gparser.h +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GPARSER_H -#define GPARSER_H - -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "generate.h" -#include "gtask.h" - -/** - * @brief The parser definition - * - * The parser is to extract all information from the .capture file. - */ -typedef struct gparser_s_ { - bool terminate; /**< The parser thread will terminate if set to true */ - generate_s *generate; /**< The generate object */ - pthread_t pthread; /**< The posix thread */ - rbuffer_s *queue; /**< A queue of task objects */ -} gparser_s; - -/** - * @brief Creates a new parser - * - * @param g The generate object - * @return The new parser object - */ -gparser_s* gparser_new(generate_s *g); - -/** - * @brief Starts the parser thread - * - * @param p The parser object - */ -void gparser_start(gparser_s *p); - -/** - * @brief Terminates the parser thread - * - * @param p The parser object - */ -void gparser_terminate(gparser_s *p); - -/** - * @brief Destroys the parser thread - * - * @param p The parser object - */ -void gparser_destroy(gparser_s *p); - -/** - * @brief Extracts information a .capture line - * - * Extracts information from a .capture line and stores it into the task - * object. - * - * @param p The parser object - * @param t The task object - */ -void gparser_extract(gparser_s *p, gtask_s *t); - -/** - * @brief Extracts information from a specific token string - * - * @param p The parser object - * @param t The task object - * @param tok The token string - * @return Returns with SUCCESS on success - */ -status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok); - -/** - * @brief Verifies the correctness of a token - * - * @param p The parser object - * @param tok The token to be verified - * @return true if token verified successfully - */ -bool gparser_token_not_ok(gparser_s *p, char *tok); - -/** - * @brief Checks whether the pidtid string is correct or not - * - * @param p The parser object - * @param pidtid The string to check - * @param pid The pointer to the resulting pid - * @param tid The pointer to the resulting tid - * @return true on success - */ -bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid); - -/** - * @brief Entry point of the parser POSIX thread - * - * @param data A pointer to the parser object - * return Always NULL - */ -void* gparser_pthread_start(void *data); - -#endif // GPARSER_H diff --git a/ioreplay/src/generate/gprocess.c b/ioreplay/src/generate/gprocess.c deleted file mode 100644 index 6a0b37a..0000000 --- a/ioreplay/src/generate/gprocess.c +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gprocess.h" - -#include "../vfd.h" -#include "gioop.h" - -void _gprocess_vfd_map_destroy_cb(void *data) -{ - vfd_destroy(data); -} - -gprocess_s* gprocess_new(const long pid, const long mapped_pid) -{ - gprocess_s* gp = Malloc(gprocess_s); - - gp->pid = pid; - gp->mapped_pid = mapped_pid; - gp->max_mapped_fd = 0; - gp->fd_map = hmap_new_l(1024); - gp->vfd_map = hmap_new_l(1024); - gp->vfd_map->data_destroy = _gprocess_vfd_map_destroy_cb; - - return gp; -} - -void gprocess_destroy(gprocess_s *gp) -{ - hmap_destroy(gp->vfd_map); - hmap_destroy(gp->fd_map); - free(gp); -} - -void gprocess_create_vfd_by_realfd(gprocess_s *gp, gtask_s *t, generate_s *g) -{ - if (t->fd < 0) - return; - - // Check whether the real FD is still open according to the .capture log - long old_mapped = (long) hmap_get_l(gp->fd_map, t->fd); - if (old_mapped) { - - // That real file descriptor is already with a mapping to a virtual - // file descriptor. This may happen when SystemTap missed to trace a - // 'close' syscall. We are inserting a close now... - - t->vfd = hmap_get_l(gp->vfd_map, old_mapped); - - hmap_remove_l(gp->fd_map, t->fd); - hmap_remove_l(gp->vfd_map, old_mapped); - - if (t->vfd) { - generate_vsize_by_path(g, t, t->vfd->path); - Gioop_write(CLOSE, "%ld|%d|close inserted", old_mapped, 0); - vsize_close(t->vsize, t->vfd); - if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) - vfd_destroy(t->vfd); - } - } - - t->vfd = rbuffer_get_next(g->vfd_buffer); - t->mapped_fd = ++g->num_mapped_fds; - if (!t->vfd) - t->vfd = vfd_new(t->fd, t->mapped_fd, t->path); - else - vfd_update(t->vfd, t->fd, t->mapped_fd, t->path); - t->vfd->free_path = t->path_r != NULL; - - hmap_insert_l(gp->vfd_map, t->mapped_fd, t->vfd); - hmap_insert_l(gp->fd_map, t->fd, (void*)t->mapped_fd); -} - -status_e gprocess_vfd_by_realfd(gprocess_s *gp, gtask_s *t) -{ - t->mapped_fd = (long) hmap_get_l(gp->fd_map, t->fd); - if (t->mapped_fd == 0) { - // No corresponding virtual fd number mapping - t->has_fd = false; - - } else { - t->vfd = hmap_get_l(gp->vfd_map, t->mapped_fd); - if (!t->vfd) { - return ERROR; - } - t->mapped_fd = t->vfd->mapped_fd; - } - - return SUCCESS; -} diff --git a/ioreplay/src/generate/gprocess.h b/ioreplay/src/generate/gprocess.h deleted file mode 100644 index 47e5037..0000000 --- a/ioreplay/src/generate/gprocess.h +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GPROCESS_H -#define GPROCESS_H - -#include "../datas/hmap.h" -#include "../defaults.h" -#include "gtask.h" -#include "generate.h" - -// Forward declarations (header include hell) -struct gtask_s_; -struct generate_s_; - -/** - * @brief Virtual process object used for generating .replay file - * - * An object of this represents a Linux process in generate context. - * In Linux every process owns * its own file descriptor table which is - * simulated here. Usually, a Linux process re-uses a FD number once not used - * anymore (e.g. after a close). However, as we want to increase concurrency - * while replaying the I/O we want * to ensure to always use unique file - * descriptor IDs for every open. Thats why we use max_mapped_fd to always - * map a real FD number to a uniq virtual FD number. - */ -typedef struct gprocess_s_ { - long pid; /**< The real PID */ - long mapped_pid; /**< The mapped PID */ - hmap_s *vfd_map; /**< All virtual file descriptors of that process */ - hmap_s *fd_map; /**< All mappings from real fd to virtual fd */ - long max_mapped_fd; /**< The max mapped fd number */ -} gprocess_s; - -/** - * @brief Creates a new gprocess object - * - * @param pid The process ID - * @param mapped_pid the mapped PID - * @return The new gprocess object - */ -gprocess_s* gprocess_new(const long pid, const long mapped_pid); - -/** - * @brief Destroys a gprocess object - * - * @param gp The gprocess object - */ -void gprocess_destroy(gprocess_s *gp); - -/** - * @brief Creates a new virtual FD from a given real FD number - * - * In ioreplay we map the real file descriptor (the fd number protocolled in - * the.capture file) to a virtual file descriptor (the fd numner written to the - * .replay file). The purpose is to increase concurrency of the I/O during - * replay. Normally, a process would reuse the same file descriptor number - * once closed earlier. However, when replaying we can't reuse the number if - * we want to replay the I/O on multiple paths in parallel. Therefore, it is - * ensured that the virtual file descriptor number in the .replay file is - * always * unique for every open! - * - * @param gp The process object - * @param t The task object (the vfd pointer will be stored to * t->vfd) - * @param g The generate object - */ -void gprocess_create_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t, - struct generate_s_ *g); - -/** - * @brief Retrieves a virtual FD from a given real FD number - * - * @param gp The process object - * @param t The task object (the vfd pointer will be stored to * t->vfd) - * @return SUCCESS if everything went smothly! - */ -status_e gprocess_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t); - -#endif // GPROCESS_H diff --git a/ioreplay/src/generate/gtask.c b/ioreplay/src/generate/gtask.c deleted file mode 100644 index 55a1124..0000000 --- a/ioreplay/src/generate/gtask.c +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gtask.h" - -gtask_s* gtask_new(void *generate) -{ - gtask_s *t = Malloc(gtask_s); - - t->generate = generate; - t->line = NULL; - t->path_r = NULL; - t->path2_r = NULL; -#ifdef LOG_FILTERED - t->original_line = NULL; -#endif - - return t; -} - -void gtask_init(gtask_s *t, char *line, const unsigned long lineno) -{ - if (t->line) - free(t->line); - t->line = Clone(line); - - if (t->path_r) - free(t->path_r); - if (t->path2_r) - free(t->path2_r); - -#ifdef LOG_FILTERED - if (t->original_line) - free(t->original_line); - t->original_line = Clone(line); - t->filtered_where = NULL; -#endif - - t->bytes = -1; - t->address = 0; - t->address2 = 0; - t->count = -1; - t->F = -1; - t->fd = -1; - t->flags = -1; - t->G = -1; - t->has_fd = false; - t->vsize = NULL; - t->vsize2 = NULL; - t->lineno = lineno; - t->mapped_fd = -1; - t->mapped_time = -1; - t->mode = -1; - t->offset = -1; - t->op = NULL; - t->path2 = NULL; - t->path2_r = NULL; - t->path = NULL; - t->path_r = NULL; - t->gprocess = NULL; - t->pid = -1; - t->pidtid = NULL; - t->ret = 0; - t->status = -1; - t->tid = -1; - t->vfd = NULL; - t->whence = -1; -} - -void gtask_destroy(gtask_s *t) -{ - if (t->line) - free(t->line); - if (t->path_r) - free(t->path_r); - if (t->path2_r) - free(t->path2_r); - free(t); -} - diff --git a/ioreplay/src/generate/gtask.h b/ioreplay/src/generate/gtask.h deleted file mode 100644 index 2f364d3..0000000 --- a/ioreplay/src/generate/gtask.h +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GTASK_H -#define GTASK_H - -#include "vsize.h" - -#include "gprocess.h" -#include "../vfd.h" -#include "../datas/amap.h" -#include "../datas/hmap.h" -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "../mounts.h" -#include "../options.h" - -/** - * @brief The generate task definition - * - * The gtask holds all possible variables required to process a particular - * .capture line and to generate the corresponding .replay line - */ -typedef struct gtask_s_ { - bool has_fd; /**< True if task has a file descriptor number */ - char *line; /**< A pointer to the remaining part of the .capture line */ - char *op; /**< Operation/syscall name */ - char *path2; /**< A second path name (e.g. for rename) */ - char *path2_r; /**< Work around to track mallocs, so it can be freed */ - char *path; /**< Path name */ - char *path_r; /**< Work around to track mallocs, so it can be freed */ - char *pidtid; /**< String representing pid:tid */ - int F; /**< Arguments for fcntl syscall */ - int G; /**< Arguments for fcntl syscall */ - int fd; /**< File descriptor number */ - int flags; /**< File open flags */ - int mode; /**< File open mode */ - int ret; /**< ioreplay process status, SUCCESS if everything is alright */ - int status; /**< Operation/syscall return status */ - long address2; /**< Another address (used by mmap related syscalls) */ - long address; /**< An address (used by mmap related syscalls) */ - long bytes; /**< Amount of bytes */ - long count; /**< A count */ - long lineno; /**< The current line number */ - long mapped_fd; /**< The mapped file descriptor number */ - long mapped_time; /**< The mapped time */ - long offset; /**< A offset */ - long pid; /**< The process ID */ - long tid; /**< The thread ID */ - long whence; /**< Whence */ - vfd_s *vfd; /**< A pointer to the virtual file descriptor */ - struct gprocess_s_ *gprocess; /**< A pointer to the process object */ - void *generate; /**< A pointer to the generate object */ - vsize_s *vsize2; /**< Pointer to a second virtual size object */ - vsize_s *vsize; /**< Pointer to the virtual size object */ -#ifdef LOG_FILTERED - char *original_line; /**< Only used for debugging purposes */ - char *filtered_where; /**< Only used for debugging purposes */ -#endif -} gtask_s; - -/** - * @brief Creates a new task object - * - * @param generate A pointer to the generate object - * @return The new task object - */ -gtask_s* gtask_new(void *generate); - -/** - * @brief Initialises a taks object - * - * This function is used in particular when we recycle/reuse an old - * gtask object. - * - * @param t The gtask object - * @param line The corresponding line from the .capture file - * @param lineno The line number - */ -void gtask_init(gtask_s *t, char *line, const unsigned long lineno); - -/** - * @brief Destroys a given task object - * - * @param t The task object - */ -void gtask_destroy(gtask_s *t); - -#endif // GTASK_H diff --git a/ioreplay/src/generate/gwriter.c b/ioreplay/src/generate/gwriter.c deleted file mode 100644 index e0d448e..0000000 --- a/ioreplay/src/generate/gwriter.c +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gwriter.h" - -#include "gtask.h" -#include "generate.h" -#include "gioop.h" -#include "../opcodes.h" - -void* gwriter_pthread_start(void *data) -{ - gwriter_s *w = data; - generate_s *g = w->generate; - gtask_s *t = NULL; - - do { - while (NULL != (t = rbuffer_get_next(w->queue))) { -#ifdef LOG_FILTERED - // Logging filtered lines - if (SUCCESS != gioop_run(w, t)) { - fprintf(g->replay_fd, "#FILTERED @%ld %s", t->lineno, - t->original_line); - } -#else - gioop_run(w, t); -#endif - rbuffer_insert(g->reuse_queue, t); - } - usleep(100); - } while (!w->terminate); - - while (NULL != (t = rbuffer_get_next(w->queue))) { -#ifdef LOG_FILTERED - if (SUCCESS != gioop_run(w, t)) { - fprintf(g->replay_fd, "#FILTERED @%ld %s\n", t->lineno, - t->original_line); - } -#else - gioop_run(w, t); -#endif - rbuffer_insert(g->reuse_queue, t); - } - - return NULL; -} - -gwriter_s* gwriter_new(generate_s *g) -{ - gwriter_s *w = Malloc(gwriter_s); - - w->generate = g; - w->terminate = false; - w->queue = rbuffer_new(1024); - - return w; -} - -void gwriter_start(gwriter_s *w) -{ - start_pthread(&w->pthread, gwriter_pthread_start, (void*)w); -} - -void gwriter_destroy(gwriter_s *w) -{ - rbuffer_destroy(w->queue); - free(w); -} - -void gwriter_terminate(gwriter_s *w) -{ - w->terminate = true; - pthread_join(w->pthread, NULL); -} diff --git a/ioreplay/src/generate/gwriter.h b/ioreplay/src/generate/gwriter.h deleted file mode 100644 index 4295580..0000000 --- a/ioreplay/src/generate/gwriter.h +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef GWRITER_H -#define GWRITER_H - -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "vsize.h" -#include "gtask.h" -#include "generate.h" - -// Forward declaration (header include hell) -struct gtask_s_; -struct generate_s_; - -/** - * @brief Definition of the writer object - * - * The writer utilises the information extracted by the parser to actually - * write the .replay file. - */ -typedef struct gwriter_s_ { - bool terminate; /**< The writer thread will terminate if set to true */ - struct generate_s_ *generate; /**< The generate object */ - pthread_t pthread; /**< The posix thread */ - rbuffer_s *queue; /**< A queue of task objects */ -} gwriter_s; - -/** - * @brief Creates a new writer - * - * @param g The generate object - * @return The new writer object - */ -gwriter_s* gwriter_new(struct generate_s_ *g); - -/** - * @brief Starts the writer thread - * - * @param w The writer object - */ -void gwriter_start(gwriter_s *w); - -/** - * @brief Terminates the writer thread - * - * @param w The writer object - */ -void gwriter_terminate(gwriter_s *w); - -/** - * @brief Destroys the writer thread - * - * @param w The writer object - */ -void gwriter_destroy(gwriter_s *w); - -/** - * @brief Writes a line to the .replay file - * - * @param w The writer object - * @param t The task object - */ -void gwriter_write(gwriter_s *w, struct gtask_s_ *t); - -/** - * @brief Entry function of the writer pthread - * - * @param data A pointer to the writer object - * @return Always returns a NULL pointer if it doesnt crash! - */ -void* gwriter_pthread_start(void *data); - -#endif // GWRITER_H diff --git a/ioreplay/src/generate/vsize.c b/ioreplay/src/generate/vsize.c deleted file mode 100644 index f2d56ba..0000000 --- a/ioreplay/src/generate/vsize.c +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "vsize.h" - -#include "generate.h" - -// Helper macros - -#define _Set_file(v) v->is_file = true; v->unsure = v->is_dir = false -#define _Set_dir(v) v->is_dir = true; v->unsure = v->is_file = false -#define _Set_unsure(v) v->unsure = true -#define _Set_inserted(v) v->inserted = true -#define _Set_renamed(v) v->renamed = true -#define _Set_required(v) v->required = true - -vsize_s* vsize_new(char *file_path, const unsigned long id, - void *generate) -{ - vsize_s *v = Malloc(vsize_s); - - v->generate = generate; - v->id = id; - v->inserted = false; - v->is_dir = false; - v->is_file = false; - v->offset = -1; - v->path = Clone(file_path); - v->renamed = false; - v->required = false; - v->unsure = false; - v->updates = 0; - v->vsize = 0; - v->vsize_deficit = 0; - - return v; -} - -void vsize_destroy(vsize_s *v) -{ - if (!v) - return; - - free(v->path); - free(v); -} - -void init_parent_dir(vsize_s *v, const char *path) -{ - generate_s *g = v->generate; - char *clone = Clone(path); - char *parent = dirname(clone); - - vsize_s *v_parent = hmap_get(g->vsize_map, parent); - if (!v_parent) { - - // Parent directory does not yet have a vsize! - // Create a vsize object for it and set it as a pre-requirement - // so that the directory can be created during init mode. - - v_parent = vsize_new(parent, ++g->num_vsizes, g); - hmap_insert(g->vsize_map, parent, v_parent); - - _Set_required(v_parent); - _Set_dir(v_parent); - - // This is for debugging purposes only - _Set_inserted(v_parent); - v_parent->updates++; - - } else if (v_parent->unsure) { - // We now know for sure that this path must be a directory! - _Set_dir(v_parent); - v_parent->updates++; - } - - free(clone); -} - -void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags) -{ - - // v->first_encounter == false means, that this is the first occurance of - // this path and we didn't initialise it (means we didn't ensure that - // we want to create all parent directories etc. - - if (v->updates == 0) { - // We may use a recycled vfd object! When opening a file we always - // assume that the offset is 0! - vfd_s *vfd_ = vfd; - vfd_->offset = 0; - init_parent_dir(v, path); - - if (Has(flags, O_DIRECTORY)) { - _Set_required(v); - _Set_dir(v); - - } else if (Hasnt(flags, O_CREAT)) { - _Set_required(v); - _Set_file(v); - _Set_unsure(v); - } - v->updates++; - - } else if (v->unsure) { - if (Has(flags, O_DIRECTORY)) { - // Now we know for sure that this path must be a directory! - _Set_dir(v); - v->updates++; - } - } -} - -void vsize_close(vsize_s *v, void* vfd) -{ - vfd_s *vfd_ = vfd; - vfd_->offset = 0; - v->updates++; -} - -void vsize_stat(vsize_s *v, const char *path) -{ - if (v->updates == 0) { - init_parent_dir(v, path); - _Set_required(v); - _Set_file(v); - - // We are not 100% sure that this is really a file, - // the path might be still a directory though! - _Set_unsure(v); - v->updates++; - } -} - -void vsize_rename(vsize_s *v, vsize_s *v2, - const char *path, const char *path2) -{ - if (v->updates == 0) { - init_parent_dir(v, path); - _Set_required(v); - _Set_file(v); - _Set_unsure(v); - v->updates++; - } - - if (v2->updates == 0) { - init_parent_dir(v2, path2); - _Set_file(v2); - - // We are not 100% sure that this is really a file, - // the path might be still a directory though! - _Set_unsure(v2); - - // For debugging purposes only - _Set_renamed(v2); - v2->updates++; - } -} - -void vsize_adjust(vsize_s *v, vfd_s* vfd) -{ - if (v->vsize >= vfd->offset) - return; - - long deficit = v->vsize - vfd->offset; - if (deficit < v->vsize_deficit) { - v->vsize_deficit = deficit; - _Set_required(v); - _Set_file(v); - } -} - -void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes) -{ - vfd_s *vfd_ = vfd; - vfd_->offset += bytes; - vsize_adjust(v, vfd_); - v->updates++; -} - -void vsize_seek(vsize_s *v, void *vfd, const long new_offset) -{ - //vfd_s *vfd_ = vfd; - - // The file's offset can be greater than the file's current size, in which - // case the next write to the file will extend the file. This is referred - // to as creating a hole in a file and is allowed. However, this behaviour - // does not suit the estimation of the file size before we want to run the - // test. - - // TODO: Implement file hole support! - //v->updates++; -} - -void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes) -{ - vfd_s *vfd_ = vfd; - vfd_->offset += bytes; - - if (v->vsize < vfd_->offset) - v->vsize = vfd_->offset; - - v->updates++; -} - -void vsize_mkdir(vsize_s *v, const char *path) -{ - if (v->updates == 0) { - init_parent_dir(v, path); - _Set_dir(v); - v->updates++; - } -} - -void vsize_rmdir(vsize_s *v, const char *path) -{ - if (v->updates == 0) { - init_parent_dir(v, path); - _Set_required(v); - _Set_dir(v); - v->updates++; - } -} - -void vsize_unlink(vsize_s *v, const char *path) -{ - if (v->updates == 0) { - init_parent_dir(v, path); - _Set_required(v); - if (!v->is_dir) { - _Set_file(v); - _Set_unsure(v); - } - v->updates++; - } -} diff --git a/ioreplay/src/generate/vsize.h b/ioreplay/src/generate/vsize.h deleted file mode 100644 index bb1008e..0000000 --- a/ioreplay/src/generate/vsize.h +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef VSIZE_H -#define VSIZE_H - -#include "../utils/utils.h" -#include "../datas/hmap.h" -#include "../vfd.h" - -/** - * @brief Definition of a virtual size object - * - * The virtual size is used to determine the expected type and size of a file. - * This piece of information will be added to the INIT section of the the - * .replay file. That file then will be created during test initialisation. - * before running the test. It is very likely the case that the test requires - * a file of a certain size already to be present, so it can be read from disk. - */ -typedef struct vsize_s_ { - char *path; /**< The path to the file/directory */ - off_t offset; /**< The current file offset */ - unsigned long id; /**< The vsize id */ - void *generate; /**< A pointer to the generate object */ - long vsize; /**< The virtual size */ - long vsize_deficit; /**< Size to use for file creating during init mode */ - bool renamed; /**< True if file/dir has been renamed */ - bool required; /**< True if init mode will create this file/dir */ - bool is_dir; /**< True if this file/dir is a directory */ - bool is_file; /**< True if this file/dir is a regular file */ - bool unsure; /**< True if the file type is not fully clear */ - long updates; /**< Amount of times this vsize has been updated */ - bool inserted; /**< For debugging purposes only */ -} vsize_s; - -/** - * @brief Creates a new vsize object - * - * @param file_path The corresponding file path - * @param id The vsize vsize aka ID - * @param generate The generate object - * @return The new vsize object - */ -vsize_s* vsize_new(char *file_path, const unsigned long id, void *generate); - -/** - * @brief Destroys a vsize object - * - * @param v The vsize object - */ -void vsize_destroy(vsize_s *v); - -/** - * @brief Ensures that the parent directory exists - * - * This function ensures that the parent directory exists as a vsize object! - * - * @param v The vsize object - * @param path The given path - */ -void init_parent_dir(vsize_s *v, const char *path); - -/** - * @brief Adjusts the vsize - * - * Compares the virtual file size of the file in the vsize - * object to the the offset in the virtual file descriptor. - * In case the offset is higher we have a size deficit and - * we need to mark it. That way ioreplay can ensure that - * during init mode it will create a file with the correct - * size prior of running the test! - * - * @param v The virtual size object - * @param vfd The virtual file descriptor object - */ -void vsize_adjust(vsize_s *v, vfd_s* vfd); - -/** - * @brief Adjust vsize on open - * - * @param v The virtual size object - * @param vfd The virtual file descriptor object - * @param path The file open path - * @param flags The file open flags - */ -void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags); - -/** - * @brief Adjust vsize on close - * - * @param v The virtual size object - * @param vfd The virtual file descriptor object - */ -void vsize_close(vsize_s *v, void *vfd); - -/** - * @brief Adjust vsize on stat - * - * @param v The virtual size object - * @param path The stat path - */ -void vsize_stat(vsize_s *v, const char *path); - -/** - * @brief Adjust vsize on rename - * - * @param v The virtual size object - * @param v2 The virtual size object of path2 - * @param path The first file path - * @param path2 The second file path - */ -void vsize_rename(vsize_s *v, vsize_s *v2, - const char *path, const char *path2); - -/** - * @brief Adjust vsize on read - * - * @param v The virtual size object - * @param vfd The virtual vile descriptor object - * @param path The file path - * @param bytes The amount of bytes read - */ -void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes); - -/** - * @brief Adjust vsize on seek - * - * @param v The virtual size object - * @param vfd The virtual vile descriptor object - * @param new_offset The new file offset after seek - */ -void vsize_seek(vsize_s *v, void *vfd, const long new_offset); - -/** - * @brief Adjust vsize on write - * - * @param v The virtual size object - * @param vfd The virtual vile descriptor object - * @param path The file path - * @param bytes The amount of bytes written - */ -void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes); - -/** - * @brief Adjust vsize on mkdir - * - * @param v The virtual size object - * @param path The directory path - */ -void vsize_mkdir(vsize_s *v, const char *path); - -/** - * @brief Adjust vsize on rmdir - * - * @param v The virtual size object - * @param path The directory path - */ -void vsize_rmdir(vsize_s *v, const char *path); - -/** - * @brief Adjust vsize on unlink - * - * @param v The virtual size object - * @param path The file path - */ -void vsize_unlink(vsize_s *v, const char *path); - -#endif // VSIZE_H - diff --git a/ioreplay/src/init/init.c b/ioreplay/src/init/init.c deleted file mode 100644 index e375379..0000000 --- a/ioreplay/src/init/init.c +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "init.h" - -#include "../datas/stack.h" -#include "itask.h" -#include "ithread.h" -#include "../meta/meta.h" -#include "../mounts.h" -#include "../utils/futils.h" - - -init_s *init_new(options_s *opts) -{ - init_s *i = Malloc(init_s); - - i->opts = opts; - i->mounts = mounts_new(opts); - i->threads_map = amap_new(i->mounts->count); - i->reuse_queue = rbuffer_new(4096); - i->replay_fd = Fopen(opts->replay_file, "r"); - - pthread_mutex_init(&i->reuse_queue_mutex, NULL); - - return i; -} - -void init_destroy(init_s *i) -{ - amap_destroy(i->threads_map); - mounts_destroy(i->mounts); - - itask_s *task = NULL; - while (NULL != (task = rbuffer_get_next(i->reuse_queue))) { - itask_destroy(task); - } - rbuffer_destroy(i->reuse_queue); - - fclose(i->replay_fd); - pthread_mutex_destroy(&i->reuse_queue_mutex); - - free(i); -} - -void init_extract_header(init_s *i, off_t *init_offset) -{ - options_s *opts = i->opts; - meta_s *m = meta_new(i->replay_fd); - meta_read_start(m); - - long version = 0; - if (meta_read_l(m, "version", &version)) { - Put("Replay version is '%ld'", version); - if (version != REPLAY_VERSION) { - Error(".replay file of incompatible version, got %x, expected %x", - (int)version, REPLAY_VERSION); - } - } - - char *user; - if (meta_read_s(m, "user", &user)) { - Put("Setting user to '%s'", user); - opts->user = user; - } - - char *name; - if (meta_read_s(m, "name", &name)) { - Put("Setting name to '%s'", name); - opts->name = name; - } - - if (meta_read_l(m, "init_offset", init_offset)) { - if (*init_offset < 0) { - Error("Offset overflow (init offset too large in .replay)"); - } - Put("Setting init offset to '%ld'", *init_offset); - } - - meta_destroy(m); -} - -status_e init_run(options_s *opts) -{ - status_e ret = SUCCESS; - init_s *i = init_new(opts); - - off_t init_offset; - init_extract_header(i, &init_offset); - - // Ensure that all ./replay/NAME directories exist - mounts_init(i->mounts); - - // Don't do messy stuff as super user - set_limits_drop_root(opts->user); - - // We need to clean up garbish from previous runs! - if (opts->purge) - mounts_purge(i->mounts); - else - mounts_trash(i->mounts); - - Out("Creating all files and directories requried for test '%s'...", - opts->name); - - // Seek to the INIT section - fseeko(i->replay_fd, init_offset, SEEK_SET); - - bool is_file = false, is_dir = false; - long vsize = 0; - char *path; - - // Stats - long dirs_created = 0; - long files_created = 0; - long files_total_size = 0; - - // Helper variables for getline - char *line = NULL; - size_t len = 0, read = 0; - char *saveptr; - - stack_s *all_threads = stack_new(); - - // Process the INIT section of the .replay file line by line. - - while ((read = getline(&line, &len, i->replay_fd)) != -1) { - char *tok = strtok_r(line, "|", &saveptr); - - for (int ntok = 0; tok; ntok++) { - switch (ntok) { - case 0: - is_dir = atoi(tok) == 1; - break; - case 1: - is_file = atoi(tok) == 1; - break; - case 2: - vsize = atol(tok); - if (vsize < 0) { - Error("Size overflow"); - } - break; - case 3: - path = tok; - break; - default: - break; - } - - tok = strtok_r(NULL, "|", &saveptr); - } - - itask_s *task = rbuffer_get_next(i->reuse_queue); - - if (!task) { - task = itask_new(); - - } else { - itask_extract_stats(task, &dirs_created, &files_created, - &files_total_size); - } - - // Set new task values - if (is_dir) { - task->is_dir = true; - - } else if (is_file) { - task->is_file = true; - task->vsize = vsize; - } - task->path = Clone(path); - - // We run one init thread per mount point - int mnr = mounts_get_mountnumber(i->mounts, path); - ithread_s *t = amap_get(i->threads_map, mnr); - - if (!t) { - t = ithread_new(i); - amap_set(i->threads_map, mnr, t); - stack_push(all_threads, t); - ithread_start(t); - } - - //itask_print(task); - while (!rbuffer_insert(t->queue, task)) - usleep(1000); - } - - ithread_s *t = NULL; - while (NULL != (t = stack_pop(all_threads))) { - ithread_terminate(t); - ithread_destroy(t); - } - stack_destroy(all_threads); - - itask_s *task = NULL; - while (NULL != (task = rbuffer_get_next(i->reuse_queue))) { - itask_extract_stats(task, &dirs_created, &files_created, - &files_total_size); - itask_destroy(task); - } - - Put("Done!"); - - Put("Created %ld files (net total size: %.2fg) and %ld directories!", - files_created, files_total_size/(1024*1024*1024.0), - dirs_created); - - init_destroy(i); - - Put("You are ready to fire up the test now"); - - return ret; -} diff --git a/ioreplay/src/init/init.h b/ioreplay/src/init/init.h deleted file mode 100644 index 3d9f9e9..0000000 --- a/ioreplay/src/init/init.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef INIT_H -#define INIT_H - -#include "../defaults.h" -#include "../options.h" -#include "../datas/amap.h" -#include "../datas/rbuffer.h" -#include "../mounts.h" - -typedef struct init_s_ { - amap_s *threads_map; - rbuffer_s *reuse_queue; - options_s *opts; - mounts_s *mounts; - FILE *replay_fd; - pthread_mutex_t reuse_queue_mutex; -} init_s; - -/** - * @brief Creates a new init object - * - * @param opts The options object - * @return The new mounts object - */ -init_s* init_new(options_s *opts); - -/** - * @brief Destroys the init object - * - * @param i The init object - */ -void init_destroy(init_s *i); - -/** - * @brief Initialises the test environment - * - * @param opts The options object - * @return SUCCESS if initialised without any issues - */ -status_e init_run(options_s *opts); - -/** - * @brief Extracts some useful information from the .replay meta header - * - * @param i The init object - * @param init_offset To store the offset of the init section - */ -void init_extract_header(init_s *i, off_t *init_offset); - -#endif // INIT_H diff --git a/ioreplay/src/init/itask.c b/ioreplay/src/init/itask.c deleted file mode 100644 index f04ce33..0000000 --- a/ioreplay/src/init/itask.c +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "itask.h" - -itask_s* itask_new() -{ - itask_s *task = Malloc(itask_s); - - task->path = NULL; - itask_reset_stats(task); - - return task; -} - -void itask_destroy(itask_s *task) -{ - if (task->path) - free(task->path); - - free(task); -} - -void itask_reset_stats(itask_s *task) -{ - task->is_dir = task->is_file = false; - task->sizes_created = task->vsize = 0; - task->dirs_created = task->files_created = 0; - - if (task->path) { - free(task->path); - task->path = NULL; - } -} - -void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created, - long *files_total_size) -{ - *dirs_created += task->dirs_created; - *files_created += task->files_created; - *files_total_size += task->sizes_created; - - if (*dirs_created < 0 || *files_created < 0 || *files_total_size < 0) { - Error("Size overflow"); - } - - itask_reset_stats(task); -} - -void itask_print(itask_s *task) -{ - Put("itask(%p): is_dir:%d is_file:%d vsize:%ld path:%s", - (void*)task, task->is_dir, task->is_file, - task->vsize, task->path); -} diff --git a/ioreplay/src/init/itask.h b/ioreplay/src/init/itask.h deleted file mode 100644 index b10d515..0000000 --- a/ioreplay/src/init/itask.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef ITASK_H -#define ITASK_H - -#include "../defaults.h" - -/** - * @brief The initialise task definition - */ -typedef struct itask_s_ { - bool is_dir; - bool is_file; - long vsize; - char *path; - long dirs_created; - long files_created; - long sizes_created; -} itask_s; - -/** - * @brief Creates a new task object - * - * @return The new task object - */ -itask_s* itask_new(); - -/** - * @brief Resets the task stats - * - * @param task The itask object - */ -void itask_reset_stats(itask_s *task); - -/** - * @brief Extract stats from a task object - * - * @param task The itask object - * @param dirs_created Adds count of dirs created to that variable - * @param files_created Adds count of files created to that variable - * @param files_total_size Adds size of files created to that variable - */ -void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created, - long *files_total_size); - -/** - * @brief Destroys a given task object - * - * @param task The task object - */ -void itask_destroy(itask_s *task); - -/** - * @brief Prints a task to stdout - * - * @param task The task object - */ -void itask_print(itask_s *task); - -#endif // ITASK_H diff --git a/ioreplay/src/init/ithread.c b/ioreplay/src/init/ithread.c deleted file mode 100644 index a580e70..0000000 --- a/ioreplay/src/init/ithread.c +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ithread.h" - -#include "itask.h" -#include "../utils/futils.h" - - -void* ithread_pthread_start(void *data) -{ - ithread_s *t = data; - init_s *i = t->init; - itask_s *task = NULL; - - do { - while (NULL != (task = rbuffer_get_next(t->queue))) { - ithread_run_task(t, task); - - // We need to mutex lock the reuse_queue as multiple threads - // can insert into it - pthread_mutex_lock(&i->reuse_queue_mutex); - int ret = rbuffer_insert(i->reuse_queue, task); - pthread_mutex_unlock(&i->reuse_queue_mutex); - if (!ret) - itask_destroy(task); - } - usleep(100); - } while (!t->terminate); - - while (NULL != (task = rbuffer_get_next(t->queue))) { - ithread_run_task(t, task); - if (!rbuffer_insert(i->reuse_queue, task)) - itask_destroy(task); - - pthread_mutex_lock(&i->reuse_queue_mutex); - int ret = rbuffer_insert(i->reuse_queue, task); - pthread_mutex_unlock(&i->reuse_queue_mutex); - if (!ret) - itask_destroy(task); - } - - return NULL; -} - -ithread_s* ithread_new(init_s *i) -{ - ithread_s *t = Malloc(ithread_s); - - t->init = i; - t->queue = rbuffer_new(1024); - t->terminate = false; - - return t; -} - -void ithread_start(ithread_s *t) -{ - start_pthread(&t->pthread, ithread_pthread_start, (void*)t); -} - -void ithread_destroy(ithread_s *t) -{ - rbuffer_destroy(t->queue); - free(t); -} - -void ithread_terminate(ithread_s *t) -{ - t->terminate = true; - pthread_join(t->pthread, NULL); -} - -void ithread_run_task(ithread_s *t, itask_s *task) -{ - if (task->is_dir) { - task->dirs_created += ensure_dir_exists(task->path); - - } else if (task->is_file) { - if (!ensure_file_exists(task->path, &task->dirs_created)) { - task->files_created++; - if (task->vsize > 0) { - append_random_to_file(task->path, task->vsize); - task->sizes_created += task->vsize; - } - } - } -} diff --git a/ioreplay/src/init/ithread.h b/ioreplay/src/init/ithread.h deleted file mode 100644 index 0884519..0000000 --- a/ioreplay/src/init/ithread.h +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef ITHREAD_H -#define ITHREAD_H - -#include "../defaults.h" -#include "../datas/rbuffer.h" - -#include "init.h" -#include "itask.h" - -#include - -/** - * @brief Definition of an init thread - * - */ -typedef struct ithread_s_ { - pthread_t pthread; /**< We run the init tasks in concurrent pthreads */ - rbuffer_s *queue; /**< The thread's task queue */ - init_s *init; /**< The responsible init object */ - bool terminate; /**< Indicates that thread can terminate */ -} ithread_s; - -/** - * @brief Creates a new thread object - * - * @param i The init object - * @return The new thread object - */ -ithread_s* ithread_new(init_s *i); - -/** - * @brief Terminates the thread - * - * This function waits (via join) for the pthread to complete all its - * current tasks from the queue. - * - * @param t The thread object - */ -void ithread_terminate(ithread_s* t); - -/** - * @brief Destroys the thread object - * - * @param t The thread object - */ -void ithread_destroy(ithread_s* t); - -/** - * @brief Executes the init task - * - * @param t The thread object - * @param task The task object - */ -void ithread_run_task(ithread_s* t, itask_s *task); - -/** - * @brief Starts the POSIX thread - * - * @param t The responsible thread object - */ -void ithread_start(ithread_s *t); - -/** - * @brief Entry point of the POSIX thread - * - * @param data Data passed to the pthread - * @return Always NULL on success - */ - -void* ithread_pthread_start(void *data); - -#endif // ITHREAD_H diff --git a/ioreplay/src/macros.h b/ioreplay/src/macros.h deleted file mode 100644 index 45e5a10..0000000 --- a/ioreplay/src/macros.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MACROS_H -#define MACROS_H - -#define Cleanup(code) ret = code; goto cleanup -#define Cleanup_unless(expr, code) \ - if (expr != code) { ret = code; goto cleanup; } - -// String helpers -#define Clone(str) notnull(strdup(str),__FILE__,__LINE__,0) -#define Eq(str1,str2) strcmp(str1,str2) == 0 - -// Number helpers -#define Abs(num) num >= 0 ? num : -num -#define Readhex(str) strtol(str, NULL, 16) -#define Perc(a, b) a > b ? b/(a/100.) : a/(b/100.) - -// Bitwise helpers -#define Has(flags, what) (flags & (what)) == (what) -#define Hasnt(flags, what) (flags & (what)) != (what) - -// Memory helpers -#define Malloc(what) \ - notnull(malloc(sizeof(what)),__FILE__,__LINE__,1) -#define Calloc(count,what) \ - notnull(calloc(count,sizeof(what)),__FILE__,__LINE__,count) -#define Mset(where,value,count,what) \ - memset(where,value,count*sizeof(what)) - -// Open helpers -#define Fopen(path, mode) fnotnull(fopen(path, mode), path, __FILE__, __LINE__) - -// Mmap helpers -#define Mmapshared(what) \ - mmapok(mmap(NULL, sizeof(what), \ - PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0), \ - __FILE__,__LINE__) -#define Cmapshared(count,what) \ - mmapok(mmap(NULL, count*sizeof(what), \ - PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0), \ - __FILE__,__LINE__) - -// Printing messages -#define Out(...) \ - fprintf(stdout, __VA_ARGS__); \ - fflush(stdout); -#define Put(...) \ - fprintf(stdout, __VA_ARGS__); \ - fprintf(stdout, "\n"); \ - fflush(stdout); - -// Printing debug messages -#define Debug(...) \ - fprintf(stderr, "%s:%d DEBUG: ", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\n"); \ - fflush(stderr); - -// Printing error messages -#define Error(...) \ - fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr,"\n"); \ - fflush(stdout); \ - fflush(stderr); \ - exit(ERROR); - -#define Error_if(expr, ...) if (expr) { Error(__VA_ARGS__); } - -#define Errno(...) \ - fprintf(stderr, "%s:%d ERROR: %s (%d). ", __FILE__, __LINE__, \ - strerror(errno), errno); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr,"\n"); \ - fflush(stdout); \ - fflush(stderr); \ - exit(ERROR); - -#define Errno_if(expr, ...) if (expr) { Errno(__VA_ARGS__); } - -#define Segfault(...) \ - fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr,"\n"); \ - fflush(stdout); \ - fflush(stderr); \ - *(int*)0 = 0; - -// Printing warn messages -#define Warn(...) \ - fprintf(stderr, "WARN: "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr,"\n"); \ - fflush(stdout); \ - fflush(stderr); - -#define Warn_if(expr, ...) if (expr) { Warn(__VA_ARGS__); } - -// Other helpers -#define Fill_with_stuff(buf, len) \ - for (int i = 0; i -#include - -#include "capture/capture.h" -#include "cleanup/cleanup.h" -#include "generate/generate.h" -#include "init/init.h" -#include "mounts.h" -#include "options.h" -#include "replay/replay.h" -#include "utests.h" -#include "utils/utils.h" - -/** - * @brief Do some architecture checks - * - * To ensure that I/O replay works correctly we have to check whether some - * data types are atomic or not. This is what this function does! - */ -static void _arch_check_atomic(void) -{ - if (sizeof(int) > sizeof(sig_atomic_t)) { - Error("int data type is not atomic on this architecture: %ld > %ld", - sizeof(int), sizeof(sig_atomic_t)); - - } else if (sizeof(bool) > sizeof(sig_atomic_t)) { - Error("bool data type is not atomic on this architecture: %ld > %ld", - sizeof(bool), sizeof(sig_atomic_t)); - } -} - -/** - * @brief Prints out version and copyright information - */ -static void _print_version(void) -{ - Put("This is I/O Replay %s - %s", IOREPLAY_VERSION, IOREPLAY_COPYRIGHT); -} - -/** - * @brief Print the synopsis - */ -static void _print_synopsis(void) -{ - _print_version(); - - Put("Synopsis:"); - Put("\tioreplay -c io.capture [-x PID] [-m MODULE]"); - Put("\tioreplay -c io.capture -r io.replay [-n str] [-u str] [-w str]"); - Put("\tioreplay -i io.replay"); - Put("\tioreplay -r io.replay [-p #] [-t #] [-D] [-s #]"); - Put("\tioreplay -R io.replay [-p #] [-t #] [-D] [-s #]"); - Put("\tioreplay -d"); - Put("\tioreplay -P"); - Put("\tioreplay -T [-n NAME]"); - Put("\tioreplay -V"); -} - -/** - * @brief Print a brief help - */ -static void _print_help(void) -{ - _print_synopsis(); - - Put("Help:"); - Put("\t-d Drop all Linux/FS caches and exit ioreplay"); - Put("\t-D Don't drop all caches (in conjunction with -r/-R):"); - Put("\t-s SPEED The speed factor (default: 0 [as fast as possible])"); - Put("\t-h Print this help"); - Put("\t-c FILE The capture file"); - Put("\t-n NAME The name (default: test0)"); - Put("\t-u USER The test run user (default: mcuser)"); - Put("\t-p #WORKERS Amount of of parallel worker processes (default: 4)"); - Put("\t-t #THREADS Threads per worker process (default: 128)"); - Put("\t-i REPLAYFILE The replay file to be initialised"); - Put("\t-r REPLAYFILE The replay file to be replayed"); - Put("\t-R REPLAYFILE Init and replay in one run (-i and -r combined)"); - Put("\t-S STATSFILE Write a stats file at the end of a test"); - Put("\t-T Trash data directories"); - Put("\t-P Purge all trash directories of all tests)"); - Put("\t-V Print I/O replay program version"); - Put("\t-w WD_BASE The working directory's base path"); - Put("\t (default: /usr/local/ioreplay)"); - Put("\t-x PID To specify a process ID (in conjunction with -c)"); - Put("\t-m MODULE To specify a module (in conjunction with -c)"); - Put("\nExample (run these commands one after another):"); - Put("\t 1.) sudo ioreplay -c io.capture"); - Put("\t 2.) sudo ioreplay -r io.replay -c io.capture -u paul -n test1"); - Put("\t 3.) sudo ioreplay -i io.replay"); - Put("\t 4.) sudo ioreplay -r io.replay -S"); -} - -/** - * @brief I/O Replay's entry point - * - * Not much more to document here though! - * @return The exit code - */ -int main(int argc, char **argv) -{ - _arch_check_atomic(); - status_e ret = UNKNOWN; - - bool dont_drop_caches = false; - options_s *opts = options_new(); - int opt = 0; - - while ((opt = getopt(argc, argv, "Vr:R:S:c:u:i:hw:n:dDs:w:p:t:UPTx:m:")) != -1) { - switch (opt) { - case 'U': - utests_run(); - Cleanup(SUCCESS); - break; - case 'V': - _print_version(); - Cleanup(SUCCESS); - break; - case 'd': - drop_caches(); - Cleanup(SUCCESS); - break; - case 'D': - dont_drop_caches = true; - break; - case 'c': - opts->capture_file = absolute_path(optarg); - Put("Capture file: %s", opts->capture_file); - break; - case 'P': - opts->purge = true; - Put("Purge option set"); - break; - case 'T': - opts->trash = true; - Put("Trash option set"); - break; - case 'i': - opts->init = true; - if (!opts->replay_file) { - opts->replay_file = absolute_path(optarg); - Put("Replay file: %s", opts->replay_file); - } - break; - case 'R': - opts->init = true; - opts->replay = true; - if (!opts->replay_file) { - opts->replay_file = absolute_path(optarg); - Put("Replay file: %s", opts->replay_file); - } - break; - case 'r': - opts->replay = true; - if (!opts->replay_file) { - opts->replay_file = absolute_path(optarg); - Put("Replay file: %s", opts->replay_file); - } - break; - case 'S': - opts->stats_file = Clone(optarg); - Put("Stats output file: %s", opts->stats_file); - break; - case 'w': - opts->wd_base = optarg; - Put("WD base: %s", opts->wd_base); - break; - case 'u': - opts->user = optarg; - Put("User: %s", opts->user); - break; - case 'm': - opts->module = Clone(optarg); - Put("Module: %s", opts->module); - break; - case 'n': - opts->name = optarg; - Put("Name: %s", opts->name); - break; - case 'h': - _print_help(); - Cleanup(SUCCESS); - case 's': - sscanf(optarg, "%lf", &opts->speed_factor); - Put("Speed factor: %lf", opts->speed_factor); - break; - case 'p': - opts->num_workers = atoi(optarg); - if (opts->num_workers < 1) - opts->num_workers = 1; - Put("Num worker processes: %d", opts->num_workers); - break; - case 't': - opts->num_threads_per_worker = atoi(optarg); - if (opts->num_threads_per_worker < 1) - opts->num_threads_per_worker = 1; - Put("Num threads per worker: %d", opts->num_threads_per_worker); - break; - case 'x': - opts->pid = atoi(optarg); - Put("PID: %d", opts->pid); - break; - default: - _print_help(); - Cleanup(ERROR); - } - } - - if (opts->purge || opts->trash) { - // Clean up all temp data of previous test runs - Cleanup(cleanup_run(opts)); - - } else if (opts->capture_file && !opts->replay_file) { - // We are going to capture I/O - Cleanup(capture_run(opts)); - - } else if (opts->capture_file && opts->replay_file) { - // We are going to generate a .replay file from the .capture file - Cleanup(generate_run(opts)); - - } else if (opts->replay_file && opts->init && !opts->replay) { - // We are going to initialise the test from the .replay file! - Cleanup(init_run(opts)); - - } else if (opts->replay_file && opts->init && opts->replay) { - // We are going to initialise the test and run the test! Run the - // initialiser in a sub-process, as it drops root privileges! - pid_t pid = fork(); - if (pid == 0) { - Cleanup(init_run(opts)); - } else { - opts->drop_caches = !dont_drop_caches; - int init_status; - waitpid(pid, &init_status, 0); - // Only proceed if initialisation was successfull! - Cleanup_unless(SUCCESS, init_status); - Cleanup(replay_run(opts)); - } - - } else if (opts->replay_file && !opts->init && opts->replay) { - // We are going to replay the I/O - opts->drop_caches = !dont_drop_caches; - Cleanup(replay_run(opts)); - - } else { - _print_help(); - Cleanup(ERROR); - } - -cleanup: - options_destroy(opts); - - return ret; -} diff --git a/ioreplay/src/meta/meta.c b/ioreplay/src/meta/meta.c deleted file mode 100644 index d56c17e..0000000 --- a/ioreplay/src/meta/meta.c +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "meta.h" - -#define _MAX_META_LEN 256 - -meta_s* meta_new(FILE *replay_fd) -{ - meta_s *m = Malloc(meta_s); - - m->replay_fd = replay_fd; - m->offset = ftello(replay_fd); - m->read_buf = NULL; - - return m; -} - -void meta_destroy(meta_s *m) -{ - if (!m) - return; - - if (m->read_buf) - free(m->read_buf); - - free(m); -} - -void meta_reserve(meta_s *m) -{ - // TODO: Use a hole in the .replay file to reserve space - char buf[_MAX_META_LEN]; - Mset(&buf, '#', _MAX_META_LEN-1, char); - fprintf(m->replay_fd, "%s\n", buf); -} - -void meta_write_start(meta_s *m) -{ - fseeko(m->replay_fd, m->offset, SEEK_SET); - // Write required '#' so that the regular worker processes - // will ignore that meta line. - fprintf(m->replay_fd, "#"); - - // Required for parsing in 'meta_read_s' - fprintf(m->replay_fd, "|"); -} - -void meta_write_s(meta_s *m, char *key, char *val) -{ - fprintf(m->replay_fd, "%s=%s|", key, val); -} - -void meta_write_l(meta_s *m, char *key, long val) -{ - char buf[1024]; - sprintf(buf, "%ld", val); - fprintf(m->replay_fd, "%s=%ld|", key, val); -} - -void meta_read_start(meta_s *m) -{ - size_t len = 0; - m->read_buf = Calloc(_MAX_META_LEN, char); - getline(&m->read_buf, &len, m->replay_fd); -} - -bool meta_read_s(meta_s *m, char *key, char **val) -{ - char *saveptr = NULL; - char *iterate_buf = Clone(m->read_buf); - int keylen = strlen(key); - - char *tok = strtok_r(iterate_buf, "|", &saveptr); - - while (tok) { - if (strncmp(tok, key, keylen) == 0 && tok[keylen] == '=') { - asprintf(val, "%s", tok+keylen+1); - free(iterate_buf); - return true; - } - tok = strtok_r(NULL, "|", &saveptr); - } - - free(iterate_buf); - return false; -} - -bool meta_read_l(meta_s *m, char *key, long *val) -{ - char *buf = NULL; - - if (meta_read_s(m, key, &buf)) { - *val = atol(buf); - free(buf); - return true; - } - - return false; -} diff --git a/ioreplay/src/meta/meta.h b/ioreplay/src/meta/meta.h deleted file mode 100644 index 10002cc..0000000 --- a/ioreplay/src/meta/meta.h +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef META_H -#define META_H - -#include "../defaults.h" - -/** - * @brief The meta information definition - * - * This is used to write or read meta information to/from the header - * of the .replay file. This information then is used by ioreplay - * in other steps. E.g. reading the amount of used file descriptors - * from the meta header in order to allocate data structures of the - * correct sizes before running the test! - */ -typedef struct meta_s_ { - FILE* replay_fd; /**< The FS of the .replay file */ - off_t offset; /**< The meta offset (usually 0) */ - char* read_buf; /**< Pointer to a read buffer */ -} meta_s; - -/** - * @brief Creates a new meta bject - * - * @return The new meta object - */ -meta_s* meta_new(); - -/** - * @brief Destroys a meta object - * - * @param m The meta object - */ -void meta_destroy(meta_s *m); - -/** - * @brief Reserves space in the .replay file for the meta header - * - * @param m The meta object - */ -void meta_reserve(meta_s *m); - -/** - * @brief Indicates that we start writing the meta header to the .replay file - * - * @param m The meta object - */ -void meta_write_start(meta_s *m); - -/** - * @brief Writes a string to the meta header - * - * @param m The meta object - * @param key The key - * @param val The string value - */ -void meta_write_s(meta_s *m, char *key, char *val); - -/** - * @brief Writes a long to the meta header - * - * @param m The meta object - * @param key The key - * @param val The long value - */ -void meta_write_l(meta_s *m, char *key, long val); - -/** - * @brief indicates that we start reading from the meta header - * - * @param m The meta object - */ -void meta_read_start(meta_s *m); - -/** - * @brief Reads a string from the meta header - * - * @param m The meta object - * @param key The key - * @param val The string val read - */ -bool meta_read_s(meta_s *m, char *key, char **val); - -/** - * @brief Reads a long from the meta header - * - * @param m The meta object - * @param key The key - * @param val The long val read - */ -bool meta_read_l(meta_s *m, char *key, long *val); - -#endif // META_H - diff --git a/ioreplay/src/mounts.c b/ioreplay/src/mounts.c deleted file mode 100644 index afa8376..0000000 --- a/ioreplay/src/mounts.c +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mounts.h" - -#include "utils/futils.h" - -#define _PATH_INSERT "/.ioreplay/" -#define _PATH_INSERT_LEN 11 // strlen of _PATH_INSERT - -void mounts_read(mounts_s *m) -{ - char *mounts = "/proc/mounts"; - size_t len = 0; - char *line = NULL; - char *saveptr = NULL; - - Put("Reading '%s'", mounts); - - FILE *fp = Fopen(mounts, "r"); - Out("Adding supported file systems to replay paths:"); - - while (getline(&line, &len, fp) != -1) { - bool ignore = true; - - char *dev = strtok_r(line, " ", &saveptr); - if (dev == NULL) { - Error("Could not parse device from %s", mounts); - } - - char *mp = strtok_r(NULL, " ", &saveptr); - if (mp == NULL) { - Error("Could not parse mountpoint from %s", mounts); - } - - char *fs = strtok_r(NULL, " ", &saveptr); - if (fs == NULL) { - Error("Could not parse file system from %s", mounts); - } -#ifdef MP_DEBUG - Debug("fs:%s", fs); -#endif - // TODO: Make file system types configurable - if (Eq(fs, "ext2")) { - ignore = false; - } else if (Eq(fs, "ext5")) { - ignore = false; - } else if (Eq(fs, "ext4")) { - ignore = false; - } else if (Eq(fs, "xfs")) { - ignore = false; - } else if (Eq(fs, "zfs")) { - ignore = false; - } else if (Eq(fs, "btrfs")) { - ignore = false; - } - - if (ignore) { - if (strcmp(mp, "/") != 0) { - m->ignore_mps[m->ignore_count] = Clone(mp); - m->ignore_count++; - } - - } else if (m->count >= MAX_MOUNTPOINTS) { - Error("Exceeded max mount points: %d\n", m->count); - - } else { - Out(" %s (%s)", mp, fs); - m->mps[m->count] = Clone(mp); - m->lengths[m->count] = strlen(mp); - m->count++; - } - } - - fclose(fp); - Out("\n"); -} - -mounts_s *mounts_new(options_s *opts) -{ - mounts_s *m = Malloc(mounts_s); - - m->opts = opts; - m->count = 0; - m->ignore_count = 0; - mounts_read(m); - - return m; -} - -void mounts_destroy(mounts_s *m) -{ - if (!m) - return; - for (int i = 0; i < m->count; i++) - free(m->mps[i]); - free(m); -} - -void mounts_trash(mounts_s *m) -{ - options_s *opts = m->opts; - set_limits_drop_root(opts->user); - Put("Moving all old files to trash (of previous tests)..."); - - struct timeval tv; - gettimeofday(&tv, NULL); - - char *wd_path = NULL; - asprintf(&wd_path, "%s/%s", opts->wd_base, opts->name); - - char *trash_path = NULL; - asprintf(&trash_path, "%s/.trash/%ld", opts->wd_base, tv.tv_sec); - - if (is_dir(wd_path)) { - ensure_dir_exists(trash_path); - chown_path(opts->user, trash_path); - if (rename(wd_path, trash_path)) { - Errno("Could not move '%s' to '%s'", wd_path, trash_path); - } - } - free(wd_path); - free(trash_path); - - for (int i = 0; i < m->count; i++) { - char *mp = m->mps[i]; - char *path = NULL; - asprintf(&path, "%s/%s/%s", mp, _PATH_INSERT, opts->name); - asprintf(&trash_path, "%s/%s/.trash/%ld", - mp, _PATH_INSERT, tv.tv_sec); - - if (is_dir(path)) { - ensure_dir_exists(trash_path); - chown_path(opts->user, trash_path); - if (rename(path, trash_path)) { - Errno("Could not move '%s' to '%s'", path, trash_path); - } - } - - free(path); - free(trash_path); - } - - Put("Done trashing!"); - Put("Once the drives fill up you may want to purge old data (-P)"); -} - -void mounts_purge(mounts_s *m) -{ - options_s *opts = m->opts; - set_limits_drop_root(opts->user); - - Out("Purging all data from the following directories:"); - - int active_purgers = 0, max_purgers = 16; - if (opts->num_workers > max_purgers) - max_purgers = opts->num_workers; - - char *purge_path = NULL; - asprintf(&purge_path, "%s", opts->wd_base); - if (is_dir(purge_path)) { - Out(" %s", purge_path); - pid_t pid = fork(); - - if (pid == 0) { - ensure_dir_empty(purge_path); - free(purge_path); - exit(0); - - } else if (pid < 0) { - Errno("\nUnable to create cleaner process! :'-("); - } - active_purgers++; - } - free(purge_path); - - int cleaner_status = SUCCESS; - - for (int i = 0; i < m->count; i++) { - char *mp = m->mps[i]; - char *purge_path = NULL; - asprintf(&purge_path, "%s/%s", mp, _PATH_INSERT); - - if (is_dir(purge_path)) { - if (active_purgers+1 >= max_purgers) { - wait(&cleaner_status); - active_purgers--; - } - - // TODO: Use threading model same way as in init/init.c - pid_t pid = fork(); - if (pid == 0) { - Out(" %s", purge_path); - ensure_dir_empty(purge_path); - free(purge_path); - exit(0); - } else if (pid < 0) { - Errno("Unable to create cleaner process! :'-("); - } - active_purgers++; - } - free(purge_path); - } - - while (wait(&cleaner_status) > 0) - active_purgers--; - Put("\nCleaning done!"); -} - -void mounts_init(mounts_s *m) -{ - options_s *opts = m->opts; - char *wd_path = NULL; - asprintf(&wd_path, "%s/%s", opts->wd_base, opts->name); - ensure_dir_exists(wd_path); - chown_path(opts->user, opts->wd_base); - chown_path(opts->user, wd_path); - - if (chdir(wd_path)) { - Errno("Could not chdir into '%s'!", wd_path); - - } else { - Put("Chdir into '%s'", wd_path); - } - - free(wd_path); - - for (int i = 0; i < m->count; i++) { - char *mp = m->mps[i]; - char *path = NULL; - - // Create .ioreplay/ directory on MP - asprintf(&path, "%s/%s", mp, _PATH_INSERT); - ensure_dir_exists(path); - chown_path(m->opts->user, path); - free(path); - path = NULL; - - // Create .ioreplay/NAME directory on MP - asprintf(&path, "%s/%s/%s", mp, _PATH_INSERT, opts->name); - ensure_dir_exists(path); - chown_path(m->opts->user, path); - free(path); - } -} - -bool mounts_ignore_path(mounts_s *m, const char *path) -{ - // CentOS 7 specific, ignore temp namespace mounts! - char *pos = strstr(path, "/tmp/namespace-"); - if (pos == path) - return true; - - // iterate backwards through all mount points. - for (int i = m->ignore_count-1; i >= 0; --i) { - char *mountpoint = m->ignore_mps[i]; - pos = strstr(path, mountpoint); - // Ignore this path as it is in the ignore mp list - if (pos == path) - return true; - } - - return false; -} - -bool mounts_transform_path(mounts_s *m, const char *name, - char *path, char **path_r) -{ - char *tmp = NULL; -#ifdef DEBUG_TRANSFORM_PATH - char *original_path = path; -#endif - bool line_ok = true; - - // First figure out whether there are '..' in any paths. If so we have to - // tokenize the path and remove '..'. Example: - // transform '/foo/bar/../' into '/foo/'. - // Also remove double '/' from paths. - - if (strstr(path, "..") || strstr(path, "//")) { - // tmp will be freed under label 'cleanup' at end of function. - tmp = Calloc(strlen(path)+1, char); - - // stack to put the tokens on - stack_s *s = stack_new(); - - // we need a copy of the path, so we can tokenize it into the stack - char* clone = Clone(path); - - char *saveptr = NULL; - char *tok = strtok_r(clone, "/", &saveptr); - - // Add each part of the path to the stack. - while (tok) { - if (strcmp(tok, "..") == 0) { - stack_pop(s); - } else { - stack_push(s, tok); - } - tok = strtok_r(NULL, "/", &saveptr); - } - - if (stack_is_empty(s)) { - strcpy(tmp, "."); - - } else { - s = stack_new_reverse_from(s); - strcpy(tmp, "/"); - strcat(tmp, (char*)stack_pop(s)); - - while(!stack_is_empty(s)) { - strcat(tmp, "/"); - strcat(tmp, (char*)stack_pop(s)); - } - } - - stack_destroy(s); - free(clone); - - // This is the path without '..' and '//' (and '///' ... etc') - path = tmp; - } - - // Now heck whether the path is on a supported file system. If not, ignore! - if (mounts_ignore_path(m, path)) { - line_ok = false; - goto cleanup; - } - - // So the path is on a valid mount point! Now we need to insert - // .ioreplay/NAME to each mount point, e.g. /usr/local/.ioreplay/NAME/... - - // Iterate backwards through all mount points. - for (int i = m->count-1; i >= 0; --i) { - char *mountpoint = m->mps[i]; - int mp_len = m->lengths[i]; - - if (strncmp(path, mountpoint, mp_len) == 0) { - // Found a path to replace - // Now insert .ioreplay/NAME/ into the file path. - *path_r = Calloc(strlen(path) + strlen(name)+1 - + _PATH_INSERT_LEN+1, char); - - if (strcmp(mountpoint, "/") == 0) { - // Root path - strcpy(*path_r, _PATH_INSERT); - strcat(*path_r, name); - strcat(*path_r, path); - - } else { - strcpy(*path_r, mountpoint); - strcat(*path_r, _PATH_INSERT); - strcat(*path_r, name); - char *pos = path; - pos += mp_len * (int) sizeof(char); - strcat(*path_r, pos); - } - - goto cleanup; - } - } - - if (tmp) - free(tmp); - - return line_ok; - -cleanup: -#ifdef DEBUG_TRANSFORM_PATH - Debug("Transform path '%s' -> '%s' -> '%s'", original_path, path, *path_r); -#endif - if (tmp) - free(tmp); - - return line_ok; -} - -int mounts_get_mountnumber(mounts_s *m, const char *path) -{ - for (int i = m->count-1; i >= 0; --i) { - char *mountpoint = m->mps[i]; - int mp_len = m->lengths[i]; - - if (strncmp(path, mountpoint, mp_len) == 0) - return i; - } - - return 0; -} diff --git a/ioreplay/src/mounts.h b/ioreplay/src/mounts.h deleted file mode 100644 index a644ddb..0000000 --- a/ioreplay/src/mounts.h +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MOUNTPOINTS_H -#define MOUNTPOINTS_H - -#include "datas/stack.h" -#include "defaults.h" -#include "options.h" - -#define MAX_MOUNTPOINTS 1024 - -/** - * @brief Represents data parsed from /proc/mounts - * - * This is used to determine the file systems and the file system types - * currently mounted on the Linux system. I/O replay only replays I/O - * on specific file systems such as ext4 or xfs and will ignore any special - * or pseudo file systems such as tmpfs, devfs, sysfs. It does not make sense - * to replay I/O on these because there is actually no underlying block device - * attached to these. - * - * A mounts object helps to determine whether a path relies on a valid file - * system or not. All I/O operations on invalid file systems are being filtered - * out! - * - * The mounts object also does more things such as purging temp test data from - * the mountpoints etc... - */ -typedef struct mounts_s_ { - int count; /**< The amount of mount points */ - char *mps[MAX_MOUNTPOINTS]; /**< The mp paths */ - int lengths[MAX_MOUNTPOINTS]; /**< The mp lenghts */ - int ignore_count; /**< The amount of ignored mount points */ - char *ignore_mps[MAX_MOUNTPOINTS]; /**< The ignored mp paths */ - options_s *opts; /**< A pointer to the options object */ -} mounts_s; - -/** - * @brief Creates a new mounts object - * - * @param opts The options object - * @return The new mounts object - */ -mounts_s *mounts_new(options_s *opts); - -/** - * @brief Destroys the mounts object - * - * @param m The mounts object - */ -void mounts_destroy(mounts_s *m); - -/** - * @brief moves all files within replay mounts to trash - * - * It moves all files of the .ioreplay/NAME directories to - * .ioreplay/NAME.trashEPOCH directories for all available mount points. - * It does the same for the working dorectory of the current test. - * - * @param m The responsible mounts object - */ -void mounts_trash(mounts_s *m); - -/** - * @brief Deletes all files within replay mounts - * - * It deletes all files from the .ioreplay/ directories for all availabe - * mount points. It also deletes the working directory of all tests. The - * function forks one sub-process per mount point, so it is cleaning all drives - * in parallel. - * - * It can take a significant amount of time to actually delete all these files. - * That's why there is also a mounts_trash function, which will not delete the - * files but move them to trash folders so they can be deleted at a later - * point. - * - * @param m The responsible mounts object - */ -void mounts_purge(mounts_s *m); - -/** - * @brief Ensures all mounts have a .ioreplay/NAME directory - * - * These directories are used by ioreplay to run the I/O replay tests in. - * The function also ensures to have the correct user permissions for these - * directories. - * - * @param m The responsible mounts object - */ -void mounts_init(mounts_s *m); - -/** - * @brief Reads /proc/mounts to determine which mounts are available - * - * @param m The mounts object - */ -void mounts_read(mounts_s *m); - -/** - * @brief Determines whether a path should be ignored - * - * ioreplay replays I/O only on known mount points of known - * file system types. This function helps to determine whether - * a path is on a valid mount point or not. - * - * @param m The responsible mounts object - * @param path The path to check - * @return true if path has to be ignored - */ -bool mounts_ignore_path(mounts_s *m, const char *path); - -/** - * @brief Inserts ./ioreplay/NAME into a path - * - * This function inserts ./ioreplay/NAME into a given file path. - * The function also checks whether the path is on a supported replay - * path or not. E.g. we want to ignore file systems such as devfs, sysfs, - * procfs.. etc. - * - * @param m The responsible mountpoint object - * @param name The name of the test - * @param path The original path - * @param path_r The tansformed path (has to be freed if not NULL) - * @return False if this path is to be ignored - */ -bool mounts_transform_path(mounts_s *m, const char *name, - char *path, char **path_r); - - -/** - * @brief Get's the mount point number of a path - * - * Used by init.c to determine which thread to use to initialise a file - * or directory on a given path. - * - * @param m The responsible mountpoint object - * @param path The file/directory path - * @return The mountpoint number - */ -int mounts_get_mountnumber(mounts_s *m, const char *path); - -#endif // MOUNTPOINTS_H diff --git a/ioreplay/src/opcodes.h b/ioreplay/src/opcodes.h deleted file mode 100644 index 3d5c114..0000000 --- a/ioreplay/src/opcodes.h +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef OPCODES_H -#define OPCODES_H - -typedef enum { - // stat() syscalls - FSTAT = 0, - FSTAT_AT, - FSTATFS, - FSTATFS64, - LSTAT, - STAT, - STATFS, - STATFS64, - - // read() syscalls - READ = 10, - READV, - READAHEAD, - READDIR, - READLINK, - READLINK_AT, - - // write() syscalls - WRITE = 20, - WRITEV, - - // open() and other syscalls which may create files - OPEN = 30, - OPEN_AT, - CREAT, - MKDIR, - MKDIR_AT, - NAME_TO_HANDLE_AT, - OPEN_BY_HANDLE_AT, - - // rename() syscalls - RENAME = 40, - RENAME_AT, - RENAME_AT2, - - // close() and unlink() syscalls - CLOSE = 50, - UNLINK, - UNLINK_AT, - RMDIR, - - // sync() syscalls - FSYNC = 60, - FDATASYNC, - SYNC, - SYNCFS, - SYNC_FILE_RANGE, - - // other syscalls - FCNTL = 70, - GETDENTS, - LSEEK, - - // mmap syscalls - MMAP2 = 80, - MUNMAP, - REMAP, - MSYNC, - - // chmod() syscalls - CHMOD = 100, - FCHMOD, - FCHMODAT, - - // chown() syscalls - CHOWN = 110, - CHOWN16, - LCHOWN, - LCOWN16, - FCHOWN, - FCHOWN16, - FCHOWNAT, - - // Meta operations (I/O replay internal use only) - // A single thread terminates - META_EXIT = 900, - // All threads of a process termiate (process termination) - META_EXIT_GROUP, - // Meta operation for lamport synchronisation (currently unused) - META_TIMELINE - -} opcode_e; - -#endif // OPCODES_H diff --git a/ioreplay/src/options.c b/ioreplay/src/options.c deleted file mode 100644 index c1dcdb9..0000000 --- a/ioreplay/src/options.c +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "options.h" - -options_s *options_new() -{ - options_s *o = Malloc(options_s); - - o->capture_file = NULL; - o->replay_file = NULL; - o->stats_file = NULL; - o->wd_base = "/usr/local/ioreplay"; - o->num_workers = 4; - o->num_threads_per_worker = 128; - o->user = "mcuser"; - o->name = "test0"; - o->init = false; - o->replay = false; - o->speed_factor = 0; - o->drop_caches = false; - o->purge = false; - o->trash = false; - o->pid = -1; - o->module = "ioreplay.ko"; - - return o; -} - -void options_destroy(options_s *o) -{ - if (o->capture_file) - free(o->capture_file); - if (o->replay_file) - free(o->replay_file); - if (o->stats_file) - free(o->stats_file); - - free(o); -} diff --git a/ioreplay/src/options.h b/ioreplay/src/options.h deleted file mode 100644 index 66cb0f7..0000000 --- a/ioreplay/src/options.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef OPTIONS_H -#define OPTIONS_H - -#include -#include "defaults.h" - -/** - * @brief The options definition, used to store user input - */ -typedef struct options_s_ { - char *capture_file; /**< The name of the .capture file */ - char *replay_file; /**< The name of the .replay file */ - char *stats_file; /**< The name of the .stats file */ - bool write_stats_file; /**< Write a stats file at the end of the test */ - char *user; /**< The user name to run the test as */ - char *name; /**< The name of the test (found in .ioreplay/name sub-dirs) */ - char *wd_base; /**< The working directory base */ - int num_workers; /**< The amount of worker processes */ - int num_threads_per_worker; /**< Max threads per worker processes */ - bool init; /**< If set ioreplay will initialise the environment */ - bool replay; /**< If set ioreplay will run/replay the test */ - bool purge; /**< If set ioreplay will purge the environment */ - bool trash; /**< If set ioreplay will trash the environment */ - bool drop_caches; /**< True if ioreplay should drop all Linux caches */ - double speed_factor; /**< Specifies how fast the test is replayed */ - int pid; /**< Specifies a process id to capture */ - char *module; /**< Specifies the kernel module for capturing */ -} options_s; - -/** - * @brief Creates a new options object - * - * The options object contains all options specified by the user as a command - * line option. It is filled with default values during creation. - * - * @return The options object - */ -options_s *options_new(); - -/** - * @brief Destroys the options object - * - * @param o The options object - */ -void options_destroy(options_s *o); - -#endif // OPTIONS_H diff --git a/ioreplay/src/replay/replay.c b/ioreplay/src/replay/replay.c deleted file mode 100644 index e4606d1..0000000 --- a/ioreplay/src/replay/replay.c +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "replay.h" - -#include "../datas/amap.h" -#include "../meta/meta.h" -#include "../mounts.h" -#include "rworker.h" -#include "rstats.h" - -void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes, - long *num_pids, long *num_fds, long *num_lines) -{ - meta_s *m = meta_new(replay_fd); - meta_read_start(m); - - long version = 0; - if (meta_read_l(m, "version", &version)) { - Put("Replay version is '%ld'", version); - if (version != REPLAY_VERSION) { - Error(".replay file of incompatible version, got %x, expected %x", - (int)version, REPLAY_VERSION); - } - } - - char *user; - if (meta_read_s(m, "user", &user)) { - Put("Setting user to '%s'", user); - opts->user = user; - } - - char *name; - if (meta_read_s(m, "name", &name)) { - Put("Setting name to '%s'", name); - opts->name = name; - } - - if (meta_read_l(m, "num_vsizes", num_vsizes)) { - if (*num_vsizes < 0) { - Error("Lamport vsize overflow"); - } - Put("Setting num of vsizes to '%ld'", *num_vsizes); - } - - if (meta_read_l(m, "num_mapped_pids", num_pids)) { - if (*num_pids < 0) { - Error("Process overflow (too many process IDs in .replay)"); - } - Put("Setting num of PIDs to '%ld'", *num_pids); - } - - if (meta_read_l(m, "num_mapped_fds", num_fds)) { - if (*num_fds < 0) { - Error("FD overflow (too many FDs in .replay)"); - } - Put("Setting num of FDs to '%ld'", *num_fds); - } - - if (meta_read_l(m, "num_lines", num_lines)) { - if (*num_fds < 0) { - Error("Overflow (too many lines in .replay)"); - } - Put("Setting num of lines to '%ld'", *num_lines); - } - - meta_destroy(m); -} - -status_e replay_run(options_s *opts) -{ - status_e status = SUCCESS; - - if (opts->drop_caches) { - drop_caches(); - //cache_file(opts->replay_file); - } - - // Extract information from the meta header - FILE *replay_fd = Fopen(opts->replay_file, "r"); - long num_vsizes = 0, num_pids = 0, num_fds = 0, num_lines = 0; - replay_extract_header(opts, replay_fd, &num_vsizes, &num_pids, - &num_fds, &num_lines); - fclose(replay_fd); - - // A map of all file descriptors used. - Out("Creating FD map..."); - amap_s *fds_map = NULL; - if (opts->num_workers > 1) { - fds_map = amap_new_mmapped(num_fds); - } else { - fds_map = amap_new(num_fds); - } - Put("done"); - - // To collect all individual worker's stats into the global - // stats object. - stack_s *all_worker_stats = stack_new(); - - // The global stats object - rstats_s *stats = rstats_new(opts); - rstats_start(stats); - - // Fork worker processes, each worker process will read the .replay file - // individually. - - if (opts->num_workers > 1) { - for (int i = 0; i < opts->num_workers; ++i) { - rworker_stats_s *worker_stats = rworker_stats_new_mmap(); - stack_push(all_worker_stats, worker_stats); - - pid_t pid = fork(); - - if (pid == 0) { - // One worker object per fork - rworker_s *w = rworker_new(i, fds_map, num_vsizes, num_pids, opts, - worker_stats); - - // Process the .replay journal line by line - status_e status = rworker_process_lines(w, num_lines); - Put("worker(%d): Exiting from %d with status %d", i, - pid, status); - rworker_destroy(w); - - // Exit sub-process - exit(status); - - } else if (pid < 0) { - Errno("worker(%d): Unable to create worker process! :'-(", i); - - } else { - Put("worker(%d): Process with pid %d forked", i, pid); - } - } - - set_limits_drop_root(opts->user); - - Put("Waiting for worker processes to finish"); - pid_t pid; - int rworker_status = SUCCESS; - - while ((pid = wait(&rworker_status)) > 0) { - if (rworker_status != SUCCESS) - status = rworker_status; - - Put("Process with pid %d exited with status %d", - pid, rworker_status); - } - - Put("All workers finished (%d)!", status); - - } else { - Put("Only one worker, don't fork sub-processes"); - - rworker_stats_s *worker_stats = rworker_stats_new_mmap(); - stack_push(all_worker_stats, worker_stats); - - rworker_s *w = rworker_new(0, fds_map, num_vsizes, num_pids, - opts, worker_stats); - status = rworker_process_lines(w, num_lines); - rworker_destroy(w); - - Put("Worker finished work!"); - } - - // Collect all statistics - rstats_stop(stats); - while (!stack_is_empty(all_worker_stats)) { - rworker_stats_s *worker_stats = stack_pop(all_worker_stats); - rstats_add_from_worker(stats, worker_stats); - rworker_stats_destroy(worker_stats); - } - stack_destroy(all_worker_stats); - - rstats_print(stats); - rstats_destroy(stats); - - amap_destroy(fds_map); - return status; -} diff --git a/ioreplay/src/replay/replay.h b/ioreplay/src/replay/replay.h deleted file mode 100644 index dcc3d84..0000000 --- a/ioreplay/src/replay/replay.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef REPLAY_H -#define REPLAY_H - -#include "../defaults.h" -#include "../utils/futils.h" -#include "../opcodes.h" -#include "../options.h" -#include "rioop.h" -#include "rprocess.h" - -/** - * @brief Replays the given .replay file - * - * @param opts The options object - * @return SUCCESS if everything went fine - */ -status_e replay_run(options_s *opts); - -/** - * @brief Extract required meta data from .replay's meta header - * - * @param opts The options object - * @param replay_fd The file handle to the .replay file - * @param num_vsizes The amount of virtual sizes/paths - * @param num_pids The amount of process IDs - * @param num_fds The amount of virtual file descriptors - * @param num_lines The amount of .replay lines with I/O ops - */ -void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes, - long *num_pids, long *num_fds,long *num_lines); - -#endif // REPLAY_H diff --git a/ioreplay/src/replay/rioop.c b/ioreplay/src/replay/rioop.c deleted file mode 100644 index 2e16c94..0000000 --- a/ioreplay/src/replay/rioop.c +++ /dev/null @@ -1,425 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rioop.h" - -#include "../vfd.h" -#include "rworker.h" - -// Printing error messages -#define _Error(...) \ - fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\nlineno:%ld path:%s\n", task->lineno, vfd->path); \ - fflush(stdout); \ - fflush(stderr); \ - exit(ERROR); - -#define _Errno(...) \ - fprintf(stderr, "%s:%d ERROR: %s (%d). ", __FILE__, __LINE__, \ - strerror(errno), errno); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, "\nlineno:%ld path:%s\n", task->lineno, vfd->path); \ - fflush(stdout); \ - fflush(stderr); \ - exit(ERROR); - -#define _Init_arg(num) int arg = atoi(task->toks[num]) -#define _Init_cmd(num) int cmd = atoi(task->toks[num]) -#define _Init_fd(num) long fd = atol(task->toks[num]) -#define _Init_flags(num) int flags = atoi(task->toks[num]) -//#define _Init_mode(num) int mode = atoi(task->toks[num]) -#define _Init_offset(num) long offset = atol(task->toks[num]) -#define _Init_op(num) int op = atoi(task->toks[num]) -#define _Init_path2(num) char *path2 = task->toks[num] -#define _Init_path(num) char *path = task->toks[num] -#define _Init_rc(num) int rc = atoi(task->toks[num]) -#define _Init_whence(num) long whence = atol(task->toks[num]) - -#define _Init_bytes(num) \ - int bytes = atoi(task->toks[num]); \ - if (bytes <= 0) return - -#define _Init_virtfd \ - vfd_s *vfd = amap_get(p->fds_map, fd); \ - if (vfd == NULL) return - -void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_op(2); - - switch (op) { - // stat() syscalls - case FSTAT: - rioop_fstat(p, t, task); - break; - case FSTATFS: - case FSTATFS64: - //Error("op(%d) not implemented", op); - break; - case FSTAT_AT: - case LSTAT: - case STAT: - rioop_stat(p, t, task); - break; - case STATFS: - case STATFS64: - //Error("op(%d) not implemented", op); - break; - - // read() syscalls - case READ: - case READV: - rioop_read(p, t, task); - break; - case READAHEAD: - //Error("op(%d) not implemented", op); - break; - case READLINK: - case READLINK_AT: - //Error("op(%d) not implemented", op); - break; - - // write() syscalls - case WRITE: - case WRITEV: - rioop_write(p, t, task); - break; - - // open() and other syscalls which may creat - case OPEN: - case OPEN_AT: - rioop_open(p, t, task, -1); - break; - case CREAT: - // A call to crat() is equivalent to calling open() with flags.. - rioop_open(p, t, task, O_CREAT|O_WRONLY|O_TRUNC); - break; - case MKDIR: - case MKDIR_AT: - rioop_mkdir(p, t, task); - break; - - // rename() syscalls - case RENAME: - case RENAME_AT: - case RENAME_AT2: - rioop_rename(p, t, task); - break; - - // close() and unlink() syscalls - case CLOSE: - rioop_close(p, t, task); - break; - case UNLINK: - case UNLINK_AT: - rioop_unlink(p, t, task); - break; - case RMDIR: - rioop_rmdir(p, t, task); - break; - - // sync() syscalls - case FSYNC: - rioop_fsync(p, t, task); - break; - case FDATASYNC: - rioop_fdatasync(p, t, task); - break; - case SYNC: - case SYNCFS: - case SYNC_FILE_RANGE: - //Error("op(%d) not implemented", op); - break; - - // Other syscalls - case FCNTL: - rioop_fcntl(p, t, task); - break; - case GETDENTS: - rioop_getdents(p, t, task); - break; - case LSEEK: - rioop_lseek(p, t, task); - break; - - // chmod() syscalls - case CHMOD: - rioop_chmod(p, t, task); - break; - case FCHMOD: - rioop_fchmod(p, t, task); - break; - - // chown() syscalls - case CHOWN: - rioop_chown(p, t, task); - break; - case FCHOWN: - case FCHOWNAT: - rioop_fchown(p, t, task); - break; - case LCHOWN: - rioop_lchown(p, t, task); - break; - - // Meta operations (I/O replay internal use only). - case META_EXIT_GROUP: - break; - case META_TIMELINE: - break; - - default: - Error("op(%d) not implemented", op); - break; - } -} - -void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - struct stat buf; - stat(path, &buf); -} - -void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - struct stat buf; - fstat(vfd->fd, &buf); -} - -void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - _Init_path2(4); - rename(path, path2); -} - -void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_bytes(4); - _Init_virtfd; - - char *buf = Calloc(bytes+1, char); - read(vfd->fd, buf, bytes); - free(buf); -} - -void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_bytes(4); - _Init_virtfd; - - char *buf = Calloc(bytes+1, char); - sprintf(buf, "%ld", task->lineno); - Fill_with_stuff(buf, bytes); - if (vfd->fd == 0) { - Debug("%d %d %ld", vfd->fd, vfd->debug, task->lineno); - _Error("ERROR"); - } - write(vfd->fd, buf, bytes); - free(buf); -} - -void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_) -{ - _Init_fd(3); - _Init_path(4); - _Init_flags(6); - - // Special case as this is creat() now - if (flags_ != -1) - flags = flags_; - - bool directory = Has(flags, O_DIRECTORY); - - if (fd > 0) { - if (directory) { - // We can not open a directory via open() otherwise! - flags &= (O_RDONLY & ~(O_RDWR|O_WRONLY|O_CREAT)); - } else { - // We don't want to open the file in read only mode. - // SystemTap could have skipped syscalls to fcntl or open - flags &= ~O_RDONLY; - } - // flags |= O_DIRECT|O_SYNC; - flags &= ~O_EXCL; - } - - int ret = open(path, flags, S_IRWXU|S_IRWXG|S_IRWXO); - - if (fd < 0 && ret > 0) { - close(ret); -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "TRACE OPEN|open+close|%s|\n", path); - fflush(t->rthread_fd); -#endif - } - - if (fd > 0 && ret > 0) { - vfd_s *vfd = vfd_new(ret, fd, path); - amap_set(p->fds_map, fd, vfd); - -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "TRACE OPEN|open|%s|\n", path); - fflush(t->rthread_fd); -#endif - } -} - -void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - - amap_unset(p->fds_map, fd); - if (vfd->dirfd) { - closedir(vfd->dirfd); -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "TRACE OPEN|closedir|%s|\n", vfd->path); - fflush(t->rthread_fd); -#endif - } else { - close(vfd->fd); -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "TRACE OPEN|close|%s|\n", vfd->path); - fflush(t->rthread_fd); -#endif - } - vfd_destroy(vfd); -} - -void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - - // getdents expects a dirfd - DIR *dirfd = fdopendir(vfd->fd); - if (dirfd) { - vfd->dirfd = dirfd; - readdir(dirfd); -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "TRACE OPEN|fdopendir|%s|\n", vfd->path); - fflush(t->rthread_fd); -#endif - } -} - -void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - mkdir(path, S_IRWXU|S_IRWXG|S_IRWXO); -} - -void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - unlink(path); -} - -void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - rmdir(path); -} - -void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_bytes(6); - _Init_virtfd; - lseek(vfd->fd, bytes, SEEK_SET); -} - -void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - fsync(vfd->fd); -} - -void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - fdatasync(vfd->fd); -} - -void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_cmd(4); - _Init_arg(5); - _Init_virtfd; - - switch (cmd) { - case F_GETFD: - case F_GETFL: - fcntl(vfd->fd, cmd); - break; - case F_SETFD: - case F_SETFL: - fcntl(vfd->fd, cmd, arg); - break; - default: - break; - } -} - -void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - chmod(path, S_IRWXU|S_IRWXG|S_IRWXO); -} - -void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - fchmod(vfd->fd, S_IRWXU|S_IRWXG|S_IRWXO); -} - -void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - rworker_s *w = t->worker; - options_s *opts = w->opts; - struct passwd *pwd = getpwnam(opts->user); - chown(path, pwd->pw_uid, -1); -} - -void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_fd(3); - _Init_virtfd; - rworker_s *w = t->worker; - options_s *opts = w->opts; - struct passwd *pwd = getpwnam(opts->user); - fchown(vfd->fd, pwd->pw_uid, -1); -} - -void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task) -{ - _Init_path(3); - rworker_s *w = t->worker; - options_s *opts = w->opts; - struct passwd *pwd = getpwnam(opts->user); - lchown(path, pwd->pw_uid, -1); -} - diff --git a/ioreplay/src/replay/rioop.h b/ioreplay/src/replay/rioop.h deleted file mode 100644 index 4db4284..0000000 --- a/ioreplay/src/replay/rioop.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef RIOOP_H -#define RIOOP_H - -#include "../defaults.h" -#include "../utils/futils.h" -#include "../opcodes.h" -#include "rprocess.h" -#include "rthread.h" - -/** - * @brief Replays the responsible I/O operation of a given task - * - * @param p The virtual replay process object - * @param t The thread object - * @param task The replay task object - */ -void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task); - -void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_); -void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task); -void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task); - -#endif // RIOOP_H diff --git a/ioreplay/src/replay/rprocess.c b/ioreplay/src/replay/rprocess.c deleted file mode 100644 index 4efd835..0000000 --- a/ioreplay/src/replay/rprocess.c +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rprocess.h" - -rprocess_s* rprocess_new(const int pid, amap_s *fds_map) -{ - rprocess_s *p = Malloc(rprocess_s); - - p->fds_map = fds_map; - p->pid = pid; - p->terminate = 0; - p->lineno = 0; - - return p; -} - -void rprocess_destroy(rprocess_s *p) -{ - if (!p) - return; - free(p); -} diff --git a/ioreplay/src/replay/rprocess.h b/ioreplay/src/replay/rprocess.h deleted file mode 100644 index 739dd89..0000000 --- a/ioreplay/src/replay/rprocess.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef RPROCESS_H -#define RPROCESS_H - -#include "../datas/hmap.h" -#include "../datas/amap.h" -#include "../defaults.h" -#include "rthread.h" - -/** - * @brief The virtual replay process object definition - * - * This defines a virtual process in replay context. - */ -typedef struct rprocess_s_ { - int terminate; /**< Indicates whether the worker is terminating or not */ - int rworker_num; /**< The worker number of the responsible worker */ - int pid; /**< The virtual process ID */ - unsigned long lineno; /**< Holding the current .replay line number */ - bool initm; /**< Indicates whether ioreplay is in init mode or not */ - amap_s *fds_map; /**< Holding all file descriptors */ -} rprocess_s; - -rprocess_s* rprocess_new(const int pid, amap_s *fds_map); -void rprocess_destroy(rprocess_s* p); - -#endif // RPROCESS_H diff --git a/ioreplay/src/replay/rstats.c b/ioreplay/src/replay/rstats.c deleted file mode 100644 index c3e6e38..0000000 --- a/ioreplay/src/replay/rstats.c +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rstats.h" - -#include - -rstats_s* rstats_new(options_s *opts) -{ - rstats_s *s = Malloc(rstats_s); - - s->opts = opts; - s->loadavg_high = 0; - s->ioops = 0; - s->duration = 0; - s->time_ahead = -1; - - if (opts->stats_file) - s->stats_fd = Fopen(opts->stats_file, "w"); - else - s->stats_fd = stdout; - - return s; -} - -void rstats_destroy(rstats_s *s) -{ - if (s->stats_fd != stdout) - fclose(s->stats_fd); - - free(s); -} - -rworker_stats_s* rworker_stats_new_mmap(options_s *opts) -{ - // Share this object between processes, so that the stats cann be - // collected by the master process! - rworker_stats_s *s = Mmapshared(rworker_stats_s); - - s->loadavg_high = 0; - s->ioops = 0; - s->time_ahead = -1; - - return s; -} - -void rworker_stats_destroy(rworker_stats_s *s) -{ - munmap(s, sizeof(rworker_stats_s)); -} - - -void rstats_start(rstats_s* s) -{ - gettimeofday(&s->start_time, NULL); -} - -void rstats_stop(rstats_s* s) -{ - gettimeofday(&s->end_time, NULL); - s->duration= ((s->end_time.tv_sec - s->start_time.tv_sec) * 1000 - + (s->end_time.tv_usec - s->start_time.tv_usec) / 1000) / 1000; - -} - -void rstats_add_from_worker(rstats_s* s, rworker_stats_s* w) -{ - if (s->loadavg_high < w->loadavg_high) - s->loadavg_high = w->loadavg_high; - - if (s->time_ahead == -1 || s->time_ahead > w->time_ahead) - s->time_ahead = w->time_ahead; - - s->ioops += w->ioops; -} - -void rstats_print(rstats_s* s) -{ - options_s *opts = s->opts; - - if (opts->stats_file) { - Put("Writing stats to '%s'", opts->stats_file); - } - - fprintf(s->stats_fd, "Stats of test '%s':\n", opts->name); - fprintf(s->stats_fd, "\tNum workers: %d\n", opts->num_workers); - fprintf(s->stats_fd, "\tThreads per worker: %d\n", opts->num_threads_per_worker); - fprintf(s->stats_fd, "\tThreads total: %d\n", - opts->num_threads_per_worker * opts->num_workers); - fprintf(s->stats_fd, "\tHighest loadavg: %.2f\n", s->loadavg_high); - fprintf(s->stats_fd, "\tPerformed ioops: %ld\n", s->ioops); - if (s->duration > 0) - fprintf(s->stats_fd, "\tAverage ioops/s: %.2f\n", s->ioops/s->duration); - fprintf(s->stats_fd, "\tTime ahead: %lds\n", s->time_ahead/1000); - fprintf(s->stats_fd, "\tTotal time: %.2fs\n", s->duration); -} - diff --git a/ioreplay/src/replay/rstats.h b/ioreplay/src/replay/rstats.h deleted file mode 100644 index 1ce3f27..0000000 --- a/ioreplay/src/replay/rstats.h +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * @file rstats.h - * @author Paul Buetow - * - * @brief For collecting replay stats - */ - -#ifndef RSTATS_H -#define RSTATS_H - -#include "../defaults.h" -#include "../options.h" - -#include - -/** - * @brief Definition of the rstats object - * - * Used to store global statistics. - */ -typedef struct rstats_s_ { - double loadavg_high; /**< Highest load average */ - long ioops; /**< Total amount if io operations */ - double duration; /**< Duration of the test */ - long time_ahead; /**< Time ahead of the original speed */ - struct timeval start_time; /**< Start time of the test */ - struct timeval end_time; /**< End time of the test */ - options_s *opts; /**< The I/O replay options object */ - FILE *stats_fd; /**< The file descriptor for writing the stats */ -} rstats_s; - -/** - * @brief Definition of the per worker stats object - * - * Used to store per worker process I/O stats - */ -typedef struct rworker_stats_s_ { - double loadavg_high; /**< Highest amount of io ops per second */ - long ioops; /**< Total amount if io operations */ - long time_ahead; /**< Time ahead of the original speed */ -} rworker_stats_s; - -/** - * @brief Creates a new stats object - * - * @return The new stats object - */ -rstats_s* rstats_new(options_s *opts); - -/** - * @brief Destroys the stats object - * - * @param s The stats object - */ -void rstats_destroy(rstats_s* s); - -/** - * @brief Creates a new per worker stats object - * - * The memory is mapped into shared memory so it can be shared across multiple - * processes. - * - * @return The new stats object - */ -rworker_stats_s* rworker_stats_new_mmap(); - -/** - * @brief Destroys the per worker stats object - * - * @param s The stats object - */ -void rworker_stats_destroy(rworker_stats_s* s); - -/** - * @brief Starts the stats - * - * @param s The stats object - */ -void rstats_start(rstats_s* s); - -/** - * @brief Finalises the stats - * - * @param s The stats object - */ -void rstats_stop(rstats_s* s); - -/** - * @brief Prints the stats - * - * @param s The stats object - */ -void rstats_print(rstats_s* s); - -/** - * @brief Adds per worker stats to the global stats object - * - * @param s The global stats object - * @param w The worker stats object - */ -void rstats_add_from_worker(rstats_s* s, rworker_stats_s* w); - -#endif diff --git a/ioreplay/src/replay/rtask.c b/ioreplay/src/replay/rtask.c deleted file mode 100644 index b1afb92..0000000 --- a/ioreplay/src/replay/rtask.c +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rtask.h" - -#include "rthread.h" -#include "rworker.h" - -rtask_s* rtask_new() -{ - rtask_s *task = Malloc(rtask_s); - - *task = (rtask_s) { - .worker = NULL, .process = NULL - }; - task->line[0] = '\0'; - -#ifdef THREAD_DEBUG - task->clone = NULL; -#endif - - return task; -} - -void rtask_destroy(rtask_s *task) -{ - if (task) - free(task); -} - -void rtask_update(rtask_s *task, void *worker, void *process, char *line, - const long lineno, const long vsize) -{ - task->worker = worker; - task->process = process; - task->lineno = lineno; - task->vsize = vsize; - strcpy(task->line, line); -} diff --git a/ioreplay/src/replay/rtask.h b/ioreplay/src/replay/rtask.h deleted file mode 100644 index 35c5714..0000000 --- a/ioreplay/src/replay/rtask.h +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef RTASK_H -#define RTASK_H - -#include "../defaults.h" - -/** - * @brief The replay task definition - * - * The rtask holds all possible variables required to process a particular - * .replay line and to replay the corresponding I/O operation. - */ -typedef struct rtask_s_ { - void *worker; /* The responsible worker object */ - void *process; /* The responsible process object */ - unsigned long lineno; /**< The current line number */ - unsigned long vsize; /**< The vsize */ - char *toks[MAX_TOKENS+1]; /**< The tokens parsed from the .replay line */ - char line[MAX_LINE_LEN]; /**< The remaining part of the .replay line */ -#ifdef RTASK_DEBUG - char *clone; /**< Used for debug purposes only */ -#endif -} rtask_s; - -/** - * @brief Creates a new thread task object - * - * This function creates a new thread task object. Such a task object is used - * by the worker to hand over I/O tasks to the corresponding threads. The - * actual I/O work is performed by the threads then. - * - * @return The new thread task object - */ -rtask_s* rtask_new(); - -/** - * @brief Destroys the replay task object - * - * @param t The thread task object to be destroyed - */ -void rtask_destroy(rtask_s* t); - -/** - * @brief Updates a reused/recycle task object - * - * @param task The task object to be updated - * @param worker The responsibe worker object - * @param process The responsible process object - * @param line The remaining line of the .replay file - * @param lineno The current line number of the .replay file - * @param vsize The vsize/path id - */ -void rtask_update(rtask_s *task, void *worker, void *process, char *line, - const long lineno, const long vsize); - -#endif // RTASK_H diff --git a/ioreplay/src/replay/rthread.c b/ioreplay/src/replay/rthread.c deleted file mode 100644 index 55364ec..0000000 --- a/ioreplay/src/replay/rthread.c +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rthread.h" - -#include - -#include "rworker.h" -#include "rprocess.h" - -#include "rioop.h" - -#ifdef THREAD_DEBUG -/** - * @brief For debugging purposes only - * - * @param t The responsible thread object - */ -static void _rthread_init_log(rthread_s *t) -{ - rworker_s *w = t->worker; - char *rthread_log = Calloc(1024, char); - snprintf(rthread_log, 1023, "/tmp/ioreplay/worker%d.thread%ld.debuglog", - w->rworker_num, (long)pthread_self()); - - ensure_dir_exists("/tmp/ioreplay"); - t->rthread_fd = Fopen(rthread_log, "a"); - - free(rthread_log); - fprintf(t->rthread_fd, "%ld: DEBUG: Created thread log\n", t->tid); -} -#endif - -void rthread_process_task(rthread_s* t, rtask_s *task, - pid_t pthread_id) -{ - char *next = task->line; - rworker_s *w = (rworker_s*) task->worker; - - // Tokenize the remaining elements of the line. - int ntoks = 0; - char *saveptr; - char *tok = strtok_r(next, "|", &saveptr); - - while (tok) { - if (ntoks > MAX_TOKENS) { - Error("worker(%d) pthread(%d): lineno:%lu, missing newline?", - w->rworker_num, pthread_id, task->lineno); - } - task->toks[ntoks++] = tok; - tok = strtok_r(NULL, "|", &saveptr); - } - // NULL marker (no more token from here) - task->toks[ntoks] = NULL; - -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "%ld(%ld): %s", - t->tid, (long)pthread_self(), task->clone); - fflush(t->rthread_fd); - free(task->clone); - task->clone = NULL; -#endif -#ifndef NO_RIOOP - // Perform the corresponding I/O operation! - rioop_run(task->process, t, task); -#endif - - // Make the task object recyclable/reusable - pthread_mutex_lock(&w->task_buffer_mutex); - if (!rbuffer_insert(w->task_buffer, task)) - // We can't recycle the task object if the buffer is full! - rtask_destroy(task); - pthread_mutex_unlock(&w->task_buffer_mutex); -} - -void *rthread_pthread_start(void *data) -{ - rthread_s* t = (rthread_s*) data; - rworker_s *w = t->worker; - rtask_s *task = NULL; - pid_t pthread_id = pthread_self(); - -#ifdef THREAD_DEBUG - _rthread_init_log(t); -#endif - - do { - while (!rbuffer_has_next(t->tasks) && !t->terminate) - usleep(100); - - while ((task = rbuffer_get_next(t->tasks)) != NULL) - rthread_process_task(t, task, pthread_id); - -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "%ld: DEBUG: Idling\n", t->tid); - fflush(t->rthread_fd); -#endif - - // Tell rworker_s that thread is not doing any work! - int inserted = false; - while (!inserted && !t->terminate) { - if (rbuffer_has_next(t->tasks)) - break; - - usleep(1000); - - if (rbuffer_has_next(t->tasks)) - break; - - // Make the rthread reusable, he is without any tasks - // for some time. - pthread_mutex_lock(&w->rthread_buffer_mutex); - inserted = rbuffer_insert(w->rthread_buffer, t); - pthread_mutex_unlock(&w->rthread_buffer_mutex); - } - -#ifdef THREAD_DEBUG - if (inserted) { - fprintf(t->rthread_fd, "%ld: DEBUG: Added to thread buffer\n", - t->tid); - } else { - fprintf(t->rthread_fd, "%ld: DEBUG: Idling thread recovered\n", - t->tid); - } - fflush(t->rthread_fd); -#endif - - } while (!t->terminate); - -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "%ld: DEBUG: Terminating\n", t->tid); - fflush(t->rthread_fd); -#endif - - // Process the very last tasks - while (NULL != (task = rbuffer_get_next(t->tasks))) - rthread_process_task(t, task, pthread_id); - -#ifdef THREAD_DEBUG - fprintf(t->rthread_fd, "%ld: DEBUG: Done terminating\n", t->tid); - fflush(t->rthread_fd); -#endif - - return NULL; -} - -rthread_s* rthread_new(const long tid, void *worker) -{ - rthread_s *t = Malloc(rthread_s); - rworker_s *w = worker; - - t->single_threaded = w->opts->num_threads_per_worker == 1; - t->tasks = rbuffer_new(TASK_BUFFER_PER_THREAD); - t->terminate = false; - t->worker = worker; - rthread_update(t, tid); - - if (t->single_threaded) { -#ifdef THREAD_DEBUG - _rthread_init_log(t); -#endif - return t; - } - - start_pthread(&t->pthread, rthread_pthread_start, (void*)t); - return t; -} - -long rthread_update(rthread_s *t, const long tid) -{ - long prev_tid = t->tid; - t->tid = tid; - - return prev_tid; -} - -void rthread_destroy(rthread_s *t) -{ - if (rbuffer_has_next(t->tasks)) { - Error("Didn't expect to have any tasks left!"); - } - rbuffer_destroy(t->tasks); - -#ifdef THREAD_DEBUG - if (t->rthread_fd) - fclose(t->rthread_fd); -#endif - - free(t); -} - -bool rthread_insert_task(rthread_s* t, rtask_s* task) -{ - if (t->single_threaded) { - rthread_process_task(t, task, pthread_self()); - return true; - } - return rbuffer_insert(t->tasks, task); -} - -void rthread_terminate(rthread_s* t) -{ - t->terminate = true; - pthread_join(t->pthread, NULL); -} diff --git a/ioreplay/src/replay/rthread.h b/ioreplay/src/replay/rthread.h deleted file mode 100644 index 9971e49..0000000 --- a/ioreplay/src/replay/rthread.h +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * @file rthread.h - * @author Paul Buetow - * - * @brief The replay thread definitiion - */ - -#ifndef RTHREAD_H -#define RTHREAD_H - -#include "../defaults.h" -#include "../datas/rbuffer.h" -#include "../datas/amap.h" -#include "../vfd.h" -#include "rtask.h" - -#include - -/** - * @brief Definition of a worker thread - * - * Every worker utilises a set of worker threads in order to parallelise the - * replaying of the I/O! Every thread comes with its own task queue. It is - * filled by the repsonsible worker. - * - * The user can specify the max amount of threads per worker per -t command - * line switch. - */ -typedef struct rthread_s_ { - void *worker; /**< The responsible worker object */ - long tid; /**< The virtual thread id */ - rbuffer_s* tasks; /**< Holds all outstanding tasks */ - bool terminate; /**< True if thread shall terminate */ - bool single_threaded; /**< Worker is single threaded or not */ - pthread_t pthread; /**< We run the tasks in concurrent pthreads */ -#ifdef RTHREAD_DEBUG - FILE *rthread_fd; /**< Used for debugging purposes only */ -#endif -} rthread_s; - -/** - * @brief Creates a new thread object - * - * @param tid The thread ID - * @param worker The worker object managing this thread - * @return The new thread object - */ -rthread_s* rthread_new(const long tid, void *worker); - -/** - * @brief Updates a thread object after recycling it - * - * @param t The thread object - * @param tid The new thread ID - */ -long rthread_update(rthread_s *t, const long tid); - -/** - * @brief Terminates the thread - * - * This function waits (via join) for the pthread to complete all its - * current tasks from the queue. - * - * @param t The thread object - */ -void rthread_terminate(rthread_s* t); - -/** - * @brief Destroys the thread object - * - * @param t The thread object - */ -void rthread_destroy(rthread_s* t); - -/** - * @brief Inserts a task into the threads work queue - * - * Inserts a task into the threads work queue. We use an atomic ring buffer - * data structure for the work queue. The ring buffer does not require any - * mutex locks. - * - * @param t The thread object - * @param task The task to be inserted - * @return Returns true on success, returns false if the task queue is full - */ -bool rthread_insert_task(rthread_s* t, rtask_s* task); - -/** - * @brief Used by the pthread to process a task - * - * In this function the pthread will attempt to process a task. It extracts all - * required information from the task object and invokes the corresponding I/O - * syscalls. - * - * @param t The responsible thread object - * @param task The task object - * @param pthread_id The current pthread id - */ -void rthread_process_task(rthread_s* t, rtask_s *task, pid_t pthread_id); - -/** - * @brief The entry function for the pthreads - * - * @param data The data structure passed to the pthread - * @return The exit code of the pthread. - */ -void *rthread_pthread_start(void *data); - -#endif // RTHREAD_H diff --git a/ioreplay/src/replay/rworker.c b/ioreplay/src/replay/rworker.c deleted file mode 100644 index 0e3fbe9..0000000 --- a/ioreplay/src/replay/rworker.c +++ /dev/null @@ -1,360 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "rworker.h" - -#include "../datas/stack.h" -#include "rprocess.h" -#include "rthread.h" - -#define _Compute_current_time(now) \ - (now.tv_sec - start_time.tv_sec) * 1000 \ - + (now.tv_usec - start_time.tv_usec) / 1000 - - -/** - * @brief A callback helper function for destroying all virtual process objects - * - * @param data The process object. - */ -static void _rprocess_destroy_cb(void *data) -{ - rprocess_destroy(data); -} - -rworker_s* rworker_new(const int rworker_num, amap_s *fds_map, - const long num_vsizes, const long num_pids, - options_s *opts, rworker_stats_s *worker_stats) -{ - rworker_s *w = Malloc(rworker_s); - -#ifdef THREAD_DEBUG - char *rworker_log = Calloc(1024, char); - snprintf(rworker_log, 1023, "/tmp/ioreplay/_worker%d.debuglog", - rworker_num); - - w->rworker_fd = Fopen(rworker_log, "a"); - free(rworker_log); - fprintf(w->rworker_fd, "DEBUG: Started worker\n"); -#endif - - w->rworker_num = rworker_num; - w->opts = opts; - w->fds_map = fds_map; - - w->rprocess_map = amap_new(num_pids); - w->rthread_map = amap_new(num_vsizes); - w->task_buffer = rbuffer_new(opts->num_threads_per_worker - *TASK_BUFFER_PER_THREAD); - w->rthread_buffer = rbuffer_new(opts->num_threads_per_worker); - w->worker_stats = worker_stats; - - // Attach a cleanup callback function to the worker map. - w->rprocess_map->data_destroy = _rprocess_destroy_cb; - - pthread_mutex_init(&w->rthread_buffer_mutex, NULL); - pthread_mutex_init(&w->task_buffer_mutex, NULL); - - // TODO: Check in the program whether the ulimit is high enough - // or not! (ulimit -n) - - return w; -} - -/** - * @brief Destroys the object - * - * Destroys the worker object (frees all memory allocated by the worker) - * - * @param w The worker object - */ -void rworker_destroy(rworker_s *w) -{ - if (!w) - return; - - if (w->rprocess_map) - amap_destroy(w->rprocess_map); - if (w->rthread_map) - amap_destroy(w->rthread_map); - - if (w->task_buffer) { - rtask_s *task = NULL; - while (NULL != (task = rbuffer_get_next(w->task_buffer))) - rtask_destroy(task); - rbuffer_destroy(w->task_buffer); - } - - if (w->rthread_buffer) - rbuffer_destroy(w->rthread_buffer); - - pthread_mutex_destroy(&w->task_buffer_mutex); - pthread_mutex_destroy(&w->rthread_buffer_mutex); - -#ifdef THREAD_DEBUG - if (w->rworker_fd) - fclose(w->rworker_fd); -#endif - - free(w); -} - -status_e rworker_process_lines(rworker_s* w, const long num_lines) -{ - Out("worker(%d): Starting to process replay lines\n", w->rworker_num); - - options_s *opts = w->opts; - FILE *replay_fd = Fopen(opts->replay_file, "r"); - - // Drop root privileges, otherwise we may overwrite other system - // files by accident in case of a bug or user error! - set_limits_drop_root(opts->user); - - // Variables required for the time based caluclations - struct timeval now, start_time; - long current_time = 0, stats_time = 0; - gettimeofday(&start_time, NULL); - - // Helper variables required for reading lines - char *line = NULL; - char *next = NULL, *next2 = NULL; - size_t len = 0, read = 0; - - // Helpers required for threading - rthread_s *t = NULL; - stack_s *all_threads = stack_new(); - rworker_stats_s *s = w->worker_stats; - - // More helper variables - //unsigned long lineno = 0, stats_ioop = 0, vsize_id = 0; - unsigned long lineno = 0, vsize_id = 0; - long pid = -1, time = -1; - - // Process the .replay file line by line. - while ((read = getline(&line, &len, replay_fd)) != -1) { - lineno++; - - if (read >= MAX_LINE_LEN) { - Error("line:%lu Exceeded max line len", lineno); - } - - // If the line begins with #: Ignore that line, it contains - // debug or meta information or comments. - - if (line[0] == '#') { - if (line[1] == 'I') { - // We stop replaying I/O once we reach the line '#INIT' - // which incitates the begin of the INIT section. - break; - } - continue; - } - -#ifdef THREAD_DEBUG - char *clone = Clone(line); -#endif - - next = strchr(line, '|'); - Error_if(!next, "lineno:%ld Could not parse time from input file", - lineno); - next[0] = '\0'; - next++; - time = atol(line); - - next2 = strchr(next, '|'); - Error_if(!next2, "Could not parse vsize_id from input file"); - next2[0] = '\0'; - next2++; - vsize_id = atol(next); - - // This worker is not responsible for this line, skip it! - if ((vsize_id % opts->num_workers) != w->rworker_num) { -#ifdef THREAD_DEBUG - free(clone); -#endif - continue; - } - - next = strchr(next2, '|'); - Error_if(!next, "Could not parse PID from input file"); - next[0] = '\0'; - next++; - pid = atol(next2); - - gettimeofday(&now, NULL); - current_time = _Compute_current_time(now); - - // Check whether the user specified a replay speed factor. If so, we - // may need to throttle down a bit. - - if (opts->speed_factor) { - s->time_ahead = time / opts->speed_factor - current_time; - if (s->time_ahead > 0) - usleep(s->time_ahead*1000); - - } else { - s->time_ahead = time - current_time; - } - - // Get the responsible process object. The process object holds data - // structures usually found in a Linux process, e.g. a table of open - // file descriptors. - - rprocess_s *p = amap_get(w->rprocess_map, pid); - if (p == NULL) { - p = rprocess_new(pid, w->fds_map); - amap_set(w->rprocess_map, pid, p); - } - p->lineno = lineno; - - if (opts->num_threads_per_worker == 1) { - // Single threaded mode? - if (!t) - t = rthread_new(vsize_id, w); - else - rthread_update(t, vsize_id); - - } else { - t = amap_get(w->rthread_map, vsize_id); - } - - if (t == NULL) { - - // First try to recycle an old (likely unused) thread - if (NULL != (t = rbuffer_get_next(w->rthread_buffer))) { - rthread_update(t, vsize_id); - -#ifdef THREAD_DEBUG - fprintf(w->rworker_fd, "DEBUG: Reused an idling thread\n"); - fflush(w->rworker_fd); -#endif - - } else if (opts->num_threads_per_worker <= all_threads->size) { - // Reached max threads, waiting until one becomes available - -#ifdef THREAD_DEBUG - fprintf(w->rworker_fd, "DEBUG: Reached max threads\n"); - fflush(w->rworker_fd); -#endif - while (NULL == (t = rbuffer_get_next(w->rthread_buffer))) - usleep(1000); - -#ifdef THREAD_DEBUG - fprintf(w->rworker_fd, "DEBUG: Reused an idling thread\n"); - fflush(w->rworker_fd); -#endif - - rthread_update(t, vsize_id); - - } else { - t = rthread_new(vsize_id, w); - - // We hold a pointer to all created threads in a stack. This - // stack is later used to terminate/join all therads. - stack_push(all_threads, t); - -#ifdef THREAD_DEBUG - fprintf(w->rworker_fd, "DEBUG: Created a new thread\n"); - fflush(w->rworker_fd); -#endif - } - - amap_set(w->rthread_map, vsize_id, t); - } - - // Create a new task for the thread. The task contains all required - // information to run an I/O operation. However, first try to - // reuse/recycle a task object! If there is no such, create a new one. - - rtask_s *task = rbuffer_get_next(w->task_buffer); - if (!task) - task = rtask_new(); - rtask_update(task, w, p, next, lineno, vsize_id); - s->ioops++; - - -#ifdef THREAD_DEBUG - task->clone = clone; - fprintf(w->rworker_fd, "DEBUG: Inserting new task\n"); - fflush(w->rworker_fd); -#endif - - // Insert that task to a ring buffer to pass it to the pthread without - // much synchronisation overhead! - - while (!rthread_insert_task(t, task)) - // The ring buffer is full. This may happen if the pthread didn't - // manage to process tasks fast enough. re-try after a short period! - usleep(1000); - -#ifdef THREAD_DEBUG - fprintf(w->rworker_fd, "DEBUG: Task inserted\n"); - fflush(w->rworker_fd); -#endif - - // The worker prints out stats every 3s - if (current_time - stats_time > 3000) { - // IDEA: Maybe refactor this block to be implemented in rstats.c - - double loadavg = get_loadavg(); - - // Determines whether we replay the I/O faster or slower than - // original speed! - char *a_b = s->time_ahead >= 0 ? "ahead" : "behind"; - - Put("worker(%d): threads:%ld %s:%lds progress:%0.2f%% " - "loadavg:%0.2f", - w->rworker_num, all_threads->size, a_b, Abs(s->time_ahead/1000), - Perc(lineno,num_lines), loadavg); - - stats_time = current_time; - //stats_ioop = lineno; - - if (s->loadavg_high < loadavg) - s->loadavg_high = loadavg; - } - } - - Put("worker(%d): Waiting for all threads to finish business...", - w->rworker_num); - - // This will wait (join) all threads one after another until all threads - // have finished their work and have terminated. - - while (!stack_is_empty(all_threads)) { - rthread_s *t = stack_pop(all_threads); - rthread_terminate(t); - rthread_destroy(t); - } - stack_destroy(all_threads); - - // Collect some stats last time - double loadavg = get_loadavg(); - if (s->loadavg_high < loadavg) - s->loadavg_high = loadavg; - - gettimeofday(&now, NULL); - current_time = _Compute_current_time(now); - if (opts->speed_factor) { - s->time_ahead = time / opts->speed_factor - current_time; - } else { - s->time_ahead = time - current_time; - } - - - Put("worker(%d): All threads terminated!", w->rworker_num); - fclose(replay_fd); - - return SUCCESS; -} diff --git a/ioreplay/src/replay/rworker.h b/ioreplay/src/replay/rworker.h deleted file mode 100644 index 26a1300..0000000 --- a/ioreplay/src/replay/rworker.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef RWORKER_H -#define RWORKER_H - -#include - -#include "../datas/amap.h" -#include "../datas/rbuffer.h" -#include "../defaults.h" -#include "../options.h" -#include "rstats.h" - -/** - * @brief Represents a worker process. - * - * This represents an I/O replay worker process. The user can specify the - * amount of worker processes via the -p command line switch. This is not - * to confuse with rprocess_s, which represents an original captured process - * and we now want to replay the I/O for! - */ -typedef struct { - int rworker_num; /**< The current worker ID */ - amap_s* fds_map; /**< Holding all file descriptors */ - amap_s* rprocess_map; /**< Holding all processes handled by this worker */ - amap_s* rthread_map; /**< Holding all threads handled by this worker */ - rbuffer_s *task_buffer; /**< Buffering thread tasks to be reused */ - pthread_mutex_t task_buffer_mutex; /**< To sync access to task_buffer */ - rbuffer_s *rthread_buffer; /**< Buffering idle threads to be reused */ - pthread_mutex_t rthread_buffer_mutex; /**< Sync access to rthread_buffer */ - options_s *opts; /**< To synchronise access to rthread_buffer */ - rworker_stats_s *worker_stats; /**< Object holding per worker statistics */ -#ifdef RTHREAD_DEBUG - FILE *rworker_fd; /**< For debugging purposes only */ -#endif -} rworker_s; - -/** - * @brief Creates a new worker object - * - * @param rworker_num The worker number - * @param fds_map A map of all virtual file descriptor objects - * @param num_vsizes The amount of virtual sizes/total file paths of the test - * @param num_pids The total amount of virtual process IDs used in this test - * @param opts A pointer to the options object - * @param worker_stats A pointer to the worker stats object - - * @return The new worker object - */ -rworker_s* rworker_new(const int rworker_num, amap_s *fds_map, - const long num_vsizes, const long num_pids, - options_s* opts, rworker_stats_s *worker_stats); - -/** - * @brief Destroys a worker object - * - * @param w The worker object to be destroyed - */ -void rworker_destroy(rworker_s* w); - -/** - * @brief Makes the worker to process all .replay lines - * - * @param w The responsible worker object - * @param num_lines The total amount of I/O op lines in the .replay file - * @return SUCCESS if everything went fine - */ -status_e rworker_process_lines(rworker_s* w, const long num_lines); - -#endif // RWORKER_H diff --git a/ioreplay/src/utests.c b/ioreplay/src/utests.c deleted file mode 100644 index 7cc4959..0000000 --- a/ioreplay/src/utests.c +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "utests.h" - -#include "datas/amap.h" -#include "datas/hmap.h" -#include "datas/list.h" -#include "datas/rbuffer.h" -#include "utils/utils.h" - -void utests_run() -{ - fprintf(stderr, "Running utils_test()\n"); - utils_test(); - - fprintf(stderr, "Running amap_test()\n"); - amap_test(); - - fprintf(stderr, "Running hmap_test()\n"); - hmap_test(); - - fprintf(stderr, "Running list_test()\n"); - list_test(); - - fprintf(stderr, "Running rbuffer_test()\n"); - rbuffer_test(); - - fprintf(stderr, "Great success, run all unit tests without any errors!\n"); -} diff --git a/ioreplay/src/utests.h b/ioreplay/src/utests.h deleted file mode 100644 index 4ad6973..0000000 --- a/ioreplay/src/utests.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef UTESTS_H -#define UTESTS_H - -#include "utils/utils.h" - -/** - * @brief This function runs all currently implemented unit tests - */ -void utests_run(); - -#endif // UTESTS_H diff --git a/ioreplay/src/utils/futils.c b/ioreplay/src/utils/futils.c deleted file mode 100644 index 5b35618..0000000 --- a/ioreplay/src/utils/futils.c +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "futils.h" - -#include -#include -#include -#include - -#include "../macros.h" - -void append_random_to_file(char *path, unsigned long bytes) -{ - char *buf = NULL; - int max_chunk = 50000000; // 50 mebibyetes - FILE *fp = Fopen(path, "a"); - - for (;;) { - if (bytes > max_chunk) { - if (!buf) - buf = Calloc(max_chunk+1, char); - - Fill_with_stuff(buf, max_chunk); - buf[max_chunk] = '\0'; - fprintf(fp, "%s", buf); - bytes -= max_chunk; - - // Print out a dot every time we wrote 'much' data to a file - Out("."); - - } else { - if (!buf) - buf = Calloc(bytes+1, char); - - Fill_with_stuff(buf, bytes); - buf[bytes] = '\0'; - fprintf(fp, "%s", buf); - - break; - } - } - - if (buf) - free(buf); - fclose(fp); -} - -long ensure_dir_exists(const char *path) -{ - long num_dirs_created = 0; - int ret = mkdir_p(path, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, - &num_dirs_created); - if (ret != 0) { - Errno("Could not create dir '%s'", path); - } - - return num_dirs_created; -} - -void ensure_parent_dir_exists(const char *path) -{ - char *clone = Clone(path); - char *parent = dirname(clone); - - int ret = mkdir_p(parent, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, NULL); - if (ret != 0) { - Errno("Could not create dir %s", parent); - } - - free(clone); -} - -void ensure_dir_empty(const char *path) -{ - DIR *dh = opendir(path); - - if (!dh) { - Errno("Unable to empty %s", path); - } - - struct dirent *de; - - while ((de = readdir(dh))) { - if (0 == strcmp(de->d_name, ".") || - 0 == strcmp(de->d_name, "..")) - continue; - - char *absolute; - asprintf(&absolute, "%s/%s", path, de->d_name); - - if (is_dir(absolute)) - ensure_dir_empty(absolute); - - if (remove(absolute) == -1) - // Don't throw an error if there is no such file or directory - if (errno != 2) { - Errno("Unable to remove %s", absolute); - } - - free(absolute); - } - - closedir(dh); -} - -int ensure_file_exists(char *path, long *num_dirs_created) -{ - if (is_reg(path)) - return SUCCESS; - - char *dirname = dirname_r(Clone(path)); - *num_dirs_created += ensure_dir_exists(dirname); - free(dirname); - - FILE *fp = fopen(path, "a"); - if (fp) { - // We only need some data, less than 1 block in size, this is answer: - fprintf(fp, "42"); - fclose(fp); - return SUCCESS; - } - - return ERROR; -} - -char* dirname_r(char *path) -{ - int len = strlen(path); - int has = 0; - int i = len-1; - - if (strcmp(path, "..") == 0) { - return path; - } - - if (path[i] == '/') { - // Root directory - if (len == 1) - return path; - - // Remove all trailing / - for (; i >= 0; --i) { - if (path[i] == '/') { - path[i] = '\0'; - has = 1; - } else { - break; - } - } - } - - // Find next / - for (; i >= 0; --i) { - if (path[i] == '/') { - path[i] = '\0'; - has = 1; - break; - } - } - - // If no / - if (has == 0) { - path[0] = '.'; - path[1] = '\0'; - } - - return path; -} - -bool is_dir(const char *path) -{ - struct stat path_stat; - if (stat(path, &path_stat) == 0 && S_ISDIR(path_stat.st_mode)) - return true; - return false; -} - -bool is_reg(const char *path) -{ - struct stat path_stat; - if (stat(path, &path_stat) == 0 && S_ISREG(path_stat.st_mode)) - return true; - return false; -} - -bool exists(const char *path) -{ - struct stat path_stat; - if (stat(path, &path_stat) == 0) - return true; - return false; -} - -int mkdir_p(const char *path, mode_t mode, long *num_dirs_created) -{ - int res = 0; - - if (is_dir(path)) - return 0; - - if (is_reg(path)) - unlink(path); - - char *top = dirname_r(Clone(path)); - if (0 != mkdir_p(top, mode, num_dirs_created)) - goto cleanup; - - if ((mkdir(path, mode) == -1) && (errno != EEXIST)) - res = -1; - - if (res != -1) - *num_dirs_created = *num_dirs_created+1; - -cleanup: - free(top); - - return res; -} - -void cache_file(const char *file) -{ - Out("Caching file %s... it can take a while", file); - FILE *fd = Fopen(file, "r"); - char *line = NULL; - size_t len = 0, read = 0; - - while ((read = getline(&line, &len, fd)) != -1); - fclose(fd); -} - -void drop_caches(void) -{ - Out("Dropping all Linux caches..."); - - if (getuid() != 0) { - Out("\n"); - Error("I need to be root to do this, aborting!"); - } - - // echo 3 > /proc/sys/vm/drop_caches - char *drop_caches = "/proc/sys/vm/drop_caches"; - FILE *fd = Fopen(drop_caches, "w"); - fprintf(fd, "3"); - fclose(fd); - - Put("done"); -} - -void chown_path(const char *user, const char *path) -{ - struct passwd *pwd = getpwnam(user); - if (!pwd) { - Errno("Unable to retrieve information about system user %s!", user); - } - - if (chown(path, pwd->pw_uid, -1) == -1) { - Errno("Could not change ownership of '%s' to '%s'!", path, user); - } -} - -char *absolute_path(const char *path) -{ - if (path[0] == '/') - return Clone(path); - - char cwd[MAX_LINE_LEN]; - getcwd(cwd, sizeof(char)*MAX_LINE_LEN); - - if (!getcwd(cwd, sizeof(cwd))) { - Errno("Could not get current working directory"); - } - - char *absolute = NULL; - if (-1 == asprintf(&absolute, "%s/%s", cwd, path)) { - Error("Could not get absolute path of '%s'", path); - } - - return absolute; -} diff --git a/ioreplay/src/utils/futils.h b/ioreplay/src/utils/futils.h deleted file mode 100644 index 9afde1a..0000000 --- a/ioreplay/src/utils/futils.h +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef FUTILS_H -#define FUTILS_H - -#include "../defaults.h" - -/** - * @brief Thread safe version of dirname() - * - * @param path The full file path - * @return The directory path - */ -char* dirname_r(char *path); - -/** - * @brief Ensures that a file exists - * - * @param path The file path - * @param num_dirs_created Holds a count of how many sub dirs have been created - * @return -1 on error, 0 on success. - */ -int ensure_file_exists(char *path, long *num_dirs_created); - -/** - * @brief Checks whether path exists - * - * @param path The path - * @return true if the path exists - */ -bool exists(const char *path); - -/** - * @brief Check if path is a directory - * - * @param path The directory path - * @return true if the path is a directory, false otherwise - */ -bool is_dir(const char *path); - -/** - * @brief Check if path is a regular file - * - * @param path The file path - * @return true if the file at path is a regular fike - */ -bool is_reg(const char *path); - -/** - * @brief Create a directory recursively - * - * @param path The directory path - * @param mode The mode - * @param num_dirs_created Counts how many directories have been created - * @return -1 on error - */ -int mkdir_p(const char *path, mode_t mode, long *num_dirs_created); - -/** - * @brief Appends data to a file - * - * @param path The file path - * @param bytes The amount of bytes - */ -void append_random_to_file(char *path, unsigned long bytes); - -/** - * @brief Ensures that a directory exists - * - * @param path The directory path - * @return The amount of directories created (including parent directories) - */ -long ensure_dir_exists(const char *path); - -/** - * @brief Ensures that a parent directory exists - * - * @param path The directory path - */ -void ensure_parent_dir_exists(const char *path); - -/** - * @brief Ensures that a directory is empty - * - * @param path The directory path - */ -void ensure_dir_empty(const char *path); - -/** - * @brief Loading a file into the file system cache - * - * @param file The path to the file - */ -void cache_file(const char *file); - -/** - * @brief Drop all Linux caches - * - * This function drops all Linux caches, which includes all file - * system caches. - */ -void drop_caches(void); - -/** - * @brief Changes owner of a path - * - * Terminates the process with an error message if failed. - * - * @param user The new owner - * @param path The path - */ -void chown_path(const char *user, const char *path); - -/** - * @brief Retrieves the absolute path of a given path - * - * @param path The path - * @return The absolute path. It must be freed manually. - */ -char *absolute_path(const char *path); - -#endif // FUTILS_H diff --git a/ioreplay/src/utils/utils.c b/ioreplay/src/utils/utils.c deleted file mode 100644 index 4b41273..0000000 --- a/ioreplay/src/utils/utils.c +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "utils.h" - -#include -#include - -void* notnull(void *p, char *file, int line, int count) -{ - if (p == NULL) { - Errno("%s:%d count:%d Could not allocate memory", file, line, count); - } - return p; -} - - -FILE* fnotnull(FILE *fd, const char *path, char *file, int line) -{ - if (fd == NULL) { - Errno("%s:%d Could not open file '%s'", file, line, path); - } - return fd; -} - -void* mmapok(void *p, char *file, int line) -{ - if (p == MAP_FAILED) { - Errno("%s:%d: Mmap failed", file, line); - } - return p; -} - -char* strtok2_r(char *str, char *delim, char **saveptr) -{ - int len = strlen(delim); - - if (str == NULL) - str = *saveptr; - - char *next = strstr(str, delim); - if (next) { - next[0] = '\0'; - for (int i = 0; i < len; ++i) - next++; - *saveptr = next; - return str; - } - - return NULL; -} - -void chreplace(char *str, char replace, char with) -{ - for (int i = 0; ; ++i) { - if (str[i] == '\0') - break; - if (str[i] == replace) - str[i] = with; - } -} - -void strunquote(char *str) -{ - int len = strlen(str); - - if (str[0] == '"') { - if (str[len-1] == '"') - str[len-1] = '\0'; - for (int i = 1; i < len; ++i) - str[i-1] = str[i]; - } -} - -void set_limits_drop_root(const char *user) -{ - if (getuid() == 0) { - struct rlimit rl; - rl.rlim_cur = rl.rlim_max = SET_RLIMIT_NOFILE; - if (0 != setrlimit(RLIMIT_NOFILE, &rl)) { - Errno("Could not set RLIMIT_NOFILE to '%lld'!", - (long long) SET_RLIMIT_NOFILE) - } - rl.rlim_cur = rl.rlim_max = SET_RLIMIT_NPROC; - if (0 != setrlimit(RLIMIT_NPROC, &rl)) { - Errno("Could not set RLIMIT_NPROC to '%lld'!", - (long long) SET_RLIMIT_NPROC) - } - - Put("Dropping root privileges to user '%s'", user); - struct passwd *pw = getpwnam(user); - - /* process is running as root, drop privileges */ - if (setgid(pw->pw_gid) != 0) { - Errno("Unable to drop group privileges!"); - } - if (setuid(pw->pw_uid) != 0) { - Errno("Unable to drop user privileges!"); - } - } - - /* - getrlimit(RLIMIT_NOFILE, &rl); - Put("Max open files: '%lld'", (long long) rl.rlim_cur); - getrlimit(RLIMIT_NPROC, &rl); - Put("Max open processes : '%lld'", (long long) rl.rlim_cur); - */ -} - -void get_loadavg_s(char *readbuf) -{ - FILE *fp = Fopen("/proc/loadavg", "r"); - fgets(readbuf, 128, fp); - char *pos = strchr(readbuf, ' '); - pos[0] = '\0'; - fclose(fp); -} - -double get_loadavg() -{ - // Not thread safe, but multi processing safe - static char buf[128]; - get_loadavg_s(buf); - - return atof(buf); -} - -bool is_number(char *str) -{ - for (int i = 0; ; ++i) { - if (str[i] == '\0') - return true; - if (isdigit(str[i]) == 0 && str[i] != '-') - return false; - } - - return true; -} - -void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data) -{ - int rc = pthread_create(thread, NULL, cb, data); - - switch (rc) { - case 0: - break; - case EAGAIN: - Error("Out of resources while creating pthread (%d)", rc); - break; - case EINVAL: - Error("Ivalid settings while creating pthread (%d)", rc); - break; - case EPERM: - Error("No permissions to configure pthread (%d)", rc); - default: - Error("Unknown error while creating pthread (%d)", rc); - break; - } -} - -void utils_test(void) -{ - if (getuid() == 0) { - set_limits_drop_root("nobody"); - struct rlimit rl; - - getrlimit(RLIMIT_NOFILE, &rl); - assert(rl.rlim_cur == SET_RLIMIT_NOFILE); - assert(rl.rlim_max == SET_RLIMIT_NOFILE); - - getrlimit(RLIMIT_NPROC, &rl); - assert(rl.rlim_cur == SET_RLIMIT_NPROC); - assert(rl.rlim_max == SET_RLIMIT_NPROC); - } -} diff --git a/ioreplay/src/utils/utils.h b/ioreplay/src/utils/utils.h deleted file mode 100644 index 3e86865..0000000 --- a/ioreplay/src/utils/utils.h +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef UTILS_H -#define UTILS_H - -// For asprintf in stdio.h -#define _GNU_SOURCE - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../macros.h" -#include "../defaults.h" - -/** - * @brief Check whether allocated memory is not NULL - * - * This function is used in conjunction with malloc() and co. It - * introduces an extra sanity check whether the memory could be - * allocated successfully or not. If not it will print out a error - * message stating the position in the source code where the memory - * allocation failed! - * - * @param p The pointer being checked - * @param file The source file name the memory was allocated in - * @param line The source line number the memory was allocated at - * @param count The amount of memory being allocated - * @return The pointer to the allocated memory - */ -void* notnull(void *p, char *file, int line, int count); - -/** - * @brief Check whether opened file handle is not NULL - * - * This function is used in conjunction with fopen(). It - * introduces an extra sanity check whether the file could be - * opened successfully or not. If not it will print out a error - * message stating the position in the source code where the open - * failed! - * - * @param fd The fd stream to be checked. - * @param path The file path opened - * @param file The source file name - * @param line The source line number - * @return The pointer to the allocated memory - */ -FILE* fnotnull(FILE *fd, const char *path, char *file, int line); - -/** - * @brief Check whether allocated memory via mmap is not null - * - * This function is used in conjunction with mmap() and co. It - * introduces an extra sanity check whether the memory could be - * allocated successfully or not. If not it will print out a error - * message stating the position in the source code where the memory - * allocation failed! - * - * @param addr The pointer being checked - * @param file The source file name the memory was allocated in - * @param line The source line number the memory was allocated at - * @return The pointer to the allocated memory - */ -void* mmapok(void *addr, char *file, int line); - -/** - * @brief A version of strtok_r supporting multi char delims - * - * @param str The input string - * @param delim The multi-char delimiter - * @param saveptr A temp storage location - * @return The next match if != NULL - */ -char* strtok2_r(char *str, char *delim, char **saveptr); - -/** - * @brief Replaces a character with another one in a string - * - * @param str The input string - * @param replace The character to be replaced - * @param with The character to replace with - */ -void chreplace(char *str, char replace, char with); - -/** - * @brief Removes quotes from a string - * - * @param str The input sting - */ -void strunquote(char *str); - -/** - * @brief Set rlimits and drop root privileges - * - * This function firsts sets the user resource limits to SET_RLIMIT_NOFILE and - * SET_RLIMIT_NPROC and then attempts to drop the root user to the specified - * one. - * - * @param user The user to switch to - */ -void set_limits_drop_root(const char *user); - -/** - * @brief Retrieve current 1 min Linux load average - * - * @param readbuf The buffer to store the load average as a string - */ -void get_loadavg_s(char *readbuf); - -/** - * @brief Retrieve current 1 min Linux load average - * - * This function is not thread safe! - * - * @return The 1 minute load average of the system - */ -double get_loadavg(); - -/** - * @brief Check whether a string represents a number - * - * @param str The input string - * @return true if all characters of the input string are a digits - */ -bool is_number(char *str); - -/** - * @brief Wrapper around pthread_create - * - * The wrapper also checks whether the thread has been created successfully - * or not! It will exit the process if not. - * - * @param thread The POSIX thread variable - * @param cb The threadss start callback routine - * @param data A data pointer passed to the thread. - */ -void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data); - -/** - * @brief Testing various of the utilities - */ -void utils_test(void); - -#endif // UTILS_H diff --git a/ioreplay/src/vfd.c b/ioreplay/src/vfd.c deleted file mode 100644 index 6e86f61..0000000 --- a/ioreplay/src/vfd.c +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "vfd.h" - -vfd_s* vfd_new(const int fd, const long mapped_fd, char *path) -{ - vfd_s *vfd = Malloc(vfd_s); - vfd->path = NULL; - vfd->debug = false; - vfd_update(vfd, fd, mapped_fd, path); - - return vfd; -} - -void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path) -{ - vfd->fd = fd; - vfd->dirfd = NULL; - vfd->mapped_fd = mapped_fd; - vfd->offset = 0; - - if (path) - free(vfd->path); - - vfd->path = Clone(path); -} - -void vfd_destroy(vfd_s *vfd) -{ - if (!vfd) - return; - - if (vfd->path) - free(vfd->path); - - free(vfd); -} - -void vfd_print(vfd_s *vfd) -{ - fprintf(stderr, "virtfd(%p) fd:%x mapped_fd:%lx path:%s\n", - (void*)vfd, vfd->fd, vfd->mapped_fd, vfd->path); -} diff --git a/ioreplay/src/vfd.h b/ioreplay/src/vfd.h deleted file mode 100644 index fd0c4fb..0000000 --- a/ioreplay/src/vfd.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2018 Mimecast Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef VFD_H -#define VFD_H - -#include "opcodes.h" -#include "defaults.h" - -/** - * @brief The virtual file descriptor definition - * - * A virtual file descriptor represents a file descriptor from ioreplay's - * point of view and is being used in various ways to simulate the real I/O - * protocolled to the .capture/.replay files. - * - * Generally speaking I/O replay maps the real FD numbers (the ones logged to - * the .capture file) to virtual FD numbers (a uniqe FD number for every open - * to increase concurrency). - */ -typedef struct vfd_s_ { - int fd; /**< the real fd */ - DIR *dirfd; /**< The real dirfd */ - long mapped_fd; /**< The mapped fd (virtual fd) */ - char *path; /**< The file path belonging to that fd */ - bool free_path; /**< True if path has to be freed or not */ - unsigned long offset; /**< The current virtual file offset in bytes */ - int debug; /**< Used for debugging purposes only */ -} vfd_s; - -/** - * @brief Creates a new virtual file descriptor object - * - * @param fd The file descriptor - * @param mapped_fd The mapped file descriptor - * @param path The path name - * @return The new fd object - */ -vfd_s* vfd_new(const int fd, const long mapped_fd, char *path); - -/** - * @brief Updates the virtfd object - * - * @param vfd The virtfd object - * @param fd The (real) file descriptor - * @param mapped_fd The mapped (virtual) file descriptor - * @param path The path name - * @return The new fd object - */ -void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path); - -/** - * @brief Destroys a file descriptor object - * - * @param vfd The file descriptor object - */ -void vfd_destroy(vfd_s *vfd); - -/** - * @brief Prints the virtual file descriptor - * @param vfd The virtual file descriptor - */ -void vfd_print(vfd_s *vfd); - -#endif // VFD_H - diff --git a/ioreplay/tags b/ioreplay/tags deleted file mode 100644 index cbad51a..0000000 --- a/ioreplay/tags +++ /dev/null @@ -1,661 +0,0 @@ -!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ -!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ -!_TAG_PROGRAM_AUTHOR Darren Hiebert /dhiebert@users.sourceforge.net/ -!_TAG_PROGRAM_NAME Exuberant Ctags // -!_TAG_PROGRAM_URL http://ctags.sourceforge.net /official site/ -!_TAG_PROGRAM_VERSION 5.8 // -AMAP_H ./src/datas/amap.h 2;" d -AMAP_MAX_ARRAY_LENGTH ./src/datas/amap.h 6;" d -Abs ./src/macros.h 13;" d -BTREE_H ./src/datas/btree.h 2;" d -CAPTURE_VERSION ./src/defaults.h 7;" d -CHMOD ./src/opcodes.h /^ CHMOD = 100,$/;" e enum:__anon1 -CHOWN ./src/opcodes.h /^ CHOWN = 110,$/;" e enum:__anon1 -CHOWN16 ./src/opcodes.h /^ CHOWN16,$/;" e enum:__anon1 -CLEANUP_H ./src/cleanup/cleanup.h 2;" d -CLOSE ./src/opcodes.h /^ CLOSE = 50,$/;" e enum:__anon1 -CREAT ./src/opcodes.h /^ CREAT,$/;" e enum:__anon1 -Calloc ./src/macros.h 23;" d -Cleanup ./src/macros.h 4;" d -Cleanup_unless ./src/macros.h 5;" d -Clone ./src/macros.h 9;" d -Cmapshared ./src/macros.h 36;" d -DEFAULTS_H ./src/defaults.h 2;" d -Debug ./src/macros.h 51;" d -ERROR ./src/defaults.h /^ ERROR, \/**< An error happened :-( *\/$/;" e enum:status_e_ -Eq ./src/macros.h 10;" d -Errno ./src/macros.h 68;" d -Errno_if ./src/macros.h 77;" d -Error ./src/macros.h 58;" d -Error_if ./src/macros.h 66;" d -F ./src/generate/gtask.h /^ int F; \/**< Arguments for fcntl syscall *\/$/;" m struct:gtask_s_ -FCHMOD ./src/opcodes.h /^ FCHMOD,$/;" e enum:__anon1 -FCHMODAT ./src/opcodes.h /^ FCHMODAT,$/;" e enum:__anon1 -FCHOWN ./src/opcodes.h /^ FCHOWN,$/;" e enum:__anon1 -FCHOWN16 ./src/opcodes.h /^ FCHOWN16,$/;" e enum:__anon1 -FCHOWNAT ./src/opcodes.h /^ FCHOWNAT,$/;" e enum:__anon1 -FCNTL ./src/opcodes.h /^ FCNTL = 70,$/;" e enum:__anon1 -FDATASYNC ./src/opcodes.h /^ FDATASYNC,$/;" e enum:__anon1 -FSTAT ./src/opcodes.h /^ FSTAT = 0,$/;" e enum:__anon1 -FSTATFS ./src/opcodes.h /^ FSTATFS,$/;" e enum:__anon1 -FSTATFS64 ./src/opcodes.h /^ FSTATFS64,$/;" e enum:__anon1 -FSTAT_AT ./src/opcodes.h /^ FSTAT_AT,$/;" e enum:__anon1 -FSYNC ./src/opcodes.h /^ FSYNC = 60,$/;" e enum:__anon1 -FUTILS_H ./src/utils/futils.h 2;" d -Fill_with_stuff ./src/macros.h 98;" d -Fopen ./src/macros.h 29;" d -G ./src/generate/gtask.h /^ int G; \/**< Arguments for fcntl syscall *\/$/;" m struct:gtask_s_ -GENERATE_H ./src/generate/generate.h 2;" d -GETDENTS ./src/opcodes.h /^ GETDENTS,$/;" e enum:__anon1 -GIOOP_H ./src/generate/gioop.h 2;" d -GPARSER_H ./src/generate/gparser.h 2;" d -GPROCESS_H ./src/generate/gprocess.h 2;" d -GTASK_H ./src/generate/gtask.h 2;" d -GWRITER_H ./src/generate/gwriter.h 2;" d -Gioop_write ./src/generate/gioop.h 12;" d -HMAP_H ./src/datas/hmap.h 2;" d -Has ./src/macros.h 17;" d -Hasnt ./src/macros.h 18;" d -INIT_H ./src/init/init.h 2;" d -IOREPLAY_COPYRIGHT ./src/defaults.h 19;" d -IOREPLAY_VERSION ./src/defaults.h 17;" d -ITASK_H ./src/init/itask.h 2;" d -ITHREAD_H ./src/init/ithread.h 2;" d -LCHOWN ./src/opcodes.h /^ LCHOWN,$/;" e enum:__anon1 -LCOWN16 ./src/opcodes.h /^ LCOWN16,$/;" e enum:__anon1 -LIST_H ./src/datas/list.h 2;" d -LSEEK ./src/opcodes.h /^ LSEEK,$/;" e enum:__anon1 -LSTAT ./src/opcodes.h /^ LSTAT,$/;" e enum:__anon1 -MACROS_H ./src/macros.h 2;" d -MAX_LINE_LEN ./src/defaults.h 13;" d -MAX_MOUNTPOINTS ./src/mounts.h 8;" d -MAX_TOKENS ./src/defaults.h 11;" d -META_EXIT ./src/opcodes.h /^ META_EXIT = 900,$/;" e enum:__anon1 -META_EXIT_GROUP ./src/opcodes.h /^ META_EXIT_GROUP,$/;" e enum:__anon1 -META_H ./src/meta/meta.h 2;" d -META_TIMELINE ./src/opcodes.h /^ META_TIMELINE$/;" e enum:__anon1 -MKDIR ./src/opcodes.h /^ MKDIR,$/;" e enum:__anon1 -MKDIR_AT ./src/opcodes.h /^ MKDIR_AT,$/;" e enum:__anon1 -MMAP2 ./src/opcodes.h /^ MMAP2 = 80,$/;" e enum:__anon1 -MOUNTPOINTS_H ./src/mounts.h 2;" d -MSYNC ./src/opcodes.h /^ MSYNC,$/;" e enum:__anon1 -MUNMAP ./src/opcodes.h /^ MUNMAP,$/;" e enum:__anon1 -Malloc ./src/macros.h 21;" d -Mmapshared ./src/macros.h 32;" d -Mset ./src/macros.h 25;" d -NAME_TO_HANDLE_AT ./src/opcodes.h /^ NAME_TO_HANDLE_AT,$/;" e enum:__anon1 -OPCODES_H ./src/opcodes.h 2;" d -OPEN ./src/opcodes.h /^ OPEN = 30,$/;" e enum:__anon1 -OPEN_AT ./src/opcodes.h /^ OPEN_AT,$/;" e enum:__anon1 -OPEN_BY_HANDLE_AT ./src/opcodes.h /^ OPEN_BY_HANDLE_AT,$/;" e enum:__anon1 -OPTIONS_H ./src/options.h 2;" d -Out ./src/macros.h 42;" d -Put ./src/macros.h 45;" d -RBUFFER_H ./src/datas/rbuffer.h 2;" d -READ ./src/opcodes.h /^ READ = 10,$/;" e enum:__anon1 -READAHEAD ./src/opcodes.h /^ READAHEAD,$/;" e enum:__anon1 -READDIR ./src/opcodes.h /^ READDIR,$/;" e enum:__anon1 -READLINK ./src/opcodes.h /^ READLINK,$/;" e enum:__anon1 -READLINK_AT ./src/opcodes.h /^ READLINK_AT,$/;" e enum:__anon1 -READV ./src/opcodes.h /^ READV,$/;" e enum:__anon1 -REMAP ./src/opcodes.h /^ REMAP,$/;" e enum:__anon1 -RENAME ./src/opcodes.h /^ RENAME = 40,$/;" e enum:__anon1 -RENAME_AT ./src/opcodes.h /^ RENAME_AT,$/;" e enum:__anon1 -RENAME_AT2 ./src/opcodes.h /^ RENAME_AT2,$/;" e enum:__anon1 -REPLAY_H ./src/replay/replay.h 2;" d -REPLAY_VERSION ./src/defaults.h 9;" d -RIOOP_H ./src/replay/rioop.h 2;" d -RMDIR ./src/opcodes.h /^ RMDIR,$/;" e enum:__anon1 -RPROCESS_H ./src/replay/rprocess.h 2;" d -RTASK_H ./src/replay/rtask.h 2;" d -RTHREAD_H ./src/replay/rthread.h 9;" d -RWORKER_H ./src/replay/rworker.h 2;" d -Readhex ./src/macros.h 14;" d -STACK_H ./src/datas/stack.h 2;" d -STAT ./src/opcodes.h /^ STAT,$/;" e enum:__anon1 -STATFS ./src/opcodes.h /^ STATFS,$/;" e enum:__anon1 -STATFS64 ./src/opcodes.h /^ STATFS64,$/;" e enum:__anon1 -SUCCESS ./src/defaults.h /^ SUCCESS, \/**< Great success! *\/$/;" e enum:status_e_ -SYNC ./src/opcodes.h /^ SYNC,$/;" e enum:__anon1 -SYNCFS ./src/opcodes.h /^ SYNCFS,$/;" e enum:__anon1 -SYNC_FILE_RANGE ./src/opcodes.h /^ SYNC_FILE_RANGE,$/;" e enum:__anon1 -Segfault ./src/macros.h 79;" d -TASK_BUFFER_PER_THREAD ./src/defaults.h 15;" d -UNKNOWN ./src/defaults.h /^ UNKNOWN, \/**< Unknown return status :-\/ *\/$/;" e enum:status_e_ -UNLINK ./src/opcodes.h /^ UNLINK,$/;" e enum:__anon1 -UNLINK_AT ./src/opcodes.h /^ UNLINK_AT,$/;" e enum:__anon1 -UTESTS_H ./src/utests.h 2;" d -UTILS_H ./src/utils/utils.h 2;" d -VFD_H ./src/vfd.h 2;" d -VSIZE_H ./src/generate/vsize.h 2;" d -WRITE ./src/opcodes.h /^ WRITE = 20,$/;" e enum:__anon1 -WRITEV ./src/opcodes.h /^ WRITEV,$/;" e enum:__anon1 -Warn ./src/macros.h 88;" d -Warn_if ./src/macros.h 95;" d -_Errno ./src/replay/rioop.c 15;" d file: -_Error ./src/replay/rioop.c 7;" d file: -_GNU_SOURCE ./src/utils/utils.h 5;" d -_Init_arg ./src/replay/rioop.c 24;" d file: -_Init_bytes ./src/replay/rioop.c 36;" d file: -_Init_cmd ./src/replay/rioop.c 25;" d file: -_Init_fd ./src/replay/rioop.c 26;" d file: -_Init_flags ./src/replay/rioop.c 27;" d file: -_Init_offset ./src/replay/rioop.c 29;" d file: -_Init_op ./src/replay/rioop.c 30;" d file: -_Init_path ./src/replay/rioop.c 32;" d file: -_Init_path2 ./src/replay/rioop.c 31;" d file: -_Init_rc ./src/replay/rioop.c 33;" d file: -_Init_virtfd ./src/replay/rioop.c 40;" d file: -_Init_whence ./src/replay/rioop.c 34;" d file: -_MAX_META_LEN ./src/meta/meta.c 3;" d file: -_MAX_PROCESSES ./src/generate/generate.c 10;" d file: -_PATH_INSERT ./src/mounts.c 5;" d file: -_PATH_INSERT_LEN ./src/mounts.c 6;" d file: -_Perc_filtered ./src/generate/generate.c 12;" d file: -_Set_dir ./src/generate/vsize.c 8;" d file: -_Set_file ./src/generate/vsize.c 7;" d file: -_Set_inserted ./src/generate/vsize.c 10;" d file: -_Set_renamed ./src/generate/vsize.c 11;" d file: -_Set_required ./src/generate/vsize.c 12;" d file: -_Set_unsure ./src/generate/vsize.c 9;" d file: -_Using_string_keys ./src/datas/hmap.c 3;" d file: -_amap_new ./src/datas/amap.c /^static amap_s *_amap_new(long size, bool mmapped)$/;" f file: -_amap_test ./src/datas/amap.c /^void _amap_test(amap_s *a)$/;" f -_arch_check_atomic ./src/main.c /^static void _arch_check_atomic(void)$/;" f file: -_gprocess_vfd_map_destroy_cb ./src/generate/gprocess.c /^void _gprocess_vfd_map_destroy_cb(void *data)$/;" f -_hmap_new ./src/datas/hmap.c /^hmap_s *_hmap_new(unsigned int init_size)$/;" f -_hmap_test ./src/datas/hmap.c /^static void _hmap_test(hmap_s *h)$/;" f file: -_hmap_test_l ./src/datas/hmap.c /^static void _hmap_test_l(hmap_s *h)$/;" f file: -_list_elem_remove ./src/datas/list.c /^void _list_elem_remove(list_s *l, list_elem_s *e)$/;" f -_print_help ./src/main.c /^static void _print_help(void)$/;" f file: -_print_synopsis ./src/main.c /^static void _print_synopsis(void)$/;" f file: -_print_version ./src/main.c /^static void _print_version(void)$/;" f file: -_rprocess_destroy_cb ./src/replay/rworker.c /^static void _rprocess_destroy_cb(void *data)$/;" f file: -_rthread_init_log ./src/replay/rthread.c /^static void _rthread_init_log(rthread_s *t)$/;" f file: -absolute_path ./src/utils/futils.c /^char *absolute_path(const char *path)$/;" f -address ./src/generate/gtask.h /^ long address; \/**< An address (used by mmap related syscalls) *\/$/;" m struct:gtask_s_ -address2 ./src/generate/gtask.h /^ long address2; \/**< Another address (used by mmap related syscalls) *\/$/;" m struct:gtask_s_ -amap_destroy ./src/datas/amap.c /^void amap_destroy(amap_s* a)$/;" f -amap_get ./src/datas/amap.c /^void* amap_get(amap_s *a, const long position)$/;" f -amap_new ./src/datas/amap.c /^amap_s* amap_new(const long size)$/;" f -amap_new_mmapped ./src/datas/amap.c /^amap_s* amap_new_mmapped(const long size)$/;" f -amap_print ./src/datas/amap.c /^void amap_print(amap_s* a)$/;" f -amap_reset ./src/datas/amap.c /^void amap_reset(amap_s* a)$/;" f -amap_run_cb ./src/datas/amap.c /^void amap_run_cb(amap_s *a, void (*cb)(void *data))$/;" f -amap_s ./src/datas/amap.h /^} amap_s;$/;" t typeref:struct:amap_s_ -amap_s_ ./src/datas/amap.h /^typedef struct amap_s_ {$/;" s -amap_set ./src/datas/amap.c /^int amap_set(amap_s *a, const long position, void* value)$/;" f -amap_test ./src/datas/amap.c /^void amap_test(void)$/;" f -amap_unset ./src/datas/amap.c /^void* amap_unset(amap_s *a, const long position)$/;" f -append_random_to_file ./src/utils/futils.c /^void append_random_to_file(char *path, unsigned long bytes)$/;" f -arrays ./src/datas/amap.h /^ void*** arrays; \/**< The pointers to the amap arrays *\/$/;" m struct:amap_s_ -btree_destroy ./src/datas/btree.c /^void btree_destroy(btree_s* b)$/;" f -btree_destroy2 ./src/datas/btree.c /^void btree_destroy2(btree_s* b)$/;" f -btree_get ./src/datas/btree.c /^void* btree_get(btree_s* b, int key)$/;" f -btree_insert ./src/datas/btree.c /^int btree_insert(btree_s* b, int key, void *data)$/;" f -btree_new ./src/datas/btree.c /^btree_s* btree_new()$/;" f -btree_print ./src/datas/btree.c /^void btree_print(btree_s* b)$/;" f -btree_s ./src/datas/btree.h /^} btree_s;$/;" t typeref:struct:btree_s_ -btree_s_ ./src/datas/btree.h /^typedef struct btree_s_ {$/;" s -btreelem_ ./src/datas/btree.h /^typedef struct btreelem_ {$/;" s -btreelem_destroy_r ./src/datas/btree.c /^void btreelem_destroy_r(btreelem_s* e)$/;" f -btreelem_destroy_r2 ./src/datas/btree.c /^void btreelem_destroy_r2(btreelem_s* e)$/;" f -btreelem_get_r ./src/datas/btree.c /^void* btreelem_get_r(btreelem_s* e, int key)$/;" f -btreelem_insert_r ./src/datas/btree.c /^int btreelem_insert_r(btreelem_s* e, int key, void *data)$/;" f -btreelem_new ./src/datas/btree.c /^btreelem_s* btreelem_new(int key, void *data)$/;" f -btreelem_print_r ./src/datas/btree.c /^void btreelem_print_r(btreelem_s* e, int depth)$/;" f -btreelem_s ./src/datas/btree.h /^} btreelem_s;$/;" t typeref:struct:btreelem_ -bytes ./src/generate/gtask.h /^ long bytes; \/**< Amount of bytes *\/$/;" m struct:gtask_s_ -cache_file ./src/utils/futils.c /^void cache_file(const char *file)$/;" f -capture_file ./src/options.h /^ char *capture_file; \/**< The name of the .capture file *\/$/;" m struct:options_s_ -chown_path ./src/utils/futils.c /^void chown_path(const char *user, const char *path)$/;" f -chreplace ./src/utils/utils.c /^void chreplace(char *str, char replace, char with)$/;" f -cleanup_run ./src/cleanup/cleanup.c /^status_e cleanup_run(options_s *opts)$/;" f -clone ./src/replay/rtask.h /^ char *clone; \/**< Used for debug purposes only *\/$/;" m struct:rtask_s_ -count ./src/generate/gtask.h /^ long count; \/**< A count *\/$/;" m struct:gtask_s_ -count ./src/mounts.h /^ int count; \/**< The amount of mount points *\/$/;" m struct:mounts_s_ -data ./src/datas/btree.h /^ void *data; \/**< A pointer to the data stored in this element *\/$/;" m struct:btreelem_ -data ./src/datas/hmap.h /^ void **data; \/**< Pointers to the stored data, NULL if nothing there *\/$/;" m struct:hmap_s_ -data ./src/datas/list.h /^ void *data; \/**< Pointer to the stored data *\/$/;" m struct:list_elem_s_ -data ./src/datas/stack.h /^ void *data; \/**< Pointer to the stored data in the current element *\/$/;" m struct:stack_elem_s_ -data_destroy ./src/datas/amap.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all elements *\/$/;" m struct:amap_s_ -data_destroy ./src/datas/hmap.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all data *\/$/;" m struct:hmap_s_ -data_destroy ./src/datas/list.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all data *\/$/;" m struct:list_s_ -debug ./src/vfd.h /^ int debug; \/**< Used for debugging purposes only *\/$/;" m struct:vfd_s_ -dirfd ./src/vfd.h /^ DIR *dirfd; \/**< The real dirfd *\/$/;" m struct:vfd_s_ -dirname_r ./src/utils/futils.c /^char* dirname_r(char *path)$/;" f -dirs_created ./src/init/itask.h /^ long dirs_created;$/;" m struct:itask_s_ -drop_caches ./src/options.h /^ bool drop_caches; \/**< True if ioreplay should drop all Linux caches *\/$/;" m struct:options_s_ -drop_caches ./src/utils/futils.c /^void drop_caches(void)$/;" f -drop_root ./src/utils/utils.c /^void drop_root(const char *user)$/;" f -ensure_dir_empty ./src/utils/futils.c /^void ensure_dir_empty(const char *path)$/;" f -ensure_dir_exists ./src/utils/futils.c /^long ensure_dir_exists(const char *path)$/;" f -ensure_file_exists ./src/utils/futils.c /^int ensure_file_exists(char *path, long *num_dirs_created)$/;" f -ensure_parent_dir_exists ./src/utils/futils.c /^void ensure_parent_dir_exists(const char *path)$/;" f -exists ./src/utils/futils.c /^bool exists(const char *path)$/;" f -fd ./src/generate/gtask.h /^ int fd; \/**< File descriptor number *\/$/;" m struct:gtask_s_ -fd ./src/vfd.h /^ int fd; \/**< the real fd *\/$/;" m struct:vfd_s_ -fd_map ./src/generate/gprocess.h /^ hmap_s *fd_map; \/**< All mappings from real fd to virtual fd *\/$/;" m struct:gprocess_s_ -fds_map ./src/replay/rprocess.h /^ amap_s *fds_map; \/**< Holding all file descriptors *\/$/;" m struct:rprocess_s_ -fds_map ./src/replay/rworker.h /^ amap_s* fds_map; \/**< Holding all file descriptors *\/$/;" m struct:__anon2 -files_created ./src/init/itask.h /^ long files_created;$/;" m struct:itask_s_ -filtered_where ./src/generate/gtask.h /^ char *filtered_where; \/**< Only used for debugging purposes *\/$/;" m struct:gtask_s_ -first ./src/datas/list.h /^ list_elem_s *first; \/**< The first element, NULL if list empty *\/$/;" m struct:list_s_ -flags ./src/generate/gtask.h /^ int flags; \/**< File open flags *\/$/;" m struct:gtask_s_ -fnotnull ./src/utils/utils.c /^FILE* fnotnull(FILE *fd, const char *path, char *file, int line)$/;" f -free_path ./src/vfd.h /^ bool free_path; \/**< True if path has to be freed or not *\/$/;" m struct:vfd_s_ -generate ./src/generate/gparser.h /^ generate_s *generate; \/**< The generate object *\/$/;" m struct:gparser_s_ -generate ./src/generate/gtask.h /^ void *generate; \/**< A pointer to the generate object *\/$/;" m struct:gtask_s_ -generate ./src/generate/gwriter.h /^ struct generate_s_ *generate; \/**< The generate object *\/$/;" m struct:gwriter_s_ typeref:struct:gwriter_s_::generate_s_ -generate ./src/generate/vsize.h /^ void *generate; \/**< A pointer to the generate object *\/$/;" m struct:vsize_s_ -generate_destroy ./src/generate/generate.c /^void generate_destroy(generate_s *g)$/;" f -generate_gprocess_by_realpid ./src/generate/generate.c /^void generate_gprocess_by_realpid(generate_s *g, gtask_s *t)$/;" f -generate_new ./src/generate/generate.c /^generate_s* generate_new(options_s *opts)$/;" f -generate_run ./src/generate/generate.c /^status_e generate_run(options_s *opts)$/;" f -generate_s ./src/generate/generate.h /^} generate_s;$/;" t typeref:struct:generate_s_ -generate_s_ ./src/generate/generate.h /^typedef struct generate_s_ {$/;" s -generate_vsize_by_path ./src/generate/generate.c /^vsize_s* generate_vsize_by_path(generate_s *g, gtask_s *t,$/;" f -generate_write_init_cb ./src/generate/generate.c /^void generate_write_init_cb(void *data)$/;" f -get_loadavg ./src/utils/utils.c /^void get_loadavg(char *readbuf)$/;" f -gioop_chmod ./src/generate/gioop.c /^status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_chown ./src/generate/gioop.c /^status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_close ./src/generate/gioop.c /^status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_close_all_vfd_cb ./src/generate/gioop.c /^void gioop_close_all_vfd_cb(void *data, void *data2)$/;" f -gioop_creat ./src/generate/gioop.c /^status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_exit_group ./src/generate/gioop.c /^status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_fchmod ./src/generate/gioop.c /^status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_fchown ./src/generate/gioop.c /^status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_fcntl ./src/generate/gioop.c /^status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_fdatasync ./src/generate/gioop.c /^status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_fstat ./src/generate/gioop.c /^status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_fstatat ./src/generate/gioop.c /^status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_fstatfs ./src/generate/gioop.c /^status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_fstatfs64 ./src/generate/gioop.c /^status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_fsync ./src/generate/gioop.c /^status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_getdents ./src/generate/gioop.c /^status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_lchown ./src/generate/gioop.c /^status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_lseek ./src/generate/gioop.c /^status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_lstat ./src/generate/gioop.c /^status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_mkdir ./src/generate/gioop.c /^status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_mkdirat ./src/generate/gioop.c /^status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_open ./src/generate/gioop.c /^status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_openat ./src/generate/gioop.c /^status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_read ./src/generate/gioop.c /^status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_readahead ./src/generate/gioop.c /^status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_readdir ./src/generate/gioop.c /^status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_readlink ./src/generate/gioop.c /^status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_readlinkat ./src/generate/gioop.c /^status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_readv ./src/generate/gioop.c /^status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_rename ./src/generate/gioop.c /^status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_renameat ./src/generate/gioop.c /^status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_renameat2 ./src/generate/gioop.c /^status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_rmdir ./src/generate/gioop.c /^status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_run ./src/generate/gioop.c /^status_e gioop_run(gwriter_s *w, gtask_s *t)$/;" f -gioop_stat ./src/generate/gioop.c /^status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_statfs ./src/generate/gioop.c /^status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_statfs64 ./src/generate/gioop.c /^status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_sync ./src/generate/gioop.c /^status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_sync_file_range ./src/generate/gioop.c /^status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_syncfs ./src/generate/gioop.c /^status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_unlink ./src/generate/gioop.c /^status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_unlinkat ./src/generate/gioop.c /^status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_write ./src/generate/gioop.c /^status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gioop_writev ./src/generate/gioop.c /^status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f -gparser_destroy ./src/generate/gparser.c /^void gparser_destroy(gparser_s *p)$/;" f -gparser_extract ./src/generate/gparser.c /^void gparser_extract(gparser_s *p, gtask_s *t)$/;" f -gparser_extract_tok ./src/generate/gparser.c /^status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok)$/;" f -gparser_get_pidtid ./src/generate/gparser.c /^bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid)$/;" f -gparser_new ./src/generate/gparser.c /^gparser_s* gparser_new(generate_s *g)$/;" f -gparser_pthread_start ./src/generate/gparser.c /^void* gparser_pthread_start(void *data)$/;" f -gparser_s ./src/generate/gparser.h /^} gparser_s;$/;" t typeref:struct:gparser_s_ -gparser_s_ ./src/generate/gparser.h /^typedef struct gparser_s_ {$/;" s -gparser_start ./src/generate/gparser.c /^void gparser_start(gparser_s *p)$/;" f -gparser_terminate ./src/generate/gparser.c /^void gparser_terminate(gparser_s *p)$/;" f -gparser_token_not_ok ./src/generate/gparser.c /^bool gparser_token_not_ok(gparser_s *p, char *tok)$/;" f -gprocess ./src/generate/gtask.h /^ struct gprocess_s_ *gprocess; \/**< A pointer to the process object *\/$/;" m struct:gtask_s_ typeref:struct:gtask_s_::gprocess_s_ -gprocess_create_vfd_by_realfd ./src/generate/gprocess.c /^void gprocess_create_vfd_by_realfd(gprocess_s *gp, gtask_s *t, generate_s *g)$/;" f -gprocess_destroy ./src/generate/gprocess.c /^void gprocess_destroy(gprocess_s *gp)$/;" f -gprocess_new ./src/generate/gprocess.c /^gprocess_s* gprocess_new(const long pid, const long mapped_pid)$/;" f -gprocess_s ./src/generate/gprocess.h /^} gprocess_s;$/;" t typeref:struct:gprocess_s_ -gprocess_s_ ./src/generate/gprocess.h /^typedef struct gprocess_s_ {$/;" s -gprocess_vfd_by_realfd ./src/generate/gprocess.c /^status_e gprocess_vfd_by_realfd(gprocess_s *gp, gtask_s *t)$/;" f -gtask_destroy ./src/generate/gtask.c /^void gtask_destroy(gtask_s *t)$/;" f -gtask_init ./src/generate/gtask.c /^void gtask_init(gtask_s *t, char *line, const unsigned long lineno)$/;" f -gtask_new ./src/generate/gtask.c /^gtask_s* gtask_new(void *generate)$/;" f -gtask_s ./src/generate/gtask.h /^} gtask_s;$/;" t typeref:struct:gtask_s_ -gtask_s_ ./src/generate/gtask.h /^typedef struct gtask_s_ {$/;" s -gwriter_destroy ./src/generate/gwriter.c /^void gwriter_destroy(gwriter_s *w)$/;" f -gwriter_new ./src/generate/gwriter.c /^gwriter_s* gwriter_new(generate_s *g)$/;" f -gwriter_pthread_start ./src/generate/gwriter.c /^void* gwriter_pthread_start(void *data)$/;" f -gwriter_s ./src/generate/gwriter.h /^} gwriter_s;$/;" t typeref:struct:gwriter_s_ -gwriter_s_ ./src/generate/gwriter.h /^typedef struct gwriter_s_ {$/;" s -gwriter_start ./src/generate/gwriter.c /^void gwriter_start(gwriter_s *w)$/;" f -gwriter_terminate ./src/generate/gwriter.c /^void gwriter_terminate(gwriter_s *w)$/;" f -has_fd ./src/generate/gtask.h /^ bool has_fd; \/**< True if task has a file descriptor number *\/$/;" m struct:gtask_s_ -hmap_destroy ./src/datas/hmap.c /^void hmap_destroy(hmap_s *h)$/;" f -hmap_get ./src/datas/hmap.c /^void* hmap_get(hmap_s *h, char *key)$/;" f -hmap_get_addr ./src/datas/hmap.c /^unsigned int hmap_get_addr(hmap_s *h, char *key)$/;" f -hmap_get_addr_l ./src/datas/hmap.c /^unsigned int hmap_get_addr_l(hmap_s *h, const long key)$/;" f -hmap_get_l ./src/datas/hmap.c /^void* hmap_get_l(hmap_s *h, const long key)$/;" f -hmap_insert ./src/datas/hmap.c /^int hmap_insert(hmap_s *h, char *key, void *data)$/;" f -hmap_insert_l ./src/datas/hmap.c /^int hmap_insert_l(hmap_s *h, const long key, void *data)$/;" f -hmap_new ./src/datas/hmap.c /^hmap_s *hmap_new(unsigned int init_size)$/;" f -hmap_new_l ./src/datas/hmap.c /^hmap_s *hmap_new_l(unsigned int init_size)$/;" f -hmap_print ./src/datas/hmap.c /^void hmap_print(hmap_s *h)$/;" f -hmap_remove ./src/datas/hmap.c /^void* hmap_remove(hmap_s *h, char *key)$/;" f -hmap_remove_l ./src/datas/hmap.c /^void* hmap_remove_l(hmap_s *h, const long key)$/;" f -hmap_run_cb ./src/datas/hmap.c /^void hmap_run_cb(hmap_s* h, void (*cb)(void *data))$/;" f -hmap_run_cb2 ./src/datas/hmap.c /^void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_)$/;" f -hmap_s ./src/datas/hmap.h /^} hmap_s;$/;" t typeref:struct:hmap_s_ -hmap_s_ ./src/datas/hmap.h /^typedef struct hmap_s_ {$/;" s -hmap_test ./src/datas/hmap.c /^void hmap_test(void)$/;" f -id ./src/generate/vsize.h /^ unsigned long id; \/**< The vsize id *\/$/;" m struct:vsize_s_ -ignore_count ./src/mounts.h /^ int ignore_count; \/**< The amount of ignored mount points *\/$/;" m struct:mounts_s_ -ignore_mps ./src/mounts.h /^ char *ignore_mps[MAX_MOUNTPOINTS]; \/**< The ignored mp paths *\/$/;" m struct:mounts_s_ -init ./src/init/ithread.h /^ init_s *init; \/**< The responsible init object *\/$/;" m struct:ithread_s_ -init ./src/options.h /^ bool init; \/**< If set ioreplay will initialise the environment *\/$/;" m struct:options_s_ -init_destroy ./src/init/init.c /^void init_destroy(init_s *i)$/;" f -init_extract_header ./src/init/init.c /^void init_extract_header(init_s *i, off_t *init_offset)$/;" f -init_new ./src/init/init.c /^init_s *init_new(options_s *opts)$/;" f -init_parent_dir ./src/generate/vsize.c /^void init_parent_dir(vsize_s *v, const char *path)$/;" f -init_run ./src/init/init.c /^status_e init_run(options_s *opts)$/;" f -init_s ./src/init/init.h /^} init_s;$/;" t typeref:struct:init_s_ -init_s_ ./src/init/init.h /^typedef struct init_s_ {$/;" s -initm ./src/replay/rprocess.h /^ bool initm; \/**< Indicates whether ioreplay is in init mode or not *\/$/;" m struct:rprocess_s_ -inserted ./src/generate/vsize.h /^ bool inserted; \/**< For debugging purposes only *\/$/;" m struct:vsize_s_ -is_dir ./src/generate/vsize.h /^ bool is_dir; \/**< True if this file\/dir is a directory *\/$/;" m struct:vsize_s_ -is_dir ./src/init/itask.h /^ bool is_dir;$/;" m struct:itask_s_ -is_dir ./src/utils/futils.c /^bool is_dir(const char *path)$/;" f -is_file ./src/generate/vsize.h /^ bool is_file; \/**< True if this file\/dir is a regular file *\/$/;" m struct:vsize_s_ -is_file ./src/init/itask.h /^ bool is_file;$/;" m struct:itask_s_ -is_number ./src/utils/utils.c /^bool is_number(char *str)$/;" f -is_reg ./src/utils/futils.c /^bool is_reg(const char *path)$/;" f -itask_destroy ./src/init/itask.c /^void itask_destroy(itask_s *task)$/;" f -itask_extract_stats ./src/init/itask.c /^void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created,$/;" f -itask_new ./src/init/itask.c /^itask_s* itask_new()$/;" f -itask_print ./src/init/itask.c /^void itask_print(itask_s *task)$/;" f -itask_reset_stats ./src/init/itask.c /^void itask_reset_stats(itask_s *task)$/;" f -itask_s ./src/init/itask.h /^} itask_s;$/;" t typeref:struct:itask_s_ -itask_s_ ./src/init/itask.h /^typedef struct itask_s_ {$/;" s -ithread_destroy ./src/init/ithread.c /^void ithread_destroy(ithread_s *t)$/;" f -ithread_new ./src/init/ithread.c /^ithread_s* ithread_new(init_s *i)$/;" f -ithread_pthread_start ./src/init/ithread.c /^void* ithread_pthread_start(void *data)$/;" f -ithread_run_task ./src/init/ithread.c /^void ithread_run_task(ithread_s *t, itask_s *task)$/;" f -ithread_s ./src/init/ithread.h /^} ithread_s;$/;" t typeref:struct:ithread_s_ -ithread_s_ ./src/init/ithread.h /^typedef struct ithread_s_ {$/;" s -ithread_start ./src/init/ithread.c /^void ithread_start(ithread_s *t)$/;" f -ithread_terminate ./src/init/ithread.c /^void ithread_terminate(ithread_s *t)$/;" f -key ./src/datas/btree.h /^ int key; \/**< The key of the element *\/$/;" m struct:btreelem_ -key ./src/datas/list.h /^ char *key; \/**< The key of the lemenet *\/$/;" m struct:list_elem_s_ -key_l ./src/datas/list.h /^ long key_l; \/**< The same as key, but for long keys *\/$/;" m struct:list_elem_s_ -keys ./src/datas/hmap.h /^ char **keys; \/**< List of all keys, NULL if nothing at a address *\/$/;" m struct:hmap_s_ -keys_l ./src/datas/hmap.h /^ int *keys_l; \/**< Same as keys, but for long keys *\/$/;" m struct:hmap_s_ -l ./src/datas/hmap.h /^ list_s **l; \/**< Pointers to the linked lists, used on hash collision *\/$/;" m struct:hmap_s_ -left ./src/datas/btree.h /^ struct btreelem_ *left; \/**< The next element to the left *\/$/;" m struct:btreelem_ typeref:struct:btreelem_::btreelem_ -lengths ./src/mounts.h /^ int lengths[MAX_MOUNTPOINTS]; \/**< The mp lenghts *\/$/;" m struct:mounts_s_ -line ./src/generate/gtask.h /^ char *line; \/**< A pointer to the remaining part of the .capture line *\/$/;" m struct:gtask_s_ -line ./src/replay/rtask.h /^ char line[MAX_LINE_LEN]; \/**< The remaining part of the .replay line *\/$/;" m struct:rtask_s_ -lineno ./src/generate/generate.h /^ long lineno; \/**< The current line number *\/$/;" m struct:generate_s_ -lineno ./src/generate/gtask.h /^ long lineno; \/**< The current line number *\/$/;" m struct:gtask_s_ -lineno ./src/replay/rprocess.h /^ unsigned long lineno; \/**< Holding the current .replay line number *\/$/;" m struct:rprocess_s_ -lineno ./src/replay/rtask.h /^ unsigned long lineno; \/**< The current line number *\/$/;" m struct:rtask_s_ -list_destroy ./src/datas/list.c /^void list_destroy(list_s *l)$/;" f -list_elem_s ./src/datas/list.h /^} list_elem_s;$/;" t typeref:struct:list_elem_s_ -list_elem_s_ ./src/datas/list.h /^typedef struct list_elem_s_ {$/;" s -list_key_get ./src/datas/list.c /^void* list_key_get(list_s *l, char *key)$/;" f -list_key_get_l ./src/datas/list.c /^void* list_key_get_l(list_s *l, const long key)$/;" f -list_key_insert ./src/datas/list.c /^int list_key_insert(list_s *l, char *key, void *data)$/;" f -list_key_insert_l ./src/datas/list.c /^int list_key_insert_l(list_s *l, const long key, void *data)$/;" f -list_key_remove ./src/datas/list.c /^void* list_key_remove(list_s *l, char *key)$/;" f -list_key_remove_l ./src/datas/list.c /^void* list_key_remove_l(list_s *l, const long key)$/;" f -list_new ./src/datas/list.c /^list_s *list_new()$/;" f -list_new_l ./src/datas/list.c /^list_s *list_new_l()$/;" f -list_print ./src/datas/list.c /^void list_print(list_s *l)$/;" f -list_run_cb ./src/datas/list.c /^void list_run_cb(list_s* l, void (*cb)(void *data))$/;" f -list_run_cb2 ./src/datas/list.c /^void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_)$/;" f -list_s ./src/datas/list.h /^} list_s;$/;" t typeref:struct:list_s_ -list_s_ ./src/datas/list.h /^typedef struct list_s_ {$/;" s -list_test ./src/datas/list.c /^void list_test(void)$/;" f -main ./src/main.c /^int main(int argc, char **argv)$/;" f -mapped_fd ./src/generate/gtask.h /^ long mapped_fd; \/**< The mapped file descriptor number *\/$/;" m struct:gtask_s_ -mapped_fd ./src/vfd.h /^ long mapped_fd; \/**< The mapped fd (virtual fd) *\/$/;" m struct:vfd_s_ -mapped_pid ./src/generate/gprocess.h /^ long mapped_pid; \/**< The mapped PID *\/$/;" m struct:gprocess_s_ -mapped_time ./src/generate/gtask.h /^ long mapped_time; \/**< The mapped time *\/$/;" m struct:gtask_s_ -max_mapped_fd ./src/generate/gprocess.h /^ long max_mapped_fd; \/**< The max mapped fd number *\/$/;" m struct:gprocess_s_ -meta_destroy ./src/meta/meta.c /^void meta_destroy(meta_s *m)$/;" f -meta_new ./src/meta/meta.c /^meta_s* meta_new(FILE *replay_fd)$/;" f -meta_read_l ./src/meta/meta.c /^bool meta_read_l(meta_s *m, char *key, long *val)$/;" f -meta_read_s ./src/meta/meta.c /^bool meta_read_s(meta_s *m, char *key, char **val)$/;" f -meta_read_start ./src/meta/meta.c /^void meta_read_start(meta_s *m)$/;" f -meta_reserve ./src/meta/meta.c /^void meta_reserve(meta_s *m)$/;" f -meta_s ./src/meta/meta.h /^} meta_s;$/;" t typeref:struct:meta_s_ -meta_s_ ./src/meta/meta.h /^typedef struct meta_s_ {$/;" s -meta_write_l ./src/meta/meta.c /^void meta_write_l(meta_s *m, char *key, long val)$/;" f -meta_write_s ./src/meta/meta.c /^void meta_write_s(meta_s *m, char *key, char *val)$/;" f -meta_write_start ./src/meta/meta.c /^void meta_write_start(meta_s *m)$/;" f -mkdir_p ./src/utils/futils.c /^int mkdir_p(const char *path, mode_t mode, long *num_dirs_created)$/;" f -mmap_map ./src/generate/generate.h /^ hmap_s *mmap_map; \/**< mmap address mappings *\/$/;" m struct:generate_s_ -mmapok ./src/utils/utils.c /^void* mmapok(void *p, char *file, int line)$/;" f -mmapped ./src/datas/amap.h /^ bool mmapped; \/**< True if amap is memory mapped *\/$/;" m struct:amap_s_ -mode ./src/generate/gtask.h /^ int mode; \/**< File open mode *\/$/;" m struct:gtask_s_ -mounts ./src/init/init.h /^ mounts_s *mounts;$/;" m struct:init_s_ -mounts_destroy ./src/mounts.c /^void mounts_destroy(mounts_s *m)$/;" f -mounts_get_mountnumber ./src/mounts.c /^int mounts_get_mountnumber(mounts_s *m, const char *path)$/;" f -mounts_ignore_path ./src/mounts.c /^bool mounts_ignore_path(mounts_s *m, const char *path)$/;" f -mounts_init ./src/mounts.c /^void mounts_init(mounts_s *m)$/;" f -mounts_new ./src/mounts.c /^mounts_s *mounts_new(options_s *opts)$/;" f -mounts_purge ./src/mounts.c /^void mounts_purge(mounts_s *m)$/;" f -mounts_read ./src/mounts.c /^void mounts_read(mounts_s *m)$/;" f -mounts_s ./src/mounts.h /^} mounts_s;$/;" t typeref:struct:mounts_s_ -mounts_s_ ./src/mounts.h /^typedef struct mounts_s_ {$/;" s -mounts_transform_path ./src/mounts.c /^bool mounts_transform_path(mounts_s *m, const char *name,$/;" f -mounts_trash ./src/mounts.c /^void mounts_trash(mounts_s *m)$/;" f -mps ./src/generate/generate.h /^ mounts_s *mps; \/**< The mounts object *\/$/;" m struct:generate_s_ -mps ./src/mounts.h /^ char *mps[MAX_MOUNTPOINTS]; \/**< The mp paths *\/$/;" m struct:mounts_s_ -name ./src/generate/generate.h /^ char *name; \/**< The name of the test specified by the user *\/$/;" m struct:generate_s_ -name ./src/options.h /^ char *name; \/**< The name of the test (found in .ioreplay\/name sub-dirs) *\/$/;" m struct:options_s_ -next ./src/datas/list.h /^ struct list_elem_s_ *next; \/**< The next element *\/$/;" m struct:list_elem_s_ typeref:struct:list_elem_s_::list_elem_s_ -next ./src/datas/stack.h /^ struct stack_elem_s_ *next; \/**< The next element *\/$/;" m struct:stack_elem_s_ typeref:struct:stack_elem_s_::stack_elem_s_ -notnull ./src/utils/utils.c /^void* notnull(void *p, char *file, int line, int count)$/;" f -num_arrays ./src/datas/amap.h /^ int num_arrays; \/**< The amount of arrays used in the amap *\/$/;" m struct:amap_s_ -num_lines_filtered ./src/generate/generate.h /^ long num_lines_filtered; \/**< The amount of lines filtered out *\/$/;" m struct:generate_s_ -num_mapped_fds ./src/generate/generate.h /^ unsigned long num_mapped_fds; \/**< The amount of mapped FDs *\/$/;" m struct:generate_s_ -num_mapped_pids ./src/generate/generate.h /^ unsigned long num_mapped_pids; \/**< The amount of mapped PIDs *\/$/;" m struct:generate_s_ -num_threads_per_worker ./src/options.h /^ int num_threads_per_worker; \/**< Max threads per worker processes *\/$/;" m struct:options_s_ -num_vsizes ./src/generate/generate.h /^ unsigned long num_vsizes; \/**< The amount of virtual sizes *\/$/;" m struct:generate_s_ -num_workers ./src/options.h /^ int num_workers; \/**< The amount of worker processes *\/$/;" m struct:options_s_ -offset ./src/generate/gtask.h /^ long offset; \/**< A offset *\/$/;" m struct:gtask_s_ -offset ./src/generate/vsize.h /^ off_t offset; \/**< The current file offset *\/$/;" m struct:vsize_s_ -offset ./src/meta/meta.h /^ off_t offset; \/**< The meta offset (usually 0) *\/$/;" m struct:meta_s_ -offset ./src/vfd.h /^ unsigned long offset; \/**< The current virtual file offset in bytes *\/$/;" m struct:vfd_s_ -op ./src/generate/gtask.h /^ char *op; \/**< Operation\/syscall name *\/$/;" m struct:gtask_s_ -opcode_e ./src/opcodes.h /^} opcode_e;$/;" t typeref:enum:__anon1 -options_destroy ./src/options.c /^void options_destroy(options_s *o)$/;" f -options_new ./src/options.c /^options_s *options_new()$/;" f -options_s ./src/options.h /^} options_s;$/;" t typeref:struct:options_s_ -options_s_ ./src/options.h /^typedef struct options_s_ {$/;" s -opts ./src/generate/generate.h /^ options_s *opts; \/**< A pointer to the options object *\/$/;" m struct:generate_s_ -opts ./src/init/init.h /^ options_s *opts;$/;" m struct:init_s_ -opts ./src/mounts.h /^ options_s *opts; \/**< A pointer to the options object *\/$/;" m struct:mounts_s_ -opts ./src/replay/rworker.h /^ options_s *opts; \/**< To synchronise access to rthread_buffer *\/$/;" m struct:__anon2 -original_line ./src/generate/gtask.h /^ char *original_line; \/**< Only used for debugging purposes *\/$/;" m struct:gtask_s_ -path ./src/generate/gtask.h /^ char *path; \/**< Path name *\/$/;" m struct:gtask_s_ -path ./src/generate/vsize.h /^ char *path; \/**< The path to the file\/directory *\/$/;" m struct:vsize_s_ -path ./src/init/itask.h /^ char *path;$/;" m struct:itask_s_ -path ./src/vfd.h /^ char *path; \/**< The file path belonging to that fd *\/$/;" m struct:vfd_s_ -path2 ./src/generate/gtask.h /^ char *path2; \/**< A second path name (e.g. for rename) *\/$/;" m struct:gtask_s_ -path2_r ./src/generate/gtask.h /^ char *path2_r; \/**< Work around to track mallocs, so it can be freed *\/$/;" m struct:gtask_s_ -path_r ./src/generate/gtask.h /^ char *path_r; \/**< Work around to track mallocs, so it can be freed *\/$/;" m struct:gtask_s_ -pid ./src/generate/gprocess.h /^ long pid; \/**< The real PID *\/$/;" m struct:gprocess_s_ -pid ./src/generate/gtask.h /^ long pid; \/**< The process ID *\/$/;" m struct:gtask_s_ -pid ./src/replay/rprocess.h /^ int pid; \/**< The virtual process ID *\/$/;" m struct:rprocess_s_ -pid_map ./src/generate/generate.h /^ amap_s *pid_map; \/**< A map of all virtual process objects *\/$/;" m struct:generate_s_ -pidtid ./src/generate/gtask.h /^ char *pidtid; \/**< String representing pid:tid *\/$/;" m struct:gtask_s_ -prev ./src/datas/list.h /^ struct list_elem_s_ *prev; \/**< The previous element *\/$/;" m struct:list_elem_s_ typeref:struct:list_elem_s_::list_elem_s_ -process ./src/replay/rtask.h /^ void *process; \/* The responsible process object *\/$/;" m struct:rtask_s_ -pthread ./src/generate/gparser.h /^ pthread_t pthread; \/**< The posix thread *\/$/;" m struct:gparser_s_ -pthread ./src/generate/gwriter.h /^ pthread_t pthread; \/**< The posix thread *\/$/;" m struct:gwriter_s_ -pthread ./src/init/ithread.h /^ pthread_t pthread; \/**< We run the init tasks in concurrent pthreads *\/$/;" m struct:ithread_s_ -pthread ./src/replay/rthread.h /^ pthread_t pthread; \/**< We run the tasks in concurrent pthreads *\/$/;" m struct:rthread_s_ -purge ./src/options.h /^ bool purge; \/**< If set ioreplay will purge the environment *\/$/;" m struct:options_s_ -queue ./src/generate/gparser.h /^ rbuffer_s *queue; \/**< A queue of task objects *\/$/;" m struct:gparser_s_ -queue ./src/generate/gwriter.h /^ rbuffer_s *queue; \/**< A queue of task objects *\/$/;" m struct:gwriter_s_ -queue ./src/init/ithread.h /^ rbuffer_s *queue; \/**< The thread's task queue *\/$/;" m struct:ithread_s_ -rbuffer_destroy ./src/datas/rbuffer.c /^void rbuffer_destroy(rbuffer_s *r)$/;" f -rbuffer_get_next ./src/datas/rbuffer.c /^void* rbuffer_get_next(rbuffer_s* r)$/;" f -rbuffer_has_next ./src/datas/rbuffer.c /^bool rbuffer_has_next(rbuffer_s* r)$/;" f -rbuffer_insert ./src/datas/rbuffer.c /^bool rbuffer_insert(rbuffer_s* r, void *data)$/;" f -rbuffer_new ./src/datas/rbuffer.c /^rbuffer_s *rbuffer_new(const int size)$/;" f -rbuffer_print ./src/datas/rbuffer.c /^void rbuffer_print(rbuffer_s* r)$/;" f -rbuffer_s ./src/datas/rbuffer.h /^} rbuffer_s;$/;" t typeref:struct:rbuffer_s_ -rbuffer_s_ ./src/datas/rbuffer.h /^typedef struct rbuffer_s_ {$/;" s -rbuffer_test ./src/datas/rbuffer.c /^void rbuffer_test(void)$/;" f -read_buf ./src/meta/meta.h /^ char* read_buf; \/**< Pointer to a read buffer *\/$/;" m struct:meta_s_ -read_pos ./src/datas/rbuffer.h /^ sig_atomic_t read_pos;$/;" m struct:rbuffer_s_ -renamed ./src/generate/vsize.h /^ bool renamed; \/**< True if file\/dir has been renamed *\/$/;" m struct:vsize_s_ -replay ./src/options.h /^ bool replay; \/**< If set ioreplay will run\/replay the test *\/$/;" m struct:options_s_ -replay_extract_header ./src/replay/replay.c /^void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes,$/;" f -replay_fd ./src/generate/generate.h /^ FILE *replay_fd; \/**< The fd of the .replay file *\/$/;" m struct:generate_s_ -replay_fd ./src/init/init.h /^ FILE *replay_fd;$/;" m struct:init_s_ -replay_fd ./src/meta/meta.h /^ FILE* replay_fd; \/**< The FS of the .replay file *\/$/;" m struct:meta_s_ -replay_file ./src/options.h /^ char *replay_file; \/**< The name of the .replay file *\/$/;" m struct:options_s_ -replay_run ./src/replay/replay.c /^status_e replay_run(options_s *opts)$/;" f -required ./src/generate/vsize.h /^ bool required; \/**< True if init mode will create this file\/dir *\/$/;" m struct:vsize_s_ -ret ./src/generate/gtask.h /^ int ret; \/**< ioreplay process status, SUCCESS if everything is alright *\/$/;" m struct:gtask_s_ -reuse_queue ./src/generate/generate.h /^ rbuffer_s *reuse_queue; \/**< A task buffer, for reusing these *\/$/;" m struct:generate_s_ -reuse_queue ./src/init/init.h /^ rbuffer_s *reuse_queue;$/;" m struct:init_s_ -reuse_queue_mutex ./src/init/init.h /^ pthread_mutex_t reuse_queue_mutex;$/;" m struct:init_s_ -right ./src/datas/btree.h /^ struct btreelem_ *right; \/**< The next element to the right *\/$/;" m struct:btreelem_ typeref:struct:btreelem_::btreelem_ -ring ./src/datas/rbuffer.h /^ void **ring;$/;" m struct:rbuffer_s_ -rioop_chmod ./src/replay/rioop.c /^void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_chown ./src/replay/rioop.c /^void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_close ./src/replay/rioop.c /^void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_fchmod ./src/replay/rioop.c /^void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_fchown ./src/replay/rioop.c /^void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_fcntl ./src/replay/rioop.c /^void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_fdatasync ./src/replay/rioop.c /^void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_fstat ./src/replay/rioop.c /^void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_fsync ./src/replay/rioop.c /^void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_getdents ./src/replay/rioop.c /^void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_lchown ./src/replay/rioop.c /^void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_lseek ./src/replay/rioop.c /^void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_mkdir ./src/replay/rioop.c /^void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_open ./src/replay/rioop.c /^void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_)$/;" f -rioop_read ./src/replay/rioop.c /^void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_rename ./src/replay/rioop.c /^void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_rmdir ./src/replay/rioop.c /^void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_run ./src/replay/rioop.c /^void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_stat ./src/replay/rioop.c /^void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_unlink ./src/replay/rioop.c /^void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -rioop_write ./src/replay/rioop.c /^void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f -root ./src/datas/btree.h /^ btreelem_s *root; \/**< The root element *\/$/;" m struct:btree_s_ -rprocess_destroy ./src/replay/rprocess.c /^void rprocess_destroy(rprocess_s *p)$/;" f -rprocess_map ./src/replay/rworker.h /^ amap_s* rprocess_map; \/**< Holding all processes handled by this worker *\/$/;" m struct:__anon2 -rprocess_new ./src/replay/rprocess.c /^rprocess_s* rprocess_new(const int pid, amap_s *fds_map)$/;" f -rprocess_s ./src/replay/rprocess.h /^} rprocess_s;$/;" t typeref:struct:rprocess_s_ -rprocess_s_ ./src/replay/rprocess.h /^typedef struct rprocess_s_ {$/;" s -rtask_destroy ./src/replay/rtask.c /^void rtask_destroy(rtask_s *task)$/;" f -rtask_new ./src/replay/rtask.c /^rtask_s* rtask_new()$/;" f -rtask_s ./src/replay/rtask.h /^} rtask_s;$/;" t typeref:struct:rtask_s_ -rtask_s_ ./src/replay/rtask.h /^typedef struct rtask_s_ {$/;" s -rtask_update ./src/replay/rtask.c /^void rtask_update(rtask_s *task, void *worker, void *process, char *line,$/;" f -rthread_buffer ./src/replay/rworker.h /^ rbuffer_s *rthread_buffer; \/**< Buffering idle threads to be reused *\/$/;" m struct:__anon2 -rthread_buffer_mutex ./src/replay/rworker.h /^ pthread_mutex_t rthread_buffer_mutex; \/**< Sync access to rthread_buffer *\/$/;" m struct:__anon2 -rthread_destroy ./src/replay/rthread.c /^void rthread_destroy(rthread_s *t)$/;" f -rthread_fd ./src/replay/rthread.h /^ FILE *rthread_fd; \/**< Used for debugging purposes only *\/$/;" m struct:rthread_s_ -rthread_insert_task ./src/replay/rthread.c /^bool rthread_insert_task(rthread_s* t, rtask_s* task)$/;" f -rthread_map ./src/replay/rworker.h /^ amap_s* rthread_map; \/**< Holding all threads handled by this worker *\/$/;" m struct:__anon2 -rthread_new ./src/replay/rthread.c /^rthread_s* rthread_new(const long tid, void *worker)$/;" f -rthread_process_task ./src/replay/rthread.c /^void rthread_process_task(rthread_s* t, rtask_s *task,$/;" f -rthread_pthread_start ./src/replay/rthread.c /^void *rthread_pthread_start(void *data)$/;" f -rthread_s ./src/replay/rthread.h /^} rthread_s;$/;" t typeref:struct:rthread_s_ -rthread_s_ ./src/replay/rthread.h /^typedef struct rthread_s_ {$/;" s -rthread_terminate ./src/replay/rthread.c /^void rthread_terminate(rthread_s* t)$/;" f -rthread_update ./src/replay/rthread.c /^long rthread_update(rthread_s *t, const long tid)$/;" f -rworker_destroy ./src/replay/rworker.c /^void rworker_destroy(rworker_s *w)$/;" f -rworker_fd ./src/replay/rworker.h /^ FILE *rworker_fd; \/**< For debugging purposes only *\/$/;" m struct:__anon2 -rworker_new ./src/replay/rworker.c /^rworker_s* rworker_new(const int rworker_num, amap_s *fds_map,$/;" f -rworker_num ./src/replay/rprocess.h /^ int rworker_num; \/**< The worker number of the responsible worker *\/$/;" m struct:rprocess_s_ -rworker_num ./src/replay/rworker.h /^ int rworker_num; \/**< The current worker ID *\/$/;" m struct:__anon2 -rworker_process_lines ./src/replay/rworker.c /^status_e rworker_process_lines(rworker_s* w, const long num_lines)$/;" f -rworker_s ./src/replay/rworker.h /^} rworker_s;$/;" t typeref:struct:__anon2 -single_threaded ./src/replay/rthread.h /^ bool single_threaded; \/**< Worker is single threaded or not *\/$/;" m struct:rthread_s_ -size ./src/datas/amap.h /^ long size; \/**< The total size\/capacity of the amap *\/$/;" m struct:amap_s_ -size ./src/datas/btree.h /^ int size; \/**< The current size of the binary tree *\/$/;" m struct:btree_s_ -size ./src/datas/hmap.h /^ unsigned int size; \/**< Size of the hmap *\/$/;" m struct:hmap_s_ -size ./src/datas/rbuffer.h /^ int size;$/;" m struct:rbuffer_s_ -size ./src/datas/stack.h /^ unsigned long size; \/**< A count how many elements are in the stack *\/$/;" m struct:stack_s_ -sizes_created ./src/init/itask.h /^ long sizes_created;$/;" m struct:itask_s_ -speed_factor ./src/options.h /^ double speed_factor; \/**< Specifies how fast the test is replayed *\/$/;" m struct:options_s_ -stack_destroy ./src/datas/stack.c /^void stack_destroy(stack_s *s)$/;" f -stack_elem_s ./src/datas/stack.h /^} stack_elem_s;$/;" t typeref:struct:stack_elem_s_ -stack_elem_s_ ./src/datas/stack.h /^typedef struct stack_elem_s_ {$/;" s -stack_is_empty ./src/datas/stack.c /^int stack_is_empty(stack_s *s)$/;" f -stack_new ./src/datas/stack.c /^stack_s *stack_new()$/;" f -stack_new_reverse_from ./src/datas/stack.c /^stack_s* stack_new_reverse_from(stack_s *s)$/;" f -stack_pop ./src/datas/stack.c /^void* stack_pop(stack_s *s)$/;" f -stack_push ./src/datas/stack.c /^void stack_push(stack_s *s, void *data)$/;" f -stack_s ./src/datas/stack.h /^} stack_s;$/;" t typeref:struct:stack_s_ -stack_s_ ./src/datas/stack.h /^typedef struct stack_s_ {$/;" s -start_pthread ./src/utils/utils.c /^void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data)$/;" f -start_time ./src/generate/generate.h /^ long start_time; \/**< The start time from the .capture file *\/$/;" m struct:generate_s_ -status ./src/generate/gtask.h /^ int status; \/**< Operation\/syscall return status *\/$/;" m struct:gtask_s_ -status_e ./src/defaults.h /^} status_e;$/;" t typeref:enum:status_e_ -status_e_ ./src/defaults.h /^typedef enum status_e_ {$/;" g -strtok2_r ./src/utils/utils.c /^char* strtok2_r(char *str, char *delim, char **saveptr)$/;" f -strunquote ./src/utils/utils.c /^void strunquote(char *str)$/;" f -task_buffer ./src/replay/rworker.h /^ rbuffer_s *task_buffer; \/**< Buffering thread tasks to be reused *\/$/;" m struct:__anon2 -task_buffer_mutex ./src/replay/rworker.h /^ pthread_mutex_t task_buffer_mutex; \/**< To sync access to task_buffer *\/$/;" m struct:__anon2 -tasks ./src/replay/rthread.h /^ rbuffer_s* tasks; \/**< Holds all outstanding tasks *\/$/;" m struct:rthread_s_ -terminate ./src/generate/gparser.h /^ bool terminate; \/**< The parser thread will terminate if set to true *\/$/;" m struct:gparser_s_ -terminate ./src/generate/gwriter.h /^ bool terminate; \/**< The writer thread will terminate if set to true *\/$/;" m struct:gwriter_s_ -terminate ./src/init/ithread.h /^ bool terminate; \/**< Indicates that thread can terminate *\/$/;" m struct:ithread_s_ -terminate ./src/replay/rprocess.h /^ int terminate; \/**< Indicates whether the worker is terminating or not *\/$/;" m struct:rprocess_s_ -terminate ./src/replay/rthread.h /^ bool terminate; \/**< True if thread shall terminate *\/$/;" m struct:rthread_s_ -threads_map ./src/init/init.h /^ amap_s *threads_map;$/;" m struct:init_s_ -tid ./src/generate/gtask.h /^ long tid; \/**< The thread ID *\/$/;" m struct:gtask_s_ -tid ./src/replay/rthread.h /^ long tid; \/**< The virtual thread id *\/$/;" m struct:rthread_s_ -toks ./src/replay/rtask.h /^ char *toks[MAX_TOKENS+1]; \/**< The tokens parsed from the .replay line *\/$/;" m struct:rtask_s_ -top ./src/datas/stack.h /^ stack_elem_s *top; \/**< The top element of the stack, NULL if empty *\/$/;" m struct:stack_s_ -trash ./src/options.h /^ bool trash; \/**< If set ioreplay will trash the environment *\/$/;" m struct:options_s_ -unsure ./src/generate/vsize.h /^ bool unsure; \/**< True if the file type is not fully clear *\/$/;" m struct:vsize_s_ -updates ./src/generate/vsize.h /^ long updates; \/**< Amount of times this vsize has been updated *\/$/;" m struct:vsize_s_ -user ./src/options.h /^ char *user; \/**< The user name to run the test as *\/$/;" m struct:options_s_ -utests_run ./src/utests.c /^void utests_run()$/;" f -vfd ./src/generate/gtask.h /^ vfd_s *vfd; \/**< A pointer to the virtual file descriptor *\/$/;" m struct:gtask_s_ -vfd_buffer ./src/generate/generate.h /^ rbuffer_s *vfd_buffer; \/**< A virtual fd buffer, for reusing these *\/$/;" m struct:generate_s_ -vfd_destroy ./src/vfd.c /^void vfd_destroy(vfd_s *vfd)$/;" f -vfd_map ./src/generate/gprocess.h /^ hmap_s *vfd_map; \/**< All virtual file descriptors of that process *\/$/;" m struct:gprocess_s_ -vfd_new ./src/vfd.c /^vfd_s* vfd_new(const int fd, const long mapped_fd, char *path)$/;" f -vfd_print ./src/vfd.c /^void vfd_print(vfd_s *vfd)$/;" f -vfd_s ./src/vfd.h /^} vfd_s;$/;" t typeref:struct:vfd_s_ -vfd_s_ ./src/vfd.h /^typedef struct vfd_s_ {$/;" s -vfd_update ./src/vfd.c /^void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path)$/;" f -vsize ./src/generate/gtask.h /^ vsize_s *vsize; \/**< Pointer to the virtual size object *\/$/;" m struct:gtask_s_ -vsize ./src/generate/vsize.h /^ long vsize; \/**< The virtual size *\/$/;" m struct:vsize_s_ -vsize ./src/init/itask.h /^ long vsize;$/;" m struct:itask_s_ -vsize ./src/replay/rtask.h /^ unsigned long vsize; \/**< The vsize *\/$/;" m struct:rtask_s_ -vsize2 ./src/generate/gtask.h /^ vsize_s *vsize2; \/**< Pointer to a second virtual size object *\/$/;" m struct:gtask_s_ -vsize_adjust ./src/generate/vsize.c /^void vsize_adjust(vsize_s *v, vfd_s* vfd)$/;" f -vsize_close ./src/generate/vsize.c /^void vsize_close(vsize_s *v, void* vfd)$/;" f -vsize_deficit ./src/generate/vsize.h /^ long vsize_deficit; \/**< Size to use for file creating during init mode *\/$/;" m struct:vsize_s_ -vsize_destroy ./src/generate/vsize.c /^void vsize_destroy(vsize_s *v)$/;" f -vsize_map ./src/generate/generate.h /^ hmap_s *vsize_map; \/**< A hash map of all virtual size objects *\/$/;" m struct:generate_s_ -vsize_mkdir ./src/generate/vsize.c /^void vsize_mkdir(vsize_s *v, const char *path)$/;" f -vsize_new ./src/generate/vsize.c /^vsize_s* vsize_new(char *file_path, const unsigned long id,$/;" f -vsize_open ./src/generate/vsize.c /^void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags)$/;" f -vsize_read ./src/generate/vsize.c /^void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes)$/;" f -vsize_rename ./src/generate/vsize.c /^void vsize_rename(vsize_s *v, vsize_s *v2,$/;" f -vsize_rmdir ./src/generate/vsize.c /^void vsize_rmdir(vsize_s *v, const char *path)$/;" f -vsize_s ./src/generate/vsize.h /^} vsize_s;$/;" t typeref:struct:vsize_s_ -vsize_s_ ./src/generate/vsize.h /^typedef struct vsize_s_ {$/;" s -vsize_seek ./src/generate/vsize.c /^void vsize_seek(vsize_s *v, void *vfd, const long new_offset)$/;" f -vsize_stat ./src/generate/vsize.c /^void vsize_stat(vsize_s *v, const char *path)$/;" f -vsize_unlink ./src/generate/vsize.c /^void vsize_unlink(vsize_s *v, const char *path)$/;" f -vsize_write ./src/generate/vsize.c /^void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes)$/;" f -wd_base ./src/options.h /^ char *wd_base; \/**< The working directory base *\/$/;" m struct:options_s_ -whence ./src/generate/gtask.h /^ long whence; \/**< Whence *\/$/;" m struct:gtask_s_ -worker ./src/replay/rtask.h /^ void *worker; \/* The responsible worker object *\/$/;" m struct:rtask_s_ -worker ./src/replay/rthread.h /^ void *worker; \/**< The responsible worker object *\/$/;" m struct:rthread_s_ -write_pos ./src/datas/rbuffer.h /^ sig_atomic_t write_pos;$/;" m struct:rbuffer_s_ -writer ./src/generate/generate.h /^ struct gwriter_s_ *writer; \/**< A pointer to the writer object *\/$/;" m struct:generate_s_ typeref:struct:generate_s_::gwriter_s_ diff --git a/ioriot/Makefile b/ioriot/Makefile new file mode 100644 index 0000000..d933d72 --- /dev/null +++ b/ioriot/Makefile @@ -0,0 +1,36 @@ +#DEBUG=-g3 -ggdb3 -pg +NAME=ioriot +LIBS=-pthread +CFLAGS=-Wall -std=gnu99 -pedantic +STATIC=#-static +DESTDIR=/opt/ioriot/bin +SRCS=$(wildcard src/*.c src/*/*.c) +HDRS=$(SRCS:.c=.h) +OBJS=$(SRCS:.c=.o) +all: compile +quick: clean ctags compile sudo_install +cshell: compile + gdb -ex='break main; run' --args ./$(NAME) +test: compile + gdb -ex=run --args ./$(NAME) -U +compile: $(OBJS) + $(CC) $(STATIC) $(DEBUG) $(LIBS) $(OBJS) -o $(NAME) +%.o: %.c %.h + $(CC) $(STATIC) $(DEBUG) $(LIBS) -c $(CFLAGS) $< -o $@ +clean: + rm -v ioriot ./src/*.o ./src/*/*.o 2>/dev/null || exit 0 +install: + test ! -d $(DESTDIR) && mkdir -p $(DESTDIR) || exit 0 + cp -v $(NAME) $(DESTDIR) + @echo "Don't forget to add $(DESTDIR) to your PATH as follows:" + @echo " export PATH=\$$PATH:$(DESTDIR)" +uninstall: + test ! -z "$(DESTDIR)" && test -f $(DESTDIR)/$(NAME) && rm -v $(DESTDIR)/$(NAME) || exit 0 +deinstall: uninstall +astyle: + astyle -n --style=linux src/*.h src/*/*.h + astyle -n --style=linux src/*.c src/*/*.c +todo: + fgrep ../TODO ./src/* +ctags: + ctags ./src/*.{h,c} ./src/*/*.{h,c} diff --git a/ioriot/src/capture/capture.c b/ioriot/src/capture/capture.c new file mode 100644 index 0000000..aaad6cb --- /dev/null +++ b/ioriot/src/capture/capture.c @@ -0,0 +1,99 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "capture.h" + +#include + +status_e capture_run(options_s *opts) +{ + int status = 0; + struct utsname uts; + + // To make it nicer we should iterate over PATH instead + char *staprun_paths[3] = { + "/usr/bin/staprun", + "/usr/local/bin/staprun", + "/bin/staprun" + }; + int num_staprun_paths = 3; + + if (0 != uname(&uts)) { + Errno("Could not identify release of currently running Kernel!"); + } + + Put("Release of currently running Kernel: %s", uts.release); + char modules_dir[128]; + sprintf(modules_dir, "/opt/ioriot/systemtap/%s", uts.release); + Put("Changing directory to module path: %s/", modules_dir); + + if (0 != chdir(modules_dir)) { + Errno("Could not change into '%s', please ensure that the compiled " + "SystemTap modules correspond to the currently running Kernel " + "and that these are installed properly!\n", + modules_dir); + } + + if (0 != access(opts->module, R_OK)) { + Errno("Module '%s/%s' can't be read, please make sure that the " + "SystemTap Kernel modules are installed!", + modules_dir, opts->module); + } + + char *staprun_path = NULL; + for (int i = 0; i < num_staprun_paths; ++i) { + if (0 == access(staprun_paths[i], X_OK)) { + staprun_path = staprun_paths[i]; + //Put("SystemTap command path: %s", staprun_path); + break; + } + } + + if (staprun_path == NULL) { + Errno("Can't find 'staprun' command, please ensure to have the SystemTap " + "runtime (usually package 'systemtap-runtime') installed!"); + } + + char staprun_command[128]; + if (opts->pid >= 0) { + sprintf(staprun_command, "%s %s -v -o %s -x %d", staprun_path, opts->module, + opts->capture_file, opts->pid); + } else { + sprintf(staprun_command, "%s %s -v -o %s", staprun_path, opts->module, + opts->capture_file); + } + + Out("NOTICE: It is good practise first to stop all processes, then to "); + Out("start capturing, and then to start all processes again. The reason "); + Out("is that processes may have already open file handles. In that case "); + Out("I/O Riot would be unable to replay these! This may be improved "); + Put("in a future release!"); + Put("To abort capturing now send Ctrl+C, otherwise wait 1h"); + Put("Capturing I/O via: '%s'", staprun_command); + + char buf[1024]; + FILE *fp; + + if ((fp = popen(staprun_command, "r")) == NULL) { + Errno("Unable to invoke staprun command!"); + } + while (fgets(buf, 1024, fp) != NULL) + Out("stapio: %s", buf); + + if (0 != pclose(fp)) { + Error("Problems invoking staprun command!"); + } + + return status; +} diff --git a/ioriot/src/capture/capture.h b/ioriot/src/capture/capture.h new file mode 100644 index 0000000..7718d3e --- /dev/null +++ b/ioriot/src/capture/capture.h @@ -0,0 +1,30 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CAPTURE_H +#define CAPTURE_H + +#include "../defaults.h" +#include "../utils/futils.h" +#include "../options.h" + +/** + * @brief Captures I/O to a .capture file by using stap from SystemTap + * + * @param opts The options object + * @return SUCCESS if everything went fine + */ +status_e capture_run(options_s *opts); + +#endif // CAPTURE_H diff --git a/ioriot/src/cleanup/cleanup.c b/ioriot/src/cleanup/cleanup.c new file mode 100644 index 0000000..570f8a7 --- /dev/null +++ b/ioriot/src/cleanup/cleanup.c @@ -0,0 +1,30 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "cleanup.h" + +#include "../mounts.h" + +status_e cleanup_run(options_s *opts) +{ + set_limits_drop_root(opts->user); + mounts_s *m = mounts_new(opts); + + if (opts->purge) + mounts_purge(m); + else + mounts_trash(m); + + return SUCCESS; +} diff --git a/ioriot/src/cleanup/cleanup.h b/ioriot/src/cleanup/cleanup.h new file mode 100644 index 0000000..127badf --- /dev/null +++ b/ioriot/src/cleanup/cleanup.h @@ -0,0 +1,29 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CLEANUP_H +#define CLEANUP_H + +#include "../defaults.h" +#include "../options.h" + +/** + * @brief Cleans up all files and directories of a given test + * + * @brief opts The options object + * @return SUCCESS in case everything went fine + */ +status_e cleanup_run(options_s *opts); + +#endif // CLEANUP_H diff --git a/ioriot/src/datas/amap.c b/ioriot/src/datas/amap.c new file mode 100644 index 0000000..806a3f8 --- /dev/null +++ b/ioriot/src/datas/amap.c @@ -0,0 +1,264 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "amap.h" + +/** + * @brief Creates a new array map + * + * @param size The array map size + * @param mmapped true if the memory should be mmapped + * @return The new amap object + */ +static amap_s *_amap_new(long size, bool mmapped) +{ + amap_s *a = NULL; + void ***arrays = NULL; + + // Calculate a multiple of 1024, but at least in size of 'size'. + if (size % 1024 != 0) { + size = 1024*(1+(long)(size/1024)); + } + + if (size < 1) { + Error("Size overflow"); + } + + int num_arrays = size / AMAP_MAX_ARRAY_LENGTH; + + if (mmapped) { + a = Mmapshared(amap_s); + arrays = Cmapshared(num_arrays, void**); + } else { + a = Malloc(amap_s); + arrays = Calloc(num_arrays, void**); + } + + for (int i = 0; i < num_arrays; ++i) { + if (mmapped) { + //Put("%d", AMAP_MAX_ARRAY_LENGTH); + arrays[i] = Cmapshared(AMAP_MAX_ARRAY_LENGTH, void*); + } else { + arrays[i] = Calloc(AMAP_MAX_ARRAY_LENGTH, void*); + } + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + arrays[i][j] = NULL; + } + } + + a->arrays = arrays; + a->num_arrays = num_arrays; + a->size = size; + a->data_destroy = NULL; + a->mmapped = mmapped; + + return a; +} + +/** + * @brief Creates a new array map + * + * @param size The array map size + * @return The new amap object + */ +amap_s* amap_new(const long size) +{ + return _amap_new(size, false); +} + +/** + * @brief Creates a new mmapped array map + * + * @param size The array map size + * @return The new amap object + */ +amap_s* amap_new_mmapped(const long size) +{ + return _amap_new(size, true); +} + +/** + * @brief Destroys a mmap object + * + * @a The new amap object + */ +void amap_destroy(amap_s* a) +{ + if (!a) { + return; + } + + // Don't bother, the mmapped version of amap will stay alive until + // process terminations. And after process termination everything + // will be cleaned up automatically by Linux. + if (a->mmapped) { + return; + } + + for (int i = 0; i < a->num_arrays; ++i) { + if (a->data_destroy) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) + if (a->arrays[i][j]) { + a->data_destroy(a->arrays[i][j]); + } + } + free(a->arrays[i]); + } + free(a->arrays); + free(a); +} + +/** + * @brief Resets a mmap object + * + * This resets all entries to NULL. + * + * @a The new amap object + */ +void amap_reset(amap_s* a) +{ + for (int i = 0; i < a->num_arrays; ++i) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + if (a->data_destroy) { + if (a->arrays[i][j]) { + a->data_destroy(a->arrays[i][j]); + } + } + a->arrays[i][j] = NULL; + } + } +} + +int amap_set(amap_s *a, const long position, void* value) +{ + if (position >= a->size) + return -1; + int which_array = position / AMAP_MAX_ARRAY_LENGTH; + int array_pos = position % AMAP_MAX_ARRAY_LENGTH; + a->arrays[which_array][array_pos] = value; + return 0; +} + +void* amap_get(amap_s *a, const long position) +{ + if (position >= a->size) + return NULL; + int which_array = position / AMAP_MAX_ARRAY_LENGTH; + int array_pos = position % AMAP_MAX_ARRAY_LENGTH; + return a->arrays[which_array][array_pos]; +} + +void* amap_unset(amap_s *a, const long position) +{ + if (position >= a->size) + return NULL; + int which_array = position / AMAP_MAX_ARRAY_LENGTH; + int array_pos = position % AMAP_MAX_ARRAY_LENGTH; + void *value = a->arrays[which_array][array_pos]; + a->arrays[which_array][array_pos] = NULL; + return value; +} + +void amap_run_cb(amap_s *a, void (*cb)(void *data)) +{ + for (int i = 0; i < a->num_arrays; ++i) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + if (a->arrays[i][j]) + cb(a->arrays[i][j]); + } + } +} + +void amap_print(amap_s* a) +{ + Put("amap_s (%p):", (void*)a); + Put("\tmmapped: %d", a->mmapped); + Put("\tmax_array_length: %d", AMAP_MAX_ARRAY_LENGTH); + Put("\tnum_arrays: %d", a->num_arrays); + Put("\tsize: %lu", a->size); + Out("\toccupied slots: "); + for (int i = 0; i < a->num_arrays; ++i) { + for (int j = 0; j < AMAP_MAX_ARRAY_LENGTH; ++j) { + if (a->arrays[i][j] != NULL) { + Out("%d:%d ", i, j); + } + } + } + Out("\n"); +} + +void _amap_test(amap_s *a) +{ + assert(0 == amap_set(a, 0, (void*)10)); + assert(0 == amap_set(a, 1, (void*)11)); + assert(0 == amap_set(a, 2, (void*)12)); + assert(0 == amap_set(a, 3, (void*)a)); + assert(10 == (long) amap_get(a, 0)); + assert(11 == (long) amap_get(a, 1)); + assert(12 == (long) amap_get(a, 2)); + assert(a == amap_get(a, 3)); + + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH-1, (void*) 23)); + assert(23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH-1)); + + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH, (void*) 42)); + assert(42 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH)); + + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2-1, (void*) (23+42))); + assert(42+23 == (long) amap_get(a, AMAP_MAX_ARRAY_LENGTH*2-1)); + assert(0 == amap_set(a, AMAP_MAX_ARRAY_LENGTH*2, (void*) 23)); + + + assert(NULL == amap_get(a, 1024*1024*9-1)); + assert(0 == amap_set(a, 1024*1024*9-1, (void*) 0x1)); + assert(0x1 == (long) amap_get(a, 1024*1024*9-1)); + assert(0x1 == (long) amap_unset(a, 1024*1024*9-1)); + assert(NULL == amap_get(a, 1024*1024*9-1)); + + assert(0 == amap_set(a, 1024*1024*9, (void*) 100)); + assert(100 == (long) amap_get(a, 1024*1024*9)); + + assert(0 == amap_set(a, 1024*1024*9+1, (void*) 101)); + assert(101 == (long) amap_get(a, 1024*1024*9+1)); + + assert(0 == amap_set(a, 1024*1024*10-2, (void*) 102)); + assert(102 == (long) amap_get(a, 1024*1024*10-2)); + + assert(0 == amap_set(a, 1024*1024*10-1, a)); + assert(a == amap_get(a, 1024*1024*10-1)); + //amap_print(a); + + assert(a == amap_unset(a, 1024*1024*10-1)); + assert(a != amap_unset(a, 1024*1024*10-1)); + //amap_print(a); +} + +void amap_test(void) +{ + // First test the non-mmapped version + amap_s* a = amap_new(1024*1024*10); + _amap_test(a); + amap_destroy(a); + + // Now test the mapped version + a = amap_new_mmapped(1024*1024*10); + _amap_test(a); + amap_destroy(a); + + // Another test with non-alligned size + a = amap_new(1024*1024*10+1); + _amap_test(a); + amap_destroy(a); +} + diff --git a/ioriot/src/datas/amap.h b/ioriot/src/datas/amap.h new file mode 100644 index 0000000..882a7c5 --- /dev/null +++ b/ioriot/src/datas/amap.h @@ -0,0 +1,49 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef AMAP_H +#define AMAP_H + +#include "../defaults.h" + +#define AMAP_MAX_ARRAY_LENGTH 1024*8 + +/** + * @brief Implements an array map data structure + * + * This array map can hold a HUGE amount of entries by allocating multiple + * smaller arrays. There are two version of the amap data structure available: + * a memory mapped (mmap) and a normal version. The memory mapped version can + * be used for IPC between various processes. + */ +typedef struct amap_s_ { + void*** arrays; /**< The pointers to the amap arrays */ + int num_arrays; /**< The amount of arrays used in the amap */ + long size; /**< The total size/capacity of the amap */ + bool mmapped; /**< True if amap is memory mapped */ + void (*data_destroy)(void *data); /**< Callback to destroy all elements */ +} amap_s; + +amap_s* amap_new(const long size); +amap_s* amap_new_mmapped(const long size); +int amap_set(amap_s *a, const long position, void* value); +void* amap_get(amap_s *a, const long position); +void* amap_unset(amap_s *a, const long position); +void amap_print(amap_s *a); +void amap_destroy(amap_s *a); +void amap_reset(amap_s *a); +void amap_run_cb(amap_s *a, void (*cb)(void *data)); +void amap_test(void); + +#endif // AMAP_H diff --git a/ioriot/src/datas/btree.c b/ioriot/src/datas/btree.c new file mode 100644 index 0000000..da5da48 --- /dev/null +++ b/ioriot/src/datas/btree.c @@ -0,0 +1,169 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "btree.h" + +btree_s* btree_new() +{ + btree_s *b = Malloc(btree_s); + *b = (btree_s) { + .root = NULL, .size = 0 + }; + return b; +} + +void btree_destroy(btree_s* b) +{ + if (b->root) + btreelem_destroy_r(b->root); + free(b); +} + +void btree_destroy2(btree_s* b) +{ + if (b->root) + btreelem_destroy_r2(b->root); + free(b); +} + +int btree_insert(btree_s* b, int key, void *data) +{ + int ret = 1; + + if (b->root == NULL) { + b->root = btreelem_new(key, data); + ret = 0; + } else { + ret = btreelem_insert_r(b->root, key, data); + } + + if (ret == 0) { + b->size++; + } + + return ret; +} + +void* btree_get(btree_s* b, int key) +{ + if (b->root == NULL) + return NULL; + + return btreelem_get_r(b->root, key); +} + +void btree_print(btree_s* b) +{ + btreelem_print_r(b->root, 0); +} + +btreelem_s* btreelem_new(int key, void *data) +{ + btreelem_s *e = Malloc(btreelem_s); + *e = (btreelem_s) { + .key = key, .data = data, .left = NULL, .right = NULL + }; + return e; +} + +void btreelem_destroy_r(btreelem_s* e) +{ + if (e->left) { + btreelem_destroy_r(e->left); + } + if (e->right) { + btreelem_destroy_r(e->right); + } + + free(e); +} + +void btreelem_destroy_r2(btreelem_s* e) +{ + if (e->left) + btreelem_destroy_r(e->left); + if (e->right) + btreelem_destroy_r(e->right); + if (e->data) + btree_destroy(e->data); + + free(e); +} + +int btreelem_insert_r(btreelem_s* e, int key, void *data) +{ + int ret = 0; + + if (e->key == key) { + ret = 1; + } + + else if (e->key > key) { + if (e->left == NULL) { + e->left = btreelem_new(key, data); + } else { + ret = btreelem_insert_r(e->left, key, data); + } + } + + else { + if (e->right == NULL) { + e->right = btreelem_new(key, data); + } else { + ret = btreelem_insert_r(e->right, key, data); + } + } + + return ret; +} + +void* btreelem_get_r(btreelem_s* e, int key) +{ + void *data = NULL; + + if (e->key == key) { + data = e->data; + } + + else if (e->key > key) { + if (e->left) { + data = btreelem_get_r(e->left, key); + } + } + + else { + if (e->right) { + data = btreelem_get_r(e->right, key); + } + } + + return data; +} + +void btreelem_print_r(btreelem_s* e, int depth) +{ + for (int i = 0; i < depth; ++i) { + Out(" "); + } + Put("%d\n", e->key); + + if (e->left) { + btreelem_print_r(e->left, depth); + } + + if (e->right) { + btreelem_print_r(e->right, depth+1); + } +} + diff --git a/ioriot/src/datas/btree.h b/ioriot/src/datas/btree.h new file mode 100644 index 0000000..55da560 --- /dev/null +++ b/ioriot/src/datas/btree.h @@ -0,0 +1,52 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BTREE_H +#define BTREE_H + +#include "../defaults.h" + +/** + * @brief This defines an element of the binary tree data structure + */ +typedef struct btreelem_ { + struct btreelem_ *left; /**< The next element to the left */ + struct btreelem_ *right; /**< The next element to the right */ + int key; /**< The key of the element */ + void *data; /**< A pointer to the data stored in this element */ +} btreelem_s; + +/** + * @brief This defines a binary tree data structure. + */ +typedef struct btree_s_ { + btreelem_s *root; /**< The root element */ + int size; /**< The current size of the binary tree */ +} btree_s; + +btree_s* btree_new(); +void btree_destroy(btree_s *b); +void btree_destroy2(btree_s *b); +int btree_insert(btree_s *b, int key, void *data); +void* btree_get(btree_s *b, int key); +void btree_print(btree_s *b); + +btreelem_s* btreelem_new(int key, void *data); +void btreelem_destroy_r(btreelem_s *e); +void btreelem_destroy_r2(btreelem_s *e); +int btreelem_insert_r(btreelem_s *e, int key, void *data); +void* btreelem_get_r(btreelem_s *e, int key); +void btreelem_print_r(btreelem_s *e, int depth); + +#endif // BTREE_H diff --git a/ioriot/src/datas/hmap.c b/ioriot/src/datas/hmap.c new file mode 100644 index 0000000..96c373e --- /dev/null +++ b/ioriot/src/datas/hmap.c @@ -0,0 +1,362 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hmap.h" + +#define _Using_string_keys h->keys != NULL + +unsigned int hmap_get_addr(hmap_s *h, char *key) +{ + unsigned long hash = 5381; + int len = strlen(key); + + for (int i = 0; i < len; ++i) { + hash = ((hash << 5) + hash) + key[i]; /* hash * 33 + c */ + } + + return (unsigned int) (hash % h->size); +} + +unsigned int hmap_get_addr_l(hmap_s *h, const long key) +{ + return (unsigned int) (key % h->size); +} + +hmap_s *_hmap_new(unsigned int init_size) +{ + hmap_s *h = Malloc(hmap_s); + + h->size = init_size; + h->data = Calloc(init_size, void*); + h->l = Calloc(init_size, list_s*); + h->data_destroy = NULL; + h->keys = NULL; + h->keys_l = NULL; + + Mset(h->data, 0, init_size, void*); + Mset(h->l, 0, init_size, list_s*); + + return h; +} + +hmap_s *hmap_new(unsigned int init_size) +{ + hmap_s *h = _hmap_new(init_size); + h->keys = Calloc(init_size, char*); + Mset(h->keys, 0, init_size, char*); + + return h; +} + +hmap_s *hmap_new_l(unsigned int init_size) +{ + hmap_s *h = _hmap_new(init_size); + h->keys_l = Calloc(init_size, int); + Mset(h->keys_l, -1, init_size, int); + + return h; +} + +void hmap_destroy(hmap_s *h) +{ + for (int i = 0; i < h->size; ++i) { + if (h->l[i]) { + list_s *l = h->l[i]; + if (h->data_destroy) + l->data_destroy = h->data_destroy; + list_destroy(h->l[i]); + } + if (h->data[i] && h->data_destroy) { + h->data_destroy(h->data[i]); + } + } + + free(h->data); + if (h->keys) + free(h->keys); + if (h->keys_l) + free(h->keys_l); + free(h->l); + free(h); + + return; +} + +int hmap_insert(hmap_s *h, char *key, void *data) +{ + if (data == NULL) { + Error("insert data can not be NULL"); + } + + int addr = hmap_get_addr(h, key); + + if (h->data[addr]) { + + if (strcmp(key, h->keys[addr]) == 0) { + // Key already exists + return 0; + } + + // There is already data, collision, create a linked list + list_s *l = h->l[addr] = list_new(); + list_key_insert(l, h->keys[addr], h->data[addr]); + list_key_insert(l, key, data); + + // Not needed anymore, as the elements are in the linked list now. + free(h->keys[addr]); + h->data[addr] = h->keys[addr] = NULL; + + return 1; + + } else if (h->l[addr]) { + // There was a collision at this address before. Insert + // the element to the linked list. Returns 0 if key is already + // in the list (no additional insert made) or 1 otherwise. + return list_key_insert(h->l[addr], key, data); + } + + // New entry on a collision free address + h->data[addr] = data; + h->keys[addr] = Clone(key); + + return 1; +} + +int hmap_insert_l(hmap_s *h, const long key, void *data) +{ + if (data == NULL) { + Error("insert data can not be NULL"); + } + + int addr = hmap_get_addr_l(h, key); + + if (h->data[addr]) { + + if (key == h->keys_l[addr]) { + // Key already exists + return 0; + } + + // There is already data, collision, create a linked list + list_s *l = h->l[addr] = list_new_l(); + list_key_insert_l(l, h->keys_l[addr], h->data[addr]); + list_key_insert_l(l, key, data); + + // Not needed anymore, as the elements are in the linked list now. + h->data[addr] = NULL; + h->keys_l[addr] = -1; + + return 1; + + } else if (h->l[addr]) { + // There was a collision at this address before. Insert + // the element to the linked list. Returns 0 if key is already + // in the list (no additional insert made) or 1 otherwise. + return list_key_insert_l(h->l[addr], key, data); + } + + // New entry on a collision free address + h->data[addr] = data; + h->keys_l[addr] = key; + + return 1; +} + +void* hmap_remove(hmap_s *h, char *key) +{ + int addr = hmap_get_addr(h, key); + + if (h->data[addr] != NULL) { + void *data = h->data[addr]; + free(h->keys[addr]); + h->data[addr] = h->keys[addr] = NULL; + return data; + + } else if (h->l[addr] != NULL) { + // There was a collision at this address before. Remove + // the element to the linked list. Returns the object if key is + // already in the list (no additional insert made) or NULL + // otherwise. + return list_key_remove(h->l[addr], key); + } + + // Key is not present + return NULL; +} + +void* hmap_remove_l(hmap_s *h, const long key) +{ + int addr = hmap_get_addr_l(h, key); + + if (h->data[addr] != NULL) { + void *data = h->data[addr]; + h->data[addr] = NULL; + h->keys_l[addr] = -1; + return data; + + } else if (h->l[addr] != NULL) { + // There was a collision at this address before. Remove + // the element to the linked list. Returns the object if key is + // already in the list (no additional insert made) or NULL + // otherwise. + return list_key_remove_l(h->l[addr], key); + } + + // Key is not present + return NULL; +} + +void* hmap_get(hmap_s *h, char *key) +{ + int addr = hmap_get_addr(h, key); + if (h->data[addr] && strcmp(h->keys[addr], key) == 0) { + return h->data[addr]; + + } else if (h->l[addr]) { + return list_key_get(h->l[addr], key); + } + + return NULL; +} + +void* hmap_get_l(hmap_s *h, const long key) +{ + int addr = hmap_get_addr_l(h, key); + if (h->data[addr] && h->keys_l[addr] == key) { + return h->data[addr]; + + } else if (h->l[addr]) { + return list_key_get_l(h->l[addr], key); + } + + return NULL; +} + +void hmap_run_cb(hmap_s* h, void (*cb)(void *data)) +{ + for (int i = 0; i < h->size; ++i) { + if (h->l[i]) { + list_s *l = h->l[i]; + list_run_cb(l, cb); + } + if (h->data[i]) { + cb(h->data[i]); + } + } +} + +void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_) +{ + for (int i = 0; i < h->size; ++i) { + if (h->l[i]) { + list_s *l = h->l[i]; + list_run_cb2(l, cb, data_); + } + if (h->data[i]) { + cb(h->data[i], data_); + } + } +} + +void hmap_print(hmap_s *h) +{ + for (int i = 0; i < h->size; ++i) { + if (h->data[i]) { + if (_Using_string_keys) { + Put("hmap:%p addr:%d key:'%s'", (void*)h, i, h->keys[i]); + } else { + Put("hmap:%p addr:%d key:%d", (void*)h, i, h->keys_l[i]); + } + } else if (h->l[i]) { + Put("hmap:%p addr:%d LIST", (void*)h, i); + list_print(h->l[i]); + } + } +} + +static void _hmap_test(hmap_s *h) +{ + void* somedata = (void*)h; + + assert(1 == hmap_insert(h, "someval", (void*)23)); + assert(1 == hmap_insert(h, "another value", (void*)123)); + + assert(1 == hmap_insert(h, "mimecast", somedata)); + assert(0 == hmap_insert(h, "mimecast", somedata)); + assert(1 == hmap_insert(h, "is", somedata)); + assert(1 == hmap_insert(h, "hiring", somedata)); + + assert(NULL != hmap_get(h, "mimecast")); + assert(NULL == hmap_get(h, "Mimecast")); + + assert(NULL != hmap_remove(h, "mimecast")); + assert(NULL == hmap_remove(h, "mimecast")); + + assert(1 == hmap_insert(h, "mimecast", somedata)); + assert(NULL != hmap_get(h, "mimecast")); + + assert(23 == (long)hmap_get(h, "someval")); + assert(23 == (long)hmap_get(h, "someval")); + + assert(123 == (long)hmap_remove(h, "another value")); + assert(0 == (long)hmap_remove(h, "another value")); + assert(NULL == hmap_get(h, "another value")); + + //hmap_print(h); +} + +static void _hmap_test_l(hmap_s *h) +{ + void* somedata = (void*)h; + + assert(1 == hmap_insert_l(h, 1, (void*)23)); + assert(1 == hmap_insert_l(h, 5, (void*)123)); + + assert(1 == hmap_insert_l(h, 3, somedata)); + assert(0 == hmap_insert_l(h, 3, somedata)); + assert(1 == hmap_insert_l(h, 4, somedata)); + assert(1 == hmap_insert_l(h, 6, somedata)); + + assert(NULL != hmap_get_l(h, 3)); + assert(NULL == hmap_get_l(h, 7)); + + assert(NULL != hmap_remove_l(h, 3)); + assert(NULL == hmap_remove_l(h, 3)); + + assert(1 == hmap_insert_l(h, 3, somedata)); + assert(NULL != hmap_get_l(h, 3)); + + assert(23 == (long)hmap_get_l(h, 1)); + assert(23 == (long)hmap_get_l(h, 1)); + + assert(123 == (long)hmap_remove_l(h, 5)); + assert(0 == (long)hmap_remove_l(h, 5)); + assert(NULL == hmap_get_l(h, 5)); +} + +void hmap_test(void) +{ + hmap_s* h = hmap_new(1024); + _hmap_test(h); + hmap_destroy(h); + + h = hmap_new(2); + _hmap_test(h); + hmap_destroy(h); + + h = hmap_new_l(1024); + _hmap_test_l(h); + hmap_print(h); + hmap_destroy(h); +} diff --git a/ioriot/src/datas/hmap.h b/ioriot/src/datas/hmap.h new file mode 100644 index 0000000..9d1978b --- /dev/null +++ b/ioriot/src/datas/hmap.h @@ -0,0 +1,56 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HMAP_H +#define HMAP_H + +#include "../defaults.h" +#include "list.h" + +/** + * @brief A hash map data structure + * + * There are two version of this hmap data structure. One version is utilising + * string keys and the other one is utilising long keys. + * + * On hash collision the data structure will make use of a "named" linked list, + * whereas every member of the linked list has either a string key or a long + * key associated. + */ +typedef struct hmap_s_ { + char **keys; /**< List of all keys, NULL if nothing at a address */ + int *keys_l; /**< Same as keys, but for long keys */ + void **data; /**< Pointers to the stored data, NULL if nothing there */ + list_s **l; /**< Pointers to the linked lists, used on hash collision */ + void (*data_destroy)(void *data); /**< Callback to destroy all data */ + unsigned int size; /**< Size of the hmap */ +} hmap_s; + +hmap_s* hmap_new(unsigned int init_size); +hmap_s* hmap_new_l(unsigned int init_size); +void hmap_destroy(hmap_s* h); +void hmap_run_cb(hmap_s* h, void (*cb)(void *data)); +void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_); +int hmap_insert_l(hmap_s* h, const long key, void *data); +int hmap_insert(hmap_s* h, char* key, void *data); +void* hmap_remove_l(hmap_s* h, const long key); +void* hmap_remove(hmap_s* h, char* key); +void* hmap_get_l(hmap_s* h, const long key); +void* hmap_get(hmap_s* h, char* key); +unsigned int hmap_get_addr_l(hmap_s* h, const long key); +unsigned int hmap_get_addr(hmap_s* h, char* key); +void hmap_print(hmap_s* h); +void hmap_test(void); + +#endif // HMAP_H diff --git a/ioriot/src/datas/list.c b/ioriot/src/datas/list.c new file mode 100644 index 0000000..9cc78db --- /dev/null +++ b/ioriot/src/datas/list.c @@ -0,0 +1,279 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "list.h" + + +list_s *list_new() +{ + list_s *l = Malloc(list_s); + *l = (list_s) { + .first = NULL, .data_destroy = NULL + }; + return l; +} + +list_s *list_new_l() +{ + return list_new(); +} + +void list_destroy(list_s *l) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key) + free(current->key); + if (current->data && l->data_destroy) + l->data_destroy(current->data); + list_elem_s *next = current->next; + free(current); + current = next; + } + + free(l); +} + +int list_key_insert(list_s *l, char *key, void *data) +{ + list_elem_s *current = l->first; + + while (current) { + // Already in the list + if (strcmp(current->key, key) == 0) + return 0; + current = current->next; + } + + list_elem_s *e = Malloc(list_elem_s); + + e->prev = NULL; + e->next = l->first; + e->key = Clone(key); + e->key_l = -1; + e->data = data; + + if (l->first) { + l->first->prev = e; + l->first = e; + + } else { + l->first = e; + } + + return 1; +} + +int list_key_insert_l(list_s *l, const long key, void *data) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key_l == key) + return 0; + current = current->next; + } + + list_elem_s *e = Malloc(list_elem_s); + + e->prev = NULL; + e->next = l->first; + e->key = NULL; + e->key_l = key; + e->data = data; + + if (l->first) { + l->first->prev = e; + l->first = e; + + } else { + l->first = e; + } + + return 1; +} + +void _list_elem_remove(list_s *l, list_elem_s *e) +{ + if (l->first == e) { + list_elem_s *first = e->next; + if (first) + first->prev = NULL; + l->first = first; + + } else { + list_elem_s *prev = e->prev; + list_elem_s *next = e->next; + + prev->next = next; + if (next) + next->prev = prev; + } + + if (e->key) + free(e->key); + free(e); +} + +void* list_key_remove(list_s *l, char *key) +{ + list_elem_s *current = l->first; + + while (current) { + if (strcmp(current->key, key) == 0) { + void *data = current->data; + _list_elem_remove(l, current); + return data; + } + current = current->next; + } + + return NULL; +} + +void* list_key_remove_l(list_s *l, const long key) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key_l == key) { + void *data = current->data; + _list_elem_remove(l, current); + return data; + } + current = current->next; + } + + return NULL; +} + +void* list_key_get(list_s *l, char *key) +{ + list_elem_s *current = l->first; + + while (current) { + if (strcmp(current->key, key) == 0) + return current->data; + current = current->next; + } + + return NULL; +} + +void* list_key_get_l(list_s *l, const long key) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key_l == key) + return current->data; + current = current->next; + } + + return NULL; +} + +void list_run_cb(list_s* l, void (*cb)(void *data)) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->data) + cb(current->data); + current = current->next; + } +} + +void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->data) + cb(current->data, data_); + current = current->next; + } +} + +void list_print(list_s *l) +{ + list_elem_s *current = l->first; + + while (current) { + if (current->key != NULL) { + Put("list:%p key:'%s' data:%p", (void*)l, + current->key, current->data); + } else { + Put("list:%p key:%ld data:%p", (void*)l, + current->key_l, current->data); + } + current = current->next; + } +} + +void list_test(void) +{ + list_s *l = list_new(); + void* somedata = (void*)l; + + assert(1 == list_key_insert(l, "foo", (void*)1)); + assert(1 == list_key_insert(l, "bar", (void*)2)); + assert(1 == list_key_insert(l, "baz", (void*)3)); + assert(2 == (long)list_key_remove(l, "bar")); + assert(1 == (long)list_key_remove(l, "foo")); + assert(3 == (long)list_key_remove(l, "baz")); + + assert(1 == list_key_insert(l, "I/O replay", somedata)); + assert(1 == list_key_insert(l, "for", somedata)); + assert(1 == list_key_insert(l, "benchmarking your server", somedata)); + assert(0 == list_key_insert(l, "for", somedata)); + + assert(NULL != list_key_get(l, "benchmarking your server")); + assert(NULL == list_key_get(l, "Mimecast")); + + assert(NULL != list_key_remove(l, "benchmarking your server")); + assert(NULL == list_key_remove(l, "benchmarking your server")); + assert(1 == list_key_insert(l, "benchmarking your server", somedata)); + + assert(1 == list_key_insert(l, "MiMecast", (void*)42)); + assert(42 == (long)list_key_get(l, "MiMecast")); + + l = list_new_l(); + + assert(1 == list_key_insert_l(l, 1, (void*)1)); + assert(1 == list_key_insert_l(l, 2, (void*)2)); + assert(1 == list_key_insert_l(l, 3, (void*)3)); + assert(1 == (long)list_key_get_l(l, 1)); + assert(1 == (long)list_key_remove_l(l, 1)); + assert(1 != (long)list_key_remove_l(l, 1)); + assert(3 == (long)list_key_remove_l(l, 3)); + + assert(1 == list_key_insert_l(l, 1234, somedata)); + assert(1 == list_key_insert_l(l, 13, somedata)); + assert(1 == list_key_insert_l(l, 666, somedata)); + assert(0 == list_key_insert_l(l, 13, somedata)); + + assert(NULL != list_key_get_l(l, 666)); + assert(NULL == list_key_get_l(l, 777)); + + assert(NULL != list_key_remove_l(l, 666)); + assert(NULL == list_key_remove_l(l, 666)); + assert(1 == list_key_insert_l(l, 666, somedata)); + + assert(1 == list_key_insert_l(l, 42, (void*)42)); + assert(42 == (long)list_key_get_l(l, 42)); + + //list_print(l); +} diff --git a/ioriot/src/datas/list.h b/ioriot/src/datas/list.h new file mode 100644 index 0000000..385333c --- /dev/null +++ b/ioriot/src/datas/list.h @@ -0,0 +1,56 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef LIST_H +#define LIST_H + +#include "../defaults.h" + +/** + * @brief Definition of a linked list element + */ +typedef struct list_elem_s_ { + struct list_elem_s_ *prev; /**< The previous element */ + struct list_elem_s_ *next; /**< The next element */ + char *key; /**< The key of the lemenet */ + long key_l; /**< The same as key, but for long keys */ + void *data; /**< Pointer to the stored data */ +} list_elem_s; + +/** + * @brief Definition of a named linked list data structure + * + * There are two version of this list data structure. One version is utilising + * string keys and the other one is utilising long keys. + */ +typedef struct list_s_ { + list_elem_s *first; /**< The first element, NULL if list empty */ + void (*data_destroy)(void *data); /**< Callback to destroy all data */ +} list_s; + +list_s* list_new(); +list_s* list_new_l(); +void list_destroy(list_s* l); +void list_run_cb(list_s* l, void (*cb)(void *data)); +void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_); +int list_key_insert(list_s* l, char *key, void *data); +int list_key_insert_l(list_s* l, const long key, void *data); +void* list_key_remove(list_s* l, char *key); +void* list_key_remove_l(list_s* l, const long key); +void* list_key_get(list_s* l, char *key); +void* list_key_get_l(list_s* l, const long key); +void list_print(list_s* l); +void list_test(); + +#endif // LIST_H diff --git a/ioriot/src/datas/rbuffer.c b/ioriot/src/datas/rbuffer.c new file mode 100644 index 0000000..c019e6c --- /dev/null +++ b/ioriot/src/datas/rbuffer.c @@ -0,0 +1,147 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rbuffer.h" + +rbuffer_s *rbuffer_new(const int size) +{ + rbuffer_s *r = Malloc(rbuffer_s); + + r->size = size; + r->read_pos = size-1; + r->write_pos = 0; + r->ring = Calloc(size, void*); + + Mset(r->ring, 0, size, void*); + + return r; +} + +void rbuffer_destroy(rbuffer_s *r) +{ + if (r) { + free(r->ring); + free(r); + } +} + +bool rbuffer_insert(rbuffer_s* r, void *data) +{ + if (r->write_pos == r->read_pos) + // Ring buffer is full + return false; + + r->ring[r->write_pos] = data; + r->write_pos = (r->write_pos+1) % r->size; + + return true; +} + +bool rbuffer_has_next(rbuffer_s* r) +{ + sig_atomic_t read_pos = (r->read_pos+1) % r->size; + + if (read_pos == r->write_pos) + // No more items to read, buffer is empty + { + return false; + } + + return true; +} + +void* rbuffer_get_next(rbuffer_s* r) +{ + sig_atomic_t read_pos = (r->read_pos+1) % r->size; + + if (read_pos == r->write_pos) + // No more items to read, buffer is empty + { + return NULL; + } + + void *data = r->ring[read_pos]; + r->ring[read_pos] = NULL; + r->read_pos = read_pos; + + return data; +} + +void rbuffer_print(rbuffer_s* r) +{ + Put("rbuffer_s (%p):", (void*)r); + Put("\tsize: %d", (int)r->size); + Put("\tread_pos: %d", r->read_pos); + Put("\twrite_pos: %d", r->write_pos); + Out("\toccupied slots: "); + for (int i = 0; i < r->size; ++i) + if (r->ring[i]) { + Out("%d:%p ", i, r->ring[i]); + } + Out("\n"); +} + +void rbuffer_test(void) +{ + rbuffer_s *r = rbuffer_new(5); + assert(NULL == rbuffer_get_next(r)); + + assert(rbuffer_insert(r, (void*)1)); + assert(rbuffer_insert(r, (void*)2)); + assert(rbuffer_insert(r, (void*)3)); + assert(rbuffer_insert(r, (void*)4)); + assert(!rbuffer_insert(r, (void*)5)); + rbuffer_print(r); + + assert(rbuffer_has_next(r)); + assert(1 == (long) rbuffer_get_next(r)); + assert(2 == (long) rbuffer_get_next(r)); + assert(3 == (long) rbuffer_get_next(r)); + assert(4 == (long) rbuffer_get_next(r)); + assert(!rbuffer_has_next(r)); + assert(NULL == rbuffer_get_next(r)); + + assert(rbuffer_insert(r, (void*)1)); + assert(1 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)2)); + assert(2 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)3)); + assert(3 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)4)); + assert(4 == (long) rbuffer_get_next(r)); + assert(rbuffer_insert(r, (void*)5)); + assert(5 == (long) rbuffer_get_next(r)); + assert(NULL == rbuffer_get_next(r)); + rbuffer_print(r); + + assert(rbuffer_insert(r, (void*)1)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)2)); + assert(1 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)3)); + assert(2 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)4)); + assert(3 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(rbuffer_insert(r, (void*)5)); + rbuffer_print(r); + assert(4 == (long) rbuffer_get_next(r)); + rbuffer_print(r); + assert(5 == (long) rbuffer_get_next(r)); + assert(NULL == rbuffer_get_next(r)); + + rbuffer_destroy(r); +} diff --git a/ioriot/src/datas/rbuffer.h b/ioriot/src/datas/rbuffer.h new file mode 100644 index 0000000..fa634de --- /dev/null +++ b/ioriot/src/datas/rbuffer.h @@ -0,0 +1,102 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RBUFFER_H +#define RBUFFER_H + +#include "signal.h" + +#include "../defaults.h" + +/** + * @brief An atomic ring buffer data type definition + * + * This data structure can be used for the common producer/consumer problem. + * As long as there is only max one producer thread and max one consumer thread + * it can be used without any mutex locking. All the operations are atomic. + */ +typedef struct rbuffer_s_ { + /** + * The positions are atomic, means the ring buffer can be accessed from + * multiple threads concurrently (one producer and one consumer thread). + * This is the current read position. + */ + sig_atomic_t read_pos; + /** + * This is the current write position. + */ + sig_atomic_t write_pos; + /** + * Holds the pointers to the actual ring data stored in the ring buffer + */ + void **ring; + /** + * Determines how many elements the ring buffer can hold. The capacity + * will be size-1 though, as we need one empty slot. + */ + int size; +} rbuffer_s; + +/** + * @brief Creates a new ring buffer + * + * @param size The size of the ring buffer + * @return The new ring buffer object + */ +rbuffer_s* rbuffer_new(const int size); + +/** + * @brief Destroys a ring buffer + * + * @param r The ring buffer object + */ +void rbuffer_destroy(rbuffer_s* r); + +/** + * @brief Inserts data pointer to the ring buffer + * + * @param r The ring buffer object + * @param data The data pointer + */ +bool rbuffer_insert(rbuffer_s* r, void *data); + +/** + * @brief Determines whether there is any data in the ring buffer + * + * @param r The ring buffer object + * @return True if there is any data, false otherwise + */ +bool rbuffer_has_next(rbuffer_s* r); + +/** + * @brief Returns and removes the next element from the ring buffer + * + * @param r The ring buffer object + * @return The data pointer + */ +void* rbuffer_get_next(rbuffer_s* r); + +/** + * @brief Prints a ring buffer + * + * @param r The ring buffer object + */ +void rbuffer_print(rbuffer_s* r); + +/** + * @brief Unit tests the ring buffer + */ +void rbuffer_test(void); + +#endif // RBUFFER_H diff --git a/ioriot/src/datas/stack.c b/ioriot/src/datas/stack.c new file mode 100644 index 0000000..94e83e3 --- /dev/null +++ b/ioriot/src/datas/stack.c @@ -0,0 +1,85 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "stack.h" + + +stack_s *stack_new() +{ + stack_s *s = Malloc(stack_s); + *s = (stack_s) { + .top = NULL, .size = 0 + }; + return s; +} + +void stack_destroy(stack_s *s) +{ + stack_elem_s *current = s->top; + + while (current) { + stack_elem_s *next = current->next; + free(current); + current = next; + } + + free(s); +} + +void stack_push(stack_s *s, void *data) +{ + stack_elem_s *new_top = Malloc(stack_elem_s); + + *new_top = (stack_elem_s) { + .next = s->top, + .data = data + }; + + s->top = new_top; + s->size++; +} + +void* stack_pop(stack_s *s) +{ + if (s->top == NULL) { + return NULL; + } + + stack_elem_s *old_top = s->top; + + void *data = old_top->data; + s->top = old_top->next; + free(old_top); + s->size--; + + return data; +} + +int stack_is_empty(stack_s *s) +{ + return s->top == NULL; +} + +stack_s* stack_new_reverse_from(stack_s *s) +{ + stack_s* r = stack_new(); + + while (!stack_is_empty(s)) { + stack_push(r, stack_pop(s)); + } + + stack_destroy(s); + + return r; +} diff --git a/ioriot/src/datas/stack.h b/ioriot/src/datas/stack.h new file mode 100644 index 0000000..87e0974 --- /dev/null +++ b/ioriot/src/datas/stack.h @@ -0,0 +1,43 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef STACK_H +#define STACK_H + +#include "../defaults.h" + +/** + * @brief Definition of a stack element + */ +typedef struct stack_elem_s_ { + struct stack_elem_s_ *next; /**< The next element */ + void *data; /**< Pointer to the stored data in the current element */ +} stack_elem_s; + +/** + * @brief Definition of a stack data structure + */ +typedef struct stack_s_ { + stack_elem_s *top; /**< The top element of the stack, NULL if empty */ + unsigned long size; /**< A count how many elements are in the stack */ +} stack_s; + +stack_s* stack_new(); +stack_s* stack_new_reverse_from(stack_s* s); +void stack_destroy(stack_s* s); +void stack_push(stack_s* s, void *data); +void* stack_pop(stack_s* s); +int stack_is_empty(stack_s* s); + +#endif // STACK_H diff --git a/ioriot/src/defaults.h b/ioriot/src/defaults.h new file mode 100644 index 0000000..6ba1f3b --- /dev/null +++ b/ioriot/src/defaults.h @@ -0,0 +1,54 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DEFAULTS_H +#define DEFAULTS_H + +#include "utils/utils.h" + +/** Version of the supported .capture format */ +#define CAPTURE_VERSION 1 +/** Version of the supported .replay format */ +#define REPLAY_VERSION 1 +/** Max amount of tokens per line in the .capture file */ +#define MAX_TOKENS 10 +/** Max line length in either .capture or .replay file */ +#define MAX_LINE_LEN 1024*8 +/** Controls how many tasks can be queued and buffered per worker thread */ +#define TASK_BUFFER_PER_THREAD 512 +/** Version of I/O Riot */ +#define IORIOT_VERSION "0.3-develop" +/** Copyright information */ +#define IORIOT_COPYRIGHT "Mimecast 2017, 2018 (c)" +/** Max open files resource user limit */ +#define SET_RLIMIT_NOFILE 369216 +/** Max processes resource user limit */ +#define SET_RLIMIT_NPROC 30768 + +// The following are for debugging purposes only + +//#define NO_IOOP +//#define THREAD_DEBUG +//#define LOG_FILTERED + +/** + * @brief Return status codes + */ +typedef enum status_e_ { + SUCCESS, /**< Great success! */ + UNKNOWN, /**< Unknown return status :-/ */ + ERROR, /**< An error happened :-( */ +} status_e; + +#endif // DEFAULTS_H diff --git a/ioriot/src/generate/generate.c b/ioriot/src/generate/generate.c new file mode 100644 index 0000000..ee7178c --- /dev/null +++ b/ioriot/src/generate/generate.c @@ -0,0 +1,235 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generate.h" + +#include "../meta/meta.h" +#include "gtask.h" +#include "gwriter.h" +#include "gparser.h" + +#include + +#define _MAX_PROCESSES 1024*1024*10 + +#define _Perc_filtered (g->num_lines_filtered / (g->lineno/100.0)) + +generate_s* generate_new(options_s *opts) +{ + generate_s *g = Malloc(generate_s); + + g->writer = NULL; + g->lineno = 0; + g->name = opts->name; + g->replay_fd = NULL; + g->mps = mounts_new(opts); + g->num_lines_filtered = 0; + g->num_vsizes = 0; + g->start_time = -1; + g->pid_map = amap_new(_MAX_PROCESSES); + g->vsize_map = hmap_new(_MAX_PROCESSES); + g->mmap_map = hmap_new(1024*1024); + g->vfd_buffer = rbuffer_new(1024); + g->num_mapped_pids = 0; + g->num_mapped_fds = 10; + g->opts = opts; + g->reuse_queue = rbuffer_new(1024); + g->replay_fd = Fopen(opts->replay_file, "w"); + + return g; +} + +void generate_destroy(generate_s *g) +{ + // TODO: Also clean the contets of these maps + amap_destroy(g->pid_map); + hmap_destroy(g->vsize_map); + hmap_destroy(g->mmap_map); + rbuffer_destroy(g->vfd_buffer); + mounts_destroy(g->mps); + + gtask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(g->reuse_queue))) + gtask_destroy(task); + rbuffer_destroy(g->reuse_queue); + + fclose(g->replay_fd); + free(g); +} + + +status_e generate_run(options_s *opts) +{ + generate_s *g = generate_new(opts); + Put("Parsing file %s, writing output to %s", opts->capture_file, + opts->replay_file); + FILE *capture_fd = Fopen(opts->capture_file, "r"); + + size_t len = 0; + ssize_t read; + char *line = NULL; + + set_limits_drop_root(opts->user); + + // Reserve first few bytes for meta information + meta_s *meta = meta_new(g->replay_fd); + meta_reserve(meta); + + // The writer will write the .replay file + gwriter_s *writer = gwriter_new(g); + + // The parser will parse every line of the .capture file + gparser_s *parser = gparser_new(g); + + g->writer = writer; + + // Start one writer and one parser thread! + gparser_start(parser); + gwriter_start(writer); + + Out("Processing, it may take a while: "); + + // Process each line of the .capture file. Determine line by line whether + // the I/O operation makes sense or not. It might be that SystemTap skipped + // some I/O ops due to system overload or other issues. The result is that + // some lines may be corrupt or contain I/O operations on unknown file + // handles. It could also be that there are operations on unknown + // file handles such as sockets etc. These will be all filtered out by + // either the parser or the writer thread! + + while ((read = getline(&line, &len, capture_fd)) != -1) { + if (0 > ++g->lineno) { + Error("lineno:%lu Line number overflow", g->lineno); + } + if (strlen(line) >= MAX_LINE_LEN) { + Error("lineno:%lu Exceeded max line length", g->lineno); + } + + // Create a new generate task (try to reuse a task object)... + gtask_s *t = rbuffer_get_next(g->reuse_queue); + if (!t) { + t = gtask_new(g); + } else if (t->ret != 0) { + g->num_lines_filtered++; + } + gtask_init(t, line, g->lineno); + + // ...pass it to the parser queue + while (!rbuffer_insert(parser->queue, t)) + usleep(100); + + if (g->lineno % 1000000 == 0) { + Out(" %lu (filtered:%.2lf%%)", g->lineno, _Perc_filtered); + } + } + + Put("\nDone reading input file!"); + + Put("Waiting for parser thread..."); + gparser_terminate(parser); + gparser_destroy(parser); + + Put("Waiting for writer thread..."); + gwriter_terminate(writer); + gwriter_destroy(writer); + + // Retrieve all left over processed tasks to collect the + // statistics! + gtask_s *t; + while (NULL != (t = rbuffer_get_next(g->reuse_queue))) { + if (t->ret != 0) + g->num_lines_filtered++; + gtask_destroy(t); + } + + Put("Processed %lu lines in total, had to filter out %.2lf%%", + g->lineno, _Perc_filtered); + + Put("Writing init section to '%s'...", opts->replay_file); + fprintf(g->replay_fd, "#INIT\n"); + off_t init_offset = ftello(g->replay_fd); + hmap_run_cb(g->vsize_map, generate_write_init_cb); + + Put("Writing meta header to '%s'...", opts->replay_file); + meta_write_start(meta); + + // The meta header is being written to the first line of the .replay + // file and used by ioriot to do various things (e.g. initializing + // the test correctly, creating the internal data structures with the + // correct sizes etc. + + meta_write_l(meta, "replay_version", REPLAY_VERSION); + meta_write_l(meta, "init_offset", init_offset); + + meta_write_s(meta, "user", opts->user); + meta_write_s(meta, "name", opts->name); + + meta_write_l(meta, "num_vsizes", g->num_vsizes); + meta_write_l(meta, "num_mapped_pids", g->num_mapped_pids); + meta_write_l(meta, "num_mapped_fds", g->num_mapped_fds); + meta_write_l(meta, "num_lines", g->lineno - g->num_lines_filtered); + + meta_destroy(meta); + fclose(capture_fd); + + Put("Generating '%s' done", opts->replay_file); + generate_destroy(g); + + return SUCCESS; +} + +void generate_write_init_cb(void *data) +{ + vsize_s *l = data; + generate_s *g = l->generate; + + if (l->required && strlen(l->path) > 0) { + fprintf(g->replay_fd, "%d|%d|%ld|%s|\n", + l->is_dir, l->is_file, -l->vsize_deficit, l->path); + } +} + +vsize_s* generate_vsize_by_path(generate_s *g, gtask_s *t, + char *path) +{ + vsize_s *v = NULL; + + if (!path && t) + path = t->path; + + Error_if(!path, "No path specified"); + v = hmap_get(g->vsize_map, path); + + if (!v) { + v = vsize_new(path, ++g->num_vsizes, g); + hmap_insert(g->vsize_map, path, v); + } + + if (t) + t->vsize = v; + + return v; +} + +void generate_gprocess_by_realpid(generate_s *g, gtask_s *t) +{ + // Get the virtual process data object from the virtual PID space. + t->gprocess = amap_get(g->pid_map, t->pid); + if (t->gprocess == NULL) { + t->gprocess = gprocess_new(t->pid, ++g->num_mapped_pids); + if (amap_set(g->pid_map, t->pid, t->gprocess)) { + Error("lineno:%lu Can not insert PID %ld", t->lineno, t->pid); + } + } +} diff --git a/ioriot/src/generate/generate.h b/ioriot/src/generate/generate.h new file mode 100644 index 0000000..6c0d97f --- /dev/null +++ b/ioriot/src/generate/generate.h @@ -0,0 +1,112 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GENERATE_H +#define GENERATE_H + +#include "gwriter.h" +#include "../datas/amap.h" +#include "../datas/hmap.h" +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "../mounts.h" +#include "../options.h" + +// Forward declarations (header include hell) +struct gtask_s_; + +/** + * @brief The generate object definition + * + * This is the general data structure required to generate a .replay file from + * the .capture file. + */ +typedef struct generate_s_ { + long lineno; /**< The current line number */ + long num_lines_filtered; /**< The amount of lines filtered out */ + long start_time; /**< The start time from the .capture file */ + char *name; /**< The name of the test specified by the user */ + FILE *replay_fd; /**< The fd of the .replay file */ + mounts_s *mps; /**< The mounts object */ + hmap_s *mmap_map; /**< mmap address mappings */ + amap_s *pid_map; /**< A map of all virtual process objects */ + unsigned long num_mapped_pids; /**< The amount of mapped PIDs */ + unsigned long num_mapped_fds; /**< The amount of mapped FDs */ + hmap_s *vsize_map; /**< A hash map of all virtual size objects */ + unsigned long num_vsizes; /**< The amount of virtual sizes */ + options_s *opts; /**< A pointer to the options object */ + rbuffer_s *vfd_buffer; /**< A virtual fd buffer, for reusing these */ + rbuffer_s *reuse_queue; /**< A task buffer, for reusing these */ + struct gwriter_s_ *writer; /**< A pointer to the writer object */ +} generate_s; + +/** + * @brief Creates a new generate object + * + * @param opts The options object + * @return The new generate object + */ +generate_s* generate_new(options_s *opts); + +/** + * @brief Destroys a generate object + * + * @param g The generate object to destroy + */ +void generate_destroy(generate_s* g); + +/** + * @brief Generates a .replay file from a .capture file + * + * @param opts The options object + * @return SUCCESS on success + */ +status_e generate_run(options_s *opts); + +/** + * @brief Callback to write the INIT section to the .replay file + * + * This function writes a list of all pre-required + * paths to the .replay file. That then can be used + * by ioriot to initialise the test enironment. + * + * @param data A pointer to the vsize timestamp object + */ +void generate_write_init_cb(void *data); + +/** + * @brief Retrieves the virtual size object of a given path + * + * A new one will be created in case there is no such virtual size object yet. + * + * @param g The generate object + * @param t The task object (vfd will be stored to t->vfd) + * @param path The file path + * @return The virtual size object + */ +vsize_s* generate_vsize_by_path(generate_s *g, struct gtask_s_ *t, + char *path); + +/** + * @brief Retrieves the virtual process object of a given real PID + * + * A new one will be created in case there is no such virtual process object + * yet. + * + * @param g The generate object + * @param t The task object (vfd will be stored to t->gprocess) + */ +void generate_gprocess_by_realpid(generate_s *g, struct gtask_s_ *t); + +#endif // GENERATE_H diff --git a/ioriot/src/generate/gioop.c b/ioriot/src/generate/gioop.c new file mode 100644 index 0000000..01701bc --- /dev/null +++ b/ioriot/src/generate/gioop.c @@ -0,0 +1,838 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gioop.h" + +status_e gioop_run(gwriter_s *w, gtask_s *t) +{ + status_e ret = SUCCESS; + + // There was already an error in the parser (parser.c) processing this + // task! Don't process it futher. + if (t->ret != SUCCESS) { + Cleanup(t->ret); + } + + generate_s *g = w->generate; + + // Get the virtual process data object from the virtual PID space and store + // a pointer to it to t->gprocess + generate_gprocess_by_realpid(g, t); + + // One of the open syscalls may openes a file handle succesfully + if (Eq(t->op, "open")) { + Cleanup(gioop_open(w, t, g)); + + } else if (Eq(t->op, "openat")) { + Cleanup(gioop_openat(w, t, g)); + + } else if (Eq(t->op, "creat")) { + Cleanup(gioop_creat(w, t, g)); + } + + // Get the virtual file descriptor of a given real fd and store a pointer + // to it to t->vfd. + if (t->has_fd) { + ret = gprocess_vfd_by_realfd(t->gprocess, t); + Cleanup_unless(SUCCESS, ret); + } + + + if (Eq(t->op, "close")) { + Cleanup(gioop_close(w, t, g)); + + } else if (Eq(t->op, "stat")) { + Cleanup(gioop_stat(w, t, g)); + + } else if (Eq(t->op, "statfs")) { + Cleanup(gioop_statfs(w, t, g)); + + } else if (Eq(t->op, "statfs64")) { + Cleanup(gioop_statfs64(w, t, g)); + + } else if (Eq(t->op, "fstat")) { + Cleanup(gioop_fstat(w, t, g)); + + } else if (Eq(t->op, "fstatat")) { + Cleanup(gioop_fstatat(w, t, g)); + + } else if (Eq(t->op, "fstatfs")) { + Cleanup(gioop_fstatfs(w, t, g)); + + } else if (Eq(t->op, "fstatfs64")) { + Cleanup(gioop_fstatfs64(w, t, g)); + + } else if (Eq(t->op, "rename")) { + Cleanup(gioop_rename(w, t, g)); + + } else if (Eq(t->op, "renameat")) { + Cleanup(gioop_renameat(w, t, g)); + + } else if (Eq(t->op, "renameat2")) { + Cleanup(gioop_renameat2(w, t, g)); + + } else if (Eq(t->op, "read")) { + Cleanup(gioop_read(w, t, g)); + + } else if (Eq(t->op, "readv")) { + Cleanup(gioop_readv(w, t, g)); + + } else if (Eq(t->op, "readahead")) { + Cleanup(gioop_readahead(w, t, g)); + + } else if (Eq(t->op, "readdir")) { + Cleanup(gioop_readdir(w, t, g)); + + } else if (Eq(t->op, "readlink")) { + Cleanup(gioop_readlink(w, t, g)); + + } else if (Eq(t->op, "readlinkat")) { + Cleanup(gioop_readlinkat(w, t, g)); + + } else if (Eq(t->op, "write")) { + Cleanup(gioop_write(w, t, g)); + + } else if (Eq(t->op, "writev")) { + Cleanup(gioop_writev(w, t, g)); + + } else if (Eq(t->op, "lseek")) { + Cleanup(gioop_lseek(w, t, g)); + + } else if (Eq(t->op, "getdents")) { + Cleanup(gioop_getdents(w, t, g)); + + } else if (Eq(t->op, "mkdir")) { + Cleanup(gioop_mkdir(w, t, g)); + + } else if (Eq(t->op, "rmdir")) { + Cleanup(gioop_rmdir(w, t, g)); + + } else if (Eq(t->op, "mkdirat")) { + Cleanup(gioop_mkdirat(w, t, g)); + + } else if (Eq(t->op, "unlink")) { + Cleanup(gioop_unlink(w, t, g)); + + } else if (Eq(t->op, "unlinkat")) { + Cleanup(gioop_unlinkat(w, t, g)); + + } else if (Eq(t->op, "lstat")) { + Cleanup(gioop_lstat(w, t, g)); + + } else if (Eq(t->op, "fsync")) { + Cleanup(gioop_fsync(w, t, g)); + + } else if (Eq(t->op, "fdatasync")) { + Cleanup(gioop_fdatasync(w, t, g)); + + } else if (Eq(t->op, "sync")) { + Cleanup(gioop_sync(w, t, g)); + + } else if (Eq(t->op, "syncfs")) { + Cleanup(gioop_syncfs(w, t, g)); + + } else if (Eq(t->op, "sync_file_range")) { + Cleanup(gioop_sync_file_range(w, t, g)); + + } else if (Eq(t->op, "fcntl")) { + Cleanup(gioop_fcntl(w, t, g)); + + } else if (Eq(t->op, "fcntl")) { + Cleanup(gioop_fcntl(w, t, g)); + + } else if (Eq(t->op, "mmap2")) { + // Support for mmap added later + + } else if (Eq(t->op, "munmap")) { + // Support for mmap added later + + } else if (Eq(t->op, "mremap")) { + // Support for mmap added later + + } else if (Eq(t->op, "msync")) { + // Support for mmap added later + + } else if (Eq(t->op, "chmod")) { + Cleanup(gioop_chmod(w, t, g)); + + } else if (Eq(t->op, "fchmodat")) { + Cleanup(gioop_chmod(w, t, g)); + + } else if (Eq(t->op, "fchmod")) { + Cleanup(gioop_fchmod(w, t, g)); + + } else if (Eq(t->op, "chown")) { + Cleanup(gioop_chown(w, t, g)); + + } else if (Eq(t->op, "chown16")) { + Cleanup(gioop_chown(w, t, g)); + + } else if (Eq(t->op, "lchown")) { + Cleanup(gioop_lchown(w, t, g)); + + } else if (Eq(t->op, "lchown16")) { + Cleanup(gioop_lchown(w, t, g)); + + } else if (Eq(t->op, "fchown")) { + Cleanup(gioop_fchown(w, t, g)); + + } else if (Eq(t->op, "fchownat")) { + Cleanup(gioop_chown(w, t, g)); + + } else if (Eq(t->op, "exit_group")) { + Cleanup(gioop_exit_group(w, t, g)); + + } else { + Cleanup(ERROR;); + } + +cleanup: + +#ifdef LOG_FILTERED + if (ret != SUCCESS) + t->filtered_where = __FILE__; +#endif + + t->ret = ret; + return ret; +} + +status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd || t->path == NULL || t->flags == -1) { + return ERROR; + } + + gprocess_create_vfd_by_realfd(t->gprocess, t, g); + generate_vsize_by_path(g, t, NULL); + + Gioop_write(OPEN, "%ld|%s|%d|%d|open", + t->mapped_fd, t->path, t->mode, t->flags); + + if (t->fd > 0) + vsize_open(t->vsize, t->vfd, t->path, t->flags); + + return SUCCESS; +} + +status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd || t->path == NULL || t->flags == -1) { + return ERROR; + } + + gprocess_create_vfd_by_realfd(t->gprocess, t, g); + generate_vsize_by_path(g, t, NULL); + Gioop_write(OPEN_AT, "%ld|%s|%d|%d|openat", + t->mapped_fd,t->path, t->mode, t->flags); + if (t->fd > 0) + vsize_open(t->vsize, t->vfd, t->path, t->flags); + + return SUCCESS; +} + +status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd || t->path == NULL || t->flags == -1) { + return ERROR; + } + + gprocess_create_vfd_by_realfd(t->gprocess, t, g); + generate_vsize_by_path(g, t, NULL); + + Gioop_write(CREAT, "%ld|%s|%d|%d|creat", + t->mapped_fd, t->path, t->mode, t->flags); + if (t->fd > 0) + vsize_open(t->vsize, t->vfd, t->path, t->flags); + + return SUCCESS; +} + + +status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(CLOSE, "%ld|%d|close", t->mapped_fd, t->status); + + if (t->status == 0) + vsize_close(t->vsize, t->vfd); + + hmap_remove_l(t->gprocess->fd_map, t->fd); + hmap_remove_l(t->gprocess->vfd_map, t->mapped_fd); + + if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) + vfd_destroy(t->vfd); + + return SUCCESS; +} + +status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(STAT, "%s|%d|stat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(STATFS, "%s|%d|statfs", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(STATFS64, "%s|%d|statfs64", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSTAT, "%ld|%d|fstat", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(FSTAT_AT, "%s|%d|fstatat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSTATFS, "%ld|%d|fstatfs", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSTATFS64, "%ld|%d|fstatfs64", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL || t->path2 == NULL ) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(RENAME, "%s|%s|%d|rename", t->path, t->path2, t->status); + + if (t->status == 0) { + t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); + vsize_rename(t->vsize, t->vsize2, t->path, t->path2); + } + + return SUCCESS; +} + +status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL || t->path2 == NULL ) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(RENAME_AT, "%s|%s|%d|renameat", t->path, t->path2, t->status); + + if (t->status == 0) { + t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); + vsize_rename(t->vsize, t->vsize2, t->path, t->path2); + } + + return SUCCESS; +} +status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL || t->path2 == NULL ) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(RENAME_AT2, "%s|%s|%d|renameat2", + t->path, t->path2, t->status); + + if (t->status == 0) { + t->vsize2 = generate_vsize_by_path(g, NULL, t->path2); + vsize_rename(t->vsize, t->vsize2, t->path, t->path2); + } + + return SUCCESS; +} + +status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READ, "%ld|%ld|read", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READ, "%ld|%ld|readv", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_read(t->vsize, t->vfd, t->vfd->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READAHEAD, "%ld|%ld|%ld|readahead", + t->mapped_fd, t->offset, t->count); + + return SUCCESS; +} + +status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(READDIR, "%ld|%d|readdir", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(READLINK, "%s|%d|readlink", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(READLINK_AT, "%s|%d|readlinkat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(WRITE, "%ld|%ld|write", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_write(t->vsize, t->vfd, t->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(WRITEV, "%ld|%ld|writev", t->mapped_fd, t->bytes); + + if (t->bytes > 0) + vsize_write(t->vsize, t->vfd, t->path, t->bytes); + + return SUCCESS; +} + +status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(LSEEK, "%ld|%ld|%ld|%ld|lseek", + t->mapped_fd, t->offset, t->whence, t->bytes); + + if (t->bytes >= 0) + vsize_seek(t->vsize, t->vfd, t->bytes); + + return SUCCESS; +} + +status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(GETDENTS, "%ld|%ld|%ld|getdents", + t->mapped_fd, t->count, t->bytes); + + return SUCCESS; +} + +status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(MKDIR, "%s|%d|%d|mkdir", t->path, t->mode, t->status); + + if (t->status == 0) + vsize_mkdir(t->vsize, t->path); + + return SUCCESS; +} +status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(MKDIR, "%s|%d|rmdir", t->path, t->status); + + if (t->status == 0) + vsize_rmdir(t->vsize, t->path); + + return SUCCESS; +} +status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(MKDIR_AT, "%s|%d|%d|mkdirat", t->path, t->mode, t->status); + + if (t->status == 0) + vsize_mkdir(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(UNLINK, "%s|%d|unlink", t->path, t->status); + + if (t->status == 0) + vsize_unlink(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(UNLINK_AT, "%s|%d|unlinkat", t->path, t->status); + + if (t->status == 0) + vsize_unlink(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(LSTAT, "%s|%d|lstat", t->path, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FSYNC, "%ld|%d|fsync", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FDATASYNC, "%ld|%d|fdatasync", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g) +{ + Gioop_write(SYNC, "%d|sync", t->status); + + return SUCCESS; +} + +status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(SYNCFS, "%ld|%d|syncfs", t->mapped_fd, t->status); + + return SUCCESS; +} + +status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(SYNC_FILE_RANGE, "%ld|%ld|%ld|%d|sync_file_range", + t->mapped_fd, t->offset, t->bytes, t->status); + + return SUCCESS; +} + +status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + switch (t->F) { + case F_GETFD: + case F_GETFL: + case F_SETFD: + case F_SETFL: + break; + default: + return ERROR; + break; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FCNTL, "%ld|%d|%d|%d|fcntl", + t->mapped_fd, t->F, t->G, t->status); + + return SUCCESS; +} + +status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + Gioop_write(CHMOD, "%s|%d|%d|chmod", t->path, t->mode, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(FCHMOD, "%ld|%d|%d|fchmod", t->mapped_fd, t->mode, t->status); + + return SUCCESS; +} + +status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + // Hmm, maybe rename t->offset, because here it is used for the user UID + Gioop_write(CHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (!t->has_fd) { + return ERROR; + } + + generate_vsize_by_path(g, t, t->vfd->path); + // Hmm, maybe rename t->offset, because here it is used for the user UID + Gioop_write(FCHOWN, "%ld|%ld|%d|%d|fchown", t->mapped_fd, t->offset, t->G, t->status); + + return SUCCESS; +} + +status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g) +{ + if (t->path == NULL) { + return ERROR; + } + + generate_vsize_by_path(g, t, NULL); + // Hmm, maybe rename t->offset, because here it is used for the user UID + Gioop_write(LCHOWN, "%s|%ld|%d|%d|chown", t->path, t->offset, t->G, t->status); + + if (t->status == 0) + vsize_stat(t->vsize, t->path); + + return SUCCESS; +} + +status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g) +{ + // It means that the process and all its threads terminate. + // Therefore close all file handles of that process! + hmap_run_cb2(t->gprocess->vfd_map, gioop_close_all_vfd_cb, t); + + // Remove virtual process from pid map and destroy it + amap_unset(g->pid_map, t->pid); + gprocess_destroy(t->gprocess); + + return SUCCESS; +} + +void gioop_close_all_vfd_cb(void *data, void *data2) +{ + gtask_s *t = data2; + t->vfd = data; + generate_s *g = t->generate; + + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(CLOSE, "%ld|%d|close on exit_group", t->vfd->mapped_fd, 0); + vsize_close(t->vsize, t->vfd); +} + diff --git a/ioriot/src/generate/gioop.h b/ioriot/src/generate/gioop.h new file mode 100644 index 0000000..ad49713 --- /dev/null +++ b/ioriot/src/generate/gioop.h @@ -0,0 +1,102 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GIOOP_H +#define GIOOP_H + +#include "../defaults.h" +#include "gwriter.h" +#include "gtask.h" +#include "generate.h" + + +// Helper macro regarding writing the .replay file! + +#define Gioop_write(op, ...) \ + fprintf(g->replay_fd, "%ld|%ld|%ld|0|0|%d|", \ + t->mapped_time, \ + (t->vsize ? t->vsize->id : 0),\ + t->gprocess->mapped_pid, \ + op); \ + fprintf(g->replay_fd, __VA_ARGS__); \ + fprintf(g->replay_fd, "@%ld", t->lineno); \ + fprintf(g->replay_fd, "|\n") + +/** + * @brief Function used when closing all virtual FDs of a virtual process + * + * This function is run on all virtual file handles whenever a virtual generate + * process object (gprocess_s) gets destroyed. This is on an exit_group + * syscall (a thread group, a process with all its threads, terminates). Upon + * process termination Linux also closes all its file descriptors! This is what + * we simulate here! + * + * @param data The pointer to the virtual file descriptor object + * @param data2 The pointer to the corresponding generate task object. + */ +void gioop_close_all_vfd_cb(void *data, void *data2); + +/** + * @brief Run a generate I/O operation on a given task + * + * @param w The writer object + * @param t The task object + * @return SUCCESS if everything went fine + */ +status_e gioop_run(gwriter_s *w, gtask_s *t); + +status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g); +status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g); + +#endif // GIOOP_H diff --git a/ioriot/src/generate/gparser.c b/ioriot/src/generate/gparser.c new file mode 100644 index 0000000..2f3b250 --- /dev/null +++ b/ioriot/src/generate/gparser.c @@ -0,0 +1,356 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gparser.h" + +#include "gtask.h" +#include "gwriter.h" + +void* gparser_pthread_start(void *data) +{ + gparser_s *p = data; + generate_s *g = p->generate; + gwriter_s *w = g->writer; + gtask_s *t = NULL; + + do { + while (NULL != (t = rbuffer_get_next(p->queue))) { + // First extract + gparser_extract(p, t); + // Second, pass the task to the writer thread + rbuffer_insert(w->queue, t); + } + usleep(100); + } while (!p->terminate); + + while (NULL != (t = rbuffer_get_next(p->queue))) { + gparser_extract(p, t); + rbuffer_insert(w->queue, t); + } + + return NULL; +} + +gparser_s* gparser_new(generate_s *g) +{ + gparser_s *p = Malloc(gparser_s); + + p->generate = g; + p->terminate = false; + p->queue = rbuffer_new(1024); + + return p; +} + +void gparser_start(gparser_s *p) +{ + start_pthread(&p->pthread, gparser_pthread_start, (void*)p); +} + +void gparser_destroy(gparser_s *p) +{ + rbuffer_destroy(p->queue); + free(p); +} + +void gparser_terminate(gparser_s *p) +{ + p->terminate = true; + pthread_join(p->pthread, NULL); +} + +void gparser_extract(gparser_s *p, gtask_s *t) +{ + status_e ret = SUCCESS; + generate_s *g = p->generate; + + char *saveptr; + char* tok = strtok2_r(t->line, ";:,", &saveptr); + int ntoks = 0; + + while (tok) { + if (++ntoks > MAX_TOKENS) { + ret = ERROR; + break; + } + ret = gparser_extract_tok(p, t, tok); + if (ret != SUCCESS) + break; + + tok = strtok2_r(NULL, ";:,", &saveptr); + } + + if (ret == SUCCESS) { + + // Check for the existance of mandatory values! + if (t->pid < 0 || t->tid < 0) { + Cleanup(ERROR); + + } else if (t->op == NULL) { + Cleanup(ERROR); + + } else if (t->mapped_time == -1) { + Cleanup(ERROR); + } + + // We are inserting ".ioriot/NAME" to the paths. This enables us to + // run multiple tests simoultaneously. + + if (t->path) { + if (!mounts_transform_path(g->mps, g->name, + t->path, &t->path_r)) { + Cleanup(ERROR); + } + if (t->path_r) + t->path = t->path_r; + } + + if (t->path2) { + if (!mounts_transform_path(g->mps, g->name, + t->path2, &t->path2_r)) { + Cleanup(ERROR); + } + if (t->path2_r) + t->path2 = t->path2_r; + } + + } + +cleanup: + + t->ret = ret; + +#ifdef LOG_FILTERED + t->filtered_where = __FILE__; +#endif +} + +status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok) +{ + status_e ret = SUCCESS; + + if (gparser_token_not_ok(p, tok)) { + Cleanup(ERROR); + } + + generate_s *g = t->generate; + + char key = tok[0]; + char *value = tok; + value += 2; + + switch (key) { + case 'a': + // Address + t->address = strtol(value, NULL, 10); + break; + + case 'A': + // Address 2 + t->address2 = strtol(value, NULL, 10); + break; + + case 'b': + // Bytes + if (t->bytes != -1) { + Cleanup(ERROR); + } + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->bytes = strtol(value, NULL, 10); + break; + + case 'c': + // Count + if (t->count != -1) { + Cleanup(ERROR); + } + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->count = strtol(value, NULL, 10); + break; + + case 'd': + // Descriptor + if (t->fd != -1) { + Cleanup(ERROR); + } + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->fd = atoi(value); + if (t->fd > 0) + t->has_fd = true; + break; + + case 'f': + // Flags + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->flags = atoi(value); + break; + + case 'i': + // PID:TID + t->pidtid = value; + // Extract PID and TID from "PID:TID" + if (!gparser_get_pidtid(p, t->pidtid, &t->pid, &t->tid)) { + Cleanup(ERROR); + } + break; + + case 'm': + // Mode + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->mode = atoi(value); + break; + + case 'o': + // Operation + t->op = value; + break; + + case 'O': + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->offset = strtol(value, NULL, 10); + break; + + case 'W': + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->whence = strtol(value, NULL, 10); + break; + + case 'p': + // File path + t->path = value; + chreplace(t->path, '|', '_'); + strunquote(t->path); + break; + + case 'P': + // File path 2 + t->path2 = value; + chreplace(t->path2, '|', '_'); + strunquote(t->path2); + break; + + case 's': + // Cleanup status + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->status = atoi(value); + break; + + case 't': + // Time + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->mapped_time = strtol(value, NULL, 10); + // Start replay time from 0 + if (g->start_time == -1) { + g->start_time = t->mapped_time; + } + t->mapped_time -= g->start_time; + break; + + case 'F': + // FCNTL function + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->F = atoi(value); + break; + + case 'G': + // FCNTL argument + if (is_number(value) == 0) { + Cleanup(ERROR); + } + t->G = atoi(value); + break; + + case 'T': + break; + + default: + // Unknown key + { + Cleanup(ERROR); + } + } + +cleanup: + if (t->path_r) { + free(t->path_r); + t->path_r = NULL; + } + if (t->path2_r) { + free(t->path2_r); + t->path2_r = NULL; + } + + return ret; +} + +bool gparser_token_not_ok(gparser_s *p, char *tok) +{ + if (strlen(tok) < 3) { + return true; + + } else if (tok[1] != '=') { + return true; + } + + return false; +} + +bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid) +{ + char *pos = strchr(pidtid, ':'); + + if (pos) { + char *tmp = pos; + tmp++; + + if (is_number(tmp)) { + *tid = atol(tmp); + } else { + return false; + } + + pos[0] = '\0'; + if (is_number(pidtid)) { + *pid = atol(pidtid); + } else { + return false; + } + } + + else { + return false; + } + + return (*pid >= 0 && *tid >= 0); +} diff --git a/ioriot/src/generate/gparser.h b/ioriot/src/generate/gparser.h new file mode 100644 index 0000000..f3e204a --- /dev/null +++ b/ioriot/src/generate/gparser.h @@ -0,0 +1,113 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GPARSER_H +#define GPARSER_H + +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "generate.h" +#include "gtask.h" + +/** + * @brief The parser definition + * + * The parser is to extract all information from the .capture file. + */ +typedef struct gparser_s_ { + bool terminate; /**< The parser thread will terminate if set to true */ + generate_s *generate; /**< The generate object */ + pthread_t pthread; /**< The posix thread */ + rbuffer_s *queue; /**< A queue of task objects */ +} gparser_s; + +/** + * @brief Creates a new parser + * + * @param g The generate object + * @return The new parser object + */ +gparser_s* gparser_new(generate_s *g); + +/** + * @brief Starts the parser thread + * + * @param p The parser object + */ +void gparser_start(gparser_s *p); + +/** + * @brief Terminates the parser thread + * + * @param p The parser object + */ +void gparser_terminate(gparser_s *p); + +/** + * @brief Destroys the parser thread + * + * @param p The parser object + */ +void gparser_destroy(gparser_s *p); + +/** + * @brief Extracts information a .capture line + * + * Extracts information from a .capture line and stores it into the task + * object. + * + * @param p The parser object + * @param t The task object + */ +void gparser_extract(gparser_s *p, gtask_s *t); + +/** + * @brief Extracts information from a specific token string + * + * @param p The parser object + * @param t The task object + * @param tok The token string + * @return Returns with SUCCESS on success + */ +status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok); + +/** + * @brief Verifies the correctness of a token + * + * @param p The parser object + * @param tok The token to be verified + * @return true if token verified successfully + */ +bool gparser_token_not_ok(gparser_s *p, char *tok); + +/** + * @brief Checks whether the pidtid string is correct or not + * + * @param p The parser object + * @param pidtid The string to check + * @param pid The pointer to the resulting pid + * @param tid The pointer to the resulting tid + * @return true on success + */ +bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid); + +/** + * @brief Entry point of the parser POSIX thread + * + * @param data A pointer to the parser object + * return Always NULL + */ +void* gparser_pthread_start(void *data); + +#endif // GPARSER_H diff --git a/ioriot/src/generate/gprocess.c b/ioriot/src/generate/gprocess.c new file mode 100644 index 0000000..6a0b37a --- /dev/null +++ b/ioriot/src/generate/gprocess.c @@ -0,0 +1,101 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gprocess.h" + +#include "../vfd.h" +#include "gioop.h" + +void _gprocess_vfd_map_destroy_cb(void *data) +{ + vfd_destroy(data); +} + +gprocess_s* gprocess_new(const long pid, const long mapped_pid) +{ + gprocess_s* gp = Malloc(gprocess_s); + + gp->pid = pid; + gp->mapped_pid = mapped_pid; + gp->max_mapped_fd = 0; + gp->fd_map = hmap_new_l(1024); + gp->vfd_map = hmap_new_l(1024); + gp->vfd_map->data_destroy = _gprocess_vfd_map_destroy_cb; + + return gp; +} + +void gprocess_destroy(gprocess_s *gp) +{ + hmap_destroy(gp->vfd_map); + hmap_destroy(gp->fd_map); + free(gp); +} + +void gprocess_create_vfd_by_realfd(gprocess_s *gp, gtask_s *t, generate_s *g) +{ + if (t->fd < 0) + return; + + // Check whether the real FD is still open according to the .capture log + long old_mapped = (long) hmap_get_l(gp->fd_map, t->fd); + if (old_mapped) { + + // That real file descriptor is already with a mapping to a virtual + // file descriptor. This may happen when SystemTap missed to trace a + // 'close' syscall. We are inserting a close now... + + t->vfd = hmap_get_l(gp->vfd_map, old_mapped); + + hmap_remove_l(gp->fd_map, t->fd); + hmap_remove_l(gp->vfd_map, old_mapped); + + if (t->vfd) { + generate_vsize_by_path(g, t, t->vfd->path); + Gioop_write(CLOSE, "%ld|%d|close inserted", old_mapped, 0); + vsize_close(t->vsize, t->vfd); + if (!(rbuffer_insert(g->vfd_buffer, t->vfd))) + vfd_destroy(t->vfd); + } + } + + t->vfd = rbuffer_get_next(g->vfd_buffer); + t->mapped_fd = ++g->num_mapped_fds; + if (!t->vfd) + t->vfd = vfd_new(t->fd, t->mapped_fd, t->path); + else + vfd_update(t->vfd, t->fd, t->mapped_fd, t->path); + t->vfd->free_path = t->path_r != NULL; + + hmap_insert_l(gp->vfd_map, t->mapped_fd, t->vfd); + hmap_insert_l(gp->fd_map, t->fd, (void*)t->mapped_fd); +} + +status_e gprocess_vfd_by_realfd(gprocess_s *gp, gtask_s *t) +{ + t->mapped_fd = (long) hmap_get_l(gp->fd_map, t->fd); + if (t->mapped_fd == 0) { + // No corresponding virtual fd number mapping + t->has_fd = false; + + } else { + t->vfd = hmap_get_l(gp->vfd_map, t->mapped_fd); + if (!t->vfd) { + return ERROR; + } + t->mapped_fd = t->vfd->mapped_fd; + } + + return SUCCESS; +} diff --git a/ioriot/src/generate/gprocess.h b/ioriot/src/generate/gprocess.h new file mode 100644 index 0000000..7fcbd77 --- /dev/null +++ b/ioriot/src/generate/gprocess.h @@ -0,0 +1,90 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GPROCESS_H +#define GPROCESS_H + +#include "../datas/hmap.h" +#include "../defaults.h" +#include "gtask.h" +#include "generate.h" + +// Forward declarations (header include hell) +struct gtask_s_; +struct generate_s_; + +/** + * @brief Virtual process object used for generating .replay file + * + * An object of this represents a Linux process in generate context. + * In Linux every process owns * its own file descriptor table which is + * simulated here. Usually, a Linux process re-uses a FD number once not used + * anymore (e.g. after a close). However, as we want to increase concurrency + * while replaying the I/O we want * to ensure to always use unique file + * descriptor IDs for every open. Thats why we use max_mapped_fd to always + * map a real FD number to a uniq virtual FD number. + */ +typedef struct gprocess_s_ { + long pid; /**< The real PID */ + long mapped_pid; /**< The mapped PID */ + hmap_s *vfd_map; /**< All virtual file descriptors of that process */ + hmap_s *fd_map; /**< All mappings from real fd to virtual fd */ + long max_mapped_fd; /**< The max mapped fd number */ +} gprocess_s; + +/** + * @brief Creates a new gprocess object + * + * @param pid The process ID + * @param mapped_pid the mapped PID + * @return The new gprocess object + */ +gprocess_s* gprocess_new(const long pid, const long mapped_pid); + +/** + * @brief Destroys a gprocess object + * + * @param gp The gprocess object + */ +void gprocess_destroy(gprocess_s *gp); + +/** + * @brief Creates a new virtual FD from a given real FD number + * + * In ioriot we map the real file descriptor (the fd number protocolled in + * the.capture file) to a virtual file descriptor (the fd numner written to the + * .replay file). The purpose is to increase concurrency of the I/O during + * replay. Normally, a process would reuse the same file descriptor number + * once closed earlier. However, when replaying we can't reuse the number if + * we want to replay the I/O on multiple paths in parallel. Therefore, it is + * ensured that the virtual file descriptor number in the .replay file is + * always * unique for every open! + * + * @param gp The process object + * @param t The task object (the vfd pointer will be stored to * t->vfd) + * @param g The generate object + */ +void gprocess_create_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t, + struct generate_s_ *g); + +/** + * @brief Retrieves a virtual FD from a given real FD number + * + * @param gp The process object + * @param t The task object (the vfd pointer will be stored to * t->vfd) + * @return SUCCESS if everything went smothly! + */ +status_e gprocess_vfd_by_realfd(gprocess_s *gp, struct gtask_s_ *t); + +#endif // GPROCESS_H diff --git a/ioriot/src/generate/gtask.c b/ioriot/src/generate/gtask.c new file mode 100644 index 0000000..55a1124 --- /dev/null +++ b/ioriot/src/generate/gtask.c @@ -0,0 +1,91 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtask.h" + +gtask_s* gtask_new(void *generate) +{ + gtask_s *t = Malloc(gtask_s); + + t->generate = generate; + t->line = NULL; + t->path_r = NULL; + t->path2_r = NULL; +#ifdef LOG_FILTERED + t->original_line = NULL; +#endif + + return t; +} + +void gtask_init(gtask_s *t, char *line, const unsigned long lineno) +{ + if (t->line) + free(t->line); + t->line = Clone(line); + + if (t->path_r) + free(t->path_r); + if (t->path2_r) + free(t->path2_r); + +#ifdef LOG_FILTERED + if (t->original_line) + free(t->original_line); + t->original_line = Clone(line); + t->filtered_where = NULL; +#endif + + t->bytes = -1; + t->address = 0; + t->address2 = 0; + t->count = -1; + t->F = -1; + t->fd = -1; + t->flags = -1; + t->G = -1; + t->has_fd = false; + t->vsize = NULL; + t->vsize2 = NULL; + t->lineno = lineno; + t->mapped_fd = -1; + t->mapped_time = -1; + t->mode = -1; + t->offset = -1; + t->op = NULL; + t->path2 = NULL; + t->path2_r = NULL; + t->path = NULL; + t->path_r = NULL; + t->gprocess = NULL; + t->pid = -1; + t->pidtid = NULL; + t->ret = 0; + t->status = -1; + t->tid = -1; + t->vfd = NULL; + t->whence = -1; +} + +void gtask_destroy(gtask_s *t) +{ + if (t->line) + free(t->line); + if (t->path_r) + free(t->path_r); + if (t->path2_r) + free(t->path2_r); + free(t); +} + diff --git a/ioriot/src/generate/gtask.h b/ioriot/src/generate/gtask.h new file mode 100644 index 0000000..0f8e5ac --- /dev/null +++ b/ioriot/src/generate/gtask.h @@ -0,0 +1,100 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GTASK_H +#define GTASK_H + +#include "vsize.h" + +#include "gprocess.h" +#include "../vfd.h" +#include "../datas/amap.h" +#include "../datas/hmap.h" +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "../mounts.h" +#include "../options.h" + +/** + * @brief The generate task definition + * + * The gtask holds all possible variables required to process a particular + * .capture line and to generate the corresponding .replay line + */ +typedef struct gtask_s_ { + bool has_fd; /**< True if task has a file descriptor number */ + char *line; /**< A pointer to the remaining part of the .capture line */ + char *op; /**< Operation/syscall name */ + char *path2; /**< A second path name (e.g. for rename) */ + char *path2_r; /**< Work around to track mallocs, so it can be freed */ + char *path; /**< Path name */ + char *path_r; /**< Work around to track mallocs, so it can be freed */ + char *pidtid; /**< String representing pid:tid */ + int F; /**< Arguments for fcntl syscall */ + int G; /**< Arguments for fcntl syscall */ + int fd; /**< File descriptor number */ + int flags; /**< File open flags */ + int mode; /**< File open mode */ + int ret; /**< ioriot process status, SUCCESS if everything is alright */ + int status; /**< Operation/syscall return status */ + long address2; /**< Another address (used by mmap related syscalls) */ + long address; /**< An address (used by mmap related syscalls) */ + long bytes; /**< Amount of bytes */ + long count; /**< A count */ + long lineno; /**< The current line number */ + long mapped_fd; /**< The mapped file descriptor number */ + long mapped_time; /**< The mapped time */ + long offset; /**< A offset */ + long pid; /**< The process ID */ + long tid; /**< The thread ID */ + long whence; /**< Whence */ + vfd_s *vfd; /**< A pointer to the virtual file descriptor */ + struct gprocess_s_ *gprocess; /**< A pointer to the process object */ + void *generate; /**< A pointer to the generate object */ + vsize_s *vsize2; /**< Pointer to a second virtual size object */ + vsize_s *vsize; /**< Pointer to the virtual size object */ +#ifdef LOG_FILTERED + char *original_line; /**< Only used for debugging purposes */ + char *filtered_where; /**< Only used for debugging purposes */ +#endif +} gtask_s; + +/** + * @brief Creates a new task object + * + * @param generate A pointer to the generate object + * @return The new task object + */ +gtask_s* gtask_new(void *generate); + +/** + * @brief Initialises a taks object + * + * This function is used in particular when we recycle/reuse an old + * gtask object. + * + * @param t The gtask object + * @param line The corresponding line from the .capture file + * @param lineno The line number + */ +void gtask_init(gtask_s *t, char *line, const unsigned long lineno); + +/** + * @brief Destroys a given task object + * + * @param t The task object + */ +void gtask_destroy(gtask_s *t); + +#endif // GTASK_H diff --git a/ioriot/src/generate/gwriter.c b/ioriot/src/generate/gwriter.c new file mode 100644 index 0000000..e0d448e --- /dev/null +++ b/ioriot/src/generate/gwriter.c @@ -0,0 +1,85 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gwriter.h" + +#include "gtask.h" +#include "generate.h" +#include "gioop.h" +#include "../opcodes.h" + +void* gwriter_pthread_start(void *data) +{ + gwriter_s *w = data; + generate_s *g = w->generate; + gtask_s *t = NULL; + + do { + while (NULL != (t = rbuffer_get_next(w->queue))) { +#ifdef LOG_FILTERED + // Logging filtered lines + if (SUCCESS != gioop_run(w, t)) { + fprintf(g->replay_fd, "#FILTERED @%ld %s", t->lineno, + t->original_line); + } +#else + gioop_run(w, t); +#endif + rbuffer_insert(g->reuse_queue, t); + } + usleep(100); + } while (!w->terminate); + + while (NULL != (t = rbuffer_get_next(w->queue))) { +#ifdef LOG_FILTERED + if (SUCCESS != gioop_run(w, t)) { + fprintf(g->replay_fd, "#FILTERED @%ld %s\n", t->lineno, + t->original_line); + } +#else + gioop_run(w, t); +#endif + rbuffer_insert(g->reuse_queue, t); + } + + return NULL; +} + +gwriter_s* gwriter_new(generate_s *g) +{ + gwriter_s *w = Malloc(gwriter_s); + + w->generate = g; + w->terminate = false; + w->queue = rbuffer_new(1024); + + return w; +} + +void gwriter_start(gwriter_s *w) +{ + start_pthread(&w->pthread, gwriter_pthread_start, (void*)w); +} + +void gwriter_destroy(gwriter_s *w) +{ + rbuffer_destroy(w->queue); + free(w); +} + +void gwriter_terminate(gwriter_s *w) +{ + w->terminate = true; + pthread_join(w->pthread, NULL); +} diff --git a/ioriot/src/generate/gwriter.h b/ioriot/src/generate/gwriter.h new file mode 100644 index 0000000..4295580 --- /dev/null +++ b/ioriot/src/generate/gwriter.h @@ -0,0 +1,86 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GWRITER_H +#define GWRITER_H + +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "vsize.h" +#include "gtask.h" +#include "generate.h" + +// Forward declaration (header include hell) +struct gtask_s_; +struct generate_s_; + +/** + * @brief Definition of the writer object + * + * The writer utilises the information extracted by the parser to actually + * write the .replay file. + */ +typedef struct gwriter_s_ { + bool terminate; /**< The writer thread will terminate if set to true */ + struct generate_s_ *generate; /**< The generate object */ + pthread_t pthread; /**< The posix thread */ + rbuffer_s *queue; /**< A queue of task objects */ +} gwriter_s; + +/** + * @brief Creates a new writer + * + * @param g The generate object + * @return The new writer object + */ +gwriter_s* gwriter_new(struct generate_s_ *g); + +/** + * @brief Starts the writer thread + * + * @param w The writer object + */ +void gwriter_start(gwriter_s *w); + +/** + * @brief Terminates the writer thread + * + * @param w The writer object + */ +void gwriter_terminate(gwriter_s *w); + +/** + * @brief Destroys the writer thread + * + * @param w The writer object + */ +void gwriter_destroy(gwriter_s *w); + +/** + * @brief Writes a line to the .replay file + * + * @param w The writer object + * @param t The task object + */ +void gwriter_write(gwriter_s *w, struct gtask_s_ *t); + +/** + * @brief Entry function of the writer pthread + * + * @param data A pointer to the writer object + * @return Always returns a NULL pointer if it doesnt crash! + */ +void* gwriter_pthread_start(void *data); + +#endif // GWRITER_H diff --git a/ioriot/src/generate/vsize.c b/ioriot/src/generate/vsize.c new file mode 100644 index 0000000..f2d56ba --- /dev/null +++ b/ioriot/src/generate/vsize.c @@ -0,0 +1,247 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "vsize.h" + +#include "generate.h" + +// Helper macros + +#define _Set_file(v) v->is_file = true; v->unsure = v->is_dir = false +#define _Set_dir(v) v->is_dir = true; v->unsure = v->is_file = false +#define _Set_unsure(v) v->unsure = true +#define _Set_inserted(v) v->inserted = true +#define _Set_renamed(v) v->renamed = true +#define _Set_required(v) v->required = true + +vsize_s* vsize_new(char *file_path, const unsigned long id, + void *generate) +{ + vsize_s *v = Malloc(vsize_s); + + v->generate = generate; + v->id = id; + v->inserted = false; + v->is_dir = false; + v->is_file = false; + v->offset = -1; + v->path = Clone(file_path); + v->renamed = false; + v->required = false; + v->unsure = false; + v->updates = 0; + v->vsize = 0; + v->vsize_deficit = 0; + + return v; +} + +void vsize_destroy(vsize_s *v) +{ + if (!v) + return; + + free(v->path); + free(v); +} + +void init_parent_dir(vsize_s *v, const char *path) +{ + generate_s *g = v->generate; + char *clone = Clone(path); + char *parent = dirname(clone); + + vsize_s *v_parent = hmap_get(g->vsize_map, parent); + if (!v_parent) { + + // Parent directory does not yet have a vsize! + // Create a vsize object for it and set it as a pre-requirement + // so that the directory can be created during init mode. + + v_parent = vsize_new(parent, ++g->num_vsizes, g); + hmap_insert(g->vsize_map, parent, v_parent); + + _Set_required(v_parent); + _Set_dir(v_parent); + + // This is for debugging purposes only + _Set_inserted(v_parent); + v_parent->updates++; + + } else if (v_parent->unsure) { + // We now know for sure that this path must be a directory! + _Set_dir(v_parent); + v_parent->updates++; + } + + free(clone); +} + +void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags) +{ + + // v->first_encounter == false means, that this is the first occurance of + // this path and we didn't initialise it (means we didn't ensure that + // we want to create all parent directories etc. + + if (v->updates == 0) { + // We may use a recycled vfd object! When opening a file we always + // assume that the offset is 0! + vfd_s *vfd_ = vfd; + vfd_->offset = 0; + init_parent_dir(v, path); + + if (Has(flags, O_DIRECTORY)) { + _Set_required(v); + _Set_dir(v); + + } else if (Hasnt(flags, O_CREAT)) { + _Set_required(v); + _Set_file(v); + _Set_unsure(v); + } + v->updates++; + + } else if (v->unsure) { + if (Has(flags, O_DIRECTORY)) { + // Now we know for sure that this path must be a directory! + _Set_dir(v); + v->updates++; + } + } +} + +void vsize_close(vsize_s *v, void* vfd) +{ + vfd_s *vfd_ = vfd; + vfd_->offset = 0; + v->updates++; +} + +void vsize_stat(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + _Set_file(v); + + // We are not 100% sure that this is really a file, + // the path might be still a directory though! + _Set_unsure(v); + v->updates++; + } +} + +void vsize_rename(vsize_s *v, vsize_s *v2, + const char *path, const char *path2) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + _Set_file(v); + _Set_unsure(v); + v->updates++; + } + + if (v2->updates == 0) { + init_parent_dir(v2, path2); + _Set_file(v2); + + // We are not 100% sure that this is really a file, + // the path might be still a directory though! + _Set_unsure(v2); + + // For debugging purposes only + _Set_renamed(v2); + v2->updates++; + } +} + +void vsize_adjust(vsize_s *v, vfd_s* vfd) +{ + if (v->vsize >= vfd->offset) + return; + + long deficit = v->vsize - vfd->offset; + if (deficit < v->vsize_deficit) { + v->vsize_deficit = deficit; + _Set_required(v); + _Set_file(v); + } +} + +void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes) +{ + vfd_s *vfd_ = vfd; + vfd_->offset += bytes; + vsize_adjust(v, vfd_); + v->updates++; +} + +void vsize_seek(vsize_s *v, void *vfd, const long new_offset) +{ + //vfd_s *vfd_ = vfd; + + // The file's offset can be greater than the file's current size, in which + // case the next write to the file will extend the file. This is referred + // to as creating a hole in a file and is allowed. However, this behaviour + // does not suit the estimation of the file size before we want to run the + // test. + + // TODO: Implement file hole support! + //v->updates++; +} + +void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes) +{ + vfd_s *vfd_ = vfd; + vfd_->offset += bytes; + + if (v->vsize < vfd_->offset) + v->vsize = vfd_->offset; + + v->updates++; +} + +void vsize_mkdir(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_dir(v); + v->updates++; + } +} + +void vsize_rmdir(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + _Set_dir(v); + v->updates++; + } +} + +void vsize_unlink(vsize_s *v, const char *path) +{ + if (v->updates == 0) { + init_parent_dir(v, path); + _Set_required(v); + if (!v->is_dir) { + _Set_file(v); + _Set_unsure(v); + } + v->updates++; + } +} diff --git a/ioriot/src/generate/vsize.h b/ioriot/src/generate/vsize.h new file mode 100644 index 0000000..8556ec0 --- /dev/null +++ b/ioriot/src/generate/vsize.h @@ -0,0 +1,180 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef VSIZE_H +#define VSIZE_H + +#include "../utils/utils.h" +#include "../datas/hmap.h" +#include "../vfd.h" + +/** + * @brief Definition of a virtual size object + * + * The virtual size is used to determine the expected type and size of a file. + * This piece of information will be added to the INIT section of the the + * .replay file. That file then will be created during test initialisation. + * before running the test. It is very likely the case that the test requires + * a file of a certain size already to be present, so it can be read from disk. + */ +typedef struct vsize_s_ { + char *path; /**< The path to the file/directory */ + off_t offset; /**< The current file offset */ + unsigned long id; /**< The vsize id */ + void *generate; /**< A pointer to the generate object */ + long vsize; /**< The virtual size */ + long vsize_deficit; /**< Size to use for file creating during init mode */ + bool renamed; /**< True if file/dir has been renamed */ + bool required; /**< True if init mode will create this file/dir */ + bool is_dir; /**< True if this file/dir is a directory */ + bool is_file; /**< True if this file/dir is a regular file */ + bool unsure; /**< True if the file type is not fully clear */ + long updates; /**< Amount of times this vsize has been updated */ + bool inserted; /**< For debugging purposes only */ +} vsize_s; + +/** + * @brief Creates a new vsize object + * + * @param file_path The corresponding file path + * @param id The vsize vsize aka ID + * @param generate The generate object + * @return The new vsize object + */ +vsize_s* vsize_new(char *file_path, const unsigned long id, void *generate); + +/** + * @brief Destroys a vsize object + * + * @param v The vsize object + */ +void vsize_destroy(vsize_s *v); + +/** + * @brief Ensures that the parent directory exists + * + * This function ensures that the parent directory exists as a vsize object! + * + * @param v The vsize object + * @param path The given path + */ +void init_parent_dir(vsize_s *v, const char *path); + +/** + * @brief Adjusts the vsize + * + * Compares the virtual file size of the file in the vsize + * object to the the offset in the virtual file descriptor. + * In case the offset is higher we have a size deficit and + * we need to mark it. That way ioriot can ensure that + * during init mode it will create a file with the correct + * size prior of running the test! + * + * @param v The virtual size object + * @param vfd The virtual file descriptor object + */ +void vsize_adjust(vsize_s *v, vfd_s* vfd); + +/** + * @brief Adjust vsize on open + * + * @param v The virtual size object + * @param vfd The virtual file descriptor object + * @param path The file open path + * @param flags The file open flags + */ +void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags); + +/** + * @brief Adjust vsize on close + * + * @param v The virtual size object + * @param vfd The virtual file descriptor object + */ +void vsize_close(vsize_s *v, void *vfd); + +/** + * @brief Adjust vsize on stat + * + * @param v The virtual size object + * @param path The stat path + */ +void vsize_stat(vsize_s *v, const char *path); + +/** + * @brief Adjust vsize on rename + * + * @param v The virtual size object + * @param v2 The virtual size object of path2 + * @param path The first file path + * @param path2 The second file path + */ +void vsize_rename(vsize_s *v, vsize_s *v2, + const char *path, const char *path2); + +/** + * @brief Adjust vsize on read + * + * @param v The virtual size object + * @param vfd The virtual vile descriptor object + * @param path The file path + * @param bytes The amount of bytes read + */ +void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes); + +/** + * @brief Adjust vsize on seek + * + * @param v The virtual size object + * @param vfd The virtual vile descriptor object + * @param new_offset The new file offset after seek + */ +void vsize_seek(vsize_s *v, void *vfd, const long new_offset); + +/** + * @brief Adjust vsize on write + * + * @param v The virtual size object + * @param vfd The virtual vile descriptor object + * @param path The file path + * @param bytes The amount of bytes written + */ +void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes); + +/** + * @brief Adjust vsize on mkdir + * + * @param v The virtual size object + * @param path The directory path + */ +void vsize_mkdir(vsize_s *v, const char *path); + +/** + * @brief Adjust vsize on rmdir + * + * @param v The virtual size object + * @param path The directory path + */ +void vsize_rmdir(vsize_s *v, const char *path); + +/** + * @brief Adjust vsize on unlink + * + * @param v The virtual size object + * @param path The file path + */ +void vsize_unlink(vsize_s *v, const char *path); + +#endif // VSIZE_H + diff --git a/ioriot/src/init/init.c b/ioriot/src/init/init.c new file mode 100644 index 0000000..e375379 --- /dev/null +++ b/ioriot/src/init/init.c @@ -0,0 +1,226 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "init.h" + +#include "../datas/stack.h" +#include "itask.h" +#include "ithread.h" +#include "../meta/meta.h" +#include "../mounts.h" +#include "../utils/futils.h" + + +init_s *init_new(options_s *opts) +{ + init_s *i = Malloc(init_s); + + i->opts = opts; + i->mounts = mounts_new(opts); + i->threads_map = amap_new(i->mounts->count); + i->reuse_queue = rbuffer_new(4096); + i->replay_fd = Fopen(opts->replay_file, "r"); + + pthread_mutex_init(&i->reuse_queue_mutex, NULL); + + return i; +} + +void init_destroy(init_s *i) +{ + amap_destroy(i->threads_map); + mounts_destroy(i->mounts); + + itask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(i->reuse_queue))) { + itask_destroy(task); + } + rbuffer_destroy(i->reuse_queue); + + fclose(i->replay_fd); + pthread_mutex_destroy(&i->reuse_queue_mutex); + + free(i); +} + +void init_extract_header(init_s *i, off_t *init_offset) +{ + options_s *opts = i->opts; + meta_s *m = meta_new(i->replay_fd); + meta_read_start(m); + + long version = 0; + if (meta_read_l(m, "version", &version)) { + Put("Replay version is '%ld'", version); + if (version != REPLAY_VERSION) { + Error(".replay file of incompatible version, got %x, expected %x", + (int)version, REPLAY_VERSION); + } + } + + char *user; + if (meta_read_s(m, "user", &user)) { + Put("Setting user to '%s'", user); + opts->user = user; + } + + char *name; + if (meta_read_s(m, "name", &name)) { + Put("Setting name to '%s'", name); + opts->name = name; + } + + if (meta_read_l(m, "init_offset", init_offset)) { + if (*init_offset < 0) { + Error("Offset overflow (init offset too large in .replay)"); + } + Put("Setting init offset to '%ld'", *init_offset); + } + + meta_destroy(m); +} + +status_e init_run(options_s *opts) +{ + status_e ret = SUCCESS; + init_s *i = init_new(opts); + + off_t init_offset; + init_extract_header(i, &init_offset); + + // Ensure that all ./replay/NAME directories exist + mounts_init(i->mounts); + + // Don't do messy stuff as super user + set_limits_drop_root(opts->user); + + // We need to clean up garbish from previous runs! + if (opts->purge) + mounts_purge(i->mounts); + else + mounts_trash(i->mounts); + + Out("Creating all files and directories requried for test '%s'...", + opts->name); + + // Seek to the INIT section + fseeko(i->replay_fd, init_offset, SEEK_SET); + + bool is_file = false, is_dir = false; + long vsize = 0; + char *path; + + // Stats + long dirs_created = 0; + long files_created = 0; + long files_total_size = 0; + + // Helper variables for getline + char *line = NULL; + size_t len = 0, read = 0; + char *saveptr; + + stack_s *all_threads = stack_new(); + + // Process the INIT section of the .replay file line by line. + + while ((read = getline(&line, &len, i->replay_fd)) != -1) { + char *tok = strtok_r(line, "|", &saveptr); + + for (int ntok = 0; tok; ntok++) { + switch (ntok) { + case 0: + is_dir = atoi(tok) == 1; + break; + case 1: + is_file = atoi(tok) == 1; + break; + case 2: + vsize = atol(tok); + if (vsize < 0) { + Error("Size overflow"); + } + break; + case 3: + path = tok; + break; + default: + break; + } + + tok = strtok_r(NULL, "|", &saveptr); + } + + itask_s *task = rbuffer_get_next(i->reuse_queue); + + if (!task) { + task = itask_new(); + + } else { + itask_extract_stats(task, &dirs_created, &files_created, + &files_total_size); + } + + // Set new task values + if (is_dir) { + task->is_dir = true; + + } else if (is_file) { + task->is_file = true; + task->vsize = vsize; + } + task->path = Clone(path); + + // We run one init thread per mount point + int mnr = mounts_get_mountnumber(i->mounts, path); + ithread_s *t = amap_get(i->threads_map, mnr); + + if (!t) { + t = ithread_new(i); + amap_set(i->threads_map, mnr, t); + stack_push(all_threads, t); + ithread_start(t); + } + + //itask_print(task); + while (!rbuffer_insert(t->queue, task)) + usleep(1000); + } + + ithread_s *t = NULL; + while (NULL != (t = stack_pop(all_threads))) { + ithread_terminate(t); + ithread_destroy(t); + } + stack_destroy(all_threads); + + itask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(i->reuse_queue))) { + itask_extract_stats(task, &dirs_created, &files_created, + &files_total_size); + itask_destroy(task); + } + + Put("Done!"); + + Put("Created %ld files (net total size: %.2fg) and %ld directories!", + files_created, files_total_size/(1024*1024*1024.0), + dirs_created); + + init_destroy(i); + + Put("You are ready to fire up the test now"); + + return ret; +} diff --git a/ioriot/src/init/init.h b/ioriot/src/init/init.h new file mode 100644 index 0000000..3d9f9e9 --- /dev/null +++ b/ioriot/src/init/init.h @@ -0,0 +1,64 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INIT_H +#define INIT_H + +#include "../defaults.h" +#include "../options.h" +#include "../datas/amap.h" +#include "../datas/rbuffer.h" +#include "../mounts.h" + +typedef struct init_s_ { + amap_s *threads_map; + rbuffer_s *reuse_queue; + options_s *opts; + mounts_s *mounts; + FILE *replay_fd; + pthread_mutex_t reuse_queue_mutex; +} init_s; + +/** + * @brief Creates a new init object + * + * @param opts The options object + * @return The new mounts object + */ +init_s* init_new(options_s *opts); + +/** + * @brief Destroys the init object + * + * @param i The init object + */ +void init_destroy(init_s *i); + +/** + * @brief Initialises the test environment + * + * @param opts The options object + * @return SUCCESS if initialised without any issues + */ +status_e init_run(options_s *opts); + +/** + * @brief Extracts some useful information from the .replay meta header + * + * @param i The init object + * @param init_offset To store the offset of the init section + */ +void init_extract_header(init_s *i, off_t *init_offset); + +#endif // INIT_H diff --git a/ioriot/src/init/itask.c b/ioriot/src/init/itask.c new file mode 100644 index 0000000..f04ce33 --- /dev/null +++ b/ioriot/src/init/itask.c @@ -0,0 +1,66 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "itask.h" + +itask_s* itask_new() +{ + itask_s *task = Malloc(itask_s); + + task->path = NULL; + itask_reset_stats(task); + + return task; +} + +void itask_destroy(itask_s *task) +{ + if (task->path) + free(task->path); + + free(task); +} + +void itask_reset_stats(itask_s *task) +{ + task->is_dir = task->is_file = false; + task->sizes_created = task->vsize = 0; + task->dirs_created = task->files_created = 0; + + if (task->path) { + free(task->path); + task->path = NULL; + } +} + +void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created, + long *files_total_size) +{ + *dirs_created += task->dirs_created; + *files_created += task->files_created; + *files_total_size += task->sizes_created; + + if (*dirs_created < 0 || *files_created < 0 || *files_total_size < 0) { + Error("Size overflow"); + } + + itask_reset_stats(task); +} + +void itask_print(itask_s *task) +{ + Put("itask(%p): is_dir:%d is_file:%d vsize:%ld path:%s", + (void*)task, task->is_dir, task->is_file, + task->vsize, task->path); +} diff --git a/ioriot/src/init/itask.h b/ioriot/src/init/itask.h new file mode 100644 index 0000000..b10d515 --- /dev/null +++ b/ioriot/src/init/itask.h @@ -0,0 +1,72 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ITASK_H +#define ITASK_H + +#include "../defaults.h" + +/** + * @brief The initialise task definition + */ +typedef struct itask_s_ { + bool is_dir; + bool is_file; + long vsize; + char *path; + long dirs_created; + long files_created; + long sizes_created; +} itask_s; + +/** + * @brief Creates a new task object + * + * @return The new task object + */ +itask_s* itask_new(); + +/** + * @brief Resets the task stats + * + * @param task The itask object + */ +void itask_reset_stats(itask_s *task); + +/** + * @brief Extract stats from a task object + * + * @param task The itask object + * @param dirs_created Adds count of dirs created to that variable + * @param files_created Adds count of files created to that variable + * @param files_total_size Adds size of files created to that variable + */ +void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created, + long *files_total_size); + +/** + * @brief Destroys a given task object + * + * @param task The task object + */ +void itask_destroy(itask_s *task); + +/** + * @brief Prints a task to stdout + * + * @param task The task object + */ +void itask_print(itask_s *task); + +#endif // ITASK_H diff --git a/ioriot/src/init/ithread.c b/ioriot/src/init/ithread.c new file mode 100644 index 0000000..a580e70 --- /dev/null +++ b/ioriot/src/init/ithread.c @@ -0,0 +1,99 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ithread.h" + +#include "itask.h" +#include "../utils/futils.h" + + +void* ithread_pthread_start(void *data) +{ + ithread_s *t = data; + init_s *i = t->init; + itask_s *task = NULL; + + do { + while (NULL != (task = rbuffer_get_next(t->queue))) { + ithread_run_task(t, task); + + // We need to mutex lock the reuse_queue as multiple threads + // can insert into it + pthread_mutex_lock(&i->reuse_queue_mutex); + int ret = rbuffer_insert(i->reuse_queue, task); + pthread_mutex_unlock(&i->reuse_queue_mutex); + if (!ret) + itask_destroy(task); + } + usleep(100); + } while (!t->terminate); + + while (NULL != (task = rbuffer_get_next(t->queue))) { + ithread_run_task(t, task); + if (!rbuffer_insert(i->reuse_queue, task)) + itask_destroy(task); + + pthread_mutex_lock(&i->reuse_queue_mutex); + int ret = rbuffer_insert(i->reuse_queue, task); + pthread_mutex_unlock(&i->reuse_queue_mutex); + if (!ret) + itask_destroy(task); + } + + return NULL; +} + +ithread_s* ithread_new(init_s *i) +{ + ithread_s *t = Malloc(ithread_s); + + t->init = i; + t->queue = rbuffer_new(1024); + t->terminate = false; + + return t; +} + +void ithread_start(ithread_s *t) +{ + start_pthread(&t->pthread, ithread_pthread_start, (void*)t); +} + +void ithread_destroy(ithread_s *t) +{ + rbuffer_destroy(t->queue); + free(t); +} + +void ithread_terminate(ithread_s *t) +{ + t->terminate = true; + pthread_join(t->pthread, NULL); +} + +void ithread_run_task(ithread_s *t, itask_s *task) +{ + if (task->is_dir) { + task->dirs_created += ensure_dir_exists(task->path); + + } else if (task->is_file) { + if (!ensure_file_exists(task->path, &task->dirs_created)) { + task->files_created++; + if (task->vsize > 0) { + append_random_to_file(task->path, task->vsize); + task->sizes_created += task->vsize; + } + } + } +} diff --git a/ioriot/src/init/ithread.h b/ioriot/src/init/ithread.h new file mode 100644 index 0000000..0884519 --- /dev/null +++ b/ioriot/src/init/ithread.h @@ -0,0 +1,86 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ITHREAD_H +#define ITHREAD_H + +#include "../defaults.h" +#include "../datas/rbuffer.h" + +#include "init.h" +#include "itask.h" + +#include + +/** + * @brief Definition of an init thread + * + */ +typedef struct ithread_s_ { + pthread_t pthread; /**< We run the init tasks in concurrent pthreads */ + rbuffer_s *queue; /**< The thread's task queue */ + init_s *init; /**< The responsible init object */ + bool terminate; /**< Indicates that thread can terminate */ +} ithread_s; + +/** + * @brief Creates a new thread object + * + * @param i The init object + * @return The new thread object + */ +ithread_s* ithread_new(init_s *i); + +/** + * @brief Terminates the thread + * + * This function waits (via join) for the pthread to complete all its + * current tasks from the queue. + * + * @param t The thread object + */ +void ithread_terminate(ithread_s* t); + +/** + * @brief Destroys the thread object + * + * @param t The thread object + */ +void ithread_destroy(ithread_s* t); + +/** + * @brief Executes the init task + * + * @param t The thread object + * @param task The task object + */ +void ithread_run_task(ithread_s* t, itask_s *task); + +/** + * @brief Starts the POSIX thread + * + * @param t The responsible thread object + */ +void ithread_start(ithread_s *t); + +/** + * @brief Entry point of the POSIX thread + * + * @param data Data passed to the pthread + * @return Always NULL on success + */ + +void* ithread_pthread_start(void *data); + +#endif // ITHREAD_H diff --git a/ioriot/src/macros.h b/ioriot/src/macros.h new file mode 100644 index 0000000..45e5a10 --- /dev/null +++ b/ioriot/src/macros.h @@ -0,0 +1,116 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MACROS_H +#define MACROS_H + +#define Cleanup(code) ret = code; goto cleanup +#define Cleanup_unless(expr, code) \ + if (expr != code) { ret = code; goto cleanup; } + +// String helpers +#define Clone(str) notnull(strdup(str),__FILE__,__LINE__,0) +#define Eq(str1,str2) strcmp(str1,str2) == 0 + +// Number helpers +#define Abs(num) num >= 0 ? num : -num +#define Readhex(str) strtol(str, NULL, 16) +#define Perc(a, b) a > b ? b/(a/100.) : a/(b/100.) + +// Bitwise helpers +#define Has(flags, what) (flags & (what)) == (what) +#define Hasnt(flags, what) (flags & (what)) != (what) + +// Memory helpers +#define Malloc(what) \ + notnull(malloc(sizeof(what)),__FILE__,__LINE__,1) +#define Calloc(count,what) \ + notnull(calloc(count,sizeof(what)),__FILE__,__LINE__,count) +#define Mset(where,value,count,what) \ + memset(where,value,count*sizeof(what)) + +// Open helpers +#define Fopen(path, mode) fnotnull(fopen(path, mode), path, __FILE__, __LINE__) + +// Mmap helpers +#define Mmapshared(what) \ + mmapok(mmap(NULL, sizeof(what), \ + PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0), \ + __FILE__,__LINE__) +#define Cmapshared(count,what) \ + mmapok(mmap(NULL, count*sizeof(what), \ + PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0), \ + __FILE__,__LINE__) + +// Printing messages +#define Out(...) \ + fprintf(stdout, __VA_ARGS__); \ + fflush(stdout); +#define Put(...) \ + fprintf(stdout, __VA_ARGS__); \ + fprintf(stdout, "\n"); \ + fflush(stdout); + +// Printing debug messages +#define Debug(...) \ + fprintf(stderr, "%s:%d DEBUG: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n"); \ + fflush(stderr); + +// Printing error messages +#define Error(...) \ + fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define Error_if(expr, ...) if (expr) { Error(__VA_ARGS__); } + +#define Errno(...) \ + fprintf(stderr, "%s:%d ERROR: %s (%d). ", __FILE__, __LINE__, \ + strerror(errno), errno); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define Errno_if(expr, ...) if (expr) { Errno(__VA_ARGS__); } + +#define Segfault(...) \ + fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); \ + *(int*)0 = 0; + +// Printing warn messages +#define Warn(...) \ + fprintf(stderr, "WARN: "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr,"\n"); \ + fflush(stdout); \ + fflush(stderr); + +#define Warn_if(expr, ...) if (expr) { Warn(__VA_ARGS__); } + +// Other helpers +#define Fill_with_stuff(buf, len) \ + for (int i = 0; i +#include + +#include "capture/capture.h" +#include "cleanup/cleanup.h" +#include "generate/generate.h" +#include "init/init.h" +#include "mounts.h" +#include "options.h" +#include "replay/replay.h" +#include "utests.h" +#include "utils/utils.h" + +/** + * @brief Do some architecture checks + * + * To ensure that I/O replay works correctly we have to check whether some + * data types are atomic or not. This is what this function does! + */ +static void _arch_check_atomic(void) +{ + if (sizeof(int) > sizeof(sig_atomic_t)) { + Error("int data type is not atomic on this architecture: %ld > %ld", + sizeof(int), sizeof(sig_atomic_t)); + + } else if (sizeof(bool) > sizeof(sig_atomic_t)) { + Error("bool data type is not atomic on this architecture: %ld > %ld", + sizeof(bool), sizeof(sig_atomic_t)); + } +} + +/** + * @brief Prints out version and copyright information + */ +static void _print_version(void) +{ + Put("This is I/O Riot %s - %s", IORIOT_VERSION, IORIOT_COPYRIGHT); +} + +/** + * @brief Print the synopsis + */ +static void _print_synopsis(void) +{ + _print_version(); + + Put("Synopsis:"); + Put("\tioriot -c io.capture [-x PID] [-m MODULE]"); + Put("\tioriot -c io.capture -r io.replay [-n str] [-u str] [-w str]"); + Put("\tioriot -i io.replay"); + Put("\tioriot -r io.replay [-p #] [-t #] [-D] [-s #]"); + Put("\tioriot -R io.replay [-p #] [-t #] [-D] [-s #]"); + Put("\tioriot -d"); + Put("\tioriot -P"); + Put("\tioriot -T [-n NAME]"); + Put("\tioriot -V"); +} + +/** + * @brief Print a brief help + */ +static void _print_help(void) +{ + _print_synopsis(); + + Put("Help:"); + Put("\t-d Drop all Linux/FS caches and exit ioriot"); + Put("\t-D Don't drop all caches (in conjunction with -r/-R):"); + Put("\t-s SPEED The speed factor (default: 0 [as fast as possible])"); + Put("\t-h Print this help"); + Put("\t-c FILE The capture file"); + Put("\t-n NAME The name (default: test0)"); + Put("\t-u USER The test run user (default: mcuser)"); + Put("\t-p #WORKERS Amount of of parallel worker processes (default: 4)"); + Put("\t-t #THREADS Threads per worker process (default: 128)"); + Put("\t-i REPLAYFILE The replay file to be initialised"); + Put("\t-r REPLAYFILE The replay file to be replayed"); + Put("\t-R REPLAYFILE Init and replay in one run (-i and -r combined)"); + Put("\t-S STATSFILE Write a stats file at the end of a test"); + Put("\t-T Trash data directories"); + Put("\t-P Purge all trash directories of all tests)"); + Put("\t-V Print I/O replay program version"); + Put("\t-w WD_BASE The working directory's base path"); + Put("\t (default: /usr/local/ioriot)"); + Put("\t-x PID To specify a process ID (in conjunction with -c)"); + Put("\t-m MODULE To specify a module (in conjunction with -c)"); + Put("\nExample (run these commands one after another):"); + Put("\t 1.) sudo ioriot -c io.capture"); + Put("\t 2.) sudo ioriot -r io.replay -c io.capture -u paul -n test1"); + Put("\t 3.) sudo ioriot -i io.replay"); + Put("\t 4.) sudo ioriot -r io.replay -S"); +} + +/** + * @brief I/O Riot's entry point + * + * Not much more to document here though! + * @return The exit code + */ +int main(int argc, char **argv) +{ + _arch_check_atomic(); + status_e ret = UNKNOWN; + + bool dont_drop_caches = false; + options_s *opts = options_new(); + int opt = 0; + + while ((opt = getopt(argc, argv, "Vr:R:S:c:u:i:hw:n:dDs:w:p:t:UPTx:m:")) != -1) { + switch (opt) { + case 'U': + utests_run(); + Cleanup(SUCCESS); + break; + case 'V': + _print_version(); + Cleanup(SUCCESS); + break; + case 'd': + drop_caches(); + Cleanup(SUCCESS); + break; + case 'D': + dont_drop_caches = true; + break; + case 'c': + opts->capture_file = absolute_path(optarg); + Put("Capture file: %s", opts->capture_file); + break; + case 'P': + opts->purge = true; + Put("Purge option set"); + break; + case 'T': + opts->trash = true; + Put("Trash option set"); + break; + case 'i': + opts->init = true; + if (!opts->replay_file) { + opts->replay_file = absolute_path(optarg); + Put("Replay file: %s", opts->replay_file); + } + break; + case 'R': + opts->init = true; + opts->replay = true; + if (!opts->replay_file) { + opts->replay_file = absolute_path(optarg); + Put("Replay file: %s", opts->replay_file); + } + break; + case 'r': + opts->replay = true; + if (!opts->replay_file) { + opts->replay_file = absolute_path(optarg); + Put("Replay file: %s", opts->replay_file); + } + break; + case 'S': + opts->stats_file = Clone(optarg); + Put("Stats output file: %s", opts->stats_file); + break; + case 'w': + opts->wd_base = optarg; + Put("WD base: %s", opts->wd_base); + break; + case 'u': + opts->user = optarg; + Put("User: %s", opts->user); + break; + case 'm': + opts->module = Clone(optarg); + Put("Module: %s", opts->module); + break; + case 'n': + opts->name = optarg; + Put("Name: %s", opts->name); + break; + case 'h': + _print_help(); + Cleanup(SUCCESS); + case 's': + sscanf(optarg, "%lf", &opts->speed_factor); + Put("Speed factor: %lf", opts->speed_factor); + break; + case 'p': + opts->num_workers = atoi(optarg); + if (opts->num_workers < 1) + opts->num_workers = 1; + Put("Num worker processes: %d", opts->num_workers); + break; + case 't': + opts->num_threads_per_worker = atoi(optarg); + if (opts->num_threads_per_worker < 1) + opts->num_threads_per_worker = 1; + Put("Num threads per worker: %d", opts->num_threads_per_worker); + break; + case 'x': + opts->pid = atoi(optarg); + Put("PID: %d", opts->pid); + break; + default: + _print_help(); + Cleanup(ERROR); + } + } + + if (opts->purge || opts->trash) { + // Clean up all temp data of previous test runs + Cleanup(cleanup_run(opts)); + + } else if (opts->capture_file && !opts->replay_file) { + // We are going to capture I/O + Cleanup(capture_run(opts)); + + } else if (opts->capture_file && opts->replay_file) { + // We are going to generate a .replay file from the .capture file + Cleanup(generate_run(opts)); + + } else if (opts->replay_file && opts->init && !opts->replay) { + // We are going to initialise the test from the .replay file! + Cleanup(init_run(opts)); + + } else if (opts->replay_file && opts->init && opts->replay) { + // We are going to initialise the test and run the test! Run the + // initialiser in a sub-process, as it drops root privileges! + pid_t pid = fork(); + if (pid == 0) { + Cleanup(init_run(opts)); + } else { + opts->drop_caches = !dont_drop_caches; + int init_status; + waitpid(pid, &init_status, 0); + // Only proceed if initialisation was successfull! + Cleanup_unless(SUCCESS, init_status); + Cleanup(replay_run(opts)); + } + + } else if (opts->replay_file && !opts->init && opts->replay) { + // We are going to replay the I/O + opts->drop_caches = !dont_drop_caches; + Cleanup(replay_run(opts)); + + } else { + _print_help(); + Cleanup(ERROR); + } + +cleanup: + options_destroy(opts); + + return ret; +} diff --git a/ioriot/src/meta/meta.c b/ioriot/src/meta/meta.c new file mode 100644 index 0000000..d56c17e --- /dev/null +++ b/ioriot/src/meta/meta.c @@ -0,0 +1,111 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "meta.h" + +#define _MAX_META_LEN 256 + +meta_s* meta_new(FILE *replay_fd) +{ + meta_s *m = Malloc(meta_s); + + m->replay_fd = replay_fd; + m->offset = ftello(replay_fd); + m->read_buf = NULL; + + return m; +} + +void meta_destroy(meta_s *m) +{ + if (!m) + return; + + if (m->read_buf) + free(m->read_buf); + + free(m); +} + +void meta_reserve(meta_s *m) +{ + // TODO: Use a hole in the .replay file to reserve space + char buf[_MAX_META_LEN]; + Mset(&buf, '#', _MAX_META_LEN-1, char); + fprintf(m->replay_fd, "%s\n", buf); +} + +void meta_write_start(meta_s *m) +{ + fseeko(m->replay_fd, m->offset, SEEK_SET); + // Write required '#' so that the regular worker processes + // will ignore that meta line. + fprintf(m->replay_fd, "#"); + + // Required for parsing in 'meta_read_s' + fprintf(m->replay_fd, "|"); +} + +void meta_write_s(meta_s *m, char *key, char *val) +{ + fprintf(m->replay_fd, "%s=%s|", key, val); +} + +void meta_write_l(meta_s *m, char *key, long val) +{ + char buf[1024]; + sprintf(buf, "%ld", val); + fprintf(m->replay_fd, "%s=%ld|", key, val); +} + +void meta_read_start(meta_s *m) +{ + size_t len = 0; + m->read_buf = Calloc(_MAX_META_LEN, char); + getline(&m->read_buf, &len, m->replay_fd); +} + +bool meta_read_s(meta_s *m, char *key, char **val) +{ + char *saveptr = NULL; + char *iterate_buf = Clone(m->read_buf); + int keylen = strlen(key); + + char *tok = strtok_r(iterate_buf, "|", &saveptr); + + while (tok) { + if (strncmp(tok, key, keylen) == 0 && tok[keylen] == '=') { + asprintf(val, "%s", tok+keylen+1); + free(iterate_buf); + return true; + } + tok = strtok_r(NULL, "|", &saveptr); + } + + free(iterate_buf); + return false; +} + +bool meta_read_l(meta_s *m, char *key, long *val) +{ + char *buf = NULL; + + if (meta_read_s(m, key, &buf)) { + *val = atol(buf); + free(buf); + return true; + } + + return false; +} diff --git a/ioriot/src/meta/meta.h b/ioriot/src/meta/meta.h new file mode 100644 index 0000000..1c160b1 --- /dev/null +++ b/ioriot/src/meta/meta.h @@ -0,0 +1,107 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef META_H +#define META_H + +#include "../defaults.h" + +/** + * @brief The meta information definition + * + * This is used to write or read meta information to/from the header + * of the .replay file. This information then is used by ioriot + * in other steps. E.g. reading the amount of used file descriptors + * from the meta header in order to allocate data structures of the + * correct sizes before running the test! + */ +typedef struct meta_s_ { + FILE* replay_fd; /**< The FS of the .replay file */ + off_t offset; /**< The meta offset (usually 0) */ + char* read_buf; /**< Pointer to a read buffer */ +} meta_s; + +/** + * @brief Creates a new meta bject + * + * @return The new meta object + */ +meta_s* meta_new(); + +/** + * @brief Destroys a meta object + * + * @param m The meta object + */ +void meta_destroy(meta_s *m); + +/** + * @brief Reserves space in the .replay file for the meta header + * + * @param m The meta object + */ +void meta_reserve(meta_s *m); + +/** + * @brief Indicates that we start writing the meta header to the .replay file + * + * @param m The meta object + */ +void meta_write_start(meta_s *m); + +/** + * @brief Writes a string to the meta header + * + * @param m The meta object + * @param key The key + * @param val The string value + */ +void meta_write_s(meta_s *m, char *key, char *val); + +/** + * @brief Writes a long to the meta header + * + * @param m The meta object + * @param key The key + * @param val The long value + */ +void meta_write_l(meta_s *m, char *key, long val); + +/** + * @brief indicates that we start reading from the meta header + * + * @param m The meta object + */ +void meta_read_start(meta_s *m); + +/** + * @brief Reads a string from the meta header + * + * @param m The meta object + * @param key The key + * @param val The string val read + */ +bool meta_read_s(meta_s *m, char *key, char **val); + +/** + * @brief Reads a long from the meta header + * + * @param m The meta object + * @param key The key + * @param val The long val read + */ +bool meta_read_l(meta_s *m, char *key, long *val); + +#endif // META_H + diff --git a/ioriot/src/mounts.c b/ioriot/src/mounts.c new file mode 100644 index 0000000..d754557 --- /dev/null +++ b/ioriot/src/mounts.c @@ -0,0 +1,400 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mounts.h" + +#include "utils/futils.h" + +#define _PATH_INSERT "/.ioriot/" +#define _PATH_INSERT_LEN 11 // strlen of _PATH_INSERT + +void mounts_read(mounts_s *m) +{ + char *mounts = "/proc/mounts"; + size_t len = 0; + char *line = NULL; + char *saveptr = NULL; + + Put("Reading '%s'", mounts); + + FILE *fp = Fopen(mounts, "r"); + Out("Adding supported file systems to replay paths:"); + + while (getline(&line, &len, fp) != -1) { + bool ignore = true; + + char *dev = strtok_r(line, " ", &saveptr); + if (dev == NULL) { + Error("Could not parse device from %s", mounts); + } + + char *mp = strtok_r(NULL, " ", &saveptr); + if (mp == NULL) { + Error("Could not parse mountpoint from %s", mounts); + } + + char *fs = strtok_r(NULL, " ", &saveptr); + if (fs == NULL) { + Error("Could not parse file system from %s", mounts); + } +#ifdef MP_DEBUG + Debug("fs:%s", fs); +#endif + // TODO: Make file system types configurable + if (Eq(fs, "ext2")) { + ignore = false; + } else if (Eq(fs, "ext5")) { + ignore = false; + } else if (Eq(fs, "ext4")) { + ignore = false; + } else if (Eq(fs, "xfs")) { + ignore = false; + } else if (Eq(fs, "zfs")) { + ignore = false; + } else if (Eq(fs, "btrfs")) { + ignore = false; + } + + if (ignore) { + if (strcmp(mp, "/") != 0) { + m->ignore_mps[m->ignore_count] = Clone(mp); + m->ignore_count++; + } + + } else if (m->count >= MAX_MOUNTPOINTS) { + Error("Exceeded max mount points: %d\n", m->count); + + } else { + Out(" %s (%s)", mp, fs); + m->mps[m->count] = Clone(mp); + m->lengths[m->count] = strlen(mp); + m->count++; + } + } + + fclose(fp); + Out("\n"); +} + +mounts_s *mounts_new(options_s *opts) +{ + mounts_s *m = Malloc(mounts_s); + + m->opts = opts; + m->count = 0; + m->ignore_count = 0; + mounts_read(m); + + return m; +} + +void mounts_destroy(mounts_s *m) +{ + if (!m) + return; + for (int i = 0; i < m->count; i++) + free(m->mps[i]); + free(m); +} + +void mounts_trash(mounts_s *m) +{ + options_s *opts = m->opts; + set_limits_drop_root(opts->user); + Put("Moving all old files to trash (of previous tests)..."); + + struct timeval tv; + gettimeofday(&tv, NULL); + + char *wd_path = NULL; + asprintf(&wd_path, "%s/%s", opts->wd_base, opts->name); + + char *trash_path = NULL; + asprintf(&trash_path, "%s/.trash/%ld", opts->wd_base, tv.tv_sec); + + if (is_dir(wd_path)) { + ensure_dir_exists(trash_path); + chown_path(opts->user, trash_path); + if (rename(wd_path, trash_path)) { + Errno("Could not move '%s' to '%s'", wd_path, trash_path); + } + } + free(wd_path); + free(trash_path); + + for (int i = 0; i < m->count; i++) { + char *mp = m->mps[i]; + char *path = NULL; + asprintf(&path, "%s/%s/%s", mp, _PATH_INSERT, opts->name); + asprintf(&trash_path, "%s/%s/.trash/%ld", + mp, _PATH_INSERT, tv.tv_sec); + + if (is_dir(path)) { + ensure_dir_exists(trash_path); + chown_path(opts->user, trash_path); + if (rename(path, trash_path)) { + Errno("Could not move '%s' to '%s'", path, trash_path); + } + } + + free(path); + free(trash_path); + } + + Put("Done trashing!"); + Put("Once the drives fill up you may want to purge old data (-P)"); +} + +void mounts_purge(mounts_s *m) +{ + options_s *opts = m->opts; + set_limits_drop_root(opts->user); + + Out("Purging all data from the following directories:"); + + int active_purgers = 0, max_purgers = 16; + if (opts->num_workers > max_purgers) + max_purgers = opts->num_workers; + + char *purge_path = NULL; + asprintf(&purge_path, "%s", opts->wd_base); + if (is_dir(purge_path)) { + Out(" %s", purge_path); + pid_t pid = fork(); + + if (pid == 0) { + ensure_dir_empty(purge_path); + free(purge_path); + exit(0); + + } else if (pid < 0) { + Errno("\nUnable to create cleaner process! :'-("); + } + active_purgers++; + } + free(purge_path); + + int cleaner_status = SUCCESS; + + for (int i = 0; i < m->count; i++) { + char *mp = m->mps[i]; + char *purge_path = NULL; + asprintf(&purge_path, "%s/%s", mp, _PATH_INSERT); + + if (is_dir(purge_path)) { + if (active_purgers+1 >= max_purgers) { + wait(&cleaner_status); + active_purgers--; + } + + // TODO: Use threading model same way as in init/init.c + pid_t pid = fork(); + if (pid == 0) { + Out(" %s", purge_path); + ensure_dir_empty(purge_path); + free(purge_path); + exit(0); + } else if (pid < 0) { + Errno("Unable to create cleaner process! :'-("); + } + active_purgers++; + } + free(purge_path); + } + + while (wait(&cleaner_status) > 0) + active_purgers--; + Put("\nCleaning done!"); +} + +void mounts_init(mounts_s *m) +{ + options_s *opts = m->opts; + char *wd_path = NULL; + asprintf(&wd_path, "%s/%s", opts->wd_base, opts->name); + ensure_dir_exists(wd_path); + chown_path(opts->user, opts->wd_base); + chown_path(opts->user, wd_path); + + if (chdir(wd_path)) { + Errno("Could not chdir into '%s'!", wd_path); + + } else { + Put("Chdir into '%s'", wd_path); + } + + free(wd_path); + + for (int i = 0; i < m->count; i++) { + char *mp = m->mps[i]; + char *path = NULL; + + // Create .ioriot/ directory on MP + asprintf(&path, "%s/%s", mp, _PATH_INSERT); + ensure_dir_exists(path); + chown_path(m->opts->user, path); + free(path); + path = NULL; + + // Create .ioriot/NAME directory on MP + asprintf(&path, "%s/%s/%s", mp, _PATH_INSERT, opts->name); + ensure_dir_exists(path); + chown_path(m->opts->user, path); + free(path); + } +} + +bool mounts_ignore_path(mounts_s *m, const char *path) +{ + // CentOS 7 specific, ignore temp namespace mounts! + char *pos = strstr(path, "/tmp/namespace-"); + if (pos == path) + return true; + + // iterate backwards through all mount points. + for (int i = m->ignore_count-1; i >= 0; --i) { + char *mountpoint = m->ignore_mps[i]; + pos = strstr(path, mountpoint); + // Ignore this path as it is in the ignore mp list + if (pos == path) + return true; + } + + return false; +} + +bool mounts_transform_path(mounts_s *m, const char *name, + char *path, char **path_r) +{ + char *tmp = NULL; +#ifdef DEBUG_TRANSFORM_PATH + char *original_path = path; +#endif + bool line_ok = true; + + // First figure out whether there are '..' in any paths. If so we have to + // tokenize the path and remove '..'. Example: + // transform '/foo/bar/../' into '/foo/'. + // Also remove double '/' from paths. + + if (strstr(path, "..") || strstr(path, "//")) { + // tmp will be freed under label 'cleanup' at end of function. + tmp = Calloc(strlen(path)+1, char); + + // stack to put the tokens on + stack_s *s = stack_new(); + + // we need a copy of the path, so we can tokenize it into the stack + char* clone = Clone(path); + + char *saveptr = NULL; + char *tok = strtok_r(clone, "/", &saveptr); + + // Add each part of the path to the stack. + while (tok) { + if (strcmp(tok, "..") == 0) { + stack_pop(s); + } else { + stack_push(s, tok); + } + tok = strtok_r(NULL, "/", &saveptr); + } + + if (stack_is_empty(s)) { + strcpy(tmp, "."); + + } else { + s = stack_new_reverse_from(s); + strcpy(tmp, "/"); + strcat(tmp, (char*)stack_pop(s)); + + while(!stack_is_empty(s)) { + strcat(tmp, "/"); + strcat(tmp, (char*)stack_pop(s)); + } + } + + stack_destroy(s); + free(clone); + + // This is the path without '..' and '//' (and '///' ... etc') + path = tmp; + } + + // Now heck whether the path is on a supported file system. If not, ignore! + if (mounts_ignore_path(m, path)) { + line_ok = false; + goto cleanup; + } + + // So the path is on a valid mount point! Now we need to insert + // .ioriot/NAME to each mount point, e.g. /usr/local/.ioriot/NAME/... + + // Iterate backwards through all mount points. + for (int i = m->count-1; i >= 0; --i) { + char *mountpoint = m->mps[i]; + int mp_len = m->lengths[i]; + + if (strncmp(path, mountpoint, mp_len) == 0) { + // Found a path to replace + // Now insert .ioriot/NAME/ into the file path. + *path_r = Calloc(strlen(path) + strlen(name)+1 + + _PATH_INSERT_LEN+1, char); + + if (strcmp(mountpoint, "/") == 0) { + // Root path + strcpy(*path_r, _PATH_INSERT); + strcat(*path_r, name); + strcat(*path_r, path); + + } else { + strcpy(*path_r, mountpoint); + strcat(*path_r, _PATH_INSERT); + strcat(*path_r, name); + char *pos = path; + pos += mp_len * (int) sizeof(char); + strcat(*path_r, pos); + } + + goto cleanup; + } + } + + if (tmp) + free(tmp); + + return line_ok; + +cleanup: +#ifdef DEBUG_TRANSFORM_PATH + Debug("Transform path '%s' -> '%s' -> '%s'", original_path, path, *path_r); +#endif + if (tmp) + free(tmp); + + return line_ok; +} + +int mounts_get_mountnumber(mounts_s *m, const char *path) +{ + for (int i = m->count-1; i >= 0; --i) { + char *mountpoint = m->mps[i]; + int mp_len = m->lengths[i]; + + if (strncmp(path, mountpoint, mp_len) == 0) + return i; + } + + return 0; +} diff --git a/ioriot/src/mounts.h b/ioriot/src/mounts.h new file mode 100644 index 0000000..1c4a3d4 --- /dev/null +++ b/ioriot/src/mounts.h @@ -0,0 +1,154 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MOUNTPOINTS_H +#define MOUNTPOINTS_H + +#include "datas/stack.h" +#include "defaults.h" +#include "options.h" + +#define MAX_MOUNTPOINTS 1024 + +/** + * @brief Represents data parsed from /proc/mounts + * + * This is used to determine the file systems and the file system types + * currently mounted on the Linux system. I/O replay only replays I/O + * on specific file systems such as ext4 or xfs and will ignore any special + * or pseudo file systems such as tmpfs, devfs, sysfs. It does not make sense + * to replay I/O on these because there is actually no underlying block device + * attached to these. + * + * A mounts object helps to determine whether a path relies on a valid file + * system or not. All I/O operations on invalid file systems are being filtered + * out! + * + * The mounts object also does more things such as purging temp test data from + * the mountpoints etc... + */ +typedef struct mounts_s_ { + int count; /**< The amount of mount points */ + char *mps[MAX_MOUNTPOINTS]; /**< The mp paths */ + int lengths[MAX_MOUNTPOINTS]; /**< The mp lenghts */ + int ignore_count; /**< The amount of ignored mount points */ + char *ignore_mps[MAX_MOUNTPOINTS]; /**< The ignored mp paths */ + options_s *opts; /**< A pointer to the options object */ +} mounts_s; + +/** + * @brief Creates a new mounts object + * + * @param opts The options object + * @return The new mounts object + */ +mounts_s *mounts_new(options_s *opts); + +/** + * @brief Destroys the mounts object + * + * @param m The mounts object + */ +void mounts_destroy(mounts_s *m); + +/** + * @brief moves all files within replay mounts to trash + * + * It moves all files of the .ioriot/NAME directories to + * .ioriot/NAME.trashEPOCH directories for all available mount points. + * It does the same for the working dorectory of the current test. + * + * @param m The responsible mounts object + */ +void mounts_trash(mounts_s *m); + +/** + * @brief Deletes all files within replay mounts + * + * It deletes all files from the .ioriot/ directories for all availabe + * mount points. It also deletes the working directory of all tests. The + * function forks one sub-process per mount point, so it is cleaning all drives + * in parallel. + * + * It can take a significant amount of time to actually delete all these files. + * That's why there is also a mounts_trash function, which will not delete the + * files but move them to trash folders so they can be deleted at a later + * point. + * + * @param m The responsible mounts object + */ +void mounts_purge(mounts_s *m); + +/** + * @brief Ensures all mounts have a .ioriot/NAME directory + * + * These directories are used by ioriot to run the I/O replay tests in. + * The function also ensures to have the correct user permissions for these + * directories. + * + * @param m The responsible mounts object + */ +void mounts_init(mounts_s *m); + +/** + * @brief Reads /proc/mounts to determine which mounts are available + * + * @param m The mounts object + */ +void mounts_read(mounts_s *m); + +/** + * @brief Determines whether a path should be ignored + * + * ioriot replays I/O only on known mount points of known + * file system types. This function helps to determine whether + * a path is on a valid mount point or not. + * + * @param m The responsible mounts object + * @param path The path to check + * @return true if path has to be ignored + */ +bool mounts_ignore_path(mounts_s *m, const char *path); + +/** + * @brief Inserts ./ioriot/NAME into a path + * + * This function inserts ./ioriot/NAME into a given file path. + * The function also checks whether the path is on a supported replay + * path or not. E.g. we want to ignore file systems such as devfs, sysfs, + * procfs.. etc. + * + * @param m The responsible mountpoint object + * @param name The name of the test + * @param path The original path + * @param path_r The tansformed path (has to be freed if not NULL) + * @return False if this path is to be ignored + */ +bool mounts_transform_path(mounts_s *m, const char *name, + char *path, char **path_r); + + +/** + * @brief Get's the mount point number of a path + * + * Used by init.c to determine which thread to use to initialise a file + * or directory on a given path. + * + * @param m The responsible mountpoint object + * @param path The file/directory path + * @return The mountpoint number + */ +int mounts_get_mountnumber(mounts_s *m, const char *path); + +#endif // MOUNTPOINTS_H diff --git a/ioriot/src/opcodes.h b/ioriot/src/opcodes.h new file mode 100644 index 0000000..3d5c114 --- /dev/null +++ b/ioriot/src/opcodes.h @@ -0,0 +1,103 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPCODES_H +#define OPCODES_H + +typedef enum { + // stat() syscalls + FSTAT = 0, + FSTAT_AT, + FSTATFS, + FSTATFS64, + LSTAT, + STAT, + STATFS, + STATFS64, + + // read() syscalls + READ = 10, + READV, + READAHEAD, + READDIR, + READLINK, + READLINK_AT, + + // write() syscalls + WRITE = 20, + WRITEV, + + // open() and other syscalls which may create files + OPEN = 30, + OPEN_AT, + CREAT, + MKDIR, + MKDIR_AT, + NAME_TO_HANDLE_AT, + OPEN_BY_HANDLE_AT, + + // rename() syscalls + RENAME = 40, + RENAME_AT, + RENAME_AT2, + + // close() and unlink() syscalls + CLOSE = 50, + UNLINK, + UNLINK_AT, + RMDIR, + + // sync() syscalls + FSYNC = 60, + FDATASYNC, + SYNC, + SYNCFS, + SYNC_FILE_RANGE, + + // other syscalls + FCNTL = 70, + GETDENTS, + LSEEK, + + // mmap syscalls + MMAP2 = 80, + MUNMAP, + REMAP, + MSYNC, + + // chmod() syscalls + CHMOD = 100, + FCHMOD, + FCHMODAT, + + // chown() syscalls + CHOWN = 110, + CHOWN16, + LCHOWN, + LCOWN16, + FCHOWN, + FCHOWN16, + FCHOWNAT, + + // Meta operations (I/O replay internal use only) + // A single thread terminates + META_EXIT = 900, + // All threads of a process termiate (process termination) + META_EXIT_GROUP, + // Meta operation for lamport synchronisation (currently unused) + META_TIMELINE + +} opcode_e; + +#endif // OPCODES_H diff --git a/ioriot/src/options.c b/ioriot/src/options.c new file mode 100644 index 0000000..01a5375 --- /dev/null +++ b/ioriot/src/options.c @@ -0,0 +1,51 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "options.h" + +options_s *options_new() +{ + options_s *o = Malloc(options_s); + + o->capture_file = NULL; + o->replay_file = NULL; + o->stats_file = NULL; + o->wd_base = "/usr/local/ioriot"; + o->num_workers = 4; + o->num_threads_per_worker = 128; + o->user = "mcuser"; + o->name = "test0"; + o->init = false; + o->replay = false; + o->speed_factor = 0; + o->drop_caches = false; + o->purge = false; + o->trash = false; + o->pid = -1; + o->module = "ioriot.ko"; + + return o; +} + +void options_destroy(options_s *o) +{ + if (o->capture_file) + free(o->capture_file); + if (o->replay_file) + free(o->replay_file); + if (o->stats_file) + free(o->stats_file); + + free(o); +} diff --git a/ioriot/src/options.h b/ioriot/src/options.h new file mode 100644 index 0000000..eca92ff --- /dev/null +++ b/ioriot/src/options.h @@ -0,0 +1,61 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPTIONS_H +#define OPTIONS_H + +#include +#include "defaults.h" + +/** + * @brief The options definition, used to store user input + */ +typedef struct options_s_ { + char *capture_file; /**< The name of the .capture file */ + char *replay_file; /**< The name of the .replay file */ + char *stats_file; /**< The name of the .stats file */ + bool write_stats_file; /**< Write a stats file at the end of the test */ + char *user; /**< The user name to run the test as */ + char *name; /**< The name of the test (found in .ioriot/name sub-dirs) */ + char *wd_base; /**< The working directory base */ + int num_workers; /**< The amount of worker processes */ + int num_threads_per_worker; /**< Max threads per worker processes */ + bool init; /**< If set ioriot will initialise the environment */ + bool replay; /**< If set ioriot will run/replay the test */ + bool purge; /**< If set ioriot will purge the environment */ + bool trash; /**< If set ioriot will trash the environment */ + bool drop_caches; /**< True if ioriot should drop all Linux caches */ + double speed_factor; /**< Specifies how fast the test is replayed */ + int pid; /**< Specifies a process id to capture */ + char *module; /**< Specifies the kernel module for capturing */ +} options_s; + +/** + * @brief Creates a new options object + * + * The options object contains all options specified by the user as a command + * line option. It is filled with default values during creation. + * + * @return The options object + */ +options_s *options_new(); + +/** + * @brief Destroys the options object + * + * @param o The options object + */ +void options_destroy(options_s *o); + +#endif // OPTIONS_H diff --git a/ioriot/src/replay/replay.c b/ioriot/src/replay/replay.c new file mode 100644 index 0000000..e4606d1 --- /dev/null +++ b/ioriot/src/replay/replay.c @@ -0,0 +1,191 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "replay.h" + +#include "../datas/amap.h" +#include "../meta/meta.h" +#include "../mounts.h" +#include "rworker.h" +#include "rstats.h" + +void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes, + long *num_pids, long *num_fds, long *num_lines) +{ + meta_s *m = meta_new(replay_fd); + meta_read_start(m); + + long version = 0; + if (meta_read_l(m, "version", &version)) { + Put("Replay version is '%ld'", version); + if (version != REPLAY_VERSION) { + Error(".replay file of incompatible version, got %x, expected %x", + (int)version, REPLAY_VERSION); + } + } + + char *user; + if (meta_read_s(m, "user", &user)) { + Put("Setting user to '%s'", user); + opts->user = user; + } + + char *name; + if (meta_read_s(m, "name", &name)) { + Put("Setting name to '%s'", name); + opts->name = name; + } + + if (meta_read_l(m, "num_vsizes", num_vsizes)) { + if (*num_vsizes < 0) { + Error("Lamport vsize overflow"); + } + Put("Setting num of vsizes to '%ld'", *num_vsizes); + } + + if (meta_read_l(m, "num_mapped_pids", num_pids)) { + if (*num_pids < 0) { + Error("Process overflow (too many process IDs in .replay)"); + } + Put("Setting num of PIDs to '%ld'", *num_pids); + } + + if (meta_read_l(m, "num_mapped_fds", num_fds)) { + if (*num_fds < 0) { + Error("FD overflow (too many FDs in .replay)"); + } + Put("Setting num of FDs to '%ld'", *num_fds); + } + + if (meta_read_l(m, "num_lines", num_lines)) { + if (*num_fds < 0) { + Error("Overflow (too many lines in .replay)"); + } + Put("Setting num of lines to '%ld'", *num_lines); + } + + meta_destroy(m); +} + +status_e replay_run(options_s *opts) +{ + status_e status = SUCCESS; + + if (opts->drop_caches) { + drop_caches(); + //cache_file(opts->replay_file); + } + + // Extract information from the meta header + FILE *replay_fd = Fopen(opts->replay_file, "r"); + long num_vsizes = 0, num_pids = 0, num_fds = 0, num_lines = 0; + replay_extract_header(opts, replay_fd, &num_vsizes, &num_pids, + &num_fds, &num_lines); + fclose(replay_fd); + + // A map of all file descriptors used. + Out("Creating FD map..."); + amap_s *fds_map = NULL; + if (opts->num_workers > 1) { + fds_map = amap_new_mmapped(num_fds); + } else { + fds_map = amap_new(num_fds); + } + Put("done"); + + // To collect all individual worker's stats into the global + // stats object. + stack_s *all_worker_stats = stack_new(); + + // The global stats object + rstats_s *stats = rstats_new(opts); + rstats_start(stats); + + // Fork worker processes, each worker process will read the .replay file + // individually. + + if (opts->num_workers > 1) { + for (int i = 0; i < opts->num_workers; ++i) { + rworker_stats_s *worker_stats = rworker_stats_new_mmap(); + stack_push(all_worker_stats, worker_stats); + + pid_t pid = fork(); + + if (pid == 0) { + // One worker object per fork + rworker_s *w = rworker_new(i, fds_map, num_vsizes, num_pids, opts, + worker_stats); + + // Process the .replay journal line by line + status_e status = rworker_process_lines(w, num_lines); + Put("worker(%d): Exiting from %d with status %d", i, + pid, status); + rworker_destroy(w); + + // Exit sub-process + exit(status); + + } else if (pid < 0) { + Errno("worker(%d): Unable to create worker process! :'-(", i); + + } else { + Put("worker(%d): Process with pid %d forked", i, pid); + } + } + + set_limits_drop_root(opts->user); + + Put("Waiting for worker processes to finish"); + pid_t pid; + int rworker_status = SUCCESS; + + while ((pid = wait(&rworker_status)) > 0) { + if (rworker_status != SUCCESS) + status = rworker_status; + + Put("Process with pid %d exited with status %d", + pid, rworker_status); + } + + Put("All workers finished (%d)!", status); + + } else { + Put("Only one worker, don't fork sub-processes"); + + rworker_stats_s *worker_stats = rworker_stats_new_mmap(); + stack_push(all_worker_stats, worker_stats); + + rworker_s *w = rworker_new(0, fds_map, num_vsizes, num_pids, + opts, worker_stats); + status = rworker_process_lines(w, num_lines); + rworker_destroy(w); + + Put("Worker finished work!"); + } + + // Collect all statistics + rstats_stop(stats); + while (!stack_is_empty(all_worker_stats)) { + rworker_stats_s *worker_stats = stack_pop(all_worker_stats); + rstats_add_from_worker(stats, worker_stats); + rworker_stats_destroy(worker_stats); + } + stack_destroy(all_worker_stats); + + rstats_print(stats); + rstats_destroy(stats); + + amap_destroy(fds_map); + return status; +} diff --git a/ioriot/src/replay/replay.h b/ioriot/src/replay/replay.h new file mode 100644 index 0000000..dcc3d84 --- /dev/null +++ b/ioriot/src/replay/replay.h @@ -0,0 +1,46 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef REPLAY_H +#define REPLAY_H + +#include "../defaults.h" +#include "../utils/futils.h" +#include "../opcodes.h" +#include "../options.h" +#include "rioop.h" +#include "rprocess.h" + +/** + * @brief Replays the given .replay file + * + * @param opts The options object + * @return SUCCESS if everything went fine + */ +status_e replay_run(options_s *opts); + +/** + * @brief Extract required meta data from .replay's meta header + * + * @param opts The options object + * @param replay_fd The file handle to the .replay file + * @param num_vsizes The amount of virtual sizes/paths + * @param num_pids The amount of process IDs + * @param num_fds The amount of virtual file descriptors + * @param num_lines The amount of .replay lines with I/O ops + */ +void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes, + long *num_pids, long *num_fds,long *num_lines); + +#endif // REPLAY_H diff --git a/ioriot/src/replay/rioop.c b/ioriot/src/replay/rioop.c new file mode 100644 index 0000000..2e16c94 --- /dev/null +++ b/ioriot/src/replay/rioop.c @@ -0,0 +1,425 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rioop.h" + +#include "../vfd.h" +#include "rworker.h" + +// Printing error messages +#define _Error(...) \ + fprintf(stderr, "%s:%d ERROR: ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\nlineno:%ld path:%s\n", task->lineno, vfd->path); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define _Errno(...) \ + fprintf(stderr, "%s:%d ERROR: %s (%d). ", __FILE__, __LINE__, \ + strerror(errno), errno); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\nlineno:%ld path:%s\n", task->lineno, vfd->path); \ + fflush(stdout); \ + fflush(stderr); \ + exit(ERROR); + +#define _Init_arg(num) int arg = atoi(task->toks[num]) +#define _Init_cmd(num) int cmd = atoi(task->toks[num]) +#define _Init_fd(num) long fd = atol(task->toks[num]) +#define _Init_flags(num) int flags = atoi(task->toks[num]) +//#define _Init_mode(num) int mode = atoi(task->toks[num]) +#define _Init_offset(num) long offset = atol(task->toks[num]) +#define _Init_op(num) int op = atoi(task->toks[num]) +#define _Init_path2(num) char *path2 = task->toks[num] +#define _Init_path(num) char *path = task->toks[num] +#define _Init_rc(num) int rc = atoi(task->toks[num]) +#define _Init_whence(num) long whence = atol(task->toks[num]) + +#define _Init_bytes(num) \ + int bytes = atoi(task->toks[num]); \ + if (bytes <= 0) return + +#define _Init_virtfd \ + vfd_s *vfd = amap_get(p->fds_map, fd); \ + if (vfd == NULL) return + +void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_op(2); + + switch (op) { + // stat() syscalls + case FSTAT: + rioop_fstat(p, t, task); + break; + case FSTATFS: + case FSTATFS64: + //Error("op(%d) not implemented", op); + break; + case FSTAT_AT: + case LSTAT: + case STAT: + rioop_stat(p, t, task); + break; + case STATFS: + case STATFS64: + //Error("op(%d) not implemented", op); + break; + + // read() syscalls + case READ: + case READV: + rioop_read(p, t, task); + break; + case READAHEAD: + //Error("op(%d) not implemented", op); + break; + case READLINK: + case READLINK_AT: + //Error("op(%d) not implemented", op); + break; + + // write() syscalls + case WRITE: + case WRITEV: + rioop_write(p, t, task); + break; + + // open() and other syscalls which may creat + case OPEN: + case OPEN_AT: + rioop_open(p, t, task, -1); + break; + case CREAT: + // A call to crat() is equivalent to calling open() with flags.. + rioop_open(p, t, task, O_CREAT|O_WRONLY|O_TRUNC); + break; + case MKDIR: + case MKDIR_AT: + rioop_mkdir(p, t, task); + break; + + // rename() syscalls + case RENAME: + case RENAME_AT: + case RENAME_AT2: + rioop_rename(p, t, task); + break; + + // close() and unlink() syscalls + case CLOSE: + rioop_close(p, t, task); + break; + case UNLINK: + case UNLINK_AT: + rioop_unlink(p, t, task); + break; + case RMDIR: + rioop_rmdir(p, t, task); + break; + + // sync() syscalls + case FSYNC: + rioop_fsync(p, t, task); + break; + case FDATASYNC: + rioop_fdatasync(p, t, task); + break; + case SYNC: + case SYNCFS: + case SYNC_FILE_RANGE: + //Error("op(%d) not implemented", op); + break; + + // Other syscalls + case FCNTL: + rioop_fcntl(p, t, task); + break; + case GETDENTS: + rioop_getdents(p, t, task); + break; + case LSEEK: + rioop_lseek(p, t, task); + break; + + // chmod() syscalls + case CHMOD: + rioop_chmod(p, t, task); + break; + case FCHMOD: + rioop_fchmod(p, t, task); + break; + + // chown() syscalls + case CHOWN: + rioop_chown(p, t, task); + break; + case FCHOWN: + case FCHOWNAT: + rioop_fchown(p, t, task); + break; + case LCHOWN: + rioop_lchown(p, t, task); + break; + + // Meta operations (I/O replay internal use only). + case META_EXIT_GROUP: + break; + case META_TIMELINE: + break; + + default: + Error("op(%d) not implemented", op); + break; + } +} + +void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + struct stat buf; + stat(path, &buf); +} + +void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + struct stat buf; + fstat(vfd->fd, &buf); +} + +void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + _Init_path2(4); + rename(path, path2); +} + +void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_bytes(4); + _Init_virtfd; + + char *buf = Calloc(bytes+1, char); + read(vfd->fd, buf, bytes); + free(buf); +} + +void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_bytes(4); + _Init_virtfd; + + char *buf = Calloc(bytes+1, char); + sprintf(buf, "%ld", task->lineno); + Fill_with_stuff(buf, bytes); + if (vfd->fd == 0) { + Debug("%d %d %ld", vfd->fd, vfd->debug, task->lineno); + _Error("ERROR"); + } + write(vfd->fd, buf, bytes); + free(buf); +} + +void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_) +{ + _Init_fd(3); + _Init_path(4); + _Init_flags(6); + + // Special case as this is creat() now + if (flags_ != -1) + flags = flags_; + + bool directory = Has(flags, O_DIRECTORY); + + if (fd > 0) { + if (directory) { + // We can not open a directory via open() otherwise! + flags &= (O_RDONLY & ~(O_RDWR|O_WRONLY|O_CREAT)); + } else { + // We don't want to open the file in read only mode. + // SystemTap could have skipped syscalls to fcntl or open + flags &= ~O_RDONLY; + } + // flags |= O_DIRECT|O_SYNC; + flags &= ~O_EXCL; + } + + int ret = open(path, flags, S_IRWXU|S_IRWXG|S_IRWXO); + + if (fd < 0 && ret > 0) { + close(ret); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|open+close|%s|\n", path); + fflush(t->rthread_fd); +#endif + } + + if (fd > 0 && ret > 0) { + vfd_s *vfd = vfd_new(ret, fd, path); + amap_set(p->fds_map, fd, vfd); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|open|%s|\n", path); + fflush(t->rthread_fd); +#endif + } +} + +void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + + amap_unset(p->fds_map, fd); + if (vfd->dirfd) { + closedir(vfd->dirfd); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|closedir|%s|\n", vfd->path); + fflush(t->rthread_fd); +#endif + } else { + close(vfd->fd); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|close|%s|\n", vfd->path); + fflush(t->rthread_fd); +#endif + } + vfd_destroy(vfd); +} + +void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + + // getdents expects a dirfd + DIR *dirfd = fdopendir(vfd->fd); + if (dirfd) { + vfd->dirfd = dirfd; + readdir(dirfd); +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "TRACE OPEN|fdopendir|%s|\n", vfd->path); + fflush(t->rthread_fd); +#endif + } +} + +void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + mkdir(path, S_IRWXU|S_IRWXG|S_IRWXO); +} + +void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + unlink(path); +} + +void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + rmdir(path); +} + +void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_bytes(6); + _Init_virtfd; + lseek(vfd->fd, bytes, SEEK_SET); +} + +void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + fsync(vfd->fd); +} + +void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + fdatasync(vfd->fd); +} + +void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_cmd(4); + _Init_arg(5); + _Init_virtfd; + + switch (cmd) { + case F_GETFD: + case F_GETFL: + fcntl(vfd->fd, cmd); + break; + case F_SETFD: + case F_SETFL: + fcntl(vfd->fd, cmd, arg); + break; + default: + break; + } +} + +void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + chmod(path, S_IRWXU|S_IRWXG|S_IRWXO); +} + +void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + fchmod(vfd->fd, S_IRWXU|S_IRWXG|S_IRWXO); +} + +void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + rworker_s *w = t->worker; + options_s *opts = w->opts; + struct passwd *pwd = getpwnam(opts->user); + chown(path, pwd->pw_uid, -1); +} + +void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_fd(3); + _Init_virtfd; + rworker_s *w = t->worker; + options_s *opts = w->opts; + struct passwd *pwd = getpwnam(opts->user); + fchown(vfd->fd, pwd->pw_uid, -1); +} + +void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task) +{ + _Init_path(3); + rworker_s *w = t->worker; + options_s *opts = w->opts; + struct passwd *pwd = getpwnam(opts->user); + lchown(path, pwd->pw_uid, -1); +} + diff --git a/ioriot/src/replay/rioop.h b/ioriot/src/replay/rioop.h new file mode 100644 index 0000000..4db4284 --- /dev/null +++ b/ioriot/src/replay/rioop.h @@ -0,0 +1,54 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RIOOP_H +#define RIOOP_H + +#include "../defaults.h" +#include "../utils/futils.h" +#include "../opcodes.h" +#include "rprocess.h" +#include "rthread.h" + +/** + * @brief Replays the responsible I/O operation of a given task + * + * @param p The virtual replay process object + * @param t The thread object + * @param task The replay task object + */ +void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task); + +void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_); +void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task); +void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task); + +#endif // RIOOP_H diff --git a/ioriot/src/replay/rprocess.c b/ioriot/src/replay/rprocess.c new file mode 100644 index 0000000..4efd835 --- /dev/null +++ b/ioriot/src/replay/rprocess.c @@ -0,0 +1,34 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rprocess.h" + +rprocess_s* rprocess_new(const int pid, amap_s *fds_map) +{ + rprocess_s *p = Malloc(rprocess_s); + + p->fds_map = fds_map; + p->pid = pid; + p->terminate = 0; + p->lineno = 0; + + return p; +} + +void rprocess_destroy(rprocess_s *p) +{ + if (!p) + return; + free(p); +} diff --git a/ioriot/src/replay/rprocess.h b/ioriot/src/replay/rprocess.h new file mode 100644 index 0000000..adba0f3 --- /dev/null +++ b/ioriot/src/replay/rprocess.h @@ -0,0 +1,40 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RPROCESS_H +#define RPROCESS_H + +#include "../datas/hmap.h" +#include "../datas/amap.h" +#include "../defaults.h" +#include "rthread.h" + +/** + * @brief The virtual replay process object definition + * + * This defines a virtual process in replay context. + */ +typedef struct rprocess_s_ { + int terminate; /**< Indicates whether the worker is terminating or not */ + int rworker_num; /**< The worker number of the responsible worker */ + int pid; /**< The virtual process ID */ + unsigned long lineno; /**< Holding the current .replay line number */ + bool initm; /**< Indicates whether ioriot is in init mode or not */ + amap_s *fds_map; /**< Holding all file descriptors */ +} rprocess_s; + +rprocess_s* rprocess_new(const int pid, amap_s *fds_map); +void rprocess_destroy(rprocess_s* p); + +#endif // RPROCESS_H diff --git a/ioriot/src/replay/rstats.c b/ioriot/src/replay/rstats.c new file mode 100644 index 0000000..c3e6e38 --- /dev/null +++ b/ioriot/src/replay/rstats.c @@ -0,0 +1,108 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rstats.h" + +#include + +rstats_s* rstats_new(options_s *opts) +{ + rstats_s *s = Malloc(rstats_s); + + s->opts = opts; + s->loadavg_high = 0; + s->ioops = 0; + s->duration = 0; + s->time_ahead = -1; + + if (opts->stats_file) + s->stats_fd = Fopen(opts->stats_file, "w"); + else + s->stats_fd = stdout; + + return s; +} + +void rstats_destroy(rstats_s *s) +{ + if (s->stats_fd != stdout) + fclose(s->stats_fd); + + free(s); +} + +rworker_stats_s* rworker_stats_new_mmap(options_s *opts) +{ + // Share this object between processes, so that the stats cann be + // collected by the master process! + rworker_stats_s *s = Mmapshared(rworker_stats_s); + + s->loadavg_high = 0; + s->ioops = 0; + s->time_ahead = -1; + + return s; +} + +void rworker_stats_destroy(rworker_stats_s *s) +{ + munmap(s, sizeof(rworker_stats_s)); +} + + +void rstats_start(rstats_s* s) +{ + gettimeofday(&s->start_time, NULL); +} + +void rstats_stop(rstats_s* s) +{ + gettimeofday(&s->end_time, NULL); + s->duration= ((s->end_time.tv_sec - s->start_time.tv_sec) * 1000 + + (s->end_time.tv_usec - s->start_time.tv_usec) / 1000) / 1000; + +} + +void rstats_add_from_worker(rstats_s* s, rworker_stats_s* w) +{ + if (s->loadavg_high < w->loadavg_high) + s->loadavg_high = w->loadavg_high; + + if (s->time_ahead == -1 || s->time_ahead > w->time_ahead) + s->time_ahead = w->time_ahead; + + s->ioops += w->ioops; +} + +void rstats_print(rstats_s* s) +{ + options_s *opts = s->opts; + + if (opts->stats_file) { + Put("Writing stats to '%s'", opts->stats_file); + } + + fprintf(s->stats_fd, "Stats of test '%s':\n", opts->name); + fprintf(s->stats_fd, "\tNum workers: %d\n", opts->num_workers); + fprintf(s->stats_fd, "\tThreads per worker: %d\n", opts->num_threads_per_worker); + fprintf(s->stats_fd, "\tThreads total: %d\n", + opts->num_threads_per_worker * opts->num_workers); + fprintf(s->stats_fd, "\tHighest loadavg: %.2f\n", s->loadavg_high); + fprintf(s->stats_fd, "\tPerformed ioops: %ld\n", s->ioops); + if (s->duration > 0) + fprintf(s->stats_fd, "\tAverage ioops/s: %.2f\n", s->ioops/s->duration); + fprintf(s->stats_fd, "\tTime ahead: %lds\n", s->time_ahead/1000); + fprintf(s->stats_fd, "\tTotal time: %.2fs\n", s->duration); +} + diff --git a/ioriot/src/replay/rstats.h b/ioriot/src/replay/rstats.h new file mode 100644 index 0000000..1ce3f27 --- /dev/null +++ b/ioriot/src/replay/rstats.h @@ -0,0 +1,117 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file rstats.h + * @author Paul Buetow + * + * @brief For collecting replay stats + */ + +#ifndef RSTATS_H +#define RSTATS_H + +#include "../defaults.h" +#include "../options.h" + +#include + +/** + * @brief Definition of the rstats object + * + * Used to store global statistics. + */ +typedef struct rstats_s_ { + double loadavg_high; /**< Highest load average */ + long ioops; /**< Total amount if io operations */ + double duration; /**< Duration of the test */ + long time_ahead; /**< Time ahead of the original speed */ + struct timeval start_time; /**< Start time of the test */ + struct timeval end_time; /**< End time of the test */ + options_s *opts; /**< The I/O replay options object */ + FILE *stats_fd; /**< The file descriptor for writing the stats */ +} rstats_s; + +/** + * @brief Definition of the per worker stats object + * + * Used to store per worker process I/O stats + */ +typedef struct rworker_stats_s_ { + double loadavg_high; /**< Highest amount of io ops per second */ + long ioops; /**< Total amount if io operations */ + long time_ahead; /**< Time ahead of the original speed */ +} rworker_stats_s; + +/** + * @brief Creates a new stats object + * + * @return The new stats object + */ +rstats_s* rstats_new(options_s *opts); + +/** + * @brief Destroys the stats object + * + * @param s The stats object + */ +void rstats_destroy(rstats_s* s); + +/** + * @brief Creates a new per worker stats object + * + * The memory is mapped into shared memory so it can be shared across multiple + * processes. + * + * @return The new stats object + */ +rworker_stats_s* rworker_stats_new_mmap(); + +/** + * @brief Destroys the per worker stats object + * + * @param s The stats object + */ +void rworker_stats_destroy(rworker_stats_s* s); + +/** + * @brief Starts the stats + * + * @param s The stats object + */ +void rstats_start(rstats_s* s); + +/** + * @brief Finalises the stats + * + * @param s The stats object + */ +void rstats_stop(rstats_s* s); + +/** + * @brief Prints the stats + * + * @param s The stats object + */ +void rstats_print(rstats_s* s); + +/** + * @brief Adds per worker stats to the global stats object + * + * @param s The global stats object + * @param w The worker stats object + */ +void rstats_add_from_worker(rstats_s* s, rworker_stats_s* w); + +#endif diff --git a/ioriot/src/replay/rtask.c b/ioriot/src/replay/rtask.c new file mode 100644 index 0000000..b1afb92 --- /dev/null +++ b/ioriot/src/replay/rtask.c @@ -0,0 +1,50 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rtask.h" + +#include "rthread.h" +#include "rworker.h" + +rtask_s* rtask_new() +{ + rtask_s *task = Malloc(rtask_s); + + *task = (rtask_s) { + .worker = NULL, .process = NULL + }; + task->line[0] = '\0'; + +#ifdef THREAD_DEBUG + task->clone = NULL; +#endif + + return task; +} + +void rtask_destroy(rtask_s *task) +{ + if (task) + free(task); +} + +void rtask_update(rtask_s *task, void *worker, void *process, char *line, + const long lineno, const long vsize) +{ + task->worker = worker; + task->process = process; + task->lineno = lineno; + task->vsize = vsize; + strcpy(task->line, line); +} diff --git a/ioriot/src/replay/rtask.h b/ioriot/src/replay/rtask.h new file mode 100644 index 0000000..35c5714 --- /dev/null +++ b/ioriot/src/replay/rtask.h @@ -0,0 +1,69 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RTASK_H +#define RTASK_H + +#include "../defaults.h" + +/** + * @brief The replay task definition + * + * The rtask holds all possible variables required to process a particular + * .replay line and to replay the corresponding I/O operation. + */ +typedef struct rtask_s_ { + void *worker; /* The responsible worker object */ + void *process; /* The responsible process object */ + unsigned long lineno; /**< The current line number */ + unsigned long vsize; /**< The vsize */ + char *toks[MAX_TOKENS+1]; /**< The tokens parsed from the .replay line */ + char line[MAX_LINE_LEN]; /**< The remaining part of the .replay line */ +#ifdef RTASK_DEBUG + char *clone; /**< Used for debug purposes only */ +#endif +} rtask_s; + +/** + * @brief Creates a new thread task object + * + * This function creates a new thread task object. Such a task object is used + * by the worker to hand over I/O tasks to the corresponding threads. The + * actual I/O work is performed by the threads then. + * + * @return The new thread task object + */ +rtask_s* rtask_new(); + +/** + * @brief Destroys the replay task object + * + * @param t The thread task object to be destroyed + */ +void rtask_destroy(rtask_s* t); + +/** + * @brief Updates a reused/recycle task object + * + * @param task The task object to be updated + * @param worker The responsibe worker object + * @param process The responsible process object + * @param line The remaining line of the .replay file + * @param lineno The current line number of the .replay file + * @param vsize The vsize/path id + */ +void rtask_update(rtask_s *task, void *worker, void *process, char *line, + const long lineno, const long vsize); + +#endif // RTASK_H diff --git a/ioriot/src/replay/rthread.c b/ioriot/src/replay/rthread.c new file mode 100644 index 0000000..ae0a597 --- /dev/null +++ b/ioriot/src/replay/rthread.c @@ -0,0 +1,216 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rthread.h" + +#include + +#include "rworker.h" +#include "rprocess.h" + +#include "rioop.h" + +#ifdef THREAD_DEBUG +/** + * @brief For debugging purposes only + * + * @param t The responsible thread object + */ +static void _rthread_init_log(rthread_s *t) +{ + rworker_s *w = t->worker; + char *rthread_log = Calloc(1024, char); + snprintf(rthread_log, 1023, "/tmp/ioriot/worker%d.thread%ld.debuglog", + w->rworker_num, (long)pthread_self()); + + ensure_dir_exists("/tmp/ioriot"); + t->rthread_fd = Fopen(rthread_log, "a"); + + free(rthread_log); + fprintf(t->rthread_fd, "%ld: DEBUG: Created thread log\n", t->tid); +} +#endif + +void rthread_process_task(rthread_s* t, rtask_s *task, + pid_t pthread_id) +{ + char *next = task->line; + rworker_s *w = (rworker_s*) task->worker; + + // Tokenize the remaining elements of the line. + int ntoks = 0; + char *saveptr; + char *tok = strtok_r(next, "|", &saveptr); + + while (tok) { + if (ntoks > MAX_TOKENS) { + Error("worker(%d) pthread(%d): lineno:%lu, missing newline?", + w->rworker_num, pthread_id, task->lineno); + } + task->toks[ntoks++] = tok; + tok = strtok_r(NULL, "|", &saveptr); + } + // NULL marker (no more token from here) + task->toks[ntoks] = NULL; + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld(%ld): %s", + t->tid, (long)pthread_self(), task->clone); + fflush(t->rthread_fd); + free(task->clone); + task->clone = NULL; +#endif +#ifndef NO_RIOOP + // Perform the corresponding I/O operation! + rioop_run(task->process, t, task); +#endif + + // Make the task object recyclable/reusable + pthread_mutex_lock(&w->task_buffer_mutex); + if (!rbuffer_insert(w->task_buffer, task)) + // We can't recycle the task object if the buffer is full! + rtask_destroy(task); + pthread_mutex_unlock(&w->task_buffer_mutex); +} + +void *rthread_pthread_start(void *data) +{ + rthread_s* t = (rthread_s*) data; + rworker_s *w = t->worker; + rtask_s *task = NULL; + pid_t pthread_id = pthread_self(); + +#ifdef THREAD_DEBUG + _rthread_init_log(t); +#endif + + do { + while (!rbuffer_has_next(t->tasks) && !t->terminate) + usleep(100); + + while ((task = rbuffer_get_next(t->tasks)) != NULL) + rthread_process_task(t, task, pthread_id); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld: DEBUG: Idling\n", t->tid); + fflush(t->rthread_fd); +#endif + + // Tell rworker_s that thread is not doing any work! + int inserted = false; + while (!inserted && !t->terminate) { + if (rbuffer_has_next(t->tasks)) + break; + + usleep(1000); + + if (rbuffer_has_next(t->tasks)) + break; + + // Make the rthread reusable, he is without any tasks + // for some time. + pthread_mutex_lock(&w->rthread_buffer_mutex); + inserted = rbuffer_insert(w->rthread_buffer, t); + pthread_mutex_unlock(&w->rthread_buffer_mutex); + } + +#ifdef THREAD_DEBUG + if (inserted) { + fprintf(t->rthread_fd, "%ld: DEBUG: Added to thread buffer\n", + t->tid); + } else { + fprintf(t->rthread_fd, "%ld: DEBUG: Idling thread recovered\n", + t->tid); + } + fflush(t->rthread_fd); +#endif + + } while (!t->terminate); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld: DEBUG: Terminating\n", t->tid); + fflush(t->rthread_fd); +#endif + + // Process the very last tasks + while (NULL != (task = rbuffer_get_next(t->tasks))) + rthread_process_task(t, task, pthread_id); + +#ifdef THREAD_DEBUG + fprintf(t->rthread_fd, "%ld: DEBUG: Done terminating\n", t->tid); + fflush(t->rthread_fd); +#endif + + return NULL; +} + +rthread_s* rthread_new(const long tid, void *worker) +{ + rthread_s *t = Malloc(rthread_s); + rworker_s *w = worker; + + t->single_threaded = w->opts->num_threads_per_worker == 1; + t->tasks = rbuffer_new(TASK_BUFFER_PER_THREAD); + t->terminate = false; + t->worker = worker; + rthread_update(t, tid); + + if (t->single_threaded) { +#ifdef THREAD_DEBUG + _rthread_init_log(t); +#endif + return t; + } + + start_pthread(&t->pthread, rthread_pthread_start, (void*)t); + return t; +} + +long rthread_update(rthread_s *t, const long tid) +{ + long prev_tid = t->tid; + t->tid = tid; + + return prev_tid; +} + +void rthread_destroy(rthread_s *t) +{ + if (rbuffer_has_next(t->tasks)) { + Error("Didn't expect to have any tasks left!"); + } + rbuffer_destroy(t->tasks); + +#ifdef THREAD_DEBUG + if (t->rthread_fd) + fclose(t->rthread_fd); +#endif + + free(t); +} + +bool rthread_insert_task(rthread_s* t, rtask_s* task) +{ + if (t->single_threaded) { + rthread_process_task(t, task, pthread_self()); + return true; + } + return rbuffer_insert(t->tasks, task); +} + +void rthread_terminate(rthread_s* t) +{ + t->terminate = true; + pthread_join(t->pthread, NULL); +} diff --git a/ioriot/src/replay/rthread.h b/ioriot/src/replay/rthread.h new file mode 100644 index 0000000..9971e49 --- /dev/null +++ b/ioriot/src/replay/rthread.h @@ -0,0 +1,123 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file rthread.h + * @author Paul Buetow + * + * @brief The replay thread definitiion + */ + +#ifndef RTHREAD_H +#define RTHREAD_H + +#include "../defaults.h" +#include "../datas/rbuffer.h" +#include "../datas/amap.h" +#include "../vfd.h" +#include "rtask.h" + +#include + +/** + * @brief Definition of a worker thread + * + * Every worker utilises a set of worker threads in order to parallelise the + * replaying of the I/O! Every thread comes with its own task queue. It is + * filled by the repsonsible worker. + * + * The user can specify the max amount of threads per worker per -t command + * line switch. + */ +typedef struct rthread_s_ { + void *worker; /**< The responsible worker object */ + long tid; /**< The virtual thread id */ + rbuffer_s* tasks; /**< Holds all outstanding tasks */ + bool terminate; /**< True if thread shall terminate */ + bool single_threaded; /**< Worker is single threaded or not */ + pthread_t pthread; /**< We run the tasks in concurrent pthreads */ +#ifdef RTHREAD_DEBUG + FILE *rthread_fd; /**< Used for debugging purposes only */ +#endif +} rthread_s; + +/** + * @brief Creates a new thread object + * + * @param tid The thread ID + * @param worker The worker object managing this thread + * @return The new thread object + */ +rthread_s* rthread_new(const long tid, void *worker); + +/** + * @brief Updates a thread object after recycling it + * + * @param t The thread object + * @param tid The new thread ID + */ +long rthread_update(rthread_s *t, const long tid); + +/** + * @brief Terminates the thread + * + * This function waits (via join) for the pthread to complete all its + * current tasks from the queue. + * + * @param t The thread object + */ +void rthread_terminate(rthread_s* t); + +/** + * @brief Destroys the thread object + * + * @param t The thread object + */ +void rthread_destroy(rthread_s* t); + +/** + * @brief Inserts a task into the threads work queue + * + * Inserts a task into the threads work queue. We use an atomic ring buffer + * data structure for the work queue. The ring buffer does not require any + * mutex locks. + * + * @param t The thread object + * @param task The task to be inserted + * @return Returns true on success, returns false if the task queue is full + */ +bool rthread_insert_task(rthread_s* t, rtask_s* task); + +/** + * @brief Used by the pthread to process a task + * + * In this function the pthread will attempt to process a task. It extracts all + * required information from the task object and invokes the corresponding I/O + * syscalls. + * + * @param t The responsible thread object + * @param task The task object + * @param pthread_id The current pthread id + */ +void rthread_process_task(rthread_s* t, rtask_s *task, pid_t pthread_id); + +/** + * @brief The entry function for the pthreads + * + * @param data The data structure passed to the pthread + * @return The exit code of the pthread. + */ +void *rthread_pthread_start(void *data); + +#endif // RTHREAD_H diff --git a/ioriot/src/replay/rworker.c b/ioriot/src/replay/rworker.c new file mode 100644 index 0000000..89bb49f --- /dev/null +++ b/ioriot/src/replay/rworker.c @@ -0,0 +1,360 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rworker.h" + +#include "../datas/stack.h" +#include "rprocess.h" +#include "rthread.h" + +#define _Compute_current_time(now) \ + (now.tv_sec - start_time.tv_sec) * 1000 \ + + (now.tv_usec - start_time.tv_usec) / 1000 + + +/** + * @brief A callback helper function for destroying all virtual process objects + * + * @param data The process object. + */ +static void _rprocess_destroy_cb(void *data) +{ + rprocess_destroy(data); +} + +rworker_s* rworker_new(const int rworker_num, amap_s *fds_map, + const long num_vsizes, const long num_pids, + options_s *opts, rworker_stats_s *worker_stats) +{ + rworker_s *w = Malloc(rworker_s); + +#ifdef THREAD_DEBUG + char *rworker_log = Calloc(1024, char); + snprintf(rworker_log, 1023, "/tmp/ioriot/_worker%d.debuglog", + rworker_num); + + w->rworker_fd = Fopen(rworker_log, "a"); + free(rworker_log); + fprintf(w->rworker_fd, "DEBUG: Started worker\n"); +#endif + + w->rworker_num = rworker_num; + w->opts = opts; + w->fds_map = fds_map; + + w->rprocess_map = amap_new(num_pids); + w->rthread_map = amap_new(num_vsizes); + w->task_buffer = rbuffer_new(opts->num_threads_per_worker + *TASK_BUFFER_PER_THREAD); + w->rthread_buffer = rbuffer_new(opts->num_threads_per_worker); + w->worker_stats = worker_stats; + + // Attach a cleanup callback function to the worker map. + w->rprocess_map->data_destroy = _rprocess_destroy_cb; + + pthread_mutex_init(&w->rthread_buffer_mutex, NULL); + pthread_mutex_init(&w->task_buffer_mutex, NULL); + + // TODO: Check in the program whether the ulimit is high enough + // or not! (ulimit -n) + + return w; +} + +/** + * @brief Destroys the object + * + * Destroys the worker object (frees all memory allocated by the worker) + * + * @param w The worker object + */ +void rworker_destroy(rworker_s *w) +{ + if (!w) + return; + + if (w->rprocess_map) + amap_destroy(w->rprocess_map); + if (w->rthread_map) + amap_destroy(w->rthread_map); + + if (w->task_buffer) { + rtask_s *task = NULL; + while (NULL != (task = rbuffer_get_next(w->task_buffer))) + rtask_destroy(task); + rbuffer_destroy(w->task_buffer); + } + + if (w->rthread_buffer) + rbuffer_destroy(w->rthread_buffer); + + pthread_mutex_destroy(&w->task_buffer_mutex); + pthread_mutex_destroy(&w->rthread_buffer_mutex); + +#ifdef THREAD_DEBUG + if (w->rworker_fd) + fclose(w->rworker_fd); +#endif + + free(w); +} + +status_e rworker_process_lines(rworker_s* w, const long num_lines) +{ + Out("worker(%d): Starting to process replay lines\n", w->rworker_num); + + options_s *opts = w->opts; + FILE *replay_fd = Fopen(opts->replay_file, "r"); + + // Drop root privileges, otherwise we may overwrite other system + // files by accident in case of a bug or user error! + set_limits_drop_root(opts->user); + + // Variables required for the time based caluclations + struct timeval now, start_time; + long current_time = 0, stats_time = 0; + gettimeofday(&start_time, NULL); + + // Helper variables required for reading lines + char *line = NULL; + char *next = NULL, *next2 = NULL; + size_t len = 0, read = 0; + + // Helpers required for threading + rthread_s *t = NULL; + stack_s *all_threads = stack_new(); + rworker_stats_s *s = w->worker_stats; + + // More helper variables + //unsigned long lineno = 0, stats_ioop = 0, vsize_id = 0; + unsigned long lineno = 0, vsize_id = 0; + long pid = -1, time = -1; + + // Process the .replay file line by line. + while ((read = getline(&line, &len, replay_fd)) != -1) { + lineno++; + + if (read >= MAX_LINE_LEN) { + Error("line:%lu Exceeded max line len", lineno); + } + + // If the line begins with #: Ignore that line, it contains + // debug or meta information or comments. + + if (line[0] == '#') { + if (line[1] == 'I') { + // We stop replaying I/O once we reach the line '#INIT' + // which incitates the begin of the INIT section. + break; + } + continue; + } + +#ifdef THREAD_DEBUG + char *clone = Clone(line); +#endif + + next = strchr(line, '|'); + Error_if(!next, "lineno:%ld Could not parse time from input file", + lineno); + next[0] = '\0'; + next++; + time = atol(line); + + next2 = strchr(next, '|'); + Error_if(!next2, "Could not parse vsize_id from input file"); + next2[0] = '\0'; + next2++; + vsize_id = atol(next); + + // This worker is not responsible for this line, skip it! + if ((vsize_id % opts->num_workers) != w->rworker_num) { +#ifdef THREAD_DEBUG + free(clone); +#endif + continue; + } + + next = strchr(next2, '|'); + Error_if(!next, "Could not parse PID from input file"); + next[0] = '\0'; + next++; + pid = atol(next2); + + gettimeofday(&now, NULL); + current_time = _Compute_current_time(now); + + // Check whether the user specified a replay speed factor. If so, we + // may need to throttle down a bit. + + if (opts->speed_factor) { + s->time_ahead = time / opts->speed_factor - current_time; + if (s->time_ahead > 0) + usleep(s->time_ahead*1000); + + } else { + s->time_ahead = time - current_time; + } + + // Get the responsible process object. The process object holds data + // structures usually found in a Linux process, e.g. a table of open + // file descriptors. + + rprocess_s *p = amap_get(w->rprocess_map, pid); + if (p == NULL) { + p = rprocess_new(pid, w->fds_map); + amap_set(w->rprocess_map, pid, p); + } + p->lineno = lineno; + + if (opts->num_threads_per_worker == 1) { + // Single threaded mode? + if (!t) + t = rthread_new(vsize_id, w); + else + rthread_update(t, vsize_id); + + } else { + t = amap_get(w->rthread_map, vsize_id); + } + + if (t == NULL) { + + // First try to recycle an old (likely unused) thread + if (NULL != (t = rbuffer_get_next(w->rthread_buffer))) { + rthread_update(t, vsize_id); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Reused an idling thread\n"); + fflush(w->rworker_fd); +#endif + + } else if (opts->num_threads_per_worker <= all_threads->size) { + // Reached max threads, waiting until one becomes available + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Reached max threads\n"); + fflush(w->rworker_fd); +#endif + while (NULL == (t = rbuffer_get_next(w->rthread_buffer))) + usleep(1000); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Reused an idling thread\n"); + fflush(w->rworker_fd); +#endif + + rthread_update(t, vsize_id); + + } else { + t = rthread_new(vsize_id, w); + + // We hold a pointer to all created threads in a stack. This + // stack is later used to terminate/join all therads. + stack_push(all_threads, t); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Created a new thread\n"); + fflush(w->rworker_fd); +#endif + } + + amap_set(w->rthread_map, vsize_id, t); + } + + // Create a new task for the thread. The task contains all required + // information to run an I/O operation. However, first try to + // reuse/recycle a task object! If there is no such, create a new one. + + rtask_s *task = rbuffer_get_next(w->task_buffer); + if (!task) + task = rtask_new(); + rtask_update(task, w, p, next, lineno, vsize_id); + s->ioops++; + + +#ifdef THREAD_DEBUG + task->clone = clone; + fprintf(w->rworker_fd, "DEBUG: Inserting new task\n"); + fflush(w->rworker_fd); +#endif + + // Insert that task to a ring buffer to pass it to the pthread without + // much synchronisation overhead! + + while (!rthread_insert_task(t, task)) + // The ring buffer is full. This may happen if the pthread didn't + // manage to process tasks fast enough. re-try after a short period! + usleep(1000); + +#ifdef THREAD_DEBUG + fprintf(w->rworker_fd, "DEBUG: Task inserted\n"); + fflush(w->rworker_fd); +#endif + + // The worker prints out stats every 3s + if (current_time - stats_time > 3000) { + // IDEA: Maybe refactor this block to be implemented in rstats.c + + double loadavg = get_loadavg(); + + // Determines whether we replay the I/O faster or slower than + // original speed! + char *a_b = s->time_ahead >= 0 ? "ahead" : "behind"; + + Put("worker(%d): threads:%ld %s:%lds progress:%0.2f%% " + "loadavg:%0.2f", + w->rworker_num, all_threads->size, a_b, Abs(s->time_ahead/1000), + Perc(lineno,num_lines), loadavg); + + stats_time = current_time; + //stats_ioop = lineno; + + if (s->loadavg_high < loadavg) + s->loadavg_high = loadavg; + } + } + + Put("worker(%d): Waiting for all threads to finish business...", + w->rworker_num); + + // This will wait (join) all threads one after another until all threads + // have finished their work and have terminated. + + while (!stack_is_empty(all_threads)) { + rthread_s *t = stack_pop(all_threads); + rthread_terminate(t); + rthread_destroy(t); + } + stack_destroy(all_threads); + + // Collect some stats last time + double loadavg = get_loadavg(); + if (s->loadavg_high < loadavg) + s->loadavg_high = loadavg; + + gettimeofday(&now, NULL); + current_time = _Compute_current_time(now); + if (opts->speed_factor) { + s->time_ahead = time / opts->speed_factor - current_time; + } else { + s->time_ahead = time - current_time; + } + + + Put("worker(%d): All threads terminated!", w->rworker_num); + fclose(replay_fd); + + return SUCCESS; +} diff --git a/ioriot/src/replay/rworker.h b/ioriot/src/replay/rworker.h new file mode 100644 index 0000000..26a1300 --- /dev/null +++ b/ioriot/src/replay/rworker.h @@ -0,0 +1,82 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef RWORKER_H +#define RWORKER_H + +#include + +#include "../datas/amap.h" +#include "../datas/rbuffer.h" +#include "../defaults.h" +#include "../options.h" +#include "rstats.h" + +/** + * @brief Represents a worker process. + * + * This represents an I/O replay worker process. The user can specify the + * amount of worker processes via the -p command line switch. This is not + * to confuse with rprocess_s, which represents an original captured process + * and we now want to replay the I/O for! + */ +typedef struct { + int rworker_num; /**< The current worker ID */ + amap_s* fds_map; /**< Holding all file descriptors */ + amap_s* rprocess_map; /**< Holding all processes handled by this worker */ + amap_s* rthread_map; /**< Holding all threads handled by this worker */ + rbuffer_s *task_buffer; /**< Buffering thread tasks to be reused */ + pthread_mutex_t task_buffer_mutex; /**< To sync access to task_buffer */ + rbuffer_s *rthread_buffer; /**< Buffering idle threads to be reused */ + pthread_mutex_t rthread_buffer_mutex; /**< Sync access to rthread_buffer */ + options_s *opts; /**< To synchronise access to rthread_buffer */ + rworker_stats_s *worker_stats; /**< Object holding per worker statistics */ +#ifdef RTHREAD_DEBUG + FILE *rworker_fd; /**< For debugging purposes only */ +#endif +} rworker_s; + +/** + * @brief Creates a new worker object + * + * @param rworker_num The worker number + * @param fds_map A map of all virtual file descriptor objects + * @param num_vsizes The amount of virtual sizes/total file paths of the test + * @param num_pids The total amount of virtual process IDs used in this test + * @param opts A pointer to the options object + * @param worker_stats A pointer to the worker stats object + + * @return The new worker object + */ +rworker_s* rworker_new(const int rworker_num, amap_s *fds_map, + const long num_vsizes, const long num_pids, + options_s* opts, rworker_stats_s *worker_stats); + +/** + * @brief Destroys a worker object + * + * @param w The worker object to be destroyed + */ +void rworker_destroy(rworker_s* w); + +/** + * @brief Makes the worker to process all .replay lines + * + * @param w The responsible worker object + * @param num_lines The total amount of I/O op lines in the .replay file + * @return SUCCESS if everything went fine + */ +status_e rworker_process_lines(rworker_s* w, const long num_lines); + +#endif // RWORKER_H diff --git a/ioriot/src/utests.c b/ioriot/src/utests.c new file mode 100644 index 0000000..7cc4959 --- /dev/null +++ b/ioriot/src/utests.c @@ -0,0 +1,41 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utests.h" + +#include "datas/amap.h" +#include "datas/hmap.h" +#include "datas/list.h" +#include "datas/rbuffer.h" +#include "utils/utils.h" + +void utests_run() +{ + fprintf(stderr, "Running utils_test()\n"); + utils_test(); + + fprintf(stderr, "Running amap_test()\n"); + amap_test(); + + fprintf(stderr, "Running hmap_test()\n"); + hmap_test(); + + fprintf(stderr, "Running list_test()\n"); + list_test(); + + fprintf(stderr, "Running rbuffer_test()\n"); + rbuffer_test(); + + fprintf(stderr, "Great success, run all unit tests without any errors!\n"); +} diff --git a/ioriot/src/utests.h b/ioriot/src/utests.h new file mode 100644 index 0000000..4ad6973 --- /dev/null +++ b/ioriot/src/utests.h @@ -0,0 +1,25 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef UTESTS_H +#define UTESTS_H + +#include "utils/utils.h" + +/** + * @brief This function runs all currently implemented unit tests + */ +void utests_run(); + +#endif // UTESTS_H diff --git a/ioriot/src/utils/futils.c b/ioriot/src/utils/futils.c new file mode 100644 index 0000000..5b35618 --- /dev/null +++ b/ioriot/src/utils/futils.c @@ -0,0 +1,291 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "futils.h" + +#include +#include +#include +#include + +#include "../macros.h" + +void append_random_to_file(char *path, unsigned long bytes) +{ + char *buf = NULL; + int max_chunk = 50000000; // 50 mebibyetes + FILE *fp = Fopen(path, "a"); + + for (;;) { + if (bytes > max_chunk) { + if (!buf) + buf = Calloc(max_chunk+1, char); + + Fill_with_stuff(buf, max_chunk); + buf[max_chunk] = '\0'; + fprintf(fp, "%s", buf); + bytes -= max_chunk; + + // Print out a dot every time we wrote 'much' data to a file + Out("."); + + } else { + if (!buf) + buf = Calloc(bytes+1, char); + + Fill_with_stuff(buf, bytes); + buf[bytes] = '\0'; + fprintf(fp, "%s", buf); + + break; + } + } + + if (buf) + free(buf); + fclose(fp); +} + +long ensure_dir_exists(const char *path) +{ + long num_dirs_created = 0; + int ret = mkdir_p(path, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, + &num_dirs_created); + if (ret != 0) { + Errno("Could not create dir '%s'", path); + } + + return num_dirs_created; +} + +void ensure_parent_dir_exists(const char *path) +{ + char *clone = Clone(path); + char *parent = dirname(clone); + + int ret = mkdir_p(parent, S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH, NULL); + if (ret != 0) { + Errno("Could not create dir %s", parent); + } + + free(clone); +} + +void ensure_dir_empty(const char *path) +{ + DIR *dh = opendir(path); + + if (!dh) { + Errno("Unable to empty %s", path); + } + + struct dirent *de; + + while ((de = readdir(dh))) { + if (0 == strcmp(de->d_name, ".") || + 0 == strcmp(de->d_name, "..")) + continue; + + char *absolute; + asprintf(&absolute, "%s/%s", path, de->d_name); + + if (is_dir(absolute)) + ensure_dir_empty(absolute); + + if (remove(absolute) == -1) + // Don't throw an error if there is no such file or directory + if (errno != 2) { + Errno("Unable to remove %s", absolute); + } + + free(absolute); + } + + closedir(dh); +} + +int ensure_file_exists(char *path, long *num_dirs_created) +{ + if (is_reg(path)) + return SUCCESS; + + char *dirname = dirname_r(Clone(path)); + *num_dirs_created += ensure_dir_exists(dirname); + free(dirname); + + FILE *fp = fopen(path, "a"); + if (fp) { + // We only need some data, less than 1 block in size, this is answer: + fprintf(fp, "42"); + fclose(fp); + return SUCCESS; + } + + return ERROR; +} + +char* dirname_r(char *path) +{ + int len = strlen(path); + int has = 0; + int i = len-1; + + if (strcmp(path, "..") == 0) { + return path; + } + + if (path[i] == '/') { + // Root directory + if (len == 1) + return path; + + // Remove all trailing / + for (; i >= 0; --i) { + if (path[i] == '/') { + path[i] = '\0'; + has = 1; + } else { + break; + } + } + } + + // Find next / + for (; i >= 0; --i) { + if (path[i] == '/') { + path[i] = '\0'; + has = 1; + break; + } + } + + // If no / + if (has == 0) { + path[0] = '.'; + path[1] = '\0'; + } + + return path; +} + +bool is_dir(const char *path) +{ + struct stat path_stat; + if (stat(path, &path_stat) == 0 && S_ISDIR(path_stat.st_mode)) + return true; + return false; +} + +bool is_reg(const char *path) +{ + struct stat path_stat; + if (stat(path, &path_stat) == 0 && S_ISREG(path_stat.st_mode)) + return true; + return false; +} + +bool exists(const char *path) +{ + struct stat path_stat; + if (stat(path, &path_stat) == 0) + return true; + return false; +} + +int mkdir_p(const char *path, mode_t mode, long *num_dirs_created) +{ + int res = 0; + + if (is_dir(path)) + return 0; + + if (is_reg(path)) + unlink(path); + + char *top = dirname_r(Clone(path)); + if (0 != mkdir_p(top, mode, num_dirs_created)) + goto cleanup; + + if ((mkdir(path, mode) == -1) && (errno != EEXIST)) + res = -1; + + if (res != -1) + *num_dirs_created = *num_dirs_created+1; + +cleanup: + free(top); + + return res; +} + +void cache_file(const char *file) +{ + Out("Caching file %s... it can take a while", file); + FILE *fd = Fopen(file, "r"); + char *line = NULL; + size_t len = 0, read = 0; + + while ((read = getline(&line, &len, fd)) != -1); + fclose(fd); +} + +void drop_caches(void) +{ + Out("Dropping all Linux caches..."); + + if (getuid() != 0) { + Out("\n"); + Error("I need to be root to do this, aborting!"); + } + + // echo 3 > /proc/sys/vm/drop_caches + char *drop_caches = "/proc/sys/vm/drop_caches"; + FILE *fd = Fopen(drop_caches, "w"); + fprintf(fd, "3"); + fclose(fd); + + Put("done"); +} + +void chown_path(const char *user, const char *path) +{ + struct passwd *pwd = getpwnam(user); + if (!pwd) { + Errno("Unable to retrieve information about system user %s!", user); + } + + if (chown(path, pwd->pw_uid, -1) == -1) { + Errno("Could not change ownership of '%s' to '%s'!", path, user); + } +} + +char *absolute_path(const char *path) +{ + if (path[0] == '/') + return Clone(path); + + char cwd[MAX_LINE_LEN]; + getcwd(cwd, sizeof(char)*MAX_LINE_LEN); + + if (!getcwd(cwd, sizeof(cwd))) { + Errno("Could not get current working directory"); + } + + char *absolute = NULL; + if (-1 == asprintf(&absolute, "%s/%s", cwd, path)) { + Error("Could not get absolute path of '%s'", path); + } + + return absolute; +} diff --git a/ioriot/src/utils/futils.h b/ioriot/src/utils/futils.h new file mode 100644 index 0000000..9afde1a --- /dev/null +++ b/ioriot/src/utils/futils.h @@ -0,0 +1,134 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef FUTILS_H +#define FUTILS_H + +#include "../defaults.h" + +/** + * @brief Thread safe version of dirname() + * + * @param path The full file path + * @return The directory path + */ +char* dirname_r(char *path); + +/** + * @brief Ensures that a file exists + * + * @param path The file path + * @param num_dirs_created Holds a count of how many sub dirs have been created + * @return -1 on error, 0 on success. + */ +int ensure_file_exists(char *path, long *num_dirs_created); + +/** + * @brief Checks whether path exists + * + * @param path The path + * @return true if the path exists + */ +bool exists(const char *path); + +/** + * @brief Check if path is a directory + * + * @param path The directory path + * @return true if the path is a directory, false otherwise + */ +bool is_dir(const char *path); + +/** + * @brief Check if path is a regular file + * + * @param path The file path + * @return true if the file at path is a regular fike + */ +bool is_reg(const char *path); + +/** + * @brief Create a directory recursively + * + * @param path The directory path + * @param mode The mode + * @param num_dirs_created Counts how many directories have been created + * @return -1 on error + */ +int mkdir_p(const char *path, mode_t mode, long *num_dirs_created); + +/** + * @brief Appends data to a file + * + * @param path The file path + * @param bytes The amount of bytes + */ +void append_random_to_file(char *path, unsigned long bytes); + +/** + * @brief Ensures that a directory exists + * + * @param path The directory path + * @return The amount of directories created (including parent directories) + */ +long ensure_dir_exists(const char *path); + +/** + * @brief Ensures that a parent directory exists + * + * @param path The directory path + */ +void ensure_parent_dir_exists(const char *path); + +/** + * @brief Ensures that a directory is empty + * + * @param path The directory path + */ +void ensure_dir_empty(const char *path); + +/** + * @brief Loading a file into the file system cache + * + * @param file The path to the file + */ +void cache_file(const char *file); + +/** + * @brief Drop all Linux caches + * + * This function drops all Linux caches, which includes all file + * system caches. + */ +void drop_caches(void); + +/** + * @brief Changes owner of a path + * + * Terminates the process with an error message if failed. + * + * @param user The new owner + * @param path The path + */ +void chown_path(const char *user, const char *path); + +/** + * @brief Retrieves the absolute path of a given path + * + * @param path The path + * @return The absolute path. It must be freed manually. + */ +char *absolute_path(const char *path); + +#endif // FUTILS_H diff --git a/ioriot/src/utils/utils.c b/ioriot/src/utils/utils.c new file mode 100644 index 0000000..4b41273 --- /dev/null +++ b/ioriot/src/utils/utils.c @@ -0,0 +1,186 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "utils.h" + +#include +#include + +void* notnull(void *p, char *file, int line, int count) +{ + if (p == NULL) { + Errno("%s:%d count:%d Could not allocate memory", file, line, count); + } + return p; +} + + +FILE* fnotnull(FILE *fd, const char *path, char *file, int line) +{ + if (fd == NULL) { + Errno("%s:%d Could not open file '%s'", file, line, path); + } + return fd; +} + +void* mmapok(void *p, char *file, int line) +{ + if (p == MAP_FAILED) { + Errno("%s:%d: Mmap failed", file, line); + } + return p; +} + +char* strtok2_r(char *str, char *delim, char **saveptr) +{ + int len = strlen(delim); + + if (str == NULL) + str = *saveptr; + + char *next = strstr(str, delim); + if (next) { + next[0] = '\0'; + for (int i = 0; i < len; ++i) + next++; + *saveptr = next; + return str; + } + + return NULL; +} + +void chreplace(char *str, char replace, char with) +{ + for (int i = 0; ; ++i) { + if (str[i] == '\0') + break; + if (str[i] == replace) + str[i] = with; + } +} + +void strunquote(char *str) +{ + int len = strlen(str); + + if (str[0] == '"') { + if (str[len-1] == '"') + str[len-1] = '\0'; + for (int i = 1; i < len; ++i) + str[i-1] = str[i]; + } +} + +void set_limits_drop_root(const char *user) +{ + if (getuid() == 0) { + struct rlimit rl; + rl.rlim_cur = rl.rlim_max = SET_RLIMIT_NOFILE; + if (0 != setrlimit(RLIMIT_NOFILE, &rl)) { + Errno("Could not set RLIMIT_NOFILE to '%lld'!", + (long long) SET_RLIMIT_NOFILE) + } + rl.rlim_cur = rl.rlim_max = SET_RLIMIT_NPROC; + if (0 != setrlimit(RLIMIT_NPROC, &rl)) { + Errno("Could not set RLIMIT_NPROC to '%lld'!", + (long long) SET_RLIMIT_NPROC) + } + + Put("Dropping root privileges to user '%s'", user); + struct passwd *pw = getpwnam(user); + + /* process is running as root, drop privileges */ + if (setgid(pw->pw_gid) != 0) { + Errno("Unable to drop group privileges!"); + } + if (setuid(pw->pw_uid) != 0) { + Errno("Unable to drop user privileges!"); + } + } + + /* + getrlimit(RLIMIT_NOFILE, &rl); + Put("Max open files: '%lld'", (long long) rl.rlim_cur); + getrlimit(RLIMIT_NPROC, &rl); + Put("Max open processes : '%lld'", (long long) rl.rlim_cur); + */ +} + +void get_loadavg_s(char *readbuf) +{ + FILE *fp = Fopen("/proc/loadavg", "r"); + fgets(readbuf, 128, fp); + char *pos = strchr(readbuf, ' '); + pos[0] = '\0'; + fclose(fp); +} + +double get_loadavg() +{ + // Not thread safe, but multi processing safe + static char buf[128]; + get_loadavg_s(buf); + + return atof(buf); +} + +bool is_number(char *str) +{ + for (int i = 0; ; ++i) { + if (str[i] == '\0') + return true; + if (isdigit(str[i]) == 0 && str[i] != '-') + return false; + } + + return true; +} + +void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data) +{ + int rc = pthread_create(thread, NULL, cb, data); + + switch (rc) { + case 0: + break; + case EAGAIN: + Error("Out of resources while creating pthread (%d)", rc); + break; + case EINVAL: + Error("Ivalid settings while creating pthread (%d)", rc); + break; + case EPERM: + Error("No permissions to configure pthread (%d)", rc); + default: + Error("Unknown error while creating pthread (%d)", rc); + break; + } +} + +void utils_test(void) +{ + if (getuid() == 0) { + set_limits_drop_root("nobody"); + struct rlimit rl; + + getrlimit(RLIMIT_NOFILE, &rl); + assert(rl.rlim_cur == SET_RLIMIT_NOFILE); + assert(rl.rlim_max == SET_RLIMIT_NOFILE); + + getrlimit(RLIMIT_NPROC, &rl); + assert(rl.rlim_cur == SET_RLIMIT_NPROC); + assert(rl.rlim_max == SET_RLIMIT_NPROC); + } +} diff --git a/ioriot/src/utils/utils.h b/ioriot/src/utils/utils.h new file mode 100644 index 0000000..3e86865 --- /dev/null +++ b/ioriot/src/utils/utils.h @@ -0,0 +1,174 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef UTILS_H +#define UTILS_H + +// For asprintf in stdio.h +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../macros.h" +#include "../defaults.h" + +/** + * @brief Check whether allocated memory is not NULL + * + * This function is used in conjunction with malloc() and co. It + * introduces an extra sanity check whether the memory could be + * allocated successfully or not. If not it will print out a error + * message stating the position in the source code where the memory + * allocation failed! + * + * @param p The pointer being checked + * @param file The source file name the memory was allocated in + * @param line The source line number the memory was allocated at + * @param count The amount of memory being allocated + * @return The pointer to the allocated memory + */ +void* notnull(void *p, char *file, int line, int count); + +/** + * @brief Check whether opened file handle is not NULL + * + * This function is used in conjunction with fopen(). It + * introduces an extra sanity check whether the file could be + * opened successfully or not. If not it will print out a error + * message stating the position in the source code where the open + * failed! + * + * @param fd The fd stream to be checked. + * @param path The file path opened + * @param file The source file name + * @param line The source line number + * @return The pointer to the allocated memory + */ +FILE* fnotnull(FILE *fd, const char *path, char *file, int line); + +/** + * @brief Check whether allocated memory via mmap is not null + * + * This function is used in conjunction with mmap() and co. It + * introduces an extra sanity check whether the memory could be + * allocated successfully or not. If not it will print out a error + * message stating the position in the source code where the memory + * allocation failed! + * + * @param addr The pointer being checked + * @param file The source file name the memory was allocated in + * @param line The source line number the memory was allocated at + * @return The pointer to the allocated memory + */ +void* mmapok(void *addr, char *file, int line); + +/** + * @brief A version of strtok_r supporting multi char delims + * + * @param str The input string + * @param delim The multi-char delimiter + * @param saveptr A temp storage location + * @return The next match if != NULL + */ +char* strtok2_r(char *str, char *delim, char **saveptr); + +/** + * @brief Replaces a character with another one in a string + * + * @param str The input string + * @param replace The character to be replaced + * @param with The character to replace with + */ +void chreplace(char *str, char replace, char with); + +/** + * @brief Removes quotes from a string + * + * @param str The input sting + */ +void strunquote(char *str); + +/** + * @brief Set rlimits and drop root privileges + * + * This function firsts sets the user resource limits to SET_RLIMIT_NOFILE and + * SET_RLIMIT_NPROC and then attempts to drop the root user to the specified + * one. + * + * @param user The user to switch to + */ +void set_limits_drop_root(const char *user); + +/** + * @brief Retrieve current 1 min Linux load average + * + * @param readbuf The buffer to store the load average as a string + */ +void get_loadavg_s(char *readbuf); + +/** + * @brief Retrieve current 1 min Linux load average + * + * This function is not thread safe! + * + * @return The 1 minute load average of the system + */ +double get_loadavg(); + +/** + * @brief Check whether a string represents a number + * + * @param str The input string + * @return true if all characters of the input string are a digits + */ +bool is_number(char *str); + +/** + * @brief Wrapper around pthread_create + * + * The wrapper also checks whether the thread has been created successfully + * or not! It will exit the process if not. + * + * @param thread The POSIX thread variable + * @param cb The threadss start callback routine + * @param data A data pointer passed to the thread. + */ +void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data); + +/** + * @brief Testing various of the utilities + */ +void utils_test(void); + +#endif // UTILS_H diff --git a/ioriot/src/vfd.c b/ioriot/src/vfd.c new file mode 100644 index 0000000..6e86f61 --- /dev/null +++ b/ioriot/src/vfd.c @@ -0,0 +1,55 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "vfd.h" + +vfd_s* vfd_new(const int fd, const long mapped_fd, char *path) +{ + vfd_s *vfd = Malloc(vfd_s); + vfd->path = NULL; + vfd->debug = false; + vfd_update(vfd, fd, mapped_fd, path); + + return vfd; +} + +void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path) +{ + vfd->fd = fd; + vfd->dirfd = NULL; + vfd->mapped_fd = mapped_fd; + vfd->offset = 0; + + if (path) + free(vfd->path); + + vfd->path = Clone(path); +} + +void vfd_destroy(vfd_s *vfd) +{ + if (!vfd) + return; + + if (vfd->path) + free(vfd->path); + + free(vfd); +} + +void vfd_print(vfd_s *vfd) +{ + fprintf(stderr, "virtfd(%p) fd:%x mapped_fd:%lx path:%s\n", + (void*)vfd, vfd->fd, vfd->mapped_fd, vfd->path); +} diff --git a/ioriot/src/vfd.h b/ioriot/src/vfd.h new file mode 100644 index 0000000..ff9c79c --- /dev/null +++ b/ioriot/src/vfd.h @@ -0,0 +1,77 @@ +// Copyright 2018 Mimecast Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef VFD_H +#define VFD_H + +#include "opcodes.h" +#include "defaults.h" + +/** + * @brief The virtual file descriptor definition + * + * A virtual file descriptor represents a file descriptor from ioriot's + * point of view and is being used in various ways to simulate the real I/O + * protocolled to the .capture/.replay files. + * + * Generally speaking I/O replay maps the real FD numbers (the ones logged to + * the .capture file) to virtual FD numbers (a uniqe FD number for every open + * to increase concurrency). + */ +typedef struct vfd_s_ { + int fd; /**< the real fd */ + DIR *dirfd; /**< The real dirfd */ + long mapped_fd; /**< The mapped fd (virtual fd) */ + char *path; /**< The file path belonging to that fd */ + bool free_path; /**< True if path has to be freed or not */ + unsigned long offset; /**< The current virtual file offset in bytes */ + int debug; /**< Used for debugging purposes only */ +} vfd_s; + +/** + * @brief Creates a new virtual file descriptor object + * + * @param fd The file descriptor + * @param mapped_fd The mapped file descriptor + * @param path The path name + * @return The new fd object + */ +vfd_s* vfd_new(const int fd, const long mapped_fd, char *path); + +/** + * @brief Updates the virtfd object + * + * @param vfd The virtfd object + * @param fd The (real) file descriptor + * @param mapped_fd The mapped (virtual) file descriptor + * @param path The path name + * @return The new fd object + */ +void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path); + +/** + * @brief Destroys a file descriptor object + * + * @param vfd The file descriptor object + */ +void vfd_destroy(vfd_s *vfd); + +/** + * @brief Prints the virtual file descriptor + * @param vfd The virtual file descriptor + */ +void vfd_print(vfd_s *vfd); + +#endif // VFD_H + diff --git a/ioriot/tags b/ioriot/tags new file mode 100644 index 0000000..65fdc50 --- /dev/null +++ b/ioriot/tags @@ -0,0 +1,661 @@ +!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ +!_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ +!_TAG_PROGRAM_AUTHOR Darren Hiebert /dhiebert@users.sourceforge.net/ +!_TAG_PROGRAM_NAME Exuberant Ctags // +!_TAG_PROGRAM_URL http://ctags.sourceforge.net /official site/ +!_TAG_PROGRAM_VERSION 5.8 // +AMAP_H ./src/datas/amap.h 2;" d +AMAP_MAX_ARRAY_LENGTH ./src/datas/amap.h 6;" d +Abs ./src/macros.h 13;" d +BTREE_H ./src/datas/btree.h 2;" d +CAPTURE_VERSION ./src/defaults.h 7;" d +CHMOD ./src/opcodes.h /^ CHMOD = 100,$/;" e enum:__anon1 +CHOWN ./src/opcodes.h /^ CHOWN = 110,$/;" e enum:__anon1 +CHOWN16 ./src/opcodes.h /^ CHOWN16,$/;" e enum:__anon1 +CLEANUP_H ./src/cleanup/cleanup.h 2;" d +CLOSE ./src/opcodes.h /^ CLOSE = 50,$/;" e enum:__anon1 +CREAT ./src/opcodes.h /^ CREAT,$/;" e enum:__anon1 +Calloc ./src/macros.h 23;" d +Cleanup ./src/macros.h 4;" d +Cleanup_unless ./src/macros.h 5;" d +Clone ./src/macros.h 9;" d +Cmapshared ./src/macros.h 36;" d +DEFAULTS_H ./src/defaults.h 2;" d +Debug ./src/macros.h 51;" d +ERROR ./src/defaults.h /^ ERROR, \/**< An error happened :-( *\/$/;" e enum:status_e_ +Eq ./src/macros.h 10;" d +Errno ./src/macros.h 68;" d +Errno_if ./src/macros.h 77;" d +Error ./src/macros.h 58;" d +Error_if ./src/macros.h 66;" d +F ./src/generate/gtask.h /^ int F; \/**< Arguments for fcntl syscall *\/$/;" m struct:gtask_s_ +FCHMOD ./src/opcodes.h /^ FCHMOD,$/;" e enum:__anon1 +FCHMODAT ./src/opcodes.h /^ FCHMODAT,$/;" e enum:__anon1 +FCHOWN ./src/opcodes.h /^ FCHOWN,$/;" e enum:__anon1 +FCHOWN16 ./src/opcodes.h /^ FCHOWN16,$/;" e enum:__anon1 +FCHOWNAT ./src/opcodes.h /^ FCHOWNAT,$/;" e enum:__anon1 +FCNTL ./src/opcodes.h /^ FCNTL = 70,$/;" e enum:__anon1 +FDATASYNC ./src/opcodes.h /^ FDATASYNC,$/;" e enum:__anon1 +FSTAT ./src/opcodes.h /^ FSTAT = 0,$/;" e enum:__anon1 +FSTATFS ./src/opcodes.h /^ FSTATFS,$/;" e enum:__anon1 +FSTATFS64 ./src/opcodes.h /^ FSTATFS64,$/;" e enum:__anon1 +FSTAT_AT ./src/opcodes.h /^ FSTAT_AT,$/;" e enum:__anon1 +FSYNC ./src/opcodes.h /^ FSYNC = 60,$/;" e enum:__anon1 +FUTILS_H ./src/utils/futils.h 2;" d +Fill_with_stuff ./src/macros.h 98;" d +Fopen ./src/macros.h 29;" d +G ./src/generate/gtask.h /^ int G; \/**< Arguments for fcntl syscall *\/$/;" m struct:gtask_s_ +GENERATE_H ./src/generate/generate.h 2;" d +GETDENTS ./src/opcodes.h /^ GETDENTS,$/;" e enum:__anon1 +GIOOP_H ./src/generate/gioop.h 2;" d +GPARSER_H ./src/generate/gparser.h 2;" d +GPROCESS_H ./src/generate/gprocess.h 2;" d +GTASK_H ./src/generate/gtask.h 2;" d +GWRITER_H ./src/generate/gwriter.h 2;" d +Gioop_write ./src/generate/gioop.h 12;" d +HMAP_H ./src/datas/hmap.h 2;" d +Has ./src/macros.h 17;" d +Hasnt ./src/macros.h 18;" d +INIT_H ./src/init/init.h 2;" d +IORIOT_COPYRIGHT ./src/defaults.h 19;" d +IORIOT_VERSION ./src/defaults.h 17;" d +ITASK_H ./src/init/itask.h 2;" d +ITHREAD_H ./src/init/ithread.h 2;" d +LCHOWN ./src/opcodes.h /^ LCHOWN,$/;" e enum:__anon1 +LCOWN16 ./src/opcodes.h /^ LCOWN16,$/;" e enum:__anon1 +LIST_H ./src/datas/list.h 2;" d +LSEEK ./src/opcodes.h /^ LSEEK,$/;" e enum:__anon1 +LSTAT ./src/opcodes.h /^ LSTAT,$/;" e enum:__anon1 +MACROS_H ./src/macros.h 2;" d +MAX_LINE_LEN ./src/defaults.h 13;" d +MAX_MOUNTPOINTS ./src/mounts.h 8;" d +MAX_TOKENS ./src/defaults.h 11;" d +META_EXIT ./src/opcodes.h /^ META_EXIT = 900,$/;" e enum:__anon1 +META_EXIT_GROUP ./src/opcodes.h /^ META_EXIT_GROUP,$/;" e enum:__anon1 +META_H ./src/meta/meta.h 2;" d +META_TIMELINE ./src/opcodes.h /^ META_TIMELINE$/;" e enum:__anon1 +MKDIR ./src/opcodes.h /^ MKDIR,$/;" e enum:__anon1 +MKDIR_AT ./src/opcodes.h /^ MKDIR_AT,$/;" e enum:__anon1 +MMAP2 ./src/opcodes.h /^ MMAP2 = 80,$/;" e enum:__anon1 +MOUNTPOINTS_H ./src/mounts.h 2;" d +MSYNC ./src/opcodes.h /^ MSYNC,$/;" e enum:__anon1 +MUNMAP ./src/opcodes.h /^ MUNMAP,$/;" e enum:__anon1 +Malloc ./src/macros.h 21;" d +Mmapshared ./src/macros.h 32;" d +Mset ./src/macros.h 25;" d +NAME_TO_HANDLE_AT ./src/opcodes.h /^ NAME_TO_HANDLE_AT,$/;" e enum:__anon1 +OPCODES_H ./src/opcodes.h 2;" d +OPEN ./src/opcodes.h /^ OPEN = 30,$/;" e enum:__anon1 +OPEN_AT ./src/opcodes.h /^ OPEN_AT,$/;" e enum:__anon1 +OPEN_BY_HANDLE_AT ./src/opcodes.h /^ OPEN_BY_HANDLE_AT,$/;" e enum:__anon1 +OPTIONS_H ./src/options.h 2;" d +Out ./src/macros.h 42;" d +Put ./src/macros.h 45;" d +RBUFFER_H ./src/datas/rbuffer.h 2;" d +READ ./src/opcodes.h /^ READ = 10,$/;" e enum:__anon1 +READAHEAD ./src/opcodes.h /^ READAHEAD,$/;" e enum:__anon1 +READDIR ./src/opcodes.h /^ READDIR,$/;" e enum:__anon1 +READLINK ./src/opcodes.h /^ READLINK,$/;" e enum:__anon1 +READLINK_AT ./src/opcodes.h /^ READLINK_AT,$/;" e enum:__anon1 +READV ./src/opcodes.h /^ READV,$/;" e enum:__anon1 +REMAP ./src/opcodes.h /^ REMAP,$/;" e enum:__anon1 +RENAME ./src/opcodes.h /^ RENAME = 40,$/;" e enum:__anon1 +RENAME_AT ./src/opcodes.h /^ RENAME_AT,$/;" e enum:__anon1 +RENAME_AT2 ./src/opcodes.h /^ RENAME_AT2,$/;" e enum:__anon1 +REPLAY_H ./src/replay/replay.h 2;" d +REPLAY_VERSION ./src/defaults.h 9;" d +RIOOP_H ./src/replay/rioop.h 2;" d +RMDIR ./src/opcodes.h /^ RMDIR,$/;" e enum:__anon1 +RPROCESS_H ./src/replay/rprocess.h 2;" d +RTASK_H ./src/replay/rtask.h 2;" d +RTHREAD_H ./src/replay/rthread.h 9;" d +RWORKER_H ./src/replay/rworker.h 2;" d +Readhex ./src/macros.h 14;" d +STACK_H ./src/datas/stack.h 2;" d +STAT ./src/opcodes.h /^ STAT,$/;" e enum:__anon1 +STATFS ./src/opcodes.h /^ STATFS,$/;" e enum:__anon1 +STATFS64 ./src/opcodes.h /^ STATFS64,$/;" e enum:__anon1 +SUCCESS ./src/defaults.h /^ SUCCESS, \/**< Great success! *\/$/;" e enum:status_e_ +SYNC ./src/opcodes.h /^ SYNC,$/;" e enum:__anon1 +SYNCFS ./src/opcodes.h /^ SYNCFS,$/;" e enum:__anon1 +SYNC_FILE_RANGE ./src/opcodes.h /^ SYNC_FILE_RANGE,$/;" e enum:__anon1 +Segfault ./src/macros.h 79;" d +TASK_BUFFER_PER_THREAD ./src/defaults.h 15;" d +UNKNOWN ./src/defaults.h /^ UNKNOWN, \/**< Unknown return status :-\/ *\/$/;" e enum:status_e_ +UNLINK ./src/opcodes.h /^ UNLINK,$/;" e enum:__anon1 +UNLINK_AT ./src/opcodes.h /^ UNLINK_AT,$/;" e enum:__anon1 +UTESTS_H ./src/utests.h 2;" d +UTILS_H ./src/utils/utils.h 2;" d +VFD_H ./src/vfd.h 2;" d +VSIZE_H ./src/generate/vsize.h 2;" d +WRITE ./src/opcodes.h /^ WRITE = 20,$/;" e enum:__anon1 +WRITEV ./src/opcodes.h /^ WRITEV,$/;" e enum:__anon1 +Warn ./src/macros.h 88;" d +Warn_if ./src/macros.h 95;" d +_Errno ./src/replay/rioop.c 15;" d file: +_Error ./src/replay/rioop.c 7;" d file: +_GNU_SOURCE ./src/utils/utils.h 5;" d +_Init_arg ./src/replay/rioop.c 24;" d file: +_Init_bytes ./src/replay/rioop.c 36;" d file: +_Init_cmd ./src/replay/rioop.c 25;" d file: +_Init_fd ./src/replay/rioop.c 26;" d file: +_Init_flags ./src/replay/rioop.c 27;" d file: +_Init_offset ./src/replay/rioop.c 29;" d file: +_Init_op ./src/replay/rioop.c 30;" d file: +_Init_path ./src/replay/rioop.c 32;" d file: +_Init_path2 ./src/replay/rioop.c 31;" d file: +_Init_rc ./src/replay/rioop.c 33;" d file: +_Init_virtfd ./src/replay/rioop.c 40;" d file: +_Init_whence ./src/replay/rioop.c 34;" d file: +_MAX_META_LEN ./src/meta/meta.c 3;" d file: +_MAX_PROCESSES ./src/generate/generate.c 10;" d file: +_PATH_INSERT ./src/mounts.c 5;" d file: +_PATH_INSERT_LEN ./src/mounts.c 6;" d file: +_Perc_filtered ./src/generate/generate.c 12;" d file: +_Set_dir ./src/generate/vsize.c 8;" d file: +_Set_file ./src/generate/vsize.c 7;" d file: +_Set_inserted ./src/generate/vsize.c 10;" d file: +_Set_renamed ./src/generate/vsize.c 11;" d file: +_Set_required ./src/generate/vsize.c 12;" d file: +_Set_unsure ./src/generate/vsize.c 9;" d file: +_Using_string_keys ./src/datas/hmap.c 3;" d file: +_amap_new ./src/datas/amap.c /^static amap_s *_amap_new(long size, bool mmapped)$/;" f file: +_amap_test ./src/datas/amap.c /^void _amap_test(amap_s *a)$/;" f +_arch_check_atomic ./src/main.c /^static void _arch_check_atomic(void)$/;" f file: +_gprocess_vfd_map_destroy_cb ./src/generate/gprocess.c /^void _gprocess_vfd_map_destroy_cb(void *data)$/;" f +_hmap_new ./src/datas/hmap.c /^hmap_s *_hmap_new(unsigned int init_size)$/;" f +_hmap_test ./src/datas/hmap.c /^static void _hmap_test(hmap_s *h)$/;" f file: +_hmap_test_l ./src/datas/hmap.c /^static void _hmap_test_l(hmap_s *h)$/;" f file: +_list_elem_remove ./src/datas/list.c /^void _list_elem_remove(list_s *l, list_elem_s *e)$/;" f +_print_help ./src/main.c /^static void _print_help(void)$/;" f file: +_print_synopsis ./src/main.c /^static void _print_synopsis(void)$/;" f file: +_print_version ./src/main.c /^static void _print_version(void)$/;" f file: +_rprocess_destroy_cb ./src/replay/rworker.c /^static void _rprocess_destroy_cb(void *data)$/;" f file: +_rthread_init_log ./src/replay/rthread.c /^static void _rthread_init_log(rthread_s *t)$/;" f file: +absolute_path ./src/utils/futils.c /^char *absolute_path(const char *path)$/;" f +address ./src/generate/gtask.h /^ long address; \/**< An address (used by mmap related syscalls) *\/$/;" m struct:gtask_s_ +address2 ./src/generate/gtask.h /^ long address2; \/**< Another address (used by mmap related syscalls) *\/$/;" m struct:gtask_s_ +amap_destroy ./src/datas/amap.c /^void amap_destroy(amap_s* a)$/;" f +amap_get ./src/datas/amap.c /^void* amap_get(amap_s *a, const long position)$/;" f +amap_new ./src/datas/amap.c /^amap_s* amap_new(const long size)$/;" f +amap_new_mmapped ./src/datas/amap.c /^amap_s* amap_new_mmapped(const long size)$/;" f +amap_print ./src/datas/amap.c /^void amap_print(amap_s* a)$/;" f +amap_reset ./src/datas/amap.c /^void amap_reset(amap_s* a)$/;" f +amap_run_cb ./src/datas/amap.c /^void amap_run_cb(amap_s *a, void (*cb)(void *data))$/;" f +amap_s ./src/datas/amap.h /^} amap_s;$/;" t typeref:struct:amap_s_ +amap_s_ ./src/datas/amap.h /^typedef struct amap_s_ {$/;" s +amap_set ./src/datas/amap.c /^int amap_set(amap_s *a, const long position, void* value)$/;" f +amap_test ./src/datas/amap.c /^void amap_test(void)$/;" f +amap_unset ./src/datas/amap.c /^void* amap_unset(amap_s *a, const long position)$/;" f +append_random_to_file ./src/utils/futils.c /^void append_random_to_file(char *path, unsigned long bytes)$/;" f +arrays ./src/datas/amap.h /^ void*** arrays; \/**< The pointers to the amap arrays *\/$/;" m struct:amap_s_ +btree_destroy ./src/datas/btree.c /^void btree_destroy(btree_s* b)$/;" f +btree_destroy2 ./src/datas/btree.c /^void btree_destroy2(btree_s* b)$/;" f +btree_get ./src/datas/btree.c /^void* btree_get(btree_s* b, int key)$/;" f +btree_insert ./src/datas/btree.c /^int btree_insert(btree_s* b, int key, void *data)$/;" f +btree_new ./src/datas/btree.c /^btree_s* btree_new()$/;" f +btree_print ./src/datas/btree.c /^void btree_print(btree_s* b)$/;" f +btree_s ./src/datas/btree.h /^} btree_s;$/;" t typeref:struct:btree_s_ +btree_s_ ./src/datas/btree.h /^typedef struct btree_s_ {$/;" s +btreelem_ ./src/datas/btree.h /^typedef struct btreelem_ {$/;" s +btreelem_destroy_r ./src/datas/btree.c /^void btreelem_destroy_r(btreelem_s* e)$/;" f +btreelem_destroy_r2 ./src/datas/btree.c /^void btreelem_destroy_r2(btreelem_s* e)$/;" f +btreelem_get_r ./src/datas/btree.c /^void* btreelem_get_r(btreelem_s* e, int key)$/;" f +btreelem_insert_r ./src/datas/btree.c /^int btreelem_insert_r(btreelem_s* e, int key, void *data)$/;" f +btreelem_new ./src/datas/btree.c /^btreelem_s* btreelem_new(int key, void *data)$/;" f +btreelem_print_r ./src/datas/btree.c /^void btreelem_print_r(btreelem_s* e, int depth)$/;" f +btreelem_s ./src/datas/btree.h /^} btreelem_s;$/;" t typeref:struct:btreelem_ +bytes ./src/generate/gtask.h /^ long bytes; \/**< Amount of bytes *\/$/;" m struct:gtask_s_ +cache_file ./src/utils/futils.c /^void cache_file(const char *file)$/;" f +capture_file ./src/options.h /^ char *capture_file; \/**< The name of the .capture file *\/$/;" m struct:options_s_ +chown_path ./src/utils/futils.c /^void chown_path(const char *user, const char *path)$/;" f +chreplace ./src/utils/utils.c /^void chreplace(char *str, char replace, char with)$/;" f +cleanup_run ./src/cleanup/cleanup.c /^status_e cleanup_run(options_s *opts)$/;" f +clone ./src/replay/rtask.h /^ char *clone; \/**< Used for debug purposes only *\/$/;" m struct:rtask_s_ +count ./src/generate/gtask.h /^ long count; \/**< A count *\/$/;" m struct:gtask_s_ +count ./src/mounts.h /^ int count; \/**< The amount of mount points *\/$/;" m struct:mounts_s_ +data ./src/datas/btree.h /^ void *data; \/**< A pointer to the data stored in this element *\/$/;" m struct:btreelem_ +data ./src/datas/hmap.h /^ void **data; \/**< Pointers to the stored data, NULL if nothing there *\/$/;" m struct:hmap_s_ +data ./src/datas/list.h /^ void *data; \/**< Pointer to the stored data *\/$/;" m struct:list_elem_s_ +data ./src/datas/stack.h /^ void *data; \/**< Pointer to the stored data in the current element *\/$/;" m struct:stack_elem_s_ +data_destroy ./src/datas/amap.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all elements *\/$/;" m struct:amap_s_ +data_destroy ./src/datas/hmap.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all data *\/$/;" m struct:hmap_s_ +data_destroy ./src/datas/list.h /^ void (*data_destroy)(void *data); \/**< Callback to destroy all data *\/$/;" m struct:list_s_ +debug ./src/vfd.h /^ int debug; \/**< Used for debugging purposes only *\/$/;" m struct:vfd_s_ +dirfd ./src/vfd.h /^ DIR *dirfd; \/**< The real dirfd *\/$/;" m struct:vfd_s_ +dirname_r ./src/utils/futils.c /^char* dirname_r(char *path)$/;" f +dirs_created ./src/init/itask.h /^ long dirs_created;$/;" m struct:itask_s_ +drop_caches ./src/options.h /^ bool drop_caches; \/**< True if ioriot should drop all Linux caches *\/$/;" m struct:options_s_ +drop_caches ./src/utils/futils.c /^void drop_caches(void)$/;" f +drop_root ./src/utils/utils.c /^void drop_root(const char *user)$/;" f +ensure_dir_empty ./src/utils/futils.c /^void ensure_dir_empty(const char *path)$/;" f +ensure_dir_exists ./src/utils/futils.c /^long ensure_dir_exists(const char *path)$/;" f +ensure_file_exists ./src/utils/futils.c /^int ensure_file_exists(char *path, long *num_dirs_created)$/;" f +ensure_parent_dir_exists ./src/utils/futils.c /^void ensure_parent_dir_exists(const char *path)$/;" f +exists ./src/utils/futils.c /^bool exists(const char *path)$/;" f +fd ./src/generate/gtask.h /^ int fd; \/**< File descriptor number *\/$/;" m struct:gtask_s_ +fd ./src/vfd.h /^ int fd; \/**< the real fd *\/$/;" m struct:vfd_s_ +fd_map ./src/generate/gprocess.h /^ hmap_s *fd_map; \/**< All mappings from real fd to virtual fd *\/$/;" m struct:gprocess_s_ +fds_map ./src/replay/rprocess.h /^ amap_s *fds_map; \/**< Holding all file descriptors *\/$/;" m struct:rprocess_s_ +fds_map ./src/replay/rworker.h /^ amap_s* fds_map; \/**< Holding all file descriptors *\/$/;" m struct:__anon2 +files_created ./src/init/itask.h /^ long files_created;$/;" m struct:itask_s_ +filtered_where ./src/generate/gtask.h /^ char *filtered_where; \/**< Only used for debugging purposes *\/$/;" m struct:gtask_s_ +first ./src/datas/list.h /^ list_elem_s *first; \/**< The first element, NULL if list empty *\/$/;" m struct:list_s_ +flags ./src/generate/gtask.h /^ int flags; \/**< File open flags *\/$/;" m struct:gtask_s_ +fnotnull ./src/utils/utils.c /^FILE* fnotnull(FILE *fd, const char *path, char *file, int line)$/;" f +free_path ./src/vfd.h /^ bool free_path; \/**< True if path has to be freed or not *\/$/;" m struct:vfd_s_ +generate ./src/generate/gparser.h /^ generate_s *generate; \/**< The generate object *\/$/;" m struct:gparser_s_ +generate ./src/generate/gtask.h /^ void *generate; \/**< A pointer to the generate object *\/$/;" m struct:gtask_s_ +generate ./src/generate/gwriter.h /^ struct generate_s_ *generate; \/**< The generate object *\/$/;" m struct:gwriter_s_ typeref:struct:gwriter_s_::generate_s_ +generate ./src/generate/vsize.h /^ void *generate; \/**< A pointer to the generate object *\/$/;" m struct:vsize_s_ +generate_destroy ./src/generate/generate.c /^void generate_destroy(generate_s *g)$/;" f +generate_gprocess_by_realpid ./src/generate/generate.c /^void generate_gprocess_by_realpid(generate_s *g, gtask_s *t)$/;" f +generate_new ./src/generate/generate.c /^generate_s* generate_new(options_s *opts)$/;" f +generate_run ./src/generate/generate.c /^status_e generate_run(options_s *opts)$/;" f +generate_s ./src/generate/generate.h /^} generate_s;$/;" t typeref:struct:generate_s_ +generate_s_ ./src/generate/generate.h /^typedef struct generate_s_ {$/;" s +generate_vsize_by_path ./src/generate/generate.c /^vsize_s* generate_vsize_by_path(generate_s *g, gtask_s *t,$/;" f +generate_write_init_cb ./src/generate/generate.c /^void generate_write_init_cb(void *data)$/;" f +get_loadavg ./src/utils/utils.c /^void get_loadavg(char *readbuf)$/;" f +gioop_chmod ./src/generate/gioop.c /^status_e gioop_chmod(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_chown ./src/generate/gioop.c /^status_e gioop_chown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_close ./src/generate/gioop.c /^status_e gioop_close(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_close_all_vfd_cb ./src/generate/gioop.c /^void gioop_close_all_vfd_cb(void *data, void *data2)$/;" f +gioop_creat ./src/generate/gioop.c /^status_e gioop_creat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_exit_group ./src/generate/gioop.c /^status_e gioop_exit_group(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fchmod ./src/generate/gioop.c /^status_e gioop_fchmod(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fchown ./src/generate/gioop.c /^status_e gioop_fchown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fcntl ./src/generate/gioop.c /^status_e gioop_fcntl(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fdatasync ./src/generate/gioop.c /^status_e gioop_fdatasync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstat ./src/generate/gioop.c /^status_e gioop_fstat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstatat ./src/generate/gioop.c /^status_e gioop_fstatat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstatfs ./src/generate/gioop.c /^status_e gioop_fstatfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fstatfs64 ./src/generate/gioop.c /^status_e gioop_fstatfs64(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_fsync ./src/generate/gioop.c /^status_e gioop_fsync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_getdents ./src/generate/gioop.c /^status_e gioop_getdents(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_lchown ./src/generate/gioop.c /^status_e gioop_lchown(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_lseek ./src/generate/gioop.c /^status_e gioop_lseek(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_lstat ./src/generate/gioop.c /^status_e gioop_lstat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_mkdir ./src/generate/gioop.c /^status_e gioop_mkdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_mkdirat ./src/generate/gioop.c /^status_e gioop_mkdirat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_open ./src/generate/gioop.c /^status_e gioop_open(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_openat ./src/generate/gioop.c /^status_e gioop_openat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_read ./src/generate/gioop.c /^status_e gioop_read(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readahead ./src/generate/gioop.c /^status_e gioop_readahead(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readdir ./src/generate/gioop.c /^status_e gioop_readdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readlink ./src/generate/gioop.c /^status_e gioop_readlink(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readlinkat ./src/generate/gioop.c /^status_e gioop_readlinkat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_readv ./src/generate/gioop.c /^status_e gioop_readv(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_rename ./src/generate/gioop.c /^status_e gioop_rename(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_renameat ./src/generate/gioop.c /^status_e gioop_renameat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_renameat2 ./src/generate/gioop.c /^status_e gioop_renameat2(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_rmdir ./src/generate/gioop.c /^status_e gioop_rmdir(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_run ./src/generate/gioop.c /^status_e gioop_run(gwriter_s *w, gtask_s *t)$/;" f +gioop_stat ./src/generate/gioop.c /^status_e gioop_stat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_statfs ./src/generate/gioop.c /^status_e gioop_statfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_statfs64 ./src/generate/gioop.c /^status_e gioop_statfs64(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_sync ./src/generate/gioop.c /^status_e gioop_sync(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_sync_file_range ./src/generate/gioop.c /^status_e gioop_sync_file_range(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_syncfs ./src/generate/gioop.c /^status_e gioop_syncfs(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_unlink ./src/generate/gioop.c /^status_e gioop_unlink(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_unlinkat ./src/generate/gioop.c /^status_e gioop_unlinkat(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_write ./src/generate/gioop.c /^status_e gioop_write(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gioop_writev ./src/generate/gioop.c /^status_e gioop_writev(gwriter_s *w, gtask_s *t, generate_s *g)$/;" f +gparser_destroy ./src/generate/gparser.c /^void gparser_destroy(gparser_s *p)$/;" f +gparser_extract ./src/generate/gparser.c /^void gparser_extract(gparser_s *p, gtask_s *t)$/;" f +gparser_extract_tok ./src/generate/gparser.c /^status_e gparser_extract_tok(gparser_s *p, gtask_s *t, char *tok)$/;" f +gparser_get_pidtid ./src/generate/gparser.c /^bool gparser_get_pidtid(gparser_s *p, char *pidtid, long *pid, long *tid)$/;" f +gparser_new ./src/generate/gparser.c /^gparser_s* gparser_new(generate_s *g)$/;" f +gparser_pthread_start ./src/generate/gparser.c /^void* gparser_pthread_start(void *data)$/;" f +gparser_s ./src/generate/gparser.h /^} gparser_s;$/;" t typeref:struct:gparser_s_ +gparser_s_ ./src/generate/gparser.h /^typedef struct gparser_s_ {$/;" s +gparser_start ./src/generate/gparser.c /^void gparser_start(gparser_s *p)$/;" f +gparser_terminate ./src/generate/gparser.c /^void gparser_terminate(gparser_s *p)$/;" f +gparser_token_not_ok ./src/generate/gparser.c /^bool gparser_token_not_ok(gparser_s *p, char *tok)$/;" f +gprocess ./src/generate/gtask.h /^ struct gprocess_s_ *gprocess; \/**< A pointer to the process object *\/$/;" m struct:gtask_s_ typeref:struct:gtask_s_::gprocess_s_ +gprocess_create_vfd_by_realfd ./src/generate/gprocess.c /^void gprocess_create_vfd_by_realfd(gprocess_s *gp, gtask_s *t, generate_s *g)$/;" f +gprocess_destroy ./src/generate/gprocess.c /^void gprocess_destroy(gprocess_s *gp)$/;" f +gprocess_new ./src/generate/gprocess.c /^gprocess_s* gprocess_new(const long pid, const long mapped_pid)$/;" f +gprocess_s ./src/generate/gprocess.h /^} gprocess_s;$/;" t typeref:struct:gprocess_s_ +gprocess_s_ ./src/generate/gprocess.h /^typedef struct gprocess_s_ {$/;" s +gprocess_vfd_by_realfd ./src/generate/gprocess.c /^status_e gprocess_vfd_by_realfd(gprocess_s *gp, gtask_s *t)$/;" f +gtask_destroy ./src/generate/gtask.c /^void gtask_destroy(gtask_s *t)$/;" f +gtask_init ./src/generate/gtask.c /^void gtask_init(gtask_s *t, char *line, const unsigned long lineno)$/;" f +gtask_new ./src/generate/gtask.c /^gtask_s* gtask_new(void *generate)$/;" f +gtask_s ./src/generate/gtask.h /^} gtask_s;$/;" t typeref:struct:gtask_s_ +gtask_s_ ./src/generate/gtask.h /^typedef struct gtask_s_ {$/;" s +gwriter_destroy ./src/generate/gwriter.c /^void gwriter_destroy(gwriter_s *w)$/;" f +gwriter_new ./src/generate/gwriter.c /^gwriter_s* gwriter_new(generate_s *g)$/;" f +gwriter_pthread_start ./src/generate/gwriter.c /^void* gwriter_pthread_start(void *data)$/;" f +gwriter_s ./src/generate/gwriter.h /^} gwriter_s;$/;" t typeref:struct:gwriter_s_ +gwriter_s_ ./src/generate/gwriter.h /^typedef struct gwriter_s_ {$/;" s +gwriter_start ./src/generate/gwriter.c /^void gwriter_start(gwriter_s *w)$/;" f +gwriter_terminate ./src/generate/gwriter.c /^void gwriter_terminate(gwriter_s *w)$/;" f +has_fd ./src/generate/gtask.h /^ bool has_fd; \/**< True if task has a file descriptor number *\/$/;" m struct:gtask_s_ +hmap_destroy ./src/datas/hmap.c /^void hmap_destroy(hmap_s *h)$/;" f +hmap_get ./src/datas/hmap.c /^void* hmap_get(hmap_s *h, char *key)$/;" f +hmap_get_addr ./src/datas/hmap.c /^unsigned int hmap_get_addr(hmap_s *h, char *key)$/;" f +hmap_get_addr_l ./src/datas/hmap.c /^unsigned int hmap_get_addr_l(hmap_s *h, const long key)$/;" f +hmap_get_l ./src/datas/hmap.c /^void* hmap_get_l(hmap_s *h, const long key)$/;" f +hmap_insert ./src/datas/hmap.c /^int hmap_insert(hmap_s *h, char *key, void *data)$/;" f +hmap_insert_l ./src/datas/hmap.c /^int hmap_insert_l(hmap_s *h, const long key, void *data)$/;" f +hmap_new ./src/datas/hmap.c /^hmap_s *hmap_new(unsigned int init_size)$/;" f +hmap_new_l ./src/datas/hmap.c /^hmap_s *hmap_new_l(unsigned int init_size)$/;" f +hmap_print ./src/datas/hmap.c /^void hmap_print(hmap_s *h)$/;" f +hmap_remove ./src/datas/hmap.c /^void* hmap_remove(hmap_s *h, char *key)$/;" f +hmap_remove_l ./src/datas/hmap.c /^void* hmap_remove_l(hmap_s *h, const long key)$/;" f +hmap_run_cb ./src/datas/hmap.c /^void hmap_run_cb(hmap_s* h, void (*cb)(void *data))$/;" f +hmap_run_cb2 ./src/datas/hmap.c /^void hmap_run_cb2(hmap_s* h, void (*cb)(void *data, void *data2), void *data_)$/;" f +hmap_s ./src/datas/hmap.h /^} hmap_s;$/;" t typeref:struct:hmap_s_ +hmap_s_ ./src/datas/hmap.h /^typedef struct hmap_s_ {$/;" s +hmap_test ./src/datas/hmap.c /^void hmap_test(void)$/;" f +id ./src/generate/vsize.h /^ unsigned long id; \/**< The vsize id *\/$/;" m struct:vsize_s_ +ignore_count ./src/mounts.h /^ int ignore_count; \/**< The amount of ignored mount points *\/$/;" m struct:mounts_s_ +ignore_mps ./src/mounts.h /^ char *ignore_mps[MAX_MOUNTPOINTS]; \/**< The ignored mp paths *\/$/;" m struct:mounts_s_ +init ./src/init/ithread.h /^ init_s *init; \/**< The responsible init object *\/$/;" m struct:ithread_s_ +init ./src/options.h /^ bool init; \/**< If set ioriot will initialise the environment *\/$/;" m struct:options_s_ +init_destroy ./src/init/init.c /^void init_destroy(init_s *i)$/;" f +init_extract_header ./src/init/init.c /^void init_extract_header(init_s *i, off_t *init_offset)$/;" f +init_new ./src/init/init.c /^init_s *init_new(options_s *opts)$/;" f +init_parent_dir ./src/generate/vsize.c /^void init_parent_dir(vsize_s *v, const char *path)$/;" f +init_run ./src/init/init.c /^status_e init_run(options_s *opts)$/;" f +init_s ./src/init/init.h /^} init_s;$/;" t typeref:struct:init_s_ +init_s_ ./src/init/init.h /^typedef struct init_s_ {$/;" s +initm ./src/replay/rprocess.h /^ bool initm; \/**< Indicates whether ioriot is in init mode or not *\/$/;" m struct:rprocess_s_ +inserted ./src/generate/vsize.h /^ bool inserted; \/**< For debugging purposes only *\/$/;" m struct:vsize_s_ +is_dir ./src/generate/vsize.h /^ bool is_dir; \/**< True if this file\/dir is a directory *\/$/;" m struct:vsize_s_ +is_dir ./src/init/itask.h /^ bool is_dir;$/;" m struct:itask_s_ +is_dir ./src/utils/futils.c /^bool is_dir(const char *path)$/;" f +is_file ./src/generate/vsize.h /^ bool is_file; \/**< True if this file\/dir is a regular file *\/$/;" m struct:vsize_s_ +is_file ./src/init/itask.h /^ bool is_file;$/;" m struct:itask_s_ +is_number ./src/utils/utils.c /^bool is_number(char *str)$/;" f +is_reg ./src/utils/futils.c /^bool is_reg(const char *path)$/;" f +itask_destroy ./src/init/itask.c /^void itask_destroy(itask_s *task)$/;" f +itask_extract_stats ./src/init/itask.c /^void itask_extract_stats(itask_s *task, long* dirs_created, long *files_created,$/;" f +itask_new ./src/init/itask.c /^itask_s* itask_new()$/;" f +itask_print ./src/init/itask.c /^void itask_print(itask_s *task)$/;" f +itask_reset_stats ./src/init/itask.c /^void itask_reset_stats(itask_s *task)$/;" f +itask_s ./src/init/itask.h /^} itask_s;$/;" t typeref:struct:itask_s_ +itask_s_ ./src/init/itask.h /^typedef struct itask_s_ {$/;" s +ithread_destroy ./src/init/ithread.c /^void ithread_destroy(ithread_s *t)$/;" f +ithread_new ./src/init/ithread.c /^ithread_s* ithread_new(init_s *i)$/;" f +ithread_pthread_start ./src/init/ithread.c /^void* ithread_pthread_start(void *data)$/;" f +ithread_run_task ./src/init/ithread.c /^void ithread_run_task(ithread_s *t, itask_s *task)$/;" f +ithread_s ./src/init/ithread.h /^} ithread_s;$/;" t typeref:struct:ithread_s_ +ithread_s_ ./src/init/ithread.h /^typedef struct ithread_s_ {$/;" s +ithread_start ./src/init/ithread.c /^void ithread_start(ithread_s *t)$/;" f +ithread_terminate ./src/init/ithread.c /^void ithread_terminate(ithread_s *t)$/;" f +key ./src/datas/btree.h /^ int key; \/**< The key of the element *\/$/;" m struct:btreelem_ +key ./src/datas/list.h /^ char *key; \/**< The key of the lemenet *\/$/;" m struct:list_elem_s_ +key_l ./src/datas/list.h /^ long key_l; \/**< The same as key, but for long keys *\/$/;" m struct:list_elem_s_ +keys ./src/datas/hmap.h /^ char **keys; \/**< List of all keys, NULL if nothing at a address *\/$/;" m struct:hmap_s_ +keys_l ./src/datas/hmap.h /^ int *keys_l; \/**< Same as keys, but for long keys *\/$/;" m struct:hmap_s_ +l ./src/datas/hmap.h /^ list_s **l; \/**< Pointers to the linked lists, used on hash collision *\/$/;" m struct:hmap_s_ +left ./src/datas/btree.h /^ struct btreelem_ *left; \/**< The next element to the left *\/$/;" m struct:btreelem_ typeref:struct:btreelem_::btreelem_ +lengths ./src/mounts.h /^ int lengths[MAX_MOUNTPOINTS]; \/**< The mp lenghts *\/$/;" m struct:mounts_s_ +line ./src/generate/gtask.h /^ char *line; \/**< A pointer to the remaining part of the .capture line *\/$/;" m struct:gtask_s_ +line ./src/replay/rtask.h /^ char line[MAX_LINE_LEN]; \/**< The remaining part of the .replay line *\/$/;" m struct:rtask_s_ +lineno ./src/generate/generate.h /^ long lineno; \/**< The current line number *\/$/;" m struct:generate_s_ +lineno ./src/generate/gtask.h /^ long lineno; \/**< The current line number *\/$/;" m struct:gtask_s_ +lineno ./src/replay/rprocess.h /^ unsigned long lineno; \/**< Holding the current .replay line number *\/$/;" m struct:rprocess_s_ +lineno ./src/replay/rtask.h /^ unsigned long lineno; \/**< The current line number *\/$/;" m struct:rtask_s_ +list_destroy ./src/datas/list.c /^void list_destroy(list_s *l)$/;" f +list_elem_s ./src/datas/list.h /^} list_elem_s;$/;" t typeref:struct:list_elem_s_ +list_elem_s_ ./src/datas/list.h /^typedef struct list_elem_s_ {$/;" s +list_key_get ./src/datas/list.c /^void* list_key_get(list_s *l, char *key)$/;" f +list_key_get_l ./src/datas/list.c /^void* list_key_get_l(list_s *l, const long key)$/;" f +list_key_insert ./src/datas/list.c /^int list_key_insert(list_s *l, char *key, void *data)$/;" f +list_key_insert_l ./src/datas/list.c /^int list_key_insert_l(list_s *l, const long key, void *data)$/;" f +list_key_remove ./src/datas/list.c /^void* list_key_remove(list_s *l, char *key)$/;" f +list_key_remove_l ./src/datas/list.c /^void* list_key_remove_l(list_s *l, const long key)$/;" f +list_new ./src/datas/list.c /^list_s *list_new()$/;" f +list_new_l ./src/datas/list.c /^list_s *list_new_l()$/;" f +list_print ./src/datas/list.c /^void list_print(list_s *l)$/;" f +list_run_cb ./src/datas/list.c /^void list_run_cb(list_s* l, void (*cb)(void *data))$/;" f +list_run_cb2 ./src/datas/list.c /^void list_run_cb2(list_s* l, void (*cb)(void *data, void *data2), void *data_)$/;" f +list_s ./src/datas/list.h /^} list_s;$/;" t typeref:struct:list_s_ +list_s_ ./src/datas/list.h /^typedef struct list_s_ {$/;" s +list_test ./src/datas/list.c /^void list_test(void)$/;" f +main ./src/main.c /^int main(int argc, char **argv)$/;" f +mapped_fd ./src/generate/gtask.h /^ long mapped_fd; \/**< The mapped file descriptor number *\/$/;" m struct:gtask_s_ +mapped_fd ./src/vfd.h /^ long mapped_fd; \/**< The mapped fd (virtual fd) *\/$/;" m struct:vfd_s_ +mapped_pid ./src/generate/gprocess.h /^ long mapped_pid; \/**< The mapped PID *\/$/;" m struct:gprocess_s_ +mapped_time ./src/generate/gtask.h /^ long mapped_time; \/**< The mapped time *\/$/;" m struct:gtask_s_ +max_mapped_fd ./src/generate/gprocess.h /^ long max_mapped_fd; \/**< The max mapped fd number *\/$/;" m struct:gprocess_s_ +meta_destroy ./src/meta/meta.c /^void meta_destroy(meta_s *m)$/;" f +meta_new ./src/meta/meta.c /^meta_s* meta_new(FILE *replay_fd)$/;" f +meta_read_l ./src/meta/meta.c /^bool meta_read_l(meta_s *m, char *key, long *val)$/;" f +meta_read_s ./src/meta/meta.c /^bool meta_read_s(meta_s *m, char *key, char **val)$/;" f +meta_read_start ./src/meta/meta.c /^void meta_read_start(meta_s *m)$/;" f +meta_reserve ./src/meta/meta.c /^void meta_reserve(meta_s *m)$/;" f +meta_s ./src/meta/meta.h /^} meta_s;$/;" t typeref:struct:meta_s_ +meta_s_ ./src/meta/meta.h /^typedef struct meta_s_ {$/;" s +meta_write_l ./src/meta/meta.c /^void meta_write_l(meta_s *m, char *key, long val)$/;" f +meta_write_s ./src/meta/meta.c /^void meta_write_s(meta_s *m, char *key, char *val)$/;" f +meta_write_start ./src/meta/meta.c /^void meta_write_start(meta_s *m)$/;" f +mkdir_p ./src/utils/futils.c /^int mkdir_p(const char *path, mode_t mode, long *num_dirs_created)$/;" f +mmap_map ./src/generate/generate.h /^ hmap_s *mmap_map; \/**< mmap address mappings *\/$/;" m struct:generate_s_ +mmapok ./src/utils/utils.c /^void* mmapok(void *p, char *file, int line)$/;" f +mmapped ./src/datas/amap.h /^ bool mmapped; \/**< True if amap is memory mapped *\/$/;" m struct:amap_s_ +mode ./src/generate/gtask.h /^ int mode; \/**< File open mode *\/$/;" m struct:gtask_s_ +mounts ./src/init/init.h /^ mounts_s *mounts;$/;" m struct:init_s_ +mounts_destroy ./src/mounts.c /^void mounts_destroy(mounts_s *m)$/;" f +mounts_get_mountnumber ./src/mounts.c /^int mounts_get_mountnumber(mounts_s *m, const char *path)$/;" f +mounts_ignore_path ./src/mounts.c /^bool mounts_ignore_path(mounts_s *m, const char *path)$/;" f +mounts_init ./src/mounts.c /^void mounts_init(mounts_s *m)$/;" f +mounts_new ./src/mounts.c /^mounts_s *mounts_new(options_s *opts)$/;" f +mounts_purge ./src/mounts.c /^void mounts_purge(mounts_s *m)$/;" f +mounts_read ./src/mounts.c /^void mounts_read(mounts_s *m)$/;" f +mounts_s ./src/mounts.h /^} mounts_s;$/;" t typeref:struct:mounts_s_ +mounts_s_ ./src/mounts.h /^typedef struct mounts_s_ {$/;" s +mounts_transform_path ./src/mounts.c /^bool mounts_transform_path(mounts_s *m, const char *name,$/;" f +mounts_trash ./src/mounts.c /^void mounts_trash(mounts_s *m)$/;" f +mps ./src/generate/generate.h /^ mounts_s *mps; \/**< The mounts object *\/$/;" m struct:generate_s_ +mps ./src/mounts.h /^ char *mps[MAX_MOUNTPOINTS]; \/**< The mp paths *\/$/;" m struct:mounts_s_ +name ./src/generate/generate.h /^ char *name; \/**< The name of the test specified by the user *\/$/;" m struct:generate_s_ +name ./src/options.h /^ char *name; \/**< The name of the test (found in .ioriot\/name sub-dirs) *\/$/;" m struct:options_s_ +next ./src/datas/list.h /^ struct list_elem_s_ *next; \/**< The next element *\/$/;" m struct:list_elem_s_ typeref:struct:list_elem_s_::list_elem_s_ +next ./src/datas/stack.h /^ struct stack_elem_s_ *next; \/**< The next element *\/$/;" m struct:stack_elem_s_ typeref:struct:stack_elem_s_::stack_elem_s_ +notnull ./src/utils/utils.c /^void* notnull(void *p, char *file, int line, int count)$/;" f +num_arrays ./src/datas/amap.h /^ int num_arrays; \/**< The amount of arrays used in the amap *\/$/;" m struct:amap_s_ +num_lines_filtered ./src/generate/generate.h /^ long num_lines_filtered; \/**< The amount of lines filtered out *\/$/;" m struct:generate_s_ +num_mapped_fds ./src/generate/generate.h /^ unsigned long num_mapped_fds; \/**< The amount of mapped FDs *\/$/;" m struct:generate_s_ +num_mapped_pids ./src/generate/generate.h /^ unsigned long num_mapped_pids; \/**< The amount of mapped PIDs *\/$/;" m struct:generate_s_ +num_threads_per_worker ./src/options.h /^ int num_threads_per_worker; \/**< Max threads per worker processes *\/$/;" m struct:options_s_ +num_vsizes ./src/generate/generate.h /^ unsigned long num_vsizes; \/**< The amount of virtual sizes *\/$/;" m struct:generate_s_ +num_workers ./src/options.h /^ int num_workers; \/**< The amount of worker processes *\/$/;" m struct:options_s_ +offset ./src/generate/gtask.h /^ long offset; \/**< A offset *\/$/;" m struct:gtask_s_ +offset ./src/generate/vsize.h /^ off_t offset; \/**< The current file offset *\/$/;" m struct:vsize_s_ +offset ./src/meta/meta.h /^ off_t offset; \/**< The meta offset (usually 0) *\/$/;" m struct:meta_s_ +offset ./src/vfd.h /^ unsigned long offset; \/**< The current virtual file offset in bytes *\/$/;" m struct:vfd_s_ +op ./src/generate/gtask.h /^ char *op; \/**< Operation\/syscall name *\/$/;" m struct:gtask_s_ +opcode_e ./src/opcodes.h /^} opcode_e;$/;" t typeref:enum:__anon1 +options_destroy ./src/options.c /^void options_destroy(options_s *o)$/;" f +options_new ./src/options.c /^options_s *options_new()$/;" f +options_s ./src/options.h /^} options_s;$/;" t typeref:struct:options_s_ +options_s_ ./src/options.h /^typedef struct options_s_ {$/;" s +opts ./src/generate/generate.h /^ options_s *opts; \/**< A pointer to the options object *\/$/;" m struct:generate_s_ +opts ./src/init/init.h /^ options_s *opts;$/;" m struct:init_s_ +opts ./src/mounts.h /^ options_s *opts; \/**< A pointer to the options object *\/$/;" m struct:mounts_s_ +opts ./src/replay/rworker.h /^ options_s *opts; \/**< To synchronise access to rthread_buffer *\/$/;" m struct:__anon2 +original_line ./src/generate/gtask.h /^ char *original_line; \/**< Only used for debugging purposes *\/$/;" m struct:gtask_s_ +path ./src/generate/gtask.h /^ char *path; \/**< Path name *\/$/;" m struct:gtask_s_ +path ./src/generate/vsize.h /^ char *path; \/**< The path to the file\/directory *\/$/;" m struct:vsize_s_ +path ./src/init/itask.h /^ char *path;$/;" m struct:itask_s_ +path ./src/vfd.h /^ char *path; \/**< The file path belonging to that fd *\/$/;" m struct:vfd_s_ +path2 ./src/generate/gtask.h /^ char *path2; \/**< A second path name (e.g. for rename) *\/$/;" m struct:gtask_s_ +path2_r ./src/generate/gtask.h /^ char *path2_r; \/**< Work around to track mallocs, so it can be freed *\/$/;" m struct:gtask_s_ +path_r ./src/generate/gtask.h /^ char *path_r; \/**< Work around to track mallocs, so it can be freed *\/$/;" m struct:gtask_s_ +pid ./src/generate/gprocess.h /^ long pid; \/**< The real PID *\/$/;" m struct:gprocess_s_ +pid ./src/generate/gtask.h /^ long pid; \/**< The process ID *\/$/;" m struct:gtask_s_ +pid ./src/replay/rprocess.h /^ int pid; \/**< The virtual process ID *\/$/;" m struct:rprocess_s_ +pid_map ./src/generate/generate.h /^ amap_s *pid_map; \/**< A map of all virtual process objects *\/$/;" m struct:generate_s_ +pidtid ./src/generate/gtask.h /^ char *pidtid; \/**< String representing pid:tid *\/$/;" m struct:gtask_s_ +prev ./src/datas/list.h /^ struct list_elem_s_ *prev; \/**< The previous element *\/$/;" m struct:list_elem_s_ typeref:struct:list_elem_s_::list_elem_s_ +process ./src/replay/rtask.h /^ void *process; \/* The responsible process object *\/$/;" m struct:rtask_s_ +pthread ./src/generate/gparser.h /^ pthread_t pthread; \/**< The posix thread *\/$/;" m struct:gparser_s_ +pthread ./src/generate/gwriter.h /^ pthread_t pthread; \/**< The posix thread *\/$/;" m struct:gwriter_s_ +pthread ./src/init/ithread.h /^ pthread_t pthread; \/**< We run the init tasks in concurrent pthreads *\/$/;" m struct:ithread_s_ +pthread ./src/replay/rthread.h /^ pthread_t pthread; \/**< We run the tasks in concurrent pthreads *\/$/;" m struct:rthread_s_ +purge ./src/options.h /^ bool purge; \/**< If set ioriot will purge the environment *\/$/;" m struct:options_s_ +queue ./src/generate/gparser.h /^ rbuffer_s *queue; \/**< A queue of task objects *\/$/;" m struct:gparser_s_ +queue ./src/generate/gwriter.h /^ rbuffer_s *queue; \/**< A queue of task objects *\/$/;" m struct:gwriter_s_ +queue ./src/init/ithread.h /^ rbuffer_s *queue; \/**< The thread's task queue *\/$/;" m struct:ithread_s_ +rbuffer_destroy ./src/datas/rbuffer.c /^void rbuffer_destroy(rbuffer_s *r)$/;" f +rbuffer_get_next ./src/datas/rbuffer.c /^void* rbuffer_get_next(rbuffer_s* r)$/;" f +rbuffer_has_next ./src/datas/rbuffer.c /^bool rbuffer_has_next(rbuffer_s* r)$/;" f +rbuffer_insert ./src/datas/rbuffer.c /^bool rbuffer_insert(rbuffer_s* r, void *data)$/;" f +rbuffer_new ./src/datas/rbuffer.c /^rbuffer_s *rbuffer_new(const int size)$/;" f +rbuffer_print ./src/datas/rbuffer.c /^void rbuffer_print(rbuffer_s* r)$/;" f +rbuffer_s ./src/datas/rbuffer.h /^} rbuffer_s;$/;" t typeref:struct:rbuffer_s_ +rbuffer_s_ ./src/datas/rbuffer.h /^typedef struct rbuffer_s_ {$/;" s +rbuffer_test ./src/datas/rbuffer.c /^void rbuffer_test(void)$/;" f +read_buf ./src/meta/meta.h /^ char* read_buf; \/**< Pointer to a read buffer *\/$/;" m struct:meta_s_ +read_pos ./src/datas/rbuffer.h /^ sig_atomic_t read_pos;$/;" m struct:rbuffer_s_ +renamed ./src/generate/vsize.h /^ bool renamed; \/**< True if file\/dir has been renamed *\/$/;" m struct:vsize_s_ +replay ./src/options.h /^ bool replay; \/**< If set ioriot will run\/replay the test *\/$/;" m struct:options_s_ +replay_extract_header ./src/replay/replay.c /^void replay_extract_header(options_s *opts, FILE *replay_fd, long *num_vsizes,$/;" f +replay_fd ./src/generate/generate.h /^ FILE *replay_fd; \/**< The fd of the .replay file *\/$/;" m struct:generate_s_ +replay_fd ./src/init/init.h /^ FILE *replay_fd;$/;" m struct:init_s_ +replay_fd ./src/meta/meta.h /^ FILE* replay_fd; \/**< The FS of the .replay file *\/$/;" m struct:meta_s_ +replay_file ./src/options.h /^ char *replay_file; \/**< The name of the .replay file *\/$/;" m struct:options_s_ +replay_run ./src/replay/replay.c /^status_e replay_run(options_s *opts)$/;" f +required ./src/generate/vsize.h /^ bool required; \/**< True if init mode will create this file\/dir *\/$/;" m struct:vsize_s_ +ret ./src/generate/gtask.h /^ int ret; \/**< ioriot process status, SUCCESS if everything is alright *\/$/;" m struct:gtask_s_ +reuse_queue ./src/generate/generate.h /^ rbuffer_s *reuse_queue; \/**< A task buffer, for reusing these *\/$/;" m struct:generate_s_ +reuse_queue ./src/init/init.h /^ rbuffer_s *reuse_queue;$/;" m struct:init_s_ +reuse_queue_mutex ./src/init/init.h /^ pthread_mutex_t reuse_queue_mutex;$/;" m struct:init_s_ +right ./src/datas/btree.h /^ struct btreelem_ *right; \/**< The next element to the right *\/$/;" m struct:btreelem_ typeref:struct:btreelem_::btreelem_ +ring ./src/datas/rbuffer.h /^ void **ring;$/;" m struct:rbuffer_s_ +rioop_chmod ./src/replay/rioop.c /^void rioop_chmod(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_chown ./src/replay/rioop.c /^void rioop_chown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_close ./src/replay/rioop.c /^void rioop_close(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fchmod ./src/replay/rioop.c /^void rioop_fchmod(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fchown ./src/replay/rioop.c /^void rioop_fchown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fcntl ./src/replay/rioop.c /^void rioop_fcntl(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fdatasync ./src/replay/rioop.c /^void rioop_fdatasync(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fstat ./src/replay/rioop.c /^void rioop_fstat(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_fsync ./src/replay/rioop.c /^void rioop_fsync(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_getdents ./src/replay/rioop.c /^void rioop_getdents(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_lchown ./src/replay/rioop.c /^void rioop_lchown(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_lseek ./src/replay/rioop.c /^void rioop_lseek(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_mkdir ./src/replay/rioop.c /^void rioop_mkdir(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_open ./src/replay/rioop.c /^void rioop_open(rprocess_s *p, rthread_s *t, rtask_s *task, int flags_)$/;" f +rioop_read ./src/replay/rioop.c /^void rioop_read(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_rename ./src/replay/rioop.c /^void rioop_rename(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_rmdir ./src/replay/rioop.c /^void rioop_rmdir(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_run ./src/replay/rioop.c /^void rioop_run(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_stat ./src/replay/rioop.c /^void rioop_stat(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_unlink ./src/replay/rioop.c /^void rioop_unlink(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +rioop_write ./src/replay/rioop.c /^void rioop_write(rprocess_s *p, rthread_s *t, rtask_s *task)$/;" f +root ./src/datas/btree.h /^ btreelem_s *root; \/**< The root element *\/$/;" m struct:btree_s_ +rprocess_destroy ./src/replay/rprocess.c /^void rprocess_destroy(rprocess_s *p)$/;" f +rprocess_map ./src/replay/rworker.h /^ amap_s* rprocess_map; \/**< Holding all processes handled by this worker *\/$/;" m struct:__anon2 +rprocess_new ./src/replay/rprocess.c /^rprocess_s* rprocess_new(const int pid, amap_s *fds_map)$/;" f +rprocess_s ./src/replay/rprocess.h /^} rprocess_s;$/;" t typeref:struct:rprocess_s_ +rprocess_s_ ./src/replay/rprocess.h /^typedef struct rprocess_s_ {$/;" s +rtask_destroy ./src/replay/rtask.c /^void rtask_destroy(rtask_s *task)$/;" f +rtask_new ./src/replay/rtask.c /^rtask_s* rtask_new()$/;" f +rtask_s ./src/replay/rtask.h /^} rtask_s;$/;" t typeref:struct:rtask_s_ +rtask_s_ ./src/replay/rtask.h /^typedef struct rtask_s_ {$/;" s +rtask_update ./src/replay/rtask.c /^void rtask_update(rtask_s *task, void *worker, void *process, char *line,$/;" f +rthread_buffer ./src/replay/rworker.h /^ rbuffer_s *rthread_buffer; \/**< Buffering idle threads to be reused *\/$/;" m struct:__anon2 +rthread_buffer_mutex ./src/replay/rworker.h /^ pthread_mutex_t rthread_buffer_mutex; \/**< Sync access to rthread_buffer *\/$/;" m struct:__anon2 +rthread_destroy ./src/replay/rthread.c /^void rthread_destroy(rthread_s *t)$/;" f +rthread_fd ./src/replay/rthread.h /^ FILE *rthread_fd; \/**< Used for debugging purposes only *\/$/;" m struct:rthread_s_ +rthread_insert_task ./src/replay/rthread.c /^bool rthread_insert_task(rthread_s* t, rtask_s* task)$/;" f +rthread_map ./src/replay/rworker.h /^ amap_s* rthread_map; \/**< Holding all threads handled by this worker *\/$/;" m struct:__anon2 +rthread_new ./src/replay/rthread.c /^rthread_s* rthread_new(const long tid, void *worker)$/;" f +rthread_process_task ./src/replay/rthread.c /^void rthread_process_task(rthread_s* t, rtask_s *task,$/;" f +rthread_pthread_start ./src/replay/rthread.c /^void *rthread_pthread_start(void *data)$/;" f +rthread_s ./src/replay/rthread.h /^} rthread_s;$/;" t typeref:struct:rthread_s_ +rthread_s_ ./src/replay/rthread.h /^typedef struct rthread_s_ {$/;" s +rthread_terminate ./src/replay/rthread.c /^void rthread_terminate(rthread_s* t)$/;" f +rthread_update ./src/replay/rthread.c /^long rthread_update(rthread_s *t, const long tid)$/;" f +rworker_destroy ./src/replay/rworker.c /^void rworker_destroy(rworker_s *w)$/;" f +rworker_fd ./src/replay/rworker.h /^ FILE *rworker_fd; \/**< For debugging purposes only *\/$/;" m struct:__anon2 +rworker_new ./src/replay/rworker.c /^rworker_s* rworker_new(const int rworker_num, amap_s *fds_map,$/;" f +rworker_num ./src/replay/rprocess.h /^ int rworker_num; \/**< The worker number of the responsible worker *\/$/;" m struct:rprocess_s_ +rworker_num ./src/replay/rworker.h /^ int rworker_num; \/**< The current worker ID *\/$/;" m struct:__anon2 +rworker_process_lines ./src/replay/rworker.c /^status_e rworker_process_lines(rworker_s* w, const long num_lines)$/;" f +rworker_s ./src/replay/rworker.h /^} rworker_s;$/;" t typeref:struct:__anon2 +single_threaded ./src/replay/rthread.h /^ bool single_threaded; \/**< Worker is single threaded or not *\/$/;" m struct:rthread_s_ +size ./src/datas/amap.h /^ long size; \/**< The total size\/capacity of the amap *\/$/;" m struct:amap_s_ +size ./src/datas/btree.h /^ int size; \/**< The current size of the binary tree *\/$/;" m struct:btree_s_ +size ./src/datas/hmap.h /^ unsigned int size; \/**< Size of the hmap *\/$/;" m struct:hmap_s_ +size ./src/datas/rbuffer.h /^ int size;$/;" m struct:rbuffer_s_ +size ./src/datas/stack.h /^ unsigned long size; \/**< A count how many elements are in the stack *\/$/;" m struct:stack_s_ +sizes_created ./src/init/itask.h /^ long sizes_created;$/;" m struct:itask_s_ +speed_factor ./src/options.h /^ double speed_factor; \/**< Specifies how fast the test is replayed *\/$/;" m struct:options_s_ +stack_destroy ./src/datas/stack.c /^void stack_destroy(stack_s *s)$/;" f +stack_elem_s ./src/datas/stack.h /^} stack_elem_s;$/;" t typeref:struct:stack_elem_s_ +stack_elem_s_ ./src/datas/stack.h /^typedef struct stack_elem_s_ {$/;" s +stack_is_empty ./src/datas/stack.c /^int stack_is_empty(stack_s *s)$/;" f +stack_new ./src/datas/stack.c /^stack_s *stack_new()$/;" f +stack_new_reverse_from ./src/datas/stack.c /^stack_s* stack_new_reverse_from(stack_s *s)$/;" f +stack_pop ./src/datas/stack.c /^void* stack_pop(stack_s *s)$/;" f +stack_push ./src/datas/stack.c /^void stack_push(stack_s *s, void *data)$/;" f +stack_s ./src/datas/stack.h /^} stack_s;$/;" t typeref:struct:stack_s_ +stack_s_ ./src/datas/stack.h /^typedef struct stack_s_ {$/;" s +start_pthread ./src/utils/utils.c /^void start_pthread(pthread_t *thread, void*(*cb)(void*), void *data)$/;" f +start_time ./src/generate/generate.h /^ long start_time; \/**< The start time from the .capture file *\/$/;" m struct:generate_s_ +status ./src/generate/gtask.h /^ int status; \/**< Operation\/syscall return status *\/$/;" m struct:gtask_s_ +status_e ./src/defaults.h /^} status_e;$/;" t typeref:enum:status_e_ +status_e_ ./src/defaults.h /^typedef enum status_e_ {$/;" g +strtok2_r ./src/utils/utils.c /^char* strtok2_r(char *str, char *delim, char **saveptr)$/;" f +strunquote ./src/utils/utils.c /^void strunquote(char *str)$/;" f +task_buffer ./src/replay/rworker.h /^ rbuffer_s *task_buffer; \/**< Buffering thread tasks to be reused *\/$/;" m struct:__anon2 +task_buffer_mutex ./src/replay/rworker.h /^ pthread_mutex_t task_buffer_mutex; \/**< To sync access to task_buffer *\/$/;" m struct:__anon2 +tasks ./src/replay/rthread.h /^ rbuffer_s* tasks; \/**< Holds all outstanding tasks *\/$/;" m struct:rthread_s_ +terminate ./src/generate/gparser.h /^ bool terminate; \/**< The parser thread will terminate if set to true *\/$/;" m struct:gparser_s_ +terminate ./src/generate/gwriter.h /^ bool terminate; \/**< The writer thread will terminate if set to true *\/$/;" m struct:gwriter_s_ +terminate ./src/init/ithread.h /^ bool terminate; \/**< Indicates that thread can terminate *\/$/;" m struct:ithread_s_ +terminate ./src/replay/rprocess.h /^ int terminate; \/**< Indicates whether the worker is terminating or not *\/$/;" m struct:rprocess_s_ +terminate ./src/replay/rthread.h /^ bool terminate; \/**< True if thread shall terminate *\/$/;" m struct:rthread_s_ +threads_map ./src/init/init.h /^ amap_s *threads_map;$/;" m struct:init_s_ +tid ./src/generate/gtask.h /^ long tid; \/**< The thread ID *\/$/;" m struct:gtask_s_ +tid ./src/replay/rthread.h /^ long tid; \/**< The virtual thread id *\/$/;" m struct:rthread_s_ +toks ./src/replay/rtask.h /^ char *toks[MAX_TOKENS+1]; \/**< The tokens parsed from the .replay line *\/$/;" m struct:rtask_s_ +top ./src/datas/stack.h /^ stack_elem_s *top; \/**< The top element of the stack, NULL if empty *\/$/;" m struct:stack_s_ +trash ./src/options.h /^ bool trash; \/**< If set ioriot will trash the environment *\/$/;" m struct:options_s_ +unsure ./src/generate/vsize.h /^ bool unsure; \/**< True if the file type is not fully clear *\/$/;" m struct:vsize_s_ +updates ./src/generate/vsize.h /^ long updates; \/**< Amount of times this vsize has been updated *\/$/;" m struct:vsize_s_ +user ./src/options.h /^ char *user; \/**< The user name to run the test as *\/$/;" m struct:options_s_ +utests_run ./src/utests.c /^void utests_run()$/;" f +vfd ./src/generate/gtask.h /^ vfd_s *vfd; \/**< A pointer to the virtual file descriptor *\/$/;" m struct:gtask_s_ +vfd_buffer ./src/generate/generate.h /^ rbuffer_s *vfd_buffer; \/**< A virtual fd buffer, for reusing these *\/$/;" m struct:generate_s_ +vfd_destroy ./src/vfd.c /^void vfd_destroy(vfd_s *vfd)$/;" f +vfd_map ./src/generate/gprocess.h /^ hmap_s *vfd_map; \/**< All virtual file descriptors of that process *\/$/;" m struct:gprocess_s_ +vfd_new ./src/vfd.c /^vfd_s* vfd_new(const int fd, const long mapped_fd, char *path)$/;" f +vfd_print ./src/vfd.c /^void vfd_print(vfd_s *vfd)$/;" f +vfd_s ./src/vfd.h /^} vfd_s;$/;" t typeref:struct:vfd_s_ +vfd_s_ ./src/vfd.h /^typedef struct vfd_s_ {$/;" s +vfd_update ./src/vfd.c /^void vfd_update(vfd_s *vfd, const int fd, const long mapped_fd, char *path)$/;" f +vsize ./src/generate/gtask.h /^ vsize_s *vsize; \/**< Pointer to the virtual size object *\/$/;" m struct:gtask_s_ +vsize ./src/generate/vsize.h /^ long vsize; \/**< The virtual size *\/$/;" m struct:vsize_s_ +vsize ./src/init/itask.h /^ long vsize;$/;" m struct:itask_s_ +vsize ./src/replay/rtask.h /^ unsigned long vsize; \/**< The vsize *\/$/;" m struct:rtask_s_ +vsize2 ./src/generate/gtask.h /^ vsize_s *vsize2; \/**< Pointer to a second virtual size object *\/$/;" m struct:gtask_s_ +vsize_adjust ./src/generate/vsize.c /^void vsize_adjust(vsize_s *v, vfd_s* vfd)$/;" f +vsize_close ./src/generate/vsize.c /^void vsize_close(vsize_s *v, void* vfd)$/;" f +vsize_deficit ./src/generate/vsize.h /^ long vsize_deficit; \/**< Size to use for file creating during init mode *\/$/;" m struct:vsize_s_ +vsize_destroy ./src/generate/vsize.c /^void vsize_destroy(vsize_s *v)$/;" f +vsize_map ./src/generate/generate.h /^ hmap_s *vsize_map; \/**< A hash map of all virtual size objects *\/$/;" m struct:generate_s_ +vsize_mkdir ./src/generate/vsize.c /^void vsize_mkdir(vsize_s *v, const char *path)$/;" f +vsize_new ./src/generate/vsize.c /^vsize_s* vsize_new(char *file_path, const unsigned long id,$/;" f +vsize_open ./src/generate/vsize.c /^void vsize_open(vsize_s *v, void *vfd, const char *path, const int flags)$/;" f +vsize_read ./src/generate/vsize.c /^void vsize_read(vsize_s *v, void *vfd, const char *path, const int bytes)$/;" f +vsize_rename ./src/generate/vsize.c /^void vsize_rename(vsize_s *v, vsize_s *v2,$/;" f +vsize_rmdir ./src/generate/vsize.c /^void vsize_rmdir(vsize_s *v, const char *path)$/;" f +vsize_s ./src/generate/vsize.h /^} vsize_s;$/;" t typeref:struct:vsize_s_ +vsize_s_ ./src/generate/vsize.h /^typedef struct vsize_s_ {$/;" s +vsize_seek ./src/generate/vsize.c /^void vsize_seek(vsize_s *v, void *vfd, const long new_offset)$/;" f +vsize_stat ./src/generate/vsize.c /^void vsize_stat(vsize_s *v, const char *path)$/;" f +vsize_unlink ./src/generate/vsize.c /^void vsize_unlink(vsize_s *v, const char *path)$/;" f +vsize_write ./src/generate/vsize.c /^void vsize_write(vsize_s *v, void *vfd, const char *path, const int bytes)$/;" f +wd_base ./src/options.h /^ char *wd_base; \/**< The working directory base *\/$/;" m struct:options_s_ +whence ./src/generate/gtask.h /^ long whence; \/**< Whence *\/$/;" m struct:gtask_s_ +worker ./src/replay/rtask.h /^ void *worker; \/* The responsible worker object *\/$/;" m struct:rtask_s_ +worker ./src/replay/rthread.h /^ void *worker; \/**< The responsible worker object *\/$/;" m struct:rthread_s_ +write_pos ./src/datas/rbuffer.h /^ sig_atomic_t write_pos;$/;" m struct:rbuffer_s_ +writer ./src/generate/generate.h /^ struct gwriter_s_ *writer; \/**< A pointer to the writer object *\/$/;" m struct:generate_s_ typeref:struct:generate_s_::gwriter_s_ diff --git a/systemtap/Makefile b/systemtap/Makefile index f027719..49f1977 100644 --- a/systemtap/Makefile +++ b/systemtap/Makefile @@ -1,15 +1,15 @@ KERNEL ?= $(shell uname -r) PROCESSOR ?= $(shell uname -p) -DESTDIR ?= /opt/ioreplay/systemtap/$(KERNEL) +DESTDIR ?= /opt/ioriot/systemtap/$(KERNEL) UPDATEURI = http://debuginfo.centos.org/7/$(PROCESSOR) DOWNLOADIR ?= ./downloads all: prepare compile prepare: - sed 's/execname() != "stapio"/pid() == target()/' ./src/ioreplay.stp > ./src/targetedioreplay.stp - sed 's/execname() != "stapio"/execname() == "java"/' ./src/ioreplay.stp > ./src/javaioreplay.stp + sed 's/execname() != "stapio"/pid() == target()/' ./src/ioriot.stp > ./src/targetedioriot.stp + sed 's/execname() != "stapio"/execname() == "java"/' ./src/ioriot.stp > ./src/javaioriot.stp compile: @echo Crosscompiling for Kernel version $(KERNEL) - for stp in ioreplay javaioreplay targetedioreplay; do \ + for stp in ioriot javaioriot targetedioriot; do \ stap -v ./src/$$stp.stp -p 4 -r $(KERNEL) -m $$stp \ -D MAXSTRINGLEN=255 -D MAXACTION=10000 -D MAXSKIPPED=10000\ -g --suppress-time-limits --suppress-handler-errors; \ @@ -18,14 +18,14 @@ testsystemtap: stap -v -e 'probe vfs.read {printf("read performed\n"); exit()}' clean: @echo Cleaning modules - test -f ioreplay.ko && rm -v ioreplay.ko || exit 0 - test -f javaioreplay.ko && rm -v javaioreplay.ko || exit 0 - test -f targetedioreplay.ko && rm -v targetedioreplay.ko || exit 0 + test -f ioriot.ko && rm -v ioriot.ko || exit 0 + test -f javaioriot.ko && rm -v javaioriot.ko || exit 0 + test -f targetedioriot.ko && rm -v targetedioriot.ko || exit 0 install: test -d $(DESTDIR) || mkdir -p $(DESTDIR) - test -f ioreplay.ko && cp -v ioreplay.ko $(DESTDIR)/ || exit 0 - test -f javaioreplay.ko && cp -v javaioreplay.ko $(DESTDIR)/ || exit 0 - test -f targetedioreplay.ko && cp -v targetedioreplay.ko $(DESTDIR)/ || exit 0 + test -f ioriot.ko && cp -v ioriot.ko $(DESTDIR)/ || exit 0 + test -f javaioriot.ko && cp -v javaioriot.ko $(DESTDIR)/ || exit 0 + test -f targetedioriot.ko && cp -v targetedioriot.ko $(DESTDIR)/ || exit 0 uninstall: test ! -z "$(DESTDIR)" && test -d $(DESTDIR)/ && find $(DESTDIR) -name \*.ko -delete || exit 0 deinstall: uninstall diff --git a/systemtap/src/ioreplay.stp b/systemtap/src/ioreplay.stp deleted file mode 100644 index c4b4f0b..0000000 --- a/systemtap/src/ioreplay.stp +++ /dev/null @@ -1,591 +0,0 @@ -#!/usr/bin/env stap - -# Copyright 2018 Mimecast Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used to capture I/O on a Linux based system in order to replay -# the same I/O via the ioreplay command line utility. -# -# The tool will generate one line per captured I/O syscall to a .capture log -# file. -# -# The key/value separator is ';:,', example line: -# -# t=1509989598023;:,i=17159:17358;:,o=open;:,d=167;:,p=/tmp/test;:,f=0;:,m=438;:, -# -# It may be that SystemTap will skip probes or interrupts probes in case of -# system overload. As a result we can have corrupt lines in the log. That's why -# we use a special field separator ';:,' to detect corrupt lines more robustly. -# -# The line uses the following format keys (we use many different of these, the -# only benefit over a more generic approach is to detect corrupt lines more -# easily): -# -# Format keys: -# t: Time -# i: PID:TID (process and thread ID) -# o: Operation name -# O: Offset or owner/user UID -# W: Whence -# d: File/dir descriptor -# p: File path -# P: File path 2 -# f: Flags -# m: Mode -# b: Bytes -# c: Count -# s: Return status -# t: Optional text -# F: FCNTL command -# G: FCNTL arg or user group UID -# T: Optional text (debugging purpose only) -# a: Address -# A: Address 2 -# - -# Return the full qualified version of path -function absolute_path (path) { - # Is it already a full qualified path? - if (substr(path,0,1) == "/") { - return path; - } - - # Look into the in Kernel task structure to look up the corresponding - # mount point and directory entry... - tc = task_current() - pwd_dentry = @cast(tc, "task_struct")->fs->pwd->dentry - pwd_mnt = @cast(tc, "task_struct")->fs->pwd->mnt - - # Construct a full qualified path from it! - return task_dentry_path(tc, pwd_dentry, pwd_mnt) . "/" . path; -} - -probe syscall.open.return, syscall.openat.return { - if (execname() != "stapio") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,m=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - $return, - absolute_path(pathname), - @entry($flags), - @entry($mode)); - } -} - -probe syscall.lseek.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%d;:,W=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($offset), - @entry($whence), - $return); - } -} - -probe syscall.fcntl.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,F=%d;:,G=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($cmd), - @entry($arg), - $return); - } -} - -probe syscall.creat.return { - if (execname() != "stapio") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - $return, - absolute_path(pathname), - @entry($mode)); - } -} - -probe syscall.write.return, syscall.writev.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.unlink.return { - if(execname() != "stapio") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.unlinkat.return { - if(execname() != "stapio") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($dfd), - absolute_path(pathname), - @entry($flag), - $return); - } -} - -probe syscall.rename.return, syscall.renameat.return, syscall.renameat2.return { - if(execname() != "stapio") { - oldname = user_string(@entry($oldname)) - newname = user_string(@entry($newname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,P=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(oldname), - absolute_path(newname), - $return); - } -} - -probe syscall.read.return, syscall.readv.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.readahead.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%ld;:,O=%ld;:,c=%ld\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return, - @entry($offset), - @entry($count)); - } -} - -probe syscall.readdir.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.readlink.return { - if(execname() != "stapio") { - pathname = user_string(@entry($path)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.readlinkat.return { - if(execname() != "stapio") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.fdatasync.return, syscall.fsync.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.sync_file_range.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%ld;:,b=%ld;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($offset), - @entry($nbytes), - $return); - } -} - -probe syscall.sync.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - $return); - } -} - -probe syscall.syncfs.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.close.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.getdents.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,c=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($count), - $return); - } -} - -probe syscall.mkdir.return { - if(execname() != "stapio") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($mode), - $return); - } -} - -probe syscall.rmdir.return { - if(execname() != "stapio") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.mkdirat.return { - if(execname() != "stapio") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($dfd), - absolute_path(pathname), - @entry($mode), - $return); - } -} - -probe syscall.stat.return { - if(execname() != "stapio") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.statfs.return, syscall.statfs64.return { - if(execname() != "stapio") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.fstatfs.return, syscall.fstatfs64.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.lstat.return { - if(execname() != "stapio") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.fstat.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.fstatat.return { - if(execname() != "stapio") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,p=%s;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($dfd), - absolute_path(pathname), - @entry($flag), - $return); - } -} - -probe syscall.chmod.return, syscall.fchmodat.return { - if(execname() != "stapio") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($mode), - $return); - } -} - -probe syscall.fchmod.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($mode), - $return); - } -} - -probe syscall.chown.return, syscall.chown16.return, - syscall.lchown.return, syscall.lchown16.return { - if(execname() != "stapio") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($user), - @entry($group), - $return); - } -} - -probe syscall.fchown.return, syscall.fchown16.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,O=%d;:,G=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($user), - @entry($group), - $return); - } -} - -probe syscall.fchownat.return { - pathname = user_string(@entry($filename)) - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($user), - @entry($group), - @entry($flag), - $return); - } -} - -probe syscall.mmap2.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,m=%d;:,f=%d;:,d=%d;:,O=%ld;:,A=%ld;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($addr), - @entry($len), - @entry($prot), - @entry($flags), - @entry($fd), - @entry($pgoff), - $return); - } -} - -probe syscall.mremap.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,A=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($new_addr), - @entry($addr), - @entry($new_len), - @entry($flags), - $return); - } -} - -probe syscall.munmap.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($addr), - @entry($len), - $return); - } -} - -probe syscall.msync.return { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($start), - @entry($len), - @entry($flags), - $return); - } -} - -probe syscall.exit_group { - if(execname() != "stapio") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name); - } -} - -# Stop probing after 1h (for safety) -probe timer.s(3600) { - exit(); -} - -# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/systemtap/src/ioriot.stp b/systemtap/src/ioriot.stp new file mode 100644 index 0000000..9ea6a68 --- /dev/null +++ b/systemtap/src/ioriot.stp @@ -0,0 +1,591 @@ +#!/usr/bin/env stap + +# Copyright 2018 Mimecast Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used to capture I/O on a Linux based system in order to replay +# the same I/O via the ioriot command line utility. +# +# The tool will generate one line per captured I/O syscall to a .capture log +# file. +# +# The key/value separator is ';:,', example line: +# +# t=1509989598023;:,i=17159:17358;:,o=open;:,d=167;:,p=/tmp/test;:,f=0;:,m=438;:, +# +# It may be that SystemTap will skip probes or interrupts probes in case of +# system overload. As a result we can have corrupt lines in the log. That's why +# we use a special field separator ';:,' to detect corrupt lines more robustly. +# +# The line uses the following format keys (we use many different of these, the +# only benefit over a more generic approach is to detect corrupt lines more +# easily): +# +# Format keys: +# t: Time +# i: PID:TID (process and thread ID) +# o: Operation name +# O: Offset or owner/user UID +# W: Whence +# d: File/dir descriptor +# p: File path +# P: File path 2 +# f: Flags +# m: Mode +# b: Bytes +# c: Count +# s: Return status +# t: Optional text +# F: FCNTL command +# G: FCNTL arg or user group UID +# T: Optional text (debugging purpose only) +# a: Address +# A: Address 2 +# + +# Return the full qualified version of path +function absolute_path (path) { + # Is it already a full qualified path? + if (substr(path,0,1) == "/") { + return path; + } + + # Look into the in Kernel task structure to look up the corresponding + # mount point and directory entry... + tc = task_current() + pwd_dentry = @cast(tc, "task_struct")->fs->pwd->dentry + pwd_mnt = @cast(tc, "task_struct")->fs->pwd->mnt + + # Construct a full qualified path from it! + return task_dentry_path(tc, pwd_dentry, pwd_mnt) . "/" . path; +} + +probe syscall.open.return, syscall.openat.return { + if (execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($flags), + @entry($mode)); + } +} + +probe syscall.lseek.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%d;:,W=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($whence), + $return); + } +} + +probe syscall.fcntl.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,F=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($cmd), + @entry($arg), + $return); + } +} + +probe syscall.creat.return { + if (execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($mode)); + } +} + +probe syscall.write.return, syscall.writev.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.unlink.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.unlinkat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.rename.return, syscall.renameat.return, syscall.renameat2.return { + if(execname() != "stapio") { + oldname = user_string(@entry($oldname)) + newname = user_string(@entry($newname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,P=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(oldname), + absolute_path(newname), + $return); + } +} + +probe syscall.read.return, syscall.readv.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readahead.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%ld;:,O=%ld;:,c=%ld\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return, + @entry($offset), + @entry($count)); + } +} + +probe syscall.readdir.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readlink.return { + if(execname() != "stapio") { + pathname = user_string(@entry($path)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.readlinkat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fdatasync.return, syscall.fsync.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.sync_file_range.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($nbytes), + $return); + } +} + +probe syscall.sync.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return); + } +} + +probe syscall.syncfs.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.close.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.getdents.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,c=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($count), + $return); + } +} + +probe syscall.mkdir.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.rmdir.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.mkdirat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.stat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.statfs.return, syscall.statfs64.return { + if(execname() != "stapio") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstatfs.return, syscall.fstatfs64.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.lstat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstat.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.fstatat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.chmod.return, syscall.fchmodat.return { + if(execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.fchmod.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($mode), + $return); + } +} + +probe syscall.chown.return, syscall.chown16.return, + syscall.lchown.return, syscall.lchown16.return { + if(execname() != "stapio") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchown.return, syscall.fchown16.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchownat.return { + pathname = user_string(@entry($filename)) + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + @entry($flag), + $return); + } +} + +probe syscall.mmap2.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,m=%d;:,f=%d;:,d=%d;:,O=%ld;:,A=%ld;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + @entry($prot), + @entry($flags), + @entry($fd), + @entry($pgoff), + $return); + } +} + +probe syscall.mremap.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,A=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($new_addr), + @entry($addr), + @entry($new_len), + @entry($flags), + $return); + } +} + +probe syscall.munmap.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + $return); + } +} + +probe syscall.msync.return { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($start), + @entry($len), + @entry($flags), + $return); + } +} + +probe syscall.exit_group { + if(execname() != "stapio") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name); + } +} + +# Stop probing after 1h (for safety) +probe timer.s(3600) { + exit(); +} + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/systemtap/src/javaioreplay.stp b/systemtap/src/javaioreplay.stp deleted file mode 100644 index 4fca0ef..0000000 --- a/systemtap/src/javaioreplay.stp +++ /dev/null @@ -1,591 +0,0 @@ -#!/usr/bin/env stap - -# Copyright 2018 Mimecast Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used to capture I/O on a Linux based system in order to replay -# the same I/O via the ioreplay command line utility. -# -# The tool will generate one line per captured I/O syscall to a .capture log -# file. -# -# The key/value separator is ';:,', example line: -# -# t=1509989598023;:,i=17159:17358;:,o=open;:,d=167;:,p=/tmp/test;:,f=0;:,m=438;:, -# -# It may be that SystemTap will skip probes or interrupts probes in case of -# system overload. As a result we can have corrupt lines in the log. That's why -# we use a special field separator ';:,' to detect corrupt lines more robustly. -# -# The line uses the following format keys (we use many different of these, the -# only benefit over a more generic approach is to detect corrupt lines more -# easily): -# -# Format keys: -# t: Time -# i: PID:TID (process and thread ID) -# o: Operation name -# O: Offset or owner/user UID -# W: Whence -# d: File/dir descriptor -# p: File path -# P: File path 2 -# f: Flags -# m: Mode -# b: Bytes -# c: Count -# s: Return status -# t: Optional text -# F: FCNTL command -# G: FCNTL arg or user group UID -# T: Optional text (debugging purpose only) -# a: Address -# A: Address 2 -# - -# Return the full qualified version of path -function absolute_path (path) { - # Is it already a full qualified path? - if (substr(path,0,1) == "/") { - return path; - } - - # Look into the in Kernel task structure to look up the corresponding - # mount point and directory entry... - tc = task_current() - pwd_dentry = @cast(tc, "task_struct")->fs->pwd->dentry - pwd_mnt = @cast(tc, "task_struct")->fs->pwd->mnt - - # Construct a full qualified path from it! - return task_dentry_path(tc, pwd_dentry, pwd_mnt) . "/" . path; -} - -probe syscall.open.return, syscall.openat.return { - if (execname() == "java") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,m=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - $return, - absolute_path(pathname), - @entry($flags), - @entry($mode)); - } -} - -probe syscall.lseek.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%d;:,W=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($offset), - @entry($whence), - $return); - } -} - -probe syscall.fcntl.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,F=%d;:,G=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($cmd), - @entry($arg), - $return); - } -} - -probe syscall.creat.return { - if (execname() == "java") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - $return, - absolute_path(pathname), - @entry($mode)); - } -} - -probe syscall.write.return, syscall.writev.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.unlink.return { - if(execname() == "java") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.unlinkat.return { - if(execname() == "java") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($dfd), - absolute_path(pathname), - @entry($flag), - $return); - } -} - -probe syscall.rename.return, syscall.renameat.return, syscall.renameat2.return { - if(execname() == "java") { - oldname = user_string(@entry($oldname)) - newname = user_string(@entry($newname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,P=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(oldname), - absolute_path(newname), - $return); - } -} - -probe syscall.read.return, syscall.readv.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.readahead.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%ld;:,O=%ld;:,c=%ld\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return, - @entry($offset), - @entry($count)); - } -} - -probe syscall.readdir.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.readlink.return { - if(execname() == "java") { - pathname = user_string(@entry($path)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.readlinkat.return { - if(execname() == "java") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.fdatasync.return, syscall.fsync.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.sync_file_range.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%ld;:,b=%ld;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($offset), - @entry($nbytes), - $return); - } -} - -probe syscall.sync.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - $return); - } -} - -probe syscall.syncfs.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.close.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.getdents.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,c=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($count), - $return); - } -} - -probe syscall.mkdir.return { - if(execname() == "java") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($mode), - $return); - } -} - -probe syscall.rmdir.return { - if(execname() == "java") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.mkdirat.return { - if(execname() == "java") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($dfd), - absolute_path(pathname), - @entry($mode), - $return); - } -} - -probe syscall.stat.return { - if(execname() == "java") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.statfs.return, syscall.statfs64.return { - if(execname() == "java") { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.fstatfs.return, syscall.fstatfs64.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.lstat.return { - if(execname() == "java") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.fstat.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.fstatat.return { - if(execname() == "java") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,p=%s;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($dfd), - absolute_path(pathname), - @entry($flag), - $return); - } -} - -probe syscall.chmod.return, syscall.fchmodat.return { - if(execname() == "java") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($mode), - $return); - } -} - -probe syscall.fchmod.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($mode), - $return); - } -} - -probe syscall.chown.return, syscall.chown16.return, - syscall.lchown.return, syscall.lchown16.return { - if(execname() == "java") { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($user), - @entry($group), - $return); - } -} - -probe syscall.fchown.return, syscall.fchown16.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,O=%d;:,G=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($user), - @entry($group), - $return); - } -} - -probe syscall.fchownat.return { - pathname = user_string(@entry($filename)) - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($user), - @entry($group), - @entry($flag), - $return); - } -} - -probe syscall.mmap2.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,m=%d;:,f=%d;:,d=%d;:,O=%ld;:,A=%ld;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($addr), - @entry($len), - @entry($prot), - @entry($flags), - @entry($fd), - @entry($pgoff), - $return); - } -} - -probe syscall.mremap.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,A=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($new_addr), - @entry($addr), - @entry($new_len), - @entry($flags), - $return); - } -} - -probe syscall.munmap.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($addr), - @entry($len), - $return); - } -} - -probe syscall.msync.return { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($start), - @entry($len), - @entry($flags), - $return); - } -} - -probe syscall.exit_group { - if(execname() == "java") { - printf("t=%d;:,i=%d:%d;:,o=%s;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name); - } -} - -# Stop probing after 1h (for safety) -probe timer.s(3600) { - exit(); -} - -# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/systemtap/src/javaioriot.stp b/systemtap/src/javaioriot.stp new file mode 100644 index 0000000..3943971 --- /dev/null +++ b/systemtap/src/javaioriot.stp @@ -0,0 +1,591 @@ +#!/usr/bin/env stap + +# Copyright 2018 Mimecast Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used to capture I/O on a Linux based system in order to replay +# the same I/O via the ioriot command line utility. +# +# The tool will generate one line per captured I/O syscall to a .capture log +# file. +# +# The key/value separator is ';:,', example line: +# +# t=1509989598023;:,i=17159:17358;:,o=open;:,d=167;:,p=/tmp/test;:,f=0;:,m=438;:, +# +# It may be that SystemTap will skip probes or interrupts probes in case of +# system overload. As a result we can have corrupt lines in the log. That's why +# we use a special field separator ';:,' to detect corrupt lines more robustly. +# +# The line uses the following format keys (we use many different of these, the +# only benefit over a more generic approach is to detect corrupt lines more +# easily): +# +# Format keys: +# t: Time +# i: PID:TID (process and thread ID) +# o: Operation name +# O: Offset or owner/user UID +# W: Whence +# d: File/dir descriptor +# p: File path +# P: File path 2 +# f: Flags +# m: Mode +# b: Bytes +# c: Count +# s: Return status +# t: Optional text +# F: FCNTL command +# G: FCNTL arg or user group UID +# T: Optional text (debugging purpose only) +# a: Address +# A: Address 2 +# + +# Return the full qualified version of path +function absolute_path (path) { + # Is it already a full qualified path? + if (substr(path,0,1) == "/") { + return path; + } + + # Look into the in Kernel task structure to look up the corresponding + # mount point and directory entry... + tc = task_current() + pwd_dentry = @cast(tc, "task_struct")->fs->pwd->dentry + pwd_mnt = @cast(tc, "task_struct")->fs->pwd->mnt + + # Construct a full qualified path from it! + return task_dentry_path(tc, pwd_dentry, pwd_mnt) . "/" . path; +} + +probe syscall.open.return, syscall.openat.return { + if (execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($flags), + @entry($mode)); + } +} + +probe syscall.lseek.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%d;:,W=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($whence), + $return); + } +} + +probe syscall.fcntl.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,F=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($cmd), + @entry($arg), + $return); + } +} + +probe syscall.creat.return { + if (execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($mode)); + } +} + +probe syscall.write.return, syscall.writev.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.unlink.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.unlinkat.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.rename.return, syscall.renameat.return, syscall.renameat2.return { + if(execname() == "java") { + oldname = user_string(@entry($oldname)) + newname = user_string(@entry($newname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,P=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(oldname), + absolute_path(newname), + $return); + } +} + +probe syscall.read.return, syscall.readv.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readahead.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%ld;:,O=%ld;:,c=%ld\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return, + @entry($offset), + @entry($count)); + } +} + +probe syscall.readdir.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readlink.return { + if(execname() == "java") { + pathname = user_string(@entry($path)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.readlinkat.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fdatasync.return, syscall.fsync.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.sync_file_range.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($nbytes), + $return); + } +} + +probe syscall.sync.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return); + } +} + +probe syscall.syncfs.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.close.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.getdents.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,c=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($count), + $return); + } +} + +probe syscall.mkdir.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.rmdir.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.mkdirat.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.stat.return { + if(execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.statfs.return, syscall.statfs64.return { + if(execname() == "java") { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstatfs.return, syscall.fstatfs64.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.lstat.return { + if(execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstat.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.fstatat.return { + if(execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.chmod.return, syscall.fchmodat.return { + if(execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.fchmod.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($mode), + $return); + } +} + +probe syscall.chown.return, syscall.chown16.return, + syscall.lchown.return, syscall.lchown16.return { + if(execname() == "java") { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchown.return, syscall.fchown16.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchownat.return { + pathname = user_string(@entry($filename)) + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + @entry($flag), + $return); + } +} + +probe syscall.mmap2.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,m=%d;:,f=%d;:,d=%d;:,O=%ld;:,A=%ld;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + @entry($prot), + @entry($flags), + @entry($fd), + @entry($pgoff), + $return); + } +} + +probe syscall.mremap.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,A=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($new_addr), + @entry($addr), + @entry($new_len), + @entry($flags), + $return); + } +} + +probe syscall.munmap.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + $return); + } +} + +probe syscall.msync.return { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($start), + @entry($len), + @entry($flags), + $return); + } +} + +probe syscall.exit_group { + if(execname() == "java") { + printf("t=%d;:,i=%d:%d;:,o=%s;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name); + } +} + +# Stop probing after 1h (for safety) +probe timer.s(3600) { + exit(); +} + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/systemtap/src/targetedioreplay.stp b/systemtap/src/targetedioreplay.stp deleted file mode 100644 index 139a89b..0000000 --- a/systemtap/src/targetedioreplay.stp +++ /dev/null @@ -1,591 +0,0 @@ -#!/usr/bin/env stap - -# Copyright 2018 Mimecast Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used to capture I/O on a Linux based system in order to replay -# the same I/O via the ioreplay command line utility. -# -# The tool will generate one line per captured I/O syscall to a .capture log -# file. -# -# The key/value separator is ';:,', example line: -# -# t=1509989598023;:,i=17159:17358;:,o=open;:,d=167;:,p=/tmp/test;:,f=0;:,m=438;:, -# -# It may be that SystemTap will skip probes or interrupts probes in case of -# system overload. As a result we can have corrupt lines in the log. That's why -# we use a special field separator ';:,' to detect corrupt lines more robustly. -# -# The line uses the following format keys (we use many different of these, the -# only benefit over a more generic approach is to detect corrupt lines more -# easily): -# -# Format keys: -# t: Time -# i: PID:TID (process and thread ID) -# o: Operation name -# O: Offset or owner/user UID -# W: Whence -# d: File/dir descriptor -# p: File path -# P: File path 2 -# f: Flags -# m: Mode -# b: Bytes -# c: Count -# s: Return status -# t: Optional text -# F: FCNTL command -# G: FCNTL arg or user group UID -# T: Optional text (debugging purpose only) -# a: Address -# A: Address 2 -# - -# Return the full qualified version of path -function absolute_path (path) { - # Is it already a full qualified path? - if (substr(path,0,1) == "/") { - return path; - } - - # Look into the in Kernel task structure to look up the corresponding - # mount point and directory entry... - tc = task_current() - pwd_dentry = @cast(tc, "task_struct")->fs->pwd->dentry - pwd_mnt = @cast(tc, "task_struct")->fs->pwd->mnt - - # Construct a full qualified path from it! - return task_dentry_path(tc, pwd_dentry, pwd_mnt) . "/" . path; -} - -probe syscall.open.return, syscall.openat.return { - if (pid() == target()) { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,m=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - $return, - absolute_path(pathname), - @entry($flags), - @entry($mode)); - } -} - -probe syscall.lseek.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%d;:,W=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($offset), - @entry($whence), - $return); - } -} - -probe syscall.fcntl.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,F=%d;:,G=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($cmd), - @entry($arg), - $return); - } -} - -probe syscall.creat.return { - if (pid() == target()) { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - $return, - absolute_path(pathname), - @entry($mode)); - } -} - -probe syscall.write.return, syscall.writev.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.unlink.return { - if(pid() == target()) { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.unlinkat.return { - if(pid() == target()) { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($dfd), - absolute_path(pathname), - @entry($flag), - $return); - } -} - -probe syscall.rename.return, syscall.renameat.return, syscall.renameat2.return { - if(pid() == target()) { - oldname = user_string(@entry($oldname)) - newname = user_string(@entry($newname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,P=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(oldname), - absolute_path(newname), - $return); - } -} - -probe syscall.read.return, syscall.readv.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.readahead.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%ld;:,O=%ld;:,c=%ld\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return, - @entry($offset), - @entry($count)); - } -} - -probe syscall.readdir.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.readlink.return { - if(pid() == target()) { - pathname = user_string(@entry($path)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.readlinkat.return { - if(pid() == target()) { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.fdatasync.return, syscall.fsync.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.sync_file_range.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%ld;:,b=%ld;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($offset), - @entry($nbytes), - $return); - } -} - -probe syscall.sync.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - $return); - } -} - -probe syscall.syncfs.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.close.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.getdents.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,c=%d;:,b=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($count), - $return); - } -} - -probe syscall.mkdir.return { - if(pid() == target()) { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($mode), - $return); - } -} - -probe syscall.rmdir.return { - if(pid() == target()) { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.mkdirat.return { - if(pid() == target()) { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($dfd), - absolute_path(pathname), - @entry($mode), - $return); - } -} - -probe syscall.stat.return { - if(pid() == target()) { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.statfs.return, syscall.statfs64.return { - if(pid() == target()) { - pathname = user_string(@entry($pathname)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.fstatfs.return, syscall.fstatfs64.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.lstat.return { - if(pid() == target()) { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - $return); - } -} - -probe syscall.fstat.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - $return); - } -} - -probe syscall.fstatat.return { - if(pid() == target()) { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,p=%s;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($dfd), - absolute_path(pathname), - @entry($flag), - $return); - } -} - -probe syscall.chmod.return, syscall.fchmodat.return { - if(pid() == target()) { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($mode), - $return); - } -} - -probe syscall.fchmod.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,m=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($mode), - $return); - } -} - -probe syscall.chown.return, syscall.chown16.return, - syscall.lchown.return, syscall.lchown16.return { - if(pid() == target()) { - pathname = user_string(@entry($filename)) - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($user), - @entry($group), - $return); - } -} - -probe syscall.fchown.return, syscall.fchown16.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,O=%d;:,G=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($fd), - @entry($user), - @entry($group), - $return); - } -} - -probe syscall.fchownat.return { - pathname = user_string(@entry($filename)) - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - absolute_path(pathname), - @entry($user), - @entry($group), - @entry($flag), - $return); - } -} - -probe syscall.mmap2.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,m=%d;:,f=%d;:,d=%d;:,O=%ld;:,A=%ld;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($addr), - @entry($len), - @entry($prot), - @entry($flags), - @entry($fd), - @entry($pgoff), - $return); - } -} - -probe syscall.mremap.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,A=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($new_addr), - @entry($addr), - @entry($new_len), - @entry($flags), - $return); - } -} - -probe syscall.munmap.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($addr), - @entry($len), - $return); - } -} - -probe syscall.msync.return { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name, - @entry($start), - @entry($len), - @entry($flags), - $return); - } -} - -probe syscall.exit_group { - if(pid() == target()) { - printf("t=%d;:,i=%d:%d;:,o=%s;:,\n", - gettimeofday_ms(), - pid(), - tid(), - name); - } -} - -# Stop probing after 1h (for safety) -probe timer.s(3600) { - exit(); -} - -# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/systemtap/src/targetedioriot.stp b/systemtap/src/targetedioriot.stp new file mode 100644 index 0000000..f72dc66 --- /dev/null +++ b/systemtap/src/targetedioriot.stp @@ -0,0 +1,591 @@ +#!/usr/bin/env stap + +# Copyright 2018 Mimecast Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used to capture I/O on a Linux based system in order to replay +# the same I/O via the ioriot command line utility. +# +# The tool will generate one line per captured I/O syscall to a .capture log +# file. +# +# The key/value separator is ';:,', example line: +# +# t=1509989598023;:,i=17159:17358;:,o=open;:,d=167;:,p=/tmp/test;:,f=0;:,m=438;:, +# +# It may be that SystemTap will skip probes or interrupts probes in case of +# system overload. As a result we can have corrupt lines in the log. That's why +# we use a special field separator ';:,' to detect corrupt lines more robustly. +# +# The line uses the following format keys (we use many different of these, the +# only benefit over a more generic approach is to detect corrupt lines more +# easily): +# +# Format keys: +# t: Time +# i: PID:TID (process and thread ID) +# o: Operation name +# O: Offset or owner/user UID +# W: Whence +# d: File/dir descriptor +# p: File path +# P: File path 2 +# f: Flags +# m: Mode +# b: Bytes +# c: Count +# s: Return status +# t: Optional text +# F: FCNTL command +# G: FCNTL arg or user group UID +# T: Optional text (debugging purpose only) +# a: Address +# A: Address 2 +# + +# Return the full qualified version of path +function absolute_path (path) { + # Is it already a full qualified path? + if (substr(path,0,1) == "/") { + return path; + } + + # Look into the in Kernel task structure to look up the corresponding + # mount point and directory entry... + tc = task_current() + pwd_dentry = @cast(tc, "task_struct")->fs->pwd->dentry + pwd_mnt = @cast(tc, "task_struct")->fs->pwd->mnt + + # Construct a full qualified path from it! + return task_dentry_path(tc, pwd_dentry, pwd_mnt) . "/" . path; +} + +probe syscall.open.return, syscall.openat.return { + if (pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($flags), + @entry($mode)); + } +} + +probe syscall.lseek.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%d;:,W=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($whence), + $return); + } +} + +probe syscall.fcntl.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,F=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($cmd), + @entry($arg), + $return); + } +} + +probe syscall.creat.return { + if (pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return, + absolute_path(pathname), + @entry($mode)); + } +} + +probe syscall.write.return, syscall.writev.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.unlink.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.unlinkat.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.rename.return, syscall.renameat.return, syscall.renameat2.return { + if(pid() == target()) { + oldname = user_string(@entry($oldname)) + newname = user_string(@entry($newname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,P=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(oldname), + absolute_path(newname), + $return); + } +} + +probe syscall.read.return, syscall.readv.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readahead.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,b=%ld;:,O=%ld;:,c=%ld\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return, + @entry($offset), + @entry($count)); + } +} + +probe syscall.readdir.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.readlink.return { + if(pid() == target()) { + pathname = user_string(@entry($path)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.readlinkat.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fdatasync.return, syscall.fsync.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.sync_file_range.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,O=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($offset), + @entry($nbytes), + $return); + } +} + +probe syscall.sync.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + $return); + } +} + +probe syscall.syncfs.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.close.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.getdents.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,c=%d;:,b=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($count), + $return); + } +} + +probe syscall.mkdir.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.rmdir.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.mkdirat.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.stat.return { + if(pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.statfs.return, syscall.statfs64.return { + if(pid() == target()) { + pathname = user_string(@entry($pathname)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstatfs.return, syscall.fstatfs64.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.lstat.return { + if(pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + $return); + } +} + +probe syscall.fstat.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + $return); + } +} + +probe syscall.fstatat.return { + if(pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,p=%s;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($dfd), + absolute_path(pathname), + @entry($flag), + $return); + } +} + +probe syscall.chmod.return, syscall.fchmodat.return { + if(pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($mode), + $return); + } +} + +probe syscall.fchmod.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%d;:,m=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($mode), + $return); + } +} + +probe syscall.chown.return, syscall.chown16.return, + syscall.lchown.return, syscall.lchown16.return { + if(pid() == target()) { + pathname = user_string(@entry($filename)) + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchown.return, syscall.fchown16.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,d=%ld;:,O=%d;:,G=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($fd), + @entry($user), + @entry($group), + $return); + } +} + +probe syscall.fchownat.return { + pathname = user_string(@entry($filename)) + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,p=%s;:,O=%d;:,G=%d;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + absolute_path(pathname), + @entry($user), + @entry($group), + @entry($flag), + $return); + } +} + +probe syscall.mmap2.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,m=%d;:,f=%d;:,d=%d;:,O=%ld;:,A=%ld;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + @entry($prot), + @entry($flags), + @entry($fd), + @entry($pgoff), + $return); + } +} + +probe syscall.mremap.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,A=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($new_addr), + @entry($addr), + @entry($new_len), + @entry($flags), + $return); + } +} + +probe syscall.munmap.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($addr), + @entry($len), + $return); + } +} + +probe syscall.msync.return { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,a=%ld;:,b=%ld;:,f=%d;:,s=%d;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name, + @entry($start), + @entry($len), + @entry($flags), + $return); + } +} + +probe syscall.exit_group { + if(pid() == target()) { + printf("t=%d;:,i=%d:%d;:,o=%s;:,\n", + gettimeofday_ms(), + pid(), + tid(), + name); + } +} + +# Stop probing after 1h (for safety) +probe timer.s(3600) { + exit(); +} + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 -- cgit v1.2.3