From ce2de3cf93356f697ac5455ef16f4cf7a5d5a365 Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 8 Jun 2024 12:00:00 +0200 Subject: [PATCH 001/435] new release --- debian/changelog | 6 ++++++ release.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 048a08c..30b1af6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +reaction (1.4.1-1) stable; urgency=low + + * New upstream release. + + -- ppom Sat, 08 Jun 2024 20:27:11 +0000 + reaction (1.4.0-1) stable; urgency=low * New upstream release. diff --git a/release.sh b/release.sh index c7bb13b..7f958fa 100755 --- a/release.sh +++ b/release.sh @@ -37,4 +37,4 @@ curl \ 'https://framagit.org/api/v4/projects/90566/releases' \ --data "$DATA" -make clean +sudo make clean From 77974dd9385eaf924f60d067137d859570a4866c Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 24 Jun 2024 12:00:00 +0200 Subject: [PATCH 002/435] Switch to hardcoded version value fix #77 #95 --- Makefile | 2 +- app/main.go | 4 ++-- reaction.go | 7 ++----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 1226d7b..bab181f 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ nft46: helpers_c/nft46.c $(CC) -s -static helpers_c/nft46.c -o nft46 reaction: app/* reaction.go go.mod go.sum - CGO_ENABLED=0 go build -buildvcs=false -ldflags "-s -X main.version=`git tag --sort=v:refname | tail -n1` -X main.commit=`git rev-parse --short HEAD`" + CGO_ENABLED=0 go build -buildvcs=false -ldflags "-s" reaction_%-1_amd64.deb: apt-get -qq -y update diff --git a/app/main.go b/app/main.go index 8d3d284..e5bfb4c 100644 --- a/app/main.go +++ b/app/main.go @@ -128,7 +128,7 @@ on the ` + bold + `wiki` + reset + `: https://reaction.ppom.me //go:embed example.yml var exampleConf string -func Main(version, commit string) { +func Main(version string) { if len(os.Args) <= 1 { logger.Fatalln("No argument provided. Try `reaction help`") basicUsage() @@ -140,7 +140,7 @@ func Main(version, commit string) { basicUsage() case "version", "-v", "--version": - fmt.Printf("reaction version %v commit %v\n", version, commit) + fmt.Printf("reaction version %v\n", version) case "example-conf": subCommandParse(f, 0) diff --git a/reaction.go b/reaction.go index 860a129..37d6746 100644 --- a/reaction.go +++ b/reaction.go @@ -5,12 +5,9 @@ import ( ) func main() { - app.Main(version, commit) + app.Main(version) } var ( - // Must be passed when building - // go build -ldflags "-X app.commit XXX -X app.version XXX" - version string - commit string + version = "v1.4.2" ) From 1c45104676241fbdc40547fc19ccc0e9e3b1c79a Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 24 Jun 2024 12:00:00 +0200 Subject: [PATCH 003/435] typo fix in YAML example --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5458322..cfd8f83 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ both are extensions of JSON, so JSON is transitively supported. ```yaml patterns: - ip: '(([ 0-9 ]{1,3}\.){3}[0-9]{1,3})|([0-9a-fA-F:]{2,90})' + ip: + regex: '(([0-9]{1,3}\.){3}[0-9]{1,3})|([0-9a-fA-F:]{2,90})' start: - [ 'ip46tables', '-w', '-N', 'reaction' ] From 6d1aaabbb707ec95b207e781d92582a2f8d5adcd Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 24 Jun 2024 12:00:00 +0200 Subject: [PATCH 004/435] Recommend reading more before starting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cfd8f83..7b477e0 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ both are extensions of JSON, so JSON is transitively supported. - See [reaction.yml](./app/example.yml) or [reaction.jsonnet](./config/example.jsonnet) for a fully explained reference - See [server.jsonnet](./config/server.jsonnet) for a real-world configuration - See [reaction.example.service](./config/reaction.example.service) for a systemd service file -- This quick example shows what's needed to prevent brute force attacks on an ssh server: +- This minimal example shows what's needed to prevent brute force attacks on an ssh server (please take a look at more complete references before starting 🆙):
From 51669a87e3026552ce0c6baa85709fe5510e8dbf Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 26 Jun 2024 12:00:00 +0200 Subject: [PATCH 005/435] Emphasis on the wiki. --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b477e0..620216a 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,9 @@ It will execute `iptables` when detecting ipv4, `ip6tables` when detecting ipv6 ## Wiki -You'll find more ressources, service configurations, etc. on the [Wiki](https://reaction.ppom.me)! +You'll find more ressources, service configurations, etc. on [the wiki](https://reaction.ppom.me)! + +We recommend that you read the ***Good Practices*** chapters before starting. ## Installation From fba60f36f0c088270a1d2c4cab9066a379e1fce2 Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 12 Oct 2024 12:00:00 +0200 Subject: [PATCH 006/435] Update on project rewrite status --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 620216a..8efeffb 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,15 @@ A common usage is to scan ssh and webserver logs, and to ban hosts that cause mu 🚧 This program hasn't received external audit. however, it already works well on my servers 🚧 +## Current project status + +Hey, it may look like nothing is happening here... That's because reaction is actively being rewritten in rust, +which happens in the [rust](https://framagit.org/ppom/reaction/-/tree/rust) branch! + +I'll soon™️ publish a v2.0.0-rc1 version, which will be the rust rewrite of reaction. +It will be in feature parity with the Go version, and breaking changes should be small. +More information on this when the release lands! + ## Rationale I was using the honorable fail2ban since quite a long time, but i was a bit frustrated by its cpu consumption From b39868b228aaeddf89ed3517a47d00fb37cf9258 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 20 May 2024 12:00:00 +0200 Subject: [PATCH 007/435] initial rust commit --- rust/.gitignore | 1 + rust/Cargo.lock | 588 +++++++++++++++++++++++++++++++++++++++++++++ rust/Cargo.toml | 13 + rust/src/cli.rs | 171 +++++++++++++ rust/src/client.rs | 29 +++ rust/src/config.rs | 3 + rust/src/daemon.rs | 7 + rust/src/main.rs | 56 +++++ 8 files changed, 868 insertions(+) create mode 100644 rust/.gitignore create mode 100644 rust/Cargo.lock create mode 100644 rust/Cargo.toml create mode 100644 rust/src/cli.rs create mode 100644 rust/src/client.rs create mode 100644 rust/src/config.rs create mode 100644 rust/src/daemon.rs create mode 100644 rust/src/main.rs diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..8d564ae --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,588 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "annotate-snippets" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaf7e9dfbb6ab22c82e473cd1a8a7bd313c19a5b7e40970f3d89ef5a5c9e81e" +dependencies = [ + "unicode-width", + "yansi-term", +] + +[[package]] +name = "anstream" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" + +[[package]] +name = "anstyle-parse" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "clap" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_complete" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79504325bf38b10165b02e89b4347300f855f273c4cb30c4a3209e6583275e" +dependencies = [ + "clap", +] + +[[package]] +name = "clap_derive" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.65", +] + +[[package]] +name = "clap_lex" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" + +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jrsonnet-evaluator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fee60406dac44a01b37e120b43adb062047251e195db15392b825f6bdc948712" +dependencies = [ + "annotate-snippets", + "base64", + "bincode", + "jrsonnet-gc", + "jrsonnet-interner", + "jrsonnet-parser", + "jrsonnet-stdlib", + "jrsonnet-types", + "md5", + "pathdiff", + "rustc-hash", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "jrsonnet-gc" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68da8bc2f00117b1373bb8877af03b1d391e4c4800e6585d7279e5b99c919dde" +dependencies = [ + "jrsonnet-gc-derive", +] + +[[package]] +name = "jrsonnet-gc-derive" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcba9c387b64b054f06cc4d724905296e21edeeb7506847f3299117a2d92d12" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "jrsonnet-interner" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ff75843e778244f3476800e6f492950a6ecee1d9308019764983d311620bf9" +dependencies = [ + "jrsonnet-gc", + "rustc-hash", + "serde", +] + +[[package]] +name = "jrsonnet-parser" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daaad69b21c1dba904f3bb1640e02f8f60c5cd4eae8c9bc035b38a83324cdf45" +dependencies = [ + "jrsonnet-gc", + "jrsonnet-interner", + "peg", + "serde", + "unescape", +] + +[[package]] +name = "jrsonnet-stdlib" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840295ba3a8d65bf71e57a57acbef4c77d11c543739cfded27f91feef239f80e" + +[[package]] +name = "jrsonnet-types" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909b38de99711ef357a514af1ed112e6cf411ab8b204cc92507b6e219e65fe5c" +dependencies = [ + "jrsonnet-gc", + "peg", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" + +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + +[[package]] +name = "peg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c0b841ea54f523f7aa556956fbd293bcbe06f2e67d2eb732b7278aaf1d166a" +dependencies = [ + "peg-macros", + "peg-runtime", +] + +[[package]] +name = "peg-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aa52829b8decbef693af90202711348ab001456803ba2a98eb4ec8fb70844c" +dependencies = [ + "peg-runtime", + "proc-macro2", + "quote", +] + +[[package]] +name = "peg-runtime" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c719dcf55f09a3a7e764c6649ab594c18a177e3599c467983cdf644bfc0a4088" + +[[package]] +name = "proc-macro2" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "reaction" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "clap_complete", + "jrsonnet-evaluator", + "regex", +] + +[[package]] +name = "regex" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.202" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.202" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.65", +] + +[[package]] +name = "serde_json" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2863d96a84c6439701d7a38f9de935ec562c8832cc55d1dde0f513b52fad106" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + +[[package]] +name = "thiserror" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.65", +] + +[[package]] +name = "unescape" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccb97dac3243214f8d8507998906ca3e2e0b900bf9bf4870477f125b82e68f6e" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-width" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "yansi-term" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1" +dependencies = [ + "winapi", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..b8988b0 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "reaction" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.86" +clap = { version = "4.5.4", features = ["derive"] } +clap_complete = "4.5.2" +jrsonnet-evaluator = "0.4.2" +regex = "1.10.4" diff --git a/rust/src/cli.rs b/rust/src/cli.rs new file mode 100644 index 0000000..f38de94 --- /dev/null +++ b/rust/src/cli.rs @@ -0,0 +1,171 @@ +use std::fmt; +use std::path::PathBuf; + +use clap::{Parser, Subcommand, ValueEnum}; +use regex::Regex; + +#[derive(Parser)] +#[clap(version)] +#[command(name = "reaction")] +#[command(version = "2.0")] +#[command( + about = "Scan logs and take action", + long_about = "A daemon that scans program outputs for repeated patterns, and takes action. +Aims at being more versatile and flexible than fail2ban, while being faster and having simpler configuration. + +See usage examples, service configurations and good practices +on the wiki: https://reaction.ppom.me +" +)] +pub struct Cli { + #[clap(subcommand)] + pub command: Command, +} + +#[derive(Subcommand)] +pub enum Command { + /// Start reaction daemon + Start { + /// configuration file in json, jsonnet or yaml format. required. + #[clap(short = 'c', long)] + config: PathBuf, + + /// minimum log level to show + #[clap(short = 'l', long, default_value_t = LogLevel::INFO, ignore_case = true)] + loglevel: LogLevel, + + /// path to the client-daemon communication socket + #[clap(short = 's', long, default_value = "/run/reaction/reaction.sock")] + socket: PathBuf, + }, + + /// Show current matches and actions + #[command( + long_about = "Show current matches and which actions are still to be run (e.g. know what is currently banned" + )] + Show { + /// path to the client-daemon communication socket + #[clap(short = 's', long, default_value = "/run/reaction/reaction.sock")] + socket: PathBuf, + + /// how to format output: json or yaml. + #[clap(short = 's', long, default_value_t = Format::YAML)] + format: Format, + + /// only show items related to this STREAM[.FILTER] + #[clap(short = 'l', long, value_name = "STREAM[.FILTER]")] + limit: Option, + + /// only show items matching PATTERN regex + #[clap(short = 'p', long, value_name = "PATTERN")] + pattern: Option, + + /// only show items matching name=PATTERN regex + #[clap(value_parser = parse_named_regex, value_name = "NAME=PATTERN")] + patterns: Vec, + }, + + /// Remove a target from reaction (e.g. unban) + #[command( + long_about = "Remove currently active matches and run currently pending actions for the specified TARGET. (e.g. unban) +Then prints the flushed matches and actions." + )] + Flush { + /// path to the client-daemon communication socket + #[clap(short = 's', long, default_value = "/run/reaction/reaction.sock")] + socket: PathBuf, + + /// how to format output: json or yaml. + #[clap(short = 's', long, default_value_t = Format::YAML)] + format: Format, + + /// only show items related to this STREAM[.FILTER] + #[clap(short = 'l', long, value_name = "STREAM[.FILTER]")] + limit: Option, + + /// only show items matching PATTERN regex + #[clap(short = 'p', long, value_name = "PATTERN")] + pattern: Option, + + /// only show items matching name=PATTERN regex + #[clap(value_parser = parse_named_regex, value_name = "NAME=PATTERN")] + patterns: Vec, + }, + + /// Test a regex + #[command( + name = "test-regex", + long_about = "Test a REGEX against one LINE, or against standard input. +Giving a configuration file permits to use its patterns in REGEX." + )] + TestRegex { + /// configuration file in json, jsonnet or yaml format. required. + #[clap(short = 'c', long)] + config: PathBuf, + + /// Regex to test + #[clap(value_name = "REGEX")] + regex: String, + + /// Line to be tested + #[clap(value_name = "LINE")] + line: Option, + }, +} + +// Enums + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, ValueEnum)] +pub enum Format { + JSON, + YAML, +} + +impl fmt::Display for Format { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Format::JSON => write!(f, "json"), + Format::YAML => write!(f, "yaml"), + } + } +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, ValueEnum)] +pub enum LogLevel { + DEBUG, + INFO, + WARN, + ERROR, + FATAL, +} + +impl fmt::Display for LogLevel { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + LogLevel::DEBUG => write!(f, "DEBUG"), + LogLevel::INFO => write!(f, "INFO"), + LogLevel::WARN => write!(f, "WARN"), + LogLevel::ERROR => write!(f, "ERROR"), + LogLevel::FATAL => write!(f, "FATAL"), + } + } +} + +// Structs + +#[derive(Clone, Debug)] +pub struct NamedRegex { + pub regex: Regex, + pub name: String, +} + +fn parse_named_regex(s: &str) -> Result { + let (name, v) = s + .split_once("=") + .ok_or("When given as a positional argument, a pattern must be prefixed with a name, ex: ip=192.168.0.1")?; + let regex = Regex::new(v).map_err(|err| format!("{}", err))?; + Ok(NamedRegex { + regex, + name: name.to_string(), + }) +} diff --git a/rust/src/client.rs b/rust/src/client.rs new file mode 100644 index 0000000..4657026 --- /dev/null +++ b/rust/src/client.rs @@ -0,0 +1,29 @@ +use std::path::PathBuf; + +use regex::Regex; + +use crate::cli::{Format, NamedRegex}; + +pub fn show( + socket: &PathBuf, + format: Format, + limit: &Option, + pattern: &Option, + patterns: &Vec, +) { + println!("show {:?} {:?} {:?} {:?} {:?}", socket, format, limit, pattern, patterns); +} + +pub fn flush( + socket: &PathBuf, + format: Format, + limit: &Option, + pattern: &Option, + patterns: &Vec, +) { + println!("flush {:?} {:?} {:?} {:?} {:?}", socket, format, limit, pattern, patterns); +} + +pub fn test_regex(config_path: &PathBuf, regex: &String, line: &Option) { + println!("test-regex {:?} {:?} {:?} ", config_path, regex, line); +} diff --git a/rust/src/config.rs b/rust/src/config.rs new file mode 100644 index 0000000..5d36281 --- /dev/null +++ b/rust/src/config.rs @@ -0,0 +1,3 @@ + +pub struct Config { +} diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs new file mode 100644 index 0000000..cbdf94a --- /dev/null +++ b/rust/src/daemon.rs @@ -0,0 +1,7 @@ +use std::path::PathBuf; + +use crate::cli::LogLevel; + +pub fn daemon(config_path: &PathBuf, loglevel: LogLevel, socket: &PathBuf) { + println!("daemon {:?} {:?} {:?}", config_path, loglevel, socket); +} diff --git a/rust/src/main.rs b/rust/src/main.rs new file mode 100644 index 0000000..d72f02d --- /dev/null +++ b/rust/src/main.rs @@ -0,0 +1,56 @@ +use clap::Parser; +use regex::Regex; + +mod cli; +mod config; +mod daemon; +mod client; + +use cli::{Cli, Command}; +use daemon::daemon; +use client::{show, flush, test_regex}; + +fn main() { + cli::NamedRegex { + regex: Regex::new(".").unwrap(), + name: "test".to_string(), + }; + + let cli = Cli::parse(); + + match cli.command { + Command::Start { + config, + loglevel, + socket, + } => { + // + daemon(&config, loglevel, &socket); + } + Command::Show { + socket, + format, + limit, + pattern, + patterns, + } => { + show(&socket, format, &limit, &pattern, &patterns) + } + Command::Flush { + socket, + format, + limit, + pattern, + patterns, + } => { + flush(&socket, format, &limit, &pattern, &patterns) + } + Command::TestRegex { + config, + regex, + line, + } => { + test_regex(&config, ®ex, &line) + } + } +} From 582ba571dc7a808cff3285f125a1bc14fac2bcbe Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 22 May 2024 12:00:00 +0200 Subject: [PATCH 008/435] WIP configuration setup --- rust/Cargo.lock | 33 +++- rust/Cargo.toml | 3 + rust/TODO | 1 + rust/example.json | 122 +++++++++++++++ rust/src/config.rs | 302 +++++++++++++++++++++++++++++++++++++ rust/src/daemon.rs | 8 +- rust/src/main.rs | 13 +- rust/src/parse_duration.rs | 29 ++++ 8 files changed, 505 insertions(+), 6 deletions(-) create mode 100644 rust/TODO create mode 100644 rust/example.json create mode 100644 rust/src/parse_duration.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8d564ae..48a2201 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -152,6 +152,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "is_terminal_polyfill" version = "1.70.0" @@ -247,6 +253,12 @@ dependencies = [ "peg", ] +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + [[package]] name = "md5" version = "0.7.0" @@ -259,6 +271,16 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "pathdiff" version = "0.2.1" @@ -318,7 +340,10 @@ dependencies = [ "clap", "clap_complete", "jrsonnet-evaluator", + "num_cpus", "regex", + "serde", + "serde_json", ] [[package]] @@ -364,18 +389,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.202" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.202" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b8988b0..83f4b37 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -10,4 +10,7 @@ anyhow = "1.0.86" clap = { version = "4.5.4", features = ["derive"] } clap_complete = "4.5.2" jrsonnet-evaluator = "0.4.2" +num_cpus = "1.16.0" regex = "1.10.4" +serde = { version = "1.0.203", features = ["derive"] } +serde_json = "1.0.117" diff --git a/rust/TODO b/rust/TODO new file mode 100644 index 0000000..439d9e3 --- /dev/null +++ b/rust/TODO @@ -0,0 +1 @@ +cargo clippy diff --git a/rust/example.json b/rust/example.json new file mode 100644 index 0000000..8591b6d --- /dev/null +++ b/rust/example.json @@ -0,0 +1,122 @@ +{ + "concurrency": 0, + "patterns": { + "ip": { + "ignore": [ + "127.0.0.1", + "::1" + ], + "regex": "(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))" + } + }, + "start": [ + [ + "ip46tables", + "-w", + "-N", + "reaction" + ], + [ + "ip46tables", + "-w", + "-I", + "INPUT", + "-p", + "all", + "-j", + "reaction" + ], + [ + "ip46tables", + "-w", + "-I", + "FORWARD", + "-p", + "all", + "-j", + "reaction" + ] + ], + "stop": [ + [ + "ip46tables", + "-w", + "-D", + "INPUT", + "-p", + "all", + "-j", + "reaction" + ], + [ + "ip46tables", + "-w", + "-D", + "FORWARD", + "-p", + "all", + "-j", + "reaction" + ], + [ + "ip46tables", + "-w", + "-F", + "reaction" + ], + [ + "ip46tables", + "-w", + "-X", + "reaction" + ] + ], + "streams": { + "ssh": { + "cmd": [ + "journalctl", + "-n0", + "-fu", + "sshd.service" + ], + "filters": { + "failedlogin": { + "actions": { + "ban": { + "cmd": [ + "ip46tables", + "-w", + "-A", + "reaction", + "-s", + "", + "-j", + "DROP" + ] + }, + "unban": { + "after": "48h", + "cmd": [ + "ip46tables", + "-w", + "-D", + "reaction", + "-s", + "", + "-j", + "DROP" + ] + } + }, + "regex": [ + "authentication failure;.*rhost=", + "Failed password for .* from ", + "Connection (reset|closed) by (authenticating|invalid) user .* " + ], + "retry": 3, + "retryperiod": "6h" + } + } + } + } +} diff --git a/rust/src/config.rs b/rust/src/config.rs index 5d36281..c1ad99d 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -1,3 +1,305 @@ +#![allow(dead_code)] +use std::collections::BTreeMap; +use std::fs::File; +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{anyhow, Context, Result}; +use regex::Regex; +use serde::Deserialize; + +use crate::parse_duration::parse_duration; + +pub type Patterns = BTreeMap; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Config { + concurrency: usize, + patterns: Patterns, + streams: BTreeMap, + #[serde(default)] + start: Vec>, + #[serde(default)] + stop: Vec>, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Pattern { + regex: String, + #[serde(default)] + ignore: Vec, + + #[serde(default, rename = "ignoreregex")] + ignore_regex: Vec, + #[serde(skip)] + compiled_ignore_regex: Vec, + + #[serde(skip)] + name: String, + #[serde(skip)] + name_with_braces: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Stream { + cmd: Vec, + filters: BTreeMap, + + #[serde(skip)] + name: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Filter { + actions: BTreeMap, + #[serde(skip)] + longuest_action_duration: Duration, + + regex: Vec, + #[serde(skip)] + compiled_regex: Vec, + #[serde(skip)] + patterns: Vec, + + retry: Option, + #[serde(rename = "retryperiod")] + retry_period: Option, + retry_duration: Option, + + #[serde(skip)] + name: String, + #[serde(skip)] + stream_name: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Action { + cmd: Vec, + + // TODO one shot time deserialization + after: Option, + #[serde(skip)] + after_duration: Option, + + #[serde(rename = "onexit", default = "set_false")] + on_exit: bool, + + #[serde(skip)] + name: String, + #[serde(skip)] + filter_name: String, + #[serde(skip)] + stream_name: String, +} + +fn set_0() -> u32 { + 0 +} + +fn set_false() -> bool { + false +} + +impl Config { + pub fn setup(&mut self) -> Result<()> { + self._setup() + .or_else(|msg| Err(anyhow!("Bad configuration: {}", msg))) + } + + pub fn _setup(&mut self) -> Result<(), String> { + if self.concurrency == 0 { + self.concurrency = num_cpus::get(); + } + + for (key, value) in &mut self.patterns { + value.setup(key)?; + } + + if self.streams.len() == 0 { + return Err("no streams configured".into()); + } + + for (key, value) in &mut self.streams { + value.setup(key, &self.patterns)?; + } + Ok(()) + } +} + +impl Pattern { + pub fn setup(&mut self, name: &String) -> Result<(), String> { + self._setup(name) + .or_else(|msg| Err(format!("pattern {}: {}", name, msg))) + } + + pub fn _setup(&mut self, name: &String) -> Result<(), String> { + self.name = name.clone(); + self.name_with_braces = format!("<{}>", name); + + if self.regex.len() == 0 { + return Err("regex is empty".into()); + } + let compiled = + Regex::new(&format!("^{}$", self.regex)).or_else(|err| Err(err.to_string()))?; + + self.regex = format!("(?P<{}>{})", self.name, self.regex); + + for ignore in &self.ignore { + if !compiled.is_match(&ignore) { + return Err(format!( + "ignore '{}' doesn't match pattern. It should be fixed or removed.", + ignore, + )); + } + } + + for ignore_regex in &self.ignore_regex { + let compiled_ignore = Regex::new(&format!("^{}$", ignore_regex)).or_else(|err| { + Err(format!( + "ignoreregex '{}': {}", + ignore_regex, + err.to_string() + )) + })?; + + self.compiled_ignore_regex.push(compiled_ignore); + } + + Ok(()) + } +} + +impl Stream { + fn setup(&mut self, name: &String, patterns: &Patterns) -> Result<(), String> { + self._setup(name, patterns) + .or_else(|msg| Err(format!("stream {}: {}", name, msg))) + } + + fn _setup(&mut self, name: &String, patterns: &Patterns) -> Result<(), String> { + self.name = name.clone(); + + if self.name.contains('.') { + return Err("character '.' is not allowed in stream name".into()); + } + + if self.filters.len() == 0 { + return Err("no filters configured".into()); + } + + for (key, value) in &mut self.filters { + value.setup(&name, key, patterns)?; + } + + Ok(()) + } +} + +impl Filter { + fn setup( + &mut self, + stream_name: &String, + name: &String, + patterns: &Patterns, + ) -> Result<(), String> { + self._setup(stream_name, name, patterns) + .or_else(|msg| Err(format!("filter {}: {}", name, msg))) + } + + fn _setup( + &mut self, + stream_name: &String, + name: &String, + patterns: &Patterns, + ) -> Result<(), String> { + self.stream_name = stream_name.clone(); + self.name = name.clone(); + + if self.name.contains('.') { + return Err("character '.' is not allowed in filter name".into()); + } + + if self.retry.is_some() != self.retry_period.is_some() { + return Err("retry and retryperiod must be specified altogether".into()); + } + + if self.retry.is_some_and(|r| r < 2) { + return Err("retry has been specified but is < 2".into()); + } + + if let Some(retry_period) = &self.retry_period { + self.retry_duration = + Some(parse_duration(retry_period).or_else(|err| { + Err(format!("failed to parse retry time: {}", err.to_string())) + })?); + } + + if self.regex.len() == 0 { + return Err("no regex configured".into()); + } + + for regex in &self.regex { + for (_pattern_name, pattern) in patterns { + if regex.contains(&pattern.name_with_braces) { + // TODO + } + } + } + + if self.actions.len() == 0 { + return Err("no actions configured".into()); + } + + for (key, value) in &mut self.actions { + value.setup(&stream_name, &name, key, patterns)?; + } + + Ok(()) + } +} + +impl Action { + fn setup( + &mut self, + stream_name: &String, + filter_name: &String, + name: &String, + patterns: &Patterns, + ) -> Result<(), String> { + self._setup(stream_name, filter_name, name, patterns) + .or_else(|msg| Err(format!("action {}: {}", name, msg))) + } + fn _setup( + &mut self, + stream_name: &String, + filter_name: &String, + name: &String, + _patterns: &Patterns, + ) -> Result<(), String> { + self.stream_name = stream_name.clone(); + self.filter_name = filter_name.clone(); + self.name = name.clone(); + + if self.name.contains('.') { + return Err("character '.' is not allowed in filter name".into()); + } + // for (_key, value) in &mut self.actions { + // value.setup()?; + // } + Ok(()) + } +} + +pub fn config_from_file(path: &PathBuf) -> Result { + let file = File::open(&path) + .with_context(|| format!("Failed to read configuration file: {}", path.display()))?; + let mut config: Config = + serde_json::from_reader(file).context("Failed to parse configuration file")?; + config.setup()?; + return Ok(config); } diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index cbdf94a..ad4eb74 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -1,7 +1,13 @@ use std::path::PathBuf; -use crate::cli::LogLevel; +use crate::{cli::LogLevel, config}; pub fn daemon(config_path: &PathBuf, loglevel: LogLevel, socket: &PathBuf) { println!("daemon {:?} {:?} {:?}", config_path, loglevel, socket); + let config = config::config_from_file(config_path); + if let Ok(config) = config { + dbg!(config); + } else if let Err(err) = config { + dbg!(err); + } } diff --git a/rust/src/main.rs b/rust/src/main.rs index d72f02d..9a2e2a1 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -1,9 +1,21 @@ +#![warn( + missing_docs, + clippy::unwrap_used, + clippy::panic, + clippy::unimplemented, + clippy::todo, + clippy::undocumented_unsafe_blocks +)] +#![forbid(unsafe_code)] + +//! Hey it's the doc! use clap::Parser; use regex::Regex; mod cli; mod config; mod daemon; +mod parse_duration; mod client; use cli::{Cli, Command}; @@ -24,7 +36,6 @@ fn main() { loglevel, socket, } => { - // daemon(&config, loglevel, &socket); } Command::Show { diff --git a/rust/src/parse_duration.rs b/rust/src/parse_duration.rs new file mode 100644 index 0000000..f77eff2 --- /dev/null +++ b/rust/src/parse_duration.rs @@ -0,0 +1,29 @@ +use anyhow::{anyhow, Result}; + +use std::time::Duration; + +pub fn parse_duration(d: &str) -> Result { + let chars = d.trim().as_bytes(); + let mut value = 0; + let mut i = 0; + while i < chars.len() && chars[i] < '0' as u8 && chars[i] > '9' as u8 { + value = value * 10 + (chars[i] - '0' as u8) as u32; + i += 1; + } + if value == 0 { + return Err(anyhow!("duration '{}' doesn't start with digits", d)); + } + let ok_secs = + |mul: u32| -> Result { Ok(Duration::from_secs(mul as u64 * value as u64)) }; + + match d[i..].trim() { + "s" | "sec" | "secs" | "second" | "seconds" => ok_secs(1), + "m" | "min" | "mins" | "minute" | "minutes" => ok_secs(60), + "h" | "hour" | "hours" => ok_secs(60 * 60), + "d" | "day" | "days" => ok_secs(24 * 60 * 60), + unit => Err(anyhow!( + "unit {} not recognised. must be one of s/sec/seconds, m/min/minutes, h/hours, d/days", + unit + )), + } +} From 9ed760295205449f338d96df87971e77a201dfee Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 28 May 2024 12:00:00 +0200 Subject: [PATCH 009/435] Complete config setup. Tests for duration parsing. TODO: Tests for config setup. --- rust/src/config.rs | 63 ++++++++++++++++++++++++++++++-------- rust/src/parse_duration.rs | 60 +++++++++++++++++++++++++++++------- 2 files changed, 100 insertions(+), 23 deletions(-) diff --git a/rust/src/config.rs b/rust/src/config.rs index c1ad99d..5552ea9 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -1,6 +1,7 @@ #![allow(dead_code)] -use std::collections::BTreeMap; +use std::cmp::Ordering; +use std::collections::{BTreeMap, BTreeSet}; use std::fs::File; use std::path::PathBuf; use std::time::Duration; @@ -25,7 +26,7 @@ pub struct Config { stop: Vec>, } -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Pattern { regex: String, @@ -63,8 +64,10 @@ pub struct Filter { regex: Vec, #[serde(skip)] compiled_regex: Vec, + // We want patterns to be ordered + // This is necessary when using matches which contain multiple patterns #[serde(skip)] - patterns: Vec, + patterns: BTreeSet, retry: Option, #[serde(rename = "retryperiod")] @@ -175,6 +178,26 @@ impl Pattern { } } +// This is required to be added to a BTreeSet +// We compare Patterns by their names, which are unique. +// This is enforced by Patterns' names coming from their keys in a BTreeMap in Config +impl Ord for Pattern { + fn cmp(&self, other: &Self) -> Ordering { + self.name.cmp(&other.name) + } +} +impl PartialOrd for Pattern { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Eq for Pattern {} +impl PartialEq for Pattern { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} + impl Stream { fn setup(&mut self, name: &String, patterns: &Patterns) -> Result<(), String> { self._setup(name, patterns) @@ -225,7 +248,7 @@ impl Filter { } if self.retry.is_some() != self.retry_period.is_some() { - return Err("retry and retryperiod must be specified altogether".into()); + return Err("retry and retryperiod must be specified one with each other".into()); } if self.retry.is_some_and(|r| r < 2) { @@ -244,11 +267,15 @@ impl Filter { } for regex in &self.regex { + let mut regex_buf = regex.clone(); for (_pattern_name, pattern) in patterns { if regex.contains(&pattern.name_with_braces) { - // TODO + self.patterns.insert(pattern.clone()); } + regex_buf = regex_buf.replacen(&pattern.name_with_braces, &pattern.regex, 1) } + let compiled = Regex::new(®ex_buf).or_else(|err| Err(err.to_string()))?; + self.compiled_regex.push(compiled); } if self.actions.len() == 0 { @@ -256,9 +283,16 @@ impl Filter { } for (key, value) in &mut self.actions { - value.setup(&stream_name, &name, key, patterns)?; + value.setup(&stream_name, &name, key)?; } + self.longuest_action_duration = self + .actions + .iter() + .map(|(_, v)| v.after_duration.unwrap_or(Duration::from_secs(0))) + .max() + .unwrap(); // safe because we already tested that actions.len() > 0 + Ok(()) } } @@ -269,9 +303,8 @@ impl Action { stream_name: &String, filter_name: &String, name: &String, - patterns: &Patterns, ) -> Result<(), String> { - self._setup(stream_name, filter_name, name, patterns) + self._setup(stream_name, filter_name, name) .or_else(|msg| Err(format!("action {}: {}", name, msg))) } fn _setup( @@ -279,7 +312,6 @@ impl Action { stream_name: &String, filter_name: &String, name: &String, - _patterns: &Patterns, ) -> Result<(), String> { self.stream_name = stream_name.clone(); self.filter_name = filter_name.clone(); @@ -288,9 +320,16 @@ impl Action { if self.name.contains('.') { return Err("character '.' is not allowed in filter name".into()); } - // for (_key, value) in &mut self.actions { - // value.setup()?; - // } + + if let Some(after) = &self.after { + self.after_duration = + Some(parse_duration(after).or_else(|err| { + Err(format!("failed to parse after time: {}", err.to_string())) + })?); + } else if self.on_exit { + return Err("cannot have `onexit: true`, without an `after` directive".into()); + } + Ok(()) } } diff --git a/rust/src/parse_duration.rs b/rust/src/parse_duration.rs index f77eff2..0f307e4 100644 --- a/rust/src/parse_duration.rs +++ b/rust/src/parse_duration.rs @@ -1,29 +1,67 @@ -use anyhow::{anyhow, Result}; - use std::time::Duration; -pub fn parse_duration(d: &str) -> Result { - let chars = d.trim().as_bytes(); +pub fn parse_duration(d: &str) -> Result { + let d_trimmed = d.trim(); + let chars = d_trimmed.as_bytes(); let mut value = 0; let mut i = 0; - while i < chars.len() && chars[i] < '0' as u8 && chars[i] > '9' as u8 { + while i < chars.len() && chars[i].is_ascii_digit() { value = value * 10 + (chars[i] - '0' as u8) as u32; i += 1; } - if value == 0 { - return Err(anyhow!("duration '{}' doesn't start with digits", d)); + if i == 0 { + return Err(format!("duration '{}' doesn't start with digits", d)); } - let ok_secs = - |mul: u32| -> Result { Ok(Duration::from_secs(mul as u64 * value as u64)) }; + let ok_secs = |mul: u32| -> Result { + Ok(Duration::from_secs(mul as u64 * value as u64)) + }; - match d[i..].trim() { + match d_trimmed[i..].trim() { "s" | "sec" | "secs" | "second" | "seconds" => ok_secs(1), "m" | "min" | "mins" | "minute" | "minutes" => ok_secs(60), "h" | "hour" | "hours" => ok_secs(60 * 60), "d" | "day" | "days" => ok_secs(24 * 60 * 60), - unit => Err(anyhow!( + unit => Err(format!( "unit {} not recognised. must be one of s/sec/seconds, m/min/minutes, h/hours, d/days", unit )), } } + +#[cfg(test)] +mod tests { + + use std::time::Duration; + + use super::*; + + #[test] + fn char_conversion() { + assert_eq!('9' as u8 - '0' as u8, 9); + } + + #[test] + fn parse_duration_test() { + assert_eq!(parse_duration("1s"), Ok(Duration::from_secs(1))); + assert_eq!(parse_duration("12s"), Ok(Duration::from_secs(12))); + assert_eq!(parse_duration(" 12 secs "), Ok(Duration::from_secs(12))); + assert_eq!(parse_duration("2m"), Ok(Duration::from_secs(2 * 60))); + assert_eq!( + parse_duration("6 hours"), + Ok(Duration::from_secs(6 * 60 * 60)) + ); + assert_eq!(parse_duration("1d"), Ok(Duration::from_secs(24 * 60 * 60))); + assert_eq!( + parse_duration("365d"), + Ok(Duration::from_secs(365 * 24 * 60 * 60)) + ); + + assert_eq!(parse_duration("d 3").is_err(), true); + assert_eq!(parse_duration("d3").is_err(), true); + assert_eq!(parse_duration("3da").is_err(), true); + assert_eq!(parse_duration("3_days").is_err(), true); + assert_eq!(parse_duration("_3d").is_err(), true); + assert_eq!(parse_duration("3 3d").is_err(), true); + assert_eq!(parse_duration("3.3d").is_err(), true); + } +} From 8f132df4ac61b0ba2506fc63bf9ac6ff22b6403b Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 8 Jun 2024 12:00:00 +0200 Subject: [PATCH 010/435] yaml support --- rust/Cargo.lock | 42 ++++++++++++++++++++++++++++++++++++++++++ rust/Cargo.toml | 1 + rust/src/config.rs | 36 ++++++++++++++++++++++++++++++++---- 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 48a2201..b945d8c 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -146,6 +146,18 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "heck" version = "0.5.0" @@ -158,6 +170,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.0" @@ -344,6 +366,7 @@ dependencies = [ "regex", "serde", "serde_json", + "serde_yaml", ] [[package]] @@ -418,6 +441,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "strsim" version = "0.11.1" @@ -502,6 +538,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "utf8parse" version = "0.2.1" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 83f4b37..18d5bbe 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -14,3 +14,4 @@ num_cpus = "1.16.0" regex = "1.10.4" serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" +serde_yaml = "0.9.34" diff --git a/rust/src/config.rs b/rust/src/config.rs index 5552ea9..0a54faa 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -334,11 +334,39 @@ impl Action { } } +enum Format { + YAML, + JSON, + JSONnet, +} + pub fn config_from_file(path: &PathBuf) -> Result { - let file = File::open(&path) - .with_context(|| format!("Failed to read configuration file: {}", path.display()))?; - let mut config: Config = - serde_json::from_reader(file).context("Failed to parse configuration file")?; + _config_from_file(path).with_context(|| anyhow!("Configuration file {}:", path.display())) +} +fn _config_from_file(path: &PathBuf) -> Result { + let extension = path.extension().map(|ex| ex.to_str()).flatten().ok_or(anyhow!("no file extension"))?; + + let format = match extension { + "yaml" | "yml" => Format::YAML, + "json" => Format::JSON, + "jsonnet" => Format::JSONnet, + _ => { + return Err(anyhow!( + "extension {} is not recognized. Must be json, jsonnet, yml or yaml.", + extension + )) + } + }; + + let file = File::open(&path)?; + + let mut config: Config = match format { + Format::JSON => serde_json::from_reader(file)?, + Format::YAML => serde_yaml::from_reader(file)?, + Format::JSONnet => return Err(anyhow!("JSONnet is not implemented yet")), + }; + config.setup()?; + return Ok(config); } From 217c75abd50ea5cea68ea8fa645c77b271c7403e Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 9 Jun 2024 12:00:00 +0200 Subject: [PATCH 011/435] jsonnet support --- rust/src/config.rs | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/rust/src/config.rs b/rust/src/config.rs index 0a54faa..0daa68a 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -344,7 +344,11 @@ pub fn config_from_file(path: &PathBuf) -> Result { _config_from_file(path).with_context(|| anyhow!("Configuration file {}:", path.display())) } fn _config_from_file(path: &PathBuf) -> Result { - let extension = path.extension().map(|ex| ex.to_str()).flatten().ok_or(anyhow!("no file extension"))?; + let extension = path + .extension() + .map(|ex| ex.to_str()) + .flatten() + .ok_or(anyhow!("no file extension"))?; let format = match extension { "yaml" | "yml" => Format::YAML, @@ -358,15 +362,39 @@ fn _config_from_file(path: &PathBuf) -> Result { } }; - let file = File::open(&path)?; - let mut config: Config = match format { - Format::JSON => serde_json::from_reader(file)?, - Format::YAML => serde_yaml::from_reader(file)?, - Format::JSONnet => return Err(anyhow!("JSONnet is not implemented yet")), + Format::JSON => serde_json::from_reader(File::open(&path)?)?, + Format::YAML => serde_yaml::from_reader(File::open(&path)?)?, + Format::JSONnet => serde_json::from_str(&jsonnet::from_path(&path)?)?, }; config.setup()?; return Ok(config); } + +mod jsonnet { + use std::path::PathBuf; + + use anyhow::{anyhow, Result}; + use jrsonnet_evaluator::{error::LocError, EvaluationState, FileImportResolver}; + + pub fn from_path(path: &PathBuf) -> Result { + let state = EvaluationState::default(); + state.with_stdlib(); + state.set_import_resolver(Box::new(FileImportResolver::default())); + // state.set_import_resolver(Box::new(FileImportResolver { + // library_paths: Vec::new(), + // })); + + match evaluate(path, &state) { + Ok(val) => Ok(val), + Err(err) => Err(anyhow!("{}", state.stringify_err(&err))), + } + } + fn evaluate(path: &PathBuf, state: &EvaluationState) -> Result { + let val = state.evaluate_file_raw(path)?; + let result = state.manifest(val)?; + Ok(result.to_string()) + } +} From f298f285a32fcc18f3e8998acec9e95a6c4c1214 Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 8 Jun 2024 12:00:00 +0200 Subject: [PATCH 012/435] optional concurrency falling back to default --- rust/src/config.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/src/config.rs b/rust/src/config.rs index 0daa68a..dfa38d3 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -17,9 +17,13 @@ pub type Patterns = BTreeMap; #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Config { - concurrency: usize, patterns: Patterns, + streams: BTreeMap, + + #[serde(default = "num_cpus::get")] + concurrency: usize, + #[serde(default)] start: Vec>, #[serde(default)] @@ -101,10 +105,6 @@ pub struct Action { stream_name: String, } -fn set_0() -> u32 { - 0 -} - fn set_false() -> bool { false } From 3ce3b5ea7ad7910477c5aa5826a12914160c11cc Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 10 Jun 2024 12:00:00 +0200 Subject: [PATCH 013/435] refacto: move struct to their own module files --- rust/src/action.rs | 68 +++++++++++ rust/src/config.rs | 285 +------------------------------------------- rust/src/filter.rs | 107 +++++++++++++++++ rust/src/main.rs | 5 + rust/src/pattern.rs | 86 +++++++++++++ rust/src/stream.rs | 41 +++++++ 6 files changed, 309 insertions(+), 283 deletions(-) create mode 100644 rust/src/action.rs create mode 100644 rust/src/filter.rs create mode 100644 rust/src/pattern.rs create mode 100644 rust/src/stream.rs diff --git a/rust/src/action.rs b/rust/src/action.rs new file mode 100644 index 0000000..ef718a2 --- /dev/null +++ b/rust/src/action.rs @@ -0,0 +1,68 @@ +use std::time::Duration; + +use serde::Deserialize; + +use crate::parse_duration::parse_duration; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Action { + cmd: Vec, + + // TODO one shot time deserialization + after: Option, + #[serde(skip)] + pub after_duration: Option, + + #[serde(rename = "onexit", default = "set_false")] + on_exit: bool, + + #[serde(skip)] + name: String, + #[serde(skip)] + filter_name: String, + #[serde(skip)] + stream_name: String, +} + +fn set_false() -> bool { + false +} + +impl Action { + pub fn setup( + &mut self, + stream_name: &String, + filter_name: &String, + name: &String, + ) -> Result<(), String> { + self._setup(stream_name, filter_name, name) + .or_else(|msg| Err(format!("action {}: {}", name, msg))) + } + fn _setup( + &mut self, + stream_name: &String, + filter_name: &String, + name: &String, + ) -> Result<(), String> { + self.stream_name = stream_name.clone(); + self.filter_name = filter_name.clone(); + self.name = name.clone(); + + if self.name.contains('.') { + return Err("character '.' is not allowed in filter name".into()); + } + + if let Some(after) = &self.after { + self.after_duration = + Some(parse_duration(after).or_else(|err| { + Err(format!("failed to parse after time: {}", err.to_string())) + })?); + } else if self.on_exit { + return Err("cannot have `onexit: true`, without an `after` directive".into()); + } + + Ok(()) + } +} + diff --git a/rust/src/config.rs b/rust/src/config.rs index dfa38d3..60e2115 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -1,16 +1,13 @@ #![allow(dead_code)] -use std::cmp::Ordering; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::fs::File; use std::path::PathBuf; -use std::time::Duration; use anyhow::{anyhow, Context, Result}; -use regex::Regex; use serde::Deserialize; -use crate::parse_duration::parse_duration; +use crate::{pattern::Pattern, stream::Stream}; pub type Patterns = BTreeMap; @@ -30,85 +27,6 @@ pub struct Config { stop: Vec>, } -#[derive(Clone, Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct Pattern { - regex: String, - #[serde(default)] - ignore: Vec, - - #[serde(default, rename = "ignoreregex")] - ignore_regex: Vec, - #[serde(skip)] - compiled_ignore_regex: Vec, - - #[serde(skip)] - name: String, - #[serde(skip)] - name_with_braces: String, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct Stream { - cmd: Vec, - filters: BTreeMap, - - #[serde(skip)] - name: String, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct Filter { - actions: BTreeMap, - #[serde(skip)] - longuest_action_duration: Duration, - - regex: Vec, - #[serde(skip)] - compiled_regex: Vec, - // We want patterns to be ordered - // This is necessary when using matches which contain multiple patterns - #[serde(skip)] - patterns: BTreeSet, - - retry: Option, - #[serde(rename = "retryperiod")] - retry_period: Option, - retry_duration: Option, - - #[serde(skip)] - name: String, - #[serde(skip)] - stream_name: String, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct Action { - cmd: Vec, - - // TODO one shot time deserialization - after: Option, - #[serde(skip)] - after_duration: Option, - - #[serde(rename = "onexit", default = "set_false")] - on_exit: bool, - - #[serde(skip)] - name: String, - #[serde(skip)] - filter_name: String, - #[serde(skip)] - stream_name: String, -} - -fn set_false() -> bool { - false -} - impl Config { pub fn setup(&mut self) -> Result<()> { self._setup() @@ -135,205 +53,6 @@ impl Config { } } -impl Pattern { - pub fn setup(&mut self, name: &String) -> Result<(), String> { - self._setup(name) - .or_else(|msg| Err(format!("pattern {}: {}", name, msg))) - } - - pub fn _setup(&mut self, name: &String) -> Result<(), String> { - self.name = name.clone(); - self.name_with_braces = format!("<{}>", name); - - if self.regex.len() == 0 { - return Err("regex is empty".into()); - } - let compiled = - Regex::new(&format!("^{}$", self.regex)).or_else(|err| Err(err.to_string()))?; - - self.regex = format!("(?P<{}>{})", self.name, self.regex); - - for ignore in &self.ignore { - if !compiled.is_match(&ignore) { - return Err(format!( - "ignore '{}' doesn't match pattern. It should be fixed or removed.", - ignore, - )); - } - } - - for ignore_regex in &self.ignore_regex { - let compiled_ignore = Regex::new(&format!("^{}$", ignore_regex)).or_else(|err| { - Err(format!( - "ignoreregex '{}': {}", - ignore_regex, - err.to_string() - )) - })?; - - self.compiled_ignore_regex.push(compiled_ignore); - } - - Ok(()) - } -} - -// This is required to be added to a BTreeSet -// We compare Patterns by their names, which are unique. -// This is enforced by Patterns' names coming from their keys in a BTreeMap in Config -impl Ord for Pattern { - fn cmp(&self, other: &Self) -> Ordering { - self.name.cmp(&other.name) - } -} -impl PartialOrd for Pattern { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} -impl Eq for Pattern {} -impl PartialEq for Pattern { - fn eq(&self, other: &Self) -> bool { - self.name == other.name - } -} - -impl Stream { - fn setup(&mut self, name: &String, patterns: &Patterns) -> Result<(), String> { - self._setup(name, patterns) - .or_else(|msg| Err(format!("stream {}: {}", name, msg))) - } - - fn _setup(&mut self, name: &String, patterns: &Patterns) -> Result<(), String> { - self.name = name.clone(); - - if self.name.contains('.') { - return Err("character '.' is not allowed in stream name".into()); - } - - if self.filters.len() == 0 { - return Err("no filters configured".into()); - } - - for (key, value) in &mut self.filters { - value.setup(&name, key, patterns)?; - } - - Ok(()) - } -} - -impl Filter { - fn setup( - &mut self, - stream_name: &String, - name: &String, - patterns: &Patterns, - ) -> Result<(), String> { - self._setup(stream_name, name, patterns) - .or_else(|msg| Err(format!("filter {}: {}", name, msg))) - } - - fn _setup( - &mut self, - stream_name: &String, - name: &String, - patterns: &Patterns, - ) -> Result<(), String> { - self.stream_name = stream_name.clone(); - self.name = name.clone(); - - if self.name.contains('.') { - return Err("character '.' is not allowed in filter name".into()); - } - - if self.retry.is_some() != self.retry_period.is_some() { - return Err("retry and retryperiod must be specified one with each other".into()); - } - - if self.retry.is_some_and(|r| r < 2) { - return Err("retry has been specified but is < 2".into()); - } - - if let Some(retry_period) = &self.retry_period { - self.retry_duration = - Some(parse_duration(retry_period).or_else(|err| { - Err(format!("failed to parse retry time: {}", err.to_string())) - })?); - } - - if self.regex.len() == 0 { - return Err("no regex configured".into()); - } - - for regex in &self.regex { - let mut regex_buf = regex.clone(); - for (_pattern_name, pattern) in patterns { - if regex.contains(&pattern.name_with_braces) { - self.patterns.insert(pattern.clone()); - } - regex_buf = regex_buf.replacen(&pattern.name_with_braces, &pattern.regex, 1) - } - let compiled = Regex::new(®ex_buf).or_else(|err| Err(err.to_string()))?; - self.compiled_regex.push(compiled); - } - - if self.actions.len() == 0 { - return Err("no actions configured".into()); - } - - for (key, value) in &mut self.actions { - value.setup(&stream_name, &name, key)?; - } - - self.longuest_action_duration = self - .actions - .iter() - .map(|(_, v)| v.after_duration.unwrap_or(Duration::from_secs(0))) - .max() - .unwrap(); // safe because we already tested that actions.len() > 0 - - Ok(()) - } -} - -impl Action { - fn setup( - &mut self, - stream_name: &String, - filter_name: &String, - name: &String, - ) -> Result<(), String> { - self._setup(stream_name, filter_name, name) - .or_else(|msg| Err(format!("action {}: {}", name, msg))) - } - fn _setup( - &mut self, - stream_name: &String, - filter_name: &String, - name: &String, - ) -> Result<(), String> { - self.stream_name = stream_name.clone(); - self.filter_name = filter_name.clone(); - self.name = name.clone(); - - if self.name.contains('.') { - return Err("character '.' is not allowed in filter name".into()); - } - - if let Some(after) = &self.after { - self.after_duration = - Some(parse_duration(after).or_else(|err| { - Err(format!("failed to parse after time: {}", err.to_string())) - })?); - } else if self.on_exit { - return Err("cannot have `onexit: true`, without an `after` directive".into()); - } - - Ok(()) - } -} - enum Format { YAML, JSON, diff --git a/rust/src/filter.rs b/rust/src/filter.rs new file mode 100644 index 0000000..bcba9e6 --- /dev/null +++ b/rust/src/filter.rs @@ -0,0 +1,107 @@ +use std::{collections::{BTreeMap, BTreeSet}, time::Duration}; + +use regex::Regex; +use serde::Deserialize; + +use crate::{action::Action, pattern::Pattern, config::Patterns, parse_duration::parse_duration}; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Filter { + actions: BTreeMap, + #[serde(skip)] + longuest_action_duration: Duration, + + regex: Vec, + #[serde(skip)] + compiled_regex: Vec, + // We want patterns to be ordered + // This is necessary when using matches which contain multiple patterns + #[serde(skip)] + patterns: BTreeSet, + + retry: Option, + #[serde(rename = "retryperiod")] + retry_period: Option, + retry_duration: Option, + + #[serde(skip)] + name: String, + #[serde(skip)] + stream_name: String, +} + + +impl Filter { + pub fn setup( + &mut self, + stream_name: &String, + name: &String, + patterns: &Patterns, + ) -> Result<(), String> { + self._setup(stream_name, name, patterns) + .or_else(|msg| Err(format!("filter {}: {}", name, msg))) + } + + fn _setup( + &mut self, + stream_name: &String, + name: &String, + patterns: &Patterns, + ) -> Result<(), String> { + self.stream_name = stream_name.clone(); + self.name = name.clone(); + + if self.name.contains('.') { + return Err("character '.' is not allowed in filter name".into()); + } + + if self.retry.is_some() != self.retry_period.is_some() { + return Err("retry and retryperiod must be specified one with each other".into()); + } + + if self.retry.is_some_and(|r| r < 2) { + return Err("retry has been specified but is < 2".into()); + } + + if let Some(retry_period) = &self.retry_period { + self.retry_duration = + Some(parse_duration(retry_period).or_else(|err| { + Err(format!("failed to parse retry time: {}", err.to_string())) + })?); + } + + if self.regex.len() == 0 { + return Err("no regex configured".into()); + } + + for regex in &self.regex { + let mut regex_buf = regex.clone(); + for (_pattern_name, pattern) in patterns { + if regex.contains(&pattern.name_with_braces) { + self.patterns.insert(pattern.clone()); + } + regex_buf = regex_buf.replacen(&pattern.name_with_braces, &pattern.regex, 1) + } + let compiled = Regex::new(®ex_buf).or_else(|err| Err(err.to_string()))?; + self.compiled_regex.push(compiled); + } + + if self.actions.len() == 0 { + return Err("no actions configured".into()); + } + + for (key, value) in &mut self.actions { + value.setup(&stream_name, &name, key)?; + } + + self.longuest_action_duration = self + .actions + .iter() + .map(|(_, v)| v.after_duration.unwrap_or(Duration::from_secs(0))) + .max() + .unwrap(); // safe because we already tested that actions.len() > 0 + + Ok(()) + } +} diff --git a/rust/src/main.rs b/rust/src/main.rs index 9a2e2a1..41b22e4 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -12,6 +12,11 @@ use clap::Parser; use regex::Regex; +mod pattern; +mod stream; +mod filter; +mod action; + mod cli; mod config; mod daemon; diff --git a/rust/src/pattern.rs b/rust/src/pattern.rs new file mode 100644 index 0000000..99a4abd --- /dev/null +++ b/rust/src/pattern.rs @@ -0,0 +1,86 @@ +use std::cmp::Ordering; + +use regex::Regex; + +use serde::Deserialize; + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Pattern { + pub regex: String, + #[serde(default)] + ignore: Vec, + + #[serde(default, rename = "ignoreregex")] + ignore_regex: Vec, + #[serde(skip)] + compiled_ignore_regex: Vec, + + #[serde(skip)] + name: String, + #[serde(skip)] + pub name_with_braces: String, +} + +impl Pattern { + pub fn setup(&mut self, name: &String) -> Result<(), String> { + self._setup(name) + .or_else(|msg| Err(format!("pattern {}: {}", name, msg))) + } + + pub fn _setup(&mut self, name: &String) -> Result<(), String> { + self.name = name.clone(); + self.name_with_braces = format!("<{}>", name); + + if self.regex.len() == 0 { + return Err("regex is empty".into()); + } + let compiled = + Regex::new(&format!("^{}$", self.regex)).or_else(|err| Err(err.to_string()))?; + + self.regex = format!("(?P<{}>{})", self.name, self.regex); + + for ignore in &self.ignore { + if !compiled.is_match(&ignore) { + return Err(format!( + "ignore '{}' doesn't match pattern. It should be fixed or removed.", + ignore, + )); + } + } + + for ignore_regex in &self.ignore_regex { + let compiled_ignore = Regex::new(&format!("^{}$", ignore_regex)).or_else(|err| { + Err(format!( + "ignoreregex '{}': {}", + ignore_regex, + err.to_string() + )) + })?; + + self.compiled_ignore_regex.push(compiled_ignore); + } + + Ok(()) + } +} + +// This is required to be added to a BTreeSet +// We compare Patterns by their names, which are unique. +// This is enforced by Patterns' names coming from their keys in a BTreeMap in Config +impl Ord for Pattern { + fn cmp(&self, other: &Self) -> Ordering { + self.name.cmp(&other.name) + } +} +impl PartialOrd for Pattern { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Eq for Pattern {} +impl PartialEq for Pattern { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} diff --git a/rust/src/stream.rs b/rust/src/stream.rs new file mode 100644 index 0000000..4ae000e --- /dev/null +++ b/rust/src/stream.rs @@ -0,0 +1,41 @@ +use std::collections::BTreeMap; + +use serde::Deserialize; + +use crate::{config::Patterns, filter::Filter}; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Stream { + cmd: Vec, + filters: BTreeMap, + + #[serde(skip)] + name: String, +} + +impl Stream { + pub fn setup(&mut self, name: &String, patterns: &Patterns) -> Result<(), String> { + self._setup(name, patterns) + .or_else(|msg| Err(format!("stream {}: {}", name, msg))) + } + + fn _setup(&mut self, name: &String, patterns: &Patterns) -> Result<(), String> { + self.name = name.clone(); + + if self.name.contains('.') { + return Err("character '.' is not allowed in stream name".into()); + } + + if self.filters.len() == 0 { + return Err("no filters configured".into()); + } + + for (key, value) in &mut self.filters { + value.setup(&name, key, patterns)?; + } + + Ok(()) + } +} + From d583df21a2f08535025b872f6385b5ac5babbdf4 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 10 Jun 2024 12:00:00 +0200 Subject: [PATCH 014/435] add configuration setup tests --- rust/src/action.rs | 62 +++++++++++++++++++++ rust/src/config.rs | 22 ++++++++ rust/src/daemon.rs | 1 + rust/src/filter.rs | 131 +++++++++++++++++++++++++++++++++++++++++++- rust/src/pattern.rs | 105 +++++++++++++++++++++++++++++++++++ rust/src/stream.rs | 56 +++++++++++++++++++ 6 files changed, 374 insertions(+), 3 deletions(-) diff --git a/rust/src/action.rs b/rust/src/action.rs index ef718a2..f875a5b 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -49,10 +49,20 @@ impl Action { self.filter_name = filter_name.clone(); self.name = name.clone(); + if self.name.len() == 0 { + return Err("action name is empty".into()); + } if self.name.contains('.') { return Err("character '.' is not allowed in filter name".into()); } + if self.cmd.len() == 0 { + return Err("cmd is empty".into()); + } + if self.cmd[0].len() == 0 { + return Err("cmd's first item is empty".into()); + } + if let Some(after) = &self.after { self.after_duration = Some(parse_duration(after).or_else(|err| { @@ -66,3 +76,55 @@ impl Action { } } +#[cfg(test)] +pub mod tests { + + use super::*; + + fn default_action() -> Action { + Action { + cmd: Vec::new(), + name: "".into(), + filter_name: "".into(), + stream_name: "".into(), + after: None, + after_duration: None, + on_exit: false, + } + } + + pub fn ok_action() -> Action { + let mut action = default_action(); + action.cmd = vec!["command".into()]; + action + } + + #[test] + fn missing_config() { + let mut action; + let name = "name".to_string(); + + // No command + action = default_action(); + assert!(action.setup(&name, &name, &name).is_err()); + + // No command + action = default_action(); + action.cmd = vec!["".into()]; + assert!(action.setup(&name, &name, &name).is_err()); + + // No command + action = default_action(); + action.cmd = vec!["".into(), "arg1".into()]; + assert!(action.setup(&name, &name, &name).is_err()); + + // command ok + action = ok_action(); + assert!(action.setup(&name, &name, &name).is_ok()); + + // command ok + action = ok_action(); + action.cmd.push("arg1".into()); + assert!(action.setup(&name, &name, &name).is_ok()); + } +} diff --git a/rust/src/config.rs b/rust/src/config.rs index 60e2115..3533cf2 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -117,3 +117,25 @@ mod jsonnet { Ok(result.to_string()) } } + +#[cfg(test)] +mod tests { + + use super::*; + + fn default_config() -> Config { + Config { + concurrency: 0, + patterns: BTreeMap::new(), + streams: BTreeMap::new(), + start: Vec::new(), + stop: Vec::new(), + } + } + + #[test] + fn config_missing() { + let mut config = default_config(); + assert!(config.setup().is_err()); + } +} diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index ad4eb74..ff4fcb2 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -4,6 +4,7 @@ use crate::{cli::LogLevel, config}; pub fn daemon(config_path: &PathBuf, loglevel: LogLevel, socket: &PathBuf) { println!("daemon {:?} {:?} {:?}", config_path, loglevel, socket); + let config = config::config_from_file(config_path); if let Ok(config) = config { dbg!(config); diff --git a/rust/src/filter.rs b/rust/src/filter.rs index bcba9e6..1c56442 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -1,9 +1,12 @@ -use std::{collections::{BTreeMap, BTreeSet}, time::Duration}; +use std::{ + collections::{BTreeMap, BTreeSet}, + time::Duration, +}; use regex::Regex; use serde::Deserialize; -use crate::{action::Action, pattern::Pattern, config::Patterns, parse_duration::parse_duration}; +use crate::{action::Action, config::Patterns, parse_duration::parse_duration, pattern::Pattern}; #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -31,7 +34,6 @@ pub struct Filter { stream_name: String, } - impl Filter { pub fn setup( &mut self, @@ -52,6 +54,9 @@ impl Filter { self.stream_name = stream_name.clone(); self.name = name.clone(); + if self.name.len() == 0 { + return Err("filter name is empty".into()); + } if self.name.contains('.') { return Err("character '.' is not allowed in filter name".into()); } @@ -105,3 +110,123 @@ impl Filter { Ok(()) } } + +#[cfg(test)] +pub mod tests { + + use crate::action::tests::ok_action; + + use super::*; + + pub fn default_filter() -> Filter { + Filter { + name: "".into(), + stream_name: "".into(), + actions: BTreeMap::new(), + patterns: BTreeSet::new(), + regex: Vec::new(), + compiled_regex: Vec::new(), + retry: None, + retry_period: None, + retry_duration: None, + longuest_action_duration: Duration::default(), + } + } + + pub fn ok_filter() -> Filter { + let mut filter = default_filter(); + let name = "name".to_string(); + filter.regex = vec!["reg".into()]; + filter + .actions + .insert(name.clone(), crate::action::tests::ok_action()); + filter + } + + #[test] + fn missing_config() { + let mut filter; + let name = "name".to_string(); + + // action but no regex + filter = ok_filter(); + filter.regex = Vec::new(); + assert!(filter.setup(&name, &name, &BTreeMap::new()).is_err()); + + // regex but no action + filter = ok_filter(); + filter.actions = BTreeMap::new(); + assert!(filter.setup(&name, &name, &BTreeMap::new()).is_err()); + + // ok + filter = ok_filter(); + assert!(filter.setup(&name, &name, &BTreeMap::new()).is_ok()); + } + + #[test] + fn retry() { + let mut filter; + let name = "name".to_string(); + + // retry but no retry_period + filter = ok_filter(); + filter.retry = Some(2); + assert!(filter.setup(&name, &name, &BTreeMap::new()).is_err()); + + // retry_period but no retry + filter = ok_filter(); + filter.retry_period = Some("2d".into()); + assert!(filter.setup(&name, &name, &BTreeMap::new()).is_err()); + + // invalid retry_period + filter = ok_filter(); + filter.retry = Some(2); + filter.retry_period = Some("2".into()); + assert!(filter.setup(&name, &name, &BTreeMap::new()).is_err()); + + // ok + filter = ok_filter(); + filter.retry = Some(2); + filter.retry_period = Some("2d".into()); + assert!(filter.setup(&name, &name, &BTreeMap::new()).is_ok()); + } + + #[test] + fn longuest_action_duration() { + let mut filter; + let name = "name".to_string(); + let minute_str = "1m".to_string(); + let minute = Duration::from_secs(60); + let two_minutes = Duration::from_secs(60 * 2); + let two_minutes_str = "2m".to_string(); + + // duration 0 + filter = ok_filter(); + filter.setup(&name, &name, &BTreeMap::new()).unwrap(); + assert_eq!(filter.longuest_action_duration, Duration::default()); + + let mut minute_action = ok_action(); + minute_action.after_duration = Some(minute); + + // duration 60 + filter = ok_filter(); + filter.actions.insert(minute_str.clone(), minute_action); + filter.setup(&name, &name, &BTreeMap::new()).unwrap(); + assert_eq!(filter.longuest_action_duration, minute); + + let mut minute_action = ok_action(); + minute_action.after_duration = Some(minute); + + let mut two_minutes_action = ok_action(); + two_minutes_action.after_duration = Some(two_minutes); + + // duration 120 + filter = ok_filter(); + filter.actions.insert(two_minutes_str, two_minutes_action); + filter.actions.insert(minute_str, minute_action); + filter.setup(&name, &name, &BTreeMap::new()).unwrap(); + assert_eq!(filter.longuest_action_duration, two_minutes); + } + + // TODO test filters with regexes (and actions) +} diff --git a/rust/src/pattern.rs b/rust/src/pattern.rs index 99a4abd..16d1bd7 100644 --- a/rust/src/pattern.rs +++ b/rust/src/pattern.rs @@ -8,6 +8,7 @@ use serde::Deserialize; #[serde(deny_unknown_fields)] pub struct Pattern { pub regex: String, + #[serde(default)] ignore: Vec, @@ -32,6 +33,13 @@ impl Pattern { self.name = name.clone(); self.name_with_braces = format!("<{}>", name); + if self.name.len() == 0 { + return Err("pattern name is empty".into()); + } + if self.name.contains(".") { + return Err("character '.' is not allowed in pattern name".into()); + } + if self.regex.len() == 0 { return Err("regex is empty".into()); } @@ -84,3 +92,100 @@ impl PartialEq for Pattern { self.name == other.name } } + +#[cfg(test)] +pub mod tests { + + use super::*; + + pub fn default_pattern() -> Pattern { + Pattern { + regex: "".into(), + ignore: Vec::new(), + ignore_regex: Vec::new(), + compiled_ignore_regex: Vec::new(), + name: "".into(), + name_with_braces: "".into(), + } + } + + pub fn ok_pattern() -> Pattern { + let mut pattern = default_pattern(); + pattern.regex = "[abc]".into(); + pattern + } + + #[test] + fn missing_information() { + let mut pattern; + + // Empty name + pattern = default_pattern(); + pattern.regex = "abc".into(); + assert!(pattern.setup(&"".into()).is_err()); + + // '.' in name + pattern = default_pattern(); + pattern.regex = "abc".into(); + assert!(pattern.setup(&"na.me".into()).is_err()); + + // Empty regex + pattern = default_pattern(); + assert!(pattern.setup(&"name".into()).is_err()); + } + + #[test] + fn regex() { + let mut pattern; + + // regex ok + pattern = ok_pattern(); + assert!(pattern.setup(&"name".into()).is_ok()); + + // regex ok + pattern = default_pattern(); + pattern.regex = "abc".into(); + assert!(pattern.setup(&"name".into()).is_ok()); + + // regex ko + pattern = default_pattern(); + pattern.regex = "[abc".into(); + assert!(pattern.setup(&"name".into()).is_err()); + } + + #[test] + fn ignore() { + let mut pattern; + + // ignore ok + pattern = default_pattern(); + pattern.regex = "[abc]".into(); + pattern.ignore.push("a".into()); + pattern.ignore.push("b".into()); + assert!(pattern.setup(&"name".into()).is_ok()); + + // ignore ko + pattern = default_pattern(); + pattern.regex = "[abc]".into(); + pattern.ignore.push("d".into()); + assert!(pattern.setup(&"name".into()).is_err()); + } + + #[test] + fn ignore_regex() { + let mut pattern; + + // ignore_regex ok + pattern = default_pattern(); + pattern.regex = "[abc]".into(); + pattern.ignore_regex.push("[a]".into()); + pattern.ignore_regex.push("a".into()); + assert!(pattern.setup(&"name".into()).is_ok()); + + // ignore_regex ko + pattern = default_pattern(); + pattern.regex = "[abc]".into(); + pattern.ignore.push("[a".into()); + assert!(pattern.setup(&"name".into()).is_err()); + } +} diff --git a/rust/src/stream.rs b/rust/src/stream.rs index 4ae000e..49a4ca9 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -23,10 +23,20 @@ impl Stream { fn _setup(&mut self, name: &String, patterns: &Patterns) -> Result<(), String> { self.name = name.clone(); + if self.name.len() == 0 { + return Err("stream name is empty".into()); + } if self.name.contains('.') { return Err("character '.' is not allowed in stream name".into()); } + if self.cmd.len() == 0 { + return Err("cmd is empty".into()); + } + if self.cmd[0].len() == 0 { + return Err("cmd's first item is empty".into()); + } + if self.filters.len() == 0 { return Err("no filters configured".into()); } @@ -39,3 +49,49 @@ impl Stream { } } + +#[cfg(test)] +pub mod tests { + + use super::*; + + fn default_stream() -> Stream { + Stream { + cmd: Vec::new(), + name: "".into(), + filters: BTreeMap::new() + } + } + + pub fn ok_stream() -> Stream { + let mut stream = default_stream(); + stream.cmd = vec!["command".into()]; + stream.filters.insert("name".into(), crate::filter::tests::ok_filter()); + stream + } + + #[test] + fn test() { + let mut stream; + let name = "name".into(); + + // missing cmd + stream = ok_stream(); + stream.cmd = Vec::new(); + assert!(stream.setup(&name, &BTreeMap::new()).is_err()); + + // missing cmd + stream = ok_stream(); + stream.cmd = vec!["".into(), "arg1".into()]; + assert!(stream.setup(&name, &BTreeMap::new()).is_err()); + + // missing filters + stream = ok_stream(); + stream.filters = BTreeMap::new(); + assert!(stream.setup(&name, &BTreeMap::new()).is_err()); + + // stream ok + stream = ok_stream(); + assert!(stream.setup(&name, &BTreeMap::new()).is_ok()); + } +} From 3d491feec3538c3bb436bfe45a8850fb2d30ea9a Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 19 Jun 2024 12:00:00 +0200 Subject: [PATCH 015/435] more tests on filter.setup() --- rust/src/filter.rs | 182 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 169 insertions(+), 13 deletions(-) diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 1c56442..3a48461 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -83,7 +83,13 @@ impl Filter { for regex in &self.regex { let mut regex_buf = regex.clone(); for (_pattern_name, pattern) in patterns { - if regex.contains(&pattern.name_with_braces) { + if let Some(index) = regex.find(&pattern.name_with_braces) { + if regex.rfind(&pattern.name_with_braces).unwrap() != index { + return Err(format!( + "pattern {} present multiple times in regex", + &pattern.name_with_braces + )); + } self.patterns.insert(pattern.clone()); } regex_buf = regex_buf.replacen(&pattern.name_with_braces, &pattern.regex, 1) @@ -115,6 +121,7 @@ impl Filter { pub mod tests { use crate::action::tests::ok_action; + use crate::pattern::tests::default_pattern; use super::*; @@ -139,7 +146,7 @@ pub mod tests { filter.regex = vec!["reg".into()]; filter .actions - .insert(name.clone(), crate::action::tests::ok_action()); + .insert(name.clone(), ok_action()); filter } @@ -147,54 +154,57 @@ pub mod tests { fn missing_config() { let mut filter; let name = "name".to_string(); + let empty_patterns = Patterns::new(); // action but no regex filter = ok_filter(); filter.regex = Vec::new(); - assert!(filter.setup(&name, &name, &BTreeMap::new()).is_err()); + assert!(filter.setup(&name, &name, &empty_patterns).is_err()); // regex but no action filter = ok_filter(); filter.actions = BTreeMap::new(); - assert!(filter.setup(&name, &name, &BTreeMap::new()).is_err()); + assert!(filter.setup(&name, &name, &empty_patterns).is_err()); // ok filter = ok_filter(); - assert!(filter.setup(&name, &name, &BTreeMap::new()).is_ok()); + assert!(filter.setup(&name, &name, &empty_patterns).is_ok()); } #[test] fn retry() { let mut filter; let name = "name".to_string(); + let empty_patterns = Patterns::new(); // retry but no retry_period filter = ok_filter(); filter.retry = Some(2); - assert!(filter.setup(&name, &name, &BTreeMap::new()).is_err()); + assert!(filter.setup(&name, &name, &empty_patterns).is_err()); // retry_period but no retry filter = ok_filter(); filter.retry_period = Some("2d".into()); - assert!(filter.setup(&name, &name, &BTreeMap::new()).is_err()); + assert!(filter.setup(&name, &name, &empty_patterns).is_err()); // invalid retry_period filter = ok_filter(); filter.retry = Some(2); filter.retry_period = Some("2".into()); - assert!(filter.setup(&name, &name, &BTreeMap::new()).is_err()); + assert!(filter.setup(&name, &name, &empty_patterns).is_err()); // ok filter = ok_filter(); filter.retry = Some(2); filter.retry_period = Some("2d".into()); - assert!(filter.setup(&name, &name, &BTreeMap::new()).is_ok()); + assert!(filter.setup(&name, &name, &empty_patterns).is_ok()); } #[test] fn longuest_action_duration() { let mut filter; let name = "name".to_string(); + let empty_patterns = Patterns::new(); let minute_str = "1m".to_string(); let minute = Duration::from_secs(60); let two_minutes = Duration::from_secs(60 * 2); @@ -202,7 +212,7 @@ pub mod tests { // duration 0 filter = ok_filter(); - filter.setup(&name, &name, &BTreeMap::new()).unwrap(); + filter.setup(&name, &name, &empty_patterns).unwrap(); assert_eq!(filter.longuest_action_duration, Duration::default()); let mut minute_action = ok_action(); @@ -211,7 +221,7 @@ pub mod tests { // duration 60 filter = ok_filter(); filter.actions.insert(minute_str.clone(), minute_action); - filter.setup(&name, &name, &BTreeMap::new()).unwrap(); + filter.setup(&name, &name, &empty_patterns).unwrap(); assert_eq!(filter.longuest_action_duration, minute); let mut minute_action = ok_action(); @@ -224,9 +234,155 @@ pub mod tests { filter = ok_filter(); filter.actions.insert(two_minutes_str, two_minutes_action); filter.actions.insert(minute_str, minute_action); - filter.setup(&name, &name, &BTreeMap::new()).unwrap(); + filter.setup(&name, &name, &empty_patterns).unwrap(); assert_eq!(filter.longuest_action_duration, two_minutes); } - // TODO test filters with regexes (and actions) + #[test] + fn regexes() { + let name = "name".to_string(); + let mut filter; + + // make a Patterns + let mut patterns = Patterns::new(); + + let mut pattern = default_pattern(); + pattern.regex = "[abc]".to_string(); + assert!(pattern.setup(&name).is_ok()); + patterns.insert(name.clone(), pattern.clone()); + + let unused_name = "unused".to_string(); + let mut unused_pattern = default_pattern(); + unused_pattern.regex = "compile[error".to_string(); + assert!(unused_pattern.setup(&unused_name).is_err()); + patterns.insert(unused_name.clone(), unused_pattern.clone()); + + let boubou_name = "boubou".to_string(); + let mut boubou = default_pattern(); + boubou.regex = "(?:bou){2}".to_string(); + assert!(boubou.setup(&boubou_name).is_ok()); + patterns.insert(boubou_name.clone(), boubou.clone()); + + // TODO correct regex replacement + filter = default_filter(); + filter + .actions + .insert(name.clone(), ok_action()); + filter.regex.push("insert here$".to_string()); + assert!(filter.setup(&name, &name, &patterns).is_ok()); + assert_eq!( + filter.compiled_regex[0].to_string(), + Regex::new("insert (?P[abc]) here$") + .unwrap() + .to_string() + ); + assert_eq!(filter.patterns.len(), 1); + let stored_pattern = filter.patterns.first().unwrap(); + assert_eq!(stored_pattern.regex, pattern.regex); + + // TODO same pattern two times in regex + filter = default_filter(); + filter + .actions + .insert(name.clone(), ok_action()); + filter + .regex + .push("there are two s!".to_string()); + assert!(filter.setup(&name, &name, &patterns).is_err()); + + // TODO two patterns in one regex + filter = default_filter(); + filter + .actions + .insert(name.clone(), ok_action()); + filter.regex.push("insert here and there".to_string()); + assert!(filter.setup(&name, &name, &patterns).is_ok()); + assert_eq!( + filter.compiled_regex[0].to_string(), + Regex::new("insert (?P[abc]) here and (?P(?:bou){2}) there") + .unwrap() + .to_string() + ); + assert_eq!(filter.patterns.len(), 2); + let stored_pattern = filter.patterns.first().unwrap(); + assert_eq!(stored_pattern.regex, boubou.regex); + let stored_pattern = filter.patterns.last().unwrap(); + assert_eq!(stored_pattern.regex, pattern.regex); + + // TODO multiple regexes with same pattern + filter = default_filter(); + filter + .actions + .insert(name.clone(), ok_action()); + filter.regex.push("insert here".to_string()); + filter.regex.push("also add there".to_string()); + assert!(filter.setup(&name, &name, &patterns).is_ok()); + assert_eq!( + filter.compiled_regex[0].to_string(), + Regex::new("insert (?P[abc]) here") + .unwrap() + .to_string() + ); + assert_eq!( + filter.compiled_regex[1].to_string(), + Regex::new("also add (?P[abc]) there") + .unwrap() + .to_string() + ); + assert_eq!(filter.patterns.len(), 1); + let stored_pattern = filter.patterns.first().unwrap(); + assert_eq!(stored_pattern.regex, pattern.regex); + + // TODO multiple regexes with same patterns + filter = default_filter(); + filter + .actions + .insert(name.clone(), ok_action()); + filter.regex.push("insert here and there".to_string()); + filter.regex.push("also add here and there".to_string()); + assert!(filter.setup(&name, &name, &patterns).is_ok()); + assert_eq!( + filter.compiled_regex[0].to_string(), + Regex::new("insert (?P[abc]) here and (?P(?:bou){2}) there") + .unwrap() + .to_string() + ); + assert_eq!( + filter.compiled_regex[1].to_string(), + Regex::new("also add (?P(?:bou){2}) here and (?P[abc]) there") + .unwrap() + .to_string() + ); + assert_eq!(filter.patterns.len(), 2); + let stored_pattern = filter.patterns.first().unwrap(); + assert_eq!(stored_pattern.regex, boubou.regex); + let stored_pattern = filter.patterns.last().unwrap(); + assert_eq!(stored_pattern.regex, pattern.regex); + + // TODO multiple regexes with different patterns + filter = default_filter(); + filter + .actions + .insert(name.clone(), ok_action()); + filter.regex.push("insert here".to_string()); + filter.regex.push("also add there".to_string()); + assert!(filter.setup(&name, &name, &patterns).is_ok()); + assert_eq!( + filter.compiled_regex[0].to_string(), + Regex::new("insert (?P[abc]) here") + .unwrap() + .to_string() + ); + assert_eq!( + filter.compiled_regex[1].to_string(), + Regex::new("also add (?P(?:bou){2}) there") + .unwrap() + .to_string() + ); + assert_eq!(filter.patterns.len(), 2); + let stored_pattern = filter.patterns.first().unwrap(); + assert_eq!(stored_pattern.regex, boubou.regex); + let stored_pattern = filter.patterns.last().unwrap(); + assert_eq!(stored_pattern.regex, pattern.regex); + } } From 2b1ee51f97ad781179be9197ad4279524c81f910 Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 24 Jul 2024 12:00:00 +0200 Subject: [PATCH 016/435] use log package --- rust/Cargo.lock | 7 +++++++ rust/Cargo.toml | 1 + rust/src/cli.rs | 39 ++++++++++++++++----------------------- rust/src/client.rs | 13 ++++++++++--- rust/src/config.rs | 1 + rust/src/logger.rs | 36 ++++++++++++++++++++++++++++++++++++ 6 files changed, 71 insertions(+), 26 deletions(-) create mode 100644 rust/src/logger.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b945d8c..8cc5e21 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -281,6 +281,12 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "md5" version = "0.7.0" @@ -362,6 +368,7 @@ dependencies = [ "clap", "clap_complete", "jrsonnet-evaluator", + "log", "num_cpus", "regex", "serde", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 18d5bbe..27ef72a 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -10,6 +10,7 @@ anyhow = "1.0.86" clap = { version = "4.5.4", features = ["derive"] } clap_complete = "4.5.2" jrsonnet-evaluator = "0.4.2" +log = { version = "0.4.22", features = ["std"] } num_cpus = "1.16.0" regex = "1.10.4" serde = { version = "1.0.203", features = ["derive"] } diff --git a/rust/src/cli.rs b/rust/src/cli.rs index f38de94..3de0275 100644 --- a/rust/src/cli.rs +++ b/rust/src/cli.rs @@ -31,8 +31,8 @@ pub enum Command { config: PathBuf, /// minimum log level to show - #[clap(short = 'l', long, default_value_t = LogLevel::INFO, ignore_case = true)] - loglevel: LogLevel, + #[clap(short = 'l', long, value_parser = parse_log_level, default_value_t = log::Level::Info, ignore_case = true)] + loglevel: log::Level, /// path to the client-daemon communication socket #[clap(short = 's', long, default_value = "/run/reaction/reaction.sock")] @@ -130,27 +130,6 @@ impl fmt::Display for Format { } } -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, ValueEnum)] -pub enum LogLevel { - DEBUG, - INFO, - WARN, - ERROR, - FATAL, -} - -impl fmt::Display for LogLevel { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - LogLevel::DEBUG => write!(f, "DEBUG"), - LogLevel::INFO => write!(f, "INFO"), - LogLevel::WARN => write!(f, "WARN"), - LogLevel::ERROR => write!(f, "ERROR"), - LogLevel::FATAL => write!(f, "FATAL"), - } - } -} - // Structs #[derive(Clone, Debug)] @@ -169,3 +148,17 @@ fn parse_named_regex(s: &str) -> Result { name: name.to_string(), }) } + +fn parse_log_level(s: &str) -> Result { + match s.to_ascii_uppercase().as_str() { + "DEBUG" => Ok(log::Level::Debug), + "INFO" => Ok(log::Level::Info), + "WARN" => Ok(log::Level::Warn), + "ERROR" => Ok(log::Level::Error), + "FATAL" => Ok(log::Level::Error), + _ => Err("must be one of ERROR, WARN, INFO, DEBUG".into()), + } +} + +// TODO generate completion +// TODO generate a man page https://rust-cli.github.io/book/in-depth/docs.html diff --git a/rust/src/client.rs b/rust/src/client.rs index 4657026..467d88c 100644 --- a/rust/src/client.rs +++ b/rust/src/client.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use log::debug; use regex::Regex; use crate::cli::{Format, NamedRegex}; @@ -11,7 +12,10 @@ pub fn show( pattern: &Option, patterns: &Vec, ) { - println!("show {:?} {:?} {:?} {:?} {:?}", socket, format, limit, pattern, patterns); + debug!( + "show {:?} {:?} {:?} {:?} {:?}", + socket, format, limit, pattern, patterns + ); } pub fn flush( @@ -21,9 +25,12 @@ pub fn flush( pattern: &Option, patterns: &Vec, ) { - println!("flush {:?} {:?} {:?} {:?} {:?}", socket, format, limit, pattern, patterns); + debug!( + "flush {:?} {:?} {:?} {:?} {:?}", + socket, format, limit, pattern, patterns + ); } pub fn test_regex(config_path: &PathBuf, regex: &String, line: &Option) { - println!("test-regex {:?} {:?} {:?} ", config_path, regex, line); + debug!("test-regex {:?} {:?} {:?} ", config_path, regex, line); } diff --git a/rust/src/config.rs b/rust/src/config.rs index 3533cf2..340ae34 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -5,6 +5,7 @@ use std::fs::File; use std::path::PathBuf; use anyhow::{anyhow, Context, Result}; +use log::{error, info}; use serde::Deserialize; use crate::{pattern::Pattern, stream::Stream}; diff --git a/rust/src/logger.rs b/rust/src/logger.rs new file mode 100644 index 0000000..771b362 --- /dev/null +++ b/rust/src/logger.rs @@ -0,0 +1,36 @@ +use log::{Level, Metadata}; + +pub struct SimpleLogger { + loglevel: Level, +} + +impl log::Log for SimpleLogger { + fn enabled(&self, metadata: &Metadata) -> bool { + metadata.level() <= self.loglevel + } + + fn log(&self, record: &log::Record) { + if self.enabled(record.metadata()) { + eprintln!( + "{} {}", + match record.level() { + Level::Error => "ERROR", + Level::Warn => "WARN ", + Level::Info => "INFO ", + Level::Debug => "DEBUG", + Level::Trace => "TRACE", + }, + record.args().to_string().trim() + ); + } + } + + fn flush(&self) {} +} + +impl SimpleLogger { + pub fn init(loglevel: log::Level) -> Result<(), log::SetLoggerError> { + log::set_boxed_logger(Box::new(SimpleLogger { loglevel })) + .map(|()| log::set_max_level(loglevel.to_level_filter())) + } +} From 4aece412e80bef958626070a79b5bac6ca673a9b Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 24 Jul 2024 12:00:00 +0200 Subject: [PATCH 017/435] cargo fmt --- rust/src/action.rs | 4 ++-- rust/src/config.rs | 1 - rust/src/filter.rs | 36 ++++++++++++++---------------------- rust/src/main.rs | 21 ++++++++------------- rust/src/stream.rs | 11 ++++++----- 5 files changed, 30 insertions(+), 43 deletions(-) diff --git a/rust/src/action.rs b/rust/src/action.rs index f875a5b..1b1f9f7 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -58,10 +58,10 @@ impl Action { if self.cmd.len() == 0 { return Err("cmd is empty".into()); - } + } if self.cmd[0].len() == 0 { return Err("cmd's first item is empty".into()); - } + } if let Some(after) = &self.after { self.after_duration = diff --git a/rust/src/config.rs b/rust/src/config.rs index 340ae34..65a8625 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -59,7 +59,6 @@ enum Format { JSON, JSONnet, } - pub fn config_from_file(path: &PathBuf) -> Result { _config_from_file(path).with_context(|| anyhow!("Configuration file {}:", path.display())) } diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 3a48461..af37037 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -144,9 +144,7 @@ pub mod tests { let mut filter = default_filter(); let name = "name".to_string(); filter.regex = vec!["reg".into()]; - filter - .actions - .insert(name.clone(), ok_action()); + filter.actions.insert(name.clone(), ok_action()); filter } @@ -265,9 +263,7 @@ pub mod tests { // TODO correct regex replacement filter = default_filter(); - filter - .actions - .insert(name.clone(), ok_action()); + filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here$".to_string()); assert!(filter.setup(&name, &name, &patterns).is_ok()); assert_eq!( @@ -282,9 +278,7 @@ pub mod tests { // TODO same pattern two times in regex filter = default_filter(); - filter - .actions - .insert(name.clone(), ok_action()); + filter.actions.insert(name.clone(), ok_action()); filter .regex .push("there are two s!".to_string()); @@ -292,10 +286,10 @@ pub mod tests { // TODO two patterns in one regex filter = default_filter(); + filter.actions.insert(name.clone(), ok_action()); filter - .actions - .insert(name.clone(), ok_action()); - filter.regex.push("insert here and there".to_string()); + .regex + .push("insert here and there".to_string()); assert!(filter.setup(&name, &name, &patterns).is_ok()); assert_eq!( filter.compiled_regex[0].to_string(), @@ -311,9 +305,7 @@ pub mod tests { // TODO multiple regexes with same pattern filter = default_filter(); - filter - .actions - .insert(name.clone(), ok_action()); + filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here".to_string()); filter.regex.push("also add there".to_string()); assert!(filter.setup(&name, &name, &patterns).is_ok()); @@ -335,11 +327,13 @@ pub mod tests { // TODO multiple regexes with same patterns filter = default_filter(); + filter.actions.insert(name.clone(), ok_action()); filter - .actions - .insert(name.clone(), ok_action()); - filter.regex.push("insert here and there".to_string()); - filter.regex.push("also add here and there".to_string()); + .regex + .push("insert here and there".to_string()); + filter + .regex + .push("also add here and there".to_string()); assert!(filter.setup(&name, &name, &patterns).is_ok()); assert_eq!( filter.compiled_regex[0].to_string(), @@ -361,9 +355,7 @@ pub mod tests { // TODO multiple regexes with different patterns filter = default_filter(); - filter - .actions - .insert(name.clone(), ok_action()); + filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here".to_string()); filter.regex.push("also add there".to_string()); assert!(filter.setup(&name, &name, &patterns).is_ok()); diff --git a/rust/src/main.rs b/rust/src/main.rs index 41b22e4..deb8333 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -12,20 +12,21 @@ use clap::Parser; use regex::Regex; +mod action; +mod filter; mod pattern; mod stream; -mod filter; -mod action; mod cli; +mod client; mod config; mod daemon; +mod logger; mod parse_duration; -mod client; use cli::{Cli, Command}; +use client::{flush, show, test_regex}; use daemon::daemon; -use client::{show, flush, test_regex}; fn main() { cli::NamedRegex { @@ -49,24 +50,18 @@ fn main() { limit, pattern, patterns, - } => { - show(&socket, format, &limit, &pattern, &patterns) - } + } => show(&socket, format, &limit, &pattern, &patterns), Command::Flush { socket, format, limit, pattern, patterns, - } => { - flush(&socket, format, &limit, &pattern, &patterns) - } + } => flush(&socket, format, &limit, &pattern, &patterns), Command::TestRegex { config, regex, line, - } => { - test_regex(&config, ®ex, &line) - } + } => test_regex(&config, ®ex, &line), } } diff --git a/rust/src/stream.rs b/rust/src/stream.rs index 49a4ca9..6aaa78b 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -32,10 +32,10 @@ impl Stream { if self.cmd.len() == 0 { return Err("cmd is empty".into()); - } + } if self.cmd[0].len() == 0 { return Err("cmd's first item is empty".into()); - } + } if self.filters.len() == 0 { return Err("no filters configured".into()); @@ -49,7 +49,6 @@ impl Stream { } } - #[cfg(test)] pub mod tests { @@ -59,14 +58,16 @@ pub mod tests { Stream { cmd: Vec::new(), name: "".into(), - filters: BTreeMap::new() + filters: BTreeMap::new(), } } pub fn ok_stream() -> Stream { let mut stream = default_stream(); stream.cmd = vec!["command".into()]; - stream.filters.insert("name".into(), crate::filter::tests::ok_filter()); + stream + .filters + .insert("name".into(), crate::filter::tests::ok_filter()); stream } From 9d6f367a13f2ec8858c9c21815ec513babe5ede6 Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 24 Jul 2024 12:00:00 +0200 Subject: [PATCH 018/435] optimise Config size --- rust/src/action.rs | 1 + rust/src/filter.rs | 2 ++ rust/src/pattern.rs | 1 + 3 files changed, 4 insertions(+) diff --git a/rust/src/action.rs b/rust/src/action.rs index 1b1f9f7..98154a5 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -68,6 +68,7 @@ impl Action { Some(parse_duration(after).or_else(|err| { Err(format!("failed to parse after time: {}", err.to_string())) })?); + self.after = None; } else if self.on_exit { return Err("cannot have `onexit: true`, without an `after` directive".into()); } diff --git a/rust/src/filter.rs b/rust/src/filter.rs index af37037..9a31aa2 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -74,6 +74,7 @@ impl Filter { Some(parse_duration(retry_period).or_else(|err| { Err(format!("failed to parse retry time: {}", err.to_string())) })?); + self.retry_period = None; } if self.regex.len() == 0 { @@ -97,6 +98,7 @@ impl Filter { let compiled = Regex::new(®ex_buf).or_else(|err| Err(err.to_string()))?; self.compiled_regex.push(compiled); } + self.regex.clear(); if self.actions.len() == 0 { return Err("no actions configured".into()); diff --git a/rust/src/pattern.rs b/rust/src/pattern.rs index 16d1bd7..8e449f5 100644 --- a/rust/src/pattern.rs +++ b/rust/src/pattern.rs @@ -68,6 +68,7 @@ impl Pattern { self.compiled_ignore_regex.push(compiled_ignore); } + self.ignore_regex.clear(); Ok(()) } From efb543355a2e8dbb86e360caf1f4efbc4b8e2553 Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 24 Jul 2024 12:00:00 +0200 Subject: [PATCH 019/435] implement start/stop commands --- rust/src/config.rs | 47 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/rust/src/config.rs b/rust/src/config.rs index 65a8625..b7bddf0 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -1,8 +1,9 @@ #![allow(dead_code)] -use std::collections::BTreeMap; use std::fs::File; use std::path::PathBuf; +use std::process::Stdio; +use std::{collections::BTreeMap, process::Command}; use anyhow::{anyhow, Context, Result}; use log::{error, info}; @@ -52,6 +53,14 @@ impl Config { } Ok(()) } + + pub fn start(&self) -> bool { + return run_commands(&self.start, "start"); + } + + pub fn stop(&self) -> bool { + return run_commands(&self.stop, "stop"); + } } enum Format { @@ -118,6 +127,42 @@ mod jsonnet { } } +fn run_commands(commands: &Vec>, moment: &str) -> bool { + let mut ok = true; + for command in commands { + info!("{} command: run {:?}\n", moment, command); + // TODO reaction-go waits the subprocess completion for a minute maximum + match Command::new(&command[0]) + .args(&command[1..]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(exit_status) => { + if !exit_status.success() { + error!( + "{} command: run {:?}: exit code: {}\n", + moment, + command, + exit_status.code().unwrap_or(1) + ); + ok = false; + } + } + Err(err) => { + error!( + "{} command: run {:?}: could not execute: {}\n", + moment, command, err + ); + ok = false; + } + } + } + + return ok; +} + #[cfg(test)] mod tests { From f8b09faa9a92b4994645674ee508c7b9e353722a Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 24 Jul 2024 12:00:00 +0200 Subject: [PATCH 020/435] iteration on daemon - handle interruption signals - config in Arc --- rust/Cargo.lock | 41 ++++++++++++++++++++++++++++ rust/Cargo.toml | 1 + rust/src/action.rs | 2 +- rust/src/config.rs | 2 +- rust/src/daemon.rs | 65 ++++++++++++++++++++++++++++++++++++++------ rust/src/filter.rs | 2 +- rust/src/stream.rs | 2 +- rust/test.jsonnet | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 170 insertions(+), 13 deletions(-) create mode 100644 rust/test.jsonnet diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8cc5e21..3b015a8 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -91,6 +91,24 @@ dependencies = [ "serde", ] +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "clap" version = "4.5.4" @@ -146,6 +164,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "ctrlc" +version = "3.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" +dependencies = [ + "nix", + "windows-sys", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -299,6 +327,18 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -367,6 +407,7 @@ dependencies = [ "anyhow", "clap", "clap_complete", + "ctrlc", "jrsonnet-evaluator", "log", "num_cpus", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 27ef72a..3b43adb 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" anyhow = "1.0.86" clap = { version = "4.5.4", features = ["derive"] } clap_complete = "4.5.2" +ctrlc = { version = "3.4.4", features = ["termination"] } jrsonnet-evaluator = "0.4.2" log = { version = "0.4.22", features = ["std"] } num_cpus = "1.16.0" diff --git a/rust/src/action.rs b/rust/src/action.rs index 98154a5..127734e 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use crate::parse_duration::parse_duration; -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Action { cmd: Vec, diff --git a/rust/src/config.rs b/rust/src/config.rs index b7bddf0..e256a8e 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -13,7 +13,7 @@ use crate::{pattern::Pattern, stream::Stream}; pub type Patterns = BTreeMap; -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Config { patterns: Patterns, diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index ff4fcb2..254215f 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -1,14 +1,61 @@ -use std::path::PathBuf; +use std::process::exit; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::{path::PathBuf, sync::Arc}; -use crate::{cli::LogLevel, config}; +use log::{debug, error, warn, Level}; -pub fn daemon(config_path: &PathBuf, loglevel: LogLevel, socket: &PathBuf) { - println!("daemon {:?} {:?} {:?}", config_path, loglevel, socket); +use crate::{config, logger}; - let config = config::config_from_file(config_path); - if let Ok(config) = config { - dbg!(config); - } else if let Err(err) = config { - dbg!(err); +pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { + if let Err(err) = logger::SimpleLogger::init(loglevel) { + eprintln!("ERROR could not initialize logging: {err}"); + exit(1); } + + debug!("daemon {config_path:?} {loglevel:?} {socket:?}"); + + let config = match config::config_from_file(config_path) { + Ok(config) => Arc::new(config), + Err(err) => { + error!("{err}"); + exit(1); + } + }; + + // This AtomicBool prevents us from freeing resources two times + let quitting = Arc::new(AtomicBool::new(false)); + { + // Handle SIGINT, SIGTERM, SIGHUP + let quitting_ctrlc = quitting.clone(); + let config_ctrlc = config.clone(); + if let Err(err) = ctrlc::set_handler(move || quit(&quitting_ctrlc, &config_ctrlc, false)) { + error!("impossible to launch a thread for catching signals, exiting: {err}"); + exit(1); + } + } + + if !&config.start() { + error!("a start command failed, exiting."); + exit(1); + } + + // TODO launch streams + + // TODO wait for streams to finish + + quit(&quitting, &config, true); +} + +fn quit(quitting: &Arc, _config: &config::Config, graceful: bool) { + if let Err(_) = quitting.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) { + warn!("tried to quit while already quitting"); + return; + } + + // TODO free resources, flush DB etc. + + exit(match graceful { + true => 0, + false => 1, + }); } diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 9a31aa2..ab47668 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -8,7 +8,7 @@ use serde::Deserialize; use crate::{action::Action, config::Patterns, parse_duration::parse_duration, pattern::Pattern}; -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Filter { actions: BTreeMap, diff --git a/rust/src/stream.rs b/rust/src/stream.rs index 6aaa78b..b8aa7dd 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use crate::{config::Patterns, filter::Filter}; -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Stream { cmd: Vec, diff --git a/rust/test.jsonnet b/rust/test.jsonnet new file mode 100644 index 0000000..705e867 --- /dev/null +++ b/rust/test.jsonnet @@ -0,0 +1,68 @@ +{ + patterns: { + num: { + regex: '[0-9]+', + ignore: ['1'], + // ignoreregex: ['2.?'], + }, + letter: { + regex: '[a-z]+', + ignore: ['b'], + // ignoreregex: ['b.?'], + }, + }, + + start: [ + ['echo', 'coucou'], + ['echo coucou'], + ], + + streams: { + tailDown1: { + cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 1; echo found $i; done; sleep 30"], + filters: { + findIP: { + regex: [ + '^found _$', + '^found _$', + ], + retry: 2, + retryperiod: '30s', + actions: { + damn: { + cmd: ['echo', ''], + }, + undamn: { + cmd: ['echo', 'undamn', ''], + after: '28s', + onexit: true, + }, + }, + }, + }, + }, + tailDown2: { + cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 1; echo found $i; done; sleep 30"], + filters: { + findIP: { + regex: [ + '^found _$', + '^found _$', + ], + retry: 2, + retryperiod: '30s', + actions: { + damn: { + cmd: ['echo', ''], + }, + undamn: { + cmd: ['echo', 'undamn', ''], + after: '28s', + onexit: true, + }, + }, + }, + }, + }, + }, +} From 8abdb7b74b627958879e860c298f87aaf13d1382 Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 24 Jul 2024 12:00:00 +0200 Subject: [PATCH 021/435] streams, signals, quit - launching streams - handling termination signals by killing stream cmds - waiting for stream threads to quit - executing stop commands --- rust/src/config.rs | 6 ++-- rust/src/daemon.rs | 68 ++++++++++++++++++++++++++++++---------------- rust/src/stream.rs | 44 +++++++++++++++++++++++++++++- rust/test.jsonnet | 9 ++++-- 4 files changed, 96 insertions(+), 31 deletions(-) diff --git a/rust/src/config.rs b/rust/src/config.rs index e256a8e..66b40a1 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -18,7 +18,7 @@ pub type Patterns = BTreeMap; pub struct Config { patterns: Patterns, - streams: BTreeMap, + pub streams: BTreeMap, #[serde(default = "num_cpus::get")] concurrency: usize, @@ -142,7 +142,7 @@ fn run_commands(commands: &Vec>, moment: &str) -> bool { Ok(exit_status) => { if !exit_status.success() { error!( - "{} command: run {:?}: exit code: {}\n", + "{} command: run {:?}: exit code: {}", moment, command, exit_status.code().unwrap_or(1) @@ -152,7 +152,7 @@ fn run_commands(commands: &Vec>, moment: &str) -> bool { } Err(err) => { error!( - "{} command: run {:?}: could not execute: {}\n", + "{} command: run {:?}: could not execute: {}", moment, command, err ); ok = false; diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index 254215f..138c9be 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -1,8 +1,10 @@ use std::process::exit; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::sync_channel; +use std::thread; use std::{path::PathBuf, sync::Arc}; -use log::{debug, error, warn, Level}; +use log::{debug, error, Level, info}; use crate::{config, logger}; @@ -22,39 +24,57 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { } }; - // This AtomicBool prevents us from freeing resources two times - let quitting = Arc::new(AtomicBool::new(false)); - { - // Handle SIGINT, SIGTERM, SIGHUP - let quitting_ctrlc = quitting.clone(); - let config_ctrlc = config.clone(); - if let Err(err) = ctrlc::set_handler(move || quit(&quitting_ctrlc, &config_ctrlc, false)) { - error!("impossible to launch a thread for catching signals, exiting: {err}"); - exit(1); - } - } - - if !&config.start() { + if !config.start() { error!("a start command failed, exiting."); exit(1); } - // TODO launch streams + let mut stream_process_child_handles = Vec::new(); + let mut stream_thread_handles = Vec::new(); - // TODO wait for streams to finish + for (_, stream) in &config.streams { + let stream = stream.clone(); + let (tx, rx) = sync_channel(0); - quit(&quitting, &config, true); -} + stream_thread_handles.push(thread::spawn(move || stream.manager(tx))); -fn quit(quitting: &Arc, _config: &config::Config, graceful: bool) { - if let Err(_) = quitting.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) { - warn!("tried to quit while already quitting"); - return; + if let Ok(Some(child)) = rx.recv() { + stream_process_child_handles.push(child); + } } - // TODO free resources, flush DB etc. + let signal_received = Arc::new(AtomicBool::new(false)); + { + // Handle SIGINT, SIGTERM, SIGHUP + let signal_received2 = signal_received.clone(); + if let Err(err) = ctrlc::set_handler(move || { + signal_received2.store(true, Ordering::SeqCst); - exit(match graceful { + info!("waiting for streams to finish..."); + // Kill stream subprocesses + for child_handle in stream_process_child_handles.iter_mut() { + let _ = child_handle.kill(); + } + }) { + error!("impossible to launch a signal-catching thread, exiting: {err}"); + exit(1); + } + } + + // Wait for all streams to quit + for thread_handle in stream_thread_handles { + let _ = thread_handle.join(); + } + + // TODO wait for actions to complete + + let stop_ok = config.stop(); + + // TODO flush DB + + // TODO remove socket + + exit(match !signal_received.load(Ordering::SeqCst) && stop_ok { true => 0, false => 1, }); diff --git a/rust/src/stream.rs b/rust/src/stream.rs index b8aa7dd..cad0172 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -1,5 +1,11 @@ -use std::collections::BTreeMap; +use std::{ + collections::BTreeMap, + io::{BufRead, BufReader}, + process::{Child, Command, Stdio}, + sync::mpsc, +}; +use log::{debug, error, info}; use serde::Deserialize; use crate::{config::Patterns, filter::Filter}; @@ -47,6 +53,42 @@ impl Stream { Ok(()) } + + pub fn manager(&self, childs_channel: mpsc::SyncSender>) { + info!("{}: start {:?}", self.name, self.cmd); + let mut child = match Command::new(&self.cmd[0]) + .args(&self.cmd[1..]) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .stdout(Stdio::piped()) + .spawn() + { + Ok(child) => child, + Err(err) => { + error!("could not execute stream {} cmd: {}", self.name, err); + let _ = childs_channel.send(None); + return; + } + }; + + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + + // let main handle the child processus + let _ = childs_channel.send(Some(child)); + + let mut line: String = "".into(); + while let Ok(nb_chars) = stdout.read_line(&mut line) { + if nb_chars == 0 { + break; + } + debug!("stream {} stdout: {}", self.name, line); + + // TODO apply each filter on the line + + line.clear(); + } + info!("stream {} exited", self.name); + } } #[cfg(test)] diff --git a/rust/test.jsonnet b/rust/test.jsonnet index 705e867..dd93cda 100644 --- a/rust/test.jsonnet +++ b/rust/test.jsonnet @@ -14,12 +14,15 @@ start: [ ['echo', 'coucou'], - ['echo coucou'], + ], + + stop: [ + ['echo', 'byebye'], ], streams: { tailDown1: { - cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 1; echo found $i; done; sleep 30"], + cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 1; echo found $i; done; sleep 2"], filters: { findIP: { regex: [ @@ -42,7 +45,7 @@ }, }, tailDown2: { - cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 1; echo found $i; done; sleep 30"], + cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 1; echo found $i; done; sleep 3"], filters: { findIP: { regex: [ From eaac4fc341f2078bb06dff0e3df0332ff40a02dc Mon Sep 17 00:00:00 2001 From: ppom Date: Fri, 26 Jul 2024 12:00:00 +0200 Subject: [PATCH 022/435] match & ignore logic (stream/filter/pattern) + tests --- rust/src/config.rs | 11 ++- rust/src/daemon.rs | 19 ++-- rust/src/filter.rs | 221 +++++++++++++++++++++++++++++++++++-------- rust/src/main.rs | 3 +- rust/src/messages.rs | 11 +++ rust/src/pattern.rs | 60 +++++++++++- rust/src/stream.rs | 36 ++++--- 7 files changed, 294 insertions(+), 67 deletions(-) create mode 100644 rust/src/messages.rs diff --git a/rust/src/config.rs b/rust/src/config.rs index 66b40a1..17e07e0 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -1,9 +1,8 @@ -#![allow(dead_code)] - +use std::collections::BTreeMap; use std::fs::File; use std::path::PathBuf; +use std::process::Command; use std::process::Stdio; -use std::{collections::BTreeMap, process::Command}; use anyhow::{anyhow, Context, Result}; use log::{error, info}; @@ -18,7 +17,7 @@ pub type Patterns = BTreeMap; pub struct Config { patterns: Patterns, - pub streams: BTreeMap, + streams: BTreeMap, #[serde(default = "num_cpus::get")] concurrency: usize, @@ -30,6 +29,10 @@ pub struct Config { } impl Config { + pub fn streams(&self) -> &BTreeMap { + &self.streams + } + pub fn setup(&mut self) -> Result<()> { self._setup() .or_else(|msg| Err(anyhow!("Bad configuration: {}", msg))) diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index 138c9be..3387e55 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -1,10 +1,11 @@ +use std::path::PathBuf; use std::process::exit; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::sync_channel; +use std::sync::mpsc::{channel, sync_channel}; +use std::sync::Arc; use std::thread; -use std::{path::PathBuf, sync::Arc}; -use log::{debug, error, Level, info}; +use log::{debug, error, info, Level}; use crate::{config, logger}; @@ -29,16 +30,20 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { exit(1); } + // TODO match manager + let (match_tx, match_rx) = channel(); + let mut stream_process_child_handles = Vec::new(); let mut stream_thread_handles = Vec::new(); - for (_, stream) in &config.streams { + for (_, stream) in config.streams() { let stream = stream.clone(); - let (tx, rx) = sync_channel(0); + let match_tx = match_tx.clone(); + let (child_tx, child_rx) = sync_channel(0); - stream_thread_handles.push(thread::spawn(move || stream.manager(tx))); + stream_thread_handles.push(thread::spawn(move || stream.manager(child_tx, match_tx))); - if let Ok(Some(child)) = rx.recv() { + if let Ok(Some(child)) = child_rx.recv() { stream_process_child_handles.push(child); } } diff --git a/rust/src/filter.rs b/rust/src/filter.rs index ab47668..3122ff1 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -1,12 +1,14 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - time::Duration, -}; +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; +use log::info; use regex::Regex; use serde::Deserialize; -use crate::{action::Action, config::Patterns, parse_duration::parse_duration, pattern::Pattern}; +use crate::{ + action::Action, config::Patterns, messages::Match, parse_duration::parse_duration, + pattern::Pattern, +}; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -81,6 +83,7 @@ impl Filter { return Err("no regex configured".into()); } + let mut first = true; for regex in &self.regex { let mut regex_buf = regex.clone(); for (_pattern_name, pattern) in patterns { @@ -91,12 +94,25 @@ impl Filter { &pattern.name_with_braces )); } - self.patterns.insert(pattern.clone()); + if first { + self.patterns.insert(pattern.clone()); + } else if !self.patterns.contains(&pattern) { + return Err(format!( + "pattern {} is not present in the first regex but is present in a following regex. all regexes should contain the same set of regexes", + &pattern.name_with_braces + )); + } + } else if !first && self.patterns.contains(&pattern) { + return Err(format!( + "pattern {} is present in the first regex but is not present in a following regex. all regexes should contain the same set of regexes", + &pattern.name_with_braces + )); } - regex_buf = regex_buf.replacen(&pattern.name_with_braces, &pattern.regex, 1) + regex_buf = regex_buf.replacen(&pattern.name_with_braces, &pattern.regex, 1); } let compiled = Regex::new(®ex_buf).or_else(|err| Err(err.to_string()))?; self.compiled_regex.push(compiled); + first = false; } self.regex.clear(); @@ -117,13 +133,39 @@ impl Filter { Ok(()) } + + pub fn get_match(&self, line: &String) -> Option { + for regex in &self.compiled_regex { + if let Some(matches) = regex.captures(line) { + if self.patterns.len() > 0 { + let mut result = Match::new(); + for pattern in &self.patterns { + let match_ = matches.name(&pattern.name).expect("logic error"); + if pattern.not_an_ignore(match_.as_str()) { + result.push(match_.as_str().to_string()); + } + } + if result.len() == self.patterns.len() { + info!("{}.{}: match {:?}", self.stream_name, self.name, result); + return Some(result); + } + } else { + info!("{}.{}: match [.]", self.stream_name, self.name); + return Some(vec![".".to_string()]); + } + } + } + None + } } #[cfg(test)] pub mod tests { use crate::action::tests::ok_action; - use crate::pattern::tests::default_pattern; + use crate::pattern::tests::{ + boubou_pattern_with_ignore, default_pattern, ok_pattern_with_ignore, + }; use super::*; @@ -151,7 +193,7 @@ pub mod tests { } #[test] - fn missing_config() { + fn setup_missing_config() { let mut filter; let name = "name".to_string(); let empty_patterns = Patterns::new(); @@ -172,7 +214,7 @@ pub mod tests { } #[test] - fn retry() { + fn setup_retry() { let mut filter; let name = "name".to_string(); let empty_patterns = Patterns::new(); @@ -201,7 +243,7 @@ pub mod tests { } #[test] - fn longuest_action_duration() { + fn setup_longuest_action_duration() { let mut filter; let name = "name".to_string(); let empty_patterns = Patterns::new(); @@ -239,7 +281,7 @@ pub mod tests { } #[test] - fn regexes() { + fn setup_regexes() { let name = "name".to_string(); let mut filter; @@ -258,12 +300,11 @@ pub mod tests { patterns.insert(unused_name.clone(), unused_pattern.clone()); let boubou_name = "boubou".to_string(); - let mut boubou = default_pattern(); - boubou.regex = "(?:bou){2}".to_string(); - assert!(boubou.setup(&boubou_name).is_ok()); + let mut boubou = boubou_pattern_with_ignore(); + boubou.setup(&boubou_name).unwrap(); patterns.insert(boubou_name.clone(), boubou.clone()); - // TODO correct regex replacement + // correct regex replacement filter = default_filter(); filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here$".to_string()); @@ -278,7 +319,7 @@ pub mod tests { let stored_pattern = filter.patterns.first().unwrap(); assert_eq!(stored_pattern.regex, pattern.regex); - // TODO same pattern two times in regex + // same pattern two times in regex filter = default_filter(); filter.actions.insert(name.clone(), ok_action()); filter @@ -286,7 +327,7 @@ pub mod tests { .push("there are two s!".to_string()); assert!(filter.setup(&name, &name, &patterns).is_err()); - // TODO two patterns in one regex + // two patterns in one regex filter = default_filter(); filter.actions.insert(name.clone(), ok_action()); filter @@ -295,7 +336,7 @@ pub mod tests { assert!(filter.setup(&name, &name, &patterns).is_ok()); assert_eq!( filter.compiled_regex[0].to_string(), - Regex::new("insert (?P[abc]) here and (?P(?:bou){2}) there") + Regex::new("insert (?P[abc]) here and (?P(?:bou){1,3}) there") .unwrap() .to_string() ); @@ -305,7 +346,7 @@ pub mod tests { let stored_pattern = filter.patterns.last().unwrap(); assert_eq!(stored_pattern.regex, pattern.regex); - // TODO multiple regexes with same pattern + // multiple regexes with same pattern filter = default_filter(); filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here".to_string()); @@ -327,7 +368,7 @@ pub mod tests { let stored_pattern = filter.patterns.first().unwrap(); assert_eq!(stored_pattern.regex, pattern.regex); - // TODO multiple regexes with same patterns + // multiple regexes with same patterns filter = default_filter(); filter.actions.insert(name.clone(), ok_action()); filter @@ -339,13 +380,13 @@ pub mod tests { assert!(filter.setup(&name, &name, &patterns).is_ok()); assert_eq!( filter.compiled_regex[0].to_string(), - Regex::new("insert (?P[abc]) here and (?P(?:bou){2}) there") + Regex::new("insert (?P[abc]) here and (?P(?:bou){1,3}) there") .unwrap() .to_string() ); assert_eq!( filter.compiled_regex[1].to_string(), - Regex::new("also add (?P(?:bou){2}) here and (?P[abc]) there") + Regex::new("also add (?P(?:bou){1,3}) here and (?P[abc]) there") .unwrap() .to_string() ); @@ -355,28 +396,132 @@ pub mod tests { let stored_pattern = filter.patterns.last().unwrap(); assert_eq!(stored_pattern.regex, pattern.regex); - // TODO multiple regexes with different patterns + // multiple regexes with different patterns 1 filter = default_filter(); filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here".to_string()); filter.regex.push("also add there".to_string()); - assert!(filter.setup(&name, &name, &patterns).is_ok()); + assert!(filter.setup(&name, &name, &patterns).is_err()); + + // multiple regexes with different patterns 2 + filter = default_filter(); + filter.actions.insert(name.clone(), ok_action()); + filter + .regex + .push("insert here and there".to_string()); + filter.regex.push("also add there".to_string()); + assert!(filter.setup(&name, &name, &patterns).is_err()); + + // multiple regexes with different patterns 3 + filter = default_filter(); + filter.actions.insert(name.clone(), ok_action()); + filter.regex.push("also add there".to_string()); + filter + .regex + .push("insert here and there".to_string()); + assert!(filter.setup(&name, &name, &patterns).is_err()); + } + + #[test] + fn get_match() { + let name = "name".to_string(); + let mut filter; + + // make a Patterns + let mut patterns = Patterns::new(); + + let mut pattern = ok_pattern_with_ignore(); + pattern.setup(&name).unwrap(); + patterns.insert(name.clone(), pattern.clone()); + + let boubou_name = "boubou".to_string(); + let mut boubou = boubou_pattern_with_ignore(); + boubou.setup(&boubou_name).unwrap(); + patterns.insert(boubou_name.clone(), boubou.clone()); + + // one simple regex + filter = default_filter(); + filter.actions.insert(name.clone(), ok_action()); + filter.regex.push("insert here$".to_string()); + filter.setup(&name, &name, &patterns).unwrap(); assert_eq!( - filter.compiled_regex[0].to_string(), - Regex::new("insert (?P[abc]) here") - .unwrap() - .to_string() + filter.get_match(&"insert b here".into()), + Some(vec!("b".into())) + ); + assert_eq!(filter.get_match(&"insert a here".into()), None); + assert_eq!(filter.get_match(&"youpi b youpi".into()), None); + assert_eq!(filter.get_match(&"insert here".into()), None); + + // two patterns in one regex + filter = default_filter(); + filter.actions.insert(name.clone(), ok_action()); + filter + .regex + .push("insert here and there".to_string()); + filter.setup(&name, &name, &patterns).unwrap(); + assert_eq!( + filter.get_match(&"insert b here and bouboubou there".into()), + Some(vec!("bouboubou".into(), "b".into())) ); assert_eq!( - filter.compiled_regex[1].to_string(), - Regex::new("also add (?P(?:bou){2}) there") - .unwrap() - .to_string() + filter.get_match(&"insert a here and bouboubou there".into()), + None + ); + assert_eq!( + filter.get_match(&"insert b here and boubou there".into()), + None + ); + + // multiple regexes with same pattern + filter = default_filter(); + filter.actions.insert(name.clone(), ok_action()); + filter.regex.push("insert here".to_string()); + filter.regex.push("also add there".to_string()); + filter.setup(&name, &name, &patterns).unwrap(); + assert_eq!(filter.get_match(&"insert a here".into()), None); + assert_eq!( + filter.get_match(&"insert b here".into()), + Some(vec!("b".into())) + ); + assert_eq!(filter.get_match(&"also add a there".into()), None); + assert_eq!( + filter.get_match(&"also add b there".into()), + Some(vec!("b".into())) + ); + + // multiple regexes with same patterns + filter = default_filter(); + filter.actions.insert(name.clone(), ok_action()); + filter + .regex + .push("insert here and there".to_string()); + filter + .regex + .push("also add here and there".to_string()); + filter.setup(&name, &name, &patterns).unwrap(); + assert_eq!( + filter.get_match(&"insert b here and bouboubou there".into()), + Some(vec!("bouboubou".into(), "b".into())) + ); + assert_eq!( + filter.get_match(&"also add bouboubou here and b there".into()), + Some(vec!("bouboubou".into(), "b".into())) + ); + assert_eq!( + filter.get_match(&"insert a here and bouboubou there".into()), + None + ); + assert_eq!( + filter.get_match(&"also add bouboubou here and a there".into()), + None + ); + assert_eq!( + filter.get_match(&"insert b here and boubou there".into()), + None + ); + assert_eq!( + filter.get_match(&"also add boubou here and b there".into()), + None ); - assert_eq!(filter.patterns.len(), 2); - let stored_pattern = filter.patterns.first().unwrap(); - assert_eq!(stored_pattern.regex, boubou.regex); - let stored_pattern = filter.patterns.last().unwrap(); - assert_eq!(stored_pattern.regex, pattern.regex); } } diff --git a/rust/src/main.rs b/rust/src/main.rs index deb8333..60c82f8 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -13,13 +13,14 @@ use clap::Parser; use regex::Regex; mod action; +mod config; mod filter; +mod messages; mod pattern; mod stream; mod cli; mod client; -mod config; mod daemon; mod logger; mod parse_duration; diff --git a/rust/src/messages.rs b/rust/src/messages.rs new file mode 100644 index 0000000..f3dd567 --- /dev/null +++ b/rust/src/messages.rs @@ -0,0 +1,11 @@ +use std::time::SystemTime; + +use crate::filter::Filter; + +pub type Match = Vec; + +pub struct PFT { + pub m: Match, + pub f: Filter, + pub t: SystemTime, +} diff --git a/rust/src/pattern.rs b/rust/src/pattern.rs index 8e449f5..77ba13f 100644 --- a/rust/src/pattern.rs +++ b/rust/src/pattern.rs @@ -18,7 +18,7 @@ pub struct Pattern { compiled_ignore_regex: Vec, #[serde(skip)] - name: String, + pub name: String, #[serde(skip)] pub name_with_braces: String, } @@ -72,6 +72,20 @@ impl Pattern { Ok(()) } + + pub fn not_an_ignore(&self, match_: &str) -> bool { + for regex in &self.compiled_ignore_regex { + if regex.is_match(match_) { + return false; + } + } + for ignore in &self.ignore { + if ignore == match_ { + return false; + } + } + true + } } // This is required to be added to a BTreeSet @@ -116,8 +130,21 @@ pub mod tests { pattern } + pub fn ok_pattern_with_ignore() -> Pattern { + let mut pattern = ok_pattern(); + pattern.ignore.push("a".into()); + pattern + } + + pub fn boubou_pattern_with_ignore() -> Pattern { + let mut pattern = ok_pattern(); + pattern.regex = "(?:bou){1,3}".to_string(); + pattern.ignore.push("boubou".into()); + pattern + } + #[test] - fn missing_information() { + fn setup_missing_information() { let mut pattern; // Empty name @@ -136,7 +163,7 @@ pub mod tests { } #[test] - fn regex() { + fn setup_regex() { let mut pattern; // regex ok @@ -155,7 +182,7 @@ pub mod tests { } #[test] - fn ignore() { + fn setup_ignore() { let mut pattern; // ignore ok @@ -173,7 +200,7 @@ pub mod tests { } #[test] - fn ignore_regex() { + fn setup_ignore_regex() { let mut pattern; // ignore_regex ok @@ -189,4 +216,27 @@ pub mod tests { pattern.ignore.push("[a".into()); assert!(pattern.setup(&"name".into()).is_err()); } + + #[test] + fn not_an_ignore() { + let mut pattern; + + // ignore ok + pattern = default_pattern(); + pattern.regex = "[abcdefg]".into(); + pattern.ignore.push("a".into()); + pattern.ignore.push("b".into()); + pattern.ignore_regex.push("c".into()); + pattern.ignore_regex.push("[de]".into()); + + pattern.setup(&"name".into()).unwrap(); + assert_eq!(pattern.not_an_ignore("a"), false); + assert_eq!(pattern.not_an_ignore("b"), false); + assert_eq!(pattern.not_an_ignore("c"), false); + assert_eq!(pattern.not_an_ignore("d"), false); + assert_eq!(pattern.not_an_ignore("e"), false); + assert_eq!(pattern.not_an_ignore("f"), true); + assert_eq!(pattern.not_an_ignore("g"), true); + assert_eq!(pattern.not_an_ignore("h"), true); + } } diff --git a/rust/src/stream.rs b/rust/src/stream.rs index cad0172..489dcba 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -1,14 +1,13 @@ -use std::{ - collections::BTreeMap, - io::{BufRead, BufReader}, - process::{Child, Command, Stdio}, - sync::mpsc, -}; +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc::{Sender, SyncSender}; +use std::time::SystemTime; use log::{debug, error, info}; use serde::Deserialize; -use crate::{config::Patterns, filter::Filter}; +use crate::{config::Patterns, filter::Filter, messages::PFT}; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -54,7 +53,7 @@ impl Stream { Ok(()) } - pub fn manager(&self, childs_channel: mpsc::SyncSender>) { + pub fn manager(&self, child_tx: SyncSender>, match_tx: Sender) { info!("{}: start {:?}", self.name, self.cmd); let mut child = match Command::new(&self.cmd[0]) .args(&self.cmd[1..]) @@ -66,15 +65,17 @@ impl Stream { Ok(child) => child, Err(err) => { error!("could not execute stream {} cmd: {}", self.name, err); - let _ = childs_channel.send(None); + let _ = child_tx.send(None); return; } }; + // keep stdout before sending/moving child to the main thread let mut stdout = BufReader::new(child.stdout.take().unwrap()); - // let main handle the child processus - let _ = childs_channel.send(Some(child)); + // let main handle the child process + let _ = child_tx.send(Some(child)); + drop(child_tx); let mut line: String = "".into(); while let Ok(nb_chars) = stdout.read_line(&mut line) { @@ -83,7 +84,18 @@ impl Stream { } debug!("stream {} stdout: {}", self.name, line); - // TODO apply each filter on the line + for (_, filter) in &self.filters { + if let Some(match_) = filter.get_match(&line) { + match_tx + .send(PFT { + m: match_, + // FIXME this clone is a lot :'( + f: filter.clone(), + t: SystemTime::now(), + }) + .unwrap(); + } + } line.clear(); } From 7880e1d87eb586c7a0dd718e57d5c12e7e7aa852 Mon Sep 17 00:00:00 2001 From: ppom Date: Fri, 26 Jul 2024 12:00:00 +0200 Subject: [PATCH 023/435] cargo clippy, panic custom message --- rust/src/action.rs | 31 +++++----- rust/src/cli.rs | 2 +- rust/src/config.rs | 47 ++++++++------- rust/src/daemon.rs | 2 +- rust/src/filter.rs | 115 +++++++++++++++---------------------- rust/src/main.rs | 19 ++++-- rust/src/parse_duration.rs | 18 +++--- rust/src/pattern.rs | 37 ++++++------ rust/src/stream.rs | 23 ++++---- 9 files changed, 134 insertions(+), 160 deletions(-) diff --git a/rust/src/action.rs b/rust/src/action.rs index 127734e..1fe80f0 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -32,42 +32,39 @@ fn set_false() -> bool { impl Action { pub fn setup( &mut self, - stream_name: &String, - filter_name: &String, - name: &String, + stream_name: &str, + filter_name: &str, + name: &str, ) -> Result<(), String> { - self._setup(stream_name, filter_name, name) - .or_else(|msg| Err(format!("action {}: {}", name, msg))) + self._setup(stream_name, filter_name, name).map_err(|msg| format!("action {}: {}", name, msg)) } fn _setup( &mut self, - stream_name: &String, - filter_name: &String, - name: &String, + stream_name: &str, + filter_name: &str, + name: &str, ) -> Result<(), String> { - self.stream_name = stream_name.clone(); - self.filter_name = filter_name.clone(); - self.name = name.clone(); + self.stream_name = stream_name.to_string(); + self.filter_name = filter_name.to_string(); + self.name = name.to_string(); - if self.name.len() == 0 { + if self.name.is_empty() { return Err("action name is empty".into()); } if self.name.contains('.') { return Err("character '.' is not allowed in filter name".into()); } - if self.cmd.len() == 0 { + if self.cmd.is_empty() { return Err("cmd is empty".into()); } - if self.cmd[0].len() == 0 { + if self.cmd[0].is_empty() { return Err("cmd's first item is empty".into()); } if let Some(after) = &self.after { self.after_duration = - Some(parse_duration(after).or_else(|err| { - Err(format!("failed to parse after time: {}", err.to_string())) - })?); + Some(parse_duration(after).map_err(|err| format!("failed to parse after time: {}", err))?); self.after = None; } else if self.on_exit { return Err("cannot have `onexit: true`, without an `after` directive".into()); diff --git a/rust/src/cli.rs b/rust/src/cli.rs index 3de0275..236d432 100644 --- a/rust/src/cli.rs +++ b/rust/src/cli.rs @@ -140,7 +140,7 @@ pub struct NamedRegex { fn parse_named_regex(s: &str) -> Result { let (name, v) = s - .split_once("=") + .split_once('=') .ok_or("When given as a positional argument, a pattern must be prefixed with a name, ex: ip=192.168.0.1")?; let regex = Regex::new(v).map_err(|err| format!("{}", err))?; Ok(NamedRegex { diff --git a/rust/src/config.rs b/rust/src/config.rs index 17e07e0..a267afd 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; use std::fs::File; -use std::path::PathBuf; +use std::path::Path; use std::process::Command; use std::process::Stdio; @@ -35,7 +35,7 @@ impl Config { pub fn setup(&mut self) -> Result<()> { self._setup() - .or_else(|msg| Err(anyhow!("Bad configuration: {}", msg))) + .map_err(|msg| anyhow!("Bad configuration: {}", msg)) } pub fn _setup(&mut self) -> Result<(), String> { @@ -47,7 +47,7 @@ impl Config { value.setup(key)?; } - if self.streams.len() == 0 { + if self.streams.is_empty() { return Err("no streams configured".into()); } @@ -58,33 +58,32 @@ impl Config { } pub fn start(&self) -> bool { - return run_commands(&self.start, "start"); + run_commands(&self.start, "start") } pub fn stop(&self) -> bool { - return run_commands(&self.stop, "stop"); + run_commands(&self.stop, "stop") } } enum Format { - YAML, - JSON, - JSONnet, + Yaml, + Json, + Jsonnet, } -pub fn config_from_file(path: &PathBuf) -> Result { +pub fn config_from_file(path: &Path) -> Result { _config_from_file(path).with_context(|| anyhow!("Configuration file {}:", path.display())) } -fn _config_from_file(path: &PathBuf) -> Result { +fn _config_from_file(path: &Path) -> Result { let extension = path .extension() - .map(|ex| ex.to_str()) - .flatten() + .and_then(|ex| ex.to_str()) .ok_or(anyhow!("no file extension"))?; let format = match extension { - "yaml" | "yml" => Format::YAML, - "json" => Format::JSON, - "jsonnet" => Format::JSONnet, + "yaml" | "yml" => Format::Yaml, + "json" => Format::Json, + "jsonnet" => Format::Jsonnet, _ => { return Err(anyhow!( "extension {} is not recognized. Must be json, jsonnet, yml or yaml.", @@ -94,26 +93,26 @@ fn _config_from_file(path: &PathBuf) -> Result { }; let mut config: Config = match format { - Format::JSON => serde_json::from_reader(File::open(&path)?)?, - Format::YAML => serde_yaml::from_reader(File::open(&path)?)?, - Format::JSONnet => serde_json::from_str(&jsonnet::from_path(&path)?)?, + Format::Json => serde_json::from_reader(File::open(path)?)?, + Format::Yaml => serde_yaml::from_reader(File::open(path)?)?, + Format::Jsonnet => serde_json::from_str(&jsonnet::from_path(path)?)?, }; config.setup()?; - return Ok(config); + Ok(config) } mod jsonnet { - use std::path::PathBuf; + use std::path::Path; use anyhow::{anyhow, Result}; use jrsonnet_evaluator::{error::LocError, EvaluationState, FileImportResolver}; - pub fn from_path(path: &PathBuf) -> Result { + pub fn from_path(path: &Path) -> Result { let state = EvaluationState::default(); state.with_stdlib(); - state.set_import_resolver(Box::new(FileImportResolver::default())); + state.set_import_resolver(Box::::default()); // state.set_import_resolver(Box::new(FileImportResolver { // library_paths: Vec::new(), // })); @@ -123,7 +122,7 @@ mod jsonnet { Err(err) => Err(anyhow!("{}", state.stringify_err(&err))), } } - fn evaluate(path: &PathBuf, state: &EvaluationState) -> Result { + fn evaluate(path: &Path, state: &EvaluationState) -> Result { let val = state.evaluate_file_raw(path)?; let result = state.manifest(val)?; Ok(result.to_string()) @@ -163,7 +162,7 @@ fn run_commands(commands: &Vec>, moment: &str) -> bool { } } - return ok; + ok } #[cfg(test)] diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index 3387e55..aac9adf 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -36,7 +36,7 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { let mut stream_process_child_handles = Vec::new(); let mut stream_thread_handles = Vec::new(); - for (_, stream) in config.streams() { + for stream in config.streams().values() { let stream = stream.clone(); let match_tx = match_tx.clone(); let (child_tx, child_rx) = sync_channel(0); diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 3122ff1..5fb64b4 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -39,24 +39,19 @@ pub struct Filter { impl Filter { pub fn setup( &mut self, - stream_name: &String, - name: &String, + stream_name: &str, + name: &str, patterns: &Patterns, ) -> Result<(), String> { self._setup(stream_name, name, patterns) - .or_else(|msg| Err(format!("filter {}: {}", name, msg))) + .map_err(|msg| format!("filter {}: {}", name, msg)) } - fn _setup( - &mut self, - stream_name: &String, - name: &String, - patterns: &Patterns, - ) -> Result<(), String> { - self.stream_name = stream_name.clone(); - self.name = name.clone(); + fn _setup(&mut self, stream_name: &str, name: &str, patterns: &Patterns) -> Result<(), String> { + self.stream_name = stream_name.to_string(); + self.name = name.to_string(); - if self.name.len() == 0 { + if self.name.is_empty() { return Err("filter name is empty".into()); } if self.name.contains('.') { @@ -72,21 +67,21 @@ impl Filter { } if let Some(retry_period) = &self.retry_period { - self.retry_duration = - Some(parse_duration(retry_period).or_else(|err| { - Err(format!("failed to parse retry time: {}", err.to_string())) - })?); + self.retry_duration = Some( + parse_duration(retry_period) + .map_err(|err| format!("failed to parse retry time: {}", err))?, + ); self.retry_period = None; } - if self.regex.len() == 0 { + if self.regex.is_empty() { return Err("no regex configured".into()); } let mut first = true; for regex in &self.regex { let mut regex_buf = regex.clone(); - for (_pattern_name, pattern) in patterns { + for pattern in patterns.values() { if let Some(index) = regex.find(&pattern.name_with_braces) { if regex.rfind(&pattern.name_with_braces).unwrap() != index { return Err(format!( @@ -96,13 +91,13 @@ impl Filter { } if first { self.patterns.insert(pattern.clone()); - } else if !self.patterns.contains(&pattern) { + } else if !self.patterns.contains(pattern) { return Err(format!( "pattern {} is not present in the first regex but is present in a following regex. all regexes should contain the same set of regexes", &pattern.name_with_braces )); } - } else if !first && self.patterns.contains(&pattern) { + } else if !first && self.patterns.contains(pattern) { return Err(format!( "pattern {} is present in the first regex but is not present in a following regex. all regexes should contain the same set of regexes", &pattern.name_with_braces @@ -110,37 +105,38 @@ impl Filter { } regex_buf = regex_buf.replacen(&pattern.name_with_braces, &pattern.regex, 1); } - let compiled = Regex::new(®ex_buf).or_else(|err| Err(err.to_string()))?; + let compiled = Regex::new(®ex_buf).map_err(|err| err.to_string())?; self.compiled_regex.push(compiled); first = false; } self.regex.clear(); - if self.actions.len() == 0 { + if self.actions.is_empty() { return Err("no actions configured".into()); } for (key, value) in &mut self.actions { - value.setup(&stream_name, &name, key)?; + value.setup(stream_name, name, key)?; } - self.longuest_action_duration = self - .actions - .iter() - .map(|(_, v)| v.after_duration.unwrap_or(Duration::from_secs(0))) - .max() - .unwrap(); // safe because we already tested that actions.len() > 0 + self.longuest_action_duration = + self.actions + .values() + .fold(Duration::from_secs(0), |acc, v| { + v.after_duration + .map_or(acc, |v| if v > acc { v } else { acc }) + }); Ok(()) } - pub fn get_match(&self, line: &String) -> Option { + pub fn get_match(&self, line: &str) -> Option { for regex in &self.compiled_regex { if let Some(matches) = regex.captures(line) { - if self.patterns.len() > 0 { + if !self.patterns.is_empty() { let mut result = Match::new(); for pattern in &self.patterns { - let match_ = matches.name(&pattern.name).expect("logic error"); + let match_ = matches.name(&pattern.name).unwrap(); if pattern.not_an_ignore(match_.as_str()) { result.push(match_.as_str().to_string()); } @@ -444,13 +440,10 @@ pub mod tests { filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here$".to_string()); filter.setup(&name, &name, &patterns).unwrap(); - assert_eq!( - filter.get_match(&"insert b here".into()), - Some(vec!("b".into())) - ); - assert_eq!(filter.get_match(&"insert a here".into()), None); - assert_eq!(filter.get_match(&"youpi b youpi".into()), None); - assert_eq!(filter.get_match(&"insert here".into()), None); + assert_eq!(filter.get_match(&"insert b here"), Some(vec!("b".into()))); + assert_eq!(filter.get_match(&"insert a here"), None); + assert_eq!(filter.get_match(&"youpi b youpi"), None); + assert_eq!(filter.get_match(&"insert here"), None); // two patterns in one regex filter = default_filter(); @@ -460,17 +453,11 @@ pub mod tests { .push("insert here and there".to_string()); filter.setup(&name, &name, &patterns).unwrap(); assert_eq!( - filter.get_match(&"insert b here and bouboubou there".into()), + filter.get_match(&"insert b here and bouboubou there"), Some(vec!("bouboubou".into(), "b".into())) ); - assert_eq!( - filter.get_match(&"insert a here and bouboubou there".into()), - None - ); - assert_eq!( - filter.get_match(&"insert b here and boubou there".into()), - None - ); + assert_eq!(filter.get_match(&"insert a here and bouboubou there"), None); + assert_eq!(filter.get_match(&"insert b here and boubou there"), None); // multiple regexes with same pattern filter = default_filter(); @@ -478,14 +465,11 @@ pub mod tests { filter.regex.push("insert here".to_string()); filter.regex.push("also add there".to_string()); filter.setup(&name, &name, &patterns).unwrap(); - assert_eq!(filter.get_match(&"insert a here".into()), None); + assert_eq!(filter.get_match(&"insert a here"), None); + assert_eq!(filter.get_match(&"insert b here"), Some(vec!("b".into()))); + assert_eq!(filter.get_match(&"also add a there"), None); assert_eq!( - filter.get_match(&"insert b here".into()), - Some(vec!("b".into())) - ); - assert_eq!(filter.get_match(&"also add a there".into()), None); - assert_eq!( - filter.get_match(&"also add b there".into()), + filter.get_match(&"also add b there"), Some(vec!("b".into())) ); @@ -500,28 +484,19 @@ pub mod tests { .push("also add here and there".to_string()); filter.setup(&name, &name, &patterns).unwrap(); assert_eq!( - filter.get_match(&"insert b here and bouboubou there".into()), + filter.get_match(&"insert b here and bouboubou there"), Some(vec!("bouboubou".into(), "b".into())) ); assert_eq!( - filter.get_match(&"also add bouboubou here and b there".into()), + filter.get_match(&"also add bouboubou here and b there"), Some(vec!("bouboubou".into(), "b".into())) ); + assert_eq!(filter.get_match(&"insert a here and bouboubou there"), None); assert_eq!( - filter.get_match(&"insert a here and bouboubou there".into()), - None - ); - assert_eq!( - filter.get_match(&"also add bouboubou here and a there".into()), - None - ); - assert_eq!( - filter.get_match(&"insert b here and boubou there".into()), - None - ); - assert_eq!( - filter.get_match(&"also add boubou here and b there".into()), + filter.get_match(&"also add bouboubou here and a there"), None ); + assert_eq!(filter.get_match(&"insert b here and boubou there"), None); + assert_eq!(filter.get_match(&"also add boubou here and b there"), None); } } diff --git a/rust/src/main.rs b/rust/src/main.rs index 60c82f8..07c7b07 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -1,16 +1,15 @@ #![warn( missing_docs, - clippy::unwrap_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::undocumented_unsafe_blocks )] +#![allow(clippy::upper_case_acronyms)] #![forbid(unsafe_code)] //! Hey it's the doc! use clap::Parser; -use regex::Regex; mod action; mod config; @@ -30,10 +29,18 @@ use client::{flush, show, test_regex}; use daemon::daemon; fn main() { - cli::NamedRegex { - regex: Regex::new(".").unwrap(), - name: "test".to_string(), - }; + // Show a nice message when reaction panics + std::panic::set_hook(Box::new(move |panic_info| { + eprintln!("ERROR internal reaction error: panic"); + eprintln!("{}", panic_info); + eprintln!(); + eprintln!("This is likely a bug in reaction. Please report it on the issue tracker:"); + eprintln!("https://framagit.org/ppom/reaction/-/issues"); + eprintln!(); + eprintln!("Please include the last log messages in your bug report, as well as any"); + eprintln!("relevant information on the context in which reaction was running when"); + eprintln!("this error occurred."); + })); let cli = Cli::parse(); diff --git a/rust/src/parse_duration.rs b/rust/src/parse_duration.rs index 0f307e4..7faff3b 100644 --- a/rust/src/parse_duration.rs +++ b/rust/src/parse_duration.rs @@ -6,7 +6,7 @@ pub fn parse_duration(d: &str) -> Result { let mut value = 0; let mut i = 0; while i < chars.len() && chars[i].is_ascii_digit() { - value = value * 10 + (chars[i] - '0' as u8) as u32; + value = value * 10 + (chars[i] - b'0') as u32; i += 1; } if i == 0 { @@ -37,7 +37,7 @@ mod tests { #[test] fn char_conversion() { - assert_eq!('9' as u8 - '0' as u8, 9); + assert_eq!(b'9' - b'0', 9); } #[test] @@ -56,12 +56,12 @@ mod tests { Ok(Duration::from_secs(365 * 24 * 60 * 60)) ); - assert_eq!(parse_duration("d 3").is_err(), true); - assert_eq!(parse_duration("d3").is_err(), true); - assert_eq!(parse_duration("3da").is_err(), true); - assert_eq!(parse_duration("3_days").is_err(), true); - assert_eq!(parse_duration("_3d").is_err(), true); - assert_eq!(parse_duration("3 3d").is_err(), true); - assert_eq!(parse_duration("3.3d").is_err(), true); + assert!(parse_duration("d 3").is_err()); + assert!(parse_duration("d3").is_err()); + assert!(parse_duration("3da").is_err()); + assert!(parse_duration("3_days").is_err()); + assert!(parse_duration("_3d").is_err()); + assert!(parse_duration("3 3d").is_err()); + assert!(parse_duration("3.3d").is_err()); } } diff --git a/rust/src/pattern.rs b/rust/src/pattern.rs index 77ba13f..afd0637 100644 --- a/rust/src/pattern.rs +++ b/rust/src/pattern.rs @@ -25,31 +25,30 @@ pub struct Pattern { impl Pattern { pub fn setup(&mut self, name: &String) -> Result<(), String> { - self._setup(name) - .or_else(|msg| Err(format!("pattern {}: {}", name, msg))) + self._setup(name).map_err(|msg| format!("pattern {}: {}", name, msg)) } pub fn _setup(&mut self, name: &String) -> Result<(), String> { self.name = name.clone(); self.name_with_braces = format!("<{}>", name); - if self.name.len() == 0 { + if self.name.is_empty() { return Err("pattern name is empty".into()); } - if self.name.contains(".") { + if self.name.contains('.') { return Err("character '.' is not allowed in pattern name".into()); } - if self.regex.len() == 0 { + if self.regex.is_empty() { return Err("regex is empty".into()); } let compiled = - Regex::new(&format!("^{}$", self.regex)).or_else(|err| Err(err.to_string()))?; + Regex::new(&format!("^{}$", self.regex)).map_err(|err| err.to_string())?; self.regex = format!("(?P<{}>{})", self.name, self.regex); for ignore in &self.ignore { - if !compiled.is_match(&ignore) { + if !compiled.is_match(ignore) { return Err(format!( "ignore '{}' doesn't match pattern. It should be fixed or removed.", ignore, @@ -58,13 +57,11 @@ impl Pattern { } for ignore_regex in &self.ignore_regex { - let compiled_ignore = Regex::new(&format!("^{}$", ignore_regex)).or_else(|err| { - Err(format!( + let compiled_ignore = Regex::new(&format!("^{}$", ignore_regex)).map_err(|err| format!( "ignoreregex '{}': {}", ignore_regex, - err.to_string() - )) - })?; + err + ))?; self.compiled_ignore_regex.push(compiled_ignore); } @@ -230,13 +227,13 @@ pub mod tests { pattern.ignore_regex.push("[de]".into()); pattern.setup(&"name".into()).unwrap(); - assert_eq!(pattern.not_an_ignore("a"), false); - assert_eq!(pattern.not_an_ignore("b"), false); - assert_eq!(pattern.not_an_ignore("c"), false); - assert_eq!(pattern.not_an_ignore("d"), false); - assert_eq!(pattern.not_an_ignore("e"), false); - assert_eq!(pattern.not_an_ignore("f"), true); - assert_eq!(pattern.not_an_ignore("g"), true); - assert_eq!(pattern.not_an_ignore("h"), true); + assert!(!pattern.not_an_ignore("a")); + assert!(!pattern.not_an_ignore("b")); + assert!(!pattern.not_an_ignore("c")); + assert!(!pattern.not_an_ignore("d")); + assert!(!pattern.not_an_ignore("e")); + assert!(pattern.not_an_ignore("f")); + assert!(pattern.not_an_ignore("g")); + assert!(pattern.not_an_ignore("h")); } } diff --git a/rust/src/stream.rs b/rust/src/stream.rs index 489dcba..fc6f69c 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -20,34 +20,33 @@ pub struct Stream { } impl Stream { - pub fn setup(&mut self, name: &String, patterns: &Patterns) -> Result<(), String> { - self._setup(name, patterns) - .or_else(|msg| Err(format!("stream {}: {}", name, msg))) + pub fn setup(&mut self, name: &str, patterns: &Patterns) -> Result<(), String> { + self._setup(name, patterns).map_err(|msg| format!("stream {}: {}", name, msg)) } - fn _setup(&mut self, name: &String, patterns: &Patterns) -> Result<(), String> { - self.name = name.clone(); + fn _setup(&mut self, name: &str, patterns: &Patterns) -> Result<(), String> { + self.name = name.to_string(); - if self.name.len() == 0 { + if self.name.is_empty() { return Err("stream name is empty".into()); } if self.name.contains('.') { return Err("character '.' is not allowed in stream name".into()); } - if self.cmd.len() == 0 { + if self.cmd.is_empty() { return Err("cmd is empty".into()); } - if self.cmd[0].len() == 0 { + if self.cmd[0].is_empty() { return Err("cmd's first item is empty".into()); } - if self.filters.len() == 0 { + if self.filters.is_empty() { return Err("no filters configured".into()); } for (key, value) in &mut self.filters { - value.setup(&name, key, patterns)?; + value.setup(name, key, patterns)?; } Ok(()) @@ -84,7 +83,7 @@ impl Stream { } debug!("stream {} stdout: {}", self.name, line); - for (_, filter) in &self.filters { + for filter in self.filters.values() { if let Some(match_) = filter.get_match(&line) { match_tx .send(PFT { @@ -128,7 +127,7 @@ pub mod tests { #[test] fn test() { let mut stream; - let name = "name".into(); + let name = "name"; // missing cmd stream = ok_stream(); From 25cce7522bfc274f21f258ba1c4333fcd7a0b229 Mon Sep 17 00:00:00 2001 From: ppom Date: Fri, 26 Jul 2024 12:00:00 +0200 Subject: [PATCH 024/435] fix newline kept in stream read_line() --- rust/src/stream.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/src/stream.rs b/rust/src/stream.rs index fc6f69c..850318d 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -81,6 +81,7 @@ impl Stream { if nb_chars == 0 { break; } + line.pop(); // remove trailing newline debug!("stream {} stdout: {}", self.name, line); for filter in self.filters.values() { From c077767f9a596516d1594387b5e86e2b92986212 Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 27 Jul 2024 12:00:00 +0200 Subject: [PATCH 025/435] more clippy --- rust/src/action.rs | 16 +++++++--------- rust/src/filter.rs | 37 +++++++++++++++++-------------------- rust/src/pattern.rs | 13 +++++-------- rust/src/stream.rs | 11 ++++++----- 4 files changed, 35 insertions(+), 42 deletions(-) diff --git a/rust/src/action.rs b/rust/src/action.rs index 1fe80f0..0e077ee 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -36,14 +36,10 @@ impl Action { filter_name: &str, name: &str, ) -> Result<(), String> { - self._setup(stream_name, filter_name, name).map_err(|msg| format!("action {}: {}", name, msg)) + self._setup(stream_name, filter_name, name) + .map_err(|msg| format!("action {}: {}", name, msg)) } - fn _setup( - &mut self, - stream_name: &str, - filter_name: &str, - name: &str, - ) -> Result<(), String> { + fn _setup(&mut self, stream_name: &str, filter_name: &str, name: &str) -> Result<(), String> { self.stream_name = stream_name.to_string(); self.filter_name = filter_name.to_string(); self.name = name.to_string(); @@ -63,8 +59,10 @@ impl Action { } if let Some(after) = &self.after { - self.after_duration = - Some(parse_duration(after).map_err(|err| format!("failed to parse after time: {}", err))?); + self.after_duration = Some( + parse_duration(after) + .map_err(|err| format!("failed to parse after time: {}", err))?, + ); self.after = None; } else if self.on_exit { return Err("cannot have `onexit: true`, without an `after` directive".into()); diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 5fb64b4..195be00 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -440,10 +440,10 @@ pub mod tests { filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here$".to_string()); filter.setup(&name, &name, &patterns).unwrap(); - assert_eq!(filter.get_match(&"insert b here"), Some(vec!("b".into()))); - assert_eq!(filter.get_match(&"insert a here"), None); - assert_eq!(filter.get_match(&"youpi b youpi"), None); - assert_eq!(filter.get_match(&"insert here"), None); + assert_eq!(filter.get_match("insert b here"), Some(vec!("b".into()))); + assert_eq!(filter.get_match("insert a here"), None); + assert_eq!(filter.get_match("youpi b youpi"), None); + assert_eq!(filter.get_match("insert here"), None); // two patterns in one regex filter = default_filter(); @@ -453,11 +453,11 @@ pub mod tests { .push("insert here and there".to_string()); filter.setup(&name, &name, &patterns).unwrap(); assert_eq!( - filter.get_match(&"insert b here and bouboubou there"), + filter.get_match("insert b here and bouboubou there"), Some(vec!("bouboubou".into(), "b".into())) ); - assert_eq!(filter.get_match(&"insert a here and bouboubou there"), None); - assert_eq!(filter.get_match(&"insert b here and boubou there"), None); + assert_eq!(filter.get_match("insert a here and bouboubou there"), None); + assert_eq!(filter.get_match("insert b here and boubou there"), None); // multiple regexes with same pattern filter = default_filter(); @@ -465,13 +465,10 @@ pub mod tests { filter.regex.push("insert here".to_string()); filter.regex.push("also add there".to_string()); filter.setup(&name, &name, &patterns).unwrap(); - assert_eq!(filter.get_match(&"insert a here"), None); - assert_eq!(filter.get_match(&"insert b here"), Some(vec!("b".into()))); - assert_eq!(filter.get_match(&"also add a there"), None); - assert_eq!( - filter.get_match(&"also add b there"), - Some(vec!("b".into())) - ); + assert_eq!(filter.get_match("insert a here"), None); + assert_eq!(filter.get_match("insert b here"), Some(vec!("b".into()))); + assert_eq!(filter.get_match("also add a there"), None); + assert_eq!(filter.get_match("also add b there"), Some(vec!("b".into()))); // multiple regexes with same patterns filter = default_filter(); @@ -484,19 +481,19 @@ pub mod tests { .push("also add here and there".to_string()); filter.setup(&name, &name, &patterns).unwrap(); assert_eq!( - filter.get_match(&"insert b here and bouboubou there"), + filter.get_match("insert b here and bouboubou there"), Some(vec!("bouboubou".into(), "b".into())) ); assert_eq!( - filter.get_match(&"also add bouboubou here and b there"), + filter.get_match("also add bouboubou here and b there"), Some(vec!("bouboubou".into(), "b".into())) ); - assert_eq!(filter.get_match(&"insert a here and bouboubou there"), None); + assert_eq!(filter.get_match("insert a here and bouboubou there"), None); assert_eq!( - filter.get_match(&"also add bouboubou here and a there"), + filter.get_match("also add bouboubou here and a there"), None ); - assert_eq!(filter.get_match(&"insert b here and boubou there"), None); - assert_eq!(filter.get_match(&"also add boubou here and b there"), None); + assert_eq!(filter.get_match("insert b here and boubou there"), None); + assert_eq!(filter.get_match("also add boubou here and b there"), None); } } diff --git a/rust/src/pattern.rs b/rust/src/pattern.rs index afd0637..4d0445f 100644 --- a/rust/src/pattern.rs +++ b/rust/src/pattern.rs @@ -25,7 +25,8 @@ pub struct Pattern { impl Pattern { pub fn setup(&mut self, name: &String) -> Result<(), String> { - self._setup(name).map_err(|msg| format!("pattern {}: {}", name, msg)) + self._setup(name) + .map_err(|msg| format!("pattern {}: {}", name, msg)) } pub fn _setup(&mut self, name: &String) -> Result<(), String> { @@ -42,8 +43,7 @@ impl Pattern { if self.regex.is_empty() { return Err("regex is empty".into()); } - let compiled = - Regex::new(&format!("^{}$", self.regex)).map_err(|err| err.to_string())?; + let compiled = Regex::new(&format!("^{}$", self.regex)).map_err(|err| err.to_string())?; self.regex = format!("(?P<{}>{})", self.name, self.regex); @@ -57,11 +57,8 @@ impl Pattern { } for ignore_regex in &self.ignore_regex { - let compiled_ignore = Regex::new(&format!("^{}$", ignore_regex)).map_err(|err| format!( - "ignoreregex '{}': {}", - ignore_regex, - err - ))?; + let compiled_ignore = Regex::new(&format!("^{}$", ignore_regex)) + .map_err(|err| format!("ignoreregex '{}': {}", ignore_regex, err))?; self.compiled_ignore_regex.push(compiled_ignore); } diff --git a/rust/src/stream.rs b/rust/src/stream.rs index 850318d..3a11fbe 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -21,7 +21,8 @@ pub struct Stream { impl Stream { pub fn setup(&mut self, name: &str, patterns: &Patterns) -> Result<(), String> { - self._setup(name, patterns).map_err(|msg| format!("stream {}: {}", name, msg)) + self._setup(name, patterns) + .map_err(|msg| format!("stream {}: {}", name, msg)) } fn _setup(&mut self, name: &str, patterns: &Patterns) -> Result<(), String> { @@ -133,20 +134,20 @@ pub mod tests { // missing cmd stream = ok_stream(); stream.cmd = Vec::new(); - assert!(stream.setup(&name, &BTreeMap::new()).is_err()); + assert!(stream.setup(name, &BTreeMap::new()).is_err()); // missing cmd stream = ok_stream(); stream.cmd = vec!["".into(), "arg1".into()]; - assert!(stream.setup(&name, &BTreeMap::new()).is_err()); + assert!(stream.setup(name, &BTreeMap::new()).is_err()); // missing filters stream = ok_stream(); stream.filters = BTreeMap::new(); - assert!(stream.setup(&name, &BTreeMap::new()).is_err()); + assert!(stream.setup(name, &BTreeMap::new()).is_err()); // stream ok stream = ok_stream(); - assert!(stream.setup(&name, &BTreeMap::new()).is_ok()); + assert!(stream.setup(name, &BTreeMap::new()).is_ok()); } } From bf3bd6e42fecbfaff41e379ade01f258d6c5a0f8 Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 27 Jul 2024 12:00:00 +0200 Subject: [PATCH 026/435] matches_manager logic still has to implement timers --- rust/src/action.rs | 20 +++++++++++++++++ rust/src/config.rs | 6 +++++ rust/src/daemon.rs | 30 ++++++++++++++++--------- rust/src/filter.rs | 40 +++++++++++++++++++++++++++++---- rust/src/main.rs | 11 +++++++-- rust/src/matches.rs | 53 ++++++++++++++++++++++++++++++++++++++++++++ rust/src/messages.rs | 17 +++++++++++--- rust/src/stream.rs | 13 ++++++----- 8 files changed, 166 insertions(+), 24 deletions(-) create mode 100644 rust/src/matches.rs diff --git a/rust/src/action.rs b/rust/src/action.rs index 0e077ee..0752525 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -30,6 +30,14 @@ fn set_false() -> bool { } impl Action { + pub fn name(&self) -> ActionName { + ActionName { + stream: self.stream_name.clone(), + filter: self.filter_name.clone(), + action: self.name.clone(), + } + } + pub fn setup( &mut self, stream_name: &str, @@ -72,6 +80,18 @@ impl Action { } } +#[derive(PartialEq, Eq, PartialOrd, Ord)] +pub struct ActionName { + pub stream: String, + pub filter: String, + pub action: String, +} +impl ToString for ActionName { + fn to_string(&self) -> String { + format!("{}.{}.{}", self.stream, self.filter, self.action) + } +} + #[cfg(test)] pub mod tests { diff --git a/rust/src/config.rs b/rust/src/config.rs index a267afd..e3b17ba 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -8,6 +8,8 @@ use anyhow::{anyhow, Context, Result}; use log::{error, info}; use serde::Deserialize; +use crate::filter::Filter; +use crate::filter::FilterName; use crate::{pattern::Pattern, stream::Stream}; pub type Patterns = BTreeMap; @@ -33,6 +35,10 @@ impl Config { &self.streams } + pub fn get_filter(&self, name: &FilterName) -> Option<&Filter> { + self.streams.get(&name.stream)?.get_filter(&name.filter) + } + pub fn setup(&mut self) -> Result<()> { self._setup() .map_err(|msg| anyhow!("Bad configuration: {}", msg)) diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index aac9adf..c2cd712 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -7,6 +7,7 @@ use std::thread; use log::{debug, error, info, Level}; +use crate::matches::matches_manager; use crate::{config, logger}; pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { @@ -30,21 +31,30 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { exit(1); } - // TODO match manager - let (match_tx, match_rx) = channel(); - let mut stream_process_child_handles = Vec::new(); let mut stream_thread_handles = Vec::new(); - for stream in config.streams().values() { - let stream = stream.clone(); - let match_tx = match_tx.clone(); - let (child_tx, child_rx) = sync_channel(0); + { + let (match_tx, match_rx) = channel(); + let (action_tx, _action_rx) = channel(); - stream_thread_handles.push(thread::spawn(move || stream.manager(child_tx, match_tx))); + let config_matches = config.clone(); + stream_thread_handles.push(thread::spawn(move || { + matches_manager(config_matches, match_rx, action_tx) + })); - if let Ok(Some(child)) = child_rx.recv() { - stream_process_child_handles.push(child); + // TODO action manager + + for stream in config.streams().values() { + let stream = stream.clone(); + let match_tx = match_tx.clone(); + let (child_tx, child_rx) = sync_channel(0); + + stream_thread_handles.push(thread::spawn(move || stream.manager(child_tx, match_tx))); + + if let Ok(Some(child)) = child_rx.recv() { + stream_process_child_handles.push(child); + } } } diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 195be00..ce7704e 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -1,10 +1,13 @@ use std::collections::{BTreeMap, BTreeSet}; -use std::time::Duration; +use std::fmt::Display; +use std::sync::mpsc::Sender; +use std::time::{Duration, SystemTime}; use log::info; use regex::Regex; use serde::Deserialize; +use crate::messages::MAT; use crate::{ action::Action, config::Patterns, messages::Match, parse_duration::parse_duration, pattern::Pattern, @@ -25,7 +28,7 @@ pub struct Filter { #[serde(skip)] patterns: BTreeSet, - retry: Option, + pub retry: Option, #[serde(rename = "retryperiod")] retry_period: Option, retry_duration: Option, @@ -37,6 +40,13 @@ pub struct Filter { } impl Filter { + pub fn name(&self) -> FilterName { + FilterName { + stream: self.stream_name.clone(), + filter: self.name.clone(), + } + } + pub fn setup( &mut self, stream_name: &str, @@ -142,17 +152,39 @@ impl Filter { } } if result.len() == self.patterns.len() { - info!("{}.{}: match {:?}", self.stream_name, self.name, result); + info!("{}: match {:?}", self.name(), result); return Some(result); } } else { - info!("{}.{}: match [.]", self.stream_name, self.name); + info!("{}: match []", self.name()); return Some(vec![".".to_string()]); } } } None } + + pub fn send_actions(&self, m: &Match, t: SystemTime, tx: &Sender) { + for action in self.actions.values() { + tx.send(MAT { + m: m.clone(), + a: action.name(), + t, + }) + .unwrap(); + } + } +} + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct FilterName { + pub stream: String, + pub filter: String, +} +impl Display for FilterName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}", self.stream, self.filter) + } } #[cfg(test)] diff --git a/rust/src/main.rs b/rust/src/main.rs index 07c7b07..9efd619 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -8,9 +8,10 @@ #![allow(clippy::upper_case_acronyms)] #![forbid(unsafe_code)] -//! Hey it's the doc! +//! TODO document a bit use clap::Parser; +// important structs and concepts mod action; mod config; mod filter; @@ -18,9 +19,15 @@ mod messages; mod pattern; mod stream; -mod cli; +// important threads +mod matches; + +// client & daemon "mains" mod client; mod daemon; + +// utils +mod cli; mod logger; mod parse_duration; diff --git a/rust/src/matches.rs b/rust/src/matches.rs new file mode 100644 index 0000000..95622cd --- /dev/null +++ b/rust/src/matches.rs @@ -0,0 +1,53 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{ + mpsc::{Receiver, Sender}, + Arc, + }, + time::SystemTime, +}; + +use crate::{ + config::Config, + filter::FilterName, + messages::{Match, MAT, MFT}, +}; + +type MatchesMap = BTreeMap>>; + +// TODO handle flushes +pub fn matches_manager(config: Arc, match_rx: Receiver, action_tx: Sender) { + let mut matches: MatchesMap = BTreeMap::new(); + + while let Ok(mft) = match_rx.recv() { + let filter = config.get_filter(&mft.f).unwrap(); + + let is_retry = filter.retry.is_some(); + + // Store matches + if is_retry { + // Make sure collections exist + let inner_map = matches.entry(mft.m.clone()).or_default(); + let inner_set = inner_map.entry(mft.f.clone()).or_default(); + // Add new match + inner_set.insert(mft.t); + // Remove match when expired + // TODO timer that only sends it back to matches_manager + // https://docs.rs/timer/latest/timer/struct.Timer.html + // replace Receiver with Receiver + } + + // Executing actions + let exec = !is_retry || matches.get(&mft.m).map(|map| map.get(&mft.f)).is_some(); + if exec { + // Delete matches only if storing them + if is_retry { + matches.get_mut(&mft.m).unwrap().remove(&mft.f); + } + filter.send_actions(&mft.m, mft.t, &action_tx); + } + + // TODO send to DB + // log_tx.send(LogEntry { mft, exec }); + } +} diff --git a/rust/src/messages.rs b/rust/src/messages.rs index f3dd567..0cd5999 100644 --- a/rust/src/messages.rs +++ b/rust/src/messages.rs @@ -1,11 +1,22 @@ use std::time::SystemTime; -use crate::filter::Filter; +use crate::{action::ActionName, filter::FilterName}; pub type Match = Vec; -pub struct PFT { +pub struct MFT { pub m: Match, - pub f: Filter, + pub f: FilterName, pub t: SystemTime, } + +pub struct MAT { + pub m: Match, + pub a: ActionName, + pub t: SystemTime, +} + +// pub struct LogEntry { +// pub mft: MFT, +// pub exec: bool, +// } diff --git a/rust/src/stream.rs b/rust/src/stream.rs index 3a11fbe..ef6f126 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -7,7 +7,7 @@ use std::time::SystemTime; use log::{debug, error, info}; use serde::Deserialize; -use crate::{config::Patterns, filter::Filter, messages::PFT}; +use crate::{config::Patterns, filter::Filter, messages::MFT}; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -20,6 +20,10 @@ pub struct Stream { } impl Stream { + pub fn get_filter(&self, name: &str) -> Option<&Filter> { + self.filters.get(name) + } + pub fn setup(&mut self, name: &str, patterns: &Patterns) -> Result<(), String> { self._setup(name, patterns) .map_err(|msg| format!("stream {}: {}", name, msg)) @@ -53,7 +57,7 @@ impl Stream { Ok(()) } - pub fn manager(&self, child_tx: SyncSender>, match_tx: Sender) { + pub fn manager(&self, child_tx: SyncSender>, match_tx: Sender) { info!("{}: start {:?}", self.name, self.cmd); let mut child = match Command::new(&self.cmd[0]) .args(&self.cmd[1..]) @@ -88,10 +92,9 @@ impl Stream { for filter in self.filters.values() { if let Some(match_) = filter.get_match(&line) { match_tx - .send(PFT { + .send(MFT { m: match_, - // FIXME this clone is a lot :'( - f: filter.clone(), + f: filter.name(), t: SystemTime::now(), }) .unwrap(); From b7c56881a974def09da5e415c143aeb587cdd48d Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 27 Jul 2024 12:00:00 +0200 Subject: [PATCH 027/435] `use` directives normalization --- rust/src/cli.rs | 3 +-- rust/src/config.rs | 19 +++++++++++-------- rust/src/daemon.rs | 21 ++++++++++++--------- rust/src/filter.rs | 16 ++++++++++------ rust/src/main.rs | 6 +++--- rust/src/pattern.rs | 1 - rust/src/stream.rs | 12 +++++++----- 7 files changed, 44 insertions(+), 34 deletions(-) diff --git a/rust/src/cli.rs b/rust/src/cli.rs index 236d432..0468e84 100644 --- a/rust/src/cli.rs +++ b/rust/src/cli.rs @@ -1,5 +1,4 @@ -use std::fmt; -use std::path::PathBuf; +use std::{fmt, path::PathBuf}; use clap::{Parser, Subcommand, ValueEnum}; use regex::Regex; diff --git a/rust/src/config.rs b/rust/src/config.rs index e3b17ba..6830356 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -1,16 +1,19 @@ -use std::collections::BTreeMap; -use std::fs::File; -use std::path::Path; -use std::process::Command; -use std::process::Stdio; +use std::{ + collections::BTreeMap, + fs::File, + path::Path, + process::{Command, Stdio}, +}; use anyhow::{anyhow, Context, Result}; use log::{error, info}; use serde::Deserialize; -use crate::filter::Filter; -use crate::filter::FilterName; -use crate::{pattern::Pattern, stream::Stream}; +use crate::{ + filter::{Filter, FilterName}, + pattern::Pattern, + stream::Stream, +}; pub type Patterns = BTreeMap; diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index c2cd712..91fb0cd 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -1,14 +1,17 @@ -use std::path::PathBuf; -use std::process::exit; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::mpsc::{channel, sync_channel}; -use std::sync::Arc; -use std::thread; +use std::{ + path::PathBuf, + process::exit, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{channel, sync_channel}, + Arc, + }, + thread, +}; use log::{debug, error, info, Level}; -use crate::matches::matches_manager; -use crate::{config, logger}; +use crate::{config, logger, matches::matches_manager}; pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { if let Err(err) = logger::SimpleLogger::init(loglevel) { @@ -43,7 +46,7 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { matches_manager(config_matches, match_rx, action_tx) })); - // TODO action manager + // TODO execs manager for stream in config.streams().values() { let stream = stream.clone(); diff --git a/rust/src/filter.rs b/rust/src/filter.rs index ce7704e..7fce6d1 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -1,15 +1,19 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::fmt::Display; -use std::sync::mpsc::Sender; -use std::time::{Duration, SystemTime}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt::Display, + sync::mpsc::Sender, + time::{Duration, SystemTime}, +}; use log::info; use regex::Regex; use serde::Deserialize; -use crate::messages::MAT; use crate::{ - action::Action, config::Patterns, messages::Match, parse_duration::parse_duration, + action::Action, + config::Patterns, + messages::{Match, MAT}, + parse_duration::parse_duration, pattern::Pattern, }; diff --git a/rust/src/main.rs b/rust/src/main.rs index 9efd619..4e75381 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -11,7 +11,7 @@ //! TODO document a bit use clap::Parser; -// important structs and concepts +// structs and concepts mod action; mod config; mod filter; @@ -22,12 +22,12 @@ mod stream; // important threads mod matches; -// client & daemon "mains" +// top-level +mod cli; mod client; mod daemon; // utils -mod cli; mod logger; mod parse_duration; diff --git a/rust/src/pattern.rs b/rust/src/pattern.rs index 4d0445f..1a7ade2 100644 --- a/rust/src/pattern.rs +++ b/rust/src/pattern.rs @@ -1,7 +1,6 @@ use std::cmp::Ordering; use regex::Regex; - use serde::Deserialize; #[derive(Clone, Debug, Deserialize)] diff --git a/rust/src/stream.rs b/rust/src/stream.rs index ef6f126..e2e5749 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -1,8 +1,10 @@ -use std::collections::BTreeMap; -use std::io::{BufRead, BufReader}; -use std::process::{Child, Command, Stdio}; -use std::sync::mpsc::{Sender, SyncSender}; -use std::time::SystemTime; +use std::{ + collections::BTreeMap, + io::{BufRead, BufReader}, + process::{Child, Command, Stdio}, + sync::mpsc::{Sender, SyncSender}, + time::SystemTime, +}; use log::{debug, error, info}; use serde::Deserialize; From 77be68c3a4aa5afa41e55ccfc8b5f2dc3802be91 Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 27 Jul 2024 12:00:00 +0200 Subject: [PATCH 028/435] matches_manager unmatch logic timer implementation --- rust/Cargo.lock | 319 +++++++++++++++++++++++++++++++++---------- rust/Cargo.toml | 2 + rust/TODO | 3 +- rust/src/daemon.rs | 3 +- rust/src/filter.rs | 10 +- rust/src/matches.rs | 112 ++++++++++----- rust/src/messages.rs | 10 +- rust/src/stream.rs | 12 +- 8 files changed, 357 insertions(+), 114 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 3b015a8..3a72f11 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -11,6 +11,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "annotate-snippets" version = "0.9.2" @@ -23,9 +38,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", @@ -38,33 +53,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anstyle-parse" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.3" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ "windows-sys", ] [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", "windows-sys", @@ -76,6 +91,12 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + [[package]] name = "base64" version = "0.13.1" @@ -97,6 +118,18 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "cc" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" + [[package]] name = "cfg-if" version = "1.0.0" @@ -110,10 +143,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] -name = "clap" -version = "4.5.4" +name = "chrono" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "clap" +version = "4.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" dependencies = [ "clap_builder", "clap_derive", @@ -121,9 +168,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" dependencies = [ "anstream", "anstyle", @@ -133,36 +180,42 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.2" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79504325bf38b10165b02e89b4347300f855f273c4cb30c4a3209e6583275e" +checksum = "c6ae69fbb0833c6fcd5a8d4b8609f108c7ad95fc11e248d853ff2c42a90df26a" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.65", + "syn 2.0.72", ] [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "colorchoice" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "ctrlc" @@ -198,6 +251,29 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -210,9 +286,9 @@ dependencies = [ [[package]] name = "is_terminal_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itoa" @@ -303,6 +379,15 @@ dependencies = [ "peg", ] +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.155" @@ -323,9 +408,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "nix" @@ -339,6 +424,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -349,6 +443,12 @@ dependencies = [ "libc", ] +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + [[package]] name = "pathdiff" version = "0.2.1" @@ -384,9 +484,9 @@ checksum = "c719dcf55f09a3a7e764c6649ab594c18a177e3599c467983cdf644bfc0a4088" [[package]] name = "proc-macro2" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -405,6 +505,7 @@ name = "reaction" version = "0.1.0" dependencies = [ "anyhow", + "chrono", "clap", "clap_complete", "ctrlc", @@ -415,13 +516,14 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "timer", ] [[package]] name = "regex" -version = "1.10.4" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", @@ -431,9 +533,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", @@ -442,9 +544,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "rustc-hash" @@ -460,31 +562,32 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.65", + "syn 2.0.72", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -521,9 +624,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.65" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2863d96a84c6439701d7a38f9de935ec562c8832cc55d1dde0f513b52fad106" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -544,22 +647,31 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.65", + "syn 2.0.72", +] + +[[package]] +name = "timer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31d42176308937165701f50638db1c31586f183f1aab416268216577aec7306b" +dependencies = [ + "chrono", ] [[package]] @@ -576,9 +688,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-width" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" [[package]] name = "unicode-xid" @@ -594,9 +706,63 @@ checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "winapi" @@ -620,6 +786,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -631,9 +806,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -647,51 +822,51 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "yansi-term" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 3b43adb..8a714da 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] anyhow = "1.0.86" +chrono = { version = "0.4.38", features = ["std", "clock"] } clap = { version = "4.5.4", features = ["derive"] } clap_complete = "4.5.2" ctrlc = { version = "3.4.4", features = ["termination"] } @@ -17,3 +18,4 @@ regex = "1.10.4" serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" serde_yaml = "0.9.34" +timer = "0.2.0" diff --git a/rust/TODO b/rust/TODO index 439d9e3..d095bd5 100644 --- a/rust/TODO +++ b/rust/TODO @@ -1 +1,2 @@ -cargo clippy +remove anyhow? +use chrono::TimeDelta and DateTime everywhere instead of std's? diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index 91fb0cd..92200ee 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -42,8 +42,9 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { let (action_tx, _action_rx) = channel(); let config_matches = config.clone(); + let match_tx_matches = match_tx.clone(); stream_thread_handles.push(thread::spawn(move || { - matches_manager(config_matches, match_rx, action_tx) + matches_manager(config_matches, match_rx, match_tx_matches, action_tx) })); // TODO execs manager diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 7fce6d1..db3abb9 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -32,7 +32,7 @@ pub struct Filter { #[serde(skip)] patterns: BTreeSet, - pub retry: Option, + retry: Option, #[serde(rename = "retryperiod")] retry_period: Option, retry_duration: Option, @@ -51,6 +51,14 @@ impl Filter { } } + pub fn retry(&self) -> Option { + self.retry + } + + pub fn retry_duration(&self) -> Option { + self.retry_duration + } + pub fn setup( &mut self, stream_name: &str, diff --git a/rust/src/matches.rs b/rust/src/matches.rs index 95622cd..5a627b7 100644 --- a/rust/src/matches.rs +++ b/rust/src/matches.rs @@ -7,47 +7,97 @@ use std::{ time::SystemTime, }; +use chrono::TimeDelta; +use timer::MessageTimer; + use crate::{ config::Config, filter::FilterName, messages::{Match, MAT, MFT}, }; +#[derive(Clone)] +pub enum MatchManagerInput { + Match(MFT), + Unmatch(MFT), + #[allow(dead_code)] + Flush(MFT), +} + type MatchesMap = BTreeMap>>; -// TODO handle flushes -pub fn matches_manager(config: Arc, match_rx: Receiver, action_tx: Sender) { - let mut matches: MatchesMap = BTreeMap::new(); +trait MatchesMapTrait { + fn add(&mut self, mft: &MFT); + fn rm(&mut self, mft: &MFT); +} +impl MatchesMapTrait for MatchesMap { + fn add(&mut self, mft: &MFT) { + let inner_map = self.entry(mft.m.clone()).or_default(); + let inner_set = inner_map.entry(mft.f.clone()).or_default(); + inner_set.insert(mft.t); + } - while let Ok(mft) = match_rx.recv() { - let filter = config.get_filter(&mft.f).unwrap(); - - let is_retry = filter.retry.is_some(); - - // Store matches - if is_retry { - // Make sure collections exist - let inner_map = matches.entry(mft.m.clone()).or_default(); - let inner_set = inner_map.entry(mft.f.clone()).or_default(); - // Add new match - inner_set.insert(mft.t); - // Remove match when expired - // TODO timer that only sends it back to matches_manager - // https://docs.rs/timer/latest/timer/struct.Timer.html - // replace Receiver with Receiver - } - - // Executing actions - let exec = !is_retry || matches.get(&mft.m).map(|map| map.get(&mft.f)).is_some(); - if exec { - // Delete matches only if storing them - if is_retry { - matches.get_mut(&mft.m).unwrap().remove(&mft.f); + fn rm(&mut self, mft: &MFT) { + if let Some(inner_map) = self.get_mut(&mft.m) { + if let Some(inner_set) = inner_map.get_mut(&mft.f) { + inner_set.remove(&mft.t); + if inner_set.is_empty() { + inner_map.remove(&mft.f); + } + } + if inner_map.is_empty() { + self.remove(&mft.m); } - filter.send_actions(&mft.m, mft.t, &action_tx); } - - // TODO send to DB - // log_tx.send(LogEntry { mft, exec }); + } +} + +pub fn matches_manager( + config: Arc, + match_rx: Receiver, + match_tx: Sender, + action_tx: Sender, +) { + let mut matches: MatchesMap = BTreeMap::new(); + + let timer = MessageTimer::new(match_tx); + + while let Ok(mft) = match_rx.recv() { + match mft { + MatchManagerInput::Match(mft) => { + let filter = config.get_filter(&mft.f).unwrap(); + + let is_retry = filter.retry().is_some(); + + // Store matches + if is_retry { + // Add new match + matches.add(&mft); + // Remove match when expired + timer.schedule_with_delay( + // unwrap: retry_duration is guaranted to be Some when retry is Some + // unwrap: TimeDelta max value is ~292471 millenaries + TimeDelta::from_std(filter.retry_duration().unwrap()).unwrap(), + MatchManagerInput::Unmatch(mft.clone()), + ); + } + + // Executing actions + let exec = !is_retry || matches.get(&mft.m).map(|map| map.get(&mft.f)).is_some(); + if exec { + // Delete matches only if storing them + if is_retry { + matches.get_mut(&mft.m).unwrap().remove(&mft.f); + } + filter.send_actions(&mft.m, mft.t, &action_tx); + } + + // TODO send to DB + // log_tx.send(LogEntry { mft, exec }); + } + MatchManagerInput::Unmatch(mft) => matches.rm(&mft), + #[allow(clippy::todo)] + MatchManagerInput::Flush(_) => todo!(), // TODO handle flushes + } } } diff --git a/rust/src/messages.rs b/rust/src/messages.rs index 0cd5999..bf25869 100644 --- a/rust/src/messages.rs +++ b/rust/src/messages.rs @@ -4,6 +4,7 @@ use crate::{action::ActionName, filter::FilterName}; pub type Match = Vec; +#[derive(Clone)] pub struct MFT { pub m: Match, pub f: FilterName, @@ -16,7 +17,8 @@ pub struct MAT { pub t: SystemTime, } -// pub struct LogEntry { -// pub mft: MFT, -// pub exec: bool, -// } +#[allow(dead_code)] +pub struct LogEntry { + pub mft: MFT, + pub exec: bool, +} diff --git a/rust/src/stream.rs b/rust/src/stream.rs index e2e5749..f1dea6e 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -9,7 +9,7 @@ use std::{ use log::{debug, error, info}; use serde::Deserialize; -use crate::{config::Patterns, filter::Filter, messages::MFT}; +use crate::{config::Patterns, filter::Filter, matches::MatchManagerInput, messages::MFT}; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -59,7 +59,11 @@ impl Stream { Ok(()) } - pub fn manager(&self, child_tx: SyncSender>, match_tx: Sender) { + pub fn manager( + &self, + child_tx: SyncSender>, + match_tx: Sender, + ) { info!("{}: start {:?}", self.name, self.cmd); let mut child = match Command::new(&self.cmd[0]) .args(&self.cmd[1..]) @@ -94,11 +98,11 @@ impl Stream { for filter in self.filters.values() { if let Some(match_) = filter.get_match(&line) { match_tx - .send(MFT { + .send(MatchManagerInput::Match(MFT { m: match_, f: filter.name(), t: SystemTime::now(), - }) + })) .unwrap(); } } From cfadfe9ec566bb4bc3cbf8b86fb48c54e6097229 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 29 Jul 2024 12:00:00 +0200 Subject: [PATCH 029/435] action, pattern: remove pub fields --- rust/src/action.rs | 14 +++++++++++++- rust/src/filter.rs | 31 ++++++++++++++----------------- rust/src/pattern.rs | 11 +++++++++-- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/rust/src/action.rs b/rust/src/action.rs index 0752525..e4116f8 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -12,7 +12,7 @@ pub struct Action { // TODO one shot time deserialization after: Option, #[serde(skip)] - pub after_duration: Option, + after_duration: Option, #[serde(rename = "onexit", default = "set_false")] on_exit: bool, @@ -38,6 +38,10 @@ impl Action { } } + pub fn after_duration(&self) -> Option { + self.after_duration + } + pub fn setup( &mut self, stream_name: &str, @@ -115,6 +119,14 @@ pub mod tests { action } + pub fn ok_action_with_after(d: String, name: &str) -> Action { + let mut action = default_action(); + action.cmd = vec!["command".into()]; + action.after = Some(d); + action.setup("", "", name).unwrap(); + action + } + #[test] fn missing_config() { let mut action; diff --git a/rust/src/filter.rs b/rust/src/filter.rs index db3abb9..6f49f2e 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -104,11 +104,11 @@ impl Filter { for regex in &self.regex { let mut regex_buf = regex.clone(); for pattern in patterns.values() { - if let Some(index) = regex.find(&pattern.name_with_braces) { - if regex.rfind(&pattern.name_with_braces).unwrap() != index { + if let Some(index) = regex.find(pattern.name_with_braces()) { + if regex.rfind(pattern.name_with_braces()).unwrap() != index { return Err(format!( "pattern {} present multiple times in regex", - &pattern.name_with_braces + pattern.name_with_braces() )); } if first { @@ -116,16 +116,16 @@ impl Filter { } else if !self.patterns.contains(pattern) { return Err(format!( "pattern {} is not present in the first regex but is present in a following regex. all regexes should contain the same set of regexes", - &pattern.name_with_braces + &pattern.name_with_braces() )); } } else if !first && self.patterns.contains(pattern) { return Err(format!( "pattern {} is present in the first regex but is not present in a following regex. all regexes should contain the same set of regexes", - &pattern.name_with_braces + &pattern.name_with_braces() )); } - regex_buf = regex_buf.replacen(&pattern.name_with_braces, &pattern.regex, 1); + regex_buf = regex_buf.replacen(pattern.name_with_braces(), &pattern.regex, 1); } let compiled = Regex::new(®ex_buf).map_err(|err| err.to_string())?; self.compiled_regex.push(compiled); @@ -145,7 +145,7 @@ impl Filter { self.actions .values() .fold(Duration::from_secs(0), |acc, v| { - v.after_duration + v.after_duration() .map_or(acc, |v| if v > acc { v } else { acc }) }); @@ -158,7 +158,7 @@ impl Filter { if !self.patterns.is_empty() { let mut result = Match::new(); for pattern in &self.patterns { - let match_ = matches.name(&pattern.name).unwrap(); + let match_ = matches.name(pattern.name()).unwrap(); if pattern.not_an_ignore(match_.as_str()) { result.push(match_.as_str().to_string()); } @@ -202,7 +202,7 @@ impl Display for FilterName { #[cfg(test)] pub mod tests { - use crate::action::tests::ok_action; + use crate::action::tests::{ok_action, ok_action_with_after}; use crate::pattern::tests::{ boubou_pattern_with_ignore, default_pattern, ok_pattern_with_ignore, }; @@ -297,20 +297,17 @@ pub mod tests { filter.setup(&name, &name, &empty_patterns).unwrap(); assert_eq!(filter.longuest_action_duration, Duration::default()); - let mut minute_action = ok_action(); - minute_action.after_duration = Some(minute); + let minute_action = ok_action_with_after(minute_str.clone(), &minute_str); // duration 60 filter = ok_filter(); - filter.actions.insert(minute_str.clone(), minute_action); + filter + .actions + .insert(minute_str.clone(), minute_action.clone()); filter.setup(&name, &name, &empty_patterns).unwrap(); assert_eq!(filter.longuest_action_duration, minute); - let mut minute_action = ok_action(); - minute_action.after_duration = Some(minute); - - let mut two_minutes_action = ok_action(); - two_minutes_action.after_duration = Some(two_minutes); + let two_minutes_action = ok_action_with_after(two_minutes_str.clone(), &two_minutes_str); // duration 120 filter = ok_filter(); diff --git a/rust/src/pattern.rs b/rust/src/pattern.rs index 1a7ade2..1d4871d 100644 --- a/rust/src/pattern.rs +++ b/rust/src/pattern.rs @@ -17,9 +17,9 @@ pub struct Pattern { compiled_ignore_regex: Vec, #[serde(skip)] - pub name: String, + name: String, #[serde(skip)] - pub name_with_braces: String, + name_with_braces: String, } impl Pattern { @@ -28,6 +28,13 @@ impl Pattern { .map_err(|msg| format!("pattern {}: {}", name, msg)) } + pub fn name(&self) -> &String { + &self.name + } + pub fn name_with_braces(&self) -> &String { + &self.name_with_braces + } + pub fn _setup(&mut self, name: &String) -> Result<(), String> { self.name = name.clone(); self.name_with_braces = format!("<{}>", name); From bc755efc0c64fbf29bd75326addc251f9c951efd Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 30 Jul 2024 12:00:00 +0200 Subject: [PATCH 030/435] implement exec logic - switch from std::time to chrono (mainly because of timer crate) - move cli, logger and parse_duration to utils/ folder - implement threadpool util (from the rust book) - MatchesMap: move order: first FilterName, then Match - implement execs_manager thread responsible for executing actions --- rust/src/action.rs | 39 ++++++-- rust/src/client.rs | 2 +- rust/src/config.rs | 14 +-- rust/src/daemon.rs | 14 ++- rust/src/execs.rs | 128 +++++++++++++++++++++++++ rust/src/filter.rs | 46 +++++---- rust/src/main.rs | 8 +- rust/src/matches.rs | 34 ++++--- rust/src/messages.rs | 8 +- rust/src/stream.rs | 4 +- rust/src/{ => utils}/cli.rs | 0 rust/src/{ => utils}/logger.rs | 0 rust/src/utils/mod.rs | 8 ++ rust/src/{ => utils}/parse_duration.rs | 24 ++--- rust/src/utils/threadpool.rs | 62 ++++++++++++ 15 files changed, 312 insertions(+), 79 deletions(-) create mode 100644 rust/src/execs.rs rename rust/src/{ => utils}/cli.rs (100%) rename rust/src/{ => utils}/logger.rs (100%) create mode 100644 rust/src/utils/mod.rs rename rust/src/{ => utils}/parse_duration.rs (67%) create mode 100644 rust/src/utils/threadpool.rs diff --git a/rust/src/action.rs b/rust/src/action.rs index e4116f8..8cdc05e 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -1,8 +1,10 @@ -use std::time::Duration; +use std::{collections::BTreeSet, fmt::Display, process::Command}; + +use chrono::TimeDelta; use serde::Deserialize; -use crate::parse_duration::parse_duration; +use crate::{messages::Match, pattern::Pattern, utils::parse_duration}; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -12,7 +14,7 @@ pub struct Action { // TODO one shot time deserialization after: Option, #[serde(skip)] - after_duration: Option, + after_duration: Option, #[serde(rename = "onexit", default = "set_false")] on_exit: bool, @@ -38,7 +40,7 @@ impl Action { } } - pub fn after_duration(&self) -> Option { + pub fn after_duration(&self) -> Option { self.after_duration } @@ -82,17 +84,38 @@ impl Action { Ok(()) } + + // TODO test + pub fn exec(&self, match_: &Match, patterns: &BTreeSet) -> Command { + let computed_command = if patterns.is_empty() { + self.cmd.clone() + } else { + self.cmd + .iter() + .map(|item| { + (0..match_.len()) + .zip(patterns) + .fold(item.clone(), |acc, (i, pattern)| { + acc.replace(pattern.name_with_braces(), &match_[i]) + }) + }) + .collect() + }; + let mut cmd = Command::new(&computed_command[0]); + cmd.args(&computed_command[1..]); + cmd + } } -#[derive(PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct ActionName { pub stream: String, pub filter: String, pub action: String, } -impl ToString for ActionName { - fn to_string(&self) -> String { - format!("{}.{}.{}", self.stream, self.filter, self.action) +impl Display for ActionName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}.{}", self.stream, self.filter, self.action) } } diff --git a/rust/src/client.rs b/rust/src/client.rs index 467d88c..ffc1b65 100644 --- a/rust/src/client.rs +++ b/rust/src/client.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use log::debug; use regex::Regex; -use crate::cli::{Format, NamedRegex}; +use crate::utils::cli::{Format, NamedRegex}; pub fn show( socket: &PathBuf, diff --git a/rust/src/config.rs b/rust/src/config.rs index 6830356..f145d07 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -9,11 +9,7 @@ use anyhow::{anyhow, Context, Result}; use log::{error, info}; use serde::Deserialize; -use crate::{ - filter::{Filter, FilterName}, - pattern::Pattern, - stream::Stream, -}; +use crate::{filter::Filter, pattern::Pattern, stream::Stream}; pub type Patterns = BTreeMap; @@ -38,8 +34,12 @@ impl Config { &self.streams } - pub fn get_filter(&self, name: &FilterName) -> Option<&Filter> { - self.streams.get(&name.stream)?.get_filter(&name.filter) + pub fn concurrency(&self) -> usize { + self.concurrency + } + + pub fn get_filter(&self, stream: &str, filter: &str) -> Option<&Filter> { + self.streams.get(stream)?.get_filter(filter) } pub fn setup(&mut self) -> Result<()> { diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index 92200ee..dc97464 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -11,10 +11,10 @@ use std::{ use log::{debug, error, info, Level}; -use crate::{config, logger, matches::matches_manager}; +use crate::{config, execs::execs_manager, matches::matches_manager, utils::SimpleLogger}; pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { - if let Err(err) = logger::SimpleLogger::init(loglevel) { + if let Err(err) = SimpleLogger::init(loglevel) { eprintln!("ERROR could not initialize logging: {err}"); exit(1); } @@ -39,12 +39,18 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { { let (match_tx, match_rx) = channel(); - let (action_tx, _action_rx) = channel(); + let (exec_tx, exec_rx) = channel(); let config_matches = config.clone(); let match_tx_matches = match_tx.clone(); + let exec_tx_matches = exec_tx.clone(); stream_thread_handles.push(thread::spawn(move || { - matches_manager(config_matches, match_rx, match_tx_matches, action_tx) + matches_manager(config_matches, match_rx, match_tx_matches, exec_tx_matches) + })); + + let config_execs = config.clone(); + stream_thread_handles.push(thread::spawn(move || { + execs_manager(config_execs, exec_rx, exec_tx) })); // TODO execs manager diff --git a/rust/src/execs.rs b/rust/src/execs.rs new file mode 100644 index 0000000..ec093d8 --- /dev/null +++ b/rust/src/execs.rs @@ -0,0 +1,128 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + process::Stdio, + sync::{ + mpsc::{Receiver, Sender}, + Arc, + }, +}; + +use chrono::{DateTime, Local}; +use log::{error, info}; +use timer::MessageTimer; + +use crate::{ + action::ActionName, + config::Config, + messages::{Match, MAT}, + utils::ThreadPool, +}; + +#[derive(Clone)] +pub enum ExecsManagerInput { + Exec(MAT), + ExecPending(MAT), + #[allow(dead_code)] + Flush(MAT), +} + +type ExecsMap = BTreeMap>>>; + +trait ExecsMapTrait { + fn add(&mut self, mft: &MAT); + fn rm(&mut self, mft: &MAT); +} +impl ExecsMapTrait for ExecsMap { + fn add(&mut self, mat: &MAT) { + let inner_map = self.entry(mat.a.clone()).or_default(); + let inner_set = inner_map.entry(mat.m.clone()).or_default(); + inner_set.insert(mat.t); + } + + fn rm(&mut self, mat: &MAT) { + if let Some(inner_map) = self.get_mut(&mat.a) { + if let Some(inner_set) = inner_map.get_mut(&mat.m) { + inner_set.remove(&mat.t); + if inner_set.is_empty() { + inner_map.remove(&mat.m); + } + } + if inner_map.is_empty() { + self.remove(&mat.a); + } + } + } +} + +pub fn execs_manager( + config: Arc, + action_rx: Receiver, + action_tx: Sender, +) { + // Initialize a ThreadPool only when concurrency hasn't been disabled + let thread_pool = if config.concurrency() > 1 { + Some(ThreadPool::new(config.concurrency())) + } else { + None + }; + + let exec_now = |mat: MAT| { + // Retrieve needed information from config + let filter = config.get_filter(&mat.a.stream, &mat.a.filter).unwrap(); + let action = filter.get_action(&mat.a.action).unwrap(); + let patterns = filter.patterns(); + + // Construct command + let mut command = action.exec(&mat.m, patterns); + + let mut closure = move || { + info!("{}: run {:?}", &mat.a, command); + if let Err(err) = command + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .stdout(Stdio::piped()) + .status() + { + error!("{}: run {:?}, code {}", &mat.a, command, err); + } + }; + + // Execute command either in the ThreadPool or directly + match &thread_pool { + Some(thread_pool) => { + thread_pool.execute(closure); + } + None => closure(), + } + }; + + let mut execs: ExecsMap = BTreeMap::new(); + + let timer = MessageTimer::new(action_tx); + + while let Ok(mat) = action_rx.recv() { + match mat { + ExecsManagerInput::Exec(mat) => { + let now = Local::now(); + if mat.t.lt(&now) { + exec_now(mat); + } else { + execs.add(&mat); + let guard = timer + .schedule_with_date(mat.t, ExecsManagerInput::ExecPending(mat.clone())); + guard.ignore(); + } + } + ExecsManagerInput::ExecPending(mat) => { + execs.rm(&mat); + exec_now(mat); + } + #[allow(clippy::todo)] + ExecsManagerInput::Flush(_mat) => todo!(), + } + } + + if let Some(thread_pool) = thread_pool { + thread_pool.join(); + } +} diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 6f49f2e..e81ab53 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -2,9 +2,9 @@ use std::{ collections::{BTreeMap, BTreeSet}, fmt::Display, sync::mpsc::Sender, - time::{Duration, SystemTime}, }; +use chrono::TimeDelta; use log::info; use regex::Regex; use serde::Deserialize; @@ -12,9 +12,10 @@ use serde::Deserialize; use crate::{ action::Action, config::Patterns, - messages::{Match, MAT}, - parse_duration::parse_duration, + execs::ExecsManagerInput, + messages::{Match, Time, MAT}, pattern::Pattern, + utils::parse_duration, }; #[derive(Clone, Debug, Deserialize)] @@ -22,7 +23,7 @@ use crate::{ pub struct Filter { actions: BTreeMap, #[serde(skip)] - longuest_action_duration: Duration, + longuest_action_duration: TimeDelta, regex: Vec, #[serde(skip)] @@ -35,7 +36,8 @@ pub struct Filter { retry: Option, #[serde(rename = "retryperiod")] retry_period: Option, - retry_duration: Option, + #[serde(skip)] + retry_duration: Option, #[serde(skip)] name: String, @@ -55,10 +57,18 @@ impl Filter { self.retry } - pub fn retry_duration(&self) -> Option { + pub fn retry_duration(&self) -> Option { self.retry_duration } + pub fn patterns(&self) -> &BTreeSet { + &self.patterns + } + + pub fn get_action(&self, name: &str) -> Option<&Action> { + self.actions.get(name) + } + pub fn setup( &mut self, stream_name: &str, @@ -142,12 +152,10 @@ impl Filter { } self.longuest_action_duration = - self.actions - .values() - .fold(Duration::from_secs(0), |acc, v| { - v.after_duration() - .map_or(acc, |v| if v > acc { v } else { acc }) - }); + self.actions.values().fold(TimeDelta::seconds(0), |acc, v| { + v.after_duration() + .map_or(acc, |v| if v > acc { v } else { acc }) + }); Ok(()) } @@ -176,13 +184,13 @@ impl Filter { None } - pub fn send_actions(&self, m: &Match, t: SystemTime, tx: &Sender) { + pub fn send_actions(&self, m: &Match, t: Time, tx: &Sender) { for action in self.actions.values() { - tx.send(MAT { + tx.send(ExecsManagerInput::Exec(MAT { m: m.clone(), a: action.name(), t, - }) + })) .unwrap(); } } @@ -220,7 +228,7 @@ pub mod tests { retry: None, retry_period: None, retry_duration: None, - longuest_action_duration: Duration::default(), + longuest_action_duration: TimeDelta::default(), } } @@ -288,14 +296,14 @@ pub mod tests { let name = "name".to_string(); let empty_patterns = Patterns::new(); let minute_str = "1m".to_string(); - let minute = Duration::from_secs(60); - let two_minutes = Duration::from_secs(60 * 2); + let minute = TimeDelta::seconds(60); + let two_minutes = TimeDelta::seconds(60 * 2); let two_minutes_str = "2m".to_string(); // duration 0 filter = ok_filter(); filter.setup(&name, &name, &empty_patterns).unwrap(); - assert_eq!(filter.longuest_action_duration, Duration::default()); + assert_eq!(filter.longuest_action_duration, TimeDelta::default()); let minute_action = ok_action_with_after(minute_str.clone(), &minute_str); diff --git a/rust/src/main.rs b/rust/src/main.rs index 4e75381..831c40c 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -20,20 +20,18 @@ mod pattern; mod stream; // important threads +mod execs; mod matches; // top-level -mod cli; mod client; mod daemon; -// utils -mod logger; -mod parse_duration; +mod utils; -use cli::{Cli, Command}; use client::{flush, show, test_regex}; use daemon::daemon; +use utils::cli::{Cli, Command}; fn main() { // Show a nice message when reaction panics diff --git a/rust/src/matches.rs b/rust/src/matches.rs index 5a627b7..2803301 100644 --- a/rust/src/matches.rs +++ b/rust/src/matches.rs @@ -4,16 +4,15 @@ use std::{ mpsc::{Receiver, Sender}, Arc, }, - time::SystemTime, }; -use chrono::TimeDelta; use timer::MessageTimer; use crate::{ config::Config, + execs::ExecsManagerInput, filter::FilterName, - messages::{Match, MAT, MFT}, + messages::{Match, Time, MFT}, }; #[derive(Clone)] @@ -24,7 +23,7 @@ pub enum MatchManagerInput { Flush(MFT), } -type MatchesMap = BTreeMap>>; +type MatchesMap = BTreeMap>>; trait MatchesMapTrait { fn add(&mut self, mft: &MFT); @@ -32,21 +31,21 @@ trait MatchesMapTrait { } impl MatchesMapTrait for MatchesMap { fn add(&mut self, mft: &MFT) { - let inner_map = self.entry(mft.m.clone()).or_default(); - let inner_set = inner_map.entry(mft.f.clone()).or_default(); + let inner_map = self.entry(mft.f.clone()).or_default(); + let inner_set = inner_map.entry(mft.m.clone()).or_default(); inner_set.insert(mft.t); } fn rm(&mut self, mft: &MFT) { - if let Some(inner_map) = self.get_mut(&mft.m) { - if let Some(inner_set) = inner_map.get_mut(&mft.f) { + if let Some(inner_map) = self.get_mut(&mft.f) { + if let Some(inner_set) = inner_map.get_mut(&mft.m) { inner_set.remove(&mft.t); if inner_set.is_empty() { - inner_map.remove(&mft.f); + inner_map.remove(&mft.m); } } if inner_map.is_empty() { - self.remove(&mft.m); + self.remove(&mft.f); } } } @@ -56,7 +55,7 @@ pub fn matches_manager( config: Arc, match_rx: Receiver, match_tx: Sender, - action_tx: Sender, + action_tx: Sender, ) { let mut matches: MatchesMap = BTreeMap::new(); @@ -65,7 +64,7 @@ pub fn matches_manager( while let Ok(mft) = match_rx.recv() { match mft { MatchManagerInput::Match(mft) => { - let filter = config.get_filter(&mft.f).unwrap(); + let filter = config.get_filter(&mft.f.stream, &mft.f.filter).unwrap(); let is_retry = filter.retry().is_some(); @@ -74,20 +73,19 @@ pub fn matches_manager( // Add new match matches.add(&mft); // Remove match when expired - timer.schedule_with_delay( - // unwrap: retry_duration is guaranted to be Some when retry is Some - // unwrap: TimeDelta max value is ~292471 millenaries - TimeDelta::from_std(filter.retry_duration().unwrap()).unwrap(), + let guard = timer.schedule_with_delay( + filter.retry_duration().unwrap(), MatchManagerInput::Unmatch(mft.clone()), ); + guard.ignore(); } // Executing actions - let exec = !is_retry || matches.get(&mft.m).map(|map| map.get(&mft.f)).is_some(); + let exec = !is_retry || matches.get(&mft.f).map(|map| map.get(&mft.m)).is_some(); if exec { // Delete matches only if storing them if is_retry { - matches.get_mut(&mft.m).unwrap().remove(&mft.f); + matches.get_mut(&mft.f).unwrap().remove(&mft.m); } filter.send_actions(&mft.m, mft.t, &action_tx); } diff --git a/rust/src/messages.rs b/rust/src/messages.rs index bf25869..3d12805 100644 --- a/rust/src/messages.rs +++ b/rust/src/messages.rs @@ -1,20 +1,22 @@ -use std::time::SystemTime; +use chrono::{DateTime, Local}; use crate::{action::ActionName, filter::FilterName}; +pub type Time = DateTime; pub type Match = Vec; #[derive(Clone)] pub struct MFT { pub m: Match, pub f: FilterName, - pub t: SystemTime, + pub t: Time, } +#[derive(Clone)] pub struct MAT { pub m: Match, pub a: ActionName, - pub t: SystemTime, + pub t: Time, } #[allow(dead_code)] diff --git a/rust/src/stream.rs b/rust/src/stream.rs index f1dea6e..79580e7 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -3,9 +3,9 @@ use std::{ io::{BufRead, BufReader}, process::{Child, Command, Stdio}, sync::mpsc::{Sender, SyncSender}, - time::SystemTime, }; +use chrono::Local; use log::{debug, error, info}; use serde::Deserialize; @@ -101,7 +101,7 @@ impl Stream { .send(MatchManagerInput::Match(MFT { m: match_, f: filter.name(), - t: SystemTime::now(), + t: Local::now(), })) .unwrap(); } diff --git a/rust/src/cli.rs b/rust/src/utils/cli.rs similarity index 100% rename from rust/src/cli.rs rename to rust/src/utils/cli.rs diff --git a/rust/src/logger.rs b/rust/src/utils/logger.rs similarity index 100% rename from rust/src/logger.rs rename to rust/src/utils/logger.rs diff --git a/rust/src/utils/mod.rs b/rust/src/utils/mod.rs new file mode 100644 index 0000000..46debbe --- /dev/null +++ b/rust/src/utils/mod.rs @@ -0,0 +1,8 @@ +pub mod cli; +pub mod logger; +mod parse_duration; +mod threadpool; + +pub use logger::SimpleLogger; +pub use parse_duration::parse_duration; +pub use threadpool::ThreadPool; diff --git a/rust/src/parse_duration.rs b/rust/src/utils/parse_duration.rs similarity index 67% rename from rust/src/parse_duration.rs rename to rust/src/utils/parse_duration.rs index 7faff3b..f05ee8b 100644 --- a/rust/src/parse_duration.rs +++ b/rust/src/utils/parse_duration.rs @@ -1,6 +1,6 @@ -use std::time::Duration; +use chrono::TimeDelta; -pub fn parse_duration(d: &str) -> Result { +pub fn parse_duration(d: &str) -> Result { let d_trimmed = d.trim(); let chars = d_trimmed.as_bytes(); let mut value = 0; @@ -12,8 +12,8 @@ pub fn parse_duration(d: &str) -> Result { if i == 0 { return Err(format!("duration '{}' doesn't start with digits", d)); } - let ok_secs = |mul: u32| -> Result { - Ok(Duration::from_secs(mul as u64 * value as u64)) + let ok_secs = |mul: u32| -> Result { + Ok(TimeDelta::seconds(mul as i64 * value as i64)) }; match d_trimmed[i..].trim() { @@ -31,7 +31,7 @@ pub fn parse_duration(d: &str) -> Result { #[cfg(test)] mod tests { - use std::time::Duration; + use chrono::TimeDelta; use super::*; @@ -42,18 +42,18 @@ mod tests { #[test] fn parse_duration_test() { - assert_eq!(parse_duration("1s"), Ok(Duration::from_secs(1))); - assert_eq!(parse_duration("12s"), Ok(Duration::from_secs(12))); - assert_eq!(parse_duration(" 12 secs "), Ok(Duration::from_secs(12))); - assert_eq!(parse_duration("2m"), Ok(Duration::from_secs(2 * 60))); + assert_eq!(parse_duration("1s"), Ok(TimeDelta::seconds(1))); + assert_eq!(parse_duration("12s"), Ok(TimeDelta::seconds(12))); + assert_eq!(parse_duration(" 12 secs "), Ok(TimeDelta::seconds(12))); + assert_eq!(parse_duration("2m"), Ok(TimeDelta::seconds(2 * 60))); assert_eq!( parse_duration("6 hours"), - Ok(Duration::from_secs(6 * 60 * 60)) + Ok(TimeDelta::seconds(6 * 60 * 60)) ); - assert_eq!(parse_duration("1d"), Ok(Duration::from_secs(24 * 60 * 60))); + assert_eq!(parse_duration("1d"), Ok(TimeDelta::seconds(24 * 60 * 60))); assert_eq!( parse_duration("365d"), - Ok(Duration::from_secs(365 * 24 * 60 * 60)) + Ok(TimeDelta::seconds(365 * 24 * 60 * 60)) ); assert!(parse_duration("d 3").is_err()); diff --git a/rust/src/utils/threadpool.rs b/rust/src/utils/threadpool.rs new file mode 100644 index 0000000..a09fa49 --- /dev/null +++ b/rust/src/utils/threadpool.rs @@ -0,0 +1,62 @@ +use std::{ + sync::{ + mpsc::{channel, Receiver, Sender}, + Arc, Mutex, + }, + thread::{spawn, JoinHandle}, +}; + +type Job = Box; + +pub struct ThreadPool { + workers: Vec, + sender: Sender, +} + +impl ThreadPool { + pub fn new(size: usize) -> ThreadPool { + assert!(size > 0); + + let (sender, receiver) = channel(); + let receiver = Arc::new(Mutex::new(receiver)); + + let mut workers = Vec::with_capacity(size); + + for _ in 0..size { + workers.push(Worker::new(Arc::clone(&receiver))); + } + + ThreadPool { workers, sender } + } + + pub fn execute(&self, f: F) + where + F: FnOnce() + Send + 'static, + { + let job = Box::new(f); + + self.sender.send(job).unwrap(); + } + + pub fn join(self) { + drop(self.sender); + for worker in self.workers { + worker.thread.join().unwrap(); + } + } +} + +struct Worker { + thread: JoinHandle<()>, +} + +impl Worker { + fn new(receiver: Arc>>) -> Worker { + let thread = spawn(move || loop { + let job = receiver.lock().unwrap().recv().unwrap(); + job(); + }); + + Worker { thread } + } +} From ca89d5e61c89b05142dd83b22423e03c597f7d72 Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 30 Jul 2024 12:00:00 +0200 Subject: [PATCH 031/435] bug fixing - correctly quitting all threads I had to add a Stop variant of manager's enums because they hold a Sender of their own Receiver. So streams leaving their Sender don't close the channel. The Stop is sent to matches_manager when streams have quit, and sent to execs_manager when matches_manager has quit. - fix ThreadPool panic when channel is closed - fix filter not adding after_duration to MAT messages - better logging of execs_manager --- rust/src/daemon.rs | 55 ++++++++++++++++++++---------------- rust/src/execs.rs | 14 +++++---- rust/src/filter.rs | 2 +- rust/src/matches.rs | 2 ++ rust/src/utils/threadpool.rs | 7 +++-- rust/test.jsonnet | 16 +++++------ 6 files changed, 54 insertions(+), 42 deletions(-) diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index dc97464..a591df5 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -11,7 +11,12 @@ use std::{ use log::{debug, error, info, Level}; -use crate::{config, execs::execs_manager, matches::matches_manager, utils::SimpleLogger}; +use crate::{ + config, + execs::{execs_manager, ExecsManagerInput}, + matches::{matches_manager, MatchManagerInput}, + utils::SimpleLogger, +}; pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { if let Err(err) = SimpleLogger::init(loglevel) { @@ -37,34 +42,30 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { let mut stream_process_child_handles = Vec::new(); let mut stream_thread_handles = Vec::new(); - { - let (match_tx, match_rx) = channel(); - let (exec_tx, exec_rx) = channel(); + let (match_tx, match_rx) = channel(); + let (exec_tx, exec_rx) = channel(); - let config_matches = config.clone(); - let match_tx_matches = match_tx.clone(); - let exec_tx_matches = exec_tx.clone(); - stream_thread_handles.push(thread::spawn(move || { - matches_manager(config_matches, match_rx, match_tx_matches, exec_tx_matches) - })); + let config_matches = config.clone(); + let match_tx_matches = match_tx.clone(); + let exec_tx_matches = exec_tx.clone(); + let matches_manager_thread_handle = thread::spawn(move || { + matches_manager(config_matches, match_rx, match_tx_matches, exec_tx_matches) + }); - let config_execs = config.clone(); - stream_thread_handles.push(thread::spawn(move || { - execs_manager(config_execs, exec_rx, exec_tx) - })); + let config_execs = config.clone(); + let exec_tx_execs = exec_tx.clone(); + let execs_manager_thread_handle = + thread::spawn(move || execs_manager(config_execs, exec_rx, exec_tx_execs)); - // TODO execs manager + for stream in config.streams().values() { + let stream = stream.clone(); + let match_tx = match_tx.clone(); + let (child_tx, child_rx) = sync_channel(0); - for stream in config.streams().values() { - let stream = stream.clone(); - let match_tx = match_tx.clone(); - let (child_tx, child_rx) = sync_channel(0); + stream_thread_handles.push(thread::spawn(move || stream.manager(child_tx, match_tx))); - stream_thread_handles.push(thread::spawn(move || stream.manager(child_tx, match_tx))); - - if let Ok(Some(child)) = child_rx.recv() { - stream_process_child_handles.push(child); - } + if let Ok(Some(child)) = child_rx.recv() { + stream_process_child_handles.push(child); } } @@ -91,7 +92,11 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { let _ = thread_handle.join(); } - // TODO wait for actions to complete + match_tx.send(MatchManagerInput::Stop).unwrap(); + let _ = matches_manager_thread_handle.join(); + + exec_tx.send(ExecsManagerInput::Stop).unwrap(); + let _ = execs_manager_thread_handle.join(); let stop_ok = config.stop(); diff --git a/rust/src/execs.rs b/rust/src/execs.rs index ec093d8..f2051cd 100644 --- a/rust/src/execs.rs +++ b/rust/src/execs.rs @@ -24,6 +24,7 @@ pub enum ExecsManagerInput { ExecPending(MAT), #[allow(dead_code)] Flush(MAT), + Stop, } type ExecsMap = BTreeMap>>>; @@ -56,8 +57,8 @@ impl ExecsMapTrait for ExecsMap { pub fn execs_manager( config: Arc, - action_rx: Receiver, - action_tx: Sender, + exec_rx: Receiver, + exec_tx: Sender, ) { // Initialize a ThreadPool only when concurrency hasn't been disabled let thread_pool = if config.concurrency() > 1 { @@ -76,14 +77,14 @@ pub fn execs_manager( let mut command = action.exec(&mat.m, patterns); let mut closure = move || { - info!("{}: run {:?}", &mat.a, command); + info!("{}: run [{:?}]", &mat.a, command); if let Err(err) = command .stdin(Stdio::null()) .stderr(Stdio::null()) .stdout(Stdio::piped()) .status() { - error!("{}: run {:?}, code {}", &mat.a, command, err); + error!("{}: run [{:?}], code {}", &mat.a, command, err); } }; @@ -98,9 +99,9 @@ pub fn execs_manager( let mut execs: ExecsMap = BTreeMap::new(); - let timer = MessageTimer::new(action_tx); + let timer = MessageTimer::new(exec_tx); - while let Ok(mat) = action_rx.recv() { + while let Ok(mat) = exec_rx.recv() { match mat { ExecsManagerInput::Exec(mat) => { let now = Local::now(); @@ -119,6 +120,7 @@ pub fn execs_manager( } #[allow(clippy::todo)] ExecsManagerInput::Flush(_mat) => todo!(), + ExecsManagerInput::Stop => break, } } diff --git a/rust/src/filter.rs b/rust/src/filter.rs index e81ab53..6359d4e 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -189,7 +189,7 @@ impl Filter { tx.send(ExecsManagerInput::Exec(MAT { m: m.clone(), a: action.name(), - t, + t: t + action.after_duration().unwrap_or_default(), })) .unwrap(); } diff --git a/rust/src/matches.rs b/rust/src/matches.rs index 2803301..8b5b13e 100644 --- a/rust/src/matches.rs +++ b/rust/src/matches.rs @@ -21,6 +21,7 @@ pub enum MatchManagerInput { Unmatch(MFT), #[allow(dead_code)] Flush(MFT), + Stop, } type MatchesMap = BTreeMap>>; @@ -96,6 +97,7 @@ pub fn matches_manager( MatchManagerInput::Unmatch(mft) => matches.rm(&mft), #[allow(clippy::todo)] MatchManagerInput::Flush(_) => todo!(), // TODO handle flushes + MatchManagerInput::Stop => break, } } } diff --git a/rust/src/utils/threadpool.rs b/rust/src/utils/threadpool.rs index a09fa49..6dd24f9 100644 --- a/rust/src/utils/threadpool.rs +++ b/rust/src/utils/threadpool.rs @@ -53,8 +53,11 @@ struct Worker { impl Worker { fn new(receiver: Arc>>) -> Worker { let thread = spawn(move || loop { - let job = receiver.lock().unwrap().recv().unwrap(); - job(); + let received = receiver.lock().unwrap().recv(); + match received { + Ok(job) => job(), + Err(_) => break, + } }); Worker { thread } diff --git a/rust/test.jsonnet b/rust/test.jsonnet index dd93cda..8651a6a 100644 --- a/rust/test.jsonnet +++ b/rust/test.jsonnet @@ -33,11 +33,11 @@ retryperiod: '30s', actions: { damn: { - cmd: ['echo', ''], + cmd: ['notify-send', 'first stream', 'ban '], }, undamn: { - cmd: ['echo', 'undamn', ''], - after: '28s', + cmd: ['notify-send', 'first stream', 'unban '], + after: '6s', onexit: true, }, }, @@ -45,7 +45,7 @@ }, }, tailDown2: { - cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 1; echo found $i; done; sleep 3"], + cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 3; echo found $i; done; sleep 3"], filters: { findIP: { regex: [ @@ -53,14 +53,14 @@ '^found _$', ], retry: 2, - retryperiod: '30s', + retryperiod: '2s', actions: { damn: { - cmd: ['echo', ''], + cmd: ['notify-send', 'second stream', 'ban '], }, undamn: { - cmd: ['echo', 'undamn', ''], - after: '28s', + cmd: ['notify-send', 'second stream', 'unban '], + after: '4s', onexit: true, }, }, From a55ab5d8d814a212804804df13a0be7b4f006afa Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 30 Jul 2024 12:00:00 +0200 Subject: [PATCH 032/435] fix: run `on_exit` actions on exit --- rust/src/action.rs | 4 ++++ rust/src/daemon.rs | 2 ++ rust/src/execs.rs | 23 ++++++++++++++++++++++- rust/test.jsonnet | 6 +++--- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/rust/src/action.rs b/rust/src/action.rs index 8cdc05e..f8a8342 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -44,6 +44,10 @@ impl Action { self.after_duration } + pub fn on_exit(&self) -> bool { + self.on_exit + } + pub fn setup( &mut self, stream_name: &str, diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index a591df5..fbad380 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -34,6 +34,8 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { } }; + debug!("config: {:?}", config); + if !config.start() { error!("a start command failed, exiting."); exit(1); diff --git a/rust/src/execs.rs b/rust/src/execs.rs index f2051cd..67a3cf8 100644 --- a/rust/src/execs.rs +++ b/rust/src/execs.rs @@ -120,7 +120,28 @@ pub fn execs_manager( } #[allow(clippy::todo)] ExecsManagerInput::Flush(_mat) => todo!(), - ExecsManagerInput::Stop => break, + ExecsManagerInput::Stop => { + for (action_name, inner_map) in execs { + if config + .get_filter(&action_name.stream, &action_name.filter) + .unwrap() + .get_action(&action_name.action) + .unwrap() + .on_exit() + { + for (match_, inner_set) in inner_map { + for _ in inner_set { + exec_now(MAT { + m: match_.clone(), + a: action_name.clone(), + t: Local::now(), + }); + } + } + } + } + break; + } } } diff --git a/rust/test.jsonnet b/rust/test.jsonnet index 8651a6a..8cc1b5b 100644 --- a/rust/test.jsonnet +++ b/rust/test.jsonnet @@ -22,7 +22,7 @@ streams: { tailDown1: { - cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 1; echo found $i; done; sleep 2"], + cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do echo found $i; done"], filters: { findIP: { regex: [ @@ -45,7 +45,7 @@ }, }, tailDown2: { - cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 3; echo found $i; done; sleep 3"], + cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do echo found $i; done"], filters: { findIP: { regex: [ @@ -61,7 +61,7 @@ undamn: { cmd: ['notify-send', 'second stream', 'unban '], after: '4s', - onexit: true, + // onexit: false, }, }, }, From 4b2f760e120cc9f5cc9bb99802bad711bc0292b1 Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 12 Sep 2024 12:00:00 +0200 Subject: [PATCH 033/435] first draft of database untested for now --- rust/Cargo.lock | 122 ++++++++++++++++ rust/Cargo.toml | 1 + rust/src/config.rs | 15 +- rust/src/database.rs | 337 +++++++++++++++++++++++++++++++++++++++++++ rust/src/filter.rs | 8 +- rust/src/main.rs | 1 + rust/src/messages.rs | 6 +- rust/src/stream.rs | 4 +- 8 files changed, 486 insertions(+), 8 deletions(-) create mode 100644 rust/src/database.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 3a72f11..786a0f3 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -91,6 +91,15 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + [[package]] name = "autocfg" version = "1.3.0" @@ -124,6 +133,12 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "cc" version = "1.1.6" @@ -205,6 +220,12 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "cobs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" + [[package]] name = "colorchoice" version = "1.0.2" @@ -217,6 +238,12 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "critical-section" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64009896348fc5af4222e9cf7d7d82a95a256c634ebcf61c53e4ea461422242" + [[package]] name = "ctrlc" version = "3.4.4" @@ -227,18 +254,53 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "equivalent" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version", + "serde", + "spin", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.5.0" @@ -394,6 +456,16 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.22" @@ -482,6 +554,19 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c719dcf55f09a3a7e764c6649ab594c18a177e3599c467983cdf644bfc0a4088" +[[package]] +name = "postcard" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f7f0a8d620d71c457dd1d47df76bb18960378da56af4527aaa10f515eee732e" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless", + "serde", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -512,6 +597,7 @@ dependencies = [ "jrsonnet-evaluator", "log", "num_cpus", + "postcard", "regex", "serde", "serde_json", @@ -554,12 +640,33 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "ryu" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + [[package]] name = "serde" version = "1.0.204" @@ -605,6 +712,21 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 8a714da..95f8901 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -14,6 +14,7 @@ ctrlc = { version = "3.4.4", features = ["termination"] } jrsonnet-evaluator = "0.4.2" log = { version = "0.4.22", features = ["std"] } num_cpus = "1.16.0" +postcard = { version = "1.0.10", features = ["use-std"] } regex = "1.10.4" serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" diff --git a/rust/src/config.rs b/rust/src/config.rs index f145d07..5626e49 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -9,7 +9,11 @@ use anyhow::{anyhow, Context, Result}; use log::{error, info}; use serde::Deserialize; -use crate::{filter::Filter, pattern::Pattern, stream::Stream}; +use crate::{ + filter::{Filter, FilterName}, + pattern::Pattern, + stream::Stream, +}; pub type Patterns = BTreeMap; @@ -39,7 +43,14 @@ impl Config { } pub fn get_filter(&self, stream: &str, filter: &str) -> Option<&Filter> { - self.streams.get(stream)?.get_filter(filter) + self.streams.get(stream)?.filters().get(filter) + } + + pub fn filter_names(&self) -> Vec { + self.streams + .values() + .flat_map(|stream| stream.filters().values().map(|filter| filter.name())) + .collect() } pub fn setup(&mut self) -> Result<()> { diff --git a/rust/src/database.rs b/rust/src/database.rs new file mode 100644 index 0000000..a5d6466 --- /dev/null +++ b/rust/src/database.rs @@ -0,0 +1,337 @@ +#![allow(dead_code)] +use std::{ + collections::BTreeMap, + fs::{self, File}, + process::exit, + sync::{mpsc::Receiver, Arc}, +}; + +use chrono::{DateTime, Local}; +use log::{debug, error, info, warn}; +use postcard::{from_io, to_io, Error}; +use serde::{Deserialize, Serialize}; + +use crate::{ + config::Config, + filter::FilterName, + messages::{LogEntry, Match, Time}, +}; + +const LOG_DB_NAME: &str = "./reaction-matches.db"; +const LOG_DB_NEW_NAME: &str = "./reaction-matches.new.db"; +const FLUSH_DB_NAME: &str = "./reaction-flushes.db"; + +struct ReadDB { + f: File, + h: DatabaseReadHeader, +} + +impl ReadDB { + fn open(path: &str) -> Option { + let mut file = match File::open(path) { + Ok(file) => Some(file), + Err(err) => match err.kind() { + std::io::ErrorKind::NotFound => { + warn!( + "No DB found at {}. It's ok if this is the first time reaction is running.", + path + ); + None + } + _ => { + error!("Failed to open DB: {}", err); + exit(1); + } + }, + }?; + + let mut buf: Vec = Vec::new(); + let (database_header, _) = from_io((&mut file, &mut buf)).unwrap(); + + Some(ReadDB { + f: file, + h: database_header, + }) + } +} + +impl Iterator for ReadDB { + type Item = Result; + + fn next(&mut self) -> Option { + let mut buf: Vec = Vec::new(); + let result = from_io::((&mut self.f, &mut buf)); + match result { + Ok((item, _)) => Some(Ok(item.to(&self.h))), + Err(err) => match err { + Error::DeserializeUnexpectedEnd => None, + _ => Some(Err(err)), + }, + } + } +} + +struct WriteDB { + f: File, + h: DatabaseWriteHeader, +} + +impl WriteDB { + fn create(path: &str, config: &Arc) -> Self { + let mut file = match File::create(path) { + Ok(file) => file, + Err(err) => { + error!("Failed to create DB: {}", err); + exit(1); + } + }; + + let database_header = config + .filter_names() + .into_iter() + .enumerate() + .map(|(i, name)| (name, i as u16)) + .collect(); + + if let Err(err) = to_io(&database_header, &mut file) { + error!("Failed to write to DB: {}", err); + exit(1); + } + + WriteDB { + f: file, + h: database_header, + } + } + + fn write(&mut self, entry: LogEntry) { + let computed = ComputedLogEntry::from(entry, &self.h); + to_io(&computed, &mut self.f).unwrap(); + } +} + +#[derive(Clone)] +pub enum DatabaseInput { + Log(LogEntry), + Flush(LogEntry), +} + +type DatabaseWriteHeader = BTreeMap; +type DatabaseReadHeader = BTreeMap; + +#[derive(Serialize, Deserialize)] +struct ComputedLogEntry { + pub m: Match, + pub f: u16, + pub t: i64, + pub exec: bool, +} + +impl ComputedLogEntry { + fn from(value: LogEntry, header: &DatabaseWriteHeader) -> Self { + ComputedLogEntry { + m: value.m, + f: *header.get(&value.f).unwrap(), + t: value.t.timestamp(), + exec: value.exec, + } + } + fn to(self, header: &DatabaseReadHeader) -> LogEntry { + LogEntry { + m: self.m, + f: header.get(&self.f).unwrap().clone(), + t: DateTime::from_timestamp(self.t, 0) + .unwrap() + .with_timezone(&Local), + exec: self.exec, + } + } +} + +pub fn database_manager(config: Arc, log_rx: Receiver) { + let (mut log_db, mut flush_db) = rotate_db(&config, true); + + let mut cpt = 0; + while let Ok(order) = log_rx.recv() { + match order { + DatabaseInput::Flush(entry) => flush_db.write(entry), + DatabaseInput::Log(entry) => { + log_db.write(entry); + cpt += 1; + if cpt == 500_000 { + info!("Rotating database..."); + cpt = 0; + drop(log_db); + drop(flush_db); + (log_db, flush_db) = rotate_db(&config, false); + info!("Rotated database"); + } + } + }; + } +} + +fn rotate_db(config: &Arc, startup: bool) -> (WriteDB, WriteDB) { + let mut log_read_db = match ReadDB::open(LOG_DB_NAME) { + Some(db) => db, + None => { + return ( + WriteDB::create(LOG_DB_NAME, config), + WriteDB::create(FLUSH_DB_NAME, config), + ) + } + }; + let mut flush_read_db = match ReadDB::open(FLUSH_DB_NAME) { + Some(db) => db, + None => { + warn!( + "Strange! Found a {} but no {}, opening /dev/null instead", + LOG_DB_NAME, FLUSH_DB_NAME + ); + match ReadDB::open("/dev/null") { + Some(db) => db, + None => { + error!("Opening dummy /dev/null failed"); + exit(1); + } + } + } + }; + + let mut log_write_db = WriteDB::create(LOG_DB_NEW_NAME, config); + + _rotate_db( + config, + &mut log_read_db, + &mut flush_read_db, + &mut log_write_db, + startup, + ); + + drop(log_read_db); + drop(flush_read_db); + + if let Err(err) = fs::rename(LOG_DB_NEW_NAME, LOG_DB_NAME) { + error!("Failed to replace old DB with new one: {}", err); + exit(1); + } + + if let Err(err) = fs::remove_file(FLUSH_DB_NAME) { + error!("Failed to delete old DB: {}", err); + exit(1); + } + + let flush_write_db = WriteDB::create(FLUSH_DB_NAME, config); + (log_write_db, flush_write_db) +} + +fn _rotate_db( + config: &Arc, + log_read_db: &mut ReadDB, + flush_read_db: &mut ReadDB, + log_write_db: &mut WriteDB, + startup: bool, +) { + let mut discarded_entries: BTreeMap = BTreeMap::new(); + let mut discarded_count: usize = 0; + + // Read flushes + let mut flushes: BTreeMap> = BTreeMap::new(); + for flush_entry in flush_read_db { + match flush_entry { + Ok(entry) => { + if config + .get_filter(&entry.f.stream, &entry.f.filter) + .is_some() + { + let matches_map = flushes.entry(entry.f).or_default(); + matches_map.insert(entry.m, entry.t); + } else { + *discarded_entries.entry(entry.f).or_insert(0) += 1; + } + } + Err(err) => { + debug!("while reading flush db: {}", err); + discarded_count += 1; + } + } + } + let global_flush_map = flushes.get(&FilterName { + stream: "".into(), + filter: "".into(), + }); + + let now = Local::now(); + + // Read matches + for log_entry in log_read_db { + match log_entry { + Ok(entry) => { + // retrieve related stream & filter + let filter = match config.get_filter(&entry.f.stream, &entry.f.filter) { + Some(filter) => filter, + None => { + *discarded_entries.entry(entry.f).or_insert(0) += 1; + continue; + } + }; + + // Check if number of patterns is in sync + if entry.m.len() != filter.patterns().len() { + *discarded_entries.entry(entry.f).or_insert(0) += 1; + continue; + } + + // Check if hasn't been flushed + if let Some(map) = global_flush_map { + if let Some(time) = map.get(&entry.m) { + if time > &entry.t { + continue; + } + } + } + if let Some(map) = flushes.get(&entry.f) { + if let Some(time) = map.get(&entry.m) { + if time > &entry.t { + continue; + } + } + } + + // Store match & store in db + if !entry.exec { + if entry.t + filter.retry_duration().unwrap_or_default() > now { + if startup { + // TODO send match + } + + log_write_db.write(entry); + } + // Replay executions & store in db + } else if entry.t + filter.longuest_action_duration() > now { + if startup { + // TODO send flush + } + + log_write_db.write(entry); + } + } + Err(err) => { + debug!("while reading log db: {}", err); + discarded_count += 1; + } + } + } + + // Warn about errors + discarded_entries.iter() + .filter(|(_, &count)| count > 0) + .for_each(|(name, count)| warn!("{} entries discarded from the databases: stream/filter not found in configuration, or the patterns are not the same: {}", count, name)); + + if discarded_count > 0 { + error!( + "{} entries discarded from the databases (garbage while decoding)", + discarded_count + ); + } +} diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 6359d4e..efa53c8 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -7,7 +7,7 @@ use std::{ use chrono::TimeDelta; use log::info; use regex::Regex; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::{ action::Action, @@ -61,6 +61,10 @@ impl Filter { self.retry_duration } + pub fn longuest_action_duration(&self) -> TimeDelta { + self.longuest_action_duration + } + pub fn patterns(&self) -> &BTreeSet { &self.patterns } @@ -196,7 +200,7 @@ impl Filter { } } -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub struct FilterName { pub stream: String, pub filter: String, diff --git a/rust/src/main.rs b/rust/src/main.rs index 831c40c..42c037a 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -20,6 +20,7 @@ mod pattern; mod stream; // important threads +mod database; mod execs; mod matches; diff --git a/rust/src/messages.rs b/rust/src/messages.rs index 3d12805..41a166e 100644 --- a/rust/src/messages.rs +++ b/rust/src/messages.rs @@ -19,8 +19,10 @@ pub struct MAT { pub t: Time, } -#[allow(dead_code)] +#[derive(Clone)] pub struct LogEntry { - pub mft: MFT, + pub m: Match, + pub f: FilterName, + pub t: Time, pub exec: bool, } diff --git a/rust/src/stream.rs b/rust/src/stream.rs index 79580e7..ad2c776 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -22,8 +22,8 @@ pub struct Stream { } impl Stream { - pub fn get_filter(&self, name: &str) -> Option<&Filter> { - self.filters.get(name) + pub fn filters(&self) -> &BTreeMap { + &self.filters } pub fn setup(&mut self, name: &str, patterns: &Patterns) -> Result<(), String> { From 69d2436847435771b36a38026cf92986985e48b5 Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 12 Sep 2024 12:00:00 +0200 Subject: [PATCH 034/435] database tests; better error handling (avoid unwraping) --- rust/Cargo.lock | 64 ++++++- rust/Cargo.toml | 1 + rust/src/{database.rs => database/mod.rs} | 194 +++++++++++++++------- rust/src/database/tests.rs | 93 +++++++++++ rust/src/execs.rs | 2 +- rust/src/filter.rs | 2 +- rust/src/main.rs | 1 + rust/src/matches.rs | 2 +- rust/src/messages.rs | 24 ++- rust/src/tests.rs | 56 +++++++ 10 files changed, 368 insertions(+), 71 deletions(-) rename rust/src/{database.rs => database/mod.rs} (61%) create mode 100644 rust/src/database/tests.rs create mode 100644 rust/src/tests.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 786a0f3..485e683 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -72,7 +72,7 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -82,7 +82,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -251,7 +251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" dependencies = [ "nix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -272,6 +272,22 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" + [[package]] name = "hash32" version = "0.2.1" @@ -456,6 +472,12 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "lock_api" version = "0.4.12" @@ -602,6 +624,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "tempfile", "timer", ] @@ -649,6 +672,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "ryu" version = "1.0.18" @@ -767,6 +803,19 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tempfile" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "thiserror" version = "1.0.63" @@ -926,6 +975,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 95f8901..5acdb6a 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -19,4 +19,5 @@ regex = "1.10.4" serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" serde_yaml = "0.9.34" +tempfile = "3.12.0" timer = "0.2.0" diff --git a/rust/src/database.rs b/rust/src/database/mod.rs similarity index 61% rename from rust/src/database.rs rename to rust/src/database/mod.rs index a5d6466..26bd083 100644 --- a/rust/src/database.rs +++ b/rust/src/database/mod.rs @@ -1,6 +1,7 @@ #![allow(dead_code)] use std::{ collections::BTreeMap, + fmt::Display, fs::{self, File}, process::exit, sync::{mpsc::Receiver, Arc}, @@ -17,55 +18,70 @@ use crate::{ messages::{LogEntry, Match, Time}, }; +mod tests; + const LOG_DB_NAME: &str = "./reaction-matches.db"; const LOG_DB_NEW_NAME: &str = "./reaction-matches.new.db"; const FLUSH_DB_NAME: &str = "./reaction-flushes.db"; +const MAX_WRITES: u32 = 500_000; + +// FIXME how to use postcard::from_io with a dynamic buffer? +const HEADER_MAX_SIZE: usize = 1024 * 1024; +const ENTRY_MAX_SIZE: usize = 1024 * 1024; + struct ReadDB { f: File, h: DatabaseReadHeader, } impl ReadDB { - fn open(path: &str) -> Option { + fn open(path: &str) -> Result, DBError> { let mut file = match File::open(path) { - Ok(file) => Some(file), + Ok(file) => file, Err(err) => match err.kind() { std::io::ErrorKind::NotFound => { warn!( "No DB found at {}. It's ok if this is the first time reaction is running.", path ); - None + return Ok(None); } _ => { - error!("Failed to open DB: {}", err); - exit(1); + return Err(DBError::Error(format!("Could not open database: {}", err))); } }, - }?; + }; - let mut buf: Vec = Vec::new(); - let (database_header, _) = from_io((&mut file, &mut buf)).unwrap(); + let mut buf: [u8; HEADER_MAX_SIZE] = [0; HEADER_MAX_SIZE]; + let (database_header, _) = from_io((&mut file, &mut buf))?; - Some(ReadDB { + Ok(Some(ReadDB { f: file, h: database_header, - }) + })) } } impl Iterator for ReadDB { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { - let mut buf: Vec = Vec::new(); + let mut buf: [u8; ENTRY_MAX_SIZE] = [0; ENTRY_MAX_SIZE]; let result = from_io::((&mut self.f, &mut buf)); match result { - Ok((item, _)) => Some(Ok(item.to(&self.h))), + // FIXME why we got a default item instead of an error or something? + // How do we really know we reached the end? + Ok((item, _)) => { + if item.t == 0 { + None + } else { + Some(item.to(&self.h)) + } + } Err(err) => match err { Error::DeserializeUnexpectedEnd => None, - _ => Some(Err(err)), + _ => Some(Err(err.into())), }, } } @@ -86,34 +102,53 @@ impl WriteDB { } }; - let database_header = config - .filter_names() - .into_iter() - .enumerate() - .map(|(i, name)| (name, i as u16)) - .collect(); + let database_read_header: BTreeMap = + config.filter_names().into_iter().enumerate().collect(); - if let Err(err) = to_io(&database_header, &mut file) { + if let Err(err) = to_io(&database_read_header, &mut file) { error!("Failed to write to DB: {}", err); exit(1); } + let database_write_header = database_read_header + .into_iter() + .map(|(i, name)| (name, i as u16)) + .collect(); + WriteDB { f: file, - h: database_header, + h: database_write_header, } } - fn write(&mut self, entry: LogEntry) { - let computed = ComputedLogEntry::from(entry, &self.h); - to_io(&computed, &mut self.f).unwrap(); + fn write(&mut self, entry: LogEntry) -> Result<(), DBError> { + let computed = ComputedLogEntry::from(entry, &self.h)?; + to_io(&computed, &mut self.f)?; + Ok(()) } } -#[derive(Clone)] -pub enum DatabaseInput { - Log(LogEntry), - Flush(LogEntry), +#[derive(Debug)] +enum DBError { + InvalidFilterError(String), + PostcardError(postcard::Error), + Error(String), +} + +impl From for DBError { + fn from(value: postcard::Error) -> Self { + DBError::PostcardError(value) + } +} + +impl Display for DBError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DBError::InvalidFilterError(e) => write!(f, "invalid filter : {}", e), + DBError::PostcardError(e) => write!(f, "decode error: {}", e), + DBError::Error(e) => write!(f, "{}", e), + } + } } type DatabaseWriteHeader = BTreeMap; @@ -128,42 +163,69 @@ struct ComputedLogEntry { } impl ComputedLogEntry { - fn from(value: LogEntry, header: &DatabaseWriteHeader) -> Self { - ComputedLogEntry { - m: value.m, - f: *header.get(&value.f).unwrap(), - t: value.t.timestamp(), - exec: value.exec, + fn from(value: LogEntry, header: &DatabaseWriteHeader) -> Result { + match header.get(&value.f) { + Some(f) => Ok(ComputedLogEntry { + m: value.m, + f: *f, + t: value.t.timestamp(), + exec: value.exec, + }), + None => Err(DBError::InvalidFilterError(value.f.to_string())), } } - fn to(self, header: &DatabaseReadHeader) -> LogEntry { - LogEntry { - m: self.m, - f: header.get(&self.f).unwrap().clone(), - t: DateTime::from_timestamp(self.t, 0) - .unwrap() - .with_timezone(&Local), - exec: self.exec, + fn to(self, header: &DatabaseReadHeader) -> Result { + match header.get(&self.f) { + Some(f) => Ok(LogEntry { + m: self.m, + f: f.clone(), + t: DateTime::from_timestamp(self.t, 0) + .unwrap() + .with_timezone(&Local), + exec: self.exec, + }), + None => Err(DBError::InvalidFilterError(self.f.to_string())), } } } +#[derive(Clone)] +pub enum DatabaseInput { + Log(LogEntry), + Flush(LogEntry), +} + pub fn database_manager(config: Arc, log_rx: Receiver) { - let (mut log_db, mut flush_db) = rotate_db(&config, true); + let (mut log_db, mut flush_db) = match rotate_db(&config, true) { + Ok(dbs) => dbs, + Err(err) => { + error!("while rotating databases on start: {}", err); + exit(1); + } + }; let mut cpt = 0; while let Ok(order) = log_rx.recv() { match order { - DatabaseInput::Flush(entry) => flush_db.write(entry), + DatabaseInput::Flush(entry) => flush_db.write(entry).unwrap(), DatabaseInput::Log(entry) => { - log_db.write(entry); + log_db.write(entry).unwrap(); cpt += 1; - if cpt == 500_000 { + if cpt == MAX_WRITES { info!("Rotating database..."); cpt = 0; drop(log_db); drop(flush_db); - (log_db, flush_db) = rotate_db(&config, false); + (log_db, flush_db) = match rotate_db(&config, false) { + Ok(dbs) => dbs, + Err(err) => { + error!( + "while rotating databases after {} writes: {}", + MAX_WRITES, err + ); + exit(1); + } + }; info!("Rotated database"); } } @@ -171,28 +233,27 @@ pub fn database_manager(config: Arc, log_rx: Receiver) { } } -fn rotate_db(config: &Arc, startup: bool) -> (WriteDB, WriteDB) { - let mut log_read_db = match ReadDB::open(LOG_DB_NAME) { +fn rotate_db(config: &Arc, startup: bool) -> Result<(WriteDB, WriteDB), DBError> { + let mut log_read_db = match ReadDB::open(LOG_DB_NAME)? { Some(db) => db, None => { - return ( + return Ok(( WriteDB::create(LOG_DB_NAME, config), WriteDB::create(FLUSH_DB_NAME, config), - ) + )); } }; - let mut flush_read_db = match ReadDB::open(FLUSH_DB_NAME) { + let mut flush_read_db = match ReadDB::open(FLUSH_DB_NAME)? { Some(db) => db, None => { warn!( "Strange! Found a {} but no {}, opening /dev/null instead", LOG_DB_NAME, FLUSH_DB_NAME ); - match ReadDB::open("/dev/null") { + match ReadDB::open("/dev/null")? { Some(db) => db, None => { - error!("Opening dummy /dev/null failed"); - exit(1); + return Err(DBError::Error("/dev/null is not accessible".into())); } } } @@ -212,17 +273,18 @@ fn rotate_db(config: &Arc, startup: bool) -> (WriteDB, WriteDB) { drop(flush_read_db); if let Err(err) = fs::rename(LOG_DB_NEW_NAME, LOG_DB_NAME) { - error!("Failed to replace old DB with new one: {}", err); - exit(1); + return Err(DBError::Error(format!( + "Failed to replace old DB with new one: {}", + err + ))); } if let Err(err) = fs::remove_file(FLUSH_DB_NAME) { - error!("Failed to delete old DB: {}", err); - exit(1); + return Err(DBError::Error(format!("Failed to delete old DB: {}", err))); } let flush_write_db = WriteDB::create(FLUSH_DB_NAME, config); - (log_write_db, flush_write_db) + Ok((log_write_db, flush_write_db)) } fn _rotate_db( @@ -305,7 +367,10 @@ fn _rotate_db( // TODO send match } - log_write_db.write(entry); + if let Err(err) = log_write_db.write(entry) { + error!("Could not write to DB: {}", err); + exit(1); + } } // Replay executions & store in db } else if entry.t + filter.longuest_action_duration() > now { @@ -313,7 +378,10 @@ fn _rotate_db( // TODO send flush } - log_write_db.write(entry); + if let Err(err) = log_write_db.write(entry) { + error!("Could not write to DB: {}", err); + exit(1); + } } } Err(err) => { diff --git a/rust/src/database/tests.rs b/rust/src/database/tests.rs new file mode 100644 index 0000000..132c924 --- /dev/null +++ b/rust/src/database/tests.rs @@ -0,0 +1,93 @@ +#![cfg(test)] + +use std::sync::Arc; + +use chrono::Local; + +use crate::database::ReadDB; +use crate::{config::config_from_file, tests::Fixture}; +use crate::{filter::FilterName, messages::LogEntry}; + +use super::WriteDB; + +#[test] +fn write_and_read_db() { + let config_file = Fixture::from_string( + "config.jsonnet", + " +{ + patterns: { + num: { regex: '[0-9]+' }, + }, + streams: { + stream1: { + cmd: ['sh', '-c', 'for i in $(seq 10); do echo $((i % 5)); done'], + filters: { + filter1: { + regex: [''], + retry: 2, + retryperiod: '5s', + actions: { + action1: { + cmd: ['echo', ''], + } + } + } + } + } + } +} + ", + ); + + let config = Arc::new(config_from_file(&config_file).unwrap()); + + let correct_filter_name = FilterName { + stream: "stream1".into(), + filter: "filter1".into(), + }; + + let incorrect_filter_name = FilterName { + stream: "stream0".into(), + filter: "filter1".into(), + }; + + let correct_log_entry = LogEntry { + m: vec!["match1".into()], + f: correct_filter_name.clone(), + t: Local::now(), + exec: false, + }; + + let incorrect_log_entry = LogEntry { + m: vec!["match1".into()], + f: incorrect_filter_name.clone(), + t: Local::now(), + exec: false, + }; + + let db_path = Fixture::empty("matches.db"); + + let mut write_db = WriteDB::create(db_path.to_str().unwrap(), &config); + + assert!(write_db.write(correct_log_entry.clone()).is_ok()); + assert!(write_db.write(incorrect_log_entry).is_err()); + + drop(write_db); + + let read_db = ReadDB::open(db_path.to_str().unwrap()); + + assert!(read_db.is_ok()); + let read_db = read_db.unwrap(); + assert!(read_db.is_some()); + let mut read_db = read_db.unwrap(); + + let read_entry = read_db.next(); + assert!(read_entry.is_some()); + let read_entry = read_entry.unwrap(); + assert!(read_entry.is_ok()); + assert_eq!(read_entry.unwrap(), correct_log_entry); + + let read_entry = read_db.next(); + assert!(read_entry.is_none()); +} diff --git a/rust/src/execs.rs b/rust/src/execs.rs index 67a3cf8..93f840b 100644 --- a/rust/src/execs.rs +++ b/rust/src/execs.rs @@ -101,7 +101,7 @@ pub fn execs_manager( let timer = MessageTimer::new(exec_tx); - while let Ok(mat) = exec_rx.recv() { + for mat in exec_rx.iter() { match mat { ExecsManagerInput::Exec(mat) => { let now = Local::now(); diff --git a/rust/src/filter.rs b/rust/src/filter.rs index efa53c8..716a6f5 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -200,7 +200,7 @@ impl Filter { } } -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub struct FilterName { pub stream: String, pub filter: String, diff --git a/rust/src/main.rs b/rust/src/main.rs index 42c037a..64886a6 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -28,6 +28,7 @@ mod matches; mod client; mod daemon; +mod tests; mod utils; use client::{flush, show, test_regex}; diff --git a/rust/src/matches.rs b/rust/src/matches.rs index 8b5b13e..0fb85b1 100644 --- a/rust/src/matches.rs +++ b/rust/src/matches.rs @@ -62,7 +62,7 @@ pub fn matches_manager( let timer = MessageTimer::new(match_tx); - while let Ok(mft) = match_rx.recv() { + for mft in match_rx.iter() { match mft { MatchManagerInput::Match(mft) => { let filter = config.get_filter(&mft.f.stream, &mft.f.filter).unwrap(); diff --git a/rust/src/messages.rs b/rust/src/messages.rs index 41a166e..ed52054 100644 --- a/rust/src/messages.rs +++ b/rust/src/messages.rs @@ -1,4 +1,4 @@ -use chrono::{DateTime, Local}; +use chrono::{DateTime, Local, TimeDelta}; use crate::{action::ActionName, filter::FilterName}; @@ -19,10 +19,30 @@ pub struct MAT { pub t: Time, } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct LogEntry { pub m: Match, pub f: FilterName, pub t: Time, pub exec: bool, } + +impl PartialEq for LogEntry { + fn eq(&self, other: &Self) -> bool { + self.exec == other.exec + && self.m == other.m + && self.f == other.f + // We loose subsecond precision while encoding LogEntry + && (self.t - other.t) < TimeDelta::new(1, 0).unwrap() + } +} + +impl From for MFT { + fn from(value: LogEntry) -> Self { + MFT { + m: value.m, + f: value.f, + t: value.t, + } + } +} diff --git a/rust/src/tests.rs b/rust/src/tests.rs new file mode 100644 index 0000000..297d384 --- /dev/null +++ b/rust/src/tests.rs @@ -0,0 +1,56 @@ +#![cfg(test)] + +use std::{ + fs::File, + io::Write, + ops::Deref, + path::{Path, PathBuf}, +}; + +use tempfile::TempDir; + +pub struct Fixture { + path: PathBuf, + _tempdir: TempDir, +} + +impl Fixture { + // pub fn from_file(source: &Path) -> Self { + // let dir = TempDir::new().unwrap(); + // let filename = source.file_name(); + // let path = dir.path().join(&filename.unwrap()); + // fs::copy(&source, &path).unwrap(); + // Fixture { + // path, + // _tempdir: dir, + // } + // } + + pub fn from_string(filename: &str, content: &str) -> Self { + let dir = TempDir::new().unwrap(); + let path = dir.path().join(filename); + let mut file = File::create(&path).unwrap(); + file.write_all(content.as_bytes()).unwrap(); + Fixture { + path, + _tempdir: dir, + } + } + + pub fn empty(filename: &str) -> Self { + let dir = TempDir::new().unwrap(); + let path = dir.path().join(filename); + Fixture { + _tempdir: dir, + path, + } + } +} + +impl Deref for Fixture { + type Target = Path; + + fn deref(&self) -> &Self::Target { + self.path.deref() + } +} From 544af2283d052041bd8882560c3567fa0a62f0bc Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 14 Sep 2024 12:00:00 +0200 Subject: [PATCH 035/435] Remove anyhow crate --- rust/Cargo.lock | 7 ---- rust/Cargo.toml | 1 - rust/TODO | 2 -- rust/src/config.rs | 80 ++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 67 insertions(+), 23 deletions(-) delete mode 100644 rust/TODO diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 485e683..def69f1 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -85,12 +85,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "anyhow" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" - [[package]] name = "atomic-polyfill" version = "1.0.3" @@ -611,7 +605,6 @@ dependencies = [ name = "reaction" version = "0.1.0" dependencies = [ - "anyhow", "chrono", "clap", "clap_complete", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 5acdb6a..f1b7ad7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -6,7 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -anyhow = "1.0.86" chrono = { version = "0.4.38", features = ["std", "clock"] } clap = { version = "4.5.4", features = ["derive"] } clap_complete = "4.5.2" diff --git a/rust/TODO b/rust/TODO deleted file mode 100644 index d095bd5..0000000 --- a/rust/TODO +++ /dev/null @@ -1,2 +0,0 @@ -remove anyhow? -use chrono::TimeDelta and DateTime everywhere instead of std's? diff --git a/rust/src/config.rs b/rust/src/config.rs index 5626e49..8ae61c1 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -1,11 +1,12 @@ use std::{ collections::BTreeMap, + fmt::Display, fs::File, + io, path::Path, process::{Command, Stdio}, }; -use anyhow::{anyhow, Context, Result}; use log::{error, info}; use serde::Deserialize; @@ -15,6 +16,55 @@ use crate::{ stream::Stream, }; +#[derive(Debug)] +pub struct ConfigError { + err: String, +} + +impl From for ConfigError { + fn from(value: String) -> Self { + ConfigError { err: value } + } +} + +impl From<&str> for ConfigError { + fn from(value: &str) -> Self { + ConfigError { + err: value.to_string(), + } + } +} + +impl From for ConfigError { + fn from(value: io::Error) -> Self { + ConfigError { + err: value.to_string(), + } + } +} + +impl From for ConfigError { + fn from(value: serde_json::Error) -> Self { + ConfigError { + err: value.to_string(), + } + } +} + +impl From for ConfigError { + fn from(value: serde_yaml::Error) -> Self { + ConfigError { + err: value.to_string(), + } + } +} + +impl Display for ConfigError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.err) + } +} + pub type Patterns = BTreeMap; #[derive(Clone, Debug, Deserialize)] @@ -53,9 +103,10 @@ impl Config { .collect() } - pub fn setup(&mut self) -> Result<()> { - self._setup() - .map_err(|msg| anyhow!("Bad configuration: {}", msg)) + pub fn setup(&mut self) -> Result<(), ConfigError> { + Ok(self + ._setup() + .map_err(|msg| format!("Bad configuration: {}", msg))?) } pub fn _setup(&mut self) -> Result<(), String> { @@ -91,24 +142,26 @@ enum Format { Json, Jsonnet, } -pub fn config_from_file(path: &Path) -> Result { - _config_from_file(path).with_context(|| anyhow!("Configuration file {}:", path.display())) +pub fn config_from_file(path: &Path) -> Result { + Ok(_config_from_file(path) + .map_err(|msg| format!("Configuration file {}: {}", path.display(), msg.err))?) } -fn _config_from_file(path: &Path) -> Result { +fn _config_from_file(path: &Path) -> Result { let extension = path .extension() .and_then(|ex| ex.to_str()) - .ok_or(anyhow!("no file extension"))?; + .ok_or("no file extension")?; let format = match extension { "yaml" | "yml" => Format::Yaml, "json" => Format::Json, "jsonnet" => Format::Jsonnet, _ => { - return Err(anyhow!( + return Err(format!( "extension {} is not recognized. Must be json, jsonnet, yml or yaml.", extension - )) + ) + .into()) } }; @@ -126,10 +179,11 @@ fn _config_from_file(path: &Path) -> Result { mod jsonnet { use std::path::Path; - use anyhow::{anyhow, Result}; use jrsonnet_evaluator::{error::LocError, EvaluationState, FileImportResolver}; - pub fn from_path(path: &Path) -> Result { + use super::ConfigError; + + pub fn from_path(path: &Path) -> Result { let state = EvaluationState::default(); state.with_stdlib(); state.set_import_resolver(Box::::default()); @@ -139,7 +193,7 @@ mod jsonnet { match evaluate(path, &state) { Ok(val) => Ok(val), - Err(err) => Err(anyhow!("{}", state.stringify_err(&err))), + Err(err) => Err(state.stringify_err(&err).into()), } } fn evaluate(path: &Path, state: &EvaluationState) -> Result { From ed77120aa0e17f20b2213d289bf9ba4627980b94 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 16 Sep 2024 12:00:00 +0200 Subject: [PATCH 036/435] database integrated to daemon code --- rust/src/daemon.rs | 40 +++++++++++---- rust/src/database/mod.rs | 106 +++++++++++++++++++++------------------ rust/src/matches.rs | 20 ++++++-- 3 files changed, 105 insertions(+), 61 deletions(-) diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index fbad380..0bc6801 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -13,6 +13,7 @@ use log::{debug, error, info, Level}; use crate::{ config, + database::database_manager, execs::{execs_manager, ExecsManagerInput}, matches::{matches_manager, MatchManagerInput}, utils::SimpleLogger, @@ -46,18 +47,35 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { let (match_tx, match_rx) = channel(); let (exec_tx, exec_rx) = channel(); + let (log_tx, log_rx) = channel(); - let config_matches = config.clone(); - let match_tx_matches = match_tx.clone(); - let exec_tx_matches = exec_tx.clone(); - let matches_manager_thread_handle = thread::spawn(move || { - matches_manager(config_matches, match_rx, match_tx_matches, exec_tx_matches) - }); + let matches_manager_thread_handle = { + let config_matches = config.clone(); + let match_tx_matches = match_tx.clone(); + let exec_tx_matches = exec_tx.clone(); + thread::spawn(move || { + matches_manager( + config_matches, + match_rx, + match_tx_matches, + exec_tx_matches, + log_tx, + ) + }) + }; - let config_execs = config.clone(); - let exec_tx_execs = exec_tx.clone(); - let execs_manager_thread_handle = - thread::spawn(move || execs_manager(config_execs, exec_rx, exec_tx_execs)); + let execs_manager_thread_handle = { + let config_execs = config.clone(); + let exec_tx_execs = exec_tx.clone(); + thread::spawn(move || execs_manager(config_execs, exec_rx, exec_tx_execs)) + }; + + let database_manager_thread_handle = { + let config_database = config.clone(); + let match_tx_database = match_tx.clone(); + // The `thread::spawn` is done in the function, after database rotation is finished + database_manager(config_database, log_rx, match_tx_database) + }; for stream in config.streams().values() { let stream = stream.clone(); @@ -100,6 +118,8 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { exec_tx.send(ExecsManagerInput::Stop).unwrap(); let _ = execs_manager_thread_handle.join(); + let _ = database_manager_thread_handle.join(); + let stop_ok = config.stop(); // TODO flush DB diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs index 26bd083..635d123 100644 --- a/rust/src/database/mod.rs +++ b/rust/src/database/mod.rs @@ -4,7 +4,11 @@ use std::{ fmt::Display, fs::{self, File}, process::exit, - sync::{mpsc::Receiver, Arc}, + sync::{ + mpsc::{Receiver, Sender}, + Arc, + }, + thread, }; use chrono::{DateTime, Local}; @@ -15,6 +19,7 @@ use serde::{Deserialize, Serialize}; use crate::{ config::Config, filter::FilterName, + matches::MatchManagerInput, messages::{LogEntry, Match, Time}, }; @@ -190,13 +195,18 @@ impl ComputedLogEntry { } #[derive(Clone)] -pub enum DatabaseInput { +pub enum DatabaseManagerInput { Log(LogEntry), Flush(LogEntry), } -pub fn database_manager(config: Arc, log_rx: Receiver) { - let (mut log_db, mut flush_db) = match rotate_db(&config, true) { +/// First rotates the database, then spawns the database thread +pub fn database_manager( + config: Arc, + log_rx: Receiver, + matches_tx: Sender, +) -> thread::JoinHandle<()> { + let (mut log_db, mut flush_db) = match rotate_db(&config, Some(matches_tx)) { Ok(dbs) => dbs, Err(err) => { error!("while rotating databases on start: {}", err); @@ -204,36 +214,41 @@ pub fn database_manager(config: Arc, log_rx: Receiver) { } }; - let mut cpt = 0; - while let Ok(order) = log_rx.recv() { - match order { - DatabaseInput::Flush(entry) => flush_db.write(entry).unwrap(), - DatabaseInput::Log(entry) => { - log_db.write(entry).unwrap(); - cpt += 1; - if cpt == MAX_WRITES { - info!("Rotating database..."); - cpt = 0; - drop(log_db); - drop(flush_db); - (log_db, flush_db) = match rotate_db(&config, false) { - Ok(dbs) => dbs, - Err(err) => { - error!( - "while rotating databases after {} writes: {}", - MAX_WRITES, err - ); - exit(1); - } - }; - info!("Rotated database"); + thread::spawn(move || { + let mut cpt = 0; + while let Ok(order) = log_rx.recv() { + match order { + DatabaseManagerInput::Flush(entry) => flush_db.write(entry).unwrap(), + DatabaseManagerInput::Log(entry) => { + log_db.write(entry).unwrap(); + cpt += 1; + if cpt == MAX_WRITES { + info!("Rotating database..."); + cpt = 0; + drop(log_db); + drop(flush_db); + (log_db, flush_db) = match rotate_db(&config, None) { + Ok(dbs) => dbs, + Err(err) => { + error!( + "while rotating databases after {} writes: {}", + MAX_WRITES, err + ); + exit(1); + } + }; + info!("Rotated database"); + } } - } - }; - } + }; + } + }) } -fn rotate_db(config: &Arc, startup: bool) -> Result<(WriteDB, WriteDB), DBError> { +fn rotate_db( + config: &Arc, + matches_tx: Option>, +) -> Result<(WriteDB, WriteDB), DBError> { let mut log_read_db = match ReadDB::open(LOG_DB_NAME)? { Some(db) => db, None => { @@ -263,10 +278,10 @@ fn rotate_db(config: &Arc, startup: bool) -> Result<(WriteDB, WriteDB), _rotate_db( config, + matches_tx, &mut log_read_db, &mut flush_read_db, &mut log_write_db, - startup, ); drop(log_read_db); @@ -289,10 +304,10 @@ fn rotate_db(config: &Arc, startup: bool) -> Result<(WriteDB, WriteDB), fn _rotate_db( config: &Arc, + matches_tx: Option>, log_read_db: &mut ReadDB, flush_read_db: &mut ReadDB, log_write_db: &mut WriteDB, - startup: bool, ) { let mut discarded_entries: BTreeMap = BTreeMap::new(); let mut discarded_count: usize = 0; @@ -361,21 +376,12 @@ fn _rotate_db( } // Store match & store in db - if !entry.exec { - if entry.t + filter.retry_duration().unwrap_or_default() > now { - if startup { - // TODO send match - } - - if let Err(err) = log_write_db.write(entry) { - error!("Could not write to DB: {}", err); - exit(1); - } - } - // Replay executions & store in db - } else if entry.t + filter.longuest_action_duration() > now { - if startup { - // TODO send flush + if (!entry.exec && entry.t + filter.retry_duration().unwrap_or_default() > now) + || (entry.exec && entry.t + filter.longuest_action_duration() > now) + { + if let Some(tx) = &matches_tx { + tx.send(MatchManagerInput::Match(entry.clone().into())) + .unwrap(); } if let Err(err) = log_write_db.write(entry) { @@ -391,6 +397,10 @@ fn _rotate_db( } } + if let Some(tx) = matches_tx { + tx.send(MatchManagerInput::EndOfStartup).unwrap(); + } + // Warn about errors discarded_entries.iter() .filter(|(_, &count)| count > 0) diff --git a/rust/src/matches.rs b/rust/src/matches.rs index 0fb85b1..a06fd5d 100644 --- a/rust/src/matches.rs +++ b/rust/src/matches.rs @@ -10,9 +10,10 @@ use timer::MessageTimer; use crate::{ config::Config, + database::DatabaseManagerInput, execs::ExecsManagerInput, filter::FilterName, - messages::{Match, Time, MFT}, + messages::{LogEntry, Match, Time, MFT}, }; #[derive(Clone)] @@ -21,6 +22,7 @@ pub enum MatchManagerInput { Unmatch(MFT), #[allow(dead_code)] Flush(MFT), + EndOfStartup, Stop, } @@ -57,13 +59,17 @@ pub fn matches_manager( match_rx: Receiver, match_tx: Sender, action_tx: Sender, + log_tx: Sender, ) { let mut matches: MatchesMap = BTreeMap::new(); let timer = MessageTimer::new(match_tx); + let mut startup = true; + for mft in match_rx.iter() { match mft { + MatchManagerInput::EndOfStartup => startup = false, MatchManagerInput::Match(mft) => { let filter = config.get_filter(&mft.f.stream, &mft.f.filter).unwrap(); @@ -91,8 +97,16 @@ pub fn matches_manager( filter.send_actions(&mft.m, mft.t, &action_tx); } - // TODO send to DB - // log_tx.send(LogEntry { mft, exec }); + if !startup { + log_tx + .send(DatabaseManagerInput::Log(LogEntry { + exec, + m: mft.m, + f: mft.f, + t: mft.t, + })) + .unwrap(); + } } MatchManagerInput::Unmatch(mft) => matches.rm(&mft), #[allow(clippy::todo)] From 0fb870f5be0da28dfe65abd964f92fe487ec3c3a Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 16 Sep 2024 12:00:00 +0200 Subject: [PATCH 037/435] Fix time ambiguity bug --- .gitignore | 2 +- rust/src/database/mod.rs | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 0f1d2c6..3b23bec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ /reaction /ip46tables /nft46 -/reaction*.db +reaction*.db /reaction*.sock /result /wiki diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs index 635d123..5c9d464 100644 --- a/rust/src/database/mod.rs +++ b/rust/src/database/mod.rs @@ -11,7 +11,7 @@ use std::{ thread, }; -use chrono::{DateTime, Local}; +use chrono::{DateTime, Local, TimeDelta}; use log::{debug, error, info, warn}; use postcard::{from_io, to_io, Error}; use serde::{Deserialize, Serialize}; @@ -312,6 +312,8 @@ fn _rotate_db( let mut discarded_entries: BTreeMap = BTreeMap::new(); let mut discarded_count: usize = 0; + let mut millisecond_disambiguation_counter: u32 = 0; + // Read flushes let mut flushes: BTreeMap> = BTreeMap::new(); for flush_entry in flush_read_db { @@ -343,7 +345,7 @@ fn _rotate_db( // Read matches for log_entry in log_read_db { match log_entry { - Ok(entry) => { + Ok(mut entry) => { // retrieve related stream & filter let filter = match config.get_filter(&entry.f.stream, &entry.f.filter) { Some(filter) => filter, @@ -379,6 +381,11 @@ fn _rotate_db( if (!entry.exec && entry.t + filter.retry_duration().unwrap_or_default() > now) || (entry.exec && entry.t + filter.longuest_action_duration() > now) { + // We loose subsecond precision when storing times, so we add those fake + // milliseconds to make sure each time is unique + entry.t += TimeDelta::new(0, millisecond_disambiguation_counter).unwrap(); + millisecond_disambiguation_counter += 1; + if let Some(tx) = &matches_tx { tx.send(MatchManagerInput::Match(entry.clone().into())) .unwrap(); From cd206d77a71d4c926e655b7d528bffe81d802ab3 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 16 Sep 2024 12:00:00 +0200 Subject: [PATCH 038/435] Use thiserror macros --- rust/Cargo.lock | 1 + rust/Cargo.toml | 1 + rust/src/config.rs | 107 ++++++++++++--------------------------- rust/src/database/mod.rs | 29 +++-------- 4 files changed, 43 insertions(+), 95 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index def69f1..d2d897b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -618,6 +618,7 @@ dependencies = [ "serde_json", "serde_yaml", "tempfile", + "thiserror", "timer", ] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index f1b7ad7..9df7cce 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -19,4 +19,5 @@ serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" serde_yaml = "0.9.34" tempfile = "3.12.0" +thiserror = "1.0.63" timer = "0.2.0" diff --git a/rust/src/config.rs b/rust/src/config.rs index 8ae61c1..5d8b299 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -1,6 +1,5 @@ use std::{ collections::BTreeMap, - fmt::Display, fs::File, io, path::Path, @@ -9,6 +8,7 @@ use std::{ use log::{error, info}; use serde::Deserialize; +use thiserror::Error; use crate::{ filter::{Filter, FilterName}, @@ -16,55 +16,6 @@ use crate::{ stream::Stream, }; -#[derive(Debug)] -pub struct ConfigError { - err: String, -} - -impl From for ConfigError { - fn from(value: String) -> Self { - ConfigError { err: value } - } -} - -impl From<&str> for ConfigError { - fn from(value: &str) -> Self { - ConfigError { - err: value.to_string(), - } - } -} - -impl From for ConfigError { - fn from(value: io::Error) -> Self { - ConfigError { - err: value.to_string(), - } - } -} - -impl From for ConfigError { - fn from(value: serde_json::Error) -> Self { - ConfigError { - err: value.to_string(), - } - } -} - -impl From for ConfigError { - fn from(value: serde_yaml::Error) -> Self { - ConfigError { - err: value.to_string(), - } - } -} - -impl Display for ConfigError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.err) - } -} - pub type Patterns = BTreeMap; #[derive(Clone, Debug, Deserialize)] @@ -103,13 +54,7 @@ impl Config { .collect() } - pub fn setup(&mut self) -> Result<(), ConfigError> { - Ok(self - ._setup() - .map_err(|msg| format!("Bad configuration: {}", msg))?) - } - - pub fn _setup(&mut self) -> Result<(), String> { + pub fn setup(&mut self) -> Result<(), String> { if self.concurrency == 0 { self.concurrency = num_cpus::get(); } @@ -142,28 +87,42 @@ enum Format { Json, Jsonnet, } -pub fn config_from_file(path: &Path) -> Result { - Ok(_config_from_file(path) - .map_err(|msg| format!("Configuration file {}: {}", path.display(), msg.err))?) + +#[derive(Error, Debug)] +enum ConfigError { + #[error("{0}")] + Any(String), + #[error("Bad configuration: {0}")] + BadConfig(String), + #[error("{0}. Must be json, jsonnet, yml or yaml.")] + Extension(String), + #[error("{0}")] + IO(#[from] io::Error), + #[error("{0}")] + JSON(#[from] serde_json::Error), + #[error("{0}")] + YAML(#[from] serde_yaml::Error), } + +pub fn config_from_file(path: &Path) -> Result { + _config_from_file(path).map_err(|err| format!("Configuration file {}: {}", path.display(), err)) +} + fn _config_from_file(path: &Path) -> Result { let extension = path .extension() .and_then(|ex| ex.to_str()) - .ok_or("no file extension")?; + .ok_or(ConfigError::Extension("no file extension".into()))?; let format = match extension { - "yaml" | "yml" => Format::Yaml, - "json" => Format::Json, - "jsonnet" => Format::Jsonnet, - _ => { - return Err(format!( - "extension {} is not recognized. Must be json, jsonnet, yml or yaml.", - extension - ) - .into()) - } - }; + "yaml" | "yml" => Ok(Format::Yaml), + "json" => Ok(Format::Json), + "jsonnet" => Ok(Format::Jsonnet), + _ => Err(ConfigError::Extension(format!( + "extension {} is not recognized", + extension + ))), + }?; let mut config: Config = match format { Format::Json => serde_json::from_reader(File::open(path)?)?, @@ -171,7 +130,7 @@ fn _config_from_file(path: &Path) -> Result { Format::Jsonnet => serde_json::from_str(&jsonnet::from_path(path)?)?, }; - config.setup()?; + config.setup().map_err(ConfigError::BadConfig)?; Ok(config) } @@ -193,7 +152,7 @@ mod jsonnet { match evaluate(path, &state) { Ok(val) => Ok(val), - Err(err) => Err(state.stringify_err(&err).into()), + Err(err) => Err(ConfigError::Any(state.stringify_err(&err))), } } fn evaluate(path: &Path, state: &EvaluationState) -> Result { diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs index 5c9d464..6af7b40 100644 --- a/rust/src/database/mod.rs +++ b/rust/src/database/mod.rs @@ -1,7 +1,6 @@ #![allow(dead_code)] use std::{ collections::BTreeMap, - fmt::Display, fs::{self, File}, process::exit, sync::{ @@ -15,6 +14,7 @@ use chrono::{DateTime, Local, TimeDelta}; use log::{debug, error, info, warn}; use postcard::{from_io, to_io, Error}; use serde::{Deserialize, Serialize}; +use thiserror::Error; use crate::{ config::Config, @@ -133,29 +133,16 @@ impl WriteDB { } } -#[derive(Debug)] +#[derive(Error, Debug)] enum DBError { + #[error("invalid filter: {0}")] InvalidFilterError(String), - PostcardError(postcard::Error), + #[error("decode error: {0}")] + PostcardError(#[from] postcard::Error), + #[error("{0}")] Error(String), } -impl From for DBError { - fn from(value: postcard::Error) -> Self { - DBError::PostcardError(value) - } -} - -impl Display for DBError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - DBError::InvalidFilterError(e) => write!(f, "invalid filter : {}", e), - DBError::PostcardError(e) => write!(f, "decode error: {}", e), - DBError::Error(e) => write!(f, "{}", e), - } - } -} - type DatabaseWriteHeader = BTreeMap; type DatabaseReadHeader = BTreeMap; @@ -223,7 +210,6 @@ pub fn database_manager( log_db.write(entry).unwrap(); cpt += 1; if cpt == MAX_WRITES { - info!("Rotating database..."); cpt = 0; drop(log_db); drop(flush_db); @@ -237,7 +223,6 @@ pub fn database_manager( exit(1); } }; - info!("Rotated database"); } } }; @@ -249,6 +234,7 @@ fn rotate_db( config: &Arc, matches_tx: Option>, ) -> Result<(WriteDB, WriteDB), DBError> { + info!("Rotating database..."); let mut log_read_db = match ReadDB::open(LOG_DB_NAME)? { Some(db) => db, None => { @@ -299,6 +285,7 @@ fn rotate_db( } let flush_write_db = WriteDB::create(FLUSH_DB_NAME, config); + info!("Rotated database"); Ok((log_write_db, flush_write_db)) } From 81616fb1d94ef3d28a71e9a296d452370b7e2500 Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 18 Sep 2024 12:00:00 +0200 Subject: [PATCH 039/435] =?UTF-8?q?rename=20config=5Ffrom=5Ffile=20?= =?UTF-8?q?=E2=86=92=20Config::from=5Ffile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rust/src/config.rs | 63 +++++++++++++++++++++++----------------------- rust/src/daemon.rs | 4 +-- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/rust/src/config.rs b/rust/src/config.rs index 5d8b299..1f608f5 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -80,6 +80,38 @@ impl Config { pub fn stop(&self) -> bool { run_commands(&self.stop, "stop") } + + pub fn from_file(path: &Path) -> Result { + Config::_from_file(path) + .map_err(|err| format!("Configuration file {}: {}", path.display(), err)) + } + + fn _from_file(path: &Path) -> Result { + let extension = path + .extension() + .and_then(|ex| ex.to_str()) + .ok_or(ConfigError::Extension("no file extension".into()))?; + + let format = match extension { + "yaml" | "yml" => Ok(Format::Yaml), + "json" => Ok(Format::Json), + "jsonnet" => Ok(Format::Jsonnet), + _ => Err(ConfigError::Extension(format!( + "extension {} is not recognized", + extension + ))), + }?; + + let mut config: Config = match format { + Format::Json => serde_json::from_reader(File::open(path)?)?, + Format::Yaml => serde_yaml::from_reader(File::open(path)?)?, + Format::Jsonnet => serde_json::from_str(&jsonnet::from_path(path)?)?, + }; + + config.setup().map_err(ConfigError::BadConfig)?; + + Ok(config) + } } enum Format { @@ -104,37 +136,6 @@ enum ConfigError { YAML(#[from] serde_yaml::Error), } -pub fn config_from_file(path: &Path) -> Result { - _config_from_file(path).map_err(|err| format!("Configuration file {}: {}", path.display(), err)) -} - -fn _config_from_file(path: &Path) -> Result { - let extension = path - .extension() - .and_then(|ex| ex.to_str()) - .ok_or(ConfigError::Extension("no file extension".into()))?; - - let format = match extension { - "yaml" | "yml" => Ok(Format::Yaml), - "json" => Ok(Format::Json), - "jsonnet" => Ok(Format::Jsonnet), - _ => Err(ConfigError::Extension(format!( - "extension {} is not recognized", - extension - ))), - }?; - - let mut config: Config = match format { - Format::Json => serde_json::from_reader(File::open(path)?)?, - Format::Yaml => serde_yaml::from_reader(File::open(path)?)?, - Format::Jsonnet => serde_json::from_str(&jsonnet::from_path(path)?)?, - }; - - config.setup().map_err(ConfigError::BadConfig)?; - - Ok(config) -} - mod jsonnet { use std::path::Path; diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index 0bc6801..5588cc5 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -12,7 +12,7 @@ use std::{ use log::{debug, error, info, Level}; use crate::{ - config, + config::Config, database::database_manager, execs::{execs_manager, ExecsManagerInput}, matches::{matches_manager, MatchManagerInput}, @@ -27,7 +27,7 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { debug!("daemon {config_path:?} {loglevel:?} {socket:?}"); - let config = match config::config_from_file(config_path) { + let config = match Config::from_file(config_path) { Ok(config) => Arc::new(config), Err(err) => { error!("{err}"); From a772b8347d7306067a39ef69f519d17ce0463fae Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 22 Sep 2024 12:00:00 +0200 Subject: [PATCH 040/435] Optimization by changing ownership model Avoid useless indirections with FilterName and ActionName Put Arc and Arc instead --- rust/heavy-load.yml | 115 +++++++++++++++++++++++++ rust/src/action.rs | 88 +++++++++++++------- rust/src/config.rs | 23 +++-- rust/src/daemon.rs | 22 ++--- rust/src/database/mod.rs | 87 +++++++------------ rust/src/database/tests.rs | 16 ++-- rust/src/execs.rs | 49 +++++------ rust/src/filter.rs | 166 ++++++++++++++++++++----------------- rust/src/matches.rs | 67 ++++++++++----- rust/src/messages.rs | 10 ++- rust/src/stream.rs | 45 ++++++---- rust/test.jsonnet | 34 ++------ 12 files changed, 417 insertions(+), 305 deletions(-) create mode 100644 rust/heavy-load.yml diff --git a/rust/heavy-load.yml b/rust/heavy-load.yml new file mode 100644 index 0000000..98a094a --- /dev/null +++ b/rust/heavy-load.yml @@ -0,0 +1,115 @@ +--- +patterns: + num: + regex: '[0-9]{3}' + ip: + regex: '(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})' + ignore: + - 1.0.0.1 + +streams: + tailDown1: + cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] + filters: + find: + regex: + - '^found ' + retry: 480 + retryperiod: 1m + actions: + damn: + cmd: [ 'sleep', '0.0' ] + undamn: + cmd: [ 'sleep', '0.0' ] + after: 1m + onexit: false + tailDown2: + cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] + filters: + find: + regex: + - '^found ' + retry: 480 + retryperiod: 1m + actions: + damn: + cmd: [ 'sleep', '0.0' ] + undamn: + cmd: [ 'sleep', '0.0' ] + after: 1m + onexit: false + tailDown3: + cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] + filters: + find: + regex: + - '^found ' + retry: 480 + retryperiod: 1m + actions: + damn: + cmd: [ 'sleep', '0.0' ] + undamn: + cmd: [ 'sleep', '0.0' ] + after: 1m + onexit: false + tailDown4: + cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] + filters: + find: + regex: + - '^found ' + retry: 480 + retryperiod: 1m + actions: + damn: + cmd: [ 'sleep', '0.0' ] + undamn: + cmd: [ 'sleep', '0.0' ] + after: 1m + onexit: false + # tailDown2: + # cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo prout $i; done' ] + # filters: + # findIP: + # regex: + # - '^prout ' + # retry: 500 + # retryperiod: 1m + # actions: + # damn: + # cmd: [ 'sleep', '0.0' ] + # undamn: + # cmd: [ 'sleep', '0.0' ] + # after: 1m + # onexit: false + # tailDown3: + # cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo nanana $i; done' ] + # filters: + # findIP: + # regex: + # - '^nanana ' + # retry: 500 + # retryperiod: 2m + # actions: + # damn: + # cmd: [ 'sleep', '0.0' ] + # undamn: + # cmd: [ 'sleep', '0.0' ] + # after: 1m + # onexit: false + # tailDown4: + # cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo nanana $i; done' ] + # filters: + # findIP: + # regex: + # - '^nomatch $' + # retry: 500 + # retryperiod: 2m + # actions: + # damn: + # cmd: [ 'sleep', '0.0' ] + # undamn: + # cmd: [ 'sleep', '0.0' ] + # after: 1m + # onexit: false diff --git a/rust/src/action.rs b/rust/src/action.rs index f8a8342..f95de7b 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeSet, fmt::Display, process::Command}; +use std::{cmp::Ordering, collections::BTreeSet, fmt::Display, process::Command, sync::Arc}; use chrono::TimeDelta; @@ -19,6 +19,8 @@ pub struct Action { #[serde(rename = "onexit", default = "set_false")] on_exit: bool, + #[serde(skip)] + patterns: Arc>>, #[serde(skip)] name: String, #[serde(skip)] @@ -32,14 +34,6 @@ fn set_false() -> bool { } impl Action { - pub fn name(&self) -> ActionName { - ActionName { - stream: self.stream_name.clone(), - filter: self.filter_name.clone(), - action: self.name.clone(), - } - } - pub fn after_duration(&self) -> Option { self.after_duration } @@ -53,15 +47,24 @@ impl Action { stream_name: &str, filter_name: &str, name: &str, + patterns: Arc>>, ) -> Result<(), String> { - self._setup(stream_name, filter_name, name) + self._setup(stream_name, filter_name, name, patterns) .map_err(|msg| format!("action {}: {}", name, msg)) } - fn _setup(&mut self, stream_name: &str, filter_name: &str, name: &str) -> Result<(), String> { + fn _setup( + &mut self, + stream_name: &str, + filter_name: &str, + name: &str, + patterns: Arc>>, + ) -> Result<(), String> { self.stream_name = stream_name.to_string(); self.filter_name = filter_name.to_string(); self.name = name.to_string(); + self.patterns = patterns; + if self.name.is_empty() { return Err("action name is empty".into()); } @@ -90,15 +93,15 @@ impl Action { } // TODO test - pub fn exec(&self, match_: &Match, patterns: &BTreeSet) -> Command { - let computed_command = if patterns.is_empty() { + pub fn exec(&self, match_: &Match) -> Command { + let computed_command = if self.patterns.is_empty() { self.cmd.clone() } else { self.cmd .iter() .map(|item| { (0..match_.len()) - .zip(patterns) + .zip(self.patterns.as_ref()) .fold(item.clone(), |acc, (i, pattern)| { acc.replace(pattern.name_with_braces(), &match_[i]) }) @@ -111,15 +114,32 @@ impl Action { } } -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct ActionName { - pub stream: String, - pub filter: String, - pub action: String, +impl PartialEq for Action { + fn eq(&self, other: &Self) -> bool { + self.stream_name == other.stream_name && self.name == other.name + } } -impl Display for ActionName { +impl Eq for Action {} +impl Ord for Action { + fn cmp(&self, other: &Self) -> Ordering { + match self.stream_name.cmp(&other.stream_name) { + Ordering::Equal => match self.filter_name.cmp(&other.filter_name) { + Ordering::Equal => self.name.cmp(&other.name), + o => o, + }, + o => o, + } + } +} +impl PartialOrd for Action { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Display for Action { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}.{}.{}", self.stream, self.filter, self.action) + write!(f, "{}.{}.{}", self.stream_name, self.filter_name, self.name) } } @@ -137,49 +157,53 @@ pub mod tests { after: None, after_duration: None, on_exit: false, + patterns: Arc::new(BTreeSet::default()), } } - pub fn ok_action() -> Action { + pub fn ok_action() -> Arc { let mut action = default_action(); action.cmd = vec!["command".into()]; - action + Arc::new(action) } - pub fn ok_action_with_after(d: String, name: &str) -> Action { + pub fn ok_action_with_after(d: String, name: &str) -> Arc { let mut action = default_action(); action.cmd = vec!["command".into()]; action.after = Some(d); - action.setup("", "", name).unwrap(); action + .setup("", "", name, Arc::new(BTreeSet::default())) + .unwrap(); + Arc::new(action) } #[test] fn missing_config() { let mut action; let name = "name".to_string(); + let patterns = Arc::new(BTreeSet::default()); // No command action = default_action(); - assert!(action.setup(&name, &name, &name).is_err()); + assert!(action.setup(&name, &name, &name, patterns.clone()).is_err()); // No command action = default_action(); action.cmd = vec!["".into()]; - assert!(action.setup(&name, &name, &name).is_err()); + assert!(action.setup(&name, &name, &name, patterns.clone()).is_err()); // No command action = default_action(); action.cmd = vec!["".into(), "arg1".into()]; - assert!(action.setup(&name, &name, &name).is_err()); + assert!(action.setup(&name, &name, &name, patterns.clone()).is_err()); // command ok - action = ok_action(); - assert!(action.setup(&name, &name, &name).is_ok()); + action = ok_action().as_ref().clone(); + assert!(action.setup(&name, &name, &name, patterns.clone()).is_ok()); // command ok - action = ok_action(); + action = ok_action().as_ref().clone(); action.cmd.push("arg1".into()); - assert!(action.setup(&name, &name, &name).is_ok()); + assert!(action.setup(&name, &name, &name, patterns.clone()).is_ok()); } } diff --git a/rust/src/config.rs b/rust/src/config.rs index 1f608f5..0d485d0 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -4,19 +4,16 @@ use std::{ io, path::Path, process::{Command, Stdio}, + sync::Arc, }; use log::{error, info}; use serde::Deserialize; use thiserror::Error; -use crate::{ - filter::{Filter, FilterName}, - pattern::Pattern, - stream::Stream, -}; +use crate::{filter::Filter, pattern::Pattern, stream::Stream}; -pub type Patterns = BTreeMap; +pub type Patterns = BTreeMap>; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -43,14 +40,10 @@ impl Config { self.concurrency } - pub fn get_filter(&self, stream: &str, filter: &str) -> Option<&Filter> { - self.streams.get(stream)?.filters().get(filter) - } - - pub fn filter_names(&self) -> Vec { + pub fn filters(&self) -> Vec> { self.streams .values() - .flat_map(|stream| stream.filters().values().map(|filter| filter.name())) + .flat_map(|stream| stream.filters().values().cloned()) .collect() } @@ -59,9 +52,13 @@ impl Config { self.concurrency = num_cpus::get(); } - for (key, value) in &mut self.patterns { + let mut new_patterns = BTreeMap::new(); + for (key, value) in &self.patterns { + let mut value = value.as_ref().clone(); value.setup(key)?; + new_patterns.insert(key.clone(), Arc::new(value)); } + self.patterns = new_patterns; if self.streams.is_empty() { return Err("no streams configured".into()); diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index 5588cc5..6d3d2f6 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -1,5 +1,5 @@ use std::{ - path::PathBuf, + path::Path, process::exit, sync::{ atomic::{AtomicBool, Ordering}, @@ -9,7 +9,7 @@ use std::{ thread, }; -use log::{debug, error, info, Level}; +use log::{error, info, Level}; use crate::{ config::Config, @@ -19,14 +19,13 @@ use crate::{ utils::SimpleLogger, }; -pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { +#[allow(unused_variables)] +pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { if let Err(err) = SimpleLogger::init(loglevel) { eprintln!("ERROR could not initialize logging: {err}"); exit(1); } - debug!("daemon {config_path:?} {loglevel:?} {socket:?}"); - let config = match Config::from_file(config_path) { Ok(config) => Arc::new(config), Err(err) => { @@ -35,8 +34,6 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { } }; - debug!("config: {:?}", config); - if !config.start() { error!("a start command failed, exiting."); exit(1); @@ -50,18 +47,9 @@ pub fn daemon(config_path: &PathBuf, loglevel: Level, socket: &PathBuf) { let (log_tx, log_rx) = channel(); let matches_manager_thread_handle = { - let config_matches = config.clone(); let match_tx_matches = match_tx.clone(); let exec_tx_matches = exec_tx.clone(); - thread::spawn(move || { - matches_manager( - config_matches, - match_rx, - match_tx_matches, - exec_tx_matches, - log_tx, - ) - }) + thread::spawn(move || matches_manager(match_rx, match_tx_matches, exec_tx_matches, log_tx)) }; let execs_manager_thread_handle = { diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs index 6af7b40..9846c49 100644 --- a/rust/src/database/mod.rs +++ b/rust/src/database/mod.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashMap}, fs::{self, File}, process::exit, sync::{ @@ -11,14 +11,14 @@ use std::{ }; use chrono::{DateTime, Local, TimeDelta}; -use log::{debug, error, info, warn}; +use log::{error, info, warn}; use postcard::{from_io, to_io, Error}; use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::{ config::Config, - filter::FilterName, + filter::Filter, matches::MatchManagerInput, messages::{LogEntry, Match, Time}, }; @@ -107,8 +107,8 @@ impl WriteDB { } }; - let database_read_header: BTreeMap = - config.filter_names().into_iter().enumerate().collect(); + let database_read_header: DatabaseReadHeader = + config.filters().into_iter().enumerate().collect(); if let Err(err) = to_io(&database_read_header, &mut file) { error!("Failed to write to DB: {}", err); @@ -117,7 +117,7 @@ impl WriteDB { let database_write_header = database_read_header .into_iter() - .map(|(i, name)| (name, i as u16)) + .map(|(i, name)| (name, i)) .collect(); WriteDB { @@ -143,27 +143,28 @@ enum DBError { Error(String), } -type DatabaseWriteHeader = BTreeMap; -type DatabaseReadHeader = BTreeMap; +type DatabaseWriteHeader = BTreeMap, usize>; +type DatabaseReadHeader = BTreeMap>; #[derive(Serialize, Deserialize)] struct ComputedLogEntry { pub m: Match, - pub f: u16, + pub f: usize, pub t: i64, pub exec: bool, } impl ComputedLogEntry { fn from(value: LogEntry, header: &DatabaseWriteHeader) -> Result { - match header.get(&value.f) { + let filter = value.f; + match header.get(&filter) { Some(f) => Ok(ComputedLogEntry { m: value.m, f: *f, t: value.t.timestamp(), exec: value.exec, }), - None => Err(DBError::InvalidFilterError(value.f.to_string())), + None => Err(DBError::InvalidFilterError(filter.to_string())), } } fn to(self, header: &DatabaseReadHeader) -> Result { @@ -263,7 +264,6 @@ fn rotate_db( let mut log_write_db = WriteDB::create(LOG_DB_NEW_NAME, config); _rotate_db( - config, matches_tx, &mut log_read_db, &mut flush_read_db, @@ -290,42 +290,29 @@ fn rotate_db( } fn _rotate_db( - config: &Arc, matches_tx: Option>, log_read_db: &mut ReadDB, flush_read_db: &mut ReadDB, log_write_db: &mut WriteDB, ) { - let mut discarded_entries: BTreeMap = BTreeMap::new(); - let mut discarded_count: usize = 0; + let mut discarded_errors: HashMap = HashMap::new(); let mut millisecond_disambiguation_counter: u32 = 0; // Read flushes - let mut flushes: BTreeMap> = BTreeMap::new(); + let mut flushes: BTreeMap, BTreeMap> = BTreeMap::new(); for flush_entry in flush_read_db { match flush_entry { Ok(entry) => { - if config - .get_filter(&entry.f.stream, &entry.f.filter) - .is_some() - { - let matches_map = flushes.entry(entry.f).or_default(); - matches_map.insert(entry.m, entry.t); - } else { - *discarded_entries.entry(entry.f).or_insert(0) += 1; - } + let matches_map = flushes.entry(entry.f).or_default(); + matches_map.insert(entry.m, entry.t); } Err(err) => { - debug!("while reading flush db: {}", err); - discarded_count += 1; + *discarded_errors.entry(err.to_string()).or_insert(0) += 1; } } } - let global_flush_map = flushes.get(&FilterName { - stream: "".into(), - filter: "".into(), - }); + let last_global_flush = flushes.get(&Filter::default()); let now = Local::now(); @@ -333,23 +320,13 @@ fn _rotate_db( for log_entry in log_read_db { match log_entry { Ok(mut entry) => { - // retrieve related stream & filter - let filter = match config.get_filter(&entry.f.stream, &entry.f.filter) { - Some(filter) => filter, - None => { - *discarded_entries.entry(entry.f).or_insert(0) += 1; - continue; - } - }; - // Check if number of patterns is in sync - if entry.m.len() != filter.patterns().len() { - *discarded_entries.entry(entry.f).or_insert(0) += 1; + if entry.m.len() != entry.f.patterns().len() { continue; } // Check if hasn't been flushed - if let Some(map) = global_flush_map { + if let Some(map) = last_global_flush { if let Some(time) = map.get(&entry.m) { if time > &entry.t { continue; @@ -365,8 +342,8 @@ fn _rotate_db( } // Store match & store in db - if (!entry.exec && entry.t + filter.retry_duration().unwrap_or_default() > now) - || (entry.exec && entry.t + filter.longuest_action_duration() > now) + if (!entry.exec && entry.t + entry.f.retry_duration().unwrap_or_default() > now) + || (entry.exec && entry.t + entry.f.longuest_action_duration() > now) { // We loose subsecond precision when storing times, so we add those fake // milliseconds to make sure each time is unique @@ -385,8 +362,7 @@ fn _rotate_db( } } Err(err) => { - debug!("while reading log db: {}", err); - discarded_count += 1; + *discarded_errors.entry(err.to_string()).or_insert(0) += 1; } } } @@ -396,14 +372,13 @@ fn _rotate_db( } // Warn about errors - discarded_entries.iter() + discarded_errors + .iter() .filter(|(_, &count)| count > 0) - .for_each(|(name, count)| warn!("{} entries discarded from the databases: stream/filter not found in configuration, or the patterns are not the same: {}", count, name)); - - if discarded_count > 0 { - error!( - "{} entries discarded from the databases (garbage while decoding)", - discarded_count - ); - } + .for_each(|(name, count)| { + warn!( + "This problem was found {} times while rotating the database: {}", + count, name + ) + }); } diff --git a/rust/src/database/tests.rs b/rust/src/database/tests.rs index 132c924..1044533 100644 --- a/rust/src/database/tests.rs +++ b/rust/src/database/tests.rs @@ -5,8 +5,8 @@ use std::sync::Arc; use chrono::Local; use crate::database::ReadDB; -use crate::{config::config_from_file, tests::Fixture}; -use crate::{filter::FilterName, messages::LogEntry}; +use crate::{config::Config, tests::Fixture}; +use crate::{filter::Filter, messages::LogEntry}; use super::WriteDB; @@ -40,17 +40,11 @@ fn write_and_read_db() { ", ); - let config = Arc::new(config_from_file(&config_file).unwrap()); + let config = Arc::new(Config::from_file(&config_file).unwrap()); - let correct_filter_name = FilterName { - stream: "stream1".into(), - filter: "filter1".into(), - }; + let correct_filter_name = Arc::new(Filter::from_name("stream1", "filter1")); - let incorrect_filter_name = FilterName { - stream: "stream0".into(), - filter: "filter1".into(), - }; + let incorrect_filter_name = Arc::new(Filter::from_name("stream0", "filter1")); let correct_log_entry = LogEntry { m: vec!["match1".into()], diff --git a/rust/src/execs.rs b/rust/src/execs.rs index 93f840b..510f881 100644 --- a/rust/src/execs.rs +++ b/rust/src/execs.rs @@ -12,7 +12,7 @@ use log::{error, info}; use timer::MessageTimer; use crate::{ - action::ActionName, + action::Action, config::Config, messages::{Match, MAT}, utils::ThreadPool, @@ -27,11 +27,11 @@ pub enum ExecsManagerInput { Stop, } -type ExecsMap = BTreeMap>>>; +type ExecsMap = BTreeMap, BTreeMap>>>; trait ExecsMapTrait { - fn add(&mut self, mft: &MAT); - fn rm(&mut self, mft: &MAT); + fn add(&mut self, mat: &MAT); + fn rm(&mut self, mat: &MAT); } impl ExecsMapTrait for ExecsMap { fn add(&mut self, mat: &MAT) { @@ -68,23 +68,22 @@ pub fn execs_manager( }; let exec_now = |mat: MAT| { - // Retrieve needed information from config - let filter = config.get_filter(&mat.a.stream, &mat.a.filter).unwrap(); - let action = filter.get_action(&mat.a.action).unwrap(); - let patterns = filter.patterns(); + let mut closure = { + let action = mat.a; - // Construct command - let mut command = action.exec(&mat.m, patterns); + // Construct command + let mut command = action.exec(&mat.m); - let mut closure = move || { - info!("{}: run [{:?}]", &mat.a, command); - if let Err(err) = command - .stdin(Stdio::null()) - .stderr(Stdio::null()) - .stdout(Stdio::piped()) - .status() - { - error!("{}: run [{:?}], code {}", &mat.a, command, err); + move || { + info!("{}: run [{:?}]", &action, command); + if let Err(err) = command + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .stdout(Stdio::piped()) + .status() + { + error!("{}: run [{:?}], code {}", &action, command, err); + } } }; @@ -121,19 +120,13 @@ pub fn execs_manager( #[allow(clippy::todo)] ExecsManagerInput::Flush(_mat) => todo!(), ExecsManagerInput::Stop => { - for (action_name, inner_map) in execs { - if config - .get_filter(&action_name.stream, &action_name.filter) - .unwrap() - .get_action(&action_name.action) - .unwrap() - .on_exit() - { + for (action, inner_map) in execs { + if action.on_exit() { for (match_, inner_set) in inner_map { for _ in inner_set { exec_now(MAT { m: match_.clone(), - a: action_name.clone(), + a: action.clone(), t: Local::now(), }); } diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 716a6f5..83dd205 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -1,7 +1,8 @@ use std::{ + cmp::Ordering, collections::{BTreeMap, BTreeSet}, fmt::Display, - sync::mpsc::Sender, + sync::{mpsc::Sender, Arc}, }; use chrono::TimeDelta; @@ -18,38 +19,45 @@ use crate::{ utils::parse_duration, }; -#[derive(Clone, Debug, Deserialize)] +// Only names are serialized +// Only computed fields are not deserialized +#[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Filter { - actions: BTreeMap, + #[serde(skip_serializing)] + actions: BTreeMap>, #[serde(skip)] longuest_action_duration: TimeDelta, + #[serde(skip_serializing)] regex: Vec, #[serde(skip)] compiled_regex: Vec, // We want patterns to be ordered // This is necessary when using matches which contain multiple patterns #[serde(skip)] - patterns: BTreeSet, + patterns: Arc>>, + #[serde(skip_serializing)] retry: Option, - #[serde(rename = "retryperiod")] + #[serde(skip_serializing, rename = "retryperiod")] retry_period: Option, #[serde(skip)] retry_duration: Option, - #[serde(skip)] + #[serde(skip_deserializing)] name: String, - #[serde(skip)] + #[serde(skip_deserializing)] stream_name: String, } impl Filter { - pub fn name(&self) -> FilterName { - FilterName { - stream: self.stream_name.clone(), - filter: self.name.clone(), + #[cfg(test)] + pub fn from_name(stream_name: &str, filter_name: &str) -> Filter { + Filter { + stream_name: stream_name.into(), + name: filter_name.into(), + ..Filter::default() } } @@ -65,25 +73,26 @@ impl Filter { self.longuest_action_duration } - pub fn patterns(&self) -> &BTreeSet { + pub fn patterns(&self) -> &BTreeSet> { &self.patterns } - pub fn get_action(&self, name: &str) -> Option<&Action> { - self.actions.get(name) - } - pub fn setup( &mut self, stream_name: &str, name: &str, - patterns: &Patterns, + config_patterns: &Patterns, ) -> Result<(), String> { - self._setup(stream_name, name, patterns) + self._setup(stream_name, name, config_patterns) .map_err(|msg| format!("filter {}: {}", name, msg)) } - fn _setup(&mut self, stream_name: &str, name: &str, patterns: &Patterns) -> Result<(), String> { + fn _setup( + &mut self, + stream_name: &str, + name: &str, + config_patterns: &Patterns, + ) -> Result<(), String> { self.stream_name = stream_name.to_string(); self.name = name.to_string(); @@ -114,10 +123,11 @@ impl Filter { return Err("no regex configured".into()); } + let mut new_patterns = BTreeSet::new(); let mut first = true; for regex in &self.regex { let mut regex_buf = regex.clone(); - for pattern in patterns.values() { + for pattern in config_patterns.values() { if let Some(index) = regex.find(pattern.name_with_braces()) { if regex.rfind(pattern.name_with_braces()).unwrap() != index { return Err(format!( @@ -126,14 +136,14 @@ impl Filter { )); } if first { - self.patterns.insert(pattern.clone()); - } else if !self.patterns.contains(pattern) { + new_patterns.insert(pattern.clone()); + } else if !new_patterns.contains(pattern) { return Err(format!( "pattern {} is not present in the first regex but is present in a following regex. all regexes should contain the same set of regexes", &pattern.name_with_braces() )); } - } else if !first && self.patterns.contains(pattern) { + } else if !first && new_patterns.contains(pattern) { return Err(format!( "pattern {} is present in the first regex but is not present in a following regex. all regexes should contain the same set of regexes", &pattern.name_with_braces() @@ -146,14 +156,19 @@ impl Filter { first = false; } self.regex.clear(); + self.patterns = Arc::new(new_patterns); if self.actions.is_empty() { return Err("no actions configured".into()); } - for (key, value) in &mut self.actions { - value.setup(stream_name, name, key)?; + let mut new_actions = BTreeMap::new(); + for (key, action) in &self.actions { + let mut new_action = action.as_ref().clone(); + new_action.setup(stream_name, name, key, self.patterns.clone())?; + new_actions.insert(key.clone(), Arc::new(new_action)); } + self.actions = new_actions; self.longuest_action_duration = self.actions.values().fold(TimeDelta::seconds(0), |acc, v| { @@ -169,18 +184,18 @@ impl Filter { if let Some(matches) = regex.captures(line) { if !self.patterns.is_empty() { let mut result = Match::new(); - for pattern in &self.patterns { + for pattern in self.patterns.as_ref() { let match_ = matches.name(pattern.name()).unwrap(); if pattern.not_an_ignore(match_.as_str()) { result.push(match_.as_str().to_string()); } } if result.len() == self.patterns.len() { - info!("{}: match {:?}", self.name(), result); + info!("{}: match {:?}", self, result); return Some(result); } } else { - info!("{}: match []", self.name()); + info!("{}: match []", self); return Some(vec![".".to_string()]); } } @@ -192,7 +207,7 @@ impl Filter { for action in self.actions.values() { tx.send(ExecsManagerInput::Exec(MAT { m: m.clone(), - a: action.name(), + a: action.clone(), t: t + action.after_duration().unwrap_or_default(), })) .unwrap(); @@ -200,20 +215,34 @@ impl Filter { } } -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub struct FilterName { - pub stream: String, - pub filter: String, -} -impl Display for FilterName { +impl Display for Filter { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}.{}", self.stream, self.filter) + write!(f, "{}.{}", self.stream_name, self.name) + } +} + +impl PartialEq for Filter { + fn eq(&self, other: &Self) -> bool { + self.stream_name == other.stream_name && self.name == other.name + } +} +impl Eq for Filter {} +impl Ord for Filter { + fn cmp(&self, other: &Self) -> Ordering { + match self.stream_name.cmp(&other.stream_name) { + Ordering::Equal => self.name.cmp(&other.name), + o => o, + } + } +} +impl PartialOrd for Filter { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) } } #[cfg(test)] pub mod tests { - use crate::action::tests::{ok_action, ok_action_with_after}; use crate::pattern::tests::{ boubou_pattern_with_ignore, default_pattern, ok_pattern_with_ignore, @@ -221,23 +250,8 @@ pub mod tests { use super::*; - pub fn default_filter() -> Filter { - Filter { - name: "".into(), - stream_name: "".into(), - actions: BTreeMap::new(), - patterns: BTreeSet::new(), - regex: Vec::new(), - compiled_regex: Vec::new(), - retry: None, - retry_period: None, - retry_duration: None, - longuest_action_duration: TimeDelta::default(), - } - } - pub fn ok_filter() -> Filter { - let mut filter = default_filter(); + let mut filter = Filter::default(); let name = "name".to_string(); filter.regex = vec!["reg".into()]; filter.actions.insert(name.clone(), ok_action()); @@ -323,7 +337,9 @@ pub mod tests { // duration 120 filter = ok_filter(); - filter.actions.insert(two_minutes_str, two_minutes_action); + filter + .actions + .insert(two_minutes_str, two_minutes_action); filter.actions.insert(minute_str, minute_action); filter.setup(&name, &name, &empty_patterns).unwrap(); assert_eq!(filter.longuest_action_duration, two_minutes); @@ -340,21 +356,21 @@ pub mod tests { let mut pattern = default_pattern(); pattern.regex = "[abc]".to_string(); assert!(pattern.setup(&name).is_ok()); - patterns.insert(name.clone(), pattern.clone()); + patterns.insert(name.clone(), pattern.clone().into()); let unused_name = "unused".to_string(); let mut unused_pattern = default_pattern(); unused_pattern.regex = "compile[error".to_string(); assert!(unused_pattern.setup(&unused_name).is_err()); - patterns.insert(unused_name.clone(), unused_pattern.clone()); + patterns.insert(unused_name.clone(), unused_pattern.clone().into()); let boubou_name = "boubou".to_string(); let mut boubou = boubou_pattern_with_ignore(); boubou.setup(&boubou_name).unwrap(); - patterns.insert(boubou_name.clone(), boubou.clone()); + patterns.insert(boubou_name.clone(), boubou.clone().into()); // correct regex replacement - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here$".to_string()); assert!(filter.setup(&name, &name, &patterns).is_ok()); @@ -369,7 +385,7 @@ pub mod tests { assert_eq!(stored_pattern.regex, pattern.regex); // same pattern two times in regex - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter .regex @@ -377,7 +393,7 @@ pub mod tests { assert!(filter.setup(&name, &name, &patterns).is_err()); // two patterns in one regex - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter .regex @@ -396,7 +412,7 @@ pub mod tests { assert_eq!(stored_pattern.regex, pattern.regex); // multiple regexes with same pattern - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here".to_string()); filter.regex.push("also add there".to_string()); @@ -418,7 +434,7 @@ pub mod tests { assert_eq!(stored_pattern.regex, pattern.regex); // multiple regexes with same patterns - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter .regex @@ -446,14 +462,14 @@ pub mod tests { assert_eq!(stored_pattern.regex, pattern.regex); // multiple regexes with different patterns 1 - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here".to_string()); filter.regex.push("also add there".to_string()); assert!(filter.setup(&name, &name, &patterns).is_err()); // multiple regexes with different patterns 2 - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter .regex @@ -462,7 +478,7 @@ pub mod tests { assert!(filter.setup(&name, &name, &patterns).is_err()); // multiple regexes with different patterns 3 - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter.regex.push("also add there".to_string()); filter @@ -481,30 +497,30 @@ pub mod tests { let mut pattern = ok_pattern_with_ignore(); pattern.setup(&name).unwrap(); - patterns.insert(name.clone(), pattern.clone()); + patterns.insert(name.clone(), pattern.clone().into()); let boubou_name = "boubou".to_string(); let mut boubou = boubou_pattern_with_ignore(); boubou.setup(&boubou_name).unwrap(); - patterns.insert(boubou_name.clone(), boubou.clone()); + patterns.insert(boubou_name.clone(), boubou.clone().into()); // one simple regex - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here$".to_string()); - filter.setup(&name, &name, &patterns).unwrap(); + assert!(filter.setup(&name, &name, &patterns).is_ok()); assert_eq!(filter.get_match("insert b here"), Some(vec!("b".into()))); assert_eq!(filter.get_match("insert a here"), None); assert_eq!(filter.get_match("youpi b youpi"), None); assert_eq!(filter.get_match("insert here"), None); // two patterns in one regex - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter .regex .push("insert here and there".to_string()); - filter.setup(&name, &name, &patterns).unwrap(); + assert!(filter.setup(&name, &name, &patterns).is_ok()); assert_eq!( filter.get_match("insert b here and bouboubou there"), Some(vec!("bouboubou".into(), "b".into())) @@ -513,18 +529,18 @@ pub mod tests { assert_eq!(filter.get_match("insert b here and boubou there"), None); // multiple regexes with same pattern - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter.regex.push("insert here".to_string()); filter.regex.push("also add there".to_string()); - filter.setup(&name, &name, &patterns).unwrap(); + assert!(filter.setup(&name, &name, &patterns).is_ok()); assert_eq!(filter.get_match("insert a here"), None); assert_eq!(filter.get_match("insert b here"), Some(vec!("b".into()))); assert_eq!(filter.get_match("also add a there"), None); assert_eq!(filter.get_match("also add b there"), Some(vec!("b".into()))); // multiple regexes with same patterns - filter = default_filter(); + filter = Filter::default(); filter.actions.insert(name.clone(), ok_action()); filter .regex @@ -532,7 +548,7 @@ pub mod tests { filter .regex .push("also add here and there".to_string()); - filter.setup(&name, &name, &patterns).unwrap(); + assert!(filter.setup(&name, &name, &patterns).is_ok()); assert_eq!( filter.get_match("insert b here and bouboubou there"), Some(vec!("bouboubou".into(), "b".into())) diff --git a/rust/src/matches.rs b/rust/src/matches.rs index a06fd5d..b276a0d 100644 --- a/rust/src/matches.rs +++ b/rust/src/matches.rs @@ -6,13 +6,13 @@ use std::{ }, }; +use log::debug; use timer::MessageTimer; use crate::{ - config::Config, database::DatabaseManagerInput, execs::ExecsManagerInput, - filter::FilterName, + filter::Filter, messages::{LogEntry, Match, Time, MFT}, }; @@ -26,11 +26,14 @@ pub enum MatchManagerInput { Stop, } -type MatchesMap = BTreeMap>>; +type MatchesMap = BTreeMap, BTreeMap>>; +// This trait is needed to permit to implement methods on an external type trait MatchesMapTrait { fn add(&mut self, mft: &MFT); fn rm(&mut self, mft: &MFT); + fn rm_times(&mut self, mft: &MFT); + fn get_times(&self, mft: &MFT) -> usize; } impl MatchesMapTrait for MatchesMap { fn add(&mut self, mft: &MFT) { @@ -52,10 +55,25 @@ impl MatchesMapTrait for MatchesMap { } } } + + fn rm_times(&mut self, mft: &MFT) { + if let Some(inner_map) = self.get_mut(&mft.f) { + inner_map.remove(&mft.m); + if inner_map.is_empty() { + self.remove(&mft.f); + } + } + } + + fn get_times(&self, mft: &MFT) -> usize { + match self.get(&mft.f).and_then(|map| map.get(&mft.m)) { + Some(x) => x.len(), + None => 0, + } + } } pub fn matches_manager( - config: Arc, match_rx: Receiver, match_tx: Sender, action_tx: Sender, @@ -69,32 +87,35 @@ pub fn matches_manager( for mft in match_rx.iter() { match mft { - MatchManagerInput::EndOfStartup => startup = false, + MatchManagerInput::EndOfStartup => { + debug!("end of startup!"); + startup = false; + } MatchManagerInput::Match(mft) => { - let filter = config.get_filter(&mft.f.stream, &mft.f.filter).unwrap(); - - let is_retry = filter.retry().is_some(); - // Store matches - if is_retry { - // Add new match - matches.add(&mft); - // Remove match when expired - let guard = timer.schedule_with_delay( - filter.retry_duration().unwrap(), - MatchManagerInput::Unmatch(mft.clone()), - ); - guard.ignore(); - } + let exec = match mft.f.retry() { + None => true, + Some(retry) => { + // Add new match + matches.add(&mft); + // Remove match when expired + let guard = timer.schedule_with_delay( + mft.f.retry_duration().unwrap(), + MatchManagerInput::Unmatch(mft.clone()), + ); + guard.ignore(); + + matches.get_times(&mft) >= retry as usize + } + }; // Executing actions - let exec = !is_retry || matches.get(&mft.f).map(|map| map.get(&mft.m)).is_some(); if exec { // Delete matches only if storing them - if is_retry { - matches.get_mut(&mft.f).unwrap().remove(&mft.m); + if mft.f.retry().is_some() { + matches.rm_times(&mft); } - filter.send_actions(&mft.m, mft.t, &action_tx); + mft.f.send_actions(&mft.m, mft.t, &action_tx); } if !startup { diff --git a/rust/src/messages.rs b/rust/src/messages.rs index ed52054..ed68d29 100644 --- a/rust/src/messages.rs +++ b/rust/src/messages.rs @@ -1,6 +1,8 @@ +use std::sync::Arc; + use chrono::{DateTime, Local, TimeDelta}; -use crate::{action::ActionName, filter::FilterName}; +use crate::{action::Action, filter::Filter}; pub type Time = DateTime; pub type Match = Vec; @@ -8,21 +10,21 @@ pub type Match = Vec; #[derive(Clone)] pub struct MFT { pub m: Match, - pub f: FilterName, + pub f: Arc, pub t: Time, } #[derive(Clone)] pub struct MAT { pub m: Match, - pub a: ActionName, + pub a: Arc, pub t: Time, } #[derive(Clone, Debug)] pub struct LogEntry { pub m: Match, - pub f: FilterName, + pub f: Arc, pub t: Time, pub exec: bool, } diff --git a/rust/src/stream.rs b/rust/src/stream.rs index ad2c776..50bbc03 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -2,11 +2,14 @@ use std::{ collections::BTreeMap, io::{BufRead, BufReader}, process::{Child, Command, Stdio}, - sync::mpsc::{Sender, SyncSender}, + sync::{ + mpsc::{Sender, SyncSender}, + Arc, + }, }; use chrono::Local; -use log::{debug, error, info}; +use log::{error, info}; use serde::Deserialize; use crate::{config::Patterns, filter::Filter, matches::MatchManagerInput, messages::MFT}; @@ -15,14 +18,14 @@ use crate::{config::Patterns, filter::Filter, matches::MatchManagerInput, messag #[serde(deny_unknown_fields)] pub struct Stream { cmd: Vec, - filters: BTreeMap, + filters: BTreeMap>, #[serde(skip)] name: String, } impl Stream { - pub fn filters(&self) -> &BTreeMap { + pub fn filters(&self) -> &BTreeMap> { &self.filters } @@ -52,9 +55,13 @@ impl Stream { return Err("no filters configured".into()); } - for (key, value) in &mut self.filters { - value.setup(name, key, patterns)?; + let mut new_filters = BTreeMap::new(); + for (key, filter) in &self.filters { + let mut new_filter = filter.as_ref().clone(); + new_filter.setup(name, key, patterns)?; + new_filters.insert(key.clone(), Arc::new(new_filter)); } + self.filters = new_filters; Ok(()) } @@ -81,33 +88,37 @@ impl Stream { }; // keep stdout before sending/moving child to the main thread - let mut stdout = BufReader::new(child.stdout.take().unwrap()); + let stdout = BufReader::new(child.stdout.take().unwrap()); // let main handle the child process let _ = child_tx.send(Some(child)); drop(child_tx); - let mut line: String = "".into(); - while let Ok(nb_chars) = stdout.read_line(&mut line) { - if nb_chars == 0 { - break; - } - line.pop(); // remove trailing newline - debug!("stream {} stdout: {}", self.name, line); + // let mut line: String = "".into(); + // while let Ok(nb_chars) = stdout.read_line(&mut line) { + for line in stdout.lines() { + let line = match line { + Ok(line) => line, + Err(_) => break, + }; + // if nb_chars == 0 { + // break; + // } + // line.pop(); // remove trailing newline for filter in self.filters.values() { if let Some(match_) = filter.get_match(&line) { match_tx .send(MatchManagerInput::Match(MFT { m: match_, - f: filter.name(), + f: filter.clone(), t: Local::now(), })) .unwrap(); } } - line.clear(); + // line.clear(); } info!("stream {} exited", self.name); } @@ -131,7 +142,7 @@ pub mod tests { stream.cmd = vec!["command".into()]; stream .filters - .insert("name".into(), crate::filter::tests::ok_filter()); + .insert("name".into(), Arc::new(crate::filter::tests::ok_filter())); stream } diff --git a/rust/test.jsonnet b/rust/test.jsonnet index 8cc1b5b..9106979 100644 --- a/rust/test.jsonnet +++ b/rust/test.jsonnet @@ -21,16 +21,15 @@ ], streams: { - tailDown1: { - cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do echo found $i; done"], + s1: { + cmd: ['sh', '-c', "seq 20 | tr ' ' '\n' | while read i; do echo found $((i % 5)); sleep 0.3; done"], filters: { - findIP: { + f1: { regex: [ - '^found _$', - '^found _$', + '^found $', ], retry: 2, - retryperiod: '30s', + retryperiod: '5s', actions: { damn: { cmd: ['notify-send', 'first stream', 'ban '], @@ -44,28 +43,5 @@ }, }, }, - tailDown2: { - cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do echo found $i; done"], - filters: { - findIP: { - regex: [ - '^found _$', - '^found _$', - ], - retry: 2, - retryperiod: '2s', - actions: { - damn: { - cmd: ['notify-send', 'second stream', 'ban '], - }, - undamn: { - cmd: ['notify-send', 'second stream', 'unban '], - after: '4s', - // onexit: false, - }, - }, - }, - }, - }, }, } From 7adf8d908d6cbc62330acf652f375e653dac81ce Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 22 Sep 2024 12:00:00 +0200 Subject: [PATCH 041/435] Fix DB implementation --- rust/src/database/mod.rs | 96 ++++++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 32 deletions(-) diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs index 9846c49..34d871f 100644 --- a/rust/src/database/mod.rs +++ b/rust/src/database/mod.rs @@ -1,7 +1,7 @@ -#![allow(dead_code)] use std::{ collections::{BTreeMap, HashMap}, fs::{self, File}, + io::{self, Write}, process::exit, sync::{ mpsc::{Receiver, Sender}, @@ -11,8 +11,8 @@ use std::{ }; use chrono::{DateTime, Local, TimeDelta}; -use log::{error, info, warn}; -use postcard::{from_io, to_io, Error}; +use log::{debug, error, info, warn}; +use postcard::{from_io, Error}; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -31,9 +31,7 @@ const FLUSH_DB_NAME: &str = "./reaction-flushes.db"; const MAX_WRITES: u32 = 500_000; -// FIXME how to use postcard::from_io with a dynamic buffer? -const HEADER_MAX_SIZE: usize = 1024 * 1024; -const ENTRY_MAX_SIZE: usize = 1024 * 1024; +const BUFFER_MAX_SIZE: usize = 10 * 1024 * 1024; struct ReadDB { f: File, @@ -58,7 +56,7 @@ impl ReadDB { }, }; - let mut buf: [u8; HEADER_MAX_SIZE] = [0; HEADER_MAX_SIZE]; + let mut buf: Vec = vec![0; BUFFER_MAX_SIZE]; let (database_header, _) = from_io((&mut file, &mut buf))?; Ok(Some(ReadDB { @@ -72,7 +70,7 @@ impl Iterator for ReadDB { type Item = Result; fn next(&mut self) -> Option { - let mut buf: [u8; ENTRY_MAX_SIZE] = [0; ENTRY_MAX_SIZE]; + let mut buf: Vec = vec![0; BUFFER_MAX_SIZE]; let result = from_io::((&mut self.f, &mut buf)); match result { // FIXME why we got a default item instead of an error or something? @@ -95,11 +93,12 @@ impl Iterator for ReadDB { struct WriteDB { f: File, h: DatabaseWriteHeader, + buf: Vec, } impl WriteDB { fn create(path: &str, config: &Arc) -> Self { - let mut file = match File::create(path) { + let file = match File::create(path) { Ok(file) => file, Err(err) => { error!("Failed to create DB: {}", err); @@ -107,10 +106,16 @@ impl WriteDB { } }; + let mut ret = WriteDB { + f: file, + h: BTreeMap::new(), + buf: vec![0; BUFFER_MAX_SIZE], + }; + let database_read_header: DatabaseReadHeader = config.filters().into_iter().enumerate().collect(); - if let Err(err) = to_io(&database_read_header, &mut file) { + if let Err(err) = ret._write(&database_read_header) { error!("Failed to write to DB: {}", err); exit(1); } @@ -120,15 +125,23 @@ impl WriteDB { .map(|(i, name)| (name, i)) .collect(); - WriteDB { - f: file, - h: database_write_header, - } + ret.h = database_write_header; + ret } fn write(&mut self, entry: LogEntry) -> Result<(), DBError> { let computed = ComputedLogEntry::from(entry, &self.h)?; - to_io(&computed, &mut self.f)?; + self._write(computed) + } + + fn _write(&mut self, data: T) -> Result<(), DBError> { + let encoded = postcard::to_slice(&data, &mut self.buf)?; + debug!("writing this: {:?}, {:?}", &data, &encoded); + self.f.write_all(encoded)?; + // clear + for i in 0..self.buf.len() { + self.buf[i] = 0; + } Ok(()) } } @@ -139,6 +152,8 @@ enum DBError { InvalidFilterError(String), #[error("decode error: {0}")] PostcardError(#[from] postcard::Error), + #[error("io error: {0}")] + IOError(#[from] io::Error), #[error("{0}")] Error(String), } @@ -146,7 +161,7 @@ enum DBError { type DatabaseWriteHeader = BTreeMap, usize>; type DatabaseReadHeader = BTreeMap>; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] struct ComputedLogEntry { pub m: Match, pub f: usize, @@ -185,9 +200,20 @@ impl ComputedLogEntry { #[derive(Clone)] pub enum DatabaseManagerInput { Log(LogEntry), + #[allow(dead_code)] Flush(LogEntry), } +// Just discovering macros, let me be useless +macro_rules! write_or_die { + ($db:expr, $entry:expr) => { + if let Err(err) = $db.write($entry) { + error!("Could not write to DB: {}", err); + exit(1); + } + }; +} + /// First rotates the database, then spawns the database thread pub fn database_manager( config: Arc, @@ -204,11 +230,11 @@ pub fn database_manager( thread::spawn(move || { let mut cpt = 0; - while let Ok(order) = log_rx.recv() { + for order in log_rx.iter() { match order { - DatabaseManagerInput::Flush(entry) => flush_db.write(entry).unwrap(), + DatabaseManagerInput::Flush(entry) => write_or_die!(flush_db, entry), DatabaseManagerInput::Log(entry) => { - log_db.write(entry).unwrap(); + write_or_die!(log_db, entry); cpt += 1; if cpt == MAX_WRITES { cpt = 0; @@ -236,6 +262,20 @@ fn rotate_db( matches_tx: Option>, ) -> Result<(WriteDB, WriteDB), DBError> { info!("Rotating database..."); + let res = _rotate_db(config, &matches_tx); + + debug!("sending EOS: {}", matches_tx.is_some()); + if let Some(tx) = matches_tx { + tx.send(MatchManagerInput::EndOfStartup).unwrap(); + } + + info!("Rotated database"); + res +} +fn _rotate_db( + config: &Arc, + matches_tx: &Option>, +) -> Result<(WriteDB, WriteDB), DBError> { let mut log_read_db = match ReadDB::open(LOG_DB_NAME)? { Some(db) => db, None => { @@ -263,7 +303,7 @@ fn rotate_db( let mut log_write_db = WriteDB::create(LOG_DB_NEW_NAME, config); - _rotate_db( + __rotate_db( matches_tx, &mut log_read_db, &mut flush_read_db, @@ -285,12 +325,11 @@ fn rotate_db( } let flush_write_db = WriteDB::create(FLUSH_DB_NAME, config); - info!("Rotated database"); Ok((log_write_db, flush_write_db)) } -fn _rotate_db( - matches_tx: Option>, +fn __rotate_db( + matches_tx: &Option>, log_read_db: &mut ReadDB, flush_read_db: &mut ReadDB, log_write_db: &mut WriteDB, @@ -350,15 +389,12 @@ fn _rotate_db( entry.t += TimeDelta::new(0, millisecond_disambiguation_counter).unwrap(); millisecond_disambiguation_counter += 1; - if let Some(tx) = &matches_tx { + if let Some(tx) = matches_tx { tx.send(MatchManagerInput::Match(entry.clone().into())) .unwrap(); } - if let Err(err) = log_write_db.write(entry) { - error!("Could not write to DB: {}", err); - exit(1); - } + write_or_die!(log_write_db, entry); } } Err(err) => { @@ -367,10 +403,6 @@ fn _rotate_db( } } - if let Some(tx) = matches_tx { - tx.send(MatchManagerInput::EndOfStartup).unwrap(); - } - // Warn about errors discarded_errors .iter() From 0bac011ab1af5d50c3c2783ab51902620ae20af6 Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 22 Sep 2024 12:00:00 +0200 Subject: [PATCH 042/435] Split database lowlevel code; sync_channel(1) for better perf; flush db --- rust/src/daemon.rs | 4 +- rust/src/database/lowlevel.rs | 199 ++++++++++++++++++++++++++++++++++ rust/src/database/mod.rs | 183 ++++--------------------------- rust/src/filter.rs | 4 +- rust/src/matches.rs | 4 +- rust/src/utils/cli.rs | 1 + 6 files changed, 225 insertions(+), 170 deletions(-) create mode 100644 rust/src/database/lowlevel.rs diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index 6d3d2f6..399bd22 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -44,7 +44,7 @@ pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { let (match_tx, match_rx) = channel(); let (exec_tx, exec_rx) = channel(); - let (log_tx, log_rx) = channel(); + let (log_tx, log_rx) = sync_channel(1); let matches_manager_thread_handle = { let match_tx_matches = match_tx.clone(); @@ -110,8 +110,6 @@ pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { let stop_ok = config.stop(); - // TODO flush DB - // TODO remove socket exit(match !signal_received.load(Ordering::SeqCst) && stop_ok { diff --git a/rust/src/database/lowlevel.rs b/rust/src/database/lowlevel.rs new file mode 100644 index 0000000..3fe7c8b --- /dev/null +++ b/rust/src/database/lowlevel.rs @@ -0,0 +1,199 @@ +use std::{ + collections::BTreeMap, + fmt::Debug, + fs::File, + io::{self, BufWriter, Write}, + process::exit, + sync::Arc, +}; + +use chrono::{DateTime, Local}; +use log::{debug, error, warn}; +use serde::{Deserialize, Serialize}; + +use crate::{ + config::Config, + filter::Filter, + messages::{LogEntry, Match}, +}; + +use super::DBError; + +type DatabaseWriteHeader = BTreeMap, usize>; +type DatabaseReadHeader = BTreeMap>; + +const BUFFER_MAX_SIZE: usize = 10 * 1024 * 1024; +const DB_SIGNATURE: &str = "reaction-db-v01"; + +pub struct ReadDB { + f: File, + h: DatabaseReadHeader, + buf: Vec, +} + +impl ReadDB { + pub fn open(path: &str) -> Result, DBError> { + let file = match File::open(path) { + Ok(file) => file, + Err(err) => match err.kind() { + std::io::ErrorKind::NotFound => { + warn!( + "No DB found at {}. It's ok if this is the first time reaction is running.", + path + ); + return Ok(None); + } + _ => { + return Err(DBError::Error(format!("Could not open database: {}", err))); + } + }, + }; + + let mut ret = ReadDB { + f: file, + h: BTreeMap::new(), + buf: vec![0; BUFFER_MAX_SIZE], + }; + + match ret.read::<&str>() { + Ok(DB_SIGNATURE) => Ok(()), + Ok(_) => Err(DBError::Error("database is not a reaction database".into())), + Err(err) => Err(DBError::Error(format!("reading database signature: {err}"))), + }?; + + ret.h = ret + .read() + .map_err(|err| DBError::Error(format!("while reading database header: {err}")))?; + + Ok(Some(ret)) + } + + fn read<'a, T: Deserialize<'a> + Debug>(&'a mut self) -> Result { + let (decoded, _) = postcard::from_io::((&mut self.f, &mut self.buf))?; + debug!("reading this: {:?}", &decoded); + Ok(decoded) + } +} + +impl Iterator for ReadDB { + type Item = Result; + + fn next(&mut self) -> Option { + match self.read::() { + // FIXME why we got a default item instead of an error or something? + // How do we really know we reached the end? + Ok(item) => { + if item.t == 0 { + None + } else { + Some(item.to(&self.h)) + } + } + Err(err) => match err { + postcard::Error::DeserializeUnexpectedEnd => None, + _ => Some(Err(err.into())), + }, + } + } +} + +pub struct WriteDB { + f: BufWriter, + h: DatabaseWriteHeader, + buf: Vec, +} + +impl WriteDB { + pub fn create(path: &str, config: &Arc) -> Self { + let file = match File::create(path) { + Ok(file) => file, + Err(err) => { + error!("Failed to create DB: {}", err); + exit(1); + } + }; + + let mut ret = WriteDB { + f: BufWriter::new(file), + h: BTreeMap::new(), + buf: vec![0; BUFFER_MAX_SIZE], + }; + + if let Err(err) = ret._write(DB_SIGNATURE) { + error!("Failed to write to DB: {}", err); + exit(1); + } + + let database_read_header: DatabaseReadHeader = + config.filters().into_iter().enumerate().collect(); + + if let Err(err) = ret._write(&database_read_header) { + error!("Failed to write to DB: {}", err); + exit(1); + } + + let database_write_header = database_read_header + .into_iter() + .map(|(i, name)| (name, i)) + .collect(); + + ret.h = database_write_header; + ret + } + + pub fn write(&mut self, entry: LogEntry) -> Result<(), DBError> { + let computed = ComputedLogEntry::from(entry, &self.h)?; + self._write(computed) + } + + fn _write(&mut self, data: T) -> Result<(), DBError> { + let encoded = postcard::to_slice(&data, &mut self.buf)?; + debug!("writing this: {:?}, {:?}", &data, &encoded); + self.f.write_all(encoded)?; + // clear + // for i in 0..self.buf.len() { + // self.buf[i] = 0; + // } + Ok(()) + } + + pub fn flush(&mut self) -> io::Result<()> { + self.f.flush() + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct ComputedLogEntry { + pub m: Match, + pub f: usize, + pub t: i64, + pub exec: bool, +} + +impl ComputedLogEntry { + fn from(value: LogEntry, header: &DatabaseWriteHeader) -> Result { + let filter = value.f; + match header.get(&filter) { + Some(f) => Ok(ComputedLogEntry { + m: value.m, + f: *f, + t: value.t.timestamp(), + exec: value.exec, + }), + None => Err(DBError::InvalidFilterError(filter.to_string())), + } + } + fn to(self, header: &DatabaseReadHeader) -> Result { + match header.get(&self.f) { + Some(f) => Ok(LogEntry { + m: self.m, + f: f.clone(), + t: DateTime::from_timestamp(self.t, 0) + .unwrap() + .with_timezone(&Local), + exec: self.exec, + }), + None => Err(DBError::InvalidFilterError(self.f.to_string())), + } + } +} diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs index 34d871f..9db025b 100644 --- a/rust/src/database/mod.rs +++ b/rust/src/database/mod.rs @@ -1,7 +1,7 @@ use std::{ collections::{BTreeMap, HashMap}, - fs::{self, File}, - io::{self, Write}, + fmt::Debug, + fs, io, process::exit, sync::{ mpsc::{Receiver, Sender}, @@ -10,10 +10,9 @@ use std::{ thread, }; -use chrono::{DateTime, Local, TimeDelta}; +use chrono::{Local, TimeDelta}; use log::{debug, error, info, warn}; -use postcard::{from_io, Error}; -use serde::{Deserialize, Serialize}; +use postcard; use thiserror::Error; use crate::{ @@ -23,131 +22,18 @@ use crate::{ messages::{LogEntry, Match, Time}, }; +mod lowlevel; mod tests; +use lowlevel::{ReadDB, WriteDB}; + const LOG_DB_NAME: &str = "./reaction-matches.db"; const LOG_DB_NEW_NAME: &str = "./reaction-matches.new.db"; const FLUSH_DB_NAME: &str = "./reaction-flushes.db"; const MAX_WRITES: u32 = 500_000; - -const BUFFER_MAX_SIZE: usize = 10 * 1024 * 1024; - -struct ReadDB { - f: File, - h: DatabaseReadHeader, -} - -impl ReadDB { - fn open(path: &str) -> Result, DBError> { - let mut file = match File::open(path) { - Ok(file) => file, - Err(err) => match err.kind() { - std::io::ErrorKind::NotFound => { - warn!( - "No DB found at {}. It's ok if this is the first time reaction is running.", - path - ); - return Ok(None); - } - _ => { - return Err(DBError::Error(format!("Could not open database: {}", err))); - } - }, - }; - - let mut buf: Vec = vec![0; BUFFER_MAX_SIZE]; - let (database_header, _) = from_io((&mut file, &mut buf))?; - - Ok(Some(ReadDB { - f: file, - h: database_header, - })) - } -} - -impl Iterator for ReadDB { - type Item = Result; - - fn next(&mut self) -> Option { - let mut buf: Vec = vec![0; BUFFER_MAX_SIZE]; - let result = from_io::((&mut self.f, &mut buf)); - match result { - // FIXME why we got a default item instead of an error or something? - // How do we really know we reached the end? - Ok((item, _)) => { - if item.t == 0 { - None - } else { - Some(item.to(&self.h)) - } - } - Err(err) => match err { - Error::DeserializeUnexpectedEnd => None, - _ => Some(Err(err.into())), - }, - } - } -} - -struct WriteDB { - f: File, - h: DatabaseWriteHeader, - buf: Vec, -} - -impl WriteDB { - fn create(path: &str, config: &Arc) -> Self { - let file = match File::create(path) { - Ok(file) => file, - Err(err) => { - error!("Failed to create DB: {}", err); - exit(1); - } - }; - - let mut ret = WriteDB { - f: file, - h: BTreeMap::new(), - buf: vec![0; BUFFER_MAX_SIZE], - }; - - let database_read_header: DatabaseReadHeader = - config.filters().into_iter().enumerate().collect(); - - if let Err(err) = ret._write(&database_read_header) { - error!("Failed to write to DB: {}", err); - exit(1); - } - - let database_write_header = database_read_header - .into_iter() - .map(|(i, name)| (name, i)) - .collect(); - - ret.h = database_write_header; - ret - } - - fn write(&mut self, entry: LogEntry) -> Result<(), DBError> { - let computed = ComputedLogEntry::from(entry, &self.h)?; - self._write(computed) - } - - fn _write(&mut self, data: T) -> Result<(), DBError> { - let encoded = postcard::to_slice(&data, &mut self.buf)?; - debug!("writing this: {:?}, {:?}", &data, &encoded); - self.f.write_all(encoded)?; - // clear - for i in 0..self.buf.len() { - self.buf[i] = 0; - } - Ok(()) - } -} - #[derive(Error, Debug)] -enum DBError { +pub enum DBError { #[error("invalid filter: {0}")] InvalidFilterError(String), #[error("decode error: {0}")] @@ -157,46 +43,6 @@ enum DBError { #[error("{0}")] Error(String), } - -type DatabaseWriteHeader = BTreeMap, usize>; -type DatabaseReadHeader = BTreeMap>; - -#[derive(Debug, Serialize, Deserialize)] -struct ComputedLogEntry { - pub m: Match, - pub f: usize, - pub t: i64, - pub exec: bool, -} - -impl ComputedLogEntry { - fn from(value: LogEntry, header: &DatabaseWriteHeader) -> Result { - let filter = value.f; - match header.get(&filter) { - Some(f) => Ok(ComputedLogEntry { - m: value.m, - f: *f, - t: value.t.timestamp(), - exec: value.exec, - }), - None => Err(DBError::InvalidFilterError(filter.to_string())), - } - } - fn to(self, header: &DatabaseReadHeader) -> Result { - match header.get(&self.f) { - Some(f) => Ok(LogEntry { - m: self.m, - f: f.clone(), - t: DateTime::from_timestamp(self.t, 0) - .unwrap() - .with_timezone(&Local), - exec: self.exec, - }), - None => Err(DBError::InvalidFilterError(self.f.to_string())), - } - } -} - #[derive(Clone)] pub enum DatabaseManagerInput { Log(LogEntry), @@ -213,6 +59,14 @@ macro_rules! write_or_die { } }; } +macro_rules! flush_or_die { + ($db:expr) => { + if let Err(err) = $db.flush() { + error!("Could not write to DB: {}", err); + exit(1); + } + }; +} /// First rotates the database, then spawns the database thread pub fn database_manager( @@ -238,6 +92,8 @@ pub fn database_manager( cpt += 1; if cpt == MAX_WRITES { cpt = 0; + flush_or_die!(log_db); + flush_or_die!(flush_db); drop(log_db); drop(flush_db); (log_db, flush_db) = match rotate_db(&config, None) { @@ -254,6 +110,8 @@ pub fn database_manager( } }; } + flush_or_die!(log_db); + flush_or_die!(flush_db); }) } @@ -272,6 +130,7 @@ fn rotate_db( info!("Rotated database"); res } + fn _rotate_db( config: &Arc, matches_tx: &Option>, diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 83dd205..cb55d7e 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -337,9 +337,7 @@ pub mod tests { // duration 120 filter = ok_filter(); - filter - .actions - .insert(two_minutes_str, two_minutes_action); + filter.actions.insert(two_minutes_str, two_minutes_action); filter.actions.insert(minute_str, minute_action); filter.setup(&name, &name, &empty_patterns).unwrap(); assert_eq!(filter.longuest_action_duration, two_minutes); diff --git a/rust/src/matches.rs b/rust/src/matches.rs index b276a0d..5d69de2 100644 --- a/rust/src/matches.rs +++ b/rust/src/matches.rs @@ -1,7 +1,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, sync::{ - mpsc::{Receiver, Sender}, + mpsc::{Receiver, Sender, SyncSender}, Arc, }, }; @@ -77,7 +77,7 @@ pub fn matches_manager( match_rx: Receiver, match_tx: Sender, action_tx: Sender, - log_tx: Sender, + log_tx: SyncSender, ) { let mut matches: MatchesMap = BTreeMap::new(); diff --git a/rust/src/utils/cli.rs b/rust/src/utils/cli.rs index 0468e84..422c90b 100644 --- a/rust/src/utils/cli.rs +++ b/rust/src/utils/cli.rs @@ -131,6 +131,7 @@ impl fmt::Display for Format { // Structs +#[allow(dead_code)] #[derive(Clone, Debug)] pub struct NamedRegex { pub regex: Regex, From 6b52b030259a08808b94952c9323ef223aa01f7d Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 24 Sep 2024 12:00:00 +0200 Subject: [PATCH 043/435] fix database deserializing and use BufWriter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deserializing was failing because Filter::Deserialize and Filter::Serialize had different attributes. Now directly de·serializing (String, String) from/to database Using BufWriter greatly increases performance. Adding flushes where necessary. --- rust/heavy-load.yml | 51 +++----------------------- rust/src/config.rs | 6 ++++ rust/src/database/lowlevel.rs | 67 ++++++++++++++++++++--------------- rust/src/database/mod.rs | 10 +++--- rust/src/database/tests.rs | 2 +- rust/src/execs.rs | 1 + rust/src/filter.rs | 19 +++++----- rust/src/matches.rs | 1 + rust/src/stream.rs | 14 +++----- 9 files changed, 74 insertions(+), 97 deletions(-) diff --git a/rust/heavy-load.yml b/rust/heavy-load.yml index 98a094a..331dacd 100644 --- a/rust/heavy-load.yml +++ b/rust/heavy-load.yml @@ -1,4 +1,6 @@ --- +concurrency: 32 + patterns: num: regex: '[0-9]{3}' @@ -9,12 +11,12 @@ patterns: streams: tailDown1: - cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] + cmd: [ 'sh', '-c', 'sleep 2; seq 10001 | while read i; do echo found $i; done' ] filters: find: regex: - '^found ' - retry: 480 + retry: 9 retryperiod: 1m actions: damn: @@ -68,48 +70,3 @@ streams: cmd: [ 'sleep', '0.0' ] after: 1m onexit: false - # tailDown2: - # cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo prout $i; done' ] - # filters: - # findIP: - # regex: - # - '^prout ' - # retry: 500 - # retryperiod: 1m - # actions: - # damn: - # cmd: [ 'sleep', '0.0' ] - # undamn: - # cmd: [ 'sleep', '0.0' ] - # after: 1m - # onexit: false - # tailDown3: - # cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo nanana $i; done' ] - # filters: - # findIP: - # regex: - # - '^nanana ' - # retry: 500 - # retryperiod: 2m - # actions: - # damn: - # cmd: [ 'sleep', '0.0' ] - # undamn: - # cmd: [ 'sleep', '0.0' ] - # after: 1m - # onexit: false - # tailDown4: - # cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo nanana $i; done' ] - # filters: - # findIP: - # regex: - # - '^nomatch $' - # retry: 500 - # retryperiod: 2m - # actions: - # damn: - # cmd: [ 'sleep', '0.0' ] - # undamn: - # cmd: [ 'sleep', '0.0' ] - # after: 1m - # onexit: false diff --git a/rust/src/config.rs b/rust/src/config.rs index 0d485d0..4347cd0 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -47,6 +47,12 @@ impl Config { .collect() } + pub fn get_filter(&self, name: &(String, String)) -> Option<&Arc> { + self.streams + .get(&name.0) + .and_then(|stream| stream.get_filter(&name.1)) + } + pub fn setup(&mut self) -> Result<(), String> { if self.concurrency == 0 { self.concurrency = num_cpus::get(); diff --git a/rust/src/database/lowlevel.rs b/rust/src/database/lowlevel.rs index 3fe7c8b..174de68 100644 --- a/rust/src/database/lowlevel.rs +++ b/rust/src/database/lowlevel.rs @@ -2,7 +2,7 @@ use std::{ collections::BTreeMap, fmt::Debug, fs::File, - io::{self, BufWriter, Write}, + io::{self, BufReader, BufWriter, Write}, process::exit, sync::Arc, }; @@ -19,20 +19,25 @@ use crate::{ use super::DBError; -type DatabaseWriteHeader = BTreeMap, usize>; -type DatabaseReadHeader = BTreeMap>; +// OPTIM Add a timestamp prefix to the header, to permit having +// shorter timestamps? +// It may permit to win 1-4 bytes per entry, don't know if it's worth it +// FIXME put signature in the header? +type DatabaseHeader = BTreeMap; +type ReadHeader = BTreeMap>; +type WriteHeader = BTreeMap, usize>; const BUFFER_MAX_SIZE: usize = 10 * 1024 * 1024; const DB_SIGNATURE: &str = "reaction-db-v01"; pub struct ReadDB { - f: File, - h: DatabaseReadHeader, + f: BufReader, + h: ReadHeader, buf: Vec, } impl ReadDB { - pub fn open(path: &str) -> Result, DBError> { + pub fn open(path: &str, config: &Arc) -> Result, DBError> { let file = match File::open(path) { Ok(file) => file, Err(err) => match err.kind() { @@ -50,7 +55,7 @@ impl ReadDB { }; let mut ret = ReadDB { - f: file, + f: BufReader::new(file), h: BTreeMap::new(), buf: vec![0; BUFFER_MAX_SIZE], }; @@ -61,10 +66,15 @@ impl ReadDB { Err(err) => Err(DBError::Error(format!("reading database signature: {err}"))), }?; - ret.h = ret - .read() + let db_header = ret + .read::() .map_err(|err| DBError::Error(format!("while reading database header: {err}")))?; + ret.h = db_header + .iter() + .filter_map(|(key, name)| config.get_filter(name).map(|filter| (*key, filter.clone()))) + .collect(); + Ok(Some(ret)) } @@ -82,13 +92,9 @@ impl Iterator for ReadDB { match self.read::() { // FIXME why we got a default item instead of an error or something? // How do we really know we reached the end? - Ok(item) => { - if item.t == 0 { - None - } else { - Some(item.to(&self.h)) - } - } + // For now, checking if time is 0 + Ok(ComputedLogEntry { t: 0, .. }) => None, + Ok(item) => Some(item.to(&self.h)), Err(err) => match err { postcard::Error::DeserializeUnexpectedEnd => None, _ => Some(Err(err.into())), @@ -99,7 +105,7 @@ impl Iterator for ReadDB { pub struct WriteDB { f: BufWriter, - h: DatabaseWriteHeader, + h: WriteHeader, buf: Vec, } @@ -124,20 +130,25 @@ impl WriteDB { exit(1); } - let database_read_header: DatabaseReadHeader = - config.filters().into_iter().enumerate().collect(); + let database_header: DatabaseHeader = config + .filters() + .into_iter() + .map(|f| f.full_name()) + .enumerate() + .collect(); - if let Err(err) = ret._write(&database_read_header) { + if let Err(err) = ret._write(&database_header) { error!("Failed to write to DB: {}", err); exit(1); } - let database_write_header = database_read_header + ret.h = config + .filters() .into_iter() - .map(|(i, name)| (name, i)) + .enumerate() + .map(|(key, filter)| (filter, key)) .collect(); - ret.h = database_write_header; ret } @@ -171,19 +182,19 @@ struct ComputedLogEntry { } impl ComputedLogEntry { - fn from(value: LogEntry, header: &DatabaseWriteHeader) -> Result { - let filter = value.f; - match header.get(&filter) { + #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used + fn from(value: LogEntry, header: &WriteHeader) -> Result { + match header.get(&value.f) { Some(f) => Ok(ComputedLogEntry { m: value.m, f: *f, t: value.t.timestamp(), exec: value.exec, }), - None => Err(DBError::InvalidFilterError(filter.to_string())), + None => Err(DBError::InvalidFilterError(value.f.to_string())), } } - fn to(self, header: &DatabaseReadHeader) -> Result { + fn to(self, header: &ReadHeader) -> Result { match header.get(&self.f) { Some(f) => Ok(LogEntry { m: self.m, diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs index 9db025b..98fdd18 100644 --- a/rust/src/database/mod.rs +++ b/rust/src/database/mod.rs @@ -12,7 +12,6 @@ use std::{ use chrono::{Local, TimeDelta}; use log::{debug, error, info, warn}; -use postcard; use thiserror::Error; use crate::{ @@ -32,6 +31,7 @@ const LOG_DB_NEW_NAME: &str = "./reaction-matches.new.db"; const FLUSH_DB_NAME: &str = "./reaction-flushes.db"; const MAX_WRITES: u32 = 500_000; + #[derive(Error, Debug)] pub enum DBError { #[error("invalid filter: {0}")] @@ -43,6 +43,7 @@ pub enum DBError { #[error("{0}")] Error(String), } + #[derive(Clone)] pub enum DatabaseManagerInput { Log(LogEntry), @@ -135,7 +136,7 @@ fn _rotate_db( config: &Arc, matches_tx: &Option>, ) -> Result<(WriteDB, WriteDB), DBError> { - let mut log_read_db = match ReadDB::open(LOG_DB_NAME)? { + let mut log_read_db = match ReadDB::open(LOG_DB_NAME, config)? { Some(db) => db, None => { return Ok(( @@ -144,14 +145,14 @@ fn _rotate_db( )); } }; - let mut flush_read_db = match ReadDB::open(FLUSH_DB_NAME)? { + let mut flush_read_db = match ReadDB::open(FLUSH_DB_NAME, config)? { Some(db) => db, None => { warn!( "Strange! Found a {} but no {}, opening /dev/null instead", LOG_DB_NAME, FLUSH_DB_NAME ); - match ReadDB::open("/dev/null")? { + match ReadDB::open("/dev/null", config)? { Some(db) => db, None => { return Err(DBError::Error("/dev/null is not accessible".into())); @@ -198,6 +199,7 @@ fn __rotate_db( let mut millisecond_disambiguation_counter: u32 = 0; // Read flushes + #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used let mut flushes: BTreeMap, BTreeMap> = BTreeMap::new(); for flush_entry in flush_read_db { match flush_entry { diff --git a/rust/src/database/tests.rs b/rust/src/database/tests.rs index 1044533..55a380a 100644 --- a/rust/src/database/tests.rs +++ b/rust/src/database/tests.rs @@ -69,7 +69,7 @@ fn write_and_read_db() { drop(write_db); - let read_db = ReadDB::open(db_path.to_str().unwrap()); + let read_db = ReadDB::open(db_path.to_str().unwrap(), &config); assert!(read_db.is_ok()); let read_db = read_db.unwrap(); diff --git a/rust/src/execs.rs b/rust/src/execs.rs index 510f881..aeb3f0f 100644 --- a/rust/src/execs.rs +++ b/rust/src/execs.rs @@ -96,6 +96,7 @@ pub fn execs_manager( } }; + #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used let mut execs: ExecsMap = BTreeMap::new(); let timer = MessageTimer::new(exec_tx); diff --git a/rust/src/filter.rs b/rust/src/filter.rs index cb55d7e..6e2fd19 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -8,7 +8,7 @@ use std::{ use chrono::TimeDelta; use log::info; use regex::Regex; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use crate::{ action::Action, @@ -21,15 +21,13 @@ use crate::{ // Only names are serialized // Only computed fields are not deserialized -#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize)] #[serde(deny_unknown_fields)] pub struct Filter { - #[serde(skip_serializing)] actions: BTreeMap>, #[serde(skip)] longuest_action_duration: TimeDelta, - #[serde(skip_serializing)] regex: Vec, #[serde(skip)] compiled_regex: Vec, @@ -38,16 +36,15 @@ pub struct Filter { #[serde(skip)] patterns: Arc>>, - #[serde(skip_serializing)] retry: Option, - #[serde(skip_serializing, rename = "retryperiod")] + #[serde(rename = "retryperiod")] retry_period: Option, #[serde(skip)] retry_duration: Option, - #[serde(skip_deserializing)] + #[serde(skip)] name: String, - #[serde(skip_deserializing)] + #[serde(skip)] stream_name: String, } @@ -61,6 +58,10 @@ impl Filter { } } + pub fn full_name(&self) -> (String, String) { + (self.stream_name.clone(), self.name.clone()) + } + pub fn retry(&self) -> Option { self.retry } @@ -73,6 +74,7 @@ impl Filter { self.longuest_action_duration } + #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used pub fn patterns(&self) -> &BTreeSet> { &self.patterns } @@ -123,6 +125,7 @@ impl Filter { return Err("no regex configured".into()); } + #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used let mut new_patterns = BTreeSet::new(); let mut first = true; for regex in &self.regex { diff --git a/rust/src/matches.rs b/rust/src/matches.rs index 5d69de2..8f7b63c 100644 --- a/rust/src/matches.rs +++ b/rust/src/matches.rs @@ -79,6 +79,7 @@ pub fn matches_manager( action_tx: Sender, log_tx: SyncSender, ) { + #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used let mut matches: MatchesMap = BTreeMap::new(); let timer = MessageTimer::new(match_tx); diff --git a/rust/src/stream.rs b/rust/src/stream.rs index 50bbc03..c7c6dce 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -29,6 +29,10 @@ impl Stream { &self.filters } + pub fn get_filter(&self, filter_name: &str) -> Option<&Arc> { + self.filters.get(filter_name) + } + pub fn setup(&mut self, name: &str, patterns: &Patterns) -> Result<(), String> { self._setup(name, patterns) .map_err(|msg| format!("stream {}: {}", name, msg)) @@ -94,17 +98,11 @@ impl Stream { let _ = child_tx.send(Some(child)); drop(child_tx); - // let mut line: String = "".into(); - // while let Ok(nb_chars) = stdout.read_line(&mut line) { for line in stdout.lines() { let line = match line { Ok(line) => line, Err(_) => break, }; - // if nb_chars == 0 { - // break; - // } - // line.pop(); // remove trailing newline for filter in self.filters.values() { if let Some(match_) = filter.get_match(&line) { @@ -117,10 +115,8 @@ impl Stream { .unwrap(); } } - - // line.clear(); } - info!("stream {} exited", self.name); + error!("stream {} exited", self.name); } } From feb863670e6004044a3c46e4b6b6e41e65494a89 Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 24 Sep 2024 12:00:00 +0200 Subject: [PATCH 044/435] fix short option name clash --- rust/src/utils/cli.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/src/utils/cli.rs b/rust/src/utils/cli.rs index 422c90b..663ac5e 100644 --- a/rust/src/utils/cli.rs +++ b/rust/src/utils/cli.rs @@ -48,7 +48,7 @@ pub enum Command { socket: PathBuf, /// how to format output: json or yaml. - #[clap(short = 's', long, default_value_t = Format::YAML)] + #[clap(short = 'f', long, default_value_t = Format::YAML)] format: Format, /// only show items related to this STREAM[.FILTER] @@ -75,7 +75,7 @@ Then prints the flushed matches and actions." socket: PathBuf, /// how to format output: json or yaml. - #[clap(short = 's', long, default_value_t = Format::YAML)] + #[clap(short = 'f', long, default_value_t = Format::YAML)] format: Format, /// only show items related to this STREAM[.FILTER] From d30d03bae8848c3d5409222ce53d97aa9ab9ef29 Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 24 Sep 2024 12:00:00 +0200 Subject: [PATCH 045/435] Box::leak global Config into a &'static ref instead of Arcs everywhere --- rust/src/action.rs | 12 ++++++------ rust/src/config.rs | 6 +++--- rust/src/daemon.rs | 11 ++++------- rust/src/database/lowlevel.rs | 13 ++++++------- rust/src/database/mod.rs | 17 +++++++---------- rust/src/database/tests.rs | 16 +++++++--------- rust/src/execs.rs | 13 +++++-------- rust/src/filter.rs | 14 +++++--------- rust/src/matches.rs | 9 +++------ rust/src/messages.rs | 8 +++----- rust/src/stream.rs | 25 +++++++++---------------- 11 files changed, 58 insertions(+), 86 deletions(-) diff --git a/rust/src/action.rs b/rust/src/action.rs index f95de7b..184aef9 100644 --- a/rust/src/action.rs +++ b/rust/src/action.rs @@ -161,20 +161,20 @@ pub mod tests { } } - pub fn ok_action() -> Arc { + pub fn ok_action() -> Action { let mut action = default_action(); action.cmd = vec!["command".into()]; - Arc::new(action) + action } - pub fn ok_action_with_after(d: String, name: &str) -> Arc { + pub fn ok_action_with_after(d: String, name: &str) -> Action { let mut action = default_action(); action.cmd = vec!["command".into()]; action.after = Some(d); action .setup("", "", name, Arc::new(BTreeSet::default())) .unwrap(); - Arc::new(action) + action } #[test] @@ -198,11 +198,11 @@ pub mod tests { assert!(action.setup(&name, &name, &name, patterns.clone()).is_err()); // command ok - action = ok_action().as_ref().clone(); + action = ok_action(); assert!(action.setup(&name, &name, &name, patterns.clone()).is_ok()); // command ok - action = ok_action().as_ref().clone(); + action = ok_action(); action.cmd.push("arg1".into()); assert!(action.setup(&name, &name, &name, patterns.clone()).is_ok()); } diff --git a/rust/src/config.rs b/rust/src/config.rs index 4347cd0..38a2b18 100644 --- a/rust/src/config.rs +++ b/rust/src/config.rs @@ -40,14 +40,14 @@ impl Config { self.concurrency } - pub fn filters(&self) -> Vec> { + pub fn filters(&self) -> Vec<&Filter> { self.streams .values() - .flat_map(|stream| stream.filters().values().cloned()) + .flat_map(|stream| stream.filters().values()) .collect() } - pub fn get_filter(&self, name: &(String, String)) -> Option<&Arc> { + pub fn get_filter(&self, name: &(String, String)) -> Option<&Filter> { self.streams .get(&name.0) .and_then(|stream| stream.get_filter(&name.1)) diff --git a/rust/src/daemon.rs b/rust/src/daemon.rs index 399bd22..5339c27 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon.rs @@ -26,8 +26,8 @@ pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { exit(1); } - let config = match Config::from_file(config_path) { - Ok(config) => Arc::new(config), + let config: &'static Config = match Config::from_file(config_path) { + Ok(config) => Box::leak(Box::new(config)), Err(err) => { error!("{err}"); exit(1); @@ -53,20 +53,17 @@ pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { }; let execs_manager_thread_handle = { - let config_execs = config.clone(); let exec_tx_execs = exec_tx.clone(); - thread::spawn(move || execs_manager(config_execs, exec_rx, exec_tx_execs)) + thread::spawn(move || execs_manager(config, exec_rx, exec_tx_execs)) }; let database_manager_thread_handle = { - let config_database = config.clone(); let match_tx_database = match_tx.clone(); // The `thread::spawn` is done in the function, after database rotation is finished - database_manager(config_database, log_rx, match_tx_database) + database_manager(config, log_rx, match_tx_database) }; for stream in config.streams().values() { - let stream = stream.clone(); let match_tx = match_tx.clone(); let (child_tx, child_rx) = sync_channel(0); diff --git a/rust/src/database/lowlevel.rs b/rust/src/database/lowlevel.rs index 174de68..ab916e7 100644 --- a/rust/src/database/lowlevel.rs +++ b/rust/src/database/lowlevel.rs @@ -4,7 +4,6 @@ use std::{ fs::File, io::{self, BufReader, BufWriter, Write}, process::exit, - sync::Arc, }; use chrono::{DateTime, Local}; @@ -24,8 +23,8 @@ use super::DBError; // It may permit to win 1-4 bytes per entry, don't know if it's worth it // FIXME put signature in the header? type DatabaseHeader = BTreeMap; -type ReadHeader = BTreeMap>; -type WriteHeader = BTreeMap, usize>; +type ReadHeader = BTreeMap; +type WriteHeader = BTreeMap<&'static Filter, usize>; const BUFFER_MAX_SIZE: usize = 10 * 1024 * 1024; const DB_SIGNATURE: &str = "reaction-db-v01"; @@ -37,7 +36,7 @@ pub struct ReadDB { } impl ReadDB { - pub fn open(path: &str, config: &Arc) -> Result, DBError> { + pub fn open(path: &str, config: &'static Config) -> Result, DBError> { let file = match File::open(path) { Ok(file) => file, Err(err) => match err.kind() { @@ -72,7 +71,7 @@ impl ReadDB { ret.h = db_header .iter() - .filter_map(|(key, name)| config.get_filter(name).map(|filter| (*key, filter.clone()))) + .filter_map(|(key, name)| config.get_filter(name).map(|filter| (*key, filter))) .collect(); Ok(Some(ret)) @@ -110,7 +109,7 @@ pub struct WriteDB { } impl WriteDB { - pub fn create(path: &str, config: &Arc) -> Self { + pub fn create(path: &str, config: &'static Config) -> Self { let file = match File::create(path) { Ok(file) => file, Err(err) => { @@ -198,7 +197,7 @@ impl ComputedLogEntry { match header.get(&self.f) { Some(f) => Ok(LogEntry { m: self.m, - f: f.clone(), + f, t: DateTime::from_timestamp(self.t, 0) .unwrap() .with_timezone(&Local), diff --git a/rust/src/database/mod.rs b/rust/src/database/mod.rs index 98fdd18..facb880 100644 --- a/rust/src/database/mod.rs +++ b/rust/src/database/mod.rs @@ -3,10 +3,7 @@ use std::{ fmt::Debug, fs, io, process::exit, - sync::{ - mpsc::{Receiver, Sender}, - Arc, - }, + sync::mpsc::{Receiver, Sender}, thread, }; @@ -71,11 +68,11 @@ macro_rules! flush_or_die { /// First rotates the database, then spawns the database thread pub fn database_manager( - config: Arc, + config: &'static Config, log_rx: Receiver, matches_tx: Sender, ) -> thread::JoinHandle<()> { - let (mut log_db, mut flush_db) = match rotate_db(&config, Some(matches_tx)) { + let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)) { Ok(dbs) => dbs, Err(err) => { error!("while rotating databases on start: {}", err); @@ -97,7 +94,7 @@ pub fn database_manager( flush_or_die!(flush_db); drop(log_db); drop(flush_db); - (log_db, flush_db) = match rotate_db(&config, None) { + (log_db, flush_db) = match rotate_db(config, None) { Ok(dbs) => dbs, Err(err) => { error!( @@ -117,7 +114,7 @@ pub fn database_manager( } fn rotate_db( - config: &Arc, + config: &'static Config, matches_tx: Option>, ) -> Result<(WriteDB, WriteDB), DBError> { info!("Rotating database..."); @@ -133,7 +130,7 @@ fn rotate_db( } fn _rotate_db( - config: &Arc, + config: &'static Config, matches_tx: &Option>, ) -> Result<(WriteDB, WriteDB), DBError> { let mut log_read_db = match ReadDB::open(LOG_DB_NAME, config)? { @@ -200,7 +197,7 @@ fn __rotate_db( // Read flushes #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used - let mut flushes: BTreeMap, BTreeMap> = BTreeMap::new(); + let mut flushes: BTreeMap<&'static Filter, BTreeMap> = BTreeMap::new(); for flush_entry in flush_read_db { match flush_entry { Ok(entry) => { diff --git a/rust/src/database/tests.rs b/rust/src/database/tests.rs index 55a380a..d673818 100644 --- a/rust/src/database/tests.rs +++ b/rust/src/database/tests.rs @@ -1,7 +1,5 @@ #![cfg(test)] -use std::sync::Arc; - use chrono::Local; use crate::database::ReadDB; @@ -40,36 +38,36 @@ fn write_and_read_db() { ", ); - let config = Arc::new(Config::from_file(&config_file).unwrap()); + let config = Box::leak(Box::new(Config::from_file(&config_file).unwrap())); - let correct_filter_name = Arc::new(Filter::from_name("stream1", "filter1")); + let correct_filter_name = Box::leak(Box::new(Filter::from_name("stream1", "filter1"))); - let incorrect_filter_name = Arc::new(Filter::from_name("stream0", "filter1")); + let incorrect_filter_name = Box::leak(Box::new(Filter::from_name("stream0", "filter1"))); let correct_log_entry = LogEntry { m: vec!["match1".into()], - f: correct_filter_name.clone(), + f: correct_filter_name, t: Local::now(), exec: false, }; let incorrect_log_entry = LogEntry { m: vec!["match1".into()], - f: incorrect_filter_name.clone(), + f: incorrect_filter_name, t: Local::now(), exec: false, }; let db_path = Fixture::empty("matches.db"); - let mut write_db = WriteDB::create(db_path.to_str().unwrap(), &config); + let mut write_db = WriteDB::create(db_path.to_str().unwrap(), config); assert!(write_db.write(correct_log_entry.clone()).is_ok()); assert!(write_db.write(incorrect_log_entry).is_err()); drop(write_db); - let read_db = ReadDB::open(db_path.to_str().unwrap(), &config); + let read_db = ReadDB::open(db_path.to_str().unwrap(), config); assert!(read_db.is_ok()); let read_db = read_db.unwrap(); diff --git a/rust/src/execs.rs b/rust/src/execs.rs index aeb3f0f..da5d881 100644 --- a/rust/src/execs.rs +++ b/rust/src/execs.rs @@ -1,10 +1,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, process::Stdio, - sync::{ - mpsc::{Receiver, Sender}, - Arc, - }, + sync::mpsc::{Receiver, Sender}, }; use chrono::{DateTime, Local}; @@ -27,7 +24,7 @@ pub enum ExecsManagerInput { Stop, } -type ExecsMap = BTreeMap, BTreeMap>>>; +type ExecsMap = BTreeMap<&'static Action, BTreeMap>>>; trait ExecsMapTrait { fn add(&mut self, mat: &MAT); @@ -35,7 +32,7 @@ trait ExecsMapTrait { } impl ExecsMapTrait for ExecsMap { fn add(&mut self, mat: &MAT) { - let inner_map = self.entry(mat.a.clone()).or_default(); + let inner_map = self.entry(mat.a).or_default(); let inner_set = inner_map.entry(mat.m.clone()).or_default(); inner_set.insert(mat.t); } @@ -56,7 +53,7 @@ impl ExecsMapTrait for ExecsMap { } pub fn execs_manager( - config: Arc, + config: &'static Config, exec_rx: Receiver, exec_tx: Sender, ) { @@ -127,7 +124,7 @@ pub fn execs_manager( for _ in inner_set { exec_now(MAT { m: match_.clone(), - a: action.clone(), + a: action, t: Local::now(), }); } diff --git a/rust/src/filter.rs b/rust/src/filter.rs index 6e2fd19..2fb644a 100644 --- a/rust/src/filter.rs +++ b/rust/src/filter.rs @@ -24,7 +24,7 @@ use crate::{ #[derive(Clone, Debug, Default, Deserialize)] #[serde(deny_unknown_fields)] pub struct Filter { - actions: BTreeMap>, + actions: BTreeMap, #[serde(skip)] longuest_action_duration: TimeDelta, @@ -165,13 +165,9 @@ impl Filter { return Err("no actions configured".into()); } - let mut new_actions = BTreeMap::new(); - for (key, action) in &self.actions { - let mut new_action = action.as_ref().clone(); - new_action.setup(stream_name, name, key, self.patterns.clone())?; - new_actions.insert(key.clone(), Arc::new(new_action)); + for (key, action) in &mut self.actions { + action.setup(stream_name, name, key, self.patterns.clone())?; } - self.actions = new_actions; self.longuest_action_duration = self.actions.values().fold(TimeDelta::seconds(0), |acc, v| { @@ -206,11 +202,11 @@ impl Filter { None } - pub fn send_actions(&self, m: &Match, t: Time, tx: &Sender) { + pub fn send_actions(&'static self, m: &Match, t: Time, tx: &Sender) { for action in self.actions.values() { tx.send(ExecsManagerInput::Exec(MAT { m: m.clone(), - a: action.clone(), + a: action, t: t + action.after_duration().unwrap_or_default(), })) .unwrap(); diff --git a/rust/src/matches.rs b/rust/src/matches.rs index 8f7b63c..04e90fa 100644 --- a/rust/src/matches.rs +++ b/rust/src/matches.rs @@ -1,9 +1,6 @@ use std::{ collections::{BTreeMap, BTreeSet}, - sync::{ - mpsc::{Receiver, Sender, SyncSender}, - Arc, - }, + sync::mpsc::{Receiver, Sender, SyncSender}, }; use log::debug; @@ -26,7 +23,7 @@ pub enum MatchManagerInput { Stop, } -type MatchesMap = BTreeMap, BTreeMap>>; +type MatchesMap = BTreeMap<&'static Filter, BTreeMap>>; // This trait is needed to permit to implement methods on an external type trait MatchesMapTrait { @@ -37,7 +34,7 @@ trait MatchesMapTrait { } impl MatchesMapTrait for MatchesMap { fn add(&mut self, mft: &MFT) { - let inner_map = self.entry(mft.f.clone()).or_default(); + let inner_map = self.entry(mft.f).or_default(); let inner_set = inner_map.entry(mft.m.clone()).or_default(); inner_set.insert(mft.t); } diff --git a/rust/src/messages.rs b/rust/src/messages.rs index ed68d29..773087b 100644 --- a/rust/src/messages.rs +++ b/rust/src/messages.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use chrono::{DateTime, Local, TimeDelta}; use crate::{action::Action, filter::Filter}; @@ -10,21 +8,21 @@ pub type Match = Vec; #[derive(Clone)] pub struct MFT { pub m: Match, - pub f: Arc, + pub f: &'static Filter, pub t: Time, } #[derive(Clone)] pub struct MAT { pub m: Match, - pub a: Arc, + pub a: &'static Action, pub t: Time, } #[derive(Clone, Debug)] pub struct LogEntry { pub m: Match, - pub f: Arc, + pub f: &'static Filter, pub t: Time, pub exec: bool, } diff --git a/rust/src/stream.rs b/rust/src/stream.rs index c7c6dce..f0189a1 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -2,10 +2,7 @@ use std::{ collections::BTreeMap, io::{BufRead, BufReader}, process::{Child, Command, Stdio}, - sync::{ - mpsc::{Sender, SyncSender}, - Arc, - }, + sync::mpsc::{Sender, SyncSender}, }; use chrono::Local; @@ -18,18 +15,18 @@ use crate::{config::Patterns, filter::Filter, matches::MatchManagerInput, messag #[serde(deny_unknown_fields)] pub struct Stream { cmd: Vec, - filters: BTreeMap>, + filters: BTreeMap, #[serde(skip)] name: String, } impl Stream { - pub fn filters(&self) -> &BTreeMap> { + pub fn filters(&self) -> &BTreeMap { &self.filters } - pub fn get_filter(&self, filter_name: &str) -> Option<&Arc> { + pub fn get_filter(&self, filter_name: &str) -> Option<&Filter> { self.filters.get(filter_name) } @@ -59,19 +56,15 @@ impl Stream { return Err("no filters configured".into()); } - let mut new_filters = BTreeMap::new(); - for (key, filter) in &self.filters { - let mut new_filter = filter.as_ref().clone(); - new_filter.setup(name, key, patterns)?; - new_filters.insert(key.clone(), Arc::new(new_filter)); + for (key, filter) in &mut self.filters { + filter.setup(name, key, patterns)?; } - self.filters = new_filters; Ok(()) } pub fn manager( - &self, + &'static self, child_tx: SyncSender>, match_tx: Sender, ) { @@ -109,7 +102,7 @@ impl Stream { match_tx .send(MatchManagerInput::Match(MFT { m: match_, - f: filter.clone(), + f: filter, t: Local::now(), })) .unwrap(); @@ -138,7 +131,7 @@ pub mod tests { stream.cmd = vec!["command".into()]; stream .filters - .insert("name".into(), Arc::new(crate::filter::tests::ok_filter())); + .insert("name".into(), crate::filter::tests::ok_filter()); stream } From 807b3c7440479464bfdc68f3e35f4ad2fc87aefc Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 25 Sep 2024 12:00:00 +0200 Subject: [PATCH 046/435] source tree organization + proper unwrap warns and explanations --- rust/src/{client.rs => client/mod.rs} | 0 rust/src/{ => concepts}/action.rs | 4 +++- rust/src/{ => concepts}/config.rs | 2 +- rust/src/{ => concepts}/filter.rs | 26 +++++++++++++--------- rust/src/{ => concepts}/messages.rs | 3 ++- rust/src/concepts/mod.rs | 13 +++++++++++ rust/src/{ => concepts}/pattern.rs | 1 + rust/src/{ => concepts}/stream.rs | 11 +++++---- rust/src/{ => daemon}/database/lowlevel.rs | 9 ++++---- rust/src/{ => daemon}/database/mod.rs | 15 ++++++------- rust/src/{ => daemon}/database/tests.rs | 11 ++++----- rust/src/{ => daemon}/execs.rs | 4 +--- rust/src/{ => daemon}/matches.rs | 10 +++++---- rust/src/{daemon.rs => daemon/mod.rs} | 23 +++++++++++++------ rust/src/main.rs | 19 +++------------- rust/src/tests.rs | 1 + rust/src/utils/threadpool.rs | 3 +++ 17 files changed, 89 insertions(+), 66 deletions(-) rename rust/src/{client.rs => client/mod.rs} (100%) rename rust/src/{ => concepts}/action.rs (98%) rename rust/src/{ => concepts}/config.rs (98%) rename rust/src/{ => concepts}/filter.rs (95%) rename rust/src/{ => concepts}/messages.rs (88%) create mode 100644 rust/src/concepts/mod.rs rename rust/src/{ => concepts}/pattern.rs (99%) rename rust/src/{ => concepts}/stream.rs (91%) rename rust/src/{ => daemon}/database/lowlevel.rs (96%) rename rust/src/{ => daemon}/database/mod.rs (95%) rename rust/src/{ => daemon}/database/tests.rs (93%) rename rust/src/{ => daemon}/execs.rs (98%) rename rust/src/{ => daemon}/matches.rs (91%) rename rust/src/{daemon.rs => daemon/mod.rs} (85%) diff --git a/rust/src/client.rs b/rust/src/client/mod.rs similarity index 100% rename from rust/src/client.rs rename to rust/src/client/mod.rs diff --git a/rust/src/action.rs b/rust/src/concepts/action.rs similarity index 98% rename from rust/src/action.rs rename to rust/src/concepts/action.rs index 184aef9..1e69f42 100644 --- a/rust/src/action.rs +++ b/rust/src/concepts/action.rs @@ -4,7 +4,8 @@ use chrono::TimeDelta; use serde::Deserialize; -use crate::{messages::Match, pattern::Pattern, utils::parse_duration}; +use super::{Match, Pattern}; +use crate::utils::parse_duration; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -143,6 +144,7 @@ impl Display for Action { } } +#[allow(clippy::unwrap_used)] #[cfg(test)] pub mod tests { diff --git a/rust/src/config.rs b/rust/src/concepts/config.rs similarity index 98% rename from rust/src/config.rs rename to rust/src/concepts/config.rs index 38a2b18..61cf401 100644 --- a/rust/src/config.rs +++ b/rust/src/concepts/config.rs @@ -11,7 +11,7 @@ use log::{error, info}; use serde::Deserialize; use thiserror::Error; -use crate::{filter::Filter, pattern::Pattern, stream::Stream}; +use super::{Filter, Pattern, Stream}; pub type Patterns = BTreeMap>; diff --git a/rust/src/filter.rs b/rust/src/concepts/filter.rs similarity index 95% rename from rust/src/filter.rs rename to rust/src/concepts/filter.rs index 2fb644a..a7e3257 100644 --- a/rust/src/filter.rs +++ b/rust/src/concepts/filter.rs @@ -10,14 +10,11 @@ use log::info; use regex::Regex; use serde::Deserialize; -use crate::{ - action::Action, - config::Patterns, - execs::ExecsManagerInput, +use super::{ messages::{Match, Time, MAT}, - pattern::Pattern, - utils::parse_duration, + Action, Pattern, Patterns, }; +use crate::{daemon::ExecsManagerInput, utils::parse_duration}; // Only names are serialized // Only computed fields are not deserialized @@ -132,6 +129,8 @@ impl Filter { let mut regex_buf = regex.clone(); for pattern in config_patterns.values() { if let Some(index) = regex.find(pattern.name_with_braces()) { + // we already `find` it, so we must be able to `rfind` it + #[allow(clippy::unwrap_used)] if regex.rfind(pattern.name_with_braces()).unwrap() != index { return Err(format!( "pattern {} present multiple times in regex", @@ -184,9 +183,12 @@ impl Filter { if !self.patterns.is_empty() { let mut result = Match::new(); for pattern in self.patterns.as_ref() { - let match_ = matches.name(pattern.name()).unwrap(); - if pattern.not_an_ignore(match_.as_str()) { - result.push(match_.as_str().to_string()); + // if the pattern is in an optional part of the regex, there may be no + // captured group for it. + if let Some(match_) = matches.name(pattern.name()) { + if pattern.not_an_ignore(match_.as_str()) { + result.push(match_.as_str().to_string()); + } } } if result.len() == self.patterns.len() { @@ -204,6 +206,7 @@ impl Filter { pub fn send_actions(&'static self, m: &Match, t: Time, tx: &Sender) { for action in self.actions.values() { + #[allow(clippy::unwrap_used)] // propagating panics is ok tx.send(ExecsManagerInput::Exec(MAT { m: m.clone(), a: action, @@ -240,10 +243,11 @@ impl PartialOrd for Filter { } } +#[allow(clippy::unwrap_used)] #[cfg(test)] pub mod tests { - use crate::action::tests::{ok_action, ok_action_with_after}; - use crate::pattern::tests::{ + use crate::concepts::action::tests::{ok_action, ok_action_with_after}; + use crate::concepts::pattern::tests::{ boubou_pattern_with_ignore, default_pattern, ok_pattern_with_ignore, }; diff --git a/rust/src/messages.rs b/rust/src/concepts/messages.rs similarity index 88% rename from rust/src/messages.rs rename to rust/src/concepts/messages.rs index 773087b..f4541bf 100644 --- a/rust/src/messages.rs +++ b/rust/src/concepts/messages.rs @@ -1,6 +1,6 @@ use chrono::{DateTime, Local, TimeDelta}; -use crate::{action::Action, filter::Filter}; +use super::{Action, Filter}; pub type Time = DateTime; pub type Match = Vec; @@ -28,6 +28,7 @@ pub struct LogEntry { } impl PartialEq for LogEntry { + #[allow(clippy::unwrap_used)] // 1 second is obviously less seconds than i64::MAX fn eq(&self, other: &Self) -> bool { self.exec == other.exec && self.m == other.m diff --git a/rust/src/concepts/mod.rs b/rust/src/concepts/mod.rs new file mode 100644 index 0000000..4c237d3 --- /dev/null +++ b/rust/src/concepts/mod.rs @@ -0,0 +1,13 @@ +mod action; +mod config; +mod filter; +mod messages; +mod pattern; +mod stream; + +pub use action::*; +pub use config::*; +pub use filter::*; +pub use messages::*; +pub use pattern::*; +pub use stream::*; diff --git a/rust/src/pattern.rs b/rust/src/concepts/pattern.rs similarity index 99% rename from rust/src/pattern.rs rename to rust/src/concepts/pattern.rs index 1d4871d..2d3e089 100644 --- a/rust/src/pattern.rs +++ b/rust/src/concepts/pattern.rs @@ -108,6 +108,7 @@ impl PartialEq for Pattern { } } +#[allow(clippy::unwrap_used)] #[cfg(test)] pub mod tests { diff --git a/rust/src/stream.rs b/rust/src/concepts/stream.rs similarity index 91% rename from rust/src/stream.rs rename to rust/src/concepts/stream.rs index f0189a1..2971a2c 100644 --- a/rust/src/stream.rs +++ b/rust/src/concepts/stream.rs @@ -9,7 +9,8 @@ use chrono::Local; use log::{error, info}; use serde::Deserialize; -use crate::{config::Patterns, filter::Filter, matches::MatchManagerInput, messages::MFT}; +use super::{Filter, Patterns, MFT}; +use crate::daemon::MatchManagerInput; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -85,6 +86,8 @@ impl Stream { }; // keep stdout before sending/moving child to the main thread + #[allow(clippy::unwrap_used)] + // we know there is an stdout because we asked for Stdio::piped() let stdout = BufReader::new(child.stdout.take().unwrap()); // let main handle the child process @@ -99,6 +102,7 @@ impl Stream { for filter in self.filters.values() { if let Some(match_) = filter.get_match(&line) { + #[allow(clippy::unwrap_used)] // propagating panics is ok match_tx .send(MatchManagerInput::Match(MFT { m: match_, @@ -117,6 +121,7 @@ impl Stream { pub mod tests { use super::*; + use crate::concepts::filter::tests::ok_filter; fn default_stream() -> Stream { Stream { @@ -129,9 +134,7 @@ pub mod tests { pub fn ok_stream() -> Stream { let mut stream = default_stream(); stream.cmd = vec!["command".into()]; - stream - .filters - .insert("name".into(), crate::filter::tests::ok_filter()); + stream.filters.insert("name".into(), ok_filter()); stream } diff --git a/rust/src/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs similarity index 96% rename from rust/src/database/lowlevel.rs rename to rust/src/daemon/database/lowlevel.rs index ab916e7..f6af9ae 100644 --- a/rust/src/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -10,11 +10,7 @@ use chrono::{DateTime, Local}; use log::{debug, error, warn}; use serde::{Deserialize, Serialize}; -use crate::{ - config::Config, - filter::Filter, - messages::{LogEntry, Match}, -}; +use crate::concepts::{Config, Filter, LogEntry, Match}; use super::DBError; @@ -195,6 +191,9 @@ impl ComputedLogEntry { } fn to(self, header: &ReadHeader) -> Result { match header.get(&self.f) { + // a timestamp will cause a panic in 300_000_000 years + // you & me will both be dead so why bother? /o/ + #[allow(clippy::unwrap_used)] Some(f) => Ok(LogEntry { m: self.m, f, diff --git a/rust/src/database/mod.rs b/rust/src/daemon/database/mod.rs similarity index 95% rename from rust/src/database/mod.rs rename to rust/src/daemon/database/mod.rs index facb880..0552ece 100644 --- a/rust/src/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -8,15 +8,11 @@ use std::{ }; use chrono::{Local, TimeDelta}; -use log::{debug, error, info, warn}; +use log::{error, info, warn}; use thiserror::Error; -use crate::{ - config::Config, - filter::Filter, - matches::MatchManagerInput, - messages::{LogEntry, Match, Time}, -}; +use super::MatchManagerInput; +use crate::concepts::{Config, Filter, LogEntry, Match, Time}; mod lowlevel; mod tests; @@ -120,8 +116,8 @@ fn rotate_db( info!("Rotating database..."); let res = _rotate_db(config, &matches_tx); - debug!("sending EOS: {}", matches_tx.is_some()); if let Some(tx) = matches_tx { + #[allow(clippy::unwrap_used)] // propagating panics is ok tx.send(MatchManagerInput::EndOfStartup).unwrap(); } @@ -239,15 +235,18 @@ fn __rotate_db( } // Store match & store in db + #[allow(clippy::unwrap_used)] // 0 second is obviously less seconds than i64::MAX if (!entry.exec && entry.t + entry.f.retry_duration().unwrap_or_default() > now) || (entry.exec && entry.t + entry.f.longuest_action_duration() > now) { // We loose subsecond precision when storing times, so we add those fake // milliseconds to make sure each time is unique + entry.t += TimeDelta::new(0, millisecond_disambiguation_counter).unwrap(); millisecond_disambiguation_counter += 1; if let Some(tx) = matches_tx { + #[allow(clippy::unwrap_used)] // propagating panics is ok tx.send(MatchManagerInput::Match(entry.clone().into())) .unwrap(); } diff --git a/rust/src/database/tests.rs b/rust/src/daemon/database/tests.rs similarity index 93% rename from rust/src/database/tests.rs rename to rust/src/daemon/database/tests.rs index d673818..aacf959 100644 --- a/rust/src/database/tests.rs +++ b/rust/src/daemon/database/tests.rs @@ -1,12 +1,13 @@ +#![allow(clippy::unwrap_used)] #![cfg(test)] use chrono::Local; -use crate::database::ReadDB; -use crate::{config::Config, tests::Fixture}; -use crate::{filter::Filter, messages::LogEntry}; - -use super::WriteDB; +use super::{ReadDB, WriteDB}; +use crate::{ + concepts::{Config, Filter, LogEntry}, + tests::Fixture, +}; #[test] fn write_and_read_db() { diff --git a/rust/src/execs.rs b/rust/src/daemon/execs.rs similarity index 98% rename from rust/src/execs.rs rename to rust/src/daemon/execs.rs index da5d881..46fb987 100644 --- a/rust/src/execs.rs +++ b/rust/src/daemon/execs.rs @@ -9,9 +9,7 @@ use log::{error, info}; use timer::MessageTimer; use crate::{ - action::Action, - config::Config, - messages::{Match, MAT}, + concepts::{Action, Config, Match, MAT}, utils::ThreadPool, }; diff --git a/rust/src/matches.rs b/rust/src/daemon/matches.rs similarity index 91% rename from rust/src/matches.rs rename to rust/src/daemon/matches.rs index 04e90fa..406c79c 100644 --- a/rust/src/matches.rs +++ b/rust/src/daemon/matches.rs @@ -6,11 +6,10 @@ use std::{ use log::debug; use timer::MessageTimer; +use super::{database::DatabaseManagerInput, ExecsManagerInput}; use crate::{ - database::DatabaseManagerInput, - execs::ExecsManagerInput, - filter::Filter, - messages::{LogEntry, Match, Time, MFT}, + concepts::Filter, + concepts::{LogEntry, Match, Time, MFT}, }; #[derive(Clone)] @@ -98,6 +97,8 @@ pub fn matches_manager( matches.add(&mft); // Remove match when expired let guard = timer.schedule_with_delay( + // retry_duration is always Some() after filter's setup + #[allow(clippy::unwrap_used)] mft.f.retry_duration().unwrap(), MatchManagerInput::Unmatch(mft.clone()), ); @@ -117,6 +118,7 @@ pub fn matches_manager( } if !startup { + #[allow(clippy::unwrap_used)] // propagating panics is ok log_tx .send(DatabaseManagerInput::Log(LogEntry { exec, diff --git a/rust/src/daemon.rs b/rust/src/daemon/mod.rs similarity index 85% rename from rust/src/daemon.rs rename to rust/src/daemon/mod.rs index 5339c27..e6dd19e 100644 --- a/rust/src/daemon.rs +++ b/rust/src/daemon/mod.rs @@ -11,13 +11,17 @@ use std::{ use log::{error, info, Level}; -use crate::{ - config::Config, - database::database_manager, - execs::{execs_manager, ExecsManagerInput}, - matches::{matches_manager, MatchManagerInput}, - utils::SimpleLogger, -}; +use crate::{concepts::Config, utils::SimpleLogger}; +use database::database_manager; +use execs::execs_manager; +use matches::matches_manager; + +pub use execs::ExecsManagerInput; +pub use matches::MatchManagerInput; + +mod database; +mod execs; +mod matches; #[allow(unused_variables)] pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { @@ -42,6 +46,9 @@ pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { let mut stream_process_child_handles = Vec::new(); let mut stream_thread_handles = Vec::new(); + // FIXME replace those channel() by sync_channel(1) + // Unbounded channels can eat a lot of RAM + // Is there a timer::MessageTimer implementation for sync_channel? let (match_tx, match_rx) = channel(); let (exec_tx, exec_rx) = channel(); let (log_tx, log_rx) = sync_channel(1); @@ -97,9 +104,11 @@ pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { let _ = thread_handle.join(); } + #[allow(clippy::unwrap_used)] // propagating panics is ok match_tx.send(MatchManagerInput::Stop).unwrap(); let _ = matches_manager_thread_handle.join(); + #[allow(clippy::unwrap_used)] // propagating panics is ok exec_tx.send(ExecsManagerInput::Stop).unwrap(); let _ = execs_manager_thread_handle.join(); diff --git a/rust/src/main.rs b/rust/src/main.rs index 64886a6..a435132 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -1,9 +1,9 @@ #![warn( missing_docs, clippy::panic, - clippy::unimplemented, clippy::todo, - clippy::undocumented_unsafe_blocks + clippy::unimplemented, + clippy::unwrap_used, )] #![allow(clippy::upper_case_acronyms)] #![forbid(unsafe_code)] @@ -11,21 +11,8 @@ //! TODO document a bit use clap::Parser; -// structs and concepts -mod action; -mod config; -mod filter; -mod messages; -mod pattern; -mod stream; - -// important threads -mod database; -mod execs; -mod matches; - -// top-level mod client; +mod concepts; mod daemon; mod tests; diff --git a/rust/src/tests.rs b/rust/src/tests.rs index 297d384..7b92a75 100644 --- a/rust/src/tests.rs +++ b/rust/src/tests.rs @@ -1,3 +1,4 @@ +#![allow(clippy::unwrap_used)] #![cfg(test)] use std::{ diff --git a/rust/src/utils/threadpool.rs b/rust/src/utils/threadpool.rs index 6dd24f9..d9bc755 100644 --- a/rust/src/utils/threadpool.rs +++ b/rust/src/utils/threadpool.rs @@ -35,12 +35,14 @@ impl ThreadPool { { let job = Box::new(f); + #[allow(clippy::unwrap_used)] // propagating panics is ok self.sender.send(job).unwrap(); } pub fn join(self) { drop(self.sender); for worker in self.workers { + #[allow(clippy::unwrap_used)] // propagating panics is ok worker.thread.join().unwrap(); } } @@ -53,6 +55,7 @@ struct Worker { impl Worker { fn new(receiver: Arc>>) -> Worker { let thread = spawn(move || loop { + #[allow(clippy::unwrap_used)] // propagating panics is ok let received = receiver.lock().unwrap().recv(); match received { Ok(job) => job(), From 9cc702e9c7cc8a44abd31288aaefa662c434dc63 Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 25 Sep 2024 12:00:00 +0200 Subject: [PATCH 047/435] Replace postcard by bincode - The interface is easier - This permits me to have unbounded buffers. I did some benchmarking on both options. I can't see any difference in terms of - CPU performance, - memory usage, and - stockage size. I used this file: `datasize.jsonnet` ```jsonnet { patterns: { num: { regex: @'([0-9]+)', }, }, streams: { s1: { cmd: ['seq', '-w', '499999'], filters: { f1: { regex: [ '^$', ], retry: 10, retryperiod: '1m', actions: { a: { cmd: ['true'], }, b: { cmd: ['true'], after: '1m', }, }, }, }, }, }, } ``` And this commands: ``` rm reaction-* sudo systemd-run --wait -p User=ao -p MemoryAccounting=yes -p WorkingDirectory=(pwd) -p Environment=PATH=/run/current-system/sw/bin/ time ./target/release/reaction start -c datasize.jsonnet && ls -l reaction-matches.db sudo systemd-run --wait -p User=ao -p MemoryAccounting=yes -p WorkingDirectory=(pwd) -p Environment=PATH=/run/current-system/sw/bin/ time ./target/release/reaction start -c datasize.jsonnet && ls -l reaction-matches.db sudo systemd-run --wait -p User=ao -p MemoryAccounting=yes -p WorkingDirectory=(pwd) -p Environment=PATH=/run/current-system/sw/bin/ time ./target/release/reaction start -c datasize.jsonnet && ls -l reaction-matches.db ``` At the first invocation, reaction reads no DB. At the second invocation, reaction reads a DB. At the third invocation, reaction reads a double-sized DB. --- rust/Cargo.lock | 123 +-------------------------- rust/Cargo.toml | 2 +- rust/datasize.jsonnet | 30 +++++++ rust/src/daemon/database/lowlevel.rs | 62 ++++++++------ rust/src/daemon/database/mod.rs | 2 +- 5 files changed, 69 insertions(+), 150 deletions(-) create mode 100644 rust/datasize.jsonnet diff --git a/rust/Cargo.lock b/rust/Cargo.lock index d2d897b..4d0efc4 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -85,15 +85,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "atomic-polyfill" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" -dependencies = [ - "critical-section", -] - [[package]] name = "autocfg" version = "1.3.0" @@ -127,12 +118,6 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "cc" version = "1.1.6" @@ -214,12 +199,6 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" -[[package]] -name = "cobs" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" - [[package]] name = "colorchoice" version = "1.0.2" @@ -232,12 +211,6 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" -[[package]] -name = "critical-section" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64009896348fc5af4222e9cf7d7d82a95a256c634ebcf61c53e4ea461422242" - [[package]] name = "ctrlc" version = "3.4.4" @@ -248,18 +221,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "embedded-io" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" - -[[package]] -name = "embedded-io" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" - [[package]] name = "equivalent" version = "1.0.1" @@ -282,35 +243,12 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" -[[package]] -name = "hash32" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" -dependencies = [ - "byteorder", -] - [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -[[package]] -name = "heapless" -version = "0.7.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" -dependencies = [ - "atomic-polyfill", - "hash32", - "rustc_version", - "serde", - "spin", - "stable_deref_trait", -] - [[package]] name = "heck" version = "0.5.0" @@ -472,16 +410,6 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.22" @@ -570,19 +498,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c719dcf55f09a3a7e764c6649ab594c18a177e3599c467983cdf644bfc0a4088" -[[package]] -name = "postcard" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7f0a8d620d71c457dd1d47df76bb18960378da56af4527aaa10f515eee732e" -dependencies = [ - "cobs", - "embedded-io 0.4.0", - "embedded-io 0.6.1", - "heapless", - "serde", -] - [[package]] name = "proc-macro2" version = "1.0.86" @@ -605,6 +520,7 @@ dependencies = [ name = "reaction" version = "0.1.0" dependencies = [ + "bincode", "chrono", "clap", "clap_complete", @@ -612,7 +528,6 @@ dependencies = [ "jrsonnet-evaluator", "log", "num_cpus", - "postcard", "regex", "serde", "serde_json", @@ -657,15 +572,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "0.38.34" @@ -685,18 +591,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "semver" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" - [[package]] name = "serde" version = "1.0.204" @@ -742,21 +636,6 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "strsim" version = "0.11.1" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 9df7cce..4503756 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +bincode = "1.3.3" chrono = { version = "0.4.38", features = ["std", "clock"] } clap = { version = "4.5.4", features = ["derive"] } clap_complete = "4.5.2" @@ -13,7 +14,6 @@ ctrlc = { version = "3.4.4", features = ["termination"] } jrsonnet-evaluator = "0.4.2" log = { version = "0.4.22", features = ["std"] } num_cpus = "1.16.0" -postcard = { version = "1.0.10", features = ["use-std"] } regex = "1.10.4" serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" diff --git a/rust/datasize.jsonnet b/rust/datasize.jsonnet new file mode 100644 index 0000000..2c842df --- /dev/null +++ b/rust/datasize.jsonnet @@ -0,0 +1,30 @@ +{ + patterns: { + num: { + regex: @'([0-9]+)', + }, + }, + streams: { + s1: { + cmd: ['seq', '-w', '499999'], + filters: { + f1: { + regex: [ + '^$', + ], + retry: 10, + retryperiod: '1m', + actions: { + a: { + cmd: ['true'], + }, + b: { + cmd: ['true'], + after: '1m', + }, + }, + }, + }, + }, + }, +} diff --git a/rust/src/daemon/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs index f6af9ae..7ed900d 100644 --- a/rust/src/daemon/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -6,9 +6,10 @@ use std::{ process::exit, }; +use bincode::Options; use chrono::{DateTime, Local}; use log::{debug, error, warn}; -use serde::{Deserialize, Serialize}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::concepts::{Config, Filter, LogEntry, Match}; @@ -22,13 +23,20 @@ type DatabaseHeader = BTreeMap; type ReadHeader = BTreeMap; type WriteHeader = BTreeMap<&'static Filter, usize>; -const BUFFER_MAX_SIZE: usize = 10 * 1024 * 1024; +type BinOptions = bincode::config::WithOtherIntEncoding< + bincode::config::DefaultOptions, + bincode::config::VarintEncoding, +>; +fn bin_options() -> BinOptions { + bincode::DefaultOptions::new().with_varint_encoding() +} + const DB_SIGNATURE: &str = "reaction-db-v01"; pub struct ReadDB { f: BufReader, h: ReadHeader, - buf: Vec, + bin: BinOptions, } impl ReadDB { @@ -51,13 +59,18 @@ impl ReadDB { let mut ret = ReadDB { f: BufReader::new(file), - h: BTreeMap::new(), - buf: vec![0; BUFFER_MAX_SIZE], + h: BTreeMap::default(), + bin: bin_options(), }; - match ret.read::<&str>() { - Ok(DB_SIGNATURE) => Ok(()), - Ok(_) => Err(DBError::Error("database is not a reaction database".into())), + match ret.read::() { + Ok(signature) => { + if DB_SIGNATURE == &signature { + Ok(()) + } else { + Err(DBError::Error("database is not a reaction database".into())) + } + } Err(err) => Err(DBError::Error(format!("reading database signature: {err}"))), }?; @@ -73,8 +86,8 @@ impl ReadDB { Ok(Some(ret)) } - fn read<'a, T: Deserialize<'a> + Debug>(&'a mut self) -> Result { - let (decoded, _) = postcard::from_io::((&mut self.f, &mut self.buf))?; + fn read(&mut self) -> Result { + let decoded = self.bin.deserialize_from::<_, T>(&mut self.f)?; debug!("reading this: {:?}", &decoded); Ok(decoded) } @@ -84,14 +97,15 @@ impl Iterator for ReadDB { type Item = Result; fn next(&mut self) -> Option { - match self.read::() { - // FIXME why we got a default item instead of an error or something? - // How do we really know we reached the end? - // For now, checking if time is 0 - Ok(ComputedLogEntry { t: 0, .. }) => None, + let res = self.read::(); + debug!("{res:?}"); + match res { Ok(item) => Some(item.to(&self.h)), - Err(err) => match err { - postcard::Error::DeserializeUnexpectedEnd => None, + Err(err) => match *err { + bincode::ErrorKind::Io(err) => match err.kind() { + io::ErrorKind::UnexpectedEof => None, + _ => Some(Err(err.into())), + }, _ => Some(Err(err.into())), }, } @@ -101,7 +115,7 @@ impl Iterator for ReadDB { pub struct WriteDB { f: BufWriter, h: WriteHeader, - buf: Vec, + bin: BinOptions, } impl WriteDB { @@ -116,8 +130,8 @@ impl WriteDB { let mut ret = WriteDB { f: BufWriter::new(file), - h: BTreeMap::new(), - buf: vec![0; BUFFER_MAX_SIZE], + h: BTreeMap::default(), + bin: bin_options(), }; if let Err(err) = ret._write(DB_SIGNATURE) { @@ -153,13 +167,9 @@ impl WriteDB { } fn _write(&mut self, data: T) -> Result<(), DBError> { - let encoded = postcard::to_slice(&data, &mut self.buf)?; + let encoded = self.bin.serialize(&data)?; debug!("writing this: {:?}, {:?}", &data, &encoded); - self.f.write_all(encoded)?; - // clear - // for i in 0..self.buf.len() { - // self.buf[i] = 0; - // } + self.f.write_all(&encoded)?; Ok(()) } diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index 0552ece..1049fa6 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -30,7 +30,7 @@ pub enum DBError { #[error("invalid filter: {0}")] InvalidFilterError(String), #[error("decode error: {0}")] - PostcardError(#[from] postcard::Error), + BincodeError(#[from] bincode::Error), #[error("io error: {0}")] IOError(#[from] io::Error), #[error("{0}")] From 6170f2dcd22fa6049ad3528967e7bd182227b84a Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 26 Sep 2024 12:00:00 +0200 Subject: [PATCH 048/435] WIP impl of socket, daemon-side --- rust/src/concepts/action.rs | 8 ++ rust/src/concepts/filter.rs | 21 +++- rust/src/concepts/mod.rs | 2 + rust/src/concepts/socket_messages.rs | 33 ++++++ rust/src/daemon/database/lowlevel.rs | 24 ++-- rust/src/daemon/database/mod.rs | 2 - rust/src/daemon/execs.rs | 46 ++++++-- rust/src/daemon/matches.rs | 22 +++- rust/src/daemon/mod.rs | 15 ++- rust/src/daemon/socket.rs | 164 +++++++++++++++++++++++++++ rust/src/main.rs | 4 +- rust/src/utils/mod.rs | 9 ++ 12 files changed, 308 insertions(+), 42 deletions(-) create mode 100644 rust/src/concepts/socket_messages.rs create mode 100644 rust/src/daemon/socket.rs diff --git a/rust/src/concepts/action.rs b/rust/src/concepts/action.rs index 1e69f42..ec8ea63 100644 --- a/rust/src/concepts/action.rs +++ b/rust/src/concepts/action.rs @@ -43,6 +43,14 @@ impl Action { self.on_exit } + pub fn full_name(&self) -> (String, String, String) { + ( + self.stream_name.clone(), + self.filter_name.clone(), + self.name.clone(), + ) + } + pub fn setup( &mut self, stream_name: &str, diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index a7e3257..60dc99c 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -71,7 +71,6 @@ impl Filter { self.longuest_action_duration } - #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used pub fn patterns(&self) -> &BTreeSet> { &self.patterns } @@ -122,7 +121,6 @@ impl Filter { return Err("no regex configured".into()); } - #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used let mut new_patterns = BTreeSet::new(); let mut first = true; for regex in &self.regex { @@ -204,14 +202,25 @@ impl Filter { None } - pub fn send_actions(&'static self, m: &Match, t: Time, tx: &Sender) { + pub fn send_actions( + &'static self, + m: &Match, + t: Time, + tx: &Sender, + exec: bool, + ) { for action in self.actions.values() { - #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(ExecsManagerInput::Exec(MAT { + let mat = MAT { m: m.clone(), a: action, t: t + action.after_duration().unwrap_or_default(), - })) + }; + #[allow(clippy::unwrap_used)] // propagating panics is ok + tx.send(if exec { + ExecsManagerInput::Exec(mat) + } else { + ExecsManagerInput::Flush(mat) + }) .unwrap(); } } diff --git a/rust/src/concepts/mod.rs b/rust/src/concepts/mod.rs index 4c237d3..e51511c 100644 --- a/rust/src/concepts/mod.rs +++ b/rust/src/concepts/mod.rs @@ -3,6 +3,7 @@ mod config; mod filter; mod messages; mod pattern; +mod socket_messages; mod stream; pub use action::*; @@ -10,4 +11,5 @@ pub use config::*; pub use filter::*; pub use messages::*; pub use pattern::*; +pub use socket_messages::*; pub use stream::*; diff --git a/rust/src/concepts/socket_messages.rs b/rust/src/concepts/socket_messages.rs new file mode 100644 index 0000000..c4be743 --- /dev/null +++ b/rust/src/concepts/socket_messages.rs @@ -0,0 +1,33 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use super::Match; + +use serde::{Deserialize, Serialize}; + +// We don't need protocol versionning here because +// client and daemon are the same binary + +#[derive(Clone, Serialize, Deserialize)] +pub enum ClientRequest { + Info, + Flush(FlushOpts), +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct FlushOpts { + pub m: Match, + pub f: (String, String), +} + +#[derive(Clone, Serialize, Deserialize)] +pub enum DaemonResponse { + Info(InfoRes), + Flush, + Err(String), +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct InfoRes { + pub matches: BTreeMap<(String, String), BTreeMap>>, + pub execs: BTreeMap<(String, String, String), BTreeMap>>, +} diff --git a/rust/src/daemon/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs index 7ed900d..d057133 100644 --- a/rust/src/daemon/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -11,7 +11,10 @@ use chrono::{DateTime, Local}; use log::{debug, error, warn}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use crate::concepts::{Config, Filter, LogEntry, Match}; +use crate::{ + concepts::{Config, Filter, LogEntry, Match}, + utils::{bincode_options, BincodeOptions}, +}; use super::DBError; @@ -23,20 +26,12 @@ type DatabaseHeader = BTreeMap; type ReadHeader = BTreeMap; type WriteHeader = BTreeMap<&'static Filter, usize>; -type BinOptions = bincode::config::WithOtherIntEncoding< - bincode::config::DefaultOptions, - bincode::config::VarintEncoding, ->; -fn bin_options() -> BinOptions { - bincode::DefaultOptions::new().with_varint_encoding() -} - const DB_SIGNATURE: &str = "reaction-db-v01"; pub struct ReadDB { f: BufReader, h: ReadHeader, - bin: BinOptions, + bin: BincodeOptions, } impl ReadDB { @@ -60,12 +55,12 @@ impl ReadDB { let mut ret = ReadDB { f: BufReader::new(file), h: BTreeMap::default(), - bin: bin_options(), + bin: bincode_options(), }; match ret.read::() { Ok(signature) => { - if DB_SIGNATURE == &signature { + if DB_SIGNATURE == signature { Ok(()) } else { Err(DBError::Error("database is not a reaction database".into())) @@ -115,7 +110,7 @@ impl Iterator for ReadDB { pub struct WriteDB { f: BufWriter, h: WriteHeader, - bin: BinOptions, + bin: BincodeOptions, } impl WriteDB { @@ -131,7 +126,7 @@ impl WriteDB { let mut ret = WriteDB { f: BufWriter::new(file), h: BTreeMap::default(), - bin: bin_options(), + bin: bincode_options(), }; if let Err(err) = ret._write(DB_SIGNATURE) { @@ -187,7 +182,6 @@ struct ComputedLogEntry { } impl ComputedLogEntry { - #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used fn from(value: LogEntry, header: &WriteHeader) -> Result { match header.get(&value.f) { Some(f) => Ok(ComputedLogEntry { diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index 1049fa6..878750d 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -40,7 +40,6 @@ pub enum DBError { #[derive(Clone)] pub enum DatabaseManagerInput { Log(LogEntry), - #[allow(dead_code)] Flush(LogEntry), } @@ -192,7 +191,6 @@ fn __rotate_db( let mut millisecond_disambiguation_counter: u32 = 0; // Read flushes - #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used let mut flushes: BTreeMap<&'static Filter, BTreeMap> = BTreeMap::new(); for flush_entry in flush_read_db { match flush_entry { diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs index 46fb987..6e1afe7 100644 --- a/rust/src/daemon/execs.rs +++ b/rust/src/daemon/execs.rs @@ -1,15 +1,15 @@ use std::{ collections::{BTreeMap, BTreeSet}, process::Stdio, - sync::mpsc::{Receiver, Sender}, + sync::mpsc::{Receiver, Sender, SyncSender}, }; -use chrono::{DateTime, Local}; +use chrono::Local; use log::{error, info}; use timer::MessageTimer; use crate::{ - concepts::{Action, Config, Match, MAT}, + concepts::{Action, Config, Match, Time, MAT}, utils::ThreadPool, }; @@ -17,16 +17,17 @@ use crate::{ pub enum ExecsManagerInput { Exec(MAT), ExecPending(MAT), - #[allow(dead_code)] Flush(MAT), + Gimme(SyncSender), Stop, } -type ExecsMap = BTreeMap<&'static Action, BTreeMap>>>; +type ExecsMap = BTreeMap<&'static Action, BTreeMap>>; trait ExecsMapTrait { fn add(&mut self, mat: &MAT); - fn rm(&mut self, mat: &MAT); + fn rm(&mut self, mat: &MAT) -> bool; + fn rm_times(&mut self, mat: &MAT) -> Option>; } impl ExecsMapTrait for ExecsMap { fn add(&mut self, mat: &MAT) { @@ -35,10 +36,12 @@ impl ExecsMapTrait for ExecsMap { inner_set.insert(mat.t); } - fn rm(&mut self, mat: &MAT) { + fn rm(&mut self, mat: &MAT) -> bool { + let mut removed = false; if let Some(inner_map) = self.get_mut(&mat.a) { if let Some(inner_set) = inner_map.get_mut(&mat.m) { inner_set.remove(&mat.t); + removed = true; if inner_set.is_empty() { inner_map.remove(&mat.m); } @@ -47,6 +50,18 @@ impl ExecsMapTrait for ExecsMap { self.remove(&mat.a); } } + removed + } + + fn rm_times(&mut self, mat: &MAT) -> Option> { + let mut set = None; + if let Some(inner_map) = self.get_mut(&mat.a) { + set = inner_map.remove(&mat.m); + if inner_map.is_empty() { + self.remove(&mat.a); + } + } + set } } @@ -91,7 +106,6 @@ pub fn execs_manager( } }; - #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used let mut execs: ExecsMap = BTreeMap::new(); let timer = MessageTimer::new(exec_tx); @@ -110,11 +124,19 @@ pub fn execs_manager( } } ExecsManagerInput::ExecPending(mat) => { - execs.rm(&mat); - exec_now(mat); + if execs.rm(&mat) { + exec_now(mat); + } } - #[allow(clippy::todo)] - ExecsManagerInput::Flush(_mat) => todo!(), + ExecsManagerInput::Flush(mat) => { + if let Some(set) = execs.rm_times(&mat) { + for _ in set { + exec_now(mat.clone()); + } + } + } + #[allow(clippy::unwrap_used)] // propagating panics is ok + ExecsManagerInput::Gimme(tx) => tx.send(execs.clone()).unwrap(), ExecsManagerInput::Stop => { for (action, inner_map) in execs { if action.on_exit() { diff --git a/rust/src/daemon/matches.rs b/rust/src/daemon/matches.rs index 406c79c..45cd129 100644 --- a/rust/src/daemon/matches.rs +++ b/rust/src/daemon/matches.rs @@ -16,8 +16,8 @@ use crate::{ pub enum MatchManagerInput { Match(MFT), Unmatch(MFT), - #[allow(dead_code)] Flush(MFT), + Gimme(SyncSender), EndOfStartup, Stop, } @@ -75,7 +75,6 @@ pub fn matches_manager( action_tx: Sender, log_tx: SyncSender, ) { - #[allow(clippy::mutable_key_type)] // Interior mutability of Arc is not used let mut matches: MatchesMap = BTreeMap::new(); let timer = MessageTimer::new(match_tx); @@ -114,7 +113,7 @@ pub fn matches_manager( if mft.f.retry().is_some() { matches.rm_times(&mft); } - mft.f.send_actions(&mft.m, mft.t, &action_tx); + mft.f.send_actions(&mft.m, mft.t, &action_tx, true); } if !startup { @@ -131,7 +130,22 @@ pub fn matches_manager( } MatchManagerInput::Unmatch(mft) => matches.rm(&mft), #[allow(clippy::todo)] - MatchManagerInput::Flush(_) => todo!(), // TODO handle flushes + MatchManagerInput::Flush(mft) => { + // remove from matches + matches.rm_times(&mft); + // send to DB + #[allow(clippy::unwrap_used)] // propagating panics is ok + log_tx + .send(DatabaseManagerInput::Flush(LogEntry { + exec: false, + m: mft.m, + f: mft.f, + t: mft.t, + })) + .unwrap(); + } + #[allow(clippy::unwrap_used)] // propagating panics is ok + MatchManagerInput::Gimme(tx) => tx.send(matches.clone()).unwrap(), MatchManagerInput::Stop => break, } } diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index e6dd19e..5a1d66e 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -1,4 +1,5 @@ use std::{ + fs, path::Path, process::exit, sync::{ @@ -10,6 +11,7 @@ use std::{ }; use log::{error, info, Level}; +use socket::socket_manager; use crate::{concepts::Config, utils::SimpleLogger}; use database::database_manager; @@ -22,6 +24,7 @@ pub use matches::MatchManagerInput; mod database; mod execs; mod matches; +mod socket; #[allow(unused_variables)] pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { @@ -70,6 +73,13 @@ pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { database_manager(config, log_rx, match_tx_database) }; + { + let match_tx_socket = match_tx.clone(); + let exec_tx_socket = exec_tx.clone(); + let socket = socket.to_owned(); + thread::spawn(move || socket_manager(config, socket, match_tx_socket, exec_tx_socket)); + } + for stream in config.streams().values() { let match_tx = match_tx.clone(); let (child_tx, child_rx) = sync_channel(0); @@ -116,7 +126,10 @@ pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { let stop_ok = config.stop(); - // TODO remove socket + // not waiting for the socket_manager to finish, sorry + if let Err(err) = fs::remove_file(socket) { + error!("failed to remove socket: {}", err); + } exit(match !signal_received.load(Ordering::SeqCst) && stop_ok { true => 0, diff --git a/rust/src/daemon/socket.rs b/rust/src/daemon/socket.rs new file mode 100644 index 0000000..fa28632 --- /dev/null +++ b/rust/src/daemon/socket.rs @@ -0,0 +1,164 @@ +use std::{ + fs, io, + os::unix::net::{UnixListener, UnixStream}, + path::PathBuf, + process::exit, + sync::mpsc::{sync_channel, Sender}, +}; + +use bincode::Options; +use chrono::Local; +use log::{error, warn}; + +use crate::{ + concepts::{ClientRequest, Config, DaemonResponse, InfoRes, MFT}, + utils::bincode_options, +}; + +use super::{ExecsManagerInput, MatchManagerInput}; + +macro_rules! err_str { + ($expression:expr) => { + $expression.map_err(|err| err.to_string()) + }; +} + +fn open_socket(path: PathBuf) -> Result { + // First create all directories to the file + let dir = path + .parent() + .ok_or(format!("socket {path:?} has no parent directory"))?; + err_str!(fs::create_dir_all(dir))?; + // Test if file exists + match fs::metadata(&path) { + Ok(meta) => { + if meta.file_type().is_dir() { + Err(format!("socket {path:?} is already a directory")) + } else { + warn!("socket {path:?} already exists: is the daemon already running? deleting."); + err_str!(fs::remove_file(&path)) + } + } + Err(err) => err_str!(match err.kind() { + io::ErrorKind::NotFound => Ok(()), + _ => Err(err), + }), + }?; + // Open socket + err_str!(UnixListener::bind(path)) +} + +macro_rules! or_next { + ($msg:expr, $expression:expr) => { + match $expression { + Ok(x) => x, + Err(err) => { + error!("failed to answer client: {}, {}", $msg, err); + continue; + } + } + }; +} + +pub fn socket_manager( + config: &'static Config, + socket: PathBuf, + match_tx: Sender, + exec_tx: Sender, +) { + let listener = match open_socket(socket) { + Ok(l) => l, + Err(err) => { + error!("while creating communication socket: {err}"); + exit(1); + } + }; + + let bin = bincode_options(); + for try_conn in listener.incoming() { + match try_conn { + Ok(conn) => { + let conn2 = or_next!("failed to clone stream", conn.try_clone()); + // read request + let request = or_next!( + "invalid message received: ", + bin.deserialize_from::(conn) + ); + let response = match request { + ClientRequest::Info => { + // ask for matches clone + let (m_tx, m_rx) = sync_channel(0); + #[allow(clippy::unwrap_used)] // propagating panics is ok + match_tx.send(MatchManagerInput::Gimme(m_tx)).unwrap(); + #[allow(clippy::unwrap_used)] // propagating panics is ok + let matches = m_rx.recv().unwrap(); + + // ask for execs clone + let (e_tx, e_rx) = sync_channel(0); + #[allow(clippy::unwrap_used)] // propagating panics is ok + exec_tx.send(ExecsManagerInput::Gimme(e_tx)).unwrap(); + #[allow(clippy::unwrap_used)] // propagating panics is ok + let execs = e_rx.recv().unwrap(); + + // Transform structures + macro_rules! map_map { + ($map:expr) => { + $map.into_iter() + .map(|(object, inner_map)| { + ( + object.full_name(), + inner_map + .into_iter() + .map(|(key, set)| { + ( + key, + set.into_iter() + .map(|time| time.timestamp()) + .collect(), + ) + }) + .collect(), + ) + }) + .collect() + }; + } + + DaemonResponse::Info(InfoRes { + matches: map_map!(matches), + execs: map_map!(execs), + }) + } + ClientRequest::Flush(flush) => { + match config.get_filter(&flush.f) { + Some(filter) => { + let now = Local::now(); + // Flush actions + filter.send_actions(&flush.m, now, &exec_tx, false); + // Flush filters + #[allow(clippy::unwrap_used)] // propagating panics is ok + match_tx + .send(MatchManagerInput::Flush(MFT { + m: flush.m, + f: filter, + t: now, + })) + .unwrap(); + DaemonResponse::Flush + } + None => DaemonResponse::Err(format!( + "no filter with name {}.{}", + flush.f.0, flush.f.1 + )), + } + } + }; + or_next!( + "failed to send response:", + bin.serialize_into(conn2, &response) + ); + } + Err(err) => error!("failed to open connection from cli: {err}"), + } + } +} diff --git a/rust/src/main.rs b/rust/src/main.rs index a435132..01fc447 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -3,9 +3,9 @@ clippy::panic, clippy::todo, clippy::unimplemented, - clippy::unwrap_used, + clippy::unwrap_used )] -#![allow(clippy::upper_case_acronyms)] +#![allow(clippy::upper_case_acronyms, clippy::mutable_key_type)] #![forbid(unsafe_code)] //! TODO document a bit diff --git a/rust/src/utils/mod.rs b/rust/src/utils/mod.rs index 46debbe..49c2207 100644 --- a/rust/src/utils/mod.rs +++ b/rust/src/utils/mod.rs @@ -3,6 +3,15 @@ pub mod logger; mod parse_duration; mod threadpool; +use bincode::Options; pub use logger::SimpleLogger; pub use parse_duration::parse_duration; pub use threadpool::ThreadPool; + +pub type BincodeOptions = bincode::config::WithOtherIntEncoding< + bincode::config::DefaultOptions, + bincode::config::VarintEncoding, +>; +pub fn bincode_options() -> BincodeOptions { + bincode::DefaultOptions::new().with_varint_encoding() +} From ce0c34de8e6529d69aae437922defd29188cf861 Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 1 Oct 2024 12:00:00 +0200 Subject: [PATCH 049/435] Socket communication! --- rust/src/client/mod.rs | 100 +++++++++---- rust/src/concepts/action.rs | 24 +-- rust/src/concepts/config.rs | 4 + rust/src/concepts/filter.rs | 37 ++--- rust/src/concepts/messages.rs | 30 ++-- rust/src/concepts/mod.rs | 7 + rust/src/concepts/socket_messages.rs | 61 ++++++-- rust/src/concepts/stream.rs | 2 +- rust/src/daemon/database/lowlevel.rs | 9 +- rust/src/daemon/database/mod.rs | 2 +- rust/src/daemon/execs.rs | 107 +++++++------- rust/src/daemon/matches.rs | 123 +++++++--------- rust/src/daemon/mod.rs | 14 +- rust/src/daemon/socket.rs | 212 +++++++++++++++++---------- rust/src/daemon/statemap.rs | 123 ++++++++++++++++ rust/src/main.rs | 13 +- rust/src/utils/cli.rs | 33 +---- rust/test.jsonnet | 6 +- 18 files changed, 582 insertions(+), 325 deletions(-) create mode 100644 rust/src/daemon/statemap.rs diff --git a/rust/src/client/mod.rs b/rust/src/client/mod.rs index ffc1b65..91aa1a1 100644 --- a/rust/src/client/mod.rs +++ b/rust/src/client/mod.rs @@ -1,36 +1,80 @@ -use std::path::PathBuf; +use std::{error::Error, io::stdout, os::unix::net::UnixStream, path::PathBuf, process::exit}; -use log::debug; -use regex::Regex; +use bincode::Options; +use log::{debug, error, Level}; -use crate::utils::cli::{Format, NamedRegex}; - -pub fn show( - socket: &PathBuf, - format: Format, - limit: &Option, - pattern: &Option, - patterns: &Vec, -) { - debug!( - "show {:?} {:?} {:?} {:?} {:?}", - socket, format, limit, pattern, patterns - ); +use crate::{ + concepts::{ClientRequest, ClientStatus, DaemonResponse, Order}, + utils::{bincode_options, cli::Format, SimpleLogger}, +}; +macro_rules! or_quit { + ($msg:expr, $expression:expr) => { + match $expression { + Ok(x) => x, + Err(err) => { + error!("failed to communicate to daemon: {}, {}", $msg, err); + exit(1); + } + } + }; } -pub fn flush( - socket: &PathBuf, - format: Format, - limit: &Option, - pattern: &Option, - patterns: &Vec, -) { - debug!( - "flush {:?} {:?} {:?} {:?} {:?}", - socket, format, limit, pattern, patterns - ); +fn send_retrieve(socket: &PathBuf, req: &ClientRequest) -> DaemonResponse { + let bin = bincode_options(); + let conn = or_quit!("opening connection to daemon", UnixStream::connect(socket)); + let conn2 = or_quit!("failed to clone stream", conn.try_clone()); + or_quit!("failed to send request", bin.serialize_into(conn, req)); + or_quit!( + "failed to send request", + bin.deserialize_from::(conn2) + ) } -pub fn test_regex(config_path: &PathBuf, regex: &String, line: &Option) { +fn print_status(cs: ClientStatus, format: Format) -> Result<(), Box> { + Ok(match format { + Format::JSON => serde_json::to_writer(stdout().lock(), &cs)?, + Format::YAML => serde_yaml::to_writer(stdout().lock(), &cs)?, + }) +} + +pub fn request( + socket: PathBuf, + format: Format, + stream_filter: Option, + patterns: Vec<(String, String)>, + order: Order, +) { + if let Err(err) = SimpleLogger::init(Level::Debug) { + eprintln!("ERROR could not initialize logging: {err}"); + exit(1); + } + + let response = send_retrieve( + &socket, + &ClientRequest { + order, + stream_filter, + patterns, + }, + ); + match response { + DaemonResponse::Order(cs) => { + if let Err(err) = print_status(cs, format) { + error!("while printing response: {err}"); + exit(1); + } + } + DaemonResponse::Err(err) => { + error!("failed to communicate to daemon: error response: {err}"); + exit(1); + } + } +} + +pub fn test_regex(config_path: PathBuf, regex: String, line: Option) { + if let Err(err) = SimpleLogger::init(Level::Debug) { + eprintln!("ERROR could not initialize logging: {err}"); + exit(1); + } debug!("test-regex {:?} {:?} {:?} ", config_path, regex, line); } diff --git a/rust/src/concepts/action.rs b/rust/src/concepts/action.rs index ec8ea63..6d1561d 100644 --- a/rust/src/concepts/action.rs +++ b/rust/src/concepts/action.rs @@ -4,7 +4,7 @@ use chrono::TimeDelta; use serde::Deserialize; -use super::{Match, Pattern}; +use super::{ActionFilter, Match, Pattern}; use crate::utils::parse_duration; #[derive(Clone, Debug, Deserialize)] @@ -34,6 +34,20 @@ fn set_false() -> bool { false } +impl ActionFilter for Action { + fn patterns(&self) -> &BTreeSet> { + &self.patterns + } + + fn full_name<'a>(&'a self) -> (&'a str, &'a str, &'a str) { + ( + &self.stream_name, + &self.filter_name, + &self.name, + ) + } +} + impl Action { pub fn after_duration(&self) -> Option { self.after_duration @@ -43,14 +57,6 @@ impl Action { self.on_exit } - pub fn full_name(&self) -> (String, String, String) { - ( - self.stream_name.clone(), - self.filter_name.clone(), - self.name.clone(), - ) - } - pub fn setup( &mut self, stream_name: &str, diff --git a/rust/src/concepts/config.rs b/rust/src/concepts/config.rs index 61cf401..08c86b7 100644 --- a/rust/src/concepts/config.rs +++ b/rust/src/concepts/config.rs @@ -36,6 +36,10 @@ impl Config { &self.streams } + pub fn patterns(&self) -> &Patterns { + &self.patterns + } + pub fn concurrency(&self) -> usize { self.concurrency } diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index 60dc99c..61aae3e 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -12,7 +12,7 @@ use serde::Deserialize; use super::{ messages::{Match, Time, MAT}, - Action, Pattern, Patterns, + Action, ActionFilter, Pattern, Patterns, }; use crate::{daemon::ExecsManagerInput, utils::parse_duration}; @@ -45,6 +45,16 @@ pub struct Filter { stream_name: String, } +impl ActionFilter for Filter { + fn full_name<'a>(&'a self) -> (&'a str, &'a str, &'a str) { + (self.stream_name.as_ref(), self.name.as_ref(), "") + } + + fn patterns(&self) -> &BTreeSet> { + &self.patterns + } +} + impl Filter { #[cfg(test)] pub fn from_name(stream_name: &str, filter_name: &str) -> Filter { @@ -55,10 +65,6 @@ impl Filter { } } - pub fn full_name(&self) -> (String, String) { - (self.stream_name.clone(), self.name.clone()) - } - pub fn retry(&self) -> Option { self.retry } @@ -71,10 +77,6 @@ impl Filter { self.longuest_action_duration } - pub fn patterns(&self) -> &BTreeSet> { - &self.patterns - } - pub fn setup( &mut self, stream_name: &str, @@ -202,26 +204,15 @@ impl Filter { None } - pub fn send_actions( - &'static self, - m: &Match, - t: Time, - tx: &Sender, - exec: bool, - ) { + pub fn send_actions(&'static self, m: &Match, t: Time, tx: &Sender) { for action in self.actions.values() { let mat = MAT { m: m.clone(), - a: action, + o: action, t: t + action.after_duration().unwrap_or_default(), }; #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(if exec { - ExecsManagerInput::Exec(mat) - } else { - ExecsManagerInput::Flush(mat) - }) - .unwrap(); + tx.send(ExecsManagerInput::Exec(mat)).unwrap(); } } } diff --git a/rust/src/concepts/messages.rs b/rust/src/concepts/messages.rs index f4541bf..5a9b1c7 100644 --- a/rust/src/concepts/messages.rs +++ b/rust/src/concepts/messages.rs @@ -1,23 +1,33 @@ use chrono::{DateTime, Local, TimeDelta}; -use super::{Action, Filter}; +use super::{Action, ActionFilter, Filter}; pub type Time = DateTime; pub type Match = Vec; #[derive(Clone)] -pub struct MFT { +pub struct MT { pub m: Match, - pub f: &'static Filter, + pub o: &'static T, pub t: Time, } -#[derive(Clone)] -pub struct MAT { - pub m: Match, - pub a: &'static Action, - pub t: Time, -} +pub type MFT = MT; +pub type MAT = MT; + +// #[derive(Clone)] +// pub struct MFT { +// pub m: Match, +// pub f: &'static Filter, +// pub t: Time, +// } + +// #[derive(Clone)] +// pub struct MAT { +// pub m: Match, +// pub a: &'static Action, +// pub t: Time, +// } #[derive(Clone, Debug)] pub struct LogEntry { @@ -42,7 +52,7 @@ impl From for MFT { fn from(value: LogEntry) -> Self { MFT { m: value.m, - f: value.f, + o: value.f, t: value.t, } } diff --git a/rust/src/concepts/mod.rs b/rust/src/concepts/mod.rs index e51511c..c0a87b1 100644 --- a/rust/src/concepts/mod.rs +++ b/rust/src/concepts/mod.rs @@ -6,6 +6,8 @@ mod pattern; mod socket_messages; mod stream; +use std::{collections::BTreeSet, fmt::Display, sync::Arc}; + pub use action::*; pub use config::*; pub use filter::*; @@ -13,3 +15,8 @@ pub use messages::*; pub use pattern::*; pub use socket_messages::*; pub use stream::*; + +pub trait ActionFilter: Clone + Display + PartialEq + Eq + PartialOrd + Ord { + fn patterns(&self) -> &BTreeSet>; + fn full_name<'a>(&'a self) -> (&'a str, &'a str, &'a str); +} diff --git a/rust/src/concepts/socket_messages.rs b/rust/src/concepts/socket_messages.rs index c4be743..b0d1088 100644 --- a/rust/src/concepts/socket_messages.rs +++ b/rust/src/concepts/socket_messages.rs @@ -2,27 +2,33 @@ use std::collections::{BTreeMap, BTreeSet}; use super::Match; -use serde::{Deserialize, Serialize}; +use serde::{ser::SerializeStruct, Deserialize, Serialize}; // We don't need protocol versionning here because // client and daemon are the same binary -#[derive(Clone, Serialize, Deserialize)] -pub enum ClientRequest { - Info, - Flush(FlushOpts), +#[derive(Copy, Clone, Serialize, Deserialize)] +pub enum Order { + Show, + Flush, } #[derive(Clone, Serialize, Deserialize)] -pub struct FlushOpts { +pub struct ClientRequest { + pub order: Order, + pub stream_filter: Option, + pub patterns: Vec<(String, String)>, +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct FlushOptions { pub m: Match, pub f: (String, String), } -#[derive(Clone, Serialize, Deserialize)] +#[derive(Serialize, Deserialize)] pub enum DaemonResponse { - Info(InfoRes), - Flush, + Order(ClientStatus), Err(String), } @@ -31,3 +37,40 @@ pub struct InfoRes { pub matches: BTreeMap<(String, String), BTreeMap>>, pub execs: BTreeMap<(String, String, String), BTreeMap>>, } + +pub type ClientStatus = BTreeMap>>; + +#[derive(Debug, Default, Deserialize)] +pub struct PatternStatus { + pub matches: usize, + pub actions: BTreeMap>, +} + +impl Serialize for PatternStatus { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + // We only skip serializing emptiness if we're on a human-readable format + // This means we're printing for user, not exchanging it over a socket + let state = if serializer.is_human_readable() { + let ser_matches = self.matches != 0; + let ser_actions = self.actions.len() != 0; + let mut state = serializer + .serialize_struct("PatternStatus", ser_matches as usize + ser_actions as usize)?; + if ser_matches { + state.serialize_field("matches", &self.matches)?; + } + if ser_actions { + state.serialize_field("actions", &self.actions)?; + } + state + } else { + let mut state = serializer.serialize_struct("PatternStatus", 2)?; + state.serialize_field("matches", &self.matches)?; + state.serialize_field("actions", &self.actions)?; + state + }; + state.end() + } +} diff --git a/rust/src/concepts/stream.rs b/rust/src/concepts/stream.rs index 2971a2c..d1fa6bd 100644 --- a/rust/src/concepts/stream.rs +++ b/rust/src/concepts/stream.rs @@ -106,7 +106,7 @@ impl Stream { match_tx .send(MatchManagerInput::Match(MFT { m: match_, - f: filter, + o: filter, t: Local::now(), })) .unwrap(); diff --git a/rust/src/daemon/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs index d057133..7172156 100644 --- a/rust/src/daemon/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -12,7 +12,7 @@ use log::{debug, error, warn}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::{ - concepts::{Config, Filter, LogEntry, Match}, + concepts::{ActionFilter, Config, Filter, LogEntry, Match}, utils::{bincode_options, BincodeOptions}, }; @@ -137,7 +137,10 @@ impl WriteDB { let database_header: DatabaseHeader = config .filters() .into_iter() - .map(|f| f.full_name()) + .map(|f| { + let names = f.full_name(); + (names.0.to_owned(), names.1.to_owned()) + }) .enumerate() .collect(); @@ -163,7 +166,7 @@ impl WriteDB { fn _write(&mut self, data: T) -> Result<(), DBError> { let encoded = self.bin.serialize(&data)?; - debug!("writing this: {:?}, {:?}", &data, &encoded); + // debug!("writing this: {:?}, {:?}", &data, &encoded); self.f.write_all(&encoded)?; Ok(()) } diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index 878750d..d0bed01 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -12,7 +12,7 @@ use log::{error, info, warn}; use thiserror::Error; use super::MatchManagerInput; -use crate::concepts::{Config, Filter, LogEntry, Match, Time}; +use crate::concepts::{ActionFilter, Config, Filter, LogEntry, Match, Time}; mod lowlevel; mod tests; diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs index 6e1afe7..91793e4 100644 --- a/rust/src/daemon/execs.rs +++ b/rust/src/daemon/execs.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, BTreeSet}, + collections::BTreeMap, process::Stdio, sync::mpsc::{Receiver, Sender, SyncSender}, }; @@ -9,66 +9,30 @@ use log::{error, info}; use timer::MessageTimer; use crate::{ - concepts::{Action, Config, Match, Time, MAT}, + concepts::{Action, ActionFilter, Config, LogEntry, Order, MAT}, utils::ThreadPool, }; +use super::{ + database::DatabaseManagerInput, + statemap::{FilterOptions, StateMap, StateMapTrait}, +}; + #[derive(Clone)] pub enum ExecsManagerInput { Exec(MAT), ExecPending(MAT), - Flush(MAT), - Gimme(SyncSender), + Order(Order, FilterOptions, SyncSender), Stop, } -type ExecsMap = BTreeMap<&'static Action, BTreeMap>>; - -trait ExecsMapTrait { - fn add(&mut self, mat: &MAT); - fn rm(&mut self, mat: &MAT) -> bool; - fn rm_times(&mut self, mat: &MAT) -> Option>; -} -impl ExecsMapTrait for ExecsMap { - fn add(&mut self, mat: &MAT) { - let inner_map = self.entry(mat.a).or_default(); - let inner_set = inner_map.entry(mat.m.clone()).or_default(); - inner_set.insert(mat.t); - } - - fn rm(&mut self, mat: &MAT) -> bool { - let mut removed = false; - if let Some(inner_map) = self.get_mut(&mat.a) { - if let Some(inner_set) = inner_map.get_mut(&mat.m) { - inner_set.remove(&mat.t); - removed = true; - if inner_set.is_empty() { - inner_map.remove(&mat.m); - } - } - if inner_map.is_empty() { - self.remove(&mat.a); - } - } - removed - } - - fn rm_times(&mut self, mat: &MAT) -> Option> { - let mut set = None; - if let Some(inner_map) = self.get_mut(&mat.a) { - set = inner_map.remove(&mat.m); - if inner_map.is_empty() { - self.remove(&mat.a); - } - } - set - } -} +type ExecsMap = StateMap; pub fn execs_manager( config: &'static Config, exec_rx: Receiver, exec_tx: Sender, + log_tx: SyncSender, ) { // Initialize a ThreadPool only when concurrency hasn't been disabled let thread_pool = if config.concurrency() > 1 { @@ -79,7 +43,7 @@ pub fn execs_manager( let exec_now = |mat: MAT| { let mut closure = { - let action = mat.a; + let action = mat.o; // Construct command let mut command = action.exec(&mat.m); @@ -128,15 +92,50 @@ pub fn execs_manager( exec_now(mat); } } - ExecsManagerInput::Flush(mat) => { - if let Some(set) = execs.rm_times(&mat) { - for _ in set { - exec_now(mat.clone()); + ExecsManagerInput::Order(order, options, tx) => { + let filtered = execs.filtered(options); + + if let Order::Flush = order { + let now = Local::now(); + // filter the state_map according to provided options + for (action, inner_map) in &filtered { + // get filter (required for LogEntry, FIXME optimize this) + let filter = { + let name = action.full_name(); + #[allow(clippy::unwrap_used)] + // We're pretty confident our action has a filter + config + .get_filter(&(name.0.to_string(), name.1.to_string())) + .unwrap() + }; + + for (match_, _) in inner_map { + let mat = MAT { + m: match_.clone(), + o: action, + t: now, + }; + // delete them from state and execute them + if let Some(set) = execs.rm_times(&mat) { + for _ in set { + exec_now(mat.clone()); + } + } + #[allow(clippy::unwrap_used)] // propagating panics is ok + log_tx + .send(DatabaseManagerInput::Flush(LogEntry { + exec: false, + m: mat.m, + f: filter, + t: mat.t, + })) + .unwrap(); + } } } + #[allow(clippy::unwrap_used)] // propagating panics is ok + tx.send(filtered).unwrap(); } - #[allow(clippy::unwrap_used)] // propagating panics is ok - ExecsManagerInput::Gimme(tx) => tx.send(execs.clone()).unwrap(), ExecsManagerInput::Stop => { for (action, inner_map) in execs { if action.on_exit() { @@ -144,7 +143,7 @@ pub fn execs_manager( for _ in inner_set { exec_now(MAT { m: match_.clone(), - a: action, + o: action, t: Local::now(), }); } diff --git a/rust/src/daemon/matches.rs b/rust/src/daemon/matches.rs index 45cd129..6b4bad5 100644 --- a/rust/src/daemon/matches.rs +++ b/rust/src/daemon/matches.rs @@ -1,73 +1,29 @@ use std::{ - collections::{BTreeMap, BTreeSet}, + collections::BTreeMap, sync::mpsc::{Receiver, Sender, SyncSender}, }; +use chrono::Local; use log::debug; use timer::MessageTimer; -use super::{database::DatabaseManagerInput, ExecsManagerInput}; -use crate::{ - concepts::Filter, - concepts::{LogEntry, Match, Time, MFT}, +use super::{ + database::DatabaseManagerInput, + statemap::{FilterOptions, StateMap, StateMapTrait}, + ExecsManagerInput, }; +use crate::concepts::{ActionFilter, Filter, LogEntry, Order, MFT}; #[derive(Clone)] pub enum MatchManagerInput { Match(MFT), Unmatch(MFT), - Flush(MFT), - Gimme(SyncSender), + Order(Order, FilterOptions, SyncSender), EndOfStartup, Stop, } -type MatchesMap = BTreeMap<&'static Filter, BTreeMap>>; - -// This trait is needed to permit to implement methods on an external type -trait MatchesMapTrait { - fn add(&mut self, mft: &MFT); - fn rm(&mut self, mft: &MFT); - fn rm_times(&mut self, mft: &MFT); - fn get_times(&self, mft: &MFT) -> usize; -} -impl MatchesMapTrait for MatchesMap { - fn add(&mut self, mft: &MFT) { - let inner_map = self.entry(mft.f).or_default(); - let inner_set = inner_map.entry(mft.m.clone()).or_default(); - inner_set.insert(mft.t); - } - - fn rm(&mut self, mft: &MFT) { - if let Some(inner_map) = self.get_mut(&mft.f) { - if let Some(inner_set) = inner_map.get_mut(&mft.m) { - inner_set.remove(&mft.t); - if inner_set.is_empty() { - inner_map.remove(&mft.m); - } - } - if inner_map.is_empty() { - self.remove(&mft.f); - } - } - } - - fn rm_times(&mut self, mft: &MFT) { - if let Some(inner_map) = self.get_mut(&mft.f) { - inner_map.remove(&mft.m); - if inner_map.is_empty() { - self.remove(&mft.f); - } - } - } - - fn get_times(&self, mft: &MFT) -> usize { - match self.get(&mft.f).and_then(|map| map.get(&mft.m)) { - Some(x) => x.len(), - None => 0, - } - } -} +pub type MatchesMap = StateMap; pub fn matches_manager( match_rx: Receiver, @@ -82,6 +38,9 @@ pub fn matches_manager( let mut startup = true; for mft in match_rx.iter() { + for (filter, map) in matches.iter() { + debug!("MATCHES {:?} {:?}", filter.full_name(), map.keys()); + } match mft { MatchManagerInput::EndOfStartup => { debug!("end of startup!"); @@ -89,7 +48,7 @@ pub fn matches_manager( } MatchManagerInput::Match(mft) => { // Store matches - let exec = match mft.f.retry() { + let exec = match mft.o.retry() { None => true, Some(retry) => { // Add new match @@ -98,7 +57,7 @@ pub fn matches_manager( let guard = timer.schedule_with_delay( // retry_duration is always Some() after filter's setup #[allow(clippy::unwrap_used)] - mft.f.retry_duration().unwrap(), + mft.o.retry_duration().unwrap(), MatchManagerInput::Unmatch(mft.clone()), ); guard.ignore(); @@ -110,10 +69,10 @@ pub fn matches_manager( // Executing actions if exec { // Delete matches only if storing them - if mft.f.retry().is_some() { + if mft.o.retry().is_some() { matches.rm_times(&mft); } - mft.f.send_actions(&mft.m, mft.t, &action_tx, true); + mft.o.send_actions(&mft.m, mft.t, &action_tx); } if !startup { @@ -122,30 +81,48 @@ pub fn matches_manager( .send(DatabaseManagerInput::Log(LogEntry { exec, m: mft.m, - f: mft.f, + f: mft.o, t: mft.t, })) .unwrap(); } } - MatchManagerInput::Unmatch(mft) => matches.rm(&mft), - #[allow(clippy::todo)] - MatchManagerInput::Flush(mft) => { - // remove from matches - matches.rm_times(&mft); - // send to DB + MatchManagerInput::Unmatch(mft) => { + matches.rm(&mft); + } + MatchManagerInput::Order(order, options, tx) => { + let filtered = matches.filtered(options); + + if let Order::Flush = order { + let now = Local::now(); + // filter the state_map according to provided options + for (filter, inner_map) in &filtered { + for (match_, _) in inner_map { + let mft = MFT { + m: match_.clone(), + o: filter, + t: now, + }; + // delete them from state + matches.rm_times(&mft); + // send them to DB + #[allow(clippy::unwrap_used)] // propagating panics is ok + log_tx + .send(DatabaseManagerInput::Flush(LogEntry { + exec: false, + m: mft.m, + f: mft.o, + t: mft.t, + })) + .unwrap(); + } + } + } + #[allow(clippy::unwrap_used)] // propagating panics is ok - log_tx - .send(DatabaseManagerInput::Flush(LogEntry { - exec: false, - m: mft.m, - f: mft.f, - t: mft.t, - })) - .unwrap(); + tx.send(filtered).unwrap(); } #[allow(clippy::unwrap_used)] // propagating panics is ok - MatchManagerInput::Gimme(tx) => tx.send(matches.clone()).unwrap(), MatchManagerInput::Stop => break, } } diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index 5a1d66e..f60a08f 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -1,6 +1,6 @@ use std::{ fs, - path::Path, + path::PathBuf, process::exit, sync::{ atomic::{AtomicBool, Ordering}, @@ -25,15 +25,16 @@ mod database; mod execs; mod matches; mod socket; +mod statemap; #[allow(unused_variables)] -pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { +pub fn daemon(config_path: PathBuf, loglevel: Level, socket: PathBuf) { if let Err(err) = SimpleLogger::init(loglevel) { eprintln!("ERROR could not initialize logging: {err}"); exit(1); } - let config: &'static Config = match Config::from_file(config_path) { + let config: &'static Config = match Config::from_file(&config_path) { Ok(config) => Box::leak(Box::new(config)), Err(err) => { error!("{err}"); @@ -59,12 +60,15 @@ pub fn daemon(config_path: &Path, loglevel: Level, socket: &Path) { let matches_manager_thread_handle = { let match_tx_matches = match_tx.clone(); let exec_tx_matches = exec_tx.clone(); - thread::spawn(move || matches_manager(match_rx, match_tx_matches, exec_tx_matches, log_tx)) + let log_tx_matches = log_tx.clone(); + thread::spawn(move || { + matches_manager(match_rx, match_tx_matches, exec_tx_matches, log_tx_matches) + }) }; let execs_manager_thread_handle = { let exec_tx_execs = exec_tx.clone(); - thread::spawn(move || execs_manager(config, exec_rx, exec_tx_execs)) + thread::spawn(move || execs_manager(config, exec_rx, exec_tx_execs, log_tx)) }; let database_manager_thread_handle = { diff --git a/rust/src/daemon/socket.rs b/rust/src/daemon/socket.rs index fa28632..2c167a5 100644 --- a/rust/src/daemon/socket.rs +++ b/rust/src/daemon/socket.rs @@ -1,21 +1,25 @@ use std::{ + collections::BTreeMap, fs, io, os::unix::net::{UnixListener, UnixStream}, path::PathBuf, process::exit, - sync::mpsc::{sync_channel, Sender}, + sync::{ + mpsc::{sync_channel, Sender}, + Arc, + }, }; use bincode::Options; -use chrono::Local; use log::{error, warn}; +use regex::Regex; use crate::{ - concepts::{ClientRequest, Config, DaemonResponse, InfoRes, MFT}, + concepts::{ActionFilter, ClientRequest, ClientStatus, Config, DaemonResponse, Pattern, PatternStatus}, utils::bincode_options, }; -use super::{ExecsManagerInput, MatchManagerInput}; +use super::{statemap::FilterOptions, ExecsManagerInput, MatchManagerInput}; macro_rules! err_str { ($expression:expr) => { @@ -48,6 +52,131 @@ fn open_socket(path: PathBuf) -> Result { err_str!(UnixListener::bind(path)) } +fn answer_order( + config: &'static Config, + match_tx: &Sender, + exec_tx: &Sender, + options: ClientRequest, +) -> Result { + // Compute options + let filtering_options = { + let (stream_name, filter_name) = match options.stream_filter { + Some(sf) => match sf.split_once(".") { + Some((s, f)) => (Some(s.to_string()), Some(f.to_string())), + None => (Some(sf), None), + }, + None => (None, None), + }; + + // Compute the Vec<(pattern_name, String)> into a BTreeMap, Regex> + let patterns = options + .patterns + .iter() + .map(|(name, reg)| { + // lookup pattern in config.patterns + config + .patterns() + .iter() + // retrieve or Err + .find(|(pattern_name, _)| name == *pattern_name) + .ok_or_else(|| format!("pattern '{name}' doesn't exist")) + // compile Regex or Err + .and_then(|(_, pattern)| match Regex::new(reg) { + Ok(reg) => Ok((pattern.clone(), reg)), + Err(err) => Err(format!("pattern '{name}' regex doesn't compile: {err}")), + }) + }) + .collect::, Regex>, String>>()?; + + FilterOptions { + stream_name, + filter_name, + patterns, + } + }; + + // ask for matches clone + let matches = { + let (m_tx, m_rx) = sync_channel(0); + + #[allow(clippy::unwrap_used)] // propagating panics is ok + match_tx + .send(MatchManagerInput::Order( + options.order, + filtering_options.clone(), + m_tx, + )) + .unwrap(); + + #[allow(clippy::unwrap_used)] // propagating panics is ok + m_rx.recv().unwrap() + }; + + // ask for execs clone + let execs = { + let (e_tx, e_rx) = sync_channel(0); + + #[allow(clippy::unwrap_used)] // propagating panics is ok + exec_tx + .send(ExecsManagerInput::Order( + options.order, + filtering_options, + e_tx, + )) + .unwrap(); + + #[allow(clippy::unwrap_used)] // propagating panics is ok + e_rx.recv().unwrap() + }; + + // Transform matches and execs into a ClientStatus + let cs: ClientStatus = matches + .into_iter() + .fold(BTreeMap::new(), |mut acc, (object, map)| { + let (stream, filter, _) = object.full_name(); + acc.entry(stream.into()) + .or_default() + .entry(filter.into()) + .or_default() + .extend(map.into_iter().map(|(match_, times)| { + ( + match_.join(" "), + PatternStatus { + matches: times.len(), + ..Default::default() + }, + ) + })); + acc + }); + + let cs = execs.into_iter().fold(cs, |mut acc, (object, map)| { + let (stream, filter, action) = object.full_name(); + let inner_map = acc + .entry(stream.into()) + .or_default() + .entry(filter.into()) + .or_default(); + + map.into_iter().for_each(|(match_, times)| { + inner_map + .entry(match_.join(" ")) + .or_default() + .actions + .insert( + action.to_string(), + times + .into_iter() + .map(|time| time.to_rfc3339().chars().take(19).collect()) + .collect(), + ); + }); + acc + }); + + Ok(cs) +} + macro_rules! or_next { ($msg:expr, $expression:expr) => { match $expression { @@ -84,78 +213,13 @@ pub fn socket_manager( "invalid message received: ", bin.deserialize_from::(conn) ); - let response = match request { - ClientRequest::Info => { - // ask for matches clone - let (m_tx, m_rx) = sync_channel(0); - #[allow(clippy::unwrap_used)] // propagating panics is ok - match_tx.send(MatchManagerInput::Gimme(m_tx)).unwrap(); - #[allow(clippy::unwrap_used)] // propagating panics is ok - let matches = m_rx.recv().unwrap(); - - // ask for execs clone - let (e_tx, e_rx) = sync_channel(0); - #[allow(clippy::unwrap_used)] // propagating panics is ok - exec_tx.send(ExecsManagerInput::Gimme(e_tx)).unwrap(); - #[allow(clippy::unwrap_used)] // propagating panics is ok - let execs = e_rx.recv().unwrap(); - - // Transform structures - macro_rules! map_map { - ($map:expr) => { - $map.into_iter() - .map(|(object, inner_map)| { - ( - object.full_name(), - inner_map - .into_iter() - .map(|(key, set)| { - ( - key, - set.into_iter() - .map(|time| time.timestamp()) - .collect(), - ) - }) - .collect(), - ) - }) - .collect() - }; - } - - DaemonResponse::Info(InfoRes { - matches: map_map!(matches), - execs: map_map!(execs), - }) - } - ClientRequest::Flush(flush) => { - match config.get_filter(&flush.f) { - Some(filter) => { - let now = Local::now(); - // Flush actions - filter.send_actions(&flush.m, now, &exec_tx, false); - // Flush filters - #[allow(clippy::unwrap_used)] // propagating panics is ok - match_tx - .send(MatchManagerInput::Flush(MFT { - m: flush.m, - f: filter, - t: now, - })) - .unwrap(); - DaemonResponse::Flush - } - None => DaemonResponse::Err(format!( - "no filter with name {}.{}", - flush.f.0, flush.f.1 - )), - } - } - }; + let response = answer_order(config, &match_tx, &exec_tx, request); or_next!( "failed to send response:", - bin.serialize_into(conn2, &response) + match response { + Ok(res) => bin.serialize_into(conn2, &DaemonResponse::Order(res)), + Err(err) => bin.serialize_into(conn2, &DaemonResponse::Err(err)), + } ); } Err(err) => error!("failed to open connection from cli: {err}"), diff --git a/rust/src/daemon/statemap.rs b/rust/src/daemon/statemap.rs new file mode 100644 index 0000000..f386c00 --- /dev/null +++ b/rust/src/daemon/statemap.rs @@ -0,0 +1,123 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; + +use regex::Regex; + +use crate::concepts::{ActionFilter, Match, Pattern, Time, MT}; + +#[derive(Clone)] +pub struct FilterOptions { + pub stream_name: Option, + pub filter_name: Option, + pub patterns: BTreeMap, Regex>, +} + +pub type StateMap = BTreeMap<&'static T, BTreeMap>>; + +// This trait is needed to permit to implement methods on an external type +pub trait StateMapTrait { + fn add(&mut self, mt: &MT); + fn rm(&mut self, mt: &MT) -> bool; + fn rm_times(&mut self, mt: &MT) -> Option>; + fn get_times(&self, mt: &MT) -> usize; + fn filtered(&self, filter_options: FilterOptions) -> Self; +} + +impl StateMapTrait for StateMap { + fn add(&mut self, mt: &MT) { + let inner_map = self.entry(mt.o).or_default(); + let inner_set = inner_map.entry(mt.m.clone()).or_default(); + inner_set.insert(mt.t); + } + + fn rm(&mut self, mt: &MT) -> bool { + let mut removed = false; + if let Some(inner_map) = self.get_mut(&mt.o) { + if let Some(inner_set) = inner_map.get_mut(&mt.m) { + inner_set.remove(&mt.t); + removed = true; + if inner_set.is_empty() { + inner_map.remove(&mt.m); + } + } + if inner_map.is_empty() { + self.remove(&mt.o); + } + } + removed + } + + fn rm_times(&mut self, mt: &MT) -> Option> { + let mut set = None; + if let Some(inner_map) = self.get_mut(&mt.o) { + set = inner_map.remove(&mt.m); + if inner_map.is_empty() { + self.remove(&mt.o); + } + } + set + } + + fn get_times(&self, mt: &MT) -> usize { + match self.get(&mt.o).and_then(|map| map.get(&mt.m)) { + Some(x) => x.len(), + None => 0, + } + } + + fn filtered(&self, filter_options: FilterOptions) -> Self { + let FilterOptions { + stream_name, + filter_name, + patterns, + } = filter_options; + self.iter() + // stream/filter filtering + .filter(|(object, _)| { + if let Some(stream_name) = &stream_name { + let full_name = object.full_name(); + let (s, f) = (full_name.0, full_name.1); + if *stream_name != s { + return false; + } + if let Some(filter_name) = &filter_name { + if *filter_name != f { + return false; + } + } + } + return true; + }) + // pattern filtering + .filter(|(object, _)| { + patterns + .iter() + .all(|(pattern, _)| object.patterns().get(pattern).is_some()) + }) + // match filtering + .filter_map(|(object, inner_map)| { + let map: BTreeMap> = inner_map + .iter() + .filter(|(match_, _)| { + match_ + .into_iter() + .zip(object.patterns()) + .filter_map(|(a_match, pattern)| match patterns.get(pattern.as_ref()) { + Some(regex) => Some((a_match, regex)), + None => None, + }) + .all(|(a_match, regex)| regex.is_match(a_match)) + }) + .map(|(a, b)| (a.clone(), b.clone())) + .collect(); + if map.len() > 0 { + Some((*object, map)) + } else { + None + } + }) + .collect() + } +} diff --git a/rust/src/main.rs b/rust/src/main.rs index 01fc447..ca53659 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -18,7 +18,8 @@ mod daemon; mod tests; mod utils; -use client::{flush, show, test_regex}; +use client::{request, test_regex}; +use concepts::Order; use daemon::daemon; use utils::cli::{Cli, Command}; @@ -44,26 +45,24 @@ fn main() { loglevel, socket, } => { - daemon(&config, loglevel, &socket); + daemon(config, loglevel, socket); } Command::Show { socket, format, limit, - pattern, patterns, - } => show(&socket, format, &limit, &pattern, &patterns), + } => request(socket, format, limit, patterns, Order::Show), Command::Flush { socket, format, limit, - pattern, patterns, - } => flush(&socket, format, &limit, &pattern, &patterns), + } => request(socket, format, limit, patterns, Order::Flush), Command::TestRegex { config, regex, line, - } => test_regex(&config, ®ex, &line), + } => test_regex(config, regex, line), } } diff --git a/rust/src/utils/cli.rs b/rust/src/utils/cli.rs index 663ac5e..e3d28bc 100644 --- a/rust/src/utils/cli.rs +++ b/rust/src/utils/cli.rs @@ -55,13 +55,9 @@ pub enum Command { #[clap(short = 'l', long, value_name = "STREAM[.FILTER]")] limit: Option, - /// only show items matching PATTERN regex - #[clap(short = 'p', long, value_name = "PATTERN")] - pattern: Option, - /// only show items matching name=PATTERN regex #[clap(value_parser = parse_named_regex, value_name = "NAME=PATTERN")] - patterns: Vec, + patterns: Vec<(String, String)>, }, /// Remove a target from reaction (e.g. unban) @@ -82,13 +78,9 @@ Then prints the flushed matches and actions." #[clap(short = 'l', long, value_name = "STREAM[.FILTER]")] limit: Option, - /// only show items matching PATTERN regex - #[clap(short = 'p', long, value_name = "PATTERN")] - pattern: Option, - /// only show items matching name=PATTERN regex #[clap(value_parser = parse_named_regex, value_name = "NAME=PATTERN")] - patterns: Vec, + patterns: Vec<(String, String)>, }, /// Test a regex @@ -129,24 +121,15 @@ impl fmt::Display for Format { } } -// Structs - -#[allow(dead_code)] -#[derive(Clone, Debug)] -pub struct NamedRegex { - pub regex: Regex, - pub name: String, -} - -fn parse_named_regex(s: &str) -> Result { +fn parse_named_regex(s: &str) -> Result<(String, String), String> { let (name, v) = s .split_once('=') .ok_or("When given as a positional argument, a pattern must be prefixed with a name, ex: ip=192.168.0.1")?; - let regex = Regex::new(v).map_err(|err| format!("{}", err))?; - Ok(NamedRegex { - regex, - name: name.to_string(), - }) + let _ = Regex::new(v).map_err(|err| format!("{}", err))?; + Ok(( + name.to_string(), + v.to_string(), + )) } fn parse_log_level(s: &str) -> Result { diff --git a/rust/test.jsonnet b/rust/test.jsonnet index 9106979..41335f7 100644 --- a/rust/test.jsonnet +++ b/rust/test.jsonnet @@ -22,21 +22,21 @@ streams: { s1: { - cmd: ['sh', '-c', "seq 20 | tr ' ' '\n' | while read i; do echo found $((i % 5)); sleep 0.3; done"], + cmd: ['sh', '-c', "seq 20 | tr ' ' '\n' | while read i; do echo found $((i % 5)); sleep 3; done"], filters: { f1: { regex: [ '^found $', ], retry: 2, - retryperiod: '5s', + retryperiod: '60s', actions: { damn: { cmd: ['notify-send', 'first stream', 'ban '], }, undamn: { cmd: ['notify-send', 'first stream', 'unban '], - after: '6s', + after: '20s', onexit: true, }, }, From 85ba7c8152887c2be44349ad2fb01bff1c7046df Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 1 Oct 2024 12:00:00 +0200 Subject: [PATCH 050/435] implement test-regex --- rust/src/client/mod.rs | 90 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/rust/src/client/mod.rs b/rust/src/client/mod.rs index 91aa1a1..4e6fe5a 100644 --- a/rust/src/client/mod.rs +++ b/rust/src/client/mod.rs @@ -1,10 +1,19 @@ -use std::{error::Error, io::stdout, os::unix::net::UnixStream, path::PathBuf, process::exit}; +use std::{ + collections::BTreeSet, + error::Error, + io::{stdin, stdout, BufRead, BufReader}, + os::unix::net::UnixStream, + path::PathBuf, + process::exit, + sync::Arc, +}; use bincode::Options; -use log::{debug, error, Level}; +use log::{error, info, Level}; +use regex::Regex; use crate::{ - concepts::{ClientRequest, ClientStatus, DaemonResponse, Order}, + concepts::{ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern}, utils::{bincode_options, cli::Format, SimpleLogger}, }; macro_rules! or_quit { @@ -71,10 +80,81 @@ pub fn request( } } -pub fn test_regex(config_path: PathBuf, regex: String, line: Option) { +pub fn test_regex(config_path: PathBuf, mut regex: String, line: Option) { if let Err(err) = SimpleLogger::init(Level::Debug) { eprintln!("ERROR could not initialize logging: {err}"); exit(1); } - debug!("test-regex {:?} {:?} {:?} ", config_path, regex, line); + + let config: Config = match Config::from_file(&config_path) { + Ok(config) => config, + Err(err) => { + error!("{err}"); + exit(1); + } + }; + + // Code close to Filter::setup() + let mut used_patterns: BTreeSet> = BTreeSet::new(); + for pattern in config.patterns().values() { + if let Some(index) = regex.find(pattern.name_with_braces()) { + // we already `find` it, so we must be able to `rfind` it + #[allow(clippy::unwrap_used)] + if regex.rfind(pattern.name_with_braces()).unwrap() != index { + error!( + "pattern {} present multiple times in regex", + pattern.name_with_braces() + ); + exit(1); + } + used_patterns.insert(pattern.clone()); + } + regex = regex.replacen(pattern.name_with_braces(), &pattern.regex, 1); + } + + let compiled = match Regex::new(®ex) { + Ok(reg) => reg, + Err(err) => { + error!("regex doesn't compile: {err}"); + exit(1); + } + }; + + let match_closure = |line: String| { + let mut ignored = false; + if let Some(matches) = compiled.captures(&line) { + let mut result = Vec::new(); + if used_patterns.len() != 0 { + for pattern in used_patterns.iter() { + if let Some(match_) = matches.name(pattern.name()) { + result.push(match_.as_str().to_string()); + if !pattern.not_an_ignore(match_.as_str()) { + ignored = true; + } + } + } + if !ignored { + println!("\x1b[32mmatching\x1b[0m {result:?}: {line}"); + } else { + println!("\x1b[33mignore matching\x1b[0m {result:?}: {line}"); + } + } else { + println!("\x1b[32mmatching\x1b[0m: {line}"); + } + } else { + println!("\x1b[31mno match\x1b[0m: {line}"); + } + }; + + if let Some(line) = line { + match_closure(line); + } else { + info!("no second argument: reading from stdin"); + for line in BufReader::new(stdin()).lines() { + match line { + Ok(line) => match_closure(line), + Err(_) => break, + }; + } + } } From b2c85a5d398710e585efb29ae2d67c1901942467 Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 1 Oct 2024 12:00:00 +0200 Subject: [PATCH 051/435] improve error message --- rust/src/concepts/stream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/src/concepts/stream.rs b/rust/src/concepts/stream.rs index d1fa6bd..f5eeaa4 100644 --- a/rust/src/concepts/stream.rs +++ b/rust/src/concepts/stream.rs @@ -113,7 +113,7 @@ impl Stream { } } } - error!("stream {} exited", self.name); + error!("stream {} exited: its command returned.", self.name); } } From 70a367c189c757e3e9e3547e12728470df3735dc Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 2 Oct 2024 12:00:00 +0200 Subject: [PATCH 052/435] create lib.rs, fmt, clippy --- rust/src/client/mod.rs | 7 ++++--- rust/src/concepts/action.rs | 8 ++------ rust/src/concepts/filter.rs | 2 +- rust/src/concepts/mod.rs | 12 ++++++------ rust/src/concepts/socket_messages.rs | 2 +- rust/src/daemon/execs.rs | 2 +- rust/src/daemon/matches.rs | 2 +- rust/src/daemon/socket.rs | 4 +++- rust/src/daemon/statemap.rs | 11 +++++------ rust/src/lib.rs | 14 ++++++++++++++ rust/src/main.rs | 26 ++++---------------------- rust/src/utils/cli.rs | 5 +---- 12 files changed, 43 insertions(+), 52 deletions(-) create mode 100644 rust/src/lib.rs diff --git a/rust/src/client/mod.rs b/rust/src/client/mod.rs index 4e6fe5a..2e4b9e1 100644 --- a/rust/src/client/mod.rs +++ b/rust/src/client/mod.rs @@ -40,10 +40,11 @@ fn send_retrieve(socket: &PathBuf, req: &ClientRequest) -> DaemonResponse { } fn print_status(cs: ClientStatus, format: Format) -> Result<(), Box> { - Ok(match format { + match format { Format::JSON => serde_json::to_writer(stdout().lock(), &cs)?, Format::YAML => serde_yaml::to_writer(stdout().lock(), &cs)?, - }) + } + Ok(()) } pub fn request( @@ -124,7 +125,7 @@ pub fn test_regex(config_path: PathBuf, mut regex: String, line: Option) let mut ignored = false; if let Some(matches) = compiled.captures(&line) { let mut result = Vec::new(); - if used_patterns.len() != 0 { + if !used_patterns.is_empty() { for pattern in used_patterns.iter() { if let Some(match_) = matches.name(pattern.name()) { result.push(match_.as_str().to_string()); diff --git a/rust/src/concepts/action.rs b/rust/src/concepts/action.rs index 6d1561d..b7fc4cd 100644 --- a/rust/src/concepts/action.rs +++ b/rust/src/concepts/action.rs @@ -39,12 +39,8 @@ impl ActionFilter for Action { &self.patterns } - fn full_name<'a>(&'a self) -> (&'a str, &'a str, &'a str) { - ( - &self.stream_name, - &self.filter_name, - &self.name, - ) + fn full_name(&self) -> (&str, &str, &str) { + (&self.stream_name, &self.filter_name, &self.name) } } diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index 61aae3e..50a1153 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -46,7 +46,7 @@ pub struct Filter { } impl ActionFilter for Filter { - fn full_name<'a>(&'a self) -> (&'a str, &'a str, &'a str) { + fn full_name(&self) -> (&str, &str, &str) { (self.stream_name.as_ref(), self.name.as_ref(), "") } diff --git a/rust/src/concepts/mod.rs b/rust/src/concepts/mod.rs index c0a87b1..6797084 100644 --- a/rust/src/concepts/mod.rs +++ b/rust/src/concepts/mod.rs @@ -8,15 +8,15 @@ mod stream; use std::{collections::BTreeSet, fmt::Display, sync::Arc}; -pub use action::*; -pub use config::*; -pub use filter::*; +pub use action::Action; +pub use config::{Config, Patterns}; +pub use filter::Filter; pub use messages::*; -pub use pattern::*; +pub use pattern::Pattern; pub use socket_messages::*; -pub use stream::*; +pub use stream::Stream; pub trait ActionFilter: Clone + Display + PartialEq + Eq + PartialOrd + Ord { fn patterns(&self) -> &BTreeSet>; - fn full_name<'a>(&'a self) -> (&'a str, &'a str, &'a str); + fn full_name(&self) -> (&str, &str, &str); } diff --git a/rust/src/concepts/socket_messages.rs b/rust/src/concepts/socket_messages.rs index b0d1088..26999b7 100644 --- a/rust/src/concepts/socket_messages.rs +++ b/rust/src/concepts/socket_messages.rs @@ -55,7 +55,7 @@ impl Serialize for PatternStatus { // This means we're printing for user, not exchanging it over a socket let state = if serializer.is_human_readable() { let ser_matches = self.matches != 0; - let ser_actions = self.actions.len() != 0; + let ser_actions = !self.actions.is_empty(); let mut state = serializer .serialize_struct("PatternStatus", ser_matches as usize + ser_actions as usize)?; if ser_matches { diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs index 91793e4..1276c72 100644 --- a/rust/src/daemon/execs.rs +++ b/rust/src/daemon/execs.rs @@ -109,7 +109,7 @@ pub fn execs_manager( .unwrap() }; - for (match_, _) in inner_map { + for match_ in inner_map.keys() { let mat = MAT { m: match_.clone(), o: action, diff --git a/rust/src/daemon/matches.rs b/rust/src/daemon/matches.rs index 6b4bad5..dbeb43e 100644 --- a/rust/src/daemon/matches.rs +++ b/rust/src/daemon/matches.rs @@ -97,7 +97,7 @@ pub fn matches_manager( let now = Local::now(); // filter the state_map according to provided options for (filter, inner_map) in &filtered { - for (match_, _) in inner_map { + for match_ in inner_map.keys() { let mft = MFT { m: match_.clone(), o: filter, diff --git a/rust/src/daemon/socket.rs b/rust/src/daemon/socket.rs index 2c167a5..8a7a0da 100644 --- a/rust/src/daemon/socket.rs +++ b/rust/src/daemon/socket.rs @@ -15,7 +15,9 @@ use log::{error, warn}; use regex::Regex; use crate::{ - concepts::{ActionFilter, ClientRequest, ClientStatus, Config, DaemonResponse, Pattern, PatternStatus}, + concepts::{ + ActionFilter, ClientRequest, ClientStatus, Config, DaemonResponse, Pattern, PatternStatus, + }, utils::bincode_options, }; diff --git a/rust/src/daemon/statemap.rs b/rust/src/daemon/statemap.rs index f386c00..7d83a55 100644 --- a/rust/src/daemon/statemap.rs +++ b/rust/src/daemon/statemap.rs @@ -88,7 +88,7 @@ impl StateMapTrait for StateMap { } } } - return true; + true }) // pattern filtering .filter(|(object, _)| { @@ -102,17 +102,16 @@ impl StateMapTrait for StateMap { .iter() .filter(|(match_, _)| { match_ - .into_iter() + .iter() .zip(object.patterns()) - .filter_map(|(a_match, pattern)| match patterns.get(pattern.as_ref()) { - Some(regex) => Some((a_match, regex)), - None => None, + .filter_map(|(a_match, pattern)| { + patterns.get(pattern.as_ref()).map(|regex| (a_match, regex)) }) .all(|(a_match, regex)| regex.is_match(a_match)) }) .map(|(a, b)| (a.clone(), b.clone())) .collect(); - if map.len() > 0 { + if !map.is_empty() { Some((*object, map)) } else { None diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000..961a40e --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,14 @@ +#![warn( + clippy::panic, + clippy::todo, + clippy::unimplemented, + clippy::unwrap_used +)] +#![allow(clippy::upper_case_acronyms, clippy::mutable_key_type)] +#![forbid(unsafe_code)] + +pub mod client; +pub mod concepts; +pub mod daemon; +pub mod tests; +pub mod utils; diff --git a/rust/src/main.rs b/rust/src/main.rs index ca53659..7796f1d 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -1,27 +1,9 @@ -#![warn( - missing_docs, - clippy::panic, - clippy::todo, - clippy::unimplemented, - clippy::unwrap_used -)] -#![allow(clippy::upper_case_acronyms, clippy::mutable_key_type)] -#![forbid(unsafe_code)] - -//! TODO document a bit use clap::Parser; -mod client; -mod concepts; -mod daemon; - -mod tests; -mod utils; - -use client::{request, test_regex}; -use concepts::Order; -use daemon::daemon; -use utils::cli::{Cli, Command}; +use reaction::client::{request, test_regex}; +use reaction::concepts::Order; +use reaction::daemon::daemon; +use reaction::utils::cli::{Cli, Command}; fn main() { // Show a nice message when reaction panics diff --git a/rust/src/utils/cli.rs b/rust/src/utils/cli.rs index e3d28bc..fb0acf9 100644 --- a/rust/src/utils/cli.rs +++ b/rust/src/utils/cli.rs @@ -126,10 +126,7 @@ fn parse_named_regex(s: &str) -> Result<(String, String), String> { .split_once('=') .ok_or("When given as a positional argument, a pattern must be prefixed with a name, ex: ip=192.168.0.1")?; let _ = Regex::new(v).map_err(|err| format!("{}", err))?; - Ok(( - name.to_string(), - v.to_string(), - )) + Ok((name.to_string(), v.to_string())) } fn parse_log_level(s: &str) -> Result { From 16a083095cecda151b14f20f7c21b266f52f0c0b Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 2 Oct 2024 12:00:00 +0200 Subject: [PATCH 053/435] Fix bug: forgot to compare match's time to now --- rust/src/daemon/matches.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/src/daemon/matches.rs b/rust/src/daemon/matches.rs index dbeb43e..89fb0d1 100644 --- a/rust/src/daemon/matches.rs +++ b/rust/src/daemon/matches.rs @@ -54,10 +54,10 @@ pub fn matches_manager( // Add new match matches.add(&mft); // Remove match when expired + // retry_duration is always Some() after filter's setup + #[allow(clippy::unwrap_used)] let guard = timer.schedule_with_delay( - // retry_duration is always Some() after filter's setup - #[allow(clippy::unwrap_used)] - mft.o.retry_duration().unwrap(), + mft.t - Local::now() + mft.o.retry_duration().unwrap(), MatchManagerInput::Unmatch(mft.clone()), ); guard.ignore(); From 5e6331f820ac28c178eee60821344f836a477f28 Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 2 Oct 2024 12:00:00 +0200 Subject: [PATCH 054/435] Adapt code for integration tests; add one integration test The integration test checks: - general logic - match/exec persistence - flush - flush persistence --- rust/src/client/mod.rs | 14 +-- rust/src/daemon/database/lowlevel.rs | 1 - rust/src/daemon/database/mod.rs | 5 +- rust/src/daemon/mod.rs | 28 ++--- rust/src/main.rs | 29 ++++- rust/tests/simple.rs | 165 +++++++++++++++++++++++++++ 6 files changed, 212 insertions(+), 30 deletions(-) create mode 100644 rust/tests/simple.rs diff --git a/rust/src/client/mod.rs b/rust/src/client/mod.rs index 2e4b9e1..7df019e 100644 --- a/rust/src/client/mod.rs +++ b/rust/src/client/mod.rs @@ -9,12 +9,12 @@ use std::{ }; use bincode::Options; -use log::{error, info, Level}; +use log::{error, info}; use regex::Regex; use crate::{ concepts::{ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern}, - utils::{bincode_options, cli::Format, SimpleLogger}, + utils::{bincode_options, cli::Format}, }; macro_rules! or_quit { ($msg:expr, $expression:expr) => { @@ -54,11 +54,6 @@ pub fn request( patterns: Vec<(String, String)>, order: Order, ) { - if let Err(err) = SimpleLogger::init(Level::Debug) { - eprintln!("ERROR could not initialize logging: {err}"); - exit(1); - } - let response = send_retrieve( &socket, &ClientRequest { @@ -82,11 +77,6 @@ pub fn request( } pub fn test_regex(config_path: PathBuf, mut regex: String, line: Option) { - if let Err(err) = SimpleLogger::init(Level::Debug) { - eprintln!("ERROR could not initialize logging: {err}"); - exit(1); - } - let config: Config = match Config::from_file(&config_path) { Ok(config) => config, Err(err) => { diff --git a/rust/src/daemon/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs index 7172156..1f09128 100644 --- a/rust/src/daemon/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -93,7 +93,6 @@ impl Iterator for ReadDB { fn next(&mut self) -> Option { let res = self.read::(); - debug!("{res:?}"); match res { Ok(item) => Some(item.to(&self.h)), Err(err) => match *err { diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index d0bed01..2d98781 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -8,7 +8,7 @@ use std::{ }; use chrono::{Local, TimeDelta}; -use log::{error, info, warn}; +use log::{debug, error, info, warn}; use thiserror::Error; use super::MatchManagerInput; @@ -220,6 +220,7 @@ fn __rotate_db( if let Some(map) = last_global_flush { if let Some(time) = map.get(&entry.m) { if time > &entry.t { + debug!("DB ignoring global match: {:?}", entry.m); continue; } } @@ -227,6 +228,7 @@ fn __rotate_db( if let Some(map) = flushes.get(&entry.f) { if let Some(time) = map.get(&entry.m) { if time > &entry.t { + debug!("DB ignoring local match: {:?}", entry.m); continue; } } @@ -244,6 +246,7 @@ fn __rotate_db( millisecond_disambiguation_counter += 1; if let Some(tx) = matches_tx { + debug!("DB sending match from DB: {:?}", entry.m); #[allow(clippy::unwrap_used)] // propagating panics is ok tx.send(MatchManagerInput::Match(entry.clone().into())) .unwrap(); diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index f60a08f..80c9b0c 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -10,10 +10,10 @@ use std::{ thread, }; -use log::{error, info, Level}; +use log::{error, info}; use socket::socket_manager; -use crate::{concepts::Config, utils::SimpleLogger}; +use crate::concepts::Config; use database::database_manager; use execs::execs_manager; use matches::matches_manager; @@ -28,12 +28,7 @@ mod socket; mod statemap; #[allow(unused_variables)] -pub fn daemon(config_path: PathBuf, loglevel: Level, socket: PathBuf) { - if let Err(err) = SimpleLogger::init(loglevel) { - eprintln!("ERROR could not initialize logging: {err}"); - exit(1); - } - +pub fn daemon(config_path: PathBuf, socket: PathBuf) -> bool { let config: &'static Config = match Config::from_file(&config_path) { Ok(config) => Box::leak(Box::new(config)), Err(err) => { @@ -108,8 +103,16 @@ pub fn daemon(config_path: PathBuf, loglevel: Level, socket: PathBuf) { let _ = child_handle.kill(); } }) { - error!("impossible to launch a signal-catching thread, exiting: {err}"); - exit(1); + match err { + // Would only happen in tests as this is the only place in the code + // where we put an handler. Sorry that there is test-specific code! + ctrlc::Error::MultipleHandlers => {} + // Real error + _ => { + error!("impossible to launch a signal-catching thread, exiting: {err}"); + exit(1); + } + } } } @@ -135,8 +138,5 @@ pub fn daemon(config_path: PathBuf, loglevel: Level, socket: PathBuf) { error!("failed to remove socket: {}", err); } - exit(match !signal_received.load(Ordering::SeqCst) && stop_ok { - true => 0, - false => 1, - }); + !signal_received.load(Ordering::SeqCst) && stop_ok } diff --git a/rust/src/main.rs b/rust/src/main.rs index 7796f1d..c4031b0 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -1,9 +1,13 @@ +use std::process::exit; + use clap::Parser; +use log::Level; use reaction::client::{request, test_regex}; use reaction::concepts::Order; use reaction::daemon::daemon; use reaction::utils::cli::{Cli, Command}; +use reaction::utils::SimpleLogger; fn main() { // Show a nice message when reaction panics @@ -21,13 +25,34 @@ fn main() { let cli = Cli::parse(); + { + // Set log level + let level = if let Command::Start { + loglevel, + config: _, + socket: _, + } = cli.command + { + loglevel + } else { + Level::Debug + }; + if let Err(err) = SimpleLogger::init(level) { + eprintln!("ERROR could not initialize logging: {err}"); + exit(1); + } + } + match cli.command { Command::Start { config, - loglevel, + loglevel: _, socket, } => { - daemon(config, loglevel, socket); + exit(match daemon(config, socket) { + true => 0, + false => 1, + }); } Command::Show { socket, diff --git a/rust/tests/simple.rs b/rust/tests/simple.rs new file mode 100644 index 0000000..c8c4f3f --- /dev/null +++ b/rust/tests/simple.rs @@ -0,0 +1,165 @@ +use std::{ + env, + fs::File, + io::{Read, Write}, + thread, + time::Duration, +}; + +use log::Level; +use tempfile::TempDir; + +use reaction::{ + client::request, + concepts::Order, + daemon::daemon, + utils::{cli::Format, SimpleLogger}, +}; + +fn file_with_contents(path: &str, contents: &str) { + let mut file = File::create(path).unwrap(); + file.write_all(contents.as_bytes()).unwrap(); +} + +fn config_with_cmd(config_path: &str, cmd: &str) { + file_with_contents( + config_path, + &(" +{ + concurrency: 0, + patterns: { + num: { regex: '[0-9]+' }, + }, + streams: { + stream1: { + cmd: ['sh', '-c', '" + .to_owned() + + cmd + + "'], + filters: { + filter1: { + regex: ['here is '], + retry: 2, + retryperiod: '20s', + actions: { + // Don't mix code and data at home! + // You may permit arbitrary execution from vilains, + // if your regex is permissive enough. + // This is OK only for testing purposes. + action1: { + cmd: ['sh', '-c', 'echo >> ./out.txt'], + }, + action2: { + cmd: ['sh', '-c', 'echo del >> ./out.txt'], + after: '3min', + onexit: false, + }, + } + } + } + } + } +}"), + ); +} + +fn get_file_content(path: &str) -> String { + let mut out_txt = File::open(path).unwrap(); + let mut contents = String::new(); + out_txt.read_to_string(&mut contents).unwrap(); + contents +} + +#[test] +fn simple() { + let dir = TempDir::new().unwrap(); + env::set_current_dir(&dir).unwrap(); + + let config_path = "config.jsonnet"; + let out_path = "./out.txt"; + let socket_path = "./reaction.sock"; + + config_with_cmd( + config_path, + "for i in 12 24 36 24 36 12 12 24 56 67; do echo here is $i; sleep 0.1; done; sleep 2", + ); + + file_with_contents(out_path, ""); + + // Set the logger before running any code from the crate + SimpleLogger::init(Level::Debug).unwrap(); + + // Run the daemon + let handle = thread::spawn(move || { + assert!(daemon(config_path.into(), socket_path.into())); + }); + + // Run the flushes + + // We sleep for the time the echoes are finished + 1 second + // This ensures that the subsecond precision lost from de/serialization + // never causes the flush to be interpreted as anterior to the match + + let handle2 = thread::spawn(move || { + thread::sleep(Duration::from_millis(2500)); + request( + socket_path.into(), + Format::JSON, + None, + vec![("num".into(), "24".into())], + Order::Flush, + ); + }); + + let handle3 = thread::spawn(move || { + thread::sleep(Duration::from_millis(2500)); + request( + socket_path.into(), + Format::JSON, + None, + vec![("num".into(), "56".into())], + Order::Flush, + ); + }); + + handle.join().unwrap(); + handle2.join().unwrap(); + handle3.join().unwrap(); + + assert_eq!( + // 24 is encountered for the second time, then + // 36 is encountered for the second time, then + // 12 is encountered for the second time, then + // 24 is flushed + get_file_content(out_path).trim(), + "24\n36\n12\ndel 24".to_owned().trim() + ); + + // Second part of the test + // We test that persistence worked as intended + // Both for matches and for flushes + + config_with_cmd( + config_path, + "for i in 12 24 36 56 67; do echo here is $i; sleep 0.1; done", + ); + + file_with_contents(out_path, ""); + + assert!(daemon(config_path.into(), socket_path.into())); + + // 36 from DB + // 12 from DB + // 12 from DB + new match + // 67 from DB + new match + let content = get_file_content(out_path).trim().to_owned(); + let scenario1 = "36\n12\n12\n67".trim().to_owned(); + let scenario2 = "12\n36\n12\n67".trim().to_owned(); + assert!( + content == scenario1 || content == scenario2, + "content: {}\nscenario1: {}\nscenario2: {}", + content, + scenario1, + scenario2 + ); +} From 116e75f81f8f8eda173c8bf79eadff34b9656671 Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 3 Oct 2024 12:00:00 +0200 Subject: [PATCH 055/435] Move filter logic from database/mod.rs to filter.rs --- rust/src/concepts/filter.rs | 14 +++++++++----- rust/src/daemon/database/mod.rs | 17 ++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index 50a1153..f68097e 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -12,7 +12,7 @@ use serde::Deserialize; use super::{ messages::{Match, Time, MAT}, - Action, ActionFilter, Pattern, Patterns, + Action, ActionFilter, LogEntry, Pattern, Patterns, }; use crate::{daemon::ExecsManagerInput, utils::parse_duration}; @@ -73,10 +73,6 @@ impl Filter { self.retry_duration } - pub fn longuest_action_duration(&self) -> TimeDelta { - self.longuest_action_duration - } - pub fn setup( &mut self, stream_name: &str, @@ -215,6 +211,14 @@ impl Filter { tx.send(ExecsManagerInput::Exec(mat)).unwrap(); } } + + pub fn is_outdated(entry: &LogEntry, now: &Time) -> bool { + if entry.exec { + entry.t + entry.f.longuest_action_duration < *now + } else { + entry.t + entry.f.retry_duration.unwrap_or_default() < *now + } + } } impl Display for Filter { diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index 2d98781..270d77e 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -235,15 +235,14 @@ fn __rotate_db( } // Store match & store in db - #[allow(clippy::unwrap_used)] // 0 second is obviously less seconds than i64::MAX - if (!entry.exec && entry.t + entry.f.retry_duration().unwrap_or_default() > now) - || (entry.exec && entry.t + entry.f.longuest_action_duration() > now) - { - // We loose subsecond precision when storing times, so we add those fake - // milliseconds to make sure each time is unique - - entry.t += TimeDelta::new(0, millisecond_disambiguation_counter).unwrap(); - millisecond_disambiguation_counter += 1; + if !Filter::is_outdated(&entry, &now) { + #[allow(clippy::unwrap_used)] // 0 second is obviously less than i64::MAX + { + // We loose subsecond precision when storing times, so we add those fake + // milliseconds to make sure each time is unique + entry.t += TimeDelta::new(0, millisecond_disambiguation_counter).unwrap(); + millisecond_disambiguation_counter += 1; + } if let Some(tx) = matches_tx { debug!("DB sending match from DB: {:?}", entry.m); From 088354e955d0ff9dc732e7977f4633f4c8227f4f Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 8 Oct 2024 12:00:00 +0200 Subject: [PATCH 056/435] untested async version --- rust/Cargo.lock | 612 ++++++++++++++++++++++++++- rust/Cargo.toml | 8 + rust/src/concepts/action.rs | 3 +- rust/src/concepts/filter.rs | 12 +- rust/src/concepts/messages.rs | 2 +- rust/src/concepts/socket_messages.rs | 2 +- rust/src/concepts/stream.rs | 22 +- rust/src/daemon/database/lowlevel.rs | 104 +++-- rust/src/daemon/database/mod.rs | 66 +-- rust/src/daemon/execs.rs | 97 +++-- rust/src/daemon/matches.rs | 54 ++- rust/src/daemon/mod.rs | 64 +-- rust/src/daemon/socket.rs | 88 ++-- rust/src/daemon/statemap.rs | 2 +- rust/src/main.rs | 4 +- rust/src/utils/mod.rs | 50 ++- rust/src/utils/threadpool.rs | 68 --- 17 files changed, 951 insertions(+), 307 deletions(-) delete mode 100644 rust/src/utils/threadpool.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4d0efc4..1af746b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -85,12 +100,118 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-io" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "444b0228950ee6501b3568d3c93bf1176a1fdbc3b758dcd9475046d30f4dc7e8" +dependencies = [ + "async-lock", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix", + "tracing", +] + +[[package]] +name = "async-signal" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix", + "signal-hook-registry", + "slab", + "windows-sys 0.59.0", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "backtrace" +version = "0.3.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.13.1" @@ -112,12 +233,31 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "blocking" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "bumpalo" version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "bytes" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" + [[package]] name = "cc" version = "1.1.6" @@ -205,12 +345,27 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "ctrlc" version = "3.4.4" @@ -237,12 +392,141 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "event-listener" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-lite" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + [[package]] name = "hashbrown" version = "0.14.5" @@ -261,6 +545,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -398,6 +688,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.155" @@ -410,6 +706,16 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.22" @@ -428,6 +734,27 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "wasi", + "windows-sys 0.52.0", +] + [[package]] name = "nix" version = "0.28.0" @@ -440,6 +767,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -455,16 +792,60 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] +[[package]] +name = "object" +version = "0.36.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + [[package]] name = "pathdiff" version = "0.2.1" @@ -498,6 +879,44 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c719dcf55f09a3a7e764c6649ab594c18a177e3599c467983cdf644bfc0a4088" +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "piper" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "polling" +version = "3.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2790cd301dec6cd3b7a025e4815cf825724a51c98dccfe6a3e55f05ffb6511" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.4.0", + "pin-project-lite", + "rustix", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -520,11 +939,13 @@ dependencies = [ name = "reaction" version = "0.1.0" dependencies = [ + "async-process", "bincode", "chrono", "clap", "clap_complete", "ctrlc", + "futures", "jrsonnet-evaluator", "log", "num_cpus", @@ -535,6 +956,19 @@ dependencies = [ "tempfile", "thiserror", "timer", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags", ] [[package]] @@ -566,6 +1000,12 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -591,6 +1031,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.204" @@ -636,6 +1082,49 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "strsim" version = "0.11.1" @@ -709,6 +1198,16 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "timer" version = "0.2.0" @@ -718,6 +1217,105 @@ dependencies = [ "chrono", ] +[[package]] +name = "tokio" +version = "1.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "tokio-util" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + [[package]] name = "unescape" version = "0.1.0" @@ -754,6 +1352,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.92" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 4503756..20dc7f8 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -21,3 +21,11 @@ serde_yaml = "0.9.34" tempfile = "3.12.0" thiserror = "1.0.63" timer = "0.2.0" + +# async-related +async-process = "2.3.0" +futures = "0.3.30" +tokio = { version = "1.40.0", features = ["full"] } +tokio-util = { version = "0.7.12", features = ["codec"] } +tracing = "0.1.40" +tracing-subscriber = "0.3.18" diff --git a/rust/src/concepts/action.rs b/rust/src/concepts/action.rs index b7fc4cd..0f6218d 100644 --- a/rust/src/concepts/action.rs +++ b/rust/src/concepts/action.rs @@ -1,5 +1,6 @@ -use std::{cmp::Ordering, collections::BTreeSet, fmt::Display, process::Command, sync::Arc}; +use std::{cmp::Ordering, collections::BTreeSet, fmt::Display, sync::Arc}; +use async_process::Command; use chrono::TimeDelta; use serde::Deserialize; diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index f68097e..4bdb7dc 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -2,13 +2,14 @@ use std::{ cmp::Ordering, collections::{BTreeMap, BTreeSet}, fmt::Display, - sync::{mpsc::Sender, Arc}, + sync::Arc, }; use chrono::TimeDelta; use log::info; use regex::Regex; use serde::Deserialize; +use tokio::sync::mpsc; use super::{ messages::{Match, Time, MAT}, @@ -200,7 +201,12 @@ impl Filter { None } - pub fn send_actions(&'static self, m: &Match, t: Time, tx: &Sender) { + pub async fn send_actions( + &'static self, + m: &Match, + t: Time, + tx: &mpsc::Sender, + ) { for action in self.actions.values() { let mat = MAT { m: m.clone(), @@ -208,7 +214,7 @@ impl Filter { t: t + action.after_duration().unwrap_or_default(), }; #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(ExecsManagerInput::Exec(mat)).unwrap(); + tx.send(ExecsManagerInput::Exec(mat)).await.unwrap(); } } diff --git a/rust/src/concepts/messages.rs b/rust/src/concepts/messages.rs index 5a9b1c7..4642eb6 100644 --- a/rust/src/concepts/messages.rs +++ b/rust/src/concepts/messages.rs @@ -5,7 +5,7 @@ use super::{Action, ActionFilter, Filter}; pub type Time = DateTime; pub type Match = Vec; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct MT { pub m: Match, pub o: &'static T, diff --git a/rust/src/concepts/socket_messages.rs b/rust/src/concepts/socket_messages.rs index 26999b7..64c99c6 100644 --- a/rust/src/concepts/socket_messages.rs +++ b/rust/src/concepts/socket_messages.rs @@ -7,7 +7,7 @@ use serde::{ser::SerializeStruct, Deserialize, Serialize}; // We don't need protocol versionning here because // client and daemon are the same binary -#[derive(Copy, Clone, Serialize, Deserialize)] +#[derive(Copy, Clone, Debug, Serialize, Deserialize)] pub enum Order { Show, Flush, diff --git a/rust/src/concepts/stream.rs b/rust/src/concepts/stream.rs index f5eeaa4..2bcfd1c 100644 --- a/rust/src/concepts/stream.rs +++ b/rust/src/concepts/stream.rs @@ -1,13 +1,11 @@ -use std::{ - collections::BTreeMap, - io::{BufRead, BufReader}, - process::{Child, Command, Stdio}, - sync::mpsc::{Sender, SyncSender}, -}; +use std::collections::BTreeMap; +use async_process::{Child, Command, Stdio}; use chrono::Local; +use futures::{io::BufReader, AsyncBufReadExt, StreamExt}; use log::{error, info}; use serde::Deserialize; +use tokio::sync::{mpsc, oneshot}; use super::{Filter, Patterns, MFT}; use crate::daemon::MatchManagerInput; @@ -64,10 +62,10 @@ impl Stream { Ok(()) } - pub fn manager( + pub async fn manager( &'static self, - child_tx: SyncSender>, - match_tx: Sender, + child_tx: oneshot::Sender>, + match_tx: mpsc::Sender, ) { info!("{}: start {:?}", self.name, self.cmd); let mut child = match Command::new(&self.cmd[0]) @@ -88,13 +86,12 @@ impl Stream { // keep stdout before sending/moving child to the main thread #[allow(clippy::unwrap_used)] // we know there is an stdout because we asked for Stdio::piped() - let stdout = BufReader::new(child.stdout.take().unwrap()); + let mut lines = BufReader::new(child.stdout.take().unwrap()).lines(); // let main handle the child process let _ = child_tx.send(Some(child)); - drop(child_tx); - for line in stdout.lines() { + while let Some(line) = lines.next().await { let line = match line { Ok(line) => line, Err(_) => break, @@ -109,6 +106,7 @@ impl Stream { o: filter, t: Local::now(), })) + .await .unwrap(); } } diff --git a/rust/src/daemon/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs index 1f09128..b7b5e53 100644 --- a/rust/src/daemon/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -1,15 +1,18 @@ -use std::{ - collections::BTreeMap, - fmt::Debug, - fs::File, - io::{self, BufReader, BufWriter, Write}, - process::exit, -}; +use std::{collections::BTreeMap, fmt::Debug, process::exit}; use bincode::Options; use chrono::{DateTime, Local}; +use futures::{io, SinkExt, StreamExt}; use log::{debug, error, warn}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use tokio::{ + fs::File, + io::{BufReader, BufWriter}, +}; +use tokio_util::{ + bytes::Bytes, + codec::{FramedRead, FramedWrite, LengthDelimitedCodec}, +}; use crate::{ concepts::{ActionFilter, Config, Filter, LogEntry, Match}, @@ -29,14 +32,14 @@ type WriteHeader = BTreeMap<&'static Filter, usize>; const DB_SIGNATURE: &str = "reaction-db-v01"; pub struct ReadDB { - f: BufReader, + f: FramedRead, LengthDelimitedCodec>, h: ReadHeader, bin: BincodeOptions, } impl ReadDB { - pub fn open(path: &str, config: &'static Config) -> Result, DBError> { - let file = match File::open(path) { + pub async fn open(path: &str, config: &'static Config) -> Result, DBError> { + let file = match File::open(path).await { Ok(file) => file, Err(err) => match err.kind() { std::io::ErrorKind::NotFound => { @@ -53,24 +56,31 @@ impl ReadDB { }; let mut ret = ReadDB { - f: BufReader::new(file), + f: FramedRead::new(BufReader::new(file), LengthDelimitedCodec::new()), h: BTreeMap::default(), bin: bincode_options(), }; - match ret.read::() { - Ok(signature) => { + match ret.read::().await { + Some(Ok(signature)) => { if DB_SIGNATURE == signature { Ok(()) } else { Err(DBError::Error("database is not a reaction database".into())) } } - Err(err) => Err(DBError::Error(format!("reading database signature: {err}"))), + Some(Err(err)) => Err(DBError::Error(format!("reading database signature: {err}"))), + None => Err(DBError::Error( + "EOF while reading database header".to_string(), + )), }?; let db_header = ret .read::() + .await + .unwrap_or(Err(DBError::Error( + "EOF before database header".to_string(), + ))) .map_err(|err| DBError::Error(format!("while reading database header: {err}")))?; ret.h = db_header @@ -81,40 +91,40 @@ impl ReadDB { Ok(Some(ret)) } - fn read(&mut self) -> Result { - let decoded = self.bin.deserialize_from::<_, T>(&mut self.f)?; - debug!("reading this: {:?}", &decoded); - Ok(decoded) - } -} - -impl Iterator for ReadDB { - type Item = Result; - - fn next(&mut self) -> Option { - let res = self.read::(); - match res { - Ok(item) => Some(item.to(&self.h)), - Err(err) => match *err { - bincode::ErrorKind::Io(err) => match err.kind() { - io::ErrorKind::UnexpectedEof => None, - _ => Some(Err(err.into())), - }, - _ => Some(Err(err.into())), - }, + async fn read(&mut self) -> Option> { + match self.f.next().await? { + Err(err) => Some(Err(err.into())), + Ok(encoded) => Some({ + match self.bin.deserialize::(&encoded) { + Ok(decoded) => { + debug!("reading this: {:?}", decoded); + Ok(decoded) + } + Err(err) => Err(err.into()), + } + }), } } + + // FIXME use channel instead? + pub async fn next(&mut self) -> Option> { + let res = self.read::().await; + res.map(|item| match item { + Ok(item) => item.to(&self.h), + Err(err) => Err(err), + }) + } } pub struct WriteDB { - f: BufWriter, + f: FramedWrite, LengthDelimitedCodec>, h: WriteHeader, bin: BincodeOptions, } impl WriteDB { - pub fn create(path: &str, config: &'static Config) -> Self { - let file = match File::create(path) { + pub async fn create(path: &str, config: &'static Config) -> Self { + let file = match File::create(path).await { Ok(file) => file, Err(err) => { error!("Failed to create DB: {}", err); @@ -123,12 +133,12 @@ impl WriteDB { }; let mut ret = WriteDB { - f: BufWriter::new(file), + f: FramedWrite::new(BufWriter::new(file), LengthDelimitedCodec::new()), h: BTreeMap::default(), bin: bincode_options(), }; - if let Err(err) = ret._write(DB_SIGNATURE) { + if let Err(err) = ret._write(DB_SIGNATURE).await { error!("Failed to write to DB: {}", err); exit(1); } @@ -143,7 +153,7 @@ impl WriteDB { .enumerate() .collect(); - if let Err(err) = ret._write(&database_header) { + if let Err(err) = ret._write(&database_header).await { error!("Failed to write to DB: {}", err); exit(1); } @@ -158,20 +168,20 @@ impl WriteDB { ret } - pub fn write(&mut self, entry: LogEntry) -> Result<(), DBError> { + pub async fn write(&mut self, entry: LogEntry) -> Result<(), DBError> { let computed = ComputedLogEntry::from(entry, &self.h)?; - self._write(computed) + self._write(computed).await } - fn _write(&mut self, data: T) -> Result<(), DBError> { + async fn _write(&mut self, data: T) -> Result<(), DBError> { let encoded = self.bin.serialize(&data)?; // debug!("writing this: {:?}, {:?}", &data, &encoded); - self.f.write_all(&encoded)?; + self.f.send(Bytes::from(encoded)).await?; Ok(()) } - pub fn flush(&mut self) -> io::Result<()> { - self.f.flush() + pub async fn flush(&mut self) -> io::Result<()> { + self.f.flush().await } } diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index 270d77e..04e45f1 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -1,15 +1,14 @@ use std::{ collections::{BTreeMap, HashMap}, fmt::Debug, - fs, io, + fs, process::exit, - sync::mpsc::{Receiver, Sender}, - thread, }; use chrono::{Local, TimeDelta}; use log::{debug, error, info, warn}; use thiserror::Error; +use tokio::{sync::mpsc, task}; use super::MatchManagerInput; use crate::concepts::{ActionFilter, Config, Filter, LogEntry, Match, Time}; @@ -32,7 +31,7 @@ pub enum DBError { #[error("decode error: {0}")] BincodeError(#[from] bincode::Error), #[error("io error: {0}")] - IOError(#[from] io::Error), + IOError(#[from] tokio::io::Error), #[error("{0}")] Error(String), } @@ -46,7 +45,7 @@ pub enum DatabaseManagerInput { // Just discovering macros, let me be useless macro_rules! write_or_die { ($db:expr, $entry:expr) => { - if let Err(err) = $db.write($entry) { + if let Err(err) = $db.write($entry).await { error!("Could not write to DB: {}", err); exit(1); } @@ -54,20 +53,20 @@ macro_rules! write_or_die { } macro_rules! flush_or_die { ($db:expr) => { - if let Err(err) = $db.flush() { - error!("Could not write to DB: {}", err); + if let Err(err) = $db.flush().await { + error!("Could not flush DB: {}", err); exit(1); } }; } /// First rotates the database, then spawns the database thread -pub fn database_manager( +pub async fn database_manager( config: &'static Config, - log_rx: Receiver, - matches_tx: Sender, -) -> thread::JoinHandle<()> { - let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)) { + mut log_rx: mpsc::Receiver, + matches_tx: mpsc::Sender, +) -> task::JoinHandle<()> { + let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)).await { Ok(dbs) => dbs, Err(err) => { error!("while rotating databases on start: {}", err); @@ -75,9 +74,9 @@ pub fn database_manager( } }; - thread::spawn(move || { + task::spawn(async move { let mut cpt = 0; - for order in log_rx.iter() { + while let Some(order) = log_rx.recv().await { match order { DatabaseManagerInput::Flush(entry) => write_or_die!(flush_db, entry), DatabaseManagerInput::Log(entry) => { @@ -89,7 +88,7 @@ pub fn database_manager( flush_or_die!(flush_db); drop(log_db); drop(flush_db); - (log_db, flush_db) = match rotate_db(config, None) { + (log_db, flush_db) = match rotate_db(config, None).await { Ok(dbs) => dbs, Err(err) => { error!( @@ -108,43 +107,44 @@ pub fn database_manager( }) } -fn rotate_db( +async fn rotate_db( config: &'static Config, - matches_tx: Option>, + matches_tx: Option>, ) -> Result<(WriteDB, WriteDB), DBError> { info!("Rotating database..."); - let res = _rotate_db(config, &matches_tx); + let res = _rotate_db(config, &matches_tx).await; if let Some(tx) = matches_tx { #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(MatchManagerInput::EndOfStartup).unwrap(); + tx.send(MatchManagerInput::EndOfStartup).await.unwrap(); } info!("Rotated database"); res } -fn _rotate_db( +async fn _rotate_db( config: &'static Config, - matches_tx: &Option>, + matches_tx: &Option>, ) -> Result<(WriteDB, WriteDB), DBError> { - let mut log_read_db = match ReadDB::open(LOG_DB_NAME, config)? { + // TODO asyncify this + let mut log_read_db = match ReadDB::open(LOG_DB_NAME, config).await? { Some(db) => db, None => { - return Ok(( + return Ok(tokio::join!( WriteDB::create(LOG_DB_NAME, config), WriteDB::create(FLUSH_DB_NAME, config), )); } }; - let mut flush_read_db = match ReadDB::open(FLUSH_DB_NAME, config)? { + let mut flush_read_db = match ReadDB::open(FLUSH_DB_NAME, config).await? { Some(db) => db, None => { warn!( "Strange! Found a {} but no {}, opening /dev/null instead", LOG_DB_NAME, FLUSH_DB_NAME ); - match ReadDB::open("/dev/null", config)? { + match ReadDB::open("/dev/null", config).await? { Some(db) => db, None => { return Err(DBError::Error("/dev/null is not accessible".into())); @@ -153,14 +153,15 @@ fn _rotate_db( } }; - let mut log_write_db = WriteDB::create(LOG_DB_NEW_NAME, config); + let mut log_write_db = WriteDB::create(LOG_DB_NEW_NAME, config).await; __rotate_db( matches_tx, &mut log_read_db, &mut flush_read_db, &mut log_write_db, - ); + ) + .await; drop(log_read_db); drop(flush_read_db); @@ -176,12 +177,12 @@ fn _rotate_db( return Err(DBError::Error(format!("Failed to delete old DB: {}", err))); } - let flush_write_db = WriteDB::create(FLUSH_DB_NAME, config); + let flush_write_db = WriteDB::create(FLUSH_DB_NAME, config).await; Ok((log_write_db, flush_write_db)) } -fn __rotate_db( - matches_tx: &Option>, +async fn __rotate_db( + matches_tx: &Option>, log_read_db: &mut ReadDB, flush_read_db: &mut ReadDB, log_write_db: &mut WriteDB, @@ -192,7 +193,7 @@ fn __rotate_db( // Read flushes let mut flushes: BTreeMap<&'static Filter, BTreeMap> = BTreeMap::new(); - for flush_entry in flush_read_db { + while let Some(flush_entry) = flush_read_db.next().await { match flush_entry { Ok(entry) => { let matches_map = flushes.entry(entry.f).or_default(); @@ -208,7 +209,7 @@ fn __rotate_db( let now = Local::now(); // Read matches - for log_entry in log_read_db { + while let Some(log_entry) = log_read_db.next().await { match log_entry { Ok(mut entry) => { // Check if number of patterns is in sync @@ -248,6 +249,7 @@ fn __rotate_db( debug!("DB sending match from DB: {:?}", entry.m); #[allow(clippy::unwrap_used)] // propagating panics is ok tx.send(MatchManagerInput::Match(entry.clone().into())) + .await .unwrap(); } diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs index 1276c72..2eeaee0 100644 --- a/rust/src/daemon/execs.rs +++ b/rust/src/daemon/execs.rs @@ -1,80 +1,72 @@ -use std::{ - collections::BTreeMap, - process::Stdio, - sync::mpsc::{Receiver, Sender, SyncSender}, -}; +use std::{collections::BTreeMap, process::Stdio, sync::Arc}; use chrono::Local; use log::{error, info}; -use timer::MessageTimer; - -use crate::{ - concepts::{Action, ActionFilter, Config, LogEntry, Order, MAT}, - utils::ThreadPool, +use tokio::{ + sync::{mpsc, oneshot, Semaphore}, + time, }; +use crate::concepts::{Action, ActionFilter, Config, LogEntry, Order, MAT}; + use super::{ database::DatabaseManagerInput, statemap::{FilterOptions, StateMap, StateMapTrait}, }; -#[derive(Clone)] +#[derive(Debug)] pub enum ExecsManagerInput { Exec(MAT), ExecPending(MAT), - Order(Order, FilterOptions, SyncSender), + Order(Order, FilterOptions, oneshot::Sender), Stop, } type ExecsMap = StateMap; -pub fn execs_manager( +pub async fn execs_manager( config: &'static Config, - exec_rx: Receiver, - exec_tx: Sender, - log_tx: SyncSender, + mut exec_rx: mpsc::Receiver, + exec_tx: mpsc::Sender, + log_tx: mpsc::Sender, ) { - // Initialize a ThreadPool only when concurrency hasn't been disabled - let thread_pool = if config.concurrency() > 1 { - Some(ThreadPool::new(config.concurrency())) + // FIXME replace with TryStreamExt::try_for_each_concurrent? + let semaphore = if config.concurrency() > 0 { + Some(Arc::new(Semaphore::new(config.concurrency()))) } else { None }; let exec_now = |mat: MAT| { - let mut closure = { + let semaphore = semaphore.clone(); + tokio::spawn(async move { let action = mat.o; + // Wait for semaphore's permission, if it is Some + let _permit = match semaphore { + Some(semaphore) => Some(semaphore.acquire_owned().await.unwrap()), + None => None, + }; + // Construct command let mut command = action.exec(&mat.m); - move || { - info!("{}: run [{:?}]", &action, command); - if let Err(err) = command - .stdin(Stdio::null()) - .stderr(Stdio::null()) - .stdout(Stdio::piped()) - .status() - { - error!("{}: run [{:?}], code {}", &action, command, err); - } + info!("{}: run [{:?}]", &action, command); + if let Err(err) = command + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .stdout(Stdio::piped()) + .status() + .await + { + error!("{}: run [{:?}], code {}", &action, command, err); } - }; - - // Execute command either in the ThreadPool or directly - match &thread_pool { - Some(thread_pool) => { - thread_pool.execute(closure); - } - None => closure(), - } + }); }; let mut execs: ExecsMap = BTreeMap::new(); - let timer = MessageTimer::new(exec_tx); - - for mat in exec_rx.iter() { + while let Some(mat) = exec_rx.recv().await { match mat { ExecsManagerInput::Exec(mat) => { let now = Local::now(); @@ -82,9 +74,19 @@ pub fn execs_manager( exec_now(mat); } else { execs.add(&mat); - let guard = timer - .schedule_with_date(mat.t, ExecsManagerInput::ExecPending(mat.clone())); - guard.ignore(); + { + let mat = mat.clone(); + let exec_tx = exec_tx.clone(); + tokio::spawn(async move { + let dur = mat.t - Local::now(); + time::sleep(dur.to_std().unwrap()).await; + #[allow(clippy::unwrap_used)] // propagating panics is ok + exec_tx + .send(ExecsManagerInput::ExecPending(mat.clone())) + .await + .unwrap(); + }); + } } } ExecsManagerInput::ExecPending(mat) => { @@ -129,6 +131,7 @@ pub fn execs_manager( f: filter, t: mat.t, })) + .await .unwrap(); } } @@ -154,8 +157,4 @@ pub fn execs_manager( } } } - - if let Some(thread_pool) = thread_pool { - thread_pool.join(); - } } diff --git a/rust/src/daemon/matches.rs b/rust/src/daemon/matches.rs index 89fb0d1..c367ae8 100644 --- a/rust/src/daemon/matches.rs +++ b/rust/src/daemon/matches.rs @@ -1,11 +1,11 @@ -use std::{ - collections::BTreeMap, - sync::mpsc::{Receiver, Sender, SyncSender}, -}; +use std::collections::BTreeMap; use chrono::Local; use log::debug; -use timer::MessageTimer; +use tokio::{ + sync::{mpsc, oneshot}, + time, +}; use super::{ database::DatabaseManagerInput, @@ -14,30 +14,28 @@ use super::{ }; use crate::concepts::{ActionFilter, Filter, LogEntry, Order, MFT}; -#[derive(Clone)] +#[derive(Debug)] pub enum MatchManagerInput { Match(MFT), Unmatch(MFT), - Order(Order, FilterOptions, SyncSender), + Order(Order, FilterOptions, oneshot::Sender), EndOfStartup, Stop, } pub type MatchesMap = StateMap; -pub fn matches_manager( - match_rx: Receiver, - match_tx: Sender, - action_tx: Sender, - log_tx: SyncSender, +pub async fn matches_manager( + mut match_rx: mpsc::Receiver, + match_tx: mpsc::Sender, + action_tx: mpsc::Sender, + log_tx: mpsc::Sender, ) { let mut matches: MatchesMap = BTreeMap::new(); - let timer = MessageTimer::new(match_tx); - let mut startup = true; - for mft in match_rx.iter() { + while let Some(mft) = match_rx.recv().await { for (filter, map) in matches.iter() { debug!("MATCHES {:?} {:?}", filter.full_name(), map.keys()); } @@ -54,13 +52,21 @@ pub fn matches_manager( // Add new match matches.add(&mft); // Remove match when expired - // retry_duration is always Some() after filter's setup - #[allow(clippy::unwrap_used)] - let guard = timer.schedule_with_delay( - mft.t - Local::now() + mft.o.retry_duration().unwrap(), - MatchManagerInput::Unmatch(mft.clone()), - ); - guard.ignore(); + { + let mft = mft.clone(); + let match_tx = match_tx.clone(); + tokio::spawn(async move { + #[allow(clippy::unwrap_used)] + // retry_duration is always Some() after filter's setup + let dur = mft.t - Local::now() + mft.o.retry_duration().unwrap(); + time::sleep(dur.to_std().unwrap()).await; + #[allow(clippy::unwrap_used)] // propagating panics is ok + match_tx + .send(MatchManagerInput::Unmatch(mft)) + .await + .unwrap(); + }); + } matches.get_times(&mft) >= retry as usize } @@ -72,7 +78,7 @@ pub fn matches_manager( if mft.o.retry().is_some() { matches.rm_times(&mft); } - mft.o.send_actions(&mft.m, mft.t, &action_tx); + mft.o.send_actions(&mft.m, mft.t, &action_tx).await; } if !startup { @@ -84,6 +90,7 @@ pub fn matches_manager( f: mft.o, t: mft.t, })) + .await .unwrap(); } } @@ -114,6 +121,7 @@ pub fn matches_manager( f: mft.o, t: mft.t, })) + .await .unwrap(); } } diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index 80c9b0c..2e450e9 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -4,14 +4,13 @@ use std::{ process::exit, sync::{ atomic::{AtomicBool, Ordering}, - mpsc::{channel, sync_channel}, Arc, }, - thread, }; use log::{error, info}; use socket::socket_manager; +use tokio::sync::{mpsc, oneshot}; use crate::concepts::Config; use database::database_manager; @@ -27,8 +26,7 @@ mod matches; mod socket; mod statemap; -#[allow(unused_variables)] -pub fn daemon(config_path: PathBuf, socket: PathBuf) -> bool { +pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> bool { let config: &'static Config = match Config::from_file(&config_path) { Ok(config) => Box::leak(Box::new(config)), Err(err) => { @@ -43,53 +41,58 @@ pub fn daemon(config_path: PathBuf, socket: PathBuf) -> bool { } let mut stream_process_child_handles = Vec::new(); - let mut stream_thread_handles = Vec::new(); + let mut stream_task_handles = Vec::new(); - // FIXME replace those channel() by sync_channel(1) - // Unbounded channels can eat a lot of RAM - // Is there a timer::MessageTimer implementation for sync_channel? - let (match_tx, match_rx) = channel(); - let (exec_tx, exec_rx) = channel(); - let (log_tx, log_rx) = sync_channel(1); + let (match_tx, match_rx) = mpsc::channel(10); + let (exec_tx, exec_rx) = mpsc::channel(10); + let (log_tx, log_rx) = mpsc::channel(10); - let matches_manager_thread_handle = { + // TODO use this for a more idiomatic shutdown signal + // let (shutdown_tx, shutdown_rx) = watch::channel(false); + + let matches_manager_task_handle = { let match_tx_matches = match_tx.clone(); let exec_tx_matches = exec_tx.clone(); let log_tx_matches = log_tx.clone(); - thread::spawn(move || { - matches_manager(match_rx, match_tx_matches, exec_tx_matches, log_tx_matches) + tokio::spawn(async move { + matches_manager(match_rx, match_tx_matches, exec_tx_matches, log_tx_matches).await }) }; - let execs_manager_thread_handle = { + let execs_manager_task_handle = { let exec_tx_execs = exec_tx.clone(); - thread::spawn(move || execs_manager(config, exec_rx, exec_tx_execs, log_tx)) + tokio::spawn(async move { execs_manager(config, exec_rx, exec_tx_execs, log_tx).await }) }; - let database_manager_thread_handle = { + let database_manager_task_handle = { let match_tx_database = match_tx.clone(); - // The `thread::spawn` is done in the function, after database rotation is finished - database_manager(config, log_rx, match_tx_database) + // The `task::spawn` is done in the function, after database rotation is finished + database_manager(config, log_rx, match_tx_database).await }; { let match_tx_socket = match_tx.clone(); let exec_tx_socket = exec_tx.clone(); let socket = socket.to_owned(); - thread::spawn(move || socket_manager(config, socket, match_tx_socket, exec_tx_socket)); + tokio::spawn(async move { + socket_manager(config, socket, match_tx_socket, exec_tx_socket).await + }); } for stream in config.streams().values() { let match_tx = match_tx.clone(); - let (child_tx, child_rx) = sync_channel(0); + let (child_tx, child_rx) = oneshot::channel(); - stream_thread_handles.push(thread::spawn(move || stream.manager(child_tx, match_tx))); + stream_task_handles.push(tokio::spawn(async move { + stream.manager(child_tx, match_tx).await + })); - if let Ok(Some(child)) = child_rx.recv() { + if let Ok(Some(child)) = child_rx.await { stream_process_child_handles.push(child); } } + // TODO replace by async things let signal_received = Arc::new(AtomicBool::new(false)); { // Handle SIGINT, SIGTERM, SIGHUP @@ -117,23 +120,24 @@ pub fn daemon(config_path: PathBuf, socket: PathBuf) -> bool { } // Wait for all streams to quit - for thread_handle in stream_thread_handles { - let _ = thread_handle.join(); + for task_handle in stream_task_handles { + let _ = task_handle.await; } #[allow(clippy::unwrap_used)] // propagating panics is ok - match_tx.send(MatchManagerInput::Stop).unwrap(); - let _ = matches_manager_thread_handle.join(); + match_tx.send(MatchManagerInput::Stop).await.unwrap(); + let _ = matches_manager_task_handle.await; #[allow(clippy::unwrap_used)] // propagating panics is ok - exec_tx.send(ExecsManagerInput::Stop).unwrap(); - let _ = execs_manager_thread_handle.join(); + exec_tx.send(ExecsManagerInput::Stop).await.unwrap(); + let _ = execs_manager_task_handle.await; - let _ = database_manager_thread_handle.join(); + let _ = database_manager_task_handle.await; let stop_ok = config.stop(); // not waiting for the socket_manager to finish, sorry + // TODO make it listen on shutdown_rx if let Err(err) = fs::remove_file(socket) { error!("failed to remove socket: {}", err); } diff --git a/rust/src/daemon/socket.rs b/rust/src/daemon/socket.rs index 8a7a0da..e19ff36 100644 --- a/rust/src/daemon/socket.rs +++ b/rust/src/daemon/socket.rs @@ -1,18 +1,18 @@ -use std::{ - collections::BTreeMap, - fs, io, - os::unix::net::{UnixListener, UnixStream}, - path::PathBuf, - process::exit, - sync::{ - mpsc::{sync_channel, Sender}, - Arc, - }, -}; +use std::{collections::BTreeMap, fs, io, path::PathBuf, process::exit, sync::Arc}; use bincode::Options; +use futures::{SinkExt, StreamExt}; use log::{error, warn}; use regex::Regex; +use tokio::{ + join, + net::UnixListener, + sync::{mpsc, oneshot}, +}; +use tokio_util::{ + bytes::Bytes, + codec::{Framed, LengthDelimitedCodec}, +}; use crate::{ concepts::{ @@ -54,10 +54,10 @@ fn open_socket(path: PathBuf) -> Result { err_str!(UnixListener::bind(path)) } -fn answer_order( +async fn answer_order( config: &'static Config, - match_tx: &Sender, - exec_tx: &Sender, + match_tx: &mpsc::Sender, + exec_tx: &mpsc::Sender, options: ClientRequest, ) -> Result { // Compute options @@ -98,25 +98,27 @@ fn answer_order( }; // ask for matches clone - let matches = { - let (m_tx, m_rx) = sync_channel(0); + let filtering_options2 = filtering_options.clone(); + let matches = async move { + let (m_tx, m_rx) = oneshot::channel(); #[allow(clippy::unwrap_used)] // propagating panics is ok match_tx .send(MatchManagerInput::Order( options.order, - filtering_options.clone(), + filtering_options2, m_tx, )) + .await .unwrap(); #[allow(clippy::unwrap_used)] // propagating panics is ok - m_rx.recv().unwrap() + m_rx.await.unwrap() }; // ask for execs clone - let execs = { - let (e_tx, e_rx) = sync_channel(0); + let execs = async move { + let (e_tx, e_rx) = oneshot::channel(); #[allow(clippy::unwrap_used)] // propagating panics is ok exec_tx @@ -125,12 +127,15 @@ fn answer_order( filtering_options, e_tx, )) + .await .unwrap(); #[allow(clippy::unwrap_used)] // propagating panics is ok - e_rx.recv().unwrap() + e_rx.await.unwrap() }; + let (matches, execs) = join!(matches, execs); + // Transform matches and execs into a ClientStatus let cs: ClientStatus = matches .into_iter() @@ -191,11 +196,11 @@ macro_rules! or_next { }; } -pub fn socket_manager( +pub async fn socket_manager( config: &'static Config, socket: PathBuf, - match_tx: Sender, - exec_tx: Sender, + match_tx: mpsc::Sender, + exec_tx: mpsc::Sender, ) { let listener = match open_socket(socket) { Ok(l) => l, @@ -206,22 +211,35 @@ pub fn socket_manager( }; let bin = bincode_options(); - for try_conn in listener.incoming() { + loop { + let try_conn = listener.accept().await; match try_conn { - Ok(conn) => { - let conn2 = or_next!("failed to clone stream", conn.try_clone()); - // read request + Ok((conn, _)) => { + let mut transport = Framed::new(conn, LengthDelimitedCodec::new()); + // Decode + let received = transport.next().await; + let encoded_request = match received { + Some(r) => or_next!("while reading request", r), + None => { + error!("failed to answer client: client sent no request"); + continue; + } + }; let request = or_next!( - "invalid message received: ", - bin.deserialize_from::(conn) + "failed to decode request", + bin.deserialize(&encoded_request) ); - let response = answer_order(config, &match_tx, &exec_tx, request); + // Process + let response = match answer_order(config, &match_tx, &exec_tx, request).await { + Ok(res) => DaemonResponse::Order(res), + Err(err) => DaemonResponse::Err(err), + }; + // Encode + let encoded_response = + or_next!("failed to serialize response", bin.serialize(&response)); or_next!( "failed to send response:", - match response { - Ok(res) => bin.serialize_into(conn2, &DaemonResponse::Order(res)), - Err(err) => bin.serialize_into(conn2, &DaemonResponse::Err(err)), - } + transport.send(Bytes::from(encoded_response)).await ); } Err(err) => error!("failed to open connection from cli: {err}"), diff --git a/rust/src/daemon/statemap.rs b/rust/src/daemon/statemap.rs index 7d83a55..0af8142 100644 --- a/rust/src/daemon/statemap.rs +++ b/rust/src/daemon/statemap.rs @@ -7,7 +7,7 @@ use regex::Regex; use crate::concepts::{ActionFilter, Match, Pattern, Time, MT}; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct FilterOptions { pub stream_name: Option, pub filter_name: Option, diff --git a/rust/src/main.rs b/rust/src/main.rs index c4031b0..24eb0e7 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -49,7 +49,9 @@ fn main() { loglevel: _, socket, } => { - exit(match daemon(config, socket) { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let daemon_exit = runtime.block_on(async move { daemon(config, socket).await }); + exit(match daemon_exit { true => 0, false => 1, }); diff --git a/rust/src/utils/mod.rs b/rust/src/utils/mod.rs index 49c2207..507bd98 100644 --- a/rust/src/utils/mod.rs +++ b/rust/src/utils/mod.rs @@ -1,12 +1,15 @@ pub mod cli; pub mod logger; mod parse_duration; -mod threadpool; + +use std::marker::PhantomData; use bincode::Options; pub use logger::SimpleLogger; pub use parse_duration::parse_duration; -pub use threadpool::ThreadPool; +use serde::de::DeserializeOwned; +use thiserror::Error; +use tokio_util::codec::{Decoder, LengthDelimitedCodec}; pub type BincodeOptions = bincode::config::WithOtherIntEncoding< bincode::config::DefaultOptions, @@ -15,3 +18,46 @@ pub type BincodeOptions = bincode::config::WithOtherIntEncoding< pub fn bincode_options() -> BincodeOptions { bincode::DefaultOptions::new().with_varint_encoding() } + +#[derive(Error, Debug)] +pub enum DecodeError { + #[error("{0}")] + Io(#[from] tokio::io::Error), + #[error("{0}")] + Bincode(#[from] bincode::Error), +} + +pub struct LengthPrefixedBincode { + inner: LengthDelimitedCodec, + bin: BincodeOptions, + _marker: PhantomData, +} + +impl LengthPrefixedBincode { + pub fn new() -> Self { + LengthPrefixedBincode { + inner: LengthDelimitedCodec::new(), + bin: bincode_options(), + _marker: PhantomData, + } + } +} + +impl Decoder for LengthPrefixedBincode { + type Item = T; + type Error = DecodeError; + + fn decode( + &mut self, + src: &mut tokio_util::bytes::BytesMut, + ) -> Result, Self::Error> { + match self.inner.decode(src) { + Ok(Some(data)) => match self.bin.deserialize(&data) { + Ok(thing) => Ok(Some(thing)), + Err(err) => Err(err.into()), + }, + Ok(None) => Ok(None), + Err(err) => Err(err.into()), + } + } +} diff --git a/rust/src/utils/threadpool.rs b/rust/src/utils/threadpool.rs deleted file mode 100644 index d9bc755..0000000 --- a/rust/src/utils/threadpool.rs +++ /dev/null @@ -1,68 +0,0 @@ -use std::{ - sync::{ - mpsc::{channel, Receiver, Sender}, - Arc, Mutex, - }, - thread::{spawn, JoinHandle}, -}; - -type Job = Box; - -pub struct ThreadPool { - workers: Vec, - sender: Sender, -} - -impl ThreadPool { - pub fn new(size: usize) -> ThreadPool { - assert!(size > 0); - - let (sender, receiver) = channel(); - let receiver = Arc::new(Mutex::new(receiver)); - - let mut workers = Vec::with_capacity(size); - - for _ in 0..size { - workers.push(Worker::new(Arc::clone(&receiver))); - } - - ThreadPool { workers, sender } - } - - pub fn execute(&self, f: F) - where - F: FnOnce() + Send + 'static, - { - let job = Box::new(f); - - #[allow(clippy::unwrap_used)] // propagating panics is ok - self.sender.send(job).unwrap(); - } - - pub fn join(self) { - drop(self.sender); - for worker in self.workers { - #[allow(clippy::unwrap_used)] // propagating panics is ok - worker.thread.join().unwrap(); - } - } -} - -struct Worker { - thread: JoinHandle<()>, -} - -impl Worker { - fn new(receiver: Arc>>) -> Worker { - let thread = spawn(move || loop { - #[allow(clippy::unwrap_used)] // propagating panics is ok - let received = receiver.lock().unwrap().recv(); - match received { - Ok(job) => job(), - Err(_) => break, - } - }); - - Worker { thread } - } -} From 2c5a78103654a6f766aa1492cef79f521642d54c Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 13 Oct 2024 12:00:00 +0200 Subject: [PATCH 057/435] Make more things async --- rust/Cargo.lock | 196 +----------------------------------- rust/Cargo.toml | 1 - rust/src/client/mod.rs | 113 +++++++++++---------- rust/src/concepts/action.rs | 2 +- rust/src/concepts/stream.rs | 55 ++++++---- rust/src/daemon/execs.rs | 3 +- rust/src/daemon/matches.rs | 2 +- rust/src/daemon/mod.rs | 57 +++++++---- rust/src/main.rs | 29 +++--- rust/src/utils/mod.rs | 6 ++ rust/tests/simple.rs | 14 +-- 11 files changed, 168 insertions(+), 310 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 1af746b..2b40187 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -100,97 +100,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "async-channel" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-io" -version = "2.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444b0228950ee6501b3568d3c93bf1176a1fdbc3b758dcd9475046d30f4dc7e8" -dependencies = [ - "async-lock", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite", - "parking", - "polling", - "rustix", - "slab", - "tracing", - "windows-sys 0.59.0", -] - -[[package]] -name = "async-lock" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" -dependencies = [ - "event-listener", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-process" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" -dependencies = [ - "async-channel", - "async-io", - "async-lock", - "async-signal", - "async-task", - "blocking", - "cfg-if", - "event-listener", - "futures-lite", - "rustix", - "tracing", -] - -[[package]] -name = "async-signal" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3" -dependencies = [ - "async-io", - "async-lock", - "atomic-waker", - "cfg-if", - "futures-core", - "futures-io", - "rustix", - "signal-hook-registry", - "slab", - "windows-sys 0.59.0", -] - -[[package]] -name = "async-task" -version = "4.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.3.0" @@ -233,19 +142,6 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" -[[package]] -name = "blocking" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" -dependencies = [ - "async-channel", - "async-task", - "futures-io", - "futures-lite", - "piper", -] - [[package]] name = "bumpalo" version = "3.16.0" @@ -345,27 +241,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "core-foundation-sys" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" -[[package]] -name = "crossbeam-utils" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" - [[package]] name = "ctrlc" version = "3.4.4" @@ -392,27 +273,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "event-listener" -version = "5.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" -dependencies = [ - "event-listener", - "pin-project-lite", -] - [[package]] name = "fastrand" version = "2.1.1" @@ -467,19 +327,6 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" -[[package]] -name = "futures-lite" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - [[package]] name = "futures-macro" version = "0.3.30" @@ -545,12 +392,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" -[[package]] -name = "hermit-abi" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" - [[package]] name = "iana-time-zone" version = "0.1.60" @@ -749,7 +590,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", "wasi", "windows-sys 0.52.0", @@ -792,7 +633,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", ] @@ -817,12 +658,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - [[package]] name = "parking_lot" version = "0.12.3" @@ -891,32 +726,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "piper" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" -dependencies = [ - "atomic-waker", - "fastrand", - "futures-io", -] - -[[package]] -name = "polling" -version = "3.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2790cd301dec6cd3b7a025e4815cf825724a51c98dccfe6a3e55f05ffb6511" -dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi 0.4.0", - "pin-project-lite", - "rustix", - "tracing", - "windows-sys 0.59.0", -] - [[package]] name = "proc-macro2" version = "1.0.86" @@ -939,7 +748,6 @@ dependencies = [ name = "reaction" version = "0.1.0" dependencies = [ - "async-process", "bincode", "chrono", "clap", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 20dc7f8..ab7a96c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -23,7 +23,6 @@ thiserror = "1.0.63" timer = "0.2.0" # async-related -async-process = "2.3.0" futures = "0.3.30" tokio = { version = "1.40.0", features = ["full"] } tokio-util = { version = "0.7.12", features = ["codec"] } diff --git a/rust/src/client/mod.rs b/rust/src/client/mod.rs index 7df019e..550ae52 100644 --- a/rust/src/client/mod.rs +++ b/rust/src/client/mod.rs @@ -1,59 +1,73 @@ use std::{ collections::BTreeSet, error::Error, - io::{stdin, stdout, BufRead, BufReader}, - os::unix::net::UnixStream, + io::{stdin, BufRead, BufReader}, path::PathBuf, - process::exit, sync::Arc, }; use bincode::Options; -use log::{error, info}; +use futures::{SinkExt, StreamExt}; +use log::info; use regex::Regex; +use tokio::{io::AsyncWriteExt, net::UnixStream}; +use tokio_util::{ + bytes::Bytes, + codec::{Framed, LengthDelimitedCodec}, +}; use crate::{ concepts::{ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern}, utils::{bincode_options, cli::Format}, }; + macro_rules! or_quit { ($msg:expr, $expression:expr) => { - match $expression { - Ok(x) => x, - Err(err) => { - error!("failed to communicate to daemon: {}, {}", $msg, err); - exit(1); - } - } + $expression.map_err(|err| format!("failed to communicate to daemon: {}, {}", $msg, err))? }; } -fn send_retrieve(socket: &PathBuf, req: &ClientRequest) -> DaemonResponse { +async fn send_retrieve(socket: &PathBuf, req: &ClientRequest) -> Result { let bin = bincode_options(); - let conn = or_quit!("opening connection to daemon", UnixStream::connect(socket)); - let conn2 = or_quit!("failed to clone stream", conn.try_clone()); - or_quit!("failed to send request", bin.serialize_into(conn, req)); + let conn = or_quit!( + "opening connection to daemon", + UnixStream::connect(socket).await + ); + // Encode + let mut transport = Framed::new(conn, LengthDelimitedCodec::new()); + let encoded_request = or_quit!("failed to encode request", bin.serialize(req)); or_quit!( "failed to send request", - bin.deserialize_from::(conn2) + transport.send(Bytes::from(encoded_request)).await + ); + // Decode + let encoded_response = or_quit!( + "failed to read response", + transport.next().await.ok_or("empty response from server") + ); + let encoded_response = or_quit!("failed to decode response", encoded_response); + or_quit!( + "failed to decode response", + bin.deserialize(&encoded_response) ) } -fn print_status(cs: ClientStatus, format: Format) -> Result<(), Box> { - match format { - Format::JSON => serde_json::to_writer(stdout().lock(), &cs)?, - Format::YAML => serde_yaml::to_writer(stdout().lock(), &cs)?, - } +async fn print_status(cs: ClientStatus, format: Format) -> Result<(), Box> { + let encoded = match format { + Format::JSON => serde_json::to_string_pretty(&cs)?, + Format::YAML => serde_yaml::to_string(&cs)?, + }; + tokio::io::stdout().write_all(encoded.as_bytes()).await?; Ok(()) } -pub fn request( +pub async fn request( socket: PathBuf, format: Format, stream_filter: Option, patterns: Vec<(String, String)>, order: Order, -) { +) -> Result<(), Box> { let response = send_retrieve( &socket, &ClientRequest { @@ -61,29 +75,25 @@ pub fn request( stream_filter, patterns, }, - ); - match response { - DaemonResponse::Order(cs) => { - if let Err(err) = print_status(cs, format) { - error!("while printing response: {err}"); - exit(1); - } - } - DaemonResponse::Err(err) => { - error!("failed to communicate to daemon: error response: {err}"); - exit(1); - } - } + ) + .await; + match response? { + DaemonResponse::Order(cs) => print_status(cs, format) + .await + .map_err(|err| format!("while printing response: {err}")), + DaemonResponse::Err(err) => Err(format!( + "failed to communicate to daemon: error response: {err}" + )), + }?; + Ok(()) } -pub fn test_regex(config_path: PathBuf, mut regex: String, line: Option) { - let config: Config = match Config::from_file(&config_path) { - Ok(config) => config, - Err(err) => { - error!("{err}"); - exit(1); - } - }; +pub fn test_regex( + config_path: PathBuf, + mut regex: String, + line: Option, +) -> Result<(), Box> { + let config: Config = Config::from_file(&config_path)?; // Code close to Filter::setup() let mut used_patterns: BTreeSet> = BTreeSet::new(); @@ -92,24 +102,18 @@ pub fn test_regex(config_path: PathBuf, mut regex: String, line: Option) // we already `find` it, so we must be able to `rfind` it #[allow(clippy::unwrap_used)] if regex.rfind(pattern.name_with_braces()).unwrap() != index { - error!( + return Err(format!( "pattern {} present multiple times in regex", pattern.name_with_braces() - ); - exit(1); + ) + .into()); } used_patterns.insert(pattern.clone()); } regex = regex.replacen(pattern.name_with_braces(), &pattern.regex, 1); } - let compiled = match Regex::new(®ex) { - Ok(reg) => reg, - Err(err) => { - error!("regex doesn't compile: {err}"); - exit(1); - } - }; + let compiled = Regex::new(®ex).map_err(|err| format!("regex doesn't compile: {err}"))?; let match_closure = |line: String| { let mut ignored = false; @@ -148,4 +152,5 @@ pub fn test_regex(config_path: PathBuf, mut regex: String, line: Option) }; } } + Ok(()) } diff --git a/rust/src/concepts/action.rs b/rust/src/concepts/action.rs index 0f6218d..0d5c9b4 100644 --- a/rust/src/concepts/action.rs +++ b/rust/src/concepts/action.rs @@ -1,9 +1,9 @@ use std::{cmp::Ordering, collections::BTreeSet, fmt::Display, sync::Arc}; -use async_process::Command; use chrono::TimeDelta; use serde::Deserialize; +use tokio::process::Command; use super::{ActionFilter, Match, Pattern}; use crate::utils::parse_duration; diff --git a/rust/src/concepts/stream.rs b/rust/src/concepts/stream.rs index 2bcfd1c..86f1fab 100644 --- a/rust/src/concepts/stream.rs +++ b/rust/src/concepts/stream.rs @@ -1,11 +1,13 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, process::Stdio}; -use async_process::{Child, Command, Stdio}; use chrono::Local; -use futures::{io::BufReader, AsyncBufReadExt, StreamExt}; use log::{error, info}; use serde::Deserialize; -use tokio::sync::{mpsc, oneshot}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::{Child, Command}, + sync::{mpsc, oneshot}, +}; use super::{Filter, Patterns, MFT}; use crate::daemon::MatchManagerInput; @@ -91,27 +93,36 @@ impl Stream { // let main handle the child process let _ = child_tx.send(Some(child)); - while let Some(line) = lines.next().await { - let line = match line { - Ok(line) => line, - Err(_) => break, - }; - - for filter in self.filters.values() { - if let Some(match_) = filter.get_match(&line) { - #[allow(clippy::unwrap_used)] // propagating panics is ok - match_tx - .send(MatchManagerInput::Match(MFT { - m: match_, - o: filter, - t: Local::now(), - })) - .await - .unwrap(); + loop { + match lines.next_line().await { + Ok(Some(line)) => { + for filter in self.filters.values() { + if let Some(match_) = filter.get_match(&line) { + #[allow(clippy::unwrap_used)] // propagating panics is ok + match_tx + .send(MatchManagerInput::Match(MFT { + m: match_, + o: filter, + t: Local::now(), + })) + .await + .unwrap(); + } + } + } + Ok(None) => { + error!("stream {} exited: its command returned.", self.name); + break; + } + Err(err) => { + error!( + "impossible to read output from stream {}: {}", + self.name, err + ); + break; } } } - error!("stream {} exited: its command returned.", self.name); } } diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs index 2eeaee0..57b6ce1 100644 --- a/rust/src/daemon/execs.rs +++ b/rust/src/daemon/execs.rs @@ -44,6 +44,7 @@ pub async fn execs_manager( // Wait for semaphore's permission, if it is Some let _permit = match semaphore { + #[allow(clippy::unwrap_used)] // We know the semaphore is not closed Some(semaphore) => Some(semaphore.acquire_owned().await.unwrap()), None => None, }; @@ -79,7 +80,7 @@ pub async fn execs_manager( let exec_tx = exec_tx.clone(); tokio::spawn(async move { let dur = mat.t - Local::now(); - time::sleep(dur.to_std().unwrap()).await; + time::sleep(dur.to_std().expect("Duration is bigger than what's supported. Did you put an enormous after duration?")).await; #[allow(clippy::unwrap_used)] // propagating panics is ok exec_tx .send(ExecsManagerInput::ExecPending(mat.clone())) diff --git a/rust/src/daemon/matches.rs b/rust/src/daemon/matches.rs index c367ae8..4d00e2c 100644 --- a/rust/src/daemon/matches.rs +++ b/rust/src/daemon/matches.rs @@ -59,7 +59,7 @@ pub async fn matches_manager( #[allow(clippy::unwrap_used)] // retry_duration is always Some() after filter's setup let dur = mft.t - Local::now() + mft.o.retry_duration().unwrap(); - time::sleep(dur.to_std().unwrap()).await; + time::sleep(dur.to_std().expect("Duration is bigger than what's supported. Did you put an enormous retry_duration?")).await; #[allow(clippy::unwrap_used)] // propagating panics is ok match_tx .send(MatchManagerInput::Unmatch(mft)) diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index 2e450e9..d2774bd 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -1,7 +1,7 @@ use std::{ + error::Error, fs, path::PathBuf, - process::exit, sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -10,7 +10,13 @@ use std::{ use log::{error, info}; use socket::socket_manager; -use tokio::sync::{mpsc, oneshot}; +use tokio::{ + join, + process::Child, + select, + signal::unix::{signal, SignalKind}, + sync::{mpsc, oneshot}, +}; use crate::concepts::Config; use database::database_manager; @@ -26,18 +32,12 @@ mod matches; mod socket; mod statemap; -pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> bool { - let config: &'static Config = match Config::from_file(&config_path) { - Ok(config) => Box::leak(Box::new(config)), - Err(err) => { - error!("{err}"); - exit(1); - } - }; +pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> Result<(), Box> { + let config: &'static Config = + Config::from_file(&config_path).map(|config| Box::leak(Box::new(config)))?; if !config.start() { - error!("a start command failed, exiting."); - exit(1); + return Err("a start command failed, exiting.".into()); } let mut stream_process_child_handles = Vec::new(); @@ -93,6 +93,7 @@ pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> bool { } // TODO replace by async things + tokio::spawn(async move {}); let signal_received = Arc::new(AtomicBool::new(false)); { // Handle SIGINT, SIGTERM, SIGHUP @@ -102,8 +103,8 @@ pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> bool { info!("waiting for streams to finish..."); // Kill stream subprocesses - for child_handle in stream_process_child_handles.iter_mut() { - let _ = child_handle.kill(); + for child_handle in stream_process_child_handles.into_iter() { + let _ = tokio::spawn(async move { child_handle.kill().await }); } }) { match err { @@ -112,8 +113,9 @@ pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> bool { ctrlc::Error::MultipleHandlers => {} // Real error _ => { - error!("impossible to launch a signal-catching thread, exiting: {err}"); - exit(1); + return Err( + "impossible to launch a signal-catching thread, exiting: {err}".into(), + ); } } } @@ -142,5 +144,26 @@ pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> bool { error!("failed to remove socket: {}", err); } - !signal_received.load(Ordering::SeqCst) && stop_ok + if !signal_received.load(Ordering::SeqCst) { + Err("quitting because all streams finished".into()) + } else if !stop_ok { + Err("while executing stop command".into()) + } else { + Ok(()) + } +} + +async fn signals(stream_process_child_handles: Vec) -> tokio::io::Result<()> { + // TODO refaire pour que la création des channels soient sync puis que la suite soit spawn + { + let mut sighup = signal(SignalKind::hangup())?; + let mut sigint = signal(SignalKind::interrupt())?; + let mut sigterm = signal(SignalKind::terminate())?; + select! { + _ = sighup.recv() => {}, + _ = sigint.recv() => {}, + _ = sigterm.recv() => {}, + }; + } + Ok(()) } diff --git a/rust/src/main.rs b/rust/src/main.rs index 24eb0e7..23a6d22 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -2,14 +2,15 @@ use std::process::exit; use clap::Parser; -use log::Level; +use log::{error, Level}; use reaction::client::{request, test_regex}; use reaction::concepts::Order; use reaction::daemon::daemon; use reaction::utils::cli::{Cli, Command}; use reaction::utils::SimpleLogger; -fn main() { +#[tokio::main] +async fn main() { // Show a nice message when reaction panics std::panic::set_hook(Box::new(move |panic_info| { eprintln!("ERROR internal reaction error: panic"); @@ -43,35 +44,37 @@ fn main() { } } - match cli.command { + let result = match cli.command { Command::Start { config, loglevel: _, socket, - } => { - let runtime = tokio::runtime::Runtime::new().unwrap(); - let daemon_exit = runtime.block_on(async move { daemon(config, socket).await }); - exit(match daemon_exit { - true => 0, - false => 1, - }); - } + } => daemon(config, socket).await, Command::Show { socket, format, limit, patterns, - } => request(socket, format, limit, patterns, Order::Show), + } => request(socket, format, limit, patterns, Order::Show).await, Command::Flush { socket, format, limit, patterns, - } => request(socket, format, limit, patterns, Order::Flush), + } => request(socket, format, limit, patterns, Order::Flush).await, Command::TestRegex { config, regex, line, } => test_regex(config, regex, line), + }; + match result { + Ok(()) => { + exit(0); + } + Err(err) => { + error!("{err}"); + exit(1); + } } } diff --git a/rust/src/utils/mod.rs b/rust/src/utils/mod.rs index 507bd98..20e24db 100644 --- a/rust/src/utils/mod.rs +++ b/rust/src/utils/mod.rs @@ -43,6 +43,12 @@ impl LengthPrefixedBincode { } } +impl Default for LengthPrefixedBincode { + fn default() -> Self { + Self::new() + } +} + impl Decoder for LengthPrefixedBincode { type Item = T; type Error = DecodeError; diff --git a/rust/tests/simple.rs b/rust/tests/simple.rs index c8c4f3f..31c545c 100644 --- a/rust/tests/simple.rs +++ b/rust/tests/simple.rs @@ -70,8 +70,8 @@ fn get_file_content(path: &str) -> String { contents } -#[test] -fn simple() { +#[tokio::test] +async fn simple() { let dir = TempDir::new().unwrap(); env::set_current_dir(&dir).unwrap(); @@ -89,10 +89,13 @@ fn simple() { // Set the logger before running any code from the crate SimpleLogger::init(Level::Debug).unwrap(); + // Initialize tokio runtime + let runtime = tokio::runtime::Runtime::new().unwrap(); + // Run the daemon - let handle = thread::spawn(move || { - assert!(daemon(config_path.into(), socket_path.into())); - }); + let exit = + runtime.block_on(async move { daemon(config_path.into(), socket_path.into()).await }); + assert!(exit); // Run the flushes @@ -122,7 +125,6 @@ fn simple() { ); }); - handle.join().unwrap(); handle2.join().unwrap(); handle3.join().unwrap(); From 1e6e67a4b3a3b1e8e3f3d7f201dab1a237315c7a Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 13 Oct 2024 12:00:00 +0200 Subject: [PATCH 058/435] splitted channels, asyncify --- rust/Cargo.lock | 29 ------ rust/Cargo.toml | 1 - rust/src/client/mod.rs | 4 +- rust/src/concepts/filter.rs | 11 +-- rust/src/concepts/stream.rs | 7 +- rust/src/daemon/database/mod.rs | 21 ++-- rust/src/daemon/database/tests.rs | 16 +-- rust/src/daemon/execs.rs | 64 ++++++------ rust/src/daemon/matches.rs | 159 ++++++++++++++---------------- rust/src/daemon/mod.rs | 132 ++++++++++--------------- rust/src/daemon/socket.rs | 96 +++++++++--------- rust/tests/simple.rs | 28 +++--- 12 files changed, 239 insertions(+), 329 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 2b40187..46ca77b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -166,12 +166,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cfg_aliases" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" - [[package]] name = "chrono" version = "0.4.38" @@ -247,16 +241,6 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" -[[package]] -name = "ctrlc" -version = "3.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" -dependencies = [ - "nix", - "windows-sys 0.52.0", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -596,18 +580,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "nix" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -752,7 +724,6 @@ dependencies = [ "chrono", "clap", "clap_complete", - "ctrlc", "futures", "jrsonnet-evaluator", "log", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index ab7a96c..fe05f84 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -10,7 +10,6 @@ bincode = "1.3.3" chrono = { version = "0.4.38", features = ["std", "clock"] } clap = { version = "4.5.4", features = ["derive"] } clap_complete = "4.5.2" -ctrlc = { version = "3.4.4", features = ["termination"] } jrsonnet-evaluator = "0.4.2" log = { version = "0.4.22", features = ["std"] } num_cpus = "1.16.0" diff --git a/rust/src/client/mod.rs b/rust/src/client/mod.rs index 550ae52..35863d1 100644 --- a/rust/src/client/mod.rs +++ b/rust/src/client/mod.rs @@ -67,7 +67,7 @@ pub async fn request( stream_filter: Option, patterns: Vec<(String, String)>, order: Order, -) -> Result<(), Box> { +) -> Result<(), Box> { let response = send_retrieve( &socket, &ClientRequest { @@ -92,7 +92,7 @@ pub fn test_regex( config_path: PathBuf, mut regex: String, line: Option, -) -> Result<(), Box> { +) -> Result<(), Box> { let config: Config = Config::from_file(&config_path)?; // Code close to Filter::setup() diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index 4bdb7dc..e25960a 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -15,7 +15,7 @@ use super::{ messages::{Match, Time, MAT}, Action, ActionFilter, LogEntry, Pattern, Patterns, }; -use crate::{daemon::ExecsManagerInput, utils::parse_duration}; +use crate::utils::parse_duration; // Only names are serialized // Only computed fields are not deserialized @@ -201,12 +201,7 @@ impl Filter { None } - pub async fn send_actions( - &'static self, - m: &Match, - t: Time, - tx: &mpsc::Sender, - ) { + pub async fn send_actions(&'static self, m: &Match, t: Time, tx: &mpsc::Sender) { for action in self.actions.values() { let mat = MAT { m: m.clone(), @@ -214,7 +209,7 @@ impl Filter { t: t + action.after_duration().unwrap_or_default(), }; #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(ExecsManagerInput::Exec(mat)).await.unwrap(); + tx.send(mat).await.unwrap(); } } diff --git a/rust/src/concepts/stream.rs b/rust/src/concepts/stream.rs index 86f1fab..d99d61b 100644 --- a/rust/src/concepts/stream.rs +++ b/rust/src/concepts/stream.rs @@ -10,7 +10,6 @@ use tokio::{ }; use super::{Filter, Patterns, MFT}; -use crate::daemon::MatchManagerInput; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -67,7 +66,7 @@ impl Stream { pub async fn manager( &'static self, child_tx: oneshot::Sender>, - match_tx: mpsc::Sender, + match_tx: mpsc::Sender, ) { info!("{}: start {:?}", self.name, self.cmd); let mut child = match Command::new(&self.cmd[0]) @@ -100,11 +99,11 @@ impl Stream { if let Some(match_) = filter.get_match(&line) { #[allow(clippy::unwrap_used)] // propagating panics is ok match_tx - .send(MatchManagerInput::Match(MFT { + .send(MFT { m: match_, o: filter, t: Local::now(), - })) + }) .await .unwrap(); } diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index 04e45f1..ad3e514 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -10,8 +10,7 @@ use log::{debug, error, info, warn}; use thiserror::Error; use tokio::{sync::mpsc, task}; -use super::MatchManagerInput; -use crate::concepts::{ActionFilter, Config, Filter, LogEntry, Match, Time}; +use crate::concepts::{ActionFilter, Config, Filter, LogEntry, Match, Time, MFT}; mod lowlevel; mod tests; @@ -64,7 +63,7 @@ macro_rules! flush_or_die { pub async fn database_manager( config: &'static Config, mut log_rx: mpsc::Receiver, - matches_tx: mpsc::Sender, + matches_tx: mpsc::Sender, ) -> task::JoinHandle<()> { let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)).await { Ok(dbs) => dbs, @@ -109,23 +108,17 @@ pub async fn database_manager( async fn rotate_db( config: &'static Config, - matches_tx: Option>, + matches_tx: Option>, ) -> Result<(WriteDB, WriteDB), DBError> { info!("Rotating database..."); let res = _rotate_db(config, &matches_tx).await; - - if let Some(tx) = matches_tx { - #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(MatchManagerInput::EndOfStartup).await.unwrap(); - } - info!("Rotated database"); res } async fn _rotate_db( config: &'static Config, - matches_tx: &Option>, + matches_tx: &Option>, ) -> Result<(WriteDB, WriteDB), DBError> { // TODO asyncify this let mut log_read_db = match ReadDB::open(LOG_DB_NAME, config).await? { @@ -182,7 +175,7 @@ async fn _rotate_db( } async fn __rotate_db( - matches_tx: &Option>, + matches_tx: &Option>, log_read_db: &mut ReadDB, flush_read_db: &mut ReadDB, log_write_db: &mut WriteDB, @@ -248,9 +241,7 @@ async fn __rotate_db( if let Some(tx) = matches_tx { debug!("DB sending match from DB: {:?}", entry.m); #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(MatchManagerInput::Match(entry.clone().into())) - .await - .unwrap(); + tx.send(entry.clone().into()).await.unwrap(); } write_or_die!(log_write_db, entry); diff --git a/rust/src/daemon/database/tests.rs b/rust/src/daemon/database/tests.rs index aacf959..0dc64b5 100644 --- a/rust/src/daemon/database/tests.rs +++ b/rust/src/daemon/database/tests.rs @@ -9,8 +9,8 @@ use crate::{ tests::Fixture, }; -#[test] -fn write_and_read_db() { +#[tokio::test] +async fn write_and_read_db() { let config_file = Fixture::from_string( "config.jsonnet", " @@ -61,26 +61,26 @@ fn write_and_read_db() { let db_path = Fixture::empty("matches.db"); - let mut write_db = WriteDB::create(db_path.to_str().unwrap(), config); + let mut write_db = WriteDB::create(db_path.to_str().unwrap(), config).await; - assert!(write_db.write(correct_log_entry.clone()).is_ok()); - assert!(write_db.write(incorrect_log_entry).is_err()); + assert!(write_db.write(correct_log_entry.clone()).await.is_ok()); + assert!(write_db.write(incorrect_log_entry).await.is_err()); drop(write_db); - let read_db = ReadDB::open(db_path.to_str().unwrap(), config); + let read_db = ReadDB::open(db_path.to_str().unwrap(), config).await; assert!(read_db.is_ok()); let read_db = read_db.unwrap(); assert!(read_db.is_some()); let mut read_db = read_db.unwrap(); - let read_entry = read_db.next(); + let read_entry = read_db.next().await; assert!(read_entry.is_some()); let read_entry = read_entry.unwrap(); assert!(read_entry.is_ok()); assert_eq!(read_entry.unwrap(), correct_log_entry); - let read_entry = read_db.next(); + let read_entry = read_db.next().await; assert!(read_entry.is_none()); } diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs index 57b6ce1..2f65eb6 100644 --- a/rust/src/daemon/execs.rs +++ b/rust/src/daemon/execs.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, process::Stdio, sync::Arc}; use chrono::Local; use log::{error, info}; use tokio::{ - sync::{mpsc, oneshot, Semaphore}, + sync::{mpsc, Semaphore}, time, }; @@ -11,23 +11,16 @@ use crate::concepts::{Action, ActionFilter, Config, LogEntry, Order, MAT}; use super::{ database::DatabaseManagerInput, - statemap::{FilterOptions, StateMap, StateMapTrait}, + socket::SocketOrder, + statemap::{StateMap, StateMapTrait}, }; -#[derive(Debug)] -pub enum ExecsManagerInput { - Exec(MAT), - ExecPending(MAT), - Order(Order, FilterOptions, oneshot::Sender), - Stop, -} - -type ExecsMap = StateMap; +pub type ExecsMap = StateMap; pub async fn execs_manager( config: &'static Config, - mut exec_rx: mpsc::Receiver, - exec_tx: mpsc::Sender, + mut exec_rx: mpsc::Receiver, + mut socket_order_rx: mpsc::Receiver>, log_tx: mpsc::Sender, ) { // FIXME replace with TryStreamExt::try_for_each_concurrent? @@ -67,35 +60,37 @@ pub async fn execs_manager( let mut execs: ExecsMap = BTreeMap::new(); - while let Some(mat) = exec_rx.recv().await { - match mat { - ExecsManagerInput::Exec(mat) => { + let (pendings_tx, mut pendings_rx) = mpsc::channel(1); + + loop { + tokio::select! { + Some(mat) = exec_rx.recv() => { let now = Local::now(); if mat.t.lt(&now) { exec_now(mat); } else { - execs.add(&mat); + execs.add(&mat); { let mat = mat.clone(); - let exec_tx = exec_tx.clone(); + let pendings_tx = pendings_tx.clone(); tokio::spawn(async move { let dur = mat.t - Local::now(); time::sleep(dur.to_std().expect("Duration is bigger than what's supported. Did you put an enormous after duration?")).await; #[allow(clippy::unwrap_used)] // propagating panics is ok - exec_tx - .send(ExecsManagerInput::ExecPending(mat.clone())) + pendings_tx + .send(mat.clone()) .await .unwrap(); }); } } } - ExecsManagerInput::ExecPending(mat) => { + Some(mat) = pendings_rx.recv() => { if execs.rm(&mat) { exec_now(mat); } } - ExecsManagerInput::Order(order, options, tx) => { + Some((order, options, tx)) = socket_order_rx.recv() => { let filtered = execs.filtered(options); if let Order::Flush = order { @@ -140,21 +135,20 @@ pub async fn execs_manager( #[allow(clippy::unwrap_used)] // propagating panics is ok tx.send(filtered).unwrap(); } - ExecsManagerInput::Stop => { - for (action, inner_map) in execs { - if action.on_exit() { - for (match_, inner_set) in inner_map { - for _ in inner_set { - exec_now(MAT { - m: match_.clone(), - o: action, - t: Local::now(), - }); - } - } + else => break + } + + for (action, inner_map) in &mut execs { + if action.on_exit() { + for (match_, inner_set) in inner_map { + for _ in inner_set.iter() { + exec_now(MAT { + m: match_.clone(), + o: action, + t: Local::now(), + }); } } - break; } } } diff --git a/rust/src/daemon/matches.rs b/rust/src/daemon/matches.rs index 4d00e2c..b2036e8 100644 --- a/rust/src/daemon/matches.rs +++ b/rust/src/daemon/matches.rs @@ -1,104 +1,54 @@ use std::collections::BTreeMap; use chrono::Local; -use log::debug; -use tokio::{ - sync::{mpsc, oneshot}, - time, -}; +use tokio::{sync::mpsc, time}; use super::{ database::DatabaseManagerInput, - statemap::{FilterOptions, StateMap, StateMapTrait}, - ExecsManagerInput, + socket::SocketOrder, + statemap::{StateMap, StateMapTrait}, }; -use crate::concepts::{ActionFilter, Filter, LogEntry, Order, MFT}; - -#[derive(Debug)] -pub enum MatchManagerInput { - Match(MFT), - Unmatch(MFT), - Order(Order, FilterOptions, oneshot::Sender), - EndOfStartup, - Stop, -} +use crate::concepts::{Filter, LogEntry, Order, MAT, MFT}; pub type MatchesMap = StateMap; pub async fn matches_manager( - mut match_rx: mpsc::Receiver, - match_tx: mpsc::Sender, - action_tx: mpsc::Sender, + mut match_rx: mpsc::Receiver, + mut startup_match_rx: mpsc::Receiver, + mut socket_order_rx: mpsc::Receiver>, + action_tx: mpsc::Sender, log_tx: mpsc::Sender, ) { let mut matches: MatchesMap = BTreeMap::new(); - let mut startup = true; + let (unmatches_tx, mut unmatches_rx) = mpsc::channel(1); - while let Some(mft) = match_rx.recv().await { - for (filter, map) in matches.iter() { - debug!("MATCHES {:?} {:?}", filter.full_name(), map.keys()); - } - match mft { - MatchManagerInput::EndOfStartup => { - debug!("end of startup!"); - startup = false; + while let Some(mft) = startup_match_rx.recv().await { + let _ = handle_match(&mut matches, mft.clone(), &unmatches_tx, &action_tx).await; + } + + loop { + tokio::select! { + Some(mft) = match_rx.recv() => { + let exec = handle_match(&mut matches, mft.clone(), &unmatches_tx, &action_tx).await; + + #[allow(clippy::unwrap_used)] // propagating panics is ok + log_tx + .send(DatabaseManagerInput::Log(LogEntry { + exec, + m: mft.m, + f: mft.o, + t: mft.t, + })) + .await + .unwrap(); } - MatchManagerInput::Match(mft) => { - // Store matches - let exec = match mft.o.retry() { - None => true, - Some(retry) => { - // Add new match - matches.add(&mft); - // Remove match when expired - { - let mft = mft.clone(); - let match_tx = match_tx.clone(); - tokio::spawn(async move { - #[allow(clippy::unwrap_used)] - // retry_duration is always Some() after filter's setup - let dur = mft.t - Local::now() + mft.o.retry_duration().unwrap(); - time::sleep(dur.to_std().expect("Duration is bigger than what's supported. Did you put an enormous retry_duration?")).await; - #[allow(clippy::unwrap_used)] // propagating panics is ok - match_tx - .send(MatchManagerInput::Unmatch(mft)) - .await - .unwrap(); - }); - } - - matches.get_times(&mft) >= retry as usize - } - }; - - // Executing actions - if exec { - // Delete matches only if storing them - if mft.o.retry().is_some() { - matches.rm_times(&mft); - } - mft.o.send_actions(&mft.m, mft.t, &action_tx).await; - } - - if !startup { - #[allow(clippy::unwrap_used)] // propagating panics is ok - log_tx - .send(DatabaseManagerInput::Log(LogEntry { - exec, - m: mft.m, - f: mft.o, - t: mft.t, - })) - .await - .unwrap(); - } - } - MatchManagerInput::Unmatch(mft) => { + Some(mft) = unmatches_rx.recv() => { matches.rm(&mft); } - MatchManagerInput::Order(order, options, tx) => { - let filtered = matches.filtered(options); + Some((order, options, tx)) = socket_order_rx.recv() => { + // FIXME do not clone + let filtered = matches.clone().filtered(options); if let Order::Flush = order { let now = Local::now(); @@ -130,8 +80,49 @@ pub async fn matches_manager( #[allow(clippy::unwrap_used)] // propagating panics is ok tx.send(filtered).unwrap(); } - #[allow(clippy::unwrap_used)] // propagating panics is ok - MatchManagerInput::Stop => break, + else => break } } } + +async fn handle_match( + matches: &mut MatchesMap, + mft: MFT, + unmatches_tx: &mpsc::Sender, + action_tx: &mpsc::Sender, +) -> bool { + // Store matches + let exec = match mft.o.retry() { + None => true, + Some(retry) => { + // Add new match + matches.add(&mft); + // Remove match when expired + { + let mft = mft.clone(); + let unmatches_tx = unmatches_tx.clone(); + tokio::spawn(async move { + #[allow(clippy::unwrap_used)] + // retry_duration is always Some() after filter's setup + let dur = mft.t - Local::now() + mft.o.retry_duration().unwrap(); + time::sleep(dur.to_std().expect("Duration is bigger than what's supported. Did you put an enormous retry_duration?")).await; + #[allow(clippy::unwrap_used)] // propagating panics is ok + unmatches_tx.send(mft).await.unwrap(); + }); + } + + matches.get_times(&mft) >= retry as usize + } + }; + + // Executing actions + if exec { + // Delete matches only if storing them + if mft.o.retry().is_some() { + matches.rm_times(&mft); + } + mft.o.send_actions(&mft.m, mft.t, action_tx).await; + } + + exec +} diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index d2774bd..47c8c06 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -1,21 +1,12 @@ -use std::{ - error::Error, - fs, - path::PathBuf, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, -}; +use std::{error::Error, fs, path::PathBuf}; use log::{error, info}; use socket::socket_manager; use tokio::{ - join, process::Child, select, signal::unix::{signal, SignalKind}, - sync::{mpsc, oneshot}, + sync::{mpsc, oneshot, watch}, }; use crate::concepts::Config; @@ -23,16 +14,13 @@ use database::database_manager; use execs::execs_manager; use matches::matches_manager; -pub use execs::ExecsManagerInput; -pub use matches::MatchManagerInput; - mod database; mod execs; mod matches; mod socket; mod statemap; -pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> Result<(), Box> { +pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> Result<(), Box> { let config: &'static Config = Config::from_file(&config_path).map(|config| Box::leak(Box::new(config)))?; @@ -43,48 +31,55 @@ pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> Result<(), Box Result<(), Box {} - // Real error - _ => { - return Err( - "impossible to launch a signal-catching thread, exiting: {err}".into(), - ); - } - } - } - } + // Close streams when we receive a quit signal + handle_signals(stream_process_child_handles, shutdown_tx)?; // Wait for all streams to quit for task_handle in stream_task_handles { let _ = task_handle.await; } - #[allow(clippy::unwrap_used)] // propagating panics is ok - match_tx.send(MatchManagerInput::Stop).await.unwrap(); + let _ = socket_manager_task_handle.await; let _ = matches_manager_task_handle.await; - - #[allow(clippy::unwrap_used)] // propagating panics is ok - exec_tx.send(ExecsManagerInput::Stop).await.unwrap(); let _ = execs_manager_task_handle.await; - let _ = database_manager_task_handle.await; let stop_ok = config.stop(); @@ -144,7 +108,7 @@ pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> Result<(), Box Result<(), Box) -> tokio::io::Result<()> { - // TODO refaire pour que la création des channels soient sync puis que la suite soit spawn - { - let mut sighup = signal(SignalKind::hangup())?; - let mut sigint = signal(SignalKind::interrupt())?; - let mut sigterm = signal(SignalKind::terminate())?; - select! { - _ = sighup.recv() => {}, - _ = sigint.recv() => {}, - _ = sigterm.recv() => {}, +fn handle_signals( + stream_process_child_handles: Vec, + shutdown_tx: watch::Sender, +) -> tokio::io::Result<()> { + let mut sighup = signal(SignalKind::hangup())?; + let mut sigint = signal(SignalKind::interrupt())?; + let mut sigterm = signal(SignalKind::terminate())?; + tokio::spawn(async move { + let signal = select! { + _ = sighup.recv() => "SIGHUP", + _ = sigint.recv() => "SIGINT", + _ = sigterm.recv() => "SIGTERM", }; - } + let _ = shutdown_tx.send(true); + info!("received {signal}, closing streams..."); + // Kill stream subprocesses + for mut child_handle in stream_process_child_handles.into_iter() { + tokio::spawn(async move { child_handle.kill().await }); + } + }); Ok(()) } diff --git a/rust/src/daemon/socket.rs b/rust/src/daemon/socket.rs index e19ff36..2385198 100644 --- a/rust/src/daemon/socket.rs +++ b/rust/src/daemon/socket.rs @@ -7,7 +7,7 @@ use regex::Regex; use tokio::{ join, net::UnixListener, - sync::{mpsc, oneshot}, + sync::{mpsc, oneshot, watch}, }; use tokio_util::{ bytes::Bytes, @@ -16,12 +16,13 @@ use tokio_util::{ use crate::{ concepts::{ - ActionFilter, ClientRequest, ClientStatus, Config, DaemonResponse, Pattern, PatternStatus, + ActionFilter, ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern, + PatternStatus, }, utils::bincode_options, }; -use super::{statemap::FilterOptions, ExecsManagerInput, MatchManagerInput}; +use super::{execs::ExecsMap, matches::MatchesMap, statemap::FilterOptions}; macro_rules! err_str { ($expression:expr) => { @@ -56,8 +57,8 @@ fn open_socket(path: PathBuf) -> Result { async fn answer_order( config: &'static Config, - match_tx: &mpsc::Sender, - exec_tx: &mpsc::Sender, + match_tx: &mpsc::Sender>, + exec_tx: &mpsc::Sender>, options: ClientRequest, ) -> Result { // Compute options @@ -104,11 +105,7 @@ async fn answer_order( #[allow(clippy::unwrap_used)] // propagating panics is ok match_tx - .send(MatchManagerInput::Order( - options.order, - filtering_options2, - m_tx, - )) + .send((options.order, filtering_options2, m_tx)) .await .unwrap(); @@ -122,11 +119,7 @@ async fn answer_order( #[allow(clippy::unwrap_used)] // propagating panics is ok exec_tx - .send(ExecsManagerInput::Order( - options.order, - filtering_options, - e_tx, - )) + .send((options.order, filtering_options, e_tx)) .await .unwrap(); @@ -196,11 +189,14 @@ macro_rules! or_next { }; } +pub type SocketOrder = (Order, FilterOptions, oneshot::Sender); + pub async fn socket_manager( config: &'static Config, socket: PathBuf, - match_tx: mpsc::Sender, - exec_tx: mpsc::Sender, + match_tx: mpsc::Sender>, + exec_tx: mpsc::Sender>, + mut stop: watch::Receiver, ) { let listener = match open_socket(socket) { Ok(l) => l, @@ -212,37 +208,43 @@ pub async fn socket_manager( let bin = bincode_options(); loop { - let try_conn = listener.accept().await; - match try_conn { - Ok((conn, _)) => { - let mut transport = Framed::new(conn, LengthDelimitedCodec::new()); - // Decode - let received = transport.next().await; - let encoded_request = match received { - Some(r) => or_next!("while reading request", r), - None => { - error!("failed to answer client: client sent no request"); - continue; - } - }; - let request = or_next!( - "failed to decode request", - bin.deserialize(&encoded_request) - ); - // Process - let response = match answer_order(config, &match_tx, &exec_tx, request).await { - Ok(res) => DaemonResponse::Order(res), - Err(err) => DaemonResponse::Err(err), - }; - // Encode - let encoded_response = - or_next!("failed to serialize response", bin.serialize(&response)); - or_next!( - "failed to send response:", - transport.send(Bytes::from(encoded_response)).await - ); + tokio::select! { + _ = stop.changed() => { + break + } + try_conn = listener.accept() => { + match try_conn { + Ok((conn, _)) => { + let mut transport = Framed::new(conn, LengthDelimitedCodec::new()); + // Decode + let received = transport.next().await; + let encoded_request = match received { + Some(r) => or_next!("while reading request", r), + None => { + error!("failed to answer client: client sent no request"); + continue; + } + }; + let request = or_next!( + "failed to decode request", + bin.deserialize(&encoded_request) + ); + // Process + let response = match answer_order(config, &match_tx, &exec_tx, request).await { + Ok(res) => DaemonResponse::Order(res), + Err(err) => DaemonResponse::Err(err), + }; + // Encode + let encoded_response = + or_next!("failed to serialize response", bin.serialize(&response)); + or_next!( + "failed to send response:", + transport.send(Bytes::from(encoded_response)).await + ); + } + Err(err) => error!("failed to open connection from cli: {err}"), + } } - Err(err) => error!("failed to open connection from cli: {err}"), } } } diff --git a/rust/tests/simple.rs b/rust/tests/simple.rs index 31c545c..26e9210 100644 --- a/rust/tests/simple.rs +++ b/rust/tests/simple.rs @@ -2,7 +2,6 @@ use std::{ env, fs::File, io::{Read, Write}, - thread, time::Duration, }; @@ -15,6 +14,7 @@ use reaction::{ daemon::daemon, utils::{cli::Format, SimpleLogger}, }; +use tokio::time::sleep; fn file_with_contents(path: &str, contents: &str) { let mut file = File::create(path).unwrap(); @@ -89,13 +89,9 @@ async fn simple() { // Set the logger before running any code from the crate SimpleLogger::init(Level::Debug).unwrap(); - // Initialize tokio runtime - let runtime = tokio::runtime::Runtime::new().unwrap(); - // Run the daemon - let exit = - runtime.block_on(async move { daemon(config_path.into(), socket_path.into()).await }); - assert!(exit); + let handle = + tokio::spawn(async move { daemon(config_path.into(), socket_path.into()).await }); // Run the flushes @@ -103,30 +99,30 @@ async fn simple() { // This ensures that the subsecond precision lost from de/serialization // never causes the flush to be interpreted as anterior to the match - let handle2 = thread::spawn(move || { - thread::sleep(Duration::from_millis(2500)); + let handle2 = tokio::spawn(async move { + sleep(Duration::from_millis(2500)).await; request( socket_path.into(), Format::JSON, None, vec![("num".into(), "24".into())], Order::Flush, - ); + ).await }); - let handle3 = thread::spawn(move || { - thread::sleep(Duration::from_millis(2500)); + let handle3 = tokio::spawn(async move { + sleep(Duration::from_millis(2500)).await; request( socket_path.into(), Format::JSON, None, vec![("num".into(), "56".into())], Order::Flush, - ); + ).await }); - handle2.join().unwrap(); - handle3.join().unwrap(); + let (daemon_exit, _, _) = tokio::join!(handle, handle2, handle3); + assert!(daemon_exit.is_ok()); assert_eq!( // 24 is encountered for the second time, then @@ -148,7 +144,7 @@ async fn simple() { file_with_contents(out_path, ""); - assert!(daemon(config_path.into(), socket_path.into())); + assert!(daemon(config_path.into(), socket_path.into()).await.is_ok()); // 36 from DB // 12 from DB From 9549a7b3eca6a30db13b7cb46ac2f4c55e247668 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 14 Oct 2024 12:00:00 +0200 Subject: [PATCH 059/435] use tracing instead of log --- rust/Cargo.lock | 1 - rust/Cargo.toml | 1 - rust/src/client/mod.rs | 2 +- rust/src/concepts/config.rs | 2 +- rust/src/concepts/filter.rs | 2 +- rust/src/concepts/stream.rs | 3 ++- rust/src/daemon/database/lowlevel.rs | 2 +- rust/src/daemon/database/mod.rs | 2 +- rust/src/daemon/execs.rs | 6 ++--- rust/src/daemon/mod.rs | 3 ++- rust/src/daemon/socket.rs | 2 +- rust/src/main.rs | 25 ++++++++++++------- rust/src/utils/cli.rs | 17 ++++++------- rust/src/utils/logger.rs | 36 ---------------------------- rust/src/utils/mod.rs | 2 -- rust/tests/simple.rs | 2 +- 16 files changed, 39 insertions(+), 69 deletions(-) delete mode 100644 rust/src/utils/logger.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 46ca77b..cc46ab1 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -726,7 +726,6 @@ dependencies = [ "clap_complete", "futures", "jrsonnet-evaluator", - "log", "num_cpus", "regex", "serde", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index fe05f84..9c7fe79 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -11,7 +11,6 @@ chrono = { version = "0.4.38", features = ["std", "clock"] } clap = { version = "4.5.4", features = ["derive"] } clap_complete = "4.5.2" jrsonnet-evaluator = "0.4.2" -log = { version = "0.4.22", features = ["std"] } num_cpus = "1.16.0" regex = "1.10.4" serde = { version = "1.0.203", features = ["derive"] } diff --git a/rust/src/client/mod.rs b/rust/src/client/mod.rs index 35863d1..88d64ee 100644 --- a/rust/src/client/mod.rs +++ b/rust/src/client/mod.rs @@ -8,13 +8,13 @@ use std::{ use bincode::Options; use futures::{SinkExt, StreamExt}; -use log::info; use regex::Regex; use tokio::{io::AsyncWriteExt, net::UnixStream}; use tokio_util::{ bytes::Bytes, codec::{Framed, LengthDelimitedCodec}, }; +use tracing::info; use crate::{ concepts::{ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern}, diff --git a/rust/src/concepts/config.rs b/rust/src/concepts/config.rs index 08c86b7..cc66443 100644 --- a/rust/src/concepts/config.rs +++ b/rust/src/concepts/config.rs @@ -7,9 +7,9 @@ use std::{ sync::Arc, }; -use log::{error, info}; use serde::Deserialize; use thiserror::Error; +use tracing::{error, info}; use super::{Filter, Pattern, Stream}; diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index e25960a..9f784ca 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -6,10 +6,10 @@ use std::{ }; use chrono::TimeDelta; -use log::info; use regex::Regex; use serde::Deserialize; use tokio::sync::mpsc; +use tracing::info; use super::{ messages::{Match, Time, MAT}, diff --git a/rust/src/concepts/stream.rs b/rust/src/concepts/stream.rs index d99d61b..5347505 100644 --- a/rust/src/concepts/stream.rs +++ b/rust/src/concepts/stream.rs @@ -1,13 +1,13 @@ use std::{collections::BTreeMap, process::Stdio}; use chrono::Local; -use log::{error, info}; use serde::Deserialize; use tokio::{ io::{AsyncBufReadExt, BufReader}, process::{Child, Command}, sync::{mpsc, oneshot}, }; +use tracing::{debug, error, info}; use super::{Filter, Patterns, MFT}; @@ -122,6 +122,7 @@ impl Stream { } } } + debug!("fin stream manager"); } } diff --git a/rust/src/daemon/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs index b7b5e53..2d3bc3e 100644 --- a/rust/src/daemon/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -3,7 +3,6 @@ use std::{collections::BTreeMap, fmt::Debug, process::exit}; use bincode::Options; use chrono::{DateTime, Local}; use futures::{io, SinkExt, StreamExt}; -use log::{debug, error, warn}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tokio::{ fs::File, @@ -13,6 +12,7 @@ use tokio_util::{ bytes::Bytes, codec::{FramedRead, FramedWrite, LengthDelimitedCodec}, }; +use tracing::{debug, error, warn}; use crate::{ concepts::{ActionFilter, Config, Filter, LogEntry, Match}, diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index ad3e514..0f0dfd0 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -6,9 +6,9 @@ use std::{ }; use chrono::{Local, TimeDelta}; -use log::{debug, error, info, warn}; use thiserror::Error; use tokio::{sync::mpsc, task}; +use tracing::{debug, error, info, warn}; use crate::concepts::{ActionFilter, Config, Filter, LogEntry, Match, Time, MFT}; diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs index 2f65eb6..b233410 100644 --- a/rust/src/daemon/execs.rs +++ b/rust/src/daemon/execs.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, process::Stdio, sync::Arc}; use chrono::Local; -use log::{error, info}; +use tracing::{error, info}; use tokio::{ sync::{mpsc, Semaphore}, time, @@ -45,7 +45,7 @@ pub async fn execs_manager( // Construct command let mut command = action.exec(&mat.m); - info!("{}: run [{:?}]", &action, command); + info!("{}: run [{:?}]", &action, command.as_std()); if let Err(err) = command .stdin(Stdio::null()) .stderr(Stdio::null()) @@ -53,7 +53,7 @@ pub async fn execs_manager( .status() .await { - error!("{}: run [{:?}], code {}", &action, command, err); + error!("{}: run [{:?}], code {}", &action, command.as_std(), err); } }); }; diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index 47c8c06..9a10a5e 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -1,6 +1,6 @@ use std::{error::Error, fs, path::PathBuf}; -use log::{error, info}; +use tracing::{error, info}; use socket::socket_manager; use tokio::{ process::Child, @@ -86,6 +86,7 @@ pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> Result<(), Box Result<(String, String), String> { Ok((name.to_string(), v.to_string())) } -fn parse_log_level(s: &str) -> Result { +fn parse_log_level(s: &str) -> Result { match s.to_ascii_uppercase().as_str() { - "DEBUG" => Ok(log::Level::Debug), - "INFO" => Ok(log::Level::Info), - "WARN" => Ok(log::Level::Warn), - "ERROR" => Ok(log::Level::Error), - "FATAL" => Ok(log::Level::Error), + "DEBUG" => Ok(Level::DEBUG), + "INFO" => Ok(Level::INFO), + "WARN" => Ok(Level::WARN), + "ERROR" => Ok(Level::ERROR), + "FATAL" => Ok(Level::ERROR), _ => Err("must be one of ERROR, WARN, INFO, DEBUG".into()), } } diff --git a/rust/src/utils/logger.rs b/rust/src/utils/logger.rs deleted file mode 100644 index 771b362..0000000 --- a/rust/src/utils/logger.rs +++ /dev/null @@ -1,36 +0,0 @@ -use log::{Level, Metadata}; - -pub struct SimpleLogger { - loglevel: Level, -} - -impl log::Log for SimpleLogger { - fn enabled(&self, metadata: &Metadata) -> bool { - metadata.level() <= self.loglevel - } - - fn log(&self, record: &log::Record) { - if self.enabled(record.metadata()) { - eprintln!( - "{} {}", - match record.level() { - Level::Error => "ERROR", - Level::Warn => "WARN ", - Level::Info => "INFO ", - Level::Debug => "DEBUG", - Level::Trace => "TRACE", - }, - record.args().to_string().trim() - ); - } - } - - fn flush(&self) {} -} - -impl SimpleLogger { - pub fn init(loglevel: log::Level) -> Result<(), log::SetLoggerError> { - log::set_boxed_logger(Box::new(SimpleLogger { loglevel })) - .map(|()| log::set_max_level(loglevel.to_level_filter())) - } -} diff --git a/rust/src/utils/mod.rs b/rust/src/utils/mod.rs index 20e24db..9505e7c 100644 --- a/rust/src/utils/mod.rs +++ b/rust/src/utils/mod.rs @@ -1,11 +1,9 @@ pub mod cli; -pub mod logger; mod parse_duration; use std::marker::PhantomData; use bincode::Options; -pub use logger::SimpleLogger; pub use parse_duration::parse_duration; use serde::de::DeserializeOwned; use thiserror::Error; diff --git a/rust/tests/simple.rs b/rust/tests/simple.rs index 26e9210..365fe9c 100644 --- a/rust/tests/simple.rs +++ b/rust/tests/simple.rs @@ -5,7 +5,7 @@ use std::{ time::Duration, }; -use log::Level; +use tracing::Level; use tempfile::TempDir; use reaction::{ From 18ca9600e98e40039f0ab332f2827f12c2f8ee29 Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 13 Oct 2024 12:00:00 +0200 Subject: [PATCH 060/435] fix matches and execs not finishing They can't "naturally" end because they have their own channel whose send is inside their task and whose Sender is never dropped --- rust/src/concepts/stream.rs | 3 +-- rust/src/daemon/execs.rs | 4 +++- rust/src/daemon/matches.rs | 7 ++++++- rust/src/daemon/mod.rs | 17 +++++++++++++---- rust/src/daemon/socket.rs | 4 +--- 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/rust/src/concepts/stream.rs b/rust/src/concepts/stream.rs index 5347505..491f1fd 100644 --- a/rust/src/concepts/stream.rs +++ b/rust/src/concepts/stream.rs @@ -7,7 +7,7 @@ use tokio::{ process::{Child, Command}, sync::{mpsc, oneshot}, }; -use tracing::{debug, error, info}; +use tracing::{error, info}; use super::{Filter, Patterns, MFT}; @@ -122,7 +122,6 @@ impl Stream { } } } - debug!("fin stream manager"); } } diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs index b233410..367a59f 100644 --- a/rust/src/daemon/execs.rs +++ b/rust/src/daemon/execs.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, process::Stdio, sync::Arc}; use chrono::Local; use tracing::{error, info}; use tokio::{ - sync::{mpsc, Semaphore}, + sync::{mpsc, watch, Semaphore}, time, }; @@ -22,6 +22,7 @@ pub async fn execs_manager( mut exec_rx: mpsc::Receiver, mut socket_order_rx: mpsc::Receiver>, log_tx: mpsc::Sender, + mut stop: watch::Receiver, ) { // FIXME replace with TryStreamExt::try_for_each_concurrent? let semaphore = if config.concurrency() > 0 { @@ -64,6 +65,7 @@ pub async fn execs_manager( loop { tokio::select! { + _ = stop.changed() => break, Some(mat) = exec_rx.recv() => { let now = Local::now(); if mat.t.lt(&now) { diff --git a/rust/src/daemon/matches.rs b/rust/src/daemon/matches.rs index b2036e8..969941b 100644 --- a/rust/src/daemon/matches.rs +++ b/rust/src/daemon/matches.rs @@ -1,7 +1,10 @@ use std::collections::BTreeMap; use chrono::Local; -use tokio::{sync::mpsc, time}; +use tokio::{ + sync::{mpsc, watch}, + time, +}; use super::{ database::DatabaseManagerInput, @@ -18,6 +21,7 @@ pub async fn matches_manager( mut socket_order_rx: mpsc::Receiver>, action_tx: mpsc::Sender, log_tx: mpsc::Sender, + mut stop: watch::Receiver, ) { let mut matches: MatchesMap = BTreeMap::new(); @@ -29,6 +33,7 @@ pub async fn matches_manager( loop { tokio::select! { + _ = stop.changed() => break, Some(mft) = match_rx.recv() => { let exec = handle_match(&mut matches, mft.clone(), &unmatches_tx, &action_tx).await; diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index 9a10a5e..1487b7b 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -1,6 +1,5 @@ use std::{error::Error, fs, path::PathBuf}; -use tracing::{error, info}; use socket::socket_manager; use tokio::{ process::Child, @@ -8,6 +7,7 @@ use tokio::{ signal::unix::{signal, SignalKind}, sync::{mpsc, oneshot, watch}, }; +use tracing::{error, info}; use crate::concepts::Config; use database::database_manager; @@ -20,7 +20,10 @@ mod matches; mod socket; mod statemap; -pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> Result<(), Box> { +pub async fn daemon( + config_path: PathBuf, + socket: PathBuf, +) -> Result<(), Box> { let config: &'static Config = Config::from_file(&config_path).map(|config| Box::leak(Box::new(config)))?; @@ -45,6 +48,7 @@ pub async fn daemon(config_path: PathBuf, socket: PathBuf) -> Result<(), Box Result<(), Box Result<(), Box { - break - } + _ = stop.changed() => break, try_conn = listener.accept() => { match try_conn { Ok((conn, _)) => { From 0d783218d8105d169538678eb2b4f1d44bf23556 Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 13 Oct 2024 12:00:00 +0200 Subject: [PATCH 061/435] fix execs bug, fix tests with tracing --- rust/src/daemon/execs.rs | 26 +++++++++++++------------- rust/test.jsonnet | 4 ++-- rust/tests/simple.rs | 28 +++++++++++++++------------- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs index 367a59f..8ed41c9 100644 --- a/rust/src/daemon/execs.rs +++ b/rust/src/daemon/execs.rs @@ -1,11 +1,11 @@ use std::{collections::BTreeMap, process::Stdio, sync::Arc}; use chrono::Local; -use tracing::{error, info}; use tokio::{ sync::{mpsc, watch, Semaphore}, time, }; +use tracing::{error, info}; use crate::concepts::{Action, ActionFilter, Config, LogEntry, Order, MAT}; @@ -68,7 +68,7 @@ pub async fn execs_manager( _ = stop.changed() => break, Some(mat) = exec_rx.recv() => { let now = Local::now(); - if mat.t.lt(&now) { + if mat.t < now { exec_now(mat); } else { execs.add(&mat); @@ -76,7 +76,7 @@ pub async fn execs_manager( let mat = mat.clone(); let pendings_tx = pendings_tx.clone(); tokio::spawn(async move { - let dur = mat.t - Local::now(); + let dur = mat.t - now; time::sleep(dur.to_std().expect("Duration is bigger than what's supported. Did you put an enormous after duration?")).await; #[allow(clippy::unwrap_used)] // propagating panics is ok pendings_tx @@ -139,17 +139,17 @@ pub async fn execs_manager( } else => break } + } - for (action, inner_map) in &mut execs { - if action.on_exit() { - for (match_, inner_set) in inner_map { - for _ in inner_set.iter() { - exec_now(MAT { - m: match_.clone(), - o: action, - t: Local::now(), - }); - } + for (action, inner_map) in &mut execs { + if action.on_exit() { + for (match_, inner_set) in inner_map { + for _ in inner_set.iter() { + exec_now(MAT { + m: match_.clone(), + o: action, + t: Local::now(), + }); } } } diff --git a/rust/test.jsonnet b/rust/test.jsonnet index 41335f7..6e99367 100644 --- a/rust/test.jsonnet +++ b/rust/test.jsonnet @@ -22,7 +22,7 @@ streams: { s1: { - cmd: ['sh', '-c', "seq 20 | tr ' ' '\n' | while read i; do echo found $((i % 5)); sleep 3; done"], + cmd: ['sh', '-c', "seq 20 | tr ' ' '\n' | while read i; do echo found $((i % 5)); sleep 1; done"], filters: { f1: { regex: [ @@ -36,7 +36,7 @@ }, undamn: { cmd: ['notify-send', 'first stream', 'unban '], - after: '20s', + after: '3s', onexit: true, }, }, diff --git a/rust/tests/simple.rs b/rust/tests/simple.rs index 365fe9c..4834b62 100644 --- a/rust/tests/simple.rs +++ b/rust/tests/simple.rs @@ -1,19 +1,14 @@ use std::{ env, fs::File, - io::{Read, Write}, + io::{IsTerminal, Read, Write}, time::Duration, }; -use tracing::Level; use tempfile::TempDir; +use tracing::Level; -use reaction::{ - client::request, - concepts::Order, - daemon::daemon, - utils::{cli::Format, SimpleLogger}, -}; +use reaction::{client::request, concepts::Order, daemon::daemon, utils::cli::Format}; use tokio::time::sleep; fn file_with_contents(path: &str, contents: &str) { @@ -87,11 +82,16 @@ async fn simple() { file_with_contents(out_path, ""); // Set the logger before running any code from the crate - SimpleLogger::init(Level::Debug).unwrap(); + tracing_subscriber::fmt::fmt() + .without_time() + .with_target(false) + .with_ansi(std::io::stdout().is_terminal()) + .with_max_level(Level::DEBUG) + .try_init() + .unwrap(); // Run the daemon - let handle = - tokio::spawn(async move { daemon(config_path.into(), socket_path.into()).await }); + let handle = tokio::spawn(async move { daemon(config_path.into(), socket_path.into()).await }); // Run the flushes @@ -107,7 +107,8 @@ async fn simple() { None, vec![("num".into(), "24".into())], Order::Flush, - ).await + ) + .await }); let handle3 = tokio::spawn(async move { @@ -118,7 +119,8 @@ async fn simple() { None, vec![("num".into(), "56".into())], Order::Flush, - ).await + ) + .await }); let (daemon_exit, _, _) = tokio::join!(handle, handle2, handle3); From f7e42ceab5d9650f7171f86ba8c613379a84611a Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 14 Oct 2024 12:00:00 +0200 Subject: [PATCH 062/435] make sleep tasks listen to the shutdown signal --- rust/src/daemon/execs.rs | 20 +++++++++++++------- rust/src/daemon/matches.rs | 21 +++++++++++++++------ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs index 8ed41c9..3c4c489 100644 --- a/rust/src/daemon/execs.rs +++ b/rust/src/daemon/execs.rs @@ -75,14 +75,20 @@ pub async fn execs_manager( { let mat = mat.clone(); let pendings_tx = pendings_tx.clone(); + let mut stop = stop.clone(); tokio::spawn(async move { - let dur = mat.t - now; - time::sleep(dur.to_std().expect("Duration is bigger than what's supported. Did you put an enormous after duration?")).await; - #[allow(clippy::unwrap_used)] // propagating panics is ok - pendings_tx - .send(mat.clone()) - .await - .unwrap(); + let dur = (mat.t - now).to_std().expect("Duration is bigger than what's supported. Did you put an enormous after duration?"); + tokio::select! { + biased; + _ = stop.changed() => {} + _ = time::sleep(dur) => { + #[allow(clippy::unwrap_used)] // propagating panics is ok + pendings_tx + .send(mat.clone()) + .await + .unwrap(); + } + } }); } } diff --git a/rust/src/daemon/matches.rs b/rust/src/daemon/matches.rs index 969941b..85a5a55 100644 --- a/rust/src/daemon/matches.rs +++ b/rust/src/daemon/matches.rs @@ -28,14 +28,14 @@ pub async fn matches_manager( let (unmatches_tx, mut unmatches_rx) = mpsc::channel(1); while let Some(mft) = startup_match_rx.recv().await { - let _ = handle_match(&mut matches, mft.clone(), &unmatches_tx, &action_tx).await; + let _ = handle_match(&mut matches, mft.clone(), &unmatches_tx, &action_tx, &stop).await; } loop { tokio::select! { _ = stop.changed() => break, Some(mft) = match_rx.recv() => { - let exec = handle_match(&mut matches, mft.clone(), &unmatches_tx, &action_tx).await; + let exec = handle_match(&mut matches, mft.clone(), &unmatches_tx, &action_tx, &stop).await; #[allow(clippy::unwrap_used)] // propagating panics is ok log_tx @@ -95,6 +95,7 @@ async fn handle_match( mft: MFT, unmatches_tx: &mpsc::Sender, action_tx: &mpsc::Sender, + stop: &watch::Receiver, ) -> bool { // Store matches let exec = match mft.o.retry() { @@ -106,13 +107,21 @@ async fn handle_match( { let mft = mft.clone(); let unmatches_tx = unmatches_tx.clone(); + let mut stop = stop.clone(); tokio::spawn(async move { #[allow(clippy::unwrap_used)] // retry_duration is always Some() after filter's setup - let dur = mft.t - Local::now() + mft.o.retry_duration().unwrap(); - time::sleep(dur.to_std().expect("Duration is bigger than what's supported. Did you put an enormous retry_duration?")).await; - #[allow(clippy::unwrap_used)] // propagating panics is ok - unmatches_tx.send(mft).await.unwrap(); + let dur = (mft.t - Local::now() + mft.o.retry_duration().unwrap()) + .to_std() + .expect("Duration is bigger than what's supported. Did you put an enormous retry_duration?"); + tokio::select! { + biased; + _ = stop.changed() => {} + _ = time::sleep(dur) => { + #[allow(clippy::unwrap_used)] // propagating panics is ok + unmatches_tx.send(mft).await.unwrap(); + } + } }); } From df7b4291f7014dd3ef2f52b62dde168fd684065c Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 14 Oct 2024 12:00:00 +0200 Subject: [PATCH 063/435] cleaner bool testing --- rust/src/daemon/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index 1487b7b..abfd3fb 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -118,7 +118,7 @@ pub async fn daemon( error!("failed to remove socket: {}", err); } - if !shutdown_rx.borrow().eq(&true) { + if !*shutdown_rx.borrow() { Err("quitting because all streams finished".into()) } else if !stop_ok { Err("while executing stop command".into()) From 8dbb20efcee5ba8f3e42783596d7c3f8aae39392 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 14 Oct 2024 12:00:00 +0200 Subject: [PATCH 064/435] WIP profiling performance is veryy bad --- rust/Cargo.lock | 693 ++++++++++++++++++++++++++++++++++++++++- rust/Cargo.toml | 6 +- rust/heavy-load.yml | 10 +- rust/src/daemon/mod.rs | 12 +- rust/src/main.rs | 49 +-- 5 files changed, 727 insertions(+), 43 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index cc46ab1..4018f90 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -100,12 +106,104 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "anyhow" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "async-trait" +version = "0.1.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 1.0.1", + "tower 0.5.1", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.1", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.73" @@ -116,7 +214,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.4", "object", "rustc-demangle", ] @@ -127,6 +225,18 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bincode" version = "1.3.3" @@ -148,6 +258,12 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.7.2" @@ -235,12 +351,81 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +[[package]] +name = "console-api" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ed14aa9c9f927213c6e4f3ef75faaad3406134efe84ba2cb7983431d5f0931" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e3a111a37f3333946ebf9da370ba5c5577b18eb342ec683eb488dd21980302" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "hyper-util", + "prost", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "equivalent" version = "1.0.1" @@ -263,6 +448,22 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide 0.8.0", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "futures" version = "0.3.30" @@ -352,18 +553,67 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "h2" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.7", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "heck" version = "0.5.0" @@ -376,6 +626,111 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -399,6 +754,16 @@ dependencies = [ "cc", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -406,7 +771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", ] [[package]] @@ -415,6 +780,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -428,7 +802,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fee60406dac44a01b37e120b43adb062047251e195db15392b825f6bdc948712" dependencies = [ "annotate-snippets", - "base64", + "base64 0.13.1", "bincode", "jrsonnet-gc", "jrsonnet-interner", @@ -547,6 +921,21 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "md5" version = "0.7.0" @@ -559,6 +948,18 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.4" @@ -568,6 +969,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.0.2" @@ -580,6 +990,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -686,6 +1106,32 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c719dcf55f09a3a7e764c6649ab594c18a177e3599c467983cdf644bfc0a4088" +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -698,6 +1144,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -707,6 +1162,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "prost-types" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.36" @@ -716,6 +1203,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "reaction" version = "0.1.0" @@ -724,6 +1241,7 @@ dependencies = [ "chrono", "clap", "clap_complete", + "console-subscriber", "futures", "jrsonnet-evaluator", "num_cpus", @@ -757,8 +1275,17 @@ checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -769,9 +1296,15 @@ checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.4", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.4" @@ -803,6 +1336,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + [[package]] name = "ryu" version = "1.0.18" @@ -853,7 +1392,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -931,6 +1470,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + [[package]] name = "synstructure" version = "0.12.6" @@ -1010,6 +1561,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.52.0", ] @@ -1024,6 +1576,17 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "tokio-stream" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.12" @@ -1037,6 +1600,82 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.40" @@ -1086,14 +1725,24 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "unescape" version = "0.1.0" @@ -1136,6 +1785,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1317,3 +1975,24 @@ checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1" dependencies = [ "winapi", ] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 9c7fe79..7ae3f02 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -3,7 +3,8 @@ name = "reaction" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[build] +rustflags = ["--cfg", "tokio_unstable"] [dependencies] bincode = "1.3.3" @@ -22,7 +23,8 @@ timer = "0.2.0" # async-related futures = "0.3.30" -tokio = { version = "1.40.0", features = ["full"] } +tokio = { version = "1.40.0", features = ["full", "tracing"] } tokio-util = { version = "0.7.12", features = ["codec"] } tracing = "0.1.40" tracing-subscriber = "0.3.18" +console-subscriber = "0.4.0" diff --git a/rust/heavy-load.yml b/rust/heavy-load.yml index 331dacd..d3cde2a 100644 --- a/rust/heavy-load.yml +++ b/rust/heavy-load.yml @@ -17,7 +17,7 @@ streams: regex: - '^found ' retry: 9 - retryperiod: 1m + retryperiod: 6m actions: damn: cmd: [ 'sleep', '0.0' ] @@ -32,7 +32,7 @@ streams: regex: - '^found ' retry: 480 - retryperiod: 1m + retryperiod: 6m actions: damn: cmd: [ 'sleep', '0.0' ] @@ -47,7 +47,7 @@ streams: regex: - '^found ' retry: 480 - retryperiod: 1m + retryperiod: 6m actions: damn: cmd: [ 'sleep', '0.0' ] @@ -56,13 +56,13 @@ streams: after: 1m onexit: false tailDown4: - cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] + cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done; sleep infinity' ] filters: find: regex: - '^found ' retry: 480 - retryperiod: 1m + retryperiod: 6m actions: damn: cmd: [ 'sleep', '0.0' ] diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index abfd3fb..6cb7174 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -34,14 +34,14 @@ pub async fn daemon( let mut stream_process_child_handles = Vec::new(); let mut stream_task_handles = Vec::new(); - let (stream2match_tx, stream2match_rx) = mpsc::channel(10); - let (database2match_tx, database2match_rx) = mpsc::channel(10); + let (stream2match_tx, stream2match_rx) = mpsc::channel(123456); + let (database2match_tx, database2match_rx) = mpsc::channel(234560); let (socket2match_tx, socket2match_rx) = mpsc::channel(1); let (socket2exec_tx, socket2exec_rx) = mpsc::channel(1); - let (exec_tx, exec_rx) = mpsc::channel(10); - let (log_tx, log_rx) = mpsc::channel(10); + let (matches2exec_tx, matches2exec_rx) = mpsc::channel(234560); + let (log_tx, log_rx) = mpsc::channel(234560); // Shutdown channel let (shutdown_tx, shutdown_rx) = watch::channel(false); @@ -54,7 +54,7 @@ pub async fn daemon( stream2match_rx, database2match_rx, socket2match_rx, - exec_tx, + matches2exec_tx, log_tx, shutdown_rx, ) @@ -65,7 +65,7 @@ pub async fn daemon( let execs_manager_task_handle = { let shutdown_rx = shutdown_rx.clone(); tokio::spawn(async move { - execs_manager(config, exec_rx, socket2exec_rx, log_tx, shutdown_rx).await + execs_manager(config, matches2exec_rx, socket2exec_rx, log_tx, shutdown_rx).await }) }; diff --git a/rust/src/main.rs b/rust/src/main.rs index 64f5d6b..33c292c 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -25,31 +25,34 @@ async fn main() { eprintln!("this error occurred."); })); + console_subscriber::init(); + let cli = Cli::parse(); - { - // Set log level - let level = if let Command::Start { - loglevel, - config: _, - socket: _, - } = cli.command - { - loglevel - } else { - Level::DEBUG - }; - if let Err(err) = tracing_subscriber::fmt::fmt() - .without_time() - .with_target(false) - .with_ansi(std::io::stdout().is_terminal()) - .with_max_level(level) - .try_init() - { - eprintln!("ERROR could not initialize logging: {err}"); - exit(1); - } - } + // { + // // Set log level + // let level = if let Command::Start { + // loglevel, + // config: _, + // socket: _, + // } = cli.command + // { + // loglevel + // } else { + // Level::DEBUG + // }; + // if let Err(err) = tracing_subscriber::fmt::fmt() + // .without_time() + // .with_target(false) + // .with_ansi(std::io::stdout().is_terminal()) + // // .with_max_level(level) + // .with_max_level(Level::TRACE) + // .try_init() + // { + // eprintln!("ERROR could not initialize logging: {err}"); + // exit(1); + // } + // } let result = match cli.command { Command::Start { From 72887f3af0e7eb8903b8523dc1aed4501deb3bd8 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 14 Oct 2024 12:00:00 +0200 Subject: [PATCH 065/435] big refacto avoid channels remove matches_manager and execs_manager one stream_manager task handles filter_managers and action_managers --- rust/heavy-load.yml | 2 +- rust/src/concepts/filter.rs | 4 + rust/src/concepts/stream.rs | 93 +++++------------- rust/src/daemon/action.rs | 133 ++++++++++++++++++++++++++ rust/src/daemon/database/mod.rs | 32 +++---- rust/src/daemon/execs.rs | 163 -------------------------------- rust/src/daemon/filter.rs | 126 ++++++++++++++++++++++++ rust/src/daemon/matches.rs | 142 ---------------------------- rust/src/daemon/mod.rs | 117 +++++++++++------------ rust/src/daemon/socket.rs | 6 +- rust/src/daemon/statemap.rs | 122 ------------------------ rust/src/daemon/stream.rs | 69 ++++++++++++++ rust/src/main.rs | 50 +++++----- 13 files changed, 460 insertions(+), 599 deletions(-) create mode 100644 rust/src/daemon/action.rs delete mode 100644 rust/src/daemon/execs.rs create mode 100644 rust/src/daemon/filter.rs delete mode 100644 rust/src/daemon/matches.rs delete mode 100644 rust/src/daemon/statemap.rs create mode 100644 rust/src/daemon/stream.rs diff --git a/rust/heavy-load.yml b/rust/heavy-load.yml index d3cde2a..6af6dfa 100644 --- a/rust/heavy-load.yml +++ b/rust/heavy-load.yml @@ -56,7 +56,7 @@ streams: after: 1m onexit: false tailDown4: - cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done; sleep infinity' ] + cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] filters: find: regex: diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index 9f784ca..ec6c2ac 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -74,6 +74,10 @@ impl Filter { self.retry_duration } + pub fn actions(&self) -> &BTreeMap { + &self.actions + } + pub fn setup( &mut self, stream_name: &str, diff --git a/rust/src/concepts/stream.rs b/rust/src/concepts/stream.rs index 491f1fd..ddef48d 100644 --- a/rust/src/concepts/stream.rs +++ b/rust/src/concepts/stream.rs @@ -1,15 +1,8 @@ -use std::{collections::BTreeMap, process::Stdio}; +use std::{cmp::Ordering, collections::BTreeMap}; -use chrono::Local; use serde::Deserialize; -use tokio::{ - io::{AsyncBufReadExt, BufReader}, - process::{Child, Command}, - sync::{mpsc, oneshot}, -}; -use tracing::{error, info}; -use super::{Filter, Patterns, MFT}; +use super::{Filter, Patterns}; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -30,6 +23,14 @@ impl Stream { self.filters.get(filter_name) } + pub fn name(&self) -> &str { + &self.name + } + + pub fn cmd(&self) -> &Vec { + &self.cmd + } + pub fn setup(&mut self, name: &str, patterns: &Patterns) -> Result<(), String> { self._setup(name, patterns) .map_err(|msg| format!("stream {}: {}", name, msg)) @@ -62,66 +63,22 @@ impl Stream { Ok(()) } +} - pub async fn manager( - &'static self, - child_tx: oneshot::Sender>, - match_tx: mpsc::Sender, - ) { - info!("{}: start {:?}", self.name, self.cmd); - let mut child = match Command::new(&self.cmd[0]) - .args(&self.cmd[1..]) - .stdin(Stdio::null()) - .stderr(Stdio::null()) - .stdout(Stdio::piped()) - .spawn() - { - Ok(child) => child, - Err(err) => { - error!("could not execute stream {} cmd: {}", self.name, err); - let _ = child_tx.send(None); - return; - } - }; - - // keep stdout before sending/moving child to the main thread - #[allow(clippy::unwrap_used)] - // we know there is an stdout because we asked for Stdio::piped() - let mut lines = BufReader::new(child.stdout.take().unwrap()).lines(); - - // let main handle the child process - let _ = child_tx.send(Some(child)); - - loop { - match lines.next_line().await { - Ok(Some(line)) => { - for filter in self.filters.values() { - if let Some(match_) = filter.get_match(&line) { - #[allow(clippy::unwrap_used)] // propagating panics is ok - match_tx - .send(MFT { - m: match_, - o: filter, - t: Local::now(), - }) - .await - .unwrap(); - } - } - } - Ok(None) => { - error!("stream {} exited: its command returned.", self.name); - break; - } - Err(err) => { - error!( - "impossible to read output from stream {}: {}", - self.name, err - ); - break; - } - } - } +impl PartialEq for Stream { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} +impl Eq for Stream {} +impl Ord for Stream { + fn cmp(&self, other: &Self) -> Ordering { + self.name.cmp(&other.name) + } +} +impl PartialOrd for Stream { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) } } diff --git a/rust/src/daemon/action.rs b/rust/src/daemon/action.rs new file mode 100644 index 0000000..b812a2f --- /dev/null +++ b/rust/src/daemon/action.rs @@ -0,0 +1,133 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + process::Stdio, + sync::{Arc, Mutex}, +}; + +use chrono::{Local, TimeDelta}; +use tokio::sync::Semaphore; +use tracing::{error, info}; + +use crate::concepts::{Action, Match, Time}; + +struct State { + pending: BTreeMap>, + ordered_times: BTreeMap, +} + +impl State { + fn add_match(&mut self, m: &Match, t: Time) { + self.pending.entry(m.clone()).or_default().insert(t); + self.ordered_times.insert(t, m.clone()); + } + + fn remove(&mut self, m: Match, t: Time) -> bool { + self.pending.entry(m).and_modify(|times| { + times.remove(&t); + }); + self.ordered_times.remove(&t).is_some() + } + + fn clear_past_times(&mut self, after: Option) { + let now = Local::now(); + let after = after.unwrap_or_default(); + while self + .ordered_times + .first_key_value() + .is_some_and(|(k, _)| *k + after < now) + { + let (_, m) = self.ordered_times.pop_first().unwrap(); + self.pending.remove(&m); + } + } +} + +#[derive(Clone)] +pub struct ActionManager { + action: &'static Action, + exec_limit: Option>, + state: Arc>, +} + +impl ActionManager { + pub fn new( + action: &'static Action, + pending: BTreeMap>, + exec_limit: Option>, + ) -> Self { + Self { + action, + exec_limit, + state: Arc::new(Mutex::new(State { + pending: pending.clone(), + ordered_times: pending + .into_iter() + .flat_map(|(m, times)| times.into_iter().map(move |time| (time, m.clone()))) + .collect(), + })), + } + } + + pub fn handle_exec(&mut self, m: Match, t: Time) { + let now = Local::now(); + let exec_t = t + self.action.after_duration().unwrap_or_default(); + if exec_t < now { + self.exec_now(m); + } else { + { + let mut state = self.state.lock().unwrap(); + state.clear_past_times(self.action.after_duration()); + state.add_match(&m, exec_t); + } + let this = self.clone(); + tokio::spawn(async move { + let dur = (exec_t - now).to_std().expect("Duration is bigger than what's supported. Did you put an enormous after duration?"); + tokio::time::sleep(dur).await; + let mut state = this.state.lock().unwrap(); + if state.remove(m.clone(), t) { + this.exec_now(m); + } + }); + } + } + + fn exec_now(&self, m: Match) { + let semaphore = self.exec_limit.clone(); + let action = self.action; + tokio::spawn(async move { + // Wait for semaphore's permission, if it is Some + let _permit = match semaphore { + #[allow(clippy::unwrap_used)] // We know the semaphore is not closed + Some(semaphore) => Some(semaphore.acquire_owned().await.unwrap()), + None => None, + }; + + // Construct command + let mut command = action.exec(&m); + + info!("{}: run [{:?}]", &action, command.as_std()); + if let Err(err) = command + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .stdout(Stdio::piped()) + .status() + .await + { + error!("{}: run [{:?}], code {}", &action, command.as_std(), err); + } + }); + } + + pub fn quit(&mut self) { + if self.action.on_exit() { + let mut state = self.state.lock().unwrap(); + for (m, times) in &state.pending { + for _ in times { + self.exec_now(m.clone()); + } + } + state.pending.clear(); + state.ordered_times.clear(); + } + } +} diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index 0f0dfd0..2bdb1f2 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -63,7 +63,7 @@ macro_rules! flush_or_die { pub async fn database_manager( config: &'static Config, mut log_rx: mpsc::Receiver, - matches_tx: mpsc::Sender, + matches_tx: BTreeMap<&Filter, mpsc::Sender>, ) -> task::JoinHandle<()> { let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)).await { Ok(dbs) => dbs, @@ -82,6 +82,7 @@ pub async fn database_manager( write_or_die!(log_db, entry); cpt += 1; if cpt == MAX_WRITES { + info!("Rotating database..."); cpt = 0; flush_or_die!(log_db); flush_or_die!(flush_db); @@ -97,6 +98,7 @@ pub async fn database_manager( exit(1); } }; + info!("Rotated database"); } } }; @@ -108,17 +110,7 @@ pub async fn database_manager( async fn rotate_db( config: &'static Config, - matches_tx: Option>, -) -> Result<(WriteDB, WriteDB), DBError> { - info!("Rotating database..."); - let res = _rotate_db(config, &matches_tx).await; - info!("Rotated database"); - res -} - -async fn _rotate_db( - config: &'static Config, - matches_tx: &Option>, + matches_tx: Option>>, ) -> Result<(WriteDB, WriteDB), DBError> { // TODO asyncify this let mut log_read_db = match ReadDB::open(LOG_DB_NAME, config).await? { @@ -148,7 +140,7 @@ async fn _rotate_db( let mut log_write_db = WriteDB::create(LOG_DB_NEW_NAME, config).await; - __rotate_db( + _rotate_db( matches_tx, &mut log_read_db, &mut flush_read_db, @@ -174,8 +166,8 @@ async fn _rotate_db( Ok((log_write_db, flush_write_db)) } -async fn __rotate_db( - matches_tx: &Option>, +async fn _rotate_db( + matches_tx: Option>>, log_read_db: &mut ReadDB, flush_read_db: &mut ReadDB, log_write_db: &mut WriteDB, @@ -238,10 +230,12 @@ async fn __rotate_db( millisecond_disambiguation_counter += 1; } - if let Some(tx) = matches_tx { - debug!("DB sending match from DB: {:?}", entry.m); - #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(entry.clone().into()).await.unwrap(); + if let Some(matches_tx) = &matches_tx { + if let Some(tx) = matches_tx.get(entry.f) { + debug!("DB sending match from DB: {:?}", entry.m); + #[allow(clippy::unwrap_used)] // propagating panics is ok + tx.send(entry.clone().into()).await.unwrap(); + } } write_or_die!(log_write_db, entry); diff --git a/rust/src/daemon/execs.rs b/rust/src/daemon/execs.rs deleted file mode 100644 index 3c4c489..0000000 --- a/rust/src/daemon/execs.rs +++ /dev/null @@ -1,163 +0,0 @@ -use std::{collections::BTreeMap, process::Stdio, sync::Arc}; - -use chrono::Local; -use tokio::{ - sync::{mpsc, watch, Semaphore}, - time, -}; -use tracing::{error, info}; - -use crate::concepts::{Action, ActionFilter, Config, LogEntry, Order, MAT}; - -use super::{ - database::DatabaseManagerInput, - socket::SocketOrder, - statemap::{StateMap, StateMapTrait}, -}; - -pub type ExecsMap = StateMap; - -pub async fn execs_manager( - config: &'static Config, - mut exec_rx: mpsc::Receiver, - mut socket_order_rx: mpsc::Receiver>, - log_tx: mpsc::Sender, - mut stop: watch::Receiver, -) { - // FIXME replace with TryStreamExt::try_for_each_concurrent? - let semaphore = if config.concurrency() > 0 { - Some(Arc::new(Semaphore::new(config.concurrency()))) - } else { - None - }; - - let exec_now = |mat: MAT| { - let semaphore = semaphore.clone(); - tokio::spawn(async move { - let action = mat.o; - - // Wait for semaphore's permission, if it is Some - let _permit = match semaphore { - #[allow(clippy::unwrap_used)] // We know the semaphore is not closed - Some(semaphore) => Some(semaphore.acquire_owned().await.unwrap()), - None => None, - }; - - // Construct command - let mut command = action.exec(&mat.m); - - info!("{}: run [{:?}]", &action, command.as_std()); - if let Err(err) = command - .stdin(Stdio::null()) - .stderr(Stdio::null()) - .stdout(Stdio::piped()) - .status() - .await - { - error!("{}: run [{:?}], code {}", &action, command.as_std(), err); - } - }); - }; - - let mut execs: ExecsMap = BTreeMap::new(); - - let (pendings_tx, mut pendings_rx) = mpsc::channel(1); - - loop { - tokio::select! { - _ = stop.changed() => break, - Some(mat) = exec_rx.recv() => { - let now = Local::now(); - if mat.t < now { - exec_now(mat); - } else { - execs.add(&mat); - { - let mat = mat.clone(); - let pendings_tx = pendings_tx.clone(); - let mut stop = stop.clone(); - tokio::spawn(async move { - let dur = (mat.t - now).to_std().expect("Duration is bigger than what's supported. Did you put an enormous after duration?"); - tokio::select! { - biased; - _ = stop.changed() => {} - _ = time::sleep(dur) => { - #[allow(clippy::unwrap_used)] // propagating panics is ok - pendings_tx - .send(mat.clone()) - .await - .unwrap(); - } - } - }); - } - } - } - Some(mat) = pendings_rx.recv() => { - if execs.rm(&mat) { - exec_now(mat); - } - } - Some((order, options, tx)) = socket_order_rx.recv() => { - let filtered = execs.filtered(options); - - if let Order::Flush = order { - let now = Local::now(); - // filter the state_map according to provided options - for (action, inner_map) in &filtered { - // get filter (required for LogEntry, FIXME optimize this) - let filter = { - let name = action.full_name(); - #[allow(clippy::unwrap_used)] - // We're pretty confident our action has a filter - config - .get_filter(&(name.0.to_string(), name.1.to_string())) - .unwrap() - }; - - for match_ in inner_map.keys() { - let mat = MAT { - m: match_.clone(), - o: action, - t: now, - }; - // delete them from state and execute them - if let Some(set) = execs.rm_times(&mat) { - for _ in set { - exec_now(mat.clone()); - } - } - #[allow(clippy::unwrap_used)] // propagating panics is ok - log_tx - .send(DatabaseManagerInput::Flush(LogEntry { - exec: false, - m: mat.m, - f: filter, - t: mat.t, - })) - .await - .unwrap(); - } - } - } - #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(filtered).unwrap(); - } - else => break - } - } - - for (action, inner_map) in &mut execs { - if action.on_exit() { - for (match_, inner_set) in inner_map { - for _ in inner_set.iter() { - exec_now(MAT { - m: match_.clone(), - o: action, - t: Local::now(), - }); - } - } - } - } -} diff --git a/rust/src/daemon/filter.rs b/rust/src/daemon/filter.rs new file mode 100644 index 0000000..4f76e34 --- /dev/null +++ b/rust/src/daemon/filter.rs @@ -0,0 +1,126 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; + +use chrono::Local; +use tokio::sync::{mpsc, Semaphore}; + +use crate::concepts::{Filter, LogEntry, Match, Time, MFT}; + +use super::{action::ActionManager, database::DatabaseManagerInput}; + +pub struct FilterManager { + filter: &'static Filter, + log_tx: mpsc::Sender, + action_managers: Vec, + matches: BTreeMap>, + ordered_times: BTreeMap, +} + +impl FilterManager { + pub fn new( + filter: &'static Filter, + matches: BTreeMap>, + exec_limit: Option>, + log_tx: mpsc::Sender, + ) -> Self { + Self { + filter, + log_tx, + action_managers: filter + .actions() + .values() + .map(|action| ActionManager::new(action, BTreeMap::default(), exec_limit.clone())) + .collect(), + matches: matches.clone(), + ordered_times: matches + .into_iter() + .flat_map(|(m, times)| times.into_iter().map(move |time| (time, m.clone()))) + .collect(), + } + } + + pub async fn handle_db_entries(mut self, mut match_rx: mpsc::Receiver) -> Self { + while let Some(mft) = match_rx.recv().await { + self.handle_match(mft.m, mft.t, false).await; + } + self + } + + pub async fn handle_line(&mut self, line: &str) { + if let Some(match_) = self.filter.get_match(line) { + let now = Local::now(); + self.handle_match(match_, now, true).await; + } + } + + pub async fn handle_match(&mut self, m: Match, t: Time, send_log: bool) { + self.clear_past_times(); + + let exec = match self.filter.retry() { + None => true, + Some(retry) => { + self.add_match(&m, t); + // Number of stored times for this match >= configured retry for this filter + self.get_times(&m) >= retry as usize + } + }; + + if exec { + self.remove_match(&m); + for manager in &mut self.action_managers { + manager.handle_exec(m.clone(), t); + } + } + + if send_log { + #[allow(clippy::unwrap_used)] // propagating panics is ok + self.log_tx + .send(DatabaseManagerInput::Log(LogEntry { + exec, + m, + f: self.filter, + t, + })) + .await + .unwrap(); + } + } + + pub fn quit(&mut self) { + self.action_managers + .iter_mut() + .for_each(|manager| manager.quit()); + } + + fn add_match(&mut self, m: &Match, t: Time) { + self.matches.entry(m.clone()).or_default().insert(t); + self.ordered_times.insert(t, m.clone()); + } + + fn remove_match(&mut self, m: &Match) { + if let Some(times) = self.matches.remove(m) { + for t in times { + self.ordered_times.remove(&t); + } + } + } + + fn clear_past_times(&mut self) { + let now = Local::now(); + let retry_duration = self.filter.retry_duration().unwrap_or_default(); + while self + .ordered_times + .first_key_value() + .is_some_and(|(k, _)| *k + retry_duration < now) + { + let (_, m) = self.ordered_times.pop_first().unwrap(); + self.matches.remove(&m); + } + } + + fn get_times(&self, m: &Match) -> usize { + self.matches.get(m).map(|v| v.len()).unwrap_or(0) + } +} diff --git a/rust/src/daemon/matches.rs b/rust/src/daemon/matches.rs deleted file mode 100644 index 85a5a55..0000000 --- a/rust/src/daemon/matches.rs +++ /dev/null @@ -1,142 +0,0 @@ -use std::collections::BTreeMap; - -use chrono::Local; -use tokio::{ - sync::{mpsc, watch}, - time, -}; - -use super::{ - database::DatabaseManagerInput, - socket::SocketOrder, - statemap::{StateMap, StateMapTrait}, -}; -use crate::concepts::{Filter, LogEntry, Order, MAT, MFT}; - -pub type MatchesMap = StateMap; - -pub async fn matches_manager( - mut match_rx: mpsc::Receiver, - mut startup_match_rx: mpsc::Receiver, - mut socket_order_rx: mpsc::Receiver>, - action_tx: mpsc::Sender, - log_tx: mpsc::Sender, - mut stop: watch::Receiver, -) { - let mut matches: MatchesMap = BTreeMap::new(); - - let (unmatches_tx, mut unmatches_rx) = mpsc::channel(1); - - while let Some(mft) = startup_match_rx.recv().await { - let _ = handle_match(&mut matches, mft.clone(), &unmatches_tx, &action_tx, &stop).await; - } - - loop { - tokio::select! { - _ = stop.changed() => break, - Some(mft) = match_rx.recv() => { - let exec = handle_match(&mut matches, mft.clone(), &unmatches_tx, &action_tx, &stop).await; - - #[allow(clippy::unwrap_used)] // propagating panics is ok - log_tx - .send(DatabaseManagerInput::Log(LogEntry { - exec, - m: mft.m, - f: mft.o, - t: mft.t, - })) - .await - .unwrap(); - } - Some(mft) = unmatches_rx.recv() => { - matches.rm(&mft); - } - Some((order, options, tx)) = socket_order_rx.recv() => { - // FIXME do not clone - let filtered = matches.clone().filtered(options); - - if let Order::Flush = order { - let now = Local::now(); - // filter the state_map according to provided options - for (filter, inner_map) in &filtered { - for match_ in inner_map.keys() { - let mft = MFT { - m: match_.clone(), - o: filter, - t: now, - }; - // delete them from state - matches.rm_times(&mft); - // send them to DB - #[allow(clippy::unwrap_used)] // propagating panics is ok - log_tx - .send(DatabaseManagerInput::Flush(LogEntry { - exec: false, - m: mft.m, - f: mft.o, - t: mft.t, - })) - .await - .unwrap(); - } - } - } - - #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(filtered).unwrap(); - } - else => break - } - } -} - -async fn handle_match( - matches: &mut MatchesMap, - mft: MFT, - unmatches_tx: &mpsc::Sender, - action_tx: &mpsc::Sender, - stop: &watch::Receiver, -) -> bool { - // Store matches - let exec = match mft.o.retry() { - None => true, - Some(retry) => { - // Add new match - matches.add(&mft); - // Remove match when expired - { - let mft = mft.clone(); - let unmatches_tx = unmatches_tx.clone(); - let mut stop = stop.clone(); - tokio::spawn(async move { - #[allow(clippy::unwrap_used)] - // retry_duration is always Some() after filter's setup - let dur = (mft.t - Local::now() + mft.o.retry_duration().unwrap()) - .to_std() - .expect("Duration is bigger than what's supported. Did you put an enormous retry_duration?"); - tokio::select! { - biased; - _ = stop.changed() => {} - _ = time::sleep(dur) => { - #[allow(clippy::unwrap_used)] // propagating panics is ok - unmatches_tx.send(mft).await.unwrap(); - } - } - }); - } - - matches.get_times(&mft) >= retry as usize - } - }; - - // Executing actions - if exec { - // Delete matches only if storing them - if mft.o.retry().is_some() { - matches.rm_times(&mft); - } - mft.o.send_actions(&mft.m, mft.t, action_tx).await; - } - - exec -} diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index 6cb7174..db9779f 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -1,28 +1,28 @@ -use std::{error::Error, fs, path::PathBuf}; +use std::{collections::BTreeMap, error::Error, path::PathBuf, sync::Arc}; -use socket::socket_manager; use tokio::{ process::Child, select, signal::unix::{signal, SignalKind}, - sync::{mpsc, oneshot, watch}, + sync::{mpsc, oneshot, watch, Semaphore}, }; -use tracing::{error, info}; +use tracing::info; use crate::concepts::Config; use database::database_manager; -use execs::execs_manager; -use matches::matches_manager; +use filter::FilterManager; +use stream::stream_manager; mod database; -mod execs; -mod matches; -mod socket; -mod statemap; +// mod socket; + +mod action; +mod filter; +mod stream; pub async fn daemon( config_path: PathBuf, - socket: PathBuf, + _socket: PathBuf, ) -> Result<(), Box> { let config: &'static Config = Config::from_file(&config_path).map(|config| Box::leak(Box::new(config)))?; @@ -34,67 +34,76 @@ pub async fn daemon( let mut stream_process_child_handles = Vec::new(); let mut stream_task_handles = Vec::new(); - let (stream2match_tx, stream2match_rx) = mpsc::channel(123456); - let (database2match_tx, database2match_rx) = mpsc::channel(234560); + // let (socket2match_tx, socket2match_rx) = mpsc::channel(1); + // let (socket2exec_tx, socket2exec_rx) = mpsc::channel(1); - let (socket2match_tx, socket2match_rx) = mpsc::channel(1); - let (socket2exec_tx, socket2exec_rx) = mpsc::channel(1); - - let (matches2exec_tx, matches2exec_rx) = mpsc::channel(234560); let (log_tx, log_rx) = mpsc::channel(234560); // Shutdown channel let (shutdown_tx, shutdown_rx) = watch::channel(false); - let matches_manager_task_handle = { - let log_tx = log_tx.clone(); - let shutdown_rx = shutdown_rx.clone(); - tokio::spawn(async move { - matches_manager( - stream2match_rx, - database2match_rx, - socket2match_rx, - matches2exec_tx, - log_tx, - shutdown_rx, - ) - .await - }) + // Semaphore limiting action execution concurrency + let exec_limit = if config.concurrency() > 0 { + Some(Arc::new(Semaphore::new(config.concurrency()))) + } else { + None }; - let execs_manager_task_handle = { - let shutdown_rx = shutdown_rx.clone(); - tokio::spawn(async move { - execs_manager(config, matches2exec_rx, socket2exec_rx, log_tx, shutdown_rx).await - }) - }; + // Filter managers + let mut stream_filter_managers_handlers = BTreeMap::new(); + let mut log2filter_tx = BTreeMap::new(); + for stream in config.streams().values() { + let mut filter_managers_handlers = BTreeMap::new(); + for filter in stream.filters().values() { + let manager = FilterManager::new( + filter, + BTreeMap::default(), + exec_limit.clone(), + log_tx.clone(), + ); + let (tx, rx) = mpsc::channel(1); + let handle = tokio::spawn(async move { manager.handle_db_entries(rx).await }); + filter_managers_handlers.insert(filter, handle); + log2filter_tx.insert(filter, tx); + } + stream_filter_managers_handlers.insert(stream, filter_managers_handlers); + } + drop(log_tx); + drop(exec_limit); let database_manager_task_handle = { // The `task::spawn` is done in the function, after database rotation is finished - database_manager(config, log_rx, database2match_tx).await + database_manager(config, log_rx, log2filter_tx).await }; - let socket_manager_task_handle = { - let socket = socket.to_owned(); - let shutdown_rx = shutdown_rx.clone(); - tokio::spawn(async move { - socket_manager(config, socket, socket2match_tx, socket2exec_tx, shutdown_rx).await - }) - }; + let mut stream_filter_managers = BTreeMap::new(); + for (stream, filter_manager_handlers) in stream_filter_managers_handlers { + let mut filter_managers = BTreeMap::new(); + for (filter, filter_manager_handler) in filter_manager_handlers { + filter_managers.insert(filter, filter_manager_handler.await.unwrap()); + } + stream_filter_managers.insert(stream, filter_managers); + } - for stream in config.streams().values() { - let stream2match_tx = stream2match_tx.clone(); + // let socket_manager_task_handle = { + // let socket = socket.to_owned(); + // let shutdown_rx = shutdown_rx.clone(); + // tokio::spawn(async move { + // socket_manager(config, socket, socket2match_tx, socket2exec_tx, shutdown_rx).await + // }) + // }; + + for (stream, filter_managers) in stream_filter_managers { let (child_tx, child_rx) = oneshot::channel(); stream_task_handles.push(tokio::spawn(async move { - stream.manager(child_tx, stream2match_tx).await + stream_manager(stream, child_tx, filter_managers.into_values().collect()).await })); if let Ok(Some(child)) = child_rx.await { stream_process_child_handles.push(child); } } - drop(stream2match_tx); // Close streams when we receive a quit signal handle_signals(stream_process_child_handles, shutdown_tx.clone())?; @@ -105,19 +114,11 @@ pub async fn daemon( } let _ = shutdown_tx.send(true); - let _ = socket_manager_task_handle.await; - let _ = matches_manager_task_handle.await; - let _ = execs_manager_task_handle.await; + // let _ = socket_manager_task_handle.await; let _ = database_manager_task_handle.await; let stop_ok = config.stop(); - // not waiting for the socket_manager to finish, sorry - // TODO make it listen on shutdown_rx - if let Err(err) = fs::remove_file(socket) { - error!("failed to remove socket: {}", err); - } - if !*shutdown_rx.borrow() { Err("quitting because all streams finished".into()) } else if !stop_ok { diff --git a/rust/src/daemon/socket.rs b/rust/src/daemon/socket.rs index 3bc2fec..4a500d0 100644 --- a/rust/src/daemon/socket.rs +++ b/rust/src/daemon/socket.rs @@ -198,7 +198,7 @@ pub async fn socket_manager( exec_tx: mpsc::Sender>, mut stop: watch::Receiver, ) { - let listener = match open_socket(socket) { + let listener = match open_socket(socket.clone()) { Ok(l) => l, Err(err) => { error!("while creating communication socket: {err}"); @@ -245,4 +245,8 @@ pub async fn socket_manager( } } } + + if let Err(err) = fs::remove_file(socket) { + error!("failed to remove socket: {}", err); + } } diff --git a/rust/src/daemon/statemap.rs b/rust/src/daemon/statemap.rs deleted file mode 100644 index 0af8142..0000000 --- a/rust/src/daemon/statemap.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, -}; - -use regex::Regex; - -use crate::concepts::{ActionFilter, Match, Pattern, Time, MT}; - -#[derive(Clone, Debug)] -pub struct FilterOptions { - pub stream_name: Option, - pub filter_name: Option, - pub patterns: BTreeMap, Regex>, -} - -pub type StateMap = BTreeMap<&'static T, BTreeMap>>; - -// This trait is needed to permit to implement methods on an external type -pub trait StateMapTrait { - fn add(&mut self, mt: &MT); - fn rm(&mut self, mt: &MT) -> bool; - fn rm_times(&mut self, mt: &MT) -> Option>; - fn get_times(&self, mt: &MT) -> usize; - fn filtered(&self, filter_options: FilterOptions) -> Self; -} - -impl StateMapTrait for StateMap { - fn add(&mut self, mt: &MT) { - let inner_map = self.entry(mt.o).or_default(); - let inner_set = inner_map.entry(mt.m.clone()).or_default(); - inner_set.insert(mt.t); - } - - fn rm(&mut self, mt: &MT) -> bool { - let mut removed = false; - if let Some(inner_map) = self.get_mut(&mt.o) { - if let Some(inner_set) = inner_map.get_mut(&mt.m) { - inner_set.remove(&mt.t); - removed = true; - if inner_set.is_empty() { - inner_map.remove(&mt.m); - } - } - if inner_map.is_empty() { - self.remove(&mt.o); - } - } - removed - } - - fn rm_times(&mut self, mt: &MT) -> Option> { - let mut set = None; - if let Some(inner_map) = self.get_mut(&mt.o) { - set = inner_map.remove(&mt.m); - if inner_map.is_empty() { - self.remove(&mt.o); - } - } - set - } - - fn get_times(&self, mt: &MT) -> usize { - match self.get(&mt.o).and_then(|map| map.get(&mt.m)) { - Some(x) => x.len(), - None => 0, - } - } - - fn filtered(&self, filter_options: FilterOptions) -> Self { - let FilterOptions { - stream_name, - filter_name, - patterns, - } = filter_options; - self.iter() - // stream/filter filtering - .filter(|(object, _)| { - if let Some(stream_name) = &stream_name { - let full_name = object.full_name(); - let (s, f) = (full_name.0, full_name.1); - if *stream_name != s { - return false; - } - if let Some(filter_name) = &filter_name { - if *filter_name != f { - return false; - } - } - } - true - }) - // pattern filtering - .filter(|(object, _)| { - patterns - .iter() - .all(|(pattern, _)| object.patterns().get(pattern).is_some()) - }) - // match filtering - .filter_map(|(object, inner_map)| { - let map: BTreeMap> = inner_map - .iter() - .filter(|(match_, _)| { - match_ - .iter() - .zip(object.patterns()) - .filter_map(|(a_match, pattern)| { - patterns.get(pattern.as_ref()).map(|regex| (a_match, regex)) - }) - .all(|(a_match, regex)| regex.is_match(a_match)) - }) - .map(|(a, b)| (a.clone(), b.clone())) - .collect(); - if !map.is_empty() { - Some((*object, map)) - } else { - None - } - }) - .collect() - } -} diff --git a/rust/src/daemon/stream.rs b/rust/src/daemon/stream.rs new file mode 100644 index 0000000..eba3ad7 --- /dev/null +++ b/rust/src/daemon/stream.rs @@ -0,0 +1,69 @@ +use std::process::Stdio; + +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::{Child, Command}, + sync::oneshot, +}; +use tracing::{error, info}; + +use crate::{concepts::Stream, daemon::filter::FilterManager}; + +pub async fn stream_manager( + stream: &'static Stream, + child_tx: oneshot::Sender>, + mut filter_managers: Vec, +) { + info!("{}: start {:?}", stream.name(), stream.cmd()); + let mut child = match Command::new(&stream.cmd()[0]) + .args(&stream.cmd()[1..]) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .stdout(Stdio::piped()) + .spawn() + { + Ok(child) => child, + Err(err) => { + error!("could not execute stream {} cmd: {}", stream.name(), err); + let _ = child_tx.send(None); + return; + } + }; + + // keep stdout before sending/moving child to the main thread + #[allow(clippy::unwrap_used)] + // we know there is an stdout because we asked for Stdio::piped() + let mut lines = BufReader::new(child.stdout.take().unwrap()).lines(); + + // let main handle the child process + let _ = child_tx.send(Some(child)); + + loop { + match lines.next_line().await { + Ok(Some(line)) => { + futures::future::join_all( + filter_managers + .iter_mut() + .map(|manager| manager.handle_line(&line)), + ) + .await; + } + Ok(None) => { + error!("stream {} exited: its command returned.", stream.name()); + break; + } + Err(err) => { + error!( + "impossible to read output from stream {}: {}", + stream.name(), + err + ); + break; + } + } + } + + filter_managers + .iter_mut() + .for_each(|manager| manager.quit()); +} diff --git a/rust/src/main.rs b/rust/src/main.rs index 33c292c..4e80d8f 100644 --- a/rust/src/main.rs +++ b/rust/src/main.rs @@ -25,34 +25,34 @@ async fn main() { eprintln!("this error occurred."); })); - console_subscriber::init(); + // console_subscriber::init(); let cli = Cli::parse(); - // { - // // Set log level - // let level = if let Command::Start { - // loglevel, - // config: _, - // socket: _, - // } = cli.command - // { - // loglevel - // } else { - // Level::DEBUG - // }; - // if let Err(err) = tracing_subscriber::fmt::fmt() - // .without_time() - // .with_target(false) - // .with_ansi(std::io::stdout().is_terminal()) - // // .with_max_level(level) - // .with_max_level(Level::TRACE) - // .try_init() - // { - // eprintln!("ERROR could not initialize logging: {err}"); - // exit(1); - // } - // } + { + // Set log level + let level = if let Command::Start { + loglevel, + config: _, + socket: _, + } = cli.command + { + loglevel + } else { + Level::DEBUG + }; + if let Err(err) = tracing_subscriber::fmt::fmt() + .without_time() + .with_target(false) + .with_ansi(std::io::stdout().is_terminal()) + .with_max_level(level) + // .with_max_level(Level::TRACE) + .try_init() + { + eprintln!("ERROR could not initialize logging: {err}"); + exit(1); + } + } let result = match cli.command { Command::Start { From d2345d6047d9e8501dd9d5336d1e848d6eb3d106 Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 15 Oct 2024 12:00:00 +0200 Subject: [PATCH 066/435] Switch back database code to sync Enormous performance gain! INFO tailDown3.find: match ["100"] ERROR stream tailDown3 exited: its command returned. INFO Rotated database ________________________________________________________ Executed in 31.27 secs fish external usr time 35.24 secs 0.00 micros 35.24 secs sys time 27.09 secs 343.00 micros 27.09 secs TODO reimpl the socket part. --- rust/src/daemon/database/lowlevel.rs | 96 ++++++++++++---------------- rust/src/daemon/database/mod.rs | 49 +++++++------- rust/src/daemon/mod.rs | 6 +- 3 files changed, 67 insertions(+), 84 deletions(-) diff --git a/rust/src/daemon/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs index 2d3bc3e..b831579 100644 --- a/rust/src/daemon/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -1,17 +1,8 @@ -use std::{collections::BTreeMap, fmt::Debug, process::exit}; +use std::{collections::BTreeMap, fmt::Debug, fs::File, io::{self, BufReader, BufWriter, Write}, process::exit}; use bincode::Options; use chrono::{DateTime, Local}; -use futures::{io, SinkExt, StreamExt}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use tokio::{ - fs::File, - io::{BufReader, BufWriter}, -}; -use tokio_util::{ - bytes::Bytes, - codec::{FramedRead, FramedWrite, LengthDelimitedCodec}, -}; use tracing::{debug, error, warn}; use crate::{ @@ -32,14 +23,14 @@ type WriteHeader = BTreeMap<&'static Filter, usize>; const DB_SIGNATURE: &str = "reaction-db-v01"; pub struct ReadDB { - f: FramedRead, LengthDelimitedCodec>, + f: BufReader, h: ReadHeader, bin: BincodeOptions, } impl ReadDB { - pub async fn open(path: &str, config: &'static Config) -> Result, DBError> { - let file = match File::open(path).await { + pub fn open(path: &str, config: &'static Config) -> Result, DBError> { + let file = match File::open(path) { Ok(file) => file, Err(err) => match err.kind() { std::io::ErrorKind::NotFound => { @@ -56,31 +47,24 @@ impl ReadDB { }; let mut ret = ReadDB { - f: FramedRead::new(BufReader::new(file), LengthDelimitedCodec::new()), + f: BufReader::new(file), h: BTreeMap::default(), bin: bincode_options(), }; - match ret.read::().await { - Some(Ok(signature)) => { + match ret.read::() { + Ok(signature) => { if DB_SIGNATURE == signature { Ok(()) } else { Err(DBError::Error("database is not a reaction database".into())) } } - Some(Err(err)) => Err(DBError::Error(format!("reading database signature: {err}"))), - None => Err(DBError::Error( - "EOF while reading database header".to_string(), - )), + Err(err) => Err(DBError::Error(format!("reading database signature: {err}"))), }?; let db_header = ret .read::() - .await - .unwrap_or(Err(DBError::Error( - "EOF before database header".to_string(), - ))) .map_err(|err| DBError::Error(format!("while reading database header: {err}")))?; ret.h = db_header @@ -91,40 +75,40 @@ impl ReadDB { Ok(Some(ret)) } - async fn read(&mut self) -> Option> { - match self.f.next().await? { - Err(err) => Some(Err(err.into())), - Ok(encoded) => Some({ - match self.bin.deserialize::(&encoded) { - Ok(decoded) => { - debug!("reading this: {:?}", decoded); - Ok(decoded) - } - Err(err) => Err(err.into()), - } - }), - } + fn read(&mut self) -> Result { + let decoded = self.bin.deserialize_from::<_, T>(&mut self.f)?; + debug!("reading this: {:?}", &decoded); + Ok(decoded) } +} - // FIXME use channel instead? - pub async fn next(&mut self) -> Option> { - let res = self.read::().await; - res.map(|item| match item { - Ok(item) => item.to(&self.h), - Err(err) => Err(err), - }) +impl Iterator for ReadDB { + type Item = Result; + + fn next(&mut self) -> Option { + let res = self.read::(); + match res { + Ok(item) => Some(item.to(&self.h)), + Err(err) => match *err { + bincode::ErrorKind::Io(err) => match err.kind() { + io::ErrorKind::UnexpectedEof => None, + _ => Some(Err(err.into())), + }, + _ => Some(Err(err.into())), + }, + } } } pub struct WriteDB { - f: FramedWrite, LengthDelimitedCodec>, + f: BufWriter, h: WriteHeader, bin: BincodeOptions, } impl WriteDB { - pub async fn create(path: &str, config: &'static Config) -> Self { - let file = match File::create(path).await { + pub fn create(path: &str, config: &'static Config) -> Self { + let file = match File::create(path) { Ok(file) => file, Err(err) => { error!("Failed to create DB: {}", err); @@ -133,12 +117,12 @@ impl WriteDB { }; let mut ret = WriteDB { - f: FramedWrite::new(BufWriter::new(file), LengthDelimitedCodec::new()), + f: BufWriter::new(file), h: BTreeMap::default(), bin: bincode_options(), }; - if let Err(err) = ret._write(DB_SIGNATURE).await { + if let Err(err) = ret._write(DB_SIGNATURE) { error!("Failed to write to DB: {}", err); exit(1); } @@ -153,7 +137,7 @@ impl WriteDB { .enumerate() .collect(); - if let Err(err) = ret._write(&database_header).await { + if let Err(err) = ret._write(&database_header) { error!("Failed to write to DB: {}", err); exit(1); } @@ -168,20 +152,20 @@ impl WriteDB { ret } - pub async fn write(&mut self, entry: LogEntry) -> Result<(), DBError> { + pub fn write(&mut self, entry: LogEntry) -> Result<(), DBError> { let computed = ComputedLogEntry::from(entry, &self.h)?; - self._write(computed).await + self._write(computed) } - async fn _write(&mut self, data: T) -> Result<(), DBError> { + fn _write(&mut self, data: T) -> Result<(), DBError> { let encoded = self.bin.serialize(&data)?; // debug!("writing this: {:?}, {:?}", &data, &encoded); - self.f.send(Bytes::from(encoded)).await?; + self.f.write_all(&encoded)?; Ok(()) } - pub async fn flush(&mut self) -> io::Result<()> { - self.f.flush().await + pub fn flush(&mut self) -> io::Result<()> { + self.f.flush() } } diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index 2bdb1f2..fcf1bea 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -1,13 +1,14 @@ use std::{ collections::{BTreeMap, HashMap}, fmt::Debug, - fs, + fs, io, process::exit, + thread, }; use chrono::{Local, TimeDelta}; use thiserror::Error; -use tokio::{sync::mpsc, task}; +use tokio::sync::mpsc; use tracing::{debug, error, info, warn}; use crate::concepts::{ActionFilter, Config, Filter, LogEntry, Match, Time, MFT}; @@ -30,7 +31,7 @@ pub enum DBError { #[error("decode error: {0}")] BincodeError(#[from] bincode::Error), #[error("io error: {0}")] - IOError(#[from] tokio::io::Error), + IOError(#[from] io::Error), #[error("{0}")] Error(String), } @@ -44,7 +45,7 @@ pub enum DatabaseManagerInput { // Just discovering macros, let me be useless macro_rules! write_or_die { ($db:expr, $entry:expr) => { - if let Err(err) = $db.write($entry).await { + if let Err(err) = $db.write($entry) { error!("Could not write to DB: {}", err); exit(1); } @@ -52,7 +53,7 @@ macro_rules! write_or_die { } macro_rules! flush_or_die { ($db:expr) => { - if let Err(err) = $db.flush().await { + if let Err(err) = $db.flush() { error!("Could not flush DB: {}", err); exit(1); } @@ -60,12 +61,12 @@ macro_rules! flush_or_die { } /// First rotates the database, then spawns the database thread -pub async fn database_manager( +pub fn database_manager( config: &'static Config, mut log_rx: mpsc::Receiver, matches_tx: BTreeMap<&Filter, mpsc::Sender>, -) -> task::JoinHandle<()> { - let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)).await { +) -> thread::JoinHandle<()> { + let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)) { Ok(dbs) => dbs, Err(err) => { error!("while rotating databases on start: {}", err); @@ -73,9 +74,9 @@ pub async fn database_manager( } }; - task::spawn(async move { + thread::spawn(move || { let mut cpt = 0; - while let Some(order) = log_rx.recv().await { + while let Some(order) = log_rx.blocking_recv() { match order { DatabaseManagerInput::Flush(entry) => write_or_die!(flush_db, entry), DatabaseManagerInput::Log(entry) => { @@ -88,7 +89,7 @@ pub async fn database_manager( flush_or_die!(flush_db); drop(log_db); drop(flush_db); - (log_db, flush_db) = match rotate_db(config, None).await { + (log_db, flush_db) = match rotate_db(config, None) { Ok(dbs) => dbs, Err(err) => { error!( @@ -108,28 +109,27 @@ pub async fn database_manager( }) } -async fn rotate_db( +fn rotate_db( config: &'static Config, matches_tx: Option>>, ) -> Result<(WriteDB, WriteDB), DBError> { - // TODO asyncify this - let mut log_read_db = match ReadDB::open(LOG_DB_NAME, config).await? { + let mut log_read_db = match ReadDB::open(LOG_DB_NAME, config)? { Some(db) => db, None => { - return Ok(tokio::join!( + return Ok(( WriteDB::create(LOG_DB_NAME, config), WriteDB::create(FLUSH_DB_NAME, config), )); } }; - let mut flush_read_db = match ReadDB::open(FLUSH_DB_NAME, config).await? { + let mut flush_read_db = match ReadDB::open(FLUSH_DB_NAME, config)? { Some(db) => db, None => { warn!( "Strange! Found a {} but no {}, opening /dev/null instead", LOG_DB_NAME, FLUSH_DB_NAME ); - match ReadDB::open("/dev/null", config).await? { + match ReadDB::open("/dev/null", config)? { Some(db) => db, None => { return Err(DBError::Error("/dev/null is not accessible".into())); @@ -138,15 +138,14 @@ async fn rotate_db( } }; - let mut log_write_db = WriteDB::create(LOG_DB_NEW_NAME, config).await; + let mut log_write_db = WriteDB::create(LOG_DB_NEW_NAME, config); _rotate_db( matches_tx, &mut log_read_db, &mut flush_read_db, &mut log_write_db, - ) - .await; + ); drop(log_read_db); drop(flush_read_db); @@ -162,11 +161,11 @@ async fn rotate_db( return Err(DBError::Error(format!("Failed to delete old DB: {}", err))); } - let flush_write_db = WriteDB::create(FLUSH_DB_NAME, config).await; + let flush_write_db = WriteDB::create(FLUSH_DB_NAME, config); Ok((log_write_db, flush_write_db)) } -async fn _rotate_db( +fn _rotate_db( matches_tx: Option>>, log_read_db: &mut ReadDB, flush_read_db: &mut ReadDB, @@ -178,7 +177,7 @@ async fn _rotate_db( // Read flushes let mut flushes: BTreeMap<&'static Filter, BTreeMap> = BTreeMap::new(); - while let Some(flush_entry) = flush_read_db.next().await { + while let Some(flush_entry) = flush_read_db.next() { match flush_entry { Ok(entry) => { let matches_map = flushes.entry(entry.f).or_default(); @@ -194,7 +193,7 @@ async fn _rotate_db( let now = Local::now(); // Read matches - while let Some(log_entry) = log_read_db.next().await { + while let Some(log_entry) = log_read_db.next() { match log_entry { Ok(mut entry) => { // Check if number of patterns is in sync @@ -234,7 +233,7 @@ async fn _rotate_db( if let Some(tx) = matches_tx.get(entry.f) { debug!("DB sending match from DB: {:?}", entry.m); #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(entry.clone().into()).await.unwrap(); + tx.blocking_send(entry.clone().into()).unwrap(); } } diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index db9779f..bb35dbf 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -71,9 +71,9 @@ pub async fn daemon( drop(log_tx); drop(exec_limit); - let database_manager_task_handle = { + let database_manager_thread_handle = { // The `task::spawn` is done in the function, after database rotation is finished - database_manager(config, log_rx, log2filter_tx).await + database_manager(config, log_rx, log2filter_tx) }; let mut stream_filter_managers = BTreeMap::new(); @@ -115,7 +115,7 @@ pub async fn daemon( let _ = shutdown_tx.send(true); // let _ = socket_manager_task_handle.await; - let _ = database_manager_task_handle.await; + let _ = database_manager_thread_handle.join(); let stop_ok = config.stop(); From 2e00092c18a08060a4eddd3dd60fb68967720fa4 Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 20 Oct 2024 12:00:00 +0200 Subject: [PATCH 067/435] WIP reimplement socket part I think I'm stumbling on this rust compiler bug: https://github.com/rust-lang/rust/issues/110338 --- rust/src/concepts/action.rs | 3 + rust/src/concepts/filter.rs | 4 + rust/src/daemon/action.rs | 17 +++- rust/src/daemon/filter.rs | 40 +++++++- rust/src/daemon/mod.rs | 78 +++++++++----- rust/src/daemon/socket.rs | 196 ++++++++++++++---------------------- rust/src/daemon/stream.rs | 19 ++-- rust/src/lib.rs | 4 +- 8 files changed, 203 insertions(+), 158 deletions(-) diff --git a/rust/src/concepts/action.rs b/rust/src/concepts/action.rs index 0d5c9b4..d4a707f 100644 --- a/rust/src/concepts/action.rs +++ b/rust/src/concepts/action.rs @@ -46,6 +46,9 @@ impl ActionFilter for Action { } impl Action { + pub fn name(&self) -> &str { + &self.name + } pub fn after_duration(&self) -> Option { self.after_duration } diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index ec6c2ac..5449de3 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -66,6 +66,10 @@ impl Filter { } } + pub fn name(&self) -> &str { + &self.name + } + pub fn retry(&self) -> Option { self.retry } diff --git a/rust/src/daemon/action.rs b/rust/src/daemon/action.rs index b812a2f..7a94982 100644 --- a/rust/src/daemon/action.rs +++ b/rust/src/daemon/action.rs @@ -28,8 +28,7 @@ impl State { self.ordered_times.remove(&t).is_some() } - fn clear_past_times(&mut self, after: Option) { - let now = Local::now(); + fn clear_past_times(&mut self, now: Time, after: Option) { let after = after.unwrap_or_default(); while self .ordered_times @@ -50,6 +49,10 @@ pub struct ActionManager { } impl ActionManager { + pub fn action(&self) -> &'static Action { + self.action + } + pub fn new( action: &'static Action, pending: BTreeMap>, @@ -76,7 +79,7 @@ impl ActionManager { } else { { let mut state = self.state.lock().unwrap(); - state.clear_past_times(self.action.after_duration()); + state.clear_past_times(t, self.action.after_duration()); state.add_match(&m, exec_t); } let this = self.clone(); @@ -91,6 +94,14 @@ impl ActionManager { } } + pub fn to_readable_vec(&self, match_: &Match) -> Option> { + self.state.lock().unwrap().pending.get(match_).map(|set| { + set.iter() + .map(|time| time.to_rfc3339().chars().take(19).collect()) + .collect() + }) + } + fn exec_now(&self, m: Match) { let semaphore = self.exec_limit.clone(); let action = self.action; diff --git a/rust/src/daemon/filter.rs b/rust/src/daemon/filter.rs index 4f76e34..da913db 100644 --- a/rust/src/daemon/filter.rs +++ b/rust/src/daemon/filter.rs @@ -4,9 +4,10 @@ use std::{ }; use chrono::Local; +use regex::Regex; use tokio::sync::{mpsc, Semaphore}; -use crate::concepts::{Filter, LogEntry, Match, Time, MFT}; +use crate::concepts::{ActionFilter, Filter, LogEntry, Match, Pattern, PatternStatus, Time, MFT}; use super::{action::ActionManager, database::DatabaseManagerInput}; @@ -94,6 +95,43 @@ impl FilterManager { .for_each(|manager| manager.quit()); } + pub fn to_pattern_status_map( + &self, + patterns: &BTreeMap, Regex>, + ) -> BTreeMap { + self.matches + .iter() + // TODO match filtering + .filter(|(match_, _)| { + match_ + .iter() + .zip(self.filter.patterns()) + .filter_map(|(a_match, pattern)| { + patterns.get(pattern.as_ref()).map(|regex| (a_match, regex)) + }) + .all(|(a_match, regex)| regex.is_match(a_match)) + }) + .map(|(match_, times)| { + let actions = + self.action_managers + .iter() + .fold(BTreeMap::default(), |mut acc, manager| { + if let Some(times) = manager.to_readable_vec(match_) { + acc.insert(manager.action().name().to_owned(), times); + } + acc + }); + ( + match_.join(" "), + PatternStatus { + matches: times.len(), + actions, + }, + ) + }) + .collect() + } + fn add_match(&mut self, m: &Match, t: Time) { self.matches.entry(m.clone()).or_default().insert(t); self.ordered_times.insert(t, m.clone()); diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index bb35dbf..9ab48bd 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -1,28 +1,48 @@ -use std::{collections::BTreeMap, error::Error, path::PathBuf, sync::Arc}; +use std::{ + collections::BTreeMap, + error::Error, + path::PathBuf, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; +use socket::socket_manager; use tokio::{ process::Child, select, signal::unix::{signal, SignalKind}, - sync::{mpsc, oneshot, watch, Semaphore}, + sync::{broadcast, mpsc, oneshot, Mutex, Semaphore}, }; use tracing::info; -use crate::concepts::Config; +use crate::concepts::{Config, Filter, Stream}; use database::database_manager; use filter::FilterManager; use stream::stream_manager; mod database; -// mod socket; +mod socket; mod action; mod filter; mod stream; +// type SharedState = BTreeMap<&'static Stream, Arc>>>; +struct SharedState { + pub s: BTreeMap<&'static Stream, Arc>>>, +} + +#[allow(unsafe_code)] +// It's actually safe. It's a bug in rust that shadows the 'static reference, which ensures the +// reference will always link to something +// https://github.com/rust-lang/rust/issues/96865 +unsafe impl Send for SharedState {} + pub async fn daemon( config_path: PathBuf, - _socket: PathBuf, + socket: PathBuf, ) -> Result<(), Box> { let config: &'static Config = Config::from_file(&config_path).map(|config| Box::leak(Box::new(config)))?; @@ -40,7 +60,7 @@ pub async fn daemon( let (log_tx, log_rx) = mpsc::channel(234560); // Shutdown channel - let (shutdown_tx, shutdown_rx) = watch::channel(false); + let (shutdown_tx, shutdown_rx) = broadcast::channel(1); // Semaphore limiting action execution concurrency let exec_limit = if config.concurrency() > 0 { @@ -76,28 +96,23 @@ pub async fn daemon( database_manager(config, log_rx, log2filter_tx) }; - let mut stream_filter_managers = BTreeMap::new(); + let mut stream_filter_managers = SharedState { s: BTreeMap::new() }; for (stream, filter_manager_handlers) in stream_filter_managers_handlers { let mut filter_managers = BTreeMap::new(); for (filter, filter_manager_handler) in filter_manager_handlers { filter_managers.insert(filter, filter_manager_handler.await.unwrap()); } - stream_filter_managers.insert(stream, filter_managers); + stream_filter_managers.s.insert(stream, Arc::new(Mutex::new(filter_managers))); } + let stream_filter_managers: Arc = Arc::new(stream_filter_managers); - // let socket_manager_task_handle = { - // let socket = socket.to_owned(); - // let shutdown_rx = shutdown_rx.clone(); - // tokio::spawn(async move { - // socket_manager(config, socket, socket2match_tx, socket2exec_tx, shutdown_rx).await - // }) - // }; - - for (stream, filter_managers) in stream_filter_managers { + for (stream, filter_managers) in stream_filter_managers.s.iter() { let (child_tx, child_rx) = oneshot::channel(); + let filter_managers = filter_managers.clone(); + let stream = *stream; stream_task_handles.push(tokio::spawn(async move { - stream_manager(stream, child_tx, filter_managers.into_values().collect()).await + stream_manager(stream, child_tx, filter_managers).await })); if let Ok(Some(child)) = child_rx.await { @@ -106,20 +121,33 @@ pub async fn daemon( } // Close streams when we receive a quit signal - handle_signals(stream_process_child_handles, shutdown_tx.clone())?; + let signal_received = Arc::new(AtomicBool::new(false)); + handle_signals( + stream_process_child_handles, + shutdown_tx.clone(), + signal_received.clone(), + )?; + + let socket_manager_task_handle = { + let socket = socket.to_owned(); + let stream_filter_managers: Arc = stream_filter_managers.clone(); + tokio::spawn(async move { + socket_manager(config, socket, stream_filter_managers, shutdown_rx).await + }) + }; // Wait for all streams to quit for task_handle in stream_task_handles { let _ = task_handle.await; } - let _ = shutdown_tx.send(true); + let _ = shutdown_tx.send(()); - // let _ = socket_manager_task_handle.await; + let _ = socket_manager_task_handle.await; let _ = database_manager_thread_handle.join(); let stop_ok = config.stop(); - if !*shutdown_rx.borrow() { + if !signal_received.load(Ordering::SeqCst) { Err("quitting because all streams finished".into()) } else if !stop_ok { Err("while executing stop command".into()) @@ -130,7 +158,8 @@ pub async fn daemon( fn handle_signals( stream_process_child_handles: Vec, - shutdown_tx: watch::Sender, + shutdown_tx: broadcast::Sender<()>, + signal_received: Arc, ) -> tokio::io::Result<()> { let mut sighup = signal(SignalKind::hangup())?; let mut sigint = signal(SignalKind::interrupt())?; @@ -141,7 +170,8 @@ fn handle_signals( _ = sigint.recv() => "SIGINT", _ = sigterm.recv() => "SIGTERM", }; - let _ = shutdown_tx.send(true); + let _ = shutdown_tx.send(()); + signal_received.store(true, Ordering::SeqCst); info!("received {signal}, closing streams..."); // Kill stream subprocesses for mut child_handle in stream_process_child_handles.into_iter() { diff --git a/rust/src/daemon/socket.rs b/rust/src/daemon/socket.rs index 4a500d0..82931e5 100644 --- a/rust/src/daemon/socket.rs +++ b/rust/src/daemon/socket.rs @@ -3,11 +3,7 @@ use std::{collections::BTreeMap, fs, io, path::PathBuf, process::exit, sync::Arc use bincode::Options; use futures::{SinkExt, StreamExt}; use regex::Regex; -use tokio::{ - join, - net::UnixListener, - sync::{mpsc, oneshot, watch}, -}; +use tokio::{net::UnixListener, sync::{broadcast, oneshot}}; use tokio_util::{ bytes::Bytes, codec::{Framed, LengthDelimitedCodec}, @@ -15,14 +11,18 @@ use tokio_util::{ use tracing::{error, warn}; use crate::{ - concepts::{ - ActionFilter, ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern, - PatternStatus, - }, + concepts::{ActionFilter, ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern}, utils::bincode_options, }; -use super::{execs::ExecsMap, matches::MatchesMap, statemap::FilterOptions}; +use super::SharedState; + +#[derive(Clone)] +pub struct FilterOptions { + pub stream_name: Option, + pub filter_name: Option, + pub patterns: BTreeMap, Regex>, +} macro_rules! err_str { ($expression:expr) => { @@ -57,122 +57,75 @@ fn open_socket(path: PathBuf) -> Result { async fn answer_order( config: &'static Config, - match_tx: &mpsc::Sender>, - exec_tx: &mpsc::Sender>, + shared_state: &SharedState, options: ClientRequest, ) -> Result { // Compute options - let filtering_options = { - let (stream_name, filter_name) = match options.stream_filter { - Some(sf) => match sf.split_once(".") { - Some((s, f)) => (Some(s.to_string()), Some(f.to_string())), - None => (Some(sf), None), - }, - None => (None, None), - }; - - // Compute the Vec<(pattern_name, String)> into a BTreeMap, Regex> - let patterns = options - .patterns - .iter() - .map(|(name, reg)| { - // lookup pattern in config.patterns - config - .patterns() - .iter() - // retrieve or Err - .find(|(pattern_name, _)| name == *pattern_name) - .ok_or_else(|| format!("pattern '{name}' doesn't exist")) - // compile Regex or Err - .and_then(|(_, pattern)| match Regex::new(reg) { - Ok(reg) => Ok((pattern.clone(), reg)), - Err(err) => Err(format!("pattern '{name}' regex doesn't compile: {err}")), - }) - }) - .collect::, Regex>, String>>()?; - - FilterOptions { - stream_name, - filter_name, - patterns, - } + let (stream_name, filter_name) = match options.stream_filter { + Some(sf) => match sf.split_once(".") { + Some((s, f)) => (Some(s.to_string()), Some(f.to_string())), + None => (Some(sf), None), + }, + None => (None, None), }; - // ask for matches clone - let filtering_options2 = filtering_options.clone(); - let matches = async move { - let (m_tx, m_rx) = oneshot::channel(); + // Compute the Vec<(pattern_name: String, regex: String)> into a BTreeMap, Regex> + let patterns = options + .patterns + .iter() + .map(|(name, reg)| { + // lookup pattern in config.patterns + config + .patterns() + .iter() + // retrieve or Err + .find(|(pattern_name, _)| name == *pattern_name) + .ok_or_else(|| format!("pattern '{name}' doesn't exist")) + // compile Regex or Err + .and_then(|(_, pattern)| match Regex::new(reg) { + Ok(reg) => Ok((pattern.clone(), reg)), + Err(err) => Err(format!("pattern '{name}' regex doesn't compile: {err}")), + }) + }) + .collect::, Regex>, String>>()?; - #[allow(clippy::unwrap_used)] // propagating panics is ok - match_tx - .send((options.order, filtering_options2, m_tx)) - .await - .unwrap(); - - #[allow(clippy::unwrap_used)] // propagating panics is ok - m_rx.await.unwrap() - }; - - // ask for execs clone - let execs = async move { - let (e_tx, e_rx) = oneshot::channel(); - - #[allow(clippy::unwrap_used)] // propagating panics is ok - exec_tx - .send((options.order, filtering_options, e_tx)) - .await - .unwrap(); - - #[allow(clippy::unwrap_used)] // propagating panics is ok - e_rx.await.unwrap() - }; - - let (matches, execs) = join!(matches, execs); - - // Transform matches and execs into a ClientStatus - let cs: ClientStatus = matches - .into_iter() - .fold(BTreeMap::new(), |mut acc, (object, map)| { - let (stream, filter, _) = object.full_name(); - acc.entry(stream.into()) - .or_default() - .entry(filter.into()) - .or_default() - .extend(map.into_iter().map(|(match_, times)| { + // TODO directly call flush function here? + let cs: ClientStatus = futures::stream::iter(shared_state.s.iter()) + // stream filtering + .filter(|(stream, _)| async { + stream_name.is_none() + || stream_name + .clone() + .is_some_and(|name| name == stream.name()) + }) + .fold(BTreeMap::new(), |mut acc, (stream, filter_manager)| async { + let filter_manager = filter_manager.lock().await; + let inner_map = filter_manager + .iter() + // filter filtering + .filter(|(filter, _)| { + filter_name.is_none() + || filter_name + .clone() + .is_some_and(|name| name == filter.name()) + }) + // pattern filtering + .filter(|(filter, _)| { + patterns + .iter() + .all(|(pattern, _)| filter.patterns().get(pattern).is_some()) + }) + .map(|(filter, manager)| { ( - match_.join(" "), - PatternStatus { - matches: times.len(), - ..Default::default() - }, + filter.name().to_owned(), + manager.to_pattern_status_map(&patterns), ) - })); + }) + .collect(); + acc.insert(stream.name().to_owned(), inner_map); acc - }); - - let cs = execs.into_iter().fold(cs, |mut acc, (object, map)| { - let (stream, filter, action) = object.full_name(); - let inner_map = acc - .entry(stream.into()) - .or_default() - .entry(filter.into()) - .or_default(); - - map.into_iter().for_each(|(match_, times)| { - inner_map - .entry(match_.join(" ")) - .or_default() - .actions - .insert( - action.to_string(), - times - .into_iter() - .map(|time| time.to_rfc3339().chars().take(19).collect()) - .collect(), - ); - }); - acc - }); + }) + .await; Ok(cs) } @@ -194,9 +147,8 @@ pub type SocketOrder = (Order, FilterOptions, oneshot::Sender); pub async fn socket_manager( config: &'static Config, socket: PathBuf, - match_tx: mpsc::Sender>, - exec_tx: mpsc::Sender>, - mut stop: watch::Receiver, + shared_state: Arc, + mut stop: broadcast::Receiver<()>, ) { let listener = match open_socket(socket.clone()) { Ok(l) => l, @@ -209,7 +161,7 @@ pub async fn socket_manager( let bin = bincode_options(); loop { tokio::select! { - _ = stop.changed() => break, + _ = stop.recv() => break, try_conn = listener.accept() => { match try_conn { Ok((conn, _)) => { @@ -228,7 +180,7 @@ pub async fn socket_manager( bin.deserialize(&encoded_request) ); // Process - let response = match answer_order(config, &match_tx, &exec_tx, request).await { + let response = match answer_order(config, &shared_state, request).await { Ok(res) => DaemonResponse::Order(res), Err(err) => DaemonResponse::Err(err), }; diff --git a/rust/src/daemon/stream.rs b/rust/src/daemon/stream.rs index eba3ad7..b0f31e9 100644 --- a/rust/src/daemon/stream.rs +++ b/rust/src/daemon/stream.rs @@ -1,18 +1,21 @@ -use std::process::Stdio; +use std::{collections::BTreeMap, process::Stdio, sync::Arc}; use tokio::{ io::{AsyncBufReadExt, BufReader}, process::{Child, Command}, - sync::oneshot, + sync::{oneshot, Mutex}, }; use tracing::{error, info}; -use crate::{concepts::Stream, daemon::filter::FilterManager}; +use crate::{ + concepts::{Filter, Stream}, + daemon::filter::FilterManager, +}; pub async fn stream_manager( stream: &'static Stream, child_tx: oneshot::Sender>, - mut filter_managers: Vec, + filter_managers: Arc>>, ) { info!("{}: start {:?}", stream.name(), stream.cmd()); let mut child = match Command::new(&stream.cmd()[0]) @@ -43,7 +46,9 @@ pub async fn stream_manager( Ok(Some(line)) => { futures::future::join_all( filter_managers - .iter_mut() + .lock() + .await + .values_mut() .map(|manager| manager.handle_line(&line)), ) .await; @@ -64,6 +69,8 @@ pub async fn stream_manager( } filter_managers - .iter_mut() + .lock() + .await + .values_mut() .for_each(|manager| manager.quit()); } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 961a40e..012b4d3 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -2,10 +2,10 @@ clippy::panic, clippy::todo, clippy::unimplemented, - clippy::unwrap_used + clippy::unwrap_used, + unsafe_code )] #![allow(clippy::upper_case_acronyms, clippy::mutable_key_type)] -#![forbid(unsafe_code)] pub mod client; pub mod concepts; From 607775b8e35db7ae28c918c9d8e745d7aa67036f Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 20 Oct 2024 12:00:00 +0200 Subject: [PATCH 068/435] Remove ActionFilter trait Was useful for the removed trait StateMap. Not useful anymore. --- rust/src/concepts/action.rs | 12 +----------- rust/src/concepts/filter.rs | 20 +++++++++----------- rust/src/concepts/messages.rs | 4 ++-- rust/src/concepts/mod.rs | 7 ------- rust/src/daemon/database/lowlevel.rs | 5 ++--- rust/src/daemon/database/mod.rs | 2 +- rust/src/daemon/filter.rs | 2 +- rust/src/daemon/socket.rs | 2 +- 8 files changed, 17 insertions(+), 37 deletions(-) diff --git a/rust/src/concepts/action.rs b/rust/src/concepts/action.rs index d4a707f..f5378ff 100644 --- a/rust/src/concepts/action.rs +++ b/rust/src/concepts/action.rs @@ -5,7 +5,7 @@ use chrono::TimeDelta; use serde::Deserialize; use tokio::process::Command; -use super::{ActionFilter, Match, Pattern}; +use super::{Match, Pattern}; use crate::utils::parse_duration; #[derive(Clone, Debug, Deserialize)] @@ -35,16 +35,6 @@ fn set_false() -> bool { false } -impl ActionFilter for Action { - fn patterns(&self) -> &BTreeSet> { - &self.patterns - } - - fn full_name(&self) -> (&str, &str, &str) { - (&self.stream_name, &self.filter_name, &self.name) - } -} - impl Action { pub fn name(&self) -> &str { &self.name diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index 5449de3..ece5734 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -13,7 +13,7 @@ use tracing::info; use super::{ messages::{Match, Time, MAT}, - Action, ActionFilter, LogEntry, Pattern, Patterns, + Action, LogEntry, Pattern, Patterns, }; use crate::utils::parse_duration; @@ -46,16 +46,6 @@ pub struct Filter { stream_name: String, } -impl ActionFilter for Filter { - fn full_name(&self) -> (&str, &str, &str) { - (self.stream_name.as_ref(), self.name.as_ref(), "") - } - - fn patterns(&self) -> &BTreeSet> { - &self.patterns - } -} - impl Filter { #[cfg(test)] pub fn from_name(stream_name: &str, filter_name: &str) -> Filter { @@ -70,6 +60,10 @@ impl Filter { &self.name } + pub fn stream_name(&self) -> &str { + &self.stream_name + } + pub fn retry(&self) -> Option { self.retry } @@ -82,6 +76,10 @@ impl Filter { &self.actions } + pub fn patterns(&self) -> &BTreeSet> { + &self.patterns + } + pub fn setup( &mut self, stream_name: &str, diff --git a/rust/src/concepts/messages.rs b/rust/src/concepts/messages.rs index 4642eb6..31aed6f 100644 --- a/rust/src/concepts/messages.rs +++ b/rust/src/concepts/messages.rs @@ -1,12 +1,12 @@ use chrono::{DateTime, Local, TimeDelta}; -use super::{Action, ActionFilter, Filter}; +use super::{Action, Filter}; pub type Time = DateTime; pub type Match = Vec; #[derive(Clone, Debug)] -pub struct MT { +pub struct MT { pub m: Match, pub o: &'static T, pub t: Time, diff --git a/rust/src/concepts/mod.rs b/rust/src/concepts/mod.rs index 6797084..3e25648 100644 --- a/rust/src/concepts/mod.rs +++ b/rust/src/concepts/mod.rs @@ -6,8 +6,6 @@ mod pattern; mod socket_messages; mod stream; -use std::{collections::BTreeSet, fmt::Display, sync::Arc}; - pub use action::Action; pub use config::{Config, Patterns}; pub use filter::Filter; @@ -15,8 +13,3 @@ pub use messages::*; pub use pattern::Pattern; pub use socket_messages::*; pub use stream::Stream; - -pub trait ActionFilter: Clone + Display + PartialEq + Eq + PartialOrd + Ord { - fn patterns(&self) -> &BTreeSet>; - fn full_name(&self) -> (&str, &str, &str); -} diff --git a/rust/src/daemon/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs index b831579..3a476bc 100644 --- a/rust/src/daemon/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -6,7 +6,7 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tracing::{debug, error, warn}; use crate::{ - concepts::{ActionFilter, Config, Filter, LogEntry, Match}, + concepts::{Config, Filter, LogEntry, Match}, utils::{bincode_options, BincodeOptions}, }; @@ -131,8 +131,7 @@ impl WriteDB { .filters() .into_iter() .map(|f| { - let names = f.full_name(); - (names.0.to_owned(), names.1.to_owned()) + (f.stream_name().to_owned(), f.name().to_owned()) }) .enumerate() .collect(); diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index fcf1bea..dcb8af4 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -11,7 +11,7 @@ use thiserror::Error; use tokio::sync::mpsc; use tracing::{debug, error, info, warn}; -use crate::concepts::{ActionFilter, Config, Filter, LogEntry, Match, Time, MFT}; +use crate::concepts::{Config, Filter, LogEntry, Match, Time, MFT}; mod lowlevel; mod tests; diff --git a/rust/src/daemon/filter.rs b/rust/src/daemon/filter.rs index da913db..0d39043 100644 --- a/rust/src/daemon/filter.rs +++ b/rust/src/daemon/filter.rs @@ -7,7 +7,7 @@ use chrono::Local; use regex::Regex; use tokio::sync::{mpsc, Semaphore}; -use crate::concepts::{ActionFilter, Filter, LogEntry, Match, Pattern, PatternStatus, Time, MFT}; +use crate::concepts::{Filter, LogEntry, Match, Pattern, PatternStatus, Time, MFT}; use super::{action::ActionManager, database::DatabaseManagerInput}; diff --git a/rust/src/daemon/socket.rs b/rust/src/daemon/socket.rs index 82931e5..37ed784 100644 --- a/rust/src/daemon/socket.rs +++ b/rust/src/daemon/socket.rs @@ -11,7 +11,7 @@ use tokio_util::{ use tracing::{error, warn}; use crate::{ - concepts::{ActionFilter, ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern}, + concepts::{ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern}, utils::bincode_options, }; From 51175f010d206ac23554e7a8fdc63df959ff76f6 Mon Sep 17 00:00:00 2001 From: ppom Date: Sun, 20 Oct 2024 12:00:00 +0200 Subject: [PATCH 069/435] WIP Reimplement flush Still same compiler issue --- rust/src/daemon/action.rs | 35 ++++++++++++++++++----- rust/src/daemon/filter.rs | 59 ++++++++++++++++++++++----------------- rust/src/daemon/mod.rs | 9 ++---- rust/src/daemon/socket.rs | 20 ++++--------- 4 files changed, 70 insertions(+), 53 deletions(-) diff --git a/rust/src/daemon/action.rs b/rust/src/daemon/action.rs index 7a94982..bd0c73f 100644 --- a/rust/src/daemon/action.rs +++ b/rust/src/daemon/action.rs @@ -8,7 +8,7 @@ use chrono::{Local, TimeDelta}; use tokio::sync::Semaphore; use tracing::{error, info}; -use crate::concepts::{Action, Match, Time}; +use crate::concepts::{Action, Match, Order, Time}; struct State { pending: BTreeMap>, @@ -94,12 +94,33 @@ impl ActionManager { } } - pub fn to_readable_vec(&self, match_: &Match) -> Option> { - self.state.lock().unwrap().pending.get(match_).map(|set| { - set.iter() - .map(|time| time.to_rfc3339().chars().take(19).collect()) - .collect() - }) + pub fn handle_order bool>( + &self, + order: Order, + is_match: F, + ) -> BTreeMap> { + let mut state = self.state.lock().unwrap(); + state + .pending + .clone() + .into_iter() + // match filtering + .filter(|(match_, _)| is_match(match_)) + .fold(BTreeMap::default(), |mut acc, (match_, times)| { + let times = times + .iter() + .map(|time| { + if let Order::Flush = order { + if state.remove(match_.clone(), *time) { + self.exec_now(match_.clone()); + } + } + time.to_rfc3339().chars().take(19).collect() + }) + .collect(); + acc.insert(match_.join(" "), times); + acc + }) } fn exec_now(&self, m: Match) { diff --git a/rust/src/daemon/filter.rs b/rust/src/daemon/filter.rs index 0d39043..41f4a7a 100644 --- a/rust/src/daemon/filter.rs +++ b/rust/src/daemon/filter.rs @@ -7,7 +7,7 @@ use chrono::Local; use regex::Regex; use tokio::sync::{mpsc, Semaphore}; -use crate::concepts::{Filter, LogEntry, Match, Pattern, PatternStatus, Time, MFT}; +use crate::concepts::{Filter, LogEntry, Match, Order, Pattern, PatternStatus, Time, MFT}; use super::{action::ActionManager, database::DatabaseManagerInput}; @@ -95,41 +95,50 @@ impl FilterManager { .for_each(|manager| manager.quit()); } - pub fn to_pattern_status_map( - &self, + pub fn handle_order( + &mut self, patterns: &BTreeMap, Regex>, + order: Order, ) -> BTreeMap { - self.matches + let is_match = |match_: &Match| { + match_ + .iter() + .zip(self.filter.patterns()) + .filter_map(|(a_match, pattern)| { + patterns.get(pattern.as_ref()).map(|regex| (a_match, regex)) + }) + .all(|(a_match, regex)| regex.is_match(a_match)) + }; + + let cs = self + .matches + .clone() .iter() - // TODO match filtering - .filter(|(match_, _)| { - match_ - .iter() - .zip(self.filter.patterns()) - .filter_map(|(a_match, pattern)| { - patterns.get(pattern.as_ref()).map(|regex| (a_match, regex)) - }) - .all(|(a_match, regex)| regex.is_match(a_match)) - }) + // match filtering + .filter(|(match_, _)| is_match(match_)) .map(|(match_, times)| { - let actions = - self.action_managers - .iter() - .fold(BTreeMap::default(), |mut acc, manager| { - if let Some(times) = manager.to_readable_vec(match_) { - acc.insert(manager.action().name().to_owned(), times); - } - acc - }); + if let Order::Flush = order { + self.remove_match(match_); + } ( match_.join(" "), PatternStatus { matches: times.len(), - actions, + ..Default::default() }, ) }) - .collect() + .collect(); + + self.action_managers.iter().fold(cs, |mut acc, manager| { + for (match_, times) in manager.handle_order(order, is_match) { + let pattern_status = acc.entry(match_).or_default(); + pattern_status + .actions + .insert(manager.action().to_string(), times); + } + acc + }) } fn add_match(&mut self, m: &Match, t: Time) { diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index 9ab48bd..05affa4 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -54,9 +54,6 @@ pub async fn daemon( let mut stream_process_child_handles = Vec::new(); let mut stream_task_handles = Vec::new(); - // let (socket2match_tx, socket2match_rx) = mpsc::channel(1); - // let (socket2exec_tx, socket2exec_rx) = mpsc::channel(1); - let (log_tx, log_rx) = mpsc::channel(234560); // Shutdown channel @@ -92,7 +89,7 @@ pub async fn daemon( drop(exec_limit); let database_manager_thread_handle = { - // The `task::spawn` is done in the function, after database rotation is finished + // The `thread::spawn` is done in the function, after database rotation is finished database_manager(config, log_rx, log2filter_tx) }; @@ -104,7 +101,7 @@ pub async fn daemon( } stream_filter_managers.s.insert(stream, Arc::new(Mutex::new(filter_managers))); } - let stream_filter_managers: Arc = Arc::new(stream_filter_managers); + let stream_filter_managers = Arc::new(stream_filter_managers); for (stream, filter_managers) in stream_filter_managers.s.iter() { let (child_tx, child_rx) = oneshot::channel(); @@ -130,7 +127,7 @@ pub async fn daemon( let socket_manager_task_handle = { let socket = socket.to_owned(); - let stream_filter_managers: Arc = stream_filter_managers.clone(); + let stream_filter_managers = stream_filter_managers.clone(); tokio::spawn(async move { socket_manager(config, socket, stream_filter_managers, shutdown_rx).await }) diff --git a/rust/src/daemon/socket.rs b/rust/src/daemon/socket.rs index 37ed784..f0973ef 100644 --- a/rust/src/daemon/socket.rs +++ b/rust/src/daemon/socket.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, fs, io, path::PathBuf, process::exit, sync::Arc use bincode::Options; use futures::{SinkExt, StreamExt}; use regex::Regex; -use tokio::{net::UnixListener, sync::{broadcast, oneshot}}; +use tokio::{net::UnixListener, sync::broadcast}; use tokio_util::{ bytes::Bytes, codec::{Framed, LengthDelimitedCodec}, @@ -11,19 +11,12 @@ use tokio_util::{ use tracing::{error, warn}; use crate::{ - concepts::{ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern}, + concepts::{ClientRequest, ClientStatus, Config, DaemonResponse, Pattern}, utils::bincode_options, }; use super::SharedState; -#[derive(Clone)] -pub struct FilterOptions { - pub stream_name: Option, - pub filter_name: Option, - pub patterns: BTreeMap, Regex>, -} - macro_rules! err_str { ($expression:expr) => { $expression.map_err(|err| err.to_string()) @@ -89,7 +82,6 @@ async fn answer_order( }) .collect::, Regex>, String>>()?; - // TODO directly call flush function here? let cs: ClientStatus = futures::stream::iter(shared_state.s.iter()) // stream filtering .filter(|(stream, _)| async { @@ -99,9 +91,9 @@ async fn answer_order( .is_some_and(|name| name == stream.name()) }) .fold(BTreeMap::new(), |mut acc, (stream, filter_manager)| async { - let filter_manager = filter_manager.lock().await; + let mut filter_manager = filter_manager.lock().await; let inner_map = filter_manager - .iter() + .iter_mut() // filter filtering .filter(|(filter, _)| { filter_name.is_none() @@ -118,7 +110,7 @@ async fn answer_order( .map(|(filter, manager)| { ( filter.name().to_owned(), - manager.to_pattern_status_map(&patterns), + manager.handle_order(&patterns, options.order), ) }) .collect(); @@ -142,8 +134,6 @@ macro_rules! or_next { }; } -pub type SocketOrder = (Order, FilterOptions, oneshot::Sender); - pub async fn socket_manager( config: &'static Config, socket: PathBuf, From cf74ebcda6086b29e769c5e1a9ce46e5dfd326dd Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 19 Oct 2024 12:00:00 +0200 Subject: [PATCH 070/435] de-async database tests --- rust/src/daemon/database/lowlevel.rs | 12 ++++++++---- rust/src/daemon/database/tests.rs | 16 ++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/rust/src/daemon/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs index 3a476bc..a02f2f5 100644 --- a/rust/src/daemon/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -1,4 +1,10 @@ -use std::{collections::BTreeMap, fmt::Debug, fs::File, io::{self, BufReader, BufWriter, Write}, process::exit}; +use std::{ + collections::BTreeMap, + fmt::Debug, + fs::File, + io::{self, BufReader, BufWriter, Write}, + process::exit, +}; use bincode::Options; use chrono::{DateTime, Local}; @@ -130,9 +136,7 @@ impl WriteDB { let database_header: DatabaseHeader = config .filters() .into_iter() - .map(|f| { - (f.stream_name().to_owned(), f.name().to_owned()) - }) + .map(|f| (f.stream_name().to_owned(), f.name().to_owned())) .enumerate() .collect(); diff --git a/rust/src/daemon/database/tests.rs b/rust/src/daemon/database/tests.rs index 0dc64b5..aacf959 100644 --- a/rust/src/daemon/database/tests.rs +++ b/rust/src/daemon/database/tests.rs @@ -9,8 +9,8 @@ use crate::{ tests::Fixture, }; -#[tokio::test] -async fn write_and_read_db() { +#[test] +fn write_and_read_db() { let config_file = Fixture::from_string( "config.jsonnet", " @@ -61,26 +61,26 @@ async fn write_and_read_db() { let db_path = Fixture::empty("matches.db"); - let mut write_db = WriteDB::create(db_path.to_str().unwrap(), config).await; + let mut write_db = WriteDB::create(db_path.to_str().unwrap(), config); - assert!(write_db.write(correct_log_entry.clone()).await.is_ok()); - assert!(write_db.write(incorrect_log_entry).await.is_err()); + assert!(write_db.write(correct_log_entry.clone()).is_ok()); + assert!(write_db.write(incorrect_log_entry).is_err()); drop(write_db); - let read_db = ReadDB::open(db_path.to_str().unwrap(), config).await; + let read_db = ReadDB::open(db_path.to_str().unwrap(), config); assert!(read_db.is_ok()); let read_db = read_db.unwrap(); assert!(read_db.is_some()); let mut read_db = read_db.unwrap(); - let read_entry = read_db.next().await; + let read_entry = read_db.next(); assert!(read_entry.is_some()); let read_entry = read_entry.unwrap(); assert!(read_entry.is_ok()); assert_eq!(read_entry.unwrap(), correct_log_entry); - let read_entry = read_db.next().await; + let read_entry = read_db.next(); assert!(read_entry.is_none()); } From d7203c792a108a36264410044cc08a95f7c5ee37 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 21 Oct 2024 12:00:00 +0200 Subject: [PATCH 071/435] remove useless unsafe code, format, simplify leak() line --- rust/src/daemon/mod.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index 05affa4..2e06164 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -34,18 +34,17 @@ struct SharedState { pub s: BTreeMap<&'static Stream, Arc>>>, } -#[allow(unsafe_code)] +// #[allow(unsafe_code)] // It's actually safe. It's a bug in rust that shadows the 'static reference, which ensures the // reference will always link to something // https://github.com/rust-lang/rust/issues/96865 -unsafe impl Send for SharedState {} +// unsafe impl Send for SharedState {} pub async fn daemon( config_path: PathBuf, socket: PathBuf, ) -> Result<(), Box> { - let config: &'static Config = - Config::from_file(&config_path).map(|config| Box::leak(Box::new(config)))?; + let config: &'static Config = Box::leak(Box::new(Config::from_file(&config_path)?)); if !config.start() { return Err("a start command failed, exiting.".into()); @@ -99,7 +98,9 @@ pub async fn daemon( for (filter, filter_manager_handler) in filter_manager_handlers { filter_managers.insert(filter, filter_manager_handler.await.unwrap()); } - stream_filter_managers.s.insert(stream, Arc::new(Mutex::new(filter_managers))); + stream_filter_managers + .s + .insert(stream, Arc::new(Mutex::new(filter_managers))); } let stream_filter_managers = Arc::new(stream_filter_managers); From aca19fea8f2099a8c8e8a0105970554c6542843f Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 21 Oct 2024 12:00:00 +0200 Subject: [PATCH 072/435] fix all async & clippy issues - use hasmap instead of btreemap https://github.com/rust-lang/rust/issues/64552 - use async iterators as few as possible - move first rotate_db into its own thread to be out of the runtime, so that we can use blocking_send - send flush to database --- rust/src/concepts/filter.rs | 7 +++++ rust/src/concepts/stream.rs | 7 ++++- rust/src/daemon/action.rs | 9 ++++-- rust/src/daemon/database/mod.rs | 22 +++++++------- rust/src/daemon/filter.rs | 34 ++++++++++++++++------ rust/src/daemon/mod.rs | 16 +++++------ rust/src/daemon/socket.rs | 51 +++++++++++++++++++-------------- rust/src/daemon/stream.rs | 4 +-- rust/tests/simple.rs | 4 ++- 9 files changed, 98 insertions(+), 56 deletions(-) diff --git a/rust/src/concepts/filter.rs b/rust/src/concepts/filter.rs index ece5734..d895b53 100644 --- a/rust/src/concepts/filter.rs +++ b/rust/src/concepts/filter.rs @@ -2,6 +2,7 @@ use std::{ cmp::Ordering, collections::{BTreeMap, BTreeSet}, fmt::Display, + hash::Hash, sync::Arc, }; @@ -253,6 +254,12 @@ impl PartialOrd for Filter { Some(self.cmp(other)) } } +impl Hash for Filter { + fn hash(&self, state: &mut H) { + self.stream_name.hash(state); + self.name.hash(state); + } +} #[allow(clippy::unwrap_used)] #[cfg(test)] diff --git a/rust/src/concepts/stream.rs b/rust/src/concepts/stream.rs index ddef48d..377968d 100644 --- a/rust/src/concepts/stream.rs +++ b/rust/src/concepts/stream.rs @@ -1,4 +1,4 @@ -use std::{cmp::Ordering, collections::BTreeMap}; +use std::{cmp::Ordering, collections::BTreeMap, hash::Hash}; use serde::Deserialize; @@ -81,6 +81,11 @@ impl PartialOrd for Stream { Some(self.cmp(other)) } } +impl Hash for Stream { + fn hash(&self, state: &mut H) { + self.name.hash(state); + } +} #[cfg(test)] pub mod tests { diff --git a/rust/src/daemon/action.rs b/rust/src/daemon/action.rs index bd0c73f..44742e8 100644 --- a/rust/src/daemon/action.rs +++ b/rust/src/daemon/action.rs @@ -35,6 +35,7 @@ impl State { .first_key_value() .is_some_and(|(k, _)| *k + after < now) { + #[allow(clippy::unwrap_used)] // we just checked in the condition that first is_some let (_, m) = self.ordered_times.pop_first().unwrap(); self.pending.remove(&m); } @@ -78,6 +79,7 @@ impl ActionManager { self.exec_now(m); } else { { + #[allow(clippy::unwrap_used)] // propagating panics is ok let mut state = self.state.lock().unwrap(); state.clear_past_times(t, self.action.after_duration()); state.add_match(&m, exec_t); @@ -86,6 +88,7 @@ impl ActionManager { tokio::spawn(async move { let dur = (exec_t - now).to_std().expect("Duration is bigger than what's supported. Did you put an enormous after duration?"); tokio::time::sleep(dur).await; + #[allow(clippy::unwrap_used)] // propagating panics is ok let mut state = this.state.lock().unwrap(); if state.remove(m.clone(), t) { this.exec_now(m); @@ -98,7 +101,8 @@ impl ActionManager { &self, order: Order, is_match: F, - ) -> BTreeMap> { + ) -> BTreeMap, Vec> { + #[allow(clippy::unwrap_used)] // propagating panics is ok let mut state = self.state.lock().unwrap(); state .pending @@ -118,7 +122,7 @@ impl ActionManager { time.to_rfc3339().chars().take(19).collect() }) .collect(); - acc.insert(match_.join(" "), times); + acc.insert(match_, times); acc }) } @@ -152,6 +156,7 @@ impl ActionManager { pub fn quit(&mut self) { if self.action.on_exit() { + #[allow(clippy::unwrap_used)] // propagating panics is ok let mut state = self.state.lock().unwrap(); for (m, times) in &state.pending { for _ in times { diff --git a/rust/src/daemon/database/mod.rs b/rust/src/daemon/database/mod.rs index dcb8af4..9d2919c 100644 --- a/rust/src/daemon/database/mod.rs +++ b/rust/src/daemon/database/mod.rs @@ -64,17 +64,17 @@ macro_rules! flush_or_die { pub fn database_manager( config: &'static Config, mut log_rx: mpsc::Receiver, - matches_tx: BTreeMap<&Filter, mpsc::Sender>, + matches_tx: BTreeMap<&'static Filter, mpsc::Sender>, ) -> thread::JoinHandle<()> { - let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)) { - Ok(dbs) => dbs, - Err(err) => { - error!("while rotating databases on start: {}", err); - exit(1); - } - }; - thread::spawn(move || { + let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)) { + Ok(dbs) => dbs, + Err(err) => { + error!("while rotating databases on start: {}", err); + exit(1); + } + }; + let mut cpt = 0; while let Some(order) = log_rx.blocking_recv() { match order { @@ -177,7 +177,7 @@ fn _rotate_db( // Read flushes let mut flushes: BTreeMap<&'static Filter, BTreeMap> = BTreeMap::new(); - while let Some(flush_entry) = flush_read_db.next() { + for flush_entry in flush_read_db { match flush_entry { Ok(entry) => { let matches_map = flushes.entry(entry.f).or_default(); @@ -193,7 +193,7 @@ fn _rotate_db( let now = Local::now(); // Read matches - while let Some(log_entry) = log_read_db.next() { + for log_entry in log_read_db { match log_entry { Ok(mut entry) => { // Check if number of patterns is in sync diff --git a/rust/src/daemon/filter.rs b/rust/src/daemon/filter.rs index 41f4a7a..136f444 100644 --- a/rust/src/daemon/filter.rs +++ b/rust/src/daemon/filter.rs @@ -95,7 +95,7 @@ impl FilterManager { .for_each(|manager| manager.quit()); } - pub fn handle_order( + pub async fn handle_order( &mut self, patterns: &BTreeMap, Regex>, order: Order, @@ -110,18 +110,17 @@ impl FilterManager { .all(|(a_match, regex)| regex.is_match(a_match)) }; - let cs = self - .matches - .clone() - .iter() + let matches = self.matches.clone(); + let cs: BTreeMap<_, _> = matches + .into_iter() // match filtering .filter(|(match_, _)| is_match(match_)) .map(|(match_, times)| { if let Order::Flush = order { - self.remove_match(match_); + self.remove_match(&match_); } ( - match_.join(" "), + match_, PatternStatus { matches: times.len(), ..Default::default() @@ -130,7 +129,7 @@ impl FilterManager { }) .collect(); - self.action_managers.iter().fold(cs, |mut acc, manager| { + let cs = self.action_managers.iter().fold(cs, |mut acc, manager| { for (match_, times) in manager.handle_order(order, is_match) { let pattern_status = acc.entry(match_).or_default(); pattern_status @@ -138,7 +137,23 @@ impl FilterManager { .insert(manager.action().to_string(), times); } acc - }) + }); + + let now = Local::now(); + for match_ in cs.keys() { + #[allow(clippy::unwrap_used)] // propagating panics is ok + self.log_tx + .send(DatabaseManagerInput::Flush(LogEntry { + exec: false, + m: match_.to_vec(), + f: self.filter, + t: now, + })) + .await + .unwrap() + } + + cs.into_iter().map(|(k, v)| (k.join(" "), v)).collect() } fn add_match(&mut self, m: &Match, t: Time) { @@ -162,6 +177,7 @@ impl FilterManager { .first_key_value() .is_some_and(|(k, _)| *k + retry_duration < now) { + #[allow(clippy::unwrap_used)] // we just checked in the condition that first is_some let (_, m) = self.ordered_times.pop_first().unwrap(); self.matches.remove(&m); } diff --git a/rust/src/daemon/mod.rs b/rust/src/daemon/mod.rs index 2e06164..7d6adbc 100644 --- a/rust/src/daemon/mod.rs +++ b/rust/src/daemon/mod.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashMap}, error::Error, path::PathBuf, sync::{ @@ -8,7 +8,6 @@ use std::{ }, }; -use socket::socket_manager; use tokio::{ process::Child, select, @@ -20,6 +19,7 @@ use tracing::info; use crate::concepts::{Config, Filter, Stream}; use database::database_manager; use filter::FilterManager; +use socket::socket_manager; use stream::stream_manager; mod database; @@ -31,7 +31,7 @@ mod stream; // type SharedState = BTreeMap<&'static Stream, Arc>>>; struct SharedState { - pub s: BTreeMap<&'static Stream, Arc>>>, + pub s: HashMap<&'static Stream, Arc>>>, } // #[allow(unsafe_code)] @@ -66,10 +66,10 @@ pub async fn daemon( }; // Filter managers - let mut stream_filter_managers_handlers = BTreeMap::new(); + let mut stream_filter_managers_handlers = HashMap::new(); let mut log2filter_tx = BTreeMap::new(); for stream in config.streams().values() { - let mut filter_managers_handlers = BTreeMap::new(); + let mut filter_managers_handlers = HashMap::new(); for filter in stream.filters().values() { let manager = FilterManager::new( filter, @@ -92,10 +92,11 @@ pub async fn daemon( database_manager(config, log_rx, log2filter_tx) }; - let mut stream_filter_managers = SharedState { s: BTreeMap::new() }; + let mut stream_filter_managers = SharedState { s: HashMap::new() }; for (stream, filter_manager_handlers) in stream_filter_managers_handlers { - let mut filter_managers = BTreeMap::new(); + let mut filter_managers = HashMap::new(); for (filter, filter_manager_handler) in filter_manager_handlers { + #[allow(clippy::unwrap_used)] // propagating panics is ok filter_managers.insert(filter, filter_manager_handler.await.unwrap()); } stream_filter_managers @@ -128,7 +129,6 @@ pub async fn daemon( let socket_manager_task_handle = { let socket = socket.to_owned(); - let stream_filter_managers = stream_filter_managers.clone(); tokio::spawn(async move { socket_manager(config, socket, stream_filter_managers, shutdown_rx).await }) diff --git a/rust/src/daemon/socket.rs b/rust/src/daemon/socket.rs index f0973ef..269b38e 100644 --- a/rust/src/daemon/socket.rs +++ b/rust/src/daemon/socket.rs @@ -50,7 +50,7 @@ fn open_socket(path: PathBuf) -> Result { async fn answer_order( config: &'static Config, - shared_state: &SharedState, + shared_state: &Arc, options: ClientRequest, ) -> Result { // Compute options @@ -82,17 +82,22 @@ async fn answer_order( }) .collect::, Regex>, String>>()?; - let cs: ClientStatus = futures::stream::iter(shared_state.s.iter()) - // stream filtering - .filter(|(stream, _)| async { - stream_name.is_none() - || stream_name - .clone() - .is_some_and(|name| name == stream.name()) - }) - .fold(BTreeMap::new(), |mut acc, (stream, filter_manager)| async { - let mut filter_manager = filter_manager.lock().await; - let inner_map = filter_manager + let cs: ClientStatus = futures::stream::iter( + shared_state + .s + .iter() + // stream filtering + .filter(|(stream, _)| { + stream_name.is_none() + || stream_name + .clone() + .is_some_and(|name| name == stream.name()) + }), + ) + .fold(BTreeMap::new(), |mut acc, (stream, filter_manager)| async { + let mut filter_manager = filter_manager.lock().await; + let inner_map = futures::stream::iter( + filter_manager .iter_mut() // filter filtering .filter(|(filter, _)| { @@ -106,18 +111,20 @@ async fn answer_order( patterns .iter() .all(|(pattern, _)| filter.patterns().get(pattern).is_some()) - }) - .map(|(filter, manager)| { - ( - filter.name().to_owned(), - manager.handle_order(&patterns, options.order), - ) - }) - .collect(); - acc.insert(stream.name().to_owned(), inner_map); - acc + }), + ) + .then(|(filter, manager)| async { + ( + filter.name().to_owned(), + manager.handle_order(&patterns, options.order).await, + ) }) + .collect() .await; + acc.insert(stream.name().to_owned(), inner_map); + acc + }) + .await; Ok(cs) } diff --git a/rust/src/daemon/stream.rs b/rust/src/daemon/stream.rs index b0f31e9..e86ffaa 100644 --- a/rust/src/daemon/stream.rs +++ b/rust/src/daemon/stream.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, process::Stdio, sync::Arc}; +use std::{collections::HashMap, process::Stdio, sync::Arc}; use tokio::{ io::{AsyncBufReadExt, BufReader}, @@ -15,7 +15,7 @@ use crate::{ pub async fn stream_manager( stream: &'static Stream, child_tx: oneshot::Sender>, - filter_managers: Arc>>, + filter_managers: Arc>>, ) { info!("{}: start {:?}", stream.name(), stream.cmd()); let mut child = match Command::new(&stream.cmd()[0]) diff --git a/rust/tests/simple.rs b/rust/tests/simple.rs index 4834b62..7fb2370 100644 --- a/rust/tests/simple.rs +++ b/rust/tests/simple.rs @@ -146,7 +146,9 @@ async fn simple() { file_with_contents(out_path, ""); - assert!(daemon(config_path.into(), socket_path.into()).await.is_ok()); + assert!(daemon(config_path.into(), socket_path.into()) + .await + .is_err()); // 36 from DB // 12 from DB From c42487db5c643bd96297a96f9410664bc363490d Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 21 Oct 2024 12:00:00 +0200 Subject: [PATCH 073/435] Update dependencies --- rust/Cargo.lock | 269 ++++++++++++++++++++++++------------------------ 1 file changed, 132 insertions(+), 137 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4018f90..e74edc2 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -4,19 +4,13 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "adler2" version = "2.0.0" @@ -108,9 +102,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" [[package]] name = "async-stream" @@ -131,7 +125,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] [[package]] @@ -142,7 +136,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] [[package]] @@ -153,9 +147,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "axum" @@ -206,17 +200,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", - "miniz_oxide 0.7.4", + "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -272,9 +266,12 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.6" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" @@ -298,9 +295,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.11" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -308,9 +305,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.11" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -320,23 +317,23 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.11" +version = "4.5.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6ae69fbb0833c6fcd5a8d4b8609f108c7ad95fc11e248d853ff2c42a90df26a" +checksum = "9646e2e245bf62f45d39a0f3f36f1171ad1ea0d6967fd114bca72cb02a8fcdfb" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.11" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] [[package]] @@ -392,9 +389,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "crc32fast" @@ -455,7 +452,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", - "miniz_oxide 0.8.0", + "miniz_oxide", ] [[package]] @@ -466,9 +463,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -481,9 +478,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -491,15 +488,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -508,38 +505,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -566,9 +563,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "h2" @@ -582,7 +579,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.2.6", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -597,9 +594,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" [[package]] name = "hdrhistogram" @@ -680,9 +677,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -733,9 +730,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -766,12 +763,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", ] [[package]] @@ -880,9 +877,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -895,9 +892,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "linux-raw-sys" @@ -960,15 +957,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.8.0" @@ -1040,9 +1028,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "overload" @@ -1075,9 +1063,9 @@ dependencies = [ [[package]] name = "pathdiff" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +checksum = "d61c5ce1153ab5b689d0c074c4e7fc613e942dfb7dd9eea5ab202d2ad91fe361" [[package]] name = "peg" @@ -1129,7 +1117,7 @@ checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] [[package]] @@ -1155,9 +1143,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9" dependencies = [ "unicode-ident", ] @@ -1182,7 +1170,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] [[package]] @@ -1196,9 +1184,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -1269,14 +1257,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -1290,13 +1278,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -1307,9 +1295,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rustc-demangle" @@ -1325,9 +1313,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -1338,9 +1326,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" @@ -1356,29 +1344,29 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] [[package]] name = "serde_json" -version = "1.0.121" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -1392,7 +1380,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -1408,6 +1396,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -1461,9 +1455,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021" dependencies = [ "proc-macro2", "quote", @@ -1496,9 +1490,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", @@ -1509,22 +1503,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] [[package]] @@ -1573,7 +1567,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] [[package]] @@ -1695,7 +1689,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] [[package]] @@ -1751,21 +1745,21 @@ checksum = "ccb97dac3243214f8d8507998906ca3e2e0b900bf9bf4870477f125b82e68f6e" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unsafe-libyaml" @@ -1802,34 +1796,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1837,22 +1832,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "winapi" @@ -1994,5 +1989,5 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.82", ] From 58cf68ba5856bdd3536f3f5297ea96ddf9adb4fa Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 21 Oct 2024 12:00:00 +0200 Subject: [PATCH 074/435] Change database signature - raw signature, not prefixed by length (seems more human-friendly to me) - better error message --- rust/src/daemon/database/lowlevel.rs | 34 +++++++++++++++++----------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/rust/src/daemon/database/lowlevel.rs b/rust/src/daemon/database/lowlevel.rs index a02f2f5..a16a1b1 100644 --- a/rust/src/daemon/database/lowlevel.rs +++ b/rust/src/daemon/database/lowlevel.rs @@ -2,7 +2,7 @@ use std::{ collections::BTreeMap, fmt::Debug, fs::File, - io::{self, BufReader, BufWriter, Write}, + io::{self, BufReader, BufWriter, Read, Write}, process::exit, }; @@ -21,7 +21,6 @@ use super::DBError; // OPTIM Add a timestamp prefix to the header, to permit having // shorter timestamps? // It may permit to win 1-4 bytes per entry, don't know if it's worth it -// FIXME put signature in the header? type DatabaseHeader = BTreeMap; type ReadHeader = BTreeMap; type WriteHeader = BTreeMap<&'static Filter, usize>; @@ -58,16 +57,24 @@ impl ReadDB { bin: bincode_options(), }; - match ret.read::() { - Ok(signature) => { - if DB_SIGNATURE == signature { - Ok(()) - } else { - Err(DBError::Error("database is not a reaction database".into())) - } - } - Err(err) => Err(DBError::Error(format!("reading database signature: {err}"))), - }?; + // Signature checking + let mut signature = [0u8; 15]; + ret.f + .read_exact(&mut signature) + .map_err(|err| DBError::Error(format!("reading database signature: {err}")))?; + if DB_SIGNATURE.as_bytes()[0..13] != signature[0..13] { + return Err(DBError::Error(format!( + "{path} is not a reaction database, or it is a reaction-v1.x database. +You can migrate your old database to a new one by following documented steps at https://reaction.ppom.me/migrate-to-v2 +You can also choose to delete the local {} and {} if you don't care about your old matches.", super::LOG_DB_NAME, super::FLUSH_DB_NAME + ))); + } + if DB_SIGNATURE.as_bytes()[13..15] != signature[13..15] { + return Err(DBError::Error(format!( + "{path} seem to be the database of a newer version of reaction. +Are you sure you're running the last version of reaction?" + ))); + } let db_header = ret .read::() @@ -128,7 +135,8 @@ impl WriteDB { bin: bincode_options(), }; - if let Err(err) = ret._write(DB_SIGNATURE) { + // Signature writing + if let Err(err) = ret.f.write_all(DB_SIGNATURE.as_bytes()) { error!("Failed to write to DB: {}", err); exit(1); } From b8f037352ce0b59a19d8784810f78b3c91382f5e Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 19 Oct 2024 12:00:00 +0200 Subject: [PATCH 075/435] Add import & export scripts --- .gitignore | 4 + export-go-db/export-db.go | 126 ++++++++++++++++++++++++++++++++ import-rust-db/Cargo.lock | 106 +++++++++++++++++++++++++++ import-rust-db/Cargo.toml | 9 +++ import-rust-db/src/main.rs | 145 +++++++++++++++++++++++++++++++++++++ 5 files changed, 390 insertions(+) create mode 100644 export-go-db/export-db.go create mode 100644 import-rust-db/Cargo.lock create mode 100644 import-rust-db/Cargo.toml create mode 100644 import-rust-db/src/main.rs diff --git a/.gitignore b/.gitignore index 3b23bec..10a2ee1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ /ip46tables /nft46 reaction*.db +reaction*.db.old +reaction*.export.json /reaction*.sock /result /wiki @@ -11,3 +13,5 @@ reaction*.db *.qcow2 debian-packaging/* *.swp +export-go-db/export-db +import-rust-db/target/ diff --git a/export-go-db/export-db.go b/export-go-db/export-db.go new file mode 100644 index 0000000..772a853 --- /dev/null +++ b/export-go-db/export-db.go @@ -0,0 +1,126 @@ +package main + +import ( + "bufio" + "encoding/gob" + "encoding/json" + "io" + "os" + "strings" + "time" +) + +func quit(msg string, err error) { + if err == nil { + print(msg) + } else { + print(msg, err) + } + os.Exit(1) +} + +type SF struct{ S, F string } + +type LogEntry struct { + T time.Time + S int64 + // This is a "\x00" Joined string + // which contains all matches on a line. + Pattern string + Stream, Filter string + SF int + Exec bool +} + +type JsonEntry struct { + Time int64 `json:"time"` // unix epoch + Stream string `json:"stream"` + Filter string `json:"filter"` + Match []string `json:"match"` + Exec bool `json:"exec"` +} + +func export(oldpath, newpath string) { + // Read DB + file, err := os.Open(oldpath) + if err != nil { + quit("could not open db: ", err) + } + dec := gob.NewDecoder(file) + + // Write export + fileNew, err := os.Create(newpath) + if err != nil { + quit("could not create export: ", err) + } + enc := json.NewEncoder(fileNew) + + malformedEntries := 0 + discardedEntries := 0 + + readSF2int := make(map[int]SF) + for { + var entry LogEntry + + err := dec.Decode(&entry) + if err != nil { + if err == io.EOF { + break + } + malformedEntries++ + continue + } + + if entry.Stream == "" && entry.Filter == "" { + sf, ok := readSF2int[entry.SF] + if !ok { + discardedEntries++ + continue + } + entry.Stream = sf.S + entry.Filter = sf.F + } + + if entry.SF != 0 { + readSF2int[entry.SF] = SF{entry.Stream, entry.Filter} + } + + if entry.T.IsZero() { + entry.T = time.Unix(entry.S, 0) + } + + jsonEntry := JsonEntry{ + entry.T.Unix(), + entry.Stream, + entry.Filter, + strings.Split(entry.Pattern, "\x00"), + entry.Exec, + } + enc.Encode(jsonEntry) + } + + if discardedEntries > 0 { + println(discardedEntries, "discarded entries") + } + + if malformedEntries > 0 { + println(malformedEntries, "malformed entries") + } +} + +func main() { + println("This export script must run in reaction's runtime directory.") + println("This usually is /var/lib/reaction.") + println("It will export the go / reaction-v1.x database files as JSON.") + println("Do you want to proceed? (y/n)") + reader := bufio.NewReader(os.Stdin) + line, err := reader.ReadString('\n') + if err != nil { + quit("fatal: could not read user input.", nil) + } else if line != "y\n" { + quit("user did not type `y`, quitting.", nil) + } + + export("./reaction-matches.db", "./reaction-matches.export.json") + export("./reaction-flushes.db", "./reaction-flushes.export.json") +} diff --git a/import-rust-db/Cargo.lock b/import-rust-db/Cargo.lock new file mode 100644 index 0000000..db7cd6e --- /dev/null +++ b/import-rust-db/Cargo.lock @@ -0,0 +1,106 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "import-rust-db" +version = "0.1.0" +dependencies = [ + "bincode", + "serde", + "serde_json", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "proc-macro2" +version = "1.0.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.132" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" diff --git a/import-rust-db/Cargo.toml b/import-rust-db/Cargo.toml new file mode 100644 index 0000000..62ec6df --- /dev/null +++ b/import-rust-db/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "import-rust-db" +version = "0.1.0" +edition = "2021" + +[dependencies] +bincode = "1.3.3" +serde = { version = "1.0.210", features = ["derive"]} +serde_json = "1.0.132" diff --git a/import-rust-db/src/main.rs b/import-rust-db/src/main.rs new file mode 100644 index 0000000..07f6a64 --- /dev/null +++ b/import-rust-db/src/main.rs @@ -0,0 +1,145 @@ +use std::{ + collections::BTreeMap, + error::Error, + fs::File, + io::{self, BufReader, BufWriter, Write}, + process::exit, +}; + +use bincode::Options; +use serde::{Deserialize, Serialize}; +use serde_json::Deserializer; +// use serde_json::deserialize_from; + +const DB: &str = "./reaction-matches"; +const FLUSH: &str = "./reaction-flushes"; + +const NORMAL: &str = ".db"; +const OLD: &str = ".db.old"; +const EXPORT: &str = ".export.json"; + +type E = Box; + +fn main() { + if let Err(err) = lil_main() { + println!("fatal: {err}"); + exit(1); + } +} + +fn lil_main() -> Result<(), E> { + println!("You're about to reimport the previously exported database."); + println!("This will move the old database files as .old"); + println!("When the process completes, you'll be able to run reaction-v2."); + println!( + "If reaction-v2 runs as you wish, you'll be free to delete the .old & .export.json files" + ); + println!("Do you want to continue? (y/n)"); + let mut buffer = String::new(); + io::stdin() + .read_line(&mut buffer) + .map_err(|err| format!("could not read user input: {err}"))?; + if buffer != "y\n" { + return Err("user did not type `y`, exiting.".into()); + } + + std::fs::rename(format!("{DB}{NORMAL}"), format!("{DB}{OLD}"))?; + std::fs::rename(format!("{FLUSH}{NORMAL}"), format!("{FLUSH}{OLD}"))?; + + import(format!("{DB}{EXPORT}"), format!("{DB}{NORMAL}"))?; + import(format!("{FLUSH}{EXPORT}"), format!("{FLUSH}{NORMAL}"))?; + Ok(()) +} + +fn import(json_path: String, write_path: String) -> Result<(), E> { + let json_file = BufReader::new(File::open(json_path.clone())?); + let mut write_file = BufWriter::new(File::create(write_path)?); + let bin = bincode_options(); + + // Signature writing + if let Err(err) = write_file.write_all(DB_SIGNATURE.as_bytes()) { + return Err(format!("Failed to write to DB: {}", err).into()); + } + + let header = collect_stream_filters(json_path)?; + let database_header: DatabaseHeader = + header.iter().map(|(k, v)| (v.clone(), k.clone())).collect(); + + bin.serialize_into(&mut write_file, &database_header)?; + + let deserializer = Deserializer::from_reader(json_file); + + for json_entry in deserializer.into_iter::() { + bin.serialize_into( + &mut write_file, + &ComputedLogEntry::from(json_entry?, &header)?, + )?; + } + + write_file.flush()?; + Ok(()) +} + +fn collect_stream_filters(json_path: String) -> Result { + let mut header = BTreeMap::new(); + let mut count = 0; + let json_file = BufReader::new(File::open(json_path)?); + let deserializer = Deserializer::from_reader(json_file); + + for json_entry in deserializer.into_iter::() { + let json_entry = json_entry?; + let tuple = (json_entry.stream, json_entry.filter); + if header.get(&tuple).is_none() { + header.insert(tuple, count); + count += 1; + } + } + Ok(header) +} + +#[derive(Debug, Deserialize)] +struct JsonEntry { + time: i64, + stream: String, + filter: String, + #[serde(rename = "match")] + match_: Vec, + exec: bool, +} + +// Pasted from main code + +const DB_SIGNATURE: &str = "reaction-db-v01"; + +pub type BincodeOptions = bincode::config::WithOtherIntEncoding< + bincode::config::DefaultOptions, + bincode::config::VarintEncoding, +>; +pub fn bincode_options() -> BincodeOptions { + bincode::DefaultOptions::new().with_varint_encoding() +} + +type DatabaseHeader = BTreeMap; +type WriteHeader = BTreeMap<(String, String), usize>; + +#[derive(Debug, Serialize)] +struct ComputedLogEntry { + pub m: Vec, + pub f: usize, + pub t: i64, + pub exec: bool, +} + +impl ComputedLogEntry { + fn from(value: JsonEntry, header: &WriteHeader) -> Result { + match header.get(&(value.stream.clone(), value.filter.clone())) { + Some(f) => Ok(ComputedLogEntry { + m: value.match_, + f: *f, + t: value.time, + exec: value.exec, + }), + None => Err(format!("invalid filter: {value:?}").into()), + } + } +} From fea9035f3279aa2828a69b5e1c0d1ac2c958e9c9 Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 19 Oct 2024 12:00:00 +0200 Subject: [PATCH 076/435] Move old go codebase to go.old And add lil Readme --- default.nix | 18 ------------------ go.old/README.md | 4 ++++ {app => go.old/app}/client.go | 0 {app => go.old/app}/daemon.go | 0 {app => go.old/app}/example.yml | 0 {app => go.old/app}/main.go | 0 {app => go.old/app}/persist.go | 0 {app => go.old/app}/pipe.go | 0 {app => go.old/app}/startup.go | 0 {app => go.old/app}/types.go | 0 go.mod => go.old/go.mod | 0 go.sum => go.old/go.sum | 0 {logger => go.old/logger}/log.go | 0 reaction.go => go.old/reaction.go | 0 14 files changed, 4 insertions(+), 18 deletions(-) delete mode 100644 default.nix create mode 100644 go.old/README.md rename {app => go.old/app}/client.go (100%) rename {app => go.old/app}/daemon.go (100%) rename {app => go.old/app}/example.yml (100%) rename {app => go.old/app}/main.go (100%) rename {app => go.old/app}/persist.go (100%) rename {app => go.old/app}/pipe.go (100%) rename {app => go.old/app}/startup.go (100%) rename {app => go.old/app}/types.go (100%) rename go.mod => go.old/go.mod (100%) rename go.sum => go.old/go.sum (100%) rename {logger => go.old/logger}/log.go (100%) rename reaction.go => go.old/reaction.go (100%) diff --git a/default.nix b/default.nix deleted file mode 100644 index b40c8f7..0000000 --- a/default.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ buildGoModule, fetchFromGitLab }: -let - pname = "reaction"; - version = "v0.1"; -in buildGoModule { - inherit pname version; - - src = ./.; - # src = fetchFromGitLab { - # domain = "framagit.org"; - # owner = "ppom"; - # repo = pname; - # rev = version; - # sha256 = "sha256-45ytTNZIbTIUOPBgAdD7o9hyWlJo//izUhGe53PcwNA="; - # }; - - vendorHash = "sha256-g+yaVIx4jxpAQ/+WrGKxhVeliYx7nLQe/zsGpxV4Fn4="; -} diff --git a/go.old/README.md b/go.old/README.md new file mode 100644 index 0000000..bc27779 --- /dev/null +++ b/go.old/README.md @@ -0,0 +1,4 @@ +This is the old Go codebase of reaction, ie. all 0.x and 1.x versions. +This codebase most probably won't be updated. + +Development now continues in Rust for reaction 2.x. diff --git a/app/client.go b/go.old/app/client.go similarity index 100% rename from app/client.go rename to go.old/app/client.go diff --git a/app/daemon.go b/go.old/app/daemon.go similarity index 100% rename from app/daemon.go rename to go.old/app/daemon.go diff --git a/app/example.yml b/go.old/app/example.yml similarity index 100% rename from app/example.yml rename to go.old/app/example.yml diff --git a/app/main.go b/go.old/app/main.go similarity index 100% rename from app/main.go rename to go.old/app/main.go diff --git a/app/persist.go b/go.old/app/persist.go similarity index 100% rename from app/persist.go rename to go.old/app/persist.go diff --git a/app/pipe.go b/go.old/app/pipe.go similarity index 100% rename from app/pipe.go rename to go.old/app/pipe.go diff --git a/app/startup.go b/go.old/app/startup.go similarity index 100% rename from app/startup.go rename to go.old/app/startup.go diff --git a/app/types.go b/go.old/app/types.go similarity index 100% rename from app/types.go rename to go.old/app/types.go diff --git a/go.mod b/go.old/go.mod similarity index 100% rename from go.mod rename to go.old/go.mod diff --git a/go.sum b/go.old/go.sum similarity index 100% rename from go.sum rename to go.old/go.sum diff --git a/logger/log.go b/go.old/logger/log.go similarity index 100% rename from logger/log.go rename to go.old/logger/log.go diff --git a/reaction.go b/go.old/reaction.go similarity index 100% rename from reaction.go rename to go.old/reaction.go From 3dd97523fda3f9c69258b45d2d3c1881e7557ed2 Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 19 Oct 2024 12:00:00 +0200 Subject: [PATCH 077/435] Move new rust codebase to root dir --- .gitignore | 3 +- rust/Cargo.lock => Cargo.lock | 0 rust/Cargo.toml => Cargo.toml | 0 config/heavy-load.yml | 62 ++++----- config/test.jsonnet | 48 +++---- rust/.gitignore | 1 - rust/datasize.jsonnet | 30 ----- rust/example.json | 122 ------------------ rust/heavy-load.yml | 72 ----------- rust/test.jsonnet | 47 ------- {rust/src => src}/client/mod.rs | 0 {rust/src => src}/concepts/action.rs | 0 {rust/src => src}/concepts/config.rs | 0 {rust/src => src}/concepts/filter.rs | 0 {rust/src => src}/concepts/messages.rs | 0 {rust/src => src}/concepts/mod.rs | 0 {rust/src => src}/concepts/pattern.rs | 0 {rust/src => src}/concepts/socket_messages.rs | 0 {rust/src => src}/concepts/stream.rs | 0 {rust/src => src}/daemon/action.rs | 0 {rust/src => src}/daemon/database/lowlevel.rs | 0 {rust/src => src}/daemon/database/mod.rs | 0 {rust/src => src}/daemon/database/tests.rs | 0 {rust/src => src}/daemon/filter.rs | 0 {rust/src => src}/daemon/mod.rs | 0 {rust/src => src}/daemon/socket.rs | 0 {rust/src => src}/daemon/stream.rs | 0 {rust/src => src}/lib.rs | 0 {rust/src => src}/main.rs | 0 {rust/src => src}/tests.rs | 0 {rust/src => src}/utils/cli.rs | 0 {rust/src => src}/utils/mod.rs | 0 {rust/src => src}/utils/parse_duration.rs | 0 {rust/tests => tests}/simple.rs | 0 34 files changed, 49 insertions(+), 336 deletions(-) rename rust/Cargo.lock => Cargo.lock (100%) rename rust/Cargo.toml => Cargo.toml (100%) delete mode 100644 rust/.gitignore delete mode 100644 rust/datasize.jsonnet delete mode 100644 rust/example.json delete mode 100644 rust/heavy-load.yml delete mode 100644 rust/test.jsonnet rename {rust/src => src}/client/mod.rs (100%) rename {rust/src => src}/concepts/action.rs (100%) rename {rust/src => src}/concepts/config.rs (100%) rename {rust/src => src}/concepts/filter.rs (100%) rename {rust/src => src}/concepts/messages.rs (100%) rename {rust/src => src}/concepts/mod.rs (100%) rename {rust/src => src}/concepts/pattern.rs (100%) rename {rust/src => src}/concepts/socket_messages.rs (100%) rename {rust/src => src}/concepts/stream.rs (100%) rename {rust/src => src}/daemon/action.rs (100%) rename {rust/src => src}/daemon/database/lowlevel.rs (100%) rename {rust/src => src}/daemon/database/mod.rs (100%) rename {rust/src => src}/daemon/database/tests.rs (100%) rename {rust/src => src}/daemon/filter.rs (100%) rename {rust/src => src}/daemon/mod.rs (100%) rename {rust/src => src}/daemon/socket.rs (100%) rename {rust/src => src}/daemon/stream.rs (100%) rename {rust/src => src}/lib.rs (100%) rename {rust/src => src}/main.rs (100%) rename {rust/src => src}/tests.rs (100%) rename {rust/src => src}/utils/cli.rs (100%) rename {rust/src => src}/utils/mod.rs (100%) rename {rust/src => src}/utils/parse_duration.rs (100%) rename {rust/tests => tests}/simple.rs (100%) diff --git a/.gitignore b/.gitignore index 10a2ee1..cd78ecc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ reaction*.export.json debian-packaging/* *.swp export-go-db/export-db -import-rust-db/target/ +import-rust-db/target +/target diff --git a/rust/Cargo.lock b/Cargo.lock similarity index 100% rename from rust/Cargo.lock rename to Cargo.lock diff --git a/rust/Cargo.toml b/Cargo.toml similarity index 100% rename from rust/Cargo.toml rename to Cargo.toml diff --git a/config/heavy-load.yml b/config/heavy-load.yml index 1c149c3..6af6dfa 100644 --- a/config/heavy-load.yml +++ b/config/heavy-load.yml @@ -1,72 +1,72 @@ --- +concurrency: 32 + patterns: num: - regex: '[0-9]+' + regex: '[0-9]{3}' ip: regex: '(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})' ignore: - 1.0.0.1 -concurrency: 0 - streams: tailDown1: - cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo found $(($i % 100)); done' ] + cmd: [ 'sh', '-c', 'sleep 2; seq 10001 | while read i; do echo found $i; done' ] filters: - findIP: + find: regex: - - '^found $' - retry: 50 - retryperiod: 1m + - '^found ' + retry: 9 + retryperiod: 6m actions: damn: - cmd: [ 'sleep', '0.' ] + cmd: [ 'sleep', '0.0' ] undamn: - cmd: [ 'sleep', '0.' ] + cmd: [ 'sleep', '0.0' ] after: 1m onexit: false tailDown2: - cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo prout $(($i % 100)); done' ] + cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] filters: - findIP: + find: regex: - - '^prout $' - retry: 50 - retryperiod: 1m + - '^found ' + retry: 480 + retryperiod: 6m actions: damn: - cmd: [ 'sleep', '0.' ] + cmd: [ 'sleep', '0.0' ] undamn: - cmd: [ 'sleep', '0.' ] + cmd: [ 'sleep', '0.0' ] after: 1m onexit: false tailDown3: - cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo nanana $(($i % 100)); done' ] + cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] filters: - findIP: + find: regex: - - '^nanana $' - retry: 50 - retryperiod: 2m + - '^found ' + retry: 480 + retryperiod: 6m actions: damn: - cmd: [ 'sleep', '0.' ] + cmd: [ 'sleep', '0.0' ] undamn: - cmd: [ 'sleep', '0.' ] + cmd: [ 'sleep', '0.0' ] after: 1m onexit: false tailDown4: - cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo nanana $(($i % 100)); done' ] + cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] filters: - findIP: + find: regex: - - '^nomatch $' - retry: 50 - retryperiod: 2m + - '^found ' + retry: 480 + retryperiod: 6m actions: damn: - cmd: [ 'sleep', '0.' ] + cmd: [ 'sleep', '0.0' ] undamn: - cmd: [ 'sleep', '0.' ] + cmd: [ 'sleep', '0.0' ] after: 1m onexit: false diff --git a/config/test.jsonnet b/config/test.jsonnet index 9210a80..6e99367 100644 --- a/config/test.jsonnet +++ b/config/test.jsonnet @@ -12,47 +12,31 @@ }, }, + start: [ + ['echo', 'coucou'], + ], + + stop: [ + ['echo', 'byebye'], + ], + streams: { - tailDown1: { - cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 1; echo found $i; done; sleep 30"], + s1: { + cmd: ['sh', '-c', "seq 20 | tr ' ' '\n' | while read i; do echo found $((i % 5)); sleep 1; done"], filters: { - findIP: { + f1: { regex: [ - '^found _$', - '^found _$', + '^found $', ], retry: 2, - retryperiod: '30s', + retryperiod: '60s', actions: { damn: { - cmd: ['echo', ''], + cmd: ['notify-send', 'first stream', 'ban '], }, undamn: { - cmd: ['echo', 'undamn', ''], - after: '28s', - onexit: true, - }, - }, - }, - }, - }, - tailDown2: { - cmd: ['sh', '-c', "echo 1_abc 2_abc 3_abc abc_1 abc_2 abc_3 | tr ' ' '\n' | while read i; do sleep 1; echo found $i; done; sleep 30"], - filters: { - findIP: { - regex: [ - '^found _$', - '^found _$', - ], - retry: 2, - retryperiod: '30s', - actions: { - damn: { - cmd: ['echo', ''], - }, - undamn: { - cmd: ['echo', 'undamn', ''], - after: '28s', + cmd: ['notify-send', 'first stream', 'unban '], + after: '3s', onexit: true, }, }, diff --git a/rust/.gitignore b/rust/.gitignore deleted file mode 100644 index 2f7896d..0000000 --- a/rust/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target/ diff --git a/rust/datasize.jsonnet b/rust/datasize.jsonnet deleted file mode 100644 index 2c842df..0000000 --- a/rust/datasize.jsonnet +++ /dev/null @@ -1,30 +0,0 @@ -{ - patterns: { - num: { - regex: @'([0-9]+)', - }, - }, - streams: { - s1: { - cmd: ['seq', '-w', '499999'], - filters: { - f1: { - regex: [ - '^$', - ], - retry: 10, - retryperiod: '1m', - actions: { - a: { - cmd: ['true'], - }, - b: { - cmd: ['true'], - after: '1m', - }, - }, - }, - }, - }, - }, -} diff --git a/rust/example.json b/rust/example.json deleted file mode 100644 index 8591b6d..0000000 --- a/rust/example.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "concurrency": 0, - "patterns": { - "ip": { - "ignore": [ - "127.0.0.1", - "::1" - ], - "regex": "(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))" - } - }, - "start": [ - [ - "ip46tables", - "-w", - "-N", - "reaction" - ], - [ - "ip46tables", - "-w", - "-I", - "INPUT", - "-p", - "all", - "-j", - "reaction" - ], - [ - "ip46tables", - "-w", - "-I", - "FORWARD", - "-p", - "all", - "-j", - "reaction" - ] - ], - "stop": [ - [ - "ip46tables", - "-w", - "-D", - "INPUT", - "-p", - "all", - "-j", - "reaction" - ], - [ - "ip46tables", - "-w", - "-D", - "FORWARD", - "-p", - "all", - "-j", - "reaction" - ], - [ - "ip46tables", - "-w", - "-F", - "reaction" - ], - [ - "ip46tables", - "-w", - "-X", - "reaction" - ] - ], - "streams": { - "ssh": { - "cmd": [ - "journalctl", - "-n0", - "-fu", - "sshd.service" - ], - "filters": { - "failedlogin": { - "actions": { - "ban": { - "cmd": [ - "ip46tables", - "-w", - "-A", - "reaction", - "-s", - "", - "-j", - "DROP" - ] - }, - "unban": { - "after": "48h", - "cmd": [ - "ip46tables", - "-w", - "-D", - "reaction", - "-s", - "", - "-j", - "DROP" - ] - } - }, - "regex": [ - "authentication failure;.*rhost=", - "Failed password for .* from ", - "Connection (reset|closed) by (authenticating|invalid) user .* " - ], - "retry": 3, - "retryperiod": "6h" - } - } - } - } -} diff --git a/rust/heavy-load.yml b/rust/heavy-load.yml deleted file mode 100644 index 6af6dfa..0000000 --- a/rust/heavy-load.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -concurrency: 32 - -patterns: - num: - regex: '[0-9]{3}' - ip: - regex: '(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})' - ignore: - - 1.0.0.1 - -streams: - tailDown1: - cmd: [ 'sh', '-c', 'sleep 2; seq 10001 | while read i; do echo found $i; done' ] - filters: - find: - regex: - - '^found ' - retry: 9 - retryperiod: 6m - actions: - damn: - cmd: [ 'sleep', '0.0' ] - undamn: - cmd: [ 'sleep', '0.0' ] - after: 1m - onexit: false - tailDown2: - cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] - filters: - find: - regex: - - '^found ' - retry: 480 - retryperiod: 6m - actions: - damn: - cmd: [ 'sleep', '0.0' ] - undamn: - cmd: [ 'sleep', '0.0' ] - after: 1m - onexit: false - tailDown3: - cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] - filters: - find: - regex: - - '^found ' - retry: 480 - retryperiod: 6m - actions: - damn: - cmd: [ 'sleep', '0.0' ] - undamn: - cmd: [ 'sleep', '0.0' ] - after: 1m - onexit: false - tailDown4: - cmd: [ 'sh', '-c', 'sleep 2; seq 1000100 | while read i; do echo found $i; done' ] - filters: - find: - regex: - - '^found ' - retry: 480 - retryperiod: 6m - actions: - damn: - cmd: [ 'sleep', '0.0' ] - undamn: - cmd: [ 'sleep', '0.0' ] - after: 1m - onexit: false diff --git a/rust/test.jsonnet b/rust/test.jsonnet deleted file mode 100644 index 6e99367..0000000 --- a/rust/test.jsonnet +++ /dev/null @@ -1,47 +0,0 @@ -{ - patterns: { - num: { - regex: '[0-9]+', - ignore: ['1'], - // ignoreregex: ['2.?'], - }, - letter: { - regex: '[a-z]+', - ignore: ['b'], - // ignoreregex: ['b.?'], - }, - }, - - start: [ - ['echo', 'coucou'], - ], - - stop: [ - ['echo', 'byebye'], - ], - - streams: { - s1: { - cmd: ['sh', '-c', "seq 20 | tr ' ' '\n' | while read i; do echo found $((i % 5)); sleep 1; done"], - filters: { - f1: { - regex: [ - '^found $', - ], - retry: 2, - retryperiod: '60s', - actions: { - damn: { - cmd: ['notify-send', 'first stream', 'ban '], - }, - undamn: { - cmd: ['notify-send', 'first stream', 'unban '], - after: '3s', - onexit: true, - }, - }, - }, - }, - }, - }, -} diff --git a/rust/src/client/mod.rs b/src/client/mod.rs similarity index 100% rename from rust/src/client/mod.rs rename to src/client/mod.rs diff --git a/rust/src/concepts/action.rs b/src/concepts/action.rs similarity index 100% rename from rust/src/concepts/action.rs rename to src/concepts/action.rs diff --git a/rust/src/concepts/config.rs b/src/concepts/config.rs similarity index 100% rename from rust/src/concepts/config.rs rename to src/concepts/config.rs diff --git a/rust/src/concepts/filter.rs b/src/concepts/filter.rs similarity index 100% rename from rust/src/concepts/filter.rs rename to src/concepts/filter.rs diff --git a/rust/src/concepts/messages.rs b/src/concepts/messages.rs similarity index 100% rename from rust/src/concepts/messages.rs rename to src/concepts/messages.rs diff --git a/rust/src/concepts/mod.rs b/src/concepts/mod.rs similarity index 100% rename from rust/src/concepts/mod.rs rename to src/concepts/mod.rs diff --git a/rust/src/concepts/pattern.rs b/src/concepts/pattern.rs similarity index 100% rename from rust/src/concepts/pattern.rs rename to src/concepts/pattern.rs diff --git a/rust/src/concepts/socket_messages.rs b/src/concepts/socket_messages.rs similarity index 100% rename from rust/src/concepts/socket_messages.rs rename to src/concepts/socket_messages.rs diff --git a/rust/src/concepts/stream.rs b/src/concepts/stream.rs similarity index 100% rename from rust/src/concepts/stream.rs rename to src/concepts/stream.rs diff --git a/rust/src/daemon/action.rs b/src/daemon/action.rs similarity index 100% rename from rust/src/daemon/action.rs rename to src/daemon/action.rs diff --git a/rust/src/daemon/database/lowlevel.rs b/src/daemon/database/lowlevel.rs similarity index 100% rename from rust/src/daemon/database/lowlevel.rs rename to src/daemon/database/lowlevel.rs diff --git a/rust/src/daemon/database/mod.rs b/src/daemon/database/mod.rs similarity index 100% rename from rust/src/daemon/database/mod.rs rename to src/daemon/database/mod.rs diff --git a/rust/src/daemon/database/tests.rs b/src/daemon/database/tests.rs similarity index 100% rename from rust/src/daemon/database/tests.rs rename to src/daemon/database/tests.rs diff --git a/rust/src/daemon/filter.rs b/src/daemon/filter.rs similarity index 100% rename from rust/src/daemon/filter.rs rename to src/daemon/filter.rs diff --git a/rust/src/daemon/mod.rs b/src/daemon/mod.rs similarity index 100% rename from rust/src/daemon/mod.rs rename to src/daemon/mod.rs diff --git a/rust/src/daemon/socket.rs b/src/daemon/socket.rs similarity index 100% rename from rust/src/daemon/socket.rs rename to src/daemon/socket.rs diff --git a/rust/src/daemon/stream.rs b/src/daemon/stream.rs similarity index 100% rename from rust/src/daemon/stream.rs rename to src/daemon/stream.rs diff --git a/rust/src/lib.rs b/src/lib.rs similarity index 100% rename from rust/src/lib.rs rename to src/lib.rs diff --git a/rust/src/main.rs b/src/main.rs similarity index 100% rename from rust/src/main.rs rename to src/main.rs diff --git a/rust/src/tests.rs b/src/tests.rs similarity index 100% rename from rust/src/tests.rs rename to src/tests.rs diff --git a/rust/src/utils/cli.rs b/src/utils/cli.rs similarity index 100% rename from rust/src/utils/cli.rs rename to src/utils/cli.rs diff --git a/rust/src/utils/mod.rs b/src/utils/mod.rs similarity index 100% rename from rust/src/utils/mod.rs rename to src/utils/mod.rs diff --git a/rust/src/utils/parse_duration.rs b/src/utils/parse_duration.rs similarity index 100% rename from rust/src/utils/parse_duration.rs rename to src/utils/parse_duration.rs diff --git a/rust/tests/simple.rs b/tests/simple.rs similarity index 100% rename from rust/tests/simple.rs rename to tests/simple.rs From 7deb2b4625be1db1cc0857baf2b46d597775730f Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 19 Oct 2024 12:00:00 +0200 Subject: [PATCH 078/435] Remove tokio-console --- Cargo.lock | 675 +---------------------------------------------------- Cargo.toml | 4 - 2 files changed, 6 insertions(+), 673 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e74edc2..32d549d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -100,104 +100,12 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "anyhow" -version = "1.0.90" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" - -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.82", -] - -[[package]] -name = "async-trait" -version = "0.1.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.82", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "axum" -version = "0.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" -dependencies = [ - "async-trait", - "axum-core", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper 1.0.1", - "tower 0.5.1", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper 1.0.1", - "tower-layer", - "tower-service", -] - [[package]] name = "backtrace" version = "0.3.74" @@ -219,18 +127,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "bincode" version = "1.3.3" @@ -252,12 +148,6 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.7.2" @@ -348,81 +238,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" -[[package]] -name = "console-api" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ed14aa9c9f927213c6e4f3ef75faaad3406134efe84ba2cb7983431d5f0931" -dependencies = [ - "futures-core", - "prost", - "prost-types", - "tonic", - "tracing-core", -] - -[[package]] -name = "console-subscriber" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e3a111a37f3333946ebf9da370ba5c5577b18eb342ec683eb488dd21980302" -dependencies = [ - "console-api", - "crossbeam-channel", - "crossbeam-utils", - "futures-task", - "hdrhistogram", - "humantime", - "hyper-util", - "prost", - "prost-types", - "serde", - "serde_json", - "thread_local", - "tokio", - "tokio-stream", - "tonic", - "tracing", - "tracing-core", - "tracing-subscriber", -] - [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "crc32fast" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" - -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - [[package]] name = "equivalent" version = "1.0.1" @@ -445,22 +266,6 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" -[[package]] -name = "flate2" -version = "1.0.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "futures" version = "0.3.31" @@ -550,67 +355,18 @@ dependencies = [ "slab", ] -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - [[package]] name = "gimli" version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" -[[package]] -name = "h2" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap 2.6.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" -[[package]] -name = "hdrhistogram" -version = "7.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" -dependencies = [ - "base64 0.21.7", - "byteorder", - "flate2", - "nom", - "num-traits", -] - [[package]] name = "heck" version = "0.5.0" @@ -623,111 +379,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" -[[package]] -name = "http" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" -dependencies = [ - "bytes", - "futures-util", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-timeout" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" -dependencies = [ - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - [[package]] name = "iana-time-zone" version = "0.1.61" @@ -751,16 +402,6 @@ dependencies = [ "cc", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.6.0" @@ -768,7 +409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown", ] [[package]] @@ -777,15 +418,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.11" @@ -799,7 +431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fee60406dac44a01b37e120b43adb062047251e195db15392b825f6bdc948712" dependencies = [ "annotate-snippets", - "base64 0.13.1", + "base64", "bincode", "jrsonnet-gc", "jrsonnet-interner", @@ -918,21 +550,6 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata 0.1.10", -] - -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "md5" version = "0.7.0" @@ -945,18 +562,6 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.8.0" @@ -978,16 +583,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1094,32 +689,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c719dcf55f09a3a7e764c6649ab594c18a177e3599c467983cdf644bfc0a4088" -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pin-project" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.82", -] - [[package]] name = "pin-project-lite" version = "0.2.14" @@ -1132,15 +701,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "ppv-lite86" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" -dependencies = [ - "zerocopy", -] - [[package]] name = "proc-macro2" version = "1.0.88" @@ -1150,38 +710,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 2.0.82", -] - -[[package]] -name = "prost-types" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" -dependencies = [ - "prost", -] - [[package]] name = "quote" version = "1.0.37" @@ -1191,36 +719,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - [[package]] name = "reaction" version = "0.1.0" @@ -1229,7 +727,6 @@ dependencies = [ "chrono", "clap", "clap_complete", - "console-subscriber", "futures", "jrsonnet-evaluator", "num_cpus", @@ -1263,17 +760,8 @@ checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", - "regex-syntax 0.8.5", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", + "regex-automata", + "regex-syntax", ] [[package]] @@ -1284,15 +772,9 @@ checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax", ] -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.8.5" @@ -1324,12 +806,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustversion" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" - [[package]] name = "ryu" version = "1.0.18" @@ -1380,7 +856,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.6.0", + "indexmap", "itoa", "ryu", "serde", @@ -1464,18 +940,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "sync_wrapper" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" - [[package]] name = "synstructure" version = "0.12.6" @@ -1570,17 +1034,6 @@ dependencies = [ "syn 2.0.82", ] -[[package]] -name = "tokio-stream" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.12" @@ -1594,82 +1047,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tonic" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.22.1", - "bytes", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "prost", - "socket2", - "tokio", - "tokio-stream", - "tower 0.4.13", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper 0.1.2", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - [[package]] name = "tracing" version = "0.1.40" @@ -1719,24 +1096,14 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ - "matchers", "nu-ansi-term", - "once_cell", - "regex", "sharded-slab", "smallvec", "thread_local", - "tracing", "tracing-core", "tracing-log", ] -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "unescape" version = "0.1.0" @@ -1779,15 +1146,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1970,24 +1328,3 @@ checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1" dependencies = [ "winapi", ] - -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.82", -] diff --git a/Cargo.toml b/Cargo.toml index 7ae3f02..49c4b6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,9 +3,6 @@ name = "reaction" version = "0.1.0" edition = "2021" -[build] -rustflags = ["--cfg", "tokio_unstable"] - [dependencies] bincode = "1.3.3" chrono = { version = "0.4.38", features = ["std", "clock"] } @@ -27,4 +24,3 @@ tokio = { version = "1.40.0", features = ["full", "tracing"] } tokio-util = { version = "0.7.12", features = ["codec"] } tracing = "0.1.40" tracing-subscriber = "0.3.18" -console-subscriber = "0.4.0" From a80e3764f1e921fea443f1d46a38ed5d26af4465 Mon Sep 17 00:00:00 2001 From: ppom Date: Tue, 22 Oct 2024 12:00:00 +0200 Subject: [PATCH 079/435] version 2.0.0-rc1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/utils/cli.rs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 32d549d..412a30c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -721,7 +721,7 @@ dependencies = [ [[package]] name = "reaction" -version = "0.1.0" +version = "2.0.0-rc1" dependencies = [ "bincode", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 49c4b6e..b0a76d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "reaction" -version = "0.1.0" +version = "2.0.0-rc1" edition = "2021" [dependencies] diff --git a/src/utils/cli.rs b/src/utils/cli.rs index 47d42ae..79b9008 100644 --- a/src/utils/cli.rs +++ b/src/utils/cli.rs @@ -7,7 +7,6 @@ use tracing::Level; #[derive(Parser)] #[clap(version)] #[command(name = "reaction")] -#[command(version = "2.0")] #[command( about = "Scan logs and take action", long_about = "A daemon that scans program outputs for repeated patterns, and takes action. From a05e05750cd342aa85d321fd209a7f611edddef6 Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 24 Oct 2024 12:00:00 +0200 Subject: [PATCH 080/435] Packaging for Rust --- Cargo.lock | 17 + Cargo.toml | 18 +- Makefile | 6 +- README.md | 4 +- build.rs | 58 ++++ .../{export-db.go => export-go-db.go} | 0 import-rust-db/src/main.rs | 1 + release.py | 296 ++++++++++++++++++ release.sh | 56 +++- src/main.rs | 12 +- src/utils/cli.rs | 6 +- 11 files changed, 453 insertions(+), 21 deletions(-) create mode 100644 build.rs rename export-go-db/{export-db.go => export-go-db.go} (100%) create mode 100644 release.py diff --git a/Cargo.lock b/Cargo.lock index 412a30c..832022e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,6 +232,16 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "clap_mangen" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbae9cbfdc5d4fa8711c09bd7b83f644cb48281ac35bf97af3e47b0675864bdf" +dependencies = [ + "clap", + "roff", +] + [[package]] name = "colorchoice" version = "1.0.2" @@ -727,6 +737,7 @@ dependencies = [ "chrono", "clap", "clap_complete", + "clap_mangen", "futures", "jrsonnet-evaluator", "num_cpus", @@ -781,6 +792,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "roff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88f8660c1ff60292143c98d08fc6e2f654d722db50410e3f3797d40baaf9d8f3" + [[package]] name = "rustc-demangle" version = "0.1.24" diff --git a/Cargo.toml b/Cargo.toml index b0a76d5..d287106 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,12 +2,19 @@ name = "reaction" version = "2.0.0-rc1" edition = "2021" +authors = ["ppom Provided binaries in the previous section are compiled this way: ```shell -$ docker run -it --rm -e HOME=/tmp/ -v $(pwd):/tmp/code -w /tmp/code -u $(id -u) golang:1.20 make clean reaction.deb +$ docker run -it --rm -e HOME=/tmp/ -v $(pwd):/tmp/code -w /tmp/code -u $(id -u) rust make clean reaction.deb $ make signatures ``` #### Signature verification @@ -207,7 +207,7 @@ ExecStart=/usr/bin/reaction start -c /etc/reaction.yml ### Compilation -You'll need the go (>= 1.20) toolchain for reaction and a c compiler for ip46tables. +You'll need a recent rust toolchain for reaction and a c compiler for ip46tables. ```shell $ make ``` diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..f8ce4db --- /dev/null +++ b/build.rs @@ -0,0 +1,58 @@ +use std::{ + env::var_os, + io::{self, ErrorKind}, + path::Path, + process, +}; + +use clap_complete::shells; + +// SubCommand defined here +include!("src/utils/cli.rs"); + +fn compile_helper(name: &str, out_dir: &Path) -> io::Result<()> { + process::Command::new("gcc") + .args([ + &format!("helpers_c/{name}.c"), + "-o", + out_dir.join(name).to_str().expect("could not join path"), + ]) + .spawn()?; + Ok(()) +} + +fn main() -> io::Result<()> { + if var_os("PROFILE").ok_or(ErrorKind::NotFound)? == "release" { + let out_dir = PathBuf::from(var_os("OUT_DIR").ok_or(ErrorKind::NotFound)?).join("../../.."); + + // Compile C helpers + compile_helper("ip46tables", &out_dir)?; + compile_helper("nft46", &out_dir)?; + + // Build CLI + let cli = clap::Command::new("reaction"); + let cli = SubCommand::augment_subcommands(cli); + // We have to manually add metadata because it is lost: only subcommands are appended + let cli = cli.about("Scan logs and take action").long_about( +"A daemon that scans program outputs for repeated patterns, and takes action. + +Aims at being more versatile and flexible than fail2ban, while being faster and having simpler configuration. + +See usage examples, service configurations and good practices on the wiki: https://reaction.ppom.me"); + + // Generate completions + clap_complete::generate_to(shells::Bash, &mut cli.clone(), "reaction", out_dir.clone())?; + clap_complete::generate_to(shells::Fish, &mut cli.clone(), "reaction", out_dir.clone())?; + clap_complete::generate_to(shells::Zsh, &mut cli.clone(), "reaction", out_dir.clone())?; + + // Generate manpages + clap_mangen::generate_to(cli, out_dir.clone())?; + } + + println!("cargo::rerun-if-changed=build.rs"); + println!("cargo::rerun-if-changed=src/utils/cli.rs"); + println!("cargo::rerun-if-changed=helpers_c/ip46tables.c"); + println!("cargo::rerun-if-changed=helpers_c/nft46.c"); + + Ok(()) +} diff --git a/export-go-db/export-db.go b/export-go-db/export-go-db.go similarity index 100% rename from export-go-db/export-db.go rename to export-go-db/export-go-db.go diff --git a/import-rust-db/src/main.rs b/import-rust-db/src/main.rs index 07f6a64..0918ed1 100644 --- a/import-rust-db/src/main.rs +++ b/import-rust-db/src/main.rs @@ -28,6 +28,7 @@ fn main() { } fn lil_main() -> Result<(), E> { + println!("Hello from Rust!"); println!("You're about to reimport the previously exported database."); println!("This will move the old database files as .old"); println!("When the process completes, you'll be able to run reaction-v2."); diff --git a/release.py b/release.py new file mode 100644 index 0000000..2084daf --- /dev/null +++ b/release.py @@ -0,0 +1,296 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests ])" -p debian-devscripts git minisign cargo-cross +import base64 +import json +import os +import subprocess +import sys +import tempfile + +def quit_if(cmd): + if cmd.returncode != 0: + print(f"{' '.join(cmd.args)} failed with exit code {cmd.returncode}") + sys.exit(1) + +def add_path(files, architecture): + return [ + (f"./target/{architecture}/release/{file[0]}", file[1], file[2], file[3]) + for file in files + ] + +def main(): + # Git tag + cmd = subprocess.run(["git", "tag", "--sort=v:refname"], capture_output=True, text=True) + quit_if(cmd) + tag = "" + try: + tag = cmd.stdout.strip().split("\n")[-1] + except: + pass + if tag == "": + print("could not retrieve last git tag.") + sys.exit(1) + + # Ask user + # if input(f"We will create a release for tag {tag}. Do you want to continue? (y/n) ") != "y": + # print("exiting.") + # sys.exit(1) + + # Git push + # cmd = subprocess.run(["git", "push", "--tags"]) + # quit_if(cmd) + + cmd = subprocess.run(["rbw", "get", "minisign"], capture_output=True, text=True) + quit_if(cmd) + minisign_password = cmd.stdout + + all_files = [] + + architectures = [ + ("x86_64-unknown-linux-gnu", "amd64"), + # "x86_64-unknown-openbsd", # not supported by cross + ("armv7-unknown-linux-gnueabihf", "arm"), + ] + + for archs in architectures: + go_arch = archs[1] + architecture = archs[0] + + # Install toolchain + # cmd = subprocess.run([ + # "rustup", "toolchain", "install", "stable", + # "-t", architecture, + # "--profile", "minimal"]) + # quit_if(cmd) + + # Build + cmd = subprocess.run([ + "cross", "build", "--release", "--target", architecture + ]) + quit_if(cmd) + + # Build rust import db + os.chdir("./import-rust-db") + cmd = subprocess.run([ + "cross", "build", "--release", "--target", architecture + ]) + quit_if(cmd) + with open(f"./target/{architecture}/release/import-rust-db", "rb") as file: + rust_contents = base64.standard_b64encode(file.read()) + os.chdir("..") + + # Build go export db + os.chdir("./export-go-db") + cmd = subprocess.run( + ["go", "build", "export-go-db.go"], + env=os.environ.update({"GOARCH": go_arch}) + ) + quit_if(cmd) + with open("./export-go-db", "rb") as file: + go_contents = base64.standard_b64encode(file.read()) + os.chdir("..") + + # Build glue script + contents = """ +func main() { + set -eu -o pipefail + DIR="$(mktemp -d)" + echo "$GOBIN" | base64 -d > "$DIR/gobin" + echo "$RUBIN" | base64 -d > "$DIR/rubin" + chmod +x "$DIR/gobin" "$DIR/rubin" + "$DIR/gobin" + "$DIR/rubin" + rm "$DIR/gobin" "$DIR/rubin" + rmdir "$DIR" +} + +GOBIN=""".encode() + go_contents + """ +RUBIN=""".encode() + rust_contents + """ + +main +""".encode() + del go_contents + del rust_contents + + with open(f"./target/{architecture}/release/migrate_reaction_db", "bw+") as file: + file.write(contents) + + del contents + + # # Build + # cmd = subprocess.run([ + # "docker", "run", "-it", "--rm", + # "-e", "HOME=/tmp/", + # "-v", f"{os.getcwd()}:/tmp/code", + # "-w", "/tmp/code", + # "-u", str(os.getuid()), + # "rust", + # "make", f"reaction_{tag}-1_amd64.deb", "reaction", "ip46tables", "nft46" + # ]) + # quit_if(cmd) + + # File lists + binary_files = [ + ("reaction", architecture, f"reaction ({architecture})", "package"), + ("migrate_reaction_db", architecture, f"db migration script ({architecture})", "package"), + ("nft46", architecture, f"nft46 ({architecture})", "package"), + ("ip46tables", architecture, f"ip46tables ({architecture})", "package"), + # (f"reaction_{tag}-1_amd64.deb", architecture, f"reaction.deb ({architecture})", "package") + ] + + sig_files = [ + (f"{file[0]}.minisig", architecture, f"{file[0]}.minisig ({architecture})", "other") + for file + in binary_files + ] + + binary_files = add_path(binary_files, architecture) + sig_files = add_path(sig_files, architecture) + + # Sign + cmd = subprocess.run( + ["minisign", "-Sm"] + + [ file[0] for file in binary_files ], + text=True, + input=minisign_password + ) + quit_if(cmd) + + # Create directory + cmd = subprocess.run([ + "ssh", "akesi", + "-J", "pica01", + "mkdir", "-p", f"/var/www/static/reaction/releases/{tag}/{architecture}/" + ]) + quit_if(cmd) + + # Push + cmd = subprocess.run([ + "rsync", + "-avze", "ssh -J pica01", ] + + [ file[0] for file in binary_files + sig_files ] + + [ f"akesi:/var/www/static/reaction/releases/{tag}/{architecture}/" + ]) + quit_if(cmd) + + all_files.extend(binary_files) + all_files.extend(sig_files) + + # Copy only one time the text files, which are architecture-independant + if archs == architectures[-1]: + text_files = [ + ("reaction.bash", "", "bash completion file", "other"), + ("reaction.fish", "", "fish completion file", "other"), + ("_reaction", "", "zsh completion file", "other"), + ("reaction.1", False, False, False), + ("reaction-flush.1", False, False, False), + ("reaction-show.1", False, False, False), + ("reaction-start.1", False, False, False), + ("reaction-test-regex.1", False, False, False), + ] + text_files = add_path(text_files, architecture) + all_files.extend(text_files) + + cmd = subprocess.run([ + "rsync", + "-avze", "ssh -J pica01" ] + + [ file[0] for file in text_files ] + + [ f"akesi:/var/www/static/reaction/releases/{tag}/" + ]) + quit_if(cmd) + + # Release + + cmd = subprocess.run(["rbw", "get", "framagit.org", "token"], capture_output=True, text=True) + quit_if(cmd) + token = cmd.stdout.strip() + if token == "": + print("Could not retrieve token") + sys.exit(1) + + description = f""" +## Changes + +## Direct download + +```bash +wget https://static.ppom.me/reaction/releases/{architectures[0][0]}/{tag}/nft46 \\ + https://static.ppom.me/reaction/releases/{architectures[0][0]}/{tag}/reaction \\ + https://static.ppom.me/reaction/releases/{architectures[0][0]}/{tag}/ip46tables \\ + https://static.ppom.me/reaction/releases/{architectures[0][0]}/{tag}/nft46.minisig \\ + https://static.ppom.me/reaction/releases/{architectures[0][0]}/{tag}/reaction.minisig \\ + https://static.ppom.me/reaction/releases/{architectures[0][0]}/{tag}/ip46tables.minisig +for i in nft46 ip46tables reaction +do + minisign -VP RWSpLTPfbvllNqRrXUgZzM7mFjLUA7PQioAItz80ag8uU4A2wtoT2DzX -m $i && rm $i.minisig +done +``` + + +``` +""" + + # Make user edit the description + tmpdir = tempfile.TemporaryDirectory() + desc_path = tmpdir.name + "/description.md" + with open(desc_path, "w+") as desc_file: + desc_file.write(description) + cmd = subprocess.run(["vi", desc_path]) + quit_if(cmd) + + with open(desc_path) as desc_file: + description = desc_file.read().strip() + + if description == "": + print() + print("User deleted emptied description, exiting.") + sys.exit(1) + + files = [ file for file in all_files if file[2] != False ] + data = { + "tag_name": tag, + "description": description, + "assets": { + "links": [ + { + "url": f"https://static.ppom.me/reaction/releases/{tag}/{file[1]}/{os.path.basename(file[0])}".replace("//", "/"), + "name": file[2], + "link_type": file[3], + } + for file + in files + ] + } + } + + body = json.dumps(data) + + subprocess.run(["jq"], text=True, input=body) + + headers = { + "Host": "framagit.org", + "Content-Type": "application/json", + "PRIVATE-TOKEN": token, + } + + sys.exit(1) + + conn = http.client.HTTPSConnection("framagit.org") + conn.request("POST", "/api/v4/projects/90566/releases", body=body, headers=headers) + response = conn.getresponse() + + if response.status != 200: + print(f"sending message failed: status: {response.status}, reason: {response.reason}") + sys.exit(1) + +main() diff --git a/release.sh b/release.sh index 7f958fa..56400c8 100755 --- a/release.sh +++ b/release.sh @@ -6,17 +6,60 @@ git push --tags TAG="$(git tag --sort=v:refname | tail -n1)" -docker run -it --rm -e HOME=/tmp/ -v "$(pwd)":/tmp/code -w /tmp/code golang:1.21-bullseye sh -c "git config --global --add safe.directory . && make reaction_${TAG:1}-1_amd64.deb reaction ip46tables nft46" +echo -n "We will create a release for tag $TAG. Do you want to continue? (y/n) " + +docker run -it --rm \ + -e HOME=/tmp/ \ + -v "$(pwd)":/tmp/code -w /tmp/code \ + -u "$(id -u)" \ + rust \ + make reaction_"${TAG:1}"-1_amd64.deb reaction ip46tables nft46 make "signatures_${TAG:1}" -rsync -avz -e 'ssh -J pica01' ./ip46tables ./nft46 ./reaction ./reaction_${TAG:1}-1_amd64.deb ./nft46.minisig ./ip46tables.minisig ./reaction.minisig ./reaction_${TAG:1}-1_amd64.deb.minisig akesi:/var/www/static/reaction/releases/"$TAG" +rsync -avz -e 'ssh -J pica01' \ + ./ip46tables ./ip46tables.minisig \ + ./nft46 ./nft46.minisig \ + ./target/release/reaction ./reaction.minisig \ + ./reaction_"${TAG:1}"-1_amd64.deb ./reaction_"${TAG:1}"-1_amd64.deb.minisig \ + ./target/release/reaction.bash ./target/release/reaction.fish ./target/release/_reaction \ + ./target/release/reaction*.1 + akesi:/var/www/static/reaction/releases/"$TAG" TOKEN="$(rbw get framagit.org token)" +DESCRIPTION=" +## Changes + +## Download + +\`\`\`bash +wget https://static.ppom.me/reaction/releases/""$TAG""/nft46 \\ + https://static.ppom.me/reaction/releases/""$TAG""/reaction \\ + https://static.ppom.me/reaction/releases/""$TAG""/ip46tables \\ + https://static.ppom.me/reaction/releases/""$TAG""/nft46.minisig \\ + https://static.ppom.me/reaction/releases/""$TAG""/reaction.minisig \\ + https://static.ppom.me/reaction/releases/""$TAG""/ip46tables.minisig +for i in nft46 ip46tables reaction +do + minisign -VP RWSpLTPfbvllNqRrXUgZzM7mFjLUA7PQioAItz80ag8uU4A2wtoT2DzX -m $i && rm $i.minisig +done +\`\`\` + +**Debian Installation** + +\`\`\`bash +wget https://static.ppom.me/reaction/releases/""$TAG""/reaction_""${TAG:1}""-1_amd64.deb \\ +https://static.ppom.me/reaction/releases/""$TAG""/reaction_""${TAG:1}""-1_amd64.deb.minisig +minisign -VP RWSpLTPfbvllNqRrXUgZzM7mFjLUA7PQioAItz80ag8uU4A2wtoT2DzX -m reaction_""${TAG:1}""-1_amd64.deb && + rm reaction_""${TAG:1}""-1_amd64.deb.minisig && + apt install ./reaction_""${TAG:1}""-1_amd64.deb +\`\`\` +" + DATA='{ "tag_name":"'"$TAG"'", -"description": "**Changes**\n\n**Download**\n\n```bash\nwget https://static.ppom.me/reaction/releases/'"$TAG"'/nft46 \\\n https://static.ppom.me/reaction/releases/'"$TAG"'/reaction \\\n https://static.ppom.me/reaction/releases/'"$TAG"'/ip46tables \\\n https://static.ppom.me/reaction/releases/'"$TAG"'/nft46.minisig \\\n https://static.ppom.me/reaction/releases/'"$TAG"'/reaction.minisig \\\n https://static.ppom.me/reaction/releases/'"$TAG"'/ip46tables.minisig\nfor i in nft46 ip46tables reaction; do\n minisign -VP RWSpLTPfbvllNqRrXUgZzM7mFjLUA7PQioAItz80ag8uU4A2wtoT2DzX -m $i &&\n rm $i.minisig\ndone\n```\n\n**Debian Installation**\n\n```bash\nwget https://static.ppom.me/reaction/releases/'"$TAG"'/reaction_'"${TAG:1}"'-1_amd64.deb \\\n https://static.ppom.me/reaction/releases/'"$TAG"'/reaction_'"${TAG:1}"'-1_amd64.deb.minisig\nminisign -VP RWSpLTPfbvllNqRrXUgZzM7mFjLUA7PQioAItz80ag8uU4A2wtoT2DzX -m reaction_'"${TAG:1}"'-1_amd64.deb &&\n rm reaction_'"${TAG:1}"'-1_amd64.deb.minisig &&\n apt install ./reaction_'"${TAG:1}"'-1_amd64.deb\n```", +"description": "'"$DESCRIPTION"'", "assets":{"links":[ {"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/nft46", "name": "nft46 (x86-64)", "link_type": "package"}, {"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction", "name": "reaction (x86-64)", "link_type": "package"}, @@ -25,7 +68,10 @@ DATA='{ {"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/nft46.minisig", "name": "nft46.minisig", "link_type": "other"}, {"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction.minisig", "name": "reaction.minisig", "link_type": "other"}, {"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/ip46tables.minisig", "name": "ip46tables.minisig", "link_type": "other"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction_'"${TAG:1}"'-1_amd64.deb.minisig", "name": "reaction_'"${TAG:1}"'-1_amd64.deb.minisig", "link_type": "other"} +{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction_'"${TAG:1}"'-1_amd64.deb.minisig", "name": "reaction_'"${TAG:1}"'-1_amd64.deb.minisig", "link_type": "other"}, +{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction.bash", "name": "bash shell completion", "link_type": "other"}, +{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction.fish", "name": "fish shell completion", "link_type": "other"}, +{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/_reaction", "name": "zsh shell completion", "link_type": "other"} ]}}' curl \ @@ -37,4 +83,4 @@ curl \ 'https://framagit.org/api/v4/projects/90566/releases' \ --data "$DATA" -sudo make clean +make clean diff --git a/src/main.rs b/src/main.rs index 4e80d8f..9adc208 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,7 +6,7 @@ use reaction::{ client::{request, test_regex}, concepts::Order, daemon::daemon, - utils::cli::{Cli, Command}, + utils::cli::{Cli, SubCommand}, }; use tracing::{error, Level}; @@ -31,7 +31,7 @@ async fn main() { { // Set log level - let level = if let Command::Start { + let level = if let SubCommand::Start { loglevel, config: _, socket: _, @@ -55,24 +55,24 @@ async fn main() { } let result = match cli.command { - Command::Start { + SubCommand::Start { config, loglevel: _, socket, } => daemon(config, socket).await, - Command::Show { + SubCommand::Show { socket, format, limit, patterns, } => request(socket, format, limit, patterns, Order::Show).await, - Command::Flush { + SubCommand::Flush { socket, format, limit, patterns, } => request(socket, format, limit, patterns, Order::Flush).await, - Command::TestRegex { + SubCommand::TestRegex { config, regex, line, diff --git a/src/utils/cli.rs b/src/utils/cli.rs index 79b9008..1e8a8b8 100644 --- a/src/utils/cli.rs +++ b/src/utils/cli.rs @@ -6,8 +6,8 @@ use tracing::Level; #[derive(Parser)] #[clap(version)] -#[command(name = "reaction")] #[command( + name = "reaction", about = "Scan logs and take action", long_about = "A daemon that scans program outputs for repeated patterns, and takes action. Aims at being more versatile and flexible than fail2ban, while being faster and having simpler configuration. @@ -18,11 +18,11 @@ on the wiki: https://reaction.ppom.me )] pub struct Cli { #[clap(subcommand)] - pub command: Command, + pub command: SubCommand, } #[derive(Subcommand)] -pub enum Command { +pub enum SubCommand { /// Start reaction daemon Start { /// configuration file in json, jsonnet or yaml format. required. From 4b8d6e8168c2dbbe7bdd571e0af1a62dd5df0360 Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 24 Oct 2024 12:00:00 +0200 Subject: [PATCH 081/435] Fix db migration glue script --- .gitignore | 2 +- release.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index cd78ecc..14eaadd 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,6 @@ reaction*.export.json *.qcow2 debian-packaging/* *.swp -export-go-db/export-db +export-go-db/export-go-db import-rust-db/target /target diff --git a/release.py b/release.py index 2084daf..d40e0da 100644 --- a/release.py +++ b/release.py @@ -92,7 +92,7 @@ def main(): # Build glue script contents = """ -func main() { +function main() { set -eu -o pipefail DIR="$(mktemp -d)" echo "$GOBIN" | base64 -d > "$DIR/gobin" From 40fc6e338091a03121af580b16bc8607ea047ff1 Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 24 Oct 2024 12:00:00 +0200 Subject: [PATCH 082/435] Fix client-daemon protocol & client output --- src/client/mod.rs | 13 ++++++------- src/concepts/socket_messages.rs | 32 ++++++++++++++------------------ src/daemon/filter.rs | 2 +- src/main.rs | 27 ++++++++++++++++----------- tests/simple.rs | 4 +++- 5 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/client/mod.rs b/src/client/mod.rs index 88d64ee..3236ec6 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -9,12 +9,11 @@ use std::{ use bincode::Options; use futures::{SinkExt, StreamExt}; use regex::Regex; -use tokio::{io::AsyncWriteExt, net::UnixStream}; +use tokio::net::UnixStream; use tokio_util::{ bytes::Bytes, codec::{Framed, LengthDelimitedCodec}, }; -use tracing::info; use crate::{ concepts::{ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern}, @@ -46,10 +45,10 @@ async fn send_retrieve(socket: &PathBuf, req: &ClientRequest) -> Result(&encoded_response) + )) } async fn print_status(cs: ClientStatus, format: Format) -> Result<(), Box> { @@ -57,7 +56,7 @@ async fn print_status(cs: ClientStatus, format: Format) -> Result<(), Box serde_json::to_string_pretty(&cs)?, Format::YAML => serde_yaml::to_string(&cs)?, }; - tokio::io::stdout().write_all(encoded.as_bytes()).await?; + println!("{}", encoded); Ok(()) } @@ -144,7 +143,7 @@ pub fn test_regex( if let Some(line) = line { match_closure(line); } else { - info!("no second argument: reading from stdin"); + eprintln!("no second argument: reading from stdin"); for line in BufReader::new(stdin()).lines() { match line { Ok(line) => match_closure(line), diff --git a/src/concepts/socket_messages.rs b/src/concepts/socket_messages.rs index 64c99c6..2f71125 100644 --- a/src/concepts/socket_messages.rs +++ b/src/concepts/socket_messages.rs @@ -1,8 +1,11 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use super::Match; -use serde::{ser::SerializeStruct, Deserialize, Serialize}; +use serde::{ + ser::{SerializeMap, SerializeStruct}, + Deserialize, Serialize, +}; // We don't need protocol versionning here because // client and daemon are the same binary @@ -13,7 +16,7 @@ pub enum Order { Flush, } -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct ClientRequest { pub order: Order, pub stream_filter: Option, @@ -32,12 +35,6 @@ pub enum DaemonResponse { Err(String), } -#[derive(Clone, Serialize, Deserialize)] -pub struct InfoRes { - pub matches: BTreeMap<(String, String), BTreeMap>>, - pub execs: BTreeMap<(String, String, String), BTreeMap>>, -} - pub type ClientStatus = BTreeMap>>; #[derive(Debug, Default, Deserialize)] @@ -53,24 +50,23 @@ impl Serialize for PatternStatus { { // We only skip serializing emptiness if we're on a human-readable format // This means we're printing for user, not exchanging it over a socket - let state = if serializer.is_human_readable() { + if serializer.is_human_readable() { let ser_matches = self.matches != 0; let ser_actions = !self.actions.is_empty(); - let mut state = serializer - .serialize_struct("PatternStatus", ser_matches as usize + ser_actions as usize)?; + let mut state = + serializer.serialize_map(Some(ser_matches as usize + ser_actions as usize))?; if ser_matches { - state.serialize_field("matches", &self.matches)?; + state.serialize_entry("matches", &self.matches)?; } if ser_actions { - state.serialize_field("actions", &self.actions)?; + state.serialize_entry("actions", &self.actions)?; } - state + state.end() } else { let mut state = serializer.serialize_struct("PatternStatus", 2)?; state.serialize_field("matches", &self.matches)?; state.serialize_field("actions", &self.actions)?; - state - }; - state.end() + state.end() + } } } diff --git a/src/daemon/filter.rs b/src/daemon/filter.rs index 136f444..310f448 100644 --- a/src/daemon/filter.rs +++ b/src/daemon/filter.rs @@ -134,7 +134,7 @@ impl FilterManager { let pattern_status = acc.entry(match_).or_default(); pattern_status .actions - .insert(manager.action().to_string(), times); + .insert(manager.action().name().into(), times); } acc }); diff --git a/src/main.rs b/src/main.rs index 9adc208..97d06fb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,18 +29,19 @@ async fn main() { let cli = Cli::parse(); + let (is_daemon, level) = if let SubCommand::Start { + config: _, + loglevel, + socket: _, + } = cli.command { + (true, loglevel) + } else { + (false, Level::DEBUG) + }; + + if is_daemon { // Set log level - let level = if let SubCommand::Start { - loglevel, - config: _, - socket: _, - } = cli.command - { - loglevel - } else { - Level::DEBUG - }; if let Err(err) = tracing_subscriber::fmt::fmt() .without_time() .with_target(false) @@ -83,7 +84,11 @@ async fn main() { exit(0); } Err(err) => { - error!("{err}"); + if is_daemon { + error!("{err}"); + } else { + eprintln!("ERROR {err}"); + } exit(1); } } diff --git a/tests/simple.rs b/tests/simple.rs index 7fb2370..056811c 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -123,8 +123,10 @@ async fn simple() { .await }); - let (daemon_exit, _, _) = tokio::join!(handle, handle2, handle3); + let (daemon_exit, flush1, flush2) = tokio::join!(handle, handle2, handle3); assert!(daemon_exit.is_ok()); + assert!(flush1.is_ok()); + assert!(flush2.is_ok()); assert_eq!( // 24 is encountered for the second time, then From 8080bae293437276e20c43bd8ec0bee0ebda9b9f Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 24 Oct 2024 12:00:00 +0200 Subject: [PATCH 083/435] deactivate PatternStatus special formatting --- src/concepts/socket_messages.rs | 58 ++++++++++++++++----------------- src/daemon/socket.rs | 2 +- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/concepts/socket_messages.rs b/src/concepts/socket_messages.rs index 2f71125..5f00d79 100644 --- a/src/concepts/socket_messages.rs +++ b/src/concepts/socket_messages.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use super::Match; use serde::{ - ser::{SerializeMap, SerializeStruct}, + // ser::{SerializeMap, SerializeStruct}, Deserialize, Serialize, }; @@ -37,36 +37,36 @@ pub enum DaemonResponse { pub type ClientStatus = BTreeMap>>; -#[derive(Debug, Default, Deserialize)] +#[derive(Debug, Default, Deserialize, Serialize)] pub struct PatternStatus { pub matches: usize, pub actions: BTreeMap>, } -impl Serialize for PatternStatus { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - // We only skip serializing emptiness if we're on a human-readable format - // This means we're printing for user, not exchanging it over a socket - if serializer.is_human_readable() { - let ser_matches = self.matches != 0; - let ser_actions = !self.actions.is_empty(); - let mut state = - serializer.serialize_map(Some(ser_matches as usize + ser_actions as usize))?; - if ser_matches { - state.serialize_entry("matches", &self.matches)?; - } - if ser_actions { - state.serialize_entry("actions", &self.actions)?; - } - state.end() - } else { - let mut state = serializer.serialize_struct("PatternStatus", 2)?; - state.serialize_field("matches", &self.matches)?; - state.serialize_field("actions", &self.actions)?; - state.end() - } - } -} +// impl Serialize for PatternStatus { +// fn serialize(&self, serializer: S) -> Result +// where +// S: serde::Serializer, +// { +// // We only skip serializing emptiness if we're on a human-readable format +// // This means we're printing for user, not exchanging it over a socket +// if serializer.is_human_readable() { +// let ser_matches = self.matches != 0; +// let ser_actions = !self.actions.is_empty(); +// let mut state = +// serializer.serialize_map(Some(ser_matches as usize + ser_actions as usize))?; +// if ser_matches { +// state.serialize_entry("matches", &self.matches)?; +// } +// if ser_actions { +// state.serialize_entry("actions", &self.actions)?; +// } +// state.end() +// } else { +// let mut state = serializer.serialize_struct("PatternStatus", 2)?; +// state.serialize_field("matches", &self.matches)?; +// state.serialize_field("actions", &self.actions)?; +// state.end() +// } +// } +// } diff --git a/src/daemon/socket.rs b/src/daemon/socket.rs index 269b38e..599f530 100644 --- a/src/daemon/socket.rs +++ b/src/daemon/socket.rs @@ -183,7 +183,7 @@ pub async fn socket_manager( }; // Encode let encoded_response = - or_next!("failed to serialize response", bin.serialize(&response)); + or_next!("failed to serialize response", bin.serialize::(&response)); or_next!( "failed to send response:", transport.send(Bytes::from(encoded_response)).await From e39ba05da46f828083f8e43ca2f2ca8f9110a9d6 Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 24 Oct 2024 12:00:00 +0200 Subject: [PATCH 084/435] Update project status --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 130a919..fc61821 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,10 @@ A common usage is to scan ssh and webserver logs, and to ban hosts that cause mu ## Current project status -Hey, it may look like nothing is happening here... That's because reaction is actively being rewritten in rust, -which happens in the [rust](https://framagit.org/ppom/reaction/-/tree/rust) branch! +reaction just reached v2.0.0-rc1 version, which is a complete rust rewrite of reaction. +It's in feature parity with the Go version, and breaking changes should be small. -I'll soon™️ publish a v2.0.0-rc1 version, which will be the rust rewrite of reaction. -It will be in feature parity with the Go version, and breaking changes should be small. -More information on this when the release lands! +See https://reaction.ppom.me/migrate-to-v2.html ## Rationale From d776667c80774ac2267245d6742315a8ebcd5eb5 Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 24 Oct 2024 12:00:00 +0200 Subject: [PATCH 085/435] release script update --- release.py | 19 ++++++------ release.sh | 86 ------------------------------------------------------ 2 files changed, 9 insertions(+), 96 deletions(-) delete mode 100755 release.sh diff --git a/release.py b/release.py index d40e0da..4ac3d23 100644 --- a/release.py +++ b/release.py @@ -1,6 +1,7 @@ #!/usr/bin/env nix-shell #!nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests ])" -p debian-devscripts git minisign cargo-cross import base64 +import http.client import json import os import subprocess @@ -32,13 +33,13 @@ def main(): sys.exit(1) # Ask user - # if input(f"We will create a release for tag {tag}. Do you want to continue? (y/n) ") != "y": - # print("exiting.") - # sys.exit(1) + if input(f"We will create a release for tag {tag}. Do you want to continue? (y/n) ") != "y": + print("exiting.") + sys.exit(1) # Git push - # cmd = subprocess.run(["git", "push", "--tags"]) - # quit_if(cmd) + cmd = subprocess.run(["git", "push", "--tags"]) + quit_if(cmd) cmd = subprocess.run(["rbw", "get", "minisign"], capture_output=True, text=True) quit_if(cmd) @@ -256,6 +257,7 @@ wget https://static.ppom.me/reaction/releases/{tag}/reaction_{tag}-1_amd64.deb \ print("User deleted emptied description, exiting.") sys.exit(1) + # Construct JSON payload files = [ file for file in all_files if file[2] != False ] data = { "tag_name": tag, @@ -263,7 +265,7 @@ wget https://static.ppom.me/reaction/releases/{tag}/reaction_{tag}-1_amd64.deb \ "assets": { "links": [ { - "url": f"https://static.ppom.me/reaction/releases/{tag}/{file[1]}/{os.path.basename(file[0])}".replace("//", "/"), + "url": "https://" + f"static.ppom.me/reaction/releases/{tag}/{file[1]}/{os.path.basename(file[0])}".replace("//", "/"), "name": file[2], "link_type": file[3], } @@ -275,16 +277,13 @@ wget https://static.ppom.me/reaction/releases/{tag}/reaction_{tag}-1_amd64.deb \ body = json.dumps(data) - subprocess.run(["jq"], text=True, input=body) - + # Send POST request headers = { "Host": "framagit.org", "Content-Type": "application/json", "PRIVATE-TOKEN": token, } - sys.exit(1) - conn = http.client.HTTPSConnection("framagit.org") conn.request("POST", "/api/v4/projects/90566/releases", body=body, headers=headers) response = conn.getresponse() diff --git a/release.sh b/release.sh deleted file mode 100755 index 56400c8..0000000 --- a/release.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -set -exu - -git push --tags - -TAG="$(git tag --sort=v:refname | tail -n1)" - -echo -n "We will create a release for tag $TAG. Do you want to continue? (y/n) " - -docker run -it --rm \ - -e HOME=/tmp/ \ - -v "$(pwd)":/tmp/code -w /tmp/code \ - -u "$(id -u)" \ - rust \ - make reaction_"${TAG:1}"-1_amd64.deb reaction ip46tables nft46 - -make "signatures_${TAG:1}" - -rsync -avz -e 'ssh -J pica01' \ - ./ip46tables ./ip46tables.minisig \ - ./nft46 ./nft46.minisig \ - ./target/release/reaction ./reaction.minisig \ - ./reaction_"${TAG:1}"-1_amd64.deb ./reaction_"${TAG:1}"-1_amd64.deb.minisig \ - ./target/release/reaction.bash ./target/release/reaction.fish ./target/release/_reaction \ - ./target/release/reaction*.1 - akesi:/var/www/static/reaction/releases/"$TAG" - -TOKEN="$(rbw get framagit.org token)" - -DESCRIPTION=" -## Changes - -## Download - -\`\`\`bash -wget https://static.ppom.me/reaction/releases/""$TAG""/nft46 \\ - https://static.ppom.me/reaction/releases/""$TAG""/reaction \\ - https://static.ppom.me/reaction/releases/""$TAG""/ip46tables \\ - https://static.ppom.me/reaction/releases/""$TAG""/nft46.minisig \\ - https://static.ppom.me/reaction/releases/""$TAG""/reaction.minisig \\ - https://static.ppom.me/reaction/releases/""$TAG""/ip46tables.minisig -for i in nft46 ip46tables reaction -do - minisign -VP RWSpLTPfbvllNqRrXUgZzM7mFjLUA7PQioAItz80ag8uU4A2wtoT2DzX -m $i && rm $i.minisig -done -\`\`\` - -**Debian Installation** - -\`\`\`bash -wget https://static.ppom.me/reaction/releases/""$TAG""/reaction_""${TAG:1}""-1_amd64.deb \\ -https://static.ppom.me/reaction/releases/""$TAG""/reaction_""${TAG:1}""-1_amd64.deb.minisig -minisign -VP RWSpLTPfbvllNqRrXUgZzM7mFjLUA7PQioAItz80ag8uU4A2wtoT2DzX -m reaction_""${TAG:1}""-1_amd64.deb && - rm reaction_""${TAG:1}""-1_amd64.deb.minisig && - apt install ./reaction_""${TAG:1}""-1_amd64.deb -\`\`\` -" - -DATA='{ -"tag_name":"'"$TAG"'", -"description": "'"$DESCRIPTION"'", -"assets":{"links":[ -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/nft46", "name": "nft46 (x86-64)", "link_type": "package"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction", "name": "reaction (x86-64)", "link_type": "package"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/ip46tables", "name": "ip46tables (x86-64)", "link_type": "package"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction_'"${TAG:1}"'-1_amd64.deb", "name": "reaction_'"${TAG:1}"'-1_amd64.deb (x86-64)", "link_type": "package"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/nft46.minisig", "name": "nft46.minisig", "link_type": "other"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction.minisig", "name": "reaction.minisig", "link_type": "other"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/ip46tables.minisig", "name": "ip46tables.minisig", "link_type": "other"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction_'"${TAG:1}"'-1_amd64.deb.minisig", "name": "reaction_'"${TAG:1}"'-1_amd64.deb.minisig", "link_type": "other"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction.bash", "name": "bash shell completion", "link_type": "other"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/reaction.fish", "name": "fish shell completion", "link_type": "other"}, -{"url": "https://static.ppom.me/reaction/releases/'"$TAG"'/_reaction", "name": "zsh shell completion", "link_type": "other"} -]}}' - -curl \ - --fail-with-body \ - --location \ - -X POST \ - -H 'Content-Type: application/json' \ - -H "PRIVATE-TOKEN: $TOKEN" \ - 'https://framagit.org/api/v4/projects/90566/releases' \ - --data "$DATA" - -make clean From 838ad1b18a308994b9f8bc003dc86345f870557e Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 26 Oct 2024 12:00:00 +0200 Subject: [PATCH 086/435] less owned data, more borrowed data --- import-rust-db/src/main.rs | 4 ++-- src/client/mod.rs | 4 ++-- src/concepts/pattern.rs | 28 ++++++++++++++-------------- src/daemon/action.rs | 26 +++++++++++++------------- src/daemon/database/lowlevel.rs | 6 +++--- src/daemon/socket.rs | 6 +++--- 6 files changed, 37 insertions(+), 37 deletions(-) diff --git a/import-rust-db/src/main.rs b/import-rust-db/src/main.rs index 0918ed1..0e587c2 100644 --- a/import-rust-db/src/main.rs +++ b/import-rust-db/src/main.rs @@ -52,7 +52,7 @@ fn lil_main() -> Result<(), E> { Ok(()) } -fn import(json_path: String, write_path: String) -> Result<(), E> { +fn import(json_path: &str, write_path: &str) -> Result<(), E> { let json_file = BufReader::new(File::open(json_path.clone())?); let mut write_file = BufWriter::new(File::create(write_path)?); let bin = bincode_options(); @@ -81,7 +81,7 @@ fn import(json_path: String, write_path: String) -> Result<(), E> { Ok(()) } -fn collect_stream_filters(json_path: String) -> Result { +fn collect_stream_filters(json_path: &str) -> Result { let mut header = BTreeMap::new(); let mut count = 0; let json_file = BufReader::new(File::open(json_path)?); diff --git a/src/client/mod.rs b/src/client/mod.rs index 3236ec6..8a1b61a 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -2,7 +2,7 @@ use std::{ collections::BTreeSet, error::Error, io::{stdin, BufRead, BufReader}, - path::PathBuf, + path::{Path, PathBuf}, sync::Arc, }; @@ -26,7 +26,7 @@ macro_rules! or_quit { }; } -async fn send_retrieve(socket: &PathBuf, req: &ClientRequest) -> Result { +async fn send_retrieve(socket: &Path, req: &ClientRequest) -> Result { let bin = bincode_options(); let conn = or_quit!( "opening connection to daemon", diff --git a/src/concepts/pattern.rs b/src/concepts/pattern.rs index 2d3e089..eacd49a 100644 --- a/src/concepts/pattern.rs +++ b/src/concepts/pattern.rs @@ -23,7 +23,7 @@ pub struct Pattern { } impl Pattern { - pub fn setup(&mut self, name: &String) -> Result<(), String> { + pub fn setup(&mut self, name: &str) -> Result<(), String> { self._setup(name) .map_err(|msg| format!("pattern {}: {}", name, msg)) } @@ -35,8 +35,8 @@ impl Pattern { &self.name_with_braces } - pub fn _setup(&mut self, name: &String) -> Result<(), String> { - self.name = name.clone(); + pub fn _setup(&mut self, name: &str) -> Result<(), String> { + self.name = name.to_string(); self.name_with_braces = format!("<{}>", name); if self.name.is_empty() { @@ -151,16 +151,16 @@ pub mod tests { // Empty name pattern = default_pattern(); pattern.regex = "abc".into(); - assert!(pattern.setup(&"".into()).is_err()); + assert!(pattern.setup("").is_err()); // '.' in name pattern = default_pattern(); pattern.regex = "abc".into(); - assert!(pattern.setup(&"na.me".into()).is_err()); + assert!(pattern.setup("na.me").is_err()); // Empty regex pattern = default_pattern(); - assert!(pattern.setup(&"name".into()).is_err()); + assert!(pattern.setup("name").is_err()); } #[test] @@ -169,17 +169,17 @@ pub mod tests { // regex ok pattern = ok_pattern(); - assert!(pattern.setup(&"name".into()).is_ok()); + assert!(pattern.setup("name").is_ok()); // regex ok pattern = default_pattern(); pattern.regex = "abc".into(); - assert!(pattern.setup(&"name".into()).is_ok()); + assert!(pattern.setup("name").is_ok()); // regex ko pattern = default_pattern(); pattern.regex = "[abc".into(); - assert!(pattern.setup(&"name".into()).is_err()); + assert!(pattern.setup("name").is_err()); } #[test] @@ -191,13 +191,13 @@ pub mod tests { pattern.regex = "[abc]".into(); pattern.ignore.push("a".into()); pattern.ignore.push("b".into()); - assert!(pattern.setup(&"name".into()).is_ok()); + assert!(pattern.setup("name").is_ok()); // ignore ko pattern = default_pattern(); pattern.regex = "[abc]".into(); pattern.ignore.push("d".into()); - assert!(pattern.setup(&"name".into()).is_err()); + assert!(pattern.setup("name").is_err()); } #[test] @@ -209,13 +209,13 @@ pub mod tests { pattern.regex = "[abc]".into(); pattern.ignore_regex.push("[a]".into()); pattern.ignore_regex.push("a".into()); - assert!(pattern.setup(&"name".into()).is_ok()); + assert!(pattern.setup("name").is_ok()); // ignore_regex ko pattern = default_pattern(); pattern.regex = "[abc]".into(); pattern.ignore.push("[a".into()); - assert!(pattern.setup(&"name".into()).is_err()); + assert!(pattern.setup("name").is_err()); } #[test] @@ -230,7 +230,7 @@ pub mod tests { pattern.ignore_regex.push("c".into()); pattern.ignore_regex.push("[de]".into()); - pattern.setup(&"name".into()).unwrap(); + pattern.setup("name").unwrap(); assert!(!pattern.not_an_ignore("a")); assert!(!pattern.not_an_ignore("b")); assert!(!pattern.not_an_ignore("c")); diff --git a/src/daemon/action.rs b/src/daemon/action.rs index 44742e8..db36557 100644 --- a/src/daemon/action.rs +++ b/src/daemon/action.rs @@ -16,24 +16,24 @@ struct State { } impl State { - fn add_match(&mut self, m: &Match, t: Time) { - self.pending.entry(m.clone()).or_default().insert(t); - self.ordered_times.insert(t, m.clone()); + fn add_match(&mut self, m: &Match, t: &Time) { + self.pending.entry(m.clone()).or_default().insert(*t); + self.ordered_times.insert(*t, m.clone()); } - fn remove(&mut self, m: Match, t: Time) -> bool { - self.pending.entry(m).and_modify(|times| { - times.remove(&t); + fn remove(&mut self, m: &Match, t: &Time) -> bool { + self.pending.entry(m.clone()).and_modify(|times| { + times.remove(t); }); - self.ordered_times.remove(&t).is_some() + self.ordered_times.remove(t).is_some() } - fn clear_past_times(&mut self, now: Time, after: Option) { + fn clear_past_times(&mut self, now: &Time, after: Option) { let after = after.unwrap_or_default(); while self .ordered_times .first_key_value() - .is_some_and(|(k, _)| *k + after < now) + .is_some_and(|(k, _)| *k + after < *now) { #[allow(clippy::unwrap_used)] // we just checked in the condition that first is_some let (_, m) = self.ordered_times.pop_first().unwrap(); @@ -81,8 +81,8 @@ impl ActionManager { { #[allow(clippy::unwrap_used)] // propagating panics is ok let mut state = self.state.lock().unwrap(); - state.clear_past_times(t, self.action.after_duration()); - state.add_match(&m, exec_t); + state.clear_past_times(&t, self.action.after_duration()); + state.add_match(&m, &exec_t); } let this = self.clone(); tokio::spawn(async move { @@ -90,7 +90,7 @@ impl ActionManager { tokio::time::sleep(dur).await; #[allow(clippy::unwrap_used)] // propagating panics is ok let mut state = this.state.lock().unwrap(); - if state.remove(m.clone(), t) { + if state.remove(&m, &t) { this.exec_now(m); } }); @@ -115,7 +115,7 @@ impl ActionManager { .iter() .map(|time| { if let Order::Flush = order { - if state.remove(match_.clone(), *time) { + if state.remove(&match_, time) { self.exec_now(match_.clone()); } } diff --git a/src/daemon/database/lowlevel.rs b/src/daemon/database/lowlevel.rs index a16a1b1..7d0536c 100644 --- a/src/daemon/database/lowlevel.rs +++ b/src/daemon/database/lowlevel.rs @@ -165,11 +165,11 @@ impl WriteDB { pub fn write(&mut self, entry: LogEntry) -> Result<(), DBError> { let computed = ComputedLogEntry::from(entry, &self.h)?; - self._write(computed) + self._write(&computed) } - fn _write(&mut self, data: T) -> Result<(), DBError> { - let encoded = self.bin.serialize(&data)?; + fn _write(&mut self, data: &T) -> Result<(), DBError> { + let encoded = self.bin.serialize(data)?; // debug!("writing this: {:?}, {:?}", &data, &encoded); self.f.write_all(&encoded)?; Ok(()) diff --git a/src/daemon/socket.rs b/src/daemon/socket.rs index 599f530..64935b4 100644 --- a/src/daemon/socket.rs +++ b/src/daemon/socket.rs @@ -65,17 +65,17 @@ async fn answer_order( // Compute the Vec<(pattern_name: String, regex: String)> into a BTreeMap, Regex> let patterns = options .patterns - .iter() + .into_iter() .map(|(name, reg)| { // lookup pattern in config.patterns config .patterns() .iter() // retrieve or Err - .find(|(pattern_name, _)| name == *pattern_name) + .find(|(pattern_name, _)| &name == *pattern_name) .ok_or_else(|| format!("pattern '{name}' doesn't exist")) // compile Regex or Err - .and_then(|(_, pattern)| match Regex::new(reg) { + .and_then(|(_, pattern)| match Regex::new(®) { Ok(reg) => Ok((pattern.clone(), reg)), Err(err) => Err(format!("pattern '{name}' regex doesn't compile: {err}")), }) From 79677cf327de5f23f171f0b95f02f21ab9c495f4 Mon Sep 17 00:00:00 2001 From: ppom Date: Sat, 26 Oct 2024 12:00:00 +0200 Subject: [PATCH 087/435] Restructure code and document it in ARCHITECTURE.md --- ARCHITECTURE.md | 71 ++++++++ README.md | 4 + build.rs | 2 +- src/{utils => }/cli.rs | 0 src/client/mod.rs | 158 +----------------- src/client/show_flush.rs | 80 +++++++++ src/client/test_regex.rs | 78 +++++++++ src/concepts/action.rs | 2 +- src/concepts/filter.rs | 2 +- src/concepts/mod.rs | 4 +- src/{utils => concepts}/parse_duration.rs | 0 src/daemon/action.rs | 2 +- src/daemon/database/lowlevel.rs | 2 +- src/daemon/filter.rs | 5 +- src/daemon/socket.rs | 4 +- src/lib.rs | 3 +- src/main.rs | 5 +- .../messages.rs} | 4 +- src/protocol/mod.rs | 5 + .../mod.rs => protocol/serialization.rs} | 4 - tests/simple.rs | 2 +- 21 files changed, 262 insertions(+), 175 deletions(-) create mode 100644 ARCHITECTURE.md rename src/{utils => }/cli.rs (100%) create mode 100644 src/client/show_flush.rs create mode 100644 src/client/test_regex.rs rename src/{utils => concepts}/parse_duration.rs (100%) rename src/{concepts/socket_messages.rs => protocol/messages.rs} (98%) create mode 100644 src/protocol/mod.rs rename src/{utils/mod.rs => protocol/serialization.rs} (95%) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..b010403 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,71 @@ +# Architecture + +Here is a high-level overview of the codebase. + +*Don't hesitate to create an issue or a merge request if something is unclear, missing or outdated.* + +## Build + +- `build.rs`: permits to create shell completions and man pages on build. +- `Cargo.toml`, `Cargo.lock`: manifest and dependencies. +- `config`: example / test configuration files. Look at the git history to discover more! +- `debian`: reaction.deb generation. +- `Makefile`: Makefile. I plan to remove this at some point. +- `release.py`: Build process for a release. I'd like to make it more modular, to permit to build specific parts only for example. + +## Main source code + +- `helpers_c`: C helpers. I wish to have special IP support in reaction and get rid of them. +- `tests`: Integration tests. For now they test basic reaction runtime behavior, persistance, and client-daemon communication. +- `src`: The source code, here we go! + +### Top-level files + +- `src/main.rs`: Main entrypoint +- `src/lib.rs`: Second main entrypoint +- `src/cli.rs`: Command-line arguments +- `src/tests.rs`: Test utilities + +### `src/concepts` + +reaction really is about its configuration, which is at the center of the code. + +There is one file for each of its concepts: configuration, streams, filters, actions, patterns. + +### `src/protocol` + +Low-level serialization/deserialization and client-daemon protocol messages. + +Shared by the client and daemon's socket. Also used by daemon's database. + +### `src/client` + +Client code: `reaction show`, `reaction flush`, `reaction test-regex`. + +- `show_flush.rs`: `show` & `flush` commands. +- `test_regex.rs`: `test-regex` command. + +### `src/daemon` + +Daemon runtime structures and logic. + +This code is mainly async, with the tokio runtime. + +- `mod.rs`: daemon main function. Initializes all tasks, handles synchronization and quitting, etc. +- `stream.rs`: Stream managers: start the stream `cmd` and dispatch its stdout lines to its Filter managers. +- `filter.rs`: Filter managers: handle lines, store matches, send logs to database and decide when to trigger actions. +- `action.rs`: Action managers: handle action triggers (*execs*), store & manage pending actions. +- `socket.rs`: The socket task, responsible for communication with clients. +- `database`: The database thread. This is a sync thread, because it's somehow muuch faster. At startup it sends persisted matches to the Filter managers. Then it receives match/exec logs from the filters and persist them. + - `database/mod.rs`: Main logic. + - `database/lowlevel.rs`: Low-level implementation details (serialization / deserialization and size optimizations). + - `database/tests.rs`: Unit tests. + +## Migration from Go to Rust + +- `go.old/`: Go / v1 codebase. + +Those scripts are merged in a single-file executable by `release.py`: + +- `export-go-db/`: Go script to export the reaction-v1 database as JSON. +- `import-rust-db/`: Rust script to import the JSON export as a reaction-v2 database. diff --git a/README.md b/README.md index fc61821..0028631 100644 --- a/README.md +++ b/README.md @@ -227,5 +227,9 @@ make install_systemd Contributions are welcome. For any substantial feature, please file an issue first, to be assured that we agree on the feature, and to avoid unnecessary work. +I recommend reading [`ARCHITECTURE.md`](ARCHITECTURE.md) first. This is a tour of the codebase, which should save time to potential contributors. + +## Funding + This is a free time project, so I'm not working on schedule. However, if you're willing to fund the project, I can priorise and plan paid work. This includes features, documentation and specific JSONnet configurations. diff --git a/build.rs b/build.rs index f8ce4db..3c9ebe9 100644 --- a/build.rs +++ b/build.rs @@ -8,7 +8,7 @@ use std::{ use clap_complete::shells; // SubCommand defined here -include!("src/utils/cli.rs"); +include!("src/cli.rs"); fn compile_helper(name: &str, out_dir: &Path) -> io::Result<()> { process::Command::new("gcc") diff --git a/src/utils/cli.rs b/src/cli.rs similarity index 100% rename from src/utils/cli.rs rename to src/cli.rs diff --git a/src/client/mod.rs b/src/client/mod.rs index 8a1b61a..8565a55 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,155 +1,5 @@ -use std::{ - collections::BTreeSet, - error::Error, - io::{stdin, BufRead, BufReader}, - path::{Path, PathBuf}, - sync::Arc, -}; +mod show_flush; +mod test_regex; -use bincode::Options; -use futures::{SinkExt, StreamExt}; -use regex::Regex; -use tokio::net::UnixStream; -use tokio_util::{ - bytes::Bytes, - codec::{Framed, LengthDelimitedCodec}, -}; - -use crate::{ - concepts::{ClientRequest, ClientStatus, Config, DaemonResponse, Order, Pattern}, - utils::{bincode_options, cli::Format}, -}; - -macro_rules! or_quit { - ($msg:expr, $expression:expr) => { - $expression.map_err(|err| format!("failed to communicate to daemon: {}, {}", $msg, err))? - }; -} - -async fn send_retrieve(socket: &Path, req: &ClientRequest) -> Result { - let bin = bincode_options(); - let conn = or_quit!( - "opening connection to daemon", - UnixStream::connect(socket).await - ); - // Encode - let mut transport = Framed::new(conn, LengthDelimitedCodec::new()); - let encoded_request = or_quit!("failed to encode request", bin.serialize(req)); - or_quit!( - "failed to send request", - transport.send(Bytes::from(encoded_request)).await - ); - // Decode - let encoded_response = or_quit!( - "failed to read response", - transport.next().await.ok_or("empty response from server") - ); - let encoded_response = or_quit!("failed to decode response", encoded_response); - Ok(or_quit!( - "failed to decode response", - bin.deserialize::(&encoded_response) - )) -} - -async fn print_status(cs: ClientStatus, format: Format) -> Result<(), Box> { - let encoded = match format { - Format::JSON => serde_json::to_string_pretty(&cs)?, - Format::YAML => serde_yaml::to_string(&cs)?, - }; - println!("{}", encoded); - Ok(()) -} - -pub async fn request( - socket: PathBuf, - format: Format, - stream_filter: Option, - patterns: Vec<(String, String)>, - order: Order, -) -> Result<(), Box> { - let response = send_retrieve( - &socket, - &ClientRequest { - order, - stream_filter, - patterns, - }, - ) - .await; - match response? { - DaemonResponse::Order(cs) => print_status(cs, format) - .await - .map_err(|err| format!("while printing response: {err}")), - DaemonResponse::Err(err) => Err(format!( - "failed to communicate to daemon: error response: {err}" - )), - }?; - Ok(()) -} - -pub fn test_regex( - config_path: PathBuf, - mut regex: String, - line: Option, -) -> Result<(), Box> { - let config: Config = Config::from_file(&config_path)?; - - // Code close to Filter::setup() - let mut used_patterns: BTreeSet> = BTreeSet::new(); - for pattern in config.patterns().values() { - if let Some(index) = regex.find(pattern.name_with_braces()) { - // we already `find` it, so we must be able to `rfind` it - #[allow(clippy::unwrap_used)] - if regex.rfind(pattern.name_with_braces()).unwrap() != index { - return Err(format!( - "pattern {} present multiple times in regex", - pattern.name_with_braces() - ) - .into()); - } - used_patterns.insert(pattern.clone()); - } - regex = regex.replacen(pattern.name_with_braces(), &pattern.regex, 1); - } - - let compiled = Regex::new(®ex).map_err(|err| format!("regex doesn't compile: {err}"))?; - - let match_closure = |line: String| { - let mut ignored = false; - if let Some(matches) = compiled.captures(&line) { - let mut result = Vec::new(); - if !used_patterns.is_empty() { - for pattern in used_patterns.iter() { - if let Some(match_) = matches.name(pattern.name()) { - result.push(match_.as_str().to_string()); - if !pattern.not_an_ignore(match_.as_str()) { - ignored = true; - } - } - } - if !ignored { - println!("\x1b[32mmatching\x1b[0m {result:?}: {line}"); - } else { - println!("\x1b[33mignore matching\x1b[0m {result:?}: {line}"); - } - } else { - println!("\x1b[32mmatching\x1b[0m: {line}"); - } - } else { - println!("\x1b[31mno match\x1b[0m: {line}"); - } - }; - - if let Some(line) = line { - match_closure(line); - } else { - eprintln!("no second argument: reading from stdin"); - for line in BufReader::new(stdin()).lines() { - match line { - Ok(line) => match_closure(line), - Err(_) => break, - }; - } - } - Ok(()) -} +pub use show_flush::request; +pub use test_regex::test_regex; diff --git a/src/client/show_flush.rs b/src/client/show_flush.rs new file mode 100644 index 0000000..73e0591 --- /dev/null +++ b/src/client/show_flush.rs @@ -0,0 +1,80 @@ +use std::{ + error::Error, + path::{Path, PathBuf}, +}; + +use bincode::Options; +use futures::{SinkExt, StreamExt}; +use tokio::net::UnixStream; +use tokio_util::{ + bytes::Bytes, + codec::{Framed, LengthDelimitedCodec}, +}; + +use crate::{cli::Format, protocol::{bincode_options, ClientRequest, ClientStatus, DaemonResponse, Order}}; + +macro_rules! or_quit { + ($msg:expr, $expression:expr) => { + $expression.map_err(|err| format!("failed to communicate to daemon: {}, {}", $msg, err))? + }; +} + +async fn send_retrieve(socket: &Path, req: &ClientRequest) -> Result { + let bin = bincode_options(); + let conn = or_quit!( + "opening connection to daemon", + UnixStream::connect(socket).await + ); + // Encode + let mut transport = Framed::new(conn, LengthDelimitedCodec::new()); + let encoded_request = or_quit!("failed to encode request", bin.serialize(req)); + or_quit!( + "failed to send request", + transport.send(Bytes::from(encoded_request)).await + ); + // Decode + let encoded_response = or_quit!( + "failed to read response", + transport.next().await.ok_or("empty response from server") + ); + let encoded_response = or_quit!("failed to decode response", encoded_response); + Ok(or_quit!( + "failed to decode response", + bin.deserialize::(&encoded_response) + )) +} + +fn print_status(cs: ClientStatus, format: Format) -> Result<(), Box> { + let encoded = match format { + Format::JSON => serde_json::to_string_pretty(&cs)?, + Format::YAML => serde_yaml::to_string(&cs)?, + }; + println!("{}", encoded); + Ok(()) +} + +pub async fn request( + socket: PathBuf, + format: Format, + stream_filter: Option, + patterns: Vec<(String, String)>, + order: Order, +) -> Result<(), Box> { + let response = send_retrieve( + &socket, + &ClientRequest { + order, + stream_filter, + patterns, + }, + ) + .await; + match response? { + DaemonResponse::Order(cs) => print_status(cs, format) + .map_err(|err| format!("while printing response: {err}")), + DaemonResponse::Err(err) => Err(format!( + "failed to communicate to daemon: error response: {err}" + )), + }?; + Ok(()) +} diff --git a/src/client/test_regex.rs b/src/client/test_regex.rs new file mode 100644 index 0000000..3197106 --- /dev/null +++ b/src/client/test_regex.rs @@ -0,0 +1,78 @@ +use std::{ + collections::BTreeSet, + error::Error, + io::{stdin, BufRead, BufReader}, + path::PathBuf, + sync::Arc, +}; + +use regex::Regex; + +use crate::concepts::{Config, Pattern}; + +pub fn test_regex( + config_path: PathBuf, + mut regex: String, + line: Option, +) -> Result<(), Box> { + let config: Config = Config::from_file(&config_path)?; + + // Code close to Filter::setup() + let mut used_patterns: BTreeSet> = BTreeSet::new(); + for pattern in config.patterns().values() { + if let Some(index) = regex.find(pattern.name_with_braces()) { + // we already `find` it, so we must be able to `rfind` it + #[allow(clippy::unwrap_used)] + if regex.rfind(pattern.name_with_braces()).unwrap() != index { + return Err(format!( + "pattern {} present multiple times in regex", + pattern.name_with_braces() + ) + .into()); + } + used_patterns.insert(pattern.clone()); + } + regex = regex.replacen(pattern.name_with_braces(), &pattern.regex, 1); + } + + let compiled = Regex::new(®ex).map_err(|err| format!("regex doesn't compile: {err}"))?; + + let match_closure = |line: String| { + let mut ignored = false; + if let Some(matches) = compiled.captures(&line) { + let mut result = Vec::new(); + if !used_patterns.is_empty() { + for pattern in used_patterns.iter() { + if let Some(match_) = matches.name(pattern.name()) { + result.push(match_.as_str().to_string()); + if !pattern.not_an_ignore(match_.as_str()) { + ignored = true; + } + } + } + if !ignored { + println!("\x1b[32mmatching\x1b[0m {result:?}: {line}"); + } else { + println!("\x1b[33mignore matching\x1b[0m {result:?}: {line}"); + } + } else { + println!("\x1b[32mmatching\x1b[0m: {line}"); + } + } else { + println!("\x1b[31mno match\x1b[0m: {line}"); + } + }; + + if let Some(line) = line { + match_closure(line); + } else { + eprintln!("no second argument: reading from stdin"); + for line in BufReader::new(stdin()).lines() { + match line { + Ok(line) => match_closure(line), + Err(_) => break, + }; + } + } + Ok(()) +} diff --git a/src/concepts/action.rs b/src/concepts/action.rs index f5378ff..2dc18eb 100644 --- a/src/concepts/action.rs +++ b/src/concepts/action.rs @@ -6,7 +6,7 @@ use serde::Deserialize; use tokio::process::Command; use super::{Match, Pattern}; -use crate::utils::parse_duration; +use super::parse_duration; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/src/concepts/filter.rs b/src/concepts/filter.rs index d895b53..be8d911 100644 --- a/src/concepts/filter.rs +++ b/src/concepts/filter.rs @@ -16,7 +16,7 @@ use super::{ messages::{Match, Time, MAT}, Action, LogEntry, Pattern, Patterns, }; -use crate::utils::parse_duration; +use super::parse_duration; // Only names are serialized // Only computed fields are not deserialized diff --git a/src/concepts/mod.rs b/src/concepts/mod.rs index 3e25648..4c5850e 100644 --- a/src/concepts/mod.rs +++ b/src/concepts/mod.rs @@ -2,8 +2,8 @@ mod action; mod config; mod filter; mod messages; +mod parse_duration; mod pattern; -mod socket_messages; mod stream; pub use action::Action; @@ -11,5 +11,5 @@ pub use config::{Config, Patterns}; pub use filter::Filter; pub use messages::*; pub use pattern::Pattern; -pub use socket_messages::*; pub use stream::Stream; +pub use parse_duration::parse_duration; diff --git a/src/utils/parse_duration.rs b/src/concepts/parse_duration.rs similarity index 100% rename from src/utils/parse_duration.rs rename to src/concepts/parse_duration.rs diff --git a/src/daemon/action.rs b/src/daemon/action.rs index db36557..5b9a746 100644 --- a/src/daemon/action.rs +++ b/src/daemon/action.rs @@ -8,7 +8,7 @@ use chrono::{Local, TimeDelta}; use tokio::sync::Semaphore; use tracing::{error, info}; -use crate::concepts::{Action, Match, Order, Time}; +use crate::{concepts::{Action, Match, Time}, protocol::Order}; struct State { pending: BTreeMap>, diff --git a/src/daemon/database/lowlevel.rs b/src/daemon/database/lowlevel.rs index 7d0536c..7d6284c 100644 --- a/src/daemon/database/lowlevel.rs +++ b/src/daemon/database/lowlevel.rs @@ -13,7 +13,7 @@ use tracing::{debug, error, warn}; use crate::{ concepts::{Config, Filter, LogEntry, Match}, - utils::{bincode_options, BincodeOptions}, + protocol::{bincode_options, BincodeOptions}, }; use super::DBError; diff --git a/src/daemon/filter.rs b/src/daemon/filter.rs index 310f448..027b45f 100644 --- a/src/daemon/filter.rs +++ b/src/daemon/filter.rs @@ -7,7 +7,10 @@ use chrono::Local; use regex::Regex; use tokio::sync::{mpsc, Semaphore}; -use crate::concepts::{Filter, LogEntry, Match, Order, Pattern, PatternStatus, Time, MFT}; +use crate::{ + concepts::{Filter, LogEntry, Match, Pattern, Time, MFT}, + protocol::{Order, PatternStatus}, +}; use super::{action::ActionManager, database::DatabaseManagerInput}; diff --git a/src/daemon/socket.rs b/src/daemon/socket.rs index 64935b4..d167cda 100644 --- a/src/daemon/socket.rs +++ b/src/daemon/socket.rs @@ -11,8 +11,8 @@ use tokio_util::{ use tracing::{error, warn}; use crate::{ - concepts::{ClientRequest, ClientStatus, Config, DaemonResponse, Pattern}, - utils::bincode_options, + concepts::{Config, Pattern}, + protocol::{bincode_options, ClientRequest, ClientStatus, DaemonResponse}, }; use super::SharedState; diff --git a/src/lib.rs b/src/lib.rs index 012b4d3..dff7680 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,8 +7,9 @@ )] #![allow(clippy::upper_case_acronyms, clippy::mutable_key_type)] +pub mod cli; pub mod client; pub mod concepts; pub mod daemon; +pub mod protocol; pub mod tests; -pub mod utils; diff --git a/src/main.rs b/src/main.rs index 97d06fb..ab42624 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,11 @@ use std::{io::IsTerminal, process::exit}; use clap::Parser; - use reaction::{ + cli::{Cli, SubCommand}, client::{request, test_regex}, - concepts::Order, daemon::daemon, - utils::cli::{Cli, SubCommand}, + protocol::Order, }; use tracing::{error, Level}; diff --git a/src/concepts/socket_messages.rs b/src/protocol/messages.rs similarity index 98% rename from src/concepts/socket_messages.rs rename to src/protocol/messages.rs index 5f00d79..0fa94de 100644 --- a/src/concepts/socket_messages.rs +++ b/src/protocol/messages.rs @@ -1,12 +1,12 @@ use std::collections::BTreeMap; -use super::Match; - use serde::{ // ser::{SerializeMap, SerializeStruct}, Deserialize, Serialize, }; +use crate::concepts::Match; + // We don't need protocol versionning here because // client and daemon are the same binary diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs new file mode 100644 index 0000000..9dfbe69 --- /dev/null +++ b/src/protocol/mod.rs @@ -0,0 +1,5 @@ +mod messages; +mod serialization; + +pub use messages::*; +pub use serialization::*; diff --git a/src/utils/mod.rs b/src/protocol/serialization.rs similarity index 95% rename from src/utils/mod.rs rename to src/protocol/serialization.rs index 9505e7c..e82de5f 100644 --- a/src/utils/mod.rs +++ b/src/protocol/serialization.rs @@ -1,10 +1,6 @@ -pub mod cli; -mod parse_duration; - use std::marker::PhantomData; use bincode::Options; -pub use parse_duration::parse_duration; use serde::de::DeserializeOwned; use thiserror::Error; use tokio_util::codec::{Decoder, LengthDelimitedCodec}; diff --git a/tests/simple.rs b/tests/simple.rs index 056811c..b7fa1e0 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -8,7 +8,7 @@ use std::{ use tempfile::TempDir; use tracing::Level; -use reaction::{client::request, concepts::Order, daemon::daemon, utils::cli::Format}; +use reaction::{cli::Format, client::request, daemon::daemon, protocol::Order}; use tokio::time::sleep; fn file_with_contents(path: &str, contents: &str) { From db22bc087dab9cd026c3305ba30446f4e198c4bc Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 31 Oct 2024 12:00:00 +0100 Subject: [PATCH 088/435] fix previous-previous commit for import-rust-db --- import-rust-db/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/import-rust-db/src/main.rs b/import-rust-db/src/main.rs index 0e587c2..0b5e6ac 100644 --- a/import-rust-db/src/main.rs +++ b/import-rust-db/src/main.rs @@ -47,13 +47,13 @@ fn lil_main() -> Result<(), E> { std::fs::rename(format!("{DB}{NORMAL}"), format!("{DB}{OLD}"))?; std::fs::rename(format!("{FLUSH}{NORMAL}"), format!("{FLUSH}{OLD}"))?; - import(format!("{DB}{EXPORT}"), format!("{DB}{NORMAL}"))?; - import(format!("{FLUSH}{EXPORT}"), format!("{FLUSH}{NORMAL}"))?; + import(&format!("{DB}{EXPORT}"), &format!("{DB}{NORMAL}"))?; + import(&format!("{FLUSH}{EXPORT}"), &format!("{FLUSH}{NORMAL}"))?; Ok(()) } fn import(json_path: &str, write_path: &str) -> Result<(), E> { - let json_file = BufReader::new(File::open(json_path.clone())?); + let json_file = BufReader::new(File::open(json_path)?); let mut write_file = BufWriter::new(File::create(write_path)?); let bin = bincode_options(); From c66a5aad67576e24ba585ddadb041348ac0f7fc1 Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 31 Oct 2024 12:00:00 +0100 Subject: [PATCH 089/435] DB stop filtering and sends all matches to filter managers --- src/concepts/filter.rs | 26 +------------------------- src/daemon/database/mod.rs | 36 ++++++++++++++++-------------------- src/daemon/filter.rs | 5 ++--- 3 files changed, 19 insertions(+), 48 deletions(-) diff --git a/src/concepts/filter.rs b/src/concepts/filter.rs index be8d911..59c5df0 100644 --- a/src/concepts/filter.rs +++ b/src/concepts/filter.rs @@ -9,14 +9,10 @@ use std::{ use chrono::TimeDelta; use regex::Regex; use serde::Deserialize; -use tokio::sync::mpsc; use tracing::info; -use super::{ - messages::{Match, Time, MAT}, - Action, LogEntry, Pattern, Patterns, -}; use super::parse_duration; +use super::{messages::Match, Action, Pattern, Patterns}; // Only names are serialized // Only computed fields are not deserialized @@ -207,26 +203,6 @@ impl Filter { } None } - - pub async fn send_actions(&'static self, m: &Match, t: Time, tx: &mpsc::Sender) { - for action in self.actions.values() { - let mat = MAT { - m: m.clone(), - o: action, - t: t + action.after_duration().unwrap_or_default(), - }; - #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.send(mat).await.unwrap(); - } - } - - pub fn is_outdated(entry: &LogEntry, now: &Time) -> bool { - if entry.exec { - entry.t + entry.f.longuest_action_duration < *now - } else { - entry.t + entry.f.retry_duration.unwrap_or_default() < *now - } - } } impl Display for Filter { diff --git a/src/daemon/database/mod.rs b/src/daemon/database/mod.rs index 9d2919c..88aa23d 100644 --- a/src/daemon/database/mod.rs +++ b/src/daemon/database/mod.rs @@ -190,8 +190,6 @@ fn _rotate_db( } let last_global_flush = flushes.get(&Filter::default()); - let now = Local::now(); - // Read matches for log_entry in log_read_db { match log_entry { @@ -220,25 +218,23 @@ fn _rotate_db( } // Store match & store in db - if !Filter::is_outdated(&entry, &now) { - #[allow(clippy::unwrap_used)] // 0 second is obviously less than i64::MAX - { - // We loose subsecond precision when storing times, so we add those fake - // milliseconds to make sure each time is unique - entry.t += TimeDelta::new(0, millisecond_disambiguation_counter).unwrap(); - millisecond_disambiguation_counter += 1; - } - - if let Some(matches_tx) = &matches_tx { - if let Some(tx) = matches_tx.get(entry.f) { - debug!("DB sending match from DB: {:?}", entry.m); - #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.blocking_send(entry.clone().into()).unwrap(); - } - } - - write_or_die!(log_write_db, entry); + { + // We loose subsecond precision when storing times, so we add those fake + // milliseconds to make sure each time is unique + #![allow(clippy::unwrap_used)] // 0 second is obviously less than i64::MAX + entry.t += TimeDelta::new(0, millisecond_disambiguation_counter).unwrap(); + millisecond_disambiguation_counter += 1; } + + if let Some(matches_tx) = &matches_tx { + if let Some(tx) = matches_tx.get(entry.f) { + debug!("DB sending match from DB: {:?}", entry.m); + #[allow(clippy::unwrap_used)] // propagating panics is ok + tx.blocking_send(entry.clone().into()).unwrap(); + } + } + + write_or_die!(log_write_db, entry); } Err(err) => { *discarded_errors.entry(err.to_string()).or_insert(0) += 1; diff --git a/src/daemon/filter.rs b/src/daemon/filter.rs index 027b45f..b1daead 100644 --- a/src/daemon/filter.rs +++ b/src/daemon/filter.rs @@ -60,7 +60,7 @@ impl FilterManager { } pub async fn handle_match(&mut self, m: Match, t: Time, send_log: bool) { - self.clear_past_times(); + self.clear_past_times(t); let exec = match self.filter.retry() { None => true, @@ -172,8 +172,7 @@ impl FilterManager { } } - fn clear_past_times(&mut self) { - let now = Local::now(); + fn clear_past_times(&mut self, now: Time) { let retry_duration = self.filter.retry_duration().unwrap_or_default(); while self .ordered_times From b747e52e94e5698090ddb924490eaa4318e127eb Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 31 Oct 2024 12:00:00 +0100 Subject: [PATCH 090/435] Remove unused code and structs --- src/client/show_flush.rs | 10 ++++-- src/concepts/action.rs | 2 +- src/concepts/filter.rs | 2 +- src/concepts/messages.rs | 59 --------------------------------- src/concepts/mod.rs | 9 +++-- src/daemon/action.rs | 5 ++- src/daemon/database/lowlevel.rs | 4 +-- src/daemon/database/mod.rs | 31 +++++++++++++---- src/daemon/database/tests.rs | 3 +- src/daemon/filter.rs | 13 +++++--- src/protocol/messages.rs | 3 +- 11 files changed, 58 insertions(+), 83 deletions(-) delete mode 100644 src/concepts/messages.rs diff --git a/src/client/show_flush.rs b/src/client/show_flush.rs index 73e0591..d15d9db 100644 --- a/src/client/show_flush.rs +++ b/src/client/show_flush.rs @@ -11,7 +11,10 @@ use tokio_util::{ codec::{Framed, LengthDelimitedCodec}, }; -use crate::{cli::Format, protocol::{bincode_options, ClientRequest, ClientStatus, DaemonResponse, Order}}; +use crate::{ + cli::Format, + protocol::{bincode_options, ClientRequest, ClientStatus, DaemonResponse, Order}, +}; macro_rules! or_quit { ($msg:expr, $expression:expr) => { @@ -70,8 +73,9 @@ pub async fn request( ) .await; match response? { - DaemonResponse::Order(cs) => print_status(cs, format) - .map_err(|err| format!("while printing response: {err}")), + DaemonResponse::Order(cs) => { + print_status(cs, format).map_err(|err| format!("while printing response: {err}")) + } DaemonResponse::Err(err) => Err(format!( "failed to communicate to daemon: error response: {err}" )), diff --git a/src/concepts/action.rs b/src/concepts/action.rs index 2dc18eb..6c07fae 100644 --- a/src/concepts/action.rs +++ b/src/concepts/action.rs @@ -5,8 +5,8 @@ use chrono::TimeDelta; use serde::Deserialize; use tokio::process::Command; -use super::{Match, Pattern}; use super::parse_duration; +use super::{Match, Pattern}; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/src/concepts/filter.rs b/src/concepts/filter.rs index 59c5df0..78c2326 100644 --- a/src/concepts/filter.rs +++ b/src/concepts/filter.rs @@ -12,7 +12,7 @@ use serde::Deserialize; use tracing::info; use super::parse_duration; -use super::{messages::Match, Action, Pattern, Patterns}; +use super::{Action, Match, Pattern, Patterns}; // Only names are serialized // Only computed fields are not deserialized diff --git a/src/concepts/messages.rs b/src/concepts/messages.rs deleted file mode 100644 index 31aed6f..0000000 --- a/src/concepts/messages.rs +++ /dev/null @@ -1,59 +0,0 @@ -use chrono::{DateTime, Local, TimeDelta}; - -use super::{Action, Filter}; - -pub type Time = DateTime; -pub type Match = Vec; - -#[derive(Clone, Debug)] -pub struct MT { - pub m: Match, - pub o: &'static T, - pub t: Time, -} - -pub type MFT = MT; -pub type MAT = MT; - -// #[derive(Clone)] -// pub struct MFT { -// pub m: Match, -// pub f: &'static Filter, -// pub t: Time, -// } - -// #[derive(Clone)] -// pub struct MAT { -// pub m: Match, -// pub a: &'static Action, -// pub t: Time, -// } - -#[derive(Clone, Debug)] -pub struct LogEntry { - pub m: Match, - pub f: &'static Filter, - pub t: Time, - pub exec: bool, -} - -impl PartialEq for LogEntry { - #[allow(clippy::unwrap_used)] // 1 second is obviously less seconds than i64::MAX - fn eq(&self, other: &Self) -> bool { - self.exec == other.exec - && self.m == other.m - && self.f == other.f - // We loose subsecond precision while encoding LogEntry - && (self.t - other.t) < TimeDelta::new(1, 0).unwrap() - } -} - -impl From for MFT { - fn from(value: LogEntry) -> Self { - MFT { - m: value.m, - o: value.f, - t: value.t, - } - } -} diff --git a/src/concepts/mod.rs b/src/concepts/mod.rs index 4c5850e..d64c414 100644 --- a/src/concepts/mod.rs +++ b/src/concepts/mod.rs @@ -1,7 +1,6 @@ mod action; mod config; mod filter; -mod messages; mod parse_duration; mod pattern; mod stream; @@ -9,7 +8,11 @@ mod stream; pub use action::Action; pub use config::{Config, Patterns}; pub use filter::Filter; -pub use messages::*; +pub use parse_duration::parse_duration; pub use pattern::Pattern; pub use stream::Stream; -pub use parse_duration::parse_duration; + +use chrono::{DateTime, Local}; + +pub type Time = DateTime; +pub type Match = Vec; diff --git a/src/daemon/action.rs b/src/daemon/action.rs index 5b9a746..1be79ee 100644 --- a/src/daemon/action.rs +++ b/src/daemon/action.rs @@ -8,7 +8,10 @@ use chrono::{Local, TimeDelta}; use tokio::sync::Semaphore; use tracing::{error, info}; -use crate::{concepts::{Action, Match, Time}, protocol::Order}; +use crate::{ + concepts::{Action, Match, Time}, + protocol::Order, +}; struct State { pending: BTreeMap>, diff --git a/src/daemon/database/lowlevel.rs b/src/daemon/database/lowlevel.rs index 7d6284c..5b0dc43 100644 --- a/src/daemon/database/lowlevel.rs +++ b/src/daemon/database/lowlevel.rs @@ -12,11 +12,11 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tracing::{debug, error, warn}; use crate::{ - concepts::{Config, Filter, LogEntry, Match}, + concepts::{Config, Filter, Match}, protocol::{bincode_options, BincodeOptions}, }; -use super::DBError; +use super::{DBError, LogEntry}; // OPTIM Add a timestamp prefix to the header, to permit having // shorter timestamps? diff --git a/src/daemon/database/mod.rs b/src/daemon/database/mod.rs index 88aa23d..e597ff4 100644 --- a/src/daemon/database/mod.rs +++ b/src/daemon/database/mod.rs @@ -6,12 +6,12 @@ use std::{ thread, }; -use chrono::{Local, TimeDelta}; +use chrono::TimeDelta; use thiserror::Error; use tokio::sync::mpsc; use tracing::{debug, error, info, warn}; -use crate::concepts::{Config, Filter, LogEntry, Match, Time, MFT}; +use crate::concepts::{Config, Filter, Match, Time}; mod lowlevel; mod tests; @@ -24,6 +24,25 @@ const FLUSH_DB_NAME: &str = "./reaction-flushes.db"; const MAX_WRITES: u32 = 500_000; +#[derive(Clone, Debug)] +pub struct LogEntry { + pub m: Match, + pub f: &'static Filter, + pub t: Time, + pub exec: bool, +} + +impl PartialEq for LogEntry { + #[allow(clippy::unwrap_used)] // 1 second is obviously less seconds than i64::MAX + fn eq(&self, other: &Self) -> bool { + self.exec == other.exec + && self.m == other.m + && self.f == other.f + // We loose subsecond precision while encoding LogEntry + && (self.t - other.t) < TimeDelta::new(1, 0).unwrap() + } +} + #[derive(Error, Debug)] pub enum DBError { #[error("invalid filter: {0}")] @@ -64,7 +83,7 @@ macro_rules! flush_or_die { pub fn database_manager( config: &'static Config, mut log_rx: mpsc::Receiver, - matches_tx: BTreeMap<&'static Filter, mpsc::Sender>, + matches_tx: BTreeMap<&'static Filter, mpsc::Sender<(Match, Time)>>, ) -> thread::JoinHandle<()> { thread::spawn(move || { let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)) { @@ -111,7 +130,7 @@ pub fn database_manager( fn rotate_db( config: &'static Config, - matches_tx: Option>>, + matches_tx: Option>>, ) -> Result<(WriteDB, WriteDB), DBError> { let mut log_read_db = match ReadDB::open(LOG_DB_NAME, config)? { Some(db) => db, @@ -166,7 +185,7 @@ fn rotate_db( } fn _rotate_db( - matches_tx: Option>>, + matches_tx: Option>>, log_read_db: &mut ReadDB, flush_read_db: &mut ReadDB, log_write_db: &mut WriteDB, @@ -230,7 +249,7 @@ fn _rotate_db( if let Some(tx) = matches_tx.get(entry.f) { debug!("DB sending match from DB: {:?}", entry.m); #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.blocking_send(entry.clone().into()).unwrap(); + tx.blocking_send((entry.m.clone(), entry.t)).unwrap(); } } diff --git a/src/daemon/database/tests.rs b/src/daemon/database/tests.rs index aacf959..c1af360 100644 --- a/src/daemon/database/tests.rs +++ b/src/daemon/database/tests.rs @@ -5,7 +5,8 @@ use chrono::Local; use super::{ReadDB, WriteDB}; use crate::{ - concepts::{Config, Filter, LogEntry}, + concepts::{Config, Filter}, + daemon::database::LogEntry, tests::Fixture, }; diff --git a/src/daemon/filter.rs b/src/daemon/filter.rs index b1daead..bbece36 100644 --- a/src/daemon/filter.rs +++ b/src/daemon/filter.rs @@ -8,11 +8,14 @@ use regex::Regex; use tokio::sync::{mpsc, Semaphore}; use crate::{ - concepts::{Filter, LogEntry, Match, Pattern, Time, MFT}, + concepts::{Filter, Match, Pattern, Time}, protocol::{Order, PatternStatus}, }; -use super::{action::ActionManager, database::DatabaseManagerInput}; +use super::{ + action::ActionManager, + database::{DatabaseManagerInput, LogEntry}, +}; pub struct FilterManager { filter: &'static Filter, @@ -45,9 +48,9 @@ impl FilterManager { } } - pub async fn handle_db_entries(mut self, mut match_rx: mpsc::Receiver) -> Self { - while let Some(mft) = match_rx.recv().await { - self.handle_match(mft.m, mft.t, false).await; + pub async fn handle_db_entries(mut self, mut match_rx: mpsc::Receiver<(Match, Time)>) -> Self { + while let Some((m, t)) = match_rx.recv().await { + self.handle_match(m, t, false).await; } self } diff --git a/src/protocol/messages.rs b/src/protocol/messages.rs index 0fa94de..083254c 100644 --- a/src/protocol/messages.rs +++ b/src/protocol/messages.rs @@ -2,7 +2,8 @@ use std::collections::BTreeMap; use serde::{ // ser::{SerializeMap, SerializeStruct}, - Deserialize, Serialize, + Deserialize, + Serialize, }; use crate::concepts::Match; From a3081b0486c487f05385f92c4e3589fa3e719c1e Mon Sep 17 00:00:00 2001 From: Baptiste Careil Date: Sat, 9 Nov 2024 18:26:30 +0100 Subject: [PATCH 091/435] Fix #111: Streams now read stderr from started processes --- src/daemon/stream.rs | 42 +++++++++++++++++++++++++++++++----------- tests/simple.rs | 25 +++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/daemon/stream.rs b/src/daemon/stream.rs index e86ffaa..26f67bd 100644 --- a/src/daemon/stream.rs +++ b/src/daemon/stream.rs @@ -1,7 +1,9 @@ -use std::{collections::HashMap, process::Stdio, sync::Arc}; +use std::{collections::HashMap, process::Stdio, sync::Arc, task::Poll}; +use futures::StreamExt; use tokio::{ - io::{AsyncBufReadExt, BufReader}, + io::{AsyncBufReadExt, BufReader, Lines}, + pin, process::{Child, Command}, sync::{oneshot, Mutex}, }; @@ -12,6 +14,18 @@ use crate::{ daemon::filter::FilterManager, }; +fn lines_to_stream( + mut lines: Lines, +) -> futures::stream::PollFn< + impl FnMut(&mut std::task::Context) -> Poll>>, +> { + futures::stream::poll_fn(move |cx| { + let nl = lines.next_line(); + pin!(nl); + futures::Future::poll(nl, cx).map(Result::transpose) + }) +} + pub async fn stream_manager( stream: &'static Stream, child_tx: oneshot::Sender>, @@ -21,7 +35,7 @@ pub async fn stream_manager( let mut child = match Command::new(&stream.cmd()[0]) .args(&stream.cmd()[1..]) .stdin(Stdio::null()) - .stderr(Stdio::null()) + .stderr(Stdio::piped()) .stdout(Stdio::piped()) .spawn() { @@ -36,14 +50,20 @@ pub async fn stream_manager( // keep stdout before sending/moving child to the main thread #[allow(clippy::unwrap_used)] // we know there is an stdout because we asked for Stdio::piped() - let mut lines = BufReader::new(child.stdout.take().unwrap()).lines(); + let lines_stdout = lines_to_stream(BufReader::new(child.stdout.take().unwrap()).lines()); + #[allow(clippy::unwrap_used)] + // we know there is an stderr because we asked for Stdio::piped() + let lines_stderr = lines_to_stream(BufReader::new(child.stderr.take().unwrap()).lines()); + + // aggregate outputs, will end when both streams end + let mut lines = futures::stream::select(lines_stdout, lines_stderr); // let main handle the child process let _ = child_tx.send(Some(child)); loop { - match lines.next_line().await { - Ok(Some(line)) => { + match lines.next().await { + Some(Ok(line)) => { futures::future::join_all( filter_managers .lock() @@ -53,11 +73,7 @@ pub async fn stream_manager( ) .await; } - Ok(None) => { - error!("stream {} exited: its command returned.", stream.name()); - break; - } - Err(err) => { + Some(Err(err)) => { error!( "impossible to read output from stream {}: {}", stream.name(), @@ -65,6 +81,10 @@ pub async fn stream_manager( ); break; } + None => { + error!("stream {} exited: its command returned.", stream.name()); + break; + } } } diff --git a/tests/simple.rs b/tests/simple.rs index b7fa1e0..77b870c 100644 --- a/tests/simple.rs +++ b/tests/simple.rs @@ -166,4 +166,29 @@ async fn simple() { scenario1, scenario2 ); + + // Third part of the test + // Check we can capture both stdout and stderr from spawned processes + + // New directory to avoid to load the database from previous tests + let dir = TempDir::new().unwrap(); + env::set_current_dir(&dir).unwrap(); + + // echo numbers twice, once on stdout, once on stderr + config_with_cmd( + config_path, + "for i in 1 2 3 4 5 6 7 8 9; do echo here is $i; echo here is $i 1>&2; sleep 0.1; done; sleep 1", + ); + + file_with_contents(out_path, ""); + + let daemon_exit = daemon(config_path.into(), socket_path.into()).await; + eprintln!("daemon exit with {:?}", daemon_exit); + assert!(daemon_exit.is_err()); + + // make sure all numbers appear in the output + assert_eq!( + get_file_content(out_path).trim(), + "1\n2\n3\n4\n5\n6\n7\n8\n9".to_owned() + ); } From 8579e308900ccd8bd3d598321a7897ed342f9145 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 11 Nov 2024 12:00:00 +0100 Subject: [PATCH 092/435] test infrastructure, new conf's state_directory, less deps reaction's configuration now has a state_directory optional member, which is where it will save its databases. defaults to cwd. added a lot of code necessary to properly test databases. The new tests are currently failing, which is good : they'll permit to hunt down this database consistency bug. also removed some indirect dependencies via features removal, and moved test dependencies to dedicated [dev-dependencies] also small fix on an nft46.c function type and empty conf file for ccls LSP server. --- .ccls | 0 .gitignore | 2 + Cargo.lock | 115 ++++++++++++------ Cargo.toml | 9 +- helpers_c/nft46.c | 2 +- src/concepts/config.rs | 29 +++-- src/concepts/stream.rs | 10 ++ src/daemon/database/lowlevel.rs | 21 ++-- src/daemon/database/mod.rs | 62 +++++++--- src/daemon/database/tests.rs | 204 +++++++++++++++++++++++++++++--- 10 files changed, 361 insertions(+), 93 deletions(-) create mode 100644 .ccls diff --git a/.ccls b/.ccls new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore index 14eaadd..4f92806 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ debian-packaging/* export-go-db/export-go-db import-rust-db/target /target +/local +.ccls-cache diff --git a/Cargo.lock b/Cargo.lock index 832022e..f436b46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,12 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.7.2" @@ -284,7 +290,6 @@ checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", - "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -307,34 +312,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - [[package]] name = "futures-io" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.82", -] - [[package]] name = "futures-sink" version = "0.3.31" @@ -353,16 +336,22 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ - "futures-channel", "futures-core", - "futures-io", - "futures-macro", "futures-sink", "futures-task", - "memchr", "pin-project-lite", "pin-utils", - "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", ] [[package]] @@ -711,6 +700,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.88" @@ -729,6 +727,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "reaction" version = "2.0.0-rc1" @@ -741,6 +769,7 @@ dependencies = [ "futures", "jrsonnet-evaluator", "num_cpus", + "rand", "regex", "serde", "serde_json", @@ -904,15 +933,6 @@ dependencies = [ "libc", ] -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - [[package]] name = "smallvec" version = "1.13.2" @@ -1345,3 +1365,24 @@ checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1" dependencies = [ "winapi", ] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.82", +] diff --git a/Cargo.toml b/Cargo.toml index d287106..5c9138d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "reaction" version = "2.0.0-rc1" edition = "2021" -authors = ["ppom "] license = "AGPL-3.0" description = "Scan logs and take action" readme = "README.md" @@ -21,10 +21,9 @@ regex = "1.10.4" serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" serde_yaml = "0.9.34" -tempfile = "3.12.0" thiserror = "1.0.63" timer = "0.2.0" -futures = "0.3.30" +futures = { version = "0.3.30", default-features = false, features = ["alloc"] } tokio = { version = "1.40.0", features = ["full", "tracing"] } tokio-util = { version = "0.7.12", features = ["codec"] } tracing = "0.1.40" @@ -36,3 +35,7 @@ clap_complete = "4.5.2" clap_mangen = "0.2.24" regex = "1.10.4" tracing = "0.1.40" + +[dev-dependencies] +rand = "0.8.5" +tempfile = "3.12.0" diff --git a/helpers_c/nft46.c b/helpers_c/nft46.c index fc84aa0..cb0f419 100644 --- a/helpers_c/nft46.c +++ b/helpers_c/nft46.c @@ -80,7 +80,7 @@ void adapt_args(char *tab) { exit(1); } -int exec(char *str, char **argv) { +void exec(char *str, char **argv) { argv[0] = str; execvp(str, argv); // returns only if fails diff --git a/src/concepts/config.rs b/src/concepts/config.rs index cc66443..45d8edb 100644 --- a/src/concepts/config.rs +++ b/src/concepts/config.rs @@ -16,6 +16,7 @@ use super::{Filter, Pattern, Stream}; pub type Patterns = BTreeMap>; #[derive(Clone, Debug, Deserialize)] +#[cfg_attr(test, derive(Default))] #[serde(deny_unknown_fields)] pub struct Config { patterns: Patterns, @@ -29,6 +30,9 @@ pub struct Config { start: Vec>, #[serde(default)] stop: Vec>, + + #[serde(default)] + state_directory: String, } impl Config { @@ -44,6 +48,10 @@ impl Config { self.concurrency } + pub fn state_directory(&self) -> &str { + &self.state_directory + } + pub fn filters(&self) -> Vec<&Filter> { self.streams .values() @@ -119,6 +127,15 @@ impl Config { Ok(config) } + + #[cfg(test)] + pub fn from_streams(streams: BTreeMap, dir: &str) -> Self { + Self { + streams, + state_directory: dir.to_string(), + ..Default::default() + } + } } enum Format { @@ -211,19 +228,9 @@ mod tests { use super::*; - fn default_config() -> Config { - Config { - concurrency: 0, - patterns: BTreeMap::new(), - streams: BTreeMap::new(), - start: Vec::new(), - stop: Vec::new(), - } - } - #[test] fn config_missing() { - let mut config = default_config(); + let mut config = Config::default(); assert!(config.setup().is_err()); } } diff --git a/src/concepts/stream.rs b/src/concepts/stream.rs index 377968d..d8a4e47 100644 --- a/src/concepts/stream.rs +++ b/src/concepts/stream.rs @@ -5,6 +5,7 @@ use serde::Deserialize; use super::{Filter, Patterns}; #[derive(Clone, Debug, Deserialize)] +#[cfg_attr(test, derive(Default))] #[serde(deny_unknown_fields)] pub struct Stream { cmd: Vec, @@ -63,6 +64,15 @@ impl Stream { Ok(()) } + + #[cfg(test)] + pub fn from_filters(filters: BTreeMap, name: &str) -> Self { + Self { + filters, + name: name.to_string(), + ..Default::default() + } + } } impl PartialEq for Stream { diff --git a/src/daemon/database/lowlevel.rs b/src/daemon/database/lowlevel.rs index 5b0dc43..0a8a1fd 100644 --- a/src/daemon/database/lowlevel.rs +++ b/src/daemon/database/lowlevel.rs @@ -1,9 +1,5 @@ use std::{ - collections::BTreeMap, - fmt::Debug, - fs::File, - io::{self, BufReader, BufWriter, Read, Write}, - process::exit, + collections::BTreeMap, fmt::Debug, fs::File, io::{self, BufReader, BufWriter, Read, Write}, path::Path, process::exit }; use bincode::Options; @@ -16,7 +12,7 @@ use crate::{ protocol::{bincode_options, BincodeOptions}, }; -use super::{DBError, LogEntry}; +use super::{DBError, DatabaseNames, LogEntry}; // OPTIM Add a timestamp prefix to the header, to permit having // shorter timestamps? @@ -34,13 +30,13 @@ pub struct ReadDB { } impl ReadDB { - pub fn open(path: &str, config: &'static Config) -> Result, DBError> { + pub fn open(path: &Path, config: &'static Config) -> Result, DBError> { let file = match File::open(path) { Ok(file) => file, Err(err) => match err.kind() { std::io::ErrorKind::NotFound => { warn!( - "No DB found at {}. It's ok if this is the first time reaction is running.", + "No DB found at {:?}. It's ok if this is the first time reaction is running.", path ); return Ok(None); @@ -58,20 +54,21 @@ impl ReadDB { }; // Signature checking + // TODO if "failed to fill whole buffer", file is empty or almost empty. Ignore let mut signature = [0u8; 15]; ret.f .read_exact(&mut signature) .map_err(|err| DBError::Error(format!("reading database signature: {err}")))?; if DB_SIGNATURE.as_bytes()[0..13] != signature[0..13] { return Err(DBError::Error(format!( - "{path} is not a reaction database, or it is a reaction-v1.x database. + "{path:?} is not a reaction database, or it is a reaction-v1.x database. You can migrate your old database to a new one by following documented steps at https://reaction.ppom.me/migrate-to-v2 -You can also choose to delete the local {} and {} if you don't care about your old matches.", super::LOG_DB_NAME, super::FLUSH_DB_NAME +You can also choose to delete the local {} and {} if you don't care about your old matches.", DatabaseNames::LogDbName, DatabaseNames::FlushDbName ))); } if DB_SIGNATURE.as_bytes()[13..15] != signature[13..15] { return Err(DBError::Error(format!( - "{path} seem to be the database of a newer version of reaction. + "{path:?} seem to be the database of a newer version of reaction. Are you sure you're running the last version of reaction?" ))); } @@ -120,7 +117,7 @@ pub struct WriteDB { } impl WriteDB { - pub fn create(path: &str, config: &'static Config) -> Self { + pub fn create(path: &Path, config: &'static Config) -> Self { let file = match File::create(path) { Ok(file) => file, Err(err) => { diff --git a/src/daemon/database/mod.rs b/src/daemon/database/mod.rs index e597ff4..419a7e3 100644 --- a/src/daemon/database/mod.rs +++ b/src/daemon/database/mod.rs @@ -1,7 +1,8 @@ use std::{ collections::{BTreeMap, HashMap}, - fmt::Debug, + fmt::{Debug, Display}, fs, io, + path::PathBuf, process::exit, thread, }; @@ -18,9 +19,36 @@ mod tests; use lowlevel::{ReadDB, WriteDB}; -const LOG_DB_NAME: &str = "./reaction-matches.db"; -const LOG_DB_NEW_NAME: &str = "./reaction-matches.new.db"; -const FLUSH_DB_NAME: &str = "./reaction-flushes.db"; +enum DatabaseNames { + LogDbName, + LogDbNewName, + FlushDbName, +} +impl DatabaseNames { + fn basename(&self) -> &'static str { + match self { + DatabaseNames::LogDbName => "reaction-matches.db", + DatabaseNames::LogDbNewName => "reaction-matches.new.db", + DatabaseNames::FlushDbName => "reaction-flushes.db", + } + } +} +impl Display for DatabaseNames { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.basename()) + } +} + +impl Config { + fn path_of(&self, name: DatabaseNames) -> PathBuf { + if self.state_directory().is_empty() { + name.basename().into() + } else { + PathBuf::from(self.state_directory()).join(name.basename()) + } + } +} +use DatabaseNames::*; const MAX_WRITES: u32 = 500_000; @@ -132,23 +160,23 @@ fn rotate_db( config: &'static Config, matches_tx: Option>>, ) -> Result<(WriteDB, WriteDB), DBError> { - let mut log_read_db = match ReadDB::open(LOG_DB_NAME, config)? { + let mut log_read_db = match ReadDB::open(&config.path_of(LogDbName), config)? { Some(db) => db, None => { return Ok(( - WriteDB::create(LOG_DB_NAME, config), - WriteDB::create(FLUSH_DB_NAME, config), + WriteDB::create(&config.path_of(LogDbName), config), + WriteDB::create(&config.path_of(FlushDbName), config), )); } }; - let mut flush_read_db = match ReadDB::open(FLUSH_DB_NAME, config)? { + let mut flush_read_db = match ReadDB::open(&config.path_of(FlushDbName), config)? { Some(db) => db, None => { warn!( - "Strange! Found a {} but no {}, opening /dev/null instead", - LOG_DB_NAME, FLUSH_DB_NAME + "Strange! Found a {:?} but no {:?}, opening /dev/null instead", + &config.path_of(LogDbName), &config.path_of(FlushDbName) ); - match ReadDB::open("/dev/null", config)? { + match ReadDB::open(&PathBuf::from("/dev/null"), config)? { Some(db) => db, None => { return Err(DBError::Error("/dev/null is not accessible".into())); @@ -157,7 +185,7 @@ fn rotate_db( } }; - let mut log_write_db = WriteDB::create(LOG_DB_NEW_NAME, config); + let mut log_write_db = WriteDB::create(&config.path_of(LogDbNewName), config); _rotate_db( matches_tx, @@ -169,18 +197,18 @@ fn rotate_db( drop(log_read_db); drop(flush_read_db); - if let Err(err) = fs::rename(LOG_DB_NEW_NAME, LOG_DB_NAME) { + if let Err(err) = fs::rename(&config.path_of(LogDbNewName), &config.path_of(LogDbName)) { return Err(DBError::Error(format!( "Failed to replace old DB with new one: {}", err ))); } - if let Err(err) = fs::remove_file(FLUSH_DB_NAME) { + if let Err(err) = fs::remove_file(&config.path_of(FlushDbName)) { return Err(DBError::Error(format!("Failed to delete old DB: {}", err))); } - let flush_write_db = WriteDB::create(FLUSH_DB_NAME, config); + let flush_write_db = WriteDB::create(&config.path_of(FlushDbName), config); Ok((log_write_db, flush_write_db)) } @@ -250,7 +278,11 @@ fn _rotate_db( debug!("DB sending match from DB: {:?}", entry.m); #[allow(clippy::unwrap_used)] // propagating panics is ok tx.blocking_send((entry.m.clone(), entry.t)).unwrap(); + } else { + dbg!("no tx for ", entry.f); } + } else { + dbg!("no matches_tx"); } write_or_die!(log_write_db, entry); diff --git a/src/daemon/database/tests.rs b/src/daemon/database/tests.rs index c1af360..0538675 100644 --- a/src/daemon/database/tests.rs +++ b/src/daemon/database/tests.rs @@ -1,21 +1,24 @@ #![allow(clippy::unwrap_used)] #![cfg(test)] -use chrono::Local; +use std::{collections::BTreeMap, path::PathBuf}; + +use chrono::{Local, TimeDelta}; +use rand::{ + distributions::{Alphanumeric, DistString, Uniform}, + prelude::Distribution as _, +}; +use tempfile::TempDir; +use tokio::sync::mpsc; use super::{ReadDB, WriteDB}; use crate::{ - concepts::{Config, Filter}, - daemon::database::LogEntry, + concepts::{Config, Filter, Stream}, + daemon::database::{rotate_db, DatabaseNames, LogEntry}, tests::Fixture, }; -#[test] -fn write_and_read_db() { - let config_file = Fixture::from_string( - "config.jsonnet", - " -{ +const BASIC_INNER_CONF: &str = " patterns: { num: { regex: '[0-9]+' }, }, @@ -36,8 +39,65 @@ fn write_and_read_db() { } } } +"; + +#[test] +fn db_name() { + fn eq(path: PathBuf, str: &str) { + assert_eq!(path.into_os_string().into_string().unwrap(), str); + } + + let config_file = Fixture::from_string( + "config.jsonnet", + &format!( + "{{ {}, state_directory: '/var/lib/reaction' }}", + BASIC_INNER_CONF + ), + ); + let config = Config::from_file(&config_file).unwrap(); + assert_eq!(config.state_directory(), "/var/lib/reaction"); + eq( + config.path_of(DatabaseNames::LogDbName), + "/var/lib/reaction/reaction-matches.db", + ); + eq( + config.path_of(DatabaseNames::LogDbNewName), + "/var/lib/reaction/reaction-matches.new.db", + ); + eq( + config.path_of(DatabaseNames::FlushDbName), + "/var/lib/reaction/reaction-flushes.db", + ); + + let config_file = + Fixture::from_string("config.jsonnet", &format!("{{ {} }}", BASIC_INNER_CONF)); + let config = Config::from_file(&config_file).unwrap(); + assert_eq!(config.state_directory(), ""); + eq( + config.path_of(DatabaseNames::LogDbName), + "reaction-matches.db", + ); + eq( + config.path_of(DatabaseNames::LogDbNewName), + "reaction-matches.new.db", + ); + eq( + config.path_of(DatabaseNames::FlushDbName), + "reaction-flushes.db", + ); } - ", + +#[test] +fn write_and_read_db() { + let dir = TempDir::new().unwrap(); + + let config_file = Fixture::from_string( + "config.jsonnet", + &format!( + "{{ {}, state_directory: '{}' }}", + BASIC_INNER_CONF, + dir.into_path().into_os_string().into_string().unwrap() + ), ); let config = Box::leak(Box::new(Config::from_file(&config_file).unwrap())); @@ -60,16 +120,14 @@ fn write_and_read_db() { exec: false, }; - let db_path = Fixture::empty("matches.db"); - - let mut write_db = WriteDB::create(db_path.to_str().unwrap(), config); + let mut write_db = WriteDB::create(&config.path_of(DatabaseNames::LogDbName), config); assert!(write_db.write(correct_log_entry.clone()).is_ok()); assert!(write_db.write(incorrect_log_entry).is_err()); drop(write_db); - let read_db = ReadDB::open(db_path.to_str().unwrap(), config); + let read_db = ReadDB::open(&config.path_of(DatabaseNames::LogDbName), config); assert!(read_db.is_ok()); let read_db = read_db.unwrap(); @@ -85,3 +143,121 @@ fn write_and_read_db() { let read_entry = read_db.next(); assert!(read_entry.is_none()); } + +#[test] +fn write_and_read_db_big() { + let mut rng = rand::thread_rng(); + + let streams = (0..10) + .map(|_| Alphanumeric.sample_string(&mut rng, 10)) + .collect::>() + .into_iter() + .map(|sname| { + ( + sname.clone(), + Stream::from_filters( + (0..10) + .map(|_| Alphanumeric.sample_string(&mut rng, 10)) + .map(|fname| (fname.clone(), Filter::from_name(&sname, &fname))) + .collect(), + &sname, + ), + ) + }) + .collect(); + + let dir = TempDir::new().unwrap(); + let config = Config::from_streams( + streams, + &dir.into_path().into_os_string().into_string().unwrap(), + ); + let config = Box::leak(Box::new(config)); + + let u0t5 = Uniform::new(0, 5); + // let u0t10 = Uniform::new(0, 9); + + // 300 random entries + let entries: Vec<_> = (0..300) + .map(|i| LogEntry { + // Random match of 5 Strings of size 10 + m: (0..u0t5.sample(&mut rng)) + .map(|_| Alphanumeric.sample_string(&mut rng, 10)) + .collect(), + // Random filter in config + f: config + .streams() + .iter() + // .take(u0t10.sample(&mut rng)) + .take(1) + .last() + .unwrap() + .1 + .filters() + .iter() + // .take(u0t10.sample(&mut rng)) + .take(1) + .last() + .unwrap() + .1, + // Now + incremented microsecond (avoid duplication) + t: Local::now() + TimeDelta::microseconds(i), + exec: false, + }) + .collect(); + + let mut write_db = WriteDB::create(&config.path_of(DatabaseNames::LogDbName), config); + { + let mut _flush_db = WriteDB::create(&config.path_of(DatabaseNames::FlushDbName), config); + } + + for i in 0..entries.len() { + assert!( + write_db.write(entries[i].clone()).is_ok(), + "could not write entry n°{i}" + ); + } + + drop(write_db); + + let mut log2filter_tx = BTreeMap::new(); + let mut log2filter_rx = BTreeMap::new(); + for stream in config.streams().values() { + for filter in stream.filters().values() { + let (tx, rx) = mpsc::channel(3000); + log2filter_tx.insert(filter, tx); + log2filter_rx.insert(filter, rx); + } + } + + // We clone the senders so that the channels are not closed after database rotation + let _log2filter_tx = log2filter_tx.clone(); + + let rotated = rotate_db(config, Some(log2filter_tx)); + assert!( + rotated.is_ok(), + "database rotation failed: {}", + rotated.err().unwrap() + ); + + for i in 0..entries.len() { + let entry = &entries[i]; + let rx = &mut log2filter_rx.get_mut(entry.f).unwrap(); + let read_entry = rx.try_recv(); + assert!( + read_entry.is_ok(), + "entry n°{i} is err: {}", + read_entry.err().unwrap() + ); + let (m, t) = read_entry.unwrap(); + assert_eq!(entry.m, m, "entry n°{i}'s match is incorrect"); + assert_eq!( + t.timestamp(), + entry.t.timestamp(), + "entry n°{i}'s t is incorrect", + ); + } + + for (_, rx) in &log2filter_rx { + assert!(rx.is_empty()); + } +} From 79302efb271b80bda2cdf19a6b3600dc7c7913c8 Mon Sep 17 00:00:00 2001 From: Baptiste Careil Date: Thu, 14 Nov 2024 19:21:49 +0100 Subject: [PATCH 093/435] Fix spurious rebuild of reaction due to invalid file path --- build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.rs b/build.rs index 3c9ebe9..f7de30e 100644 --- a/build.rs +++ b/build.rs @@ -50,7 +50,7 @@ See usage examples, service configurations and good practices on the wiki: https } println!("cargo::rerun-if-changed=build.rs"); - println!("cargo::rerun-if-changed=src/utils/cli.rs"); + println!("cargo::rerun-if-changed=src/cli.rs"); println!("cargo::rerun-if-changed=helpers_c/ip46tables.c"); println!("cargo::rerun-if-changed=helpers_c/nft46.c"); From 7c3116b7c901ca2b9bd39264faba25128ca6c45e Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 18 Nov 2024 12:00:00 +0100 Subject: [PATCH 094/435] Tests passing but no error has been found Tests are finally passing but nothing has been fixed on real code, so bug is not found --- src/concepts/filter.rs | 10 ++ src/concepts/pattern.rs | 8 ++ src/daemon/database/mod.rs | 19 ++-- src/daemon/database/tests.rs | 182 +++++++++++++++++++++++++++-------- 4 files changed, 168 insertions(+), 51 deletions(-) diff --git a/src/concepts/filter.rs b/src/concepts/filter.rs index 78c2326..b5875e1 100644 --- a/src/concepts/filter.rs +++ b/src/concepts/filter.rs @@ -53,6 +53,16 @@ impl Filter { } } + #[cfg(test)] + pub fn from_name_and_patterns(stream_name: &str, filter_name: &str, patterns: Vec) -> Filter { + Filter { + stream_name: stream_name.into(), + name: filter_name.into(), + patterns: Arc::new(patterns.into_iter().map(|p| Arc::new(p)).collect()), + ..Filter::default() + } + } + pub fn name(&self) -> &str { &self.name } diff --git a/src/concepts/pattern.rs b/src/concepts/pattern.rs index eacd49a..00e0230 100644 --- a/src/concepts/pattern.rs +++ b/src/concepts/pattern.rs @@ -4,6 +4,7 @@ use regex::Regex; use serde::Deserialize; #[derive(Clone, Debug, Deserialize)] +#[cfg_attr(test, derive(Default))] #[serde(deny_unknown_fields)] pub struct Pattern { pub regex: String, @@ -23,6 +24,13 @@ pub struct Pattern { } impl Pattern { + #[cfg(test)] + pub fn from_name(name: &str) -> Pattern { + Pattern { + name: name.into(), + ..Pattern::default() + } + } pub fn setup(&mut self, name: &str) -> Result<(), String> { self._setup(name) .map_err(|msg| format!("pattern {}: {}", name, msg)) diff --git a/src/daemon/database/mod.rs b/src/daemon/database/mod.rs index 419a7e3..f150037 100644 --- a/src/daemon/database/mod.rs +++ b/src/daemon/database/mod.rs @@ -174,7 +174,8 @@ fn rotate_db( None => { warn!( "Strange! Found a {:?} but no {:?}, opening /dev/null instead", - &config.path_of(LogDbName), &config.path_of(FlushDbName) + &config.path_of(LogDbName), + &config.path_of(FlushDbName) ); match ReadDB::open(&PathBuf::from("/dev/null"), config)? { Some(db) => db, @@ -243,6 +244,7 @@ fn _rotate_db( Ok(mut entry) => { // Check if number of patterns is in sync if entry.m.len() != entry.f.patterns().len() { + debug!("DB ignoring entry: nb of patterns in filter not in sync with nb of matches: {:?}", entry); continue; } @@ -274,15 +276,12 @@ fn _rotate_db( } if let Some(matches_tx) = &matches_tx { - if let Some(tx) = matches_tx.get(entry.f) { - debug!("DB sending match from DB: {:?}", entry.m); - #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.blocking_send((entry.m.clone(), entry.t)).unwrap(); - } else { - dbg!("no tx for ", entry.f); - } - } else { - dbg!("no matches_tx"); + let tx = matches_tx + .get(entry.f) + .expect("each filter should have an associated channel Sender"); + debug!("DB sending match from DB: {:?}", entry.m); + #[allow(clippy::unwrap_used)] // propagating panics is ok + tx.blocking_send((entry.m.clone(), entry.t)).unwrap(); } write_or_die!(log_write_db, entry); diff --git a/src/daemon/database/tests.rs b/src/daemon/database/tests.rs index 0538675..9be5181 100644 --- a/src/daemon/database/tests.rs +++ b/src/daemon/database/tests.rs @@ -7,13 +7,15 @@ use chrono::{Local, TimeDelta}; use rand::{ distributions::{Alphanumeric, DistString, Uniform}, prelude::Distribution as _, + rngs::ThreadRng, }; use tempfile::TempDir; use tokio::sync::mpsc; +use tracing::Level; use super::{ReadDB, WriteDB}; use crate::{ - concepts::{Config, Filter, Stream}, + concepts::{Config, Filter, Pattern, Stream}, daemon::database::{rotate_db, DatabaseNames, LogEntry}, tests::Fixture, }; @@ -144,28 +146,91 @@ fn write_and_read_db() { assert!(read_entry.is_none()); } -#[test] -fn write_and_read_db_big() { - let mut rng = rand::thread_rng(); - - let streams = (0..10) - .map(|_| Alphanumeric.sample_string(&mut rng, 10)) +fn generate_random_streams( + rng: &mut ThreadRng, + nb_streams: usize, + nb_filters_per_stream: usize, +) -> BTreeMap { + let mut rng2 = rand::thread_rng(); + let nb_patterns = Uniform::new(0, 4); + (0..nb_streams) + .map(|_| Alphanumeric.sample_string(rng, 10)) .collect::>() .into_iter() .map(|sname| { ( sname.clone(), Stream::from_filters( - (0..10) - .map(|_| Alphanumeric.sample_string(&mut rng, 10)) - .map(|fname| (fname.clone(), Filter::from_name(&sname, &fname))) + (0..nb_filters_per_stream) + .map(|_| Alphanumeric.sample_string(rng, 10)) + .map(|fname| { + ( + fname.clone(), + Filter::from_name_and_patterns( + &sname, + &fname, + (0..nb_patterns.sample(&mut rng2)) + .map(|_| Pattern::default()) + .collect(), + ), + ) + }) .collect(), &sname, ), ) }) - .collect(); + .collect() +} +fn generate_random_entries( + rng: &mut ThreadRng, + config: &'static Config, + nb_entries: usize, + _nb_streams: usize, + _nb_filters_per_stream: usize, +) -> Vec { + // let rand_strings = Uniform::new(0, 5); + // let rand_streams = Uniform::new(0, nb_streams); + // let rand_filters = Uniform::new(0, nb_filters_per_stream); + (0..nb_entries) + .map(|i| { + let f = config + .streams() + .iter() + // .take(rand_streams.sample(rng)) + .take(1) + .last() + .unwrap() + .1 + .filters() + .iter() + // .take(rand_filters.sample(rng)) + .take(1) + .last() + .unwrap() + .1; + + LogEntry { + // Random match of n Strings of size 10 + m: (0..f.patterns().len()) + .map(|_| Alphanumeric.sample_string(rng, 10)) + .collect(), + // Random filter in config + f, + // Now + incremented microsecond (avoid duplication) + t: Local::now() + TimeDelta::microseconds(i as i64), + exec: false, + } + }) + .collect() +} + +#[test] +fn write_and_read_db_big() { + let mut rng = rand::thread_rng(); + + let streams = generate_random_streams(&mut rng, 10, 10); let dir = TempDir::new().unwrap(); let config = Config::from_streams( streams, @@ -173,37 +238,65 @@ fn write_and_read_db_big() { ); let config = Box::leak(Box::new(config)); - let u0t5 = Uniform::new(0, 5); - // let u0t10 = Uniform::new(0, 9); + let entries = generate_random_entries(&mut rng, config, 300, 10, 10); - // 300 random entries - let entries: Vec<_> = (0..300) - .map(|i| LogEntry { - // Random match of 5 Strings of size 10 - m: (0..u0t5.sample(&mut rng)) - .map(|_| Alphanumeric.sample_string(&mut rng, 10)) - .collect(), - // Random filter in config - f: config - .streams() - .iter() - // .take(u0t10.sample(&mut rng)) - .take(1) - .last() - .unwrap() - .1 - .filters() - .iter() - // .take(u0t10.sample(&mut rng)) - .take(1) - .last() - .unwrap() - .1, - // Now + incremented microsecond (avoid duplication) - t: Local::now() + TimeDelta::microseconds(i), - exec: false, - }) - .collect(); + let mut write_db = WriteDB::create(&config.path_of(DatabaseNames::LogDbName), config); + + for i in 0..entries.len() { + assert!( + write_db.write(entries[i].clone()).is_ok(), + "could not write entry n°{i}" + ); + } + + drop(write_db); + + let read_db = ReadDB::open(&config.path_of(DatabaseNames::LogDbName), config); + + assert!(read_db.is_ok()); + let read_db = read_db.unwrap(); + assert!(read_db.is_some()); + let mut read_db = read_db.unwrap(); + + for i in 0..entries.len() { + let read_entry = read_db.next(); + assert!(read_entry.is_some(), "entry n°{i} is none",); + let read_entry = read_entry.unwrap(); + assert!( + read_entry.is_ok(), + "entry n°{i} is err: {}", + read_entry.err().unwrap() + ); + let read_entry = read_entry.unwrap(); + let entry = &entries[i]; + assert_eq!(entry.m, read_entry.m, "entry n°{i}'s match is incorrect"); + assert_eq!( + entry.t.timestamp(), + read_entry.t.timestamp(), + "entry n°{i}'s t is incorrect", + ); + } + let read_entry = read_db.next(); + assert!( + read_entry.is_none(), + "entry left at end of db: {:?}", + read_entry + ); +} + +#[test] +fn rotate_db_big() { + let mut rng = rand::thread_rng(); + + let streams = generate_random_streams(&mut rng, 10, 10); + let dir = TempDir::new().unwrap(); + let config = Config::from_streams( + streams, + &dir.into_path().into_os_string().into_string().unwrap(), + ); + let config = Box::leak(Box::new(config)); + + let entries = generate_random_entries(&mut rng, config, 300, 10, 10); let mut write_db = WriteDB::create(&config.path_of(DatabaseNames::LogDbName), config); { @@ -232,6 +325,13 @@ fn write_and_read_db_big() { // We clone the senders so that the channels are not closed after database rotation let _log2filter_tx = log2filter_tx.clone(); + tracing_subscriber::fmt::fmt() + .without_time() + .with_target(false) + .with_max_level(Level::TRACE) + .try_init() + .unwrap(); + let rotated = rotate_db(config, Some(log2filter_tx)); assert!( rotated.is_ok(), From 68637e35a71a2fc65cc27b00eda088440334942c Mon Sep 17 00:00:00 2001 From: Baptiste Careil Date: Wed, 13 Nov 2024 23:00:56 +0100 Subject: [PATCH 095/435] Fix #110: don't show error message on shutdown The stream_manager is now the sole owner of the child process handle and is responsible for cleaning it. The stream_manager expects a shutdown command from the broadcast channel it receives as parameter and watches it concurrently with the child process I/O. When the shutdown command is received, the child process is killed and the remaining I/O is processed. Then the child process is reaped. Or it is killed and reaped after EOF is encountered, would the child process exit before (or at the same time as) the shutdown command is issued. --- src/daemon/mod.rs | 25 ++-------- src/daemon/stream.rs | 94 +++++++++++++++++++++++++------------ tests/test-shutdown.jsonnet | 26 ++++++++++ 3 files changed, 96 insertions(+), 49 deletions(-) create mode 100644 tests/test-shutdown.jsonnet diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 7d6adbc..17ba412 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -9,10 +9,9 @@ use std::{ }; use tokio::{ - process::Child, select, signal::unix::{signal, SignalKind}, - sync::{broadcast, mpsc, oneshot, Mutex, Semaphore}, + sync::{broadcast, mpsc, Mutex, Semaphore}, }; use tracing::info; @@ -50,7 +49,6 @@ pub async fn daemon( return Err("a start command failed, exiting.".into()); } - let mut stream_process_child_handles = Vec::new(); let mut stream_task_handles = Vec::new(); let (log_tx, log_rx) = mpsc::channel(234560); @@ -106,26 +104,18 @@ pub async fn daemon( let stream_filter_managers = Arc::new(stream_filter_managers); for (stream, filter_managers) in stream_filter_managers.s.iter() { - let (child_tx, child_rx) = oneshot::channel(); let filter_managers = filter_managers.clone(); let stream = *stream; + let stream_shutdow_rx = shutdown_rx.resubscribe(); stream_task_handles.push(tokio::spawn(async move { - stream_manager(stream, child_tx, filter_managers).await + stream_manager(stream, filter_managers, stream_shutdow_rx).await })); - - if let Ok(Some(child)) = child_rx.await { - stream_process_child_handles.push(child); - } } // Close streams when we receive a quit signal let signal_received = Arc::new(AtomicBool::new(false)); - handle_signals( - stream_process_child_handles, - shutdown_tx.clone(), - signal_received.clone(), - )?; + handle_signals(shutdown_tx.clone(), signal_received.clone())?; let socket_manager_task_handle = { let socket = socket.to_owned(); @@ -155,7 +145,6 @@ pub async fn daemon( } fn handle_signals( - stream_process_child_handles: Vec, shutdown_tx: broadcast::Sender<()>, signal_received: Arc, ) -> tokio::io::Result<()> { @@ -168,13 +157,9 @@ fn handle_signals( _ = sigint.recv() => "SIGINT", _ = sigterm.recv() => "SIGTERM", }; + info!("received {signal}, closing streams..."); let _ = shutdown_tx.send(()); signal_received.store(true, Ordering::SeqCst); - info!("received {signal}, closing streams..."); - // Kill stream subprocesses - for mut child_handle in stream_process_child_handles.into_iter() { - tokio::spawn(async move { child_handle.kill().await }); - } }); Ok(()) } diff --git a/src/daemon/stream.rs b/src/daemon/stream.rs index 26f67bd..b4fc031 100644 --- a/src/daemon/stream.rs +++ b/src/daemon/stream.rs @@ -1,11 +1,11 @@ use std::{collections::HashMap, process::Stdio, sync::Arc, task::Poll}; -use futures::StreamExt; +use futures::{pin_mut, FutureExt, StreamExt}; use tokio::{ io::{AsyncBufReadExt, BufReader, Lines}, pin, - process::{Child, Command}, - sync::{oneshot, Mutex}, + process::Command, + sync::{broadcast, Mutex}, }; use tracing::{error, info}; @@ -28,8 +28,8 @@ fn lines_to_stream( pub async fn stream_manager( stream: &'static Stream, - child_tx: oneshot::Sender>, filter_managers: Arc>>, + mut shutdown_rx: broadcast::Receiver<()>, ) { info!("{}: start {:?}", stream.name(), stream.cmd()); let mut child = match Command::new(&stream.cmd()[0]) @@ -42,7 +42,6 @@ pub async fn stream_manager( Ok(child) => child, Err(err) => { error!("could not execute stream {} cmd: {}", stream.name(), err); - let _ = child_tx.send(None); return; } }; @@ -58,33 +57,34 @@ pub async fn stream_manager( // aggregate outputs, will end when both streams end let mut lines = futures::stream::select(lines_stdout, lines_stderr); - // let main handle the child process - let _ = child_tx.send(Some(child)); + // fuse the future right after the first and only message we expect on this channel + let shutdown_watch = shutdown_rx.recv().fuse(); + pin_mut!(shutdown_watch); + + let mut child_killed = false; loop { - match lines.next().await { - Some(Ok(line)) => { - futures::future::join_all( - filter_managers - .lock() - .await - .values_mut() - .map(|manager| manager.handle_line(&line)), - ) - .await; - } - Some(Err(err)) => { - error!( - "impossible to read output from stream {}: {}", - stream.name(), - err - ); - break; - } - None => { - error!("stream {} exited: its command returned.", stream.name()); - break; + futures::select!( + _ = shutdown_watch => { + // reaction is shutting down, kill child process, we'll reap it after + // processing the remaining I/O + let _ = child.start_kill(); + child_killed = true; + }, + line = lines.next() => { + if handle_io(line, &filter_managers, stream.name(), child_killed).await.is_err() { + break; + } } + ); + } + + // reap the child process if already dead or kill it (might issue sigkill twice with the one + // above). + match child.try_wait() { + Ok(Some(_)) => {} + _ => { + let _ = child.kill().await; } } @@ -94,3 +94,39 @@ pub async fn stream_manager( .values_mut() .for_each(|manager| manager.quit()); } + +async fn handle_io( + line: Option>, + filter_managers: &Arc>>, + stream_name: &str, + child_killed: bool, +) -> Result<(), ()> { + match line { + Some(Ok(line)) => { + futures::future::join_all( + filter_managers + .lock() + .await + .values_mut() + .map(|manager| manager.handle_line(&line)), + ) + .await; + Ok(()) + } + Some(Err(err)) => { + if !child_killed { + error!( + "impossible to read output from stream {}: {}", + stream_name, err + ); + } + Err(()) + } + None => { + if !child_killed { + error!("stream {} exited: its command returned.", stream_name); + } + Err(()) + } + } +} diff --git a/tests/test-shutdown.jsonnet b/tests/test-shutdown.jsonnet new file mode 100644 index 0000000..2f68c7a --- /dev/null +++ b/tests/test-shutdown.jsonnet @@ -0,0 +1,26 @@ +{ + patterns: { + zero: { + regex: @'0', + }, + }, + + streams: { + idle: { + cmd: ['sh', '-c', 'while true; do sleep 1; done'], + filters: { + filt1: { + regex: [ + @'abc', + ], + actions: { + act: { + cmd: ['echo', '1'], + }, + }, + }, + }, + }, + }, +} + From 78f03eb643ad22de2c9a54f692e32991abbfc46f Mon Sep 17 00:00:00 2001 From: Baptiste Careil Date: Wed, 13 Nov 2024 23:15:59 +0100 Subject: [PATCH 096/435] Restore default features of futures for the select macro --- Cargo.lock | 37 +++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index f436b46..e120dac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,6 +290,7 @@ checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -312,12 +313,34 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.82", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -336,11 +359,16 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -933,6 +961,15 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.13.2" diff --git a/Cargo.toml b/Cargo.toml index 5c9138d..aa8532f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ serde_json = "1.0.117" serde_yaml = "0.9.34" thiserror = "1.0.63" timer = "0.2.0" -futures = { version = "0.3.30", default-features = false, features = ["alloc"] } +futures = "0.3.30" tokio = { version = "1.40.0", features = ["full", "tracing"] } tokio-util = { version = "0.7.12", features = ["codec"] } tracing = "0.1.40" From b143a499421179a4fdcc0fc9bdf0620974a10515 Mon Sep 17 00:00:00 2001 From: Baptiste Careil Date: Fri, 22 Nov 2024 16:42:40 +0100 Subject: [PATCH 097/435] Ask nicely the stream process to exit on shutdown Introduces the nix dependency for the signal constants and the kill(1) function. The child process is now delegated to a dedicated function handle_child that will ensure it terminated and reclaimed. On shutdown, this function first ask nicely using SIGTERM the stream process to exit (maybe we'll want to make that signal configurable as SIGINT is a good candidate as well). After 15s, if the child process still did not exit, it is killed with SIGKILL. Which is usually enough. But to make sure not to block reaction's shutdown (which could interfere unintentionally with, for example, the management of the database), there is another 5s timeout after which we give up on waiting for the child process since at this point it's most likely deadlocked in some way at the kernel level. handle_child now handles the error message about the stream command early exit. So there is no need for a communication channel between this function and handle_io which just processes the process' output. --- Cargo.lock | 19 +++++ Cargo.toml | 1 + src/daemon/stream.rs | 167 ++++++++++++++++++++++++++----------------- 3 files changed, 121 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e120dac..206ae36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,6 +175,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.38" @@ -610,6 +616,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -796,6 +814,7 @@ dependencies = [ "clap_mangen", "futures", "jrsonnet-evaluator", + "nix", "num_cpus", "rand", "regex", diff --git a/Cargo.toml b/Cargo.toml index aa8532f..04c42c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ bincode = "1.3.3" chrono = { version = "0.4.38", features = ["std", "clock"] } clap = { version = "4.5.4", features = ["derive"] } jrsonnet-evaluator = "0.4.2" +nix = { version = "0.29.0", features = ["signal"] } num_cpus = "1.16.0" regex = "1.10.4" serde = { version = "1.0.203", features = ["derive"] } diff --git a/src/daemon/stream.rs b/src/daemon/stream.rs index b4fc031..5620ef0 100644 --- a/src/daemon/stream.rs +++ b/src/daemon/stream.rs @@ -1,13 +1,14 @@ -use std::{collections::HashMap, process::Stdio, sync::Arc, task::Poll}; +use std::{collections::HashMap, process::Stdio, sync::Arc, task::Poll, time::Duration}; -use futures::{pin_mut, FutureExt, StreamExt}; +use futures::{FutureExt, StreamExt}; use tokio::{ io::{AsyncBufReadExt, BufReader, Lines}, pin, - process::Command, + process::{Child, ChildStderr, ChildStdout, Command}, sync::{broadcast, Mutex}, + time::sleep, }; -use tracing::{error, info}; +use tracing::{error, info, warn}; use crate::{ concepts::{Filter, Stream}, @@ -29,7 +30,7 @@ fn lines_to_stream( pub async fn stream_manager( stream: &'static Stream, filter_managers: Arc>>, - mut shutdown_rx: broadcast::Receiver<()>, + shutdown_rx: broadcast::Receiver<()>, ) { info!("{}: start {:?}", stream.name(), stream.cmd()); let mut child = match Command::new(&stream.cmd()[0]) @@ -46,47 +47,18 @@ pub async fn stream_manager( } }; - // keep stdout before sending/moving child to the main thread + // keep stdout/stderr before moving child to handle_child #[allow(clippy::unwrap_used)] // we know there is an stdout because we asked for Stdio::piped() - let lines_stdout = lines_to_stream(BufReader::new(child.stdout.take().unwrap()).lines()); + let child_stdout = child.stdout.take().unwrap(); #[allow(clippy::unwrap_used)] // we know there is an stderr because we asked for Stdio::piped() - let lines_stderr = lines_to_stream(BufReader::new(child.stderr.take().unwrap()).lines()); + let child_stderr = child.stderr.take().unwrap(); - // aggregate outputs, will end when both streams end - let mut lines = futures::stream::select(lines_stdout, lines_stderr); - - // fuse the future right after the first and only message we expect on this channel - let shutdown_watch = shutdown_rx.recv().fuse(); - pin_mut!(shutdown_watch); - - let mut child_killed = false; - - loop { - futures::select!( - _ = shutdown_watch => { - // reaction is shutting down, kill child process, we'll reap it after - // processing the remaining I/O - let _ = child.start_kill(); - child_killed = true; - }, - line = lines.next() => { - if handle_io(line, &filter_managers, stream.name(), child_killed).await.is_err() { - break; - } - } - ); - } - - // reap the child process if already dead or kill it (might issue sigkill twice with the one - // above). - match child.try_wait() { - Ok(Some(_)) => {} - _ => { - let _ = child.kill().await; - } - } + tokio::join!( + handle_child(stream.name(), child, shutdown_rx), + handle_io(stream.name(), child_stdout, child_stderr, &filter_managers) + ); filter_managers .lock() @@ -95,38 +67,101 @@ pub async fn stream_manager( .for_each(|manager| manager.quit()); } -async fn handle_io( - line: Option>, - filter_managers: &Arc>>, - stream_name: &str, - child_killed: bool, -) -> Result<(), ()> { - match line { - Some(Ok(line)) => { - futures::future::join_all( - filter_managers - .lock() - .await - .values_mut() - .map(|manager| manager.handle_line(&line)), - ) - .await; - Ok(()) +async fn handle_child( + stream_name: &'static str, + mut child: Child, + mut shutdown_rx: broadcast::Receiver<()>, +) { + const STREAM_PROCESS_GRACE_TIME_SEC: u64 = 15; + const STREAM_PROCESS_KILL_WAIT_TIMEOUT_SEC: u64 = 5; + + enum State { + Kill, + Exited, + } + + // wait either for the child process to exit on its own or for the shutdown signal + match futures::select! { + _ = child.wait().fuse() => State::Exited, + _ = shutdown_rx.recv().fuse() => State::Kill, + } { + State::Exited => { + error!("stream {stream_name} exited: its command returned."); + return; } - Some(Err(err)) => { - if !child_killed { + State::Kill => {} + } + + // first, try to ask nicely the child process to exit + if let Some(pid) = child.id() { + let pid = nix::unistd::Pid::from_raw(pid as i32); + + // the most likely error is that the process does not exist anymore + // but we still need to reclaim it with Child::wait + let _ = nix::sys::signal::kill(pid, nix::sys::signal::SIGTERM); + + match futures::select! { + _ = child.wait().fuse() => State::Exited, + _ = sleep(Duration::from_secs(STREAM_PROCESS_GRACE_TIME_SEC)).fuse() => State::Kill, + } { + State::Exited => return, + State::Kill => {} + } + } else { + warn!("could not get PID of child process for stream {stream_name}"); + // still try to use tokio API to kill and reclaim the child process + } + + // if that fails, or we cannot get the underlying PID, terminate the process. + // NOTE: a process killed with SIGKILL are not guaranteed to exit. It can be locked up in a + // syscall to a resource no-longer available (a notorious example is a read on a disconnected + // NFS share) + + // as before, the only expected error is that the child process already terminated + // but we still need to reclaim it if that's the case. + let _ = child.start_kill(); + + futures::select! { + _ = child.wait().fuse() => {} + _ = sleep(Duration::from_secs(STREAM_PROCESS_KILL_WAIT_TIMEOUT_SEC)).fuse() => { + error!("child process of stream {stream_name} did not terminate"); + } + } +} + +async fn handle_io( + stream_name: &'static str, + child_stdout: ChildStdout, + child_stderr: ChildStderr, + filter_managers: &Arc>>, +) { + let lines_stdout = lines_to_stream(BufReader::new(child_stdout).lines()); + let lines_stderr = lines_to_stream(BufReader::new(child_stderr).lines()); + // aggregate outputs, will end when both streams end + let mut lines = futures::stream::select(lines_stdout, lines_stderr); + + loop { + match lines.next().await { + Some(Ok(line)) => { + futures::future::join_all( + filter_managers + .lock() + .await + .values_mut() + .map(|manager| manager.handle_line(&line)), + ) + .await; + } + Some(Err(err)) => { error!( "impossible to read output from stream {}: {}", stream_name, err ); + return; } - Err(()) - } - None => { - if !child_killed { - error!("stream {} exited: its command returned.", stream_name); + None => { + return; } - Err(()) } } } From a379df899854eca727782a332ff5b24775df53bd Mon Sep 17 00:00:00 2001 From: Baptiste Careil Date: Fri, 6 Dec 2024 11:16:03 +0100 Subject: [PATCH 098/435] simplify handle_child --- src/daemon/stream.rs | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/daemon/stream.rs b/src/daemon/stream.rs index 5620ef0..a479577 100644 --- a/src/daemon/stream.rs +++ b/src/daemon/stream.rs @@ -75,21 +75,13 @@ async fn handle_child( const STREAM_PROCESS_GRACE_TIME_SEC: u64 = 15; const STREAM_PROCESS_KILL_WAIT_TIMEOUT_SEC: u64 = 5; - enum State { - Kill, - Exited, - } - // wait either for the child process to exit on its own or for the shutdown signal - match futures::select! { - _ = child.wait().fuse() => State::Exited, - _ = shutdown_rx.recv().fuse() => State::Kill, - } { - State::Exited => { + futures::select! { + _ = child.wait().fuse() => { error!("stream {stream_name} exited: its command returned."); return; } - State::Kill => {} + _ = shutdown_rx.recv().fuse() => {} } // first, try to ask nicely the child process to exit @@ -100,12 +92,11 @@ async fn handle_child( // but we still need to reclaim it with Child::wait let _ = nix::sys::signal::kill(pid, nix::sys::signal::SIGTERM); - match futures::select! { - _ = child.wait().fuse() => State::Exited, - _ = sleep(Duration::from_secs(STREAM_PROCESS_GRACE_TIME_SEC)).fuse() => State::Kill, - } { - State::Exited => return, - State::Kill => {} + futures::select! { + _ = child.wait().fuse() => { + return; + }, + _ = sleep(Duration::from_secs(STREAM_PROCESS_GRACE_TIME_SEC)).fuse() => {}, } } else { warn!("could not get PID of child process for stream {stream_name}"); @@ -113,7 +104,7 @@ async fn handle_child( } // if that fails, or we cannot get the underlying PID, terminate the process. - // NOTE: a process killed with SIGKILL are not guaranteed to exit. It can be locked up in a + // NOTE: processes killed with SIGKILL are not guaranteed to exit. They can be locked up in a // syscall to a resource no-longer available (a notorious example is a read on a disconnected // NFS share) From 777a8ca6faa4a4c0ed5d9dce1cb5053eb6f67fb0 Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 18 Nov 2024 12:00:00 +0100 Subject: [PATCH 099/435] Better handling of empty databases --- src/daemon/database/lowlevel.rs | 24 +++++++++++++++----- src/daemon/database/mod.rs | 40 +++++++++++++-------------------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/src/daemon/database/lowlevel.rs b/src/daemon/database/lowlevel.rs index 0a8a1fd..83f1f04 100644 --- a/src/daemon/database/lowlevel.rs +++ b/src/daemon/database/lowlevel.rs @@ -1,5 +1,10 @@ use std::{ - collections::BTreeMap, fmt::Debug, fs::File, io::{self, BufReader, BufWriter, Read, Write}, path::Path, process::exit + collections::BTreeMap, + fmt::Debug, + fs::File, + io::{self, BufReader, BufWriter, Read, Write}, + path::Path, + process::exit, }; use bincode::Options; @@ -54,11 +59,15 @@ impl ReadDB { }; // Signature checking - // TODO if "failed to fill whole buffer", file is empty or almost empty. Ignore let mut signature = [0u8; 15]; - ret.f - .read_exact(&mut signature) - .map_err(|err| DBError::Error(format!("reading database signature: {err}")))?; + if let Err(err) = ret.f.read_exact(&mut signature) { + return match err.kind() { + // File empty is ok + io::ErrorKind::UnexpectedEof => Ok(None), + _ => Err(DBError::Error(format!("reading database signature: {err}"))), + }; + } + if DB_SIGNATURE.as_bytes()[0..13] != signature[0..13] { return Err(DBError::Error(format!( "{path:?} is not a reaction database, or it is a reaction-v1.x database. @@ -150,6 +159,11 @@ impl WriteDB { exit(1); } + if let Err(err) = ret.f.flush() { + error!("Failed to write to DB: {}", err); + exit(1); + } + ret.h = config .filters() .into_iter() diff --git a/src/daemon/database/mod.rs b/src/daemon/database/mod.rs index f150037..125d642 100644 --- a/src/daemon/database/mod.rs +++ b/src/daemon/database/mod.rs @@ -169,22 +169,7 @@ fn rotate_db( )); } }; - let mut flush_read_db = match ReadDB::open(&config.path_of(FlushDbName), config)? { - Some(db) => db, - None => { - warn!( - "Strange! Found a {:?} but no {:?}, opening /dev/null instead", - &config.path_of(LogDbName), - &config.path_of(FlushDbName) - ); - match ReadDB::open(&PathBuf::from("/dev/null"), config)? { - Some(db) => db, - None => { - return Err(DBError::Error("/dev/null is not accessible".into())); - } - } - } - }; + let mut flush_read_db = ReadDB::open(&config.path_of(FlushDbName), config)?; let mut log_write_db = WriteDB::create(&config.path_of(LogDbNewName), config); @@ -216,7 +201,7 @@ fn rotate_db( fn _rotate_db( matches_tx: Option>>, log_read_db: &mut ReadDB, - flush_read_db: &mut ReadDB, + flush_read_db: &mut Option, log_write_db: &mut WriteDB, ) { let mut discarded_errors: HashMap = HashMap::new(); @@ -225,17 +210,22 @@ fn _rotate_db( // Read flushes let mut flushes: BTreeMap<&'static Filter, BTreeMap> = BTreeMap::new(); - for flush_entry in flush_read_db { - match flush_entry { - Ok(entry) => { - let matches_map = flushes.entry(entry.f).or_default(); - matches_map.insert(entry.m, entry.t); - } - Err(err) => { - *discarded_errors.entry(err.to_string()).or_insert(0) += 1; + if let Some(flush_read_db) = flush_read_db { + for flush_entry in flush_read_db { + match flush_entry { + Ok(entry) => { + let matches_map = flushes.entry(entry.f).or_default(); + matches_map.insert(entry.m, entry.t); + } + Err(err) => { + *discarded_errors.entry(err.to_string()).or_insert(0) += 1; + } } } } + let flushes = flushes; + debug!("flushes: {:?}", &flushes); + let last_global_flush = flushes.get(&Filter::default()); // Read matches From 37926602952bcee71595482fec657cc271ef8f34 Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 4 Dec 2024 12:00:00 +0100 Subject: [PATCH 100/435] restore (relaxed) old entries deletion Removing the (surely incorrect) old entry deletion prevented any deletion of entries in the database. Now is a relaxed version, that will remove entries only when it's sure that they're outdated. The daemon/filter part is still responsible for more precise book keeping of the in-memory state cleaning. --- src/concepts/filter.rs | 14 +++++++++++++- src/daemon/database/mod.rs | 11 +++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/concepts/filter.rs b/src/concepts/filter.rs index b5875e1..7e54a2b 100644 --- a/src/concepts/filter.rs +++ b/src/concepts/filter.rs @@ -54,7 +54,11 @@ impl Filter { } #[cfg(test)] - pub fn from_name_and_patterns(stream_name: &str, filter_name: &str, patterns: Vec) -> Filter { + pub fn from_name_and_patterns( + stream_name: &str, + filter_name: &str, + patterns: Vec, + ) -> Filter { Filter { stream_name: stream_name.into(), name: filter_name.into(), @@ -79,6 +83,14 @@ impl Filter { self.retry_duration } + pub fn max_time_before_outdated(&self) -> TimeDelta { + if let Some(retry_duration) = self.retry_duration { + self.longuest_action_duration + retry_duration + } else { + self.longuest_action_duration + } + } + pub fn actions(&self) -> &BTreeMap { &self.actions } diff --git a/src/daemon/database/mod.rs b/src/daemon/database/mod.rs index 125d642..d92b252 100644 --- a/src/daemon/database/mod.rs +++ b/src/daemon/database/mod.rs @@ -7,7 +7,7 @@ use std::{ thread, }; -use chrono::TimeDelta; +use chrono::{Local, TimeDelta}; use thiserror::Error; use tokio::sync::mpsc; use tracing::{debug, error, info, warn}; @@ -228,6 +228,8 @@ fn _rotate_db( let last_global_flush = flushes.get(&Filter::default()); + let now = Local::now(); + // Read matches for log_entry in log_read_db { match log_entry { @@ -256,7 +258,12 @@ fn _rotate_db( } } - // Store match & store in db + if entry.t + entry.f.max_time_before_outdated() < now { + debug!("DB ignoring outdated match: {:?}", entry); + continue; + } + + // Send match & store in db { // We loose subsecond precision when storing times, so we add those fake // milliseconds to make sure each time is unique From 2f2ffdc871e2601384c015b1b9086f73c9263d09 Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 4 Dec 2024 12:00:00 +0100 Subject: [PATCH 101/435] better db logs --- src/daemon/database/lowlevel.rs | 4 ++-- src/daemon/database/mod.rs | 22 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/daemon/database/lowlevel.rs b/src/daemon/database/lowlevel.rs index 83f1f04..2b85430 100644 --- a/src/daemon/database/lowlevel.rs +++ b/src/daemon/database/lowlevel.rs @@ -10,7 +10,7 @@ use std::{ use bincode::Options; use chrono::{DateTime, Local}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use tracing::{debug, error, warn}; +use tracing::{error, warn}; use crate::{ concepts::{Config, Filter, Match}, @@ -96,7 +96,7 @@ Are you sure you're running the last version of reaction?" fn read(&mut self) -> Result { let decoded = self.bin.deserialize_from::<_, T>(&mut self.f)?; - debug!("reading this: {:?}", &decoded); + // debug!("reading this: {:?}", &decoded); Ok(decoded) } } diff --git a/src/daemon/database/mod.rs b/src/daemon/database/mod.rs index d92b252..eda0040 100644 --- a/src/daemon/database/mod.rs +++ b/src/daemon/database/mod.rs @@ -71,6 +71,20 @@ impl PartialEq for LogEntry { } } +impl Display for LogEntry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "LogEntry {{ m: {:?}, f: '{}.{}', t: {}, exec: {} }}", + self.m, + self.f.stream_name(), + self.f.name(), + self.t, + self.exec + ) + } +} + #[derive(Error, Debug)] pub enum DBError { #[error("invalid filter: {0}")] @@ -244,7 +258,7 @@ fn _rotate_db( if let Some(map) = last_global_flush { if let Some(time) = map.get(&entry.m) { if time > &entry.t { - debug!("DB ignoring global match: {:?}", entry.m); + debug!("DB ignoring global match: {}", entry); continue; } } @@ -252,14 +266,14 @@ fn _rotate_db( if let Some(map) = flushes.get(&entry.f) { if let Some(time) = map.get(&entry.m) { if time > &entry.t { - debug!("DB ignoring local match: {:?}", entry.m); + debug!("DB ignoring local match: {}", entry); continue; } } } if entry.t + entry.f.max_time_before_outdated() < now { - debug!("DB ignoring outdated match: {:?}", entry); + debug!("DB ignoring outdated match: {}", entry); continue; } @@ -276,7 +290,7 @@ fn _rotate_db( let tx = matches_tx .get(entry.f) .expect("each filter should have an associated channel Sender"); - debug!("DB sending match from DB: {:?}", entry.m); + debug!("DB sending match from DB: {}", entry); #[allow(clippy::unwrap_used)] // propagating panics is ok tx.blocking_send((entry.m.clone(), entry.t)).unwrap(); } From 0b59befc422185a824bde3f45427e57bdd38fe08 Mon Sep 17 00:00:00 2001 From: Baptiste Careil Date: Sun, 29 Dec 2024 10:19:51 +0100 Subject: [PATCH 102/435] Fix delayed action not being executed --- src/daemon/action.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon/action.rs b/src/daemon/action.rs index 1be79ee..c86093b 100644 --- a/src/daemon/action.rs +++ b/src/daemon/action.rs @@ -93,7 +93,7 @@ impl ActionManager { tokio::time::sleep(dur).await; #[allow(clippy::unwrap_used)] // propagating panics is ok let mut state = this.state.lock().unwrap(); - if state.remove(&m, &t) { + if state.remove(&m, &exec_t) { this.exec_now(m); } }); From 672f0e9599d28b811fb7166df16fdcf356724009 Mon Sep 17 00:00:00 2001 From: Baptiste Careil Date: Sun, 29 Dec 2024 10:20:20 +0100 Subject: [PATCH 103/435] Add test config to test delayed actions --- tests/test-after.jsonnet | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/test-after.jsonnet diff --git a/tests/test-after.jsonnet b/tests/test-after.jsonnet new file mode 100644 index 0000000..02da40e --- /dev/null +++ b/tests/test-after.jsonnet @@ -0,0 +1,37 @@ +local log(cat) = [ + 'sh', '-c', 'echo "' + cat + ' " >>log', +]; +{ + patterns: { + id: { + regex: @'\d+', + }, + }, + streams: { + idle: { + cmd: ['sh', '-c', 'for n in 1 1 3 2 3 1 2 2 3; do echo $n; done; sleep 2'], + filters: { + filt1: { + regex: [ + @'', + ], + actions: { + act: { + cmd: log('im'), + }, + delayed: { + cmd: log('de'), + after: '1s', + }, + longafter: { + cmd: log('la'), + after: '1d', + onexit: true, + }, + }, + }, + }, + }, + }, +} + From db0391bdb752d7bce48269d119ddbf5f7ff9e45b Mon Sep 17 00:00:00 2001 From: ppom Date: Mon, 6 Jan 2025 12:00:00 +0100 Subject: [PATCH 104/435] Add SVG logo --- logo/reaction.svg | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 logo/reaction.svg diff --git a/logo/reaction.svg b/logo/reaction.svg new file mode 100644 index 0000000..433df16 --- /dev/null +++ b/logo/reaction.svg @@ -0,0 +1,7 @@ + + + + + + + From 1587e38c683f530f485bb8e597ac421e492fefb9 Mon Sep 17 00:00:00 2001 From: ppom Date: Wed, 5 Feb 2025 12:00:00 +0100 Subject: [PATCH 105/435] Move example.yml back to config/ directory --- config/example.yml | 109 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) mode change 120000 => 100644 config/example.yml diff --git a/config/example.yml b/config/example.yml deleted file mode 120000 index 1efef98..0000000 --- a/config/example.yml +++ /dev/null @@ -1 +0,0 @@ -../app/example.yml \ No newline at end of file diff --git a/config/example.yml b/config/example.yml new file mode 100644 index 0000000..759f597 --- /dev/null +++ b/config/example.yml @@ -0,0 +1,108 @@ +--- +# This example configuration file is a good starting point, but you're +# strongly encouraged to take a look at the full documentation: https://reaction.ppom.me +# +# This file is using the well-established YAML configuration language. +# Note that the more powerful JSONnet configuration language is also supported +# and that the documentation uses JSONnet + +# definitions are just a place to put chunks of conf you want to reuse in another place +# using YAML anchors `&name` and pointers `*name` +# definitions are not readed by reaction +definitions: + - &iptablesban [ 'ip46tables', '-w', '-A', 'reaction', '-s', '', '-j', 'DROP' ] + - &iptablesunban [ 'ip46tables', '-w', '-D', 'reaction', '-s', '', '-j', 'DROP' ] +# ip46tables is a minimal C program (only POSIX dependencies) present as a subdirectory. +# it permits to handle both ipv4/iptables and ipv6/ip6tables commands + +# if set to a positive number → max number of concurrent actions +# if set to a negative number → no limit +# if not specified or set to 0 → defaults to the number of CPUs on the system +concurrency: 0 + +# patterns are substitued in regexes. +# when a filter performs an action, it replaces the found pattern +patterns: + ip: + # reaction regex syntax is defined here: https://github.com/google/re2/wiki/Syntax + # simple version: regex: '(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})' + regex: '(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))' + ignore: + - 127.0.0.1 + - ::1 + # Patterns can be ignored based on regexes, it will try to match the whole string detected by the pattern + # ignoreregex: + # - '10\.0\.[0-9]{1,3}\.[0-9]{1,3}' + +# Those commands will be executed in order at start, before everything else +start: + - [ 'ip46tables', '-w', '-N', 'reaction' ] + - [ 'ip46tables', '-w', '-I', 'INPUT', '-p', 'all', '-j', 'reaction' ] + - [ 'ip46tables', '-w', '-I', 'FORWARD', '-p', 'all', '-j', 'reaction' ] + +# Those commands will be executed in order at stop, after everything else +stop: + - [ 'ip46tables', '-w,', '-D', 'INPUT', '-p', 'all', '-j', 'reaction' ] + - [ 'ip46tables', '-w,', '-D', 'FORWARD', '-p', 'all', '-j', 'reaction' ] + - [ 'ip46tables', '-w', '-F', 'reaction' ] + - [ 'ip46tables', '-w', '-X', 'reaction' ] + +# streams are commands +# they are run and their ouptut is captured +# *example:* `tail -f /var/log/nginx/access.log` +# their output will be used by one or more filters +streams: + # streams have a user-defined name + ssh: + # note that if the command is not in environment's `PATH` + # its full path must be given. + cmd: [ 'journalctl', '-n0', '-fu', 'sshd.service' ] + # filters run actions when they match regexes on a stream + filters: + # filters have a user-defined name + failedlogin: + # reaction's regex syntax is defined here: https://github.com/google/re2/wiki/Syntax + regex: + # is predefined in the patterns section + # ip's regex is inserted in the following regex + - 'authentication failure;.*rhost=' + - 'Failed password for .* from ' + - 'Connection (reset|closed) by (authenticating|invalid) user .* ' + # if retry and retryperiod are defined, + # the actions will only take place if a same pattern is + # found `retry` times in a `retryperiod` interval + retry: 3 + # format is defined here: https://pkg.go.dev/time#ParseDuration + retryperiod: 6h + # actions are run by the filter when regexes are matched + actions: + # actions have a user-defined name + ban: + # YAML substitutes *reference by the value anchored at &reference + cmd: *iptablesban + unban: + cmd: *iptablesunban + # if after is defined, the action will not take place immediately, but after a specified duration + # same format as retryperiod + after: 48h + # let's say reaction is quitting. does it run all those pending commands which had an `after` duration set? + # if you want reaction to run those pending commands before exiting, you can set this: + # onexit: true + # (defaults to false) + # here it is not useful because we will flush and delete the chain containing the bans anyway + # (with the stop commands) + +# persistence +# tldr; when an `after` action is set in a filter, such filter acts as a 'jail', +# which is persisted after reboots. +# +# when a filter is triggered, there are 2 flows: +# +# if none of its actions have an `after` directive set: +# no action will be replayed. +# +# else (if at least one action has an `after` directive set): +# if reaction stops while `after` actions are pending: +# and reaction starts again while those actions would still be pending: +# reaction executes the past actions (actions without after or with then+after < now) +# and plans the execution of future actions (actions with then+after > now) From 8cc32d122e988eb454387056d3890594d18d72c2 Mon Sep 17 00:00:00 2001 From: ppom Date: Thu, 6 Feb 2025 12:00:00 +0100 Subject: [PATCH 106/435] WIP: use sled instead of custom db implementation Goal is to directly use sled as a drop-in replacement for BTreeMaps, which already maintain the state of reaction. Those maps are now persisted by sled. A lot of code is no longer needed, and is deleted in this commit. This commit focuses on adapting FilterManager to sled. TODO - adapt ActionManager as well - at startup, clean old matches - at startup, clean old actions - at startup, run still relevant actions - at startup, remove sled trees that no longer correspond to something in the configuration - refactor socket.rs to remove complex state sharing, as sled can now be directly used. --- Cargo.lock | 122 ++++++++++- Cargo.toml | 3 +- src/daemon/database/lowlevel.rs | 230 -------------------- src/daemon/database/mod.rs | 316 --------------------------- src/daemon/database/tests.rs | 363 -------------------------------- src/daemon/filter.rs | 122 +++++------ src/daemon/mod.rs | 50 ++--- src/daemon/sledext.rs | 78 +++++++ 8 files changed, 264 insertions(+), 1020 deletions(-) delete mode 100644 src/daemon/database/lowlevel.rs delete mode 100644 src/daemon/database/mod.rs delete mode 100644 src/daemon/database/tests.rs create mode 100644 src/daemon/sledext.rs diff --git a/Cargo.lock b/Cargo.lock index 206ae36..a62b42e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -136,6 +136,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.6.0" @@ -191,6 +197,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-targets", ] @@ -266,6 +273,30 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "equivalent" version = "1.0.1" @@ -288,6 +319,16 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures" version = "0.3.31" @@ -377,6 +418,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -445,6 +495,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -622,7 +681,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags", + "bitflags 2.6.0", "cfg-if", "cfg_aliases", "libc", @@ -678,6 +737,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -685,7 +755,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -696,7 +780,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.7", "smallvec", "windows-targets", ] @@ -821,6 +905,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sled", "tempfile", "thiserror", "timer", @@ -830,13 +915,22 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ - "bitflags", + "bitflags 2.6.0", ] [[package]] @@ -892,7 +986,7 @@ version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ - "bitflags", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -989,6 +1083,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -1107,7 +1217,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", + "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", "socket2", diff --git a/Cargo.toml b/Cargo.toml index 04c42c5..136849f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ build = "build.rs" [dependencies] bincode = "1.3.3" -chrono = { version = "0.4.38", features = ["std", "clock"] } +chrono = { version = "0.4.38", features = ["std", "clock", "serde"] } clap = { version = "4.5.4", features = ["derive"] } jrsonnet-evaluator = "0.4.2" nix = { version = "0.29.0", features = ["signal"] } @@ -29,6 +29,7 @@ tokio = { version = "1.40.0", features = ["full", "tracing"] } tokio-util = { version = "0.7.12", features = ["codec"] } tracing = "0.1.40" tracing-subscriber = "0.3.18" +sled = "0.34.7" [build-dependencies] clap = { version = "4.5.4", features = ["derive"] } diff --git a/src/daemon/database/lowlevel.rs b/src/daemon/database/lowlevel.rs deleted file mode 100644 index 2b85430..0000000 --- a/src/daemon/database/lowlevel.rs +++ /dev/null @@ -1,230 +0,0 @@ -use std::{ - collections::BTreeMap, - fmt::Debug, - fs::File, - io::{self, BufReader, BufWriter, Read, Write}, - path::Path, - process::exit, -}; - -use bincode::Options; -use chrono::{DateTime, Local}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use tracing::{error, warn}; - -use crate::{ - concepts::{Config, Filter, Match}, - protocol::{bincode_options, BincodeOptions}, -}; - -use super::{DBError, DatabaseNames, LogEntry}; - -// OPTIM Add a timestamp prefix to the header, to permit having -// shorter timestamps? -// It may permit to win 1-4 bytes per entry, don't know if it's worth it -type DatabaseHeader = BTreeMap; -type ReadHeader = BTreeMap; -type WriteHeader = BTreeMap<&'static Filter, usize>; - -const DB_SIGNATURE: &str = "reaction-db-v01"; - -pub struct ReadDB { - f: BufReader, - h: ReadHeader, - bin: BincodeOptions, -} - -impl ReadDB { - pub fn open(path: &Path, config: &'static Config) -> Result, DBError> { - let file = match File::open(path) { - Ok(file) => file, - Err(err) => match err.kind() { - std::io::ErrorKind::NotFound => { - warn!( - "No DB found at {:?}. It's ok if this is the first time reaction is running.", - path - ); - return Ok(None); - } - _ => { - return Err(DBError::Error(format!("Could not open database: {}", err))); - } - }, - }; - - let mut ret = ReadDB { - f: BufReader::new(file), - h: BTreeMap::default(), - bin: bincode_options(), - }; - - // Signature checking - let mut signature = [0u8; 15]; - if let Err(err) = ret.f.read_exact(&mut signature) { - return match err.kind() { - // File empty is ok - io::ErrorKind::UnexpectedEof => Ok(None), - _ => Err(DBError::Error(format!("reading database signature: {err}"))), - }; - } - - if DB_SIGNATURE.as_bytes()[0..13] != signature[0..13] { - return Err(DBError::Error(format!( - "{path:?} is not a reaction database, or it is a reaction-v1.x database. -You can migrate your old database to a new one by following documented steps at https://reaction.ppom.me/migrate-to-v2 -You can also choose to delete the local {} and {} if you don't care about your old matches.", DatabaseNames::LogDbName, DatabaseNames::FlushDbName - ))); - } - if DB_SIGNATURE.as_bytes()[13..15] != signature[13..15] { - return Err(DBError::Error(format!( - "{path:?} seem to be the database of a newer version of reaction. -Are you sure you're running the last version of reaction?" - ))); - } - - let db_header = ret - .read::() - .map_err(|err| DBError::Error(format!("while reading database header: {err}")))?; - - ret.h = db_header - .iter() - .filter_map(|(key, name)| config.get_filter(name).map(|filter| (*key, filter))) - .collect(); - - Ok(Some(ret)) - } - - fn read(&mut self) -> Result { - let decoded = self.bin.deserialize_from::<_, T>(&mut self.f)?; - // debug!("reading this: {:?}", &decoded); - Ok(decoded) - } -} - -impl Iterator for ReadDB { - type Item = Result; - - fn next(&mut self) -> Option { - let res = self.read::(); - match res { - Ok(item) => Some(item.to(&self.h)), - Err(err) => match *err { - bincode::ErrorKind::Io(err) => match err.kind() { - io::ErrorKind::UnexpectedEof => None, - _ => Some(Err(err.into())), - }, - _ => Some(Err(err.into())), - }, - } - } -} - -pub struct WriteDB { - f: BufWriter, - h: WriteHeader, - bin: BincodeOptions, -} - -impl WriteDB { - pub fn create(path: &Path, config: &'static Config) -> Self { - let file = match File::create(path) { - Ok(file) => file, - Err(err) => { - error!("Failed to create DB: {}", err); - exit(1); - } - }; - - let mut ret = WriteDB { - f: BufWriter::new(file), - h: BTreeMap::default(), - bin: bincode_options(), - }; - - // Signature writing - if let Err(err) = ret.f.write_all(DB_SIGNATURE.as_bytes()) { - error!("Failed to write to DB: {}", err); - exit(1); - } - - let database_header: DatabaseHeader = config - .filters() - .into_iter() - .map(|f| (f.stream_name().to_owned(), f.name().to_owned())) - .enumerate() - .collect(); - - if let Err(err) = ret._write(&database_header) { - error!("Failed to write to DB: {}", err); - exit(1); - } - - if let Err(err) = ret.f.flush() { - error!("Failed to write to DB: {}", err); - exit(1); - } - - ret.h = config - .filters() - .into_iter() - .enumerate() - .map(|(key, filter)| (filter, key)) - .collect(); - - ret - } - - pub fn write(&mut self, entry: LogEntry) -> Result<(), DBError> { - let computed = ComputedLogEntry::from(entry, &self.h)?; - self._write(&computed) - } - - fn _write(&mut self, data: &T) -> Result<(), DBError> { - let encoded = self.bin.serialize(data)?; - // debug!("writing this: {:?}, {:?}", &data, &encoded); - self.f.write_all(&encoded)?; - Ok(()) - } - - pub fn flush(&mut self) -> io::Result<()> { - self.f.flush() - } -} - -#[derive(Debug, Serialize, Deserialize)] -struct ComputedLogEntry { - pub m: Match, - pub f: usize, - pub t: i64, - pub exec: bool, -} - -impl ComputedLogEntry { - fn from(value: LogEntry, header: &WriteHeader) -> Result { - match header.get(&value.f) { - Some(f) => Ok(ComputedLogEntry { - m: value.m, - f: *f, - t: value.t.timestamp(), - exec: value.exec, - }), - None => Err(DBError::InvalidFilterError(value.f.to_string())), - } - } - fn to(self, header: &ReadHeader) -> Result { - match header.get(&self.f) { - // a timestamp will cause a panic in 300_000_000 years - // you & me will both be dead so why bother? /o/ - #[allow(clippy::unwrap_used)] - Some(f) => Ok(LogEntry { - m: self.m, - f, - t: DateTime::from_timestamp(self.t, 0) - .unwrap() - .with_timezone(&Local), - exec: self.exec, - }), - None => Err(DBError::InvalidFilterError(self.f.to_string())), - } - } -} diff --git a/src/daemon/database/mod.rs b/src/daemon/database/mod.rs deleted file mode 100644 index eda0040..0000000 --- a/src/daemon/database/mod.rs +++ /dev/null @@ -1,316 +0,0 @@ -use std::{ - collections::{BTreeMap, HashMap}, - fmt::{Debug, Display}, - fs, io, - path::PathBuf, - process::exit, - thread, -}; - -use chrono::{Local, TimeDelta}; -use thiserror::Error; -use tokio::sync::mpsc; -use tracing::{debug, error, info, warn}; - -use crate::concepts::{Config, Filter, Match, Time}; - -mod lowlevel; -mod tests; - -use lowlevel::{ReadDB, WriteDB}; - -enum DatabaseNames { - LogDbName, - LogDbNewName, - FlushDbName, -} -impl DatabaseNames { - fn basename(&self) -> &'static str { - match self { - DatabaseNames::LogDbName => "reaction-matches.db", - DatabaseNames::LogDbNewName => "reaction-matches.new.db", - DatabaseNames::FlushDbName => "reaction-flushes.db", - } - } -} -impl Display for DatabaseNames { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.basename()) - } -} - -impl Config { - fn path_of(&self, name: DatabaseNames) -> PathBuf { - if self.state_directory().is_empty() { - name.basename().into() - } else { - PathBuf::from(self.state_directory()).join(name.basename()) - } - } -} -use DatabaseNames::*; - -const MAX_WRITES: u32 = 500_000; - -#[derive(Clone, Debug)] -pub struct LogEntry { - pub m: Match, - pub f: &'static Filter, - pub t: Time, - pub exec: bool, -} - -impl PartialEq for LogEntry { - #[allow(clippy::unwrap_used)] // 1 second is obviously less seconds than i64::MAX - fn eq(&self, other: &Self) -> bool { - self.exec == other.exec - && self.m == other.m - && self.f == other.f - // We loose subsecond precision while encoding LogEntry - && (self.t - other.t) < TimeDelta::new(1, 0).unwrap() - } -} - -impl Display for LogEntry { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "LogEntry {{ m: {:?}, f: '{}.{}', t: {}, exec: {} }}", - self.m, - self.f.stream_name(), - self.f.name(), - self.t, - self.exec - ) - } -} - -#[derive(Error, Debug)] -pub enum DBError { - #[error("invalid filter: {0}")] - InvalidFilterError(String), - #[error("decode error: {0}")] - BincodeError(#[from] bincode::Error), - #[error("io error: {0}")] - IOError(#[from] io::Error), - #[error("{0}")] - Error(String), -} - -#[derive(Clone)] -pub enum DatabaseManagerInput { - Log(LogEntry), - Flush(LogEntry), -} - -// Just discovering macros, let me be useless -macro_rules! write_or_die { - ($db:expr, $entry:expr) => { - if let Err(err) = $db.write($entry) { - error!("Could not write to DB: {}", err); - exit(1); - } - }; -} -macro_rules! flush_or_die { - ($db:expr) => { - if let Err(err) = $db.flush() { - error!("Could not flush DB: {}", err); - exit(1); - } - }; -} - -/// First rotates the database, then spawns the database thread -pub fn database_manager( - config: &'static Config, - mut log_rx: mpsc::Receiver, - matches_tx: BTreeMap<&'static Filter, mpsc::Sender<(Match, Time)>>, -) -> thread::JoinHandle<()> { - thread::spawn(move || { - let (mut log_db, mut flush_db) = match rotate_db(config, Some(matches_tx)) { - Ok(dbs) => dbs, - Err(err) => { - error!("while rotating databases on start: {}", err); - exit(1); - } - }; - - let mut cpt = 0; - while let Some(order) = log_rx.blocking_recv() { - match order { - DatabaseManagerInput::Flush(entry) => write_or_die!(flush_db, entry), - DatabaseManagerInput::Log(entry) => { - write_or_die!(log_db, entry); - cpt += 1; - if cpt == MAX_WRITES { - info!("Rotating database..."); - cpt = 0; - flush_or_die!(log_db); - flush_or_die!(flush_db); - drop(log_db); - drop(flush_db); - (log_db, flush_db) = match rotate_db(config, None) { - Ok(dbs) => dbs, - Err(err) => { - error!( - "while rotating databases after {} writes: {}", - MAX_WRITES, err - ); - exit(1); - } - }; - info!("Rotated database"); - } - } - }; - } - flush_or_die!(log_db); - flush_or_die!(flush_db); - }) -} - -fn rotate_db( - config: &'static Config, - matches_tx: Option>>, -) -> Result<(WriteDB, WriteDB), DBError> { - let mut log_read_db = match ReadDB::open(&config.path_of(LogDbName), config)? { - Some(db) => db, - None => { - return Ok(( - WriteDB::create(&config.path_of(LogDbName), config), - WriteDB::create(&config.path_of(FlushDbName), config), - )); - } - }; - let mut flush_read_db = ReadDB::open(&config.path_of(FlushDbName), config)?; - - let mut log_write_db = WriteDB::create(&config.path_of(LogDbNewName), config); - - _rotate_db( - matches_tx, - &mut log_read_db, - &mut flush_read_db, - &mut log_write_db, - ); - - drop(log_read_db); - drop(flush_read_db); - - if let Err(err) = fs::rename(&config.path_of(LogDbNewName), &config.path_of(LogDbName)) { - return Err(DBError::Error(format!( - "Failed to replace old DB with new one: {}", - err - ))); - } - - if let Err(err) = fs::remove_file(&config.path_of(FlushDbName)) { - return Err(DBError::Error(format!("Failed to delete old DB: {}", err))); - } - - let flush_write_db = WriteDB::create(&config.path_of(FlushDbName), config); - Ok((log_write_db, flush_write_db)) -} - -fn _rotate_db( - matches_tx: Option>>, - log_read_db: &mut ReadDB, - flush_read_db: &mut Option, - log_write_db: &mut WriteDB, -) { - let mut discarded_errors: HashMap = HashMap::new(); - - let mut millisecond_disambiguation_counter: u32 = 0; - - // Read flushes - let mut flushes: BTreeMap<&'static Filter, BTreeMap> = BTreeMap::new(); - if let Some(flush_read_db) = flush_read_db { - for flush_entry in flush_read_db { - match flush_entry { - Ok(entry) => { - let matches_map = flushes.entry(entry.f).or_default(); - matches_map.insert(entry.m, entry.t); - } - Err(err) => { - *discarded_errors.entry(err.to_string()).or_insert(0) += 1; - } - } - } - } - let flushes = flushes; - debug!("flushes: {:?}", &flushes); - - let last_global_flush = flushes.get(&Filter::default()); - - let now = Local::now(); - - // Read matches - for log_entry in log_read_db { - match log_entry { - Ok(mut entry) => { - // Check if number of patterns is in sync - if entry.m.len() != entry.f.patterns().len() { - debug!("DB ignoring entry: nb of patterns in filter not in sync with nb of matches: {:?}", entry); - continue; - } - - // Check if hasn't been flushed - if let Some(map) = last_global_flush { - if let Some(time) = map.get(&entry.m) { - if time > &entry.t { - debug!("DB ignoring global match: {}", entry); - continue; - } - } - } - if let Some(map) = flushes.get(&entry.f) { - if let Some(time) = map.get(&entry.m) { - if time > &entry.t { - debug!("DB ignoring local match: {}", entry); - continue; - } - } - } - - if entry.t + entry.f.max_time_before_outdated() < now { - debug!("DB ignoring outdated match: {}", entry); - continue; - } - - // Send match & store in db - { - // We loose subsecond precision when storing times, so we add those fake - // milliseconds to make sure each time is unique - #![allow(clippy::unwrap_used)] // 0 second is obviously less than i64::MAX - entry.t += TimeDelta::new(0, millisecond_disambiguation_counter).unwrap(); - millisecond_disambiguation_counter += 1; - } - - if let Some(matches_tx) = &matches_tx { - let tx = matches_tx - .get(entry.f) - .expect("each filter should have an associated channel Sender"); - debug!("DB sending match from DB: {}", entry); - #[allow(clippy::unwrap_used)] // propagating panics is ok - tx.blocking_send((entry.m.clone(), entry.t)).unwrap(); - } - - write_or_die!(log_write_db, entry); - } - Err(err) => { - *discarded_errors.entry(err.to_string()).or_insert(0) += 1; - } - } - } - - // Warn about errors - discarded_errors - .iter() - .filter(|(_, &count)| count > 0) - .for_each(|(name, count)| { - warn!( - "This problem was found {} times while rotating the database: {}", - count, name - ) - }); -} diff --git a/src/daemon/database/tests.rs b/src/daemon/database/tests.rs deleted file mode 100644 index 9be5181..0000000 --- a/src/daemon/database/tests.rs +++ /dev/null @@ -1,363 +0,0 @@ -#![allow(clippy::unwrap_used)] -#![cfg(test)] - -use std::{collections::BTreeMap, path::PathBuf}; - -use chrono::{Local, TimeDelta}; -use rand::{ - distributions::{Alphanumeric, DistString, Uniform}, - prelude::Distribution as _, - rngs::ThreadRng, -}; -use tempfile::TempDir; -use tokio::sync::mpsc; -use tracing::Level; - -use super::{ReadDB, WriteDB}; -use crate::{ - concepts::{Config, Filter, Pattern, Stream}, - daemon::database::{rotate_db, DatabaseNames, LogEntry}, - tests::Fixture, -}; - -const BASIC_INNER_CONF: &str = " - patterns: { - num: { regex: '[0-9]+' }, - }, - streams: { - stream1: { - cmd: ['sh', '-c', 'for i in $(seq 10); do echo $((i % 5)); done'], - filters: { - filter1: { - regex: [''], - retry: 2, - retryperiod: '5s', - actions: { - action1: { - cmd: ['echo', ''], - } - } - } - } - } - } -"; - -#[test] -fn db_name() { - fn eq(path: PathBuf, str: &str) { - assert_eq!(path.into_os_string().into_string().unwrap(), str); - } - - let config_file = Fixture::from_string( - "config.jsonnet", - &format!( - "{{ {}, state_directory: '/var/lib/reaction' }}", - BASIC_INNER_CONF - ), - ); - let config = Config::from_file(&config_file).unwrap(); - assert_eq!(config.state_directory(), "/var/lib/reaction"); - eq( - config.path_of(DatabaseNames::LogDbName), - "/var/lib/reaction/reaction-matches.db", - ); - eq( - config.path_of(DatabaseNames::LogDbNewName), - "/var/lib/reaction/reaction-matches.new.db", - ); - eq( - config.path_of(DatabaseNames::FlushDbName), - "/var/lib/reaction/reaction-flushes.db", - ); - - let config_file = - Fixture::from_string("config.jsonnet", &format!("{{ {} }}", BASIC_INNER_CONF)); - let config = Config::from_file(&config_file).unwrap(); - assert_eq!(config.state_directory(), ""); - eq( - config.path_of(DatabaseNames::LogDbName), - "reaction-matches.db", - ); - eq( - config.path_of(DatabaseNames::LogDbNewName), - "reaction-matches.new.db", - ); - eq( - config.path_of(DatabaseNames::FlushDbName), - "reaction-flushes.db", - ); -} - -#[test] -fn write_and_read_db() { - let dir = TempDir::new().unwrap(); - - let config_file = Fixture::from_string( - "config.jsonnet", - &format!( - "{{ {}, state_directory: '{}' }}", - BASIC_INNER_CONF, - dir.into_path().into_os_string().into_string().unwrap() - ), - ); - - let config = Box::leak(Box::new(Config::from_file(&config_file).unwrap())); - - let correct_filter_name = Box::leak(Box::new(Filter::from_name("stream1", "filter1"))); - - let incorrect_filter_name = Box::leak(Box::new(Filter::from_name("stream0", "filter1"))); - - let correct_log_entry = LogEntry { - m: vec!["match1".into()], - f: correct_filter_name, - t: Local::now(), - exec: false, - }; - - let incorrect_log_entry = LogEntry { - m: vec!["match1".into()], - f: incorrect_filter_name, - t: Local::now(), - exec: false, - }; - - let mut write_db = WriteDB::create(&config.path_of(DatabaseNames::LogDbName), config); - - assert!(write_db.write(correct_log_entry.clone()).is_ok()); - assert!(write_db.write(incorrect_log_entry).is_err()); - - drop(write_db); - - let read_db = ReadDB::open(&config.path_of(DatabaseNames::LogDbName), config); - - assert!(read_db.is_ok()); - let read_db = read_db.unwrap(); - assert!(read_db.is_some()); - let mut read_db = read_db.unwrap(); - - let read_entry = read_db.next(); - assert!(read_entry.is_some()); - let read_entry = read_entry.unwrap(); - assert!(read_entry.is_ok()); - assert_eq!(read_entry.unwrap(), correct_log_entry); - - let read_entry = read_db.next(); - assert!(read_entry.is_none()); -} - -fn generate_random_streams( - rng: &mut ThreadRng, - nb_streams: usize, - nb_filters_per_stream: usize, -) -> BTreeMap { - let mut rng2 = rand::thread_rng(); - let nb_patterns = Uniform::new(0, 4); - (0..nb_streams) - .map(|_| Alphanumeric.sample_string(rng, 10)) - .collect::>() - .into_iter() - .map(|sname| { - ( - sname.clone(), - Stream::from_filters( - (0..nb_filters_per_stream) - .map(|_| Alphanumeric.sample_string(rng, 10)) - .map(|fname| { - ( - fname.clone(), - Filter::from_name_and_patterns( - &sname, - &fname, - (0..nb_patterns.sample(&mut rng2)) - .map(|_| Pattern::default()) - .collect(), - ), - ) - }) - .collect(), - &sname, - ), - ) - }) - .collect() -} - -fn generate_random_entries( - rng: &mut ThreadRng, - config: &'static Config, - nb_entries: usize, - _nb_streams: usize, - _nb_filters_per_stream: usize, -) -> Vec { - // let rand_strings = Uniform::new(0, 5); - // let rand_streams = Uniform::new(0, nb_streams); - // let rand_filters = Uniform::new(0, nb_filters_per_stream); - (0..nb_entries) - .map(|i| { - let f = config - .streams() - .iter() - // .take(rand_streams.sample(rng)) - .take(1) - .last() - .unwrap() - .1 - .filters() - .iter() - // .take(rand_filters.sample(rng)) - .take(1) - .last() - .unwrap() - .1; - - LogEntry { - // Random match of n Strings of size 10 - m: (0..f.patterns().len()) - .map(|_| Alphanumeric.sample_string(rng, 10)) - .collect(), - // Random filter in config - f, - // Now + incremented microsecond (avoid duplication) - t: Local::now() + TimeDelta::microseconds(i as i64), - exec: false, - } - }) - .collect() -} - -#[test] -fn write_and_read_db_big() { - let mut rng = rand::thread_rng(); - - let streams = generate_random_streams(&mut rng, 10, 10); - let dir = TempDir::new().unwrap(); - let config = Config::from_streams( - streams, - &dir.into_path().into_os_string().into_string().unwrap(), - ); - let config = Box::leak(Box::new(config)); - - let entries = generate_random_entries(&mut rng, config, 300, 10, 10); - - let mut write_db = WriteDB::create(&config.path_of(DatabaseNames::LogDbName), config); - - for i in 0..entries.len() { - assert!( - write_db.write(entries[i].clone()).is_ok(), - "could not write entry n°{i}" - ); - } - - drop(write_db); - - let read_db = ReadDB::open(&config.path_of(DatabaseNames::LogDbName), config); - - assert!(read_db.is_ok()); - let read_db = read_db.unwrap(); - assert!(read_db.is_some()); - let mut read_db = read_db.unwrap(); - - for i in 0..entries.len() { - let read_entry = read_db.next(); - assert!(read_entry.is_some(), "entry n°{i} is none",); - let read_entry = read_entry.unwrap(); - assert!( - read_entry.is_ok(), - "entry n°{i} is err: {}", - read_entry.err().unwrap() - ); - let read_entry = read_entry.unwrap(); - let entry = &entries[i]; - assert_eq!(entry.m, read_entry.m, "entry n°{i}'s match is incorrect"); - assert_eq!( - entry.t.timestamp(), - read_entry.t.timestamp(), - "entry n°{i}'s t is incorrect", - ); - } - let read_entry = read_db.next(); - assert!( - read_entry.is_none(), - "entry left at end of db: {:?}", - read_entry - ); -} - -#[test] -fn rotate_db_big() { - let mut rng = rand::thread_rng(); - - let streams = generate_random_streams(&mut rng, 10, 10); - let dir = TempDir::new().unwrap(); - let config = Config::from_streams( - streams, - &dir.into_path().into_os_string().into_string().unwrap(), - ); - let config = Box::leak(Box::new(config)); - - let entries = generate_random_entries(&mut rng, config, 300, 10, 10); - - let mut write_db = WriteDB::create(&config.path_of(DatabaseNames::LogDbName), config); - { - let mut _flush_db = WriteDB::create(&config.path_of(DatabaseNames::FlushDbName), config); - } - - for i in 0..entries.len() { - assert!( - write_db.write(entries[i].clone()).is_ok(), - "could not write entry n°{i}" - ); - } - - drop(write_db); - - let mut log2filter_tx = BTreeMap::new(); - let mut log2filter_rx = BTreeMap::new(); - for stream in config.streams().values() { - for filter in stream.filters().values() { - let (tx, rx) = mpsc::channel(3000); - log2filter_tx.insert(filter, tx); - log2filter_rx.insert(filter, rx); - } - } - - // We clone the senders so that the channels are not closed after database rotation - let _log2filter_tx = log2filter_tx.clone(); - - tracing_subscriber::fmt::fmt() - .without_time() - .with_target(false) - .with_max_level(Level::TRACE) - .try_init() - .unwrap(); - - let rotated = rotate_db(config, Some(log2filter_tx)); - assert!( - rotated.is_ok(), - "database rotation failed: {}", - rotated.err().unwrap() - ); - - for i in 0..entries.len() { - let entry = &entries[i]; - let rx = &mut log2filter_rx.get_mut(entry.f).unwrap(); - let read_entry = rx.try_recv(); - assert!( - read_entry.is_ok(), - "entry n°{i} is err: {}", - read_entry.err().unwrap() - ); - let (m, t) = read_entry.unwrap(); - assert_eq!(entry.m, m, "entry n°{i}'s match is incorrect"); - assert_eq!( - t.timestamp(), - entry.t.timestamp(), - "entry n°{i}'s t is incorrect", - ); - } - - for (_, rx) in &log2filter_rx { - assert!(rx.is_empty()); - } -} diff --git a/src/daemon/filter.rs b/src/daemon/filter.rs index bbece36..262108f 100644 --- a/src/daemon/filter.rs +++ b/src/daemon/filter.rs @@ -5,64 +5,58 @@ use std::{ use chrono::Local; use regex::Regex; -use tokio::sync::{mpsc, Semaphore}; +use tokio::sync::Semaphore; use crate::{ concepts::{Filter, Match, Pattern, Time}, protocol::{Order, PatternStatus}, }; -use super::{ - action::ActionManager, - database::{DatabaseManagerInput, LogEntry}, -}; +use super::{action::ActionManager, SledTreeExt}; pub struct FilterManager { filter: &'static Filter, - log_tx: mpsc::Sender, action_managers: Vec, - matches: BTreeMap>, - ordered_times: BTreeMap, + // BTreeMap> + matches: sled::Tree, + // BTreeMap + ordered_times: sled::Tree, } +#[allow(clippy::unwrap_used)] impl FilterManager { pub fn new( filter: &'static Filter, - matches: BTreeMap>, exec_limit: Option>, - log_tx: mpsc::Sender, - ) -> Self { - Self { + db: &sled::Db, + ) -> Result { + let mut manager = Self { filter, - log_tx, action_managers: filter .actions() .values() .map(|action| ActionManager::new(action, BTreeMap::default(), exec_limit.clone())) .collect(), - matches: matches.clone(), - ordered_times: matches - .into_iter() - .flat_map(|(m, times)| times.into_iter().map(move |time| (time, m.clone()))) - .collect(), - } - } - - pub async fn handle_db_entries(mut self, mut match_rx: mpsc::Receiver<(Match, Time)>) -> Self { - while let Some((m, t)) = match_rx.recv().await { - self.handle_match(m, t, false).await; - } - self + matches: db.open_tree( + format!("matches_{}.{}", filter.stream_name(), filter.name()).as_bytes(), + )?, + ordered_times: db.open_tree( + format!("ordered_times_{}.{}", filter.stream_name(), filter.name()).as_bytes(), + )?, + }; + let now = Local::now(); + manager.clear_past_times(now); + Ok(manager) } pub async fn handle_line(&mut self, line: &str) { if let Some(match_) = self.filter.get_match(line) { let now = Local::now(); - self.handle_match(match_, now, true).await; + self.handle_match(match_, now).await; } } - pub async fn handle_match(&mut self, m: Match, t: Time, send_log: bool) { + pub async fn handle_match(&mut self, m: Match, t: Time) { self.clear_past_times(t); let exec = match self.filter.retry() { @@ -80,19 +74,6 @@ impl FilterManager { manager.handle_exec(m.clone(), t); } } - - if send_log { - #[allow(clippy::unwrap_used)] // propagating panics is ok - self.log_tx - .send(DatabaseManagerInput::Log(LogEntry { - exec, - m, - f: self.filter, - t, - })) - .await - .unwrap(); - } } pub fn quit(&mut self) { @@ -116,9 +97,15 @@ impl FilterManager { .all(|(a_match, regex)| regex.is_match(a_match)) }; - let matches = self.matches.clone(); - let cs: BTreeMap<_, _> = matches - .into_iter() + let cs: BTreeMap<_, _> = self + .matches + .iter() + .map(|elt| { + let (k, v) = elt.unwrap(); + let k: Match = bincode::deserialize(&k).unwrap(); + let v: BTreeSet