Merge pull request #1669 from tursodatabase/tokio-runtime-metrics

expose basic tokio runtime metrics

Author: sivukhin

.github/workflows/nemesis.yml

@@ -18,7 +18,7 @@ jobs:
     if: github.repository == 'tursodatabase/libsql'
     name: Run Nemesis Tests
     env:
-      RUSTFLAGS: -D warnings
+      RUSTFLAGS: -D warnings --cfg tokio_unstable
     steps:
       - uses: hecrj/setup-rust-action@v2
 

.github/workflows/rust.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Run Checks
     env:
-      RUSTFLAGS: -D warnings
+      RUSTFLAGS: -D warnings --cfg tokio_unstable
     steps:
       - uses: hecrj/setup-rust-action@v2
 
@@ -80,15 +80,15 @@ jobs:
       - uses: taiki-e/install-action@cargo-udeps
       - uses: Swatinem/rust-cache@v2
       - run: cargo +nightly hack udeps -p libsql --each-feature
-      - run: RUSTFLAGS="-D warnings" cargo check -p libsql --no-default-features --features core 
-      - run: RUSTFLAGS="-D warnings" cargo check -p libsql --no-default-features --features replication 
-      - run: RUSTFLAGS="-D warnings" cargo check -p libsql --no-default-features --features remote
+      - run: RUSTFLAGS="-D warnings --cfg tokio_unstable" cargo check -p libsql --no-default-features --features core 
+      - run: RUSTFLAGS="-D warnings --cfg tokio_unstable" cargo check -p libsql --no-default-features --features replication 
+      - run: RUSTFLAGS="-D warnings --cfg tokio_unstable" cargo check -p libsql --no-default-features --features remote
 
   test:
     runs-on: ubuntu-latest
     name: Run Tests
     env:
-      RUSTFLAGS: -D warnings
+      RUSTFLAGS: -D warnings --cfg tokio_unstable
     steps:
     - uses: hecrj/setup-rust-action@v2
 

libsql-server/Cargo.toml

@@ -8,6 +8,9 @@ default-run = "sqld"
 name = "sqld"
 path = "src/main.rs"
 
+[build]
+rustflags = ["--cfg", "tokio_unstable"]
+
 [dependencies]
 anyhow = "1.0.66"
 async-lock = "2.6.0"

libsql-server/src/http/admin/mod.rs

@@ -93,9 +93,29 @@ where
 
         tokio::task::spawn(async move {
             loop {
-                tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+                let runtime = tokio::runtime::Handle::current();
+                let metrics = runtime.metrics();
+                crate::metrics::TOKIO_RUNTIME_BLOCKING_QUEUE_DEPTH
+                    .set(metrics.blocking_queue_depth() as f64);
+                crate::metrics::TOKIO_RUNTIME_INJECTION_QUEUE_DEPTH
+                    .set(metrics.injection_queue_depth() as f64);
+                crate::metrics::TOKIO_RUNTIME_NUM_BLOCKING_THREADS
+                    .set(metrics.num_blocking_threads() as f64);
+                crate::metrics::TOKIO_RUNTIME_NUM_IDLE_BLOCKING_THREADS
+                    .set(metrics.num_idle_blocking_threads() as f64);
+                crate::metrics::TOKIO_RUNTIME_NUM_WORKERS.set(metrics.num_workers() as f64);
+
+                crate::metrics::TOKIO_RUNTIME_IO_DRIVER_FD_DEREGISTERED_COUNT
+                    .absolute(metrics.io_driver_fd_deregistered_count() as u64);
+                crate::metrics::TOKIO_RUNTIME_IO_DRIVER_FD_REGISTERED_COUNT
+                    .absolute(metrics.io_driver_fd_registered_count() as u64);
+                crate::metrics::TOKIO_RUNTIME_IO_DRIVER_READY_COUNT
+                    .absolute(metrics.io_driver_ready_count() as u64);
+                crate::metrics::TOKIO_RUNTIME_REMOTE_SCHEDULE_COUNT
+                    .absolute(metrics.remote_schedule_count() as u64);
 
                 crate::metrics::SERVER_COUNT.set(1.0);
+                tokio::time::sleep(std::time::Duration::from_secs(1)).await;
             }
         });
 

libsql-server/src/metrics.rs

@@ -158,3 +158,57 @@ pub static QUERY_CANCELED: Lazy<Counter> = Lazy::new(|| {
     describe_counter!(NAME, "Number of canceled queries");
     register_counter!(NAME)
 });
+
+pub static TOKIO_RUNTIME_BLOCKING_QUEUE_DEPTH: Lazy<Gauge> = Lazy::new(|| {
+    const NAME: &str = "tokio_runtime_blocking_queue_depth";
+    describe_gauge!(NAME, "tokio runtime blocking_queue_depth");
+    register_gauge!(NAME)
+});
+
+pub static TOKIO_RUNTIME_INJECTION_QUEUE_DEPTH: Lazy<Gauge> = Lazy::new(|| {
+    const NAME: &str = "tokio_runtime_injection_queue_depth";
+    describe_gauge!(NAME, "tokio runtime injection_queue_depth");
+    register_gauge!(NAME)
+});
+
+pub static TOKIO_RUNTIME_NUM_BLOCKING_THREADS: Lazy<Gauge> = Lazy::new(|| {
+    const NAME: &str = "tokio_runtime_num_blocking_threads";
+    describe_gauge!(NAME, "tokio runtime num_blocking_threads");
+    register_gauge!(NAME)
+});
+
+pub static TOKIO_RUNTIME_NUM_IDLE_BLOCKING_THREADS: Lazy<Gauge> = Lazy::new(|| {
+    const NAME: &str = "tokio_runtime_num_idle_blocking_threads";
+    describe_gauge!(NAME, "tokio runtime num_idle_blocking_threads");
+    register_gauge!(NAME)
+});
+
+pub static TOKIO_RUNTIME_NUM_WORKERS: Lazy<Gauge> = Lazy::new(|| {
+    const NAME: &str = "tokio_runtime_num_workers";
+    describe_gauge!(NAME, "tokio runtime num_workers");
+    register_gauge!(NAME)
+});
+
+pub static TOKIO_RUNTIME_IO_DRIVER_FD_DEREGISTERED_COUNT: Lazy<Counter> = Lazy::new(|| {
+    const NAME: &str = "tokio_runtime_io_driver_fd_deregistered_count";
+    describe_counter!(NAME, "tokio runtime io_driver_fd_deregistered_count");
+    register_counter!(NAME)
+});
+
+pub static TOKIO_RUNTIME_IO_DRIVER_FD_REGISTERED_COUNT: Lazy<Counter> = Lazy::new(|| {
+    const NAME: &str = "tokio_runtime_io_driver_fd_registered_count";
+    describe_counter!(NAME, "tokio runtime io_driver_fd_registered_count");
+    register_counter!(NAME)
+});
+
+pub static TOKIO_RUNTIME_IO_DRIVER_READY_COUNT: Lazy<Counter> = Lazy::new(|| {
+    const NAME: &str = "tokio_runtime_io_driver_ready_count";
+    describe_counter!(NAME, "tokio runtime io_driver_ready_count");
+    register_counter!(NAME)
+});
+
+pub static TOKIO_RUNTIME_REMOTE_SCHEDULE_COUNT: Lazy<Counter> = Lazy::new(|| {
+    const NAME: &str = "tokio_runtime_remote_schedule_count";
+    describe_gauge!(NAME, "tokio runtime remote_schedule_count");
+    register_counter!(NAME)
+});