Merge pull request #1643 from tursodatabase/libsql-footer

libsql footer

Author: MarinPostma

libsql-wal/src/lib.rs

@@ -15,6 +15,22 @@ const LIBSQL_MAGIC: u64 = u64::from_be_bytes(*b"LIBSQL\0\0");
 const LIBSQL_PAGE_SIZE: u16 = 4096;
 const LIBSQL_WAL_VERSION: u16 = 1;
 
+use zerocopy::byteorder::big_endian::{U16 as bu16, U64 as bu64};
+/// LibsqlFooter is located at the end of the libsql file. I contains libsql specific metadata,
+/// while remaining fully compatible with sqlite (which just ignores that footer)
+///
+/// The fields are in big endian to remain coherent with sqlite
+#[derive(Copy, Clone, Debug, zerocopy::FromBytes, zerocopy::FromZeroes, zerocopy::AsBytes)]
+#[repr(C)]
+pub struct LibsqlFooter {
+    pub magic: bu64,
+    pub version: bu16,
+    /// Replication index checkpointed into this file.
+    /// only valid if there are no outstanding segments to checkpoint, since a checkpoint could be
+    /// partial.
+    pub replication_index: bu64,
+}
+
 #[cfg(any(debug_assertions, test))]
 pub mod test {
     use std::fs::OpenOptions;

libsql-wal/src/segment/list.rs

@@ -15,6 +15,7 @@ use crate::error::Result;
 use crate::io::buf::{ZeroCopyBoxIoBuf, ZeroCopyBuf};
 use crate::io::FileExt;
 use crate::segment::Frame;
+use crate::{LibsqlFooter, LIBSQL_MAGIC, LIBSQL_PAGE_SIZE, LIBSQL_WAL_VERSION};
 
 use super::Segment;
 
@@ -157,6 +158,21 @@ where
             buf = read_buf.into_inner();
         }
 
+        // update the footer at the end of the db file.
+        let footer = LibsqlFooter {
+            magic: LIBSQL_MAGIC.into(),
+            version: LIBSQL_WAL_VERSION.into(),
+            replication_index: last_replication_index.into(),
+        };
+
+        let footer_offset = size_after as usize * LIBSQL_PAGE_SIZE as usize;
+        let (_, ret) = db_file
+            .write_all_at_async(ZeroCopyBuf::new_init(footer), footer_offset as u64)
+            .await;
+        ret?;
+
+        // todo: truncate if necessary
+
         //// todo: make async
         db_file.sync_all()?;
 
@@ -185,7 +201,7 @@ where
         Ok(Some(last_replication_index))
     }
 
-    /// returnsstream pages from the sealed segment list, and what's the lowest replication index
+    /// returns a stream of pages from the sealed segment list, and what's the lowest replication index
     /// that was covered. If the returned index is less than start frame_no, the missing frames
     /// must be read somewhere else.
     pub async fn stream_pages_from<'a>(