From f5fccc04694065f7b25a7744920c345b9cb8ba74 Mon Sep 17 00:00:00 2001 From: Martin Dobrev Date: Tue, 10 Nov 2026 18:19:17 +0000 Subject: [PATCH] Keep future-dated records at init instead of sweeping them away init() calls sweep(), which deleted every record whose timestamp is ahead of microStore::time(). On a device without a real-time clock that clock is millis() plus an offset the application persists periodically (microReticulum writes it every 10 minutes), so after a reboot it resumes from the last saved offset. Every record written after that save is then "in the future" and is deleted at boot. Those are the newest records in the store: on an ESP32-S3 running microReticulum, the paths and known destinations learned in the last minutes before each restart, or all of them if the device restarts before the first offset is saved. Clamp those timestamps to the current time in the in-memory index instead of erasing the records. A record with an impossible timestamp now ages from the boot that noticed it, so TTL expiry stays bounded and nothing is lost because the clock went backwards. Nothing is written to flash, so crash safety is unchanged. Adds test_file_store_future_timestamps_survive_reload. --- CHANGELOG.md | 1 + docs/microstore_user_guide.md | 2 ++ include/microStore/FileStore.h | 30 +++++++++++------------- test/test_file_store/test_file_store.cpp | 29 +++++++++++++++++++++++ 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c5099e..0d6d2ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Fixed `FileStore::init()` deleting records whose timestamp lies ahead of the current clock. On devices without a real-time clock the clock restarts from the last persisted offset at boot, so records written after that point were lost on every restart; their timestamps are now clamped to the current time instead. - Fixed `FileStore` compaction running with the active segment still open, which LittleFS and FatFS refuse to delete, and fixed the store being left without a writable segment (or with a stray out-of-range segment file) after a failed compaction (#6, #11). ## [0.1.7] - 2026-06-27 diff --git a/docs/microstore_user_guide.md b/docs/microstore_user_guide.md index 8f2205f..d553306 100644 --- a/docs/microstore_user_guide.md +++ b/docs/microstore_user_guide.md @@ -396,6 +396,8 @@ store.put("session.token", token, /*ttl=*/3600); // 1 hour store.set_ttl_secs(86400); // 24 hours ``` +Records whose timestamp lies ahead of the current clock (for example after a reboot on a device without a real-time clock, whose clock resumes from an earlier persisted offset) are kept at `init()`: their timestamp is clamped to the current time in the in-memory index, so their TTL counts from that boot. Nothing is written to flash. + Expired records are evicted lazily — they are detected on `get`/`exists` (which then remove them from the index) and pruned at compaction. They do not consume RAM forever, but they do hold flash until the next compaction sweep. ### 6.5 Size-based eviction diff --git a/include/microStore/FileStore.h b/include/microStore/FileStore.h index 814274a..5ddf61c 100644 --- a/include/microStore/FileStore.h +++ b/include/microStore/FileStore.h @@ -964,29 +964,27 @@ USTORE_LOG("[ustore] set_max_recs: %u\n", max_recs); /* -------- POLICY HELPERS -------- */ - // Clean out any records with timestamp in the future. + // Pull records whose timestamp lies in the future back to the current + // time. On a device without a real-time clock microStore::time() is + // millis() plus an offset, so after a reboot it resumes from whatever + // offset the application last persisted (or from zero if it persists + // none). Every record written after that point is "in the future", and + // those are the newest records in the store. Clamping keeps them and + // still bounds TTL expiry: such a record ages from this boot. Only the + // in-memory index changes; nothing is written. size_t sweep() { - size_t evicted = 0; + size_t clamped = 0; uint32_t current_time = microStore::time(); - using KTSAlloc = rebind_alloc; - KTSAlloc kts_alloc(_alloc); - std::vector evict(kts_alloc); - evict.reserve(_index.size()); for (auto& kv : _index) { if (kv.second.timestamp > current_time) { - USTORE_LOG("[ustore] sweep: removing record with timestamp %u seconds in the future\n", kv.second.timestamp - current_time); - evict.push_back(kv.first); + USTORE_LOG("[ustore] sweep: clamping record with timestamp %u seconds in the future\n", kv.second.timestamp - current_time); + kv.second.timestamp = current_time; + ++clamped; } } - for (auto& key : evict) { - _index.erase(key); - ++evicted; - } - // Record(s) evicted so increment _dead_since_compact - _dead_since_compact += evicted; - if (evicted > 0) USTORE_LOG("[ustore] sweep: evicted %lu records\n", evicted); - return evicted; + if (clamped > 0) USTORE_LOG("[ustore] sweep: clamped %lu records\n", clamped); + return clamped; } bool is_ttl_expired(uint32_t ts, uint32_t record_ttl) const diff --git a/test/test_file_store/test_file_store.cpp b/test/test_file_store/test_file_store.cpp index e694f14..81bd1be 100644 --- a/test/test_file_store/test_file_store.cpp +++ b/test/test_file_store/test_file_store.cpp @@ -802,6 +802,34 @@ void test_file_store_rotation_compact_failure_no_stray_segment() { assert_survives_rebuild("z", "v9"); } +// Records written before a reboot must survive it. On a device with no +// real-time clock microStore::time() restarts near zero, so every stored +// record looks "in the future" at init; the store must keep them (clamping +// their timestamp) instead of sweeping them away. +void test_file_store_future_timestamps_survive_reload() { + reset_ram_fs(); + + uint32_t now = microStore::time(); + { + microStore::FileStore writer; + auto fs = make_ram_fs(); + writer.init(fs, "/p"); + writer.put("a", "v1", /*ttl=*/0, now + 3600); // "written before the clock reset" + writer.put("b", "v2", /*ttl=*/0, now + 7200); + } + + microStore::FileStore reader; + auto fs = make_ram_fs(); + reader.init(fs, "/p"); // init() sweeps + + TEST_ASSERT_EQUAL(2u, reader.size()); + TEST_ASSERT_TRUE(reader.exists("a")); + TEST_ASSERT_TRUE(reader.exists("b")); + uint8_t buf[32]; uint16_t sz = sizeof(buf); + TEST_ASSERT_TRUE(reader.get("a", buf, &sz)); + TEST_ASSERT_EQUAL(0, memcmp(buf, "v1", 2)); +} + /* =========================================================================== * Directory-mode prefix tests * @@ -933,6 +961,7 @@ int runUnityTests(void) { // Additional edge-case tests RUN_TEST(test_file_store_ttl_exists_expires); RUN_TEST(test_file_store_compact_basic); + RUN_TEST(test_file_store_future_timestamps_survive_reload); RUN_TEST(test_file_store_compact_keeps_store_writable); RUN_TEST(test_file_store_compact_failure_keeps_store_writable); RUN_TEST(test_file_store_threshold_compact_failure_keeps_store_writable);