diff --git a/CHANGELOG.md b/CHANGELOG.md index ad1a945..5a1d7a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Fixed `FileStore::init()` deleting records whose timestamp lies ahead of the current clock. On devices without a real-time clock the clock restarts from the last persisted offset at boot, so records written after that point were lost on every restart; their timestamps are now clamped to the current time instead. - Fixed `FileStore` compaction running with the active segment still open, which LittleFS and FatFS refuse to delete, and fixed the store being left without a writable segment (or with a stray out-of-range segment file) after a failed compaction (#6, #11). - Fixed compaction recovery paths that could lose data: a power loss after the compacted segment was moved into place but before the journal was cleared deleted all data on the next boot; a failed rename while finalizing deleted all data and still reported success; recovery from a compaction interrupted mid-way kept a stale index, and a later compaction attempt could truncate the only copy of the interrupted compaction's records. A compaction interrupted by a write failure is now recovered immediately rather than at the next boot. diff --git a/docs/microstore_user_guide.md b/docs/microstore_user_guide.md index f821e13..e4597f7 100644 --- a/docs/microstore_user_guide.md +++ b/docs/microstore_user_guide.md @@ -396,6 +396,8 @@ store.put("session.token", token, /*ttl=*/3600); // 1 hour store.set_ttl_secs(86400); // 24 hours ``` +Records whose timestamp lies ahead of the current clock (for example after a reboot on a device without a real-time clock, whose clock resumes from an earlier persisted offset) are kept at `init()`: their timestamp is clamped to the current time in the in-memory index, so their TTL counts from that boot. Nothing is written to flash. + Expired records are evicted lazily — they are detected on `get`/`exists` (which then remove them from the index) and pruned at compaction. They do not consume RAM forever, but they do hold flash until the next compaction sweep. ### 6.5 Size-based eviction diff --git a/include/microStore/FileStore.h b/include/microStore/FileStore.h index 264b275..9b469b1 100644 --- a/include/microStore/FileStore.h +++ b/include/microStore/FileStore.h @@ -964,29 +964,27 @@ USTORE_LOG("[ustore] set_max_recs: %u\n", max_recs); /* -------- POLICY HELPERS -------- */ - // Clean out any records with timestamp in the future. + // Pull records whose timestamp lies in the future back to the current + // time. On a device without a real-time clock microStore::time() is + // millis() plus an offset, so after a reboot it resumes from whatever + // offset the application last persisted (or from zero if it persists + // none). Every record written after that point is "in the future", and + // those are the newest records in the store. Clamping keeps them and + // still bounds TTL expiry: such a record ages from this boot. Only the + // in-memory index changes; nothing is written. size_t sweep() { - size_t evicted = 0; + size_t clamped = 0; uint32_t current_time = microStore::time(); - using KTSAlloc = rebind_alloc; - KTSAlloc kts_alloc(_alloc); - std::vector evict(kts_alloc); - evict.reserve(_index.size()); for (auto& kv : _index) { if (kv.second.timestamp > current_time) { - USTORE_LOG("[ustore] sweep: removing record with timestamp %u seconds in the future\n", kv.second.timestamp - current_time); - evict.push_back(kv.first); + USTORE_LOG("[ustore] sweep: clamping record with timestamp %u seconds in the future\n", kv.second.timestamp - current_time); + kv.second.timestamp = current_time; + ++clamped; } } - for (auto& key : evict) { - _index.erase(key); - ++evicted; - } - // Record(s) evicted so increment _dead_since_compact - _dead_since_compact += evicted; - if (evicted > 0) USTORE_LOG("[ustore] sweep: evicted %lu records\n", evicted); - return evicted; + if (clamped > 0) USTORE_LOG("[ustore] sweep: clamped %lu records\n", clamped); + return clamped; } bool is_ttl_expired(uint32_t ts, uint32_t record_ttl) const diff --git a/test/test_file_store/test_file_store.cpp b/test/test_file_store/test_file_store.cpp index f8ce6f4..bad8d00 100644 --- a/test/test_file_store/test_file_store.cpp +++ b/test/test_file_store/test_file_store.cpp @@ -814,6 +814,34 @@ void test_file_store_rotation_compact_failure_no_stray_segment() { assert_survives_rebuild("z", "v9"); } +// Records written before a reboot must survive it. On a device with no +// real-time clock microStore::time() restarts near zero, so every stored +// record looks "in the future" at init; the store must keep them (clamping +// their timestamp) instead of sweeping them away. +void test_file_store_future_timestamps_survive_reload() { + reset_ram_fs(); + + uint32_t now = microStore::time(); + { + microStore::FileStore writer; + auto fs = make_ram_fs(); + writer.init(fs, "/p"); + writer.put("a", "v1", /*ttl=*/0, now + 3600); // "written before the clock reset" + writer.put("b", "v2", /*ttl=*/0, now + 7200); + } + + microStore::FileStore reader; + auto fs = make_ram_fs(); + reader.init(fs, "/p"); // init() sweeps + + TEST_ASSERT_EQUAL(2u, reader.size()); + TEST_ASSERT_TRUE(reader.exists("a")); + TEST_ASSERT_TRUE(reader.exists("b")); + uint8_t buf[32]; uint16_t sz = sizeof(buf); + TEST_ASSERT_TRUE(reader.get("a", buf, &sz)); + TEST_ASSERT_EQUAL(0, memcmp(buf, "v1", 2)); +} + /* =========================================================================== * Compaction recovery tests * @@ -1127,6 +1155,7 @@ int runUnityTests(void) { // Additional edge-case tests RUN_TEST(test_file_store_ttl_exists_expires); RUN_TEST(test_file_store_compact_basic); + RUN_TEST(test_file_store_future_timestamps_survive_reload); RUN_TEST(test_file_store_compact_keeps_store_writable); RUN_TEST(test_file_store_compact_failure_keeps_store_writable); RUN_TEST(test_file_store_threshold_compact_failure_keeps_store_writable);