From 68f2aad670d86d8325c34f0e017fd0a237409f80 Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Wed, 16 Sep 2026 16:37:40 -0500 Subject: [PATCH 01/12] Remove DescendingCursor.java and modify RangeCursor as newer versions of H2 support cursors with reverse iteration. --- sensorhub-datastore-h2/build.gradle | 7 +- .../java/org/h2/mvstore/DescendingCursor.java | 177 ------------------ .../main/java/org/h2/mvstore/RangeCursor.java | 26 ++- 3 files changed, 17 insertions(+), 193 deletions(-) delete mode 100644 sensorhub-datastore-h2/src/main/java/org/h2/mvstore/DescendingCursor.java diff --git a/sensorhub-datastore-h2/build.gradle b/sensorhub-datastore-h2/build.gradle index 6dfa92750..14bd46965 100644 --- a/sensorhub-datastore-h2/build.gradle +++ b/sensorhub-datastore-h2/build.gradle @@ -4,7 +4,7 @@ ext.details = 'Storage module backed by a H2 MVStore object database' dependencies { implementation project(':sensorhub-core') implementation project(':sensorhub-utils-kryo') - embeddedApi 'com.h2database:h2-mvstore:1.4.200' + embeddedApi 'com.h2database:h2-mvstore:2.5.250' embeddedApi 'org.apache.lucene:lucene-analyzers-common:8.11.2' testImplementation(testFixtures(project(':sensorhub-core'))) } @@ -34,3 +34,8 @@ ext.pom >>= { } } } + +// TEMP (H2 port): show all compiler errors; remove before merging +tasks.withType(JavaCompile) { + options.compilerArgs << "-Xmaxerrs" << "2000" +} diff --git a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/DescendingCursor.java b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/DescendingCursor.java deleted file mode 100644 index 0b352df47..000000000 --- a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/DescendingCursor.java +++ /dev/null @@ -1,177 +0,0 @@ -/***************************** BEGIN LICENSE BLOCK *************************** - -The contents of this file are subject to the Mozilla Public License, v. 2.0. -If a copy of the MPL was not distributed with this file, You can obtain one -at http://mozilla.org/MPL/2.0/. - -Software distributed under the License is distributed on an "AS IS" basis, -WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -for the specific language governing rights and limitations under the License. - -Copyright (C) 2026 GeoRobotix Inc. All Rights Reserved. - -******************************* END LICENSE BLOCK ***************************/ - -package org.h2.mvstore; - -import java.util.Iterator; -import java.util.NoSuchElementException; - - -/** - * A cursor to iterate over elements in ascending order. - * - * @param the key type - * @param the value type - */ -public class DescendingCursor implements Iterator { - private final K to; - private CursorPos cursorPos; - private CursorPos keeper; - private K current; - private K last; - private V lastValue; - private Page lastPage; - - public DescendingCursor(Page root, K from) { - this(root, from, null); - } - - public DescendingCursor(Page root, K from, K to) { - this.cursorPos = traverseDown(root, from); - this.to = to; - } - - @Override - @SuppressWarnings("unchecked") - public boolean hasNext() { - if (cursorPos != null) { - while (current == null) { - Page page = cursorPos.page; - int index = cursorPos.index; - if (index < 0) { - CursorPos tmp = cursorPos; - cursorPos = cursorPos.parent; - tmp.parent = keeper; - keeper = tmp; - if(cursorPos == null) - { - return false; - } - } else { - while (!page.isLeaf()) { - page = page.getChildPage(index); - int numKeys = page.isLeaf() ? page.getKeyCount() : page.map.getChildPageCount(page); - if (keeper == null) { - cursorPos = new CursorPos(page, numKeys-1, cursorPos); - } else { - CursorPos tmp = keeper; - keeper = keeper.parent; - tmp.parent = cursorPos; - tmp.page = page; - tmp.index = numKeys-1; - cursorPos = tmp; - } - index = numKeys-1; - } - if (index >= 0) { - K key = (K) page.getKey(index); - if (to != null && page.map.getKeyType().compare(key, to) < 0) { - return false; - } - current = last = key; - lastValue = (V) page.getValue(index); - lastPage = page; - } - } - --cursorPos.index; - } - } - return current != null; - } - - @Override - public K next() { - if(!hasNext()) { - throw new NoSuchElementException(); - } - current = null; - return last; - } - - /** - * Get the last read key if there was one. - * - * @return the key or null - */ - public K getKey() { - return last; - } - - /** - * Get the last read value if there was one. - * - * @return the value or null - */ - public V getValue() { - return lastValue; - } - - /** - * Get the page where last retrieved key is located. - * - * @return the page - */ - Page getPage() { - return lastPage; - } - - /** - * Skip over that many entries. This method is relatively fast (for this map - * implementation) even if many entries need to be skipped. - * - * @param n the number of entries to skip - */ - public void skip(long n) { - if (n < 10) { - while (n-- > 0 && hasNext()) { - next(); - } - } else if(hasNext()) { - assert cursorPos != null; - CursorPos cp = cursorPos; - CursorPos parent; - while ((parent = cp.parent) != null) cp = parent; - Page root = cp.page; - @SuppressWarnings("unchecked") - MVMap map = (MVMap) root.map; - long index = map.getKeyIndex(next()); - last = map.getKey(index + n); - this.cursorPos = traverseDown(root, last); - } - } - - @Override - public void remove() { - throw DataUtils.newUnsupportedOperationException( - "Removal is not supported"); - } - - /** - * Fetch the next entry that is equal or larger than the given key, starting - * from the given page. This method retains the stack. - * - * @param p the page to start from - * @param key the key to search, null means search for the first key - */ - private static CursorPos traverseDown(Page p, Object key) { - CursorPos cursorPos = key == null ? p.getPrependCursorPos(null) : CursorPos.traverseDown(p, key); - if (cursorPos.index < 0) { - cursorPos.index = -cursorPos.index - 1; - // select previous instead of next - cursorPos.index--; - } - return cursorPos; - } -} - diff --git a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RangeCursor.java b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RangeCursor.java index 5ac45156d..a4348ad97 100644 --- a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RangeCursor.java +++ b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RangeCursor.java @@ -25,7 +25,11 @@ /** *

- * Wrapper for native H2 Cursor and DescendingCursor + * Wrapper for native H2 Cursor adding support for descending iteration + * and key/value/entry streams. + *

+ * When iterating in descending order, {@code from} must be greater than + * or equal to {@code to}. Both bounds are inclusive in either direction. *

* * @author Alex Robin @@ -35,7 +39,7 @@ */ public class RangeCursor implements Iterator { - final Iterator cursor; + final Cursor cursor; final MVMap map; final boolean descending; @@ -54,21 +58,19 @@ public RangeCursor(MVMap map, K from, K to) public RangeCursor(MVMap map, K from, K to, boolean descending) { - this.cursor = descending ? - new DescendingCursor(map.getRootPage(), from, to) : - new Cursor(map.getRootPage(), from, to); + this.cursor = map.cursor(from, to, descending); this.map = map; this.descending = descending; } - + @Override public boolean hasNext() { return cursor.hasNext(); } - + @Override public K next() { @@ -76,21 +78,15 @@ public K next() } - @SuppressWarnings("unchecked") public K getKey() { - return descending ? - ((DescendingCursor)cursor).getKey() : - ((Cursor)cursor).getKey(); + return cursor.getKey(); } - @SuppressWarnings("unchecked") public V getValue() { - return descending ? - ((DescendingCursor)cursor).getValue() : - ((Cursor)cursor).getValue(); + return cursor.getValue(); } From 5e99b993545612032d9fbc06a6efc5814199aef3 Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Wed, 16 Sep 2026 17:48:34 -0500 Subject: [PATCH 02/12] =?UTF-8?q?Updated=20KryoDataType.java=20to=20use=20?= =?UTF-8?q?BasicDataType,=20H2=E2=80=99s=20bulk=20method=20signatu?= =?UTF-8?q?res,=20and=20createStorage().?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/datastore/h2/kryo/KryoDataType.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/kryo/KryoDataType.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/kryo/KryoDataType.java index 44ebbfe40..8dca68868 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/kryo/KryoDataType.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/kryo/KryoDataType.java @@ -20,7 +20,7 @@ import java.util.function.Consumer; import java.util.function.Supplier; import org.h2.mvstore.WriteBuffer; -import org.h2.mvstore.type.DataType; +import org.h2.mvstore.type.BasicDataType; import org.objenesis.strategy.StdInstantiatorStrategy; import org.sensorhub.impl.datastore.h2.H2Utils; import org.sensorhub.impl.serialization.kryo.VersionedSerializer; @@ -36,7 +36,7 @@ import com.google.common.collect.ImmutableMap; -public class KryoDataType implements DataType +public class KryoDataType extends BasicDataType { static Logger log = LoggerFactory.getLogger(KryoDataType.class); @@ -176,8 +176,9 @@ public Object read(ByteBuffer buff) @Override - public void read(ByteBuffer buff, Object[] obj, int len, boolean key) + public void read(ByteBuffer buff, Object storage, int len) { + Object[] obj = cast(storage); KryoInstance kryoI = getReadKryo(); for (int i = 0; i < len; i++) obj[i] = read(buff, kryoI); @@ -207,8 +208,9 @@ public void write(WriteBuffer buff, Object obj) @Override - public void write(WriteBuffer buff, Object[] objects, int len, boolean key) + public void write(WriteBuffer buff, Object storage, int len) { + Object[] objects = cast(storage); for (int i = 0; i < len; i++) { var obj = objects[i]; @@ -281,4 +283,10 @@ public int compare(Object a, Object b) return 0; } + @Override + public Object[] createStorage(int size) + { + return new Object[size]; + } + } From dc59da687cf2a96a2dfff456b8d0af2b2a7965fc Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Wed, 16 Sep 2026 17:51:43 -0500 Subject: [PATCH 03/12] Updated MVVarLongDataType.java to use BasicDataType, inherited bulk methods, and createStorage(). --- .../java/org/h2/mvstore/MVVarLongDataType.java | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVVarLongDataType.java b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVVarLongDataType.java index 110443558..4e860ff94 100644 --- a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVVarLongDataType.java +++ b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVVarLongDataType.java @@ -15,7 +15,7 @@ package org.h2.mvstore; import java.nio.ByteBuffer; -import org.h2.mvstore.type.DataType; +import org.h2.mvstore.type.BasicDataType; /** @@ -26,7 +26,7 @@ * @author Alex Robin * @date Oct 26, 2018 */ -public class MVVarLongDataType implements DataType +public class MVVarLongDataType extends BasicDataType { @@ -51,14 +51,6 @@ public final void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (Object o: obj) - write(wbuf, o); - } - - @Override public final Object read(ByteBuffer buf) { @@ -67,9 +59,8 @@ public final Object read(ByteBuffer buf) @Override - public void read(ByteBuffer buf, Object[] obj, int len, boolean key) + public Object[] createStorage(int size) { - for (int i=0; i Date: Wed, 16 Sep 2026 18:15:05 -0500 Subject: [PATCH 04/12] Add ObjectArrayDataType to share createStorage() across serializers. Migrate to BasicDataType and remove obsolete bulk methods. --- .../org/h2/mvstore/FullTextKeyDataType.java | 18 ++--------------- .../org/h2/mvstore/MVVarLongDataType.java | 7 +------ .../org/h2/mvstore/ObjectArrayDataType.java | 13 ++++++++++++ ...mmandSeriesKeyByCommandStreamDataType.java | 20 +++---------------- .../h2/MVCommandStatusKeyDataType.java | 20 +++---------------- .../h2/MVCommandStreamKeyDataType.java | 20 +++---------------- .../datastore/h2/MVDataStreamKeyDataType.java | 20 +++---------------- .../datastore/h2/MVFeatureKeyDataType.java | 20 +++---------------- .../h2/MVFeatureParentKeyByIdDataType.java | 3 +-- .../h2/MVFeatureParentKeyDataType.java | 19 +++--------------- .../datastore/h2/MVFeatureRefDataType.java | 20 +++---------------- .../MVObsSeriesKeyByDataStreamDataType.java | 20 +++---------------- .../datastore/h2/MVPropertyKeyDataType.java | 20 +++---------------- .../h2/MVTimeSeriesInfoDataType.java | 20 +++---------------- .../h2/MVTimeSeriesRecordKeyDataType.java | 20 +++---------------- .../h2/MVTimeSeriesSystemKeyDataType.java | 20 +++---------------- .../impl/datastore/h2/MVVoidDataType.java | 20 +++++-------------- .../impl/datastore/h2/kryo/KryoDataType.java | 9 ++------- 18 files changed, 60 insertions(+), 249 deletions(-) create mode 100644 sensorhub-datastore-h2/src/main/java/org/h2/mvstore/ObjectArrayDataType.java diff --git a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/FullTextKeyDataType.java b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/FullTextKeyDataType.java index 98e80e23b..f977e9a76 100644 --- a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/FullTextKeyDataType.java +++ b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/FullTextKeyDataType.java @@ -15,10 +15,11 @@ package org.h2.mvstore; import java.nio.ByteBuffer; +import org.h2.mvstore.type.BasicDataType; import org.h2.mvstore.type.DataType; -public class FullTextKeyDataType implements DataType +public class FullTextKeyDataType extends BasicDataType implements ObjectArrayDataType { DataType refKeyDataType; @@ -64,14 +65,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i +public class MVVarLongDataType extends BasicDataType implements ObjectArrayDataType { @@ -58,9 +58,4 @@ public final Object read(ByteBuffer buf) } - @Override - public Object[] createStorage(int size) - { - return new Object[size]; - } } diff --git a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/ObjectArrayDataType.java b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/ObjectArrayDataType.java new file mode 100644 index 000000000..cc1793d08 --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/ObjectArrayDataType.java @@ -0,0 +1,13 @@ +package org.h2.mvstore; + +import org.h2.mvstore.type.DataType; + + +public interface ObjectArrayDataType extends DataType +{ + @Override + default Object[] createStorage(int size) + { + return new Object[size]; + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVCommandSeriesKeyByCommandStreamDataType.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVCommandSeriesKeyByCommandStreamDataType.java index 17900a935..0694cbbba 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVCommandSeriesKeyByCommandStreamDataType.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVCommandSeriesKeyByCommandStreamDataType.java @@ -15,9 +15,10 @@ package org.sensorhub.impl.datastore.h2; import java.nio.ByteBuffer; +import org.h2.mvstore.ObjectArrayDataType; import org.h2.mvstore.DataUtils; import org.h2.mvstore.WriteBuffer; -import org.h2.mvstore.type.DataType; +import org.h2.mvstore.type.BasicDataType; /** @@ -29,7 +30,7 @@ * @author Alex Robin * @date Mar 25, 2021 */ -class MVCommandSeriesKeyByCommandStreamDataType implements DataType +class MVCommandSeriesKeyByCommandStreamDataType extends BasicDataType implements ObjectArrayDataType { private static final int MEM_SIZE = 8+8; // long ID, long receiver ID @@ -66,14 +67,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { int idScope; @@ -88,14 +89,6 @@ public static void encode(WriteBuffer wbuf, MVCommandStatusKey key) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { final int idScope; @@ -69,14 +70,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { final int idScope; @@ -68,14 +69,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { Comparator timeCompare = Comparator.nullsFirst(Comparator.naturalOrder()); int idScope; @@ -77,14 +78,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { final int idScope; final boolean storeValidTime; @@ -87,14 +88,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { private static final int MIN_MEM_SIZE = 8+12+12; @@ -60,14 +61,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { private static final int MEM_SIZE = 8+12+8; // long ID + instant + long ID @@ -73,14 +74,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { final int idScope; @@ -69,14 +70,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { private static final int MEM_SIZE = 8; // long ID @@ -56,14 +57,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { final int idScope; @@ -88,14 +89,6 @@ public static void encode(WriteBuffer wbuf, MVTimeSeriesRecordKey key) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { private static final int MIN_MEM_SIZE = 8+8+4; @@ -74,14 +75,6 @@ public void write(WriteBuffer wbuf, Object obj) } - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i implements ObjectArrayDataType { private final static Object VALUE = new Object(); @@ -53,26 +54,15 @@ public void write(WriteBuffer wbuf, Object obj) { } - - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - } @Override public Object read(ByteBuffer buf) { // always return a value even though nothing is actually written to file - // this is needed because MVMap.contains() method checks that value is not null + // this is needed because MVMap.containsKey() checks that value is not null return VALUE; } - - @Override - public void read(ByteBuffer buf, Object[] obj, int len, boolean key) - { - for (int i=0; i +public class KryoDataType extends BasicDataType implements ObjectArrayDataType { static Logger log = LoggerFactory.getLogger(KryoDataType.class); @@ -283,10 +284,4 @@ public int compare(Object a, Object b) return 0; } - @Override - public Object[] createStorage(int size) - { - return new Object[size]; - } - } From 859f9e77fe7ede6624bb419948f0942969542491 Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Wed, 16 Sep 2026 18:25:31 -0500 Subject: [PATCH 05/12] Parameterize H2 datatype factories and spatial index constructor. Replace DataType signatures to preserve type inference when building MVStore maps. --- .../main/java/org/h2/mvstore/MVBTreeMap.java | 88 ++++++++----------- .../datastore/h2/MVBaseFeatureStoreImpl.java | 2 +- .../datastore/h2/MVBaseResourceStoreImpl.java | 4 +- .../datastore/h2/MVDeploymentStoreImpl.java | 2 +- .../datastore/h2/MVProcedureStoreImpl.java | 2 +- .../datastore/h2/MVPropertyStoreImpl.java | 4 +- .../impl/datastore/h2/MVSystemStoreImpl.java | 2 +- .../impl/datastore/h2/index/SpatialIndex.java | 2 +- 8 files changed, 46 insertions(+), 60 deletions(-) diff --git a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVBTreeMap.java b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVBTreeMap.java index 51e6e123b..4acae6ed9 100644 --- a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVBTreeMap.java +++ b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVBTreeMap.java @@ -20,7 +20,6 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.h2.mvstore.type.DataType; -import org.h2.mvstore.type.ObjectDataType; /** @@ -36,9 +35,23 @@ public class MVBTreeMap extends MVMap { - protected MVBTreeMap(Map config) + protected MVBTreeMap(Map config, DataType keyType, DataType valueType) { - super(config); + super(config, keyType, valueType); + } + + + protected MVBTreeMap(MVBTreeMap source) + { + super(source); + } + + + @Override + protected MVBTreeMap cloneIt() + { + // H2 uses this hook to create read-only version snapshots. + return new MVBTreeMap<>(this); } @@ -53,9 +66,9 @@ public K getFullKey(Object key) * This can be used when a partial key is used for retrieval */ @SuppressWarnings({ "unchecked" }) - private K getFullKey(Page p, Object key) + private K getFullKey(Page p, Object key) { - int x = p.binarySearch(key); + int x = p.binarySearch((K)key); if (!p.isLeaf()) { if (x < 0) { x = -x - 1; @@ -66,7 +79,7 @@ private K getFullKey(Page p, Object key) return getFullKey(p, key); } if (x >= 0) { - return (K)p.getKey(x); + return p.getKey(x); } return null; } @@ -82,9 +95,9 @@ public Entry getEntry(Object key) * Same code as binarySearch method but returning full entry */ @SuppressWarnings({ "unchecked" }) - private Entry getEntry(Page p, Object key) + private Entry getEntry(Page p, Object key) { - int x = p.binarySearch(key); + int x = p.binarySearch((K)key); if (!p.isLeaf()) { if (x < 0) { x = -x - 1; @@ -95,7 +108,7 @@ private Entry getEntry(Page p, Object key) return getEntry(p, key); } if (x >= 0) { - return new SimpleEntry<>((K)p.getKey(x), (V)p.getValue(x)); + return new SimpleEntry<>(p.getKey(x), p.getValue(x)); } return null; } @@ -128,8 +141,7 @@ public Entry lowerEntry(K key) /* * Same code as getMinMax method but returning full entry */ - @SuppressWarnings("unchecked") - private Entry getMinMaxEntry(Page p, K key, boolean min, boolean excluding) { + private Entry getMinMaxEntry(Page p, K key, boolean min, boolean excluding) { int x = p.binarySearch(key); if (p.isLeaf()) { if (x < 0) { @@ -140,7 +152,7 @@ private Entry getMinMaxEntry(Page p, K key, boolean min, boolean excluding if (x < 0 || x >= p.getKeyCount()) { return null; } - return new SimpleEntry<>((K)p.getKey(x), (V)p.getValue(x)); + return new SimpleEntry<>(p.getKey(x), p.getValue(x)); } if (x++ < 0) { x = -x; @@ -171,62 +183,36 @@ public Stream keyStream() * @param the key type * @param the value type */ - public static class Builder implements MapBuilder, K, V> + public static class Builder extends MVMap.BasicBuilder, K, V> { - protected DataType keyType; - protected DataType valueType; - - - public Builder keyType(DataType keyType) + public Builder() { - this.keyType = keyType; - return this; } - public Builder valueType(DataType valueType) - { - this.valueType = valueType; - return this; - } @Override - public DataType getKeyType() + public Builder keyType(DataType keyType) { - return keyType; + setKeyType(keyType); + return this; } - @Override - public DataType getValueType() - { - return valueType; - } @Override - public void setKeyType(DataType dataType) + public Builder valueType(DataType valueType) { - this.keyType = dataType; + setValueType(valueType); + return this; } - @Override - public void setValueType(DataType dataType) - { - this.valueType = dataType; - } @Override - public MVBTreeMap create(MVStore store, Map config) + protected MVBTreeMap create(Map config) { - if (keyType == null) - keyType = new ObjectDataType(); - - if (valueType == null) - valueType = new ObjectDataType(); - - config.put("store", store); - config.put("key", keyType); - config.put("val", valueType); - - return new MVBTreeMap<>(config); + Object type = config.get("type"); + if (type == null) + return new MVBTreeMap<>(config, getKeyType(), getValueType()); + throw new IllegalArgumentException("Incompatible map type"); } } } diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseFeatureStoreImpl.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseFeatureStoreImpl.java index 5313b9c7c..c97db290b 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseFeatureStoreImpl.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseFeatureStoreImpl.java @@ -196,7 +196,7 @@ protected SpatialKey getSpatialKey(MVFeatureParentKey key, IFeature f) } - protected DataType getFeatureDataType(MVMap kryoClassMap) + protected DataType getFeatureDataType(MVMap kryoClassMap) { return new FeatureDataType(kryoClassMap); } diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseResourceStoreImpl.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseResourceStoreImpl.java index 1787e29fd..8e315dc7d 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseResourceStoreImpl.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseResourceStoreImpl.java @@ -106,9 +106,9 @@ protected MVBaseResourceStoreImpl init(MVStore mvStore, int idScope } - protected abstract DataType getResourceKeyDataType(int idScope); + protected abstract DataType getResourceKeyDataType(int idScope); - protected abstract DataType getResourceDataType(MVMap kryoClassMap, int idScope); + protected abstract DataType getResourceDataType(MVMap kryoClassMap, int idScope); protected abstract K generateKey(V f); diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVDeploymentStoreImpl.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVDeploymentStoreImpl.java index 9d0480d1a..70ef8e310 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVDeploymentStoreImpl.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVDeploymentStoreImpl.java @@ -87,7 +87,7 @@ public static MVDeploymentStoreImpl open(MVStore mvStore, int idScope, IdProvide @Override - protected DataType getFeatureDataType(MVMap kryoClassMap) + protected DataType getFeatureDataType(MVMap kryoClassMap) { return new SensorMLDataType(kryoClassMap); } diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVProcedureStoreImpl.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVProcedureStoreImpl.java index af952e37e..9bf2cd352 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVProcedureStoreImpl.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVProcedureStoreImpl.java @@ -83,7 +83,7 @@ public static MVProcedureStoreImpl open(MVStore mvStore, int idScope, IdProvider @Override - protected DataType getFeatureDataType(MVMap kryoClassMap) + protected DataType getFeatureDataType(MVMap kryoClassMap) { return new SensorMLDataType(kryoClassMap); } diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVPropertyStoreImpl.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVPropertyStoreImpl.java index 88153ed8a..deddfd552 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVPropertyStoreImpl.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVPropertyStoreImpl.java @@ -104,14 +104,14 @@ protected MVPropertyStoreImpl init(MVStore mvStore, int idScope, IdProviderType @Override - protected DataType getResourceKeyDataType(int idScope) + protected DataType getResourceKeyDataType(int idScope) { return new MVPropertyKeyDataType(idScope); } @Override - protected DataType getResourceDataType(MVMap kryoClassMap, int idScope) + protected DataType getResourceDataType(MVMap kryoClassMap, int idScope) { return new PropertyDataType(kryoClassMap); } diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVSystemStoreImpl.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVSystemStoreImpl.java index 11d97f6e3..a3fea3b97 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVSystemStoreImpl.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVSystemStoreImpl.java @@ -91,7 +91,7 @@ public static MVSystemStoreImpl open(MVStore mvStore, int idScope, IdProviderTyp @Override - protected DataType getFeatureDataType(MVMap kryoClassMap) + protected DataType getFeatureDataType(MVMap kryoClassMap) { return new SensorMLDataType(kryoClassMap); } diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/SpatialIndex.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/SpatialIndex.java index 92e03a946..f3a00371c 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/SpatialIndex.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/SpatialIndex.java @@ -49,7 +49,7 @@ public abstract class SpatialIndex> protected abstract SpatialKey getSpatialKey(K key, T resource); - public SpatialIndex(MVStore mvStore, String mapName, DataType valueType) + public SpatialIndex(MVStore mvStore, String mapName, DataType valueType) { this.rTreeMap = mvStore.openMap(mapName, new MVRTreeMap.Builder() .dimensions(3) From d8946296340806dcb10ab2d10784456dc6fc1f1d Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Wed, 16 Sep 2026 19:48:48 -0500 Subject: [PATCH 06/12] Replace SpatialKey with Spatial and create bounds through a private SpatialDataType factory. --- .../java/org/h2/mvstore/RTreeEntryCursor.java | 32 +++++++++---------- .../sensorhub/impl/datastore/h2/H2Utils.java | 29 +++++++++++++---- .../datastore/h2/MVBaseFeatureStoreImpl.java | 4 +-- .../impl/datastore/h2/index/SpatialIndex.java | 16 +++++----- 4 files changed, 49 insertions(+), 32 deletions(-) diff --git a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RTreeEntryCursor.java b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RTreeEntryCursor.java index 742c73876..4d150654c 100644 --- a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RTreeEntryCursor.java +++ b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RTreeEntryCursor.java @@ -17,7 +17,7 @@ import java.util.Iterator; import java.util.Map.Entry; import java.util.NoSuchElementException; -import org.h2.mvstore.rtree.SpatialKey; +import org.h2.mvstore.rtree.Spatial; /** @@ -31,16 +31,16 @@ * @author Alex Robin * @since Jan 23, 2022 */ -public class RTreeEntryCursor implements Iterator> { +public class RTreeEntryCursor implements Iterator> { - private final SpatialKey filter; - private CursorPos pos; - private SpatialKey currentKey, lastKey; + private final Spatial filter; + private CursorPos pos; + private Spatial currentKey, lastKey; private V currentVal, lastVal; - private final Page root; + private final Page root; private boolean initialized; - protected RTreeEntryCursor(Page root, SpatialKey filter) { + protected RTreeEntryCursor(Page root, Spatial filter) { this.root = root; this.filter = filter; } @@ -49,7 +49,7 @@ protected RTreeEntryCursor(Page root, SpatialKey filter) { public boolean hasNext() { if (!initialized) { // init - pos = new CursorPos(root, 0, null); + pos = new CursorPos<>(root, 0, null); fetchNext(); initialized = true; } @@ -69,7 +69,7 @@ public void skip(long n) { } @Override - public Entry next() { + public Entry next() { if (!hasNext()) { throw new NoSuchElementException(); } @@ -91,14 +91,14 @@ public void remove() { @SuppressWarnings("unchecked") protected void fetchNext() { while (pos != null) { - Page p = pos.page; + Page p = pos.page; if (p.isLeaf()) { while (pos.index < p.getKeyCount()) { int index = pos.index++; - SpatialKey c = (SpatialKey) p.getKey(index); + Spatial c = p.getKey(index); if (filter == null || check(true, c, filter)) { currentKey = c; - currentVal = (V)p.getValue(index); + currentVal = p.getValue(index); return; } } @@ -106,10 +106,10 @@ protected void fetchNext() { boolean found = false; while (pos.index < p.getKeyCount()) { int index = pos.index++; - SpatialKey c = (SpatialKey) p.getKey(index); + Spatial c = p.getKey(index); if (filter == null || check(false, c, filter)) { - Page child = pos.page.getChildPage(index); - pos = new CursorPos(child, 0, pos); + Page child = pos.page.getChildPage(index); + pos = new CursorPos<>(child, 0, pos); found = true; break; } @@ -133,7 +133,7 @@ protected void fetchNext() { * @return true if there is a match */ @SuppressWarnings("unused") - protected boolean check(boolean leaf, SpatialKey key, SpatialKey test) { + protected boolean check(boolean leaf, Spatial key, Spatial test) { return true; } diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java index 9663a3ad1..7853352de 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java @@ -21,7 +21,8 @@ import org.h2.mvstore.MVMap; import org.h2.mvstore.MVStore; import org.h2.mvstore.WriteBuffer; -import org.h2.mvstore.rtree.SpatialKey; +import org.h2.mvstore.rtree.Spatial; +import org.h2.mvstore.rtree.SpatialDataType; import org.sensorhub.api.datastore.TemporalFilter; import org.sensorhub.impl.datastore.h2.kryo.KryoDataType; import org.vast.util.Asserts; @@ -40,12 +41,28 @@ public class H2Utils static final String DATASTORES_MAP_NAME = "@datastores"; static final String GEOM_DIM_ERROR = "Only 2D and 3D geometries are supported"; + private static final SpatialFactory SPATIAL_FACTORY = new SpatialFactory(); public static final Range ALL_TIMES_RANGE = Range.closed(Instant.MIN, Instant.MAX); public static final TemporalFilter ALL_TIMES_FILTER = new TemporalFilter.Builder() .withRange(Instant.MIN, Instant.MAX) .build(); + private static final class SpatialFactory extends SpatialDataType + { + private SpatialFactory() + { + super(3); + } + + @Override + public Spatial create(long id, float... minMax) + { + return super.create(id, minMax); + } + } + + static class Holder { public T value; @@ -219,7 +236,7 @@ else if (lower == null) } - public static SpatialKey getBoundingRectangle(long id, AbstractGeometry geom) + public static Spatial getBoundingRectangle(long id, AbstractGeometry geom) { float[] minMaxCoords = null; int numDims = -1; @@ -261,21 +278,21 @@ else if (geom instanceof Polygon) } if (minMaxCoords != null) - return new SpatialKey(id, minMaxCoords); + return SPATIAL_FACTORY.create(id, minMaxCoords); else return null; } - public static SpatialKey getBoundingRectangle(long id, Geometry geom) + public static Spatial getBoundingRectangle(long id, Geometry geom) { return getBoundingRectangle(id, geom.getEnvelopeInternal()); } - public static SpatialKey getBoundingRectangle(long id, Envelope env) + public static Spatial getBoundingRectangle(long id, Envelope env) { - return new SpatialKey(id, + return SPATIAL_FACTORY.create(id, Math.nextDown((float)env.getMinX()), Math.nextUp((float)env.getMaxX()), Math.nextDown((float)env.getMinY()), diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseFeatureStoreImpl.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseFeatureStoreImpl.java index c97db290b..64db46e3c 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseFeatureStoreImpl.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVBaseFeatureStoreImpl.java @@ -33,7 +33,7 @@ import org.h2.mvstore.MVMap.Decision; import org.h2.mvstore.MVStore; import org.h2.mvstore.RangeCursor; -import org.h2.mvstore.rtree.SpatialKey; +import org.h2.mvstore.rtree.Spatial; import org.h2.mvstore.type.DataType; import org.sensorhub.api.common.BigId; import org.sensorhub.api.datastore.DataStoreException; @@ -178,7 +178,7 @@ protected MVBaseFeatureStoreImpl init(MVStore mvStore, int idScope, Id mapName = dataStoreInfo.getName() + ":" + FEATURE_SPATIAL_INDEX_MAP_NAME; this.spatialIndex = new SpatialIndex<>(mvStore, mapName, new MVFeatureParentKeyDataType(idScope, true)) { @Override - protected SpatialKey getSpatialKey(MVFeatureParentKey key, IFeature f) + protected Spatial getSpatialKey(MVFeatureParentKey key, IFeature f) { if (f.getGeometry() == null) return null; diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/SpatialIndex.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/SpatialIndex.java index f3a00371c..ff83dd4dd 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/SpatialIndex.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/SpatialIndex.java @@ -22,7 +22,7 @@ import org.h2.mvstore.Page; import org.h2.mvstore.RTreeEntryCursor; import org.h2.mvstore.rtree.MVRTreeMap; -import org.h2.mvstore.rtree.SpatialKey; +import org.h2.mvstore.rtree.Spatial; import org.h2.mvstore.rtree.SpatialDataType; import org.h2.mvstore.type.DataType; import org.sensorhub.api.datastore.SpatialFilter; @@ -46,7 +46,7 @@ public abstract class SpatialIndex> protected MVRTreeMap rTreeMap; - protected abstract SpatialKey getSpatialKey(K key, T resource); + protected abstract Spatial getSpatialKey(K key, T resource); public SpatialIndex(MVStore mvStore, String mapName, DataType valueType) @@ -59,7 +59,7 @@ public SpatialIndex(MVStore mvStore, String mapName, DataType valueTy public void add(K key, T res) { - SpatialKey spatialKey = getSpatialKey(key, res); + Spatial spatialKey = getSpatialKey(key, res); if (spatialKey != null) rTreeMap.put(spatialKey, key); } @@ -74,7 +74,7 @@ public void update(K key, T old, T new_) public void remove(K key, T res) { - SpatialKey spatialKey = getSpatialKey(key, res); + Spatial spatialKey = getSpatialKey(key, res); if (spatialKey != null) rTreeMap.remove(spatialKey); } @@ -82,7 +82,7 @@ public void remove(K key, T res) public Stream selectKeys(SpatialFilter filter) { - SpatialKey bbox = H2Utils.getBoundingRectangle(0, filter.getRoi()); + Spatial bbox = H2Utils.getBoundingRectangle(0, filter.getRoi()); /*final RTreeCursor geoCursor = rTreeMap.findIntersectingKeys(bbox); return StreamSupport.stream(Spliterators.spliteratorUnknownSize(geoCursor, Spliterator.DISTINCT), true) @@ -91,7 +91,7 @@ public Stream selectKeys(SpatialFilter filter) var rTreeKeyType = ((SpatialDataType)rTreeMap.getKeyType()); var geoCursor = new RTreeEntryCursor(rTreeMap.getRoot().root, bbox) { @Override - protected boolean check(boolean leaf, SpatialKey key, SpatialKey test) + protected boolean check(boolean leaf, Spatial key, Spatial test) { return rTreeKeyType.isOverlap(key, test); } @@ -109,10 +109,10 @@ public Bbox getFullExtent() { Bbox extent = new Bbox(); - Page root = rTreeMap.getRoot().root; + Page root = rTreeMap.getRoot().root; for (int i = 0; i < root.getKeyCount(); i++) { - SpatialKey key = (SpatialKey)root.getKey(i); + Spatial key = root.getKey(i); extent.add(new Bbox(key.min(0), key.min(1), key.min(2), key.max(0), key.max(1), key.max(2))); } From 166dff50ee3e9429e5fe056c2a3f9044161af6f2 Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Wed, 16 Sep 2026 19:55:36 -0500 Subject: [PATCH 07/12] remove unused radix files and classes and clean up old references to them --- .../java/org/h2/mvstore/MVRadixTreeMap.java | 1015 ----------------- .../org/h2/mvstore/RadixKeyBytesDataType.java | 188 --- .../java/org/h2/mvstore/RadixKeyDataType.java | 81 -- .../java/org/h2/mvstore/RadixTreeCursor.java | 182 --- .../datastore/h2/index/FullTextIndex.java | 26 - .../h2/index/ResourceRadixKeyDataType.java | 86 -- .../impl/datastore/h2/PrintDbInfo.java | 1 - 7 files changed, 1579 deletions(-) delete mode 100644 sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVRadixTreeMap.java delete mode 100644 sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RadixKeyBytesDataType.java delete mode 100644 sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RadixKeyDataType.java delete mode 100644 sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RadixTreeCursor.java delete mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/ResourceRadixKeyDataType.java diff --git a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVRadixTreeMap.java b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVRadixTreeMap.java deleted file mode 100644 index 67d50cbc6..000000000 --- a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/MVRadixTreeMap.java +++ /dev/null @@ -1,1015 +0,0 @@ -/***************************** BEGIN LICENSE BLOCK *************************** - -The contents of this file are subject to the Mozilla Public License, v. 2.0. -If a copy of the MPL was not distributed with this file, You can obtain one -at http://mozilla.org/MPL/2.0/. - -Software distributed under the License is distributed on an "AS IS" basis, -WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -for the specific language governing rights and limitations under the License. - -Copyright (C) 2020 Sensia Software LLC. All Rights Reserved. - -******************************* END LICENSE BLOCK ***************************/ - -package org.h2.mvstore; - -import java.util.AbstractSet; -import java.util.Deque; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.Map; -import java.util.Set; -import org.h2.mvstore.Page.PageReference; -import org.h2.mvstore.type.DataType; -import org.h2.mvstore.type.ObjectDataType; -import org.vast.util.Asserts; - - -/** - *

- * Implementation of a radix tree for use as an H2 MVStore map.
- * The last child of a page is used to store the reference to a binary tree - * that contains the values. - * - * WARNING: This is NOT working since H2 MVStore version 1.4.198 - *

- * - * @author Alex Robin - * @param Type of key usable with this map - * @param Type of value usable with this map - * @date Oct 9, 2018 - */ -public class MVRadixTreeMap extends MVMap -{ - - static class SearchNode - { - Page page; - int childIndex; - - SearchNode(Page page, int childIndex) - { - this.page = page; - this.childIndex = childIndex; - } - - public String toString() - { - return new String((byte[])page.getKey(childIndex)); - } - } - - /* - * Class used to store compare and search results - */ - static class SearchContext - { - // current offset in full key where to start comparison with prefix - int offset; - - // comparison result flag: 0 if prefix match, -1 if smaller, 1 if bigger - int compare; - - // length of match between prefix and full key, starting at offset - int matchLength; - - // length remaining in key for further matching - int keyRemaining; - - // true if node key is present in its entirety in full key at offset - boolean fullNodeMatch; - - // found index within page (negative if no exact match found) - int index; - - // found page - // this is the page attached to index if index>=0 - // otherwise it is the parent page - Page page; - - // stacks containing info of each node along the path to the selected key - Deque nodeStack = new LinkedList<>(); - - Page getSelectedPage() - { - if (!nodeStack.isEmpty()) - { - SearchNode lastNode = nodeStack.peek(); - if (lastNode.childIndex < 0) - return lastNode.page; - else - return lastNode.page.getChildPage(lastNode.childIndex); - } - else - return page.getChildPage(index); - } - - boolean isExactMatch() - { - return fullNodeMatch && index >= 0; - } - } - - - MVRadixTreeMap(Map config) - { - super(config); - } - - - /*@Override - void setRootPos(long rootPos, long version) - { - root = rootPos == 0 ? createEmptyNode(-1) : readPage(rootPos); - root.setVersion(version); - }*/ - - - @Override - public V get(Page root, Object key) - { - Page valuePage = getValuePage(root, key); - - if (valuePage != null) - return getFirstValue(valuePage); - else - return null; - } - - - /* - * Look for value key in B-Tree pages - */ - protected boolean containsValueKey(Page p, Object key) { - int x = p.binarySearch(key); - if (!p.isLeaf()) { - if (x < 0) { - x = -x - 1; - } else { - x++; - } - p = p.getChildPage(x); - return containsValueKey(p, key); - } - if (x >= 0) { - return true; - } - return false; - } - - - public boolean containsValue(Object key, V value) - { - Page valuePage = getValuePage(getRootPage(), key); - if (valuePage == null) - return false; - - return containsValueKey(valuePage, getValueKey(value)); - } - - - protected Page getValuePage(Page root, Object key) - { - // use prefix search to look for matching key index and page - SearchContext context = new SearchContext(); - prefixBinarySearch(root, key, context, false); - - // if exact match was found, return the first leaf value, if any - if (context.isExactMatch()) - return getValuePage(context.getSelectedPage()); - else - return null; - } - - - protected Page getValuePage(Page p) - { - // values are always in the last child page - int valuesNodeIndex = p.getKeyCount(); - if (p.getCounts(valuesNodeIndex) > 0) - return p.getChildPage(valuesNodeIndex); - - return null; - } - - - protected boolean hasValues(Page p) - { - // values are always in the last child page (after the last key) - int valuesNodeIndex = p.getKeyCount(); - return (p.getCounts(valuesNodeIndex) > 0); - } - - - @SuppressWarnings("unchecked") - protected V getFirstValue(Page valuePage) - { - while (!valuePage.isLeaf()) - valuePage = valuePage.getChildPage(0); - - if (valuePage.getKeyCount() == 0) - return null; - - return (V)valuePage.getValue(0); - } - - - /* - * Look recursively for the given prefix in the page tree - */ - protected void prefixBinarySearch(Page p, Object key, SearchContext context, boolean recordStack) - { - checkKeyNotNull(key); - - int x = prefixBinarySearchInPage(p, key, context); - context.index = x; - context.page = p; - - // return if key cannot match anymore - if (x < 0) - return; - - // add result to context - Object childPrefix = p.getKey(x); - Page childPage = p.getChildPage(x); - - // record stack for traversal - if (recordStack) - context.nodeStack.push(new SearchNode(p, x)); - - // stop here if done matching prefix - if (context.keyRemaining == 0 || !context.fullNodeMatch) - return; - - // increase offset for next prefix comparison - context.offset = context.offset + getPrefixLength(childPrefix); - prefixBinarySearch(childPage, key, context, recordStack); - } - - - /* - * Find the closest key in the page using binary search - * A positive result indicates an exact match with a key at the given index - * A negative result gives the index where the key could be inserted, minus one - */ - protected int prefixBinarySearchInPage(Page p, Object key, SearchContext context) - { - int low = 0; - int high = p.getKeyCount() - 1; - - while (low <= high) - { - int x = (low + high) >>> 1; - comparePrefix(key, p.getKey(x), context); - if (context.compare > 0) - low = x + 1; - else if (context.compare < 0) - high = x - 1; - else - return x; - } - - return -(low + 1); - } - - - public long getChildCount(K key) - { - SearchContext context = new SearchContext(); - prefixBinarySearch(getRootPage(), key, context, false); - if (context.keyRemaining == 0 && context.index >= 0) - return context.page.getCounts(context.index); - else - return 0; - } - - - /*@Override - protected K getFirstLast(Page root, boolean first) - { - SearchContext context = new SearchContext(); - if (getFirstLast(getRootPage(), first, true, context)) - return buildFullKey(context); - else - return null; - }*/ - - - /* - * Find first or last key in a tree branch starting from the given page - */ - protected boolean getFirstLast(Page p, boolean first, boolean excludeSelf, SearchContext context) - { - boolean exclude = excludeSelf; - - while (p != null) - { - if (!exclude && (first || p.getKeyCount() == 0) && hasValues(p)) - { - context.nodeStack.push(new SearchNode(p, -1)); - return true; - } - - exclude = false; - - // stop here if no more children - if (p.getKeyCount() == 0) - return false; - - // get next child index to scan - int childIndex = first ? 0 : p.getKeyCount()-1; - - // add to stack as we go down - context.nodeStack.push(new SearchNode(p, childIndex)); - - p = p.getChildPage(childIndex); - } - - return false; - } - - - /*@Override - protected K getMinMax(K key, boolean min, boolean excluding) - { - SearchContext context = new SearchContext(); - return getPreviousOrNext(key, min, excluding, context); - }*/ - - - protected K getPreviousOrNext(K key, boolean previous, boolean excluding, SearchContext context) - { - // use prefix search to look for matching key index and page - prefixBinarySearch(getRootPage(), key, context, true); - boolean exactMatch = context.isExactMatch(); - boolean found = false; - - // exact match - if (exactMatch) - { - Page selectedPage = context.getSelectedPage(); - - // if next requested, we just go down - if (!previous || !excluding && hasValues(selectedPage)) - found = getFirstLast(selectedPage, true, excluding, context); - - // else we need to search siblings or parents - else - found = getPreviousOrNext(context, previous); - } - - // partial match - else if (context.index >= 0) - { - Page selectedPage = context.getSelectedPage(); - - // need to compare reminder of prefix to know what direction to take - Object[] split = splitKey(context.page.getKey(context.index), context.matchLength); - Object prefixEnd = split[1]; - - if (context.keyRemaining > 0) - { - context.offset += context.matchLength; - comparePrefix(key, prefixEnd, context); - } - - // key is after prefix - if (context.compare > 0 && previous) - found = getFirstLast(selectedPage, false, false, context); - - // key is before prefix - else if ((context.keyRemaining == 0 || context.compare < 0) && !previous) - found = getFirstLast(selectedPage, true, false, context); - - else - found = getPreviousOrNext(context, previous); - } - - // else need to navigate the tree - else - { - int index = -context.index + (previous ? -1 : -2); - context.nodeStack.push(new SearchNode(context.page, index)); - found = getPreviousOrNext(context, previous); - } - - // return full key if something found - if (found) - return buildFullKey(context); - else - return null; - } - - - /* - * Navigate the tree to find next or previous key - */ - protected boolean getPreviousOrNext(SearchContext context, boolean previous) - { - SearchNode node = context.nodeStack.peek(); - if (node == null) - return false; - - Page p = node.page; - int nextIndex = previous ? node.childIndex-1 : node.childIndex+1; - - if (nextIndex >= 0 && nextIndex < p.getKeyCount()) - { - node.childIndex = nextIndex; - Page childPage = p.getChildPage(nextIndex); - return getFirstLast(childPage, !previous, false, context); - } - else if (previous && nextIndex == -1 && hasValues(p)) - { - // select this node since it has values - context.nodeStack.pop(); - return true; - } - else - { - if (!getPreviousOrNextPageUp(context, previous)) - return false; - return getPreviousOrNext(context, previous); - } - } - - - /* - * Navigate up the tree until we find a branch going in the right direction. - */ - protected boolean getPreviousOrNextPageUp(SearchContext context, boolean previous) - { - while (!context.nodeStack.isEmpty()) - { - SearchNode node = context.nodeStack.peek(); - Page p = node.page; - int nextIndex = previous ? node.childIndex-1 : node.childIndex+1; - - if (nextIndex >= 0 && nextIndex < p.getKeyCount()) - return true; - else if (previous && nextIndex == -1 && hasValues(p)) - return true; - - context.nodeStack.pop(); - } - - return false; - } - - - @Override - @SuppressWarnings("unchecked") - public synchronized V put(K key, V value) - { - DataUtils.checkArgument(value != null, "The value may not be null"); - beforeWrite(); - RootReference rootReference = flushAndGetRoot(); - long v = rootReference.version; - Page p = rootReference.root.copy(); - Object result = put(p, v, key, value, new SearchContext()); - rootReference.updateRootPage(p, 1); - return (V) result; - } - - - protected Object put(Page p, long writeVersion, Object key, Object value, SearchContext context) - { - checkKeyNotNull(key); - - int index = prefixBinarySearchInPage(p, key, context); - Page childPage; - - // no prefix match, need to create a new node - if (index < 0) - { - index = -index - 1; - childPage = createEmptyNode(writeVersion); - addValue(childPage, value); - - Object keyPart = tailKey(key, context.offset); - if (keyPart == null) - return null; - - p.insertNode(index, keyPart, childPage); - return null; - } - - // else found prefix match - else - { - // if entire node key was matched - if (context.fullNodeMatch) - { - // copy selected page because it's going to be modified - childPage = p.getChildPage(index).copy(); - - // if exact match, add value to this page - // else if end of key still has to be processed, keep going down - if (context.keyRemaining == 0) - { - addValue(childPage, value); - p.setChild(index, childPage); - return null; - } - else - { - // increase offset for next prefix comparison - context.offset = context.offset + getPrefixLength(p.getKey(index)); - Object val = put(childPage, writeVersion, key, value, context); - p.setChild(index, childPage); // do this here to properly update total count - return val; - } - } - - // else need to split node key - else - { - Page oldPage = p.getChildPage(index); - Object[] split = splitKey(p.getKey(index), context.matchLength); - Object prefixBegin = split[0]; - Object prefixEnd = split[1]; - - // create new page to insert - childPage = createEmptyNode(writeVersion); - childPage.insertNode(0, prefixEnd, oldPage); - - // insert at selected index with shorter key - p.setChild(index, childPage); - p.setKey(index, prefixBegin); - - return put(p, writeVersion, key, value, context); - } - } - } - - - protected void addValue(Page p, Object value) - { - int valuesNodeIndex = p.getKeyCount(); // it's always the last child page - long valueCount = p.getCounts(valuesNodeIndex); - Page valuePage; - - // create the leaf page if needed - if (valueCount == 0) - { - valuePage = Page.createLeaf( - this, - new Object[] {getValueKey(value)}, - new Object[] {value}, - 0); - - p.setChild(valuesNodeIndex, valuePage); - } - else - { - valuePage = p.getChildPage(valuesNodeIndex).copy(); - - // split if value page is still a leaf page and reached memory threshold - // we need this because super.put doesn't handle splitting a top level leaf page - if (valuePage.isLeaf() && valuePage.getMemory() > store.getPageSplitSize() && valuePage.getKeyCount() > 1) - { - long numValues = valuePage.getTotalCount(); - Page c1 = valuePage; - - int at = c1.getKeyCount() / 2; - Object k = c1.getKey(at); - Page c2 = c1.split(at); - valuePage = Page.createNode( - this, - new Object[] {k}, - new PageReference[] { - new PageReference(c1), - new PageReference(c2) - }, - numValues, 0); - } - - var valueKey = getValueKey(value); - int index = valuePage.binarySearch(valueKey); - valuePage.insertLeaf(-index - 1, valueKey, value); - - p.setChild(valuesNodeIndex, valuePage); // need to do that to properly update counts - } - } - - - @Override - @SuppressWarnings("unchecked") - public V remove(final Object key) - { - return (V) remove(key, null, false); - } - - - public boolean remove(final Object key, final boolean removeSubKeys) - { - return (remove(key, null, removeSubKeys) != null); - } - - - @Override - public synchronized boolean remove(final Object key, final Object value) - { - return (remove(key, value, false) != null); - } - - - protected synchronized Object remove(final Object key, final Object value, final boolean removeSubKeys) - { - checkKeyNotNull(key); - - // use prefix search to look for matching key index and page - SearchContext context = new SearchContext(); - prefixBinarySearch(getRootPage(), key, context, true); - - // continue only if exact match found - if (context.isExactMatch()) - { - beforeWrite(); - RootReference rootReference = flushAndGetRoot(); - long v = rootReference.version; - - synchronized (this) - { - Page p = context.getSelectedPage(); - Page newPage = null; - Object oldValue = null; - - // remove only value if specified - if (value != null && hasValues(p)) - { - newPage = p.copy(); - Page newValuePage = getValuePage(newPage).copy(); - int index = newValuePage.binarySearch(key); - oldValue = newValuePage.getValue(index); - newValuePage.remove(index); - newPage.setChild(newPage.getKeyCount(), newValuePage); - } - - // or remove key and all subkeys - else if (removeSubKeys) - { - oldValue = Boolean.TRUE; - p.removeAllRecursive(v); - } - - // otherwise remove all values for the selected key - else if (p.getKeyCount() > 0) - { - Page valuePage = getValuePage(p); - if (valuePage != null) - { - oldValue = getFirstValue(valuePage); - newPage = removeAllValues(p, v); - } - } - - // copy and update parent pages all the way to root - Page newChild = newPage; - Page parent = null; - for (SearchNode node: context.nodeStack) - { - parent = node.page.copy(); - if (newChild != null) - parent.setChild(node.childIndex, newChild); - else - parent.remove(node.childIndex); - newChild = parent; - } - - rootReference.updateRootPage(parent, 1); - return oldValue; - } - } - - return null; - } - - - protected Page removeAllValues(Page p, long version) - { - p = p.copy(); - - Page valuePage = getValuePage(p); - if (valuePage != null) - { - p.setChild(p.getKeyCount(), null); - valuePage.removeAllRecursive(version); - } - - return p; - } - - - /*@Override - public Cursor cursor(K from) - { - throw new UnsupportedOperationException("Use prefixCursor() or entryCursor() instead"); - }*/ - - - /** - * Get a cursor that iterates through all keys starting at the - * given key or key prefix - * @param startPrefix - * @return the cursor object to iterate through keys - */ - public RadixTreeCursor prefixCursor(K startPrefix) - { - return new RadixTreeCursor<>(this, getRootPage(), startPrefix); - } - - - /** - * Get a cursor that iterates through all entries associated to keys - * starting at the given key or key prefix (a key can be reported multiple - * times if several values are attached to it) - * @param startPrefix - * @return the cursor object to iterate through keys - */ - public RadixTreeCursor entryCursor(K startPrefix) - { - return new RadixTreeCursor<>(this, getRootPage(), startPrefix, true); - } - - - /*@Override - public Set> entrySet() - { - return new AbstractSet>() - { - @Override - public Iterator> iterator() - { - final RadixTreeCursor cursor = entryCursor(firstKey()); - return new Iterator>() - { - @Override - public boolean hasNext() - { - return cursor.hasNext(); - } - - @Override - public Entry next() - { - K k = cursor.next(); - return new SimpleEntry<>(k, cursor.getValue()); - } - - @Override - public void remove() - { - cursor.remove(); - } - }; - } - - @Override - public int size() - { - return MVRadixTreeMap.this.size(); - } - - @Override - public boolean contains(Object o) - { - return MVRadixTreeMap.this.containsKey(o); - } - }; - }*/ - - - @Override - public Set keySet() - { - return new AbstractSet() - { - @Override - public Iterator iterator() - { - return prefixCursor(firstKey()); - } - - @Override - public int size() - { - return MVRadixTreeMap.this.size(); - } - - @Override - public boolean contains(Object o) - { - return MVRadixTreeMap.this.containsKey(o); - } - }; - } - - - protected Object[] splitKey(Object prefix, int splitIndex) - { - return ((RadixKeyDataType) getKeyType()).splitKey(prefix, splitIndex); - } - - - protected Object tailKey(Object key, int offset) - { - return ((RadixKeyDataType) getKeyType()).tailKey(key, offset); - } - - - protected int getPrefixLength(Object prefix) - { - return ((RadixKeyDataType)getKeyType()).getPrefixLength(prefix); - } - - - protected void comparePrefix(Object fullKey, Object prefix, SearchContext context) - { - ((RadixKeyDataType)getKeyType()).comparePrefix(fullKey, prefix, context); - } - - - @SuppressWarnings("unchecked") - protected K buildFullKey(SearchContext context) - { - return (K) ((RadixKeyDataType)getKeyType()).buildFullKey(context); - } - - - protected Object getValueKey(Object value) - { - return ((RadixKeyDataType)getKeyType()).getValueKey(value); - } - - - protected Page createEmptyNode(long version) - { - return Page.createEmptyNode(this); - } - - - protected final void checkKeyNotNull(Object k) - { - Asserts.checkNotNull(k, "Key"); - } - - - protected String keyAsString(Object key) - { - if (key instanceof String) - return (String)key; - else if (key instanceof byte[]) - return new String((byte[])key); - else - return key.toString(); - } - - - public String asString() - { - StringBuilder buf = new StringBuilder(); - toString(getRootPage(), buf, 0); - return buf.toString(); - } - - - protected void toString(Page p, StringBuilder buf, int indent) - { - for (int i=0; i the key type - * @param the value type - */ - public static class Builder implements MapBuilder, K, V> - { - protected RadixKeyDataType keyType; - protected DataType valueType; - - - public Builder keyType(RadixKeyDataType keyType) - { - this.keyType = keyType; - return this; - } - - public Builder valueType(DataType valueType) - { - this.valueType = valueType; - return this; - } - - @Override - public DataType getKeyType() - { - return keyType; - } - - @Override - public DataType getValueType() - { - return valueType; - } - - @Override - public void setKeyType(DataType dataType) - { - this.keyType = (RadixKeyDataType)dataType; - } - - @Override - public void setValueType(DataType dataType) - { - this.valueType = dataType; - } - - @Override - public MVRadixTreeMap create(MVStore store, Map config) - { - Asserts.checkNotNull(keyType, RadixKeyDataType.class); - - if (valueType == null) - valueType = new ObjectDataType(); - - config.put("store", store); - config.put("key", keyType); - config.put("val", valueType); - - return new MVRadixTreeMap<>(config); - } - } - -} diff --git a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RadixKeyBytesDataType.java b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RadixKeyBytesDataType.java deleted file mode 100644 index 432950d74..000000000 --- a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RadixKeyBytesDataType.java +++ /dev/null @@ -1,188 +0,0 @@ -/***************************** BEGIN LICENSE BLOCK *************************** - -The contents of this file are subject to the Mozilla Public License, v. 2.0. -If a copy of the MPL was not distributed with this file, You can obtain one -at http://mozilla.org/MPL/2.0/. - -Software distributed under the License is distributed on an "AS IS" basis, -WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -for the specific language governing rights and limitations under the License. - -Copyright (C) 2020 Sensia Software LLC. All Rights Reserved. - -******************************* END LICENSE BLOCK ***************************/ - -package org.h2.mvstore; - -import java.nio.ByteBuffer; -import java.util.Iterator; -import org.h2.mvstore.MVRadixTreeMap.SearchContext; -import org.h2.mvstore.MVRadixTreeMap.SearchNode; -import com.google.common.primitives.UnsignedBytes; - - -/** - *

- * Base implementation for byte array prefix keys. - *

- * - * @author Alex Robin - * @date Oct 8, 2018 - */ -public abstract class RadixKeyBytesDataType implements RadixKeyDataType -{ - //Comparator comparator = UnsignedBytes.lexicographicalComparator(); - - - @Override - public int compare(Object key1, Object key2) - { - byte[] k1 = (byte[])key1; - byte[] k2 = (byte[])key2; - - int minLength = Math.min(k1.length, k2.length); - for (int i = 0; i < minLength; i++) - { - int result = UnsignedBytes.compare(k1[i], k2[i]); - if (result != 0) - return result; - } - - return k1.length - k2.length; - } - - - @Override - public void comparePrefix(Object fullKey, Object nodeKey, SearchContext context) - { - byte[] k = (byte[])fullKey; // full key to test - byte[] kp = (byte[])nodeKey; // key part contained in tree node - - // compare key with prefix lexicographically starting at offset - int i = context.offset, j = 0; - for (; j < kp.length && i < k.length && k[i] == kp[j]; i++, j++); - - if (k.length == 0) - context.compare = -1; - else if (kp.length == 0) - context.compare = 1; - else if (j == 0) - context.compare = k[i] < kp[0] ? -1 : 1; - else - context.compare = 0; - - context.matchLength = j; - context.keyRemaining = k.length - i; - context.fullNodeMatch = (j == kp.length); - } - - - @Override - public Object[] splitKey(Object nodeKey, int splitIndex) - { - byte[] key = (byte[])nodeKey; - if (splitIndex >= key.length) - throw new IllegalArgumentException("Split index out of bounds"); - - byte[] begin = new byte[splitIndex]; - System.arraycopy(key, 0, begin, 0, begin.length); - - byte[] end = new byte[key.length - splitIndex]; - System.arraycopy(key, splitIndex, end, 0, end.length); - - return new Object[] {begin, end}; - } - - - @Override - public Object tailKey(Object fullKey, int offset) - { - byte[] key = (byte[])fullKey; - if (key.length <= offset) - return null; - byte[] keyPart = new byte[key.length-offset]; - System.arraycopy(key, offset, keyPart, 0, keyPart.length); - return keyPart; - } - - - @Override - public int getPrefixLength(Object keyPrefix) - { - return ((byte[])keyPrefix).length; - } - - - @Override - public Object buildFullKey(SearchContext context) - { - int keySize = 0; - for (SearchNode node: context.nodeStack) - { - if (node.childIndex >= 0) - keySize += ((byte[])node.page.getKey(node.childIndex)).length; - } - - int destPos = 0; - byte[] fullKey = new byte[keySize]; - Iterator it = context.nodeStack.descendingIterator(); - while (it.hasNext()) - { - SearchNode node = it.next(); - if (node.childIndex >= 0) - { - byte[] prefixB = (byte[])node.page.getKey(node.childIndex); - System.arraycopy(prefixB, 0, fullKey, destPos, prefixB.length); - destPos += prefixB.length; - } - } - - return fullKey; - } - - - @Override - public int getMemory(Object obj) - { - return ((byte[])obj).length + 2; - } - - - @Override - public void write(WriteBuffer wbuf, Object obj) - { - byte[] key = (byte[])obj; - - // write array length, then byte[] - wbuf.putVarInt(key.length); - wbuf.put(key); - } - - - @Override - public void write(WriteBuffer wbuf, Object[] obj, int len, boolean key) - { - for (int i=0; i - * Base interface for Radix-tree Key DataTypes - *

- * - * @author Alex Robin - * @date Nov 2, 2018 - */ -public interface RadixKeyDataType extends DataType -{ - - /** - * Compare the part of the key starting at offset with a tree node prefix - * @param fullKey - * @param prefix - * @param context search context provides the offset and receives comparison - * results (compare flag and match length) - */ - void comparePrefix(Object fullKey, Object prefix, SearchContext context); - - - /** - * @param prefix Prefix stored in tree node - * @return Length of key prefix object - */ - int getPrefixLength(Object prefix); - - - /** - * Split prefix at given index - * @param prefix - * @param splitIndex - * @return array of length 2 containing begin and end of key/prefix - */ - Object[] splitKey(Object prefix, int splitIndex); - - - /** - * @param fullKey - * @param offset - * @return return end of key starting at offset - */ - Object tailKey(Object fullKey, int offset); - - - /** - * Generates the complete key from individual prefixes obtained from the - * search context (matching node stack) - * @param context - * @return The complete key - */ - Object buildFullKey(SearchContext context); - - - /** - * @param value value object being inserted or removed from the map - * @return the key to use to store this value in B-Tree page - */ - Object getValueKey(Object value); - -} diff --git a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RadixTreeCursor.java b/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RadixTreeCursor.java deleted file mode 100644 index 3b1f0c2fb..000000000 --- a/sensorhub-datastore-h2/src/main/java/org/h2/mvstore/RadixTreeCursor.java +++ /dev/null @@ -1,182 +0,0 @@ -/***************************** BEGIN LICENSE BLOCK *************************** - -The contents of this file are subject to the Mozilla Public License, v. 2.0. -If a copy of the MPL was not distributed with this file, You can obtain one -at http://mozilla.org/MPL/2.0/. - -Software distributed under the License is distributed on an "AS IS" basis, -WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -for the specific language governing rights and limitations under the License. - -Copyright (C) 2020 Sensia Software LLC. All Rights Reserved. - -******************************* END LICENSE BLOCK ***************************/ - -package org.h2.mvstore; - -import java.util.Iterator; -import java.util.NoSuchElementException; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import org.h2.mvstore.MVRadixTreeMap.SearchContext; -import org.vast.util.Asserts; - - -public class RadixTreeCursor implements Iterator -{ - private final MVRadixTreeMap map; - private final SearchContext context; - private final K from; - private K currentKey, nextKey; - private boolean initialized; - private boolean allValues; - private Cursor valueCursor; - - - RadixTreeCursor(MVRadixTreeMap map, Page root, K from) - { - this.map = map; - this.from = from; - this.context = new SearchContext(); - } - - - RadixTreeCursor(MVRadixTreeMap map, Page root, K from, boolean allValues) - { - this(map, root, from); - this.allValues = allValues; - } - - - @Override - public boolean hasNext() - { - if (nextKey == null) - fetchNext(); - - return nextKey != null; - } - - - @Override - public K next() - { - if (nextKey == null && !hasNext()) - throw new NoSuchElementException(); - - currentKey = nextKey; - nextKey = null; - return currentKey; - } - - - public K getKey() - { - return currentKey; - } - - - public V getValue() - { - Asserts.checkNotNull(currentKey); - - if (valueCursor == null) - return null; - else - return valueCursor.getValue(); - } - - - @Override - public void remove() - { - map.remove(getKey()); - } - - - private void fetchNext() - { - // get next value if any - if (valueCursor != null && valueCursor.hasNext()) - { - valueCursor.next(); - nextKey = currentKey; - return; - } - - // else go to next key - K key = null; - if (!initialized) - { - key = map.getPreviousOrNext(from, false, false, context); - initialized = true; - } - else - { - if (map.getPreviousOrNext(context, false)) - key = map.buildFullKey(context); - } - - if (key != null) - { - context.offset = 0; - map.comparePrefix(key, from, context); - if (context.fullNodeMatch) - { - nextKey = key; - setValueCursor(); - } - } - } - - - @SuppressWarnings("unchecked") - private void setValueCursor() - { - if (allValues) - { - Page selectedPage = context.getSelectedPage(); - Page valuePage = map.getValuePage(selectedPage); - valueCursor = new Cursor<>(valuePage, (K)valuePage.getKey(0)); - if (valueCursor.hasNext()) - valueCursor.next(); - } - } - - - public Stream keyStream() - { - allValues = false; - Spliterator it = Spliterators.spliteratorUnknownSize(this, Spliterator.ORDERED | Spliterator.DISTINCT); - return StreamSupport.stream(it, false); - } - - - /** - * @return The stream of selected values. - * (The returned stream can contain duplicated elements; call its distinct() - * method if duplicates cannot be handled by client) - */ - public Stream valueStream() - { - Spliterator it = Spliterators.spliteratorUnknownSize(new Iterator() { - @Override - public boolean hasNext() - { - return RadixTreeCursor.this.hasNext(); - } - - @Override - public V next() - { - RadixTreeCursor.this.next(); - return getValue(); - } - }, Spliterator.ORDERED); - - return StreamSupport.stream(it, false); - } - -} \ No newline at end of file diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/FullTextIndex.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/FullTextIndex.java index 5f4d285b0..4f642460b 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/FullTextIndex.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/FullTextIndex.java @@ -49,7 +49,6 @@ public class FullTextIndex> { protected static final Pattern NUMBER_PATTERN = Pattern.compile("^\\.?\\d+(\\.\\d+)?"); - //MVRadixTreeMap radixTreeMap; MVBTreeMap, Boolean> btreeMap; /* Default analyzer */ @@ -58,9 +57,6 @@ public class FullTextIndex> public FullTextIndex(MVStore mvStore, String mapName, DataType valueType) { - //this.radixTreeMap = mvStore.openMap(mapName, new MVRadixTreeMap.Builder() - // .keyType(new ResourceRadixKeyDataType()) - // .valueType(valueType)); this.btreeMap = mvStore.openMap(mapName, new MVBTreeMap.Builder, Boolean>() .keyType(new FullTextKeyDataType(valueType)) .valueType(new MVVoidDataType())); @@ -73,7 +69,6 @@ public void add(K key, T res) HashSet tokenSet = new HashSet<>(); addToTokenSet(res, tokenSet); for (String token: tokenSet) - //radixTreeMap.put(token.getBytes(), key); btreeMap.put(new FullTextKey<>(token, key), Boolean.TRUE); } @@ -90,17 +85,12 @@ public void remove(K key, T res) HashSet tokenSet = new HashSet<>(); addToTokenSet(res, tokenSet); for (String token: tokenSet) - //radixTreeMap.remove(token.getBytes(), key); btreeMap.put(new FullTextKey<>(token, key), Boolean.TRUE); } public Stream selectKeys(FullTextFilter filter) { - //return getKeywordStream(filter) - // .flatMap(w -> radixTreeMap.entryCursor(w.getBytes()).valueStream()) - // .distinct(); // remove duplicate IDs - return getKeywordStream(filter) .flatMap(w -> { var startKey = new FullTextKey(w, null); @@ -153,21 +143,6 @@ protected Stream getKeywordStream(FullTextFilter filter) public Stream addFullTextPostFilter(Stream pkStream, FullTextFilter filter) { - /*Set keywordSet = getKeywordStream(filter) - .map(String::getBytes) - .collect(Collectors.toSet()); - - return pkStream.filter(k -> { - if (k == null) - return false; - for (byte[] w: keywordSet) - { - if (radixTreeMap.containsValue(w, k)) - return true; - } - return false; - });*/ - var keywordSet = getKeywordStream(filter) .collect(Collectors.toSet()); @@ -187,7 +162,6 @@ public Stream addFullTextPostFilter(Stream pkStream, FullTextFilter filter public void clear() { - //radixTreeMap.clear(); btreeMap.clear(); } diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/ResourceRadixKeyDataType.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/ResourceRadixKeyDataType.java deleted file mode 100644 index 6ff32dec7..000000000 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/index/ResourceRadixKeyDataType.java +++ /dev/null @@ -1,86 +0,0 @@ -/***************************** BEGIN LICENSE BLOCK *************************** - -The contents of this file are subject to the Mozilla Public License, v. 2.0. -If a copy of the MPL was not distributed with this file, You can obtain one -at http://mozilla.org/MPL/2.0/. - -Software distributed under the License is distributed on an "AS IS" basis, -WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License -for the specific language governing rights and limitations under the License. - -Copyright (C) 2020 Sensia Software LLC. All Rights Reserved. - -******************************* END LICENSE BLOCK ***************************/ - -package org.sensorhub.impl.datastore.h2.index; - -import org.h2.mvstore.RadixKeyBytesDataType; -import org.sensorhub.api.resource.ResourceKey; - - -/** - *

- * Implementation with prefix key as byte array and value key as long. - *

- * - * @author Alex Robin - * @date Nov 1, 2018 - */ -public class ResourceRadixKeyDataType extends RadixKeyBytesDataType -{ - - @Override - public Object getValueKey(Object value) - { - // extract long id from value object - long id; - if (value instanceof ResourceKey) - id = ((ResourceKey)value).getInternalID().getIdAsLong(); - else if (value instanceof Long) - id = (Long)value; - else - throw new IllegalArgumentException("Unsupported value type"); - - // convert to byte[] - byte[] bytes = new byte[8]; - bytes[0] = (byte) (id >> 56); - bytes[1] = (byte) (id >> 48); - bytes[2] = (byte) (id >> 40); - bytes[3] = (byte) (id >> 32); - bytes[4] = (byte) (id >> 24); - bytes[5] = (byte) (id >> 16); - bytes[6] = (byte) (id >> 8); - bytes[7] = (byte) (id /*>> 0*/); - return bytes; - } - - - /*@Override - public void write(WriteBuffer wbuf, Object obj) - { - if (obj instanceof Long) - { - wbuf.put((byte)0); - wbuf.putVarLong((Long)obj); - } - else - super.write(wbuf, obj); - } - - - @Override - public Object read(ByteBuffer buf) - { - // read array length, then byte[] - int l = DataUtils.readVarInt(buf); - - // case of value key as long - if (l == 0) - return DataUtils.readVarLong(buf); - - byte[] key = new byte[l]; - buf.get(key); - return key; - }*/ - -} diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/PrintDbInfo.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/PrintDbInfo.java index 6781bf659..a504d1293 100644 --- a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/PrintDbInfo.java +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/PrintDbInfo.java @@ -37,7 +37,6 @@ public static void main(String[] args) throws Exception scanPages(db.foiStore.featuresIndex); scanPages(db.foiStore.idsIndex); scanPages(db.foiStore.uidsIndex); - //scanPages(db.foiStore.fullTextIndex.radixTreeMap); //scanPages(db.foiStore.spatialIndex.rTreeMap); //Thread.sleep(100000); From af35d7820a132117eef86dc97288b341106426ac Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Tue, 29 Sep 2026 16:15:01 -0500 Subject: [PATCH 08/12] Add db version marker and compatibility check --- .../sensorhub/impl/datastore/h2/H2Utils.java | 101 ++++++ .../impl/datastore/h2/MVFeatureDatabase.java | 5 +- .../datastore/h2/MVObsSystemDatabase.java | 22 +- .../impl/datastore/h2/TestMVCommandStore.java | 10 +- .../h2/TestMVCommandStreamStore.java | 10 +- .../datastore/h2/TestMVDataStreamStore.java | 10 +- .../datastore/h2/TestMVDatabaseVersion.java | 290 ++++++++++++++++++ .../impl/datastore/h2/TestMVFeatureStore.java | 10 +- .../impl/datastore/h2/TestMVObsStore.java | 10 +- 9 files changed, 424 insertions(+), 44 deletions(-) create mode 100644 sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVDatabaseVersion.java diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java index 7853352de..caa72db8f 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java @@ -14,15 +14,21 @@ package org.sensorhub.impl.datastore.h2; +import java.io.File; import java.nio.ByteBuffer; import java.time.Instant; +import java.util.Arrays; import java.util.Map; +import org.h2.engine.Constants; import org.h2.mvstore.DataUtils; import org.h2.mvstore.MVMap; import org.h2.mvstore.MVStore; +import org.h2.mvstore.MVStoreException; import org.h2.mvstore.WriteBuffer; import org.h2.mvstore.rtree.Spatial; import org.h2.mvstore.rtree.SpatialDataType; +import org.h2.mvstore.type.StringDataType; +import org.sensorhub.api.datastore.DataStoreException; import org.sensorhub.api.datastore.TemporalFilter; import org.sensorhub.impl.datastore.h2.kryo.KryoDataType; import org.vast.util.Asserts; @@ -38,6 +44,12 @@ public class H2Utils { public static final int CURRENT_VERSION = 5; + + // Compatibility bounds are deliberate, not derived from the running H2 version. + static final String MIN_SUPPORTED_H2_VERSION = "2.5.250"; + static final String MAX_SUPPORTED_H2_VERSION = "2.5.250"; + static final String DATABASE_MAP_NAME = "@database"; + static final String H2_VERSION_KEY = "h2.version"; static final String DATASTORES_MAP_NAME = "@datastores"; static final String GEOM_DIM_ERROR = "Only 2D and 3D geometries are supported"; @@ -80,6 +92,95 @@ static class StoreInfoDataType extends KryoDataType }; } } + + + /** + * Opens a database and checks its H2 version before any application maps are + * opened. Missing metadata in an existing file requires migration. + * @param storagePath database file path + * @param builder storage options (the file name and autocommit delay are set here) + * @param autoCommitDelay autocommit delay in milliseconds after validation + * @return the compatible store; the caller is responsible for closing it + * @throws DataStoreException if the file format or version is unsupported + */ + public static MVStore openMVStore(String storagePath, MVStore.Builder builder, int autoCommitDelay) + throws DataStoreException + { + var file = new File(storagePath); + boolean newFile = !file.exists() || file.length() == 0; + MVStore store; + try + { + store = builder.fileName(storagePath).autoCommitDisabled().open(); + } + catch (MVStoreException e) + { + if (e.getErrorCode() == DataUtils.ERROR_UNSUPPORTED_FORMAT) + throw new DataStoreException("Unsupported H2 file format: " + storagePath, e); + throw e; + } + + boolean compatible = false; + try + { + var metadataBuilder = new MVMap.Builder() + .keyType(StringDataType.INSTANCE) + .valueType(StringDataType.INSTANCE); + if (newFile) + { + store.openMap(DATABASE_MAP_NAME, metadataBuilder).put(H2_VERSION_KEY, Constants.VERSION); + store.commit(); + } + else + { + String version = store.hasMap(DATABASE_MAP_NAME) + ? store.openMap(DATABASE_MAP_NAME, metadataBuilder).get(H2_VERSION_KEY) : null; + if (version == null) + migrateDatabase(storagePath, null); + + var components = parseH2Version(version); + if (Arrays.compare(components, parseH2Version(MIN_SUPPORTED_H2_VERSION)) < 0) + migrateDatabase(storagePath, version); + if (Arrays.compare(components, parseH2Version(MAX_SUPPORTED_H2_VERSION)) > 0) + throw new DataStoreException("Unsupported H2 database version " + version + + " in " + storagePath + "; supported range is " + MIN_SUPPORTED_H2_VERSION + + " to " + MAX_SUPPORTED_H2_VERSION); + } + + store.setAutoCommitDelay(autoCommitDelay); + compatible = true; + return store; + } + finally + { + if (!compatible) + store.closeImmediately(); + } + } + + + private static int[] parseH2Version(String version) throws DataStoreException + { + if (version == null || !version.matches("[0-9]+\\.[0-9]+\\.[0-9]+")) + throw new DataStoreException("Invalid H2 database version: " + version); + try + { + return Arrays.stream(version.split("\\.")).mapToInt(Integer::parseInt).toArray(); + } + catch (NumberFormatException e) + { + throw new DataStoreException("Invalid H2 database version: " + version, e); + } + } + + + private static void migrateDatabase(String storagePath, String sourceVersion) throws DataStoreException + { + // Migration of legacy files is not part of this change; files that need it are rejected here. + throw new DataStoreException("Database requires migration: " + storagePath + + " (H2 version " + (sourceVersion == null ? "missing" : sourceVersion) + + "; supported range is " + MIN_SUPPORTED_H2_VERSION + " to " + MAX_SUPPORTED_H2_VERSION + ")"); + } public static MVDataStoreInfo getDataStoreInfo(MVStore mvStore, String dataStoreName) diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java index e7f9ecb7d..323ee3c9e 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java @@ -49,7 +49,7 @@ protected void doStart() throws SensorHubException if (!FileUtils.isSafeFilePath(config.storagePath)) throw new DataStoreException("Storage path contains illegal characters: " + config.storagePath); - MVStore.Builder builder = new MVStore.Builder().fileName(config.storagePath); + MVStore.Builder builder = new MVStore.Builder(); if (config.readOnly) builder.readOnly(); @@ -63,8 +63,7 @@ protected void doStart() throws SensorHubException if (config.useCompression) builder.compress(); - mvStore = builder.open(); - mvStore.setAutoCommitDelay(config.autoCommitPeriod*1000); + mvStore = H2Utils.openMVStore(config.storagePath, builder, config.autoCommitPeriod*1000); mvStore.setVersionsToKeep(0); // open feature store diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java index 2aa65b300..51217c00c 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java @@ -82,7 +82,7 @@ protected void doStart() throws SensorHubException { try { - MVStore.Builder builder = new MVStore.Builder().fileName(config.storagePath); + MVStore.Builder builder = new MVStore.Builder(); builder.backgroundExceptionHandler((t, e) -> { getLogger().error("Error in H2 background thread {}", t.getName(), e); }); @@ -102,8 +102,7 @@ protected void doStart() throws SensorHubException if (config.useCompression) builder.compress(); - mvStore = builder.open(); - mvStore.setAutoCommitDelay(config.autoCommitPeriod*1000); + mvStore = H2Utils.openMVStore(config.storagePath, builder, config.autoCommitPeriod*1000); mvStore.setVersionsToKeep(0); var idScope = getDatabaseNum() != null ? getDatabaseNum() : 0; @@ -177,14 +176,15 @@ protected void beforeStop() @Override protected void doStop() throws SensorHubException { - if (mvStore != null) - { - // must call commit first to make sure kryo persistent class resolver - // is updated before we serialize it again in close - mvStore.commit(); - mvStore.close(); - mvStore = null; - } + // Startup may have rejected the file before acquiring a compatible store. + if (mvStore == null) + return; + + // must call commit first to make sure kryo persistent class resolver + // is updated before we serialize it again in close + mvStore.commit(); + mvStore.close(); + mvStore = null; // log store info if debug is enabled if (config.printDebugInfoOnClose && getLogger().isDebugEnabled() && new File(config.storagePath).exists()) diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVCommandStore.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVCommandStore.java index cfa9db9e6..ee30cf0c6 100644 --- a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVCommandStore.java +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVCommandStore.java @@ -39,13 +39,13 @@ protected MVCommandStoreImpl initStore() throws Exception } - protected void forceReadBackFromStorage() + protected void forceReadBackFromStorage() throws Exception { this.cmdStore = openMVStore(); } - private MVCommandStoreImpl openMVStore() + private MVCommandStoreImpl openMVStore() throws Exception { if (mvStore != null) { @@ -54,11 +54,9 @@ private MVCommandStoreImpl openMVStore() System.out.println("MVStore flushed to disk"); } - mvStore = new MVStore.Builder() - .fileName(dbFile.getAbsolutePath()) + mvStore = H2Utils.openMVStore(dbFile.getAbsolutePath(), new MVStore.Builder() .autoCommitBufferSize(10) - .cacheSize(10) - .open(); + .cacheSize(10), 1000); return MVCommandStoreImpl.open(mvStore, DATABASE_NUM, IdProviderType.SEQUENTIAL, MVDataStoreInfo.builder() diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVCommandStreamStore.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVCommandStreamStore.java index 6e4d78e66..92b26785d 100644 --- a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVCommandStreamStore.java +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVCommandStreamStore.java @@ -37,13 +37,13 @@ protected MVCommandStreamStoreImpl initStore() throws Exception } - protected void forceReadBackFromStorage() + protected void forceReadBackFromStorage() throws Exception { this.cmdStreamStore = openMVStore(); } - private MVCommandStreamStoreImpl openMVStore() + private MVCommandStreamStoreImpl openMVStore() throws Exception { if (mvStore != null) { @@ -52,11 +52,9 @@ private MVCommandStreamStoreImpl openMVStore() System.out.println("MVStore flushed to disk"); } - mvStore = new MVStore.Builder() - .fileName(dbFile.getAbsolutePath()) + mvStore = H2Utils.openMVStore(dbFile.getAbsolutePath(), new MVStore.Builder() .autoCommitBufferSize(10) - .cacheSize(10) - .open(); + .cacheSize(10), 1000); var cmdStore = MVCommandStoreImpl.open(mvStore, DATABASE_NUM, IdProviderType.SEQUENTIAL, MVDataStoreInfo.builder() diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVDataStreamStore.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVDataStreamStore.java index 0d12fca1f..f8321d788 100644 --- a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVDataStreamStore.java +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVDataStreamStore.java @@ -37,13 +37,13 @@ protected MVDataStreamStoreImpl initStore() throws Exception } - protected void forceReadBackFromStorage() + protected void forceReadBackFromStorage() throws Exception { this.dataStreamStore = openMVStore(); } - private MVDataStreamStoreImpl openMVStore() + private MVDataStreamStoreImpl openMVStore() throws Exception { if (mvStore != null) { @@ -52,11 +52,9 @@ private MVDataStreamStoreImpl openMVStore() System.out.println("MVStore flushed to disk"); } - mvStore = new MVStore.Builder() - .fileName(dbFile.getAbsolutePath()) + mvStore = H2Utils.openMVStore(dbFile.getAbsolutePath(), new MVStore.Builder() .autoCommitBufferSize(10) - .cacheSize(10) - .open(); + .cacheSize(10), 1000); var obsStore = MVObsStoreImpl.open(mvStore, DATABASE_NUM, IdProviderType.SEQUENTIAL, MVDataStoreInfo.builder() diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVDatabaseVersion.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVDatabaseVersion.java new file mode 100644 index 000000000..69d871ba7 --- /dev/null +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVDatabaseVersion.java @@ -0,0 +1,290 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2; + +import static org.junit.Assert.*; +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import org.h2.engine.Constants; +import org.h2.mvstore.DataUtils; +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.MVStoreException; +import org.h2.mvstore.type.StringDataType; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sensorhub.api.datastore.DataStoreException; +import org.sensorhub.impl.module.AbstractModule; + + +public class TestMVDatabaseVersion +{ + @Rule + public TemporaryFolder files = new TemporaryFolder(); + + + @Test + public void testNewFilesAndReadOnlyReopen() throws Exception + { + for (File file: new File[] {new File(files.getRoot(), "new.dat"), files.newFile()}) + { + try (var store = H2Utils.openMVStore(file.getPath(), new MVStore.Builder(), 1234)) + { + assertEquals(Constants.VERSION, metadata(store).get(H2Utils.H2_VERSION_KEY)); + assertEquals(1234, store.getAutoCommitDelay()); + store.openMap("payload").put("key", "value"); + } + try (var store = H2Utils.openMVStore(file.getPath(), new MVStore.Builder(), 0)) + { + assertEquals(Constants.VERSION, metadata(store).get(H2Utils.H2_VERSION_KEY)); + assertEquals("value", store.openMap("payload").get("key")); + } + byte[] before = Files.readAllBytes(file.toPath()); + try (var store = H2Utils.openMVStore(file.getPath(), new MVStore.Builder().readOnly(), 0)) + { + assertEquals(Constants.VERSION, metadata(store).get(H2Utils.H2_VERSION_KEY)); + } + assertArrayEquals(before, Files.readAllBytes(file.toPath())); + } + } + + + @Test + public void testMissingMetadataMapRequiresMigration() throws Exception + { + assertRejected(createFile(null, false), "Database requires migration"); + } + + + @Test + public void testMissingVersionKeyRequiresMigration() throws Exception + { + assertRejected(createFile(null, true), "Database requires migration"); + } + + + @Test + public void testEmptyExistingMVStoreRequiresMigration() throws Exception + { + File file = files.newFile(); + try (var store = new MVStore.Builder().fileName(file.getPath()).open()) + { + assertTrue(store.getMapNames().isEmpty()); + } + assertTrue(file.length() > 0); + assertRejected(file, "Database requires migration"); + } + + + @Test + public void testOlderVersionsRequireMigration() throws Exception + { + for (String version: new String[] {"2.5.249", "2.4.999", "1.4.200"}) + assertRejected(createFile(version, true), "Database requires migration"); + } + + + @Test + public void testNewerVersionsAreRejected() throws Exception + { + for (String version: new String[] {"2.5.251", "2.10.1", "3.0.0"}) + assertRejected(createFile(version, true), "Unsupported H2 database version " + version); + } + + + @Test + public void testMalformedVersionsAreRejected() throws Exception + { + for (String version: new String[] {"", "bad", "2.5", "2.5.250.extra", "-2.5.250", "2.5.999999999999"}) + assertRejected(createFile(version, true), "Invalid H2 database version"); + } + + + @Test + public void testUnsupportedPhysicalFormatsAreNotMigrated() throws Exception + { + for (int format: new int[] {1, 4}) + { + File file = files.newFile(); + // Two checksummed H2 headers, declaring an old or future native format. + String header = "H:2,blockSize:1000,format:" + format + ",created:0"; + byte[] bytes = header.getBytes(StandardCharsets.ISO_8859_1); + int checksum = DataUtils.getFletcher32(bytes, 0, bytes.length); + bytes = (header + ",fletcher:" + Integer.toHexString(checksum) + "\n") + .getBytes(StandardCharsets.ISO_8859_1); + var buffer = ByteBuffer.allocate(8192); + buffer.put(bytes); + buffer.position(4096); + buffer.put(bytes); + Files.write(file.toPath(), buffer.array()); + + var error = assertThrows(DataStoreException.class, + () -> H2Utils.openMVStore(file.getPath(), new MVStore.Builder(), 0)); + assertTrue(error.getMessage().contains("Unsupported H2 file format")); + assertTrue(error.getCause() instanceof MVStoreException); + assertEquals(DataUtils.ERROR_UNSUPPORTED_FORMAT, ((MVStoreException)error.getCause()).getErrorCode()); + assertArrayEquals(buffer.array(), Files.readAllBytes(file.toPath())); + try (var channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE); + var lock = channel.tryLock()) + { + assertNotNull(lock); + } + } + } + + + @Test + public void testBothDatabaseModulesCreateAndReopen() throws Exception + { + for (boolean featureDatabase: new boolean[] {false, true}) + { + File file = files.newFile(); + var database = database(file, featureDatabase, false); + try + { + database.init(); + database.start(); + assertTrue(database.isStarted()); + } + finally + { + database.stop(); + } + byte[] before = Files.readAllBytes(file.toPath()); + database = database(file, featureDatabase, true); + try + { + database.init(); + database.start(); + assertTrue(database.isStarted()); + } + finally + { + database.stop(); + } + assertArrayEquals(before, Files.readAllBytes(file.toPath())); + } + } + + + @Test + public void testBothDatabaseModulesRejectBeforeReadingApplicationMaps() throws Exception + { + for (boolean featureDatabase: new boolean[] {false, true}) + { + File file = createFile(null, false); + byte[] before = Files.readAllBytes(file.toPath()); + var database = database(file, featureDatabase, false); + try + { + database.init(); + database.start(); + assertFalse(database.isStarted()); + Throwable error = database.getCurrentError(); + assertNotNull(error); + while (error.getCause() != null) + error = error.getCause(); + assertTrue(error.getMessage(), error.getMessage().contains("Database requires migration")); + } + finally + { + database.stop(); + } + assertUnchangedAndUnlocked(file, before); + } + } + + + private MVMap metadata(MVStore store) + { + return store.openMap(H2Utils.DATABASE_MAP_NAME, new MVMap.Builder() + .keyType(StringDataType.INSTANCE).valueType(StringDataType.INSTANCE)); + } + + + private File createFile(String version, boolean metadataMap) throws Exception + { + File file = files.newFile(); + try (var store = new MVStore.Builder().fileName(file.getPath()).open()) + { + if (metadataMap) + { + var metadata = metadata(store); + if (version != null) + metadata.put(H2Utils.H2_VERSION_KEY, version); + } + // Deliberately incompatible application metadata: validation must precede decoding it. + var datastores = store.openMap(H2Utils.DATASTORES_MAP_NAME); + datastores.put("sys_store", "not Kryo data"); + datastores.put("feature_store", "not Kryo data"); + } + return file; + } + + + private void assertRejected(File file, String message) throws Exception + { + byte[] before = Files.readAllBytes(file.toPath()); + for (boolean readOnly: new boolean[] {false, true}) + { + var builder = new MVStore.Builder(); + if (readOnly) + builder.readOnly(); + var error = assertThrows(DataStoreException.class, + () -> H2Utils.openMVStore(file.getPath(), builder, 1000)); + assertTrue(error.getMessage(), error.getMessage().contains(message)); + assertUnchangedAndUnlocked(file, before); + } + } + + + private void assertUnchangedAndUnlocked(File file, byte[] before) throws Exception + { + assertArrayEquals(before, Files.readAllBytes(file.toPath())); + // A writable reopen proves rejection released the lock, without committing changes. + var store = new MVStore.Builder().fileName(file.getPath()).autoCommitDisabled().open(); + store.closeImmediately(); + assertArrayEquals(before, Files.readAllBytes(file.toPath())); + } + + + private AbstractModule database(File file, boolean featureDatabase, boolean readOnly) + { + if (featureDatabase) + { + var database = new MVFeatureDatabase(); + var config = new MVFeatureDatabaseConfig(); + config.databaseNum = 1; + config.storagePath = file.getPath(); + config.readOnly = readOnly; + database.setConfiguration(config); + return database; + } + var database = new MVObsSystemDatabase(); + var config = new MVObsSystemDatabaseConfig(); + config.databaseNum = 1; + config.storagePath = file.getPath(); + config.readOnly = readOnly; + // In particular, failed startup must not compact an incompatible file. + config.compactOnClose = !readOnly; + database.setConfiguration(config); + return database; + } +} diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVFeatureStore.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVFeatureStore.java index cd38ef61e..39bd1e4d2 100644 --- a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVFeatureStore.java +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVFeatureStore.java @@ -37,13 +37,13 @@ protected MVFeatureStoreImpl initStore() throws Exception } - protected void forceReadBackFromStorage() + protected void forceReadBackFromStorage() throws Exception { this.featureStore = openMVStore(); } - private MVFeatureStoreImpl openMVStore() + private MVFeatureStoreImpl openMVStore() throws Exception { if (mvStore != null) { @@ -52,11 +52,9 @@ private MVFeatureStoreImpl openMVStore() System.out.println("MVStore flushed to disk"); } - mvStore = new MVStore.Builder() - .fileName(dbFile.getAbsolutePath()) + mvStore = H2Utils.openMVStore(dbFile.getAbsolutePath(), new MVStore.Builder() .autoCommitBufferSize(100) - .cacheSize(1) - .open(); + .cacheSize(1), 1000); return MVFeatureStoreImpl.open(mvStore, DATABASE_NUM, IdProviderType.SEQUENTIAL, MVDataStoreInfo.builder() .withName(DATASTORE_NAME) diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVObsStore.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVObsStore.java index 1de63964f..578554bbe 100644 --- a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVObsStore.java +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestMVObsStore.java @@ -39,13 +39,13 @@ protected MVObsStoreImpl initStore() throws Exception } - protected void forceReadBackFromStorage() + protected void forceReadBackFromStorage() throws Exception { this.obsStore = openMVStore(); } - private MVObsStoreImpl openMVStore() + private MVObsStoreImpl openMVStore() throws Exception { if (mvStore != null) { @@ -54,11 +54,9 @@ private MVObsStoreImpl openMVStore() System.out.println("MVStore flushed to disk"); } - mvStore = new MVStore.Builder() - .fileName(dbFile.getAbsolutePath()) + mvStore = H2Utils.openMVStore(dbFile.getAbsolutePath(), new MVStore.Builder() .autoCommitBufferSize(10) - .cacheSize(1) - .open(); + .cacheSize(1), 1000); return MVObsStoreImpl.open(mvStore, DATABASE_NUM, IdProviderType.SEQUENTIAL, MVDataStoreInfo.builder() From f26795d8d27e1391890411526b49e9780989be95 Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Tue, 29 Sep 2026 16:21:10 -0500 Subject: [PATCH 09/12] remove temporary compiler setting and ignore IDE run configs --- .gitignore | 1 + sensorhub-datastore-h2/build.gradle | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index b6e3e768f..2ef653757 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ **/.classpath **/.idea **/*.iml +**/.run/ diff --git a/sensorhub-datastore-h2/build.gradle b/sensorhub-datastore-h2/build.gradle index 14bd46965..bff7f5316 100644 --- a/sensorhub-datastore-h2/build.gradle +++ b/sensorhub-datastore-h2/build.gradle @@ -34,8 +34,3 @@ ext.pom >>= { } } } - -// TEMP (H2 port): show all compiler errors; remove before merging -tasks.withType(JavaCompile) { - options.compilerArgs << "-Xmaxerrs" << "2000" -} From 3a96490593aa52b9204a1bda2a1aae65e1fb71d7 Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Wed, 30 Sep 2026 10:04:14 -0500 Subject: [PATCH 10/12] Add migration tooling and make a few constants public for reuse in the tool. The migration tool will eventually be separated from this module, but I am packaging it here so that we can easily automigrate old files during this transitory period. --- sensorhub-datastore-h2/build.gradle | 78 +++ .../legacy/LegacyStoreFactoryImpl.java | 105 ++++ .../h2/migrate/legacy/LegacyStoreImpl.java | 261 ++++++++++ .../DatabaseMigrationRequiredException.java | 40 ++ .../sensorhub/impl/datastore/h2/H2Utils.java | 25 +- .../impl/datastore/h2/MVFeatureDatabase.java | 2 +- .../datastore/h2/MVObsSystemDatabase.java | 16 +- .../impl/datastore/h2/migrate/FileOps.java | 212 +++++++++ .../impl/datastore/h2/migrate/H2Migrate.java | 152 ++++++ .../datastore/h2/migrate/LegacyLoader.java | 242 ++++++++++ .../h2/migrate/MigrationException.java | 37 ++ .../impl/datastore/h2/migrate/Migrator.java | 448 ++++++++++++++++++ .../datastore/h2/migrate/RawDataType.java | 126 +++++ .../datastore/h2/migrate/SourceReader.java | 269 +++++++++++ .../datastore/h2/migrate/StoreLayout.java | 226 +++++++++ .../impl/datastore/h2/migrate/Verifier.java | 270 +++++++++++ .../datastore/h2/migrate/spi/LegacyStore.java | 157 ++++++ .../h2/migrate/TestLegacyLoader.java | 134 ++++++ .../h2/migrate/TestMigrateFailures.java | 274 +++++++++++ .../h2/migrate/TestMigrateNative.java | 120 +++++ .../h2/migrate/TestMigrateRoundTrip.java | 226 +++++++++ .../datastore/h2/migrate/TestSupport.java | 404 ++++++++++++++++ 22 files changed, 3808 insertions(+), 16 deletions(-) create mode 100644 sensorhub-datastore-h2/src/legacy/java/org/sensorhub/impl/datastore/h2/migrate/legacy/LegacyStoreFactoryImpl.java create mode 100644 sensorhub-datastore-h2/src/legacy/java/org/sensorhub/impl/datastore/h2/migrate/legacy/LegacyStoreImpl.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/DatabaseMigrationRequiredException.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/FileOps.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/H2Migrate.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/LegacyLoader.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/MigrationException.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/Migrator.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/RawDataType.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/SourceReader.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/StoreLayout.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/Verifier.java create mode 100644 sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/spi/LegacyStore.java create mode 100644 sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestLegacyLoader.java create mode 100644 sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateFailures.java create mode 100644 sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateNative.java create mode 100644 sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateRoundTrip.java create mode 100644 sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestSupport.java diff --git a/sensorhub-datastore-h2/build.gradle b/sensorhub-datastore-h2/build.gradle index bff7f5316..b7cac40b3 100644 --- a/sensorhub-datastore-h2/build.gradle +++ b/sensorhub-datastore-h2/build.gradle @@ -1,20 +1,98 @@ description = 'OSH Datastore (H2)' ext.details = 'Storage module backed by a H2 MVStore object database' +// Legacy H2 1.4 adapters used by the migration tool (org.sensorhub.impl.datastore.h2.migrate). +// Compiled against H2 1.4.200 only and loaded at runtime in an isolated classloader; the 1.4 +// jar is never on a classpath. By default both jars are embedded as resources under +// META-INF/h2migrate so migrations work from a plain node install. Build with +// -PembedH2MigrationLibs=false to leave them out; the tool then needs a legacy-libs/ directory +// next to the node's lib/ (see the h2migrateLegacyLibs task) or --legacy-libs. +ext.embedH2MigrationLibs = !project.hasProperty('embedH2MigrationLibs') || + Boolean.parseBoolean(project.property('embedH2MigrationLibs').toString()) + +sourceSets { + legacy { + java.srcDir 'src/legacy/java' + compileClasspath += files(sourceSets.main.java.classesDirectory) + } +} + +configurations { + legacyH2 +} + dependencies { implementation project(':sensorhub-core') implementation project(':sensorhub-utils-kryo') embeddedApi 'com.h2database:h2-mvstore:2.5.250' embeddedApi 'org.apache.lucene:lucene-analyzers-common:8.11.2' testImplementation(testFixtures(project(':sensorhub-core'))) + + legacyCompileOnly 'com.h2database:h2-mvstore:1.4.200' + legacyH2 'com.h2database:h2-mvstore:1.4.200' +} + +compileLegacyJava.dependsOn compileJava + +tasks.register('legacyJar', Jar) { + description = 'Packages the H2 1.4 migration adapters (published with classifier h2migrate-legacy)' + from sourceSets.legacy.output + archiveClassifier = 'h2migrate-legacy' +} + +def legacyLibsDir = layout.buildDirectory.dir('h2migrate/legacy-libs') + +tasks.register('h2migrateLegacyLibs', Sync) { + description = 'Assembles the H2 1.4 jar and the migration adapters into build/h2migrate/legacy-libs ' + + '(copy that directory next to a node\'s lib/ as legacy-libs/, or pass it with --legacy-libs)' + from(tasks.named('legacyJar')) + from(configurations.legacyH2) + into legacyLibsDir +} + +if (embedH2MigrationLibs) { + def embeddedLegacyDir = layout.buildDirectory.dir('generated-resources/h2migrate') + tasks.register('embedLegacyLibs', Sync) { + description = 'Embeds the H2 1.4 jar and the migration adapters as resources of this module' + from(tasks.named('legacyJar')) + from(configurations.legacyH2) + into embeddedLegacyDir.map { it.dir('META-INF/h2migrate') } + doLast { + def dir = embeddedLegacyDir.get().dir('META-INF/h2migrate').asFile + new File(dir, 'index.txt').text = dir.listFiles().findAll { it.name.endsWith('.jar') }.collect { it.name }.sort().join('\n') + '\n' + } + } + sourceSets.main.output.dir(embeddedLegacyDir, builtBy: 'embedLegacyLibs') +} + +tasks.register('migrate', JavaExec) { + description = 'Runs the H2 database migration tool; pass arguments with -PmigrateArgs=" [options]"' + classpath = sourceSets.main.runtimeClasspath + mainClass = 'org.sensorhub.impl.datastore.h2.migrate.H2Migrate' + if (!embedH2MigrationLibs) { + dependsOn 'h2migrateLegacyLibs' + systemProperty 'h2migrate.legacyLibs', legacyLibsDir.get().asFile.absolutePath + } + if (project.hasProperty('migrateArgs')) + args project.migrateArgs.split('\\s+') } test { + // default builds exercise the embedded libraries; an opt-out build uses the directory + if (!embedH2MigrationLibs) { + dependsOn 'h2migrateLegacyLibs' + systemProperty 'h2migrate.legacyLibs', legacyLibsDir.get().asFile.absolutePath + } filter { excludeTestsMatching 'org.sensorhub.impl.datastore.h2.TestMVFeatureStore.test*Throughput' } } +// publish the adapters next to the module so operators can fetch them without a source build +afterEvaluate { + publishing.publications.mavenJava.artifact(tasks.legacyJar) +} + // add info to OSGI manifest osgi { manifest { diff --git a/sensorhub-datastore-h2/src/legacy/java/org/sensorhub/impl/datastore/h2/migrate/legacy/LegacyStoreFactoryImpl.java b/sensorhub-datastore-h2/src/legacy/java/org/sensorhub/impl/datastore/h2/migrate/legacy/LegacyStoreFactoryImpl.java new file mode 100644 index 000000000..0b5d11068 --- /dev/null +++ b/sensorhub-datastore-h2/src/legacy/java/org/sensorhub/impl/datastore/h2/migrate/legacy/LegacyStoreFactoryImpl.java @@ -0,0 +1,105 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate.legacy; + +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.rtree.MVRTreeMap; +import org.h2.mvstore.rtree.SpatialKey; +import org.sensorhub.impl.datastore.h2.migrate.spi.LegacyStore; + + +/** + * Instantiated reflectively by the tool inside the legacy classloader. + */ +public final class LegacyStoreFactoryImpl implements LegacyStore.Factory +{ + public LegacyStoreFactoryImpl() + { + } + + + @Override + public LegacyStore openReadOnly(String fileName) + { + return new LegacyStoreImpl(fileName); + } + + + @Override + public LegacyStore.Writer openForWrite(String fileName) + { + return new WriterImpl(fileName); + } + + + @Override + public void compact(String fileName) + { + org.h2.mvstore.MVStoreTool.compact(fileName, false); + } + + + /** + * Writes a genuine H2 1.4 (format 1) file from raw entries. Test fixture support. + */ + static final class WriterImpl implements LegacyStore.Writer + { + private final MVStore store; + + + WriterImpl(String fileName) + { + this.store = new MVStore.Builder() + .fileName(fileName) + .autoCommitDisabled() + .open(); + } + + + @Override + public MapWriter openMap(String mapName, LegacyStore.EntryComparator keyComparator) + { + MVMap map = store.openMap(mapName, new MVMap.Builder() + .keyType(new LegacyStoreImpl.RecordingDataType(null, keyComparator)) + .valueType(new LegacyStoreImpl.RecordingDataType(null, null))); + return (key, decodedKey, value) -> map.put(new LegacyStoreImpl.RawEntry(key, decodedKey), new LegacyStoreImpl.RawEntry(value, null)); + } + + + @Override + public RTreeMapWriter openRTreeMap(String mapName, int dimensions) + { + MVRTreeMap map = store.openMap(mapName, new MVRTreeMap.Builder() + .dimensions(dimensions) + .valueType(new LegacyStoreImpl.RecordingDataType(null, null))); + return (id, minMax, value) -> map.put(new SpatialKey(id, minMax), new LegacyStoreImpl.RawEntry(value, null)); + } + + + @Override + public void commit() + { + store.commit(); + } + + + @Override + public void close() + { + store.close(); + } + } +} diff --git a/sensorhub-datastore-h2/src/legacy/java/org/sensorhub/impl/datastore/h2/migrate/legacy/LegacyStoreImpl.java b/sensorhub-datastore-h2/src/legacy/java/org/sensorhub/impl/datastore/h2/migrate/legacy/LegacyStoreImpl.java new file mode 100644 index 000000000..aebe0d3e2 --- /dev/null +++ b/sensorhub-datastore-h2/src/legacy/java/org/sensorhub/impl/datastore/h2/migrate/legacy/LegacyStoreImpl.java @@ -0,0 +1,261 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate.legacy; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import org.h2.engine.Constants; +import org.h2.mvstore.Cursor; +import org.h2.mvstore.DataUtils; +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.Page; +import org.h2.mvstore.WriteBuffer; +import org.h2.mvstore.rtree.MVRTreeMap; +import org.h2.mvstore.rtree.SpatialKey; +import org.h2.mvstore.type.DataType; +import org.sensorhub.impl.datastore.h2.migrate.spi.LegacyStore; + + +/** + * Read-only H2 1.4 store. Every map is opened with {@link RecordingDataType}s + * so entries come out as exact bytes. + */ +final class LegacyStoreImpl implements LegacyStore +{ + private final String fileName; + private final MVStore store; + + + LegacyStoreImpl(String fileName) + { + this.fileName = fileName; + this.store = new MVStore.Builder() + .fileName(fileName) + .readOnly() + .open(); + } + + + @Override + public String getFileName() + { + return fileName; + } + + + @Override + public String getH2Version() + { + return Constants.VERSION; + } + + + @Override + public Set getMapNames() + { + return Collections.unmodifiableSet(new TreeSet<>(store.getMapNames())); + } + + + @Override + public Map getMapMeta(String mapName) + { + String id = store.getMetaMap().get(DataUtils.META_NAME + mapName); + if (id == null) + throw new IllegalArgumentException("Unknown map: " + mapName); + String config = store.getMetaMap().get("map." + id); + if (config == null) + throw new IllegalStateException("Missing metadata for map: " + mapName); + return DataUtils.parseMap(config); + } + + + @Override + public boolean isRTree(String mapName) + { + return "rtree".equals(getMapMeta(mapName).get("type")); + } + + + @Override + public long readMap(String mapName, LegacyStore.EntryDecoder keyDecoder, LegacyStore.EntryDecoder valueDecoder, LegacyStore.RawEntrySink sink) + { + if (isRTree(mapName)) + throw new IllegalArgumentException("Map is an R-tree, use readRTreeMap: " + mapName); + + MVMap map = store.openMap(mapName, new MVMap.Builder() + .keyType(new RecordingDataType(keyDecoder, null)) + .valueType(new RecordingDataType(valueDecoder, null))); + + long count = 0; + Cursor cursor = map.cursor(null); + while (cursor.hasNext()) + { + RawEntry key = (RawEntry)cursor.next(); + RawEntry value = (RawEntry)cursor.getValue(); + sink.entry(key.bytes, key.decoded, value.bytes, value.decoded); + count++; + } + return count; + } + + + @Override + public long readRTreeMap(String mapName, int dimensions, LegacyStore.EntryDecoder valueDecoder, LegacyStore.SpatialEntrySink sink) + { + // no metadata check here: the caller knows the map's type from the store layout, + // and files compacted by H2 1.4 have lost the type:rtree marker + MVRTreeMap map = store.openMap(mapName, new MVRTreeMap.Builder() + .dimensions(dimensions) + .valueType(new RecordingDataType(valueDecoder, null))); + + return walk(map.getRootPage(), dimensions, sink); + } + + + private long walk(Page page, int dimensions, LegacyStore.SpatialEntrySink sink) + { + long count = 0; + if (page.isLeaf()) + { + for (int i = 0; i < page.getKeyCount(); i++) + { + SpatialKey key = (SpatialKey)page.getKey(i); + RawEntry value = (RawEntry)page.getValue(i); + float[] minMax = new float[2 * dimensions]; + for (int d = 0; d < dimensions; d++) + { + minMax[2 * d] = key.min(d); + minMax[2 * d + 1] = key.max(d); + } + sink.entry(key.getId(), minMax, value.bytes, value.decoded); + count++; + } + } + else + { + // R-tree node pages hold one child per key; the extra raw slot is an empty placeholder + for (int i = 0; i < page.getKeyCount(); i++) + count += walk(page.getChildPage(i), dimensions, sink); + } + return count; + } + + + @Override + public void close() + { + store.closeImmediately(); + } + + + /** + * Key or value as stored in an H2 1.4 page: the exact bytes plus, when the + * entry was read through a decoder, the decoded object. + */ + static final class RawEntry + { + final byte[] bytes; + final Object decoded; + + + RawEntry(byte[] bytes, Object decoded) + { + this.bytes = bytes; + this.decoded = decoded; + } + } + + + /** + * H2 1.4 {@link DataType} that finds entry boundaries by delegating the + * decode to the caller (the real OSH datatype on the H2 2.5 side) and + * records the exact bytes consumed. Writes emit recorded bytes verbatim. + */ + static final class RecordingDataType implements DataType + { + private final LegacyStore.EntryDecoder decoder; + private final LegacyStore.EntryComparator comparator; + + + RecordingDataType(LegacyStore.EntryDecoder decoder, LegacyStore.EntryComparator comparator) + { + this.decoder = decoder; + this.comparator = comparator; + } + + + @Override + public int compare(Object a, Object b) + { + if (comparator == null) + throw new UnsupportedOperationException("No key comparator configured for this map"); + RawEntry ra = (RawEntry)a; + RawEntry rb = (RawEntry)b; + return comparator.compare(ra.bytes, ra.decoded, rb.bytes, rb.decoded); + } + + + @Override + public int getMemory(Object obj) + { + return ((RawEntry)obj).bytes.length + 32; + } + + + @Override + public void write(WriteBuffer buff, Object obj) + { + buff.put(((RawEntry)obj).bytes); + } + + + @Override + public void write(WriteBuffer buff, Object[] obj, int len, boolean key) + { + for (int i = 0; i < len; i++) + write(buff, obj[i]); + } + + + @Override + public Object read(ByteBuffer buff) + { + if (decoder == null) + throw new UnsupportedOperationException("No decoder configured for this map"); + int start = buff.position(); + Object decoded = decoder.decode(buff); + int end = buff.position(); + if (end < start) + throw new IllegalStateException("Decoder moved buffer backwards"); + byte[] bytes = new byte[end - start]; + buff.position(start); + buff.get(bytes); + return new RawEntry(bytes, decoded); + } + + + @Override + public void read(ByteBuffer buff, Object[] obj, int len, boolean key) + { + for (int i = 0; i < len; i++) + obj[i] = read(buff); + } + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/DatabaseMigrationRequiredException.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/DatabaseMigrationRequiredException.java new file mode 100644 index 000000000..f0156348e --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/DatabaseMigrationRequiredException.java @@ -0,0 +1,40 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2; + +import org.sensorhub.api.datastore.DataStoreException; + + +/** + * Thrown when a database file was written by an older H2 version or lacks the + * version marker, so it must be migrated before the module can open it. The + * message tells the operator how (autoMigrate flag or command line tool). + */ +public class DatabaseMigrationRequiredException extends DataStoreException +{ + private static final long serialVersionUID = 1L; + + + public DatabaseMigrationRequiredException(String message) + { + super(message); + } + + + public DatabaseMigrationRequiredException(String message, Throwable cause) + { + super(message, cause); + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java index caa72db8f..26b82337e 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/H2Utils.java @@ -48,10 +48,10 @@ public class H2Utils // Compatibility bounds are deliberate, not derived from the running H2 version. static final String MIN_SUPPORTED_H2_VERSION = "2.5.250"; static final String MAX_SUPPORTED_H2_VERSION = "2.5.250"; - static final String DATABASE_MAP_NAME = "@database"; - static final String H2_VERSION_KEY = "h2.version"; + public static final String DATABASE_MAP_NAME = "@database"; + public static final String H2_VERSION_KEY = "h2.version"; - static final String DATASTORES_MAP_NAME = "@datastores"; + public static final String DATASTORES_MAP_NAME = "@datastores"; static final String GEOM_DIM_ERROR = "Only 2D and 3D geometries are supported"; private static final SpatialFactory SPATIAL_FACTORY = new SpatialFactory(); @@ -116,7 +116,8 @@ public static MVStore openMVStore(String storagePath, MVStore.Builder builder, i catch (MVStoreException e) { if (e.getErrorCode() == DataUtils.ERROR_UNSUPPORTED_FORMAT) - throw new DataStoreException("Unsupported H2 file format: " + storagePath, e); + throw new DatabaseMigrationRequiredException("Unsupported H2 file format: " + storagePath + + " (written by an older H2 version). " + migrationHint(storagePath), e); throw e; } @@ -176,10 +177,20 @@ private static int[] parseH2Version(String version) throws DataStoreException private static void migrateDatabase(String storagePath, String sourceVersion) throws DataStoreException { - // Migration of legacy files is not part of this change; files that need it are rejected here. - throw new DataStoreException("Database requires migration: " + storagePath + + // Migration (backup, interim copy, verification, atomic swap) is implemented in + // org.sensorhub.impl.datastore.h2.migrate. It runs at startup only when the database + // config enables autoMigrate; otherwise the operator runs it from the command line. + throw new DatabaseMigrationRequiredException("Database requires migration: " + storagePath + " (H2 version " + (sourceVersion == null ? "missing" : sourceVersion) + - "; supported range is " + MIN_SUPPORTED_H2_VERSION + " to " + MAX_SUPPORTED_H2_VERSION + ")"); + "; supported range is " + MIN_SUPPORTED_H2_VERSION + " to " + MAX_SUPPORTED_H2_VERSION + "). " + + migrationHint(storagePath)); + } + + + static String migrationHint(String storagePath) + { + return "Enable 'autoMigrate' in the database module config to migrate it at startup, " + + "or stop the node and run: java -cp 'lib/*' org.sensorhub.impl.datastore.h2.migrate.H2Migrate " + storagePath; } diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java index 323ee3c9e..7bee71f4c 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java @@ -34,7 +34,7 @@ */ public class MVFeatureDatabase extends AbstractModule implements IFeatureDatabase { - final static String FEATURE_STORE_NAME = "feature_store"; + public final static String FEATURE_STORE_NAME = "feature_store"; MVStore mvStore; MVFeatureStoreImpl featureStore; diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java index 51217c00c..7d4cef854 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java @@ -47,14 +47,14 @@ */ public class MVObsSystemDatabase extends AbstractModule implements IObsSystemDatabase, IProcedureDatabase, IObsSystemDatabaseModule { - final static String KRYO_CLASS_MAP_NAME = "kryo_class_map"; - final static String SYSTEM_STORE_NAME = "sys_store"; - final static String DEPL_STORE_NAME = "depl_store"; - final static String FOI_STORE_NAME = "foi_store"; - final static String OBS_STORE_NAME = "obs_store"; - final static String CMD_STORE_NAME = "cmd_store"; - final static String PROC_STORE_NAME = "proc_store"; - final static String PROP_STORE_NAME = "prop_store"; + public final static String KRYO_CLASS_MAP_NAME = "kryo_class_map"; + public final static String SYSTEM_STORE_NAME = "sys_store"; + public final static String DEPL_STORE_NAME = "depl_store"; + public final static String FOI_STORE_NAME = "foi_store"; + public final static String OBS_STORE_NAME = "obs_store"; + public final static String CMD_STORE_NAME = "cmd_store"; + public final static String PROC_STORE_NAME = "proc_store"; + public final static String PROP_STORE_NAME = "prop_store"; MVStore mvStore; MVSystemStoreImpl sysStore; diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/FileOps.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/FileOps.java new file mode 100644 index 000000000..6f36217bd --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/FileOps.java @@ -0,0 +1,212 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.HexFormat; +import java.util.regex.Pattern; + + +/** + * File-level steps of the migration: header inspection, lock probing, + * verified backup, instatement by rename (the original is retired as .old). + */ +final class FileOps +{ + private static final Pattern FORMAT = Pattern.compile("(?:^|,)format:(\\d+)(?:,|$)"); + private static final DateTimeFormatter TS = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'"); + + + private FileOps() + { + } + + + /** + * Reads the H2 file header and returns its "format" field. + * H2 1.4 writes format 1; H2 2.x writes format 2 or 3. + */ + static int readH2Format(Path file) throws IOException, MigrationException + { + byte[] head = new byte[8192]; + int n; + try (InputStream is = Files.newInputStream(file)) + { + n = is.readNBytes(head, 0, head.length); + } + String text = new String(head, 0, n, StandardCharsets.ISO_8859_1); + int start = text.indexOf("H:2,"); + if (start < 0) + throw new MigrationException("Not an H2 MVStore file (no header found): " + file); + int end = text.indexOf('\n', start); + String header = end < 0 ? text.substring(start) : text.substring(start, end); + var m = FORMAT.matcher(header); + if (!m.find()) + throw new MigrationException("H2 header has no format field: " + file); + return Integer.parseInt(m.group(1)); + } + + + /** + * Fails if another process holds a lock on the file (a running node). + */ + static void checkNotLocked(Path file) throws IOException, MigrationException + { + try (FileChannel ch = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) + { + FileLock lock = ch.tryLock(); + if (lock == null) + throw new MigrationException("Database file is locked by another process: " + file); + lock.release(); + } + catch (OverlappingFileLockException e) + { + throw new MigrationException("Database file is locked by this process: " + file); + } + } + + + /** + * Copies the file next to itself (or into backupDir) and verifies the copy + * by length and SHA-256. + * @return path of the verified backup + */ + static Path backup(Path file, Path backupDir, String tag) throws IOException, MigrationException + { + Path dir = backupDir != null ? backupDir : file.toAbsolutePath().getParent(); + Files.createDirectories(dir); + String stamp = ZonedDateTime.now(ZoneOffset.UTC).format(TS); + Path backup = dir.resolve(file.getFileName() + "." + tag + "-backup-" + stamp); + if (Files.exists(backup)) + throw new MigrationException("Backup target already exists: " + backup); + + Files.copy(file, backup, StandardCopyOption.COPY_ATTRIBUTES); + fsync(backup); + + long srcLen = Files.size(file); + long dstLen = Files.size(backup); + if (srcLen != dstLen) + throw new MigrationException("Backup length mismatch: " + srcLen + " vs " + dstLen + " for " + backup); + String srcHash = sha256(file); + String dstHash = sha256(backup); + if (!srcHash.equals(dstHash)) + throw new MigrationException("Backup checksum mismatch for " + backup); + return backup; + } + + + static String sha256(Path file) throws IOException + { + MessageDigest md; + try + { + md = MessageDigest.getInstance("SHA-256"); + } + catch (NoSuchAlgorithmException e) + { + throw new IllegalStateException(e); + } + try (FileChannel ch = FileChannel.open(file, StandardOpenOption.READ)) + { + ByteBuffer buf = ByteBuffer.allocate(1 << 20); + while (ch.read(buf) >= 0) + { + buf.flip(); + md.update(buf); + buf.clear(); + } + } + return HexFormat.of().formatHex(md.digest()); + } + + + /** + * Puts the interim file in place without ever overwriting or deleting the + * original: the original is renamed to {@code retired} (its bytes untouched), + * then the interim file is renamed onto the now free target path. If the second + * rename fails, the original is renamed back. + */ + static void instate(Path interim, Path target, Path retired) throws IOException + { + if (Files.exists(retired)) + throw new IOException("Cannot retire the original, file already exists: " + retired); + fsync(interim); + Files.move(target, retired, StandardCopyOption.ATOMIC_MOVE); + try + { + Files.move(interim, target, StandardCopyOption.ATOMIC_MOVE); + } + catch (IOException e) + { + Files.move(retired, target, StandardCopyOption.ATOMIC_MOVE); + throw e; + } + fsyncDirectory(target.toAbsolutePath().getParent()); + } + + + static void fsync(Path file) throws IOException + { + try (FileChannel ch = FileChannel.open(file, StandardOpenOption.READ)) + { + ch.force(true); + } + } + + + static void fsyncDirectory(Path dir) + { + if (dir == null) + return; + try (FileChannel ch = FileChannel.open(dir, StandardOpenOption.READ)) + { + ch.force(true); + } + catch (IOException e) + { + // directories cannot be opened for sync on every platform; best effort + } + } + + + static void deleteQuietly(Path file) + { + if (file == null) + return; + try + { + Files.deleteIfExists(file); + } + catch (IOException e) + { + // best effort cleanup + } + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/H2Migrate.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/H2Migrate.java new file mode 100644 index 000000000..3020542a0 --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/H2Migrate.java @@ -0,0 +1,152 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import java.nio.file.Paths; +import java.util.Locale; + + +/** + * Command line entry point. + * 
+ * h2migrate <db-file> [--type obs|feature] [--backup-dir DIR] [--legacy-libs DIR]
+ *                     [--keep-interim] [--dry-run] [--quiet]
+ *
+ */ +public final class H2Migrate +{ + static final String USAGE = String.join("\n", + "Usage: h2migrate [options]", + "", + "Migrates an OpenSensorHub H2 MVStore database written by H2 1.4.200 (file format 1),", + "or an unmarked H2 2.x file, to the H2 2.5.250 format with a version marker.", + "", + "Steps: preflight, optional backup copy, interim copy, byte comparison, decode check, then the", + "original is renamed to .old and the migrated file takes its place.", + "", + "Options:", + " --type obs|feature Database layout (default: detected from map names)", + " --backup Also keep a verified backup copy (.h2v1-backup-);", + " the original is kept unmodified as .old either way", + " --backup-dir DIR Where to write that copy (default: next to the database)", + " --legacy-libs DIR Directory holding h2-mvstore-1.4.200.jar and the legacy adapters", + " (default: -Dh2migrate.legacyLibs or /legacy-libs)", + " --keep-interim Keep the .migrating file if migration fails", + " --dry-run Do everything except retiring the original and moving the migrated file in", + " --quiet Only print the final report or the error", + " -h, --help Show this help", + "", + "Exit codes: 0 success, 1 migration failed (original untouched), 2 usage error"); + + + private H2Migrate() + { + } + + + public static void main(String[] args) + { + System.exit(run(args)); + } + + + static int run(String[] args) + { + Migrator.Options opts = new Migrator.Options(); + boolean quiet = false; + try + { + for (int i = 0; i < args.length; i++) + { + String a = args[i]; + switch (a) + { + case "-h", "--help" -> { + System.out.println(USAGE); + return 0; + } + case "--type" -> opts.type = parseType(next(args, ++i, a)); + case "--backup" -> opts.backup = true; + case "--backup-dir" -> opts.backupDir = Paths.get(next(args, ++i, a)); + case "--legacy-libs" -> opts.legacyLibs = Paths.get(next(args, ++i, a)); + case "--keep-interim" -> opts.keepInterim = true; + case "--dry-run" -> opts.dryRun = true; + case "--quiet" -> quiet = true; + default -> { + if (a.startsWith("-") || opts.dbFile != null) + throw new IllegalArgumentException("Unexpected argument: " + a); + opts.dbFile = Paths.get(a); + } + } + } + if (opts.dbFile == null) + throw new IllegalArgumentException("Missing "); + } + catch (IllegalArgumentException e) + { + System.err.println("Error: " + e.getMessage()); + System.err.println(); + System.err.println(USAGE); + return 2; + } + + if (!quiet) + opts.log = System.out::println; + + try + { + Migrator.Report report = Migrator.run(opts); + System.out.println(); + System.out.print(Migrator.describe(report)); + return 0; + } + catch (MigrationException e) + { + System.err.println("Migration failed: " + e.getMessage()); + if (e.getCause() != null) + System.err.println("Cause: " + e.getCause()); + System.err.println("The original database file was not modified."); + return 1; + } + catch (Exception e) + { + System.err.println("Migration failed with an unexpected error. The original database file was not modified."); + e.printStackTrace(System.err); + return 1; + } + } + + + private static String next(String[] args, int i, String option) + { + if (i >= args.length) + throw new IllegalArgumentException("Option " + option + " requires a value"); + return args[i]; + } + + + private static StoreLayout parseType(String s) + { + switch (s.toLowerCase(Locale.ROOT)) + { + case "obs", "obs_system", "obs-system": + return StoreLayout.OBS_SYSTEM; + case "feature", "features": + return StoreLayout.FEATURE; + default: + throw new IllegalArgumentException("Unknown database type: " + s + " (expected obs or feature)"); + } + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/LegacyLoader.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/LegacyLoader.java new file mode 100644 index 000000000..3baeb982c --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/LegacyLoader.java @@ -0,0 +1,242 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.sensorhub.impl.datastore.h2.migrate.spi.LegacyStore; + + +/** + * Child-first classloader that isolates the H2 1.4 jar and the legacy + * adapters from the H2 2.5 classes on the tool's own classpath. Everything + * else, including the spi interfaces and the OSH datatypes, is delegated to + * the parent so objects can be shared across the boundary. + */ +public final class LegacyLoader extends URLClassLoader +{ + public static final String LIBS_PROPERTY = "h2migrate.legacyLibs"; + /** Resource directory holding the embedded H2 1.4 jar and adapters, listed in index.txt */ + static final String EMBEDDED_DIR = "META-INF/h2migrate/"; + static final String LEGACY_PACKAGE = "org.sensorhub.impl.datastore.h2.migrate.legacy."; + static final String H2_PACKAGE = "org.h2."; + + + private LegacyLoader(URL[] urls, ClassLoader parent) + { + super("h2migrate-legacy", urls, parent); + } + + + /** Directory name looked up next to the directory holding this module's jar */ + public static final String LIBS_DIR_NAME = "legacy-libs"; + + + /** + * Creates the loader from the default location, in this order: an explicit + * directory, the {@value #LIBS_PROPERTY} system property, a {@value #LIBS_DIR_NAME} + * directory next to the directory holding this module's jar (a node's lib/), + * then the jars embedded in this module under {@value #EMBEDDED_DIR} when the + * module was built with them. + */ + public static LegacyLoader createDefault(Path explicit) throws MigrationException + { + return create(resolveLibsDir(explicit)); + } + + + public static Path resolveLibsDir(Path explicit) throws MigrationException + { + List candidates = new ArrayList<>(); + if (explicit != null) + candidates.add(explicit); + String prop = System.getProperty(LIBS_PROPERTY); + if (prop != null && !prop.isBlank()) + candidates.add(Paths.get(prop)); + Path installed = installedLibsDir(); + if (installed != null) + candidates.add(installed); + + for (Path dir: candidates) + { + if (Files.isDirectory(dir) && hasLegacyH2Jar(dir)) + return dir; + } + if (explicit != null || (prop != null && !prop.isBlank())) + throw new MigrationException("No h2-mvstore-1.4.x.jar found in " + candidates); + if (hasEmbedded()) + return extractEmbedded(); + throw new MigrationException("The H2 1.4 migration libraries were not found. Looked in " + candidates + + " and this build of the datastore module does not embed them. Put h2-mvstore-1.4.200.jar and " + + "the sensorhub-datastore-h2 h2migrate-legacy jar in a '" + LIBS_DIR_NAME + "' directory next to the " + + "node's lib/ directory (a source checkout builds it with ./gradlew :sensorhub-datastore-h2:h2migrateLegacyLibs), " + + "or pass --legacy-libs / -D" + LIBS_PROPERTY); + } + + + private static Path installedLibsDir() + { + try + { + var src = LegacyLoader.class.getProtectionDomain().getCodeSource(); + if (src == null) + return null; + Path location = Paths.get(src.getLocation().toURI()); + Path libDir = Files.isDirectory(location) ? location : location.getParent(); + return libDir == null || libDir.getParent() == null ? null : libDir.getParent().resolve(LIBS_DIR_NAME); + } + catch (Exception e) + { + return null; + } + } + + + static boolean hasEmbedded() + { + return LegacyLoader.class.getClassLoader().getResource(EMBEDDED_DIR + "index.txt") != null; + } + + + /** + * Copies the embedded legacy jars to a temporary directory that is removed on exit. + */ + static Path extractEmbedded() throws MigrationException + { + ClassLoader cl = LegacyLoader.class.getClassLoader(); + try (var index = cl.getResourceAsStream(EMBEDDED_DIR + "index.txt")) + { + if (index == null) + throw new MigrationException("This build does not embed the H2 1.4 migration libraries (" + + EMBEDDED_DIR + "index.txt missing); use --legacy-libs or -D" + LIBS_PROPERTY); + Path dir = Files.createTempDirectory("h2migrate-legacy-"); + dir.toFile().deleteOnExit(); + for (String name: new String(index.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).split("\\R")) + { + if (name.isBlank()) + continue; + try (var in = cl.getResourceAsStream(EMBEDDED_DIR + name)) + { + if (in == null) + throw new MigrationException("Embedded legacy library missing: " + EMBEDDED_DIR + name); + Path target = dir.resolve(name); + Files.copy(in, target); + target.toFile().deleteOnExit(); + } + } + if (!hasLegacyH2Jar(dir)) + throw new MigrationException("Embedded legacy libraries do not include h2-mvstore-1.4.x.jar"); + return dir; + } + catch (IOException e) + { + throw new MigrationException("Cannot extract embedded legacy libraries", e); + } + } + + + private static boolean hasLegacyH2Jar(Path dir) + { + try (Stream files = Files.list(dir)) + { + return files.anyMatch(f -> f.getFileName().toString().matches("h2-mvstore-1\\.4\\.\\d+\\.jar")); + } + catch (IOException e) + { + return false; + } + } + + + public static LegacyLoader create(Path libsDir) throws MigrationException + { + try (Stream files = Files.list(libsDir)) + { + List urls = new ArrayList<>(); + for (Path f: files.sorted().toList()) + { + if (f.getFileName().toString().endsWith(".jar")) + urls.add(f.toUri().toURL()); + } + if (urls.isEmpty()) + throw new MigrationException("No jars found in " + libsDir); + return new LegacyLoader(urls.toArray(new URL[0]), LegacyLoader.class.getClassLoader()); + } + catch (IOException e) + { + throw new MigrationException("Cannot list legacy libs in " + libsDir, e); + } + } + + + public LegacyStore.Factory factory() throws MigrationException + { + try + { + Class impl = Class.forName(LegacyStore.Factory.IMPL_CLASS, true, this); + return (LegacyStore.Factory)impl.getDeclaredConstructor().newInstance(); + } + catch (ReflectiveOperationException | ClassCastException e) + { + throw new MigrationException("Cannot load legacy adapters from " + List.of(getURLs()), e); + } + } + + + static boolean isIsolated(String className) + { + return className.startsWith(H2_PACKAGE) || className.startsWith(LEGACY_PACKAGE); + } + + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException + { + synchronized (getClassLoadingLock(name)) + { + Class c = findLoadedClass(name); + if (c == null) + { + if (!isIsolated(name)) + return super.loadClass(name, resolve); + c = findClass(name); // never fall back to the parent's H2 2.5 classes + } + if (resolve) + resolveClass(c); + return c; + } + } + + + @Override + public URL getResource(String name) + { + // resources under org/h2 must also come from the 1.4 jar + if (name.startsWith("org/h2/")) + { + URL url = findResource(name); + if (url != null) + return url; + } + return super.getResource(name); + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/MigrationException.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/MigrationException.java new file mode 100644 index 000000000..b14d2ef96 --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/MigrationException.java @@ -0,0 +1,37 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + + +/** + * Raised when the migration cannot proceed or a verification step fails. + * The original database file is untouched whenever this is thrown. + */ +public class MigrationException extends Exception +{ + private static final long serialVersionUID = 1L; + + + public MigrationException(String message) + { + super(message); + } + + + public MigrationException(String message, Throwable cause) + { + super(message, cause); + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/Migrator.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/Migrator.java new file mode 100644 index 000000000..f3649ba67 --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/Migrator.java @@ -0,0 +1,448 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import org.sensorhub.impl.datastore.h2.MVObsSystemDatabase; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Consumer; +import org.h2.mvstore.MVBTreeMap; +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.rtree.MVRTreeMap; +import org.h2.mvstore.rtree.Spatial; +import org.h2.mvstore.rtree.SpatialDataType; +import org.sensorhub.impl.datastore.h2.H2Utils; + + +/** + * The migration pipeline: preflight, verified backup, interim copy, + * byte-level and semantic verification, instatement. + * The original file is never written to: it is only read, copied for the + * backup, and finally renamed to .old when the migrated file takes its + * path. Neither the backup nor the retired original is ever deleted here. + */ +public final class Migrator +{ + /** Suffix under which the untouched original is kept after instatement */ + public static final String RETIRED_SUFFIX = ".old"; + + /** Commit the interim store whenever this much unsaved memory accumulates */ + private static final int COMMIT_THRESHOLD_BYTES = 32 * 1024 * 1024; + + private static final Set NEVER_COPIED = Set.of(H2Utils.DATABASE_MAP_NAME); + + + public static final class Options + { + public Path dbFile; + public StoreLayout type; // null = detect from map names + public boolean backup; // also keep a verified copy of the original (it survives as .old anyway) + public Path backupDir; // null = next to the database file + public Path legacyLibs; // null = system property, then the jars embedded in this module + public boolean keepInterim; + public boolean dryRun; + public Consumer log = s -> {}; + } + + + public static final class Report + { + public Path dbFile; + public Path backup; + public Path interim; + /** the untouched original after instatement (.old), null for a dry run */ + public Path retiredOriginal; + public int sourceFormat; + public String sourceDescription; + public StoreLayout type; + public Map copied = new LinkedHashMap<>(); + /** byte-identical duplicate keys found in the source and stored once, per map */ + public Map duplicatesCollapsed = new LinkedHashMap<>(); + public boolean instated; + public Duration elapsed; + } + + + private static final class SpatialFactory extends SpatialDataType + { + SpatialFactory(int dimensions) + { + super(dimensions); + } + + @Override + public Spatial create(long id, float... minMax) + { + return super.create(id, minMax); + } + } + + + private Migrator() + { + } + + + public static Report run(Options options) throws Exception + { + Instant t0 = Instant.now(); + Consumer log = options.log; + Path dbFile = options.dbFile.toAbsolutePath(); + Report report = new Report(); + report.dbFile = dbFile; + + // 1. preflight + if (!Files.isRegularFile(dbFile) || Files.size(dbFile) == 0) + throw new MigrationException("Database file is missing or empty: " + dbFile); + FileOps.checkNotLocked(dbFile); + int format = FileOps.readH2Format(dbFile); + report.sourceFormat = format; + log.accept("Source file: " + dbFile + " (H2 file format " + format + ")"); + + Path interim = dbFile.resolveSibling(dbFile.getFileName() + ".migrating"); + Path scratch = dbFile.resolveSibling(dbFile.getFileName() + ".migrating-types"); + if (Files.exists(interim)) + throw new MigrationException("Interim file already exists, remove it first: " + interim); + report.interim = interim; + Path retired = dbFile.resolveSibling(dbFile.getFileName() + RETIRED_SUFFIX); + if (!options.dryRun && Files.exists(retired)) + throw new MigrationException("A retired original already exists, move it away first: " + retired); + + LegacyLoader loader = null; + if (format == 1) + { + Path libs = LegacyLoader.resolveLibsDir(options.legacyLibs); + log.accept("Legacy H2 libraries: " + libs); + loader = LegacyLoader.create(libs); + } + + boolean success = false; + try (SourceReader source = openSource(format, loader, dbFile)) + { + report.sourceDescription = source.describe(); + log.accept("Reading with " + source.describe()); + + if (source instanceof SourceReader.Native native_) + { + String marker = native_.getVersionMarker(); + if (marker != null) + throw new MigrationException("File already carries version marker " + marker + + "; nothing to migrate: " + dbFile); + } + + StoreLayout type = options.type != null ? options.type : StoreLayout.detect(source.getMapNames()); + report.type = type; + log.accept("Database type: " + type); + + // 2. harvest datatypes on a scratch store seeded with the class map, and make sure + // the layout knows every source map before anything is written next to the original + try (StoreLayout.Types types = StoreLayout.Types.harvest(source, type, scratch)) + { + List unknown = new ArrayList<>(); + for (String name: source.getMapNames()) + { + if (!NEVER_COPIED.contains(name) && types.get(name) == null) + unknown.add(name); + } + if (!unknown.isEmpty()) + throw new MigrationException("Source contains maps unknown to the " + type + " layout: " + unknown + + ". This tool cannot migrate them; a layout extension is required."); + + // 3. optional backup copy; the original itself is retired as .old at the end + if (options.backup) + { + report.backup = FileOps.backup(dbFile, options.backupDir, format == 1 ? "h2v1" : "h2v2"); + log.accept("Backup verified: " + report.backup); + } + else + log.accept("No backup copy requested; the original will be kept unmodified as " + retired); + + // 4. copy every map into the interim file, class map first + MVStore target = H2Utils.openMVStore(interim.toString(), new MVStore.Builder(), 0); + try + { + for (String name: copyOrder(source.getMapNames())) + { + var result = copyMap(source, target, types.get(name), log); + report.copied.put(name, result.stored); + if (result.duplicates > 0) + report.duplicatesCollapsed.put(name, result.duplicates); + target.commit(); + } + target.commit(); + } + finally + { + target.close(); + } + log.accept("Interim file written: " + interim); + + // 7. pass one: compare bytes + log.accept("Verifying bytes against source..."); + try (SourceReader migrated = new SourceReader.Native(interim, 3)) + { + Verifier.compareBytes(source, migrated, types.getSpecs(), NEVER_COPIED, log); + } + } + + // 8. pass two: decode through the store implementations + log.accept("Verifying decoding through store implementations..."); + Map decoded = Verifier.decodeAll(interim, type, log); + for (var e: report.copied.entrySet()) + { + Long n = decoded.get(e.getKey()); + if (n == null) + throw new MigrationException("Map " + e.getKey() + " was not opened by the store implementations"); + // the class map may legitimately grow while the stores read pages, and opening the stores + // registers every store the file predates in @datastores; data maps may not change + boolean mayGrow = e.getKey().equals(MVObsSystemDatabase.KRYO_CLASS_MAP_NAME) || e.getKey().equals(H2Utils.DATASTORES_MAP_NAME); + boolean ok = mayGrow ? n >= e.getValue() : n.equals(e.getValue()); + if (!ok) + throw new MigrationException("Decoded count for " + e.getKey() + " is " + n + ", expected " + e.getValue()); + } + + // 9. instate + if (options.dryRun) + { + log.accept("Dry run: original left in place, interim kept at " + interim); + } + else + { + FileOps.instate(interim, dbFile, retired); + report.instated = true; + report.retiredOriginal = retired; + log.accept("Original kept unmodified as " + retired); + log.accept("Migrated database instated at " + dbFile); + } + success = true; + } + catch (DuplicateConflictException e) + { + throw new MigrationException(e.getMessage(), e); + } + finally + { + if (loader != null) + loader.close(); + FileOps.deleteQuietly(scratch); + if (!success && !options.keepInterim) + FileOps.deleteQuietly(interim); + if (success && report.instated) + FileOps.deleteQuietly(interim); // already moved; no-op unless move left a copy + } + + report.elapsed = Duration.between(t0, Instant.now()); + return report; + } + + + /** + * Tells whether a database file must be migrated before the datastore can open it: + * an H2 1.4 (format 1) file, or an H2 2.x file without the version marker. + * Missing or empty files are new databases and return false. + */ + public static boolean needsMigration(Path dbFile) throws Exception + { + if (!Files.isRegularFile(dbFile) || Files.size(dbFile) == 0) + return false; + int format = FileOps.readH2Format(dbFile); + if (format == 1) + return true; + try (SourceReader.Native reader = new SourceReader.Native(dbFile, format)) + { + return reader.getVersionMarker() == null; + } + } + + + /** + * Startup hook for the database modules: migrates the file in place (with + * verification; the original is kept as .old) when it needs it, + * otherwise does nothing. + * @param backup also keep a verified backup copy of the original + * @return the report, or null if no migration was needed + */ + public static Report migrateIfNeeded(Path dbFile, boolean backup, Consumer log) throws Exception + { + if (!needsMigration(dbFile)) + return null; + log.accept("Database file " + dbFile + " uses a legacy H2 format; migrating it (autoMigrate is enabled)"); + Options options = new Options(); + options.dbFile = dbFile; + options.backup = backup; + options.log = log; + Report report = run(options); + log.accept(describe(report)); + return report; + } + + + private static SourceReader openSource(int format, LegacyLoader loader, Path dbFile) throws Exception + { + if (format == 1) + return new SourceReader.Legacy(loader.factory(), dbFile); + return new SourceReader.Native(dbFile, format); + } + + + /** Class map first, datastore registry second, then everything else sorted */ + static List copyOrder(Set names) + { + List order = new ArrayList<>(); + if (names.contains(MVObsSystemDatabase.KRYO_CLASS_MAP_NAME)) + order.add(MVObsSystemDatabase.KRYO_CLASS_MAP_NAME); + if (names.contains(H2Utils.DATASTORES_MAP_NAME)) + order.add(H2Utils.DATASTORES_MAP_NAME); + for (String n: new TreeSet<>(names)) + { + if (!order.contains(n) && !NEVER_COPIED.contains(n)) + order.add(n); + } + return order; + } + + + record CopyResult(long stored, long duplicates) + { + } + + + private static CopyResult copyMap(SourceReader source, MVStore target, StoreLayout.MapSpec spec, Consumer log) throws Exception + { + long n; + long duplicates = 0; + if (spec.rtree()) + { + if (!source.isRTree(spec.name())) + log.accept(" note: " + spec.name() + " has no R-tree marker in the source metadata " + + "(dropped by H2 1.4 compaction); it is read as an R-tree and the marker is restored"); + var factory = new SpatialFactory(spec.dimensions()); + MVRTreeMap map = target.openMap(spec.name(), new MVRTreeMap.Builder() + .dimensions(spec.dimensions()) + .valueType(new RawDataType(spec.valueType()))); + n = source.readRTreeMap(spec.name(), spec.dimensions(), spec.valueType(), (id, minMax, v, dv) -> { + map.put(factory.create(id, minMax), new RawDataType.Entry(v)); + commitIfNeeded(target); + }); + } + else + { + MVMap map = target.openMap(spec.name(), new MVBTreeMap.Builder() + .keyType(new RawDataType(spec.keyType())) + .valueType(new RawDataType(spec.valueType()))); + var dedupe = new DuplicateFilter(spec.name()); + source.readMap(spec.name(), spec.keyType(), spec.valueType(), (k, dk, v, dv) -> { + if (dedupe.isDuplicate(k, v)) + return; + map.put(new RawDataType.Entry(k, dk), new RawDataType.Entry(v)); + commitIfNeeded(target); + }); + n = dedupe.kept; + duplicates = dedupe.duplicates; + } + log.accept(" copied " + spec.name() + ": " + n + " entries" + (spec.rtree() ? " (rtree)" : "") + + (duplicates > 0 ? ", collapsed " + duplicates + " byte-identical duplicate key(s)" : "")); + return new CopyResult(n, duplicates); + } + + + private static void commitIfNeeded(MVStore target) + { + if (target.getUnsavedMemory() > COMMIT_THRESHOLD_BYTES) + target.commit(); + } + + + /** + * Legacy B-trees can hold consecutive entries whose key bytes are identical. + * The datastore compares some keys on fields it never persists (for example the + * full-text index compares feature valid time but stores only the feature id), + * so two versions of one feature produced two on-disk entries that decode to the + * same key. They collapse to one entry here; identical keys with different values + * are refused because that would silently drop data. + */ + /** Unchecked so it can cross the sink callbacks; converted to MigrationException in run() */ + static final class DuplicateConflictException extends RuntimeException + { + private static final long serialVersionUID = 1L; + + DuplicateConflictException(String message) + { + super(message); + } + } + + + static final class DuplicateFilter + { + private final String mapName; + private byte[] lastKey; + private byte[] lastValue; + long kept; + long duplicates; + + DuplicateFilter(String mapName) + { + this.mapName = mapName; + } + + boolean isDuplicate(byte[] key, byte[] value) + { + if (lastKey != null && java.util.Arrays.equals(lastKey, key)) + { + if (!java.util.Arrays.equals(lastValue, value)) + throw new DuplicateConflictException("Map " + mapName + " holds byte-identical duplicate keys with " + + "different values; refusing to choose one (key " + java.util.HexFormat.of().formatHex(key) + ")"); + duplicates++; + return true; + } + lastKey = key; + lastValue = value; + kept++; + return false; + } + } + + + static String describe(Report r) + { + StringBuilder sb = new StringBuilder(); + sb.append("Database: ").append(r.dbFile).append('\n'); + sb.append("Source: ").append(r.sourceDescription).append('\n'); + sb.append("Type: ").append(r.type).append('\n'); + sb.append("Backup: ").append(r.backup != null ? r.backup : "none requested").append('\n'); + sb.append("Result: ").append(r.instated ? "instated at " + r.dbFile : "dry run, interim at " + r.interim).append('\n'); + if (r.retiredOriginal != null) + sb.append("Original: kept unmodified as ").append(r.retiredOriginal).append('\n'); + sb.append("Elapsed: ").append(r.elapsed).append('\n'); + sb.append("Maps:\n"); + for (var e: r.copied.entrySet()) + { + Long dup = r.duplicatesCollapsed.get(e.getKey()); + sb.append(String.format(" %-40s %10d%s%n", e.getKey(), e.getValue(), + dup == null ? "" : " (+" + dup + " byte-identical duplicate key(s) collapsed)")); + } + return sb.toString(); + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/RawDataType.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/RawDataType.java new file mode 100644 index 000000000..d204ca555 --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/RawDataType.java @@ -0,0 +1,126 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import java.nio.ByteBuffer; +import org.h2.mvstore.WriteBuffer; +import org.h2.mvstore.type.BasicDataType; +import org.h2.mvstore.type.DataType; + + +/** + * H2 2.5 datatype that moves entries as exact bytes. Reads delegate to the + * real OSH datatype to find the entry boundary and keep the decoded object; + * writes emit the recorded bytes verbatim; key comparison decodes lazily and + * uses the real datatype's ordering, so the resulting B-tree is identical to + * one built through the store implementations. + */ +public final class RawDataType extends BasicDataType +{ + public static final class Entry + { + private final byte[] bytes; + private Object decoded; + private boolean isDecoded; + + + public Entry(byte[] bytes) + { + this.bytes = bytes; + } + + + public Entry(byte[] bytes, Object decoded) + { + this.bytes = bytes; + this.decoded = decoded; + this.isDecoded = true; + } + + + public byte[] bytes() + { + return bytes; + } + + + public Object decoded(DataType real) + { + if (!isDecoded) + { + decoded = real.read(ByteBuffer.wrap(bytes)); + isDecoded = true; + } + return decoded; + } + } + + + private final DataType real; + + + @SuppressWarnings("unchecked") + public RawDataType(DataType real) + { + this.real = (DataType)real; + } + + + public DataType getRealType() + { + return real; + } + + + @Override + public int compare(Object a, Object b) + { + return real.compare(((Entry)a).decoded(real), ((Entry)b).decoded(real)); + } + + + @Override + public int getMemory(Object obj) + { + return ((Entry)obj).bytes.length + 32; + } + + + @Override + public void write(WriteBuffer buff, Object obj) + { + buff.put(((Entry)obj).bytes); + } + + + @Override + public Object read(ByteBuffer buff) + { + int start = buff.position(); + Object decoded = real.read(buff); + int end = buff.position(); + byte[] bytes = new byte[end - start]; + buff.position(start); + buff.get(bytes); + return new Entry(bytes, decoded); + } + + + @Override + public Object[] createStorage(int size) + { + return new Object[size]; + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/SourceReader.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/SourceReader.java new file mode 100644 index 000000000..0fee49421 --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/SourceReader.java @@ -0,0 +1,269 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import org.h2.engine.Constants; +import org.h2.mvstore.Cursor; +import org.h2.mvstore.DataUtils; +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.Page; +import org.h2.mvstore.rtree.MVRTreeMap; +import org.h2.mvstore.rtree.Spatial; +import org.h2.mvstore.type.DataType; +import org.h2.mvstore.type.StringDataType; +import org.sensorhub.impl.datastore.h2.H2Utils; +import org.sensorhub.impl.datastore.h2.migrate.spi.LegacyStore; + + +/** + * Uniform raw access to a database file regardless of the H2 version that + * wrote it. Entries are delivered as the exact bytes stored in the file plus + * the objects decoded with the supplied real datatypes. + */ +public interface SourceReader extends AutoCloseable +{ + /** @return human readable description, e.g. "H2 1.4.200 file (format 1)" */ + String describe(); + + Set getMapNames(); + + boolean isRTree(String mapName); + + long readMap(String mapName, DataType keyType, DataType valueType, LegacyStore.RawEntrySink sink) throws Exception; + + long readRTreeMap(String mapName, int dimensions, DataType valueType, LegacyStore.SpatialEntrySink sink) throws Exception; + + @Override + void close(); + + + /** + * Reads an H2 1.4 file through the isolated legacy classloader. The real + * H2 2.5 datatypes do the decoding; only byte buffers and JDK objects cross + * the classloader boundary. + */ + public static final class Legacy implements SourceReader + { + private final LegacyStore store; + + + public Legacy(LegacyStore.Factory factory, Path file) throws MigrationException + { + this.store = factory.openReadOnly(file.toString()); + String version = store.getH2Version(); + if (!version.startsWith("1.")) + { + store.close(); + throw new MigrationException("Legacy classloader loaded H2 " + version + " instead of a 1.x release"); + } + } + + + @Override + public String describe() + { + return "H2 " + store.getH2Version() + " file (format 1)"; + } + + + @Override + public Set getMapNames() + { + return store.getMapNames(); + } + + + @Override + public boolean isRTree(String mapName) + { + return store.isRTree(mapName); + } + + + @Override + public long readMap(String mapName, DataType keyType, DataType valueType, LegacyStore.RawEntrySink sink) + { + return store.readMap(mapName, decoder(keyType), decoder(valueType), sink); + } + + + @Override + public long readRTreeMap(String mapName, int dimensions, DataType valueType, LegacyStore.SpatialEntrySink sink) + { + return store.readRTreeMap(mapName, dimensions, decoder(valueType), sink); + } + + + @SuppressWarnings("unchecked") + private static LegacyStore.EntryDecoder decoder(DataType type) + { + var t = (DataType)type; + return t::read; + } + + + @Override + public void close() + { + store.close(); + } + } + + + /** + * Reads a file the embedded H2 2.5 library can open natively. Used for + * unmarked 2.x sources and to read the interim file back during verification. + */ + public static final class Native implements SourceReader + { + private final MVStore store; + private final int format; + + + public Native(Path file, int format) + { + this.format = format; + this.store = new MVStore.Builder() + .fileName(file.toString()) + .readOnly() + .open(); + } + + + @Override + public String describe() + { + return "H2 " + Constants.VERSION + " readable file (format " + format + ")"; + } + + + @Override + public Set getMapNames() + { + return Collections.unmodifiableSet(new TreeSet<>(store.getMapNames())); + } + + + public Map getMapMeta(String mapName) + { + String id = store.getMetaMap().get(DataUtils.META_NAME + mapName); + if (id == null) + throw new IllegalArgumentException("Unknown map: " + mapName); + String config = store.getMetaMap().get("map." + id); + if (config == null) + throw new IllegalStateException("Missing metadata for map: " + mapName); + return DataUtils.parseMap(config); + } + + + @Override + public boolean isRTree(String mapName) + { + return "rtree".equals(getMapMeta(mapName).get("type")); + } + + + /** + * Reads the version marker if present. + * @return marker value or null if the map or key is missing + */ + public String getVersionMarker() + { + if (!store.hasMap(H2Utils.DATABASE_MAP_NAME)) + return null; + MVMap map = store.openMap(H2Utils.DATABASE_MAP_NAME, new MVMap.Builder() + .keyType(StringDataType.INSTANCE) + .valueType(StringDataType.INSTANCE)); + return map.get(H2Utils.H2_VERSION_KEY); + } + + + @Override + public long readMap(String mapName, DataType keyType, DataType valueType, LegacyStore.RawEntrySink sink) + { + if (isRTree(mapName)) + throw new IllegalArgumentException("Map is an R-tree, use readRTreeMap: " + mapName); + + MVMap map = store.openMap(mapName, new MVMap.Builder() + .keyType(new RawDataType(keyType)) + .valueType(new RawDataType(valueType))); + + long count = 0; + Cursor cursor = map.cursor(null); + while (cursor.hasNext()) + { + var key = (RawDataType.Entry)cursor.next(); + var value = (RawDataType.Entry)cursor.getValue(); + sink.entry(key.bytes(), key.decoded(map.getKeyType()), value.bytes(), value.decoded(map.getValueType())); + count++; + } + return count; + } + + + @Override + public long readRTreeMap(String mapName, int dimensions, DataType valueType, LegacyStore.SpatialEntrySink sink) + { + // no metadata check: the store layout decides the type, see LegacyStore.readRTreeMap + MVRTreeMap map = store.openMap(mapName, new MVRTreeMap.Builder() + .dimensions(dimensions) + .valueType(new RawDataType(valueType))); + + return walk(map.getRootPage(), map.getValueType(), dimensions, sink); + } + + + private static long walk(Page page, DataType valueType, int dimensions, LegacyStore.SpatialEntrySink sink) + { + long count = 0; + if (page.isLeaf()) + { + for (int i = 0; i < page.getKeyCount(); i++) + { + Spatial key = page.getKey(i); + var value = (RawDataType.Entry)page.getValue(i); + float[] minMax = new float[2 * dimensions]; + for (int d = 0; d < dimensions; d++) + { + minMax[2 * d] = key.min(d); + minMax[2 * d + 1] = key.max(d); + } + sink.entry(key.getId(), minMax, value.bytes(), value.decoded(valueType)); + count++; + } + } + else + { + // R-tree node pages hold one child per key; the extra raw slot is an empty placeholder + for (int i = 0; i < page.getKeyCount(); i++) + count += walk(page.getChildPage(i), valueType, dimensions, sink); + } + return count; + } + + + @Override + public void close() + { + store.closeImmediately(); + } + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/StoreLayout.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/StoreLayout.java new file mode 100644 index 000000000..0bad01490 --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/StoreLayout.java @@ -0,0 +1,226 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import org.h2.mvstore.DataUtils; +import org.h2.mvstore.MVBTreeMap; +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.rtree.MVRTreeMap; +import org.h2.mvstore.type.DataType; +import org.h2.mvstore.type.ObjectDataType; +import org.sensorhub.impl.datastore.h2.H2Utils; +import org.sensorhub.impl.datastore.h2.MVCommandStoreImpl; +import org.sensorhub.impl.datastore.h2.MVDataStoreInfo; +import org.sensorhub.impl.datastore.h2.MVDatabaseConfig.IdProviderType; +import org.sensorhub.impl.datastore.h2.MVDeploymentStoreImpl; +import org.sensorhub.impl.datastore.h2.MVFeatureDatabase; +import org.sensorhub.impl.datastore.h2.MVFeatureStoreImpl; +import org.sensorhub.impl.datastore.h2.MVFoiStoreImpl; +import org.sensorhub.impl.datastore.h2.MVObsStoreImpl; +import org.sensorhub.impl.datastore.h2.MVObsSystemDatabase; +import org.sensorhub.impl.datastore.h2.MVProcedureStoreImpl; +import org.sensorhub.impl.datastore.h2.MVPropertyStoreImpl; +import org.sensorhub.impl.datastore.h2.MVSystemStoreImpl; + + +/** + * The two database layouts the datastore module produces, and how the tool + * learns the real datatype of every map: opening the stores on an empty + * scratch MVStore creates each map exactly as the database modules do. + */ +public enum StoreLayout +{ + OBS_SYSTEM(List.of(MVObsSystemDatabase.SYSTEM_STORE_NAME, MVObsSystemDatabase.DEPL_STORE_NAME, MVObsSystemDatabase.FOI_STORE_NAME, + MVObsSystemDatabase.OBS_STORE_NAME, MVObsSystemDatabase.CMD_STORE_NAME, MVObsSystemDatabase.PROC_STORE_NAME, MVObsSystemDatabase.PROP_STORE_NAME)), + FEATURE(List.of(MVFeatureDatabase.FEATURE_STORE_NAME)); + + // ID scope is only applied when decoding IDs, never written, so 0 is safe here + private static final int ID_SCOPE = 0; + + private final List storeNames; + + + private StoreLayout(List storeNames) + { + this.storeNames = storeNames; + } + + + public List getStoreNames() + { + return storeNames; + } + + + /** + * Opens (and creates if needed) every store of this layout, exactly as the + * database modules do at startup, minus the cross-store links. + */ + public void openStores(MVStore store) + { + var idProvider = IdProviderType.SEQUENTIAL; + switch (this) + { + case OBS_SYSTEM: + MVSystemStoreImpl.open(store, ID_SCOPE, idProvider, info(MVObsSystemDatabase.SYSTEM_STORE_NAME)); + MVDeploymentStoreImpl.open(store, ID_SCOPE, idProvider, info(MVObsSystemDatabase.DEPL_STORE_NAME)); + MVFoiStoreImpl.open(store, ID_SCOPE, idProvider, info(MVObsSystemDatabase.FOI_STORE_NAME)); + MVObsStoreImpl.open(store, ID_SCOPE, idProvider, info(MVObsSystemDatabase.OBS_STORE_NAME)); + MVCommandStoreImpl.open(store, ID_SCOPE, idProvider, info(MVObsSystemDatabase.CMD_STORE_NAME)); + MVProcedureStoreImpl.open(store, ID_SCOPE, idProvider, info(MVObsSystemDatabase.PROC_STORE_NAME)); + MVPropertyStoreImpl.open(store, ID_SCOPE, idProvider, info(MVObsSystemDatabase.PROP_STORE_NAME)); + break; + + case FEATURE: + MVFeatureStoreImpl.open(store, ID_SCOPE, idProvider, info(MVFeatureDatabase.FEATURE_STORE_NAME)); + break; + } + } + + + private static MVDataStoreInfo info(String name) + { + return MVDataStoreInfo.builder().withName(name).build(); + } + + + /** + * Picks the layout from the map names found in a source file. + * @throws MigrationException if the names match neither or both layouts + */ + public static StoreLayout detect(Set mapNames) throws MigrationException + { + boolean feature = mapNames.stream().anyMatch(n -> n.startsWith(MVFeatureDatabase.FEATURE_STORE_NAME + ":")); + boolean obs = mapNames.stream().anyMatch(n -> n.startsWith(MVObsSystemDatabase.SYSTEM_STORE_NAME + ":") || n.startsWith(MVObsSystemDatabase.OBS_STORE_NAME + ":")); + if (feature && !obs) + return FEATURE; + if (obs && !feature) + return OBS_SYSTEM; + throw new MigrationException("Cannot detect database type from map names " + mapNames + + "; use --type obs or --type feature"); + } + + + /** Dimensions used by every OSH spatial index (see SpatialIndex) */ + public static final int SPATIAL_DIMENSIONS = 3; + + + /** + * The real H2 2.5 datatypes an OSH store uses for one map, learned by letting + * the store implementations open the map themselves. + */ + public record MapSpec(String name, DataType keyType, DataType valueType, boolean rtree, int dimensions) + { + } + + + /** + * Learns the real datatype of every map by letting the store implementations + * open an empty scratch database. The scratch store first receives the source + * database's Kryo class map, because the Kryo datatypes resolve class ids + * through the class map of the store they were opened on. The scratch store + * stays open for as long as the harvested datatypes are in use. + */ + public static final class Types implements AutoCloseable + { + private final MVStore scratch; + private final Path scratchPath; + private final Map specs; + + + private Types(MVStore scratch, Path scratchPath, Map specs) + { + this.scratch = scratch; + this.scratchPath = scratchPath; + this.specs = Collections.unmodifiableMap(specs); + } + + + public static Types harvest(SourceReader source, StoreLayout layout, Path scratchPath) throws Exception + { + Files.deleteIfExists(scratchPath); + MVStore scratch = H2Utils.openMVStore(scratchPath.toString(), new MVStore.Builder(), 0); + try + { + if (source.getMapNames().contains(MVObsSystemDatabase.KRYO_CLASS_MAP_NAME)) + { + // typed copy: String -> Integer through H2's default datatype, whose + // encoding is identical in 1.4 and 2.5 + MVMap classMap = scratch.openMap(MVObsSystemDatabase.KRYO_CLASS_MAP_NAME, + new MVBTreeMap.Builder()); + var od = new ObjectDataType(); + source.readMap(MVObsSystemDatabase.KRYO_CLASS_MAP_NAME, od, od, + (k, dk, v, dv) -> classMap.put((String)dk, (Integer)dv)); + scratch.commit(); + } + + layout.openStores(scratch); + scratch.commit(); + + Map specs = new TreeMap<>(); + for (String name: scratch.getMapNames()) + { + String id = scratch.getMetaMap().get(DataUtils.META_NAME + name); + MVMap map = scratch.getMap(DataUtils.parseHexInt(id)); + if (map == null) + throw new MigrationException("Store implementations did not open map " + name); + + boolean rtree = map instanceof MVRTreeMap; + @SuppressWarnings("unchecked") + var keyType = (DataType)map.getKeyType(); + @SuppressWarnings("unchecked") + var valueType = (DataType)map.getValueType(); + specs.put(name, new MapSpec(name, keyType, valueType, rtree, rtree ? SPATIAL_DIMENSIONS : 0)); + } + return new Types(scratch, scratchPath, specs); + } + catch (Exception | Error e) + { + scratch.closeImmediately(); + Files.deleteIfExists(scratchPath); + throw e; + } + } + + + public Map getSpecs() + { + return specs; + } + + + public MapSpec get(String mapName) + { + return specs.get(mapName); + } + + + @Override + public void close() throws IOException + { + scratch.closeImmediately(); + Files.deleteIfExists(scratchPath); + } + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/Verifier.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/Verifier.java new file mode 100644 index 000000000..7a3f680db --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/Verifier.java @@ -0,0 +1,270 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import org.sensorhub.impl.datastore.h2.MVObsSystemDatabase; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Consumer; +import org.h2.mvstore.Cursor; +import org.h2.mvstore.DataUtils; +import org.h2.mvstore.MVBTreeMap; +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.Page; +import org.h2.mvstore.rtree.MVRTreeMap; +import org.h2.mvstore.rtree.Spatial; +import org.sensorhub.impl.datastore.h2.H2Utils; + + +/** + * Two independent checks of the migrated file. + * Pass one compares the source and the interim file entry by entry as bytes: + * map name sets, per-map counts and a SHA-256 over every key and value + * (ordered for B-trees, order-independent for R-trees). + * Pass two opens the interim file through the real store implementations and + * decodes every entry with the real datatypes. + */ +final class Verifier +{ + private Verifier() + { + } + + + /** Per-map streaming digest of length-prefixed key and value bytes */ + private static final class MapDigest + { + final MessageDigest md = newDigest(); + final List unordered = new ArrayList<>(); + long count; + + void add(byte[] key, byte[] value, boolean ordered) + { + count++; + if (ordered) + { + update(md, key); + update(md, value); + } + else + { + MessageDigest one = newDigest(); + update(one, key); + update(one, value); + unordered.add(one.digest()); + } + } + + byte[] result() + { + if (!unordered.isEmpty()) + { + unordered.sort(Arrays::compare); + for (byte[] d: unordered) + md.update(d); + } + return md.digest(); + } + + static void update(MessageDigest md, byte[] bytes) + { + md.update(ByteBuffer.allocate(4).putInt(bytes.length).array()); + md.update(bytes); + } + } + + + private static MessageDigest newDigest() + { + try + { + return MessageDigest.getInstance("SHA-256"); + } + catch (NoSuchAlgorithmException e) + { + throw new IllegalStateException(e); + } + } + + + private static byte[] spatialKeyBytes(long id, float[] minMax) + { + ByteBuffer b = ByteBuffer.allocate(8 + 4 * minMax.length); + b.putLong(id); + for (float f: minMax) + b.putFloat(f); + return b.array(); + } + + + /** + * Pass one. + * @param skip source maps that are intentionally not copied + * @return per-map entry counts as seen in the interim file + */ + static Map compareBytes(SourceReader source, SourceReader target, Map specs, + Set skip, Consumer log) throws Exception + { + Set expected = new TreeSet<>(source.getMapNames()); + expected.removeAll(skip); + expected.remove(H2Utils.DATABASE_MAP_NAME); + Set actual = new TreeSet<>(target.getMapNames()); + actual.remove(H2Utils.DATABASE_MAP_NAME); + if (!expected.equals(actual)) + throw new MigrationException("Map sets differ: source " + expected + " vs migrated " + actual); + + Map counts = new TreeMap<>(); + for (String name: expected) + { + StoreLayout.MapSpec spec = specs.get(name); + if (spec == null) + throw new MigrationException("No datatype known for map " + name); + // the store layout decides the map type; metadata may only disagree in one + // direction (marker dropped by H2 1.4 compaction), never claim an R-tree for a B-tree + if (!spec.rtree() && (source.isRTree(name) || target.isRTree(name))) + throw new MigrationException("Map " + name + " is marked as an R-tree in the file but the store layout uses a B-tree"); + if (spec.rtree() && !target.isRTree(name)) + throw new MigrationException("Migrated map " + name + " lost its R-tree marker"); + + MapDigest src = new MapDigest(); + MapDigest dst = new MapDigest(); + if (spec.rtree()) + { + source.readRTreeMap(name, spec.dimensions(), spec.valueType(), + (id, mm, v, dv) -> src.add(spatialKeyBytes(id, mm), v, false)); + target.readRTreeMap(name, spec.dimensions(), spec.valueType(), + (id, mm, v, dv) -> dst.add(spatialKeyBytes(id, mm), v, false)); + } + else + { + // the source may hold byte-identical duplicate keys that the copy stored once + var dedupe = new Migrator.DuplicateFilter(name); + source.readMap(name, spec.keyType(), spec.valueType(), (k, dk, v, dv) -> { + if (!dedupe.isDuplicate(k, v)) + src.add(k, v, true); + }); + target.readMap(name, spec.keyType(), spec.valueType(), (k, dk, v, dv) -> dst.add(k, v, true)); + if (dedupe.duplicates > 0) + log.accept(" note: " + name + " has " + dedupe.duplicates + " byte-identical duplicate key(s) in the source"); + } + + if (src.count != dst.count) + throw new MigrationException("Entry count differs for " + name + ": source " + src.count + + " vs migrated " + dst.count); + if (!Arrays.equals(src.result(), dst.result())) + throw new MigrationException("Entry bytes differ for " + name + " (" + src.count + " entries)"); + counts.put(name, dst.count); + log.accept(" verified " + name + ": " + dst.count + " entries, bytes identical"); + } + return counts; + } + + + /** + * Pass two: open through the store implementations and decode everything. + * @return per-map entry counts decoded with the real datatypes + */ + static Map decodeAll(Path file, StoreLayout type, Consumer log) throws Exception + { + Map counts = new TreeMap<>(); + MVStore store = H2Utils.openMVStore(file.toString(), new MVStore.Builder().readOnly(), 0); + try + { + // The datastore's Kryo datatypes may register additional class mappings while + // reading pages (getMemory serializes values). Record what the open adds so the + // caller can tell it apart from a copy defect. + Set classesBefore = new TreeSet<>(); + if (store.hasMap(MVObsSystemDatabase.KRYO_CLASS_MAP_NAME)) + classesBefore.addAll(store.openMap(MVObsSystemDatabase.KRYO_CLASS_MAP_NAME, new MVBTreeMap.Builder()).keySet()); + + type.openStores(store); + for (String name: new TreeSet<>(store.getMapNames())) + { + String id = store.getMetaMap().get(DataUtils.META_NAME + name); + MVMap map = store.getMap(DataUtils.parseHexInt(id)); + if (map == null) + throw new MigrationException("Store implementations did not open map " + name); + + long n; + if (map instanceof MVRTreeMap) + { + @SuppressWarnings("unchecked") + var rtree = (MVRTreeMap)(MVMap)map; + n = countLeaves(rtree.getRootPage()); + } + else + { + n = 0; + Cursor cursor = map.cursor(null); + while (cursor.hasNext()) + { + cursor.next(); + cursor.getValue(); + n++; + } + } + counts.put(name, n); + log.accept(" decoded " + name + ": " + n + " entries"); + } + + if (store.hasMap(MVObsSystemDatabase.KRYO_CLASS_MAP_NAME)) + { + Set added = new TreeSet<>(store.openMap(MVObsSystemDatabase.KRYO_CLASS_MAP_NAME, + new MVBTreeMap.Builder()).keySet()); + added.removeAll(classesBefore); + if (!added.isEmpty()) + log.accept(" note: opening the stores registered " + added.size() + " additional Kryo class mapping(s): " + + added + " (datastore behavior on first read, not a migration defect)"); + } + } + finally + { + store.closeImmediately(); + } + return counts; + } + + + private static long countLeaves(Page page) + { + long n = 0; + if (page.isLeaf()) + { + for (int i = 0; i < page.getKeyCount(); i++) + { + page.getKey(i); + page.getValue(i); + n++; + } + } + else + { + // R-tree node pages hold one child per key; the extra raw slot is an empty placeholder + for (int i = 0; i < page.getKeyCount(); i++) + n += countLeaves(page.getChildPage(i)); + } + return n; + } +} diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/spi/LegacyStore.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/spi/LegacyStore.java new file mode 100644 index 000000000..ca20100ee --- /dev/null +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/migrate/spi/LegacyStore.java @@ -0,0 +1,157 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate.spi; + +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.Set; + + +/** + * Contract between the H2 2.5 side of the migration and the H2 1.4 adapters that + * run in an isolated classloader. Everything in here uses JDK types only, so the + * two sides can share these interfaces while sharing no H2 class. The adapters + * implement {@link Factory}, which the tool loads by name ({@link Factory#IMPL_CLASS}). + */ +public interface LegacyStore extends AutoCloseable +{ + String getFileName(); + + /** @return H2 library version used to open the file (e.g. "1.4.200") */ + String getH2Version(); + + Set getMapNames(); + + /** @return the parsed "map.<id>" metadata for the named map */ + Map getMapMeta(String mapName); + + /** @return true when the map metadata carries the R-tree marker (type:rtree) */ + boolean isRTree(String mapName); + + /** + * Iterates a B-tree map in key order. + * @return number of entries delivered to the sink + */ + long readMap(String mapName, EntryDecoder keyDecoder, EntryDecoder valueDecoder, RawEntrySink sink); + + /** + * Iterates every leaf entry of an R-tree map by walking its pages. The map is + * opened as an R-tree whether or not its metadata still says so: H2 1.4's + * compaction rewrites maps generically and drops the type:rtree marker, while + * the datastore keeps opening such maps as R-trees. + * @return number of entries delivered to the sink + */ + long readRTreeMap(String mapName, int dimensions, EntryDecoder valueDecoder, SpatialEntrySink sink); + + @Override + void close(); + + + /** + * Entry point implemented inside the legacy classloader. The tool locates the + * implementation by name ({@link #IMPL_CLASS}) and casts it to this interface. + */ + interface Factory + { + String IMPL_CLASS = "org.sensorhub.impl.datastore.h2.migrate.legacy.LegacyStoreFactoryImpl"; + + LegacyStore openReadOnly(String fileName); + + Writer openForWrite(String fileName); + + /** + * Compacts a file with H2 1.4's own MVStoreTool, exactly as the pre-upgrade + * datastore's compactOnClose did. Test fixture support: 1.4 compaction drops + * the R-tree type marker from map metadata. + */ + void compact(String fileName); + } + + + /** + * Writes an H2 1.4 MVStore file from raw entries. This exists to build + * genuine format-1 test fixtures from the current datatypes; it is not part + * of the forward migration path. + */ + interface Writer extends AutoCloseable + { + interface MapWriter + { + void put(byte[] key, Object decodedKey, byte[] value); + } + + interface RTreeMapWriter + { + void put(long id, float[] minMax, byte[] value); + } + + MapWriter openMap(String mapName, EntryComparator keyComparator); + + RTreeMapWriter openRTreeMap(String mapName, int dimensions); + + void commit(); + + @Override + void close(); + } + + + /** + * Decodes one key or value from its position in a buffer, advancing the + * buffer position past exactly the bytes that belong to that entry. + * Implemented on the H2 2.5 side by delegating to the real OSH datatype. + * Only JDK types cross the classloader boundary. + */ + interface EntryDecoder + { + Object decode(ByteBuffer buff); + } + + + /** + * Orders two raw keys. Implemented on the H2 2.5 side with the real + * datatype's compare method. Used only when writing legacy files. + */ + interface EntryComparator + { + int compare(byte[] keyA, Object decodedA, byte[] keyB, Object decodedB); + } + + + /** + * Receives B-tree entries as the exact bytes stored in the source file, + * together with the decoded objects produced by the {@link EntryDecoder}s. + */ + interface RawEntrySink + { + void entry(byte[] key, Object decodedKey, byte[] value, Object decodedValue); + } + + + /** + * Receives R-tree leaf entries. The spatial key is transferred structurally + * (id plus min/max per dimension) and the value as exact bytes. + */ + interface SpatialEntrySink + { + /** + * @param id spatial key id + * @param minMax bounds as min0, max0, min1, max1, ... + * @param value exact value bytes + * @param decodedValue value decoded by the supplied {@link EntryDecoder} + */ + void entry(long id, float[] minMax, byte[] value, Object decodedValue); + } +} diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestLegacyLoader.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestLegacyLoader.java new file mode 100644 index 000000000..052fdfe6e --- /dev/null +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestLegacyLoader.java @@ -0,0 +1,134 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import static org.junit.Assert.*; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.TreeSet; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.type.DataType; +import org.h2.mvstore.type.ObjectDataType; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sensorhub.impl.datastore.h2.migrate.spi.LegacyStore; + + +/** + * Spikes for the central mechanism: an H2 1.4 store running in the isolated + * classloader while H2 2.5 is on the parent classpath, with entry boundaries + * found by parent-side datatypes. + */ +public class TestLegacyLoader +{ + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + + @Test + public void testClassIsolation() throws Exception + { + try (LegacyLoader loader = TestSupport.legacyLoader()) + { + Class legacyStore = Class.forName("org.h2.mvstore.MVStore", true, loader); + assertNotSame("legacy loader must define its own org.h2 classes", MVStore.class, legacyStore); + assertSame(loader, legacyStore.getClassLoader()); + + Class spi = Class.forName(LegacyStore.Factory.class.getName(), true, loader); + assertSame("spi interfaces must be shared with the parent", LegacyStore.Factory.class, spi); + + Class impl = Class.forName(LegacyStore.Factory.IMPL_CLASS, true, loader); + assertSame(loader, impl.getClassLoader()); + assertTrue(LegacyStore.Factory.class.isAssignableFrom(impl)); + } + } + + + @Test + public void testWriteAndReadFormat1File() throws Exception + { + Path file = tmp.getRoot().toPath().resolve("legacy.dat"); + var od = new ObjectDataType(); + @SuppressWarnings("unchecked") + DataType type = (DataType)(DataType)od; + + List ids = new ArrayList<>(); + for (int i = 0; i < 2000; i++) + ids.add(i); + Collections.shuffle(ids, new Random(42)); + + try (LegacyLoader loader = TestSupport.legacyLoader()) + { + var writer = loader.factory().openForWrite(file.toString()); + try + { + var m = writer.openMap("plain", (ka, da, kb, db) -> od.compare(da, db)); + for (int i: ids) + { + String key = String.format("key-%05d", i); + m.put(TestSupport.encode(type, key), key, TestSupport.encode(type, i)); + } + var r = writer.openRTreeMap("spatial", 3); + for (int i = 0; i < 100; i++) + r.put(i, new float[] {i, i + 1, -i, -i + 1, 0, 0}, TestSupport.encode(type, i * 10)); + writer.commit(); + } + finally + { + writer.close(); + } + } + + assertEquals("H2 1.4 must write file format 1", 1, FileOps.readH2Format(file)); + + try (LegacyLoader loader = TestSupport.legacyLoader(); + SourceReader.Legacy reader = new SourceReader.Legacy(loader.factory(), file)) + { + assertTrue(reader.describe(), reader.describe().contains("1.4.200")); + assertEquals(new TreeSet<>(List.of("plain", "spatial")), new TreeSet<>(reader.getMapNames())); + assertFalse(reader.isRTree("plain")); + assertTrue(reader.isRTree("spatial")); + + List keys = new ArrayList<>(); + long n = reader.readMap("plain", od, od, (k, dk, v, dv) -> { + String key = (String)dk; + keys.add(key); + assertArrayEquals("key bytes must be exactly what was written", TestSupport.encode(type, key), k); + int expected = Integer.parseInt(key.substring(4)); + assertEquals(expected, dv); + assertArrayEquals(TestSupport.encode(type, expected), v); + }); + assertEquals(2000, n); + List sorted = new ArrayList<>(keys); + Collections.sort(sorted); + assertEquals("cursor must yield keys in order", sorted, keys); + + TreeSet seen = new TreeSet<>(); + long rn = reader.readRTreeMap("spatial", 3, od, (id, mm, v, dv) -> { + seen.add(id); + assertEquals(6, mm.length); + assertEquals((float)id, mm[0], 0f); + assertEquals((float)(id + 1), mm[1], 0f); + assertEquals((int)(id * 10), dv); + }); + assertEquals(100, rn); + assertEquals(100, seen.size()); + } + } +} diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateFailures.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateFailures.java new file mode 100644 index 000000000..f122fc7c9 --- /dev/null +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateFailures.java @@ -0,0 +1,274 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import static org.junit.Assert.*; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.stream.Stream; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.type.ObjectDataType; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sensorhub.impl.datastore.h2.H2Utils; + + +/** + * Every failure must leave the original file untouched. + */ +public class TestMigrateFailures +{ + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + + private Path legacyFixture(String name) throws Exception + { + Path original = tmp.getRoot().toPath().resolve(name + "-25.dat"); + Path legacy = tmp.getRoot().toPath().resolve(name + ".dat"); + TestSupport.populateObsDatabase(original, 3, 50, 2); + TestSupport.exportToLegacy(original, legacy, StoreLayout.OBS_SYSTEM); + return legacy; + } + + + private long backupCount(Path file) throws Exception + { + try (Stream s = Files.list(file.getParent())) + { + return s.filter(p -> p.getFileName().toString().contains("-backup-")).count(); + } + } + + + @Test + public void testLockedFileIsRefusedBeforeBackup() throws Exception + { + Path legacy = legacyFixture("locked"); + byte[] before = Files.readAllBytes(legacy); + try (FileChannel ch = FileChannel.open(legacy, StandardOpenOption.READ, StandardOpenOption.WRITE)) + { + ch.lock(); + try + { + Migrator.run(TestSupport.options(legacy)); + fail(); + } + catch (MigrationException e) + { + assertTrue(e.getMessage(), e.getMessage().contains("locked")); + } + } + assertArrayEquals(before, Files.readAllBytes(legacy)); + assertEquals(0, backupCount(legacy)); + } + + + @Test + public void testNonH2FileIsRefused() throws Exception + { + Path bogus = tmp.getRoot().toPath().resolve("bogus.dat"); + Files.writeString(bogus, "this is not a database", StandardCharsets.UTF_8); + try + { + Migrator.run(TestSupport.options(bogus)); + fail(); + } + catch (MigrationException e) + { + assertTrue(e.getMessage(), e.getMessage().contains("Not an H2 MVStore file")); + } + assertEquals(0, backupCount(bogus)); + } + + + @Test + public void testAlreadyMarkedFileIsRefused() throws Exception + { + Path marked = tmp.getRoot().toPath().resolve("marked.dat"); + TestSupport.populateObsDatabase(marked, 2, 10, 1); + byte[] before = Files.readAllBytes(marked); + try + { + Migrator.run(TestSupport.options(marked)); + fail(); + } + catch (MigrationException e) + { + assertTrue(e.getMessage(), e.getMessage().contains("already carries version marker")); + } + assertArrayEquals(before, Files.readAllBytes(marked)); + assertEquals(0, backupCount(marked)); + } + + + @Test + public void testDryRunLeavesOriginalAndKeepsInterim() throws Exception + { + Path legacy = legacyFixture("dryrun"); + byte[] before = Files.readAllBytes(legacy); + var opts = TestSupport.options(legacy); + opts.dryRun = true; + var report = Migrator.run(opts); + + assertFalse(report.instated); + assertArrayEquals(before, Files.readAllBytes(legacy)); + assertEquals(1, FileOps.readH2Format(legacy)); + assertTrue(Files.exists(report.interim)); + assertEquals(3, FileOps.readH2Format(report.interim)); + assertEquals(FileOps.sha256(legacy), FileOps.sha256(report.backup)); + + // the interim file is a complete, marked database + try (MVStore s = H2Utils.openMVStore(report.interim.toString(), new MVStore.Builder().readOnly(), 0)) + { + assertTrue(s.hasMap("obs_store:obs_records")); + } + } + + + @Test + public void testUnknownMapAbortsAndCleansUp() throws Exception + { + Path legacy = legacyFixture("unknown"); + + // add a map the layout does not know, using the legacy writer on the same file + try (LegacyLoader loader = TestSupport.legacyLoader()) + { + var w = loader.factory().openForWrite(legacy.toString()); + try + { + var od = new ObjectDataType(); + @SuppressWarnings("unchecked") + var type = (org.h2.mvstore.type.DataType)(org.h2.mvstore.type.DataType)od; + var m = w.openMap("addon_store:custom", (ka, da, kb, db) -> od.compare(da, db)); + m.put(TestSupport.encode(type, "k"), "k", TestSupport.encode(type, 1)); + w.commit(); + } + finally + { + w.close(); + } + } + byte[] before = Files.readAllBytes(legacy); + + try + { + Migrator.run(TestSupport.options(legacy)); + fail(); + } + catch (MigrationException e) + { + assertTrue(e.getMessage(), e.getMessage().contains("addon_store:custom")); + } + assertArrayEquals(before, Files.readAllBytes(legacy)); + assertFalse(Files.exists(legacy.resolveSibling(legacy.getFileName() + ".migrating"))); + assertFalse(Files.exists(legacy.resolveSibling(legacy.getFileName() + ".migrating-types"))); + assertEquals("layout is checked before the backup is written", 0, backupCount(legacy)); + } + + + @Test + public void testBackupDirAndKeepInterim() throws Exception + { + Path legacy = legacyFixture("options"); + Path backupDir = tmp.getRoot().toPath().resolve("backups/nested"); + byte[] before = Files.readAllBytes(legacy); + + // --backup-dir: the verified backup lands in the requested directory + var opts = TestSupport.options(legacy); + opts.dryRun = true; + opts.backupDir = backupDir; + var report = Migrator.run(opts); + assertEquals(backupDir, report.backup.getParent()); + assertArrayEquals(before, Files.readAllBytes(report.backup)); + assertEquals("no backup next to the database", 0, backupCount(legacy)); + Files.delete(report.interim); + + // --keep-interim: a failure after the interim was written leaves it for inspection + Path interim = legacy.resolveSibling(legacy.getFileName() + ".migrating"); + Files.writeString(interim, "stale", StandardCharsets.UTF_8); + try + { + Migrator.run(TestSupport.options(legacy)); + fail("a pre-existing interim file must be refused"); + } + catch (MigrationException e) + { + assertTrue(e.getMessage(), e.getMessage().contains("Interim file already exists")); + } + assertTrue("existing interim file must not be deleted", Files.exists(interim)); + Files.delete(interim); + + opts = TestSupport.options(legacy); + opts.keepInterim = true; + opts.type = StoreLayout.FEATURE; // wrong layout: fails at the unknown-map check, before any interim + try + { + Migrator.run(opts); + fail(); + } + catch (MigrationException e) + { + assertTrue(e.getMessage(), e.getMessage().contains("unknown to the FEATURE layout")); + } + assertArrayEquals(before, Files.readAllBytes(legacy)); + } + + + @Test + public void testExistingRetiredOriginalIsRefusedBeforeBackup() throws Exception + { + Path legacy = legacyFixture("retired"); + Path retired = legacy.resolveSibling(legacy.getFileName() + Migrator.RETIRED_SUFFIX); + Files.writeString(retired, "an earlier original", StandardCharsets.UTF_8); + byte[] before = Files.readAllBytes(legacy); + try + { + Migrator.run(TestSupport.options(legacy)); + fail(); + } + catch (MigrationException e) + { + assertTrue(e.getMessage(), e.getMessage().contains("retired original already exists")); + } + assertArrayEquals(before, Files.readAllBytes(legacy)); + assertEquals("an earlier original", Files.readString(retired, StandardCharsets.UTF_8)); + assertEquals(0, backupCount(legacy)); + + // a dry run does not retire anything, so it still works + Files.delete(retired); + var opts = TestSupport.options(legacy); + opts.dryRun = true; + var report = Migrator.run(opts); + assertNull(report.retiredOriginal); + assertArrayEquals(before, Files.readAllBytes(legacy)); + } + + + @Test + public void testCliUsageErrors() + { + assertEquals(2, H2Migrate.run(new String[0])); + assertEquals(2, H2Migrate.run(new String[] {"--type"})); + assertEquals(2, H2Migrate.run(new String[] {"--type", "nope", "x.dat"})); + assertEquals(2, H2Migrate.run(new String[] {"--backup", "--backup-dir"})); + assertEquals(0, H2Migrate.run(new String[] {"--help"})); + assertEquals(1, H2Migrate.run(new String[] {"--quiet", tmp.getRoot().toPath().resolve("missing.dat").toString()})); + } +} diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateNative.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateNative.java new file mode 100644 index 000000000..3c9549951 --- /dev/null +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateNative.java @@ -0,0 +1,120 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import static org.junit.Assert.*; +import java.nio.file.Path; +import org.h2.mvstore.MVStore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + + +/** + * An H2 2.5 file written before the version marker existed goes through the + * same pipeline in-process and comes out marked. + */ +public class TestMigrateNative +{ + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + + @Test + public void testUnmarkedFileGetsMigratedAndMarked() throws Exception + { + Path marked = tmp.getRoot().toPath().resolve("marked.dat"); + Path unmarked = tmp.getRoot().toPath().resolve("unmarked.dat"); + var exp = TestSupport.populateObsDatabase(marked, 5, 200, 5); + + // strip the marker: copy every map except @database into a plain 2.5 store + try (SourceReader.Native src = new SourceReader.Native(marked, 3); + StoreLayout.Types types = StoreLayout.Types.harvest(src, StoreLayout.OBS_SYSTEM, + tmp.getRoot().toPath().resolve("types.dat"))) + { + MVStore dst = new MVStore.Builder().fileName(unmarked.toString()).open(); + try + { + for (String name: Migrator.copyOrder(src.getMapNames())) + { + StoreLayout.MapSpec spec = types.get(name); + if (spec.rtree()) + { + var m = dst.openMap(name, new org.h2.mvstore.rtree.MVRTreeMap.Builder() + .dimensions(spec.dimensions()).valueType(new RawDataType(spec.valueType()))); + var f = new org.h2.mvstore.rtree.SpatialDataType(spec.dimensions()) { + public org.h2.mvstore.rtree.Spatial mk(long id, float[] mm) { return create(id, mm); } + }; + src.readRTreeMap(name, spec.dimensions(), spec.valueType(), + (id, mm, v, dv) -> m.put(f.mk(id, mm), new RawDataType.Entry(v, dv))); + } + else + { + var m = dst.openMap(name, new org.h2.mvstore.MVBTreeMap.Builder() + .keyType(new RawDataType(spec.keyType())).valueType(new RawDataType(spec.valueType()))); + src.readMap(name, spec.keyType(), spec.valueType(), + (k, dk, v, dv) -> m.put(new RawDataType.Entry(k, dk), new RawDataType.Entry(v, dv))); + } + } + dst.commit(); + } + finally + { + dst.close(); + } + } + assertEquals(3, FileOps.readH2Format(unmarked)); + try (SourceReader.Native r = new SourceReader.Native(unmarked, 3)) + { + assertNull(r.getVersionMarker()); + } + + // the datastore refuses it (the module records the error instead of throwing) + var rejected = TestSupport.openObsDatabase(unmarked); + try + { + assertFalse(rejected.isStarted()); + Throwable error = rejected.getCurrentError(); + assertNotNull(error); + while (error.getCause() != null) + error = error.getCause(); + assertTrue(error.getMessage(), error.getMessage().contains("migration")); + } + finally + { + rejected.stop(); + } + + var report = Migrator.run(TestSupport.options(unmarked)); + assertTrue(report.instated); + assertEquals(3, report.sourceFormat); + assertEquals((Long)(long)exp.numObs, report.copied.get("obs_store:obs_records")); + + TestMigrateRoundTrip.assertIdenticalEntries(marked, unmarked, StoreLayout.OBS_SYSTEM); + + var db = TestSupport.openObsDatabase(unmarked); + try + { + assertEquals(exp.numObs, db.getObservationStore().getNumRecords()); + assertEquals(exp.numFois, db.getFoiStore().getNumRecords()); + assertEquals(exp.spatialMatches, TestSupport.spatialMatches(db)); + assertEquals(exp.keywordMatches, TestSupport.keywordMatches(db)); + } + finally + { + db.stop(); + } + } +} diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateRoundTrip.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateRoundTrip.java new file mode 100644 index 000000000..eeb53e7fa --- /dev/null +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestMigrateRoundTrip.java @@ -0,0 +1,226 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import static org.junit.Assert.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sensorhub.api.datastore.obs.ObsFilter; + + +/** + * Populates a real database with H2 2.5, exports it to a genuine H2 1.4 file, + * migrates that file and checks the result through the store APIs and by + * comparing bytes with the original 2.5 database. + */ +public class TestMigrateRoundTrip +{ + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + + @Test + public void testObsSystemDatabase() throws Exception + { + Path original = tmp.getRoot().toPath().resolve("obsdb.dat"); + Path legacy = tmp.getRoot().toPath().resolve("obsdb-v1.dat"); + var exp = TestSupport.populateObsDatabase(original, 25, 3000, 40); + + TestSupport.exportToLegacy(original, legacy, StoreLayout.OBS_SYSTEM); + assertEquals(1, FileOps.readH2Format(legacy)); + byte[] legacyBytes = Files.readAllBytes(legacy); + + var report = Migrator.run(TestSupport.options(legacy)); + assertTrue(report.instated); + assertEquals(StoreLayout.OBS_SYSTEM, report.type); + assertEquals(1, report.sourceFormat); + assertEquals(3, FileOps.readH2Format(legacy)); + assertEquals((Long)(long)exp.numObs, report.copied.get("obs_store:obs_records")); + assertEquals((Long)(long)exp.numFois, report.copied.get("foi_store:feature_records")); + assertEquals((Long)(long)exp.numFois, report.copied.get("foi_store:feature_geom")); + assertEquals((Long)(long)exp.numCommands, report.copied.get("cmd_store:cmd_records")); + assertEquals((Long)(long)exp.numSystemVersions, report.copied.get("sys_store:feature_records")); + assertTrue("legacy full-text index duplicates must be reported", + report.duplicatesCollapsed.getOrDefault("sys_store:feature_text", 0L) > 0); + + // backup is byte-identical to the legacy file, and the original itself survives + // untouched under the .old name + assertNotNull(report.backup); + assertArrayEquals(legacyBytes, Files.readAllBytes(report.backup)); + assertEquals(legacy.resolveSibling("obsdb-v1.dat.old"), report.retiredOriginal); + assertArrayEquals(legacyBytes, Files.readAllBytes(report.retiredOriginal)); + assertFalse(Files.exists(report.interim)); + + // independent byte comparison of the migrated file against the original 2.5 file, + // before the module opens it (opening persists Kryo class mappings added on read) + assertIdenticalEntries(original, legacy, StoreLayout.OBS_SYSTEM); + + // migrated file opens through the module and holds the same data + var db = TestSupport.openObsDatabase(legacy); + try + { + assertEquals(exp.numSystemVersions, db.getSystemDescStore().getNumRecords()); + var sys0 = db.getSystemDescStore().getCurrentVersion(TestSupport.SYS_UID_PREFIX + 0); + assertNotNull(sys0); + assertTrue(sys0.getDescription(), sys0.getDescription().contains("updated")); + assertEquals(exp.numFois, db.getFoiStore().getNumRecords()); + assertEquals(exp.numDataStreams, db.getObservationStore().getDataStreams().getNumRecords()); + assertEquals(exp.numObs, db.getObservationStore().getNumRecords()); + assertEquals(exp.numCommandStreams, db.getCommandStore().getCommandStreams().getNumRecords()); + assertEquals(exp.numCommands, db.getCommandStore().getNumRecords()); + assertEquals(exp.numStatus, db.getCommandStore().getStatusReports().getNumRecords()); + + for (var e: exp.obsValues.entrySet()) + { + var obs = db.getObservationStore().get(e.getKey()); + assertNotNull("observation missing: " + e.getKey(), obs); + assertEquals(exp.dataStreamId, obs.getDataStreamID()); + assertEquals(exp.firstFoiId, obs.getFoiID()); + for (int s = 0; s < 5; s++) + assertEquals(e.getValue()[s], obs.getResult().getDoubleValue(s), 0.0); + } + + try (Stream s = db.getObservationStore().select(new ObsFilter.Builder() + .withDataStreams(exp.dataStreamId).build())) + { + assertEquals(exp.numObs, s.count()); + } + + // spatial and full-text indexes answer exactly as the original did + assertEquals(exp.spatialMatches, TestSupport.spatialMatches(db)); + assertEquals(exp.keywordMatches, TestSupport.keywordMatches(db)); + } + finally + { + db.stop(); + } + } + + + @Test + public void testFeatureDatabase() throws Exception + { + Path original = tmp.getRoot().toPath().resolve("featdb.dat"); + Path legacy = tmp.getRoot().toPath().resolve("featdb-v1.dat"); + var exp = TestSupport.populateFeatureDatabase(original, 300); + int numFeatures = exp.numFois; + + TestSupport.exportToLegacy(original, legacy, StoreLayout.FEATURE); + assertEquals(1, FileOps.readH2Format(legacy)); + byte[] legacyBytes = Files.readAllBytes(legacy); + + var opts = TestSupport.options(legacy); + opts.type = null; // must be detected + opts.backup = false; // default: no copy, the retired original is the safeguard + var report = Migrator.run(opts); + assertEquals(StoreLayout.FEATURE, report.type); + assertTrue(report.instated); + assertNull(report.backup); + assertArrayEquals(legacyBytes, Files.readAllBytes(report.retiredOriginal)); + try (Stream files = Files.list(legacy.getParent())) + { + assertEquals("no backup copy without --backup", 0, + files.filter(p -> p.getFileName().toString().contains("-backup-")).count()); + } + assertEquals((Long)(long)numFeatures, report.copied.get("feature_store:feature_records")); + assertEquals((Long)(long)numFeatures, report.copied.get("feature_store:feature_geom")); + + assertIdenticalEntries(original, legacy, StoreLayout.FEATURE); + + var db = TestSupport.openFeatureDatabase(legacy); + try + { + assertEquals(numFeatures, db.getFeatureStore().getNumRecords()); + assertEquals(exp.spatialMatches, TestSupport.spatialMatches(db)); + assertEquals(exp.keywordMatches, TestSupport.keywordMatches(db)); + assertNotNull(db.getFeatureStore().getCurrentVersion(TestSupport.FOI_UID_PREFIX + 42)); + } + finally + { + db.stop(); + } + } + + + static void assertIdenticalEntries(Path a, Path b, StoreLayout type) throws Exception + { + Path scratch = a.resolveSibling(a.getFileName() + ".cmp-types"); + try (SourceReader.Native ra = new SourceReader.Native(a, 3); + SourceReader.Native rb = new SourceReader.Native(b, 3); + StoreLayout.Types types = StoreLayout.Types.harvest(ra, type, scratch)) + { + var counts = Verifier.compareBytes(ra, rb, types.getSpecs(), java.util.Set.of(), s -> {}); + assertFalse(counts.isEmpty()); + } + } + + + /** + * H2 1.4's MVStoreTool.compact (the old build's compactOnClose) rewrites every map as a + * plain map and drops the type:rtree marker from the metadata. The old datastore never + * looked at it; the migration must not either. + */ + @Test + public void testLegacyFileCompactedByH2v14LosesRTreeMarker() throws Exception + { + Path original = tmp.getRoot().toPath().resolve("compacted.dat"); + Path legacy = tmp.getRoot().toPath().resolve("compacted-v1.dat"); + var exp = TestSupport.populateObsDatabase(original, 40, 500, 5); + TestSupport.exportToLegacy(original, legacy, StoreLayout.OBS_SYSTEM); + + try (LegacyLoader loader = TestSupport.legacyLoader()) + { + var factory = loader.factory(); + try (var before = factory.openReadOnly(legacy.toString())) + { + assertTrue(before.isRTree("foi_store:feature_geom")); + } + factory.compact(legacy.toString()); + assertEquals(1, FileOps.readH2Format(legacy)); + try (var after = factory.openReadOnly(legacy.toString())) + { + assertFalse("1.4 compaction is expected to drop the marker", after.isRTree("foi_store:feature_geom")); + assertNull(after.getMapMeta("foi_store:feature_geom").get("type")); + } + } + + var report = Migrator.run(TestSupport.options(legacy)); + assertTrue(report.instated); + assertEquals((Long)(long)exp.numFois, report.copied.get("foi_store:feature_geom")); + assertEquals((Long)(long)exp.numObs, report.copied.get("obs_store:obs_records")); + + // the migrated file carries the marker again and the spatial index answers as before + try (SourceReader.Native migrated = new SourceReader.Native(legacy, 3)) + { + assertTrue(migrated.isRTree("foi_store:feature_geom")); + } + assertIdenticalEntries(original, legacy, StoreLayout.OBS_SYSTEM); + var db = TestSupport.openObsDatabase(legacy); + try + { + assertEquals(exp.numFois, db.getFoiStore().getNumRecords()); + assertEquals(exp.spatialMatches, TestSupport.spatialMatches(db)); + assertEquals(exp.keywordMatches, TestSupport.keywordMatches(db)); + } + finally + { + db.stop(); + } + } +} diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestSupport.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestSupport.java new file mode 100644 index 000000000..6df626b8b --- /dev/null +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestSupport.java @@ -0,0 +1,404 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Stream; +import javax.xml.namespace.QName; +import org.h2.mvstore.WriteBuffer; +import org.h2.mvstore.type.DataType; +import org.sensorhub.api.command.CommandData; +import org.sensorhub.api.command.CommandStatus; +import org.sensorhub.api.command.CommandStreamInfo; +import org.sensorhub.api.common.BigId; +import org.sensorhub.api.data.DataStreamInfo; +import org.sensorhub.api.data.ObsData; +import org.sensorhub.api.database.IObsSystemDatabase; +import org.sensorhub.api.datastore.feature.FeatureFilter; +import org.sensorhub.api.datastore.feature.FeatureFilterBase; +import org.sensorhub.api.datastore.feature.FoiFilter; +import org.sensorhub.api.datastore.feature.IFeatureStoreBase; +import org.sensorhub.api.feature.FeatureId; +import org.sensorhub.impl.datastore.h2.MVFeatureDatabase; +import org.sensorhub.impl.datastore.h2.MVFeatureDatabaseConfig; +import org.sensorhub.impl.datastore.h2.MVObsSystemDatabase; +import org.sensorhub.impl.datastore.h2.MVObsSystemDatabaseConfig; +import org.sensorhub.impl.system.wrapper.SystemWrapper; +import org.sensorhub.impl.datastore.h2.migrate.spi.LegacyStore; +import org.vast.data.DataBlockDouble; +import org.vast.data.TextEncodingImpl; +import org.vast.ogc.gml.GenericFeatureImpl; +import org.vast.sensorML.SMLHelper; +import org.vast.swe.SWEHelper; +import org.vast.util.Bbox; +import org.vast.util.TimeExtent; +import net.opengis.gml.v32.impl.GMLFactory; + + +/** + * Shared helpers: populate real databases through the store APIs, and export + * an H2 2.5 database to a genuine H2 1.4 (format 1) file through the legacy + * classloader so migration tests run against real legacy files. + */ +final class TestSupport +{ + static final int DATABASE_NUM = 1; + static final String SYS_UID_PREFIX = "urn:osh:test:sensor:"; + static final String FOI_UID_PREFIX = "urn:osh:test:foi:"; + static final Instant T0 = Instant.parse("2024-01-01T00:00:00Z"); + + + /** Everything the population step wrote, so tests can check it survived */ + static final class Expected + { + int numSystems, numSystemVersions, numFois, numDataStreams, numObs, numCommandStreams, numCommands, numStatus; + final Map obsValues = new LinkedHashMap<>(); + BigId dataStreamId; + BigId firstFoiId; + long spatialMatches; // features inside SPATIAL_QUERY, computed on the original database + long keywordMatches; // features matching KEYWORD_QUERY, computed on the original database + } + + static final Bbox SPATIAL_QUERY = new Bbox(-1, -1, 4.6, 2.3); + static final String KEYWORD_QUERY = "number"; + + + @SuppressWarnings({"rawtypes", "unchecked"}) + static long countFeatures(IFeatureStoreBase store, FeatureFilterBase filter) + { + try (Stream s = store.select(filter)) + { + return s.count(); + } + } + + + static long spatialMatches(IObsSystemDatabase db) + { + return countFeatures(db.getFoiStore(), new FoiFilter.Builder().withLocationWithin(SPATIAL_QUERY).build()); + } + + + static long keywordMatches(IObsSystemDatabase db) + { + return countFeatures(db.getFoiStore(), new FoiFilter.Builder().withKeywords(KEYWORD_QUERY).build()); + } + + + static long spatialMatches(MVFeatureDatabase db) + { + return countFeatures(db.getFeatureStore(), new FeatureFilter.Builder().withLocationWithin(SPATIAL_QUERY).build()); + } + + + static long keywordMatches(MVFeatureDatabase db) + { + return countFeatures(db.getFeatureStore(), new FeatureFilter.Builder().withKeywords(KEYWORD_QUERY).build()); + } + + + private TestSupport() + { + } + + + /** Legacy jars come from the module's embedded resources unless a directory is given */ + static LegacyLoader legacyLoader() throws MigrationException + { + return LegacyLoader.createDefault(null); + } + + + static MVObsSystemDatabase openObsDatabase(Path file) throws Exception + { + var db = new MVObsSystemDatabase(); + var config = new MVObsSystemDatabaseConfig(); + config.storagePath = file.toString(); + config.databaseNum = DATABASE_NUM; + db.setConfiguration(config); + db.init(); + db.start(); + return db; + } + + + static MVFeatureDatabase openFeatureDatabase(Path file) throws Exception + { + var db = new MVFeatureDatabase(); + var config = new MVFeatureDatabaseConfig(); + config.storagePath = file.toString(); + config.databaseNum = DATABASE_NUM; + db.setConfiguration(config); + db.init(); + db.start(); + return db; + } + + + /** + * Writes systems, geo-referenced FOIs, a datastream with observations, and a + * command stream with commands and status reports through the public APIs. + */ + static Expected populateObsDatabase(Path file, int numFois, int numObs, int numCommands) throws Exception + { + var exp = new Expected(); + var db = openObsDatabase(file); + try + { + var sml = new SMLHelper(); + BigId[] sysIds = new BigId[3]; + for (int i = 0; i < sysIds.length; i++) + { + var proc = sml.createPhysicalComponent() + .uniqueID(SYS_UID_PREFIX + i) + .name("System " + i) + .description("Test system number " + i) + .build(); + sysIds[i] = db.getSystemDescStore().add(new SystemWrapper(proc)).getInternalID(); + exp.numSystems++; + } + + exp.numSystemVersions = exp.numSystems; + + var gml = new GMLFactory(true); + QName fType = new QName("http://mydomain/features", "MyPointFeature"); + for (int i = 0; i < numFois; i++) + { + var f = new GenericFeatureImpl(fType); + var p = gml.newPoint(); + p.setPos(new double[] {i * 0.5, i * 0.25, 0.0}); + f.setGeometry(p); + f.setId("F" + i); + f.setUniqueIdentifier(FOI_UID_PREFIX + i); + f.setName("Feature " + i); + f.setDescription("Sampling location number " + i); + var key = db.getFoiStore().add(f); + if (i == 0) + exp.firstFoiId = key.getInternalID(); + exp.numFois++; + } + + var swe = new SWEHelper(); + var rec = swe.createRecord().name("out1"); + for (int i = 0; i < 5; i++) + rec.addField("comp" + i, swe.createQuantity().build()); + var dsInfo = new DataStreamInfo.Builder() + .withName("out1") + .withSystem(new FeatureId(sysIds[0], SYS_UID_PREFIX + 0)) + .withRecordDescription(rec.build()) + .withRecordEncoding(new TextEncodingImpl()) + .build(); + exp.dataStreamId = db.getObservationStore().getDataStreams().add(dsInfo).getInternalID(); + exp.numDataStreams++; + + for (int i = 0; i < numObs; i++) + { + var data = new DataBlockDouble(5); + double[] vals = new double[5]; + for (int s = 0; s < 5; s++) + { + vals[s] = i + s * 0.1; + data.setDoubleValue(s, vals[s]); + } + var obs = new ObsData.Builder() + .withDataStream(exp.dataStreamId) + .withFoi(exp.firstFoiId) + .withPhenomenonTime(T0.plusSeconds(i)) + .withResult(data) + .build(); + var key = db.getObservationStore().add(obs); + exp.obsValues.put(key, vals); + exp.numObs++; + } + + var cmdRec = swe.createRecord().name("cmd1"); + for (int i = 0; i < 3; i++) + cmdRec.addField("param" + i, swe.createQuantity().build()); + var csInfo = new CommandStreamInfo.Builder() + .withName("cmd1") + .withSystem(new FeatureId(sysIds[1], SYS_UID_PREFIX + 1)) + .withRecordDescription(cmdRec.build()) + .withRecordEncoding(new TextEncodingImpl()) + .build(); + var csId = db.getCommandStore().getCommandStreams().add(csInfo).getInternalID(); + exp.numCommandStreams++; + + for (int i = 0; i < numCommands; i++) + { + var params = new DataBlockDouble(3); + for (int s = 0; s < 3; s++) + params.setDoubleValue(s, i + s); + var ts = T0.plusSeconds(i); + var cmd = new CommandData.Builder() + .withCommandStream(csId) + .withIssueTime(ts) + .withParams(params) + .withSender("test" + i) + .build(); + var cmdKey = db.getCommandStore().add(cmd); + exp.numCommands++; + db.getCommandStore().getStatusReports().add(CommandStatus.completed(cmdKey, TimeExtent.instant(ts))); + exp.numStatus++; + } + + } + finally + { + db.stop(); + } + + // A second description version of system 0, written in a new session like a node + // restart would. The full-text index compares keys by valid time in memory but does + // not persist it, so the reloaded key compares unequal and a byte-identical duplicate + // entry lands on disk. Real legacy databases show exactly this. + db = openObsDatabase(file); + try + { + var sml = new SMLHelper(); + var proc0v2 = sml.createPhysicalComponent() + .uniqueID(SYS_UID_PREFIX + 0) + .name("System 0") + .description("Test system number 0, updated") + .validFrom(OffsetDateTime.of(2025, 6, 1, 0, 0, 0, 0, ZoneOffset.UTC)) + .build(); + db.getSystemDescStore().add(new SystemWrapper(proc0v2)); + exp.numSystemVersions++; + + exp.spatialMatches = spatialMatches(db); + exp.keywordMatches = keywordMatches(db); + assertTrue("spatial index must select something on the original", exp.spatialMatches > 0); + assertTrue("full-text index must select something on the original", exp.keywordMatches > 0); + } + finally + { + db.stop(); + } + return exp; + } + + + static Expected populateFeatureDatabase(Path file, int numFeatures) throws Exception + { + var exp = new Expected(); + var db = openFeatureDatabase(file); + try + { + var gml = new GMLFactory(true); + QName fType = new QName("http://mydomain/features", "MyPointFeature"); + for (int i = 0; i < numFeatures; i++) + { + var f = new GenericFeatureImpl(fType); + var p = gml.newPoint(); + p.setPos(new double[] {-i, i * 2.0, 0.0}); + f.setGeometry(p); + f.setId("F" + i); + f.setUniqueIdentifier(FOI_UID_PREFIX + i); + f.setName("Feature " + i); + f.setDescription("Generic feature number " + i); + db.getFeatureStore().add(f); + exp.numFois++; + } + exp.spatialMatches = spatialMatches(db); + exp.keywordMatches = keywordMatches(db); + assertTrue("spatial index must select something on the original", exp.spatialMatches > 0); + assertTrue("full-text index must select something on the original", exp.keywordMatches > 0); + } + finally + { + db.stop(); + } + return exp; + } + + + /** + * Reverse direction, test support only: writes every map of an H2 2.5 + * database into a genuine H2 1.4 file with identical entry bytes. + */ + static void exportToLegacy(Path source, Path dest, StoreLayout type) throws Exception + { + Files.deleteIfExists(dest); + LegacyLoader loader = legacyLoader(); + try (SourceReader.Native src = new SourceReader.Native(source, 3); + StoreLayout.Types types = StoreLayout.Types.harvest(src, type, dest.resolveSibling(dest.getFileName() + ".types"))) + { + LegacyStore.Writer writer = loader.factory().openForWrite(dest.toString()); + try + { + for (String name: Migrator.copyOrder(src.getMapNames())) + { + StoreLayout.MapSpec spec = types.get(name); + assertNotNull("no spec for " + name, spec); + if (spec.rtree()) + { + var mw = writer.openRTreeMap(name, spec.dimensions()); + src.readRTreeMap(name, spec.dimensions(), spec.valueType(), (id, mm, v, dv) -> mw.put(id, mm, v)); + } + else + { + // Preserve byte-identical duplicate keys exactly as the datastore leaves them on disk: + // a distinct entry that compares equal is ordered after the existing one instead of + // replacing it, so the legacy fixture carries the same duplicates a real node produced. + var mw = writer.openMap(name, (ka, da, kb, dB) -> { + int c = spec.keyType().compare(da, dB); + return c != 0 ? c : (ka == kb ? 0 : 1); + }); + src.readMap(name, spec.keyType(), spec.valueType(), (k, dk, v, dv) -> mw.put(k, dk, v)); + } + writer.commit(); + } + } + finally + { + writer.close(); + } + } + finally + { + loader.close(); + } + } + + + static byte[] encode(DataType type, Object value) + { + WriteBuffer b = new WriteBuffer(); + type.write(b, value); + ByteBuffer bb = b.getBuffer(); + bb.flip(); + byte[] out = new byte[bb.remaining()]; + bb.get(out); + return out; + } + + + static Migrator.Options options(Path dbFile) + { + var o = new Migrator.Options(); + o.dbFile = dbFile; + o.backup = true; + o.log = s -> System.out.println("[h2migrate] " + s); + return o; + } +} From b954096c293aca451914a32fc9451cd35a101c96 Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Wed, 30 Sep 2026 10:08:02 -0500 Subject: [PATCH 11/12] Add config and hooks for automatic database migration. This feature may be moved to a different location in the future to keep this module more lightweight. --- .../impl/datastore/h2/MVDatabaseConfig.java | 11 ++ .../impl/datastore/h2/MVFeatureDatabase.java | 12 ++ .../datastore/h2/MVObsSystemDatabase.java | 11 ++ .../datastore/h2/migrate/TestAutoMigrate.java | 175 ++++++++++++++++++ 4 files changed, 209 insertions(+) create mode 100644 sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestAutoMigrate.java diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVDatabaseConfig.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVDatabaseConfig.java index 1577526d3..0d72ce5b5 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVDatabaseConfig.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVDatabaseConfig.java @@ -76,6 +76,17 @@ public enum IdProviderType public boolean readOnly = false; + @DisplayInfo(desc = "Set to migrate a database file written by an older H2 version at startup. " + + "The original file is kept unmodified next to the migrated one with a .old suffix. " + + "When unset, such files are rejected with an error") + public boolean autoMigrate = false; + + + @DisplayInfo(desc = "Set to also keep a verified backup copy of the original file when autoMigrate " + + "migrates it (the original is kept as .old either way)") + public boolean backupOnMigrate = false; + + @DisplayInfo(desc = "Set to compact the database file when the database module is stopped or restarted") public boolean compactOnClose = false; diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java index 7bee71f4c..c372126dc 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVFeatureDatabase.java @@ -14,6 +14,8 @@ package org.sensorhub.impl.datastore.h2; +import java.nio.file.Paths; + import java.util.concurrent.Callable; import org.h2.mvstore.MVStore; import org.sensorhub.api.common.SensorHubException; @@ -21,6 +23,7 @@ import org.sensorhub.api.datastore.DataStoreException; import org.sensorhub.impl.module.AbstractModule; import org.sensorhub.utils.FileUtils; +import org.sensorhub.impl.datastore.h2.migrate.Migrator; /** @@ -63,6 +66,9 @@ protected void doStart() throws SensorHubException if (config.useCompression) builder.compress(); + if (config.autoMigrate && !config.readOnly) + Migrator.migrateIfNeeded(Paths.get(config.storagePath), config.backupOnMigrate, msg -> getLogger().info(msg)); + mvStore = H2Utils.openMVStore(config.storagePath, builder, config.autoCommitPeriod*1000); mvStore.setVersionsToKeep(0); @@ -72,6 +78,12 @@ protected void doStart() throws SensorHubException .withName(FEATURE_STORE_NAME) .build()); } + catch (DatabaseMigrationRequiredException e) + { + // one plain line for the operator, in addition to the module error state + getLogger().error("Cannot open database file: {}", e.getMessage()); + throw e; + } catch (Exception e) { throw new DataStoreException("Error while starting MVStore", e); diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java index 7d4cef854..f322dc9dd 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/MVObsSystemDatabase.java @@ -15,6 +15,7 @@ package org.sensorhub.impl.datastore.h2; import java.io.File; +import java.nio.file.Paths; import java.io.StringWriter; import java.util.concurrent.Callable; import org.h2.mvstore.MVStore; @@ -33,6 +34,7 @@ import org.sensorhub.api.datastore.system.ISystemDescStore; import org.sensorhub.impl.module.AbstractModule; import org.sensorhub.utils.FileUtils; +import org.sensorhub.impl.datastore.h2.migrate.Migrator; /** @@ -102,6 +104,9 @@ protected void doStart() throws SensorHubException if (config.useCompression) builder.compress(); + if (config.autoMigrate && !config.readOnly) + Migrator.migrateIfNeeded(Paths.get(config.storagePath), config.backupOnMigrate, msg -> getLogger().info(msg)); + mvStore = H2Utils.openMVStore(config.storagePath, builder, config.autoCommitPeriod*1000); mvStore.setVersionsToKeep(0); @@ -150,6 +155,12 @@ protected void doStart() throws SensorHubException obsStore.getDataStreams().linkTo(sysStore); cmdStore.getCommandStreams().linkTo(sysStore); } + catch (DatabaseMigrationRequiredException e) + { + // one plain line for the operator, in addition to the module error state + getLogger().error("Cannot open database file: {}", e.getMessage()); + throw e; + } catch (Exception e) { throw new DataStoreException("Error while starting MVStore", e); diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestAutoMigrate.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestAutoMigrate.java new file mode 100644 index 000000000..6fde63100 --- /dev/null +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/migrate/TestAutoMigrate.java @@ -0,0 +1,175 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2.migrate; + +import static org.junit.Assert.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sensorhub.impl.datastore.h2.MVFeatureDatabase; +import org.sensorhub.impl.datastore.h2.MVFeatureDatabaseConfig; +import org.sensorhub.impl.datastore.h2.MVObsSystemDatabase; +import org.sensorhub.impl.datastore.h2.MVObsSystemDatabaseConfig; + + +/** + * The database module migrates a legacy file at startup only when autoMigrate is set; + * otherwise it refuses it with a message that tells the operator what to do. + */ +public class TestAutoMigrate +{ + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + + private MVObsSystemDatabase database(Path file, boolean autoMigrate) throws Exception + { + var db = new MVObsSystemDatabase(); + var config = new MVObsSystemDatabaseConfig(); + config.storagePath = file.toString(); + config.databaseNum = TestSupport.DATABASE_NUM; + config.autoMigrate = autoMigrate; + db.setConfiguration(config); + db.init(); + db.start(); + return db; + } + + + private MVFeatureDatabase featureDatabase(Path file, boolean autoMigrate) throws Exception + { + var db = new MVFeatureDatabase(); + var config = new MVFeatureDatabaseConfig(); + config.storagePath = file.toString(); + config.databaseNum = TestSupport.DATABASE_NUM; + config.autoMigrate = autoMigrate; + db.setConfiguration(config); + db.init(); + db.start(); + return db; + } + + + /** All messages in the cause chain; the migration hint sits on the wrapping exception */ + private static String chainMessages(Throwable error) + { + var sb = new StringBuilder(); + for (Throwable t = error; t != null; t = t.getCause()) + sb.append(t.getMessage()).append(" | "); + return sb.toString(); + } + + + @Test + public void testLegacyFileRefusedUnlessAutoMigrate() throws Exception + { + Path original = tmp.getRoot().toPath().resolve("obs-25.dat"); + Path legacy = tmp.getRoot().toPath().resolve("obs.dat"); + var exp = TestSupport.populateObsDatabase(original, 4, 100, 3); + TestSupport.exportToLegacy(original, legacy, StoreLayout.OBS_SYSTEM); + byte[] before = Files.readAllBytes(legacy); + + // default: refused, file untouched, message names the config flag and the command + var refused = database(legacy, false); + try + { + assertFalse(refused.isStarted()); + String msg = chainMessages(refused.getCurrentError()); + assertTrue(msg, msg.contains("autoMigrate")); + assertTrue(msg, msg.contains(H2Migrate.class.getName())); + } + finally + { + refused.stop(); + } + assertArrayEquals(before, Files.readAllBytes(legacy)); + assertEquals(1, FileOps.readH2Format(legacy)); + + // autoMigrate: migrated in place, original retired as .old, then started normally + var db = database(legacy, true); + try + { + assertTrue(String.valueOf(db.getCurrentError()), db.isStarted()); + assertEquals(exp.numObs, db.getObservationStore().getNumRecords()); + assertEquals(exp.numFois, db.getFoiStore().getNumRecords()); + assertEquals(exp.spatialMatches, TestSupport.spatialMatches(db)); + } + finally + { + db.stop(); + } + assertEquals(3, FileOps.readH2Format(legacy)); + assertArrayEquals("original must survive untouched as .old", before, + Files.readAllBytes(legacy.resolveSibling("obs.dat" + Migrator.RETIRED_SUFFIX))); + try (Stream files = Files.list(legacy.getParent())) + { + assertEquals("no backup copy unless backupOnMigrate is set", 0, + files.filter(p -> p.getFileName().toString().contains("-backup-")).count()); + } + + // second start: nothing to migrate, still opens + db = database(legacy, true); + try + { + assertTrue(db.isStarted()); + assertEquals(exp.numObs, db.getObservationStore().getNumRecords()); + } + finally + { + db.stop(); + } + } + + + @Test + public void testFeatureDatabaseAutoMigrate() throws Exception + { + Path original = tmp.getRoot().toPath().resolve("feat-25.dat"); + Path legacy = tmp.getRoot().toPath().resolve("feat.dat"); + var exp = TestSupport.populateFeatureDatabase(original, 50); + TestSupport.exportToLegacy(original, legacy, StoreLayout.FEATURE); + byte[] before = Files.readAllBytes(legacy); + + var refused = featureDatabase(legacy, false); + try + { + assertFalse(refused.isStarted()); + String msg = chainMessages(refused.getCurrentError()); + assertTrue(msg, msg.contains("autoMigrate")); + } + finally + { + refused.stop(); + } + assertArrayEquals(before, Files.readAllBytes(legacy)); + + var db = featureDatabase(legacy, true); + try + { + assertTrue(String.valueOf(db.getCurrentError()), db.isStarted()); + assertEquals(exp.numFois, db.getFeatureStore().getNumRecords()); + assertEquals(exp.spatialMatches, TestSupport.spatialMatches(db)); + assertEquals(exp.keywordMatches, TestSupport.keywordMatches(db)); + } + finally + { + db.stop(); + } + assertEquals(3, FileOps.readH2Format(legacy)); + } +} From 95c8f13dcfbc3774abaef8ab0faf77d54d2a4db7 Mon Sep 17 00:00:00 2001 From: Ian Patterson Date: Thu, 1 Oct 2026 14:07:39 -0500 Subject: [PATCH 12/12] Fix Kryo reads of unsaved H2 pages where the KryoDataType ignored the slice's array offset. --- .../impl/datastore/h2/kryo/KryoDataType.java | 6 ++- .../impl/datastore/h2/TestKryoDataType.java | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestKryoDataType.java diff --git a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/kryo/KryoDataType.java b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/kryo/KryoDataType.java index e8f77f8ea..0cae906ba 100644 --- a/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/kryo/KryoDataType.java +++ b/sensorhub-datastore-h2/src/main/java/org/sensorhub/impl/datastore/h2/kryo/KryoDataType.java @@ -192,9 +192,11 @@ protected Object read(ByteBuffer buff, KryoInstance kryoI) Kryo kryo = kryoI.kryo; Input input = kryoI.input; - input.setBuffer(buff.array(), buff.position(), buff.remaining()); + // H2 2.x passes slices of an unsaved chunk's buffer, so the page may not start at array index 0 + int offset = buff.arrayOffset(); + input.setBuffer(buff.array(), offset + buff.position(), buff.remaining()); Object obj = kryo.readClassAndObject(input); - buff.position(input.position()); + buff.position(input.position() - offset); return obj; } diff --git a/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestKryoDataType.java b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestKryoDataType.java new file mode 100644 index 000000000..c6ef1b97d --- /dev/null +++ b/sensorhub-datastore-h2/src/test/java/org/sensorhub/impl/datastore/h2/TestKryoDataType.java @@ -0,0 +1,52 @@ +/***************************** BEGIN LICENSE BLOCK *************************** + +The contents of this file are subject to the Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this file, You can obtain one +at http://mozilla.org/MPL/2.0/. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +for the specific language governing rights and limitations under the License. + +Copyright (C) 2026 GeoRobotix Innovative Research, LLC. All Rights Reserved. + +******************************* END LICENSE BLOCK ***************************/ + +package org.sensorhub.impl.datastore.h2; + +import static org.junit.Assert.*; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.h2.mvstore.WriteBuffer; +import org.junit.Test; +import org.sensorhub.impl.datastore.h2.kryo.KryoDataType; + + +public class TestKryoDataType +{ + + /* + * H2 2.x reads pages of a chunk that is serialized but not yet saved from a + * slice of the chunk buffer, so the page does not start at array index 0 + */ + @Test + public void testReadFromSlicedBuffer() + { + var dataType = new KryoDataType(); + var wbuf = new WriteBuffer(); + dataType.write(wbuf, "first"); + dataType.write(wbuf, "second"); + var page = wbuf.getBuffer(); + page.flip(); + + int offset = 4096; + var chunk = new byte[offset + page.remaining()]; + Arrays.fill(chunk, 0, offset, (byte)0x7f); + page.get(chunk, offset, page.remaining()); + var buff = ByteBuffer.wrap(chunk).position(offset).slice(); + + assertEquals("first", dataType.read(buff)); + assertEquals("second", dataType.read(buff)); + assertFalse(buff.hasRemaining()); + } +}