From 48414dba1090da9313358585eecfc8a10af1dff2 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Sun, 13 Sep 2026 15:23:57 +0300 Subject: [PATCH 1/2] HIVE-30045: K8s operator: missing LLAP scheduler config breaks task affinity and caps cache hits A standalone Tez AM configures its LLAP task scheduler, task communicator and split location provider from the AM pod's tez-site.xml. It never loads hive-site, so hive.llap.task.*, hive.llap.client.* and hive.llap.daemon.communicator.* set in the HiveServer2 configOverrides are absent from the AM's configuration and every such key runs at its HiveConf default. That includes hive.llap.task.scheduler.locality.delay, which defaults to 0: a task whose consistent-hash daemon is busy goes to the next host immediately. Under saturation only about a third of tasks land on the daemon holding their data, and since a task reads from the cache of the daemon it runs on, the cache hit rate is pinned to the same figure no matter how large the cache is -- measured 38% data-local and a 37% hit rate on a 10 TB TPC-DS scan. Copy the keys those plugins read into tez-site, keeping tezAm overrides authoritative. With the delay reaching the scheduler, the same scan placed 1421 of 1422 tasks locally and hit 98.5% in cache. --- .../hive/kubernetes/operator/util/ConfigUtils.java | 14 ++++++++++++++ .../operator/util/HiveConfigBuilder.java | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java index 640b7d3abf75..a8a8950cb2d8 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java @@ -148,6 +148,20 @@ public static String tezAmComponentKey(String llapName) { public static final String HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY = "hive.llap.task.scheduler.locality.delay"; + /** Prefixes of the hive.* keys the Tez AM's LLAP plugins read from tez-site. */ + private static final String[] TEZ_AM_PLUGIN_KEY_PREFIXES = { + "hive.llap.task.", "hive.llap.client.", "hive.llap.daemon.communicator." + }; + + public static boolean isTezAmPluginKey(String key) { + for (String prefix : TEZ_AM_PLUGIN_KEY_PREFIXES) { + if (key.startsWith(prefix)) { + return true; + } + } + return false; + } + public static final String METASTORE_SERVER_TRANSPORT_MODE_KEY = "metastore.server.thrift.transport.mode"; public static final String METASTORE_SERVER_TRANSPORT_MODE_DEFAULT = "http"; diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java index b4957a24d86e..a7a751d8d156 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java @@ -201,6 +201,17 @@ public static Map getTezSite(HiveClusterSpec spec, LlapSpec llap tezProps.put(ConfigUtils.HIVE_LLAP_DAEMON_UMBILICAL_PORT_KEY, ConfigUtils.HIVE_LLAP_DAEMON_UMBILICAL_PORT_DEFAULT); + // A standalone AM's LLAP plugins -- task scheduler, task communicator, split location + // provider -- read tez-site, not hive-site. Users set these hive.llap keys in the + // HiveServer2 overrides, so copy them across; an explicit tezAm override below wins. + if (spec.hiveServer2().configOverrides() != null) { + spec.hiveServer2().configOverrides().forEach((key, value) -> { + if (ConfigUtils.isTezAmPluginKey(key)) { + tezProps.put(key, value); + } + }); + } + if (spec.tezAm().configOverrides() != null) { tezProps.putAll(spec.tezAm().configOverrides()); } From db117491f16f7cf6fd99df6dc1d0701c7af842fd Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Tue, 15 Sep 2026 21:20:15 +0300 Subject: [PATCH 2/2] HIVE-30045: address review comments Widen hive.llap.daemon.communicator. to hive.llap.daemon., which is what LlapTaskCommunicator actually reads, and drop the separately seeded hive.llap.daemon.umbilical.port: it now arrives through the same copy when set, and LlapTaskCommunicator reads it with HiveConf.getVar, which supplies the same "0" default when it is not. The copy runs after the keys the operator derives, so an explicit user setting wins, and before tezAm.configOverrides, which stays the last word. Drop the redundant llap.isEnabled(): the reconciler skips disabled clusters and defaultLlapCluster filters on it. --- .../kubernetes/operator/util/ConfigUtils.java | 20 +++++--------- .../operator/util/HiveConfigBuilder.java | 27 +++++++++---------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java index a8a8950cb2d8..5aebfadc979d 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java @@ -19,6 +19,7 @@ package org.apache.hive.kubernetes.operator.util; +import java.util.List; import java.util.Map; import com.fasterxml.jackson.core.JsonProcessingException; @@ -143,23 +144,14 @@ public static String tezAmComponentKey(String llapName) { public static final String HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_KEY = "hive.llap.daemon.output.service.port"; public static final int HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_DEFAULT = 15003; - public static final String HIVE_LLAP_DAEMON_UMBILICAL_PORT_KEY = "hive.llap.daemon.umbilical.port"; - public static final String HIVE_LLAP_DAEMON_UMBILICAL_PORT_DEFAULT = "0"; - public static final String HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY = "hive.llap.task.scheduler.locality.delay"; - /** Prefixes of the hive.* keys the Tez AM's LLAP plugins read from tez-site. */ - private static final String[] TEZ_AM_PLUGIN_KEY_PREFIXES = { - "hive.llap.task.", "hive.llap.client.", "hive.llap.daemon.communicator." - }; + /** Prefixes of the HiveServer2 hive.llap settings carried over to a standalone Tez AM. */ + private static final List TEZ_AM_LLAP_KEY_PREFIXES = + List.of("hive.llap.task.", "hive.llap.client.", "hive.llap.daemon."); - public static boolean isTezAmPluginKey(String key) { - for (String prefix : TEZ_AM_PLUGIN_KEY_PREFIXES) { - if (key.startsWith(prefix)) { - return true; - } - } - return false; + public static boolean isTezAmLlapKey(String key) { + return TEZ_AM_LLAP_KEY_PREFIXES.stream().anyMatch(key::startsWith); } public static final String METASTORE_SERVER_TRANSPORT_MODE_KEY = "metastore.server.thrift.transport.mode"; diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java index a7a751d8d156..bc7232ae477d 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java @@ -195,21 +195,18 @@ public static Map getTezSite(HiveClusterSpec spec, LlapSpec llap if (llap != null) { tezProps.put(ConfigUtils.HIVE_LLAP_DAEMON_SERVICE_HOSTS_KEY, llap.serviceHosts()); - } - - // Required by LlapTaskCommunicator — Tez's Configuration doesn't get HiveConf defaults - tezProps.put(ConfigUtils.HIVE_LLAP_DAEMON_UMBILICAL_PORT_KEY, - ConfigUtils.HIVE_LLAP_DAEMON_UMBILICAL_PORT_DEFAULT); - // A standalone AM's LLAP plugins -- task scheduler, task communicator, split location - // provider -- read tez-site, not hive-site. Users set these hive.llap keys in the - // HiveServer2 overrides, so copy them across; an explicit tezAm override below wins. - if (spec.hiveServer2().configOverrides() != null) { - spec.hiveServer2().configOverrides().forEach((key, value) -> { - if (ConfigUtils.isTezAmPluginKey(key)) { - tezProps.put(key, value); - } - }); + // A standalone Tez AM loads tez-site.xml, never hive-site.xml, so LLAP settings from + // the HiveServer2 overrides reach it only by being copied here, after the derived + // keys above so an explicit user setting wins. + Map hs2Overrides = spec.hiveServer2().configOverrides(); + if (hs2Overrides != null) { + hs2Overrides.forEach((key, value) -> { + if (ConfigUtils.isTezAmLlapKey(key)) { + tezProps.put(key, value); + } + }); + } } if (spec.tezAm().configOverrides() != null) { @@ -217,7 +214,7 @@ public static Map getTezSite(HiveClusterSpec spec, LlapSpec llap } // Disable Infinite locality Delay when LLAP Auto-scaling is enabled, as they are mutually exclusive. - if (llap != null && llap.isEnabled() && llap.autoscaling().isEnabled() && + if (llap != null && llap.autoscaling().isEnabled() && ConfigUtils.getTimeMs(tezProps, ConfigUtils.HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY, 0) == -1) { tezProps.put(ConfigUtils.HIVE_LLAP_TASK_SCHEDULER_LOCALITY_DELAY_KEY, "0ms"); }