OpenVINO backend: GPU compile-time memory optimizations

ggml/src/ggml-openvino/ggml-decoder.cpp

@@ -16,6 +16,7 @@
 #include <iomanip>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <openvino/core/dimension.hpp>
 #include <openvino/core/except.hpp>
 #include <openvino/core/node.hpp>
@@ -31,6 +32,7 @@
 #include <set>
 #include <stdexcept>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph,
@@ -779,6 +781,42 @@ std::map<std::string, std::shared_ptr<ov::Node>> GgmlOvDecoder::create_weight_no
     return model_weights;
 }
 
+// Process-lifetime cache for weight nodes built from NON-OpenVINO buffers (e.g. the
+// token_embd.weight copy that lives in a CPU/mmap buffer and feeds GET_ROWS). Such
+// tensors have no OV buffer context to own a cached extra, so without this they are
+// re-extracted/re-requantized on every (re)compile — for token_embd that is a ~1-2 GB
+// F32 dequant each time. Keyed by tensor->data, which is stable for the process and
+// uniquely identifies the immutable weight bytes. OV-buffer weights keep using the
+// per-tensor extra cache and never reach here.
+static std::mutex g_nonov_weight_cache_mutex;
+static std::unordered_map<const void *, std::shared_ptr<ov::Node>> g_nonov_weight_cache;
+
+std::set<std::string> GgmlOvDecoder::collect_weight_names(ggml_cgraph * cgraph) {
+    // Mirrors the name-selection logic of create_weight_nodes() but builds no nodes,
+    // so topology checks don't trigger weight extraction/requantization.
+    std::set<std::string> names;
+    for (int node_i = 0; node_i < cgraph->n_nodes; node_i++) {
+        auto * node = cgraph->nodes[node_i];
+        for (int i = 0; i < GGML_MAX_SRC; i++) {
+            auto * src = node->src[i];
+            if (src == nullptr) {
+                continue;
+            }
+            std::string src_name(src->name);
+            if (is_rope_freqs_weight(src, node)) {
+                src_name = "rope_freqs.weight";
+            }
+            if (!src->view_src) {
+                ggml_backend_buffer * buffer = src->buffer;
+                if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type)) {
+                    names.insert(src_name);
+                }
+            }
+        }
+    }
+    return names;
+}
+
 std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor, bool naive) {
     const bool is_ov_buffer = ggml_backend_buffer_is_openvino(tensor->buffer);
 
@@ -818,6 +856,21 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
         return weight_node;
     }
 
+    // Non-OV-buffer weights (CPU/mmap, e.g. the GET_ROWS token_embd copy) have no buffer
+    // context to cache an extra in, so memoize them here keyed by their (stable) data
+    // pointer to avoid re-extracting on every recompile. Opt-in via
+    // GGML_OPENVINO_REDUCE_COMPILE_MEM. Skip for `naive` (test/naive path) since use_bias
+    // changes the produced node.
+    const bool cacheable_nonov = ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") != 0 && !is_ov_buffer &&
+                                 !naive && tensor->data != nullptr;
+    if (cacheable_nonov) {
+        std::lock_guard<std::mutex> lock(g_nonov_weight_cache_mutex);
+        auto it = g_nonov_weight_cache.find(tensor->data);
+        if (it != g_nonov_weight_cache.end()) {
+            return it->second;
+        }
+    }
+
     // There are three cases where we need to create a new weight node:
     // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor
     // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used
@@ -855,6 +908,12 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
 
     ov_weight.weight_node->set_friendly_name(tensor->name);
     if (!is_ov_buffer) {
+        if (cacheable_nonov) {
+            std::lock_guard<std::mutex> lock(g_nonov_weight_cache_mutex);
+            // Another thread may have inserted concurrently; keep the first.
+            auto [it, inserted] = g_nonov_weight_cache.emplace(tensor->data, ov_weight.weight_node);
+            return it->second;
+        }
         return ov_weight.weight_node;
     }
 

ggml/src/ggml-openvino/ggml-decoder.h

@@ -11,6 +11,8 @@
 #include <memory>
 #include <openvino/core/partial_shape.hpp>
 #include <optional>
+#include <set>
+#include <string>
 #include <vector>
 
 struct ModelParams {
@@ -237,6 +239,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
     static std::map<std::string, std::shared_ptr<ov::Node>> create_weight_nodes(ggml_cgraph * cgraph,
                                                                                 bool naive = false);
 
+    // Collect just the set of weight-tensor names referenced by the graph, without
+    // building (or requantizing) any OV weight nodes. Used by topology checks like
+    // is_model_splitted that only need name membership.
+    static std::set<std::string> collect_weight_names(ggml_cgraph * cgraph);
+
     const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const;
 
     const ggml_tensor * get_tensor_from_name(const std::string & name) const;

ggml/src/ggml-openvino/ggml-openvino-extra.cpp

@@ -45,6 +45,9 @@ void ggml_openvino_device_config::init() {
         "GGML_OPENVINO_DISABLE_CACHE",
         "GGML_OPENVINO_DISABLE_KV_SLICE",
         "GGML_OPENVINO_MANUAL_GQA_ATTN",
+        "GGML_OPENVINO_RELEASE_WEIGHTS",
+        "GGML_OPENVINO_REDUCE_COMPILE_MEM",
+        "GGML_OPENVINO_MODEL_CACHE_DIR",
     };
 
     for (const char * const & env_var : env_var_names) {

ggml/src/ggml-openvino/ggml-openvino-extra.h

@@ -99,6 +99,14 @@ int ggml_openvino_getenv_int(const char * var, int default_value = 0);
 // Check if running on NPU
 bool ggml_openvino_is_npu();
 
+// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS, GPU only).
+// register: record a host weight buffer (idempotent per data pointer).
+// release:  madvise(MADV_DONTNEED) all registered buffers, dropping their RSS.
+// released: true once release has run (used to fail-fast on post-release recompile).
+void ggml_openvino_register_weight_buffer(void * data, size_t size);
+void ggml_openvino_release_weight_buffers();
+bool ggml_openvino_weight_buffers_released();
+
 // Get requantization type for a tensor type (returns nullopt if no requant needed)
 std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false);
 

ggml/src/ggml-openvino/ggml-openvino.cpp

@@ -32,6 +32,7 @@
 #    endif
 #    include <windows.h>
 #else
+#    include <sys/mman.h>
 #    include <unistd.h>
 #endif
 
@@ -135,6 +136,81 @@ struct ggml_backend_openvino_buffer_type_context {
     std::string name;
 };
 
+// =====================================================
+// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS)
+// =====================================================
+// The OpenVINO weight Constants are zero-copy views into the host buffers
+// allocated here (ggml_aligned_malloc, anonymous memory). On GPU the plugin
+// holds its own device copy after compile_model, so the host pages are dead
+// weight for inference and can be dropped to reclaim RSS (~weights size).
+//
+// We do NOT free the buffer (ggml owns its lifetime and tensors still point
+// into it); instead madvise(MADV_DONTNEED) drops the resident pages while
+// keeping the mapping valid. A later recompile would re-read these Constants
+// from now-zeroed memory and produce garbage, so once released we fail fast
+// if the cache-miss compile branch is reached again (see utils.cpp).
+namespace {
+struct ov_weight_buffer_registry {
+    std::mutex mutex;
+    // (data, size) of every non-remote weight buffer, for madvise.
+    std::vector<std::pair<void *, size_t>> buffers;
+    bool released = false;
+};
+
+ov_weight_buffer_registry & ov_weight_registry() {
+    static ov_weight_buffer_registry reg;
+    return reg;
+}
+}  // namespace
+
+void ggml_openvino_register_weight_buffer(void * data, size_t size) {
+    if (data == nullptr || size == 0) {
+        return;
+    }
+    auto & reg = ov_weight_registry();
+    std::lock_guard<std::mutex> lock(reg.mutex);
+    for (const auto & b : reg.buffers) {
+        if (b.first == data) {
+            return;  // already registered
+        }
+    }
+    reg.buffers.emplace_back(data, size);
+}
+
+bool ggml_openvino_weight_buffers_released() {
+    auto & reg = ov_weight_registry();
+    std::lock_guard<std::mutex> lock(reg.mutex);
+    return reg.released;
+}
+
+void ggml_openvino_release_weight_buffers() {
+    auto & reg = ov_weight_registry();
+    std::lock_guard<std::mutex> lock(reg.mutex);
+    if (reg.released) {
+        return;
+    }
+    size_t total = 0;
+#if !defined(_WIN32)
+    for (const auto & b : reg.buffers) {
+        // Align down/up to page boundaries so madvise only drops whole pages
+        // fully owned by this buffer.
+        const long page = sysconf(_SC_PAGESIZE);
+        uintptr_t start = reinterpret_cast<uintptr_t>(b.first);
+        uintptr_t end = start + b.second;
+        uintptr_t astart = (start + page - 1) & ~(uintptr_t) (page - 1);
+        uintptr_t aend = end & ~(uintptr_t) (page - 1);
+        if (aend > astart) {
+            if (madvise(reinterpret_cast<void *>(astart), aend - astart, MADV_DONTNEED) == 0) {
+                total += aend - astart;
+            }
+        }
+    }
+#endif
+    reg.released = true;
+    GGML_LOG_INFO("%s: released %zu MB of host weight buffers (%zu buffers)\n", __func__, total / 1024 / 1024,
+                  reg.buffers.size());
+}
+
 // Buffer interface functions
 static void ggml_backend_openvino_buffer_free_buffer(ggml_backend_buffer_t buffer) {
     ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;
@@ -275,6 +351,22 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer
             ctx->tensor_extras[tensor] = extra;
             tensor->extra = extra;
 
+            // Register the host buffer so its pages can be dropped after the GPU
+            // plugin has its own device copy (GGML_OPENVINO_RELEASE_WEIGHTS).
+            if (!ctx->is_remote) {
+                // Weights are set once at model load. Setting a weight after a release
+                // means a second model is loading while the first's compiled graph is
+                // pinned — that graph would be wrongly reused with this model's key.
+                // Fail loud rather than return silently-wrong results.
+                if (ggml_openvino_weight_buffers_released()) {
+                    GGML_ABORT(
+                        "ggml-openvino: loading a new model while GGML_OPENVINO_RELEASE_WEIGHTS pinned a previous "
+                        "model's compiled graph. This mode supports a single model per process; unset it for "
+                        "multi-model runs.");
+                }
+                ggml_openvino_register_weight_buffer(ctx->data, ctx->size);
+            }
+
         } catch (const std::exception & e) {
             GGML_LOG_ERROR("%s: failed to process weight tensor for %s: %s\n", __func__, tensor->name, e.what());
             memcpy((char *) tensor->data + offset, data, size);
@@ -620,7 +712,13 @@ static void ggml_backend_openvino_free(ggml_backend_t backend) {
     if (ctx->runtime_context) {
         auto r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context);
         if (--r_ctx->backend_count == 0) {
-            r_ctx->clear_caches();
+            // If host weight buffers were released (GGML_OPENVINO_RELEASE_WEIGHTS), the
+            // dropped pages can never be repopulated, so a recompile is impossible. Keep
+            // the compiled-model cache alive across backend teardown so the next context
+            // reuses it instead of recompiling against zeroed weights.
+            if (!ggml_openvino_weight_buffers_released()) {
+                r_ctx->clear_caches();
+            }
         }
     }