diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 0cd69eeaaa4e..aa9eb8f66718 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include #include GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, @@ -779,6 +781,42 @@ std::map> GgmlOvDecoder::create_weight_no return model_weights; } +// Process-lifetime cache for weight nodes built from NON-OpenVINO buffers (e.g. the +// token_embd.weight copy that lives in a CPU/mmap buffer and feeds GET_ROWS). Such +// tensors have no OV buffer context to own a cached extra, so without this they are +// re-extracted/re-requantized on every (re)compile — for token_embd that is a ~1-2 GB +// F32 dequant each time. Keyed by tensor->data, which is stable for the process and +// uniquely identifies the immutable weight bytes. OV-buffer weights keep using the +// per-tensor extra cache and never reach here. +static std::mutex g_nonov_weight_cache_mutex; +static std::unordered_map> g_nonov_weight_cache; + +std::set GgmlOvDecoder::collect_weight_names(ggml_cgraph * cgraph) { + // Mirrors the name-selection logic of create_weight_nodes() but builds no nodes, + // so topology checks don't trigger weight extraction/requantization. + std::set names; + for (int node_i = 0; node_i < cgraph->n_nodes; node_i++) { + auto * node = cgraph->nodes[node_i]; + for (int i = 0; i < GGML_MAX_SRC; i++) { + auto * src = node->src[i]; + if (src == nullptr) { + continue; + } + std::string src_name(src->name); + if (is_rope_freqs_weight(src, node)) { + src_name = "rope_freqs.weight"; + } + if (!src->view_src) { + ggml_backend_buffer * buffer = src->buffer; + if (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type)) { + names.insert(src_name); + } + } + } + } + return names; +} + std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor, bool naive) { const bool is_ov_buffer = ggml_backend_buffer_is_openvino(tensor->buffer); @@ -818,6 +856,21 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor return weight_node; } + // Non-OV-buffer weights (CPU/mmap, e.g. the GET_ROWS token_embd copy) have no buffer + // context to cache an extra in, so memoize them here keyed by their (stable) data + // pointer to avoid re-extracting on every recompile. Opt-in via + // GGML_OPENVINO_REDUCE_COMPILE_MEM. Skip for `naive` (test/naive path) since use_bias + // changes the produced node. + const bool cacheable_nonov = ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") != 0 && !is_ov_buffer && + !naive && tensor->data != nullptr; + if (cacheable_nonov) { + std::lock_guard lock(g_nonov_weight_cache_mutex); + auto it = g_nonov_weight_cache.find(tensor->data); + if (it != g_nonov_weight_cache.end()) { + return it->second; + } + } + // There are three cases where we need to create a new weight node: // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used @@ -855,6 +908,12 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor ov_weight.weight_node->set_friendly_name(tensor->name); if (!is_ov_buffer) { + if (cacheable_nonov) { + std::lock_guard lock(g_nonov_weight_cache_mutex); + // Another thread may have inserted concurrently; keep the first. + auto [it, inserted] = g_nonov_weight_cache.emplace(tensor->data, ov_weight.weight_node); + return it->second; + } return ov_weight.weight_node; } diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 695676acd6ba..c27d3306f07f 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include struct ModelParams { @@ -237,6 +239,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { static std::map> create_weight_nodes(ggml_cgraph * cgraph, bool naive = false); + // Collect just the set of weight-tensor names referenced by the graph, without + // building (or requantizing) any OV weight nodes. Used by topology checks like + // is_model_splitted that only need name membership. + static std::set collect_weight_names(ggml_cgraph * cgraph); + const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const; const ggml_tensor * get_tensor_from_name(const std::string & name) const; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 18df24c77e64..cb62a0022313 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -45,6 +45,9 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_DISABLE_CACHE", "GGML_OPENVINO_DISABLE_KV_SLICE", "GGML_OPENVINO_MANUAL_GQA_ATTN", + "GGML_OPENVINO_RELEASE_WEIGHTS", + "GGML_OPENVINO_REDUCE_COMPILE_MEM", + "GGML_OPENVINO_MODEL_CACHE_DIR", }; for (const char * const & env_var : env_var_names) { diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index c2654fbfa1b8..e62d966accc0 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -99,6 +99,14 @@ int ggml_openvino_getenv_int(const char * var, int default_value = 0); // Check if running on NPU bool ggml_openvino_is_npu(); +// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS, GPU only). +// register: record a host weight buffer (idempotent per data pointer). +// release: madvise(MADV_DONTNEED) all registered buffers, dropping their RSS. +// released: true once release has run (used to fail-fast on post-release recompile). +void ggml_openvino_register_weight_buffer(void * data, size_t size); +void ggml_openvino_release_weight_buffers(); +bool ggml_openvino_weight_buffers_released(); + // Get requantization type for a tensor type (returns nullopt if no requant needed) std::optional ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false); diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 6c88a7405cf4..a137aaab2512 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -32,6 +32,7 @@ # endif # include #else +# include # include #endif @@ -135,6 +136,81 @@ struct ggml_backend_openvino_buffer_type_context { std::string name; }; +// ===================================================== +// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS) +// ===================================================== +// The OpenVINO weight Constants are zero-copy views into the host buffers +// allocated here (ggml_aligned_malloc, anonymous memory). On GPU the plugin +// holds its own device copy after compile_model, so the host pages are dead +// weight for inference and can be dropped to reclaim RSS (~weights size). +// +// We do NOT free the buffer (ggml owns its lifetime and tensors still point +// into it); instead madvise(MADV_DONTNEED) drops the resident pages while +// keeping the mapping valid. A later recompile would re-read these Constants +// from now-zeroed memory and produce garbage, so once released we fail fast +// if the cache-miss compile branch is reached again (see utils.cpp). +namespace { +struct ov_weight_buffer_registry { + std::mutex mutex; + // (data, size) of every non-remote weight buffer, for madvise. + std::vector> buffers; + bool released = false; +}; + +ov_weight_buffer_registry & ov_weight_registry() { + static ov_weight_buffer_registry reg; + return reg; +} +} // namespace + +void ggml_openvino_register_weight_buffer(void * data, size_t size) { + if (data == nullptr || size == 0) { + return; + } + auto & reg = ov_weight_registry(); + std::lock_guard lock(reg.mutex); + for (const auto & b : reg.buffers) { + if (b.first == data) { + return; // already registered + } + } + reg.buffers.emplace_back(data, size); +} + +bool ggml_openvino_weight_buffers_released() { + auto & reg = ov_weight_registry(); + std::lock_guard lock(reg.mutex); + return reg.released; +} + +void ggml_openvino_release_weight_buffers() { + auto & reg = ov_weight_registry(); + std::lock_guard lock(reg.mutex); + if (reg.released) { + return; + } + size_t total = 0; +#if !defined(_WIN32) + for (const auto & b : reg.buffers) { + // Align down/up to page boundaries so madvise only drops whole pages + // fully owned by this buffer. + const long page = sysconf(_SC_PAGESIZE); + uintptr_t start = reinterpret_cast(b.first); + uintptr_t end = start + b.second; + uintptr_t astart = (start + page - 1) & ~(uintptr_t) (page - 1); + uintptr_t aend = end & ~(uintptr_t) (page - 1); + if (aend > astart) { + if (madvise(reinterpret_cast(astart), aend - astart, MADV_DONTNEED) == 0) { + total += aend - astart; + } + } + } +#endif + reg.released = true; + GGML_LOG_INFO("%s: released %zu MB of host weight buffers (%zu buffers)\n", __func__, total / 1024 / 1024, + reg.buffers.size()); +} + // Buffer interface functions static void ggml_backend_openvino_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context; @@ -275,6 +351,22 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer ctx->tensor_extras[tensor] = extra; tensor->extra = extra; + // Register the host buffer so its pages can be dropped after the GPU + // plugin has its own device copy (GGML_OPENVINO_RELEASE_WEIGHTS). + if (!ctx->is_remote) { + // Weights are set once at model load. Setting a weight after a release + // means a second model is loading while the first's compiled graph is + // pinned — that graph would be wrongly reused with this model's key. + // Fail loud rather than return silently-wrong results. + if (ggml_openvino_weight_buffers_released()) { + GGML_ABORT( + "ggml-openvino: loading a new model while GGML_OPENVINO_RELEASE_WEIGHTS pinned a previous " + "model's compiled graph. This mode supports a single model per process; unset it for " + "multi-model runs."); + } + ggml_openvino_register_weight_buffer(ctx->data, ctx->size); + } + } catch (const std::exception & e) { GGML_LOG_ERROR("%s: failed to process weight tensor for %s: %s\n", __func__, tensor->name, e.what()); memcpy((char *) tensor->data + offset, data, size); @@ -620,7 +712,13 @@ static void ggml_backend_openvino_free(ggml_backend_t backend) { if (ctx->runtime_context) { auto r_ctx = std::static_pointer_cast(ctx->runtime_context); if (--r_ctx->backend_count == 0) { - r_ctx->clear_caches(); + // If host weight buffers were released (GGML_OPENVINO_RELEASE_WEIGHTS), the + // dropped pages can never be repopulated, so a recompile is impossible. Keep + // the compiled-model cache alive across backend teardown so the next context + // reuses it instead of recompiling against zeroed weights. + if (!ggml_openvino_weight_buffers_released()) { + r_ctx->clear_caches(); + } } } diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index d4e4d8f660b0..fa48197d94c5 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -2,6 +2,7 @@ #include "ggml-common.h" #include "ggml-impl.h" +#include "ggml-openvino-extra.h" #include "ggml.h" #include @@ -779,28 +780,75 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, ov::Tensor & scales, ov::Tensor & zp) { int64_t n_elements = ggml_nelements(tensor); + const int64_t ne0 = tensor->ne[0]; // elements per row + const int64_t n_rows = n_elements / ne0; + const auto * type_traits = ggml_get_type_traits(tensor->type); + const size_t src_row_bytes = ggml_row_size(tensor->type, ne0); - // First dequantize to F32 - std::vector weights_f32(n_elements); - ggml_get_type_traits(tensor->type)->to_float(data, weights_f32.data(), n_elements); - - // Handle F16 case - just convert and create constant - if (requant_type == ExtraQuantType::F16) { - ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements); - auto result = std::make_shared(weights); - result->set_friendly_name(tensor->name); - return result; - } - - // Requantize to target quantized format bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128); - if (is_u4) { - quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); - } else if (requant_type == ExtraQuantType::Q8_1_C) { - quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size); + // Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM): instead of + // materializing the full n_elements F32 array (e.g. ~1 GB for token_embd), dequantize + // a chunk of complete rows into a small scratch and quantize/convert it straight into + // the output buffers, capping the transient F32 footprint at CHUNK_ROWS*ne0 floats. + // + // Only valid (and only used) for the Q8_0_C / Q8_1_C / F16 targets whose block size + // divides a row (channel-wise _C uses block_size == ne0) so no target block straddles + // a row boundary, and Q8/F16 have no cross-block packing. The u4 (Q4_0) path packs two + // weights per byte with running zp ORs that assume a single whole-array call, so it is + // never streamed. When the flag is off, behavior is identical to the original + // full-materialization path. + const bool stream_requant = ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") != 0 && !is_u4 && + !(block_size > 0 && ne0 % block_size != 0); + + if (!stream_requant) { + // Full materialization (original behavior): dequantize the whole tensor to F32, + // then convert/quantize in one call. + std::vector weights_f32(n_elements); + type_traits->to_float(data, weights_f32.data(), n_elements); + if (requant_type == ExtraQuantType::F16) { + ggml_get_type_traits(GGML_TYPE_F16)->from_float_ref(weights_f32.data(), weights.data(), n_elements); + auto result = std::make_shared(weights); + result->set_friendly_name(tensor->name); + return result; + } + if (is_u4) { + quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } else if (requant_type == ExtraQuantType::Q8_1_C) { + quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } else { + quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); + } } else { - quantize_q8_0(weights_f32.data(), weights, scales, zp, n_elements, block_size); + // Streaming path for Q8_0_C / Q8_1_C / F16 (covers token_embd, output.weight, + // and per-layer Q6_K/Q5_K requant — the large transient cases). + const int64_t CHUNK_ROWS = std::min(n_rows, 256); + std::vector scratch(CHUNK_ROWS * ne0); + // F16 destination: 2 bytes/element, advanced per chunk by r0*ne0 elements. + auto * f16_base = static_cast(weights.data()); + for (int64_t r0 = 0; r0 < n_rows; r0 += CHUNK_ROWS) { + const int64_t rows = std::min(CHUNK_ROWS, n_rows - r0); + const int64_t elems = rows * ne0; + const auto * src = static_cast(data) + r0 * src_row_bytes; + type_traits->to_float(src, scratch.data(), elems); + + if (requant_type == ExtraQuantType::F16) { + ggml_get_type_traits(GGML_TYPE_F16) + ->from_float_ref(scratch.data(), f16_base + (r0 * ne0) * sizeof(uint16_t), elems); + } else { + const int64_t block_offset = (r0 * ne0) / block_size; + if (requant_type == ExtraQuantType::Q8_1_C) { + quantize_q8_1(scratch.data(), weights, scales, zp, elems, block_size, block_offset); + } else { + quantize_q8_0(scratch.data(), weights, scales, zp, elems, block_size, block_offset); + } + } + } + if (requant_type == ExtraQuantType::F16) { + auto result = std::make_shared(weights); + result->set_friendly_name(tensor->name); + return result; + } } // Create the OpenVINO weight subgraph @@ -1055,16 +1103,21 @@ void quantize_q8_0(const float * x, ov::Tensor & scales_arr, ov::Tensor & zp_arr, int64_t k, - int64_t qk) { + int64_t qk, + int64_t block_offset) { assert(k % qk == 0); const int nb = k / qk; - auto * weights = static_cast(weights_arr.data()); - auto * scales = scales_arr.data::value_type>(); + // block_offset lets a caller quantize a chunk of blocks into the right place in the + // output buffers (used for streaming requant). x points at this chunk's first block; + // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no + // nibble packing), so any block boundary is safe. + auto * weights = static_cast(weights_arr.data()) + block_offset * qk; + auto * scales = scales_arr.data::value_type>() + block_offset; bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path if (!is_symmetric) { - auto * zp = static_cast(zp_arr.data()); + auto * zp = static_cast(zp_arr.data()) + block_offset; for (int i = 0; i < nb; i++) { float amax = 0.0f; for (int j = 0; j < qk; j++) { @@ -1106,13 +1159,15 @@ void quantize_q8_1(const float * x, ov::Tensor & scales_arr, ov::Tensor & zp_arr, int64_t k, - int64_t qk) { + int64_t qk, + int64_t block_offset) { assert(k % qk == 0); const int nb = k / qk; - auto * weights = static_cast(weights_arr.data()); - auto * scales = scales_arr.data::value_type>(); - auto * zp = static_cast(zp_arr.data()); + // See quantize_q8_0: block_offset places this chunk's output at the right block. + auto * weights = static_cast(weights_arr.data()) + block_offset * qk; + auto * scales = scales_arr.data::value_type>() + block_offset; + auto * zp = static_cast(zp_arr.data()) + block_offset; for (int i = 0; i < nb; i++) { float min = std::numeric_limits::max(); float max = std::numeric_limits::lowest(); diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h index 1b89fd887e16..fddbd9fd2b32 100644 --- a/ggml/src/ggml-openvino/ggml-quants.h +++ b/ggml/src/ggml-openvino/ggml-quants.h @@ -146,13 +146,15 @@ void quantize_q8_1(const float * x, ov::Tensor & scales_arr, ov::Tensor & zp_arr, int64_t k, - int64_t qk); + int64_t qk, + int64_t block_offset = 0); void quantize_q8_0(const float * x, ov::Tensor & weights_arr, ov::Tensor & scales_arr, ov::Tensor & zp_arr, int64_t k, - int64_t qk); + int64_t qk, + int64_t block_offset = 0); namespace ov { namespace op { diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp new file mode 100644 index 000000000000..2c2e5ac14d0f --- /dev/null +++ b/ggml/src/ggml-openvino/model-cache.cpp @@ -0,0 +1,272 @@ +#include "model-cache.h" + +#include "ggml-backend-impl.h" +#include "ggml-backend.h" +#include "ggml-impl.h" +#include "ggml-openvino-extra.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# include +#endif + +namespace { + +// 64-bit FNV-1a, the mixing primitive for all fingerprints here. +inline uint64_t fnv1a(uint64_t h, const void * data, size_t n) { + const uint8_t * p = static_cast(data); + for (size_t i = 0; i < n; ++i) { + h ^= p[i]; + h *= 0x100000001b3ull; + } + return h; +} + +inline uint64_t fnv1a_u64(uint64_t h, uint64_t v) { + return fnv1a(h, &v, sizeof(v)); +} + +constexpr uint64_t FNV_OFFSET = 0xcbf29ce484222325ull; + +// Bytes sampled from each end of a weight tensor for the sampled hash. The whole +// model is never hashed (that would cost seconds every run); instead we sample a +// bounded window from the head and tail of each weight's bytes. The manifest +// re-verify (same sample) guards the residual collision risk. +constexpr size_t WEIGHT_SAMPLE_BYTES = 4096; + +// Is this src a model weight, mirroring create_weight_nodes()'s selection: +// non-view tensor whose buffer is USAGE_WEIGHTS or whose type is quantized. +bool is_weight_src(const ggml_tensor * src) { + if (src == nullptr || src->view_src != nullptr || src->buffer == nullptr) { + return false; + } + return src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS || ggml_is_quantized(src->type); +} + +// Per-weight sampled fingerprint: identity (name/shape/type) + a bounded byte +// sample. Returns FNV offset basis if data is unavailable (kept deterministic). +uint64_t weight_fingerprint(const ggml_tensor * t) { + uint64_t h = FNV_OFFSET; + h = fnv1a(h, t->name, strlen(t->name)); + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + h = fnv1a_u64(h, static_cast(t->ne[i])); + } + h = fnv1a_u64(h, static_cast(t->type)); + const size_t nbytes = ggml_nbytes(t); + h = fnv1a_u64(h, nbytes); + if (t->data != nullptr && nbytes > 0) { + const size_t head = nbytes < WEIGHT_SAMPLE_BYTES ? nbytes : WEIGHT_SAMPLE_BYTES; + h = fnv1a(h, t->data, head); + if (nbytes > WEIGHT_SAMPLE_BYTES) { + const size_t tail = nbytes < 2 * WEIGHT_SAMPLE_BYTES ? nbytes - WEIGHT_SAMPLE_BYTES : WEIGHT_SAMPLE_BYTES; + h = fnv1a(h, static_cast(t->data) + (nbytes - tail), tail); + } + } + return h; +} + +// Walk the cgraph and invoke fn(weight_tensor) for each distinct weight, in node +// order. De-duplicates by tensor pointer so a weight used by several nodes is +// fingerprinted once, deterministically. +template +void for_each_weight(const ggml_cgraph * cgraph, F && fn) { + std::vector seen; + for (int i = 0; i < cgraph->n_nodes; ++i) { + const ggml_tensor * node = cgraph->nodes[i]; + for (int s = 0; s < GGML_MAX_SRC; ++s) { + const ggml_tensor * src = node->src[s]; + if (!is_weight_src(src)) { + continue; + } + bool dup = false; + for (const auto * p : seen) { + if (p == src) { + dup = true; + break; + } + } + if (dup) { + continue; + } + seen.push_back(src); + fn(src); + } + } +} + +std::string ov_version_string() { + const ov::Version v = ov::get_openvino_version(); + return std::string(v.buildNumber ? v.buildNumber : "unknown"); +} + +std::string hex64(uint64_t v) { + char buf[17]; + snprintf(buf, sizeof(buf), "%016llx", static_cast(v)); + return std::string(buf); +} + +// Portable mkdir for a single path component. Returns true if the directory +// exists after the call (created now or already present). +bool make_dir(const std::string & path) { +#if defined(_WIN32) + int rc = _mkdir(path.c_str()); +#else + int rc = ::mkdir(path.c_str(), 0755); +#endif + if (rc == 0 || errno == EEXIST) { + return true; + } + return false; +} + +// Create `path` and any missing parents (like `mkdir -p`). Best-effort: +// returns true only if the full directory exists afterwards. +bool make_dirs(const std::string & path) { + if (path.empty()) { + return false; + } + std::string acc; + for (size_t i = 0; i < path.size(); ++i) { + const char c = path[i]; + acc.push_back(c); + const bool sep = (c == '/' +#if defined(_WIN32) + || c == '\\' +#endif + ); + // Create each intermediate component (skip a leading "/" root). + if (sep && acc.size() > 1) { + std::string component = acc.substr(0, acc.size() - 1); + if (!make_dir(component)) { + return false; + } + } + } + return make_dir(path); +} + +} // namespace + +std::string ggml_openvino_model_cache_dir() { + const char * dir = ggml_openvino_getenv_str("GGML_OPENVINO_MODEL_CACHE_DIR"); + if (!dir || strlen(dir) == 0) { + return std::string(); + } + std::string path(dir); + // Create the cache directory (and parents) on first use so callers don't + // have to pre-create it; a missing dir would otherwise silently disable the + // cache (manifest/blob writes fail with no directory to write into). + if (!make_dirs(path)) { + GGML_LOG_WARN("ggml-openvino: could not create model cache dir '%s' (errno=%d); caching disabled\n", + path.c_str(), errno); + return std::string(); + } + return path; +} + +uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph, + const std::string & device, + bool fa, + const int32_t * rope_params, + int rope_len, + uint64_t extra_cfg) { + uint64_t h = FNV_OFFSET; + + // Topology: node count + each node's op and name (cheap, and distinguishes + // graphs that share weights but differ structurally). + h = fnv1a_u64(h, static_cast(cgraph->n_nodes)); + for (int i = 0; i < cgraph->n_nodes; ++i) { + const ggml_tensor * node = cgraph->nodes[i]; + h = fnv1a_u64(h, static_cast(node->op)); + h = fnv1a(h, node->name, strlen(node->name)); + } + + // Weights: the model identity. + for_each_weight(cgraph, [&](const ggml_tensor * t) { h = fnv1a_u64(h, weight_fingerprint(t)); }); + + // Config that changes the produced blob. + h = fnv1a(h, device.data(), device.size()); + h = fnv1a_u64(h, fa ? 1u : 0u); + if (rope_params && rope_len > 0) { + h = fnv1a(h, rope_params, sizeof(int32_t) * static_cast(rope_len)); + } + h = fnv1a_u64(h, extra_cfg); + const std::string ver = ov_version_string(); + h = fnv1a(h, ver.data(), ver.size()); + + return h; +} + +std::string ggml_openvino_model_cache_blob_path(const std::string & dir, uint64_t fingerprint) { + return dir + "/" + hex64(fingerprint) + ".blob"; +} + +std::string ggml_openvino_model_cache_manifest_path(const std::string & dir, uint64_t fingerprint) { + return dir + "/" + hex64(fingerprint) + ".manifest"; +} + +bool ggml_openvino_model_cache_write_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint) { + std::ofstream f(path, std::ios::trunc); + if (!f.is_open()) { + return false; + } + f << "fingerprint " << hex64(fingerprint) << "\n"; + f << "ov_version " << ov_version_string() << "\n"; + for_each_weight(cgraph, [&](const ggml_tensor * t) { + f << t->name << " " << t->ne[0] << " " << t->ne[1] << " " << t->ne[2] << " " << t->ne[3] << " " + << static_cast(t->type) << " " << hex64(weight_fingerprint(t)) << "\n"; + }); + return f.good(); +} + +bool ggml_openvino_model_cache_verify_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint) { + std::ifstream f(path); + if (!f.is_open()) { + return false; + } + std::string tag, val; + // header: fingerprint + if (!(f >> tag >> val) || tag != "fingerprint" || val != hex64(fingerprint)) { + return false; + } + // header: ov_version + if (!(f >> tag >> val) || tag != "ov_version" || val != ov_version_string()) { + return false; + } + + // Build the expected per-weight lines from the live cgraph, then require an + // exact match (same set, same order) against the manifest. + std::vector expected; + for_each_weight(cgraph, [&](const ggml_tensor * t) { + expected.push_back(std::string(t->name) + " " + std::to_string(t->ne[0]) + " " + std::to_string(t->ne[1]) + + " " + std::to_string(t->ne[2]) + " " + std::to_string(t->ne[3]) + " " + + std::to_string(static_cast(t->type)) + " " + hex64(weight_fingerprint(t))); + }); + + size_t idx = 0; + std::string line; + std::getline(f, line); // consume rest of ov_version line + while (std::getline(f, line)) { + if (line.empty()) { + continue; + } + if (idx >= expected.size() || line != expected[idx]) { + return false; + } + ++idx; + } + return idx == expected.size(); +} diff --git a/ggml/src/ggml-openvino/model-cache.h b/ggml/src/ggml-openvino/model-cache.h new file mode 100644 index 000000000000..ec46e827a448 --- /dev/null +++ b/ggml/src/ggml-openvino/model-cache.h @@ -0,0 +1,56 @@ +#pragma once + +// Frontend-level model cache (GGML_OPENVINO_MODEL_CACHE_DIR). +// +// The OpenVINO plugin's own ov::cache_dir caches the compiled blob keyed by the +// *OV model*, but producing that model still runs the full frontend every time: +// weight requantization (incl. the large token_embd F32 transient) and the +// ggml->OV graph conversion. This cache keys off a fingerprint computed directly +// from the ggml cgraph, so a hit skips requant + convert + compile entirely and +// instead imports a previously exported CompiledModel blob. +// +// Opt-in and independent from GGML_OPENVINO_CACHE_DIR. Default off. + +#include "ggml.h" + +#include +#include + +// Returns the model-cache directory from GGML_OPENVINO_MODEL_CACHE_DIR, or empty +// if unset/disabled. When empty, callers must not use the cache. +std::string ggml_openvino_model_cache_dir(); + +// Compute a stable 64-bit fingerprint identifying the model+config that a cgraph +// would compile to. Combines graph topology, a sampled hash of every weight +// tensor (name/shape/dtype + bounded byte sample), and the config that changes +// the produced blob (device, flash-attention, rope params, the compile-memory +// flags, stateful, and the OpenVINO version). `device` is the resolved device +// string; `fa` is the flash-attention flag; `rope_params`/`rope_len` cover the +// model's rope configuration; `extra_cfg` folds in any other blob-affecting bits. +uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph, + const std::string & device, + bool fa, + const int32_t * rope_params, + int rope_len, + uint64_t extra_cfg); + +// Path to the compiled-blob file for a fingerprint (/.blob). +std::string ggml_openvino_model_cache_blob_path(const std::string & dir, uint64_t fingerprint); + +// Path to the sidecar manifest (/.manifest) holding the per-weight +// fingerprints, used to re-verify a hit before trusting the blob. +std::string ggml_openvino_model_cache_manifest_path(const std::string & dir, uint64_t fingerprint); + +// Write/read the manifest. The manifest is a newline-separated list of +// "name ne0 ne1 ne2 ne3 type sample_hash" lines plus a header line with the +// fingerprint and OV version. Returns false on I/O error. +bool ggml_openvino_model_cache_write_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint); + +// Verify that the cgraph's weights still match the stored manifest (guards the +// sampled-hash collision risk: a blob is only trusted if every weight's +// name/shape/type/sample-hash matches what was cached). Returns true on match. +bool ggml_openvino_model_cache_verify_manifest(const std::string & path, + const ggml_cgraph * cgraph, + uint64_t fingerprint); diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 70af08bdf182..04d1cad90414 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -4,6 +4,7 @@ #include "ggml-openvino-extra.h" #include "ggml-openvino/ggml-decoder.h" #include "ggml.h" +#include "model-cache.h" #include "openvino/frontend.h" #include "openvino/input_model.h" @@ -286,13 +287,101 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< conversion_end_time = decoder_end_time; compile_end_time = decoder_end_time; } else { + // Fail fast: a cache-miss recompile feeds weight data to compile_model, but + // GGML_OPENVINO_RELEASE_WEIGHTS may have already dropped the host weight pages + // (they would read as zeros). That mode requires stable graph shapes. + if (ggml_openvino_weight_buffers_released()) { + GGML_ABORT( + "ggml-openvino: a new graph needs to be compiled but host weight buffers were already " + "released via GGML_OPENVINO_RELEASE_WEIGHTS. This mode requires stable graph shapes; " + "unset GGML_OPENVINO_RELEASE_WEIGHTS for dynamic workloads."); + } if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); r_ctx->infer_request_cache.erase(key); } bool model_is_splitted = is_model_splitted(cgraph); + // Frontend-level model cache (GGML_OPENVINO_MODEL_CACHE_DIR): if this model + // was compiled before, import the saved blob and skip requant + convert + + // compile. Only the dynamic single-model path is cached (split models compile + // two graphs and are left to the plugin-level ov::cache_dir). The decoder is + // still needed for I/O mapping, but can be built without weight nodes since + // the weights are baked into the imported CompiledModel. + const std::string model_cache_dir = ggml_openvino_model_cache_dir(); + uint64_t model_fp = 0; + std::string blob_path, manifest_path; + bool imported = false; + // When the frontend model cache is active it supersedes the plugin-level + // ov::cache_dir: a blob exported from a model compiled WITH cache_dir cannot + // be re-imported (import returns an uninitialized model). Strip cache_dir / + // cache_mode from the config used for the cached compile and the import. + ov::AnyMap mc_config = config; + if (!model_cache_dir.empty()) { + mc_config.erase("CACHE_DIR"); + mc_config.erase("CACHE_MODE"); + } + if (!model_cache_dir.empty() && !model_is_splitted) { + uint64_t extra_cfg = 0; + extra_cfg = extra_cfg * 131 + (stateful ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM") ? 2u : 0u); + model_fp = ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, + 15, extra_cfg); + blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); + manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp); + + std::ifstream blob_in(blob_path, std::ios::binary); + bool blob_ok = blob_in.is_open(); + bool manifest_ok = blob_ok && ggml_openvino_model_cache_verify_manifest(manifest_path, cgraph, model_fp); + if (blob_ok && manifest_ok) { + int64_t import_start = ggml_time_us(); + try { + ov::CompiledModel cm; + auto remote_context = ggml_openvino_get_remote_context(); + if (remote_context.has_value()) { + cm = core.import_model(blob_in, remote_context.value(), mc_config); + } else { + cm = core.import_model(blob_in, device, mc_config); + } + // Lightweight decoder: names-only weight map (membership is all the + // decoder needs; weights live in the imported model). + std::map> weight_names; + for (const auto & n : GgmlOvDecoder::collect_weight_names(cgraph)) { + weight_names[n] = nullptr; + } + ggml_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, + is_static, stateful, model_is_splitted); + infer_request = std::make_shared(cm.create_infer_request()); + entry->ptr = ggml_decoder; + // Names must match the decoder's ggml-tensor keys. The non-cached + // path keys off Parameter/Result *friendly names* (set by the + // frontend); export_model preserves these, and each compiled-model + // port's node is exactly that Parameter/Result. Use the port nodes + // directly (NOT get_runtime_model(), whose graph differs and is + // unsafe to deref this way). + for (const auto & p : cm.inputs()) { + ov_input_names.push_back(p.get_node()->get_friendly_name()); + } + for (const auto & o : cm.outputs()) { + ov_output_names.push_back(o.get_node()->get_friendly_name()); + } + imported = true; + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { + GGML_LOG_INFO(" - Model cache import time: %.3f ms \n", + (ggml_time_us() - import_start) / 1000.0); + } + GGML_LOG_INFO("ggml-openvino: model cache HIT %s\n", blob_path.c_str()); + } catch (const std::exception & e) { + GGML_LOG_WARN("ggml-openvino: model cache import failed (%s), recompiling\n", e.what()); + imported = false; + } + } + } + std::shared_ptr model; + if (imported) { + decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); + } else { auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, @@ -311,14 +400,43 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< ov::serialize(model, timestamped_filename); } + // Use the cache-stripped config when the frontend model cache is active, so + // the resulting CompiledModel can be exported and later re-imported. + const ov::AnyMap & compile_config = model_cache_dir.empty() ? config : mc_config; ov::CompiledModel compiled_model; auto remote_context = ggml_openvino_get_remote_context(); if (remote_context.has_value()) { - compiled_model = core.compile_model(model, remote_context.value(), config); + compiled_model = core.compile_model(model, remote_context.value(), compile_config); } else { - compiled_model = core.compile_model(model, device, config); + compiled_model = core.compile_model(model, device, compile_config); } compile_end_time = ggml_time_us(); + + // Export to the frontend model cache for next time. Write the blob to a + // temp file then rename (atomic) so a concurrent/crashed run never sees a + // half-written blob; write the manifest first so a present blob always has + // a verifiable manifest. + if (!model_cache_dir.empty() && !model_is_splitted && model_fp != 0) { + try { + if (ggml_openvino_model_cache_write_manifest(manifest_path, cgraph, model_fp)) { + const std::string tmp = blob_path + ".tmp"; + std::ofstream blob_out(tmp, std::ios::binary | std::ios::trunc); + if (blob_out.is_open()) { + compiled_model.export_model(blob_out); + blob_out.close(); + if (blob_out.good()) { + std::rename(tmp.c_str(), blob_path.c_str()); + GGML_LOG_INFO("ggml-openvino: model cache WROTE %s\n", blob_path.c_str()); + } else { + std::remove(tmp.c_str()); + } + } + } + } catch (const std::exception & e) { + GGML_LOG_WARN("ggml-openvino: model cache export failed: %s\n", e.what()); + } + } + infer_request = std::make_shared(compiled_model.create_infer_request()); entry->ptr = ggml_decoder; @@ -328,6 +446,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< for (const auto & ov_output : model->get_results()) { ov_output_names.push_back(ov_output->get_friendly_name()); } + } // end non-imported (compile) path if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); @@ -390,6 +509,20 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< } } + // GGML_OPENVINO_RELEASE_WEIGHTS: on GPU the plugin holds its own device copy of + // every weight after compile, so the host weight buffers can be dropped to reclaim + // RSS. The GPU backend uses a single dynamic-shape model for both prefill and decode, + // so once a graph is compiled it is reused for the whole session — the only thing + // that forces a recompile is clear_caches() on backend teardown. We therefore release + // on the first cache-hit (model compiled, plugin has its copy) and, crucially, pin the + // compiled-model cache so it survives backend teardown (see ggml_backend_openvino_free). + // Without the pin, a later test/context would recompile against the now-dropped pages. + // A genuinely new graph still fails fast at the cache-miss compile branch. + if (cache_hit && device == "GPU" && ggml_openvino_getenv_int("GGML_OPENVINO_RELEASE_WEIGHTS") && + !ggml_openvino_weight_buffers_released()) { + ggml_openvino_release_weight_buffers(); + } + return GGML_STATUS_SUCCESS; } @@ -670,7 +803,17 @@ bool is_model_splitted(ggml_cgraph * cgraph) { } } // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check. - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, true); + // Only weight-name membership is needed below. With GGML_OPENVINO_REDUCE_COMPILE_MEM + // use the name-only collector (no weight extraction); otherwise keep the original + // behavior of building (naive) weight nodes and take their names. + std::set model_weights; + if (ggml_openvino_getenv_int("GGML_OPENVINO_REDUCE_COMPILE_MEM")) { + model_weights = GgmlOvDecoder::collect_weight_names(cgraph); + } else { + for (const auto & kv : GgmlOvDecoder::create_weight_nodes(cgraph, true)) { + model_weights.insert(kv.first); + } + } std::set model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes); // leaf nodes std::set model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs);