writerslogic_warm/core/
schema.rs

1// Copyright 2024-2026 WritersLogic Contributors
2// SPDX-License-Identifier: Apache-2.0
3
4#[cfg(not(target_arch = "wasm32"))]
5use anyhow::{bail, Result};
6use serde::{Deserialize, Serialize};
7#[cfg(not(target_arch = "wasm32"))]
8use std::path::Path;
9
10#[cfg(not(target_arch = "wasm32"))]
11use super::config::WarmConfig;
12
13#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
14pub struct EmbeddingSpace {
15    pub model: String,
16    pub revision: String,
17    pub dimensions: usize,
18    pub normalization: String,
19    pub metric: String,
20}
21
22#[cfg(any(feature = "local-embed", feature = "local-models"))]
23impl EmbeddingSpace {
24    pub(crate) fn effective(
25        model: String,
26        dimensions: usize,
27        normalization: &str,
28        descriptor: &impl Serialize,
29    ) -> Result<Self> {
30        use sha2::{Digest, Sha256};
31        let digest = Sha256::digest(serde_json::to_vec(descriptor)?);
32        Ok(Self {
33            model,
34            revision: format!(
35                "hms-embedding-v1:{}",
36                digest
37                    .iter()
38                    .map(|b| format!("{b:02x}"))
39                    .collect::<String>()
40            ),
41            dimensions,
42            normalization: normalization.into(),
43            metric: "cosine".into(),
44        })
45    }
46}
47
48#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
49#[cfg(not(target_arch = "wasm32"))]
50struct StoreSchema {
51    version: u32,
52    dimensions: usize,
53    text_encoder: String,
54    vector_encoder: String,
55    metric: String,
56    encrypted: bool,
57    meaning_enabled: bool,
58    embedding_space: Option<EmbeddingSpace>,
59}
60
61/// Checks the store's `store.json` against `config` (writing it for a new store) and returns
62/// its schema version, which fixes the journal format the store is written in.
63#[cfg(not(target_arch = "wasm32"))]
64pub(crate) fn validate_store(path: &Path, dimensions: usize, config: &WarmConfig) -> Result<u32> {
65    let mut expected = StoreSchema {
66        version: super::format::STORE_SCHEMA_VERSION,
67        dimensions,
68        text_encoder: config.text_encoder.name().into(),
69        vector_encoder: "signed-projection-v2".into(),
70        metric: "jaccard".into(),
71        encrypted: config.security.encryption_enabled,
72        meaning_enabled: config.meaning.enabled,
73        embedding_space: config.embedding_space.clone(),
74    };
75    let schema_path = path.join("store.json");
76    if schema_path.exists() {
77        let bytes = std::fs::read(schema_path)?;
78        if let Some(version) = serde_json::from_slice::<serde_json::Value>(&bytes)
79            .ok()
80            .and_then(|v| v["version"].as_u64())
81            .filter(|&v| v > u64::from(super::format::STORE_SCHEMA_VERSION))
82        {
83            bail!(
84                "store schema version {version} is newer than the supported version {}; open it with a release that supports it",
85                super::format::STORE_SCHEMA_VERSION
86            );
87        }
88        let actual: StoreSchema = serde_json::from_slice(&bytes)?;
89        // Every schema from the oldest supported one opens; only the version differs between
90        // them (it selects the journal format).
91        if (super::format::STORE_SCHEMA_OLDEST..=super::format::STORE_SCHEMA_VERSION)
92            .contains(&actual.version)
93        {
94            expected.version = actual.version;
95        }
96        if actual != expected {
97            bail!("incompatible store schema: dimensions, encoder, embedding model, encryption, or meaning configuration changed; reopen with the original configuration or re-encode source documents into a new store");
98        }
99    } else {
100        if path.join("vectors_data.bin").exists() {
101            bail!("legacy store has no embedding schema; preserve it and use warm-admin reencode with original source documents to create a new store");
102        }
103        super::durable_file::atomic_write(&schema_path, &serde_json::to_vec_pretty(&expected)?)?;
104    }
105    Ok(expected.version)
106}
writerslogic_warm/core/schema.rs

writerslogic_warm/core/
schema.rs
