config.json

{
  "architectures": ["DeepXEmbedding"],
  "model_type": "deepx-embedding",
  "hidden_size": 1536,
  "num_layers": 24,
  "num_attention_heads": 24,
  "num_key_value_heads": 8,
  "head_dim": 64,
  "intermediate_size": 4608,
  "vocab_size": 186303,
  "max_position_embeddings": 131072,
  "rope_theta": 1000000.0,
  "rope_scaling": {
    "type": "yarn",
    "factor": 8.0,
    "original_max_position_embeddings": 16384
  },
  "rms_norm_eps": 1e-6,
  "hidden_dropout": 0.0,
  "pooling_strategy": "attention",
  "use_hyperloop": true,
  "begin_layers": 4,
  "loop_block_layers": 2,
  "num_loops": 8,
  "middle_anchor_layers": 1,
  "end_layers": 4,
  "use_rode": true,
  "depth_rotary_dims": 8,
  "attention_type": "gated_linear_attention",
  "state_space_type": "mamba2",
  "torch_dtype": "float16",
  "library_name": "sentence-transformers",
  "tags": [
    "sentence-transformers",
    "feature-extraction",
    "embedding",
    "retrieval",
    "vietnamese",
    "legal",
    "linear-attention",
    "gated-deltanet",
    "matryoshka"
  ]
}