feat: add fts support

.gitattributes

@@ -0,0 +1,3 @@
+# Auto-generated files — collapsed in GitHub PR diffs
+src/db/index/column/fts_column/gen/** linguist-generated=true
+src/db/sqlengine/antlr/gen/** linguist-generated=true

.gitmodules

@@ -40,3 +40,12 @@
 [submodule "thirdparty/RaBitQ-Library/RaBitQ-Library-0.1"]
 	path = thirdparty/RaBitQ-Library/RaBitQ-Library-0.1
 	url = https://github.com/VectorDB-NTU/RaBitQ-Library.git
+[submodule "thirdparty/cppjieba/cppjieba-5.6.7"]
+	path = thirdparty/cppjieba/cppjieba-5.6.7
+	url = https://github.com/yanyiwu/cppjieba.git
+[submodule "thirdparty/FastPFOR/FastPFOR-0.4.0"]
+	path = thirdparty/FastPFOR/FastPFOR-0.4.0
+	url = https://github.com/fast-pack/FastPFOR.git
+[submodule "thirdparty/limonp/limonp-v1.0.2"]
+	path = thirdparty/limonp/limonp-v1.0.2
+	url = https://github.com/yanyiwu/limonp.git

python/tests/test_collection_fts.py

@@ -0,0 +1,188 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""End-to-end tests for FTS-only collections (no vector field).
+
+The schema validation rule "must have at least one vector field" has been
+lifted; these tests pin the new behavior so insert / query / delete /
+optimize all work on a vector-less collection.
+"""
+
+from __future__ import annotations
+
+import pytest
+import zvec
+from zvec import (
+    Collection,
+    CollectionOption,
+    DataType,
+    Doc,
+    FieldSchema,
+    FtsIndexParam,
+    OptimizeOption,
+)
+from zvec.model.param.query import Fts, Query
+
+
+# ==================== Fixtures ====================
+
+
+@pytest.fixture(scope="function")
+def fts_collection(tmp_path_factory) -> Collection:
+    """FTS-only collection: a STRING field for forward + an FTS-indexed STRING."""
+    temp_dir = tmp_path_factory.mktemp("zvec_fts_only")
+    collection_path = temp_dir / "fts_collection"
+
+    schema = zvec.CollectionSchema(
+        name="fts_only",
+        fields=[
+            FieldSchema("title", DataType.STRING, nullable=False),
+            FieldSchema(
+                "content",
+                DataType.STRING,
+                nullable=False,
+                index_param=FtsIndexParam(
+                    tokenizer_name="standard",
+                    filters=["lowercase"],
+                ),
+            ),
+        ],
+        # vectors omitted on purpose — schema validation must accept this.
+    )
+
+    coll = zvec.create_and_open(
+        path=str(collection_path),
+        schema=schema,
+        option=CollectionOption(read_only=False, enable_mmap=True),
+    )
+    assert coll is not None
+
+    try:
+        yield coll
+    finally:
+        try:
+            coll.destroy()
+        except Exception as e:
+            print(f"Warning: failed to destroy collection: {e}")
+
+
+def _make_docs() -> list[Doc]:
+    """5-doc corpus where 4 contain 'hello' and doc 4 is the only outlier."""
+    return [
+        Doc(id="pk_0", fields={"title": "intro", "content": "hello world"}),
+        Doc(id="pk_1", fields={"title": "guide", "content": "hello foo bar"}),
+        Doc(id="pk_2", fields={"title": "tips", "content": "hello baz"}),
+        Doc(id="pk_3", fields={"title": "more", "content": "hello hello"}),
+        Doc(id="pk_4", fields={"title": "other", "content": "nothing relevant"}),
+    ]
+
+
+def _fts_query(coll: Collection, term: str) -> list[Doc]:
+    """Run a single-term FTS match query against the `content` field."""
+    return coll.query(
+        queries=Query(field_name="content", fts=Fts(match_string=term)),
+        topk=10,
+    )
+
+
+# ==================== Tests ====================
+
+
+class TestFtsOnlyCollectionSchema:
+    def test_create_and_open_without_vectors(self, fts_collection: Collection):
+        """Schema with zero vector fields must be accepted by validate()."""
+        assert fts_collection.schema.name == "fts_only"
+        assert {f.name for f in fts_collection.schema.fields} == {"title", "content"}
+        # Empty vectors is the whole point of the test.
+        assert list(fts_collection.schema.vectors) == []
+        assert fts_collection.stats.doc_count == 0
+
+    def test_create_schema_omitting_vectors_kwarg(self):
+        """Constructing CollectionSchema without `vectors=` argument is valid."""
+        schema = zvec.CollectionSchema(
+            name="bare_fts",
+            fields=[
+                FieldSchema(
+                    "content",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=FtsIndexParam(),
+                ),
+            ],
+        )
+        assert list(schema.vectors) == []
+        assert {f.name for f in schema.fields} == {"content"}
+
+
+class TestFtsOnlyCollectionLifecycle:
+    def test_insert_and_fts_query(self, fts_collection: Collection):
+        """FTS-only collection supports insert + FTS query end-to-end."""
+        results = fts_collection.insert(_make_docs())
+        assert all(r.ok() for r in results)
+        assert fts_collection.stats.doc_count == 5
+
+        hits = _fts_query(fts_collection, "hello")
+        assert len(hits) == 4
+        assert {doc.id for doc in hits} == {"pk_0", "pk_1", "pk_2", "pk_3"}
+
+        # Term that nothing in the surviving corpus contains.
+        assert _fts_query(fts_collection, "missing_term_xyz") == []
+
+    def test_delete_then_query(self, fts_collection: Collection):
+        """Tombstone filter must drop deleted docs from FTS results."""
+        fts_collection.insert(_make_docs())
+        statuses = fts_collection.delete(["pk_0", "pk_4"])
+        assert all(s.ok() for s in statuses)
+        assert fts_collection.stats.doc_count == 3
+
+        hits = _fts_query(fts_collection, "hello")
+        assert len(hits) == 3
+        assert {doc.id for doc in hits} == {"pk_1", "pk_2", "pk_3"}
+        # pk_4's unique term is filtered out post-delete.
+        assert _fts_query(fts_collection, "nothing") == []
+
+    def test_optimize_rebuilds_fts(self, fts_collection: Collection):
+        """Optimize with >30% deletes triggers ReduceFts; recall unchanged."""
+        fts_collection.insert(_make_docs())
+        # 40% delete ratio — above COMPACT_DELETE_RATIO_THRESHOLD=0.3, so
+        # build_compact_task picks the rebuild path and ReduceFts runs.
+        fts_collection.delete(["pk_0", "pk_4"])
+
+        before = {doc.id for doc in _fts_query(fts_collection, "hello")}
+        assert before == {"pk_1", "pk_2", "pk_3"}
+
+        fts_collection.optimize(option=OptimizeOption())
+        assert fts_collection.stats.doc_count == 3
+
+        after = {doc.id for doc in _fts_query(fts_collection, "hello")}
+        assert after == before
+        assert _fts_query(fts_collection, "nothing") == []
+
+
+class TestFtsOnlyCollectionQueryValidation:
+    def test_vector_query_rejected(self, fts_collection: Collection):
+        """Vector query on a no-vector collection must raise."""
+        with pytest.raises(ValueError, match="vector or id"):
+            fts_collection.query(
+                queries=Query(field_name="content", vector=[0.1, 0.2, 0.3]),
+                topk=5,
+            )
+
+    def test_id_query_rejected(self, fts_collection: Collection):
+        """ID-based query on a no-vector collection must raise."""
+        fts_collection.insert(_make_docs()[:1])
+        with pytest.raises(ValueError, match="vector or id"):
+            fts_collection.query(
+                queries=Query(field_name="content", id="pk_0"),
+                topk=5,
+            )

python/tests/test_fts_query.py

@@ -0,0 +1,158 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for FTS (Full-Text Search) query support in the Python SDK."""
+
+import pickle
+
+import pytest
+
+from zvec.model.param.query import Fts, Query
+
+
+class TestFtsQueryValidation:
+    """Test FTS parameter validation in Query dataclass."""
+
+    def test_fts_query_string_only(self):
+        """Query with only query_string in Fts should be valid."""
+        q = Query(
+            field_name="content", fts=Fts(query_string='+hello -world "exact phrase"')
+        )
+        q._validate()
+        assert q.fts.query_string == '+hello -world "exact phrase"'
+        assert q.fts.match_string is None
+        assert q.has_fts() is True
+
+    def test_fts_match_string_only(self):
+        """Query with only match_string in Fts should be valid."""
+        q = Query(field_name="content", fts=Fts(match_string="machine learning"))
+        q._validate()
+        assert q.fts.match_string == "machine learning"
+        assert q.fts.query_string is None
+        assert q.has_fts() is True
+
+    def test_fts_query_string_and_match_string_mutually_exclusive(self):
+        """Cannot provide both query_string and match_string in Fts."""
+        q = Query(
+            field_name="content",
+            fts=Fts(query_string="+hello", match_string="hello world"),
+        )
+        with pytest.raises(ValueError, match="mutually exclusive"):
+            q._validate()
+
+    def test_no_fts(self):
+        """Query without FTS fields should have has_fts() == False."""
+        q = Query(field_name="embedding", vector=[0.1, 0.2, 0.3])
+        assert q.has_fts() is False
+
+    def test_vector_and_fts_mutually_exclusive(self):
+        """Cannot combine vector search with FTS in a single Query."""
+        q = Query(
+            field_name="embedding",
+            vector=[0.1, 0.2, 0.3],
+            fts=Fts(match_string="deep learning"),
+        )
+        with pytest.raises(ValueError, match="Cannot combine fts with vector search"):
+            q._validate()
+
+    def test_fts_without_vector_or_id(self):
+        """Query with only FTS (no vector, no id) should be valid."""
+        q = Query(field_name="content", fts=Fts(query_string="hello"))
+        q._validate()
+        assert q.has_vector() is False
+        assert q.has_id() is False
+        assert q.has_fts() is True
+
+
+class TestFtsQueryBinding:
+    """Test FTS binding layer (_Fts)."""
+
+    def test_import_fts_query(self):
+        """_Fts should be importable from _zvec.param."""
+        from _zvec.param import _Fts
+
+        fts = _Fts()
+        assert fts.query_string == ""
+        assert fts.match_string == ""
+
+    def test_fts_query_set_fields(self):
+        """Setting fields on _Fts should work."""
+        from _zvec.param import _Fts
+
+        fts = _Fts()
+        fts.query_string = "+hello -world"
+        assert fts.query_string == "+hello -world"
+
+        fts2 = _Fts()
+        fts2.match_string = "machine learning"
+        assert fts2.match_string == "machine learning"
+
+    def test_fts_query_pickle(self):
+        """_Fts should support pickling."""
+        from _zvec.param import _Fts
+
+        fts = _Fts()
+        fts.query_string = "+vector search"
+        fts.match_string = ""
+
+        data = pickle.dumps(fts)
+        restored = pickle.loads(data)
+        assert restored.query_string == "+vector search"
+        assert restored.match_string == ""
+
+    def test_vector_query_fts_field(self):
+        """_VectorQuery should have fts field."""
+        from _zvec.param import _Fts, _VectorQuery
+
+        vq = _VectorQuery()
+        # fts should be None by default (optional)
+        assert vq.fts is None
+
+        # set fts
+        fts = _Fts()
+        fts.query_string = "hello"
+        vq.fts = fts
+        assert vq.fts is not None
+        assert vq.fts.query_string == "hello"
+
+    def test_vector_query_pickle_with_fts(self):
+        """_VectorQuery with fts should survive pickling."""
+        from _zvec.param import _Fts, _VectorQuery
+
+        vq = _VectorQuery()
+        vq.topk = 10
+        vq.field_name = "embedding"
+        fts = _Fts()
+        fts.match_string = "test query"
+        vq.fts = fts
+
+        data = pickle.dumps(vq)
+        restored = pickle.loads(data)
+        assert restored.topk == 10
+        assert restored.field_name == "embedding"
+        assert restored.fts is not None
+        assert restored.fts.match_string == "test query"
+
+    def test_vector_query_pickle_without_fts(self):
+        """_VectorQuery without fts should survive pickling."""
+        from _zvec.param import _VectorQuery
+
+        vq = _VectorQuery()
+        vq.topk = 5
+        vq.field_name = "vec"
+
+        data = pickle.dumps(vq)
+        restored = pickle.loads(data)
+        assert restored.topk == 5
+        assert restored.field_name == "vec"
+        assert restored.fts is None

python/tests/test_query_executor.py

@@ -225,7 +225,9 @@ def test_init(self):
     def test_do_validate_with_queries(self):
         schema = MockCollectionSchema()
         executor = NoVectorQueryExecutor(schema)
-        ctx = QueryContext(topk=10, queries=[Query(field_name="test")])
+        ctx = QueryContext(
+            topk=10, queries=[Query(field_name="test", vector=[0.1, 0.2, 0.3])]
+        )
 
         with pytest.raises(
             ValueError, match="Collection does not support query with vector or id"

python/zvec/__init__.py

@@ -56,11 +56,14 @@
 from .model.doc import Doc
 
 # —— Query & index parameters ——
+# —— FTS params (C++ binding) ——
 from .model.param import (
     AddColumnOption,
     AlterColumnOption,
     CollectionOption,
     FlatIndexParam,
+    FtsIndexParam,
+    FtsQueryParam,
     HnswIndexParam,
     HnswQueryParam,
     HnswRabitqIndexParam,
@@ -73,7 +76,7 @@
     VamanaIndexParam,
     VamanaQueryParam,
 )
-from .model.param.query import Query, VectorQuery
+from .model.param.query import Fts, Query, VectorQuery
 
 # —— Schema & field definitions ——
 from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema