diff --git a/JS/edgechains/arakoodev/README.md b/JS/edgechains/arakoodev/README.md index 81237e662..9395f36a7 100644 --- a/JS/edgechains/arakoodev/README.md +++ b/JS/edgechains/arakoodev/README.md @@ -3,3 +3,45 @@ Installation ``` npm install arakoodev ``` + +## Qdrant vector database + +The vector-db package includes a Qdrant REST wrapper. It uses Qdrant's HTTP API directly and does +not require a Qdrant client package. + +```ts +import { Qdrant } from "@arakoodev/edgechains.js/vector-db"; + +const qdrant = new Qdrant({ + url: process.env.QDRANT_URL, + apiKey: process.env.QDRANT_API_KEY, +}); + +await qdrant.createCollection({ + collectionName: "documents", + vectors: { size: 1536, distance: "Cosine" }, +}); + +await qdrant.upsertPoints({ + collectionName: "documents", + points: [ + { + id: 1, + vector: [0.1, 0.2, 0.3], + payload: { + raw_text: "Document text", + namespace: "docs", + filename: "example.pdf", + }, + }, + ], +}); + +const results = await qdrant.search({ + collectionName: "documents", + vector: [0.1, 0.2, 0.3], + limit: 5, + filter: { must: [{ key: "namespace", match: { value: "docs" } }] }, + withPayload: true, +}); +``` diff --git a/JS/edgechains/arakoodev/src/vector-db/src/index.ts b/JS/edgechains/arakoodev/src/vector-db/src/index.ts index 557104a14..6d1a39fb3 100644 --- a/JS/edgechains/arakoodev/src/vector-db/src/index.ts +++ b/JS/edgechains/arakoodev/src/vector-db/src/index.ts @@ -1 +1,14 @@ export { Supabase } from "./lib/supabase/supabase.js"; +export { Qdrant } from "./lib/qdrant/qdrant.js"; +export type { + QdrantConstructorOptions, + QdrantCreateCollectionArgs, + QdrantDeletePointsArgs, + QdrantDistance, + QdrantGetPointArgs, + QdrantPoint, + QdrantSearchArgs, + QdrantUpdatePayloadArgs, + QdrantUpsertPointsArgs, + QdrantVectorConfig, +} from "./lib/qdrant/qdrant.js"; diff --git a/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts b/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts new file mode 100644 index 000000000..75bd090eb --- /dev/null +++ b/JS/edgechains/arakoodev/src/vector-db/src/lib/qdrant/qdrant.ts @@ -0,0 +1,303 @@ +import axios, { AxiosInstance, AxiosRequestConfig } from "axios"; +import retry from "retry"; +import { config } from "dotenv"; +config(); + +type QdrantDistance = "Cosine" | "Dot" | "Euclid" | "Manhattan"; +type QdrantPointId = string | number; +type QdrantSearchVector = + | number[] + | { name: string; vector: number[] } + | Record; + +interface QdrantConstructorOptions { + url?: string; + apiKey?: string; + timeout?: number; +} + +interface QdrantVectorConfig { + size: number; + distance?: QdrantDistance; +} + +interface QdrantCreateCollectionArgs { + collectionName: string; + vectors: QdrantVectorConfig | Record; +} + +interface QdrantPoint { + id: QdrantPointId; + vector: number[] | Record; + payload?: Record; +} + +interface QdrantUpsertPointsArgs { + collectionName: string; + points: QdrantPoint[]; + wait?: boolean; +} + +interface QdrantSearchArgs { + collectionName: string; + vector: QdrantSearchVector; + limit?: number; + filter?: Record; + withPayload?: boolean | string[]; + withVector?: boolean | string[]; + scoreThreshold?: number; + using?: string; +} + +interface QdrantGetPointArgs { + collectionName: string; + id: QdrantPointId; + withPayload?: boolean | string[]; + withVector?: boolean | string[]; +} + +interface QdrantDeletePointsArgs { + collectionName: string; + ids: QdrantPointId[]; + wait?: boolean; +} + +interface QdrantUpdatePayloadArgs { + collectionName: string; + ids: QdrantPointId[]; + payload: Record; + wait?: boolean; +} + +export class Qdrant { + QDRANT_URL: string; + QDRANT_API_KEY: string; + private client: AxiosInstance; + + constructor(options: QdrantConstructorOptions = {}) { + this.QDRANT_URL = (options.url || process.env.QDRANT_URL || "").replace( + /\/$/, + "", + ); + this.QDRANT_API_KEY = options.apiKey || process.env.QDRANT_API_KEY || ""; + this.client = axios.create({ + baseURL: this.QDRANT_URL, + timeout: options.timeout || 30000, + headers: this.buildHeaders(), + }); + } + + createClient(): AxiosInstance { + return this.client; + } + + async createCollection({ + collectionName, + vectors, + }: QdrantCreateCollectionArgs): Promise { + return this.request({ + method: "put", + url: `/collections/${collectionName}`, + data: { vectors }, + }); + } + + async upsertPoints({ + collectionName, + points, + wait = true, + }: QdrantUpsertPointsArgs): Promise { + return this.request({ + method: "put", + url: `/collections/${collectionName}/points`, + params: { wait }, + data: { points }, + }); + } + + async search({ + collectionName, + vector, + limit = 10, + filter, + withPayload = true, + withVector = false, + scoreThreshold, + using, + }: QdrantSearchArgs): Promise { + const data: Record = { + vector: this.formatSearchVector(vector, using), + limit, + filter, + with_payload: withPayload, + with_vector: withVector, + score_threshold: scoreThreshold, + }; + + return this.request({ + method: "post", + url: `/collections/${collectionName}/points/search`, + data: this.removeUndefined(data), + }); + } + + async getPoint({ + collectionName, + id, + withPayload = true, + withVector = false, + }: QdrantGetPointArgs): Promise { + const response = await this.request({ + method: "post", + url: `/collections/${collectionName}/points`, + data: { + ids: [id], + with_payload: withPayload, + with_vector: withVector, + }, + }); + + if (Array.isArray(response?.result)) { + return { + ...response, + result: response.result[0] || null, + }; + } + + return response; + } + + async updatePayload({ + collectionName, + ids, + payload, + wait = true, + }: QdrantUpdatePayloadArgs): Promise { + return this.request({ + method: "post", + url: `/collections/${collectionName}/points/payload`, + params: { wait }, + data: { + payload, + points: ids, + }, + }); + } + + async deletePoints({ + collectionName, + ids, + wait = true, + }: QdrantDeletePointsArgs): Promise { + return this.request({ + method: "post", + url: `/collections/${collectionName}/points/delete`, + params: { wait }, + data: { + points: ids, + }, + }); + } + + private buildHeaders(): Record { + const headers: Record = { + "content-type": "application/json", + }; + + if (this.QDRANT_API_KEY) { + headers["api-key"] = this.QDRANT_API_KEY; + } + + return headers; + } + + private async request(requestConfig: AxiosRequestConfig): Promise { + return new Promise((resolve, reject) => { + const operation = retry.operation({ + retries: 5, + factor: 3, + minTimeout: 1000, + maxTimeout: 60000, + randomize: true, + }); + + operation.attempt(async () => { + try { + const response = await this.client.request(requestConfig); + resolve(response.data); + } catch (error: any) { + if (operation.retry(error)) return; + + const status = error.response?.status; + const message = error.response?.data?.status?.error || error.message; + reject( + new Error( + `Qdrant request failed${status ? ` with status ${status}` : ""}: ${message}`, + ), + ); + } + }); + }); + } + + private removeUndefined(data: Record): Record { + return Object.fromEntries( + Object.entries(data).filter(([, value]) => value !== undefined), + ); + } + + private formatSearchVector( + vector: QdrantSearchVector, + using?: string, + ): number[] | { name: string; vector: number[] } { + if (Array.isArray(vector)) { + return using ? { name: using, vector } : vector; + } + + if (this.isNamedSearchVector(vector)) { + return vector; + } + + const vectorMap = vector as Record; + if (using) { + const namedVector = vectorMap[using]; + if (Array.isArray(namedVector)) { + return { name: using, vector: namedVector }; + } + + throw new Error(`Search vector map does not include "${using}"`); + } + + const entries = Object.entries(vectorMap).filter(([, value]) => + Array.isArray(value), + ) as [string, number[]][]; + if (entries.length === 1) { + const [name, namedVector] = entries[0]; + return { name, vector: namedVector }; + } + + throw new Error( + "Named-vector search requires a single vector name or a using value", + ); + } + + private isNamedSearchVector( + vector: Exclude, + ): vector is { name: string; vector: number[] } { + return typeof vector.name === "string" && Array.isArray(vector.vector); + } +} + +export type { + QdrantConstructorOptions, + QdrantCreateCollectionArgs, + QdrantDeletePointsArgs, + QdrantDistance, + QdrantGetPointArgs, + QdrantPoint, + QdrantSearchArgs, + QdrantSearchVector, + QdrantUpdatePayloadArgs, + QdrantUpsertPointsArgs, + QdrantVectorConfig, +}; diff --git a/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts b/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts new file mode 100644 index 000000000..3efa4dc8a --- /dev/null +++ b/JS/edgechains/arakoodev/src/vector-db/src/tests/qdrant/qdrant.test.ts @@ -0,0 +1,194 @@ +import axios from "axios"; +import { Qdrant } from "../../../../../dist/vector-db/src/lib/qdrant/qdrant.js"; + +jest.mock("axios"); + +const mockedAxios = axios as jest.Mocked; +const requestMock = jest.fn(); + +describe("Qdrant", () => { + beforeEach(() => { + requestMock.mockResolvedValue({ data: { result: "ok" } }); + mockedAxios.create.mockReturnValue({ request: requestMock } as any); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it("creates an axios client with api-key auth when provided", () => { + const qdrant = new Qdrant({ + url: "https://qdrant.example.com/", + apiKey: "test-key", + timeout: 5000, + }); + + expect(qdrant.createClient()).toEqual({ request: requestMock }); + expect(mockedAxios.create).toHaveBeenCalledWith({ + baseURL: "https://qdrant.example.com", + timeout: 5000, + headers: { + "content-type": "application/json", + "api-key": "test-key", + }, + }); + }); + + it("creates a collection with vector configuration", async () => { + const qdrant = new Qdrant({ url: "https://qdrant.example.com" }); + + await qdrant.createCollection({ + collectionName: "documents", + vectors: { size: 1536, distance: "Cosine" }, + }); + + expect(requestMock).toHaveBeenCalledWith({ + method: "put", + url: "/collections/documents", + data: { + vectors: { size: 1536, distance: "Cosine" }, + }, + }); + }); + + it("upserts points into a collection", async () => { + const qdrant = new Qdrant({ url: "https://qdrant.example.com" }); + + await qdrant.upsertPoints({ + collectionName: "documents", + points: [ + { + id: 1, + vector: [0.1, 0.2, 0.3], + payload: { raw_text: "hello" }, + }, + ], + }); + + expect(requestMock).toHaveBeenCalledWith({ + method: "put", + url: "/collections/documents/points", + params: { wait: true }, + data: { + points: [ + { + id: 1, + vector: [0.1, 0.2, 0.3], + payload: { raw_text: "hello" }, + }, + ], + }, + }); + }); + + it("searches points with vector and payload options", async () => { + const qdrant = new Qdrant({ url: "https://qdrant.example.com" }); + + await qdrant.search({ + collectionName: "documents", + vector: [0.1, 0.2, 0.3], + limit: 3, + filter: { must: [{ key: "namespace", match: { value: "docs" } }] }, + withPayload: ["raw_text", "filename"], + scoreThreshold: 0.7, + }); + + expect(requestMock).toHaveBeenCalledWith({ + method: "post", + url: "/collections/documents/points/search", + data: { + vector: [0.1, 0.2, 0.3], + limit: 3, + filter: { must: [{ key: "namespace", match: { value: "docs" } }] }, + with_payload: ["raw_text", "filename"], + with_vector: false, + score_threshold: 0.7, + }, + }); + }); + + it("formats named vector searches with a using value", async () => { + const qdrant = new Qdrant({ url: "https://qdrant.example.com" }); + + await qdrant.search({ + collectionName: "documents", + vector: [0.1, 0.2, 0.3], + using: "image-embeddings", + }); + + expect(requestMock).toHaveBeenCalledWith({ + method: "post", + url: "/collections/documents/points/search", + data: { + vector: { name: "image-embeddings", vector: [0.1, 0.2, 0.3] }, + limit: 10, + with_payload: true, + with_vector: false, + }, + }); + }); + + it("formats single-entry vector maps as named vector searches", async () => { + const qdrant = new Qdrant({ url: "https://qdrant.example.com" }); + + await qdrant.search({ + collectionName: "documents", + vector: { "text-embeddings": [0.1, 0.2, 0.3] }, + }); + + expect(requestMock).toHaveBeenCalledWith({ + method: "post", + url: "/collections/documents/points/search", + data: { + vector: { name: "text-embeddings", vector: [0.1, 0.2, 0.3] }, + limit: 10, + with_payload: true, + with_vector: false, + }, + }); + }); + + it("gets, updates, and deletes points", async () => { + const qdrant = new Qdrant({ url: "https://qdrant.example.com" }); + + await qdrant.getPoint({ + collectionName: "documents", + id: 1, + withPayload: false, + withVector: true, + }); + await qdrant.updatePayload({ + collectionName: "documents", + ids: [1], + payload: { filename: "example.pdf" }, + }); + await qdrant.deletePoints({ collectionName: "documents", ids: [1] }); + + expect(requestMock).toHaveBeenNthCalledWith(1, { + method: "post", + url: "/collections/documents/points", + data: { + ids: [1], + with_payload: false, + with_vector: true, + }, + }); + expect(requestMock).toHaveBeenNthCalledWith(2, { + method: "post", + url: "/collections/documents/points/payload", + params: { wait: true }, + data: { + payload: { filename: "example.pdf" }, + points: [1], + }, + }); + expect(requestMock).toHaveBeenNthCalledWith(3, { + method: "post", + url: "/collections/documents/points/delete", + params: { wait: true }, + data: { + points: [1], + }, + }); + }); +});