change orchestrator behavior to directly take content to remove path dependency

afterrburn · afterrburn · commit 44eaee9871c4 · 2025-06-16T21:51:32.000-06:00
diff --git a/.github/workflows/@sync-docs.yml b/.github/workflows/@sync-docs.yml
@@ -26,35 +26,52 @@ jobs:
             head_commit=${commit_arr[0]}
           fi
           echo "Comparing $base_commit..$head_commit on main branch"
-          echo "changed=$(git diff --name-only $base_commit $head_commit -- 'content/**/*.mdx' | xargs)" >> $GITHUB_OUTPUT
-          echo "removed=$(git diff --name-only --diff-filter=D $base_commit $head_commit -- 'content/**/*.mdx' | xargs)" >> $GITHUB_OUTPUT
-
-      - name: Print files being sent to agent
-        run: |
-          echo "Changed files: ${{ steps.files.outputs.changed }}"
-          echo "Removed files: ${{ steps.files.outputs.removed }}"
-
-      - name: Trigger Agentuity Sync Agent
-        run: |
-          changed_json=$(jq -n --arg files "${{ steps.files.outputs.changed }}" '$files | split(" ") | map(select(length > 0))')
-          removed_json=$(jq -n --arg files "${{ steps.files.outputs.removed }}" '$files | split(" ") | map(select(length > 0))')
-          echo "Sending to agent API:"
-          echo "Changed files JSON: $changed_json"
-          echo "Removed files JSON: $removed_json"
           
-          # Build the complete JSON payload using jq
+          # Get changed files (relative to content directory)
+          CHANGED_FILES=$(git diff --name-only $base_commit $head_commit -- 'content/**/*.mdx' | sed 's|^content/||')
+          REMOVED_FILES=$(git diff --name-only --diff-filter=D $base_commit $head_commit -- 'content/**/*.mdx' | sed 's|^content/||')
+          
+          echo "Changed files: $CHANGED_FILES"
+          echo "Removed files: $REMOVED_FILES"
+          
+          # Build JSON payload with file contents
           payload=$(jq -n \
-            --arg changed "${{ steps.files.outputs.changed }}" \
-            --arg removed "${{ steps.files.outputs.removed }}" \
-            '{
-              changedFiles: ($changed | split(" ") | map(select(length > 0))),
-              removedFiles: ($removed | split(" ") | map(select(length > 0)))
-            }')
+            --arg commit "$head_commit" \
+            --arg repo "${{ github.repository }}" \
+            --argjson changed "$(
+              if [ -n "$CHANGED_FILES" ]; then
+                for f in $CHANGED_FILES; do
+                  if [ -f "content/$f" ]; then
+                    jq -n \
+                      --arg path "$f" \
+                      --arg content "$(base64 -w0 < "content/$f")" \
+                      '{path: $path, content: $content}'
+                  fi
+                done | jq -s '.'
+              else
+                echo '[]'
+              fi
+            )" \
+            --argjson removed "$(
+              if [ -n "$REMOVED_FILES" ]; then
+                printf '%s\n' $REMOVED_FILES | jq -R -s -c 'split("\n") | map(select(length > 0))'
+              else
+                echo '[]'
+              fi
+            )" \
+            '{commit: $commit, repo: $repo, changed: $changed, removed: $removed}'
+          )
           
-          echo "Complete JSON payload:"
-          echo "$payload"
+          echo "payload<<EOF" >> $GITHUB_OUTPUT
+          echo "$payload" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - name: Trigger Agentuity Sync Agent
+        run: |
+          echo "Sending payload to agent:"
+          echo '${{ steps.files.outputs.payload }}' | jq '.'
           
           curl https://agentuity.ai/webhook/f61d5ce9d6ed85695cc992c55ccdc2a6 \
             -X POST \
             -H "Content-Type: application/json" \
-            -d "$payload" 
+            -d '${{ steps.files.outputs.payload }}' 
diff --git a/.github/workflows/sync-docs.yml b/.github/workflows/sync-docs.yml
@@ -17,19 +17,55 @@ jobs:
         id: files
         run: |
           git fetch origin ${{ github.event.before }}
-          echo "changed=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} -- 'content/**/*.mdx' | xargs)" >> $GITHUB_OUTPUT
-          echo "removed=$(git diff --name-only --diff-filter=D ${{ github.event.before }} ${{ github.sha }} -- 'content/**/*.mdx' | xargs)" >> $GITHUB_OUTPUT
+          
+          # Get changed files (relative to content directory)
+          CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} -- 'content/**/*.mdx' | sed 's|^content/||')
+          REMOVED_FILES=$(git diff --name-only --diff-filter=D ${{ github.event.before }} ${{ github.sha }} -- 'content/**/*.mdx' | sed 's|^content/||')
+          
+          echo "Changed files: $CHANGED_FILES"
+          echo "Removed files: $REMOVED_FILES"
+          
+          # Build JSON payload with file contents
+          payload=$(jq -n \
+            --arg commit "${{ github.sha }}" \
+            --arg repo "${{ github.repository }}" \
+            --argjson changed "$(
+              if [ -n "$CHANGED_FILES" ]; then
+                for f in $CHANGED_FILES; do
+                  if [ -f "content/$f" ]; then
+                    jq -n \
+                      --arg path "$f" \
+                      --arg content "$(base64 -w0 < "content/$f")" \
+                      '{path: $path, content: $content}'
+                  fi
+                done | jq -s '.'
+              else
+                echo '[]'
+              fi
+            )" \
+            --argjson removed "$(
+              if [ -n "$REMOVED_FILES" ]; then
+                printf '%s\n' $REMOVED_FILES | jq -R -s -c 'split("\n") | map(select(length > 0))'
+              else
+                echo '[]'
+              fi
+            )" \
+            '{commit: $commit, repo: $repo, changed: $changed, removed: $removed}'
+          )
+          
+          echo "payload<<EOF" >> $GITHUB_OUTPUT
+          echo "$payload" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Trigger Agentuity Sync Agent
         env:
           AGENTUITY_TOKEN: ${{ secrets.AGENTUITY_TOKEN }}
         run: |
-          # Convert space-separated lists to JSON arrays
-          changed_json=$(jq -n --arg files "${{ steps.files.outputs.changed }}" '$files | split(" ") | map(select(length > 0))')
-          removed_json=$(jq -n --arg files "${{ steps.files.outputs.removed }}" '$files | split(" ") | map(select(length > 0))')
-          # POST to Agentuity agent webhook
+          echo "Sending payload to agent:"
+          echo '${{ steps.files.outputs.payload }}' | jq '.'
+          
           curl https://agentuity.ai/webhook/f61d5ce9d6ed85695cc992c55ccdc2a6 \
             -X POST \
             -H "Authorization: Bearer $AGENTUITY_TOKEN" \
             -H "Content-Type: application/json" \
-            -d '{"changedFiles":'$changed_json', "removedFiles":'$removed_json'}'
+            -d '${{ steps.files.outputs.payload }}'
diff --git a/agent-docs/src/agents/doc-processing/docs-orchestrator.ts b/agent-docs/src/agents/doc-processing/docs-orchestrator.ts
@@ -1,105 +1,102 @@
-// Node.js built-in modules
-import { promises as fs } from 'fs';
-import * as path from 'path';
-
 import type { AgentContext } from '@agentuity/sdk';
-
 import { processDoc } from './docs-processor';
 import { VECTOR_STORE_NAME } from './config';
 
-export const DEFAULT_CONTENT_DIR = path.resolve(__dirname, '../../../../content');
-
-/**
- * Checks if a document has changed by comparing its hash.
- * @param docPath Absolute path to the .mdx file
- * @param newHash Newly computed hash
- * @returns true if changed, false if unchanged
- */
-export async function hasDocChanged(ctx: AgentContext, docPath: string, newHash: string): Promise<boolean> {
-    const vectors = await ctx.vector.search(VECTOR_STORE_NAME, {
-        query: ' ',
-        limit: 1,
-        metadata: { path: docPath },
-    });
-    if (vectors.length > 0) {
-        const storedHash = vectors[0]?.metadata?.hash;
-        return storedHash !== newHash;
-    }
-    return true;
+interface FilePayload {
+  path: string;
+  content: string; // base64-encoded
 }
 
-interface OrchestratorOptions {
-    changedFiles: string[]; // Absolute file paths
-    removedFiles: string[]; // Absolute file paths
-    contentDir?: string;    // Defaults to /content
+interface SyncPayload {
+  commit?: string;
+  repo?: string;
+  changed: FilePayload[];
+  removed: string[];
 }
 
 interface SyncStats {
-    processed: number;
-    deleted: number;
-    errors: number;
-    errorFiles: string[];
+  processed: number;
+  deleted: number;
+  errors: number;
+  errorFiles: string[];
 }
 
 /**
- * Helper to remove all vectors for a given relative path from the vector store.
+ * Helper to remove all vectors for a given logical path from the vector store.
  */
-async function removeVectorsByRelativePath(ctx: AgentContext, relativePath: string, vectorStoreName: string) {
-    ctx.logger.info('Removing vectors for path: %s, store: %s, metadata: %o', relativePath, vectorStoreName, { path: relativePath });
-    const vectors = await ctx.vector.search(vectorStoreName, {
-        query: ' ',
-        limit: 1000,
-        metadata: { path: relativePath },
-    });
-    if (vectors.length > 0) {
-        // Delete vectors one by one to avoid issues with large batches
-        for (const vector of vectors) {
-            await ctx.vector.delete(vectorStoreName, vector.key);
-        }
-    } else {
-        ctx.logger.info('No vectors found for path: %s, store: %s, metadata: %o', relativePath, vectorStoreName, { path: relativePath });
+async function removeVectorsByPath(ctx: AgentContext, logicalPath: string, vectorStoreName: string) {
+  ctx.logger.info('Removing vectors for path: %s', logicalPath);
+  const vectors = await ctx.vector.search(vectorStoreName, {
+    query: ' ',
+    limit: 1000,
+    metadata: { path: logicalPath },
+  });
+  
+  if (vectors.length > 0) {
+    // Delete vectors one by one to avoid issues with large batches
+    for (const vector of vectors) {
+      await ctx.vector.delete(vectorStoreName, vector.key);
     }
+    ctx.logger.info('Removed %d vectors for path: %s', vectors.length, logicalPath);
+  } else {
+    ctx.logger.info('No vectors found for path: %s', logicalPath);
+  }
 }
 
-export async function syncDocs(ctx: AgentContext, options: OrchestratorOptions): Promise<SyncStats> {
-    const { changedFiles, removedFiles } = options;
-    const contentDir = options.contentDir || DEFAULT_CONTENT_DIR;
-    let processed = 0, deleted = 0, errors = 0;
-    const errorFiles: string[] = [];
+/**
+ * Process documentation sync from embedded payload - completely filesystem-free
+ */
+export async function syncDocsFromPayload(ctx: AgentContext, payload: SyncPayload): Promise<SyncStats> {
+  const { changed, removed } = payload;
+  let processed = 0, deleted = 0, errors = 0;
+  const errorFiles: string[] = [];
 
-    for (const absolutePath of changedFiles) {
-        try {
-            // Compute relative path for metadata and vector store
-            const relativePath = path.relative(contentDir, absolutePath);
-            await removeVectorsByRelativePath(ctx, relativePath, VECTOR_STORE_NAME);
-            const content = await fs.readFile(absolutePath, 'utf-8');
-            const chunks = await processDoc(content);
-            for (const chunk of chunks) {
-                chunk.metadata = {
-                    ...chunk.metadata,
-                    path: relativePath,
-                };
-                await ctx.vector.upsert(VECTOR_STORE_NAME, chunk);
-            }
-            processed++;
-        } catch (err) {
-            errors++;
-            errorFiles.push(absolutePath);
-            ctx.logger.error('Error processing file %s: %o', absolutePath, err);
-        }
+  // Process changed files with embedded content
+  for (const file of changed) {
+    try {
+      const { path: logicalPath, content: base64Content } = file;
+      
+      // Base64-decode the content
+      const content = Buffer.from(base64Content, 'base64').toString('utf-8');
+      
+      // Remove existing vectors for this path
+      await removeVectorsByPath(ctx, logicalPath, VECTOR_STORE_NAME);
+      
+      // Process the document content into chunks
+      const chunks = await processDoc(content);
+      
+      // Upsert chunks with path metadata
+      for (const chunk of chunks) {
+        chunk.metadata = {
+          ...chunk.metadata,
+          path: logicalPath,
+        };
+        await ctx.vector.upsert(VECTOR_STORE_NAME, chunk);
+      }
+      
+      processed++;
+      ctx.logger.info('Successfully processed file: %s (%d chunks)', logicalPath, chunks.length);
+    } catch (err) {
+      errors++;
+      errorFiles.push(file.path);
+      ctx.logger.error('Error processing file %s: %o', file.path, err);
     }
+  }
 
-    for (const absolutePath of removedFiles) {
-        try {
-            const relativePath = path.relative(contentDir, absolutePath);
-            await removeVectorsByRelativePath(ctx, relativePath, VECTOR_STORE_NAME);
-            deleted++;
-        } catch (err) {
-            errors++;
-            errorFiles.push(absolutePath);
-            ctx.logger.error('Error deleting file %s: %o', absolutePath, err);
-        }
+  // Process removed files
+  for (const logicalPath of removed) {
+    try {
+      await removeVectorsByPath(ctx, logicalPath, VECTOR_STORE_NAME);
+      deleted++;
+      ctx.logger.info('Successfully removed file: %s', logicalPath);
+    } catch (err) {
+      errors++;
+      errorFiles.push(logicalPath);
+      ctx.logger.error('Error deleting file %s: %o', logicalPath, err);
     }
+  }
 
-    return { processed, deleted, errors, errorFiles };
+  const stats = { processed, deleted, errors, errorFiles };
+  ctx.logger.info('Sync completed: %o', stats);
+  return stats;
 } 
diff --git a/agent-docs/src/agents/doc-processing/index.ts b/agent-docs/src/agents/doc-processing/index.ts
