Skip to content

Updating ingest terraform module to use awscc provider for s3 vectors resources #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
donald-quixote wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Updating ingest terraform module to use awscc provider for s3 vectors resources #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 21 additions & 46 deletions terraform/3_ingestion/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ terraform {
source = "hashicorp/aws"
version = "~> 5.0"
}
awscc = {
source = "hashicorp/awscc"
version = "~> 1.0"
}
}

# Using local backend - state will be stored in terraform.tfstate in this directory
Expand All @@ -16,47 +20,31 @@ provider "aws" {
region = var.aws_region
}

provider "awscc" {
region = var.aws_region
}

# Data source for current caller identity
data "aws_caller_identity" "current" {}

# ========================================
# S3 Vectors Bucket
# ========================================

resource "aws_s3_bucket" "vectors" {
bucket = "alex-vectors-${data.aws_caller_identity.current.account_id}"

tags = {
Project = "alex"
Part = "3"
}
}

resource "aws_s3_bucket_versioning" "vectors" {
bucket = aws_s3_bucket.vectors.id

versioning_configuration {
status = "Enabled"
}
# S3 Vectors Bucket
resource "awscc_s3vectors_vector_bucket" "vectors" {
vector_bucket_name = "alex-vectors-${data.aws_caller_identity.current.account_id}"
}

resource "aws_s3_bucket_server_side_encryption_configuration" "vectors" {
bucket = aws_s3_bucket.vectors.id

rule {
apply_server_side_encryption_by_default {
sse_algorithm = "AES256"
}
}
}
# S3 Vectors Index
resource "awscc_s3vectors_index" "index" {
index_name = "financial-research"
vector_bucket_name = awscc_s3vectors_vector_bucket.vectors.vector_bucket_name

resource "aws_s3_bucket_public_access_block" "vectors" {
bucket = aws_s3_bucket.vectors.id

block_public_acls = true
block_public_policy = true
ignore_public_acls = true
restrict_public_buckets = true
# Required attributes
data_type = "float32"
dimension = 384
distance_metric = "cosine"
}

# ========================================
Expand Down Expand Up @@ -103,19 +91,6 @@ resource "aws_iam_role_policy" "lambda_policy" {
]
Resource = "arn:aws:logs:${var.aws_region}:${data.aws_caller_identity.current.account_id}:*"
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject",
"s3:ListBucket"
]
Resource = [
aws_s3_bucket.vectors.arn,
"${aws_s3_bucket.vectors.arn}/*"
]
},
{
Effect = "Allow"
Action = [
Expand All @@ -131,7 +106,7 @@ resource "aws_iam_role_policy" "lambda_policy" {
"s3vectors:GetVectors",
"s3vectors:DeleteVectors"
]
Resource = "arn:aws:s3vectors:${var.aws_region}:${data.aws_caller_identity.current.account_id}:bucket/${aws_s3_bucket.vectors.id}/index/*"
Resource = "arn:aws:s3vectors:${var.aws_region}:${data.aws_caller_identity.current.account_id}:bucket/${awscc_s3vectors_vector_bucket.vectors.vector_bucket_name}/index/*"
}
]
})
Expand All @@ -153,7 +128,7 @@ resource "aws_lambda_function" "ingest" {

environment {
variables = {
VECTOR_BUCKET = aws_s3_bucket.vectors.id
VECTOR_BUCKET = awscc_s3vectors_vector_bucket.vectors.vector_bucket_name
SAGEMAKER_ENDPOINT = var.sagemaker_endpoint_name
}
}
Expand Down
4 changes: 2 additions & 2 deletions terraform/3_ingestion/outputs.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
output "vector_bucket_name" {
description = "Name of the S3 Vectors bucket"
value = aws_s3_bucket.vectors.id
value = awscc_s3vectors_vector_bucket.vectors.vector_bucket_name
}

output "api_endpoint" {
Expand All @@ -26,7 +26,7 @@ output "setup_instructions" {
✅ Ingestion pipeline deployed successfully!

Add the following to your .env file:
VECTOR_BUCKET=${aws_s3_bucket.vectors.id}
VECTOR_BUCKET=${awscc_s3vectors_vector_bucket.vectors.vector_bucket_name}
ALEX_API_ENDPOINT=${aws_api_gateway_stage.api.invoke_url}/ingest

To get your API key value:
Expand Down