diff --git a/codebundles/gcp-bigquery-job-health/.runwhen/generation-rules/gcp-bigquery-job-health.yaml b/codebundles/gcp-bigquery-job-health/.runwhen/generation-rules/gcp-bigquery-job-health.yaml new file mode 100644 index 00000000..8427207f --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.runwhen/generation-rules/gcp-bigquery-job-health.yaml @@ -0,0 +1,22 @@ +apiVersion: runwhen.com/v1 +kind: GenerationRules +spec: + platform: gcp + generationRules: + - resourceTypes: + - project + matchRules: + - type: pattern + pattern: ".+" + properties: [name] + mode: substring + slxs: + - baseName: gcp-bigquery-job-health + qualifiers: ["project"] + baseTemplateName: gcp-bigquery-job-health + levelOfDetail: basic + outputItems: + - type: slx + - type: sli + - type: runbook + templateName: gcp-bigquery-job-health-taskset.yaml \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-sli.yaml b/codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-sli.yaml new file mode 100644 index 00000000..e15eff3c --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-sli.yaml @@ -0,0 +1,50 @@ +apiVersion: runwhen.com/v1 +kind: ServiceLevelIndicator +metadata: + name: {{slx_name}} + labels: + {% include "common-labels.yaml" %} + annotations: + {% include "common-annotations.yaml" %} +spec: + displayUnitsLong: Health Score + displayUnitsShort: score + locations: + - {{default_location}} + description: Measures BigQuery job health by scoring success rate, slow jobs, and slot contention for project {{project.name}}. + codeBundle: + {% if repo_url %} + repoUrl: {{repo_url}} + {% else %} + repoUrl: https://github.com/runwhen-contrib/rw-cli-codecollection.git + {% endif %} + {% if ref %} + ref: {{ref}} + {% else %} + ref: main + {% endif %} + pathToRobot: codebundles/gcp-bigquery-job-health/sli.robot + intervalStrategy: intermezzo + intervalSeconds: 300 + configProvided: + - name: GCP_PROJECT_ID + value: "{{project.name}}" + - name: JOB_LOOKBACK_HOURS + value: "24" + - name: SUCCESS_RATE_THRESHOLD + value: "95" + - name: SLOW_JOB_DURATION_MINUTES + value: "30" + - name: SLOT_CONTENTION_THRESHOLD + value: "80" + secretsProvided: + {% if wb_version %} + {% include "gcp-auth.yaml" ignore missing %} + {% else %} + - name: gcp_credentials + workspaceKey: AUTH DETAILS NOT FOUND + {% endif %} + alertConfig: + tasks: + persona: eager-edgar + sessionTTL: 10m \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-slx.yaml b/codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-slx.yaml new file mode 100644 index 00000000..c1fc2563 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-slx.yaml @@ -0,0 +1,29 @@ +apiVersion: runwhen.com/v1 +kind: ServiceLevelX +metadata: + name: {{slx_name}} + labels: + {% include "common-labels.yaml" %} + annotations: + {% include "common-annotations.yaml" %} +spec: + imageURL: https://storage.googleapis.com/runwhen-nonprod-shared-images/icons/gcp/bigquery/bigquery.svg + alias: "{{match_resource.resource.name}} GCP BigQuery Job Health" + asMeasuredBy: A combined score based on job success rate, slow job count, and slot contention. A score of 1 is healthy, less than 1 indicates issues with BigQuery job execution. + configProvided: + - name: GCP_PROJECT_ID + value: "{{project.name}}" + owners: + - {{workspace.owner_email}} + statement: BigQuery jobs should complete successfully with acceptable performance and without resource contention. + additionalContext: + {% include "gcp-hierarchy.yaml" ignore missing %} + qualified_name: "{{ match_resource.qualified_name }}" + tags: + {% include "gcp-tags.yaml" ignore missing %} + - name: service + value: bigquery + - name: category + value: analytics + - name: access + value: read-only \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-taskset.yaml b/codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-taskset.yaml new file mode 100644 index 00000000..17b54782 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-taskset.yaml @@ -0,0 +1,41 @@ +apiVersion: runwhen.com/v1 +kind: Runbook +metadata: + name: {{slx_name}} + labels: + {% include "common-labels.yaml" %} + annotations: + {% include "common-annotations.yaml" %} +spec: + location: {{default_location}} + description: Runs tasks to validate GCP BigQuery job health, including success rate, error patterns, slow jobs, and slot contention for project {{project.name}}. + codeBundle: + {% if repo_url %} + repoUrl: {{repo_url}} + {% else %} + repoUrl: https://github.com/runwhen-contrib/rw-cli-codecollection.git + {% endif %} + {% if ref %} + ref: {{ref}} + {% else %} + ref: main + {% endif %} + pathToRobot: codebundles/gcp-bigquery-job-health/runbook.robot + configProvided: + - name: GCP_PROJECT_ID + value: "{{project.name}}" + - name: JOB_LOOKBACK_HOURS + value: "24" + - name: SUCCESS_RATE_THRESHOLD + value: "95" + - name: SLOW_JOB_DURATION_MINUTES + value: "30" + - name: SLOT_CONTENTION_THRESHOLD + value: "80" + secretsProvided: + {% if wb_version %} + {% include "gcp-auth.yaml" ignore missing %} + {% else %} + - name: gcp_credentials + workspaceKey: AUTH DETAILS NOT FOUND + {% endif %} \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.test/README.md b/codebundles/gcp-bigquery-job-health/.test/README.md new file mode 100644 index 00000000..06d1f302 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.test/README.md @@ -0,0 +1,43 @@ +# Test Infrastructure for gcp-bigquery-job-health + +This directory contains test infrastructure for the GCP BigQuery Job Health CodeBundle. + +## Test Scenarios + +### healthy_project +A GCP project with no failed jobs, fast execution, and sufficient slot capacity. +- Expected issues: 0 + +### failed_jobs_detected +A GCP project with failed jobs due to quota exceedance and invalid queries. +- Expected issues: 3 +- Expected severities: [2, 2, 3] + +### slot_contention +A GCP project with high slot utilization and slow queued jobs. +- Expected issues: 2 +- Expected severities: [2, 3] + +## Running Tests + +```bash +# Build infrastructure +task build-infra + +# Run full test suite +task default + +# Clean up +task clean +``` + +## Terraform + +The Terraform configuration creates: +- A GCP project with BigQuery datasets and tables +- Service account with appropriate roles +- Test queries to simulate job patterns + +Set the following variables in `terraform.tfvars`: +- `gcp_project_id`: Target project ID +- `service_account_key`: Path to service account JSON key \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.test/Taskfile.yaml b/codebundles/gcp-bigquery-job-health/.test/Taskfile.yaml new file mode 100644 index 00000000..d949dcc3 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.test/Taskfile.yaml @@ -0,0 +1,77 @@ +version: "3" + +tasks: + default: + desc: "Run complete test suite" + cmds: + - task: check-unpushed-commits + - task: generate-rwl-config + + clean: + desc: "Run cleanup tasks" + cmds: + - task: check-and-cleanup-terraform + - task: delete-slxs + - task: clean-rwl-discovery + + build-infra: + desc: "Build test infrastructure" + cmds: + - task: build-terraform-infra + + check-unpushed-commits: + desc: Check for uncommitted/unpushed changes + vars: + BASE_DIR: "../" + cmds: + - | + UNCOMMITTED=$(git diff --name-only HEAD | grep -E "^${BASE_DIR}" | grep -v "/\.test/" || true) + if [ -n "$UNCOMMITTED" ]; then + echo "Uncommitted changes found. Commit and push before testing." + exit 1 + fi + silent: true + + build-terraform-infra: + desc: "Build Terraform test resources" + dir: terraform + cmds: + - | + source tf.secret + terraform init + terraform plan -out=tfplan + terraform apply tfplan + + check-and-cleanup-terraform: + desc: "Destroy Terraform resources" + dir: terraform + cmds: + - | + if [ -f "terraform.tfstate" ]; then + source tf.secret + terraform destroy -auto-approve + fi + + generate-rwl-config: + desc: "Generate RWL config for testing" + cmds: + - | + echo "Generating RWL config..." + mkdir -p .rwl-config + cat > .rwl-config/config.yaml << 'EOF' + gcp_project_id: "${GCP_PROJECT_ID}" + job_lookback_hours: 24 + success_rate_threshold: 95 + slow_job_duration_minutes: 30 + slot_contention_threshold: 80 + EOF + + delete-slxs: + desc: "Delete SLX configurations" + cmds: + - rm -rf .rwl-config + + clean-rwl-discovery: + desc: "Clean RWL discovery artifacts" + cmds: + - rm -rf .rwl-discovery \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.test/terraform/main.tf b/codebundles/gcp-bigquery-job-health/.test/terraform/main.tf new file mode 100644 index 00000000..e880d6d7 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.test/terraform/main.tf @@ -0,0 +1,25 @@ +provider "google" { + credentials = file(var.service_account_key) + project = var.gcp_project_id + region = "us-central1" +} + +resource "google_bigquery_dataset" "test_dataset" { + dataset_id = "bq_job_health_test" + description = "Test dataset for BigQuery job health monitoring" + location = "US" +} + +resource "google_bigquery_table" "test_table" { + dataset_id = google_bigquery_dataset.test_dataset.dataset_id + table_id = "test_data" + + schema = jsonencode([ + { name = "id", type = "INTEGER", mode = "REQUIRED" }, + { name = "name", type = "STRING", mode = "NULLABLE" }, + { name = "value", type = "FLOAT", mode = "NULLABLE" }, + { name = "timestamp", type = "TIMESTAMP", mode = "NULLABLE" }, + ]) + + depends_on = [google_bigquery_dataset.test_dataset] +} \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.test/terraform/outputs.tf b/codebundles/gcp-bigquery-job-health/.test/terraform/outputs.tf new file mode 100644 index 00000000..25969304 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.test/terraform/outputs.tf @@ -0,0 +1,9 @@ +output "dataset_id" { + description = "The ID of the test BigQuery dataset" + value = google_bigquery_dataset.test_dataset.dataset_id +} + +output "table_id" { + description = "The ID of the test BigQuery table" + value = google_bigquery_table.test_table.table_id +} \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.test/terraform/providers.tf b/codebundles/gcp-bigquery-job-health/.test/terraform/providers.tf new file mode 100644 index 00000000..8183f662 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.test/terraform/providers.tf @@ -0,0 +1,5 @@ +provider "google" { + credentials = file(var.service_account_key) + project = var.gcp_project_id + region = var.region +} \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.test/terraform/terraform.tfvars b/codebundles/gcp-bigquery-job-health/.test/terraform/terraform.tfvars new file mode 100644 index 00000000..7bb78c98 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.test/terraform/terraform.tfvars @@ -0,0 +1,3 @@ +# Example values - replace with real values before running +# gcp_project_id = "my-test-project" +# service_account_key = "/path/to/service-account-key.json" \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.test/terraform/variables.tf b/codebundles/gcp-bigquery-job-health/.test/terraform/variables.tf new file mode 100644 index 00000000..2c64921f --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.test/terraform/variables.tf @@ -0,0 +1,15 @@ +variable "gcp_project_id" { + description = "The GCP project ID to create test resources in" + type = string +} + +variable "service_account_key" { + description = "Path to the GCP service account JSON key file" + type = string +} + +variable "region" { + description = "GCP region" + type = string + default = "us-central1" +} \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/.test/validate-all-tests.sh b/codebundles/gcp-bigquery-job-health/.test/validate-all-tests.sh new file mode 100644 index 00000000..752d6748 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/.test/validate-all-tests.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Validate all test scenarios for gcp-bigquery-job-health +set -euo pipefail + +echo "=== Validating gcp-bigquery-job-health Test Scenarios ===" + +# Check that all required scripts exist +for script in check_job_success_rate.sh analyze_failed_jobs.sh identify_slow_jobs.sh check_slot_contention.sh generate_job_summary.sh; do + if [ -f "../$script" ]; then + echo "PASS: $script exists" + else + echo "FAIL: $script is missing" + exit 1 + fi +done + +# Check runbook.robot exists +if [ -f "../runbook.robot" ]; then + echo "PASS: runbook.robot exists" +else + echo "FAIL: runbook.robot is missing" + exit 1 +fi + +# Check sli.robot exists +if [ -f "../sli.robot" ]; then + echo "PASS: sli.robot exists" +else + echo "FAIL: sli.robot is missing" + exit 1 +fi + +# Check README exists +if [ -f "../README.md" ]; then + echo "PASS: README.md exists" +else + echo "FAIL: README.md is missing" + exit 1 +fi + +echo "" +echo "=== All validation checks passed ===" \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/README.md b/codebundles/gcp-bigquery-job-health/README.md new file mode 100644 index 00000000..52e760a0 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/README.md @@ -0,0 +1,67 @@ +# GCP BigQuery Job Health + +Monitors BigQuery job execution health by analyzing success/failure rates, error patterns, and performance anomalies. Helps operators identify failed queries, stuck jobs, and systemic issues affecting BigQuery workloads. + +## Overview + +- **Job Success Rate**: Calculates the job success rate over a configurable lookback window and raises issues if below threshold +- **Error Pattern Analysis**: Categorizes failed jobs by error reason (quotaExceeded, invalidQuery, timeout, accessDenied, etc.) +- **Slow Job Detection**: Identifies jobs exceeding a configurable duration threshold +- **Slot Contention Analysis**: Detects periods where slot demand exceeds reservation capacity +- **Health Summary Report**: Produces a consolidated health summary with total jobs, success rate, failure breakdown, and slot utilization + +## Configuration + +### Required Variables + +- `GCP_PROJECT_ID`: The GCP project ID that contains the BigQuery jobs to monitor + +### Optional Variables + +- `JOB_LOOKBACK_HOURS`: Number of hours to look back for job analysis (default: `24`) +- `SUCCESS_RATE_THRESHOLD`: Minimum acceptable job success rate percentage (default: `95`) +- `SLOW_JOB_DURATION_MINUTES`: Duration in minutes above which a job is considered slow (default: `30`) +- `SLOT_CONTENTION_THRESHOLD`: Slot utilization percentage indicating contention (default: `80`) + +### Secrets + +- `gcp_credentials`: GCP service account JSON key used to authenticate with GCP APIs. Format: JSON object containing type, project_id, private_key_id, private_key, client_email, client_id, auth_uri, token_uri + +## Prerequisites + +### Required GCP Permissions + +The service account needs the following roles: +- `roles/bigquery.jobUser` - to run queries against INFORMATION_SCHEMA +- `roles/bigquery.metadataViewer` - to access BigQuery metadata + +### Required Tools + +- `gcloud` CLI (Google Cloud SDK) +- `bq` command-line tool (included with gcloud) +- `jq` for JSON processing +- `bc` for numeric comparisons +- Bash 4.0 or higher + +## Tasks Overview + +### Check BigQuery Job Success Rate +Queries `INFORMATION_SCHEMA.JOBS_BY_PROJECT` to calculate the percentage of successful jobs. Raises a severity 3 issue if the success rate falls below the configured threshold. Severity 2 or 3 depending on severity of failure rate. + +### Analyze Failed BigQuery Job Error Patterns +Categorizes failed jobs by error reason using `INFORMATION_SCHEMA.JOBS_BY_PROJECT`. Detects frequent error categories like quotaExceeded, invalidQuery, timeout, accessDenied, and provides targeted remediation guidance for each category. + +### Identify Slow Running BigQuery Jobs +Detects jobs exceeding the configured duration threshold via `INFORMATION_SCHEMA.JOBS_BY_PROJECT`. Raises issues based on the number of slow jobs: severity 4 for widespread issues (>20 jobs), severity 3 for moderate issues (>5 jobs), severity 2 for isolated cases. + +### Check BigQuery Job Slot Contention +Analyzes slot usage from `INFORMATION_SCHEMA.JOBS_TIMELINE` to detect contention periods. Raises severity 3 for significant contention (>10 periods) or severity 2 for minor contention. + +### Generate BigQuery Job Health Summary Report +Produces a consolidated health summary including total jobs, success rate, failure breakdown by error reason, average and p95 duration, total data processed, slot hours, unique users, and slot utilization. Appends results to the workspace report. + +## Related Resources + +- [BigQuery INFORMATION_SCHEMA Documentation](https://cloud.google.com/bigquery/docs/reference/standard-sql/information-schema) +- [BigQuery Monitoring](https://cloud.google.com/bigquery/docs/monitoring) +- [BigQuery Slots and Reservations](https://cloud.google.com/bigquery/docs/reservations-intro) \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/analyze_failed_jobs.sh b/codebundles/gcp-bigquery-job-health/analyze_failed_jobs.sh new file mode 100644 index 00000000..804d957c --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/analyze_failed_jobs.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" +: "${JOB_LOOKBACK_HOURS:=24}" + +OUTPUT_FILE="failed_jobs_analysis_output.json" +issues_json='[]' + +echo "Analyzing failed BigQuery job error patterns for project: $GCP_PROJECT_ID" + +if ! query_result=$(bq query --project_id="$GCP_PROJECT_ID" --format=json --use_legacy_sql=false \ + "SELECT + error_result.reason as error_reason, + COUNT(*) as error_count + FROM \`$GCP_PROJECT_ID.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT\` + WHERE creation_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL $JOB_LOOKBACK_HOURS HOUR) + AND error_result IS NOT NULL + AND job_type = 'QUERY' + GROUP BY error_reason + ORDER BY error_count DESC" 2>err.log); then + err_msg=$(cat err.log) + rm -f err.log + issues_json=$(echo "$issues_json" | jq \ + --arg title "Failed Jobs Analysis Query Failed for \`$GCP_PROJECT_ID\`" \ + --arg details "Query failed: $err_msg" \ + --arg severity "3" \ + --arg next_steps "Verify BigQuery permissions and INFORMATION_SCHEMA access." \ + '. += [{ + "title": $title, + "details": $details, + "severity": ($severity | tonumber), + "next_steps": $next_steps + }]') + echo "$issues_json" > "$OUTPUT_FILE" + exit 0 +fi +rm -f err.log + +error_categories=$(echo "$query_result" | jq -c '.[]') + +severity_map() { + local reason="$1" + case "$reason" in + quotaExceeded|quota_exceeded) echo "3" ;; + accessDenied|access_denied) echo "3" ;; + invalidQuery|invalid_query) echo "2" ;; + timeout|timedOut) echo "3" ;; + rateLimitExceeded|rate_limit_exceeded) echo "2" ;; + datasetNotFound|tableNotFound|notFound) echo "2" ;; + resourcesExceeded|resources_exceeded) echo "3" ;; + *) echo "2" ;; + esac +} + +severity_map_expected() { + local reason="$1" + case "$reason" in + quotaExceeded|quota_exceeded) echo "BigQuery quotas and limits should not be exceeded." ;; + accessDenied|access_denied) echo "All queries should have proper access permissions." ;; + invalidQuery|invalid_query) echo "Queries should be syntactically valid." ;; + timeout|timedOut) echo "Queries should complete within the configured timeout." ;; + rateLimitExceeded|rate_limit_exceeded) echo "API rate limits should not be exceeded." ;; + datasetNotFound|tableNotFound|notFound) echo "All referenced datasets and tables should exist." ;; + resourcesExceeded|resources_exceeded) echo "Query resources (memory/shuffle) should be sufficient." ;; + *) echo "Jobs should complete without errors." ;; + esac +} + +severity_map_actual() { + local reason="$1" + case "$reason" in + quotaExceeded|quota_exceeded) echo "Quota limit has been exceeded for one or more BigQuery resources." ;; + accessDenied|access_denied) echo "Access denied errors detected, possibly due to missing permissions." ;; + invalidQuery|invalid_query) echo "Invalid queries detected, possibly due to schema changes or syntax errors." ;; + timeout|timedOut) echo "Queries are timing out, possibly due to complexity or resource contention." ;; + rateLimitExceeded|rate_limit_exceeded) echo "API rate limit has been exceeded." ;; + datasetNotFound|tableNotFound|notFound) echo "Referenced datasets or tables are missing." ;; + resourcesExceeded|resources_exceeded) echo "Query exceeded available resources." ;; + *) echo "Unexpected error patterns detected." ;; + esac +} + +severity_map_next_steps() { + local reason="$1" + case "$reason" in + quotaExceeded|quota_exceeded) echo "1. Request quota increase in GCP Console. 2. Optimize query patterns to reduce consumption. 3. Consider slot reservations for predictable workloads." ;; + accessDenied|access_denied) echo "1. Review IAM permissions for the service account. 2. Check dataset-level ACLs. 3. Verify authorized views and routines." ;; + invalidQuery|invalid_query) echo "1. Review query syntax in the failing jobs. 2. Check for schema changes that may break existing queries. 3. Test queries in BigQuery console." ;; + timeout|timedOut) echo "1. Optimize query performance with clustering/partitioning. 2. Increase query timeout if appropriate. 3. Reduce query complexity or use materialized views." ;; + rateLimitExceeded|rate_limit_exceeded) echo "1. Implement exponential backoff in applications. 2. Reduce concurrent query volume. 3. Distribute queries across time." ;; + datasetNotFound|tableNotFound|notFound) echo "1. Verify dataset and table names. 2. Check if tables have been deleted or renamed. 3. Update references in queries and views." ;; + resourcesExceeded|resources_exceeded) echo "1. Optimize JOINs and GROUP BY operations. 2. Increase slot capacity. 3. Use approximate aggregation functions where possible." ;; + *) echo "1. Review the failed job details in BigQuery console. 2. Check error messages for guidance." ;; + esac +} + +echo "$error_categories" | while read -r category; do + reason=$(echo "$category" | jq -r '.error_reason') + count=$(echo "$category" | jq -r '.error_count') + severity=$(severity_map "$reason") + expected=$(severity_map_expected "$reason") + actual=$(severity_map_actual "$reason") + next_steps=$(severity_map_next_steps "$reason") + + issues_json=$(echo "$issues_json" | jq \ + --arg title "Frequent BigQuery Error: \`$reason\` in \`$GCP_PROJECT_ID\` ($count occurrences)" \ + --arg details "Error reason: $reason occurred $count times in the last $JOB_LOOKBACK_HOURS hours." \ + --arg severity "$severity" \ + --arg expected "$expected" \ + --arg actual "$actual" \ + --arg next_steps "$next_steps" \ + '. += [{ + "title": $title, + "details": $details, + "severity": ($severity | tonumber), + "expected": $expected, + "actual": $actual, + "next_steps": $next_steps + }]') +done + +tmpfile=$(mktemp) +echo "$issues_json" > "$tmpfile" +mv "$tmpfile" "$OUTPUT_FILE" +echo "Analysis completed. Results saved to $OUTPUT_FILE" \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/check_job_success_rate.sh b/codebundles/gcp-bigquery-job-health/check_job_success_rate.sh new file mode 100644 index 00000000..7aa1115e --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/check_job_success_rate.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" +: "${JOB_LOOKBACK_HOURS:=24}" +: "${SUCCESS_RATE_THRESHOLD:=95}" + +OUTPUT_FILE="job_success_rate_output.json" +issues_json='[]' + +echo "Checking BigQuery job success rate for project: $GCP_PROJECT_ID" + +if ! query_result=$(bq query --project_id="$GCP_PROJECT_ID" --format=json --use_legacy_sql=false \ + "SELECT + COUNT(*) as total_jobs, + COUNTIF(error_result IS NULL) as successful_jobs, + COUNTIF(error_result IS NOT NULL) as failed_jobs, + ROUND(SAFE_DIVIDE(COUNTIF(error_result IS NULL), COUNT(*)) * 100, 2) as success_rate + FROM \`$GCP_PROJECT_ID.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT\` + WHERE creation_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL $JOB_LOOKBACK_HOURS HOUR) + AND job_type = 'QUERY'" 2>err.log); then + err_msg=$(cat err.log) + rm -f err.log + issues_json=$(echo "$issues_json" | jq \ + --arg title "BigQuery Job Success Rate Check Failed for \`$GCP_PROJECT_ID\`" \ + --arg details "Query failed: $err_msg" \ + --arg severity "3" \ + --arg next_steps "Verify BigQuery permissions (roles/bigquery.jobUser, roles/bigquery.metadataViewer) and ensure INFORMATION_SCHEMA is accessible." \ + '. += [{ + "title": $title, + "details": $details, + "severity": ($severity | tonumber), + "next_steps": $next_steps + }]') + echo "$issues_json" > "$OUTPUT_FILE" + exit 0 +fi +rm -f err.log + +total_jobs=$(echo "$query_result" | jq -r '.[0].total_jobs // 0') +success_rate=$(echo "$query_result" | jq -r '.[0].success_rate // 100') +failed_jobs=$(echo "$query_result" | jq -r '.[0].failed_jobs // 0') + +echo "Project: $GCP_PROJECT_ID" +echo "Total jobs: $total_jobs" +echo "Success rate: $success_rate%" +echo "Failed jobs: $failed_jobs" + +threshold=$SUCCESS_RATE_THRESHOLD +if (( $(echo "$success_rate < $threshold" | bc -l) )); then + if (( total_jobs > 0 )); then + issues_json=$(echo "$issues_json" | jq \ + --arg title "Low BigQuery Job Success Rate in \`$GCP_PROJECT_ID\`" \ + --arg details "Job success rate is $success_rate% across $total_jobs jobs in the last $JOB_LOOKBACK_HOURS hours. Threshold is $threshold%. Failed jobs: $failed_jobs." \ + --arg severity "3" \ + --arg expected "Job success rate should be at least $threshold%" \ + --arg actual "Current success rate is $success_rate%" \ + --arg next_steps "1. Run the failed job analysis task for details on error patterns. 2. Check BigQuery quota and reservation status. 3. Review recent query changes or deployments that may have introduced errors." \ + '. += [{ + "title": $title, + "details": $details, + "severity": ($severity | tonumber), + "expected": $expected, + "actual": $actual, + "next_steps": $next_steps + }]') + fi +fi + +echo "$issues_json" > "$OUTPUT_FILE" +echo "Analysis completed. Results saved to $OUTPUT_FILE" \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/check_slot_contention.sh b/codebundles/gcp-bigquery-job-health/check_slot_contention.sh new file mode 100644 index 00000000..e42b75d3 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/check_slot_contention.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" +: "${JOB_LOOKBACK_HOURS:=24}" +: "${SLOT_CONTENTION_THRESHOLD:=80}" + +OUTPUT_FILE="slot_contention_output.json" +issues_json='[]' + +echo "Checking BigQuery slot contention for project: $GCP_PROJECT_ID" + +if ! query_result=$(bq query --project_id="$GCP_PROJECT_ID" --format=json --use_legacy_sql=false \ + "SELECT + TIMESTAMP_TRUNC(period_start, HOUR) as hour, + SUM(slot_usage) / SUM(slot_capacity) * 100 as slot_utilization_pct, + SUM(jobs_queued) as total_queued_jobs, + SUM(slot_usage) as total_slot_usage, + SUM(slot_capacity) as total_slot_capacity + FROM \`$GCP_PROJECT_ID.region-us.INFORMATION_SCHEMA.JOBS_TIMELINE\` + WHERE period_start >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL $JOB_LOOKBACK_HOURS HOUR) + GROUP BY hour + ORDER BY hour DESC + LIMIT 48" 2>err.log); then + err_msg=$(cat err.log) + rm -f err.log + issues_json=$(echo "$issues_json" | jq \ + --arg title "Slot Contention Query Failed for \`$GCP_PROJECT_ID\`" \ + --arg details "Query failed. This may happen if JOBS_TIMELINE is not available or if there is no reservation. Error: $err_msg" \ + --arg severity "2" \ + --arg next_steps "Slot contention analysis requires a reservation. If no reservation exists, this is informational only." \ + '. += [{ + "title": $title, + "details": $details, + "severity": ($severity | tonumber), + "next_steps": $next_steps + }]') + echo "$issues_json" > "$OUTPUT_FILE" + exit 0 +fi +rm -f err.log + +contention_periods=$(echo "$query_result" | jq -c "[.[] | select(.slot_utilization_pct > $SLOT_CONTENTION_THRESHOLD)]") +contention_count=$(echo "$contention_periods" | jq -r 'length') + +echo "Found $contention_count time periods with slot utilization > ${SLOT_CONTENTION_THRESHOLD}%" + +if [ "$contention_count" -gt 0 ]; then + peak_utilization=$(echo "$query_result" | jq -r '[.[].slot_utilization_pct] | max // 0') + avg_utilization=$(echo "$query_result" | jq -r '[.[].slot_utilization_pct] | add / length // 0') + total_queued=$(echo "$contention_periods" | jq -r '[.[].total_queued_jobs] | add // 0') + + if [ "$contention_count" -gt 10 ]; then + severity=3 + expected="Slot utilization should remain below ${SLOT_CONTENTION_THRESHOLD}% to avoid job queuing and performance degradation." + actual="Slot contention detected across $contention_count time periods with $total_queued queued jobs and peak utilization of ${peak_utilization}%. Average utilization is ${avg_utilization}%." + title="Significant BigQuery Slot Contention in \`$GCP_PROJECT_ID\`" + next_steps="1. Increase slot reservation capacity. 2. Distribute large queries across different times. 3. Use workload management to prioritize critical queries. 4. Consider purchasing flex slots for peak periods." + else + severity=2 + expected="Slot utilization should remain below ${SLOT_CONTENTION_THRESHOLD}% to maintain consistent job performance." + actual="Slot contention detected across $contention_count time periods. Peak utilization was ${peak_utilization}%." + title="BigQuery Slot Contention Detected in \`$GCP_PROJECT_ID\`" + next_steps="1. Monitor slot utilization trends. 2. Review if slot capacity is appropriate for workload. 3. Optimize queries that consume excessive slots." + fi + + issues_json=$(echo "$issues_json" | jq \ + --arg title "$title" \ + --arg details "Slot contention detected in $contention_count periods over the last $JOB_LOOKBACK_HOURS hours. Peak utilization: ${peak_utilization}%. Avg utilization: ${avg_utilization}%. Total queued jobs: $total_queued." \ + --arg severity "$severity" \ + --arg expected "$expected" \ + --arg actual "$actual" \ + --arg next_steps "$next_steps" \ + '. += [{ + "title": $title, + "details": $details, + "severity": ($severity | tonumber), + "expected": $expected, + "actual": $actual, + "next_steps": $next_steps + }]') +fi + +echo "$issues_json" > "$OUTPUT_FILE" +echo "Analysis completed. Results saved to $OUTPUT_FILE" \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/generate_job_summary.sh b/codebundles/gcp-bigquery-job-health/generate_job_summary.sh new file mode 100644 index 00000000..67b3ec35 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/generate_job_summary.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" +: "${JOB_LOOKBACK_HOURS:=24}" + +OUTPUT_FILE="job_summary_output.json" + +echo "Generating BigQuery job health summary for project: $GCP_PROJECT_ID" + +summary_data='{}' + +if query_result=$(bq query --project_id="$GCP_PROJECT_ID" --format=json --use_legacy_sql=false \ + "SELECT + COUNT(*) as total_jobs, + COUNTIF(error_result IS NULL) as successful_jobs, + COUNTIF(error_result IS NOT NULL) as failed_jobs, + ROUND(SAFE_DIVIDE(COUNTIF(error_result IS NULL), COUNT(*)) * 100, 2) as success_rate, + ROUND(AVG(TIMESTAMP_DIFF(end_time, start_time, SECOND)), 2) as avg_duration_seconds, + ROUND(APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, start_time, SECOND), 100)[OFFSET(95)], 2) as p95_duration_seconds, + ROUND(SUM(total_bytes_processed) / POWER(1024, 4), 2) as total_data_processed_tb, + ROUND(SUM(total_slot_ms) / 1000 / 60 / 60, 2) as total_slot_hours, + COUNT(DISTINCT user_email) as unique_users + FROM \`$GCP_PROJECT_ID.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT\` + WHERE creation_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL $JOB_LOOKBACK_HOURS HOUR) + AND job_type = 'QUERY'" 2>err.log); then + rm -f err.log + summary_data=$(echo "$query_result" | jq '.[0]') +fi + +if error_query=$(bq query --project_id="$GCP_PROJECT_ID" --format=json --use_legacy_sql=false \ + "SELECT + error_result.reason as error_reason, + COUNT(*) as count + FROM \`$GCP_PROJECT_ID.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT\` + WHERE creation_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL $JOB_LOOKBACK_HOURS HOUR) + AND error_result IS NOT NULL + AND job_type = 'QUERY' + GROUP BY error_reason + ORDER BY count DESC" 2>/dev/null); then + error_breakdown=$(echo "$error_query" | jq -c '.') +else + error_breakdown='[]' +fi + +if slot_query=$(bq query --project_id="$GCP_PROJECT_ID" --format=json --use_legacy_sql=false \ + "SELECT + ROUND(AVG(slot_usage) / NULLIF(AVG(slot_capacity), 0) * 100, 2) as avg_slot_utilization_pct, + SUM(jobs_queued) as total_queued_jobs + FROM \`$GCP_PROJECT_ID.region-us.INFORMATION_SCHEMA.JOBS_TIMELINE\` + WHERE period_start >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL $JOB_LOOKBACK_HOURS HOUR)" 2>/dev/null); then + slot_info=$(echo "$slot_query" | jq '.[0]') +else + slot_info='{"avg_slot_utilization_pct": null, "total_queued_jobs": null}' +fi + +summary_data=$(echo "$summary_data" | jq \ + --argjson error_breakdown "$error_breakdown" \ + --argjson slot_info "$slot_info" \ + --arg project_id "$GCP_PROJECT_ID" \ + --arg lookback "$JOB_LOOKBACK_HOURS" \ + '{ + project_id: $project_id, + lookback_hours: ($lookback | tonumber), + total_jobs: (.total_jobs // 0), + successful_jobs: (.successful_jobs // 0), + failed_jobs: (.failed_jobs // 0), + success_rate: (.success_rate // 100), + avg_duration_seconds: (.avg_duration_seconds // 0), + p95_duration_seconds: (.p95_duration_seconds // 0), + total_data_processed_tb: (.total_data_processed_tb // 0), + total_slot_hours: (.total_slot_hours // 0), + unique_users: (.unique_users // 0), + error_breakdown: $error_breakdown, + slot_utilization: $slot_info + }') + +echo "$summary_data" > "$OUTPUT_FILE" +echo "Summary report generated:" +echo "$summary_data" | jq '.' +echo "Report saved to $OUTPUT_FILE" \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/identify_slow_jobs.sh b/codebundles/gcp-bigquery-job-health/identify_slow_jobs.sh new file mode 100644 index 00000000..931d377c --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/identify_slow_jobs.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" +: "${JOB_LOOKBACK_HOURS:=24}" +: "${SLOW_JOB_DURATION_MINUTES:=30}" + +OUTPUT_FILE="slow_jobs_output.json" +issues_json='[]' + +echo "Identifying slow BigQuery jobs for project: $GCP_PROJECT_ID" + +if ! query_result=$(bq query --project_id="$GCP_PROJECT_ID" --format=json --use_legacy_sql=false \ + "SELECT + job_id, + user_email, + query, + TIMESTAMP_DIFF(end_time, start_time, SECOND) as duration_seconds, + total_bytes_processed, + total_slot_ms, + state, + error_result.reason as error_reason, + creation_time, + start_time, + end_time + FROM \`$GCP_PROJECT_ID.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT\` + WHERE creation_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL $JOB_LOOKBACK_HOURS HOUR) + AND TIMESTAMP_DIFF(end_time, start_time, SECOND) > ($SLOW_JOB_DURATION_MINUTES * 60) + AND job_type = 'QUERY' + ORDER BY duration_seconds DESC + LIMIT 50" 2>err.log); then + err_msg=$(cat err.log) + rm -f err.log + issues_json=$(echo "$issues_json" | jq \ + --arg title "Slow Jobs Query Failed for \`$GCP_PROJECT_ID\`" \ + --arg details "Query failed: $err_msg" \ + --arg severity "3" \ + --arg next_steps "Verify BigQuery permissions and INFORMATION_SCHEMA access." \ + '. += [{ + "title": $title, + "details": $details, + "severity": ($severity | tonumber), + "next_steps": $next_steps + }]') + echo "$issues_json" > "$OUTPUT_FILE" + exit 0 +fi +rm -f err.log + +slow_job_count=$(echo "$query_result" | jq -r 'length') + +echo "Found $slow_job_count slow jobs (exceeding ${SLOW_JOB_DURATION_MINUTES} minutes)" + +if [ "$slow_job_count" -gt 0 ]; then + slow_threshold=$SLOW_JOB_DURATION_MINUTES + + if [ "$slow_job_count" -gt 20 ]; then + severity=4 + expected="Fewer than 10% of BigQuery jobs should exceed the ${slow_threshold} minute duration threshold." + actual="$slow_job_count jobs exceeded ${slow_threshold} minutes, indicating widespread performance issues." + next_steps="1. Check for systemic performance issues like slot contention. 2. Review slot reservation and capacity. 3. Consider tuning query performance with clustering and partitioning. 4. Investigate if a recent deployment or data change caused the slowdown." + elif [ "$slow_job_count" -gt 5 ]; then + severity=3 + expected="BigQuery jobs should complete within the configured duration threshold of ${slow_threshold} minutes." + actual="$slow_job_count jobs exceeded ${slow_threshold} minutes, indicating moderate performance concerns." + next_steps="1. Review the identified slow jobs for optimization opportunities. 2. Check slot utilization during the affected period. 3. Consider materialized views or query rewriting for repeated slow queries." + else + severity=2 + expected="All BigQuery jobs should complete within the configured duration threshold of ${slow_threshold} minutes." + actual="$slow_job_count jobs exceeded ${slow_threshold} minutes." + next_steps="1. Review each slow job individually for optimization. 2. Check if these are expected batch jobs. 3. Consider if the threshold needs adjustment." + fi + + title="BigQuery Slow Jobs Detected in \`$GCP_PROJECT_ID\` ($slow_job_count jobs over ${slow_threshold}min)" + + details="Found $slow_job_count jobs exceeding ${slow_threshold} minutes in the last $JOB_LOOKBACK_HOURS hours." + details+=" Slow job details:" + while IFS= read -r job; do + job_id=$(echo "$job" | jq -r '.job_id // "unknown"') + user=$(echo "$job" | jq -r '.user_email // "unknown"') + dur=$(echo "$job" | jq -r '.duration_seconds // 0') + proc=$(echo "$job" | jq -r '.total_bytes_processed // 0') + details+="\n- Job: $job_id | User: $user | Duration: ${dur}s | Bytes: $proc" + done < <(echo "$query_result" | jq -c '.[]') + + issues_json=$(echo "$issues_json" | jq \ + --arg title "$title" \ + --arg details "$details" \ + --arg severity "$severity" \ + --arg expected "$expected" \ + --arg actual "$actual" \ + --arg next_steps "$next_steps" \ + '. += [{ + "title": $title, + "details": $details, + "severity": ($severity | tonumber), + "expected": $expected, + "actual": $actual, + "next_steps": $next_steps + }]') +fi + +echo "$issues_json" > "$OUTPUT_FILE" +echo "Analysis completed. Results saved to $OUTPUT_FILE" \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/runbook.robot b/codebundles/gcp-bigquery-job-health/runbook.robot new file mode 100644 index 00000000..44753f38 --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/runbook.robot @@ -0,0 +1,229 @@ +*** Settings *** +Documentation Monitors BigQuery job execution health by analyzing success/failure rates, error patterns, performance anomalies, and slot contention. +Metadata Author rw-codebundle-agent +Metadata Display Name GCP BigQuery Job Health +Metadata Supports GCP BigQuery Job Health Query Performance +Force Tags GCP BigQuery Job Health Query Performance + +Library String +Library BuiltIn +Library RW.Core +Library RW.CLI +Library RW.platform +Library OperatingSystem +Library Collections +Suite Setup Suite Initialization + + +*** Tasks *** +Check BigQuery Job Success Rate for `${GCP_PROJECT_ID}` + [Documentation] Queries BigQuery job history (INFORMATION_SCHEMA) to calculate the job success rate over a configurable lookback window. Raises an issue if the success rate falls below the configured threshold. + [Tags] GCP BigQuery Job Health Success Rate access:read-only data:metrics + ${success_rate_result}= RW.CLI.Run Bash File + ... bash_file=check_job_success_rate.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=./check_job_success_rate.sh + ${success_issues_json}= RW.CLI.Run Cli + ... cmd=cat job_success_rate_output.json + ... env=${env} + ... timeout_seconds=30 + ... include_in_history=false + TRY + ${success_issues_list}= Evaluate json.loads(r'''${success_issues_json.stdout}''') json + EXCEPT + Log Failed to parse JSON for success rate check, defaulting to empty list. WARN + ${success_issues_list}= Create List + END + IF len(@{success_issues_list}) > 0 + FOR ${issue} IN @{success_issues_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${success_rate_result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + RW.Core.Add Pre To Report BigQuery Job Success Rate Analysis:\n${success_rate_result.stdout} + +Analyze Failed BigQuery Job Error Patterns for `${GCP_PROJECT_ID}` + [Documentation] Categorizes failed BigQuery jobs by error reason (quotaExceeded, invalidQuery, timeout, accessDenied, etc.) and raises issues for the most frequent error categories. + [Tags] GCP BigQuery Error Analysis Job Failures access:read-only data:logs-config + ${failed_jobs_result}= RW.CLI.Run Bash File + ... bash_file=analyze_failed_jobs.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=./analyze_failed_jobs.sh + ${failed_issues_json}= RW.CLI.Run Cli + ... cmd=cat failed_jobs_analysis_output.json + ... env=${env} + ... timeout_seconds=30 + ... include_in_history=false + TRY + ${failed_issues_list}= Evaluate json.loads(r'''${failed_issues_json.stdout}''') json + EXCEPT + Log Failed to parse JSON for failed jobs analysis, defaulting to empty list. WARN + ${failed_issues_list}= Create List + END + IF len(@{failed_issues_list}) > 0 + FOR ${issue} IN @{failed_issues_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${failed_jobs_result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + RW.Core.Add Pre To Report Failed Jobs Error Pattern Analysis:\n${failed_jobs_result.stdout} + +Identify Slow Running BigQuery Jobs for `${GCP_PROJECT_ID}` + [Documentation] Detects BigQuery jobs that exceed a configurable duration threshold (default: 30 minutes). Raises issues for jobs that are slow, indicating potential performance or resource problems. + [Tags] GCP BigQuery Performance Slow Jobs access:read-only data:metrics + ${slow_jobs_result}= RW.CLI.Run Bash File + ... bash_file=identify_slow_jobs.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=./identify_slow_jobs.sh + ${slow_issues_json}= RW.CLI.Run Cli + ... cmd=cat slow_jobs_output.json + ... env=${env} + ... timeout_seconds=30 + ... include_in_history=false + TRY + ${slow_issues_list}= Evaluate json.loads(r'''${slow_issues_json.stdout}''') json + EXCEPT + Log Failed to parse JSON for slow jobs analysis, defaulting to empty list. WARN + ${slow_issues_list}= Create List + END + IF len(@{slow_issues_list}) > 0 + FOR ${issue} IN @{slow_issues_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${slow_jobs_result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + RW.Core.Add Pre To Report Slow Running Jobs Analysis:\n${slow_jobs_result.stdout} + +Check BigQuery Job Slot Contention for `${GCP_PROJECT_ID}` + [Documentation] Analyzes slot usage from INFORMATION_SCHEMA to detect contention periods where slot demand exceeds reservation capacity. Raises issues when slot contention is detected. + [Tags] GCP BigQuery Slot Contention Performance access:read-only data:metrics + ${slot_result}= RW.CLI.Run Bash File + ... bash_file=check_slot_contention.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=./check_slot_contention.sh + ${slot_issues_json}= RW.CLI.Run Cli + ... cmd=cat slot_contention_output.json + ... env=${env} + ... timeout_seconds=30 + ... include_in_history=false + TRY + ${slot_issues_list}= Evaluate json.loads(r'''${slot_issues_json.stdout}''') json + EXCEPT + Log Failed to parse JSON for slot contention check, defaulting to empty list. WARN + ${slot_issues_list}= Create List + END + IF len(@{slot_issues_list}) > 0 + FOR ${issue} IN @{slot_issues_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${slot_result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + RW.Core.Add Pre To Report Slot Contention Analysis:\n${slot_result.stdout} + +Generate BigQuery Job Health Summary Report for `${GCP_PROJECT_ID}` + [Documentation] Produces a consolidated health summary for BigQuery jobs including total jobs, success rate, failure breakdown, average duration, and slot utilization. Appends to the workspace report. + [Tags] GCP BigQuery Summary Reporting access:read-only data:config + ${summary_result}= RW.CLI.Run Bash File + ... bash_file=generate_job_summary.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=./generate_job_summary.sh + RW.Core.Add Pre To Report BigQuery Job Health Summary Report:\n${summary_result.stdout} + ${summary_json}= RW.CLI.Run Cli + ... cmd=cat job_summary_output.json + ... env=${env} + ... timeout_seconds=30 + ... include_in_history=false + TRY + ${summary_data}= Evaluate json.loads(r'''${summary_json.stdout}''') json + ${total_jobs}= Set Variable ${summary_data['total_jobs']} + ${success_rate}= Set Variable ${summary_data['success_rate']} + RW.Core.Add To Report BigQuery Job Health Summary - Project: ${GCP_PROJECT_ID} - Total Jobs: ${total_jobs} - Success Rate: ${success_rate}% + EXCEPT + Log Failed to parse summary JSON. WARN + END + + +*** Keywords *** +Suite Initialization + ${gcp_credentials}= RW.Core.Import Secret gcp_credentials + ... type=string + ... description=GCP service account json used to authenticate with GCP APIs. + ... pattern=\w* + ... example={"type": "service_account","project_id":"myproject-ID", ... super secret stuff ...} + ${GCP_PROJECT_ID}= RW.Core.Import User Variable GCP_PROJECT_ID + ... type=string + ... description=The GCP project ID that contains the BigQuery jobs to monitor. + ... pattern=\w* + ... example=my-gcp-project + ${JOB_LOOKBACK_HOURS}= RW.Core.Import User Variable JOB_LOOKBACK_HOURS + ... type=string + ... description=Number of hours to look back for job analysis. + ... pattern=\d+ + ... default=24 + ${SUCCESS_RATE_THRESHOLD}= RW.Core.Import User Variable SUCCESS_RATE_THRESHOLD + ... type=string + ... description=Minimum acceptable job success rate (percentage). + ... pattern=\d+ + ... default=95 + ${SLOW_JOB_DURATION_MINUTES}= RW.Core.Import User Variable SLOW_JOB_DURATION_MINUTES + ... type=string + ... description=Duration in minutes above which a job is considered slow. + ... pattern=\d+ + ... default=30 + ${SLOT_CONTENTION_THRESHOLD}= RW.Core.Import User Variable SLOT_CONTENTION_THRESHOLD + ... type=string + ... description=Slot utilization percentage indicating contention. + ... pattern=\d+ + ... default=80 + ${OS_PATH}= Get Environment Variable PATH + Set Suite Variable ${GCP_PROJECT_ID} ${GCP_PROJECT_ID} + Set Suite Variable ${JOB_LOOKBACK_HOURS} ${JOB_LOOKBACK_HOURS} + Set Suite Variable ${SUCCESS_RATE_THRESHOLD} ${SUCCESS_RATE_THRESHOLD} + Set Suite Variable ${SLOW_JOB_DURATION_MINUTES} ${SLOW_JOB_DURATION_MINUTES} + Set Suite Variable ${SLOT_CONTENTION_THRESHOLD} ${SLOT_CONTENTION_THRESHOLD} + Set Suite Variable ${gcp_credentials} ${gcp_credentials} + Set Suite Variable ${env} {"GOOGLE_APPLICATION_CREDENTIALS":"./${gcp_credentials.key}","PATH":"$PATH:${OS_PATH}","GCP_PROJECT_ID":"${GCP_PROJECT_ID}","JOB_LOOKBACK_HOURS":"${JOB_LOOKBACK_HOURS}","SUCCESS_RATE_THRESHOLD":"${SUCCESS_RATE_THRESHOLD}","SLOW_JOB_DURATION_MINUTES":"${SLOW_JOB_DURATION_MINUTES}","SLOT_CONTENTION_THRESHOLD":"${SLOT_CONTENTION_THRESHOLD}"} \ No newline at end of file diff --git a/codebundles/gcp-bigquery-job-health/sli.robot b/codebundles/gcp-bigquery-job-health/sli.robot new file mode 100644 index 00000000..b2d4b9ee --- /dev/null +++ b/codebundles/gcp-bigquery-job-health/sli.robot @@ -0,0 +1,133 @@ +*** Settings *** +Documentation This SLI measures BigQuery job health by scoring success rate, slow job count, slot contention, and error patterns. Produces a value between 0 (completely failing) and 1 (fully passing). +Metadata Author rw-codebundle-agent +Metadata Display Name GCP BigQuery Job Health +Metadata Supports GCP BigQuery Job Health + +Library BuiltIn +Library RW.Core +Library RW.CLI +Library RW.platform +Library OperatingSystem + +Suite Setup Suite Initialization + + +*** Tasks *** +Check BigQuery Job Success Rate Score for `${GCP_PROJECT_ID}` + [Documentation] Checks the job success rate and scores it as 1 if above threshold, 0 otherwise. + [Tags] GCP BigQuery Job Health access:read-only data:metrics + ${result}= RW.CLI.Run Bash File + ... bash_file=check_job_success_rate.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ${issues_json}= RW.CLI.Run Cli + ... cmd=cat job_success_rate_output.json + ... env=${env} + ... timeout_seconds=30 + TRY + ${issues}= Evaluate json.loads(r'''${issues_json.stdout}''') json + ${has_issues}= Evaluate len($issues) > 0 + EXCEPT + Log Failed to parse JSON, defaulting to 0. WARN + ${has_issues}= Set Variable ${TRUE} + END + ${success_score}= Evaluate 0 if $has_issues else 1 + Set Global Variable ${success_score} + RW.Core.Push Metric ${success_score} sub_name=job_success_rate + +Check Slow Jobs Score for `${GCP_PROJECT_ID}` + [Documentation] Checks for slow running jobs and scores 1 if none found, 0 otherwise. + [Tags] GCP BigQuery Performance access:read-only data:metrics + ${result}= RW.CLI.Run Bash File + ... bash_file=identify_slow_jobs.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ${issues_json}= RW.CLI.Run Cli + ... cmd=cat slow_jobs_output.json + ... env=${env} + ... timeout_seconds=30 + TRY + ${issues}= Evaluate json.loads(r'''${issues_json.stdout}''') json + ${has_issues}= Evaluate len($issues) > 0 + EXCEPT + Log Failed to parse JSON, defaulting to 0. WARN + ${has_issues}= Set Variable ${TRUE} + END + ${slow_jobs_score}= Evaluate 0 if $has_issues else 1 + Set Global Variable ${slow_jobs_score} + RW.Core.Push Metric ${slow_jobs_score} sub_name=slow_jobs + +Check Slot Contention Score for `${GCP_PROJECT_ID}` + [Documentation] Checks for slot contention and scores 1 if none found, 0 otherwise. + [Tags] GCP BigQuery Slot Contention access:read-only data:metrics + ${result}= RW.CLI.Run Bash File + ... bash_file=check_slot_contention.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ${issues_json}= RW.CLI.Run Cli + ... cmd=cat slot_contention_output.json + ... env=${env} + ... timeout_seconds=30 + TRY + ${issues}= Evaluate json.loads(r'''${issues_json.stdout}''') json + ${has_issues}= Evaluate len($issues) > 0 + EXCEPT + Log Failed to parse JSON, defaulting to 0. WARN + ${has_issues}= Set Variable ${TRUE} + END + ${slot_score}= Evaluate 0 if $has_issues else 1 + Set Global Variable ${slot_score} + RW.Core.Push Metric ${slot_score} sub_name=slot_contention + +Generate Aggregate BigQuery Job Health Score for `${GCP_PROJECT_ID}` + [Documentation] Averages sub-scores into the final 0-1 health metric. + ${health_score}= Evaluate (${success_score} + ${slow_jobs_score} + ${slot_score}) / 3 + ${health_score}= Convert to Number ${health_score} 2 + RW.Core.Add to Report BigQuery Job Health Score for ${GCP_PROJECT_ID}: ${health_score} + RW.Core.Push Metric ${health_score} + + +*** Keywords *** +Suite Initialization + ${gcp_credentials}= RW.Core.Import Secret gcp_credentials + ... type=string + ... description=GCP service account json used to authenticate with GCP APIs. + ... pattern=\w* + ... example={"type": "service_account","project_id":"myproject-ID", ... super secret stuff ...} + ${GCP_PROJECT_ID}= RW.Core.Import User Variable GCP_PROJECT_ID + ... type=string + ... description=The GCP project ID that contains the BigQuery jobs to monitor. + ... pattern=\w* + ... example=my-gcp-project + ${JOB_LOOKBACK_HOURS}= RW.Core.Import User Variable JOB_LOOKBACK_HOURS + ... type=string + ... description=Number of hours to look back for job analysis. + ... pattern=\d+ + ... default=24 + ${SUCCESS_RATE_THRESHOLD}= RW.Core.Import User Variable SUCCESS_RATE_THRESHOLD + ... type=string + ... description=Minimum acceptable job success rate (percentage). + ... pattern=\d+ + ... default=95 + ${SLOW_JOB_DURATION_MINUTES}= RW.Core.Import User Variable SLOW_JOB_DURATION_MINUTES + ... type=string + ... description=Duration in minutes above which a job is considered slow. + ... pattern=\d+ + ... default=30 + ${SLOT_CONTENTION_THRESHOLD}= RW.Core.Import User Variable SLOT_CONTENTION_THRESHOLD + ... type=string + ... description=Slot utilization percentage indicating contention. + ... pattern=\d+ + ... default=80 + ${OS_PATH}= Get Environment Variable PATH + Set Suite Variable ${GCP_PROJECT_ID} ${GCP_PROJECT_ID} + Set Suite Variable ${JOB_LOOKBACK_HOURS} ${JOB_LOOKBACK_HOURS} + Set Suite Variable ${SUCCESS_RATE_THRESHOLD} ${SUCCESS_RATE_THRESHOLD} + Set Suite Variable ${SLOW_JOB_DURATION_MINUTES} ${SLOW_JOB_DURATION_MINUTES} + Set Suite Variable ${SLOT_CONTENTION_THRESHOLD} ${SLOT_CONTENTION_THRESHOLD} + Set Suite Variable ${gcp_credentials} ${gcp_credentials} + Set Suite Variable ${env} {"GOOGLE_APPLICATION_CREDENTIALS":"./${gcp_credentials.key}","PATH":"$PATH:${OS_PATH}","GCP_PROJECT_ID":"${GCP_PROJECT_ID}","JOB_LOOKBACK_HOURS":"${JOB_LOOKBACK_HOURS}","SUCCESS_RATE_THRESHOLD":"${SUCCESS_RATE_THRESHOLD}","SLOW_JOB_DURATION_MINUTES":"${SLOW_JOB_DURATION_MINUTES}","SLOT_CONTENTION_THRESHOLD":"${SLOT_CONTENTION_THRESHOLD}"} \ No newline at end of file