[Creator] [design-spec] gcp-bigquery-job-health

codebundles/gcp-bigquery-job-health/.runwhen/generation-rules/gcp-bigquery-job-health.yaml

@@ -0,0 +1,22 @@
+apiVersion: runwhen.com/v1
+kind: GenerationRules
+spec:
+  platform: gcp
+  generationRules:
+    - resourceTypes:
+        - project
+      matchRules:
+        - type: pattern
+          pattern: ".+"
+          properties: [name]
+          mode: substring
+      slxs:
+        - baseName: gcp-bigquery-job-health
+          qualifiers: ["project"]
+          baseTemplateName: gcp-bigquery-job-health
+          levelOfDetail: basic
+          outputItems:
+            - type: slx
+            - type: sli
+            - type: runbook
+              templateName: gcp-bigquery-job-health-taskset.yaml

codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-sli.yaml

@@ -0,0 +1,50 @@
+apiVersion: runwhen.com/v1
+kind: ServiceLevelIndicator
+metadata:
+  name: {{slx_name}}
+  labels:
+    {% include "common-labels.yaml" %}
+  annotations:
+    {% include "common-annotations.yaml" %}
+spec:
+  displayUnitsLong: Health Score
+  displayUnitsShort: score
+  locations:
+    - {{default_location}}
+  description: Measures BigQuery job health by scoring success rate, slow jobs, and slot contention for project {{project.name}}.
+  codeBundle:
+    {% if repo_url %}
+    repoUrl: {{repo_url}}
+    {% else %}
+    repoUrl: https://github.com/runwhen-contrib/rw-cli-codecollection.git
+    {% endif %}
+    {% if ref %}
+    ref: {{ref}}
+    {% else %}
+    ref: main
+    {% endif %}
+    pathToRobot: codebundles/gcp-bigquery-job-health/sli.robot
+  intervalStrategy: intermezzo
+  intervalSeconds: 300
+  configProvided:
+    - name: GCP_PROJECT_ID
+      value: "{{project.name}}"
+    - name: JOB_LOOKBACK_HOURS
+      value: "24"
+    - name: SUCCESS_RATE_THRESHOLD
+      value: "95"
+    - name: SLOW_JOB_DURATION_MINUTES
+      value: "30"
+    - name: SLOT_CONTENTION_THRESHOLD
+      value: "80"
+  secretsProvided:
+  {% if wb_version %}
+    {% include "gcp-auth.yaml" ignore missing %}
+  {% else %}
+    - name: gcp_credentials
+      workspaceKey: AUTH DETAILS NOT FOUND
+  {% endif %}
+  alertConfig:
+    tasks:
+      persona: eager-edgar
+      sessionTTL: 10m

codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-slx.yaml

@@ -0,0 +1,29 @@
+apiVersion: runwhen.com/v1
+kind: ServiceLevelX
+metadata:
+  name: {{slx_name}}
+  labels:
+    {% include "common-labels.yaml" %}
+  annotations:
+    {% include "common-annotations.yaml" %}
+spec:
+  imageURL: https://storage.googleapis.com/runwhen-nonprod-shared-images/icons/gcp/bigquery/bigquery.svg
+  alias: "{{match_resource.resource.name}} GCP BigQuery Job Health"
+  asMeasuredBy: A combined score based on job success rate, slow job count, and slot contention. A score of 1 is healthy, less than 1 indicates issues with BigQuery job execution.
+  configProvided:
+    - name: GCP_PROJECT_ID
+      value: "{{project.name}}"
+  owners:
+    - {{workspace.owner_email}}
+  statement: BigQuery jobs should complete successfully with acceptable performance and without resource contention.
+  additionalContext:
+    {% include "gcp-hierarchy.yaml" ignore missing %}
+    qualified_name: "{{ match_resource.qualified_name }}"
+  tags:
+    {% include "gcp-tags.yaml" ignore missing %}
+    - name: service
+      value: bigquery
+    - name: category
+      value: analytics
+    - name: access
+      value: read-only

codebundles/gcp-bigquery-job-health/.runwhen/templates/gcp-bigquery-job-health-taskset.yaml

@@ -0,0 +1,41 @@
+apiVersion: runwhen.com/v1
+kind: Runbook
+metadata:
+  name: {{slx_name}}
+  labels:
+    {% include "common-labels.yaml" %}
+  annotations:
+    {% include "common-annotations.yaml" %}
+spec:
+  location: {{default_location}}
+  description: Runs tasks to validate GCP BigQuery job health, including success rate, error patterns, slow jobs, and slot contention for project {{project.name}}.
+  codeBundle:
+    {% if repo_url %}
+    repoUrl: {{repo_url}}
+    {% else %}
+    repoUrl: https://github.com/runwhen-contrib/rw-cli-codecollection.git
+    {% endif %}
+    {% if ref %}
+    ref: {{ref}}
+    {% else %}
+    ref: main
+    {% endif %}
+    pathToRobot: codebundles/gcp-bigquery-job-health/runbook.robot
+  configProvided:
+    - name: GCP_PROJECT_ID
+      value: "{{project.name}}"
+    - name: JOB_LOOKBACK_HOURS
+      value: "24"
+    - name: SUCCESS_RATE_THRESHOLD
+      value: "95"
+    - name: SLOW_JOB_DURATION_MINUTES
+      value: "30"
+    - name: SLOT_CONTENTION_THRESHOLD
+      value: "80"
+  secretsProvided:
+  {% if wb_version %}
+    {% include "gcp-auth.yaml" ignore missing %}
+  {% else %}
+    - name: gcp_credentials
+      workspaceKey: AUTH DETAILS NOT FOUND
+  {% endif %}

codebundles/gcp-bigquery-job-health/.test/README.md

@@ -0,0 +1,43 @@
+# Test Infrastructure for gcp-bigquery-job-health
+
+This directory contains test infrastructure for the GCP BigQuery Job Health CodeBundle.
+
+## Test Scenarios
+
+### healthy_project
+A GCP project with no failed jobs, fast execution, and sufficient slot capacity.
+- Expected issues: 0
+
+### failed_jobs_detected
+A GCP project with failed jobs due to quota exceedance and invalid queries.
+- Expected issues: 3
+- Expected severities: [2, 2, 3]
+
+### slot_contention
+A GCP project with high slot utilization and slow queued jobs.
+- Expected issues: 2
+- Expected severities: [2, 3]
+
+## Running Tests
+
+```bash
+# Build infrastructure
+task build-infra
+
+# Run full test suite
+task default
+
+# Clean up
+task clean
+```
+
+## Terraform
+
+The Terraform configuration creates:
+- A GCP project with BigQuery datasets and tables
+- Service account with appropriate roles
+- Test queries to simulate job patterns
+
+Set the following variables in `terraform.tfvars`:
+- `gcp_project_id`: Target project ID
+- `service_account_key`: Path to service account JSON key

codebundles/gcp-bigquery-job-health/.test/Taskfile.yaml

@@ -0,0 +1,77 @@
+version: "3"
+
+tasks:
+  default:
+    desc: "Run complete test suite"
+    cmds:
+      - task: check-unpushed-commits
+      - task: generate-rwl-config
+
+  clean:
+    desc: "Run cleanup tasks"
+    cmds:
+      - task: check-and-cleanup-terraform
+      - task: delete-slxs
+      - task: clean-rwl-discovery
+
+  build-infra:
+    desc: "Build test infrastructure"
+    cmds:
+      - task: build-terraform-infra
+
+  check-unpushed-commits:
+    desc: Check for uncommitted/unpushed changes
+    vars:
+      BASE_DIR: "../"
+    cmds:
+      - |
+        UNCOMMITTED=$(git diff --name-only HEAD | grep -E "^${BASE_DIR}" | grep -v "/\.test/" || true)
+        if [ -n "$UNCOMMITTED" ]; then
+          echo "Uncommitted changes found. Commit and push before testing."
+          exit 1
+        fi
+    silent: true
+
+  build-terraform-infra:
+    desc: "Build Terraform test resources"
+    dir: terraform
+    cmds:
+      - |
+        source tf.secret
+        terraform init
+        terraform plan -out=tfplan
+        terraform apply tfplan
+
+  check-and-cleanup-terraform:
+    desc: "Destroy Terraform resources"
+    dir: terraform
+    cmds:
+      - |
+        if [ -f "terraform.tfstate" ]; then
+          source tf.secret
+          terraform destroy -auto-approve
+        fi
+
+  generate-rwl-config:
+    desc: "Generate RWL config for testing"
+    cmds:
+      - |
+        echo "Generating RWL config..."
+        mkdir -p .rwl-config
+        cat > .rwl-config/config.yaml << 'EOF'
+        gcp_project_id: "${GCP_PROJECT_ID}"
+        job_lookback_hours: 24
+        success_rate_threshold: 95
+        slow_job_duration_minutes: 30
+        slot_contention_threshold: 80
+        EOF
+
+  delete-slxs:
+    desc: "Delete SLX configurations"
+    cmds:
+      - rm -rf .rwl-config
+
+  clean-rwl-discovery:
+    desc: "Clean RWL discovery artifacts"
+    cmds:
+      - rm -rf .rwl-discovery

codebundles/gcp-bigquery-job-health/.test/terraform/main.tf

@@ -0,0 +1,25 @@
+provider "google" {
+  credentials = file(var.service_account_key)
+  project     = var.gcp_project_id
+  region      = "us-central1"
+}
+
+resource "google_bigquery_dataset" "test_dataset" {
+  dataset_id  = "bq_job_health_test"
+  description = "Test dataset for BigQuery job health monitoring"
+  location    = "US"
+}
+
+resource "google_bigquery_table" "test_table" {
+  dataset_id = google_bigquery_dataset.test_dataset.dataset_id
+  table_id   = "test_data"
+
+  schema = jsonencode([
+    { name = "id", type = "INTEGER", mode = "REQUIRED" },
+    { name = "name", type = "STRING", mode = "NULLABLE" },
+    { name = "value", type = "FLOAT", mode = "NULLABLE" },
+    { name = "timestamp", type = "TIMESTAMP", mode = "NULLABLE" },
+  ])
+
+  depends_on = [google_bigquery_dataset.test_dataset]
+}

codebundles/gcp-bigquery-job-health/.test/terraform/outputs.tf

@@ -0,0 +1,9 @@
+output "dataset_id" {
+  description = "The ID of the test BigQuery dataset"
+  value       = google_bigquery_dataset.test_dataset.dataset_id
+}
+
+output "table_id" {
+  description = "The ID of the test BigQuery table"
+  value       = google_bigquery_table.test_table.table_id
+}

codebundles/gcp-bigquery-job-health/.test/terraform/providers.tf

@@ -0,0 +1,5 @@
+provider "google" {
+  credentials = file(var.service_account_key)
+  project     = var.gcp_project_id
+  region      = var.region
+}

codebundles/gcp-bigquery-job-health/.test/terraform/terraform.tfvars

@@ -0,0 +1,3 @@
+# Example values - replace with real values before running
+# gcp_project_id = "my-test-project"
+# service_account_key = "/path/to/service-account-key.json"

codebundles/gcp-bigquery-job-health/.test/terraform/variables.tf

@@ -0,0 +1,15 @@
+variable "gcp_project_id" {
+  description = "The GCP project ID to create test resources in"
+  type        = string
+}
+
+variable "service_account_key" {
+  description = "Path to the GCP service account JSON key file"
+  type        = string
+}
+
+variable "region" {
+  description = "GCP region"
+  type        = string
+  default     = "us-central1"
+}

codebundles/gcp-bigquery-job-health/.test/validate-all-tests.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Validate all test scenarios for gcp-bigquery-job-health
+set -euo pipefail
+
+echo "=== Validating gcp-bigquery-job-health Test Scenarios ==="
+
+# Check that all required scripts exist
+for script in check_job_success_rate.sh analyze_failed_jobs.sh identify_slow_jobs.sh check_slot_contention.sh generate_job_summary.sh; do
+    if [ -f "../$script" ]; then
+        echo "PASS: $script exists"
+    else
+        echo "FAIL: $script is missing"
+        exit 1
+    fi
+done
+
+# Check runbook.robot exists
+if [ -f "../runbook.robot" ]; then
+    echo "PASS: runbook.robot exists"
+else
+    echo "FAIL: runbook.robot is missing"
+    exit 1
+fi
+
+# Check sli.robot exists
+if [ -f "../sli.robot" ]; then
+    echo "PASS: sli.robot exists"
+else
+    echo "FAIL: sli.robot is missing"
+    exit 1
+fi
+
+# Check README exists
+if [ -f "../README.md" ]; then
+    echo "PASS: README.md exists"
+else
+    echo "FAIL: README.md is missing"
+    exit 1
+fi
+
+echo ""
+echo "=== All validation checks passed ==="