Cluster Management#

Monitor and manage your Antfly cluster's health and status.

Cluster Health#

The cluster health endpoint provides real-time status of all stores and shards:

healthy: All nodes and shards operational
degraded: Some shards unavailable but cluster functional
unhealthy: Critical failures affecting availability
error: Cluster-wide failures

Monitoring#

Health check endpoint: GET /healthz
Prometheus metrics: GET /metrics on port 4200

Use these endpoints to integrate with monitoring systems and alerting infrastructure.

Common questions about this section

How do I check if my cluster is healthy?
What do the different health statuses mean?
How do I integrate Antfly with Prometheus?

Get cluster status#

GET/status

Returns the current health and status of all stores and shards in the cluster

Security#

Provide your bearer token in the Authorization header when making requests to protected resources.

Example: Authorization: Bearer YOUR_API_KEY

Code Examples#

curl -X GET "/api/v1/status" \
    -H "Authorization: Bearer YOUR_API_KEY"

const response = await fetch('/api/v1/status', {
    method: 'GET',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY'
    }
});

const data = await response.json();

fetch('/api/v1/status', {
    method: 'GET',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY'
    }
})
    .then(response => response.json())
    .then(data => console.log(data));

import requests

headers = {
    'Authorization': 'Bearer YOUR_API_KEY'
}

response = requests.get('/api/v1/status', headers=headers)

data = response.json()

package main

import (
    "bytes"
    "encoding/json"
    "net/http"
)

func main() {
    req, _ := http.NewRequest("GET", "/api/v1/status", nil)
    req.Header.Set("Authorization", "Bearer YOUR_API_KEY")

    client := &http.Client{}
    resp, _ := client.Do(req)
    defer resp.Body.Close()
}

Responses#

{
  "health": "unknown",
  "message": "string",
  "auth_enabled": true,
  "swarm_mode": true
}

{
  "error": "An error message"
}

List secrets status#

GET/secrets

List all configured secret names and their status (keystore, env var, or both). Never returns secret values — only names and configuration status.

Security#

Provide your bearer token in the Authorization header when making requests to protected resources.

Example: Authorization: Bearer YOUR_API_KEY

Code Examples#

curl -X GET "/api/v1/secrets" \
    -H "Authorization: Bearer YOUR_API_KEY"

const response = await fetch('/api/v1/secrets', {
    method: 'GET',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY'
    }
});

const data = await response.json();

fetch('/api/v1/secrets', {
    method: 'GET',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY'
    }
})
    .then(response => response.json())
    .then(data => console.log(data));

import requests

headers = {
    'Authorization': 'Bearer YOUR_API_KEY'
}

response = requests.get('/api/v1/secrets', headers=headers)

data = response.json()

package main

import (
    "bytes"
    "encoding/json"
    "net/http"
)

func main() {
    req, _ := http.NewRequest("GET", "/api/v1/secrets", nil)
    req.Header.Set("Authorization", "Bearer YOUR_API_KEY")

    client := &http.Client{}
    resp, _ := client.Do(req)
    defer resp.Body.Close()
}

Responses#

{
  "secrets": [
    {
      "key": "string",
      "status": "configured_keystore",
      "env_var": "string",
      "created_at": "2024-01-01T00:00:00Z",
      "updated_at": "2024-01-01T00:00:00Z"
    }
  ]
}

{
  "error": "An error message"
}

Store a secret#

PUT/secrets/{key}

Store a secret in the keystore. Only available in swarm (single-node) mode. Returns 503 in multi-node mode.

Security#

Provide your bearer token in the Authorization header when making requests to protected resources.

Example: Authorization: Bearer YOUR_API_KEY

Request Body#

Example:

{
    "value": "string"
}

Code Examples#

curl -X PUT "/api/v1/secrets/{key}" \
    -H "Authorization: Bearer YOUR_API_KEY" \
    -H "Content-Type: application/json" \
    -d '{
    "value": "string"
}'

const response = await fetch('/api/v1/secrets/{key}', {
    method: 'PUT',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY',
        'Content-Type': 'application/json'
    },
    body: JSON.stringify({
        "value": "string"
    })
});

const data = await response.json();

fetch('/api/v1/secrets/{key}', {
    method: 'PUT',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY',
        'Content-Type': 'application/json'
    },
    body: JSON.stringify({
        "value": "string"
    })
})
    .then(response => response.json())
    .then(data => console.log(data));

import requests

headers = {
    'Authorization': 'Bearer YOUR_API_KEY'
}

response = requests.put(
    '/api/v1/secrets/{key}',
    headers=headers,
    json={
        "value": "string"
    }
)

data = response.json()

package main

import (
    "bytes"
    "encoding/json"
    "net/http"
)

func main() {
    body := []byte(`{
    "value": "string"
}`)
    req, _ := http.NewRequest("PUT", "/api/v1/secrets/{key}", bytes.NewBuffer(body))
    req.Header.Set("Authorization", "Bearer YOUR_API_KEY")
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{}
    resp, _ := client.Do(req)
    defer resp.Body.Close()
}

Responses#

{
  "key": "string",
  "status": "configured_keystore",
  "env_var": "string",
  "created_at": "2024-01-01T00:00:00Z",
  "updated_at": "2024-01-01T00:00:00Z"
}

{
  "error": "An error message"
}

{
  "error": "An error message"
}

Delete a secret#

DELETE/secrets/{key}

Remove a secret from the keystore. Only available in swarm (single-node) mode. Returns 503 in multi-node mode.

Security#

Provide your bearer token in the Authorization header when making requests to protected resources.

Example: Authorization: Bearer YOUR_API_KEY

Code Examples#

curl -X DELETE "/api/v1/secrets/{key}" \
    -H "Authorization: Bearer YOUR_API_KEY"

const response = await fetch('/api/v1/secrets/{key}', {
    method: 'DELETE',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY'
    }
});

const data = await response.json();

fetch('/api/v1/secrets/{key}', {
    method: 'DELETE',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY'
    }
})
    .then(response => response.json())
    .then(data => console.log(data));

import requests

headers = {
    'Authorization': 'Bearer YOUR_API_KEY'
}

response = requests.delete('/api/v1/secrets/{key}', headers=headers)

data = response.json()

package main

import (
    "bytes"
    "encoding/json"
    "net/http"
)

func main() {
    req, _ := http.NewRequest("DELETE", "/api/v1/secrets/{key}", nil)
    req.Header.Set("Authorization", "Bearer YOUR_API_KEY")

    client := &http.Client{}
    resp, _ := client.Do(req)
    defer resp.Body.Close()
}

Responses#

No response body

{
  "error": "An error message"
}

{
  "error": "An error message"
}

Backup all tables or selected tables#

POST/backup

Creates a backup of all tables or specified tables. Each table's backup includes:

Table metadata (schema, indexes, shard configuration)
All shard data (compressed with zstd)

The backup creates a cluster-level manifest that tracks all included tables and their individual backup locations.

Storage Locations:

Local filesystem: file:///path/to/backup
Amazon S3: s3://bucket-name/path/to/backup

Backup Structure:

{location}/
├── {backup_id}-cluster-metadata.json   (cluster manifest)
├── {table1}-{backup_id}-metadata.json  (table metadata)
├── shard-1-{table1}-{backup_id}.tar.zst
├── shard-2-{table1}-{backup_id}.tar.zst
├── {table2}-{backup_id}-metadata.json
└── ...

Security#

Provide your bearer token in the Authorization header when making requests to protected resources.

Example: Authorization: Bearer YOUR_API_KEY

Request Body#

Example:

{
    "backup_id": "cluster-backup-2025-01-15",
    "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
    "table_names": [
        "users",
        "products"
    ]
}

Code Examples#

curl -X POST "/api/v1/backup" \
    -H "Authorization: Bearer YOUR_API_KEY" \
    -H "Content-Type: application/json" \
    -d '{
    "backup_id": "cluster-backup-2025-01-15",
    "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
    "table_names": [
        "users",
        "products"
    ]
}'

const response = await fetch('/api/v1/backup', {
    method: 'POST',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY',
        'Content-Type': 'application/json'
    },
    body: JSON.stringify({
        "backup_id": "cluster-backup-2025-01-15",
        "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
        "table_names": [
            "users",
            "products"
        ]
    })
});

const data = await response.json();

fetch('/api/v1/backup', {
    method: 'POST',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY',
        'Content-Type': 'application/json'
    },
    body: JSON.stringify({
        "backup_id": "cluster-backup-2025-01-15",
        "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
        "table_names": [
            "users",
            "products"
        ]
    })
})
    .then(response => response.json())
    .then(data => console.log(data));

import requests

headers = {
    'Authorization': 'Bearer YOUR_API_KEY'
}

response = requests.post(
    '/api/v1/backup',
    headers=headers,
    json={
        "backup_id": "cluster-backup-2025-01-15",
        "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
        "table_names": [
            "users",
            "products"
        ]
    }
)

data = response.json()

package main

import (
    "bytes"
    "encoding/json"
    "net/http"
)

func main() {
    body := []byte(`{
    "backup_id": "cluster-backup-2025-01-15",
    "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
    "table_names": [
        "users",
        "products"
    ]
}`)
    req, _ := http.NewRequest("POST", "/api/v1/backup", bytes.NewBuffer(body))
    req.Header.Set("Authorization", "Bearer YOUR_API_KEY")
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{}
    resp, _ := client.Do(req)
    defer resp.Body.Close()
}

Responses#

{
  "backup_id": "cluster-backup-2025-01-15",
  "tables": [
    {
      "name": "users",
      "status": "completed",
      "error": "string"
    }
  ],
  "status": "completed"
}

Restore multiple tables from a backup#

POST/restore

Restores tables from a cluster backup. Can restore all tables or a subset.

Restore Modes:

fail_if_exists: Abort if any target table already exists (default)
skip_if_exists: Skip existing tables and restore the rest
overwrite: Drop existing tables and restore from backup

The restore is asynchronous - this endpoint triggers the restore process and returns immediately. The actual data restoration happens via the reconciliation loop as shards are started.

Security#

Provide your bearer token in the Authorization header when making requests to protected resources.

Example: Authorization: Bearer YOUR_API_KEY

Request Body#

Example:

{
    "backup_id": "cluster-backup-2025-01-15",
    "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
    "table_names": [
        "users",
        "products"
    ],
    "restore_mode": "skip_if_exists"
}

Code Examples#

curl -X POST "/api/v1/restore" \
    -H "Authorization: Bearer YOUR_API_KEY" \
    -H "Content-Type: application/json" \
    -d '{
    "backup_id": "cluster-backup-2025-01-15",
    "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
    "table_names": [
        "users",
        "products"
    ],
    "restore_mode": "skip_if_exists"
}'

const response = await fetch('/api/v1/restore', {
    method: 'POST',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY',
        'Content-Type': 'application/json'
    },
    body: JSON.stringify({
        "backup_id": "cluster-backup-2025-01-15",
        "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
        "table_names": [
            "users",
            "products"
        ],
        "restore_mode": "skip_if_exists"
    })
});

const data = await response.json();

fetch('/api/v1/restore', {
    method: 'POST',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY',
        'Content-Type': 'application/json'
    },
    body: JSON.stringify({
        "backup_id": "cluster-backup-2025-01-15",
        "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
        "table_names": [
            "users",
            "products"
        ],
        "restore_mode": "skip_if_exists"
    })
})
    .then(response => response.json())
    .then(data => console.log(data));

import requests

headers = {
    'Authorization': 'Bearer YOUR_API_KEY'
}

response = requests.post(
    '/api/v1/restore',
    headers=headers,
    json={
        "backup_id": "cluster-backup-2025-01-15",
        "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
        "table_names": [
            "users",
            "products"
        ],
        "restore_mode": "skip_if_exists"
    }
)

data = response.json()

package main

import (
    "bytes"
    "encoding/json"
    "net/http"
)

func main() {
    body := []byte(`{
    "backup_id": "cluster-backup-2025-01-15",
    "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
    "table_names": [
        "users",
        "products"
    ],
    "restore_mode": "skip_if_exists"
}`)
    req, _ := http.NewRequest("POST", "/api/v1/restore", bytes.NewBuffer(body))
    req.Header.Set("Authorization", "Bearer YOUR_API_KEY")
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{}
    resp, _ := client.Do(req)
    defer resp.Body.Close()
}

Responses#

{
  "tables": [
    {
      "name": "users",
      "status": "triggered",
      "error": "string"
    }
  ],
  "status": "triggered"
}

List available backups#

GET/backups

Lists all cluster-level backups available at the specified location. Returns metadata about each backup including the tables included, timestamp, and Antfly version.

Security#

Provide your bearer token in the Authorization header when making requests to protected resources.

Example: Authorization: Bearer YOUR_API_KEY

Parameters#

Name	Type	Location	Required	Description
`location`	string	query	Yes	Storage location to search for backups.

Local filesystem: file:///path/to/backup
Amazon S3: s3://bucket-name/path/to/backup |

Code Examples#

curl -X GET "/api/v1/backups?location=s3://mybucket/antfly-backups/" \
    -H "Authorization: Bearer YOUR_API_KEY"

const response = await fetch('/api/v1/backups?location=s3://mybucket/antfly-backups/', {
    method: 'GET',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY'
    }
});

const data = await response.json();

fetch('/api/v1/backups?location=s3://mybucket/antfly-backups/', {
    method: 'GET',
    headers: {
        'Authorization': 'Bearer YOUR_API_KEY'
    }
})
    .then(response => response.json())
    .then(data => console.log(data));

import requests

headers = {
    'Authorization': 'Bearer YOUR_API_KEY'
}

response = requests.get('/api/v1/backups?location=s3://mybucket/antfly-backups/', headers=headers)

data = response.json()

package main

import (
    "bytes"
    "encoding/json"
    "net/http"
)

func main() {
    req, _ := http.NewRequest("GET", "/api/v1/backups?location=s3://mybucket/antfly-backups/", nil)
    req.Header.Set("Authorization", "Bearer YOUR_API_KEY")

    client := &http.Client{}
    resp, _ := client.Do(req)
    defer resp.Body.Close()
}

Responses#

{
  "backups": [
    {
      "backup_id": "cluster-backup-2025-01-15",
      "timestamp": "2025-01-15T10:30:00Z",
      "tables": [
        "users",
        "products"
      ],
      "location": "s3://mybucket/antfly-backups/cluster/2025-01-15",
      "antfly_version": "v1.0.0"
    }
  ]
}