Web to API: turn a website task into a reliable endpoint

curl -X POST https://api.valendata.com/v1/skills \
  -H "Authorization: Bearer $VALENDATA_API_KEY" \
  -H "Content-Type: application/json" \
  -H "Idempotency-Key: dentists-austin-001" \
  -d '{
    "task": "Search for dentists in Austin and list them",
    "start_url": "https://www.google.com/maps",
    "inputs": {"city": "Austin"},
    "output_fields": {"name": "business name", "rating": "star rating", "phone": "phone number"}
  }'

import os
import requests

API = "https://api.valendata.com"
HEADERS = {"Authorization": f"Bearer {os.environ['VALENDATA_API_KEY']}"}

resp = requests.post(
    f"{API}/v1/skills",
    headers={**HEADERS, "Idempotency-Key": "dentists-austin-001"},
    json={
        "task": "Search for dentists in Austin and list them",
        "start_url": "https://www.google.com/maps",
        "inputs": {"city": "Austin"},
        "output_fields": {"name": "business name", "rating": "star rating", "phone": "phone number"},
    },
)
resp.raise_for_status()
creation = resp.json()
print(creation["creation_id"], creation["status"])

const API = "https://api.valendata.com";
const headers = {
  Authorization: `Bearer ${process.env.VALENDATA_API_KEY}`,
  "Content-Type": "application/json",
};

const resp = await fetch(`${API}/v1/skills`, {
  method: "POST",
  headers: { ...headers, "Idempotency-Key": "dentists-austin-001" },
  body: JSON.stringify({
    task: "Search for dentists in Austin and list them",
    start_url: "https://www.google.com/maps",
    inputs: { city: "Austin" },
    output_fields: { name: "business name", rating: "star rating", phone: "phone number" },
  }),
});
if (!resp.ok) throw new Error(`${resp.status} ${await resp.text()}`);
const creation = await resp.json();
console.log(creation.creation_id, creation.status);

import time

while creation["status"] not in ("ready", "failed"):
    time.sleep(5)
    creation = requests.get(f"{API}{creation['status_url']}", headers=HEADERS).json()

if creation["status"] == "failed":
    raise RuntimeError(creation["error"])
slug = creation["slug"]

let status = creation;
while (!["ready", "failed"].includes(status.status)) {
  await new Promise((r) => setTimeout(r, 5000));
  status = await (await fetch(`${API}${status.status_url}`, { headers })).json();
}
if (status.status === "failed") throw new Error(status.error);
const slug: string = status.slug;

curl -X POST "https://api.valendata.com/v1/skills/austin-dentists/run?max_results=20" \
  -H "Authorization: Bearer $VALENDATA_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"city": "Dallas"}'

resp = requests.post(
    f"{API}/v1/skills/{slug}/run",
    headers=HEADERS,
    params={"max_results": 20},
    json={"city": "Dallas"},
    timeout=600,
)
resp.raise_for_status()
result = resp.json()
for row in result["data"]:
    print(row)

const run = await fetch(`${API}/v1/skills/${slug}/run?max_results=20`, {
  method: "POST",
  headers,
  body: JSON.stringify({ city: "Dallas" }),
});
if (!run.ok) throw new Error(`${run.status} ${await run.text()}`);
const result = await run.json();
console.log(result.count, result.data);

curl -X POST https://api.valendata.com/v1/skills/austin-dentists/runs \
  -H "Authorization: Bearer $VALENDATA_API_KEY" \
  -H "Content-Type: application/json" \
  -H "Idempotency-Key: order-1234" \
  -d '{"inputs": {"city": "Houston"}, "max_results": 50, "webhook_url": "https://example.com/hooks/valendata"}'

# → 202 {"run_id": "run_8c1d2e3f4a5b", "status": "queued", "poll_url": "/v1/runs/run_8c1d2e3f4a5b", "idempotent_replay": false}

curl https://api.valendata.com/v1/runs/run_8c1d2e3f4a5b \
  -H "Authorization: Bearer $VALENDATA_API_KEY"

start = requests.post(
    f"{API}/v1/skills/{slug}/runs",
    headers={**HEADERS, "Idempotency-Key": "order-1234"},
    json={"inputs": {"city": "Houston"}, "max_results": 50},
)
start.raise_for_status()
poll_url = start.json()["poll_url"]

while True:
    run = requests.get(f"{API}{poll_url}", headers=HEADERS).json()
    if run["status"] not in ("queued", "running"):
        break
    time.sleep(3)

print(run["status"], run["count"], run["rows"][:3])

const start = await fetch(`${API}/v1/skills/${slug}/runs`, {
  method: "POST",
  headers: { ...headers, "Idempotency-Key": "order-1234" },
  body: JSON.stringify({ inputs: { city: "Houston" }, max_results: 50 }),
});
const { poll_url } = await start.json();

let runStatus: any;
do {
  await new Promise((r) => setTimeout(r, 3000));
  runStatus = await (await fetch(`${API}${poll_url}`, { headers })).json();
} while (["queued", "running"].includes(runStatus.status));

console.log(runStatus.status, runStatus.count, runStatus.rows);

import hashlib
import hmac
import time


def verify_valendata(secret: str, timestamp: str, raw_body: bytes, signature: str, tolerance_s: int = 300) -> bool:
    try:
        ts = int(timestamp)
    except (TypeError, ValueError):
        return False
    if abs(int(time.time()) - ts) > tolerance_s:
        return False
    digest = hmac.new(secret.encode(), f"{ts}.".encode() + raw_body, hashlib.sha256).hexdigest()
    expected = f"v1={digest}"
    return any(hmac.compare_digest(expected, s.strip()) for s in (signature or "").split(","))


# Flask example
# @app.post("/hooks/valendata")
# def hook():
#     ok = verify_valendata(
#         SECRET,
#         request.headers.get("X-Valendata-Timestamp"),
#         request.get_data(),
#         request.headers.get("X-Valendata-Signature"),
#     )
#     if not ok:
#         abort(401)
#     run = request.get_json()
#     ...
#     return "", 204

import { createHmac, timingSafeEqual } from "node:crypto";

export function verifyValendata(
  secret: string,
  timestamp: string | null,
  rawBody: Buffer,
  signature: string | null,
  toleranceS = 300,
): boolean {
  const ts = Number(timestamp);
  if (!Number.isInteger(ts) || !signature) return false;
  if (Math.abs(Math.floor(Date.now() / 1000) - ts) > toleranceS) return false;
  const digest = createHmac("sha256", secret)
    .update(Buffer.concat([Buffer.from(`${ts}.`), rawBody]))
    .digest("hex");
  const expected = Buffer.from(`v1=${digest}`);
  return signature.split(",").some((value) => {
    const given = Buffer.from(value.trim());
    return expected.length === given.length && timingSafeEqual(expected, given);
  });
}

// Express example: use express.raw() so you get the exact bytes that were signed.
// app.post("/hooks/valendata", express.raw({ type: "application/json" }), (req, res) => {
//   if (!verifyValendata(SECRET, req.get("X-Valendata-Timestamp") ?? null, req.body, req.get("X-Valendata-Signature") ?? null)) {
//     return res.sendStatus(401);
//   }
//   const run = JSON.parse(req.body.toString("utf8"));
//   res.sendStatus(204);
// });