Documentation Index
Fetch the complete documentation index at: https://webscraping.titannet.io/docs/llms.txt
Use this file to discover all available pages before exploring further.
The search action type is for discovery-first work: turn queries and ranking into candidate URLs or documents before you lock onto specific pages. It pairs with scrape or crawl when the hard part is “what should we read?” rather than only “extract these known URLs.”
Examples use TITAN_API_URL (Task Service origin, no /api/v1 suffix) and TITAN_TOKEN. Tab titles match other integration pages. Rust: ureq + serde_json. You need tasks:write (and related scopes for run and read paths).
When to use search
- You have questions or keywords, not a fixed PDP list.
- You want fresh candidates from the open web on each run.
- You will downstream either crawl for more coverage or scrape into a stable schema.
- With
input_source: static_urls, seeds usually live in the task’s urls list (validated per action rules).
- In multi-step plans, a later
scrape step often sets input_source: previous_step so discovered URLs flow into extraction.
Single-action example (POST /api/v1/tasks)
The exact limits and payload keys depend on your template and script contract—treat the JSON below as structural, not a guarantee of field names.
cURL
Go
TypeScript
Python
Rust
curl -sS -X POST "$TITAN_API_URL/api/v1/tasks" \
-H "Authorization: Bearer $TITAN_TOKEN" \
-H "Content-Type: application/json" \
-d '{
"name": "Nightly patent news discovery",
"objective": "Find recent articles matching our keywords and return candidate URLs with titles",
"execution_type": "single",
"urls": ["https://example.com/seed-blog"],
"action_type": "search",
"input_source": "static_urls",
"template_slug": "example-search-template",
"limits": { "max_results": 25 }
}'
body := map[string]any{
"name": "Nightly patent news discovery",
"objective": "Find recent articles matching our keywords and return candidate URLs with titles",
"execution_type": "single",
"urls": []string{"https://example.com/seed-blog"},
"action_type": "search",
"input_source": "static_urls",
"template_slug": "example-search-template",
"limits": map[string]any{"max_results": 25},
}
b, _ := json.Marshal(body)
req, _ := http.NewRequest("POST", os.Getenv("TITAN_API_URL")+"/api/v1/tasks", bytes.NewReader(b))
req.Header.Set("Authorization", "Bearer "+os.Getenv("TITAN_TOKEN"))
req.Header.Set("Content-Type", "application/json")
resp, _ := http.DefaultClient.Do(req)
defer resp.Body.Close()
out, _ := io.ReadAll(resp.Body)
fmt.Println(string(out))
const base = process.env.TITAN_API_URL!;
const token = process.env.TITAN_TOKEN!;
const res = await fetch(`${base}/api/v1/tasks`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
name: "Nightly patent news discovery",
objective:
"Find recent articles matching our keywords and return candidate URLs with titles",
execution_type: "single",
urls: ["https://example.com/seed-blog"],
action_type: "search",
input_source: "static_urls",
template_slug: "example-search-template",
limits: { max_results: 25 },
}),
});
console.log(await res.text());
import json
import os
import urllib.request
payload = {
"name": "Nightly patent news discovery",
"objective": "Find recent articles matching our keywords and return candidate URLs with titles",
"execution_type": "single",
"urls": ["https://example.com/seed-blog"],
"action_type": "search",
"input_source": "static_urls",
"template_slug": "example-search-template",
"limits": {"max_results": 25},
}
req = urllib.request.Request(
f"{os.environ['TITAN_API_URL']}/api/v1/tasks",
data=json.dumps(payload).encode(),
headers={
"Authorization": f"Bearer {os.environ['TITAN_TOKEN']}",
"Content-Type": "application/json",
},
method="POST",
)
with urllib.request.urlopen(req) as resp:
print(resp.read().decode())
use serde_json::json;
let base = std::env::var("TITAN_API_URL").expect("TITAN_API_URL");
let token = std::env::var("TITAN_TOKEN").expect("TITAN_TOKEN");
let body = json!({
"name": "Nightly patent news discovery",
"objective": "Find recent articles matching our keywords and return candidate URLs with titles",
"execution_type": "single",
"urls": ["https://example.com/seed-blog"],
"action_type": "search",
"input_source": "static_urls",
"template_slug": "example-search-template",
"limits": { "max_results": 25 }
});
let resp = ureq::post(format!("{base}/api/v1/tasks"))
.set("Authorization", &format!("Bearer {token}"))
.set("Content-Type", "application/json")
.send_json(body)
.expect("create");
println!("{}", resp.into_string().expect("body"));
Multi-step example (search → scrape)
Create a task whose execution_plan has two steps: search with static_urls, then scrape with previous_step and an output_schema carried on the scrape step (self-contained plan). Sketch:
{
"name": "Discover then extract headlines",
"objective": "Search for product mentions, then scrape top results into records",
"execution_type": "single",
"execution_plan": {
"steps": [
{
"step_id": "discover",
"action_type": "search",
"input_source": "static_urls",
"template_slug": "vertical-search",
"limits": { "max_results": 15 }
},
{
"step_id": "extract",
"action_type": "scrape",
"input_source": "previous_step",
"template_slug": "article-headline-scrape",
"output_schema": {
"type": "object",
"properties": {
"url": { "type": "string" },
"title": { "type": "string" }
},
"required": ["url", "title"]
}
}
]
}
}
Do not send top-level action_type or output_schema when the body is an execution_plan-only create; the plan must include everything each step needs.
Submit the multi-step task (same body via API)
cURL
Go
TypeScript
Python
Rust
curl -sS -X POST "$TITAN_API_URL/api/v1/tasks" \
-H "Authorization: Bearer $TITAN_TOKEN" \
-H "Content-Type: application/json" \
-d @search-scrape-plan.json
// Load JSON from file or embed as []byte; then:
req, _ := http.NewRequest("POST", os.Getenv("TITAN_API_URL")+"/api/v1/tasks", bytes.NewReader(planJSON))
req.Header.Set("Authorization", "Bearer "+os.Getenv("TITAN_TOKEN"))
req.Header.Set("Content-Type", "application/json")
resp, _ := http.DefaultClient.Do(req)
defer resp.Body.Close()
out, _ := io.ReadAll(resp.Body)
fmt.Println(string(out))
import * as fs from "node:fs";
const base = process.env.TITAN_API_URL!;
const token = process.env.TITAN_TOKEN!;
const plan = fs.readFileSync("search-scrape-plan.json", "utf8");
const res = await fetch(`${base}/api/v1/tasks`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: plan,
});
console.log(await res.text());
import os
import urllib.request
with open("search-scrape-plan.json", "rb") as f:
raw = f.read()
req = urllib.request.Request(
f"{os.environ['TITAN_API_URL']}/api/v1/tasks",
data=raw,
headers={
"Authorization": f"Bearer {os.environ['TITAN_TOKEN']}",
"Content-Type": "application/json",
},
method="POST",
)
with urllib.request.urlopen(req) as resp:
print(resp.read().decode())
let base = std::env::var("TITAN_API_URL").expect("TITAN_API_URL");
let token = std::env::var("TITAN_TOKEN").expect("TITAN_TOKEN");
let plan_json = std::fs::read_to_string("search-scrape-plan.json").expect("read plan");
let body: serde_json::Value = serde_json::from_str(&plan_json).expect("json");
let resp = ureq::post(format!("{base}/api/v1/tasks"))
.set("Authorization", &format!("Bearer {token}"))
.set("Content-Type", "application/json")
.send_json(body)
.expect("create");
println!("{}", resp.into_string().expect("body"));
Save the JSON sketch as search-scrape-plan.json, or build the same object in memory in your language.
Run and inspect
Run task
cURL
Go
TypeScript
Python
Rust
curl -sS -X POST "$TITAN_API_URL/api/v1/tasks/$TASK_ID/run" \
-H "Authorization: Bearer $TITAN_TOKEN" \
-H "Content-Type: application/json" \
-d '{}'
req, _ := http.NewRequest(
"POST",
fmt.Sprintf("%s/api/v1/tasks/%s/run", os.Getenv("TITAN_API_URL"), os.Getenv("TASK_ID")),
bytes.NewReader([]byte("{}")),
)
req.Header.Set("Authorization", "Bearer "+os.Getenv("TITAN_TOKEN"))
req.Header.Set("Content-Type", "application/json")
resp, _ := http.DefaultClient.Do(req)
defer resp.Body.Close()
out, _ := io.ReadAll(resp.Body)
fmt.Println(string(out))
const base = process.env.TITAN_API_URL!;
const token = process.env.TITAN_TOKEN!;
const taskId = process.env.TASK_ID!;
const res = await fetch(`${base}/api/v1/tasks/${taskId}/run`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: "{}",
});
console.log(await res.text());
import os
import urllib.request
tid = os.environ["TASK_ID"]
req = urllib.request.Request(
f"{os.environ['TITAN_API_URL']}/api/v1/tasks/{tid}/run",
data=b"{}",
headers={
"Authorization": f"Bearer {os.environ['TITAN_TOKEN']}",
"Content-Type": "application/json",
},
method="POST",
)
with urllib.request.urlopen(req) as resp:
print(resp.read().decode())
let base = std::env::var("TITAN_API_URL").expect("TITAN_API_URL");
let token = std::env::var("TITAN_TOKEN").expect("TITAN_TOKEN");
let task_id = std::env::var("TASK_ID").expect("TASK_ID");
let resp = ureq::post(format!("{base}/api/v1/tasks/{task_id}/run"))
.set("Authorization", &format!("Bearer {token}"))
.set("Content-Type", "application/json")
.send_json(serde_json::json!({}))
.expect("run");
println!("{}", resp.into_string().expect("body"));
Get task
cURL
Go
TypeScript
Python
Rust
curl -sS "$TITAN_API_URL/api/v1/tasks/$TASK_ID" \
-H "Authorization: Bearer $TITAN_TOKEN"
req, _ := http.NewRequest(
"GET",
fmt.Sprintf("%s/api/v1/tasks/%s", os.Getenv("TITAN_API_URL"), os.Getenv("TASK_ID")),
nil,
)
req.Header.Set("Authorization", "Bearer "+os.Getenv("TITAN_TOKEN"))
resp, _ := http.DefaultClient.Do(req)
defer resp.Body.Close()
out, _ := io.ReadAll(resp.Body)
fmt.Println(string(out))
const base = process.env.TITAN_API_URL!;
const token = process.env.TITAN_TOKEN!;
const taskId = process.env.TASK_ID!;
const res = await fetch(`${base}/api/v1/tasks/${taskId}`, {
headers: { Authorization: `Bearer ${token}` },
});
console.log(await res.text());
import os
import urllib.request
tid = os.environ["TASK_ID"]
req = urllib.request.Request(
f"{os.environ['TITAN_API_URL']}/api/v1/tasks/{tid}",
headers={"Authorization": f"Bearer {os.environ['TITAN_TOKEN']}"},
method="GET",
)
with urllib.request.urlopen(req) as resp:
print(resp.read().decode())
let base = std::env::var("TITAN_API_URL").expect("TITAN_API_URL");
let token = std::env::var("TITAN_TOKEN").expect("TITAN_TOKEN");
let task_id = std::env::var("TASK_ID").expect("TASK_ID");
let resp = ureq::get(format!("{base}/api/v1/tasks/{task_id}"))
.set("Authorization", &format!("Bearer {token}"))
.call()
.expect("get");
println!("{}", resp.into_string().expect("body"));
Dry-run a plan without persisting a task:
cURL
Go
TypeScript
Python
Rust
curl -sS -X POST "$TITAN_API_URL/api/v1/executions/preview" \
-H "Authorization: Bearer $TITAN_TOKEN" \
-H "Content-Type: application/json" \
-d '{
"objective": "Probe a two-step search→scrape graph",
"execution_plan": { "steps": [] }
}'
body := map[string]any{
"objective": "Probe a two-step search→scrape graph",
"execution_plan": map[string]any{"steps": []any{}},
}
b, _ := json.Marshal(body)
req, _ := http.NewRequest("POST", os.Getenv("TITAN_API_URL")+"/api/v1/executions/preview", bytes.NewReader(b))
req.Header.Set("Authorization", "Bearer "+os.Getenv("TITAN_TOKEN"))
req.Header.Set("Content-Type", "application/json")
resp, _ := http.DefaultClient.Do(req)
defer resp.Body.Close()
out, _ := io.ReadAll(resp.Body)
fmt.Println(string(out))
const base = process.env.TITAN_API_URL!;
const token = process.env.TITAN_TOKEN!;
const res = await fetch(`${base}/api/v1/executions/preview`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
objective: "Probe a two-step search→scrape graph",
execution_plan: { steps: [] },
}),
});
console.log(await res.text());
import json
import os
import urllib.request
payload = {
"objective": "Probe a two-step search→scrape graph",
"execution_plan": {"steps": []},
}
req = urllib.request.Request(
f"{os.environ['TITAN_API_URL']}/api/v1/executions/preview",
data=json.dumps(payload).encode(),
headers={
"Authorization": f"Bearer {os.environ['TITAN_TOKEN']}",
"Content-Type": "application/json",
},
method="POST",
)
with urllib.request.urlopen(req) as resp:
print(resp.read().decode())
use serde_json::json;
let base = std::env::var("TITAN_API_URL").expect("TITAN_API_URL");
let token = std::env::var("TITAN_TOKEN").expect("TITAN_TOKEN");
let body = json!({
"objective": "Probe a two-step search→scrape graph",
"execution_plan": { "steps": [] }
});
let resp = ureq::post(format!("{base}/api/v1/executions/preview"))
.set("Authorization", &format!("Bearer {token}"))
.set("Content-Type", "application/json")
.send_json(body)
.expect("preview");
println!("{}", resp.into_string().expect("body"));
Replace the empty steps array with a real plan when you call this for real.