cURL
curl -X POST https://reygrid.com/api/v1/agents/{AgentId}/sources/website \
-H "Authorization: Bearer $REYGRID_API_KEY" \
-H "Content-Type: application/json" \
-d '{"url": "https://docs.example.com"}'from reygrid import ReyGrid
client = ReyGrid(api_key="YOUR_API_KEY")
res = client.crawl_website("{AgentId}", "https://docs.example.com")
print(res.pagesCrawled, [page.path for page in res.pages])import { ReyGrid } from "reygrid";
const client = new ReyGrid({ apiKey: "YOUR_API_KEY" });
const { pagesCrawled, pages } = await client.crawlWebsite("{AgentId}", "https://docs.example.com");
console.log(pagesCrawled, pages.map((page) => page.path));{
"success": true,
"pagesCrawled": 3,
"pages": [
{
"id": "0f3c9a7e-2b1d-4c6e-8a5f-7d9b1e3c5a7f",
"path": "/"
},
{
"id": "9a1b3c5d-7e9f-4a2b-8c4d-6e8f0a2b4c6d",
"path": "/pricing"
},
{
"id": "2d4f6a8c-0e2a-4c6e-9a1b-3c5d7e9f1a3b",
"path": "/getting-started"
}
]
}{
"success": false,
"error": {
"code": "INVALID_BODY",
"message": "Invalid body",
"list": {
"title": [
"Title too short"
]
}
}
}{
"error": "Unauthorized",
"code": "AUTH_REQUIRED"
}{
"success": false,
"error": {
"code": "PLAN_LIMIT_REACHED",
"message": "You need to upgrade your plan"
}
}{
"success": false,
"error": {
"code": "SOURCE_NOT_FOUND",
"message": "This source does not exist"
}
}Knowledge
Crawl a website
Crawls the website from the given URL, following links on the same domain, and indexes each page. Pages already crawled are skipped. The crawl stops at your plan’s page or storage limit. The request returns when the crawl is finished.
POST
/
agents
/
{agentId}
/
sources
/
website
cURL
curl -X POST https://reygrid.com/api/v1/agents/{AgentId}/sources/website \
-H "Authorization: Bearer $REYGRID_API_KEY" \
-H "Content-Type: application/json" \
-d '{"url": "https://docs.example.com"}'from reygrid import ReyGrid
client = ReyGrid(api_key="YOUR_API_KEY")
res = client.crawl_website("{AgentId}", "https://docs.example.com")
print(res.pagesCrawled, [page.path for page in res.pages])import { ReyGrid } from "reygrid";
const client = new ReyGrid({ apiKey: "YOUR_API_KEY" });
const { pagesCrawled, pages } = await client.crawlWebsite("{AgentId}", "https://docs.example.com");
console.log(pagesCrawled, pages.map((page) => page.path));{
"success": true,
"pagesCrawled": 3,
"pages": [
{
"id": "0f3c9a7e-2b1d-4c6e-8a5f-7d9b1e3c5a7f",
"path": "/"
},
{
"id": "9a1b3c5d-7e9f-4a2b-8c4d-6e8f0a2b4c6d",
"path": "/pricing"
},
{
"id": "2d4f6a8c-0e2a-4c6e-9a1b-3c5d7e9f1a3b",
"path": "/getting-started"
}
]
}{
"success": false,
"error": {
"code": "INVALID_BODY",
"message": "Invalid body",
"list": {
"title": [
"Title too short"
]
}
}
}{
"error": "Unauthorized",
"code": "AUTH_REQUIRED"
}{
"success": false,
"error": {
"code": "PLAN_LIMIT_REACHED",
"message": "You need to upgrade your plan"
}
}{
"success": false,
"error": {
"code": "SOURCE_NOT_FOUND",
"message": "This source does not exist"
}
}Authorizations
ReyGrid API key sent as a Bearer token.
Path Parameters
The ID of the agent.
Body
application/json
Example:
"https://docs.example.com"